mirror of
https://github.com/prowler-cloud/prowler.git
synced 2026-08-19 17:40:25 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
447bbd5777 | ||
|
|
4ae7c67d3f | ||
|
|
a76ba156d5 | ||
|
|
cd90a91158 | ||
|
|
e1b23e2526 | ||
|
|
537c3ea71e | ||
|
|
b6f74c7284 | ||
|
|
8cd008ba91 | ||
|
|
1f13e1d348 | ||
|
|
87a15d7bb8 | ||
|
|
f5bdacd07a | ||
|
|
ce80fcd430 | ||
|
|
c1c080b072 | ||
|
|
587187419f | ||
|
|
050a5915ca | ||
|
|
d4e4d12c5a | ||
|
|
72cf2a65a6 | ||
|
|
48db27481d | ||
|
|
1247c5fb33 | ||
|
|
69321418a3 | ||
|
|
301d13a4b9 | ||
|
|
3f8c1e822f | ||
|
|
1e1c1c018b | ||
|
|
a212916a49 | ||
|
|
883ffa1fdb | ||
|
|
21d9d6192e | ||
|
|
fd38a0ac03 | ||
|
|
af6918d57b | ||
|
|
9a9cbc997b | ||
|
|
aec500ee3b | ||
|
|
8fbc721223 | ||
|
|
c3ce3d2b3c | ||
|
|
c46cbaaa4a | ||
|
|
34e8e3ca61 | ||
|
|
5dac8a0a53 | ||
|
|
ed1fec8866 | ||
|
|
2abcb05e22 | ||
|
|
aba43440ca | ||
|
|
d47cbb4f8c | ||
|
|
36006de8ce | ||
|
|
5ccb044b85 | ||
|
|
e40e9a6483 | ||
|
|
717d48b0e0 | ||
|
|
4e7e2f7eab | ||
|
|
5404863a3e | ||
|
|
9d4b6c4d16 | ||
|
|
cd56985480 | ||
|
|
58eb0fa095 | ||
|
|
d850349a1c | ||
|
|
36a609f2ee | ||
|
|
0c5ceb7e72 | ||
|
|
ed04257e6c | ||
|
|
e2b2e568a6 | ||
|
|
dc432c8c3a | ||
|
|
6ffbb8373e | ||
|
|
6bea847232 | ||
|
|
78b94b7043 | ||
|
|
92634d4261 | ||
|
|
007e32c690 | ||
|
|
fe7e6675e0 | ||
|
|
d6f5f060ca | ||
|
|
9d013910e6 | ||
|
|
4c281aa464 | ||
|
|
086805df1d | ||
|
|
5793cd7e38 | ||
|
|
9b8b77cec0 | ||
|
|
5b9824c379 | ||
|
|
2b7db88694 | ||
|
|
fbddeea254 | ||
|
|
7785829969 | ||
|
|
d62abeb407 | ||
|
|
4e00cfd1b6 | ||
|
|
917e5d07ff | ||
|
|
76286f1186 | ||
|
|
93dd696a4f | ||
|
|
36be63af07 | ||
|
|
dc228e8b36 | ||
|
|
058a1dc8fe | ||
|
|
de7da3e960 | ||
|
|
3b0124d3fd | ||
|
|
fb995a79bf | ||
|
|
9d8c060c49 | ||
|
|
0cabceb09c | ||
|
|
3ee24fba51 | ||
|
|
48acb3bd2e | ||
|
|
c6c07957a6 | ||
|
|
0610866b73 | ||
|
|
2afa18d3da | ||
|
|
a0fdc96649 | ||
|
|
b6caaa4268 | ||
|
|
04e6e330a7 | ||
|
|
29329f6203 | ||
|
|
bdd44a0dce | ||
|
|
10d9fc35e6 | ||
|
|
6826422a6a | ||
|
|
ca48fd0719 | ||
|
|
b9298b4023 | ||
|
|
2375f1d962 | ||
|
|
5ee8b9680d | ||
|
|
45cfe4e411 | ||
|
|
30d737c7d7 | ||
|
|
869f0726f5 | ||
|
|
6dda1ae485 | ||
|
|
13f51de5c1 | ||
|
|
5d5f0676e0 | ||
|
|
8a1d7bcd6b | ||
|
|
ccc1f161d2 | ||
|
|
a7917f779a | ||
|
|
7f96d895bb | ||
|
|
bf3b5c2ba7 | ||
|
|
218f64595a | ||
|
|
6d8d553610 | ||
|
|
e10cf34ad6 | ||
|
|
bbf54011ea | ||
|
|
9e173978dc | ||
|
|
d27ec7d62e | ||
|
|
151dcd2895 | ||
|
|
d961d7efe4 | ||
|
|
99285d4656 | ||
|
|
19629e9bb8 | ||
|
|
b89b427a86 | ||
|
|
908d2ce766 | ||
|
|
853610bbbf | ||
|
|
751c7fc29f | ||
|
|
7dd08bc6bf | ||
|
|
2111d083df | ||
|
|
82d37c4978 | ||
|
|
aee3b392a7 | ||
|
|
ddbf3405a0 | ||
|
|
2293cab72c | ||
|
|
5a761f341b | ||
|
|
3c68a121e5 | ||
|
|
c0ae8b9739 | ||
|
|
5ec4a1cbba | ||
|
|
bae74b8181 | ||
|
|
5ecfd6ea20 | ||
|
|
e8ffe59ce2 | ||
|
|
e2ce41a492 | ||
|
|
6546d51a6c | ||
|
|
73059ffc7e | ||
|
|
f1a30f706a | ||
|
|
aa60dc3e17 | ||
|
|
54518bd127 | ||
|
|
8d4ec561c2 | ||
|
|
0463cd1559 | ||
|
|
ca97d7d983 | ||
|
|
7b8ce51263 | ||
|
|
262dfda0aa | ||
|
|
8bc42a5ded | ||
|
|
406712ffa3 | ||
|
|
c2d7187a0b | ||
|
|
e690e5e86b | ||
|
|
e4d5ca11b3 | ||
|
|
181197177c | ||
|
|
f21304c6a8 | ||
|
|
0cf48a2c35 | ||
|
|
6b4fb934f8 | ||
|
|
d1ed1eddef | ||
|
|
0c9f4f6578 | ||
|
|
e1f20487ce | ||
|
|
26b8c6b663 | ||
|
|
3960827a9c | ||
|
|
e419771b04 | ||
|
|
94ce76d679 | ||
|
|
28c064a9b7 | ||
|
|
eeb02453d1 | ||
|
|
cb4b889b20 | ||
|
|
f1e42d1681 | ||
|
|
ca7ce5a8c3 | ||
|
|
810d8d7686 | ||
|
|
dd1895d2c4 | ||
|
|
b5bb85c956 | ||
|
|
36fe48dbc5 | ||
|
|
e5bbffd47c | ||
|
|
566167489b | ||
|
|
3cb360e9ae | ||
|
|
24e3182329 | ||
|
|
49309b43d3 | ||
|
|
6db8ce672c | ||
|
|
9465b82747 | ||
|
|
383d2b218f | ||
|
|
dccd674cf9 | ||
|
|
a679865cce | ||
|
|
15bfa39b23 | ||
|
|
dc3433aaf0 | ||
|
|
25fc285966 | ||
|
|
9022a3a138 | ||
|
|
ca443b8ff1 | ||
|
|
79e066d3f5 | ||
|
|
56831a7392 | ||
|
|
2e82f1564f | ||
|
|
a394c0fdf6 | ||
|
|
20eca78767 | ||
|
|
bba594a1db | ||
|
|
65f00a197b | ||
|
|
ce27053c2d | ||
|
|
610febb5d5 | ||
|
|
c4378d5992 | ||
|
|
f1d741214a | ||
|
|
285974b7d4 | ||
|
|
989c3b174e | ||
|
|
75f95559d6 | ||
|
|
e085e14247 | ||
|
|
368d3a2661 | ||
|
|
3c8fde25ee | ||
|
|
ec0bb53839 | ||
|
|
bfb3fcea4c | ||
|
|
61cd4aea3f | ||
|
|
01b49f0743 | ||
|
|
4a5a49b5bb | ||
|
|
a21cb64a94 | ||
|
|
9a50dffaa0 | ||
|
|
e710ebff1c | ||
|
|
b3caee88e4 | ||
|
|
d9f90e50b8 | ||
|
|
58efb719fa | ||
|
|
355b7071aa | ||
|
|
b994b0b14e | ||
|
|
6c559fbb8d | ||
|
|
b2d74711d9 | ||
|
|
7e60e8f8da | ||
|
|
62955dd16b | ||
|
|
1f7caa6394 | ||
|
|
662e7e9e18 | ||
|
|
e3013d9918 | ||
|
|
0ea2f6d67e | ||
|
|
7692a1d76a | ||
|
|
1c9afc714e | ||
|
|
466f1a3d73 | ||
|
|
061fbaa7bb | ||
|
|
28b045302f | ||
|
|
5a2226c02c | ||
|
|
6f172a5c19 | ||
|
|
a7d180ea5b | ||
|
|
d4bbc8b5ad | ||
|
|
a5bc226f11 | ||
|
|
3a3d9d6146 | ||
|
|
bcd282d3d0 | ||
|
|
eb7949c884 | ||
|
|
e60a4462e5 | ||
|
|
f7f8747512 | ||
|
|
d573af911d | ||
|
|
cf9beb8234 | ||
|
|
7f67eac1bf | ||
|
|
a652e28b4a | ||
|
|
1b17304c4a | ||
|
|
c2cef99b33 | ||
|
|
a769e37615 | ||
|
|
9d2a8d9108 | ||
|
|
e05519ff9f | ||
|
|
67b26072f8 | ||
|
|
2222082631 | ||
|
|
8b0cb4b981 | ||
|
|
9422eff8ab | ||
|
|
e3c4368d32 | ||
|
|
2a641b39c8 | ||
|
|
02b713572b | ||
|
|
74251350bc | ||
|
|
8f745cdbe6 | ||
|
|
81226cd837 | ||
|
|
a2824f7166 | ||
|
|
edbbd86828 | ||
|
|
c58dad2ca4 | ||
|
|
b4befe3a10 | ||
|
|
d98933c2e7 | ||
|
|
03dfa3816d | ||
|
|
ad1261ce54 | ||
|
|
3252f9cf19 | ||
|
|
f1cdf3df15 | ||
|
|
03ddb8a708 | ||
|
|
2678c6bc9f | ||
|
|
48c071297f | ||
|
|
7e9a16d022 | ||
|
|
84b388f649 | ||
|
|
671d0c746c | ||
|
|
0e4b117161 | ||
|
|
a70bc3c1c7 | ||
|
|
723d161c63 | ||
|
|
d560020592 | ||
|
|
00451f8239 | ||
|
|
329dfdf8e6 | ||
|
|
4c59af93eb | ||
|
|
6ca8e726f7 | ||
|
|
546eb2d85a | ||
|
|
ec3efc94f5 | ||
|
|
6cffd0d17f | ||
|
|
528d32601b | ||
|
|
56b3044aae | ||
|
|
3a096b1750 | ||
|
|
6f01041178 | ||
|
|
13e2ede763 | ||
|
|
c53ddfd532 | ||
|
|
f86bd7b52e | ||
|
|
6177fc6286 | ||
|
|
0fd952ae2b | ||
|
|
74622dd576 | ||
|
|
4dfa2b9748 | ||
|
|
435424a680 | ||
|
|
dbbefd0558 | ||
|
|
e55d1d470e | ||
|
|
ab69f3b665 | ||
|
|
a28f4994a8 | ||
|
|
349611d52d | ||
|
|
10b965e3c7 | ||
|
|
554a5024c1 | ||
|
|
7d03bc5e17 | ||
|
|
c660b35ed6 | ||
|
|
f3bac38a55 | ||
|
|
61330937f7 | ||
|
|
5ac978b9a3 | ||
|
|
b4159bd590 | ||
|
|
ef4d45d409 | ||
|
|
f210c26c2f | ||
|
|
a55a736363 | ||
|
|
9f2af5abc2 | ||
|
|
fee98a58eb | ||
|
|
1ab8f2f0ac | ||
|
|
e7fbc8b391 | ||
|
|
8caab36c3f | ||
|
|
0c4794b060 | ||
|
|
782e3f238b | ||
|
|
e1c7e0a99b | ||
|
|
6ef70484c7 | ||
|
|
621170d9c9 | ||
|
|
b6e2255e9e | ||
|
|
3ce8eae72f | ||
|
|
81aa1883fd | ||
|
|
534dedb608 | ||
|
|
cff1704d7b | ||
|
|
0ca444895f | ||
|
|
a9865209a1 | ||
|
|
8526e8b4a6 | ||
|
|
a52ef3c04a | ||
|
|
1f3f5c2e27 | ||
|
|
6eebfcfe77 | ||
|
|
9d8b69abda | ||
|
|
60aa601e92 | ||
|
|
fc1fd538bd | ||
|
|
40c1761840 | ||
|
|
0ab0e8671d | ||
|
|
7a7c828fc7 | ||
|
|
5cbe473eb9 | ||
|
|
caf2f61563 | ||
|
|
9dc4deccb6 | ||
|
|
476e7d1010 |
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"name": "prowler-plugins",
|
||||
"description": "Prowler Cloud Security for Claude Code",
|
||||
"owner": {
|
||||
"name": "Prowler",
|
||||
"email": "support@prowler.com"
|
||||
},
|
||||
"plugins": [
|
||||
{
|
||||
"name": "prowler",
|
||||
"source": "./claude_plugins/prowler",
|
||||
"description": "Prowler for Claude Code — cloud security and compliance skills powered by the Prowler MCP server. Bundles compliance triage and remediation; more skills coming.",
|
||||
"category": "security",
|
||||
"homepage": "https://prowler.com"
|
||||
}
|
||||
]
|
||||
}
|
||||
+8
-1
@@ -11,7 +11,14 @@ envs = "wt step copy-ignored"
|
||||
[[pre-start]]
|
||||
deps = "uv sync"
|
||||
|
||||
# Block 3: reminder - last visible output before `wt switch` returns.
|
||||
# Block 3: prepare pnpm via corepack.
|
||||
[[pre-start]]
|
||||
corepack-enable = "corepack enable"
|
||||
|
||||
[[pre-start]]
|
||||
corepack-install = "cd ui && corepack install"
|
||||
|
||||
# Block 4: reminder - last visible output before `wt switch` returns.
|
||||
# Hooks can't mutate the parent shell, so venv activation is manual.
|
||||
[[pre-start]]
|
||||
reminder = "echo '>> Reminder: activate the venv in this shell with: source .venv/bin/activate'"
|
||||
|
||||
@@ -6,14 +6,20 @@
|
||||
PROWLER_UI_VERSION="stable"
|
||||
AUTH_URL=http://localhost:3000
|
||||
API_BASE_URL=http://prowler-api:8080/api/v1
|
||||
# deprecated, use UI_API_BASE_URL
|
||||
NEXT_PUBLIC_API_BASE_URL=${API_BASE_URL}
|
||||
UI_API_BASE_URL=${API_BASE_URL}
|
||||
# deprecated, use UI_API_DOCS_URL
|
||||
NEXT_PUBLIC_API_DOCS_URL=http://prowler-api:8080/api/v1/docs
|
||||
UI_API_DOCS_URL=http://prowler-api:8080/api/v1/docs
|
||||
AUTH_TRUST_HOST=true
|
||||
UI_PORT=3000
|
||||
# openssl rand -base64 32
|
||||
AUTH_SECRET="N/c6mnaS5+SWq81+819OrzQZlmx1Vxtp/orjttJSmw8="
|
||||
# Google Tag Manager ID
|
||||
# Google Tag Manager ID (empty/unset ⇒ GTM not loaded, zero egress)
|
||||
# deprecated, use UI_GOOGLE_TAG_MANAGER_ID
|
||||
NEXT_PUBLIC_GOOGLE_TAG_MANAGER_ID=""
|
||||
UI_GOOGLE_TAG_MANAGER_ID=""
|
||||
|
||||
#### MCP Server ####
|
||||
PROWLER_MCP_VERSION=stable
|
||||
@@ -139,13 +145,19 @@ DJANGO_BROKER_VISIBILITY_TIMEOUT=86400
|
||||
DJANGO_SENTRY_DSN=
|
||||
DJANGO_THROTTLE_TOKEN_OBTAIN=50/minute
|
||||
|
||||
# Sentry settings
|
||||
SENTRY_ENVIRONMENT=local
|
||||
# Sentry for the web app (server + browser). Empty/unset UI_SENTRY_DSN ⇒
|
||||
# Sentry disabled, zero egress. SENTRY_RELEASE (unprefixed) feeds the web app's
|
||||
# server/edge SDKs.
|
||||
UI_SENTRY_DSN=
|
||||
UI_SENTRY_ENVIRONMENT=local
|
||||
SENTRY_RELEASE=local
|
||||
NEXT_PUBLIC_SENTRY_ENVIRONMENT=${SENTRY_ENVIRONMENT}
|
||||
# Reserved runtime public config (registered now; no UI consumer yet)
|
||||
# POSTHOG_KEY=
|
||||
# POSTHOG_HOST=
|
||||
# REO_DEV_CLIENT_ID=
|
||||
|
||||
#### Prowler release version ####
|
||||
NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v5.27.0
|
||||
NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v5.32.0
|
||||
|
||||
# Social login credentials
|
||||
SOCIAL_GOOGLE_OAUTH_CALLBACK_URL="${AUTH_URL}/api/auth/callback/google"
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SDK
|
||||
/* @prowler-cloud/detection-remediation
|
||||
/prowler/ @prowler-cloud/detection-remediation
|
||||
/prowler/compliance/ @prowler-cloud/compliance
|
||||
/tests/ @prowler-cloud/detection-remediation
|
||||
/dashboard/ @prowler-cloud/detection-remediation
|
||||
/docs/ @prowler-cloud/detection-remediation
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
name: 'OSV-Scanner'
|
||||
description: 'Install osv-scanner and scan a lockfile, failing on HIGH/CRITICAL/UNKNOWN severity findings. Posts/updates a PR comment with findings on pull_request events (requires pull-requests: write).'
|
||||
description: 'Install osv-scanner and scan a lockfile, failing on CRITICAL severity findings. Posts/updates a PR comment with findings on pull_request events (requires pull-requests: write).'
|
||||
author: 'Prowler'
|
||||
|
||||
inputs:
|
||||
@@ -7,9 +7,9 @@ inputs:
|
||||
description: 'Path to the lockfile to scan, relative to the repository root (e.g. uv.lock, api/uv.lock, ui/pnpm-lock.yaml).'
|
||||
required: true
|
||||
severity-levels:
|
||||
description: 'Comma-separated severity levels that fail the scan. Default: HIGH,CRITICAL,UNKNOWN.'
|
||||
description: 'Comma-separated severity levels that fail the scan. Default: CRITICAL.'
|
||||
required: false
|
||||
default: 'HIGH,CRITICAL,UNKNOWN'
|
||||
default: 'CRITICAL'
|
||||
version:
|
||||
description: 'osv-scanner release tag to install. When overriding, you MUST also override binary-sha256.'
|
||||
required: false
|
||||
|
||||
@@ -43,8 +43,17 @@ runs:
|
||||
if: github.repository_owner == 'prowler-cloud' && github.repository != 'prowler-cloud/prowler'
|
||||
shell: bash
|
||||
working-directory: ${{ inputs.working-directory }}
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
LATEST_COMMIT=$(curl -s "https://api.github.com/repos/prowler-cloud/prowler/commits/master" | jq -r '.sha')
|
||||
LATEST_COMMIT=$(curl -sf --retry 3 --retry-all-errors --retry-delay 2 --retry-max-time 60 \
|
||||
-H "Authorization: Bearer ${GITHUB_TOKEN}" \
|
||||
-H "Accept: application/vnd.github+json" \
|
||||
"https://api.github.com/repos/prowler-cloud/prowler/commits/master" \
|
||||
| jq -er '.sha') || {
|
||||
echo "::error::Failed to fetch latest prowler/master commit from the GitHub API (HTTP error or missing .sha). Check the GITHUB_TOKEN and API rate limits."
|
||||
exit 1
|
||||
}
|
||||
echo "Latest commit hash: $LATEST_COMMIT"
|
||||
sed -i "s|\(git = \"https://github\.com/prowler-cloud/prowler\.git?rev=master\)#[a-f0-9]\{40\}\"|\1#${LATEST_COMMIT}\"|g" uv.lock
|
||||
echo "Updated uv.lock entry:"
|
||||
@@ -54,8 +63,17 @@ runs:
|
||||
if: github.event_name == 'push' && github.ref == 'refs/heads/master' && github.repository == 'prowler-cloud/prowler'
|
||||
shell: bash
|
||||
working-directory: ${{ inputs.working-directory }}
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
LATEST_COMMIT=$(curl -s "https://api.github.com/repos/prowler-cloud/prowler/commits/master" | jq -r '.sha')
|
||||
LATEST_COMMIT=$(curl -sf --retry 3 --retry-all-errors --retry-delay 2 --retry-max-time 60 \
|
||||
-H "Authorization: Bearer ${GITHUB_TOKEN}" \
|
||||
-H "Accept: application/vnd.github+json" \
|
||||
"https://api.github.com/repos/prowler-cloud/prowler/commits/master" \
|
||||
| jq -er '.sha') || {
|
||||
echo "::error::Failed to fetch latest prowler/master commit from the GitHub API (HTTP error or missing .sha). Check the GITHUB_TOKEN and API rate limits."
|
||||
exit 1
|
||||
}
|
||||
echo "Latest commit hash: $LATEST_COMMIT"
|
||||
sed -i "s|\(git = \"https://github\.com/prowler-cloud/prowler\.git?rev=master\)#[a-f0-9]\{40\}\"|\1#${LATEST_COMMIT}\"|g" uv.lock
|
||||
echo "Updated uv.lock entry:"
|
||||
|
||||
@@ -63,7 +63,7 @@ runs:
|
||||
exit-code: '0'
|
||||
scanners: 'vuln'
|
||||
timeout: '5m'
|
||||
version: 'v0.69.2'
|
||||
version: 'v0.71.2'
|
||||
|
||||
- name: Run Trivy vulnerability scan (SARIF)
|
||||
if: inputs.upload-sarif == 'true' && github.event_name == 'push'
|
||||
@@ -76,7 +76,7 @@ runs:
|
||||
exit-code: '0'
|
||||
scanners: 'vuln'
|
||||
timeout: '5m'
|
||||
version: 'v0.69.2'
|
||||
version: 'v0.71.2'
|
||||
|
||||
- name: Upload Trivy results to GitHub Security tab
|
||||
if: inputs.upload-sarif == 'true' && github.event_name == 'push'
|
||||
|
||||
+46
-44
@@ -6,17 +6,17 @@
|
||||
version: 2
|
||||
updates:
|
||||
# v5
|
||||
- package-ecosystem: "pip"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
open-pull-requests-limit: 25
|
||||
target-branch: master
|
||||
labels:
|
||||
- "dependencies"
|
||||
- "pip"
|
||||
cooldown:
|
||||
default-days: 7
|
||||
# - package-ecosystem: "pip"
|
||||
# directory: "/"
|
||||
# schedule:
|
||||
# interval: "monthly"
|
||||
# open-pull-requests-limit: 25
|
||||
# target-branch: master
|
||||
# labels:
|
||||
# - "dependencies"
|
||||
# - "pip"
|
||||
# cooldown:
|
||||
# default-days: 7
|
||||
|
||||
# Dependabot Updates are temporary disabled - 2025/03/19
|
||||
# - package-ecosystem: "pip"
|
||||
@@ -30,17 +30,18 @@ updates:
|
||||
# - "pip"
|
||||
# - "component/api"
|
||||
|
||||
- package-ecosystem: "github-actions"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
open-pull-requests-limit: 25
|
||||
target-branch: master
|
||||
labels:
|
||||
- "dependencies"
|
||||
- "github_actions"
|
||||
cooldown:
|
||||
default-days: 7
|
||||
# Dependabot version updates disabled - migrated to Renovate - 2026/07/02
|
||||
# - package-ecosystem: "github-actions"
|
||||
# directory: "/"
|
||||
# schedule:
|
||||
# interval: "monthly"
|
||||
# open-pull-requests-limit: 25
|
||||
# target-branch: master
|
||||
# labels:
|
||||
# - "dependencies"
|
||||
# - "github_actions"
|
||||
# cooldown:
|
||||
# default-days: 7
|
||||
|
||||
# Dependabot Updates are temporary disabled - 2025/03/19
|
||||
# - package-ecosystem: "npm"
|
||||
@@ -54,29 +55,30 @@ updates:
|
||||
# - "npm"
|
||||
# - "component/ui"
|
||||
|
||||
- package-ecosystem: "docker"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
open-pull-requests-limit: 25
|
||||
target-branch: master
|
||||
labels:
|
||||
- "dependencies"
|
||||
- "docker"
|
||||
cooldown:
|
||||
default-days: 7
|
||||
# Dependabot version updates disabled - migrated to Renovate - 2026/07/02
|
||||
# - package-ecosystem: "docker"
|
||||
# directory: "/"
|
||||
# schedule:
|
||||
# interval: "monthly"
|
||||
# open-pull-requests-limit: 25
|
||||
# target-branch: master
|
||||
# labels:
|
||||
# - "dependencies"
|
||||
# - "docker"
|
||||
# cooldown:
|
||||
# default-days: 7
|
||||
|
||||
- package-ecosystem: "pre-commit"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
open-pull-requests-limit: 25
|
||||
target-branch: master
|
||||
labels:
|
||||
- "dependencies"
|
||||
- "pre-commit"
|
||||
cooldown:
|
||||
default-days: 7
|
||||
# - package-ecosystem: "pre-commit"
|
||||
# directory: "/"
|
||||
# schedule:
|
||||
# interval: "monthly"
|
||||
# open-pull-requests-limit: 25
|
||||
# target-branch: master
|
||||
# labels:
|
||||
# - "dependencies"
|
||||
# - "pre-commit"
|
||||
# cooldown:
|
||||
# default-days: 7
|
||||
|
||||
# Dependabot Updates are temporary disabled - 2025/04/15
|
||||
# v4.6
|
||||
|
||||
@@ -77,6 +77,11 @@ provider/okta:
|
||||
- any-glob-to-any-file: "prowler/providers/okta/**"
|
||||
- any-glob-to-any-file: "tests/providers/okta/**"
|
||||
|
||||
provider/linode:
|
||||
- changed-files:
|
||||
- any-glob-to-any-file: "prowler/providers/linode/**"
|
||||
- any-glob-to-any-file: "tests/providers/linode/**"
|
||||
|
||||
github_actions:
|
||||
- changed-files:
|
||||
- any-glob-to-any-file: ".github/workflows/*"
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
{
|
||||
"$schema": "https://docs.renovatebot.com/renovate-schema.json",
|
||||
"extends": [
|
||||
"config:best-practices",
|
||||
":enablePreCommit",
|
||||
":semanticCommits",
|
||||
":enableVulnerabilityAlertsWithLabel(security)",
|
||||
"docker:enableMajor",
|
||||
"helpers:pinGitHubActionDigestsToSemver",
|
||||
"helpers:disableTypesNodeMajor",
|
||||
"security:openssf-scorecard",
|
||||
"customManagers:githubActionsVersions",
|
||||
"customManagers:dockerfileVersions"
|
||||
],
|
||||
"timezone": "Europe/Madrid",
|
||||
"baseBranchPatterns": [
|
||||
"master"
|
||||
],
|
||||
"labels": [
|
||||
"dependencies"
|
||||
],
|
||||
"dependencyDashboardTitle": "Dependency Dashboard",
|
||||
"prConcurrentLimit": 20,
|
||||
"prHourlyLimit": 10,
|
||||
"vulnerabilityAlerts": {
|
||||
"prHourlyLimit": 0,
|
||||
"prConcurrentLimit": 0
|
||||
},
|
||||
"configMigration": true,
|
||||
"minimumReleaseAge": "7 days",
|
||||
"rangeStrategy": "pin",
|
||||
"packageRules": [
|
||||
{
|
||||
"description": "Patches: 1st of every month, Madrid overnight window (22:00-06:00)",
|
||||
"matchUpdateTypes": [
|
||||
"patch"
|
||||
],
|
||||
"schedule": [
|
||||
"* 22-23,0-5 1 * *"
|
||||
],
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"description": "Minors: 8th of every 3 months, Madrid overnight window (22:00-06:00)",
|
||||
"matchUpdateTypes": [
|
||||
"minor"
|
||||
],
|
||||
"schedule": [
|
||||
"* 22-23,0-5 8 */3 *"
|
||||
],
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"description": "Majors: 15th of every 3 months, Madrid overnight window",
|
||||
"matchUpdateTypes": [
|
||||
"major"
|
||||
],
|
||||
"schedule": [
|
||||
"* 22-23,0-5 15 */3 *"
|
||||
],
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"description": "GitHub Actions - single grouped PR, no changelog, scope=ci",
|
||||
"matchManagers": [
|
||||
"github-actions"
|
||||
],
|
||||
"groupName": "github-actions",
|
||||
"semanticCommitScope": "ci",
|
||||
"addLabels": [
|
||||
"no-changelog"
|
||||
]
|
||||
},
|
||||
{
|
||||
"description": "Docker images - single grouped PR, no changelog, scope=docker",
|
||||
"matchManagers": [
|
||||
"dockerfile",
|
||||
"docker-compose"
|
||||
],
|
||||
"groupName": "docker",
|
||||
"semanticCommitScope": "docker",
|
||||
"addLabels": [
|
||||
"no-changelog"
|
||||
]
|
||||
},
|
||||
{
|
||||
"description": "Pre-commit hooks - single grouped PR, scope=pre-commit",
|
||||
"matchManagers": [
|
||||
"pre-commit"
|
||||
],
|
||||
"groupName": "pre-commit hooks",
|
||||
"semanticCommitScope": "pre-commit",
|
||||
"addLabels": [
|
||||
"no-changelog"
|
||||
]
|
||||
},
|
||||
{
|
||||
"description": "UI - scope=ui",
|
||||
"matchFileNames": [
|
||||
"ui/**"
|
||||
],
|
||||
"semanticCommitScope": "ui"
|
||||
},
|
||||
{
|
||||
"description": "API - scope=api",
|
||||
"matchFileNames": [
|
||||
"api/**"
|
||||
],
|
||||
"semanticCommitScope": "api"
|
||||
},
|
||||
{
|
||||
"description": "MCP server - scope=mcp",
|
||||
"matchFileNames": [
|
||||
"mcp_server/**"
|
||||
],
|
||||
"semanticCommitScope": "mcp"
|
||||
},
|
||||
{
|
||||
"description": "Python SDK (root) - scope=sdk",
|
||||
"matchFileNames": [
|
||||
"pyproject.toml",
|
||||
"poetry.lock",
|
||||
"util/prowler-bulk-provisioning/**"
|
||||
],
|
||||
"semanticCommitScope": "sdk"
|
||||
},
|
||||
{
|
||||
"description": "UI devDependencies - no changelog",
|
||||
"matchFileNames": [
|
||||
"ui/**"
|
||||
],
|
||||
"matchDepTypes": [
|
||||
"devDependencies"
|
||||
],
|
||||
"addLabels": [
|
||||
"no-changelog"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -6,8 +6,7 @@
|
||||
# - .github/workflows/api-security.yml, sdk-security.yml, ui-security.yml
|
||||
#
|
||||
# Severity levels (comma-separated) are read from OSV_SEVERITY_LEVELS.
|
||||
# Default: HIGH,CRITICAL,UNKNOWN — preserves prior .safety-policy.yml policy
|
||||
# (ignore-cvss-severity-below: 7 + ignore-cvss-unknown-severity: False).
|
||||
# Default: CRITICAL — only CVSS >= 9.0 findings fail the scan.
|
||||
# osv-scanner has no native CVSS threshold (google/osv-scanner#1400, closed
|
||||
# not-planned). Severity is derived from $group.max_severity (numeric CVSS
|
||||
# score string) which osv-scanner emits per group.
|
||||
@@ -33,7 +32,7 @@ set -euo pipefail
|
||||
|
||||
ROOT="$(git rev-parse --show-toplevel)"
|
||||
CONFIG="${ROOT}/osv-scanner.toml"
|
||||
SEVERITY_LEVELS="${OSV_SEVERITY_LEVELS:-HIGH,CRITICAL,UNKNOWN}"
|
||||
SEVERITY_LEVELS="${OSV_SEVERITY_LEVELS:-CRITICAL}"
|
||||
|
||||
for bin in osv-scanner jq; do
|
||||
if ! command -v "${bin}" >/dev/null 2>&1; then
|
||||
|
||||
@@ -35,7 +35,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -53,7 +53,7 @@ jobs:
|
||||
|
||||
- name: Check for API changes
|
||||
id: check-changes
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
api/**
|
||||
|
||||
@@ -44,7 +44,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -61,12 +61,12 @@ jobs:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@89a39a4e59826350b863aa6b6252a07ad50cf83e # v4.32.4
|
||||
uses: github/codeql-action/init@68bde559dea0fdcac2102bfdf6230c5f70eb485e # v4.35.4
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
config-file: ./.github/codeql/api-codeql-config.yml
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@89a39a4e59826350b863aa6b6252a07ad50cf83e # v4.32.4
|
||||
uses: github/codeql-action/analyze@68bde559dea0fdcac2102bfdf6230c5f70eb485e # v4.35.4
|
||||
with:
|
||||
category: '/language:${{ matrix.language }}'
|
||||
|
||||
@@ -46,7 +46,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
|
||||
@@ -65,7 +65,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -108,7 +108,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -122,6 +122,7 @@ jobs:
|
||||
github.com:443
|
||||
powershellinfraartifacts-gkhedzdeaghdezhr.z01.azurefd.net:443
|
||||
production.cloudflare.docker.com:443
|
||||
production.cloudfront.docker.com:443
|
||||
pypi.org:443
|
||||
registry-1.docker.io:443
|
||||
release-assets.githubusercontent.com:443
|
||||
@@ -132,14 +133,18 @@ jobs:
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Pin prowler SDK to latest master commit
|
||||
if: github.event_name == 'push'
|
||||
- name: Refresh prowler SDK pin to current branch tip
|
||||
run: |
|
||||
LATEST_SHA=$(git ls-remote https://github.com/prowler-cloud/prowler.git refs/heads/master | cut -f1)
|
||||
sed -i "s|prowler-cloud/prowler.git@master|prowler-cloud/prowler.git@${LATEST_SHA}|" api/pyproject.toml
|
||||
# api/pyproject.toml has `@master` on master and `@v5.X` on release
|
||||
# branches (set by prepare-release.yml). uv lock --upgrade-package
|
||||
# re-resolves whichever ref is present against the current branch tip
|
||||
# and writes the SHA into api/uv.lock. The Dockerfile runs
|
||||
# `uv sync --locked`, which is what actually drives the install.
|
||||
pip install --no-cache-dir "uv==0.11.14"
|
||||
(cd api && uv lock --upgrade-package prowler)
|
||||
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
@@ -150,7 +155,7 @@ jobs:
|
||||
- name: Build and push API container for ${{ matrix.arch }}
|
||||
id: container-push
|
||||
if: github.event_name == 'push' || github.event_name == 'release' || github.event_name == 'workflow_dispatch'
|
||||
uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
with:
|
||||
context: ${{ env.WORKING_DIRECTORY }}
|
||||
push: true
|
||||
@@ -170,7 +175,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -179,8 +184,9 @@ jobs:
|
||||
registry-1.docker.io:443
|
||||
auth.docker.io:443
|
||||
production.cloudflare.docker.com:443
|
||||
production.cloudfront.docker.com:443
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
@@ -209,7 +215,7 @@ jobs:
|
||||
|
||||
- name: Install regctl
|
||||
if: always()
|
||||
uses: regclient/actions/regctl-installer@da9319db8e44e8b062b3a147e1dfb2f574d41a03 # main
|
||||
uses: regclient/actions/regctl-installer@9a2d4216180dbb3e2dccfa60d2dd4afd98e42ec5 # main
|
||||
|
||||
- name: Cleanup intermediate architecture tags
|
||||
if: always()
|
||||
@@ -230,7 +236,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -266,27 +272,3 @@ jobs:
|
||||
payload-file-path: "./.github/scripts/slack-messages/container-release-completed.json"
|
||||
step-outcome: ${{ steps.outcome.outputs.outcome }}
|
||||
update-ts: ${{ needs.notify-release-started.outputs.message-ts }}
|
||||
|
||||
trigger-deployment:
|
||||
needs: [setup, container-build-push]
|
||||
if: always() && github.event_name == 'push' && needs.setup.result == 'success' && needs.container-build-push.result == 'success'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
api.github.com:443
|
||||
|
||||
- name: Trigger API deployment
|
||||
uses: peter-evans/repository-dispatch@28959ce8df70de7be546dd1250a005dd32156697 # v4.0.1
|
||||
with:
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
repository: ${{ secrets.CLOUD_DISPATCH }}
|
||||
event-type: api-prowler-deployment
|
||||
client-payload: '{"sha": "${{ github.sha }}", "short_sha": "${{ needs.setup.outputs.short-sha }}"}'
|
||||
|
||||
@@ -12,9 +12,6 @@ on:
|
||||
branches:
|
||||
- 'master'
|
||||
- 'v5.*'
|
||||
paths:
|
||||
- 'api/**'
|
||||
- '.github/workflows/api-container-checks.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
@@ -36,7 +33,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -50,7 +47,7 @@ jobs:
|
||||
|
||||
- name: Check if Dockerfile changed
|
||||
id: dockerfile-changed
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: api/Dockerfile
|
||||
|
||||
@@ -72,7 +69,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -83,6 +80,7 @@ jobs:
|
||||
registry-1.docker.io:443
|
||||
auth.docker.io:443
|
||||
production.cloudflare.docker.com:443
|
||||
production.cloudfront.docker.com:443
|
||||
debian.map.fastlydns.net:80
|
||||
release-assets.githubusercontent.com:443
|
||||
objects.githubusercontent.com:443
|
||||
@@ -103,7 +101,7 @@ jobs:
|
||||
|
||||
- name: Check for API changes
|
||||
id: check-changes
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: api/**
|
||||
files_ignore: |
|
||||
@@ -118,7 +116,7 @@ jobs:
|
||||
|
||||
- name: Build container
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
with:
|
||||
context: ${{ env.API_WORKING_DIR }}
|
||||
push: false
|
||||
@@ -133,5 +131,5 @@ jobs:
|
||||
with:
|
||||
image-name: ${{ env.IMAGE_NAME }}
|
||||
image-tag: ${{ github.sha }}
|
||||
fail-on-critical: 'false'
|
||||
fail-on-critical: 'true'
|
||||
severity: 'CRITICAL'
|
||||
|
||||
@@ -16,13 +16,6 @@ on:
|
||||
branches:
|
||||
- "master"
|
||||
- "v5.*"
|
||||
paths:
|
||||
- 'api/**'
|
||||
- '.github/workflows/api-tests.yml'
|
||||
- '.github/workflows/api-security.yml'
|
||||
- '.github/actions/setup-python-uv/**'
|
||||
- '.github/actions/osv-scanner/**'
|
||||
- '.github/scripts/osv-scan.sh'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
@@ -50,7 +43,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -73,7 +66,7 @@ jobs:
|
||||
|
||||
- name: Check for API changes
|
||||
id: check-changes
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
api/**
|
||||
|
||||
@@ -48,7 +48,7 @@ jobs:
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:17@sha256:2cd82735a36356842d5eb1ef80db3ae8f1154172f0f653db48fde079b2a0b7f7
|
||||
image: postgres:17@sha256:5c855ad7b85e68e48a62f34662853f38b57c1c1d80f3a927ab58034fd6d31c5e
|
||||
env:
|
||||
POSTGRES_HOST: ${{ env.POSTGRES_HOST }}
|
||||
POSTGRES_PORT: ${{ env.POSTGRES_PORT }}
|
||||
@@ -63,7 +63,7 @@ jobs:
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
valkey:
|
||||
image: valkey/valkey:7-alpine3.19
|
||||
image: valkey/valkey:7-alpine3.19@sha256:4054fe7fc607b9326ac7c4691ed26e9670d2ff17a9fb28c2577adecf928acbcc
|
||||
env:
|
||||
VALKEY_HOST: ${{ env.VALKEY_HOST }}
|
||||
VALKEY_PORT: ${{ env.VALKEY_PORT }}
|
||||
@@ -78,7 +78,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -102,7 +102,7 @@ jobs:
|
||||
|
||||
- name: Check for API changes
|
||||
id: check-changes
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
api/**
|
||||
|
||||
@@ -30,7 +30,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
|
||||
@@ -29,7 +29,7 @@ jobs:
|
||||
patch_version: ${{ steps.detect.outputs.patch_version }}
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -75,7 +75,7 @@ jobs:
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -139,6 +139,17 @@ jobs:
|
||||
sed -i "s|version = \"${CURRENT_API_VERSION}\"|version = \"${NEXT_API_VERSION}\"|" api/pyproject.toml
|
||||
sed -i "s| version: ${CURRENT_API_VERSION}| version: ${NEXT_API_VERSION}|" api/src/backend/api/specs/v1.yaml
|
||||
|
||||
- name: Regenerate lockfiles after version bump
|
||||
run: |
|
||||
set -e
|
||||
# The bumps above edit pyproject.toml / api/pyproject.toml but leave
|
||||
# uv.lock / api/uv.lock stale, which makes `uv sync --locked` fail in
|
||||
# the container builds. Refresh both with the uv version the images
|
||||
# pin (plain `uv lock`, no --upgrade: only the version line changes).
|
||||
pip install --no-cache-dir "uv==0.11.14"
|
||||
uv lock
|
||||
(cd api && uv lock)
|
||||
|
||||
- name: Bump UI version (.env)
|
||||
run: |
|
||||
set -e
|
||||
@@ -155,7 +166,7 @@ jobs:
|
||||
run: git --no-pager diff
|
||||
|
||||
- name: Create PR for next versions to master
|
||||
uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0 # v8.1.0
|
||||
uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
|
||||
with:
|
||||
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
@@ -191,7 +202,7 @@ jobs:
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -240,6 +251,17 @@ jobs:
|
||||
sed -i "s|version = \"${CURRENT_API_VERSION}\"|version = \"${FIRST_API_PATCH_VERSION}\"|" api/pyproject.toml
|
||||
sed -i "s| version: ${CURRENT_API_VERSION}| version: ${FIRST_API_PATCH_VERSION}|" api/src/backend/api/specs/v1.yaml
|
||||
|
||||
- name: Regenerate lockfiles after version bump
|
||||
run: |
|
||||
set -e
|
||||
# The bumps above edit pyproject.toml / api/pyproject.toml but leave
|
||||
# uv.lock / api/uv.lock stale, which makes `uv sync --locked` fail in
|
||||
# the container builds. Refresh both with the uv version the images
|
||||
# pin (plain `uv lock`, no --upgrade: only the version line changes).
|
||||
pip install --no-cache-dir "uv==0.11.14"
|
||||
uv lock
|
||||
(cd api && uv lock)
|
||||
|
||||
- name: Bump UI version (.env)
|
||||
run: |
|
||||
set -e
|
||||
@@ -249,7 +271,7 @@ jobs:
|
||||
run: git --no-pager diff
|
||||
|
||||
- name: Create PR for first patch versions to version branch
|
||||
uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0 # v8.1.0
|
||||
uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
|
||||
with:
|
||||
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
@@ -285,7 +307,7 @@ jobs:
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -341,6 +363,17 @@ jobs:
|
||||
sed -i "s|version = \"${CURRENT_API_VERSION}\"|version = \"${NEXT_API_PATCH_VERSION}\"|" api/pyproject.toml
|
||||
sed -i "s| version: ${CURRENT_API_VERSION}| version: ${NEXT_API_PATCH_VERSION}|" api/src/backend/api/specs/v1.yaml
|
||||
|
||||
- name: Regenerate lockfiles after version bump
|
||||
run: |
|
||||
set -e
|
||||
# The bumps above edit pyproject.toml / api/pyproject.toml but leave
|
||||
# uv.lock / api/uv.lock stale, which makes `uv sync --locked` fail in
|
||||
# the container builds. Refresh both with the uv version the images
|
||||
# pin (plain `uv lock`, no --upgrade: only the version line changes).
|
||||
pip install --no-cache-dir "uv==0.11.14"
|
||||
uv lock
|
||||
(cd api && uv lock)
|
||||
|
||||
- name: Bump UI version (.env)
|
||||
run: |
|
||||
set -e
|
||||
@@ -350,7 +383,7 @@ jobs:
|
||||
run: git --no-pager diff
|
||||
|
||||
- name: Create PR for next patch versions to version branch
|
||||
uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0 # v8.1.0
|
||||
uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
|
||||
with:
|
||||
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
|
||||
@@ -36,7 +36,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -51,6 +51,6 @@ jobs:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Run zizmor
|
||||
uses: zizmorcore/zizmor-action@71321a20a9ded102f6e9ce5718a2fcec2c4f70d8 # v0.5.2
|
||||
uses: zizmorcore/zizmor-action@a16621b09c6db4281f81a93cb393b05dcd7b7165 # v0.5.5
|
||||
with:
|
||||
token: ${{ github.token }}
|
||||
|
||||
@@ -22,7 +22,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -26,7 +26,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -25,7 +25,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -25,14 +25,15 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
# We can't block as Trufflehog needs to verify secrets against vendors
|
||||
egress-policy: audit
|
||||
# allowed-endpoints: >
|
||||
# github.com:443
|
||||
# ghcr.io:443
|
||||
# pkg-containers.githubusercontent.com:443
|
||||
allowed-endpoints: >
|
||||
github.com:443
|
||||
ghcr.io:443
|
||||
pkg-containers.githubusercontent.com:443
|
||||
www.formbucket.com:443
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
@@ -44,6 +45,6 @@ jobs:
|
||||
|
||||
- name: Scan diff for secrets with TruffleHog
|
||||
# Action auto-injects --since-commit/--branch from event payload; passing them in extra_args produces duplicate flags.
|
||||
uses: trufflesecurity/trufflehog@ef6e76c3c4023279497fab4721ffa071a722fd05 # v3.92.4
|
||||
uses: trufflesecurity/trufflehog@37b77001d0174ebec2fcca2bd83ff83a6d45a3ab # v3.95.3
|
||||
with:
|
||||
extra_args: --results=verified,unknown
|
||||
|
||||
@@ -33,7 +33,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -26,7 +26,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -22,7 +22,7 @@ jobs:
|
||||
issues: write
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
|
||||
Generated
+12
-12
@@ -66,12 +66,12 @@ jobs:
|
||||
title: ${{ steps.compute-text.outputs.title }}
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Setup Scripts
|
||||
uses: github/gh-aw/actions/setup@9382be3ca9ac18917e111a99d4e6bbff58d0dccc # v0.43.23
|
||||
uses: github/gh-aw/actions/setup@4d44d0e89851a877f4ddc0cb6c0197e42b1016c5 # v0.73.0
|
||||
with:
|
||||
destination: /opt/gh-aw/actions
|
||||
- name: Check workflow file timestamps
|
||||
@@ -135,12 +135,12 @@ jobs:
|
||||
secret_verification_result: ${{ steps.validate-secret.outputs.verification_result }}
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Setup Scripts
|
||||
uses: github/gh-aw/actions/setup@9382be3ca9ac18917e111a99d4e6bbff58d0dccc # v0.43.23
|
||||
uses: github/gh-aw/actions/setup@4d44d0e89851a877f4ddc0cb6c0197e42b1016c5 # v0.73.0
|
||||
with:
|
||||
destination: /opt/gh-aw/actions
|
||||
- name: Checkout repository
|
||||
@@ -870,12 +870,12 @@ jobs:
|
||||
total_count: ${{ steps.missing_tool.outputs.total_count }}
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Setup Scripts
|
||||
uses: github/gh-aw/actions/setup@9382be3ca9ac18917e111a99d4e6bbff58d0dccc # v0.43.23
|
||||
uses: github/gh-aw/actions/setup@4d44d0e89851a877f4ddc0cb6c0197e42b1016c5 # v0.73.0
|
||||
with:
|
||||
destination: /opt/gh-aw/actions
|
||||
- name: Download agent output artifact
|
||||
@@ -982,12 +982,12 @@ jobs:
|
||||
success: ${{ steps.parse_results.outputs.success }}
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Setup Scripts
|
||||
uses: github/gh-aw/actions/setup@9382be3ca9ac18917e111a99d4e6bbff58d0dccc # v0.43.23
|
||||
uses: github/gh-aw/actions/setup@4d44d0e89851a877f4ddc0cb6c0197e42b1016c5 # v0.73.0
|
||||
with:
|
||||
destination: /opt/gh-aw/actions
|
||||
- name: Download agent artifacts
|
||||
@@ -1091,12 +1091,12 @@ jobs:
|
||||
activated: ${{ (steps.check_membership.outputs.is_team_member == 'true') && (steps.check_rate_limit.outputs.rate_limit_ok == 'true') }}
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Setup Scripts
|
||||
uses: github/gh-aw/actions/setup@9382be3ca9ac18917e111a99d4e6bbff58d0dccc # v0.43.23
|
||||
uses: github/gh-aw/actions/setup@4d44d0e89851a877f4ddc0cb6c0197e42b1016c5 # v0.73.0
|
||||
with:
|
||||
destination: /opt/gh-aw/actions
|
||||
- name: Add eyes reaction for immediate feedback
|
||||
@@ -1164,12 +1164,12 @@ jobs:
|
||||
process_safe_outputs_temporary_id_map: ${{ steps.process_safe_outputs.outputs.temporary_id_map }}
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Setup Scripts
|
||||
uses: github/gh-aw/actions/setup@9382be3ca9ac18917e111a99d4e6bbff58d0dccc # v0.43.23
|
||||
uses: github/gh-aw/actions/setup@4d44d0e89851a877f4ddc0cb6c0197e42b1016c5 # v0.73.0
|
||||
with:
|
||||
destination: /opt/gh-aw/actions
|
||||
- name: Download agent output artifact
|
||||
|
||||
@@ -27,12 +27,12 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Apply labels to PR
|
||||
uses: actions/labeler@634933edcd8ababfe52f92936142cc22ac488b1b # v6.0.1
|
||||
uses: actions/labeler@f27b608878404679385c85cfa523b85ccb86e213 # v6.1.0
|
||||
with:
|
||||
sync-labels: true
|
||||
|
||||
@@ -46,7 +46,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
name: 'Docs: Markdown Lint'
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- 'master'
|
||||
- 'v5.*'
|
||||
pull_request:
|
||||
branches:
|
||||
- 'master'
|
||||
- 'v5.*'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
markdown-lint:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
api.github.com:443
|
||||
github.com:443
|
||||
registry.npmjs.org:443
|
||||
release-assets.githubusercontent.com:443
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
|
||||
with:
|
||||
node-version-file: ui/.nvmrc
|
||||
|
||||
- name: Setup pnpm
|
||||
uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0
|
||||
with:
|
||||
package_json_file: ui/package.json
|
||||
run_install: false
|
||||
|
||||
- name: Run markdownlint
|
||||
# Pin must match .pre-commit-config.yaml so prek and CI behave identically.
|
||||
# pnpm dlx doesn't accept --ignore-scripts as a flag; the env var
|
||||
# disables postinstall scripts on transitives the same way.
|
||||
env:
|
||||
pnpm_config_ignore_scripts: 'true'
|
||||
run: pnpm dlx markdownlint-cli@0.45.0 '**/*.md'
|
||||
@@ -45,7 +45,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
|
||||
@@ -64,7 +64,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -106,7 +106,7 @@ jobs:
|
||||
packages: write
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -114,6 +114,7 @@ jobs:
|
||||
registry-1.docker.io:443
|
||||
auth.docker.io:443
|
||||
production.cloudflare.docker.com:443
|
||||
production.cloudfront.docker.com:443
|
||||
ghcr.io:443
|
||||
pkg-containers.githubusercontent.com:443
|
||||
files.pythonhosted.org:443
|
||||
@@ -125,7 +126,7 @@ jobs:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
@@ -136,7 +137,7 @@ jobs:
|
||||
- name: Build and push MCP container for ${{ matrix.arch }}
|
||||
id: container-push
|
||||
if: github.event_name == 'push' || github.event_name == 'release' || github.event_name == 'workflow_dispatch'
|
||||
uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
with:
|
||||
context: ${{ env.WORKING_DIRECTORY }}
|
||||
push: true
|
||||
@@ -164,18 +165,19 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
registry-1.docker.io:443
|
||||
auth.docker.io:443
|
||||
production.cloudflare.docker.com:443
|
||||
production.cloudfront.docker.com:443
|
||||
github.com:443
|
||||
release-assets.githubusercontent.com:443
|
||||
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
@@ -204,7 +206,7 @@ jobs:
|
||||
|
||||
- name: Install regctl
|
||||
if: always()
|
||||
uses: regclient/actions/regctl-installer@da9319db8e44e8b062b3a147e1dfb2f574d41a03 # main
|
||||
uses: regclient/actions/regctl-installer@9a2d4216180dbb3e2dccfa60d2dd4afd98e42ec5 # main
|
||||
|
||||
- name: Cleanup intermediate architecture tags
|
||||
if: always()
|
||||
@@ -225,7 +227,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -261,27 +263,3 @@ jobs:
|
||||
payload-file-path: "./.github/scripts/slack-messages/container-release-completed.json"
|
||||
step-outcome: ${{ steps.outcome.outputs.outcome }}
|
||||
update-ts: ${{ needs.notify-release-started.outputs.message-ts }}
|
||||
|
||||
trigger-deployment:
|
||||
needs: [setup, container-build-push]
|
||||
if: always() && github.event_name == 'push' && needs.setup.result == 'success' && needs.container-build-push.result == 'success'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
api.github.com:443
|
||||
|
||||
- name: Trigger MCP deployment
|
||||
uses: peter-evans/repository-dispatch@28959ce8df70de7be546dd1250a005dd32156697 # v4.0.1
|
||||
with:
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
repository: ${{ secrets.CLOUD_DISPATCH }}
|
||||
event-type: mcp-prowler-deployment
|
||||
client-payload: '{"sha": "${{ github.sha }}", "short_sha": "${{ needs.setup.outputs.short-sha }}"}'
|
||||
|
||||
@@ -12,9 +12,6 @@ on:
|
||||
branches:
|
||||
- 'master'
|
||||
- 'v5.*'
|
||||
paths:
|
||||
- 'mcp_server/**'
|
||||
- '.github/workflows/mcp-container-checks.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
@@ -36,7 +33,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -50,7 +47,7 @@ jobs:
|
||||
|
||||
- name: Check if Dockerfile changed
|
||||
id: dockerfile-changed
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: mcp_server/Dockerfile
|
||||
|
||||
@@ -71,7 +68,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -99,7 +96,7 @@ jobs:
|
||||
|
||||
- name: Check for MCP changes
|
||||
id: check-changes
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: mcp_server/**
|
||||
files_ignore: |
|
||||
@@ -112,7 +109,7 @@ jobs:
|
||||
|
||||
- name: Build MCP container
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
with:
|
||||
context: ${{ env.MCP_WORKING_DIR }}
|
||||
push: false
|
||||
@@ -127,5 +124,5 @@ jobs:
|
||||
with:
|
||||
image-name: ${{ env.IMAGE_NAME }}
|
||||
image-tag: ${{ github.sha }}
|
||||
fail-on-critical: 'false'
|
||||
fail-on-critical: 'true'
|
||||
severity: 'CRITICAL'
|
||||
|
||||
@@ -29,7 +29,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -67,7 +67,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -113,7 +113,7 @@ jobs:
|
||||
|
||||
- name: Publish prowler-mcp package to PyPI
|
||||
if: steps.pypi-check.outputs.skip != 'true'
|
||||
uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0
|
||||
uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0
|
||||
with:
|
||||
packages-dir: ${{ env.WORKING_DIRECTORY }}/dist/
|
||||
print-hash: true
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
name: 'MCP: Security'
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- 'master'
|
||||
- 'v5.*'
|
||||
paths:
|
||||
- 'mcp_server/pyproject.toml'
|
||||
- 'mcp_server/uv.lock'
|
||||
- '.github/workflows/mcp-security.yml'
|
||||
- '.github/actions/osv-scanner/**'
|
||||
- '.github/scripts/osv-scan.sh'
|
||||
pull_request:
|
||||
branches:
|
||||
- 'master'
|
||||
- 'v5.*'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
mcp-security-scans:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write # osv-scanner action posts/updates a PR comment with findings
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
github.com:443
|
||||
api.github.com:443
|
||||
objects.githubusercontent.com:443
|
||||
release-assets.githubusercontent.com:443
|
||||
api.osv.dev:443
|
||||
api.deps.dev:443
|
||||
osv-vulnerabilities.storage.googleapis.com:443
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
# zizmor: ignore[artipacked]
|
||||
persist-credentials: true # Required by tj-actions/changed-files to fetch PR branch
|
||||
|
||||
- name: Check for MCP dependency changes
|
||||
id: check-changes
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
mcp_server/pyproject.toml
|
||||
mcp_server/uv.lock
|
||||
.github/workflows/mcp-security.yml
|
||||
.github/actions/osv-scanner/**
|
||||
.github/scripts/osv-scan.sh
|
||||
|
||||
- name: Dependency vulnerability scan with osv-scanner
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: ./.github/actions/osv-scanner
|
||||
with:
|
||||
lockfile: mcp_server/uv.lock
|
||||
@@ -48,7 +48,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -61,7 +61,7 @@ jobs:
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
|
||||
- name: Build ${{ matrix.component }} container (linux/arm64)
|
||||
uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
with:
|
||||
context: ${{ matrix.context }}
|
||||
file: ${{ matrix.dockerfile }}
|
||||
@@ -83,7 +83,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -31,7 +31,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -52,7 +52,7 @@ jobs:
|
||||
|
||||
- name: Get changed files
|
||||
id: changed-files
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
api/**
|
||||
|
||||
@@ -35,7 +35,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -56,7 +56,7 @@ jobs:
|
||||
|
||||
- name: Get changed files
|
||||
id: changed-files
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
prowler/providers/**/services/**/*.metadata.json
|
||||
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -37,8 +37,7 @@ jobs:
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
fetch-depth: 1
|
||||
# zizmor: ignore[artipacked]
|
||||
persist-credentials: true # Required by tj-actions/changed-files to fetch PR branch
|
||||
persist-credentials: false # No write token in the untrusted PR-head tree; public repo so base fetch/changed-files work unauthenticated
|
||||
|
||||
- name: Fetch PR base ref for tj-actions/changed-files
|
||||
env:
|
||||
@@ -47,9 +46,11 @@ jobs:
|
||||
|
||||
- name: Get changed files
|
||||
id: changed-files
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: '**'
|
||||
safe_output: false # Raw paths (list read via env var, injection-safe); default escaping backslash-quotes chars like () and breaks the -f test
|
||||
separator: "\n" # Newline-delimited so the reader tolerates spaces and glob chars in paths
|
||||
|
||||
- name: Check for conflict markers
|
||||
id: conflict-check
|
||||
@@ -59,19 +60,18 @@ jobs:
|
||||
CONFLICT_FILES=""
|
||||
HAS_CONFLICTS=false
|
||||
|
||||
# Check each changed file for conflict markers
|
||||
for file in ${STEPS_CHANGED_FILES_OUTPUTS_ALL_CHANGED_FILES}; do
|
||||
if [ -f "$file" ]; then
|
||||
echo "Checking file: $file"
|
||||
# Read newline-delimited paths so spaces/globs neither word-split nor glob-expand
|
||||
while IFS= read -r file; do
|
||||
[ -n "$file" ] || continue
|
||||
[ -f "$file" ] || continue
|
||||
echo "Checking file: $file"
|
||||
|
||||
# Look for conflict markers (more precise regex)
|
||||
if grep -qE '^(<<<<<<<|=======|>>>>>>>)' "$file" 2>/dev/null; then
|
||||
echo "Conflict markers found in: $file"
|
||||
CONFLICT_FILES="${CONFLICT_FILES}- \`${file}\`"$'\n'
|
||||
HAS_CONFLICTS=true
|
||||
fi
|
||||
if grep -qE '^(<<<<<<<|=======|>>>>>>>)' "$file" 2>/dev/null; then
|
||||
echo "Conflict markers found in: $file"
|
||||
CONFLICT_FILES="${CONFLICT_FILES}- \`${file}\`"$'\n'
|
||||
HAS_CONFLICTS=true
|
||||
fi
|
||||
done
|
||||
done <<< "$STEPS_CHANGED_FILES_OUTPUTS_ALL_CHANGED_FILES"
|
||||
|
||||
if [ "$HAS_CONFLICTS" = true ]; then
|
||||
echo "has_conflicts=true" >> $GITHUB_OUTPUT
|
||||
@@ -88,18 +88,49 @@ jobs:
|
||||
env:
|
||||
STEPS_CHANGED_FILES_OUTPUTS_ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }}
|
||||
|
||||
- name: Check base-branch mergeability
|
||||
id: merge-check
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
REPO: ${{ github.repository }}
|
||||
run: |
|
||||
MERGEABLE=null
|
||||
|
||||
# GitHub computes mergeability async, so .mergeable is null until ready; poll until resolved
|
||||
for attempt in 1 2 3 4 5; do
|
||||
MERGEABLE=$(gh api "repos/${REPO}/pulls/${PR_NUMBER}" --jq '.mergeable')
|
||||
if [ "$MERGEABLE" != "null" ]; then
|
||||
break
|
||||
fi
|
||||
echo "Attempt ${attempt}: mergeability not computed yet, retrying..."
|
||||
sleep 3
|
||||
done
|
||||
|
||||
# Keep 'unknown' distinct from 'clean' so we never assert a clean merge we could not confirm
|
||||
case "$MERGEABLE" in
|
||||
false) STATUS=conflict; echo "PR branch cannot be merged cleanly into its base branch" ;;
|
||||
true) STATUS=clean; echo "PR branch merges cleanly into its base branch" ;;
|
||||
*) STATUS=unknown; echo "::warning::Mergeability did not resolve after retries; leaving it undetermined" ;;
|
||||
esac
|
||||
|
||||
echo "merge_status=${STATUS}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Manage conflict label
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
HAS_CONFLICTS: ${{ steps.conflict-check.outputs.has_conflicts }}
|
||||
MERGE_STATUS: ${{ steps.merge-check.outputs.merge_status }}
|
||||
run: |
|
||||
LABEL_NAME="has-conflicts"
|
||||
|
||||
# Add or remove label based on conflict status
|
||||
if [ "$HAS_CONFLICTS" = "true" ]; then
|
||||
if [ "$HAS_CONFLICTS" = "true" ] || [ "$MERGE_STATUS" = "conflict" ]; then
|
||||
echo "Adding conflict label to PR #${PR_NUMBER}..."
|
||||
gh pr edit "$PR_NUMBER" --add-label "$LABEL_NAME" --repo ${{ github.repository }} || true
|
||||
elif [ "$MERGE_STATUS" = "unknown" ]; then
|
||||
# Don't drop the label on an undetermined merge state; a later run will settle it
|
||||
echo "Mergeability undetermined; leaving label unchanged"
|
||||
else
|
||||
echo "Removing conflict label from PR #${PR_NUMBER}..."
|
||||
gh pr edit "$PR_NUMBER" --remove-label "$LABEL_NAME" --repo ${{ github.repository }} || true
|
||||
@@ -121,20 +152,25 @@ jobs:
|
||||
edit-mode: replace
|
||||
body: |
|
||||
<!-- conflict-checker-comment -->
|
||||
${{ steps.conflict-check.outputs.has_conflicts == 'true' && '⚠️ **Conflict Markers Detected**' || '✅ **Conflict Markers Resolved**' }}
|
||||
|
||||
${{ steps.conflict-check.outputs.has_conflicts == 'true' && format('This pull request contains unresolved conflict markers in the following files:
|
||||
${{ (steps.conflict-check.outputs.has_conflicts == 'true' || steps.merge-check.outputs.merge_status == 'conflict') && '⚠️ **Conflicts Detected**' || (steps.merge-check.outputs.merge_status == 'unknown' && 'ℹ️ **Conflict Check Incomplete**' || '✅ **No Conflicts**') }}
|
||||
${{ steps.conflict-check.outputs.has_conflicts == 'true' && format('
|
||||
**Conflict markers** are present in the following files:
|
||||
|
||||
{0}
|
||||
|
||||
Please resolve these conflicts by:
|
||||
1. Locating the conflict markers: `<<<<<<<`, `=======`, and `>>>>>>>`
|
||||
2. Manually editing the files to resolve the conflicts
|
||||
3. Removing all conflict markers
|
||||
4. Committing and pushing the changes', steps.conflict-check.outputs.conflict_files) || 'All conflict markers have been successfully resolved in this pull request.' }}
|
||||
Resolve them by removing every `<<<<<<<`, `=======`, and `>>>>>>>` marker, then commit and push.', steps.conflict-check.outputs.conflict_files) || '' }}
|
||||
${{ steps.merge-check.outputs.merge_status == 'conflict' && '
|
||||
**Merge conflict with the base branch.** This PR cannot be merged cleanly. Update your branch with the latest base (rebase or merge) and resolve the conflicts.' || '' }}
|
||||
${{ steps.merge-check.outputs.merge_status == 'unknown' && '
|
||||
GitHub had not finished computing mergeability, so base-branch conflict status could not be verified on this run.' || '' }}
|
||||
${{ (steps.conflict-check.outputs.has_conflicts != 'true' && steps.merge-check.outputs.merge_status == 'clean') && '
|
||||
No conflict markers, and the branch merges cleanly into its base.' || '' }}
|
||||
|
||||
- name: Fail workflow if conflicts detected
|
||||
if: steps.conflict-check.outputs.has_conflicts == 'true'
|
||||
if: steps.conflict-check.outputs.has_conflicts == 'true' || steps.merge-check.outputs.merge_status == 'conflict'
|
||||
env:
|
||||
HAS_CONFLICTS: ${{ steps.conflict-check.outputs.has_conflicts }}
|
||||
MERGE_STATUS: ${{ steps.merge-check.outputs.merge_status }}
|
||||
run: |
|
||||
echo "::error::Workflow failed due to conflict markers detected in the PR"
|
||||
[ "$HAS_CONFLICTS" = "true" ] && echo "::error::Conflict markers detected in changed files"
|
||||
[ "$MERGE_STATUS" = "conflict" ] && echo "::error::PR branch has merge conflicts with the base branch"
|
||||
exit 1
|
||||
|
||||
@@ -26,7 +26,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -56,6 +56,6 @@ jobs:
|
||||
"PROWLER_PR_BODY": ${{ toJson(github.event.pull_request.body) }},
|
||||
"PROWLER_PR_URL": ${{ toJson(github.event.pull_request.html_url) }},
|
||||
"PROWLER_PR_MERGED_BY": "${{ github.event.pull_request.merged_by.login }}",
|
||||
"PROWLER_PR_BASE_BRANCH": "${{ github.event.pull_request.base.ref }}",
|
||||
"PROWLER_PR_HEAD_BRANCH": "${{ github.event.pull_request.head.ref }}"
|
||||
"PROWLER_PR_BASE_BRANCH": ${{ toJson(github.event.pull_request.base.ref) }},
|
||||
"PROWLER_PR_HEAD_BRANCH": ${{ toJson(github.event.pull_request.head.ref) }}
|
||||
}
|
||||
|
||||
@@ -29,7 +29,7 @@ jobs:
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -338,7 +338,7 @@ jobs:
|
||||
|
||||
- name: Create PR for API dependency update
|
||||
if: ${{ env.PATCH_VERSION == '0' }}
|
||||
uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0 # v8.1.0
|
||||
uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
|
||||
with:
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
commit-message: 'chore(api): update prowler dependency to ${{ env.BRANCH_NAME }} for release ${{ env.PROWLER_VERSION }}'
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
name: 'CI: Renovate Config Validate'
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- 'master'
|
||||
paths:
|
||||
- '.github/renovate.json'
|
||||
- '.pre-commit-config.yaml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions: {}
|
||||
|
||||
env:
|
||||
# renovate: datasource=pypi depName=prek
|
||||
PREK_VERSION: '0.4.0'
|
||||
|
||||
jobs:
|
||||
validate:
|
||||
name: Validate Renovate config
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
api.github.com:443
|
||||
github.com:443
|
||||
objects.githubusercontent.com:443
|
||||
codeload.github.com:443
|
||||
release-assets.githubusercontent.com:443
|
||||
pypi.org:443
|
||||
files.pythonhosted.org:443
|
||||
registry.npmjs.org:443
|
||||
nodejs.org:443
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up uv
|
||||
uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098 # v7.3.1
|
||||
|
||||
- name: Install prek
|
||||
run: uv tool install "prek==${PREK_VERSION}"
|
||||
|
||||
- name: Validate Renovate config
|
||||
run: prek run renovate-config-validator --files .github/renovate.json
|
||||
@@ -25,7 +25,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
|
||||
@@ -29,10 +29,11 @@ jobs:
|
||||
- '3.10'
|
||||
- '3.11'
|
||||
- '3.12'
|
||||
- '3.13'
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -48,7 +49,7 @@ jobs:
|
||||
|
||||
- name: Check for SDK changes
|
||||
id: check-changes
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: ./**
|
||||
files_ignore: |
|
||||
|
||||
@@ -51,7 +51,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -66,12 +66,12 @@ jobs:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@89a39a4e59826350b863aa6b6252a07ad50cf83e # v4.32.4
|
||||
uses: github/codeql-action/init@68bde559dea0fdcac2102bfdf6230c5f70eb485e # v4.35.4
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
config-file: ./.github/codeql/sdk-codeql-config.yml
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@89a39a4e59826350b863aa6b6252a07ad50cf83e # v4.32.4
|
||||
uses: github/codeql-action/analyze@68bde559dea0fdcac2102bfdf6230c5f70eb485e # v4.35.4
|
||||
with:
|
||||
category: '/language:${{ matrix.language }}'
|
||||
|
||||
@@ -60,7 +60,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -98,7 +98,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -138,17 +138,21 @@ jobs:
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
id-token: write
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
api.ecr-public.us-east-1.amazonaws.com:443
|
||||
public.ecr.aws:443
|
||||
sts.amazonaws.com:443
|
||||
sts.us-east-1.amazonaws.com:443
|
||||
registry-1.docker.io:443
|
||||
production.cloudflare.docker.com:443
|
||||
production.cloudfront.docker.com:443
|
||||
auth.docker.io:443
|
||||
debian.map.fastlydns.net:80
|
||||
github.com:443
|
||||
@@ -167,19 +171,21 @@ jobs:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Login to Public ECR
|
||||
uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0
|
||||
- name: Configure AWS credentials (OIDC)
|
||||
uses: aws-actions/configure-aws-credentials@d979d5b3a71173a29b74b5b88418bfda9437d885 # v6.1.1
|
||||
with:
|
||||
registry: public.ecr.aws
|
||||
username: ${{ secrets.PUBLIC_ECR_AWS_ACCESS_KEY_ID }}
|
||||
password: ${{ secrets.PUBLIC_ECR_AWS_SECRET_ACCESS_KEY }}
|
||||
env:
|
||||
AWS_REGION: ${{ env.AWS_REGION }}
|
||||
aws-region: us-east-1
|
||||
role-to-assume: ${{ secrets.PUBLIC_ECR_IAM_ROLE_ARN }}
|
||||
|
||||
- name: Login to Public ECR
|
||||
uses: aws-actions/amazon-ecr-login@d539f0932e70871a027e9d5a9d8fc38589180a64 # v2.1.6
|
||||
with:
|
||||
registry-type: public
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
@@ -187,7 +193,7 @@ jobs:
|
||||
- name: Build and push SDK container for ${{ matrix.arch }}
|
||||
id: container-push
|
||||
if: github.event_name == 'push' || github.event_name == 'release' || github.event_name == 'workflow_dispatch'
|
||||
uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
with:
|
||||
context: .
|
||||
file: ${{ env.DOCKERFILE_PATH }}
|
||||
@@ -205,10 +211,11 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -216,25 +223,30 @@ jobs:
|
||||
auth.docker.io:443
|
||||
public.ecr.aws:443
|
||||
production.cloudflare.docker.com:443
|
||||
production.cloudfront.docker.com:443
|
||||
github.com:443
|
||||
release-assets.githubusercontent.com:443
|
||||
api.ecr-public.us-east-1.amazonaws.com:443
|
||||
sts.amazonaws.com:443
|
||||
sts.us-east-1.amazonaws.com:443
|
||||
|
||||
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Login to Public ECR
|
||||
uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0
|
||||
- name: Configure AWS credentials (OIDC)
|
||||
uses: aws-actions/configure-aws-credentials@d979d5b3a71173a29b74b5b88418bfda9437d885 # v6.1.1
|
||||
with:
|
||||
registry: public.ecr.aws
|
||||
username: ${{ secrets.PUBLIC_ECR_AWS_ACCESS_KEY_ID }}
|
||||
password: ${{ secrets.PUBLIC_ECR_AWS_SECRET_ACCESS_KEY }}
|
||||
env:
|
||||
AWS_REGION: ${{ env.AWS_REGION }}
|
||||
aws-region: us-east-1
|
||||
role-to-assume: ${{ secrets.PUBLIC_ECR_IAM_ROLE_ARN }}
|
||||
|
||||
- name: Login to Public ECR
|
||||
uses: aws-actions/amazon-ecr-login@d539f0932e70871a027e9d5a9d8fc38589180a64 # v2.1.6
|
||||
with:
|
||||
registry-type: public
|
||||
|
||||
- name: Create and push manifests for push event
|
||||
if: github.event_name == 'push'
|
||||
@@ -265,7 +277,7 @@ jobs:
|
||||
# Push to toniblyx/prowler only for current version (latest/stable/release tags)
|
||||
- name: Login to DockerHub (toniblyx)
|
||||
if: needs.setup.outputs.latest_tag == 'latest'
|
||||
uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
with:
|
||||
username: ${{ secrets.TONIBLYX_DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.TONIBLYX_DOCKERHUB_PASSWORD }}
|
||||
@@ -290,14 +302,14 @@ jobs:
|
||||
# Re-login as prowlercloud for cleanup of intermediate tags
|
||||
- name: Login to DockerHub (prowlercloud)
|
||||
if: always()
|
||||
uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Install regctl
|
||||
if: always()
|
||||
uses: regclient/actions/regctl-installer@da9319db8e44e8b062b3a147e1dfb2f574d41a03 # main
|
||||
uses: regclient/actions/regctl-installer@9a2d4216180dbb3e2dccfa60d2dd4afd98e42ec5 # main
|
||||
|
||||
- name: Cleanup intermediate architecture tags
|
||||
if: always()
|
||||
@@ -318,7 +330,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -15,12 +15,6 @@ on:
|
||||
branches:
|
||||
- 'master'
|
||||
- 'v5.*'
|
||||
paths:
|
||||
- 'prowler/**'
|
||||
- 'Dockerfile*'
|
||||
- 'pyproject.toml'
|
||||
- 'uv.lock'
|
||||
- '.github/workflows/sdk-container-checks.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
@@ -41,7 +35,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -55,7 +49,7 @@ jobs:
|
||||
|
||||
- name: Check if Dockerfile changed
|
||||
id: dockerfile-changed
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: Dockerfile
|
||||
|
||||
@@ -77,7 +71,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -85,6 +79,7 @@ jobs:
|
||||
registry-1.docker.io:443
|
||||
auth.docker.io:443
|
||||
production.cloudflare.docker.com:443
|
||||
production.cloudfront.docker.com:443
|
||||
api.github.com:443
|
||||
mirror.gcr.io:443
|
||||
check.trivy.dev:443
|
||||
@@ -108,27 +103,16 @@ jobs:
|
||||
|
||||
- name: Check for SDK changes
|
||||
id: check-changes
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: ./**
|
||||
files: |
|
||||
prowler/**
|
||||
Dockerfile*
|
||||
pyproject.toml
|
||||
uv.lock
|
||||
.github/workflows/sdk-container-checks.yml
|
||||
files_ignore: |
|
||||
.github/**
|
||||
prowler/CHANGELOG.md
|
||||
docs/**
|
||||
permissions/**
|
||||
api/**
|
||||
ui/**
|
||||
dashboard/**
|
||||
mcp_server/**
|
||||
skills/**
|
||||
README.md
|
||||
mkdocs.yml
|
||||
.backportrc.json
|
||||
.env
|
||||
docker-compose*
|
||||
examples/**
|
||||
.gitignore
|
||||
contrib/**
|
||||
**/AGENTS.md
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
@@ -137,7 +121,7 @@ jobs:
|
||||
|
||||
- name: Build SDK container
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
with:
|
||||
context: .
|
||||
push: false
|
||||
@@ -152,5 +136,5 @@ jobs:
|
||||
with:
|
||||
image-name: ${{ env.IMAGE_NAME }}
|
||||
image-tag: ${{ github.sha }}
|
||||
fail-on-critical: 'false'
|
||||
fail-on-critical: 'true'
|
||||
severity: 'CRITICAL'
|
||||
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -66,7 +66,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -85,7 +85,7 @@ jobs:
|
||||
run: uv build
|
||||
|
||||
- name: Publish Prowler package to PyPI
|
||||
uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0
|
||||
uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0
|
||||
with:
|
||||
print-hash: true
|
||||
|
||||
@@ -102,7 +102,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -129,6 +129,6 @@ jobs:
|
||||
run: uv build
|
||||
|
||||
- name: Publish prowler-cloud package to PyPI
|
||||
uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0
|
||||
uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0
|
||||
with:
|
||||
print-hash: true
|
||||
|
||||
@@ -27,7 +27,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -47,7 +47,7 @@ jobs:
|
||||
run: pip install boto3
|
||||
|
||||
- name: Configure AWS credentials
|
||||
uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0
|
||||
uses: aws-actions/configure-aws-credentials@d979d5b3a71173a29b74b5b88418bfda9437d885 # v6.1.1
|
||||
with:
|
||||
aws-region: ${{ env.AWS_REGION }}
|
||||
role-to-assume: ${{ secrets.DEV_IAM_ROLE_ARN }}
|
||||
@@ -58,7 +58,7 @@ jobs:
|
||||
|
||||
- name: Create pull request
|
||||
id: create-pr
|
||||
uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0 # v8.1.0
|
||||
uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
|
||||
with:
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
author: 'prowler-bot <179230569+prowler-bot@users.noreply.github.com>'
|
||||
|
||||
@@ -25,7 +25,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -55,7 +55,7 @@ jobs:
|
||||
|
||||
- name: Create pull request
|
||||
id: create-pr
|
||||
uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0 # v8.1.0
|
||||
uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
|
||||
with:
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
author: 'prowler-bot <179230569+prowler-bot@users.noreply.github.com>'
|
||||
|
||||
@@ -19,16 +19,6 @@ on:
|
||||
branches:
|
||||
- 'master'
|
||||
- 'v5.*'
|
||||
paths:
|
||||
- 'prowler/**'
|
||||
- 'tests/**'
|
||||
- 'pyproject.toml'
|
||||
- 'uv.lock'
|
||||
- '.github/workflows/sdk-tests.yml'
|
||||
- '.github/workflows/sdk-security.yml'
|
||||
- '.github/actions/setup-python-uv/**'
|
||||
- '.github/actions/osv-scanner/**'
|
||||
- '.github/scripts/osv-scan.sh'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
@@ -47,7 +37,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -69,29 +59,20 @@ jobs:
|
||||
|
||||
- name: Check for SDK changes
|
||||
id: check-changes
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files:
|
||||
./**
|
||||
files: |
|
||||
prowler/**
|
||||
tests/**
|
||||
pyproject.toml
|
||||
uv.lock
|
||||
.github/workflows/sdk-tests.yml
|
||||
.github/workflows/sdk-security.yml
|
||||
.github/actions/setup-python-uv/**
|
||||
.github/actions/osv-scanner/**
|
||||
.github/scripts/osv-scan.sh
|
||||
files_ignore: |
|
||||
.github/**
|
||||
prowler/CHANGELOG.md
|
||||
docs/**
|
||||
permissions/**
|
||||
api/**
|
||||
ui/**
|
||||
dashboard/**
|
||||
mcp_server/**
|
||||
skills/**
|
||||
README.md
|
||||
mkdocs.yml
|
||||
.backportrc.json
|
||||
.env
|
||||
docker-compose*
|
||||
examples/**
|
||||
.gitignore
|
||||
contrib/**
|
||||
**/AGENTS.md
|
||||
|
||||
- name: Setup Python with uv
|
||||
|
||||
+119
-18
@@ -29,10 +29,11 @@ jobs:
|
||||
- '3.10'
|
||||
- '3.11'
|
||||
- '3.12'
|
||||
- '3.13'
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -46,6 +47,7 @@ jobs:
|
||||
schema.ocsf.io:443
|
||||
registry-1.docker.io:443
|
||||
production.cloudflare.docker.com:443
|
||||
production.cloudfront.docker.com:443
|
||||
powershellinfraartifacts-gkhedzdeaghdezhr.z01.azurefd.net:443
|
||||
o26192.ingest.us.sentry.io:443
|
||||
management.azure.com:443
|
||||
@@ -69,7 +71,7 @@ jobs:
|
||||
|
||||
- name: Check for SDK changes
|
||||
id: check-changes
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: ./**
|
||||
files_ignore: |
|
||||
@@ -102,7 +104,7 @@ jobs:
|
||||
- name: Check if AWS files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: changed-aws
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
./prowler/**/aws/**
|
||||
@@ -232,7 +234,7 @@ jobs:
|
||||
- name: Check if Azure files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: changed-azure
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
./prowler/**/azure/**
|
||||
@@ -256,7 +258,7 @@ jobs:
|
||||
- name: Check if GCP files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: changed-gcp
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
./prowler/**/gcp/**
|
||||
@@ -280,7 +282,7 @@ jobs:
|
||||
- name: Check if Kubernetes files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: changed-kubernetes
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
./prowler/**/kubernetes/**
|
||||
@@ -304,7 +306,7 @@ jobs:
|
||||
- name: Check if GitHub files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: changed-github
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
./prowler/**/github/**
|
||||
@@ -328,7 +330,7 @@ jobs:
|
||||
- name: Check if Okta files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: changed-okta
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
./prowler/**/okta/**
|
||||
@@ -352,7 +354,7 @@ jobs:
|
||||
- name: Check if NHN files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: changed-nhn
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
./prowler/**/nhn/**
|
||||
@@ -376,7 +378,7 @@ jobs:
|
||||
- name: Check if M365 files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: changed-m365
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
./prowler/**/m365/**
|
||||
@@ -400,7 +402,7 @@ jobs:
|
||||
- name: Check if IaC files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: changed-iac
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
./prowler/**/iac/**
|
||||
@@ -424,7 +426,7 @@ jobs:
|
||||
- name: Check if MongoDB Atlas files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: changed-mongodbatlas
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
./prowler/**/mongodbatlas/**
|
||||
@@ -448,7 +450,7 @@ jobs:
|
||||
- name: Check if OCI files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: changed-oraclecloud
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
./prowler/**/oraclecloud/**
|
||||
@@ -472,7 +474,7 @@ jobs:
|
||||
- name: Check if OpenStack files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: changed-openstack
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
./prowler/**/openstack/**
|
||||
@@ -496,7 +498,7 @@ jobs:
|
||||
- name: Check if Google Workspace files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: changed-googleworkspace
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
./prowler/**/googleworkspace/**
|
||||
@@ -520,7 +522,7 @@ jobs:
|
||||
- name: Check if Vercel files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: changed-vercel
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
./prowler/**/vercel/**
|
||||
@@ -540,11 +542,110 @@ jobs:
|
||||
flags: prowler-py${{ matrix.python-version }}-vercel
|
||||
files: ./vercel_coverage.xml
|
||||
|
||||
# Scaleway Provider
|
||||
- name: Check if Scaleway files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: changed-scaleway
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
./prowler/**/scaleway/**
|
||||
./tests/**/scaleway/**
|
||||
./uv.lock
|
||||
|
||||
- name: Run Scaleway tests
|
||||
if: steps.changed-scaleway.outputs.any_changed == 'true'
|
||||
run: uv run pytest -n auto --cov=./prowler/providers/scaleway --cov-report=xml:scaleway_coverage.xml tests/providers/scaleway
|
||||
|
||||
- name: Upload Scaleway coverage to Codecov
|
||||
if: steps.changed-scaleway.outputs.any_changed == 'true'
|
||||
uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2
|
||||
env:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
with:
|
||||
flags: prowler-py${{ matrix.python-version }}-scaleway
|
||||
files: ./scaleway_coverage.xml
|
||||
|
||||
# StackIT Provider
|
||||
- name: Check if StackIT files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: changed-stackit
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
./prowler/**/stackit/**
|
||||
./tests/**/stackit/**
|
||||
./uv.lock
|
||||
|
||||
- name: Run StackIT tests
|
||||
if: steps.changed-stackit.outputs.any_changed == 'true'
|
||||
run: uv run pytest -n auto --cov=./prowler/providers/stackit --cov-report=xml:stackit_coverage.xml tests/providers/stackit
|
||||
|
||||
- name: Upload StackIT coverage to Codecov
|
||||
if: steps.changed-stackit.outputs.any_changed == 'true'
|
||||
uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2
|
||||
env:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
with:
|
||||
flags: prowler-py${{ matrix.python-version }}-stackit
|
||||
files: ./stackit_coverage.xml
|
||||
|
||||
# Linode Provider
|
||||
- name: Check if Linode files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: changed-linode
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
./prowler/**/linode/**
|
||||
./tests/**/linode/**
|
||||
./uv.lock
|
||||
|
||||
- name: Run Linode tests
|
||||
if: steps.changed-linode.outputs.any_changed == 'true'
|
||||
run: uv run pytest -n auto --cov=./prowler/providers/linode --cov-report=xml:linode_coverage.xml tests/providers/linode
|
||||
|
||||
- name: Upload Linode coverage to Codecov
|
||||
if: steps.changed-linode.outputs.any_changed == 'true'
|
||||
uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2
|
||||
env:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
with:
|
||||
flags: prowler-py${{ matrix.python-version }}-linode
|
||||
files: ./linode_coverage.xml
|
||||
|
||||
# External Provider (dynamic loading)
|
||||
- name: Check if External Provider files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: changed-external
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
with:
|
||||
files: |
|
||||
./prowler/providers/common/**
|
||||
./prowler/config/**
|
||||
./prowler/lib/**
|
||||
./tests/providers/external/**
|
||||
./uv.lock
|
||||
|
||||
- name: Run External Provider tests
|
||||
if: steps.changed-external.outputs.any_changed == 'true'
|
||||
run: uv run pytest -n auto --cov=./prowler/providers/common --cov=./prowler/config --cov=./prowler/lib --cov-report=xml:external_coverage.xml tests/providers/external
|
||||
|
||||
- name: Upload External Provider coverage to Codecov
|
||||
if: steps.changed-external.outputs.any_changed == 'true'
|
||||
|
||||
uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2
|
||||
env:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
with:
|
||||
flags: prowler-py${{ matrix.python-version }}-external
|
||||
files: ./external_coverage.xml
|
||||
|
||||
# Lib
|
||||
- name: Check if Lib files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: changed-lib
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
./prowler/lib/**
|
||||
@@ -568,7 +669,7 @@ jobs:
|
||||
- name: Check if Config files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: changed-config
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
./prowler/config/**
|
||||
|
||||
@@ -52,7 +52,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -68,12 +68,12 @@ jobs:
|
||||
|
||||
- name: Get changed files
|
||||
id: changed-files
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: '3.12'
|
||||
python-version: '3.12.13'
|
||||
|
||||
- name: Install PyYAML
|
||||
run: pip install pyyaml
|
||||
|
||||
@@ -47,7 +47,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -62,12 +62,12 @@ jobs:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@89a39a4e59826350b863aa6b6252a07ad50cf83e # v4.32.4
|
||||
uses: github/codeql-action/init@68bde559dea0fdcac2102bfdf6230c5f70eb485e # v4.35.4
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
config-file: ./.github/codeql/ui-codeql-config.yml
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@89a39a4e59826350b863aa6b6252a07ad50cf83e # v4.32.4
|
||||
uses: github/codeql-action/analyze@68bde559dea0fdcac2102bfdf6230c5f70eb485e # v4.35.4
|
||||
with:
|
||||
category: '/language:${{ matrix.language }}'
|
||||
|
||||
@@ -32,9 +32,6 @@ env:
|
||||
PROWLERCLOUD_DOCKERHUB_REPOSITORY: prowlercloud
|
||||
PROWLERCLOUD_DOCKERHUB_IMAGE: prowler-ui
|
||||
|
||||
# Build args
|
||||
NEXT_PUBLIC_API_BASE_URL: http://prowler-api:8080/api/v1
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
@@ -48,7 +45,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -67,7 +64,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -110,12 +107,13 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
registry-1.docker.io:443
|
||||
production.cloudflare.docker.com:443
|
||||
production.cloudfront.docker.com:443
|
||||
auth.docker.io:443
|
||||
registry.npmjs.org:443
|
||||
dl-cdn.alpinelinux.org:443
|
||||
@@ -129,7 +127,7 @@ jobs:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
@@ -140,12 +138,11 @@ jobs:
|
||||
- name: Build and push UI container for ${{ matrix.arch }}
|
||||
id: container-push
|
||||
if: github.event_name == 'push' || github.event_name == 'release' || github.event_name == 'workflow_dispatch'
|
||||
uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
with:
|
||||
context: ${{ env.WORKING_DIRECTORY }}
|
||||
build-args: |
|
||||
NEXT_PUBLIC_PROWLER_RELEASE_VERSION=${{ (github.event_name == 'release' || github.event_name == 'workflow_dispatch') && format('v{0}', env.RELEASE_TAG) || needs.setup.outputs.short-sha }}
|
||||
NEXT_PUBLIC_API_BASE_URL=${{ env.NEXT_PUBLIC_API_BASE_URL }}
|
||||
push: true
|
||||
platforms: ${{ matrix.platform }}
|
||||
tags: |
|
||||
@@ -163,7 +160,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -172,9 +169,10 @@ jobs:
|
||||
registry-1.docker.io:443
|
||||
auth.docker.io:443
|
||||
production.cloudflare.docker.com:443
|
||||
production.cloudfront.docker.com:443
|
||||
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
@@ -203,7 +201,7 @@ jobs:
|
||||
|
||||
- name: Install regctl
|
||||
if: always()
|
||||
uses: regclient/actions/regctl-installer@da9319db8e44e8b062b3a147e1dfb2f574d41a03 # main
|
||||
uses: regclient/actions/regctl-installer@9a2d4216180dbb3e2dccfa60d2dd4afd98e42ec5 # main
|
||||
|
||||
- name: Cleanup intermediate architecture tags
|
||||
if: always()
|
||||
@@ -224,7 +222,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -260,27 +258,3 @@ jobs:
|
||||
payload-file-path: "./.github/scripts/slack-messages/container-release-completed.json"
|
||||
step-outcome: ${{ steps.outcome.outputs.outcome }}
|
||||
update-ts: ${{ needs.notify-release-started.outputs.message-ts }}
|
||||
|
||||
trigger-deployment:
|
||||
needs: [setup, container-build-push]
|
||||
if: always() && github.event_name == 'push' && needs.setup.result == 'success' && needs.container-build-push.result == 'success'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
api.github.com:443
|
||||
|
||||
- name: Trigger UI deployment
|
||||
uses: peter-evans/repository-dispatch@28959ce8df70de7be546dd1250a005dd32156697 # v4.0.1
|
||||
with:
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
repository: ${{ secrets.CLOUD_DISPATCH }}
|
||||
event-type: ui-prowler-deployment
|
||||
client-payload: '{"sha": "${{ github.sha }}", "short_sha": "${{ needs.setup.outputs.short-sha }}"}'
|
||||
|
||||
@@ -12,9 +12,6 @@ on:
|
||||
branches:
|
||||
- 'master'
|
||||
- 'v5.*'
|
||||
paths:
|
||||
- 'ui/**'
|
||||
- '.github/workflows/ui-container-checks.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
@@ -36,7 +33,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -50,7 +47,7 @@ jobs:
|
||||
|
||||
- name: Check if Dockerfile changed
|
||||
id: dockerfile-changed
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: ui/Dockerfile
|
||||
|
||||
@@ -72,7 +69,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -100,7 +97,7 @@ jobs:
|
||||
|
||||
- name: Check for UI changes
|
||||
id: check-changes
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: ui/**
|
||||
files_ignore: |
|
||||
@@ -114,7 +111,7 @@ jobs:
|
||||
|
||||
- name: Build UI container
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
with:
|
||||
context: ${{ env.UI_WORKING_DIR }}
|
||||
target: prod
|
||||
@@ -132,5 +129,5 @@ jobs:
|
||||
with:
|
||||
image-name: ${{ env.IMAGE_NAME }}
|
||||
image-tag: ${{ github.sha }}
|
||||
fail-on-critical: 'false'
|
||||
fail-on-critical: 'true'
|
||||
severity: 'CRITICAL'
|
||||
|
||||
@@ -40,7 +40,8 @@ jobs:
|
||||
AUTH_SECRET: 'fallback-ci-secret-for-testing'
|
||||
AUTH_TRUST_HOST: true
|
||||
NEXTAUTH_URL: 'http://localhost:3000'
|
||||
NEXT_PUBLIC_API_BASE_URL: 'http://localhost:8080/api/v1'
|
||||
AUTH_URL: 'http://localhost:3000'
|
||||
UI_API_BASE_URL: 'http://localhost:8080/api/v1'
|
||||
E2E_ADMIN_USER: ${{ secrets.E2E_ADMIN_USER }}
|
||||
E2E_ADMIN_PASSWORD: ${{ secrets.E2E_ADMIN_PASSWORD }}
|
||||
E2E_AWS_PROVIDER_ACCOUNT_ID: ${{ secrets.E2E_AWS_PROVIDER_ACCOUNT_ID }}
|
||||
@@ -77,6 +78,14 @@ jobs:
|
||||
E2E_ALIBABACLOUD_ACCESS_KEY_ID: ${{ secrets.E2E_ALIBABACLOUD_ACCESS_KEY_ID }}
|
||||
E2E_ALIBABACLOUD_ACCESS_KEY_SECRET: ${{ secrets.E2E_ALIBABACLOUD_ACCESS_KEY_SECRET }}
|
||||
E2E_ALIBABACLOUD_ROLE_ARN: ${{ secrets.E2E_ALIBABACLOUD_ROLE_ARN }}
|
||||
E2E_OKTA_DOMAIN: ${{ secrets.E2E_OKTA_DOMAIN }}
|
||||
E2E_OKTA_CLIENT_ID: ${{ secrets.E2E_OKTA_CLIENT_ID }}
|
||||
E2E_OKTA_BASE64_PRIVATE_KEY: ${{ secrets.E2E_OKTA_BASE64_PRIVATE_KEY }}
|
||||
E2E_GOOGLEWORKSPACE_CUSTOMER_ID: ${{ secrets.E2E_GOOGLEWORKSPACE_CUSTOMER_ID }}
|
||||
E2E_GOOGLEWORKSPACE_SERVICE_ACCOUNT_JSON: ${{ secrets.E2E_GOOGLEWORKSPACE_SERVICE_ACCOUNT_JSON }}
|
||||
E2E_GOOGLEWORKSPACE_DELEGATED_USER: ${{ secrets.E2E_GOOGLEWORKSPACE_DELEGATED_USER }}
|
||||
E2E_VERCEL_TEAM_ID: ${{ secrets.E2E_VERCEL_TEAM_ID }}
|
||||
E2E_VERCEL_API_TOKEN: ${{ secrets.E2E_VERCEL_API_TOKEN }}
|
||||
# Pass E2E paths from impact analysis
|
||||
E2E_TEST_PATHS: ${{ needs.impact-analysis.outputs.ui-e2e }}
|
||||
RUN_ALL_TESTS: ${{ needs.impact-analysis.outputs.run-all }}
|
||||
@@ -85,7 +94,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -134,7 +143,17 @@ jobs:
|
||||
# docker-compose.yml references prowlercloud/prowler-api:latest from the registry,
|
||||
# which lags behind PR changes; build locally so E2E exercises the API image
|
||||
# produced by this PR.
|
||||
run: docker build -t prowlercloud/prowler-api:latest ./api
|
||||
#
|
||||
# The image installs the SDK from git@master (api/uv.lock), so a PR changing BOTH the SDK
|
||||
# and the API would run against the OLD SDK and crash on startup. Overlay the checkout's
|
||||
# SDK source so both run together. New SDK dependencies still need an api/uv.lock bump.
|
||||
run: |
|
||||
docker build -t prowlercloud/prowler-api:pr-base ./api
|
||||
docker build -t prowlercloud/prowler-api:latest -f - prowler <<'DOCKERFILE'
|
||||
FROM prowlercloud/prowler-api:pr-base
|
||||
RUN rm -rf /home/prowler/.venv/lib/python3.12/site-packages/prowler
|
||||
COPY --chown=prowler:prowler . /home/prowler/.venv/lib/python3.12/site-packages/prowler
|
||||
DOCKERFILE
|
||||
|
||||
- name: Start API services
|
||||
run: |
|
||||
@@ -147,7 +166,7 @@ jobs:
|
||||
timeout=150
|
||||
elapsed=0
|
||||
while [ $elapsed -lt $timeout ]; do
|
||||
if curl -s ${NEXT_PUBLIC_API_BASE_URL}/docs >/dev/null 2>&1; then
|
||||
if curl -s ${UI_API_BASE_URL}/docs >/dev/null 2>&1; then
|
||||
echo "Prowler API is ready!"
|
||||
exit 0
|
||||
fi
|
||||
@@ -172,7 +191,7 @@ jobs:
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
|
||||
with:
|
||||
node-version: '24.13.0'
|
||||
node-version-file: 'ui/.nvmrc'
|
||||
|
||||
- name: Setup pnpm
|
||||
uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0
|
||||
@@ -184,7 +203,7 @@ jobs:
|
||||
run: echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV
|
||||
|
||||
- name: Setup pnpm and Next.js cache
|
||||
uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
with:
|
||||
path: |
|
||||
${{ env.STORE_PATH }}
|
||||
@@ -204,7 +223,7 @@ jobs:
|
||||
run: pnpm run build
|
||||
|
||||
- name: Cache Playwright browsers
|
||||
uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
id: playwright-cache
|
||||
with:
|
||||
path: ~/.cache/ms-playwright
|
||||
@@ -295,7 +314,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -15,12 +15,6 @@ on:
|
||||
branches:
|
||||
- 'master'
|
||||
- 'v5.*'
|
||||
paths:
|
||||
- 'ui/package.json'
|
||||
- 'ui/pnpm-lock.yaml'
|
||||
- '.github/workflows/ui-security.yml'
|
||||
- '.github/actions/osv-scanner/**'
|
||||
- '.github/scripts/osv-scan.sh'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
@@ -30,7 +24,6 @@ permissions: {}
|
||||
|
||||
jobs:
|
||||
ui-security-scans:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
@@ -39,7 +32,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -59,7 +52,7 @@ jobs:
|
||||
|
||||
- name: Check for UI dependency changes
|
||||
id: check-changes
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
ui/package.json
|
||||
|
||||
@@ -16,7 +16,6 @@ concurrency:
|
||||
|
||||
env:
|
||||
UI_WORKING_DIR: ./ui
|
||||
NODE_VERSION: "24.13.0"
|
||||
|
||||
permissions: {}
|
||||
|
||||
@@ -32,14 +31,18 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
github.com:443
|
||||
registry.npmjs.org:443
|
||||
nodejs.org:443
|
||||
fonts.googleapis.com:443
|
||||
fonts.gstatic.com:443
|
||||
api.iconify.design:443
|
||||
api.simplesvg.com:443
|
||||
api.unisvg.com:443
|
||||
api.github.com:443
|
||||
release-assets.githubusercontent.com:443
|
||||
cdn.playwright.dev:443
|
||||
@@ -54,7 +57,7 @@ jobs:
|
||||
|
||||
- name: Check for UI changes
|
||||
id: check-changes
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
ui/**
|
||||
@@ -67,7 +70,7 @@ jobs:
|
||||
- name: Get changed source files for targeted tests
|
||||
id: changed-source
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
ui/**/*.ts
|
||||
@@ -83,7 +86,7 @@ jobs:
|
||||
- name: Check for critical path changes (run all tests)
|
||||
id: critical-changes
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
ui/lib/**
|
||||
@@ -93,11 +96,11 @@ jobs:
|
||||
ui/vitest.config.ts
|
||||
ui/vitest.setup.ts
|
||||
|
||||
- name: Setup Node.js ${{ env.NODE_VERSION }}
|
||||
- name: Setup Node.js
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
|
||||
with:
|
||||
node-version: ${{ env.NODE_VERSION }}
|
||||
node-version-file: 'ui/.nvmrc'
|
||||
|
||||
- name: Setup pnpm
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
@@ -113,7 +116,7 @@ jobs:
|
||||
|
||||
- name: Setup pnpm and Next.js cache
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
with:
|
||||
path: |
|
||||
${{ env.STORE_PATH }}
|
||||
@@ -132,6 +135,10 @@ jobs:
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
run: pnpm run healthcheck
|
||||
|
||||
- name: Check product-tour alignment
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
run: pnpm run tour:check
|
||||
|
||||
- name: Run pnpm audit
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
run: pnpm run audit
|
||||
@@ -162,7 +169,7 @@ jobs:
|
||||
- name: Cache Playwright browsers
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: playwright-cache
|
||||
uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
with:
|
||||
path: ~/.cache/ms-playwright
|
||||
key: ${{ runner.os }}-playwright-chromium-${{ hashFiles('ui/pnpm-lock.yaml') }}
|
||||
|
||||
+4
-1
@@ -60,7 +60,6 @@ htmlcov/
|
||||
**/mcp-config.json
|
||||
**/mcpServers.json
|
||||
.mcp/
|
||||
.mcp.json
|
||||
|
||||
# AI Coding Assistants - Cursor
|
||||
.cursorignore
|
||||
@@ -170,3 +169,7 @@ GEMINI.md
|
||||
|
||||
# Claude Code
|
||||
.claude/*
|
||||
|
||||
# Docker
|
||||
docker-compose.override.yml
|
||||
docker-compose-dev.override.yml
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"extends": "markdownlint/style/prettier",
|
||||
"first-line-h1": false,
|
||||
"no-duplicate-heading": {
|
||||
"siblings_only": true
|
||||
},
|
||||
"no-inline-html": false,
|
||||
"line-length": false,
|
||||
"no-bare-urls": false
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
node_modules/
|
||||
ui/node_modules/
|
||||
.git/
|
||||
.venv/
|
||||
**/.venv/
|
||||
dist/
|
||||
build/
|
||||
htmlcov/
|
||||
.next/
|
||||
ui/.next/
|
||||
ui/out/
|
||||
contrib/
|
||||
|
||||
# Auto-generated content (keepachangelog format legitimately repeats section headings).
|
||||
# Revisit with the team — see beads task on markdownlint rule triage.
|
||||
**/CHANGELOG.md
|
||||
+51
-9
@@ -7,6 +7,10 @@
|
||||
# P50 — dependency validation
|
||||
|
||||
default_install_hook_types: [pre-commit]
|
||||
# Hooks run on commit only by default;
|
||||
# NOTE: default_stages does NOT override a hook's manifest stages, so fixers shipping pre-push in their
|
||||
# manifest need an explicit stages: ["pre-commit"] below to stay off push.
|
||||
default_stages: [pre-commit]
|
||||
|
||||
repos:
|
||||
## GENERAL (prek built-in — no external repo needed)
|
||||
@@ -21,13 +25,16 @@ repos:
|
||||
- id: check-json
|
||||
priority: 10
|
||||
- id: end-of-file-fixer
|
||||
stages: ["pre-commit"]
|
||||
priority: 0
|
||||
- id: trailing-whitespace
|
||||
stages: ["pre-commit"]
|
||||
priority: 0
|
||||
- id: no-commit-to-branch
|
||||
priority: 10
|
||||
- id: pretty-format-json
|
||||
args: ["--autofix", --no-sort-keys, --no-ensure-ascii]
|
||||
stages: ["pre-commit"]
|
||||
priority: 10
|
||||
|
||||
## TOML
|
||||
@@ -49,6 +56,14 @@ repos:
|
||||
files: ^\.github/(workflows|actions)/.+\.ya?ml$|^\.github/dependabot\.ya?ml$
|
||||
priority: 30
|
||||
|
||||
## RENOVATE
|
||||
- repo: https://github.com/renovatebot/pre-commit-hooks
|
||||
rev: 43.150.0
|
||||
hooks:
|
||||
- id: renovate-config-validator
|
||||
files: ^\.github/renovate\.json$
|
||||
priority: 10
|
||||
|
||||
## BASH
|
||||
- repo: https://github.com/koalaman/shellcheck-precommit
|
||||
rev: v0.11.0
|
||||
@@ -74,6 +89,7 @@ repos:
|
||||
name: "SDK - isort"
|
||||
files: { glob: ["{prowler,tests,dashboard,util,scripts}/**/*.py"] }
|
||||
args: ["--profile", "black"]
|
||||
stages: ["pre-commit"]
|
||||
priority: 20
|
||||
|
||||
- repo: https://github.com/psf/black
|
||||
@@ -94,17 +110,36 @@ repos:
|
||||
priority: 30
|
||||
|
||||
## PYTHON — API + MCP Server (ruff)
|
||||
- repo: https://github.com/astral-sh/ruff-pre-commit
|
||||
rev: v0.15.11
|
||||
# Run ruff through `uv run` against each project so prek uses the exact ruff
|
||||
# version pinned in that project's uv.lock — the same version GitHub Actions
|
||||
# runs via `uv run ruff`. This removes the drift between the local hooks and
|
||||
# CI. api/ and mcp_server/ are separate uv projects, so they need separate
|
||||
# hooks (each `uv run --project` resolves its own pinned ruff + config).
|
||||
- repo: local
|
||||
hooks:
|
||||
- id: ruff
|
||||
name: "API + MCP - ruff check"
|
||||
files: { glob: ["{api,mcp_server}/**/*.py"] }
|
||||
args: ["--fix"]
|
||||
- id: ruff-check-api
|
||||
name: "API - ruff check"
|
||||
entry: uv run --project ./api ruff check --fix
|
||||
language: system
|
||||
files: { glob: ["api/**/*.py"] }
|
||||
priority: 30
|
||||
- id: ruff-format
|
||||
name: "API + MCP - ruff format"
|
||||
files: { glob: ["{api,mcp_server}/**/*.py"] }
|
||||
- id: ruff-format-api
|
||||
name: "API - ruff format"
|
||||
entry: uv run --project ./api ruff format
|
||||
language: system
|
||||
files: { glob: ["api/**/*.py"] }
|
||||
priority: 20
|
||||
- id: ruff-check-mcp
|
||||
name: "MCP - ruff check"
|
||||
entry: uv run --project ./mcp_server ruff check --fix
|
||||
language: system
|
||||
files: { glob: ["mcp_server/**/*.py"] }
|
||||
priority: 30
|
||||
- id: ruff-format-mcp
|
||||
name: "MCP - ruff format"
|
||||
entry: uv run --project ./mcp_server ruff format
|
||||
language: system
|
||||
files: { glob: ["mcp_server/**/*.py"] }
|
||||
priority: 20
|
||||
|
||||
## PYTHON — uv (API + SDK)
|
||||
@@ -125,6 +160,13 @@ repos:
|
||||
pass_filenames: false
|
||||
priority: 50
|
||||
|
||||
## MARKDOWN
|
||||
- repo: https://github.com/igorshubovych/markdownlint-cli
|
||||
rev: v0.45.0
|
||||
hooks:
|
||||
- id: markdownlint
|
||||
priority: 30
|
||||
|
||||
## CONTAINERS
|
||||
- repo: https://github.com/hadolint/hadolint
|
||||
rev: v2.14.0
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
# Trivy ignore file for prowlercloud/prowler SDK container image.
|
||||
# Each entry below documents (a) the affected package and why it ships in the
|
||||
# image, (b) why the CVE is not exploitable in Prowler's runtime, and (c) the
|
||||
# upstream fix status. Entries carry an expiry so they auto-force re-review.
|
||||
# Entries are scoped per-package so suppressions cannot drift onto unrelated
|
||||
# packages that may be assigned the same CVE in the future.
|
||||
#
|
||||
# Scanned by: .github/actions/trivy-scan via .github/workflows/sdk-container-checks.yml
|
||||
|
||||
# CVE-2026-42496 — perl-archive-tar path traversal via crafted symlinks.
|
||||
# CVE-2026-8376 — perl heap buffer overflow when compiling regex.
|
||||
# Packages: perl, perl-base, perl-modules-5.36, libperl5.36.
|
||||
# Why ignored: perl-base is part of Debian's "Essential: yes" set; it cannot be
|
||||
# removed without breaking dpkg. The Prowler SDK does not invoke perl at runtime;
|
||||
# neither vulnerable code path (Archive::Tar parsing or regex compilation of
|
||||
# attacker-controlled input) is reachable from Prowler. No Debian bookworm fix
|
||||
# is available yet.
|
||||
CVE-2026-42496 pkg:perl exp:2026-07-15
|
||||
CVE-2026-42496 pkg:perl-base exp:2026-07-15
|
||||
CVE-2026-42496 pkg:perl-modules-5.36 exp:2026-07-15
|
||||
CVE-2026-42496 pkg:libperl5.36 exp:2026-07-15
|
||||
CVE-2026-8376 pkg:perl exp:2026-07-15
|
||||
CVE-2026-8376 pkg:perl-base exp:2026-07-15
|
||||
CVE-2026-8376 pkg:perl-modules-5.36 exp:2026-07-15
|
||||
CVE-2026-8376 pkg:libperl5.36 exp:2026-07-15
|
||||
|
||||
# CVE-2025-7458 — SQLite integer overflow.
|
||||
# Package: libsqlite3-0.
|
||||
# Why ignored: transitive dependency of CPython's stdlib sqlite3 module. The
|
||||
# Prowler SDK does not open user-supplied SQLite databases; SQLite usage is
|
||||
# internal and bounded. No Debian bookworm fix is available.
|
||||
CVE-2025-7458 pkg:libsqlite3-0 exp:2026-07-15
|
||||
|
||||
# CVE-2026-43185 — Linux kernel ksmbd signedness bug.
|
||||
# Package: linux-libc-dev.
|
||||
# Why ignored: linux-libc-dev ships kernel headers for build-time compilation,
|
||||
# not a running kernel. Containers execute against the host kernel, so these
|
||||
# headers are inert at runtime. The upstream fix landed in kernel 7.0-rc2 and
|
||||
# has not been backported to Debian's 6.1 LTS line.
|
||||
CVE-2026-43185 pkg:linux-libc-dev exp:2026-07-15
|
||||
|
||||
# CVE-2023-45853 — zlib MiniZip integer overflow / heap overflow in
|
||||
# zipOpenNewFileInZip4_64.
|
||||
# Packages: zlib1g, zlib1g-dev.
|
||||
# Why ignored: Debian Security Tracker status for bookworm is <ignored>, with
|
||||
# the published rationale "contrib/minizip not built and src:zlib not producing
|
||||
# binary packages" — i.e. the vulnerable symbol is not present in the libz.so
|
||||
# shipped by Debian. Real-not-affected, not unpatched. Upstream fix is in
|
||||
# zlib 1.3.1, available in Debian trixie (13); migrating the base image would
|
||||
# clear it fully.
|
||||
# Ref: https://security-tracker.debian.org/tracker/CVE-2023-45853
|
||||
CVE-2023-45853 pkg:zlib1g exp:2026-07-15
|
||||
CVE-2023-45853 pkg:zlib1g-dev exp:2026-07-15
|
||||
|
||||
# CVE-2026-55200 — libssh2 out-of-bounds write in ssh2_transport_read() due to
|
||||
# an unchecked packet_length field in transport.c (heap corruption, possible RCE).
|
||||
# Package: libssh2-1.
|
||||
# Why ignored: libssh2-1 is pulled in only as a transitive dependency of libcurl4
|
||||
# (installed in the SDK Dockerfile for the networking/PowerShell stack). The
|
||||
# vulnerable path is reached exclusively when libssh2 acts as an SSH/SCP/SFTP
|
||||
# client parsing transport packets from a server. Prowler never uses libcurl's
|
||||
# SSH/SCP/SFTP transports; it talks to cloud provider HTTPS endpoints only, so the
|
||||
# affected code is unreachable at runtime. Fixed upstream in libssh2 commit
|
||||
# 97acf3df (PR #2052); no Debian bookworm fix is available yet.
|
||||
# Ref: https://security-tracker.debian.org/tracker/CVE-2026-55200
|
||||
CVE-2026-55200 pkg:libssh2-1 exp:2026-07-15
|
||||
|
||||
# --- API container image (api/Dockerfile) ---
|
||||
# The entries below are specific to the Prowler API image, which ships
|
||||
# PowerShell and additional build tooling on top of the same bookworm base.
|
||||
|
||||
# CVE-2026-7210 — CPython/Expat hash-flooding denial of service in
|
||||
# `xml.parsers.expat` and `xml.etree.ElementTree`.
|
||||
# Packages: the Debian system Python 3.11 (python3.11*, libpython3.11*).
|
||||
# Why ignored: the API runs under the Python 3.12 interpreter shipped in its
|
||||
# `.venv`; the system `python3.11` is only present because `python3-dev` is
|
||||
# pulled in to compile native extensions (xmlsec, lxml) and is never executed
|
||||
# at runtime. The vulnerable path requires parsing attacker-controlled XML with
|
||||
# the affected interpreter, which Prowler does not do with the system Python.
|
||||
# Full mitigation also needs libexpat >= 2.8.0; no Debian bookworm fix yet.
|
||||
CVE-2026-7210 pkg:python3.11 exp:2026-07-15
|
||||
CVE-2026-7210 pkg:python3.11-dev exp:2026-07-15
|
||||
CVE-2026-7210 pkg:python3.11-minimal exp:2026-07-15
|
||||
CVE-2026-7210 pkg:libpython3.11 exp:2026-07-15
|
||||
CVE-2026-7210 pkg:libpython3.11-dev exp:2026-07-15
|
||||
CVE-2026-7210 pkg:libpython3.11-minimal exp:2026-07-15
|
||||
CVE-2026-7210 pkg:libpython3.11-stdlib exp:2026-07-15
|
||||
|
||||
# CVE-2026-33278 — Unbound DNSSEC validator use-after-free (DoS, possible RCE).
|
||||
# CVE-2026-42960 — Unbound DNS cache poisoning via promiscuous additional records.
|
||||
# Package: libunbound8.
|
||||
# Why ignored: libunbound8 is a transitive apt dependency of the TLS/networking
|
||||
# stack (GnuTLS DANE support); only the shared library ships in the image. Both
|
||||
# vulnerabilities require operating a live Unbound recursive DNSSEC validator
|
||||
# that processes attacker-influenced DNS responses. Prowler never starts an
|
||||
# Unbound resolver, so neither code path is reachable. No Debian bookworm fix yet.
|
||||
CVE-2026-33278 pkg:libunbound8 exp:2026-07-15
|
||||
CVE-2026-42960 pkg:libunbound8 exp:2026-07-15
|
||||
@@ -1,2 +1,3 @@
|
||||
.envrc
|
||||
ui/.env.local
|
||||
openspec/
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
Use these skills for detailed patterns on-demand:
|
||||
|
||||
### Generic Skills (Any Project)
|
||||
|
||||
| Skill | Description | URL |
|
||||
|-------|-------------|-----|
|
||||
| `typescript` | Const types, flat interfaces, utility types | [SKILL.md](skills/typescript/SKILL.md) |
|
||||
@@ -28,6 +29,7 @@ Use these skills for detailed patterns on-demand:
|
||||
| `tdd` | Test-Driven Development workflow | [SKILL.md](skills/tdd/SKILL.md) |
|
||||
|
||||
### Prowler-Specific Skills
|
||||
|
||||
| Skill | Description | URL |
|
||||
|-------|-------------|-----|
|
||||
| `prowler` | Project overview, component navigation | [SKILL.md](skills/prowler/SKILL.md) |
|
||||
@@ -49,6 +51,7 @@ Use these skills for detailed patterns on-demand:
|
||||
| `django-migration-psql` | Django migration best practices for PostgreSQL | [SKILL.md](skills/django-migration-psql/SKILL.md) |
|
||||
| `postgresql-indexing` | PostgreSQL indexing, EXPLAIN, monitoring, maintenance | [SKILL.md](skills/postgresql-indexing/SKILL.md) |
|
||||
| `prowler-attack-paths-query` | Create Attack Paths openCypher queries | [SKILL.md](skills/prowler-attack-paths-query/SKILL.md) |
|
||||
| `prowler-tour` | Keep product-tour definitions aligned with the UI | [SKILL.md](skills/prowler-tour/SKILL.md) |
|
||||
| `gh-aw` | GitHub Agentic Workflows (gh-aw) | [SKILL.md](skills/gh-aw/SKILL.md) |
|
||||
| `skill-creator` | Create new AI agent skills | [SKILL.md](skills/skill-creator/SKILL.md) |
|
||||
|
||||
@@ -65,10 +68,12 @@ When performing these actions, ALWAYS invoke the corresponding skill FIRST:
|
||||
| Adding new providers | `prowler-provider` |
|
||||
| Adding privilege escalation detection queries | `prowler-attack-paths-query` |
|
||||
| Adding services to existing providers | `prowler-provider` |
|
||||
| Adding, updating, or removing a tour definition (*.tour.ts) | `prowler-tour` |
|
||||
| After creating/modifying a skill | `skill-sync` |
|
||||
| App Router / Server Actions | `nextjs-16` |
|
||||
| Auditing check-to-requirement mappings as a cloud auditor | `prowler-compliance` |
|
||||
| Building AI chat features | `ai-sdk-5` |
|
||||
| Changing button labels or section headings on a tour-covered page | `prowler-tour` |
|
||||
| Committing changes | `prowler-commit` |
|
||||
| Configuring MCP servers in agentic workflows | `gh-aw` |
|
||||
| Create PR that requires changelog entry | `prowler-changelog` |
|
||||
@@ -87,6 +92,7 @@ When performing these actions, ALWAYS invoke the corresponding skill FIRST:
|
||||
| Creating/updating compliance frameworks | `prowler-compliance` |
|
||||
| Debug why a GitHub Actions job is failing | `prowler-ci` |
|
||||
| Debugging gh-aw compilation errors | `gh-aw` |
|
||||
| Editing a UI file containing data-tour-id attributes | `prowler-tour` |
|
||||
| Fill .github/pull_request_template.md (Context/Description/Steps to review/Checklist) | `prowler-pr` |
|
||||
| Fixing bug | `tdd` |
|
||||
| Fixing compliance JSON bugs (duplicate IDs, empty Section, stale refs) | `prowler-compliance` |
|
||||
@@ -103,9 +109,12 @@ When performing these actions, ALWAYS invoke the corresponding skill FIRST:
|
||||
| Modifying gh-aw workflow frontmatter or safe-outputs | `gh-aw` |
|
||||
| Refactoring code | `tdd` |
|
||||
| Regenerate AGENTS.md Auto-invoke tables (sync.sh) | `skill-sync` |
|
||||
| Renaming or removing a data-tour-id attribute value | `prowler-tour` |
|
||||
| Restructuring routes or layouts covered by a tour | `prowler-tour` |
|
||||
| Review PR requirements: template, title conventions, changelog gate | `prowler-pr` |
|
||||
| Review changelog format and conventions | `prowler-changelog` |
|
||||
| Reviewing JSON:API compliance | `jsonapi` |
|
||||
| Reviewing Prowler UI components | `prowler-ui` |
|
||||
| Reviewing compliance framework PRs | `prowler-compliance-review` |
|
||||
| Running makemigrations or pgmakemigrations | `django-migration-psql` |
|
||||
| Syncing compliance framework with upstream catalog | `prowler-compliance` |
|
||||
|
||||
+4
-3
@@ -1,4 +1,4 @@
|
||||
# Do you want to learn on how to...
|
||||
# Do you want to learn on how to
|
||||
|
||||
- [Contribute with your code or fixes to Prowler](https://docs.prowler.com/developer-guide/introduction)
|
||||
- [Create a new provider](https://docs.prowler.com/developer-guide/provider)
|
||||
@@ -32,5 +32,6 @@ Provider-specific developer notes:
|
||||
|
||||
Want some swag as appreciation for your contribution?
|
||||
|
||||
# Prowler Developer Guide
|
||||
https://goto.prowler.com/devguide
|
||||
## Prowler Developer Guide
|
||||
|
||||
<https://goto.prowler.com/devguide>
|
||||
|
||||
+21
-9
@@ -1,4 +1,4 @@
|
||||
FROM python:3.12.11-slim-bookworm@sha256:519591d6871b7bc437060736b9f7456b8731f1499a57e22e6c285135ae657bf7 AS build
|
||||
FROM python:3.12.13-slim-bookworm@sha256:8a7e7cc04fd3e2bd787f7f24e22d5d119aa590d429b50c95dfe12b3abe52f48b AS build
|
||||
|
||||
LABEL maintainer="https://github.com/prowler-cloud/prowler"
|
||||
LABEL org.opencontainers.image.source="https://github.com/prowler-cloud/prowler"
|
||||
@@ -6,7 +6,7 @@ LABEL org.opencontainers.image.source="https://github.com/prowler-cloud/prowler"
|
||||
ARG POWERSHELL_VERSION=7.5.0
|
||||
ENV POWERSHELL_VERSION=${POWERSHELL_VERSION}
|
||||
|
||||
ARG TRIVY_VERSION=0.70.0
|
||||
ARG TRIVY_VERSION=0.71.2
|
||||
ENV TRIVY_VERSION=${TRIVY_VERSION}
|
||||
|
||||
ARG ZIZMOR_VERSION=1.24.1
|
||||
@@ -76,11 +76,11 @@ USER prowler
|
||||
WORKDIR /home/prowler
|
||||
|
||||
# Copy necessary files
|
||||
COPY prowler/ /home/prowler/prowler/
|
||||
COPY dashboard/ /home/prowler/dashboard/
|
||||
COPY pyproject.toml uv.lock /home/prowler/
|
||||
COPY README.md /home/prowler/
|
||||
COPY prowler/providers/m365/lib/powershell/m365_powershell.py /home/prowler/prowler/providers/m365/lib/powershell/m365_powershell.py
|
||||
COPY --chown=prowler:prowler prowler/ /home/prowler/prowler/
|
||||
COPY --chown=prowler:prowler dashboard/ /home/prowler/dashboard/
|
||||
COPY --chown=prowler:prowler pyproject.toml uv.lock /home/prowler/
|
||||
COPY --chown=prowler:prowler README.md /home/prowler/
|
||||
COPY --chown=prowler:prowler prowler/providers/m365/lib/powershell/m365_powershell.py /home/prowler/prowler/providers/m365/lib/powershell/m365_powershell.py
|
||||
|
||||
# Install Python dependencies
|
||||
ENV HOME='/home/prowler'
|
||||
@@ -89,15 +89,27 @@ ENV PATH="${HOME}/.local/bin:${PATH}"
|
||||
RUN pip install --no-cache-dir --upgrade pip && \
|
||||
pip install --no-cache-dir uv==0.11.14
|
||||
|
||||
RUN uv sync --compile-bytecode && \
|
||||
RUN uv sync --locked --compile-bytecode && \
|
||||
rm -rf ~/.cache/uv
|
||||
|
||||
# Install PowerShell modules
|
||||
RUN .venv/bin/python prowler/providers/m365/lib/powershell/m365_powershell.py
|
||||
|
||||
USER root
|
||||
|
||||
# Remove build-only packages from the final image after Python dependencies are installed.
|
||||
RUN apt-get purge -y --auto-remove \
|
||||
build-essential \
|
||||
pkg-config \
|
||||
libzstd-dev \
|
||||
zlib1g-dev \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
USER prowler
|
||||
|
||||
# Remove deprecated dash dependencies
|
||||
RUN pip uninstall dash-html-components -y && \
|
||||
pip uninstall dash-core-components -y
|
||||
|
||||
USER prowler
|
||||
ENTRYPOINT [".venv/bin/prowler"]
|
||||
ENTRYPOINT ["/home/prowler/.venv/bin/prowler"]
|
||||
|
||||
@@ -1,5 +1,34 @@
|
||||
.DEFAULT_GOAL:=help
|
||||
|
||||
DEV_LOCAL := ./scripts/development/dev-local.sh
|
||||
|
||||
.PHONY: dev dev-setup dev-attach dev-launch dev-stop dev-clean dev-wipe dev-status
|
||||
|
||||
##@ Local Development
|
||||
dev: ## Start local API, worker, and database logs
|
||||
$(DEV_LOCAL) all
|
||||
|
||||
dev-setup: ## Bootstrap local dependencies, migrations, and fixtures
|
||||
$(DEV_LOCAL) setup
|
||||
|
||||
dev-attach: ## Attach to the local tmux development session
|
||||
$(DEV_LOCAL) attach
|
||||
|
||||
dev-launch: ## Start the local stack on fixed ports and attach
|
||||
$(DEV_LOCAL) launch
|
||||
|
||||
dev-stop: ## Stop the local tmux session and containers
|
||||
$(DEV_LOCAL) kill
|
||||
|
||||
dev-clean: ## Remove stopped local development containers
|
||||
$(DEV_LOCAL) clean
|
||||
|
||||
dev-wipe: ## Stop everything and delete local development data
|
||||
$(DEV_LOCAL) wipe
|
||||
|
||||
dev-status: ## Show local development container status
|
||||
$(DEV_LOCAL) status
|
||||
|
||||
##@ Testing
|
||||
test: ## Test with pytest
|
||||
rm -rf .coverage && \
|
||||
@@ -16,18 +45,41 @@ coverage-html: ## Show Test Coverage
|
||||
coverage html && \
|
||||
open htmlcov/index.html
|
||||
|
||||
##@ Linting
|
||||
format: ## Format Code
|
||||
@echo "Running black..."
|
||||
black .
|
||||
##@ Code Quality
|
||||
# `make` is the single entrypoint and mirrors CI exactly (uv run + same flags):
|
||||
# SDK (prowler/, util/) -> flake8 + black + pylint
|
||||
# API & MCP server -> ruff (rules live in each project's pyproject.toml)
|
||||
# `format` applies fixes (incl. ruff's import/upgrade autofixes); `lint` only
|
||||
# verifies and is what CI gates on.
|
||||
.PHONY: format format-sdk format-api format-mcp lint lint-sdk lint-api lint-mcp
|
||||
|
||||
lint: ## Lint Code
|
||||
@echo "Running flake8..."
|
||||
flake8 . --ignore=E266,W503,E203,E501,W605,E128 --exclude .venv,contrib
|
||||
@echo "Running black... "
|
||||
black --check .
|
||||
@echo "Running pylint..."
|
||||
pylint --disable=W,C,R,E -j 0 prowler util
|
||||
format: format-sdk format-api format-mcp ## Format & autofix all components (SDK, API, MCP)
|
||||
|
||||
lint: lint-sdk lint-api lint-mcp ## Lint all components (SDK, API, MCP) — mirrors CI
|
||||
|
||||
format-sdk: ## Format SDK code (black)
|
||||
uv run black --exclude "\.venv|api|ui|skills|mcp_server" .
|
||||
|
||||
lint-sdk: ## Lint SDK code (flake8, black --check, pylint)
|
||||
uv run flake8 . --ignore=E266,W503,E203,E501,W605,E128 --exclude .venv,contrib,ui,api,skills,mcp_server
|
||||
uv run black --exclude "\.venv|api|ui|skills|mcp_server" --check .
|
||||
uv run pylint --disable=W,C,R,E -j 0 -rn -sn prowler/
|
||||
|
||||
format-api: ## Format & autofix API code (ruff)
|
||||
cd api && uv run ruff check . --exclude contrib --fix
|
||||
cd api && uv run ruff format . --exclude contrib
|
||||
|
||||
lint-api: ## Lint API code (ruff check + format --check)
|
||||
cd api && uv run ruff check . --exclude contrib
|
||||
cd api && uv run ruff format --check . --exclude contrib
|
||||
|
||||
format-mcp: ## Format & autofix MCP server code (ruff)
|
||||
cd mcp_server && uv run ruff check . --fix
|
||||
cd mcp_server && uv run ruff format .
|
||||
|
||||
lint-mcp: ## Lint MCP server code (ruff check + format --check)
|
||||
cd mcp_server && uv run ruff check .
|
||||
cd mcp_server && uv run ruff format --check .
|
||||
|
||||
##@ PyPI
|
||||
pypi-clean: ## Delete the distribution files
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
<p align="center">
|
||||
<img align="center" src="https://github.com/prowler-cloud/prowler/blob/master/docs/img/prowler-logo-black.png#gh-light-mode-only" width="50%" height="50%">
|
||||
<img align="center" src="https://github.com/prowler-cloud/prowler/blob/master/docs/img/prowler-logo-white.png#gh-dark-mode-only" width="50%" height="50%">
|
||||
<img align="center" alt="Prowler logo" src="https://github.com/prowler-cloud/prowler/blob/master/docs/img/prowler-logo-black.png#gh-light-mode-only" width="50%" height="50%">
|
||||
<img align="center" alt="Prowler logo" src="https://github.com/prowler-cloud/prowler/blob/master/docs/img/prowler-logo-white.png#gh-dark-mode-only" width="50%" height="50%">
|
||||
</p>
|
||||
<p align="center">
|
||||
<b><i>Prowler</b> is the Open Cloud Security Platform trusted by thousands to automate security and compliance in any cloud environment. With hundreds of ready-to-use checks and compliance frameworks, Prowler delivers real-time, customizable monitoring and seamless integrations, making cloud security simple, scalable, and cost-effective for organizations of any size.
|
||||
@@ -22,8 +22,8 @@
|
||||
<a href="https://pypistats.org/packages/prowler"><img alt="PyPI Downloads" src="https://img.shields.io/pypi/dw/prowler.svg?label=downloads"></a>
|
||||
<a href="https://hub.docker.com/r/toniblyx/prowler"><img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/toniblyx/prowler"></a>
|
||||
<a href="https://gallery.ecr.aws/prowler-cloud/prowler"><img width="120" height=19" alt="AWS ECR Gallery" src="https://user-images.githubusercontent.com/3985464/151531396-b6535a68-c907-44eb-95a1-a09508178616.png"></a>
|
||||
<a href="https://codecov.io/gh/prowler-cloud/prowler"><img src="https://codecov.io/gh/prowler-cloud/prowler/graph/badge.svg?token=OflBGsdpDl"/></a>
|
||||
<a href="https://insights.linuxfoundation.org/project/prowler-cloud-prowler"><img src="https://insights.linuxfoundation.org/api/badge/health-score?project=prowler-cloud-prowler"/></a>
|
||||
<a href="https://codecov.io/gh/prowler-cloud/prowler"><img alt="Codecov coverage" src="https://codecov.io/gh/prowler-cloud/prowler/graph/badge.svg?token=OflBGsdpDl"/></a>
|
||||
<a href="https://insights.linuxfoundation.org/project/prowler-cloud-prowler"><img alt="Linux Foundation insights health score" src="https://insights.linuxfoundation.org/api/badge/health-score?project=prowler-cloud-prowler"/></a>
|
||||
</p>
|
||||
<p align="center">
|
||||
<a href="https://github.com/prowler-cloud/prowler/releases"><img alt="Version" src="https://img.shields.io/github/v/release/prowler-cloud/prowler"></a>
|
||||
@@ -36,7 +36,7 @@
|
||||
</p>
|
||||
<hr>
|
||||
<p align="center">
|
||||
<img align="center" src="/docs/img/prowler-cloud.gif" width="100%" height="100%">
|
||||
<img align="center" alt="Prowler Cloud demo" src="/docs/img/prowler-cloud.gif" width="100%" height="100%">
|
||||
</p>
|
||||
|
||||
# Description
|
||||
@@ -83,16 +83,35 @@ prowler dashboard
|
||||
|
||||
## Attack Paths
|
||||
|
||||
Attack Paths automatically extends every completed AWS scan with a Neo4j graph that combines Cartography's cloud inventory with Prowler findings. The feature runs in the API worker after each scan and therefore requires:
|
||||
Attack Paths automatically extends every completed AWS scan with a graph that combines Cartography's cloud inventory with Prowler findings. The feature runs in the API worker after each scan.
|
||||
|
||||
- An accessible Neo4j instance (the Docker Compose files already ships a `neo4j` service).
|
||||
- The following environment variables so Django and Celery can connect:
|
||||
Two graph backends are supported as the long-lived sink:
|
||||
|
||||
| Variable | Description | Default |
|
||||
| --- | --- | --- |
|
||||
| `NEO4J_HOST` | Hostname used by the API containers. | `neo4j` |
|
||||
| `NEO4J_PORT` | Bolt port exposed by Neo4j. | `7687` |
|
||||
| `NEO4J_USER` / `NEO4J_PASSWORD` | Credentials with rights to create per-tenant databases. | `neo4j` / `neo4j_password` |
|
||||
- **Neo4j** (default; the Docker Compose files already ship a `neo4j` service).
|
||||
- **Amazon Neptune** (cloud-managed; opt-in).
|
||||
|
||||
Select the sink with `ATTACK_PATHS_SINK_DATABASE` (`neo4j` or `neptune`; default `neo4j`).
|
||||
|
||||
> Note: Cartography ingestion always uses a temporary Neo4j database, regardless of the configured sink. The `NEO4J_*` variables below must remain set even when `ATTACK_PATHS_SINK_DATABASE=neptune`.
|
||||
|
||||
### Neo4j sink
|
||||
|
||||
| Variable | Description | Default |
|
||||
| --- | --- | --- |
|
||||
| `NEO4J_HOST` | Hostname used by the API containers. | `neo4j` |
|
||||
| `NEO4J_PORT` | Bolt port exposed by Neo4j. | `7687` |
|
||||
| `NEO4J_USER` / `NEO4J_PASSWORD` | Credentials with rights to create per-tenant databases. | `neo4j` / `neo4j_password` |
|
||||
|
||||
### Neptune sink
|
||||
|
||||
| Variable | Description | Default |
|
||||
| --- | --- | --- |
|
||||
| `NEPTUNE_WRITER_ENDPOINT` | Bolt host for the Neptune writer instance. Required when sink is `neptune`. | _empty_ |
|
||||
| `NEPTUNE_READER_ENDPOINT` | Optional reader endpoint for read-only queries. Falls back to the writer when unset. | _empty_ |
|
||||
| `NEPTUNE_PORT` | Bolt port exposed by Neptune. | `8182` |
|
||||
| `AWS_REGION` | Region the Neptune cluster lives in. Required when sink is `neptune`. | _empty_ |
|
||||
|
||||
Neptune authenticates with SigV4 using the standard boto3 credential chain. The worker's IAM role (or `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY`) supplies the credentials. There is no Neptune password variable.
|
||||
|
||||
Every AWS provider scan will enqueue an Attack Paths ingestion job automatically. Other cloud providers will be added in future iterations.
|
||||
|
||||
@@ -104,25 +123,27 @@ Every AWS provider scan will enqueue an Attack Paths ingestion job automatically
|
||||
|
||||
| Provider | Checks | Services | [Compliance Frameworks](https://docs.prowler.com/projects/prowler-open-source/en/latest/tutorials/compliance/) | [Categories](https://docs.prowler.com/projects/prowler-open-source/en/latest/tutorials/misc/#categories) | Support | Interface |
|
||||
|---|---|---|---|---|---|---|
|
||||
| AWS | 600 | 84 | 44 | 18 | Official | UI, API, CLI |
|
||||
| Azure | 167 | 22 | 19 | 16 | Official | UI, API, CLI |
|
||||
| GCP | 102 | 18 | 17 | 12 | Official | UI, API, CLI |
|
||||
| Kubernetes | 83 | 7 | 7 | 11 | Official | UI, API, CLI |
|
||||
| GitHub | 24 | 3 | 1 | 5 | Official | UI, API, CLI |
|
||||
| M365 | 102 | 10 | 4 | 10 | Official | UI, API, CLI |
|
||||
| OCI | 51 | 14 | 4 | 10 | Official | UI, API, CLI |
|
||||
| Alibaba Cloud | 63 | 9 | 4 | 9 | Official | UI, API, CLI |
|
||||
| Cloudflare | 29 | 3 | 0 | 5 | Official | UI, API, CLI |
|
||||
| AWS | 615 | 86 | 47 | 19 | Official | UI, API, CLI |
|
||||
| Azure | 190 | 22 | 21 | 16 | Official | UI, API, CLI |
|
||||
| GCP | 109 | 20 | 19 | 12 | Official | UI, API, CLI |
|
||||
| Kubernetes | 90 | 7 | 8 | 11 | Official | UI, API, CLI |
|
||||
| GitHub | 24 | 3 | 2 | 5 | Official | UI, API, CLI |
|
||||
| M365 | 109 | 10 | 6 | 10 | Official | UI, API, CLI |
|
||||
| OCI | 52 | 14 | 5 | 10 | Official | UI, API, CLI |
|
||||
| Alibaba Cloud | 63 | 9 | 6 | 9 | Official | UI, API, CLI |
|
||||
| Cloudflare | 29 | 3 | 2 | 5 | Official | UI, API, CLI |
|
||||
| IaC | [See `trivy` docs.](https://trivy.dev/latest/docs/coverage/iac/) | N/A | N/A | N/A | Official | UI, API, CLI |
|
||||
| MongoDB Atlas | 10 | 3 | 0 | 8 | Official | UI, API, CLI |
|
||||
| MongoDB Atlas | 10 | 3 | 1 | 8 | Official | UI, API, CLI |
|
||||
| LLM | [See `promptfoo` docs.](https://www.promptfoo.dev/docs/red-team/plugins/) | N/A | N/A | N/A | Official | CLI |
|
||||
| Image | N/A | N/A | N/A | N/A | Official | CLI, API |
|
||||
| Google Workspace | 39 | 5 | 2 | 5 | Official | UI, API, CLI |
|
||||
| OpenStack | 34 | 5 | 0 | 9 | Official | UI, API, CLI |
|
||||
| Vercel | 26 | 6 | 0 | 8 | Official | UI, API, CLI |
|
||||
| Okta | 1 | 1 | 0 | 1 | Official | CLI |
|
||||
| Scaleway [Contact us](https://prowler.com/contact) | 1 | 1 | 0 | 1 | Unofficial | CLI |
|
||||
| NHN | 6 | 2 | 1 | 0 | Unofficial | CLI |
|
||||
| Google Workspace | 65 | 11 | 3 | 6 | Official | UI, API, CLI |
|
||||
| OpenStack | 34 | 5 | 1 | 9 | Official | UI, API, CLI |
|
||||
| Vercel | 26 | 6 | 1 | 8 | Official | UI, API, CLI |
|
||||
| Okta | 29 | 8 | 2 | 2 | Official | UI, API, CLI |
|
||||
| Linode [Contact us](https://prowler.com/contact) | 10 | 3 | 1 | 4 | Unofficial | CLI |
|
||||
| Scaleway [Contact us](https://prowler.com/contact) | 1 | 1 | 1 | 1 | Unofficial | CLI |
|
||||
| StackIT [Contact us](https://prowler.com/contact) | 7 | 2 | 1 | 3 | Unofficial | CLI |
|
||||
| NHN | 6 | 2 | 2 | 0 | Unofficial | CLI |
|
||||
|
||||
> [!Note]
|
||||
> The numbers in the table are updated periodically.
|
||||
@@ -146,11 +167,13 @@ Prowler App offers flexible installation methods tailored to various environment
|
||||
|
||||
### Docker Compose
|
||||
|
||||
**Requirements**
|
||||
#### Requirements
|
||||
|
||||
* `Docker Compose` installed: https://docs.docker.com/compose/install/.
|
||||
- `Docker Compose` installed: https://docs.docker.com/compose/install/.
|
||||
|
||||
**Commands**
|
||||
#### Commands
|
||||
|
||||
_macOS/Linux:_
|
||||
|
||||
``` console
|
||||
VERSION=$(curl -s https://api.github.com/repos/prowler-cloud/prowler/releases/latest | jq -r .tag_name)
|
||||
@@ -160,6 +183,16 @@ curl -sLO "https://raw.githubusercontent.com/prowler-cloud/prowler/refs/tags/${V
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
_Windows PowerShell:_
|
||||
|
||||
``` powershell
|
||||
$VERSION = (Invoke-RestMethod -Uri "https://api.github.com/repos/prowler-cloud/prowler/releases/latest").tag_name
|
||||
Invoke-WebRequest -Uri "https://raw.githubusercontent.com/prowler-cloud/prowler/refs/tags/$VERSION/docker-compose.yml" -OutFile "docker-compose.yml"
|
||||
# Environment variables can be customized in the .env file. Using default values in production environments is not recommended.
|
||||
Invoke-WebRequest -Uri "https://raw.githubusercontent.com/prowler-cloud/prowler/refs/tags/$VERSION/.env" -OutFile ".env"
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
> [!WARNING]
|
||||
> 🔒 For a secure setup, the API auto-generates a unique key pair, `DJANGO_TOKEN_SIGNING_KEY` and `DJANGO_TOKEN_VERIFYING_KEY`, and stores it in `~/.config/prowler-api` (non-container) or the bound Docker volume in `_data/api` (container). Never commit or reuse static/default keys. To rotate keys, delete the stored key files and restart the API.
|
||||
|
||||
@@ -175,14 +208,14 @@ You can find more information in the [Troubleshooting](./docs/troubleshooting.md
|
||||
|
||||
### From GitHub
|
||||
|
||||
**Requirements**
|
||||
#### Requirements
|
||||
|
||||
* `git` installed.
|
||||
* `uv` installed: [uv installation](https://docs.astral.sh/uv/getting-started/installation/).
|
||||
* `pnpm` installed: [pnpm installation](https://pnpm.io/installation).
|
||||
* `Docker Compose` installed: https://docs.docker.com/compose/install/.
|
||||
- `git` installed.
|
||||
- `uv` installed: [uv installation](https://docs.astral.sh/uv/getting-started/installation/).
|
||||
- `pnpm` installed: [pnpm installation](https://pnpm.io/installation).
|
||||
- `Docker Compose` installed: https://docs.docker.com/compose/install/.
|
||||
|
||||
**Commands to run the API**
|
||||
#### Commands to run the API
|
||||
|
||||
``` console
|
||||
git clone https://github.com/prowler-cloud/prowler
|
||||
@@ -199,7 +232,7 @@ gunicorn -c config/guniconf.py config.wsgi:application
|
||||
|
||||
> After completing the setup, access the API documentation at http://localhost:8080/api/v1/docs.
|
||||
|
||||
**Commands to run the API Worker**
|
||||
#### Commands to run the API Worker
|
||||
|
||||
``` console
|
||||
git clone https://github.com/prowler-cloud/prowler
|
||||
@@ -212,7 +245,7 @@ cd src/backend
|
||||
python -m celery -A config.celery worker -l info -E
|
||||
```
|
||||
|
||||
**Commands to run the API Scheduler**
|
||||
#### Commands to run the API Scheduler
|
||||
|
||||
``` console
|
||||
git clone https://github.com/prowler-cloud/prowler
|
||||
@@ -225,7 +258,7 @@ cd src/backend
|
||||
python -m celery -A config.celery beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler
|
||||
```
|
||||
|
||||
**Commands to run the UI**
|
||||
#### Commands to run the UI
|
||||
|
||||
``` console
|
||||
git clone https://github.com/prowler-cloud/prowler
|
||||
@@ -237,7 +270,7 @@ pnpm start
|
||||
|
||||
> Once configured, access the Prowler App at http://localhost:3000. Sign up using your email and password to get started.
|
||||
|
||||
**Pre-commit Hooks Setup**
|
||||
#### Pre-commit Hooks Setup
|
||||
|
||||
Some pre-commit hooks require tools installed on your system:
|
||||
|
||||
@@ -257,14 +290,14 @@ prowler -v
|
||||
|
||||
### Containers
|
||||
|
||||
**Available Versions of Prowler CLI**
|
||||
#### Available Versions of Prowler CLI
|
||||
|
||||
The following versions of Prowler CLI are available, depending on your requirements:
|
||||
|
||||
- `latest`: Synchronizes with the `master` branch. Note that this version is not stable.
|
||||
- `v4-latest`: Synchronizes with the `v4` branch. Note that this version is not stable.
|
||||
- `v3-latest`: Synchronizes with the `v3` branch. Note that this version is not stable.
|
||||
- `<x.y.z>` (release): Stable releases corresponding to specific versions. You can find the complete list of releases [here](https://github.com/prowler-cloud/prowler/releases).
|
||||
- `<x.y.z>` (release): Stable releases corresponding to specific versions. See the [complete list of Prowler releases](https://github.com/prowler-cloud/prowler/releases).
|
||||
- `stable`: Always points to the latest release.
|
||||
- `v4-stable`: Always points to the latest release for v4.
|
||||
- `v3-stable`: Always points to the latest release for v3.
|
||||
@@ -293,7 +326,7 @@ python prowler-cli.py -v
|
||||
|
||||
# 🛡️ GitHub Action
|
||||
|
||||
The official **Prowler GitHub Action** runs Prowler scans in your GitHub workflows using the official [`prowlercloud/prowler`](https://hub.docker.com/r/prowlercloud/prowler) Docker image. Scans run on any [supported provider](https://docs.prowler.com/user-guide/providers/), with optional [`--push-to-cloud`](https://docs.prowler.com/user-guide/tutorials/prowler-app-import-findings) to send findings to Prowler Cloud and optional SARIF upload so findings show up in the repo's **Security → Code scanning** tab and as inline PR annotations.
|
||||
The official **Prowler GitHub Action** runs Prowler scans in your GitHub workflows using the official [`prowlercloud/prowler`](https://hub.docker.com/r/prowlercloud/prowler) Docker image. Scans run on any [supported provider](https://docs.prowler.com/user-guide/providers/), with optional [`--push-to-cloud`](https://docs.prowler.com/user-guide/tutorials/prowler-import-findings) to send findings to Prowler Cloud and optional SARIF upload so findings show up in the repo's **Security → Code scanning** tab and as inline PR annotations.
|
||||
|
||||
```yaml
|
||||
name: Prowler IaC Scan
|
||||
@@ -338,7 +371,7 @@ Full configuration, per-provider authentication, and SARIF examples: [Prowler Gi
|
||||
|
||||
## Prowler CLI
|
||||
|
||||
**Running Prowler**
|
||||
### Running Prowler
|
||||
|
||||
Prowler can be executed across various environments, offering flexibility to meet your needs. It can be run from:
|
||||
|
||||
|
||||
+2
-2
@@ -22,7 +22,7 @@ inputs:
|
||||
required: false
|
||||
default: json-ocsf
|
||||
push-to-cloud:
|
||||
description: Push scan findings to Prowler Cloud. Requires the PROWLER_CLOUD_API_KEY environment variable. See https://docs.prowler.com/user-guide/tutorials/prowler-app-import-findings#using-the-cli
|
||||
description: Push scan findings to Prowler Cloud. Requires the PROWLER_CLOUD_API_KEY environment variable. See https://docs.prowler.com/user-guide/tutorials/prowler-import-findings#using-the-cli
|
||||
required: false
|
||||
default: "false"
|
||||
flags:
|
||||
@@ -299,7 +299,7 @@ runs:
|
||||
echo ""
|
||||
echo "**Get started in 3 steps:**"
|
||||
echo "1. Create an account at [cloud.prowler.com](https://cloud.prowler.com)"
|
||||
echo "2. Generate a Prowler Cloud API key ([docs](https://docs.prowler.com/user-guide/tutorials/prowler-app-import-findings#using-the-cli))"
|
||||
echo "2. Generate a Prowler Cloud API key ([docs](https://docs.prowler.com/user-guide/tutorials/prowler-import-findings#using-the-cli))"
|
||||
echo "3. Add \`PROWLER_CLOUD_API_KEY\` to your GitHub secrets and set \`push-to-cloud: true\` on this action"
|
||||
echo ""
|
||||
echo "See [prowler.com/pricing](https://prowler.com/pricing) for plan details."
|
||||
|
||||
@@ -24,6 +24,9 @@ DJANGO_THROTTLE_TOKEN_OBTAIN=50/minute
|
||||
# Decide whether to allow Django manage database table partitions
|
||||
DJANGO_MANAGE_DB_PARTITIONS=[True|False]
|
||||
DJANGO_CELERY_DEADLOCK_ATTEMPTS=5
|
||||
# Optional: bound Celery's prefork pool size. Unset → Celery uses os.cpu_count().
|
||||
# Useful on Kubernetes nodes with many CPUs where unbounded prefork balloons memory.
|
||||
# DJANGO_CELERY_WORKER_CONCURRENCY=4
|
||||
DJANGO_BROKER_VISIBILITY_TIMEOUT=86400
|
||||
DJANGO_SENTRY_DSN=
|
||||
|
||||
|
||||
+4
-4
@@ -10,7 +10,7 @@
|
||||
> - [`jsonapi`](../skills/jsonapi/SKILL.md) - Strict JSON:API v1.1 spec compliance
|
||||
> - [`pytest`](../skills/pytest/SKILL.md) - Generic pytest patterns
|
||||
|
||||
### Auto-invoke Skills
|
||||
## Auto-invoke Skills
|
||||
|
||||
When performing these actions, ALWAYS invoke the corresponding skill FIRST:
|
||||
|
||||
@@ -81,7 +81,7 @@ When performing these actions, ALWAYS invoke the corresponding skill FIRST:
|
||||
## DECISION TREES
|
||||
|
||||
### Serializer Selection
|
||||
```
|
||||
```text
|
||||
Read → <Model>Serializer
|
||||
Create → <Model>CreateSerializer
|
||||
Update → <Model>UpdateSerializer
|
||||
@@ -89,7 +89,7 @@ Nested read → <Model>IncludeSerializer
|
||||
```
|
||||
|
||||
### Task vs View
|
||||
```
|
||||
```text
|
||||
< 100ms → View
|
||||
> 100ms or external API → Celery task
|
||||
Needs retry → Celery task
|
||||
@@ -105,7 +105,7 @@ Django 5.1.x | DRF 3.15.x | djangorestframework-jsonapi 7.x | Celery 5.4.x | Pos
|
||||
|
||||
## PROJECT STRUCTURE
|
||||
|
||||
```
|
||||
```text
|
||||
api/src/backend/
|
||||
├── api/ # Main Django app
|
||||
│ ├── v1/ # API version 1 (views, serializers, urls)
|
||||
|
||||
+151
-1
@@ -2,6 +2,157 @@
|
||||
|
||||
All notable changes to the **Prowler API** are documented in this file.
|
||||
|
||||
## [1.33.0] (Prowler v5.32.0)
|
||||
|
||||
### 🚀 Added
|
||||
|
||||
- Timestamp precision support for `/api/v1/findings` `inserted_at` and `updated_at` filters [(#11754)](https://github.com/prowler-cloud/prowler/pull/11754)
|
||||
|
||||
### 🔄 Changed
|
||||
|
||||
- Attack Paths: AWS Neptune is now supported as a persistent sink database, selectable via `ATTACK_PATHS_SINK_DATABASE=neptune` (default `neo4j`), Cartography's (bumped to 0.138.1) per-scan ingest database stays on Neo4j [(#11524)](https://github.com/prowler-cloud/prowler/pull/11524)
|
||||
- Attack Paths: Scan task now checks the ingest Neo4j database and configured graph sink before starting graph ingestion [(#11743)](https://github.com/prowler-cloud/prowler/pull/11743)
|
||||
- Disable PowerShell telemetry in the API container image [(#11746)](https://github.com/prowler-cloud/prowler/pull/11746)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- Attack Paths: Provider graph cleanup now deletes Neo4j and Neptune relationships in directed batches before deleting nodes [(#11755)](https://github.com/prowler-cloud/prowler/pull/11755)
|
||||
- `scan-perform` no longer reports an error when a provider is deleted during a running scan [(#11696)](https://github.com/prowler-cloud/prowler/pull/11696)
|
||||
|
||||
|
||||
---
|
||||
|
||||
## [1.32.1] (Prowler v5.31.1)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- API key auth no longer mutates `TenantAPIKey.objects` during admin DB lookups [(#11686)](https://github.com/prowler-cloud/prowler/pull/11686)
|
||||
|
||||
---
|
||||
|
||||
## [1.32.0] (Prowler v5.31.0)
|
||||
|
||||
### 🚀 Added
|
||||
|
||||
- Provider group filters for API endpoints that support cloud provider filtering, including exact and `__in` variants [(#11573)](https://github.com/prowler-cloud/prowler/pull/11573)
|
||||
- Provider filters for `GET /api/v1/compliance-overviews`, `/metadata`, and `/requirements`, using latest completed scans per matching provider [(#11587)](https://github.com/prowler-cloud/prowler/pull/11587)
|
||||
- Server-Sent Events (SSE) infrastructure for the API: a base viewset, a tenant-aware channel manager, and channel-name helpers backed by `django-eventstream` over Valkey Pub/Sub and served through the Gunicorn ASGI worker, so feature endpoints can stream events to clients over a single long-lived connection [(#11556)](https://github.com/prowler-cloud/prowler/pull/11556)
|
||||
- `DJANGO_CELERY_WORKER_CONCURRENCY` to configure Celery workers concurrency. Unset for default behaviour [(#11075)](https://github.com/prowler-cloud/prowler/pull/11075)
|
||||
|
||||
### 🔄 Changed
|
||||
|
||||
- Gunicorn worker timeout raised from the 30s default to 120s, so long-running requests are no longer killed prematurely [(#11631)](https://github.com/prowler-cloud/prowler/pull/11631)
|
||||
- Sentry now drops ASGI's `RequestAborted` errors from health-check probe disconnects on `/health/live` [(#11632)](https://github.com/prowler-cloud/prowler/pull/11632)
|
||||
- Gunicorn keep-alive timeout now exceeds the load balancer idle timeout, stopping 502s from reused connections [(#11647)](https://github.com/prowler-cloud/prowler/pull/11647)
|
||||
- API runs under the Uvicorn worker so keep-alive outlives the load balancer idle timeout, fixing Gunicorn's intermittent 502s [(#11663)](https://github.com/prowler-cloud/prowler/pull/11663)
|
||||
- SAML logins no longer wipe a user's roles when the IdP does not send the `userType` attribute; existing roles are kept, and when `userType` names a role that does not exist it is now created with read-only access (visibility over all providers, no management permissions) instead of no permissions at all [(#11520)](https://github.com/prowler-cloud/prowler/pull/11520)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- Database connections no longer leak under the ASGI worker, which previously exhausted the read replica's connection slots and caused 500s on read endpoints [(#11640)](https://github.com/prowler-cloud/prowler/pull/11640)
|
||||
|
||||
### 🔐 Security
|
||||
|
||||
- `aiohttp` to 3.14.0 and `idna` to 3.15, patching known CVEs [(#11596)](https://github.com/prowler-cloud/prowler/pull/11596)
|
||||
- Container base image to `python:3.12.13-slim-bookworm` and `trivy` to 0.71.0, patching OS and Go module CVEs [(#11596)](https://github.com/prowler-cloud/prowler/pull/11596)
|
||||
- `trivy` binary bumped to 0.71.0 patching embedded `golang.org/x/crypto`, `golang.org/x/net`, and Go `stdlib` CVEs [(#11592)](https://github.com/prowler-cloud/prowler/pull/11592)
|
||||
|
||||
---
|
||||
|
||||
## [1.31.3] (Prowler v5.30.3)
|
||||
|
||||
### 🔐 Security
|
||||
|
||||
- SAML logins now link to an existing account only when the asserted email domain matches the ACS endpoint and the user is already a member of that domain's tenant, fixing a cross-tenant account takeover [(GHSA-h8m9-jgf8-vwvp)](https://github.com/prowler-cloud/prowler/security/advisories/GHSA-h8m9-jgf8-vwvp)
|
||||
|
||||
---
|
||||
|
||||
## [1.31.2] (Prowler v5.30.2)
|
||||
|
||||
### 🔄 Changed
|
||||
|
||||
- `scan-compliance-overviews` task now streams the findings aggregation and the requirement-row writes so it runs faster and its peak memory no longer grows with the number of regions and frameworks [(#11591)](https://github.com/prowler-cloud/prowler/pull/11591)
|
||||
|
||||
---
|
||||
|
||||
## [1.31.1] (Prowler v5.30.1)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- `compliance-overviews/attributes` now resolves the provider from the scan, so multi-provider universal frameworks (e.g. CSA CCM) return the check IDs of the scan's provider and Azure/GCP requirement details show their findings instead of appearing empty [(#11546)](https://github.com/prowler-cloud/prowler/pull/11546)
|
||||
- Attack Paths: `drop_subgraph` now deletes relationships first and then nodes in batches, using less memory on Neo4j when clearing a dense provider graph [(#11557)](https://github.com/prowler-cloud/prowler/pull/11557)
|
||||
- OCI scans now use API key credentials with the configured region instead of falling back to `/home/prowler/.oci/config` [(#11558)](https://github.com/prowler-cloud/prowler/pull/11558)
|
||||
|
||||
---
|
||||
|
||||
## [1.31.0] (Prowler v5.30.0)
|
||||
|
||||
### 🚀 Added
|
||||
|
||||
- Opt-in automatic recovery of allowlisted idempotent background tasks whose worker died during a deploy or crash: when enabled via `DJANGO_TASK_RECOVERY_ENABLED` (off by default), stuck summary and deletion tasks are detected and re-run instead of staying pending forever (scan and Jira tasks are excluded), with a `reconcile_orphan_tasks` management command for on-demand recovery [(#11416)](https://github.com/prowler-cloud/prowler/pull/11416)
|
||||
- DORA compliance framework support [(#11131)](https://github.com/prowler-cloud/prowler/pull/11131)
|
||||
- Label Postgres connections with `application_name="<component>:<alias>"` (component injected per process via `DJANGO_APP_COMPONENT`) so connections are attributable by component in `pg_stat_activity` [(#11494)](https://github.com/prowler-cloud/prowler/pull/11494)
|
||||
- DISA Okta IDaaS STIG V1R2 compliance framework export support for the Okta provider [(#11428)](https://github.com/prowler-cloud/prowler/pull/11428)
|
||||
|
||||
### 🔄 Changed
|
||||
|
||||
- Allowlisted idempotent background tasks are no longer lost when a worker is stopped or crashes mid-task; tasks with external side effects are marked terminal instead of blindly re-running [(#11416)](https://github.com/prowler-cloud/prowler/pull/11416)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- Workers now shut down gracefully on deploy or restart, finishing or re-queueing in-flight tasks instead of being force-killed and leaving them stuck [(#11416)](https://github.com/prowler-cloud/prowler/pull/11416)
|
||||
- Resource `name` is now stored and refreshed on every scan, so resources no longer keep an empty name [(#11476)](https://github.com/prowler-cloud/prowler/pull/11476)
|
||||
- Compliance catalog now warms in background during startup. `compliance-overviews/attributes` returns `503` while warming, so the first request after a deploy no longer trips the API timeout [(#11530)](https://github.com/prowler-cloud/prowler/pull/11530)
|
||||
|
||||
### 🔐 Security
|
||||
|
||||
- `dulwich` from 0.23.0 to 1.2.5 and `pyjwt` from 2.12.1 to 2.13.0, patching `GHSA-897w-fcg9-f6xj` (arbitrary file write) and `PYSEC-2026-179` (HMAC/JWK key confusion) [(#11499)](https://github.com/prowler-cloud/prowler/pull/11499)
|
||||
|
||||
---
|
||||
|
||||
## [1.30.3] (Prowler v5.29.3)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- API startup no longer crashes when Neo4j is unreachable, as the Neo4j driver now connects lazily on first use rather than during app initialization [(#11491)](https://github.com/prowler-cloud/prowler/pull/11491)
|
||||
|
||||
---
|
||||
|
||||
## [1.30.1] (Prowler v5.29.1)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- `GET /api/v1/findings` N+1 query loading `resources__tags` when listing findings [(#11420)](https://github.com/prowler-cloud/prowler/pull/11420)
|
||||
- Clean up the scan tmp output directory when `scan-report` fails so partial files do not accumulate and fill the worker disk (`No space left on device`) [(#11421)](https://github.com/prowler-cloud/prowler/pull/11421)
|
||||
|
||||
---
|
||||
|
||||
## [1.30.0] (Prowler v5.29.0)
|
||||
|
||||
### 🔄 Changed
|
||||
|
||||
- Scan finding ingestion: bulk-resolve `Resource`/`ResourceTag` rows, replace per-mapping `SELECT FOR UPDATE` with deferred `ResourceTagMapping.bulk_create(ignore_conflicts=True)`, wrap each micro-batch in a single `rls_transaction`, and raise `SCAN_DB_BATCH_SIZE` to 1000 [(#11249)](https://github.com/prowler-cloud/prowler/pull/11249)
|
||||
- Faster `GET /api/v1/finding-groups/latest` aggregation on tenants where one recent scan holds most findings [(#11380)](https://github.com/prowler-cloud/prowler/pull/11380)
|
||||
|
||||
---
|
||||
|
||||
## [1.29.1] (Prowler v5.28.1)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- `finding-groups` slow response with finding-level filters such as `region`; check title and description are now read from the daily summaries, which drops sorting by `check_title` [(#11326)](https://github.com/prowler-cloud/prowler/pull/11326)
|
||||
|
||||
---
|
||||
|
||||
## [1.29.0] (Prowler v5.28.0)
|
||||
|
||||
### 🚀 Added
|
||||
|
||||
- `okta` provider support [(#11184)](https://github.com/prowler-cloud/prowler/pull/11184)
|
||||
- `resource.metadata` attribute included in `/api/v1/findings?include=resources` [(#11187)](https://github.com/prowler-cloud/prowler/pull/11187)
|
||||
|
||||
---
|
||||
|
||||
## [1.28.0] (Prowler v5.27.0)
|
||||
|
||||
### 🚀 Added
|
||||
@@ -20,7 +171,6 @@ All notable changes to the **Prowler API** are documented in this file.
|
||||
- `perform_scan_task` and `perform_scheduled_scan_task` now short-circuit with a warning and `return None` when the target provider no longer exists, instead of letting `handle_provider_deletion` raise `ProviderDeletedException`. `perform_scheduled_scan_task` also removes any orphan `PeriodicTask` it finds so beat stops re-firing scans for deleted providers. Prevents queued messages for deleted providers from being recorded as `FAILURE` [(#11185)](https://github.com/prowler-cloud/prowler/pull/11185)
|
||||
- Attack Paths: `BEDROCK-001` and `BEDROCK-002` now target roles trusting `bedrock-agentcore.amazonaws.com` instead of `bedrock.amazonaws.com`, eliminating false positives against regular Bedrock service roles (Agents, Knowledge Bases, model invocation) [(#11141)](https://github.com/prowler-cloud/prowler/pull/11141)
|
||||
|
||||
|
||||
---
|
||||
|
||||
## [1.27.1] (Prowler v5.26.1)
|
||||
|
||||
+25
-6
@@ -1,11 +1,13 @@
|
||||
FROM python:3.12.10-slim-bookworm@sha256:fd95fa221297a88e1cf49c55ec1828edd7c5a428187e67b5d1805692d11588db AS build
|
||||
FROM python:3.12.13-slim-bookworm@sha256:8a7e7cc04fd3e2bd787f7f24e22d5d119aa590d429b50c95dfe12b3abe52f48b AS build
|
||||
|
||||
LABEL maintainer="https://github.com/prowler-cloud/api"
|
||||
|
||||
ARG POWERSHELL_VERSION=7.5.0
|
||||
ENV POWERSHELL_VERSION=${POWERSHELL_VERSION}
|
||||
# Opt out of PowerShell telemetry (Application Insights -> dc.services.visualstudio.com)
|
||||
ENV POWERSHELL_TELEMETRY_OPTOUT=1
|
||||
|
||||
ARG TRIVY_VERSION=0.70.0
|
||||
ARG TRIVY_VERSION=0.71.2
|
||||
ENV TRIVY_VERSION=${TRIVY_VERSION}
|
||||
|
||||
ARG ZIZMOR_VERSION=1.24.1
|
||||
@@ -89,7 +91,7 @@ WORKDIR /home/prowler
|
||||
# Ensure output directory exists
|
||||
RUN mkdir -p /tmp/prowler_api_output
|
||||
|
||||
COPY pyproject.toml uv.lock ./
|
||||
COPY --chown=prowler:prowler pyproject.toml uv.lock ./
|
||||
|
||||
RUN pip install --no-cache-dir --upgrade pip && \
|
||||
pip install --no-cache-dir uv==0.11.14
|
||||
@@ -97,13 +99,30 @@ RUN pip install --no-cache-dir --upgrade pip && \
|
||||
ENV PATH="/home/prowler/.local/bin:$PATH"
|
||||
|
||||
# Add `--no-install-project` to avoid installing the current project as a package
|
||||
RUN uv sync --no-install-project && \
|
||||
RUN uv sync --locked --no-install-project && \
|
||||
rm -rf ~/.cache/uv
|
||||
|
||||
RUN .venv/bin/python .venv/lib/python3.12/site-packages/prowler/providers/m365/lib/powershell/m365_powershell.py
|
||||
|
||||
COPY src/backend/ ./backend/
|
||||
COPY docker-entrypoint.sh ./docker-entrypoint.sh
|
||||
USER root
|
||||
|
||||
# Remove build-only packages from the final image after Python dependencies are installed.
|
||||
RUN apt-get purge -y --auto-remove \
|
||||
gcc \
|
||||
g++ \
|
||||
make \
|
||||
libxml2-dev \
|
||||
libxmlsec1-dev \
|
||||
pkg-config \
|
||||
libtool \
|
||||
libxslt1-dev \
|
||||
python3-dev \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
USER prowler
|
||||
|
||||
COPY --chown=prowler:prowler src/backend/ ./backend/
|
||||
COPY --chown=prowler:prowler docker-entrypoint.sh ./docker-entrypoint.sh
|
||||
|
||||
WORKDIR /home/prowler/backend
|
||||
|
||||
|
||||
+65
-29
@@ -2,7 +2,7 @@
|
||||
|
||||
This repository contains the JSON API and Task Runner components for Prowler, which facilitate a complete backend that interacts with the Prowler SDK and is used by the Prowler UI.
|
||||
|
||||
# Components
|
||||
## Components
|
||||
The Prowler API is composed of the following components:
|
||||
|
||||
- The JSON API, which is an API built with Django Rest Framework.
|
||||
@@ -10,13 +10,13 @@ The Prowler API is composed of the following components:
|
||||
- The PostgreSQL database, which is used to store the data.
|
||||
- The Valkey database, which is an in-memory database which is used as a message broker for the Celery workers.
|
||||
|
||||
## Note about Valkey
|
||||
### Note about Valkey
|
||||
|
||||
[Valkey](https://valkey.io/) is an open source (BSD) high performance key/value datastore.
|
||||
|
||||
Valkey exposes a Redis 7.2 compliant API. Any service that exposes the Redis API can be used with Prowler API.
|
||||
|
||||
# Modify environment variables
|
||||
## Modify environment variables
|
||||
|
||||
Under the root path of the project, you can find a file called `.env`. This file shows all the environment variables that the project uses. You should review it and set the values for the variables you want to change.
|
||||
|
||||
@@ -24,7 +24,7 @@ If you don’t set `DJANGO_TOKEN_SIGNING_KEY` or `DJANGO_TOKEN_VERIFYING_KEY`, t
|
||||
|
||||
**Important note**: Every Prowler version (or repository branches and tags) could have different variables set in its `.env` file. Please use the `.env` file that corresponds with each version.
|
||||
|
||||
## Local deployment
|
||||
### Local deployment
|
||||
Keep in mind if you export the `.env` file to use it with local deployment that you will have to do it within the context of the virtual environment, not before. Otherwise, variables will not be loaded properly.
|
||||
|
||||
To do this, you can run:
|
||||
@@ -34,12 +34,12 @@ set -a
|
||||
source .env
|
||||
```
|
||||
|
||||
# 🚀 Production deployment
|
||||
## Docker deployment
|
||||
## 🚀 Production deployment
|
||||
### Docker deployment
|
||||
|
||||
This method requires `docker` and `docker compose`.
|
||||
|
||||
### Clone the repository
|
||||
#### Clone the repository
|
||||
|
||||
```console
|
||||
# HTTPS
|
||||
@@ -50,13 +50,13 @@ git clone git@github.com:prowler-cloud/api.git
|
||||
|
||||
```
|
||||
|
||||
### Build the base image
|
||||
#### Build the base image
|
||||
|
||||
```console
|
||||
docker compose --profile prod build
|
||||
```
|
||||
|
||||
### Run the production service
|
||||
#### Run the production service
|
||||
|
||||
This command will start the Django production server and the Celery worker and also the Valkey and PostgreSQL databases.
|
||||
|
||||
@@ -68,7 +68,7 @@ You can access the server in `http://localhost:8080`.
|
||||
|
||||
> **NOTE:** notice how the port is different. When developing using docker, the port will be `8080` to prevent conflicts.
|
||||
|
||||
### View the Production Server Logs
|
||||
#### View the Production Server Logs
|
||||
|
||||
To view the logs for any component (e.g., Django, Celery worker), you can use the following command with a wildcard. This command will follow logs for any container that matches the specified pattern:
|
||||
|
||||
@@ -133,13 +133,13 @@ gunicorn -c config/guniconf.py config.wsgi:application
|
||||
|
||||
> By default, the Gunicorn server will try to use as many workers as your machine can handle. You can manually change that in the `src/backend/config/guniconf.py` file.
|
||||
|
||||
# 🧪 Development guide
|
||||
## 🧪 Development guide
|
||||
|
||||
## Local deployment
|
||||
### Local deployment
|
||||
|
||||
To use this method, you'll need to set up a Python virtual environment (version ">=3.11,<3.13") and keep dependencies updated. Additionally, ensure that `uv` and `docker compose` are installed.
|
||||
|
||||
### Clone the repository
|
||||
#### Clone the repository
|
||||
|
||||
```console
|
||||
# HTTPS
|
||||
@@ -150,7 +150,7 @@ git clone git@github.com:prowler-cloud/api.git
|
||||
|
||||
```
|
||||
|
||||
### Start the PostgreSQL Database and Valkey
|
||||
#### Start the PostgreSQL Database and Valkey
|
||||
|
||||
The PostgreSQL database (version 16.3) and Valkey (version 7) are required for the development environment. To make development easier, we have provided a `docker-compose` file that will start these components for you.
|
||||
|
||||
@@ -161,7 +161,7 @@ The PostgreSQL database (version 16.3) and Valkey (version 7) are required for t
|
||||
docker compose up postgres valkey -d
|
||||
```
|
||||
|
||||
### Install the Python dependencies
|
||||
#### Install the Python dependencies
|
||||
|
||||
> You must have uv installed
|
||||
|
||||
@@ -169,7 +169,7 @@ docker compose up postgres valkey -d
|
||||
uv sync
|
||||
```
|
||||
|
||||
### Apply migrations
|
||||
#### Apply migrations
|
||||
|
||||
For migrations, you need to force the `admin` database router. Assuming you have the correct environment variables and Python virtual environment, run:
|
||||
|
||||
@@ -178,7 +178,7 @@ cd src/backend
|
||||
python manage.py migrate --database admin
|
||||
```
|
||||
|
||||
### Run the Django development server
|
||||
#### Run the Django development server
|
||||
|
||||
```console
|
||||
cd src/backend
|
||||
@@ -188,7 +188,7 @@ python manage.py runserver
|
||||
You can access the server in `http://localhost:8000`.
|
||||
All changes in the code will be automatically reloaded in the server.
|
||||
|
||||
### Run the Celery worker
|
||||
#### Run the Celery worker
|
||||
|
||||
```console
|
||||
python -m celery -A config.celery worker -l info -E
|
||||
@@ -196,11 +196,47 @@ python -m celery -A config.celery worker -l info -E
|
||||
|
||||
The Celery worker does not detect and reload changes in the code, so you need to restart it manually when you make changes.
|
||||
|
||||
## Docker deployment
|
||||
### Makefile-Assisted Local Deployment
|
||||
|
||||
This method is an additional local development workflow. It does not replace the manual local deployment or the Docker deployment described in this guide.
|
||||
|
||||
PostgreSQL, Valkey, and Neo4j run with Docker Compose, while Django and the Celery worker run natively through `uv`. Additionally, this workflow creates a `tmux` session with panes for the API, worker, and PostgreSQL logs.
|
||||
|
||||
Before using this method, ensure `docker compose`, `tmux`, and `uv` are installed.
|
||||
|
||||
This workflow is designed for macOS and should also work on Linux when Docker, `tmux`, and `uv` are available. Windows requires script changes before it can be supported.
|
||||
|
||||
From the repository root, run:
|
||||
|
||||
```console
|
||||
make dev
|
||||
```
|
||||
|
||||
The API will be available at:
|
||||
|
||||
```console
|
||||
http://localhost:8080/api/v1
|
||||
```
|
||||
|
||||
Use these commands to manage the local stack:
|
||||
|
||||
```console
|
||||
make dev-setup # Bootstrap dependencies, migrations, and fixtures
|
||||
make dev-attach # Attach to the tmux session
|
||||
make dev-launch # Start the stack on fixed ports and attach
|
||||
make dev-stop # Stop the tmux session and containers
|
||||
make dev-clean # Remove stopped development containers
|
||||
make dev-wipe # Stop everything and delete local development data
|
||||
make dev-status # Show development container status
|
||||
```
|
||||
|
||||
This workflow does not start the UI. Start it separately from the `ui/` directory when needed.
|
||||
|
||||
### Docker deployment
|
||||
|
||||
This method requires `docker` and `docker compose`.
|
||||
|
||||
### Clone the repository
|
||||
#### Clone the repository
|
||||
|
||||
```console
|
||||
# HTTPS
|
||||
@@ -211,13 +247,13 @@ git clone git@github.com:prowler-cloud/api.git
|
||||
|
||||
```
|
||||
|
||||
### Build the base image
|
||||
#### Build the base image
|
||||
|
||||
```console
|
||||
docker compose --profile dev build
|
||||
```
|
||||
|
||||
### Run the development service
|
||||
#### Run the development service
|
||||
|
||||
This command will start the Django development server and the Celery worker and also the Valkey and PostgreSQL databases.
|
||||
|
||||
@@ -230,7 +266,7 @@ All changes in the code will be automatically reloaded in the server.
|
||||
|
||||
> **NOTE:** notice how the port is different. When developing using docker, the port will be `8080` to prevent conflicts.
|
||||
|
||||
### View the development server logs
|
||||
#### View the development server logs
|
||||
|
||||
To view the logs for any component (e.g., Django, Celery worker), you can use the following command with a wildcard. This command will follow logs for any container that matches the specified pattern:
|
||||
|
||||
@@ -238,7 +274,7 @@ To view the logs for any component (e.g., Django, Celery worker), you can use th
|
||||
docker logs -f $(docker ps --format "{{.Names}}" | grep 'api-')
|
||||
```
|
||||
|
||||
## Applying migrations
|
||||
### Applying migrations
|
||||
|
||||
For migrations, you need to force the `admin` database router. Assuming you have the correct environment variables and Python virtual environment, run:
|
||||
|
||||
@@ -247,7 +283,7 @@ cd src/backend
|
||||
uv run python manage.py migrate --database admin
|
||||
```
|
||||
|
||||
## Apply fixtures
|
||||
### Apply fixtures
|
||||
|
||||
Fixtures are used to populate the database with initial development data.
|
||||
|
||||
@@ -258,7 +294,7 @@ uv run python manage.py loaddata api/fixtures/0_dev_users.json --database admin
|
||||
|
||||
> The default credentials are `dev@prowler.com:Thisisapassword123@` or `dev2@prowler.com:Thisisapassword123@`
|
||||
|
||||
## Run tests
|
||||
### Run tests
|
||||
|
||||
Note that the tests will fail if you use the same `.env` file as the development environment.
|
||||
|
||||
@@ -269,7 +305,7 @@ cd src/backend
|
||||
uv run pytest
|
||||
```
|
||||
|
||||
# Custom commands
|
||||
## Custom commands
|
||||
|
||||
Django provides a way to create custom commands that can be run from the command line.
|
||||
|
||||
@@ -281,7 +317,7 @@ To run a custom command, you need to be in the `prowler/api/src/backend` directo
|
||||
uv run python manage.py <command_name>
|
||||
```
|
||||
|
||||
## Generate dummy data
|
||||
### Generate dummy data
|
||||
|
||||
```console
|
||||
python manage.py findings --tenant
|
||||
@@ -298,7 +334,7 @@ This command creates, for a given tenant, a provider, scan and a set of findings
|
||||
>
|
||||
> The last step is required to access the findings details, since the UI needs that to print all the information.
|
||||
|
||||
### Example
|
||||
#### Example
|
||||
|
||||
```console
|
||||
~/backend $ uv run python manage.py findings --tenant
|
||||
|
||||
@@ -21,13 +21,19 @@ apply_fixtures() {
|
||||
}
|
||||
|
||||
start_dev_server() {
|
||||
echo "Starting the development server..."
|
||||
uv run python manage.py runserver 0.0.0.0:"${DJANGO_PORT:-8080}"
|
||||
echo "Starting the development server (Gunicorn ASGI, debug + reload)..."
|
||||
# Same server/worker as prod (config.asgi via the native `asgi` worker), so
|
||||
# SSE streams run on the event loop exactly as they do in production. DEBUG is
|
||||
# on so guniconf's `reload = DEBUG` hot-reloads edited code (and flips
|
||||
# `preload_app` off so reload actually takes).
|
||||
export DJANGO_DEBUG="${DJANGO_DEBUG:-True}"
|
||||
export DJANGO_BIND_ADDRESS="${DJANGO_BIND_ADDRESS:-0.0.0.0}"
|
||||
exec uv run gunicorn -c config/guniconf.py config.asgi:application
|
||||
}
|
||||
|
||||
start_prod_server() {
|
||||
echo "Starting the Gunicorn server..."
|
||||
uv run gunicorn -c config/guniconf.py config.wsgi:application
|
||||
exec uv run gunicorn -c config/guniconf.py config.asgi:application
|
||||
}
|
||||
|
||||
resolve_worker_hostname() {
|
||||
@@ -47,7 +53,7 @@ resolve_worker_hostname() {
|
||||
|
||||
start_worker() {
|
||||
echo "Starting the worker..."
|
||||
uv run python -m celery -A config.celery worker \
|
||||
exec uv run python -m celery -A config.celery worker \
|
||||
-n "$(resolve_worker_hostname)" \
|
||||
-l "${DJANGO_LOGGING_LEVEL:-info}" \
|
||||
-Q celery,scans,scan-reports,deletion,backfill,overview,integrations,compliance,attack-paths-scans \
|
||||
@@ -56,7 +62,7 @@ start_worker() {
|
||||
|
||||
start_worker_beat() {
|
||||
echo "Starting the worker-beat..."
|
||||
uv run python -m celery -A config.celery beat -l "${DJANGO_LOGGING_LEVEL:-info}" --scheduler django_celery_beat.schedulers:DatabaseScheduler
|
||||
exec uv run python -m celery -A config.celery beat -l "${DJANGO_LOGGING_LEVEL:-info}" --scheduler django_celery_beat.schedulers:DatabaseScheduler
|
||||
}
|
||||
|
||||
manage_db_partitions() {
|
||||
@@ -68,6 +74,15 @@ manage_db_partitions() {
|
||||
fi
|
||||
}
|
||||
|
||||
# Identify this process to Postgres (application_name=<component>:<alias>) so
|
||||
# connections are attributable by component in pg_stat_activity. Web tiers
|
||||
# report "api"; everything else uses the launch subcommand.
|
||||
case "$1" in
|
||||
prod|dev) DJANGO_APP_COMPONENT="api" ;;
|
||||
*) DJANGO_APP_COMPONENT="$1" ;;
|
||||
esac
|
||||
export DJANGO_APP_COMPONENT
|
||||
|
||||
case "$1" in
|
||||
dev)
|
||||
apply_migrations
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
# Orphan Celery task recovery
|
||||
|
||||
When a worker is terminated mid-task (a deploy, an OOM kill, a node eviction), the
|
||||
task it was running can be left non-terminal forever: the `TaskResult` stays
|
||||
`STARTED` and nothing re-runs it. This page describes the mechanisms that detect and
|
||||
recover allowlisted idempotent orphans so pending-task alerts do not fire. Scan tasks
|
||||
are not auto-recovered (re-running a scan is not safe to do automatically); the
|
||||
watchdog covers the summary/aggregation and deletion tasks.
|
||||
|
||||
## How recovery works
|
||||
|
||||
1. **Durable delivery.** The broker is configured so a task message is acknowledged
|
||||
only after the task finishes (`task_acks_late`), one task is reserved at a time
|
||||
(`worker_prefetch_multiplier = 1`), and an abruptly-lost worker re-queues its task
|
||||
(`task_reject_on_worker_lost`). On `SIGTERM` the worker is given a soft-shutdown
|
||||
window (`worker_soft_shutdown_timeout`) to finish or re-queue in-flight work
|
||||
before it is force-killed. `scan-perform`, `scan-perform-scheduled` and
|
||||
`integration-jira` opt out of redelivery with `acks_late=False`, so a crash drops
|
||||
them rather than re-running and duplicating findings or Jira issues. Other
|
||||
non-recovered side-effect tasks keep `acks_late=True`, so the broker can still
|
||||
re-deliver them after a worker loss: the S3 upload rebuilds from worker-local files
|
||||
that did not survive the crash and so no-ops, but Security Hub re-reads findings from
|
||||
the DB and re-sends them to AWS.
|
||||
|
||||
2. **Periodic watchdog.** A Beat task, `reconcile-orphan-tasks`, runs every couple of
|
||||
minutes (a `django_celery_beat` periodic task created by migration). For each
|
||||
in-flight task result with an allowlisted idempotent task name, it pings the
|
||||
worker recorded on the task's `TaskResult`:
|
||||
- worker responds -> the task is still running, leave it alone;
|
||||
- worker is gone (and the task started before a short grace window) -> it is a
|
||||
real orphan: the stale task is revoked and marked terminal (clearing the
|
||||
pending/started alert), and the task is re-enqueued from its stored name and
|
||||
kwargs.
|
||||
|
||||
The re-run is safe because only tasks with proven idempotency are allowlisted: the
|
||||
summary/aggregation tasks clear and re-write their own rows, and deletions are
|
||||
idempotent. Scan tasks and external side effects are excluded: re-running a scan is
|
||||
not safe to do automatically, Jira sends would create duplicate issues, the S3
|
||||
upload rebuilds from worker-local files that do not survive a crash, and
|
||||
report/Security Hub recovery is out of scope.
|
||||
|
||||
3. **Recovery cap.** A per-task Valkey counter limits how often the same task is
|
||||
re-enqueued. After `--max-attempts` recoveries (default 3) the orphan is marked
|
||||
terminal instead of re-enqueued, so a task that repeatedly kills its worker cannot
|
||||
loop forever.
|
||||
|
||||
A Postgres advisory lock ensures that, even with multiple API/worker replicas, only
|
||||
one reconciliation runs at a time; the others no-op.
|
||||
|
||||
## On-demand command
|
||||
|
||||
The same logic is available as a management command, useful right after a deploy or
|
||||
for manual intervention:
|
||||
|
||||
```bash
|
||||
python manage.py reconcile_orphan_tasks # recover now
|
||||
python manage.py reconcile_orphan_tasks --dry-run # report orphans, change nothing
|
||||
python manage.py reconcile_orphan_tasks --grace-minutes 5 --max-attempts 3
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
All settings have safe defaults; override via environment variables.
|
||||
|
||||
| Env var | Default | Purpose |
|
||||
| --- | --- | --- |
|
||||
| `DJANGO_CELERY_WORKER_PREFETCH_MULTIPLIER` | `1` | Tasks reserved per worker process. |
|
||||
| `DJANGO_CELERY_WORKER_CONCURRENCY` | unset | Optional Celery prefork pool size. When unset, Celery uses its CPU-based default. Set this on worker containers to bound idle memory on hosts with many CPUs. |
|
||||
| `DJANGO_CELERY_WORKER_SOFT_SHUTDOWN_TIMEOUT` | `60` | Seconds the worker drains/re-queues on `SIGTERM` before force-kill. |
|
||||
| `DJANGO_CELERY_TASK_TIME_LIMIT` | `21600` (6h) | Hard limit for most tasks; connection checks are capped at 120s. |
|
||||
| `DJANGO_CELERY_TASK_SOFT_TIME_LIMIT` | hard - 600 | Soft limit; raises `SoftTimeLimitExceeded` for cleanup. |
|
||||
| `DJANGO_CELERY_LONG_TASK_TIME_LIMIT` | `172800` (48h) | Hard limit for scans and provider/tenant deletions, which can legitimately run for more than a day. |
|
||||
| `DJANGO_CELERY_LONG_TASK_SOFT_TIME_LIMIT` | long hard - 600 | Soft limit for the long-running tasks above. |
|
||||
| `DJANGO_TASK_RECOVERY_ENABLED` | `false` | Master switch for orphan-task recovery, disabled by default (opt-in); set to `true` to enable. When off, no orphan is detected, marked terminal, or re-enqueued (attack-paths stale cleanup still runs). |
|
||||
| `DJANGO_TASK_RECOVERY_SUMMARIES_ENABLED` | `true` | Auto re-enqueue orphaned scan summary/aggregation tasks. |
|
||||
| `DJANGO_TASK_RECOVERY_DELETIONS_ENABLED` | `true` | Auto re-enqueue orphaned provider/tenant deletion tasks. |
|
||||
|
||||
Recovery is opt-in: with the master flag off (the default) the sweep does nothing.
|
||||
Once enabled, the per-group flags default to on, so every group recovers unless you
|
||||
turn one off; a task whose group flag is off is marked terminal instead of
|
||||
re-enqueued.
|
||||
|
||||
Turning recovery off only disables this watchdog sweep; it does not change Celery's
|
||||
broker-level redelivery (`task_acks_late`/`task_reject_on_worker_lost`), which still
|
||||
re-delivers tasks that keep `acks_late=True` on worker loss, independently of this flag.
|
||||
|
||||
`task_acks_late` and `task_reject_on_worker_lost` are enabled in `config/celery.py`.
|
||||
|
||||
## Deployment requirement
|
||||
|
||||
Two conditions must both hold for the soft shutdown to actually drain work:
|
||||
|
||||
1. **The worker must receive `SIGTERM`.** The container entrypoint `exec`s the
|
||||
Celery process so it runs as PID 1; otherwise `SIGTERM` from `docker stop`/ECS
|
||||
hits the entrypoint shell, never reaches Celery, and the worker is hard-killed
|
||||
(SIGKILL) at the grace deadline without draining. Custom entrypoints must
|
||||
preserve the `exec`.
|
||||
2. **The orchestrator must give the worker enough time** before force-killing it.
|
||||
Set the stop grace period to exceed `DJANGO_CELERY_WORKER_SOFT_SHUTDOWN_TIMEOUT`
|
||||
plus a margin:
|
||||
- **docker-compose:** `stop_grace_period` on the worker services (set to `120s`).
|
||||
- **AWS ECS:** the worker container `stopTimeout` (configured in the deployment
|
||||
repository).
|
||||
|
||||
If either condition is missing, long tasks are still recovered by the watchdog,
|
||||
but they are cut mid-run on every deploy instead of draining.
|
||||
+61
-23
@@ -14,7 +14,7 @@ dev = [
|
||||
"pytest-env==1.1.3",
|
||||
"pytest-randomly==3.15.0",
|
||||
"pytest-xdist==3.6.1",
|
||||
"ruff==0.5.0",
|
||||
"ruff==0.15.11",
|
||||
"tqdm==4.67.1",
|
||||
"vulture==2.14",
|
||||
"prek==0.3.9"
|
||||
@@ -41,9 +41,11 @@ dependencies = [
|
||||
"drf-spectacular==0.27.2",
|
||||
"drf-spectacular-jsonapi==0.5.1",
|
||||
"defusedxml==0.7.1",
|
||||
"gunicorn==23.0.0",
|
||||
"django-eventstream==5.3.3",
|
||||
"gunicorn==26.0.0",
|
||||
"uvloop==0.22.1",
|
||||
"lxml==6.1.0",
|
||||
"prowler @ git+https://github.com/prowler-cloud/prowler.git@master",
|
||||
"prowler @ git+https://github.com/prowler-cloud/prowler.git@v5.32",
|
||||
"psycopg2-binary==2.9.9",
|
||||
"pytest-celery[redis] (==1.3.0)",
|
||||
"sentry-sdk[django] (==2.56.0)",
|
||||
@@ -56,11 +58,12 @@ dependencies = [
|
||||
"matplotlib (==3.10.8)",
|
||||
"reportlab (==4.4.10)",
|
||||
"neo4j (==6.1.0)",
|
||||
"cartography (==0.135.0)",
|
||||
"cartography (==0.138.1)",
|
||||
"gevent (==25.9.1)",
|
||||
"werkzeug (==3.1.7)",
|
||||
"sqlparse (==0.5.5)",
|
||||
"fonttools (==4.62.1)"
|
||||
"fonttools (==4.62.1)",
|
||||
"uvicorn-worker (==0.4.0)",
|
||||
]
|
||||
description = "Prowler's API (Django/DRF)"
|
||||
license = "Apache-2.0"
|
||||
@@ -68,7 +71,24 @@ name = "prowler-api"
|
||||
package-mode = false
|
||||
# Needed for the SDK compatibility
|
||||
requires-python = ">=3.11,<3.13"
|
||||
version = "1.28.0"
|
||||
version = "1.33.0"
|
||||
|
||||
# Shared ruff baseline (kept in sync with mcp_server/pyproject.toml).
|
||||
# target-version tracks this project's lowest supported Python.
|
||||
[tool.ruff]
|
||||
src = ["src"]
|
||||
target-version = "py311"
|
||||
|
||||
[tool.ruff.lint]
|
||||
# Defaults (E4/E7/E9, F) plus import sorting, modern-syntax upgrades, and
|
||||
# comprehension lints — all mechanically auto-fixable. flake8-bugbear (B) is a
|
||||
# good next step but needs manual cleanup (e.g. B904 raise-from), so it is left
|
||||
# out of the shared baseline for now.
|
||||
extend-select = [
|
||||
"I", # isort — import ordering (prek's isort hook covers only the SDK)
|
||||
"UP", # pyupgrade — modern syntax for the min supported Python
|
||||
"C4" # flake8-comprehensions
|
||||
]
|
||||
|
||||
[tool.uv]
|
||||
# Transitive pins matching master to avoid silent drift; bump deliberately.
|
||||
@@ -79,7 +99,7 @@ constraint-dependencies = [
|
||||
"aiobotocore==2.25.1",
|
||||
"aiofiles==24.1.0",
|
||||
"aiohappyeyeballs==2.6.1",
|
||||
"aiohttp==3.13.5",
|
||||
"aiohttp==3.14.0",
|
||||
"aioitertools==0.13.0",
|
||||
"aiosignal==1.4.0",
|
||||
"alibabacloud-actiontrail20200706==2.4.1",
|
||||
@@ -124,9 +144,8 @@ constraint-dependencies = [
|
||||
"astroid==3.2.4",
|
||||
"async-timeout==5.0.1",
|
||||
"attrs==25.4.0",
|
||||
"authlib==1.6.9",
|
||||
"authlib==1.6.12",
|
||||
"autopep8==2.3.2",
|
||||
"awsipranges==0.3.3",
|
||||
"azure-cli-core==2.83.0",
|
||||
"azure-cli-telemetry==1.1.0",
|
||||
"azure-common==1.1.28",
|
||||
@@ -174,7 +193,7 @@ constraint-dependencies = [
|
||||
"blinker==1.9.0",
|
||||
"boto3==1.40.61",
|
||||
"botocore==1.40.61",
|
||||
"cartography==0.135.0",
|
||||
"cartography==0.138.1",
|
||||
"celery==5.6.2",
|
||||
"certifi==2026.1.4",
|
||||
"cffi==2.0.0",
|
||||
@@ -199,7 +218,6 @@ constraint-dependencies = [
|
||||
"debugpy==1.8.20",
|
||||
"decorator==5.2.1",
|
||||
"defusedxml==0.7.1",
|
||||
"detect-secrets==1.5.0",
|
||||
"dill==0.4.1",
|
||||
"distro==1.9.0",
|
||||
"dj-rest-auth==7.0.1",
|
||||
@@ -209,6 +227,7 @@ constraint-dependencies = [
|
||||
"django-celery-results==2.6.0",
|
||||
"django-cors-headers==4.4.0",
|
||||
"django-environ==0.11.2",
|
||||
"django-eventstream==5.3.3",
|
||||
"django-filter==24.3",
|
||||
"django-guid==3.5.0",
|
||||
"django-postgres-extra==2.0.9",
|
||||
@@ -226,7 +245,7 @@ constraint-dependencies = [
|
||||
"drf-simple-apikey==2.2.1",
|
||||
"drf-spectacular==0.27.2",
|
||||
"drf-spectacular-jsonapi==0.5.1",
|
||||
"dulwich==0.23.0",
|
||||
"dulwich==1.2.5",
|
||||
"duo-client==5.5.0",
|
||||
"durationpy==0.10",
|
||||
"email-validator==2.2.0",
|
||||
@@ -253,7 +272,7 @@ constraint-dependencies = [
|
||||
"grpc-google-iam-v1==0.14.3",
|
||||
"grpcio==1.76.0",
|
||||
"grpcio-status==1.76.0",
|
||||
"gunicorn==23.0.0",
|
||||
"gunicorn==26.0.0",
|
||||
"h11==0.16.0",
|
||||
"h2==4.3.0",
|
||||
"hpack==4.1.0",
|
||||
@@ -262,8 +281,8 @@ constraint-dependencies = [
|
||||
"httpx==0.28.1",
|
||||
"humanfriendly==10.0",
|
||||
"hyperframe==6.1.0",
|
||||
"iamdata==0.1.202602021",
|
||||
"idna==3.11",
|
||||
"iamdata==0.1.202605131",
|
||||
"idna==3.15",
|
||||
"importlib-metadata==8.7.1",
|
||||
"inflection==0.5.1",
|
||||
"iniconfig==2.3.0",
|
||||
@@ -281,6 +300,7 @@ constraint-dependencies = [
|
||||
"jsonschema==4.23.0",
|
||||
"jsonschema-specifications==2025.9.1",
|
||||
"keystoneauth1==5.13.0",
|
||||
"kingfisher-bin==1.104.0",
|
||||
"kiwisolver==1.4.9",
|
||||
"knack==0.11.0",
|
||||
"kombu==5.6.2",
|
||||
@@ -315,7 +335,7 @@ constraint-dependencies = [
|
||||
"neo4j==6.1.0",
|
||||
"nest-asyncio==1.6.0",
|
||||
"nltk==3.9.4",
|
||||
"numpy==2.0.2",
|
||||
"numpy==2.2.6",
|
||||
"oauthlib==3.3.1",
|
||||
"oci==2.169.0",
|
||||
"openai==1.109.1",
|
||||
@@ -344,7 +364,7 @@ constraint-dependencies = [
|
||||
"psutil==7.2.2",
|
||||
"psycopg2-binary==2.9.9",
|
||||
"py-deviceid==0.1.1",
|
||||
"py-iam-expand==0.1.0",
|
||||
"py-iam-expand==0.3.0",
|
||||
"py-ocsf-models==0.8.1",
|
||||
"pyasn1==0.6.3",
|
||||
"pyasn1-modules==0.4.2",
|
||||
@@ -354,7 +374,7 @@ constraint-dependencies = [
|
||||
"pydantic-core==2.41.5",
|
||||
"pygithub==2.8.0",
|
||||
"pygments==2.20.0",
|
||||
"pyjwt==2.12.1",
|
||||
"pyjwt==2.13.0",
|
||||
"pylint==3.2.5",
|
||||
"pymsalruntime==0.18.1",
|
||||
"pynacl==1.6.2",
|
||||
@@ -390,7 +410,7 @@ constraint-dependencies = [
|
||||
"rpds-py==0.30.0",
|
||||
"rsa==4.9.1",
|
||||
"ruamel-yaml==0.19.1",
|
||||
"ruff==0.5.0",
|
||||
"ruff==0.15.11",
|
||||
"s3transfer==0.14.0",
|
||||
"scaleway==2.10.3",
|
||||
"scaleway-core==2.10.3",
|
||||
@@ -420,12 +440,14 @@ constraint-dependencies = [
|
||||
"uritemplate==4.2.0",
|
||||
"urllib3==2.7.0",
|
||||
"uuid6==2024.7.10",
|
||||
"uvicorn==0.49.0",
|
||||
"uvloop==0.22.1",
|
||||
"vine==5.1.0",
|
||||
"vulture==2.14",
|
||||
"wcwidth==0.5.3",
|
||||
"websocket-client==1.9.0",
|
||||
"werkzeug==3.1.7",
|
||||
"workos==6.0.4",
|
||||
"workos==6.0.8",
|
||||
"wrapt==1.17.3",
|
||||
"xlsxwriter==3.2.9",
|
||||
"xmlsec==1.3.17",
|
||||
@@ -436,14 +458,30 @@ constraint-dependencies = [
|
||||
"zope-interface==8.2",
|
||||
"zstd==1.5.7.3"
|
||||
]
|
||||
# prowler@master needs okta==3.4.2; cartography 0.135.0 declares okta<1.0.0 for an
|
||||
# integration prowler does not import.
|
||||
# prowler@master needs okta==3.4.2, but cartography 0.138.1 requires okta<1.0.0.
|
||||
# Attack Paths does not ingest Okta today, so override the Cartography
|
||||
# dependency to the Prowler pin.
|
||||
#
|
||||
# prowler@master needs azure-mgmt-containerservice==34.1.0, but cartography
|
||||
# 0.138.1 requires azure-mgmt-containerservice>=41.0.0. Attack Paths does not
|
||||
# ingest Azure today, so override the Cartography dependency to the Prowler pin.
|
||||
#
|
||||
# prowler@master hard-pins microsoft-kiota-abstractions==1.9.2 in [project.dependencies].
|
||||
# The microsoft-kiota-http security bump to 1.9.9 (GHSA-7j59-v9qr-6fq9) requires
|
||||
# microsoft-kiota-abstractions>=1.9.9, which a constraint cannot satisfy against the
|
||||
# SDK's hard pin; override it to the patched, kiota-aligned version.
|
||||
#
|
||||
# prowler@master hard-pins dulwich==0.23.0 and pyjwt==2.12.1 in [project.dependencies].
|
||||
# dulwich 1.2.5 patches GHSA-897w-fcg9-f6xj (arbitrary file write) and pyjwt 2.13.0
|
||||
# patches PYSEC-2026-179 (HMAC/JWK key-confusion); a constraint cannot satisfy these
|
||||
# against the SDK's hard pins, so override them to the patched versions until the SDK
|
||||
# bump propagates to the pinned master rev. pyjwt keeps the [crypto] extra because an
|
||||
# override replaces the whole requirement; bare pyjwt would drop it from the consumers
|
||||
# that request pyjwt[crypto] and leave cryptography (needed for RS256) only transitive.
|
||||
override-dependencies = [
|
||||
"okta==3.4.2",
|
||||
"microsoft-kiota-abstractions==1.9.9"
|
||||
"azure-mgmt-containerservice==34.1.0",
|
||||
"microsoft-kiota-abstractions==1.9.9",
|
||||
"dulwich==1.2.5",
|
||||
"pyjwt[crypto]==2.13.0"
|
||||
]
|
||||
|
||||
@@ -1,9 +1,15 @@
|
||||
from allauth.socialaccount.adapter import DefaultSocialAccountAdapter
|
||||
from django.db import transaction
|
||||
|
||||
from api.db_router import MainRouter
|
||||
from api.db_utils import rls_transaction
|
||||
from api.models import Membership, Role, Tenant, User, UserRoleRelationship
|
||||
from api.models import (
|
||||
Membership,
|
||||
Role,
|
||||
SAMLConfiguration,
|
||||
Tenant,
|
||||
User,
|
||||
UserRoleRelationship,
|
||||
)
|
||||
from django.db import transaction
|
||||
|
||||
|
||||
class ProwlerSocialAccountAdapter(DefaultSocialAccountAdapter):
|
||||
@@ -18,7 +24,42 @@ class ProwlerSocialAccountAdapter(DefaultSocialAccountAdapter):
|
||||
# Link existing accounts with the same email address
|
||||
email = sociallogin.account.extra_data.get("email")
|
||||
if sociallogin.provider.id == "saml":
|
||||
# For SAML, the asserted NameID email cannot be trusted on its own:
|
||||
# any tenant can claim any email domain in its SAML configuration. To
|
||||
# prevent cross-tenant account takeover (GHSA-h8m9-jgf8-vwvp), only link
|
||||
# the incoming SAML session to an existing account when (1) the email
|
||||
# domain matches the tenant whose ACS endpoint is being used and (2) the
|
||||
# existing user is already a member of that tenant.
|
||||
email = sociallogin.user.email
|
||||
if not email:
|
||||
return
|
||||
|
||||
domain = email.rsplit("@", 1)[-1].lower()
|
||||
resolver_match = getattr(request, "resolver_match", None)
|
||||
organization_slug = (
|
||||
(resolver_match.kwargs or {}).get("organization_slug", "")
|
||||
if resolver_match
|
||||
else ""
|
||||
).lower()
|
||||
# The ACS endpoint is scoped per email domain; reject mismatches so an
|
||||
# attacker cannot replay an assertion through another tenant's endpoint.
|
||||
if organization_slug != domain:
|
||||
return
|
||||
|
||||
try:
|
||||
saml_config = SAMLConfiguration.objects.using(MainRouter.admin_db).get(
|
||||
email_domain=domain
|
||||
)
|
||||
except SAMLConfiguration.DoesNotExist:
|
||||
return
|
||||
|
||||
existing_user = self.get_user_by_email(email)
|
||||
if existing_user and existing_user.is_member_of_tenant(
|
||||
str(saml_config.tenant_id)
|
||||
):
|
||||
sociallogin.connect(request, existing_user)
|
||||
return
|
||||
|
||||
if email:
|
||||
existing_user = self.get_user_by_email(email)
|
||||
if existing_user:
|
||||
|
||||
@@ -28,9 +28,10 @@ class ApiConfig(AppConfig):
|
||||
name = "api"
|
||||
|
||||
def ready(self):
|
||||
from api import schema_extensions # noqa: F401
|
||||
from api import signals # noqa: F401
|
||||
from api.attack_paths import database as graph_database
|
||||
from api import (
|
||||
schema_extensions, # noqa: F401
|
||||
signals, # noqa: F401
|
||||
)
|
||||
|
||||
# Generate required cryptographic keys if not present, but only if:
|
||||
# `"manage.py" not in sys.argv[0]`: If an external server (e.g., Gunicorn) is running the app
|
||||
@@ -41,38 +42,6 @@ class ApiConfig(AppConfig):
|
||||
):
|
||||
self._ensure_crypto_keys()
|
||||
|
||||
# Commands that don't need Neo4j
|
||||
SKIP_NEO4J_DJANGO_COMMANDS = [
|
||||
"makemigrations",
|
||||
"migrate",
|
||||
"pgpartition",
|
||||
"check",
|
||||
"help",
|
||||
"showmigrations",
|
||||
"check_and_fix_socialaccount_sites_migration",
|
||||
]
|
||||
|
||||
# Skip eager Neo4j init for tests, some Django commands, and Celery (prefork pool: driver must stay lazy, no post_fork hook)
|
||||
if getattr(settings, "TESTING", False) or (
|
||||
len(sys.argv) > 1
|
||||
and (
|
||||
(
|
||||
"manage.py" in sys.argv[0]
|
||||
and sys.argv[1] in SKIP_NEO4J_DJANGO_COMMANDS
|
||||
)
|
||||
or "celery" in sys.argv[0]
|
||||
)
|
||||
):
|
||||
logger.info(
|
||||
"Skipping eager Neo4j init: tests, some Django commands, or Celery prefork pool (driver stays lazy)"
|
||||
)
|
||||
|
||||
else:
|
||||
graph_database.init_driver()
|
||||
|
||||
# Neo4j driver is initialized at API startup (see api.attack_paths.database)
|
||||
# It remains lazy for Celery workers and selected Django commands
|
||||
|
||||
def _ensure_crypto_keys(self):
|
||||
"""
|
||||
Orchestrator method that ensures all required cryptographic keys are present.
|
||||
|
||||
@@ -5,7 +5,6 @@ from api.attack_paths.queries import (
|
||||
get_query_by_id,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"AttackPathsQueryDefinition",
|
||||
"AttackPathsQueryParameterDefinition",
|
||||
|
||||
@@ -4,10 +4,10 @@ Cypher sanitizer for custom (user-supplied) Attack Paths queries.
|
||||
Two responsibilities:
|
||||
|
||||
1. **Validation** - reject queries containing SSRF or dangerous procedure
|
||||
patterns (defense-in-depth; the primary control is ``neo4j.READ_ACCESS``).
|
||||
patterns (defense-in-depth; the primary control is `neo4j.READ_ACCESS`).
|
||||
|
||||
2. **Provider-scoped label injection** - inject a dynamic
|
||||
``_Provider_{uuid}`` label into every node pattern so the database can
|
||||
`_Provider_{uuid}` label into every node pattern so the database can
|
||||
use its native label index for provider isolation.
|
||||
|
||||
Label-injection pipeline:
|
||||
@@ -22,18 +22,16 @@ Label-injection pipeline:
|
||||
import re
|
||||
|
||||
from rest_framework.exceptions import ValidationError
|
||||
|
||||
from tasks.jobs.attack_paths.config import get_provider_label
|
||||
|
||||
|
||||
# Step 1 - String / comment protection
|
||||
# Single combined regex: strings first, then line comments.
|
||||
# Single combined regex: strings first, then line comments
|
||||
# The regex engine finds the leftmost match, so a string like 'https://prowler.com'
|
||||
# is consumed as a string before the // inside it can match as a comment.
|
||||
# is consumed as a string before the // inside it can match as a comment
|
||||
_PROTECTED_RE = re.compile(r"'(?:[^'\\]|\\.)*'|\"(?:[^\"\\]|\\.)*\"|//[^\n]*")
|
||||
|
||||
# Step 2 - Clause splitting
|
||||
# OPTIONAL MATCH must come before MATCH to avoid partial matching.
|
||||
# `OPTIONAL MATCH` must come before `MATCH` to avoid partial matching
|
||||
_CLAUSE_RE = re.compile(
|
||||
r"\b(OPTIONAL\s+MATCH|MATCH|WHERE|RETURN|WITH|ORDER\s+BY"
|
||||
r"|SKIP|LIMIT|UNION|UNWIND|CALL)\b",
|
||||
@@ -41,10 +39,10 @@ _CLAUSE_RE = re.compile(
|
||||
)
|
||||
|
||||
# Pass A - Labeled node patterns (all segments)
|
||||
# Matches node patterns that have at least one :Label.
|
||||
# (?<!\w)\( - open paren NOT preceded by a word char (excludes function calls).
|
||||
# Group 1: optional variable + one or more :Label
|
||||
# Group 2: optional {properties} + closing paren
|
||||
# Matches node patterns that have at least one `:Label`
|
||||
# `(?<!\w)\(` - open paren NOT preceded by a word char, excludes function calls
|
||||
# Group 1: optional variable + one or more `:Label`
|
||||
# Group 2: optional `{`properties`}` + closing paren
|
||||
_LABELED_NODE_RE = re.compile(
|
||||
r"(?<!\w)\("
|
||||
r"("
|
||||
@@ -57,9 +55,9 @@ _LABELED_NODE_RE = re.compile(
|
||||
r")"
|
||||
)
|
||||
|
||||
# Pass B - Bare node patterns (MATCH segments only)
|
||||
# Matches (identifier) or (identifier {properties}) without any :Label.
|
||||
# Only applied in MATCH/OPTIONAL MATCH segments.
|
||||
# Pass B - Bare node patterns (`MATCH` segments only)
|
||||
# Matches (identifier) or (identifier {properties}) without any `:Label`
|
||||
# Only applied in `MATCH` / `OPTIONAL MATCH` segments
|
||||
_BARE_NODE_RE = re.compile(
|
||||
r"(?<!\w)\(" r"(\s*[a-zA-Z_]\w*)" r"(\s*(?:\{[^}]*\})?)" r"\s*\)"
|
||||
)
|
||||
@@ -98,6 +96,11 @@ def inject_provider_label(cypher: str, provider_id: str) -> str:
|
||||
node pattern.
|
||||
"""
|
||||
label = get_provider_label(provider_id)
|
||||
return inject_label(cypher, label)
|
||||
|
||||
|
||||
def inject_label(cypher: str, label: str) -> str:
|
||||
"""Rewrite a Cypher query to append a label to every node pattern."""
|
||||
|
||||
# Step 1: Protect strings and comments (single pass, leftmost-first)
|
||||
protected: list[str] = []
|
||||
@@ -136,9 +139,7 @@ def inject_provider_label(cypher: str, provider_id: str) -> str:
|
||||
return work
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Validation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Patterns that indicate SSRF or dangerous procedure calls
|
||||
# Defense-in-depth layer - the primary control is `neo4j.READ_ACCESS`
|
||||
|
||||
@@ -1,233 +1,32 @@
|
||||
import atexit
|
||||
import logging
|
||||
import threading
|
||||
from contextlib import contextmanager
|
||||
from typing import Any, Iterator
|
||||
"""Backwards-compatible facade over the ingest and sink modules.
|
||||
|
||||
Historically this module owned a single Neo4j driver used for both the
|
||||
cartography temp database and the per-tenant sink database. The port to AWS
|
||||
Neptune split those roles: the cartography ingest (temp) database is always
|
||||
Neo4j and lives in `api.attack_paths.ingest`; the sink is configurable
|
||||
(Neo4j or Neptune) and lives in `api.attack_paths.sink`. This shim preserves
|
||||
the public API that `tasks/` and `api/v1/views.py` already depend on, and
|
||||
dispatches to the right module by database-name prefix.
|
||||
|
||||
A database name starting with `db-tmp-scan-` is a cartography temp DB and
|
||||
routes to ingest. Everything else routes to the configured sink.
|
||||
"""
|
||||
|
||||
from contextlib import AbstractContextManager
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
import neo4j
|
||||
import neo4j.exceptions
|
||||
import neo4j # noqa: F401 - kept for tests that patch api.attack_paths.database.neo4j
|
||||
from api.attack_paths import ingest
|
||||
from api.attack_paths import sink as sink_module
|
||||
from config.env import env
|
||||
from django.conf import settings
|
||||
from tasks.jobs.attack_paths.config import (
|
||||
BATCH_SIZE,
|
||||
PROVIDER_RESOURCE_LABEL,
|
||||
get_provider_label,
|
||||
from django.conf import (
|
||||
settings, # noqa: F401 - kept for tests that patch ...database.settings
|
||||
)
|
||||
|
||||
from api.attack_paths.retryable_session import RetryableSession
|
||||
|
||||
# Without this Celery goes crazy with Neo4j logging
|
||||
logging.getLogger("neo4j").setLevel(logging.ERROR)
|
||||
logging.getLogger("neo4j").propagate = False
|
||||
|
||||
SERVICE_UNAVAILABLE_MAX_RETRIES = env.int(
|
||||
"ATTACK_PATHS_SERVICE_UNAVAILABLE_MAX_RETRIES", default=3
|
||||
)
|
||||
READ_QUERY_TIMEOUT_SECONDS = env.int(
|
||||
"ATTACK_PATHS_READ_QUERY_TIMEOUT_SECONDS", default=30
|
||||
)
|
||||
MAX_CUSTOM_QUERY_NODES = env.int("ATTACK_PATHS_MAX_CUSTOM_QUERY_NODES", default=250)
|
||||
CONN_ACQUISITION_TIMEOUT = env.int("NEO4J_CONN_ACQUISITION_TIMEOUT", default=15)
|
||||
READ_EXCEPTION_CODES = [
|
||||
"Neo.ClientError.Statement.AccessMode",
|
||||
"Neo.ClientError.Procedure.ProcedureNotFound",
|
||||
]
|
||||
CLIENT_STATEMENT_EXCEPTION_PREFIX = "Neo.ClientError.Statement."
|
||||
|
||||
# Module-level process-wide driver singleton
|
||||
_driver: neo4j.Driver | None = None
|
||||
_lock = threading.Lock()
|
||||
|
||||
# Base Neo4j functions
|
||||
|
||||
|
||||
def get_uri() -> str:
|
||||
host = settings.DATABASES["neo4j"]["HOST"]
|
||||
port = settings.DATABASES["neo4j"]["PORT"]
|
||||
return f"bolt://{host}:{port}"
|
||||
|
||||
|
||||
def init_driver() -> neo4j.Driver:
|
||||
global _driver
|
||||
if _driver is not None:
|
||||
return _driver
|
||||
|
||||
with _lock:
|
||||
if _driver is None:
|
||||
uri = get_uri()
|
||||
config = settings.DATABASES["neo4j"]
|
||||
|
||||
_driver = neo4j.GraphDatabase.driver(
|
||||
uri,
|
||||
auth=(config["USER"], config["PASSWORD"]),
|
||||
keep_alive=True,
|
||||
max_connection_lifetime=7200,
|
||||
connection_acquisition_timeout=CONN_ACQUISITION_TIMEOUT,
|
||||
max_connection_pool_size=50,
|
||||
)
|
||||
_driver.verify_connectivity()
|
||||
|
||||
# Register cleanup handler (only runs once since we're inside the _driver is None block)
|
||||
atexit.register(close_driver)
|
||||
|
||||
return _driver
|
||||
|
||||
|
||||
def get_driver() -> neo4j.Driver:
|
||||
return init_driver()
|
||||
|
||||
|
||||
def close_driver() -> None: # TODO: Use it
|
||||
global _driver
|
||||
with _lock:
|
||||
if _driver is not None:
|
||||
try:
|
||||
_driver.close()
|
||||
|
||||
finally:
|
||||
_driver = None
|
||||
|
||||
|
||||
@contextmanager
|
||||
def get_session(
|
||||
database: str | None = None, default_access_mode: str | None = None
|
||||
) -> Iterator[RetryableSession]:
|
||||
session_wrapper: RetryableSession | None = None
|
||||
|
||||
try:
|
||||
session_wrapper = RetryableSession(
|
||||
session_factory=lambda: get_driver().session(
|
||||
database=database, default_access_mode=default_access_mode
|
||||
),
|
||||
max_retries=SERVICE_UNAVAILABLE_MAX_RETRIES,
|
||||
)
|
||||
yield session_wrapper
|
||||
|
||||
except neo4j.exceptions.Neo4jError as exc:
|
||||
if (
|
||||
default_access_mode == neo4j.READ_ACCESS
|
||||
and exc.code
|
||||
and exc.code in READ_EXCEPTION_CODES
|
||||
):
|
||||
message = "Read query not allowed"
|
||||
code = READ_EXCEPTION_CODES[0]
|
||||
raise WriteQueryNotAllowedException(message=message, code=code)
|
||||
|
||||
message = exc.message if exc.message is not None else str(exc)
|
||||
|
||||
if exc.code and exc.code.startswith(CLIENT_STATEMENT_EXCEPTION_PREFIX):
|
||||
raise ClientStatementException(message=message, code=exc.code)
|
||||
|
||||
raise GraphDatabaseQueryException(message=message, code=exc.code)
|
||||
|
||||
finally:
|
||||
if session_wrapper is not None:
|
||||
session_wrapper.close()
|
||||
|
||||
|
||||
def execute_read_query(
|
||||
database: str,
|
||||
cypher: str,
|
||||
parameters: dict[str, Any] | None = None,
|
||||
) -> neo4j.graph.Graph:
|
||||
with get_session(database, default_access_mode=neo4j.READ_ACCESS) as session:
|
||||
|
||||
def _run(tx: neo4j.ManagedTransaction) -> neo4j.graph.Graph:
|
||||
result = tx.run(
|
||||
cypher, parameters or {}, timeout=READ_QUERY_TIMEOUT_SECONDS
|
||||
)
|
||||
return result.graph()
|
||||
|
||||
return session.execute_read(_run)
|
||||
|
||||
|
||||
def create_database(database: str) -> None:
|
||||
query = "CREATE DATABASE $database IF NOT EXISTS"
|
||||
parameters = {"database": database}
|
||||
|
||||
with get_session() as session:
|
||||
session.run(query, parameters)
|
||||
|
||||
|
||||
def drop_database(database: str) -> None:
|
||||
query = f"DROP DATABASE `{database}` IF EXISTS DESTROY DATA"
|
||||
|
||||
with get_session() as session:
|
||||
session.run(query)
|
||||
|
||||
|
||||
def drop_subgraph(database: str, provider_id: str) -> int:
|
||||
"""
|
||||
Delete all nodes for a provider from the tenant database.
|
||||
|
||||
Uses batched deletion to avoid memory issues with large graphs.
|
||||
Silently returns 0 if the database doesn't exist.
|
||||
"""
|
||||
provider_label = get_provider_label(provider_id)
|
||||
deleted_nodes = 0
|
||||
|
||||
try:
|
||||
with get_session(database) as session:
|
||||
deleted_count = 1
|
||||
while deleted_count > 0:
|
||||
result = session.run(
|
||||
f"""
|
||||
MATCH (n:{PROVIDER_RESOURCE_LABEL}:`{provider_label}`)
|
||||
WITH n LIMIT $batch_size
|
||||
DETACH DELETE n
|
||||
RETURN COUNT(n) AS deleted_nodes_count
|
||||
""",
|
||||
{"batch_size": BATCH_SIZE},
|
||||
)
|
||||
deleted_count = result.single().get("deleted_nodes_count", 0)
|
||||
deleted_nodes += deleted_count
|
||||
|
||||
except GraphDatabaseQueryException as exc:
|
||||
if exc.code == "Neo.ClientError.Database.DatabaseNotFound":
|
||||
return 0
|
||||
raise
|
||||
|
||||
return deleted_nodes
|
||||
|
||||
|
||||
def has_provider_data(database: str, provider_id: str) -> bool:
|
||||
"""
|
||||
Check if any ProviderResource node exists for this provider.
|
||||
|
||||
Returns `False` if the database doesn't exist.
|
||||
"""
|
||||
provider_label = get_provider_label(provider_id)
|
||||
query = f"MATCH (n:{PROVIDER_RESOURCE_LABEL}:`{provider_label}`) RETURN 1 LIMIT 1"
|
||||
|
||||
try:
|
||||
with get_session(database, default_access_mode=neo4j.READ_ACCESS) as session:
|
||||
result = session.run(query)
|
||||
return result.single() is not None
|
||||
|
||||
except GraphDatabaseQueryException as exc:
|
||||
if exc.code == "Neo.ClientError.Database.DatabaseNotFound":
|
||||
return False
|
||||
raise
|
||||
|
||||
|
||||
def clear_cache(database: str) -> None:
|
||||
query = "CALL db.clearQueryCaches()"
|
||||
|
||||
try:
|
||||
with get_session(database) as session:
|
||||
session.run(query)
|
||||
|
||||
except GraphDatabaseQueryException as exc:
|
||||
logging.warning(f"Failed to clear query cache for database `{database}`: {exc}")
|
||||
|
||||
|
||||
# Neo4j functions related to Prowler + Cartography
|
||||
|
||||
|
||||
def get_database_name(entity_id: str | UUID, temporary: bool = False) -> str:
|
||||
prefix = "tmp-scan" if temporary else "tenant"
|
||||
return f"db-{prefix}-{str(entity_id).lower()}"
|
||||
TEMP_DB_PREFIX = "db-tmp-scan-"
|
||||
|
||||
|
||||
# Exceptions
|
||||
@@ -242,7 +41,6 @@ class GraphDatabaseQueryException(Exception):
|
||||
def __str__(self) -> str:
|
||||
if self.code:
|
||||
return f"{self.code}: {self.message}"
|
||||
|
||||
return self.message
|
||||
|
||||
|
||||
@@ -252,3 +50,177 @@ class WriteQueryNotAllowedException(GraphDatabaseQueryException):
|
||||
|
||||
class ClientStatementException(GraphDatabaseQueryException):
|
||||
pass
|
||||
|
||||
|
||||
# Routing
|
||||
|
||||
|
||||
def _is_ingest_database(database: str | None) -> bool:
|
||||
return bool(database) and database.startswith(TEMP_DB_PREFIX)
|
||||
|
||||
|
||||
# Driver lifecycle
|
||||
|
||||
|
||||
def init_driver() -> Any:
|
||||
"""Initialize the configured sink backend.
|
||||
|
||||
The ingest driver (Neo4j for cartography temp DBs) stays lazy: it is
|
||||
only initialized when a temp-DB operation actually runs, which never
|
||||
happens on API pods.
|
||||
"""
|
||||
return sink_module.init()
|
||||
|
||||
|
||||
def close_driver() -> None:
|
||||
"""Close every driver held by this process."""
|
||||
sink_module.close()
|
||||
ingest.close_driver()
|
||||
|
||||
|
||||
def get_driver() -> neo4j.Driver:
|
||||
"""Return the sink backend's underlying driver.
|
||||
|
||||
Only meaningful for the Neo4j sink (where the backend has a single Neo4j
|
||||
driver). On Neptune this returns the writer driver. Kept for tests and
|
||||
legacy call-sites; prefer `get_session` for new code.
|
||||
"""
|
||||
backend = sink_module.get_backend()
|
||||
|
||||
# Neo4jSink exposes get_driver(); NeptuneSink exposes get_writer()
|
||||
if hasattr(backend, "get_driver"):
|
||||
return backend.get_driver()
|
||||
|
||||
if hasattr(backend, "get_writer"):
|
||||
return backend.get_writer()
|
||||
|
||||
raise RuntimeError("Active sink backend does not expose a driver handle")
|
||||
|
||||
|
||||
def verify_connectivity() -> None:
|
||||
"""Raise if the configured graph database is unreachable on the API read path.
|
||||
|
||||
Backend-agnostic entry point for the readiness probe: Neo4j verifies its
|
||||
driver, Neptune verifies the reader endpoint.
|
||||
"""
|
||||
sink_module.get_backend().verify_connectivity()
|
||||
|
||||
|
||||
def verify_scan_databases_available() -> None:
|
||||
"""Raise if either graph database needed by an Attack Paths scan is unavailable."""
|
||||
errors: list[str] = []
|
||||
first_error: Exception | None = None
|
||||
|
||||
try:
|
||||
ingest.get_driver().verify_connectivity()
|
||||
except Exception as exc:
|
||||
errors.append(f"ingest Neo4j: {exc}")
|
||||
first_error = exc
|
||||
|
||||
try:
|
||||
get_driver().verify_connectivity()
|
||||
except Exception as exc:
|
||||
errors.append(f"sink {settings.ATTACK_PATHS_SINK_DATABASE}: {exc}")
|
||||
if first_error is None:
|
||||
first_error = exc
|
||||
|
||||
if errors:
|
||||
raise RuntimeError(
|
||||
"Attack Paths graph database unavailable before scan start: "
|
||||
+ "; ".join(errors)
|
||||
) from first_error
|
||||
|
||||
|
||||
def get_uri() -> str:
|
||||
"""Return the sink URI. Retained for backwards compatibility."""
|
||||
if settings.ATTACK_PATHS_SINK_DATABASE == "neptune":
|
||||
cfg = settings.DATABASES["neptune"]
|
||||
return f"bolt+s://{cfg['WRITER_ENDPOINT']}:{cfg['PORT']}"
|
||||
|
||||
cfg = settings.DATABASES["neo4j"]
|
||||
return f"bolt://{cfg['HOST']}:{cfg['PORT']}"
|
||||
|
||||
|
||||
def get_ingest_uri() -> str:
|
||||
"""Neo4j URI for the cartography temp (ingest) database, which is always
|
||||
Neo4j regardless of the configured sink."""
|
||||
return ingest.get_uri()
|
||||
|
||||
|
||||
# Session API
|
||||
|
||||
|
||||
def get_session(
|
||||
database: str | None = None,
|
||||
default_access_mode: str | None = None,
|
||||
) -> AbstractContextManager:
|
||||
"""Return a session against the right backend.
|
||||
|
||||
- `database` names starting with `db-tmp-scan-` always go to ingest.
|
||||
- No database name → ingest (used for CREATE / DROP DATABASE admin ops).
|
||||
- Any other name → sink.
|
||||
"""
|
||||
if _is_ingest_database(database) or database is None:
|
||||
return ingest.get_session(
|
||||
database=database, default_access_mode=default_access_mode
|
||||
)
|
||||
|
||||
return sink_module.get_backend().get_session(
|
||||
database=database, default_access_mode=default_access_mode
|
||||
)
|
||||
|
||||
|
||||
def execute_read_query(
|
||||
database: str,
|
||||
cypher: str,
|
||||
parameters: dict[str, Any] | None = None,
|
||||
) -> neo4j.graph.Graph:
|
||||
"""Read-only query against the sink."""
|
||||
return sink_module.get_backend().execute_read_query(database, cypher, parameters)
|
||||
|
||||
|
||||
def create_database(database: str) -> None:
|
||||
"""Create a database. Temp DBs always land on ingest (Neo4j).
|
||||
|
||||
On the Neo4j sink, tenant DBs also route to ingest because both drivers
|
||||
connect to the same Neo4j cluster. On the Neptune sink, tenant DB creates
|
||||
are no-ops.
|
||||
"""
|
||||
if _is_ingest_database(database):
|
||||
ingest.create_database(database)
|
||||
return
|
||||
|
||||
sink_module.get_backend().create_database(database)
|
||||
|
||||
|
||||
def drop_database(database: str) -> None:
|
||||
"""Drop a database. Mirrors `create_database` routing."""
|
||||
if _is_ingest_database(database):
|
||||
ingest.drop_database(database)
|
||||
return
|
||||
|
||||
sink_module.get_backend().drop_database(database)
|
||||
|
||||
|
||||
def drop_subgraph(database: str, provider_id: str) -> int:
|
||||
return sink_module.get_backend().drop_subgraph(database, provider_id)
|
||||
|
||||
|
||||
def has_provider_data(database: str, provider_id: str) -> bool:
|
||||
return sink_module.get_backend().has_provider_data(database, provider_id)
|
||||
|
||||
|
||||
def clear_cache(database: str) -> None:
|
||||
if _is_ingest_database(database):
|
||||
ingest.clear_cache(database)
|
||||
return
|
||||
|
||||
sink_module.get_backend().clear_cache(database)
|
||||
|
||||
|
||||
# Name helper
|
||||
|
||||
|
||||
def get_database_name(entity_id: str | UUID, temporary: bool = False) -> str:
|
||||
prefix = "tmp-scan" if temporary else "tenant"
|
||||
return f"db-{prefix}-{str(entity_id).lower()}"
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
"""Cartography ingest layer.
|
||||
|
||||
Public surface for the per-scan Neo4j temp database driver. Implementation
|
||||
lives in `api.attack_paths.ingest.driver`.
|
||||
"""
|
||||
|
||||
from api.attack_paths.ingest.driver import (
|
||||
clear_cache,
|
||||
close_driver,
|
||||
create_database,
|
||||
drop_database,
|
||||
get_driver,
|
||||
get_session,
|
||||
get_uri,
|
||||
init_driver,
|
||||
run_cypher,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"clear_cache",
|
||||
"close_driver",
|
||||
"create_database",
|
||||
"drop_database",
|
||||
"get_driver",
|
||||
"get_session",
|
||||
"get_uri",
|
||||
"init_driver",
|
||||
"run_cypher",
|
||||
]
|
||||
@@ -0,0 +1,187 @@
|
||||
"""Cartography ingest driver: per-scan throw-away Neo4j database.
|
||||
|
||||
Cartography writes each scan's graph into a throw-away Neo4j database named
|
||||
`db-tmp-scan-{scan_uuid}`. This is always Neo4j, regardless of the configured
|
||||
sink: Neptune is single-database and cannot host per-scan throw-away
|
||||
databases. This module owns the Neo4j driver used for those temp DBs and the
|
||||
admin ops they need (CREATE / DROP DATABASE).
|
||||
"""
|
||||
|
||||
import atexit
|
||||
import logging
|
||||
import threading
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
from typing import Any
|
||||
|
||||
import neo4j
|
||||
import neo4j.exceptions
|
||||
from api.attack_paths.retryable_session import RetryableSession
|
||||
from config.env import env
|
||||
from django.conf import settings
|
||||
|
||||
logging.getLogger("neo4j").setLevel(logging.ERROR)
|
||||
logging.getLogger("neo4j").propagate = False
|
||||
|
||||
SERVICE_UNAVAILABLE_MAX_RETRIES = env.int(
|
||||
"ATTACK_PATHS_SERVICE_UNAVAILABLE_MAX_RETRIES", default=3
|
||||
)
|
||||
CONN_ACQUISITION_TIMEOUT = env.int("NEO4J_CONN_ACQUISITION_TIMEOUT", default=15)
|
||||
# TCP connect timeout, ordered below the acquisition timeout so an unreachable
|
||||
# host can't pin a worker on a temp-DB op longer than this.
|
||||
CONNECTION_TIMEOUT = env.int("NEO4J_CONNECTION_TIMEOUT", default=5)
|
||||
MAX_CONNECTION_LIFETIME = env.int("NEO4J_MAX_CONNECTION_LIFETIME", default=7200)
|
||||
MAX_CONNECTION_POOL_SIZE = env.int("NEO4J_MAX_CONNECTION_POOL_SIZE", default=50)
|
||||
|
||||
_driver: neo4j.Driver | None = None
|
||||
_lock = threading.Lock()
|
||||
|
||||
|
||||
def _neo4j_config() -> dict:
|
||||
return settings.DATABASES["neo4j"]
|
||||
|
||||
|
||||
def get_uri() -> str:
|
||||
"""Bolt URI for the Neo4j temp (ingest) database. Always Neo4j."""
|
||||
config = _neo4j_config()
|
||||
host = config["HOST"]
|
||||
port = config["PORT"]
|
||||
if not host or not port:
|
||||
raise RuntimeError(
|
||||
"NEO4J_HOST / NEO4J_PORT must be set to use the attack-paths "
|
||||
"temp database. Workers require Neo4j env even when the sink is Neptune."
|
||||
)
|
||||
|
||||
return f"bolt://{host}:{port}"
|
||||
|
||||
|
||||
def init_driver() -> neo4j.Driver:
|
||||
"""Initialize the temp-database Neo4j driver. Idempotent."""
|
||||
global _driver
|
||||
if _driver is not None:
|
||||
return _driver
|
||||
|
||||
with _lock:
|
||||
if _driver is None:
|
||||
config = _neo4j_config()
|
||||
_driver = neo4j.GraphDatabase.driver(
|
||||
get_uri(),
|
||||
auth=(config["USER"], config["PASSWORD"]),
|
||||
keep_alive=True,
|
||||
max_connection_lifetime=MAX_CONNECTION_LIFETIME,
|
||||
connection_timeout=CONNECTION_TIMEOUT,
|
||||
connection_acquisition_timeout=CONN_ACQUISITION_TIMEOUT,
|
||||
max_connection_pool_size=MAX_CONNECTION_POOL_SIZE,
|
||||
)
|
||||
# Best-effort connectivity check: a Neo4j that is down at boot must
|
||||
# not crash the worker. The driver reconnects lazily on first use.
|
||||
try:
|
||||
_driver.verify_connectivity()
|
||||
|
||||
except Exception:
|
||||
logging.warning(
|
||||
"Neo4j temp-database unreachable at init; continuing with a "
|
||||
"lazily-reconnecting driver",
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
atexit.register(close_driver)
|
||||
|
||||
return _driver
|
||||
|
||||
|
||||
def get_driver() -> neo4j.Driver:
|
||||
return init_driver()
|
||||
|
||||
|
||||
def close_driver() -> None:
|
||||
global _driver
|
||||
with _lock:
|
||||
if _driver is not None:
|
||||
try:
|
||||
_driver.close()
|
||||
finally:
|
||||
_driver = None
|
||||
|
||||
|
||||
@contextmanager
|
||||
def get_session(
|
||||
database: str | None = None,
|
||||
default_access_mode: str | None = None,
|
||||
) -> Iterator[RetryableSession]:
|
||||
"""Session against the Neo4j temp-database cluster. Used for temp DB sessions
|
||||
and for admin operations (CREATE / DROP DATABASE) when `database` is None."""
|
||||
from api.attack_paths.database import (
|
||||
ClientStatementException,
|
||||
GraphDatabaseQueryException,
|
||||
WriteQueryNotAllowedException,
|
||||
)
|
||||
|
||||
READ_EXCEPTION_CODES = [
|
||||
"Neo.ClientError.Statement.AccessMode",
|
||||
"Neo.ClientError.Procedure.ProcedureNotFound",
|
||||
]
|
||||
CLIENT_STATEMENT_EXCEPTION_PREFIX = "Neo.ClientError.Statement."
|
||||
|
||||
session_wrapper: RetryableSession | None = None
|
||||
try:
|
||||
session_wrapper = RetryableSession(
|
||||
session_factory=lambda: get_driver().session(
|
||||
database=database, default_access_mode=default_access_mode
|
||||
),
|
||||
max_retries=SERVICE_UNAVAILABLE_MAX_RETRIES,
|
||||
)
|
||||
yield session_wrapper
|
||||
|
||||
except neo4j.exceptions.Neo4jError as exc:
|
||||
if (
|
||||
default_access_mode == neo4j.READ_ACCESS
|
||||
and exc.code
|
||||
and exc.code in READ_EXCEPTION_CODES
|
||||
):
|
||||
raise WriteQueryNotAllowedException(
|
||||
message="Read query not allowed", code=READ_EXCEPTION_CODES[0]
|
||||
)
|
||||
|
||||
message = exc.message if exc.message is not None else str(exc)
|
||||
if exc.code and exc.code.startswith(CLIENT_STATEMENT_EXCEPTION_PREFIX):
|
||||
raise ClientStatementException(message=message, code=exc.code)
|
||||
raise GraphDatabaseQueryException(message=message, code=exc.code)
|
||||
|
||||
finally:
|
||||
if session_wrapper is not None:
|
||||
session_wrapper.close()
|
||||
|
||||
|
||||
def create_database(database: str) -> None:
|
||||
"""Create a database on the Neo4j cluster. Used for temp scan DBs."""
|
||||
with get_session() as session:
|
||||
session.run("CREATE DATABASE $database IF NOT EXISTS", {"database": database})
|
||||
|
||||
|
||||
def drop_database(database: str) -> None:
|
||||
"""Drop a database on the Neo4j cluster. Used for temp scan DBs."""
|
||||
with get_session() as session:
|
||||
session.run(f"DROP DATABASE `{database}` IF EXISTS DESTROY DATA")
|
||||
|
||||
|
||||
def clear_cache(database: str) -> None:
|
||||
"""Best-effort cache clear for a Neo4j database."""
|
||||
from api.attack_paths.database import GraphDatabaseQueryException
|
||||
|
||||
try:
|
||||
with get_session(database) as session:
|
||||
session.run("CALL db.clearQueryCaches()")
|
||||
|
||||
except GraphDatabaseQueryException as exc:
|
||||
logging.warning(f"Failed to clear query cache for database `{database}`: {exc}")
|
||||
|
||||
|
||||
def run_cypher(
|
||||
database: str | None,
|
||||
cypher: str,
|
||||
parameters: dict[str, Any] | None = None,
|
||||
) -> Any:
|
||||
"""Execute Cypher directly without the context manager. Thin helper."""
|
||||
with get_session(database) as session:
|
||||
return session.run(cypher, parameters or {})
|
||||
@@ -1,12 +1,11 @@
|
||||
from api.attack_paths.queries.types import (
|
||||
AttackPathsQueryDefinition,
|
||||
AttackPathsQueryParameterDefinition,
|
||||
)
|
||||
from api.attack_paths.queries.registry import (
|
||||
get_queries_for_provider,
|
||||
get_query_by_id,
|
||||
)
|
||||
|
||||
from api.attack_paths.queries.types import (
|
||||
AttackPathsQueryDefinition,
|
||||
AttackPathsQueryParameterDefinition,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"AttackPathsQueryDefinition",
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,13 +1,14 @@
|
||||
from api.attack_paths.queries.types import AttackPathsQueryDefinition
|
||||
from api.attack_paths.queries.aws import AWS_QUERIES
|
||||
|
||||
# TODO: drop after Neptune cutover
|
||||
from api.attack_paths.queries.aws_deprecated import AWS_DEPRECATED_QUERIES
|
||||
from api.attack_paths.queries.types import AttackPathsQueryDefinition
|
||||
|
||||
# Query definitions organized by provider
|
||||
# Query definitions for scans synced with the current schema.
|
||||
_QUERY_DEFINITIONS: dict[str, list[AttackPathsQueryDefinition]] = {
|
||||
"aws": AWS_QUERIES,
|
||||
}
|
||||
|
||||
# Flat lookup by query ID for O(1) access
|
||||
_QUERIES_BY_ID: dict[str, AttackPathsQueryDefinition] = {
|
||||
definition.id: definition
|
||||
for definitions in _QUERY_DEFINITIONS.values()
|
||||
@@ -15,11 +16,45 @@ _QUERIES_BY_ID: dict[str, AttackPathsQueryDefinition] = {
|
||||
}
|
||||
|
||||
|
||||
def get_queries_for_provider(provider: str) -> list[AttackPathsQueryDefinition]:
|
||||
"""Get all attack path queries for a specific provider."""
|
||||
return _QUERY_DEFINITIONS.get(provider, [])
|
||||
# TODO: drop after Neptune cutover
|
||||
#
|
||||
# Query definitions for pre-cutover scans (`AttackPathsScan.is_migrated=False`)
|
||||
# whose graph data was written under the previous schema. Both maps expose the
|
||||
# same query IDs so the API contract is identical regardless of which set is
|
||||
# routed to.
|
||||
_DEPRECATED_QUERY_DEFINITIONS: dict[str, list[AttackPathsQueryDefinition]] = {
|
||||
"aws": AWS_DEPRECATED_QUERIES,
|
||||
}
|
||||
|
||||
_DEPRECATED_QUERIES_BY_ID: dict[str, AttackPathsQueryDefinition] = {
|
||||
definition.id: definition
|
||||
for definitions in _DEPRECATED_QUERY_DEFINITIONS.values()
|
||||
for definition in definitions
|
||||
}
|
||||
|
||||
|
||||
def get_query_by_id(query_id: str) -> AttackPathsQueryDefinition | None:
|
||||
"""Get a specific attack path query by its ID."""
|
||||
return _QUERIES_BY_ID.get(query_id)
|
||||
def get_queries_for_provider(
|
||||
provider: str,
|
||||
is_migrated: bool = True,
|
||||
) -> list[AttackPathsQueryDefinition]:
|
||||
"""Get all attack path queries for a provider.
|
||||
|
||||
`is_migrated` selects the catalog: True for scans synced with the current
|
||||
schema, False for pre-cutover scans still using the legacy graph shape.
|
||||
# TODO: drop the `is_migrated` parameter after Neptune cutover
|
||||
"""
|
||||
catalog = _QUERY_DEFINITIONS if is_migrated else _DEPRECATED_QUERY_DEFINITIONS
|
||||
return catalog.get(provider, [])
|
||||
|
||||
|
||||
def get_query_by_id(
|
||||
query_id: str,
|
||||
is_migrated: bool = True,
|
||||
) -> AttackPathsQueryDefinition | None:
|
||||
"""Get a specific attack path query by ID.
|
||||
|
||||
`is_migrated` selects the catalog (see `get_queries_for_provider`).
|
||||
# TODO: drop the `is_migrated` parameter after Neptune cutover
|
||||
"""
|
||||
by_id = _QUERIES_BY_ID if is_migrated else _DEPRECATED_QUERIES_BY_ID
|
||||
return by_id.get(query_id)
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import logging
|
||||
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
"""Attack-paths sink database layer.
|
||||
|
||||
The sink is the persistent store where attack-paths graphs live after a scan
|
||||
finishes. Currently selectable between Neo4j (OSS / local dev default) and
|
||||
AWS Neptune (hosted dev/staging/prod). Backend is picked by the
|
||||
`ATTACK_PATHS_SINK_DATABASE` setting at process init.
|
||||
|
||||
This package exposes the public factory API; the implementation lives in
|
||||
`api.attack_paths.sink.factory`.
|
||||
"""
|
||||
|
||||
from api.attack_paths.sink.factory import (
|
||||
SinkBackend,
|
||||
close,
|
||||
get_backend,
|
||||
get_backend_for_name,
|
||||
get_backend_for_scan,
|
||||
init,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"SinkBackend",
|
||||
"close",
|
||||
"get_backend",
|
||||
"get_backend_for_name",
|
||||
"get_backend_for_scan",
|
||||
"init",
|
||||
]
|
||||
@@ -0,0 +1,92 @@
|
||||
"""Protocol every sink backend must implement."""
|
||||
|
||||
from contextlib import AbstractContextManager
|
||||
from typing import Any, Protocol
|
||||
|
||||
import neo4j
|
||||
|
||||
|
||||
class SinkDatabase(Protocol):
|
||||
"""Contract for the persistent attack-paths graph store.
|
||||
|
||||
The `database` argument is an opaque identifier passed through from the
|
||||
legacy `database.py` API surface. On Neo4j it is the per-tenant database
|
||||
name (e.g. `db-tenant-{uuid}`). On Neptune it is ignored (the cluster
|
||||
has a single graph, and isolation is label-based).
|
||||
"""
|
||||
|
||||
def init(self) -> None: ...
|
||||
|
||||
def close(self) -> None: ...
|
||||
|
||||
def verify_connectivity(self) -> None:
|
||||
"""Raise if the backend the API read path uses is unreachable.
|
||||
|
||||
Neo4j verifies its single driver. Neptune verifies the reader
|
||||
driver (the endpoint the API serves reads from); on single-endpoint
|
||||
clusters the reader aliases the writer, so that path is covered too.
|
||||
Used by the readiness probe; must not block longer than the caller's
|
||||
probe budget.
|
||||
"""
|
||||
...
|
||||
|
||||
def get_session(
|
||||
self,
|
||||
database: str | None = None,
|
||||
default_access_mode: str | None = None,
|
||||
) -> AbstractContextManager: ...
|
||||
|
||||
def execute_read_query(
|
||||
self,
|
||||
database: str,
|
||||
cypher: str,
|
||||
parameters: dict[str, Any] | None = None,
|
||||
) -> neo4j.graph.Graph: ...
|
||||
|
||||
def create_database(self, database: str) -> None: ...
|
||||
|
||||
def drop_database(self, database: str) -> None: ...
|
||||
|
||||
def drop_subgraph(self, database: str, provider_id: str) -> int: ...
|
||||
|
||||
def has_provider_data(self, database: str, provider_id: str) -> bool: ...
|
||||
|
||||
def clear_cache(self, database: str) -> None: ...
|
||||
|
||||
def ensure_sync_indexes(self, database: str) -> None:
|
||||
"""Create any index needed for the sync write path.
|
||||
|
||||
Called once at the start of each provider sync; must be idempotent.
|
||||
Neo4j creates a `_provider_element_id` index on `_ProviderResource`;
|
||||
Neptune is a no-op (its `~id` lookup needs no index).
|
||||
"""
|
||||
...
|
||||
|
||||
def write_nodes(
|
||||
self,
|
||||
database: str,
|
||||
labels: str,
|
||||
rows: list[dict[str, Any]],
|
||||
) -> None:
|
||||
"""Upsert a batch of nodes into the sink.
|
||||
|
||||
`labels` is a pre-rendered Cypher label string ready to drop after
|
||||
the node variable (e.g. `` `AWSUser`:`_ProviderResource`:`_Tenant_x` ``).
|
||||
Each row carries `provider_element_id` and `props`.
|
||||
"""
|
||||
...
|
||||
|
||||
def write_relationships(
|
||||
self,
|
||||
database: str,
|
||||
rel_type: str,
|
||||
provider_id: str,
|
||||
rows: list[dict[str, Any]],
|
||||
) -> None:
|
||||
"""Upsert a batch of relationships into the sink.
|
||||
|
||||
Each row carries `start_element_id`, `end_element_id`,
|
||||
`provider_element_id` and `props`. `rel_type` is the relationship
|
||||
type (already a valid Cypher identifier).
|
||||
"""
|
||||
...
|
||||
@@ -0,0 +1,78 @@
|
||||
"""Shared batched deletion helpers for sink backends."""
|
||||
|
||||
import logging
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
RELATIONSHIP_DELETE_QUERY_TEMPLATES = {
|
||||
"outgoing relationship": """
|
||||
MATCH (n:`{provider_label}`)-[r]->()
|
||||
WITH r LIMIT $batch_size
|
||||
DELETE r
|
||||
RETURN COUNT(r) AS deleted_rels_count
|
||||
""",
|
||||
"incoming relationship": """
|
||||
MATCH (n:`{provider_label}`)<-[r]-()
|
||||
WITH r LIMIT $batch_size
|
||||
DELETE r
|
||||
RETURN COUNT(r) AS deleted_rels_count
|
||||
""",
|
||||
}
|
||||
|
||||
NODE_DELETE_QUERY_TEMPLATE = """
|
||||
MATCH (n:{provider_resource_label}:`{provider_label}`)
|
||||
WITH n LIMIT $batch_size
|
||||
DELETE n
|
||||
RETURN COUNT(n) AS deleted_nodes_count
|
||||
"""
|
||||
|
||||
|
||||
def delete_batches(
|
||||
*,
|
||||
session: Any,
|
||||
logger: logging.Logger,
|
||||
log_target: str,
|
||||
provider_id: str,
|
||||
query: str,
|
||||
phase: str,
|
||||
count_key: str,
|
||||
total_key: str,
|
||||
deleted_key: str,
|
||||
initial_total: int,
|
||||
batch_size: int,
|
||||
drop_t0: float,
|
||||
) -> tuple[int, int]:
|
||||
deleted_total = initial_total
|
||||
batches = 0
|
||||
while True:
|
||||
logger.info(
|
||||
"Deleting %s batch from %s "
|
||||
"(provider=%s, batch=%s, total_%s=%s, elapsed=%.3fs)",
|
||||
phase,
|
||||
log_target,
|
||||
provider_id,
|
||||
batches + 1,
|
||||
total_key,
|
||||
deleted_total,
|
||||
time.perf_counter() - drop_t0,
|
||||
)
|
||||
record = session.run(query, {"batch_size": batch_size}).single()
|
||||
deleted = (record[count_key] if record else 0) or 0
|
||||
if deleted == 0:
|
||||
return deleted_total, batches
|
||||
|
||||
batches += 1
|
||||
deleted_total += deleted
|
||||
logger.info(
|
||||
"Deleted %s batch from %s "
|
||||
"(provider=%s, batch=%s, %s=%s, total_%s=%s, elapsed=%.3fs)",
|
||||
phase,
|
||||
log_target,
|
||||
provider_id,
|
||||
batches,
|
||||
deleted_key,
|
||||
deleted,
|
||||
total_key,
|
||||
deleted_total,
|
||||
time.perf_counter() - drop_t0,
|
||||
)
|
||||
@@ -0,0 +1,134 @@
|
||||
"""Sink backend factory and process-wide handle cache.
|
||||
|
||||
Picks the active backend from `settings.ATTACK_PATHS_SINK_DATABASE` at first
|
||||
use, holds the active backend plus any secondary backends needed to serve
|
||||
scans written under the previous configuration, and tears them all down on
|
||||
process shutdown. Imported via `from api.attack_paths import sink as
|
||||
sink_module`.
|
||||
"""
|
||||
|
||||
import threading
|
||||
from enum import StrEnum, auto
|
||||
|
||||
from api.attack_paths.sink.base import SinkDatabase
|
||||
from api.models import AttackPathsScan
|
||||
from django.conf import settings
|
||||
|
||||
# Backend names
|
||||
|
||||
|
||||
class SinkBackend(StrEnum):
|
||||
NEO4J = auto()
|
||||
NEPTUNE = auto()
|
||||
|
||||
|
||||
# Backend cache
|
||||
|
||||
_backend: SinkDatabase | None = None
|
||||
_secondary_backends: dict[SinkBackend, SinkDatabase] = {}
|
||||
_lock = threading.Lock()
|
||||
|
||||
|
||||
def _resolve_setting() -> SinkBackend:
|
||||
raw = settings.ATTACK_PATHS_SINK_DATABASE.lower()
|
||||
try:
|
||||
return SinkBackend(raw)
|
||||
|
||||
except ValueError:
|
||||
valid = sorted(b.value for b in SinkBackend)
|
||||
raise RuntimeError(
|
||||
f"ATTACK_PATHS_SINK_DATABASE must be one of {valid}; got {raw!r}"
|
||||
)
|
||||
|
||||
|
||||
def _build_backend(name: SinkBackend) -> SinkDatabase:
|
||||
if name is SinkBackend.NEO4J:
|
||||
from api.attack_paths.sink.neo4j import Neo4jSink
|
||||
|
||||
return Neo4jSink()
|
||||
|
||||
if name is SinkBackend.NEPTUNE:
|
||||
from api.attack_paths.sink.neptune import NeptuneSink
|
||||
|
||||
return NeptuneSink()
|
||||
|
||||
raise RuntimeError(f"Unknown sink backend {name!r}")
|
||||
|
||||
|
||||
# Lifecycle
|
||||
|
||||
|
||||
def init(name: SinkBackend | str | None = None) -> SinkDatabase:
|
||||
"""Initialize the configured sink backend. Idempotent."""
|
||||
global _backend
|
||||
if _backend is not None:
|
||||
return _backend
|
||||
|
||||
with _lock:
|
||||
if _backend is None:
|
||||
resolved = SinkBackend(name) if name else _resolve_setting()
|
||||
backend = _build_backend(resolved)
|
||||
backend.init()
|
||||
_backend = backend
|
||||
|
||||
return _backend
|
||||
|
||||
|
||||
def close() -> None:
|
||||
"""Close the active backend and every cached secondary backend."""
|
||||
global _backend
|
||||
with _lock:
|
||||
backends = [
|
||||
b for b in (_backend, *_secondary_backends.values()) if b is not None
|
||||
]
|
||||
_backend = None
|
||||
_secondary_backends.clear()
|
||||
|
||||
for backend in backends:
|
||||
try:
|
||||
backend.close()
|
||||
|
||||
except Exception: # pragma: no cover - best-effort
|
||||
pass
|
||||
|
||||
|
||||
def get_backend() -> SinkDatabase:
|
||||
"""Return the active sink. Initializes on first call."""
|
||||
return init()
|
||||
|
||||
|
||||
# Per-scan routing
|
||||
|
||||
|
||||
def get_backend_for_scan(scan: AttackPathsScan) -> SinkDatabase:
|
||||
"""Route reads by the sink that stores this scan's graph."""
|
||||
raw_backend = getattr(scan, "sink_backend", SinkBackend.NEO4J.value)
|
||||
if not isinstance(raw_backend, str):
|
||||
raw_backend = SinkBackend.NEO4J.value
|
||||
return get_backend_for_name(raw_backend)
|
||||
|
||||
|
||||
def get_backend_for_name(name: SinkBackend | str) -> SinkDatabase:
|
||||
"""Return the backend named by persisted scan metadata."""
|
||||
resolved = SinkBackend(name)
|
||||
if resolved is _resolve_setting():
|
||||
return get_backend()
|
||||
|
||||
return _build_backend_cached(resolved)
|
||||
|
||||
|
||||
def _build_backend_cached(name: SinkBackend) -> SinkDatabase:
|
||||
# TODO: drop after Neptune cutover
|
||||
# Needed only during cutover to serve Neo4j-written scans from a Neptune-
|
||||
# configured API pod (and vice versa). Once every scan is on Neptune,
|
||||
# `get_backend_for_scan` becomes a one-liner returning `get_backend()`.
|
||||
if name in _secondary_backends:
|
||||
return _secondary_backends[name]
|
||||
|
||||
with _lock:
|
||||
if name not in _secondary_backends:
|
||||
backend = _build_backend(name)
|
||||
backend.init()
|
||||
_secondary_backends[name] = backend
|
||||
|
||||
return _secondary_backends[name]
|
||||
@@ -0,0 +1,417 @@
|
||||
"""Neo4j sink implementation.
|
||||
|
||||
Owns a Neo4j driver independent from the staging driver. On OSS and local dev
|
||||
this is the only sink; on hosted deployments it runs only as a legacy read
|
||||
path while phase-1 drains tenant DBs.
|
||||
"""
|
||||
|
||||
import atexit
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Iterator
|
||||
from contextlib import AbstractContextManager, contextmanager
|
||||
from typing import Any
|
||||
|
||||
import neo4j
|
||||
import neo4j.exceptions
|
||||
from api.attack_paths.retryable_session import RetryableSession
|
||||
from api.attack_paths.sink.base import SinkDatabase
|
||||
from api.attack_paths.sink.drop import (
|
||||
NODE_DELETE_QUERY_TEMPLATE,
|
||||
RELATIONSHIP_DELETE_QUERY_TEMPLATES,
|
||||
delete_batches,
|
||||
)
|
||||
from config.env import env
|
||||
from django.conf import settings
|
||||
|
||||
logging.getLogger("neo4j").setLevel(logging.ERROR)
|
||||
logging.getLogger("neo4j").propagate = False
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SERVICE_UNAVAILABLE_MAX_RETRIES = env.int(
|
||||
"ATTACK_PATHS_SERVICE_UNAVAILABLE_MAX_RETRIES", default=3
|
||||
)
|
||||
READ_QUERY_TIMEOUT_SECONDS = env.int(
|
||||
"ATTACK_PATHS_READ_QUERY_TIMEOUT_SECONDS", default=30
|
||||
)
|
||||
CONN_ACQUISITION_TIMEOUT = env.int("NEO4J_CONN_ACQUISITION_TIMEOUT", default=15)
|
||||
# TCP connect timeout, ordered below the acquisition timeout so an unreachable
|
||||
# host can't pin a request or the readiness probe longer than this.
|
||||
CONNECTION_TIMEOUT = env.int("NEO4J_CONNECTION_TIMEOUT", default=5)
|
||||
MAX_CONNECTION_LIFETIME = env.int("NEO4J_MAX_CONNECTION_LIFETIME", default=7200)
|
||||
MAX_CONNECTION_POOL_SIZE = env.int("NEO4J_MAX_CONNECTION_POOL_SIZE", default=50)
|
||||
|
||||
READ_EXCEPTION_CODES = [
|
||||
"Neo.ClientError.Statement.AccessMode",
|
||||
"Neo.ClientError.Procedure.ProcedureNotFound",
|
||||
]
|
||||
CLIENT_STATEMENT_EXCEPTION_PREFIX = "Neo.ClientError.Statement."
|
||||
DATABASE_NOT_FOUND_CODE = "Neo.ClientError.Database.DatabaseNotFound"
|
||||
|
||||
|
||||
class Neo4jSink(SinkDatabase):
|
||||
"""Neo4j-backed sink. Multi-database cluster; tenant isolation is physical."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._driver: neo4j.Driver | None = None
|
||||
self._lock = threading.Lock()
|
||||
self._atexit_registered = False
|
||||
|
||||
# Driver
|
||||
|
||||
def _config(self) -> dict:
|
||||
return settings.DATABASES["neo4j"]
|
||||
|
||||
def _uri(self) -> str:
|
||||
cfg = self._config()
|
||||
host = cfg["HOST"]
|
||||
port = cfg["PORT"]
|
||||
if not host or not port:
|
||||
raise RuntimeError(
|
||||
"NEO4J_HOST / NEO4J_PORT must be set when ATTACK_PATHS_SINK_DATABASE=neo4j"
|
||||
)
|
||||
return f"bolt://{host}:{port}"
|
||||
|
||||
def init(self) -> neo4j.Driver:
|
||||
if self._driver is not None:
|
||||
return self._driver
|
||||
with self._lock:
|
||||
if self._driver is None:
|
||||
cfg = self._config()
|
||||
self._driver = neo4j.GraphDatabase.driver(
|
||||
self._uri(),
|
||||
auth=(cfg["USER"], cfg["PASSWORD"]),
|
||||
keep_alive=True,
|
||||
max_connection_lifetime=MAX_CONNECTION_LIFETIME,
|
||||
connection_timeout=CONNECTION_TIMEOUT,
|
||||
connection_acquisition_timeout=CONN_ACQUISITION_TIMEOUT,
|
||||
max_connection_pool_size=MAX_CONNECTION_POOL_SIZE,
|
||||
)
|
||||
# Eager connectivity check is best-effort:
|
||||
# A Neo4j that is down at boot must not crash the process, same degradation model as Postgres
|
||||
# The driver reconnects lazily on first use
|
||||
# /health/ready surfaces the outage until it recovers
|
||||
try:
|
||||
self._driver.verify_connectivity()
|
||||
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Neo4j sink unreachable at init; continuing with a lazily-reconnecting driver",
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
if not self._atexit_registered:
|
||||
atexit.register(self.close)
|
||||
self._atexit_registered = True
|
||||
return self._driver
|
||||
|
||||
def _get_driver(self) -> neo4j.Driver:
|
||||
return self.init()
|
||||
|
||||
def verify_connectivity(self) -> None:
|
||||
self._get_driver().verify_connectivity()
|
||||
|
||||
def close(self) -> None:
|
||||
with self._lock:
|
||||
if self._driver is not None:
|
||||
try:
|
||||
self._driver.close()
|
||||
finally:
|
||||
self._driver = None
|
||||
|
||||
# Sessions
|
||||
|
||||
@contextmanager
|
||||
def get_session(
|
||||
self,
|
||||
database: str | None = None,
|
||||
default_access_mode: str | None = None,
|
||||
) -> Iterator[RetryableSession]:
|
||||
from api.attack_paths.database import (
|
||||
ClientStatementException,
|
||||
GraphDatabaseQueryException,
|
||||
WriteQueryNotAllowedException,
|
||||
)
|
||||
|
||||
session_wrapper: RetryableSession | None = None
|
||||
try:
|
||||
session_wrapper = RetryableSession(
|
||||
session_factory=lambda: self._get_driver().session(
|
||||
database=database, default_access_mode=default_access_mode
|
||||
),
|
||||
max_retries=SERVICE_UNAVAILABLE_MAX_RETRIES,
|
||||
)
|
||||
yield session_wrapper
|
||||
|
||||
except neo4j.exceptions.Neo4jError as exc:
|
||||
if (
|
||||
default_access_mode == neo4j.READ_ACCESS
|
||||
and exc.code
|
||||
and exc.code in READ_EXCEPTION_CODES
|
||||
):
|
||||
raise WriteQueryNotAllowedException(
|
||||
message="Read query not allowed", code=READ_EXCEPTION_CODES[0]
|
||||
)
|
||||
|
||||
message = exc.message if exc.message is not None else str(exc)
|
||||
if exc.code and exc.code.startswith(CLIENT_STATEMENT_EXCEPTION_PREFIX):
|
||||
raise ClientStatementException(message=message, code=exc.code)
|
||||
raise GraphDatabaseQueryException(message=message, code=exc.code)
|
||||
|
||||
finally:
|
||||
if session_wrapper is not None:
|
||||
session_wrapper.close()
|
||||
|
||||
# Operations
|
||||
|
||||
def execute_read_query(
|
||||
self,
|
||||
database: str,
|
||||
cypher: str,
|
||||
parameters: dict[str, Any] | None = None,
|
||||
) -> neo4j.graph.Graph:
|
||||
with self.get_session(
|
||||
database, default_access_mode=neo4j.READ_ACCESS
|
||||
) as session:
|
||||
|
||||
def _run(tx: neo4j.ManagedTransaction) -> neo4j.graph.Graph:
|
||||
result = tx.run(
|
||||
cypher, parameters or {}, timeout=READ_QUERY_TIMEOUT_SECONDS
|
||||
)
|
||||
return result.graph()
|
||||
|
||||
return session.execute_read(_run)
|
||||
|
||||
def create_database(self, database: str) -> None:
|
||||
with self.get_session() as session:
|
||||
session.run(
|
||||
"CREATE DATABASE $database IF NOT EXISTS", {"database": database}
|
||||
)
|
||||
|
||||
def drop_database(self, database: str) -> None:
|
||||
with self.get_session() as session:
|
||||
session.run(f"DROP DATABASE `{database}` IF EXISTS DESTROY DATA")
|
||||
|
||||
def drop_subgraph(self, database: str, provider_id: str) -> int:
|
||||
"""Delete all nodes for a provider from a tenant database, batched.
|
||||
|
||||
Deletes relationships then nodes in batches (not `DETACH DELETE`) so a
|
||||
dense provider's graph cannot exceed Neo4j's transaction memory limit.
|
||||
Silently returns 0 if the database doesn't exist.
|
||||
"""
|
||||
from api.attack_paths.database import GraphDatabaseQueryException
|
||||
from tasks.jobs.attack_paths.config import (
|
||||
BATCH_SIZE,
|
||||
PROVIDER_RESOURCE_LABEL,
|
||||
get_provider_label,
|
||||
)
|
||||
|
||||
provider_label = get_provider_label(provider_id)
|
||||
deleted_nodes = deleted_relationships = 0
|
||||
relationship_batches = node_batches = 0
|
||||
drop_t0 = time.perf_counter()
|
||||
|
||||
logger.info(
|
||||
"Dropping provider graph from Neo4j sink database %s "
|
||||
"(provider=%s, provider_label=%s)",
|
||||
database,
|
||||
provider_id,
|
||||
provider_label,
|
||||
)
|
||||
|
||||
try:
|
||||
logger.info(
|
||||
"Opening Neo4j sink session for provider graph drop "
|
||||
"(database=%s, provider=%s)",
|
||||
database,
|
||||
provider_id,
|
||||
)
|
||||
with self.get_session(database) as session:
|
||||
logger.info(
|
||||
"Opened Neo4j sink session for provider graph drop "
|
||||
"(database=%s, provider=%s)",
|
||||
database,
|
||||
provider_id,
|
||||
)
|
||||
log_target = f"Neo4j sink database {database}"
|
||||
for (
|
||||
phase,
|
||||
query_template,
|
||||
) in RELATIONSHIP_DELETE_QUERY_TEMPLATES.items():
|
||||
deleted_relationships, phase_batches = delete_batches(
|
||||
session=session,
|
||||
logger=logger,
|
||||
log_target=log_target,
|
||||
provider_id=provider_id,
|
||||
query=query_template.format(provider_label=provider_label),
|
||||
phase=phase,
|
||||
count_key="deleted_rels_count",
|
||||
total_key="rels",
|
||||
deleted_key="deleted_rels",
|
||||
initial_total=deleted_relationships,
|
||||
batch_size=BATCH_SIZE,
|
||||
drop_t0=drop_t0,
|
||||
)
|
||||
relationship_batches += phase_batches
|
||||
|
||||
deleted_nodes, node_batches = delete_batches(
|
||||
session=session,
|
||||
logger=logger,
|
||||
log_target=log_target,
|
||||
provider_id=provider_id,
|
||||
query=NODE_DELETE_QUERY_TEMPLATE.format(
|
||||
provider_label=provider_label,
|
||||
provider_resource_label=PROVIDER_RESOURCE_LABEL,
|
||||
),
|
||||
phase="node",
|
||||
count_key="deleted_nodes_count",
|
||||
total_key="nodes",
|
||||
deleted_key="deleted_nodes",
|
||||
initial_total=0,
|
||||
batch_size=BATCH_SIZE,
|
||||
drop_t0=drop_t0,
|
||||
)
|
||||
|
||||
except GraphDatabaseQueryException as exc:
|
||||
if exc.code == DATABASE_NOT_FOUND_CODE:
|
||||
logger.info(
|
||||
"Skipped provider graph drop from Neo4j sink database %s "
|
||||
"(provider=%s, reason=database_not_found, elapsed=%.3fs)",
|
||||
database,
|
||||
provider_id,
|
||||
time.perf_counter() - drop_t0,
|
||||
)
|
||||
return 0
|
||||
raise
|
||||
|
||||
logger.info(
|
||||
"Finished dropping provider graph from Neo4j sink database %s "
|
||||
"(provider=%s, relationship_batches=%s, deleted_rels=%s, "
|
||||
"node_batches=%s, deleted_nodes=%s, elapsed=%.3fs)",
|
||||
database,
|
||||
provider_id,
|
||||
relationship_batches,
|
||||
deleted_relationships,
|
||||
node_batches,
|
||||
deleted_nodes,
|
||||
time.perf_counter() - drop_t0,
|
||||
)
|
||||
return deleted_nodes
|
||||
|
||||
def has_provider_data(self, database: str, provider_id: str) -> bool:
|
||||
from api.attack_paths.database import GraphDatabaseQueryException
|
||||
from tasks.jobs.attack_paths.config import (
|
||||
PROVIDER_RESOURCE_LABEL,
|
||||
get_provider_label,
|
||||
)
|
||||
|
||||
provider_label = get_provider_label(provider_id)
|
||||
query = (
|
||||
f"MATCH (n:{PROVIDER_RESOURCE_LABEL}:`{provider_label}`) RETURN 1 LIMIT 1"
|
||||
)
|
||||
try:
|
||||
with self.get_session(
|
||||
database, default_access_mode=neo4j.READ_ACCESS
|
||||
) as session:
|
||||
result = session.run(query)
|
||||
return result.single() is not None
|
||||
|
||||
except GraphDatabaseQueryException as exc:
|
||||
if exc.code == DATABASE_NOT_FOUND_CODE:
|
||||
return False
|
||||
raise
|
||||
|
||||
def clear_cache(self, database: str) -> None:
|
||||
from api.attack_paths.database import GraphDatabaseQueryException
|
||||
|
||||
try:
|
||||
with self.get_session(database) as session:
|
||||
session.run("CALL db.clearQueryCaches()")
|
||||
except GraphDatabaseQueryException as exc:
|
||||
logger.warning(
|
||||
f"Failed to clear query cache for database `{database}`: {exc}"
|
||||
)
|
||||
|
||||
# Sync write path
|
||||
|
||||
def ensure_sync_indexes(self, database: str) -> None:
|
||||
"""Create the `_provider_element_id` lookup index on `_ProviderResource`.
|
||||
|
||||
Every synced node carries the `_ProviderResource` label, so a single
|
||||
index covers both node-upserts and relationship endpoint MATCHes.
|
||||
Without this index the rel sync degrades to a label scan per row and
|
||||
large provider syncs become unworkable.
|
||||
"""
|
||||
from tasks.jobs.attack_paths.config import (
|
||||
PROVIDER_ELEMENT_ID_PROPERTY,
|
||||
PROVIDER_RESOURCE_LABEL,
|
||||
)
|
||||
|
||||
query = (
|
||||
f"CREATE INDEX provider_element_id_idx IF NOT EXISTS "
|
||||
f"FOR (n:`{PROVIDER_RESOURCE_LABEL}`) "
|
||||
f"ON (n.`{PROVIDER_ELEMENT_ID_PROPERTY}`)"
|
||||
)
|
||||
with self.get_session(database) as session:
|
||||
session.run(query).consume()
|
||||
|
||||
def write_nodes(
|
||||
self,
|
||||
database: str,
|
||||
labels: str,
|
||||
rows: list[dict[str, Any]],
|
||||
) -> None:
|
||||
if not rows:
|
||||
return
|
||||
from tasks.jobs.attack_paths.config import (
|
||||
PROVIDER_ELEMENT_ID_PROPERTY,
|
||||
PROVIDER_RESOURCE_LABEL,
|
||||
)
|
||||
|
||||
query = f"""
|
||||
UNWIND $rows AS row
|
||||
MERGE (n:`{PROVIDER_RESOURCE_LABEL}` {{`{PROVIDER_ELEMENT_ID_PROPERTY}`: row.provider_element_id}})
|
||||
SET n:{labels}
|
||||
SET n += row.props
|
||||
"""
|
||||
with self.get_session(database) as session:
|
||||
session.run(query, {"rows": rows}).consume()
|
||||
|
||||
def write_relationships(
|
||||
self,
|
||||
database: str,
|
||||
rel_type: str,
|
||||
provider_id: str,
|
||||
rows: list[dict[str, Any]],
|
||||
) -> None:
|
||||
if not rows:
|
||||
return
|
||||
from tasks.jobs.attack_paths.config import (
|
||||
PROVIDER_ELEMENT_ID_PROPERTY,
|
||||
PROVIDER_RESOURCE_LABEL,
|
||||
get_provider_label,
|
||||
)
|
||||
|
||||
provider_label = get_provider_label(provider_id)
|
||||
query = f"""
|
||||
UNWIND $rows AS row
|
||||
MATCH (s:`{PROVIDER_RESOURCE_LABEL}`:`{provider_label}` {{`{PROVIDER_ELEMENT_ID_PROPERTY}`: row.start_element_id}})
|
||||
MATCH (t:`{PROVIDER_RESOURCE_LABEL}`:`{provider_label}` {{`{PROVIDER_ELEMENT_ID_PROPERTY}`: row.end_element_id}})
|
||||
MERGE (s)-[r:`{rel_type}` {{`{PROVIDER_ELEMENT_ID_PROPERTY}`: row.provider_element_id}}]->(t)
|
||||
SET r += row.props
|
||||
"""
|
||||
with self.get_session(database) as session:
|
||||
session.run(query, {"rows": rows}).consume()
|
||||
|
||||
# For compatibility with test harnesses that patch the concrete driver
|
||||
def get_driver(self) -> neo4j.Driver:
|
||||
return self._get_driver()
|
||||
|
||||
|
||||
# Helper for tests / external callers that want a writer session specifically
|
||||
def get_read_session(
|
||||
sink: Neo4jSink, database: str
|
||||
) -> AbstractContextManager[RetryableSession]:
|
||||
return sink.get_session(database, default_access_mode=neo4j.READ_ACCESS)
|
||||
@@ -0,0 +1,491 @@
|
||||
"""AWS Neptune sink implementation.
|
||||
|
||||
Dual Bolt drivers: one against the writer endpoint for workers, one against
|
||||
the reader endpoint for the API read path. If `NEPTUNE_READER_ENDPOINT` is
|
||||
unset the reader falls back to the writer driver so single-node clusters work.
|
||||
|
||||
Neptune is single-database. The `database` argument on the SinkDatabase
|
||||
protocol is ignored; tenant / provider isolation is enforced by labels that
|
||||
the sync step already writes on every node (see tasks/jobs/attack_paths/sync.py).
|
||||
|
||||
SigV4 auth lives at the bottom of this file as `neptune_auth_provider`. The
|
||||
neo4j driver invokes the returned callable on each token refresh.
|
||||
"""
|
||||
|
||||
import atexit
|
||||
import datetime
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Callable, Iterator
|
||||
from contextlib import contextmanager
|
||||
from typing import Any
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
import neo4j
|
||||
import neo4j.exceptions
|
||||
from api.attack_paths.retryable_session import RetryableSession
|
||||
from api.attack_paths.sink.base import SinkDatabase
|
||||
from api.attack_paths.sink.drop import (
|
||||
NODE_DELETE_QUERY_TEMPLATE,
|
||||
RELATIONSHIP_DELETE_QUERY_TEMPLATES,
|
||||
delete_batches,
|
||||
)
|
||||
from botocore.auth import SigV4Auth
|
||||
from botocore.awsrequest import AWSRequest
|
||||
from botocore.session import Session as BotoSession
|
||||
from config.env import env
|
||||
from django.conf import settings
|
||||
from neo4j.auth_management import AuthManagers, ExpiringAuth
|
||||
|
||||
logging.getLogger("neo4j").setLevel(logging.ERROR)
|
||||
logging.getLogger("neo4j").propagate = False
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SERVICE_UNAVAILABLE_MAX_RETRIES = env.int(
|
||||
"ATTACK_PATHS_SERVICE_UNAVAILABLE_MAX_RETRIES", default=3
|
||||
)
|
||||
READ_QUERY_TIMEOUT_SECONDS = env.int(
|
||||
"ATTACK_PATHS_READ_QUERY_TIMEOUT_SECONDS", default=30
|
||||
)
|
||||
# Neptune serverless cold-start can be >30s; give the driver room
|
||||
CONN_ACQUISITION_TIMEOUT = env.int("NEPTUNE_CONN_ACQUISITION_TIMEOUT", default=60)
|
||||
# TCP connect timeout, ordered below the acquisition timeout so an unreachable
|
||||
# endpoint can't pin a request or the readiness probe longer than this. Kept
|
||||
# generous: cold-start delays query execution, not the socket connect.
|
||||
CONNECTION_TIMEOUT = env.int("NEPTUNE_CONNECTION_TIMEOUT", default=10)
|
||||
# Roll connections hourly so SigV4 rotations and cert refreshes don't strand long-lived pool entries
|
||||
MAX_CONNECTION_LIFETIME = env.int("NEPTUNE_MAX_CONNECTION_LIFETIME", default=3600)
|
||||
MAX_CONNECTION_POOL_SIZE = env.int("NEPTUNE_MAX_CONNECTION_POOL_SIZE", default=50)
|
||||
|
||||
READ_EXCEPTION_CODES = [
|
||||
"Neo.ClientError.Statement.AccessMode",
|
||||
"Neo.ClientError.Procedure.ProcedureNotFound",
|
||||
]
|
||||
CLIENT_STATEMENT_EXCEPTION_PREFIX = "Neo.ClientError.Statement."
|
||||
|
||||
# Refresh 60s before the 5-minute SigV4 window closes
|
||||
SIGV4_TOKEN_LIFETIME_MINUTES = 4
|
||||
|
||||
|
||||
class NeptuneSink(SinkDatabase):
|
||||
"""Neptune-backed sink. Single database; isolation is label-based."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._writer: neo4j.Driver | None = None
|
||||
self._reader: neo4j.Driver | None = None
|
||||
self._lock = threading.Lock()
|
||||
self._atexit_registered = False
|
||||
|
||||
# Config
|
||||
|
||||
def _config(self) -> dict:
|
||||
return settings.DATABASES["neptune"]
|
||||
|
||||
def _bolt_uri(self, endpoint: str, port: str) -> str:
|
||||
return f"bolt+s://{endpoint}:{port}"
|
||||
|
||||
def _https_url(self, endpoint: str, port: str) -> str:
|
||||
return f"https://{endpoint}:{port}"
|
||||
|
||||
def _build_driver(self, endpoint: str) -> neo4j.Driver:
|
||||
cfg = self._config()
|
||||
port = cfg["PORT"]
|
||||
region = cfg["REGION"]
|
||||
if not endpoint or not region:
|
||||
raise RuntimeError(
|
||||
"NEPTUNE_WRITER_ENDPOINT and AWS_REGION must be set when "
|
||||
"ATTACK_PATHS_SINK_DATABASE=neptune"
|
||||
)
|
||||
return neo4j.GraphDatabase.driver(
|
||||
self._bolt_uri(endpoint, port),
|
||||
auth=AuthManagers.bearer(
|
||||
neptune_auth_provider(region, self._https_url(endpoint, port))
|
||||
),
|
||||
keep_alive=True,
|
||||
max_connection_lifetime=MAX_CONNECTION_LIFETIME,
|
||||
connection_timeout=CONNECTION_TIMEOUT,
|
||||
connection_acquisition_timeout=CONN_ACQUISITION_TIMEOUT,
|
||||
max_connection_pool_size=MAX_CONNECTION_POOL_SIZE,
|
||||
max_transaction_retry_time=0,
|
||||
)
|
||||
|
||||
# Lifecycle
|
||||
|
||||
def init(self) -> None:
|
||||
if self._writer is not None:
|
||||
return
|
||||
with self._lock:
|
||||
if self._writer is None:
|
||||
cfg = self._config()
|
||||
writer_endpoint = cfg["WRITER_ENDPOINT"]
|
||||
reader_endpoint = cfg["READER_ENDPOINT"] or writer_endpoint
|
||||
|
||||
# Eager connectivity checks are best-effort
|
||||
# A Neptune that is down at boot must not crash the process, same degradation model as Postgres
|
||||
# Drivers reconnect lazily on first use
|
||||
# /health/ready surfaces the outage until it recovers
|
||||
self._writer = self._build_driver(writer_endpoint)
|
||||
self._verify_best_effort(self._writer, "writer")
|
||||
|
||||
if reader_endpoint == writer_endpoint:
|
||||
self._reader = self._writer
|
||||
|
||||
else:
|
||||
self._reader = self._build_driver(reader_endpoint)
|
||||
self._verify_best_effort(self._reader, "reader")
|
||||
|
||||
if not self._atexit_registered:
|
||||
atexit.register(self.close)
|
||||
self._atexit_registered = True
|
||||
|
||||
def close(self) -> None:
|
||||
with self._lock:
|
||||
# `Driver.close()` is idempotent, so closing the same driver twice
|
||||
# (when reader aliases writer on single-endpoint configs) is safe
|
||||
for driver in (self._reader, self._writer):
|
||||
if driver is None:
|
||||
continue
|
||||
try:
|
||||
driver.close()
|
||||
except Exception: # pragma: no cover - best-effort
|
||||
pass
|
||||
self._writer = None
|
||||
self._reader = None
|
||||
|
||||
# Sessions
|
||||
|
||||
def _get_writer(self) -> neo4j.Driver:
|
||||
self.init()
|
||||
assert self._writer is not None
|
||||
return self._writer
|
||||
|
||||
def _get_reader(self) -> neo4j.Driver:
|
||||
self.init()
|
||||
assert self._reader is not None
|
||||
return self._reader
|
||||
|
||||
@staticmethod
|
||||
def _verify_best_effort(driver: neo4j.Driver, role: str) -> None:
|
||||
try:
|
||||
driver.verify_connectivity()
|
||||
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Neptune %s endpoint unreachable at init; continuing with a lazily-reconnecting driver",
|
||||
role,
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
def verify_connectivity(self) -> None:
|
||||
# The API read path uses the reader driver
|
||||
# On single-endpoint clusters it aliases the writer, so this also covers the writer
|
||||
# A writer-only outage is a workers' concern (no HTTP probe there) and deliberately does not fail API readiness
|
||||
self._get_reader().verify_connectivity()
|
||||
|
||||
@contextmanager
|
||||
def get_session(
|
||||
self,
|
||||
database: str | None = None, # noqa: ARG002 - ignored on Neptune
|
||||
default_access_mode: str | None = None,
|
||||
) -> Iterator[RetryableSession]:
|
||||
from api.attack_paths.database import (
|
||||
ClientStatementException,
|
||||
GraphDatabaseQueryException,
|
||||
WriteQueryNotAllowedException,
|
||||
)
|
||||
|
||||
driver = (
|
||||
self._get_reader()
|
||||
if default_access_mode == neo4j.READ_ACCESS
|
||||
else self._get_writer()
|
||||
)
|
||||
|
||||
session_wrapper: RetryableSession | None = None
|
||||
try:
|
||||
session_wrapper = RetryableSession(
|
||||
session_factory=lambda: driver.session(
|
||||
default_access_mode=default_access_mode
|
||||
),
|
||||
max_retries=SERVICE_UNAVAILABLE_MAX_RETRIES,
|
||||
)
|
||||
yield session_wrapper
|
||||
|
||||
except neo4j.exceptions.Neo4jError as exc:
|
||||
if (
|
||||
default_access_mode == neo4j.READ_ACCESS
|
||||
and exc.code
|
||||
and exc.code in READ_EXCEPTION_CODES
|
||||
):
|
||||
raise WriteQueryNotAllowedException(
|
||||
message="Read query not allowed", code=READ_EXCEPTION_CODES[0]
|
||||
)
|
||||
|
||||
message = exc.message if exc.message is not None else str(exc)
|
||||
if exc.code and exc.code.startswith(CLIENT_STATEMENT_EXCEPTION_PREFIX):
|
||||
raise ClientStatementException(message=message, code=exc.code)
|
||||
raise GraphDatabaseQueryException(message=message, code=exc.code)
|
||||
|
||||
finally:
|
||||
if session_wrapper is not None:
|
||||
session_wrapper.close()
|
||||
|
||||
# Operations
|
||||
|
||||
def execute_read_query(
|
||||
self,
|
||||
database: str, # noqa: ARG002 - ignored on Neptune
|
||||
cypher: str,
|
||||
parameters: dict[str, Any] | None = None,
|
||||
) -> neo4j.graph.Graph:
|
||||
with self.get_session(default_access_mode=neo4j.READ_ACCESS) as session:
|
||||
|
||||
def _run(tx: neo4j.ManagedTransaction) -> neo4j.graph.Graph:
|
||||
result = tx.run(
|
||||
cypher, parameters or {}, timeout=READ_QUERY_TIMEOUT_SECONDS
|
||||
)
|
||||
return result.graph()
|
||||
|
||||
return session.execute_read(_run)
|
||||
|
||||
def create_database(self, database: str) -> None: # noqa: ARG002
|
||||
# Neptune clusters are single-database; there is nothing to create.
|
||||
return None
|
||||
|
||||
def drop_database(self, database: str) -> None: # noqa: ARG002
|
||||
# Neptune clusters are single-database; there is nothing to drop.
|
||||
return None
|
||||
|
||||
def drop_subgraph(self, database: str, provider_id: str) -> int: # noqa: ARG002
|
||||
"""Delete a provider's subgraph in two bounded phases.
|
||||
|
||||
Neptune write transactions are capped at ~2 minutes. A naive
|
||||
`DETACH DELETE` on a label-scanned batch grows unbounded with graph
|
||||
density (one node can drag thousands of relationships into the same
|
||||
transaction). Instead:
|
||||
|
||||
1. Delete relationships incident to provider nodes, one fixed-size
|
||||
batch per transaction.
|
||||
2. Delete the now-orphaned nodes, one fixed-size batch per transaction.
|
||||
|
||||
Each transaction does work proportional to `batch_size`, never to the
|
||||
graph's branching factor.
|
||||
"""
|
||||
from tasks.jobs.attack_paths.config import (
|
||||
BATCH_SIZE,
|
||||
PROVIDER_RESOURCE_LABEL,
|
||||
get_provider_label,
|
||||
)
|
||||
|
||||
provider_label = get_provider_label(provider_id)
|
||||
deleted_relationships = 0
|
||||
relationship_batches = 0
|
||||
node_batches = 0
|
||||
drop_t0 = time.perf_counter()
|
||||
|
||||
logger.info(
|
||||
"Dropping provider graph from Neptune sink "
|
||||
"(provider=%s, provider_label=%s)",
|
||||
provider_id,
|
||||
provider_label,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"Opening Neptune writer session for provider graph drop (provider=%s)",
|
||||
provider_id,
|
||||
)
|
||||
with self.get_session() as session:
|
||||
logger.info(
|
||||
"Opened Neptune writer session for provider graph drop (provider=%s)",
|
||||
provider_id,
|
||||
)
|
||||
for phase, query_template in RELATIONSHIP_DELETE_QUERY_TEMPLATES.items():
|
||||
deleted_relationships, phase_batches = delete_batches(
|
||||
session=session,
|
||||
logger=logger,
|
||||
log_target="Neptune sink",
|
||||
provider_id=provider_id,
|
||||
query=query_template.format(provider_label=provider_label),
|
||||
phase=phase,
|
||||
count_key="deleted_rels_count",
|
||||
total_key="rels",
|
||||
deleted_key="deleted_rels",
|
||||
initial_total=deleted_relationships,
|
||||
batch_size=BATCH_SIZE,
|
||||
drop_t0=drop_t0,
|
||||
)
|
||||
relationship_batches += phase_batches
|
||||
|
||||
deleted_nodes, node_batches = delete_batches(
|
||||
session=session,
|
||||
logger=logger,
|
||||
log_target="Neptune sink",
|
||||
provider_id=provider_id,
|
||||
query=NODE_DELETE_QUERY_TEMPLATE.format(
|
||||
provider_label=provider_label,
|
||||
provider_resource_label=PROVIDER_RESOURCE_LABEL,
|
||||
),
|
||||
phase="node",
|
||||
count_key="deleted_nodes_count",
|
||||
total_key="nodes",
|
||||
deleted_key="deleted_nodes",
|
||||
initial_total=0,
|
||||
batch_size=BATCH_SIZE,
|
||||
drop_t0=drop_t0,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"Finished dropping provider graph from Neptune sink "
|
||||
"(provider=%s, relationship_batches=%s, deleted_rels=%s, "
|
||||
"node_batches=%s, deleted_nodes=%s, elapsed=%.3fs)",
|
||||
provider_id,
|
||||
relationship_batches,
|
||||
deleted_relationships,
|
||||
node_batches,
|
||||
deleted_nodes,
|
||||
time.perf_counter() - drop_t0,
|
||||
)
|
||||
return deleted_nodes
|
||||
|
||||
def has_provider_data(self, database: str, provider_id: str) -> bool: # noqa: ARG002
|
||||
from tasks.jobs.attack_paths.config import (
|
||||
PROVIDER_RESOURCE_LABEL,
|
||||
get_provider_label,
|
||||
)
|
||||
|
||||
provider_label = get_provider_label(provider_id)
|
||||
query = (
|
||||
f"MATCH (n:{PROVIDER_RESOURCE_LABEL}:`{provider_label}`) RETURN 1 LIMIT 1"
|
||||
)
|
||||
with self.get_session(default_access_mode=neo4j.READ_ACCESS) as session:
|
||||
result = session.run(query)
|
||||
return result.single() is not None
|
||||
|
||||
def clear_cache(self, database: str) -> None: # noqa: ARG002
|
||||
# Neptune has no user-facing cache-clear procedure; no-op.
|
||||
return None
|
||||
|
||||
# Sync write path
|
||||
|
||||
def ensure_sync_indexes(self, database: str) -> None: # noqa: ARG002
|
||||
# Neptune routes node and relationship lookups through `~id`, which is the cluster's primary key
|
||||
# No additional index is needed or supported
|
||||
return None
|
||||
|
||||
def write_nodes(
|
||||
self,
|
||||
database: str, # noqa: ARG002
|
||||
labels: str,
|
||||
rows: list[dict[str, Any]],
|
||||
) -> None:
|
||||
if not rows:
|
||||
return
|
||||
from tasks.jobs.attack_paths.config import (
|
||||
PROVIDER_ELEMENT_ID_PROPERTY,
|
||||
PROVIDER_RESOURCE_LABEL,
|
||||
)
|
||||
|
||||
# MERGE on `~id` is the documented and engine-optimized idempotent
|
||||
# upsert pattern for Neptune openCypher. The label inside the MERGE
|
||||
# matters: Neptune assigns a default `vertex` label to any node
|
||||
# created without an explicit one, so we pin `_ProviderResource`
|
||||
# (which every synced node carries anyway) at MERGE-time. Additional
|
||||
# labels are added after
|
||||
#
|
||||
# We also write `_provider_element_id` as a regular property so
|
||||
# non-sync code (drop_subgraph, query helpers) keeps a stable contract
|
||||
# that doesn't know about `~id`
|
||||
query = f"""
|
||||
UNWIND $rows AS row
|
||||
MERGE (n:`{PROVIDER_RESOURCE_LABEL}` {{`~id`: row.provider_element_id}})
|
||||
SET n:{labels}
|
||||
SET n += row.props
|
||||
SET n.`{PROVIDER_ELEMENT_ID_PROPERTY}` = row.provider_element_id
|
||||
"""
|
||||
with self.get_session() as session:
|
||||
session.run(query, {"rows": rows}).consume()
|
||||
|
||||
def write_relationships(
|
||||
self,
|
||||
database: str, # noqa: ARG002
|
||||
rel_type: str,
|
||||
provider_id: str, # noqa: ARG002 - encoded in start/end `~id` already
|
||||
rows: list[dict[str, Any]],
|
||||
) -> None:
|
||||
if not rows:
|
||||
return
|
||||
from tasks.jobs.attack_paths.config import PROVIDER_ELEMENT_ID_PROPERTY
|
||||
|
||||
# `id(n) = $value` is Neptune's parameterized fast path; both endpoint
|
||||
# MATCHes resolve in O(1) via the system `~id`, so per-row work stays
|
||||
# bounded regardless of batch size
|
||||
query = f"""
|
||||
UNWIND $rows AS row
|
||||
MATCH (s) WHERE id(s) = row.start_element_id
|
||||
MATCH (e) WHERE id(e) = row.end_element_id
|
||||
MERGE (s)-[r:`{rel_type}` {{`{PROVIDER_ELEMENT_ID_PROPERTY}`: row.provider_element_id}}]->(e)
|
||||
SET r += row.props
|
||||
"""
|
||||
with self.get_session() as session:
|
||||
session.run(query, {"rows": rows}).consume()
|
||||
|
||||
# Test helpers
|
||||
|
||||
def get_writer(self) -> neo4j.Driver:
|
||||
return self._get_writer()
|
||||
|
||||
def get_reader(self) -> neo4j.Driver:
|
||||
return self._get_reader()
|
||||
|
||||
|
||||
# SigV4 auth provider
|
||||
|
||||
|
||||
class _NeptuneAuthToken(neo4j.Auth):
|
||||
"""Neo4j Auth backed by a SigV4-signed GET to `/opencypher`."""
|
||||
|
||||
def __init__(self, region: str, url: str) -> None:
|
||||
session = BotoSession()
|
||||
credentials = session.get_credentials()
|
||||
if credentials is None:
|
||||
raise RuntimeError(
|
||||
"No AWS credentials available for Neptune SigV4 signing. "
|
||||
"Ensure the boto3 credential chain can resolve."
|
||||
)
|
||||
credentials = credentials.get_frozen_credentials()
|
||||
|
||||
request = AWSRequest(method="GET", url=url + "/opencypher")
|
||||
# SigV4 canonical Host must carry the real `host:port`
|
||||
# Neptune runs on a non-default port (8182), so `.hostname` would drop it and break signing
|
||||
request.headers.add_header("Host", urlsplit(url).netloc)
|
||||
SigV4Auth(credentials, "neptune-db", region).add_auth(request)
|
||||
|
||||
auth_obj = {
|
||||
header: request.headers[header]
|
||||
for header in (
|
||||
"Authorization",
|
||||
"X-Amz-Date",
|
||||
"X-Amz-Security-Token",
|
||||
"Host",
|
||||
)
|
||||
if header in request.headers
|
||||
}
|
||||
auth_obj["HttpMethod"] = "GET"
|
||||
|
||||
super().__init__("basic", "username", json.dumps(auth_obj))
|
||||
|
||||
|
||||
def neptune_auth_provider(region: str, https_url: str) -> Callable[[], ExpiringAuth]:
|
||||
"""Return a callable the neo4j driver can invoke to refresh credentials."""
|
||||
|
||||
def _provider() -> ExpiringAuth:
|
||||
token = _NeptuneAuthToken(region, https_url)
|
||||
expires_at = (
|
||||
datetime.datetime.now(datetime.UTC)
|
||||
+ datetime.timedelta(minutes=SIGV4_TOKEN_LIFETIME_MINUTES)
|
||||
).timestamp()
|
||||
return ExpiringAuth(auth=token, expires_at=expires_at)
|
||||
|
||||
return _provider
|
||||
@@ -1,12 +1,11 @@
|
||||
import logging
|
||||
|
||||
from typing import Any, Iterable
|
||||
from collections.abc import Iterable
|
||||
from typing import Any
|
||||
|
||||
import neo4j
|
||||
|
||||
from rest_framework.exceptions import APIException, PermissionDenied, ValidationError
|
||||
|
||||
from api.attack_paths import database as graph_database, AttackPathsQueryDefinition
|
||||
from api.attack_paths import AttackPathsQueryDefinition
|
||||
from api.attack_paths import database as graph_database
|
||||
from api.attack_paths import sink as sink_module
|
||||
from api.attack_paths.cypher_sanitizer import (
|
||||
inject_provider_label,
|
||||
validate_custom_query,
|
||||
@@ -16,7 +15,10 @@ from api.attack_paths.queries.schema import (
|
||||
RAW_SCHEMA_URL,
|
||||
get_cartography_schema_query,
|
||||
)
|
||||
from api.models import AttackPathsScan
|
||||
from config.custom_logging import BackendLogger
|
||||
from config.env import env
|
||||
from rest_framework.exceptions import APIException, PermissionDenied, ValidationError
|
||||
from tasks.jobs.attack_paths.config import (
|
||||
INTERNAL_LABELS,
|
||||
INTERNAL_PROPERTIES,
|
||||
@@ -27,6 +29,10 @@ from tasks.jobs.attack_paths.config import (
|
||||
logger = logging.getLogger(BackendLogger.API)
|
||||
|
||||
|
||||
def _custom_query_timeout_ms() -> int:
|
||||
return env.int("ATTACK_PATHS_READ_QUERY_TIMEOUT_SECONDS", default=30) * 1000
|
||||
|
||||
|
||||
# Predefined query helpers
|
||||
|
||||
|
||||
@@ -103,13 +109,13 @@ def execute_query(
|
||||
definition: AttackPathsQueryDefinition,
|
||||
parameters: dict[str, Any],
|
||||
provider_id: str,
|
||||
scan: AttackPathsScan,
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
graph = graph_database.execute_read_query(
|
||||
database=database_name,
|
||||
cypher=definition.cypher,
|
||||
parameters=parameters,
|
||||
)
|
||||
# TODO: drop after Neptune cutover
|
||||
# Route reads by the scan row's recorded sink, not by current settings.
|
||||
backend = sink_module.get_backend_for_scan(scan)
|
||||
graph = backend.execute_read_query(database_name, definition.cypher, parameters)
|
||||
return _serialize_graph(graph, provider_id)
|
||||
|
||||
except graph_database.WriteQueryNotAllowedException:
|
||||
@@ -143,22 +149,31 @@ def execute_custom_query(
|
||||
database_name: str,
|
||||
cypher: str,
|
||||
provider_id: str,
|
||||
scan: AttackPathsScan,
|
||||
) -> dict[str, Any]:
|
||||
# Defense-in-depth for custom queries:
|
||||
# 1. neo4j.READ_ACCESS — prevents mutations at the driver level
|
||||
# 2. inject_provider_label() — regex-based label injection scopes node patterns
|
||||
# 3. _serialize_graph() — post-query filter drops nodes without the provider label
|
||||
# 1. `neo4j.READ_ACCESS` — prevents mutations at the driver level
|
||||
# 2. `inject_provider_label()` — regex-based label injection scopes node patterns
|
||||
# 3. `_serialize_graph()` — post-query filter drops nodes without the provider label
|
||||
# 4. `USING QUERY:TIMEOUTMILLISECONDS` on Neptune — server-side runaway cutoff
|
||||
#
|
||||
# Layer 2 is best-effort (regex can't fully parse Cypher);
|
||||
# layer 3 is the safety net that guarantees provider isolation.
|
||||
validate_custom_query(cypher)
|
||||
cypher = inject_provider_label(cypher, provider_id)
|
||||
|
||||
# TODO: drop after Neptune cutover
|
||||
backend = sink_module.get_backend_for_scan(scan)
|
||||
|
||||
# Neptune enforces a cluster-level query timeout; prepending the hint
|
||||
# makes the limit explicit and matches the client-side read timeout.
|
||||
# Applies only when the scan's graph lives in Neptune.
|
||||
if getattr(scan, "sink_backend", None) == "neptune":
|
||||
timeout_ms = _custom_query_timeout_ms()
|
||||
cypher = f"USING QUERY:TIMEOUTMILLISECONDS {timeout_ms}\n{cypher}"
|
||||
|
||||
try:
|
||||
graph = graph_database.execute_read_query(
|
||||
database=database_name,
|
||||
cypher=cypher,
|
||||
)
|
||||
graph = backend.execute_read_query(database_name, cypher, None)
|
||||
serialized = _serialize_graph(graph, provider_id)
|
||||
return _truncate_graph(serialized)
|
||||
|
||||
@@ -181,10 +196,11 @@ def execute_custom_query(
|
||||
|
||||
|
||||
def get_cartography_schema(
|
||||
database_name: str, provider_id: str
|
||||
database_name: str, provider_id: str, scan: AttackPathsScan
|
||||
) -> dict[str, str] | None:
|
||||
try:
|
||||
with graph_database.get_session(
|
||||
backend = sink_module.get_backend_for_scan(scan)
|
||||
with backend.get_session(
|
||||
database_name, default_access_mode=neo4j.READ_ACCESS
|
||||
) as session:
|
||||
result = session.run(get_cartography_schema_query(provider_id))
|
||||
|
||||
@@ -1,18 +1,19 @@
|
||||
from typing import Optional, Tuple
|
||||
from math import isfinite
|
||||
from uuid import UUID
|
||||
|
||||
from api.db_router import MainRouter
|
||||
from api.models import TenantAPIKey, TenantAPIKeyManager
|
||||
from cryptography.fernet import InvalidToken
|
||||
from django.core.exceptions import ObjectDoesNotExist
|
||||
from django.utils import timezone
|
||||
from drf_simple_apikey.backends import APIKeyAuthentication as BaseAPIKeyAuth
|
||||
from drf_simple_apikey.crypto import get_crypto
|
||||
from drf_simple_apikey.settings import package_settings
|
||||
from rest_framework.authentication import BaseAuthentication
|
||||
from rest_framework.exceptions import AuthenticationFailed
|
||||
from rest_framework.request import Request
|
||||
from rest_framework_simplejwt.authentication import JWTAuthentication
|
||||
|
||||
from api.db_router import MainRouter
|
||||
from api.models import TenantAPIKey, TenantAPIKeyManager
|
||||
|
||||
|
||||
class TenantAPIKeyAuthentication(BaseAPIKeyAuth):
|
||||
model = TenantAPIKey
|
||||
@@ -23,18 +24,49 @@ class TenantAPIKeyAuthentication(BaseAPIKeyAuth):
|
||||
def _authenticate_credentials(self, request, key):
|
||||
"""
|
||||
Override to use admin connection, bypassing RLS during authentication.
|
||||
Delegates to parent after temporarily routing model queries to admin DB.
|
||||
"""
|
||||
# Temporarily point the model's manager to admin database
|
||||
original_objects = self.model.objects
|
||||
self.model.objects = self.model.objects.using(MainRouter.admin_db)
|
||||
try:
|
||||
payload = self.key_crypto.decrypt(key)
|
||||
except ValueError:
|
||||
raise AuthenticationFailed("Invalid API Key.")
|
||||
|
||||
if not isinstance(payload, dict):
|
||||
raise AuthenticationFailed("Invalid API Key.")
|
||||
|
||||
payload_pk = payload.get("_pk")
|
||||
payload_exp = payload.get("_exp")
|
||||
if (
|
||||
not isinstance(payload_pk, str)
|
||||
or isinstance(payload_exp, bool)
|
||||
or not isinstance(payload_exp, (int, float))
|
||||
or not isfinite(payload_exp)
|
||||
):
|
||||
raise AuthenticationFailed("Invalid API Key.")
|
||||
|
||||
try:
|
||||
# Call parent method which will now use admin database
|
||||
return super()._authenticate_credentials(request, key)
|
||||
finally:
|
||||
# Restore original manager
|
||||
self.model.objects = original_objects
|
||||
api_key_pk = UUID(payload_pk)
|
||||
except ValueError:
|
||||
raise AuthenticationFailed("Invalid API Key.")
|
||||
|
||||
if payload_exp < timezone.now().timestamp():
|
||||
raise AuthenticationFailed("API Key has already expired.")
|
||||
|
||||
try:
|
||||
api_key = self.model.objects.using(MainRouter.admin_db).get(id=api_key_pk)
|
||||
except ObjectDoesNotExist:
|
||||
raise AuthenticationFailed("No entity matching this api key.")
|
||||
|
||||
if api_key.revoked:
|
||||
raise AuthenticationFailed("This API Key has been revoked.")
|
||||
|
||||
client_ip = request.META.get(package_settings.IP_ADDRESS_HEADER)
|
||||
if api_key.blacklisted_ips and client_ip in api_key.blacklisted_ips:
|
||||
raise AuthenticationFailed("Access denied from blacklisted IP.")
|
||||
|
||||
if api_key.whitelisted_ips and client_ip not in api_key.whitelisted_ips:
|
||||
raise AuthenticationFailed("Access restricted to specific IP addresses.")
|
||||
|
||||
return api_key.entity, key
|
||||
|
||||
def authenticate(self, request: Request):
|
||||
prefixed_key = self.get_key(request)
|
||||
@@ -81,7 +113,7 @@ class CombinedJWTOrAPIKeyAuthentication(BaseAuthentication):
|
||||
jwt_auth = JWTAuthentication()
|
||||
api_key_auth = TenantAPIKeyAuthentication()
|
||||
|
||||
def authenticate(self, request: Request) -> Optional[Tuple[object, dict]]:
|
||||
def authenticate(self, request: Request) -> tuple[object, dict] | None:
|
||||
auth_header = request.headers.get("Authorization", "")
|
||||
|
||||
# Prioritize JWT authentication if both are present
|
||||
@@ -93,3 +125,30 @@ class CombinedJWTOrAPIKeyAuthentication(BaseAuthentication):
|
||||
|
||||
# Default fallback
|
||||
return self.jwt_auth.authenticate(request)
|
||||
|
||||
|
||||
class SSEAuthentication(CombinedJWTOrAPIKeyAuthentication):
|
||||
"""JWT/API-Key auth that also accepts `?access_token=<jwt>`.
|
||||
|
||||
Browser `EventSource` is the only widely available SSE client API
|
||||
and it cannot set the `Authorization` header (its constructor takes
|
||||
only a URL and `withCredentials`). To keep browser SSE clients on
|
||||
the same auth stack as the rest of the API, SSE endpoints additionally
|
||||
accept a JWT via the `?access_token=<jwt>` query parameter — the
|
||||
standard parameter name defined in RFC 6750 Section 2.3 for bearer tokens.
|
||||
"""
|
||||
|
||||
def authenticate(self, request: Request):
|
||||
auth_header = request.headers.get("Authorization", "")
|
||||
if auth_header:
|
||||
return super().authenticate(request)
|
||||
|
||||
raw_token = request.query_params.get("access_token")
|
||||
if not raw_token:
|
||||
# No header and no query token — let the default path raise
|
||||
# the canonical AuthenticationFailed via the parent class.
|
||||
return super().authenticate(request)
|
||||
|
||||
validated_token = self.jwt_auth.get_validated_token(raw_token)
|
||||
user = self.jwt_auth.get_user(validated_token)
|
||||
return user, validated_token
|
||||
|
||||
@@ -1,3 +1,9 @@
|
||||
from api.authentication import CombinedJWTOrAPIKeyAuthentication
|
||||
from api.db_router import MainRouter, reset_read_db_alias, set_read_db_alias
|
||||
from api.db_utils import POSTGRES_USER_VAR, rls_transaction
|
||||
from api.filters import CustomDjangoFilterBackend
|
||||
from api.models import Role, UserRoleRelationship
|
||||
from api.rbac.permissions import HasPermissions
|
||||
from django.conf import settings
|
||||
from django.db import transaction
|
||||
from rest_framework import permissions
|
||||
@@ -8,13 +14,6 @@ from rest_framework.response import Response
|
||||
from rest_framework_json_api import filters
|
||||
from rest_framework_json_api.views import ModelViewSet
|
||||
|
||||
from api.authentication import CombinedJWTOrAPIKeyAuthentication
|
||||
from api.db_router import MainRouter, reset_read_db_alias, set_read_db_alias
|
||||
from api.db_utils import POSTGRES_USER_VAR, rls_transaction
|
||||
from api.filters import CustomDjangoFilterBackend
|
||||
from api.models import Role, UserRoleRelationship
|
||||
from api.rbac.permissions import HasPermissions
|
||||
|
||||
|
||||
class BaseViewSet(ModelViewSet):
|
||||
authentication_classes = [CombinedJWTOrAPIKeyAuthentication]
|
||||
|
||||
@@ -1,11 +1,26 @@
|
||||
import logging
|
||||
import threading
|
||||
from collections.abc import Iterable, Mapping
|
||||
|
||||
from api.models import Provider
|
||||
from prowler.lib.check.compliance_models import Compliance
|
||||
from prowler.lib.check.compliance_models import (
|
||||
get_bulk_compliance_frameworks_universal,
|
||||
)
|
||||
from prowler.lib.check.models import CheckMetadata
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
AVAILABLE_COMPLIANCE_FRAMEWORKS = {}
|
||||
|
||||
# Per-process readiness flags for the background compliance warm-up.
|
||||
# `STARTED` is set as soon as warming begins (only happens under Gunicorn via
|
||||
# the post_fork hook); `WARMED` is set when it finishes. The attributes
|
||||
# endpoint checks both: it returns 503 only while warming is in progress.
|
||||
# Under `runserver` warming never runs, so `STARTED` stays clear and the
|
||||
# endpoint keeps lazy-loading as before.
|
||||
COMPLIANCE_WARMING_STARTED = threading.Event()
|
||||
COMPLIANCE_WARMED = threading.Event()
|
||||
|
||||
|
||||
class LazyComplianceTemplate(Mapping):
|
||||
"""Lazy-load compliance templates per provider on first access."""
|
||||
@@ -94,25 +109,22 @@ PROWLER_CHECKS = LazyChecksMapping()
|
||||
|
||||
|
||||
def get_compliance_frameworks(provider_type: Provider.ProviderChoices) -> list[str]:
|
||||
"""List compliance frameworks the API can load for `provider_type`.
|
||||
"""List compliance framework identifiers available for `provider_type`.
|
||||
|
||||
The list is sourced from `Compliance.get_bulk` so that the names
|
||||
returned here are guaranteed to be loadable by the bulk loader. This
|
||||
prevents downstream key mismatches (e.g. CSV report generation iterating
|
||||
framework names and looking them up in the bulk dict).
|
||||
Includes both per-provider frameworks and universal top-level frameworks
|
||||
(e.g. ``dora_2022_2554``, ``csa_ccm_4.0``).
|
||||
|
||||
Args:
|
||||
provider_type (Provider.ProviderChoices): The cloud provider type for which to retrieve
|
||||
available compliance frameworks (e.g., "aws", "azure", "gcp", "m365").
|
||||
provider_type (Provider.ProviderChoices): The cloud provider type
|
||||
(e.g., "aws", "azure", "gcp", "m365").
|
||||
|
||||
Returns:
|
||||
list[str]: A list of framework identifiers (e.g., "cis_1.4_aws", "mitre_attack_azure") available
|
||||
for the given provider.
|
||||
list[str]: Framework identifiers (e.g., "cis_1.4_aws", "dora_2022_2554").
|
||||
"""
|
||||
global AVAILABLE_COMPLIANCE_FRAMEWORKS
|
||||
if provider_type not in AVAILABLE_COMPLIANCE_FRAMEWORKS:
|
||||
AVAILABLE_COMPLIANCE_FRAMEWORKS[provider_type] = list(
|
||||
Compliance.get_bulk(provider_type).keys()
|
||||
get_bulk_compliance_frameworks_universal(provider_type).keys()
|
||||
)
|
||||
|
||||
return AVAILABLE_COMPLIANCE_FRAMEWORKS[provider_type]
|
||||
@@ -139,18 +151,14 @@ def get_prowler_provider_compliance(provider_type: Provider.ProviderChoices) ->
|
||||
"""
|
||||
Retrieve the Prowler compliance data for a specified provider type.
|
||||
|
||||
This function fetches the compliance frameworks and their associated
|
||||
requirements for the given cloud provider.
|
||||
|
||||
Args:
|
||||
provider_type (Provider.ProviderChoices): The provider type
|
||||
(e.g., 'aws', 'azure') for which to retrieve compliance data.
|
||||
|
||||
Returns:
|
||||
dict: A dictionary mapping compliance framework names to their respective
|
||||
Compliance objects for the specified provider.
|
||||
dict: Mapping of framework name to `ComplianceFramework` for the provider.
|
||||
"""
|
||||
return Compliance.get_bulk(provider_type)
|
||||
return get_bulk_compliance_frameworks_universal(provider_type)
|
||||
|
||||
|
||||
def _load_provider_assets(provider_type: Provider.ProviderChoices) -> tuple[dict, dict]:
|
||||
@@ -179,6 +187,56 @@ def _ensure_provider_loaded(provider_type: Provider.ProviderChoices) -> None:
|
||||
PROWLER_CHECKS._cache[provider_type] = checks
|
||||
|
||||
|
||||
def warm_compliance_caches(
|
||||
provider_types: Iterable[str] | None = None,
|
||||
) -> list[str]:
|
||||
"""
|
||||
Eagerly populate the per-process compliance caches at server startup.
|
||||
|
||||
Moves the cold-cache catalog load off the request thread so the first
|
||||
request does not trip the Gunicorn worker timeout. Reads only on-disk
|
||||
metadata (no database access). Each provider is warmed in isolation;
|
||||
failures are logged and fall back to lazy loading.
|
||||
|
||||
Args:
|
||||
provider_types (Iterable[str] | None): Subset to warm. Defaults to all.
|
||||
|
||||
Returns:
|
||||
list[str]: Provider types that could not be warmed.
|
||||
"""
|
||||
if provider_types is None:
|
||||
provider_types = Provider.ProviderChoices.values
|
||||
provider_types = list(provider_types)
|
||||
|
||||
COMPLIANCE_WARMING_STARTED.set()
|
||||
logger.info("Compliance cache warm-up started for providers: %s", provider_types)
|
||||
|
||||
failed = []
|
||||
for provider_type in provider_types:
|
||||
try:
|
||||
get_compliance_frameworks(provider_type)
|
||||
_ensure_provider_loaded(provider_type)
|
||||
# Prowler check loading may sys.exit (SystemExit, not Exception).
|
||||
except (Exception, SystemExit):
|
||||
logger.warning(
|
||||
"Failed to warm compliance caches for provider '%s'; "
|
||||
"loading lazily on first request",
|
||||
provider_type,
|
||||
exc_info=True,
|
||||
)
|
||||
failed.append(provider_type)
|
||||
|
||||
# Mark as warmed even when some providers failed: a failed provider falls
|
||||
# back to a single-provider lazy load, which stays under the worker timeout.
|
||||
COMPLIANCE_WARMED.set()
|
||||
logger.info(
|
||||
"Compliance cache warm-up finished (providers warmed: %d, failed: %s)",
|
||||
len(provider_types) - len(failed),
|
||||
failed,
|
||||
)
|
||||
return failed
|
||||
|
||||
|
||||
def load_prowler_checks(
|
||||
prowler_compliance, provider_types: Iterable[str] | None = None
|
||||
):
|
||||
@@ -209,8 +267,8 @@ def load_prowler_checks(
|
||||
for compliance_name, compliance_data in prowler_compliance.get(
|
||||
provider_type, {}
|
||||
).items():
|
||||
for requirement in compliance_data.Requirements:
|
||||
for check in requirement.Checks:
|
||||
for requirement in compliance_data.requirements:
|
||||
for check in requirement.checks.get(provider_type, []):
|
||||
try:
|
||||
checks[provider_type][check].add(compliance_name)
|
||||
except KeyError:
|
||||
@@ -290,24 +348,40 @@ def generate_compliance_overview_template(
|
||||
requirements_status = {"passed": 0, "failed": 0, "manual": 0}
|
||||
total_requirements = 0
|
||||
|
||||
for requirement in compliance_data.Requirements:
|
||||
for requirement in compliance_data.requirements:
|
||||
total_requirements += 1
|
||||
total_checks = len(requirement.Checks)
|
||||
checks_dict = {check: None for check in requirement.Checks}
|
||||
provider_check_list = list(requirement.checks.get(provider_type, []))
|
||||
total_checks = len(provider_check_list)
|
||||
checks_dict = dict.fromkeys(provider_check_list)
|
||||
|
||||
req_status_val = "MANUAL" if total_checks == 0 else "PASS"
|
||||
|
||||
# MITRE attrs are wrapped under `_raw_attributes` by the
|
||||
# universal adapter — unwrap so consumers see the flat list.
|
||||
requirement_attributes = requirement.attributes
|
||||
if (
|
||||
isinstance(requirement_attributes, dict)
|
||||
and "_raw_attributes" in requirement_attributes
|
||||
):
|
||||
attributes_payload = list(requirement_attributes["_raw_attributes"])
|
||||
elif isinstance(requirement_attributes, dict):
|
||||
attributes_payload = (
|
||||
[dict(requirement_attributes)] if requirement_attributes else []
|
||||
)
|
||||
else:
|
||||
attributes_payload = [
|
||||
dict(attribute) for attribute in requirement_attributes
|
||||
]
|
||||
|
||||
# Build requirement dictionary
|
||||
requirement_dict = {
|
||||
"name": requirement.Name or requirement.Id,
|
||||
"description": requirement.Description,
|
||||
"tactics": getattr(requirement, "Tactics", []),
|
||||
"subtechniques": getattr(requirement, "SubTechniques", []),
|
||||
"platforms": getattr(requirement, "Platforms", []),
|
||||
"technique_url": getattr(requirement, "TechniqueURL", ""),
|
||||
"attributes": [
|
||||
dict(attribute) for attribute in requirement.Attributes
|
||||
],
|
||||
"name": requirement.name or requirement.id,
|
||||
"description": requirement.description,
|
||||
"tactics": requirement.tactics or [],
|
||||
"subtechniques": requirement.sub_techniques or [],
|
||||
"platforms": requirement.platforms or [],
|
||||
"technique_url": requirement.technique_url or "",
|
||||
"attributes": attributes_payload,
|
||||
"checks": checks_dict,
|
||||
"checks_status": {
|
||||
"pass": 0,
|
||||
@@ -325,15 +399,15 @@ def generate_compliance_overview_template(
|
||||
requirements_status["passed"] += 1
|
||||
|
||||
# Add requirement to compliance requirements
|
||||
compliance_requirements[requirement.Id] = requirement_dict
|
||||
compliance_requirements[requirement.id] = requirement_dict
|
||||
|
||||
# Build compliance dictionary
|
||||
compliance_dict = {
|
||||
"framework": compliance_data.Framework,
|
||||
"name": compliance_data.Name,
|
||||
"version": compliance_data.Version,
|
||||
"framework": compliance_data.framework,
|
||||
"name": compliance_data.name,
|
||||
"version": compliance_data.version,
|
||||
"provider": provider_type,
|
||||
"description": compliance_data.Description,
|
||||
"description": compliance_data.description,
|
||||
"requirements": compliance_requirements,
|
||||
"requirements_status": requirements_status,
|
||||
"total_requirements": total_requirements,
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user