mirror of
https://github.com/prowler-cloud/prowler.git
synced 2026-08-19 17:40:25 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9b0c4a0073 | ||
|
|
1459046985 | ||
|
|
84c3c9ce0a | ||
|
|
8271659fda | ||
|
|
9916e1ac66 | ||
|
|
ccec96ac5f | ||
|
|
f5ea116763 | ||
|
|
99ca260855 | ||
|
|
bfc6b9e577 | ||
|
|
bc3c922177 | ||
|
|
0e20d2388e | ||
|
|
e0ddc0de27 | ||
|
|
cf840588c7 | ||
|
|
e1c2e9373c | ||
|
|
641a11e4a0 | ||
|
|
59b13778e2 | ||
|
|
7d8c64d35b | ||
|
|
a022ad5c7a | ||
|
|
0f31f1a3c2 | ||
|
|
1ee71c1f20 | ||
|
|
0dc424031b | ||
|
|
0d899b3076 | ||
|
|
3fd9fca32f | ||
|
|
07d7e9d5bb | ||
|
|
31b8ab9b4b | ||
|
|
60c4e9b257 | ||
|
|
8e9af708f8 | ||
|
|
24b670ac36 | ||
|
|
cc6c731af6 | ||
|
|
c53acd4184 | ||
|
|
faa74f4c6c | ||
|
|
077dff7f68 | ||
|
|
a4752d6a27 | ||
|
|
a9f6e04a84 | ||
|
|
d9224e682f | ||
|
|
9822fd97d7 | ||
|
|
bd72ec91ea | ||
|
|
fa9b2c707b | ||
|
|
22401a54a0 | ||
|
|
9c15796c84 | ||
|
|
db9356ba86 | ||
|
|
54237d824a | ||
|
|
7df1054966 | ||
|
|
53772e2931 | ||
|
|
d37d5058bb | ||
|
|
8debf70d5c | ||
|
|
1a1e804c00 | ||
|
|
c7583a5633 | ||
|
|
470e218bb8 | ||
|
|
488f3d1bed | ||
|
|
0f6137dd04 | ||
|
|
f5f6e5768d | ||
|
|
c6dae3a711 | ||
|
|
4242297e72 | ||
|
|
8d4be0f586 | ||
|
|
d78e0189e0 | ||
|
|
cef131812c | ||
|
|
3c78a0df43 | ||
|
|
991e8b8061 | ||
|
|
dbdbd8a379 | ||
|
|
a0a0883578 | ||
|
|
a4bf70eecf | ||
|
|
5bc41b2609 | ||
|
|
58fd4170b5 | ||
|
|
3369e48260 | ||
|
|
106026614d | ||
|
|
3f2e5929d9 | ||
|
|
c93ea035fc | ||
|
|
847997672d | ||
|
|
08ee83c572 | ||
|
|
892cc2bc07 | ||
|
|
2a077cae5e | ||
|
|
1ee4de18be | ||
|
|
b44f8ebf5f | ||
|
|
6f72785a28 | ||
|
|
be78fe8374 | ||
|
|
01c004a7af | ||
|
|
0b36d08b92 | ||
|
|
bf9c53a4c3 | ||
|
|
80c5363649 | ||
|
|
9d899ae0e2 | ||
|
|
13bd6fc0bf | ||
|
|
18b3c49234 | ||
|
|
84ea68927f | ||
|
|
1af9cdd351 | ||
|
|
25bcbac309 | ||
|
|
20aad80a78 | ||
|
|
52875b5c7c | ||
|
|
c665005790 | ||
|
|
b2532ebfe5 | ||
|
|
3461f9ac49 | ||
|
|
d874fc573d | ||
|
|
991c204a88 | ||
|
|
eee17f0e8b | ||
|
|
80608bfdbc | ||
|
|
6c5f54808d | ||
|
|
bb6608c1d7 | ||
|
|
18e1bf5195 |
@@ -72,8 +72,8 @@ NEO4J_APOC_IMPORT_FILE_ENABLED=false
|
||||
NEO4J_APOC_IMPORT_FILE_USE_NEO4J_CONFIG=true
|
||||
NEO4J_APOC_TRIGGER_ENABLED=false
|
||||
NEO4J_DBMS_CONNECTOR_BOLT_LISTEN_ADDRESS=0.0.0.0:7687
|
||||
# Neo4j Prowler settings
|
||||
ATTACK_PATHS_BATCH_SIZE=1000
|
||||
# Attack Paths graph settings
|
||||
ATTACK_PATHS_GRAPH_MUTATION_BATCH_SIZE=1000
|
||||
ATTACK_PATHS_SERVICE_UNAVAILABLE_MAX_RETRIES=3
|
||||
ATTACK_PATHS_READ_QUERY_TIMEOUT_SECONDS=30
|
||||
ATTACK_PATHS_MAX_CUSTOM_QUERY_NODES=250
|
||||
@@ -146,7 +146,7 @@ DJANGO_SENTRY_DSN=
|
||||
DJANGO_THROTTLE_TOKEN_OBTAIN=50/minute
|
||||
|
||||
# Sentry for the web app (server + browser). The UI_SENTRY_* values load only
|
||||
# when UI_SENTRY_ENABLE="true"; without it they are ignored (default off, zero
|
||||
# when UI_SENTRY_ENABLED="true"; without it they are ignored (default off, zero
|
||||
# egress). The deprecated NEXT_PUBLIC_SENTRY_DSN still activates Sentry without
|
||||
# the flag. SENTRY_RELEASE (unprefixed) feeds the web app's server/edge SDKs.
|
||||
UI_SENTRY_DSN=
|
||||
@@ -158,7 +158,7 @@ SENTRY_RELEASE=local
|
||||
# REO_DEV_CLIENT_ID=
|
||||
|
||||
#### Prowler release version ####
|
||||
NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v5.33.2
|
||||
NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v5.35.0
|
||||
|
||||
# Social login credentials
|
||||
SOCIAL_GOOGLE_OAUTH_CALLBACK_URL="${AUTH_URL}/api/auth/callback/google"
|
||||
|
||||
@@ -5,10 +5,20 @@
|
||||
"version": "v8",
|
||||
"sha": "ed597411d8f924073f98dfc5c65a23a2325f34cd"
|
||||
},
|
||||
"github/gh-aw-actions/setup@v0.81.6": {
|
||||
"repo": "github/gh-aw-actions/setup",
|
||||
"version": "v0.81.6",
|
||||
"sha": "ba6380cc6e5be5d21677bebe04d52fb48e3abec7"
|
||||
},
|
||||
"github/gh-aw/actions/setup@v0.43.23": {
|
||||
"repo": "github/gh-aw/actions/setup",
|
||||
"version": "v0.43.23",
|
||||
"sha": "9382be3ca9ac18917e111a99d4e6bbff58d0dccc"
|
||||
},
|
||||
"step-security/harden-runner@v2.20.0": {
|
||||
"repo": "step-security/harden-runner",
|
||||
"version": "v2.20.0",
|
||||
"sha": "bf7454d06d71f1098171f2acdf0cd4708d7b5920"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,6 +20,7 @@ Please add a detailed description of how to review this PR.
|
||||
|
||||
- [ ] This feature/issue is listed in the [open issues](https://github.com/prowler-cloud/prowler/issues?q=sort%3Aupdated-desc+is%3Aissue+is%3Aopen) or roadmap.prowler.com
|
||||
- [ ] Is it assigned to me, if not, request it via the [open issues](https://github.com/prowler-cloud/prowler/issues?q=sort%3Aupdated-desc+is%3Aissue+is%3Aopen) or [Prowler Community Slack](https://goto.prowler.com/slack)
|
||||
- [ ] I have reviewed the [open pull requests](https://github.com/prowler-cloud/prowler/pulls?q=sort%3Aupdated-desc+is%3Apr+is%3Aopen) and confirmed there is no existing PR that implements the same outcome
|
||||
|
||||
</details>
|
||||
|
||||
@@ -27,7 +28,7 @@ Please add a detailed description of how to review this PR.
|
||||
- [ ] Review if the code is being covered by tests.
|
||||
- [ ] Review if code is being documented following this specification https://github.com/google/styleguide/blob/gh-pages/pyguide.md#38-comments-and-docstrings
|
||||
- [ ] Review if backport is needed.
|
||||
- [ ] Review if is needed to change the [Readme.md](https://github.com/prowler-cloud/prowler/blob/master/README.md)
|
||||
- [ ] Review if is needed to change the [README.md](https://github.com/prowler-cloud/prowler/blob/master/README.md)
|
||||
- [ ] Ensure a changelog fragment is added under [prowler/changelog.d/](https://github.com/prowler-cloud/prowler/tree/master/prowler/changelog.d), if applicable.
|
||||
|
||||
#### SDK/CLI
|
||||
|
||||
@@ -23,6 +23,10 @@
|
||||
"prConcurrentLimit": 20,
|
||||
"prHourlyLimit": 10,
|
||||
"vulnerabilityAlerts": {
|
||||
"labels": [
|
||||
"dependencies",
|
||||
"security"
|
||||
],
|
||||
"prHourlyLimit": 0,
|
||||
"prConcurrentLimit": 0
|
||||
},
|
||||
@@ -60,6 +64,13 @@
|
||||
],
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"description": "gh-aw compiled lock files - generated by 'gh aw compile', action pins must match the compiler version, never bump directly",
|
||||
"matchFileNames": [
|
||||
".github/workflows/*.lock.yml"
|
||||
],
|
||||
"enabled": false
|
||||
},
|
||||
{
|
||||
"description": "GitHub Actions - single grouped PR, no changelog, scope=ci",
|
||||
"matchManagers": [
|
||||
|
||||
@@ -35,7 +35,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
|
||||
@@ -46,7 +46,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
|
||||
@@ -46,7 +46,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
|
||||
@@ -65,7 +65,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -108,7 +108,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -175,7 +175,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -236,7 +236,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -33,7 +33,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -69,7 +69,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -92,6 +92,8 @@ jobs:
|
||||
_http._tcp.deb.debian.org:443
|
||||
powershellinfraartifacts-gkhedzdeaghdezhr.z01.azurefd.net:443
|
||||
get.trivy.dev:443
|
||||
raw.githubusercontent.com:443
|
||||
releases.astral.sh:443
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
@@ -43,7 +43,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
|
||||
@@ -78,7 +78,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
|
||||
@@ -30,7 +30,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
|
||||
@@ -29,7 +29,7 @@ jobs:
|
||||
patch_version: ${{ steps.detect.outputs.patch_version }}
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -75,7 +75,7 @@ jobs:
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -202,7 +202,7 @@ jobs:
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -307,7 +307,7 @@ jobs:
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
name: 'Tools: Check Test Init Files'
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- 'master'
|
||||
- 'v5.*'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
check-test-init-files:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Check for __init__.py files in test directories
|
||||
run: python3 scripts/check_test_init_files.py .
|
||||
@@ -36,7 +36,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
|
||||
@@ -22,7 +22,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -54,7 +54,7 @@ jobs:
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Harden the runner (Block outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
|
||||
@@ -26,7 +26,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -25,7 +25,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
name: 'Docs: Check Provider Cards Snippet'
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- 'master'
|
||||
- 'v5.*'
|
||||
paths:
|
||||
- 'docs/user-guide/providers/**/getting-started-*.mdx'
|
||||
- 'docs/scripts/generate_provider_cards.py'
|
||||
- 'docs/snippets/provider-cards.mdx'
|
||||
- 'api/src/backend/api/models.py'
|
||||
- '.github/workflows/docs-check-provider-cards.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
check-provider-cards:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
github.com:443
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Verify provider cards snippet is up to date
|
||||
run: |
|
||||
if ! python3 docs/scripts/generate_provider_cards.py; then
|
||||
echo "::error::docs/snippets/provider-cards.mdx is out of sync with the provider getting-started pages or the API ProviderChoices enum."
|
||||
echo "Run 'python3 docs/scripts/generate_provider_cards.py' locally and commit the regenerated snippet."
|
||||
echo "--- diff ---"
|
||||
git diff docs/snippets/provider-cards.mdx
|
||||
exit 1
|
||||
fi
|
||||
@@ -25,7 +25,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
# We can't block as Trufflehog needs to verify secrets against vendors
|
||||
egress-policy: audit
|
||||
|
||||
@@ -33,7 +33,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -26,7 +26,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -22,7 +22,7 @@ jobs:
|
||||
issues: write
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
|
||||
Generated
+1301
-703
File diff suppressed because one or more lines are too long
@@ -12,8 +12,8 @@ if: contains(toJson(github.event.issue.labels), 'status/needs-triage')
|
||||
|
||||
timeout-minutes: 12
|
||||
|
||||
rate-limit:
|
||||
max: 5
|
||||
user-rate-limit:
|
||||
max-runs-per-window: 5
|
||||
window: 60
|
||||
|
||||
concurrency:
|
||||
@@ -30,6 +30,12 @@ permissions:
|
||||
engine: copilot
|
||||
strict: false
|
||||
|
||||
pre-steps:
|
||||
- name: Harden the runner
|
||||
uses: step-security/harden-runner@v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
imports:
|
||||
- ../agents/issue-triage.md
|
||||
|
||||
@@ -108,7 +114,7 @@ Triage the following GitHub issue using the Prowler Issue Triage Agent persona.
|
||||
|
||||
## Sanitized Issue Content
|
||||
|
||||
${{ needs.activation.outputs.text }}
|
||||
${{ steps.sanitized.outputs.text }}
|
||||
|
||||
## Instructions
|
||||
|
||||
|
||||
@@ -27,7 +27,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -46,7 +46,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -93,7 +93,7 @@ jobs:
|
||||
fi
|
||||
|
||||
- name: Add community label
|
||||
if: steps.check_membership.outputs.is_member == 'false'
|
||||
if: steps.check_membership.outputs.is_member == 'false' && github.event.pull_request.user.type != 'Bot'
|
||||
env:
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
|
||||
@@ -26,7 +26,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
|
||||
@@ -45,7 +45,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
|
||||
@@ -64,7 +64,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -106,7 +106,7 @@ jobs:
|
||||
packages: write
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -165,7 +165,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -227,7 +227,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -33,7 +33,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -68,7 +68,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
|
||||
@@ -29,7 +29,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -67,7 +67,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -32,7 +32,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
|
||||
@@ -48,7 +48,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -83,7 +83,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -85,7 +85,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
|
||||
@@ -35,7 +35,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -26,7 +26,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
|
||||
@@ -29,7 +29,7 @@ jobs:
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -28,12 +28,13 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
api.github.com:443
|
||||
github.com:443
|
||||
raw.githubusercontent.com:443
|
||||
objects.githubusercontent.com:443
|
||||
codeload.github.com:443
|
||||
release-assets.githubusercontent.com:443
|
||||
@@ -41,6 +42,7 @@ jobs:
|
||||
files.pythonhosted.org:443
|
||||
registry.npmjs.org:443
|
||||
nodejs.org:443
|
||||
releases.astral.sh:443
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
@@ -25,7 +25,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
|
||||
@@ -33,7 +33,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
|
||||
@@ -53,7 +53,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
|
||||
@@ -60,7 +60,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -98,7 +98,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -142,7 +142,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -180,7 +180,7 @@ jobs:
|
||||
uses: aws-actions/configure-aws-credentials@d979d5b3a71173a29b74b5b88418bfda9437d885 # v6.1.1
|
||||
with:
|
||||
aws-region: us-east-1
|
||||
role-to-assume: ${{ secrets.PUBLIC_ECR_IAM_ROLE_ARN }}
|
||||
role-to-assume: ${{ secrets.PUBLIC_ECR_PUSH_ROLE_ARN }}
|
||||
|
||||
- name: Login to Public ECR
|
||||
uses: aws-actions/amazon-ecr-login@d539f0932e70871a027e9d5a9d8fc38589180a64 # v2.1.6
|
||||
@@ -215,7 +215,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -241,7 +241,7 @@ jobs:
|
||||
uses: aws-actions/configure-aws-credentials@d979d5b3a71173a29b74b5b88418bfda9437d885 # v6.1.1
|
||||
with:
|
||||
aws-region: us-east-1
|
||||
role-to-assume: ${{ secrets.PUBLIC_ECR_IAM_ROLE_ARN }}
|
||||
role-to-assume: ${{ secrets.PUBLIC_ECR_PUSH_ROLE_ARN }}
|
||||
|
||||
- name: Login to Public ECR
|
||||
uses: aws-actions/amazon-ecr-login@d539f0932e70871a027e9d5a9d8fc38589180a64 # v2.1.6
|
||||
@@ -330,7 +330,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -35,7 +35,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -71,7 +71,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -94,6 +94,8 @@ jobs:
|
||||
_http._tcp.deb.debian.org:443
|
||||
powershellinfraartifacts-gkhedzdeaghdezhr.z01.azurefd.net:443
|
||||
get.trivy.dev:443
|
||||
raw.githubusercontent.com:443
|
||||
releases.astral.sh:443
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -66,7 +66,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -102,7 +102,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -27,7 +27,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -25,7 +25,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -37,7 +37,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
|
||||
@@ -33,7 +33,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -615,6 +615,30 @@ jobs:
|
||||
flags: prowler-py${{ matrix.python-version }}-linode
|
||||
files: ./linode_coverage.xml
|
||||
|
||||
# E2E Networks Provider
|
||||
- name: Check if E2E Networks files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: changed-e2enetworks
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
./prowler/**/e2enetworks/**
|
||||
./tests/**/e2enetworks/**
|
||||
./uv.lock
|
||||
|
||||
- name: Run E2E Networks tests
|
||||
if: steps.changed-e2enetworks.outputs.any_changed == 'true'
|
||||
run: uv run pytest -n auto --cov=./prowler/providers/e2enetworks --cov-report=xml:e2enetworks_coverage.xml tests/providers/e2enetworks
|
||||
|
||||
- name: Upload E2E Networks coverage to Codecov
|
||||
if: steps.changed-e2enetworks.outputs.any_changed == 'true'
|
||||
uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2
|
||||
env:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
with:
|
||||
flags: prowler-py${{ matrix.python-version }}-e2enetworks
|
||||
files: ./e2enetworks_coverage.xml
|
||||
|
||||
# External Provider (dynamic loading)
|
||||
- name: Check if External Provider files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
|
||||
@@ -52,7 +52,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
|
||||
@@ -49,7 +49,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
|
||||
@@ -45,7 +45,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -64,7 +64,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -107,7 +107,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -160,7 +160,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -222,7 +222,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -33,7 +33,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -69,7 +69,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
|
||||
@@ -96,7 +96,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -316,7 +316,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -32,7 +32,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
|
||||
@@ -31,7 +31,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
|
||||
@@ -144,7 +144,7 @@ repos:
|
||||
|
||||
- id: generate-provider-cards
|
||||
name: "Docs - regenerate provider cards snippet"
|
||||
entry: python docs/scripts/generate_provider_cards.py
|
||||
entry: python3 docs/scripts/generate_provider_cards.py
|
||||
language: system
|
||||
files: { glob: ["docs/user-guide/providers/**/getting-started-*.mdx", "docs/scripts/generate_provider_cards.py", "docs/snippets/provider-cards.mdx", "api/src/backend/api/models.py"] }
|
||||
pass_filenames: false
|
||||
|
||||
+33
-22
@@ -15,21 +15,32 @@
|
||||
# neither vulnerable code path (Archive::Tar parsing or regex compilation of
|
||||
# attacker-controlled input) is reachable from Prowler. No Debian bookworm fix
|
||||
# is available yet.
|
||||
CVE-2026-42496 pkg:perl exp:2026-07-15
|
||||
CVE-2026-42496 pkg:perl-base exp:2026-07-15
|
||||
CVE-2026-42496 pkg:perl-modules-5.36 exp:2026-07-15
|
||||
CVE-2026-42496 pkg:libperl5.36 exp:2026-07-15
|
||||
CVE-2026-8376 pkg:perl exp:2026-07-15
|
||||
CVE-2026-8376 pkg:perl-base exp:2026-07-15
|
||||
CVE-2026-8376 pkg:perl-modules-5.36 exp:2026-07-15
|
||||
CVE-2026-8376 pkg:libperl5.36 exp:2026-07-15
|
||||
CVE-2026-42496 pkg:perl exp:2026-08-15
|
||||
CVE-2026-42496 pkg:perl-base exp:2026-08-15
|
||||
CVE-2026-42496 pkg:perl-modules-5.36 exp:2026-08-15
|
||||
CVE-2026-42496 pkg:libperl5.36 exp:2026-08-15
|
||||
CVE-2026-8376 pkg:perl exp:2026-08-15
|
||||
CVE-2026-8376 pkg:perl-base exp:2026-08-15
|
||||
CVE-2026-8376 pkg:perl-modules-5.36 exp:2026-08-15
|
||||
CVE-2026-8376 pkg:libperl5.36 exp:2026-08-15
|
||||
|
||||
# CVE-2026-13221 - Perl regex trie overflow.
|
||||
# Packages: perl, perl-base, perl-modules-5.36, libperl5.36.
|
||||
# Why ignored: upstream confirms Perl 5.36.0 is not affected; the regression
|
||||
# was introduced after this version. Debian currently marks bookworm as
|
||||
# vulnerable, which causes Trivy to report a false positive.
|
||||
# Ref: https://github.com/Perl/perl5/issues/23388
|
||||
CVE-2026-13221 pkg:perl exp:2026-08-15
|
||||
CVE-2026-13221 pkg:perl-base exp:2026-08-15
|
||||
CVE-2026-13221 pkg:perl-modules-5.36 exp:2026-08-15
|
||||
CVE-2026-13221 pkg:libperl5.36 exp:2026-08-15
|
||||
|
||||
# CVE-2025-7458 — SQLite integer overflow.
|
||||
# Package: libsqlite3-0.
|
||||
# Why ignored: transitive dependency of CPython's stdlib sqlite3 module. The
|
||||
# Prowler SDK does not open user-supplied SQLite databases; SQLite usage is
|
||||
# internal and bounded. No Debian bookworm fix is available.
|
||||
CVE-2025-7458 pkg:libsqlite3-0 exp:2026-07-15
|
||||
CVE-2025-7458 pkg:libsqlite3-0 exp:2026-08-15
|
||||
|
||||
# CVE-2026-43185 — Linux kernel ksmbd signedness bug.
|
||||
# Package: linux-libc-dev.
|
||||
@@ -37,7 +48,7 @@ CVE-2025-7458 pkg:libsqlite3-0 exp:2026-07-15
|
||||
# not a running kernel. Containers execute against the host kernel, so these
|
||||
# headers are inert at runtime. The upstream fix landed in kernel 7.0-rc2 and
|
||||
# has not been backported to Debian's 6.1 LTS line.
|
||||
CVE-2026-43185 pkg:linux-libc-dev exp:2026-07-15
|
||||
CVE-2026-43185 pkg:linux-libc-dev exp:2026-08-15
|
||||
|
||||
# CVE-2023-45853 — zlib MiniZip integer overflow / heap overflow in
|
||||
# zipOpenNewFileInZip4_64.
|
||||
@@ -49,8 +60,8 @@ CVE-2026-43185 pkg:linux-libc-dev exp:2026-07-15
|
||||
# zlib 1.3.1, available in Debian trixie (13); migrating the base image would
|
||||
# clear it fully.
|
||||
# Ref: https://security-tracker.debian.org/tracker/CVE-2023-45853
|
||||
CVE-2023-45853 pkg:zlib1g exp:2026-07-15
|
||||
CVE-2023-45853 pkg:zlib1g-dev exp:2026-07-15
|
||||
CVE-2023-45853 pkg:zlib1g exp:2026-08-15
|
||||
CVE-2023-45853 pkg:zlib1g-dev exp:2026-08-15
|
||||
|
||||
# CVE-2026-55200 — libssh2 out-of-bounds write in ssh2_transport_read() due to
|
||||
# an unchecked packet_length field in transport.c (heap corruption, possible RCE).
|
||||
@@ -63,7 +74,7 @@ CVE-2023-45853 pkg:zlib1g-dev exp:2026-07-15
|
||||
# affected code is unreachable at runtime. Fixed upstream in libssh2 commit
|
||||
# 97acf3df (PR #2052); no Debian bookworm fix is available yet.
|
||||
# Ref: https://security-tracker.debian.org/tracker/CVE-2026-55200
|
||||
CVE-2026-55200 pkg:libssh2-1 exp:2026-07-15
|
||||
CVE-2026-55200 pkg:libssh2-1 exp:2026-08-15
|
||||
|
||||
# --- API container image (api/Dockerfile) ---
|
||||
# The entries below are specific to the Prowler API image, which ships
|
||||
@@ -78,13 +89,13 @@ CVE-2026-55200 pkg:libssh2-1 exp:2026-07-15
|
||||
# at runtime. The vulnerable path requires parsing attacker-controlled XML with
|
||||
# the affected interpreter, which Prowler does not do with the system Python.
|
||||
# Full mitigation also needs libexpat >= 2.8.0; no Debian bookworm fix yet.
|
||||
CVE-2026-7210 pkg:python3.11 exp:2026-07-15
|
||||
CVE-2026-7210 pkg:python3.11-dev exp:2026-07-15
|
||||
CVE-2026-7210 pkg:python3.11-minimal exp:2026-07-15
|
||||
CVE-2026-7210 pkg:libpython3.11 exp:2026-07-15
|
||||
CVE-2026-7210 pkg:libpython3.11-dev exp:2026-07-15
|
||||
CVE-2026-7210 pkg:libpython3.11-minimal exp:2026-07-15
|
||||
CVE-2026-7210 pkg:libpython3.11-stdlib exp:2026-07-15
|
||||
CVE-2026-7210 pkg:python3.11 exp:2026-08-15
|
||||
CVE-2026-7210 pkg:python3.11-dev exp:2026-08-15
|
||||
CVE-2026-7210 pkg:python3.11-minimal exp:2026-08-15
|
||||
CVE-2026-7210 pkg:libpython3.11 exp:2026-08-15
|
||||
CVE-2026-7210 pkg:libpython3.11-dev exp:2026-08-15
|
||||
CVE-2026-7210 pkg:libpython3.11-minimal exp:2026-08-15
|
||||
CVE-2026-7210 pkg:libpython3.11-stdlib exp:2026-08-15
|
||||
|
||||
# CVE-2026-33278 — Unbound DNSSEC validator use-after-free (DoS, possible RCE).
|
||||
# CVE-2026-42960 — Unbound DNS cache poisoning via promiscuous additional records.
|
||||
@@ -94,5 +105,5 @@ CVE-2026-7210 pkg:libpython3.11-stdlib exp:2026-07-15
|
||||
# vulnerabilities require operating a live Unbound recursive DNSSEC validator
|
||||
# that processes attacker-influenced DNS responses. Prowler never starts an
|
||||
# Unbound resolver, so neither code path is reachable. No Debian bookworm fix yet.
|
||||
CVE-2026-33278 pkg:libunbound8 exp:2026-07-15
|
||||
CVE-2026-42960 pkg:libunbound8 exp:2026-07-15
|
||||
CVE-2026-33278 pkg:libunbound8 exp:2026-08-15
|
||||
CVE-2026-42960 pkg:libunbound8 exp:2026-08-15
|
||||
|
||||
@@ -5,6 +5,8 @@ LABEL org.opencontainers.image.source="https://github.com/prowler-cloud/prowler"
|
||||
|
||||
ARG POWERSHELL_VERSION=7.5.0
|
||||
ENV POWERSHELL_VERSION=${POWERSHELL_VERSION}
|
||||
# Opt out of PowerShell telemetry (Application Insights -> dc.services.visualstudio.com)
|
||||
ENV POWERSHELL_TELEMETRY_OPTOUT=1
|
||||
|
||||
ARG TRIVY_VERSION=0.71.2
|
||||
ENV TRIVY_VERSION=${TRIVY_VERSION}
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
<img align="center" alt="Prowler logo" src="https://github.com/prowler-cloud/prowler/blob/master/docs/img/prowler-logo-white.png#gh-dark-mode-only" width="50%" height="50%">
|
||||
</p>
|
||||
<p align="center">
|
||||
<b><i>Prowler</b> is the Open Cloud Security Platform trusted by thousands to automate security and compliance in any cloud environment. With hundreds of ready-to-use checks and compliance frameworks, Prowler delivers real-time, customizable monitoring and seamless integrations, making cloud security simple, scalable, and cost-effective for organizations of any size.
|
||||
<b><i>Prowler</b> is the Open Cloud Security Platform trusted by thousands to automate security and compliance in any cloud environment. With thousands of ready-to-use checks and compliance frameworks, Prowler delivers real-time, customizable monitoring and seamless integrations, making cloud security simple, scalable, and cost-effective for organizations of any size.
|
||||
</p>
|
||||
<p align="center">
|
||||
<b>Secure ANY cloud at AI Speed at <a href="https://prowler.com">prowler.com</i></b>
|
||||
@@ -21,7 +21,7 @@
|
||||
<a href="https://pypi.python.org/pypi/prowler/"><img alt="Python Version" src="https://img.shields.io/pypi/pyversions/prowler.svg"></a>
|
||||
<a href="https://pypistats.org/packages/prowler"><img alt="PyPI Downloads" src="https://img.shields.io/pypi/dw/prowler.svg?label=downloads"></a>
|
||||
<a href="https://hub.docker.com/r/toniblyx/prowler"><img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/toniblyx/prowler"></a>
|
||||
<a href="https://gallery.ecr.aws/prowler-cloud/prowler"><img width="120" height=19" alt="AWS ECR Gallery" src="https://user-images.githubusercontent.com/3985464/151531396-b6535a68-c907-44eb-95a1-a09508178616.png"></a>
|
||||
<a href="https://gallery.ecr.aws/prowler-cloud/prowler"><img width="120" height="19" alt="AWS ECR Gallery" src="https://user-images.githubusercontent.com/3985464/151531396-b6535a68-c907-44eb-95a1-a09508178616.png"></a>
|
||||
<a href="https://codecov.io/gh/prowler-cloud/prowler"><img alt="Codecov coverage" src="https://codecov.io/gh/prowler-cloud/prowler/graph/badge.svg?token=OflBGsdpDl"/></a>
|
||||
<a href="https://insights.linuxfoundation.org/project/prowler-cloud-prowler"><img alt="Linux Foundation insights health score" src="https://insights.linuxfoundation.org/api/badge/health-score?project=prowler-cloud-prowler"/></a>
|
||||
</p>
|
||||
@@ -41,7 +41,7 @@
|
||||
|
||||
# Description
|
||||
|
||||
**Prowler** is the world’s most widely used _Open-Source Cloud Security Platform_ that automates security and compliance across **any cloud environment**. With hundreds of ready-to-use security checks, remediation guidance, and compliance frameworks, Prowler is built to _“Secure ANY Cloud at AI Speed”_. Prowler delivers **AI-driven**, **customizable**, and **easy-to-use** assessments, dashboards, reports, and integrations, making cloud security **simple**, **scalable**, and **cost-effective** for organizations of any size.
|
||||
**Prowler** is the world’s most widely used _Open-Source Cloud Security Platform_ that automates security and compliance across **any cloud environment**. With thousands of ready-to-use security checks, remediation guidance, and compliance frameworks, Prowler is built to _“Secure ANY Cloud at AI Speed”_. Prowler delivers **AI-driven**, **customizable**, and **easy-to-use** assessments, dashboards, reports, and integrations, making cloud security **simple**, **scalable**, and **cost-effective** for organizations of any size.
|
||||
|
||||
Prowler includes hundreds of built-in controls to ensure compliance with standards and frameworks, including:
|
||||
|
||||
@@ -54,16 +54,16 @@ Prowler includes hundreds of built-in controls to ensure compliance with standar
|
||||
- **National Security Standards:** ENS (Spanish National Security Scheme) and KISA ISMS-P (Korean)
|
||||
- **Custom Security Frameworks:** Tailored to your needs
|
||||
|
||||
## Prowler App / Prowler Cloud
|
||||
## Prowler Cloud & Prowler Local Server
|
||||
|
||||
Prowler App / [Prowler Cloud](https://cloud.prowler.com/) is a web-based application that simplifies running Prowler across your cloud provider accounts. It provides a user-friendly interface to visualize the results and streamline your security assessments.
|
||||
[Prowler Cloud](https://cloud.prowler.com/) and Prowler Local Server, its self-hosted open-source version, are web applications that simplify running Prowler across your cloud provider accounts. They provide a user-friendly interface to visualize the results and streamline your security assessments.
|
||||
|
||||

|
||||

|
||||

|
||||

|
||||
|
||||
|
||||
>For more details, refer to the [Prowler App Documentation](https://docs.prowler.com/projects/prowler-open-source/en/latest/#prowler-app-installation)
|
||||
>For more details, refer to the [Prowler Local Server documentation](https://docs.prowler.com/getting-started/installation/prowler-app)
|
||||
|
||||
## Prowler CLI
|
||||
|
||||
@@ -73,12 +73,12 @@ prowler <provider>
|
||||

|
||||
|
||||
|
||||
## Prowler Dashboard
|
||||
## Prowler Local Dashboard
|
||||
|
||||
```console
|
||||
prowler dashboard
|
||||
```
|
||||

|
||||

|
||||
|
||||
|
||||
## Attack Paths
|
||||
@@ -121,7 +121,7 @@ Every AWS provider scan will enqueue an Attack Paths ingestion job automatically
|
||||
> For the most accurate and up-to-date information about checks, services, frameworks, and categories, visit [**Prowler Hub**](https://hub.prowler.com).
|
||||
|
||||
|
||||
| Provider | Checks | Services | [Compliance Frameworks](https://docs.prowler.com/projects/prowler-open-source/en/latest/tutorials/compliance/) | [Categories](https://docs.prowler.com/projects/prowler-open-source/en/latest/tutorials/misc/#categories) | Support | Interface |
|
||||
| Provider | Checks | Services | [Compliance Frameworks](https://docs.prowler.com/user-guide/compliance/tutorials/compliance) | [Categories](https://docs.prowler.com/user-guide/cli/tutorials/misc#categories) | Support | Interface |
|
||||
|---|---|---|---|---|---|---|
|
||||
| AWS | 615 | 86 | 47 | 19 | Official | UI, API, CLI |
|
||||
| Azure | 190 | 22 | 21 | 16 | Official | UI, API, CLI |
|
||||
@@ -141,6 +141,7 @@ Every AWS provider scan will enqueue an Attack Paths ingestion job automatically
|
||||
| Vercel | 26 | 6 | 1 | 8 | Official | UI, API, CLI |
|
||||
| Okta | 29 | 8 | 2 | 2 | Official | UI, API, CLI |
|
||||
| Linode [Contact us](https://prowler.com/contact) | 10 | 3 | 1 | 4 | Unofficial | CLI |
|
||||
| E2E Networks [Contact us](https://prowler.com/contact) | 27 | 6 | 0 | 2 | Unofficial | CLI |
|
||||
| Scaleway [Contact us](https://prowler.com/contact) | 1 | 1 | 1 | 1 | Unofficial | CLI |
|
||||
| StackIT [Contact us](https://prowler.com/contact) | 7 | 2 | 1 | 3 | Unofficial | CLI |
|
||||
| NHN | 6 | 2 | 2 | 0 | Unofficial | CLI |
|
||||
@@ -159,11 +160,11 @@ Every AWS provider scan will enqueue an Attack Paths ingestion job automatically
|
||||
|
||||
# 💻 Installation
|
||||
|
||||
## Prowler App
|
||||
## Prowler Local Server
|
||||
|
||||
Prowler App offers flexible installation methods tailored to various environments:
|
||||
Prowler Local Server offers flexible installation methods tailored to various environments:
|
||||
|
||||
> For detailed instructions on using Prowler App, refer to the [Prowler App Usage Guide](https://docs.prowler.com/projects/prowler-open-source/en/latest/tutorials/prowler-app/).
|
||||
> For detailed instructions on using Prowler Local Server, refer to the [usage guide](https://docs.prowler.com/user-guide/tutorials/prowler-app).
|
||||
|
||||
### Docker Compose
|
||||
|
||||
@@ -196,7 +197,7 @@ docker compose up -d
|
||||
> [!WARNING]
|
||||
> 🔒 For a secure setup, the API auto-generates a unique key pair, `DJANGO_TOKEN_SIGNING_KEY` and `DJANGO_TOKEN_VERIFYING_KEY`, and stores it in `~/.config/prowler-api` (non-container) or the bound Docker volume in `_data/api` (container). Never commit or reuse static/default keys. To rotate keys, delete the stored key files and restart the API.
|
||||
|
||||
Once configured, access the Prowler App at http://localhost:3000. Sign up using your email and password to get started.
|
||||
Once configured, access Prowler Local Server at http://localhost:3000. Sign up using your email and password to get started.
|
||||
|
||||
### Common Issues with Docker Pull Installation
|
||||
|
||||
@@ -268,7 +269,7 @@ pnpm run build
|
||||
pnpm start
|
||||
```
|
||||
|
||||
> Once configured, access the Prowler App at http://localhost:3000. Sign up using your email and password to get started.
|
||||
> Once configured, access Prowler Local Server at http://localhost:3000. Sign up using your email and password to get started.
|
||||
|
||||
#### Pre-commit Hooks Setup
|
||||
|
||||
@@ -286,7 +287,7 @@ Prowler CLI is available as a project in [PyPI](https://pypi.org/project/prowler
|
||||
pip install prowler
|
||||
prowler -v
|
||||
```
|
||||
>For further guidance, refer to [https://docs.prowler.com](https://docs.prowler.com/projects/prowler-open-source/en/latest/#prowler-cli-installation)
|
||||
>For further guidance, refer to [https://docs.prowler.com](https://docs.prowler.com/getting-started/installation/prowler-cli)
|
||||
|
||||
### Containers
|
||||
|
||||
@@ -306,7 +307,7 @@ The container images are available here:
|
||||
- Prowler CLI:
|
||||
- [DockerHub](https://hub.docker.com/r/prowlercloud/prowler/tags)
|
||||
- [AWS Public ECR](https://gallery.ecr.aws/prowler-cloud/prowler)
|
||||
- Prowler App:
|
||||
- Prowler Local Server:
|
||||
- [DockerHub - Prowler UI](https://hub.docker.com/r/prowlercloud/prowler-ui/tags)
|
||||
- [DockerHub - Prowler API](https://hub.docker.com/r/prowlercloud/prowler-api/tags)
|
||||
|
||||
@@ -356,17 +357,55 @@ Full configuration, per-provider authentication, and SARIF examples: [Prowler Gi
|
||||
|
||||
# ✏️ High level architecture
|
||||
|
||||
## Prowler App
|
||||
**Prowler App** is composed of four key components:
|
||||
## Prowler Local Server
|
||||
**Prowler Local Server** is composed of four key components:
|
||||
|
||||
- **Prowler UI**: A web-based interface, built with Next.js, providing a user-friendly experience for executing Prowler scans and visualizing results.
|
||||
- **Prowler API**: A backend service, developed with Django REST Framework, responsible for running Prowler scans and storing the generated results.
|
||||
- **Prowler SDK**: A Python SDK designed to extend the functionality of the Prowler CLI for advanced capabilities.
|
||||
- **Prowler MCP Server**: A Model Context Protocol server that provides AI tools for Lighthouse, the AI-powered security assistant. This is a critical dependency for Lighthouse functionality.
|
||||
|
||||

|
||||
```mermaid
|
||||
flowchart TB
|
||||
user([User / Security Team])
|
||||
cli([Prowler CLI])
|
||||
|
||||
<!-- Diagram source: docs/images/products/prowler-app-architecture.mmd — edit there, re-render at https://mermaid.live, and replace the PNG. -->
|
||||
subgraph APP["Prowler Local Server"]
|
||||
ui["Prowler UI<br/>(Next.js)"]
|
||||
api["Prowler API<br/>(Django REST Framework)"]
|
||||
worker["API Worker<br/>(Celery)"]
|
||||
beat["API Scheduler<br/>(Celery Beat)"]
|
||||
mcp["Prowler MCP Server<br/>(Lighthouse AI tools)"]
|
||||
end
|
||||
|
||||
sdk["Prowler SDK<br/>(Python)"]
|
||||
|
||||
subgraph DATA["Data Layer"]
|
||||
pg[("PostgreSQL")]
|
||||
valkey[("Valkey / Redis")]
|
||||
neo4j[("Neo4j")]
|
||||
end
|
||||
|
||||
providers["Providers"]
|
||||
|
||||
user --> ui
|
||||
user --> cli
|
||||
ui -->|REST| api
|
||||
ui -->|MCP HTTP| mcp
|
||||
mcp -->|REST| api
|
||||
api --> pg
|
||||
api --> valkey
|
||||
beat -->|enqueue jobs| valkey
|
||||
valkey -->|dispatch| worker
|
||||
worker --> pg
|
||||
worker -->|Attack Paths| neo4j
|
||||
worker -->|invokes| sdk
|
||||
cli --> sdk
|
||||
|
||||
sdk --> providers
|
||||
```
|
||||
|
||||
<!-- Diagram source: docs/images/products/prowler-app-architecture.mmd — keep this inline block, the docs page getting-started/products/prowler-app.mdx, and the .mmd file in sync. -->
|
||||
|
||||
|
||||
## Prowler CLI
|
||||
|
||||
@@ -4,6 +4,35 @@ All notable changes to the **Prowler API** are documented in this file.
|
||||
|
||||
<!-- changelog: release notes start -->
|
||||
|
||||
## [1.36.0] (Prowler v5.35.0)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- `attack-paths-scan-perform` Celery tasks now use the configurable long-task time limits instead of the six-hour defaults [(#12009)](https://github.com/prowler-cloud/prowler/pull/12009)
|
||||
- Attack Paths scans handle provider deletion races cleanly, detect stale tasks after 16 hours, use backend-specific graph synchronization batches, and report exhausted Neptune write retries with the original database error [(#12019)](https://github.com/prowler-cloud/prowler/pull/12019)
|
||||
|
||||
### 🔐 Security
|
||||
|
||||
- Jira integration credentials only accept bare Atlassian site names containing letters, numbers, and hyphens [(#12012)](https://github.com/prowler-cloud/prowler/pull/12012)
|
||||
- Social account linking requires a verified matching email from both the identity provider and the existing user account without sending account connection notifications [(#12013)](https://github.com/prowler-cloud/prowler/pull/12013)
|
||||
|
||||
---
|
||||
|
||||
## [1.35.0] (Prowler v5.34.0)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- `rls_transaction` now falls back directly to the primary DB for connection-level mid-query read replica failures via `execute_wrapper`, reducing non-streaming read crashes during replica recovery [(#10379)](https://github.com/prowler-cloud/prowler/pull/10379)
|
||||
- RBAC permission gates now combine permissions from every role assigned to a user in the active tenant [(#11979)](https://github.com/prowler-cloud/prowler/pull/11979)
|
||||
- `attack-paths-cleanup-stale-scans` now retries worker pings and checks recent scan activity before failing scans and removing temporary databases [(#11986)](https://github.com/prowler-cloud/prowler/pull/11986)
|
||||
|
||||
### 🔐 Security
|
||||
|
||||
- User role relationship updates are limited to the active tenant to preserve role assignments in other tenants [(#11903)](https://github.com/prowler-cloud/prowler/pull/11903)
|
||||
- `api` container image removes the unused Debian `libxml2` runtime package and scopes the `CVE-2026-13221` Trivy exception to unaffected Perl 5.36 packages [(#11991)](https://github.com/prowler-cloud/prowler/pull/11991)
|
||||
|
||||
---
|
||||
|
||||
## [1.34.2] (Prowler v5.33.2)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
@@ -113,6 +113,7 @@ RUN apt-get purge -y --auto-remove \
|
||||
make \
|
||||
libxml2-dev \
|
||||
libxmlsec1-dev \
|
||||
libxmlsec1-openssl \
|
||||
pkg-config \
|
||||
libtool \
|
||||
libxslt1-dev \
|
||||
|
||||
+2
-2
@@ -45,7 +45,7 @@ dependencies = [
|
||||
"gunicorn==26.0.0",
|
||||
"uvloop==0.22.1",
|
||||
"lxml==6.1.0",
|
||||
"prowler @ git+https://github.com/prowler-cloud/prowler.git@v5.33",
|
||||
"prowler @ git+https://github.com/prowler-cloud/prowler.git@v5.35",
|
||||
"psycopg2-binary==2.9.9",
|
||||
"pytest-celery[redis] (==1.3.0)",
|
||||
"sentry-sdk[django] (==2.56.0)",
|
||||
@@ -71,7 +71,7 @@ name = "prowler-api"
|
||||
package-mode = false
|
||||
# Needed for the SDK compatibility
|
||||
requires-python = ">=3.11,<3.13"
|
||||
version = "1.34.2"
|
||||
version = "1.36.0"
|
||||
|
||||
# Shared ruff baseline (kept in sync with mcp_server/pyproject.toml).
|
||||
# target-version tracks this project's lowest supported Python.
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
from allauth.account.models import EmailAddress
|
||||
from allauth.core.exceptions import ImmediateHttpResponse
|
||||
from allauth.socialaccount.adapter import DefaultSocialAccountAdapter
|
||||
from api.db_router import MainRouter
|
||||
from api.db_utils import rls_transaction
|
||||
@@ -11,6 +13,7 @@ from api.models import (
|
||||
)
|
||||
from api.utils import accept_invitation_for_user
|
||||
from django.db import transaction
|
||||
from django.http import HttpResponseForbidden
|
||||
|
||||
|
||||
class ProwlerSocialAccountAdapter(DefaultSocialAccountAdapter):
|
||||
@@ -38,8 +41,13 @@ class ProwlerSocialAccountAdapter(DefaultSocialAccountAdapter):
|
||||
return None
|
||||
|
||||
def pre_social_login(self, request, sociallogin):
|
||||
# Link existing accounts with the same email address
|
||||
email = sociallogin.account.extra_data.get("email")
|
||||
# The provider account is already bound, so no email-based linking is needed.
|
||||
if sociallogin.account.pk:
|
||||
return
|
||||
|
||||
# Prefer the normalized email populated by allauth. GitHub can return the
|
||||
# primary email separately from the profile stored in extra_data.
|
||||
email = sociallogin.user.email or sociallogin.account.extra_data.get("email")
|
||||
if sociallogin.provider.id == "saml":
|
||||
# For SAML, the asserted NameID email cannot be trusted on its own:
|
||||
# any tenant can claim any email domain in its SAML configuration. To
|
||||
@@ -80,6 +88,17 @@ class ProwlerSocialAccountAdapter(DefaultSocialAccountAdapter):
|
||||
if email:
|
||||
existing_user = self.get_user_by_email(email)
|
||||
if existing_user:
|
||||
email_is_verified = EmailAddress.objects.filter(
|
||||
user=existing_user,
|
||||
email__iexact=email,
|
||||
verified=True,
|
||||
).exists()
|
||||
provider_verified_email = any(
|
||||
address.verified and address.email.casefold() == email.casefold()
|
||||
for address in sociallogin.email_addresses
|
||||
)
|
||||
if not email_is_verified or not provider_verified_email:
|
||||
raise ImmediateHttpResponse(HttpResponseForbidden())
|
||||
sociallogin.connect(request, existing_user)
|
||||
|
||||
def save_user(self, request, sociallogin, form=None):
|
||||
|
||||
@@ -27,6 +27,7 @@ from django.conf import (
|
||||
MAX_CUSTOM_QUERY_NODES = env.int("ATTACK_PATHS_MAX_CUSTOM_QUERY_NODES", default=250)
|
||||
|
||||
TEMP_DB_PREFIX = "db-tmp-scan-"
|
||||
DATABASE_NOT_FOUND_CODE = "Neo.ClientError.Database.DatabaseNotFound"
|
||||
|
||||
|
||||
# Exceptions
|
||||
@@ -44,6 +45,10 @@ class GraphDatabaseQueryException(Exception):
|
||||
return self.message
|
||||
|
||||
|
||||
class NeptuneWriteRetryExhaustedException(GraphDatabaseQueryException):
|
||||
pass
|
||||
|
||||
|
||||
class WriteQueryNotAllowedException(GraphDatabaseQueryException):
|
||||
pass
|
||||
|
||||
|
||||
@@ -10,6 +10,28 @@ import neo4j.exceptions
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class RetryExhaustedError(Exception):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
retry_context: str,
|
||||
method_name: str,
|
||||
attempts: int,
|
||||
elapsed_seconds: float,
|
||||
last_error: Exception,
|
||||
) -> None:
|
||||
self.retry_context = retry_context
|
||||
self.method_name = method_name
|
||||
self.attempts = attempts
|
||||
self.elapsed_seconds = elapsed_seconds
|
||||
self.last_error = last_error
|
||||
last_message = getattr(last_error, "message", None) or str(last_error)
|
||||
super().__init__(
|
||||
f"{retry_context} {method_name} failed after {attempts} attempts over "
|
||||
f"{elapsed_seconds:.3f}s. Last error: {last_message}"
|
||||
)
|
||||
|
||||
|
||||
class RetryableSession:
|
||||
"""Wrapper around ``neo4j.Session`` with a refreshable retry policy."""
|
||||
|
||||
@@ -19,11 +41,13 @@ class RetryableSession:
|
||||
max_retries: int,
|
||||
retry_if: Callable[[Exception], bool] | None = None,
|
||||
initial_retry_delay_seconds: float = 0,
|
||||
retry_context: str | None = None,
|
||||
) -> None:
|
||||
self._session_factory = session_factory
|
||||
self._max_retries = max(0, max_retries)
|
||||
self._retry_if = retry_if
|
||||
self._initial_retry_delay_seconds = max(0.0, initial_retry_delay_seconds)
|
||||
self._retry_context = retry_context
|
||||
self._session = self._session_factory()
|
||||
|
||||
def close(self) -> None:
|
||||
@@ -54,6 +78,7 @@ class RetryableSession:
|
||||
def _call_with_retry(self, method_name: str, *args: Any, **kwargs: Any) -> Any:
|
||||
attempt = 0
|
||||
last_exc: Exception | None = None
|
||||
started_at = time.monotonic()
|
||||
|
||||
while attempt <= self._max_retries:
|
||||
try:
|
||||
@@ -68,17 +93,38 @@ class RetryableSession:
|
||||
attempt += 1
|
||||
|
||||
if attempt > self._max_retries:
|
||||
if self._retry_context is not None:
|
||||
raise RetryExhaustedError(
|
||||
retry_context=self._retry_context,
|
||||
method_name=method_name,
|
||||
attempts=attempt,
|
||||
elapsed_seconds=time.monotonic() - started_at,
|
||||
last_error=exc,
|
||||
) from exc
|
||||
raise
|
||||
|
||||
delay = self._retry_delay(attempt)
|
||||
logger.warning(
|
||||
"Graph session %s failed with %s; retry %s/%s in %.3fs",
|
||||
method_name,
|
||||
type(exc).__name__,
|
||||
attempt,
|
||||
self._max_retries,
|
||||
delay,
|
||||
)
|
||||
if self._retry_context is not None:
|
||||
error_message = getattr(exc, "message", None) or str(exc)
|
||||
logger.warning(
|
||||
"%s %s failed with %s: %s; retry %s/%s in %.3fs",
|
||||
self._retry_context,
|
||||
method_name,
|
||||
type(exc).__name__,
|
||||
error_message,
|
||||
attempt,
|
||||
self._max_retries,
|
||||
delay,
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
"Graph session %s failed with %s; retry %s/%s in %.3fs",
|
||||
method_name,
|
||||
type(exc).__name__,
|
||||
attempt,
|
||||
self._max_retries,
|
||||
delay,
|
||||
)
|
||||
self._refresh_session()
|
||||
if delay:
|
||||
time.sleep(delay)
|
||||
|
||||
@@ -15,6 +15,8 @@ class SinkDatabase(Protocol):
|
||||
has a single graph, and isolation is label-based).
|
||||
"""
|
||||
|
||||
sync_batch_size: int
|
||||
|
||||
def init(self) -> None: ...
|
||||
|
||||
def close(self) -> None: ...
|
||||
|
||||
@@ -54,6 +54,8 @@ DATABASE_NOT_FOUND_CODE = "Neo.ClientError.Database.DatabaseNotFound"
|
||||
class Neo4jSink(SinkDatabase):
|
||||
"""Neo4j-backed sink. Multi-database cluster; tenant isolation is physical."""
|
||||
|
||||
sync_batch_size = env.int("ATTACK_PATHS_NEO4J_SYNC_BATCH_SIZE", default=1000)
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._driver: neo4j.Driver | None = None
|
||||
self._lock = threading.Lock()
|
||||
@@ -203,7 +205,7 @@ class Neo4jSink(SinkDatabase):
|
||||
"""
|
||||
from api.attack_paths.database import GraphDatabaseQueryException
|
||||
from tasks.jobs.attack_paths.config import (
|
||||
BATCH_SIZE,
|
||||
GRAPH_MUTATION_BATCH_SIZE,
|
||||
PROVIDER_RESOURCE_LABEL,
|
||||
get_provider_label,
|
||||
)
|
||||
@@ -251,7 +253,7 @@ class Neo4jSink(SinkDatabase):
|
||||
total_key="rels",
|
||||
deleted_key="deleted_rels",
|
||||
initial_total=deleted_relationships,
|
||||
batch_size=BATCH_SIZE,
|
||||
batch_size=GRAPH_MUTATION_BATCH_SIZE,
|
||||
drop_t0=drop_t0,
|
||||
)
|
||||
relationship_batches += phase_batches
|
||||
@@ -270,7 +272,7 @@ class Neo4jSink(SinkDatabase):
|
||||
total_key="nodes",
|
||||
deleted_key="deleted_nodes",
|
||||
initial_total=0,
|
||||
batch_size=BATCH_SIZE,
|
||||
batch_size=GRAPH_MUTATION_BATCH_SIZE,
|
||||
drop_t0=drop_t0,
|
||||
)
|
||||
|
||||
|
||||
@@ -25,7 +25,7 @@ from urllib.parse import urlsplit
|
||||
|
||||
import neo4j
|
||||
import neo4j.exceptions
|
||||
from api.attack_paths.retryable_session import RetryableSession
|
||||
from api.attack_paths.retryable_session import RetryableSession, RetryExhaustedError
|
||||
from api.attack_paths.sink.base import SinkDatabase
|
||||
from api.attack_paths.sink.drop import (
|
||||
NODE_DELETE_QUERY_TEMPLATE,
|
||||
@@ -66,7 +66,7 @@ READ_EXCEPTION_CODES = [
|
||||
"Neo.ClientError.Procedure.ProcedureNotFound",
|
||||
]
|
||||
CLIENT_STATEMENT_EXCEPTION_PREFIX = "Neo.ClientError.Statement."
|
||||
RETRYABLE_WRITE_ERROR_PREFIXES = (
|
||||
RETRYABLE_WRITE_ERROR_FRAGMENTS = (
|
||||
"Operation failed due to conflicting concurrent operations",
|
||||
"Operation terminated (deadline exceeded)",
|
||||
)
|
||||
@@ -78,12 +78,15 @@ SIGV4_TOKEN_LIFETIME_MINUTES = 4
|
||||
def _is_retryable_write_error(exc: Exception) -> bool:
|
||||
if not isinstance(exc, neo4j.exceptions.Neo4jError):
|
||||
return False
|
||||
return bool(exc.message and exc.message.startswith(RETRYABLE_WRITE_ERROR_PREFIXES))
|
||||
message = exc.message or ""
|
||||
return any(fragment in message for fragment in RETRYABLE_WRITE_ERROR_FRAGMENTS)
|
||||
|
||||
|
||||
class NeptuneSink(SinkDatabase):
|
||||
"""Neptune-backed sink. Single database; isolation is label-based."""
|
||||
|
||||
sync_batch_size = env.int("ATTACK_PATHS_NEPTUNE_SYNC_BATCH_SIZE", default=500)
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._writer: neo4j.Driver | None = None
|
||||
self._reader: neo4j.Driver | None = None
|
||||
@@ -205,6 +208,7 @@ class NeptuneSink(SinkDatabase):
|
||||
from api.attack_paths.database import (
|
||||
ClientStatementException,
|
||||
GraphDatabaseQueryException,
|
||||
NeptuneWriteRetryExhaustedException,
|
||||
WriteQueryNotAllowedException,
|
||||
)
|
||||
|
||||
@@ -226,9 +230,17 @@ class NeptuneSink(SinkDatabase):
|
||||
initial_retry_delay_seconds=(
|
||||
NEPTUNE_WRITE_RETRY_DELAY_SECONDS if is_write_session else 0
|
||||
),
|
||||
retry_context="Neptune write" if is_write_session else None,
|
||||
)
|
||||
yield session_wrapper
|
||||
|
||||
except RetryExhaustedError as exc:
|
||||
last_error = exc.last_error
|
||||
raise NeptuneWriteRetryExhaustedException(
|
||||
message=str(exc),
|
||||
code=getattr(last_error, "code", None),
|
||||
) from last_error
|
||||
|
||||
except neo4j.exceptions.Neo4jError as exc:
|
||||
if (
|
||||
default_access_mode == neo4j.READ_ACCESS
|
||||
@@ -290,7 +302,7 @@ class NeptuneSink(SinkDatabase):
|
||||
graph's branching factor.
|
||||
"""
|
||||
from tasks.jobs.attack_paths.config import (
|
||||
BATCH_SIZE,
|
||||
GRAPH_MUTATION_BATCH_SIZE,
|
||||
PROVIDER_RESOURCE_LABEL,
|
||||
get_provider_label,
|
||||
)
|
||||
@@ -329,7 +341,7 @@ class NeptuneSink(SinkDatabase):
|
||||
total_key="rels",
|
||||
deleted_key="deleted_rels",
|
||||
initial_total=deleted_relationships,
|
||||
batch_size=BATCH_SIZE,
|
||||
batch_size=GRAPH_MUTATION_BATCH_SIZE,
|
||||
drop_t0=drop_t0,
|
||||
)
|
||||
relationship_batches += phase_batches
|
||||
@@ -348,7 +360,7 @@ class NeptuneSink(SinkDatabase):
|
||||
total_key="nodes",
|
||||
deleted_key="deleted_nodes",
|
||||
initial_total=0,
|
||||
batch_size=BATCH_SIZE,
|
||||
batch_size=GRAPH_MUTATION_BATCH_SIZE,
|
||||
drop_t0=drop_t0,
|
||||
)
|
||||
|
||||
|
||||
@@ -152,10 +152,10 @@ def execute_custom_query(
|
||||
scan: AttackPathsScan,
|
||||
) -> dict[str, Any]:
|
||||
# Defense-in-depth for custom queries:
|
||||
# 1. `neo4j.READ_ACCESS` — prevents mutations at the driver level
|
||||
# 2. `inject_provider_label()` — regex-based label injection scopes node patterns
|
||||
# 3. `_serialize_graph()` — post-query filter drops nodes without the provider label
|
||||
# 4. `USING QUERY:TIMEOUTMILLISECONDS` on Neptune — server-side runaway cutoff
|
||||
# 1. `neo4j.READ_ACCESS` - prevents mutations at the driver level
|
||||
# 2. `inject_provider_label()` - regex-based label injection scopes node patterns
|
||||
# 3. `_serialize_graph()` - post-query filter drops nodes without the provider label
|
||||
# 4. `USING QUERY:TIMEOUTMILLISECONDS` on Neptune - server-side runaway cutoff
|
||||
#
|
||||
# Layer 2 is best-effort (regex can't fully parse Cypher);
|
||||
# layer 3 is the safety net that guarantees provider isolation.
|
||||
|
||||
+273
-50
@@ -2,7 +2,7 @@ import re
|
||||
import secrets
|
||||
import time
|
||||
import uuid
|
||||
from contextlib import contextmanager
|
||||
from contextlib import ExitStack, contextmanager, nullcontext
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from api.db_router import (
|
||||
@@ -48,6 +48,140 @@ REPLICA_MAX_ATTEMPTS = env.int("POSTGRES_REPLICA_MAX_ATTEMPTS", default=3)
|
||||
REPLICA_RETRY_BASE_DELAY = env.float("POSTGRES_REPLICA_RETRY_BASE_DELAY", default=0.5)
|
||||
|
||||
SET_CONFIG_QUERY = "SELECT set_config(%s, %s::text, TRUE);"
|
||||
SET_TRANSACTION_READ_ONLY_QUERY = "SET TRANSACTION READ ONLY;"
|
||||
|
||||
REPLICA_CONNECTION_SQLSTATE_PREFIXES = ("08",)
|
||||
REPLICA_CONNECTION_SQLSTATES = {"57P01", "57P02", "57P03"}
|
||||
REPLICA_NON_FAILOVER_SQLSTATES = {"57014", "40001", "40P01"}
|
||||
REPLICA_CONNECTION_ERROR_MESSAGES = (
|
||||
"ssl syscall",
|
||||
"eof detected",
|
||||
"server closed the connection",
|
||||
"connection already closed",
|
||||
"connection not open",
|
||||
"could not connect to server",
|
||||
"connection refused",
|
||||
"connection reset",
|
||||
"connection timed out",
|
||||
"lost synchronization",
|
||||
"terminating connection",
|
||||
"database system is starting up",
|
||||
"database system is shutting down",
|
||||
"database system is in recovery mode",
|
||||
)
|
||||
REPLICA_NON_FAILOVER_ERROR_MESSAGES = (
|
||||
"canceling statement due to user request",
|
||||
"deadlock detected",
|
||||
"could not serialize access",
|
||||
)
|
||||
|
||||
|
||||
def _iter_exception_chain(error: BaseException):
|
||||
seen = set()
|
||||
pending = [error]
|
||||
while pending:
|
||||
current = pending.pop(0)
|
||||
if current is None or id(current) in seen:
|
||||
continue
|
||||
seen.add(id(current))
|
||||
yield current
|
||||
|
||||
cause = getattr(current, "__cause__", None)
|
||||
context = getattr(current, "__context__", None)
|
||||
if cause is not None:
|
||||
pending.append(cause)
|
||||
if context is not None:
|
||||
pending.append(context)
|
||||
for arg in getattr(current, "args", ()):
|
||||
if isinstance(arg, BaseException):
|
||||
pending.append(arg)
|
||||
|
||||
|
||||
def _get_exception_sqlstate(error: BaseException) -> str | None:
|
||||
for attr in ("pgcode", "sqlstate"):
|
||||
sqlstate = getattr(error, attr, None)
|
||||
if sqlstate:
|
||||
return sqlstate
|
||||
|
||||
diag = getattr(error, "diag", None)
|
||||
if diag is not None:
|
||||
sqlstate = getattr(diag, "sqlstate", None)
|
||||
if sqlstate:
|
||||
return sqlstate
|
||||
return None
|
||||
|
||||
|
||||
def _is_replica_connection_failure(error: BaseException) -> bool:
|
||||
"""
|
||||
Return True only for replica failures where retrying on primary is safe.
|
||||
|
||||
Query cancellations, serialization failures, and deadlocks should surface to
|
||||
callers because replaying them can hide real query or concurrency problems.
|
||||
"""
|
||||
messages = []
|
||||
sqlstates = set()
|
||||
|
||||
for chained_error in _iter_exception_chain(error):
|
||||
sqlstate = _get_exception_sqlstate(chained_error)
|
||||
if sqlstate:
|
||||
sqlstates.add(sqlstate)
|
||||
messages.append(str(chained_error).lower())
|
||||
|
||||
if sqlstates & REPLICA_NON_FAILOVER_SQLSTATES:
|
||||
return False
|
||||
if any(
|
||||
sqlstate.startswith(REPLICA_CONNECTION_SQLSTATE_PREFIXES)
|
||||
or sqlstate in REPLICA_CONNECTION_SQLSTATES
|
||||
for sqlstate in sqlstates
|
||||
):
|
||||
return True
|
||||
|
||||
message = " ".join(messages)
|
||||
if any(marker in message for marker in REPLICA_NON_FAILOVER_ERROR_MESSAGES):
|
||||
return False
|
||||
|
||||
return any(marker in message for marker in REPLICA_CONNECTION_ERROR_MESSAGES)
|
||||
|
||||
|
||||
def _strip_leading_sql_comments(sql: str) -> str:
|
||||
if not isinstance(sql, str):
|
||||
return ""
|
||||
|
||||
sql_text = sql.lstrip()
|
||||
while True:
|
||||
if sql_text.startswith("--"):
|
||||
newline_index = sql_text.find("\n")
|
||||
if newline_index == -1:
|
||||
return ""
|
||||
sql_text = sql_text[newline_index + 1 :].lstrip()
|
||||
continue
|
||||
|
||||
if sql_text.startswith("/*"):
|
||||
comment_end_index = sql_text.find("*/", 2)
|
||||
if comment_end_index == -1:
|
||||
return ""
|
||||
sql_text = sql_text[comment_end_index + 2 :].lstrip()
|
||||
continue
|
||||
|
||||
return sql_text
|
||||
|
||||
|
||||
def _is_safe_primary_replay(sql: str, many: bool) -> bool:
|
||||
if many:
|
||||
return False
|
||||
|
||||
sql_text = _strip_leading_sql_comments(sql)
|
||||
if not re.match(r"(?is)^SELECT\b", sql_text):
|
||||
return False
|
||||
|
||||
return not any(
|
||||
re.search(pattern, sql_text, re.IGNORECASE | re.DOTALL)
|
||||
for pattern in (
|
||||
r"\bINTO\b",
|
||||
r"\bFOR\s+(?:NO\s+KEY\s+)?UPDATE\b",
|
||||
r"\bFOR\s+(?:KEY\s+)?SHARE\b",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@contextmanager
|
||||
@@ -77,14 +211,36 @@ def rls_transaction(
|
||||
retry_on_replica: bool = True,
|
||||
):
|
||||
"""
|
||||
Creates a new database transaction setting the given configuration value for Postgres RLS. It validates the
|
||||
if the value is a valid UUID.
|
||||
Context manager that opens an RLS-scoped database transaction.
|
||||
|
||||
Sets a Postgres configuration variable (``set_config``) so that Row-Level
|
||||
Security policies can filter by tenant. When *using* points to a read
|
||||
replica and *retry_on_replica* is True, replica failures are handled in two
|
||||
places:
|
||||
|
||||
1. **Pre-yield** (connection-setup failures): the function retries
|
||||
up to ``REPLICA_MAX_ATTEMPTS`` times on the replica, then falls
|
||||
back to the primary DB.
|
||||
2. **Post-yield** (mid-query failures): an ``execute_wrapper``
|
||||
intercepts connection-level ``OperationalError`` during
|
||||
``cursor.execute()`` calls and falls back directly to the primary DB
|
||||
for single ``SELECT`` statements. The primary fallback transaction is
|
||||
read-only, and unsafe statements keep raising the original error.
|
||||
The wrapper swaps the inner cursor so ``fetchall()`` / ``fetchone()``
|
||||
read from the new connection transparently.
|
||||
|
||||
Limitation: server-side cursors (``.iterator()``) fetch rows via
|
||||
``fetchmany()``, which the wrapper does not intercept. Call sites
|
||||
that iterate large result sets with ``.iterator()`` on the replica
|
||||
should add their own retry logic.
|
||||
|
||||
Args:
|
||||
value (str): Database configuration parameter value.
|
||||
parameter (str): Database configuration parameter name, by default is 'api.tenant_id'.
|
||||
using (str | None): Optional database alias to run the transaction against. Defaults to the
|
||||
active read alias (if any) or Django's default connection.
|
||||
value: Database configuration parameter value (must be a valid UUID).
|
||||
parameter: Database configuration parameter name.
|
||||
using: Optional database alias. Defaults to the active read
|
||||
alias or Django's default connection.
|
||||
retry_on_replica: Whether replica setup failures can retry and
|
||||
connection-level mid-query failures can fall back to primary.
|
||||
"""
|
||||
requested_alias = using or get_read_db_alias()
|
||||
db_alias = requested_alias or DEFAULT_DB_ALIAS
|
||||
@@ -92,54 +248,121 @@ def rls_transaction(
|
||||
db_alias = DEFAULT_DB_ALIAS
|
||||
|
||||
alias = db_alias
|
||||
is_replica = READ_REPLICA_ALIAS and alias == READ_REPLICA_ALIAS
|
||||
max_attempts = REPLICA_MAX_ATTEMPTS if is_replica and retry_on_replica else 1
|
||||
is_replica = bool(READ_REPLICA_ALIAS and alias == READ_REPLICA_ALIAS)
|
||||
can_failover = is_replica and retry_on_replica
|
||||
replica_alias = alias # captured before the loop mutates alias
|
||||
max_attempts = (REPLICA_MAX_ATTEMPTS + 1) if can_failover else 1
|
||||
|
||||
for attempt in range(1, max_attempts + 1):
|
||||
router_token = None
|
||||
yielded_cursor = False
|
||||
# State shared between the generator and the _query_failover closure.
|
||||
# The fallback transaction.atomic() is registered into fallback_stack
|
||||
# via enter_context so its __exit__ runs when the outer with-ExitStack
|
||||
# block exits, with the right exc_info. No manual __enter__/__exit__.
|
||||
_fallback = {"succeeded": False, "token": None, "caller_exited_cleanly": False}
|
||||
|
||||
# On final attempt, fallback to primary
|
||||
if attempt == max_attempts and is_replica:
|
||||
logger.warning(
|
||||
f"RLS transaction failed after {attempt - 1} attempts on replica, "
|
||||
f"falling back to primary DB"
|
||||
)
|
||||
alias = DEFAULT_DB_ALIAS
|
||||
with ExitStack() as fallback_stack:
|
||||
|
||||
conn = connections[alias]
|
||||
try:
|
||||
if alias != DEFAULT_DB_ALIAS:
|
||||
router_token = set_read_db_alias(alias)
|
||||
def _query_failover(execute, sql, params, many, context):
|
||||
"""execute_wrapper: replay failed replica queries on the primary DB."""
|
||||
try:
|
||||
return execute(sql, params, many, context)
|
||||
except OperationalError as err:
|
||||
if not _is_replica_connection_failure(err):
|
||||
raise
|
||||
if not _is_safe_primary_replay(sql, many):
|
||||
raise
|
||||
|
||||
with transaction.atomic(using=alias):
|
||||
with conn.cursor() as cursor:
|
||||
try:
|
||||
# just in case the value is a UUID object
|
||||
uuid.UUID(str(value))
|
||||
except ValueError:
|
||||
raise ValidationError("Must be a valid UUID")
|
||||
cursor.execute(SET_CONFIG_QUERY, [parameter, value])
|
||||
yielded_cursor = True
|
||||
yield cursor
|
||||
return
|
||||
except OperationalError as e:
|
||||
if yielded_cursor:
|
||||
raise
|
||||
# If on primary or max attempts reached, raise
|
||||
if not is_replica or attempt == max_attempts:
|
||||
raise
|
||||
try:
|
||||
connections[replica_alias].close()
|
||||
except Exception:
|
||||
pass # Best-effort; connection may already be dead
|
||||
|
||||
# Retry with exponential backoff
|
||||
delay = REPLICA_RETRY_BASE_DELAY * (2 ** (attempt - 1))
|
||||
logger.info(
|
||||
f"RLS transaction failed on replica (attempt {attempt}/{max_attempts}), "
|
||||
f"retrying in {delay}s. Error: {e}"
|
||||
)
|
||||
time.sleep(delay)
|
||||
finally:
|
||||
if router_token is not None:
|
||||
reset_read_db_alias(router_token)
|
||||
logger.warning(
|
||||
"Mid-query replica connection failure, falling back to primary DB"
|
||||
)
|
||||
primary = connections[DEFAULT_DB_ALIAS]
|
||||
primary.ensure_connection()
|
||||
fallback_stack.enter_context(transaction.atomic(using=DEFAULT_DB_ALIAS))
|
||||
|
||||
fallback_cursor = primary.cursor()
|
||||
fallback_stack.callback(fallback_cursor.close)
|
||||
fallback_cursor.execute(SET_TRANSACTION_READ_ONLY_QUERY)
|
||||
fallback_cursor.execute(SET_CONFIG_QUERY, [parameter, value])
|
||||
_fallback["token"] = set_read_db_alias(DEFAULT_DB_ALIAS)
|
||||
|
||||
fallback_cursor.execute(sql, params)
|
||||
|
||||
context["cursor"].db = primary
|
||||
context["cursor"].cursor = fallback_cursor.cursor
|
||||
_fallback["succeeded"] = True
|
||||
return None
|
||||
|
||||
for attempt in range(1, max_attempts + 1):
|
||||
router_token = None
|
||||
yielded_cursor = False
|
||||
|
||||
# On final attempt, fall back to primary
|
||||
if attempt == max_attempts and can_failover:
|
||||
if attempt > 1:
|
||||
logger.warning(
|
||||
f"RLS transaction failed after {attempt - 1} attempts on replica, "
|
||||
f"falling back to primary DB"
|
||||
)
|
||||
alias = DEFAULT_DB_ALIAS
|
||||
|
||||
conn = connections[alias]
|
||||
try:
|
||||
if alias != DEFAULT_DB_ALIAS:
|
||||
router_token = set_read_db_alias(alias)
|
||||
|
||||
with transaction.atomic(using=alias):
|
||||
with conn.cursor() as cursor:
|
||||
try:
|
||||
uuid.UUID(str(value))
|
||||
except ValueError:
|
||||
raise ValidationError("Must be a valid UUID")
|
||||
cursor.execute(SET_CONFIG_QUERY, [parameter, value])
|
||||
|
||||
wrapper_cm = (
|
||||
conn.execute_wrapper(_query_failover)
|
||||
if can_failover and alias == replica_alias
|
||||
else nullcontext()
|
||||
)
|
||||
with wrapper_cm:
|
||||
yielded_cursor = True
|
||||
yield cursor
|
||||
_fallback["caller_exited_cleanly"] = True
|
||||
return
|
||||
except OperationalError as e:
|
||||
if yielded_cursor:
|
||||
if _fallback["succeeded"] and _fallback["caller_exited_cleanly"]:
|
||||
# Caller's queries succeeded on primary via failover.
|
||||
# This error is transaction.atomic() cleanup on the
|
||||
# dead replica connection, suppress it.
|
||||
return
|
||||
raise
|
||||
|
||||
if not can_failover or attempt == max_attempts:
|
||||
raise
|
||||
|
||||
try:
|
||||
connections[alias].close()
|
||||
except Exception:
|
||||
pass # Best-effort; connection may already be dead
|
||||
|
||||
# Retry with exponential backoff
|
||||
delay = REPLICA_RETRY_BASE_DELAY * (2 ** (attempt - 1))
|
||||
logger.info(
|
||||
f"RLS transaction failed on replica (attempt {attempt}/{max_attempts}), "
|
||||
f"retrying in {delay}s. Error: {e}"
|
||||
)
|
||||
time.sleep(delay)
|
||||
finally:
|
||||
if _fallback["token"] is not None:
|
||||
reset_read_db_alias(_fallback["token"])
|
||||
_fallback["token"] = None
|
||||
|
||||
if router_token is not None:
|
||||
reset_read_db_alias(router_token)
|
||||
|
||||
|
||||
class CustomUserManager(BaseUserManager):
|
||||
|
||||
@@ -1,12 +1,13 @@
|
||||
import uuid
|
||||
from functools import wraps
|
||||
|
||||
from api.attack_paths.database import GraphDatabaseQueryException
|
||||
from api.db_router import READ_REPLICA_ALIAS
|
||||
from api.db_utils import POSTGRES_TENANT_VAR, SET_CONFIG_QUERY, rls_transaction
|
||||
from api.exceptions import ProviderDeletedException
|
||||
from api.models import Provider, Scan
|
||||
from api.models import Membership, Provider, Scan, Tenant
|
||||
from django.core.exceptions import ObjectDoesNotExist
|
||||
from django.db import DatabaseError, connection, transaction
|
||||
from django.db import DEFAULT_DB_ALIAS, DatabaseError, connection, transaction
|
||||
from rest_framework_json_api.serializers import ValidationError
|
||||
|
||||
|
||||
@@ -75,9 +76,11 @@ def handle_provider_deletion(func):
|
||||
"""
|
||||
Decorator that raises `ProviderDeletedException` if provider was deleted during execution.
|
||||
|
||||
Catches `ObjectDoesNotExist` and `DatabaseError` (including `IntegrityError`), checks if
|
||||
provider still exists, and raises `ProviderDeletedException` if not. Otherwise,
|
||||
re-raises original exception.
|
||||
Catches `ObjectDoesNotExist`, `DatabaseError` (including `IntegrityError`), and
|
||||
`GraphDatabaseQueryException`, checks if provider still exists, and raises
|
||||
`ProviderDeletedException` if not. Graph database errors also check whether the
|
||||
tenant still exists and has memberships. Otherwise, re-raises the original
|
||||
exception.
|
||||
|
||||
Requires `tenant_id` and `provider_id` in kwargs.
|
||||
|
||||
@@ -92,11 +95,16 @@ def handle_provider_deletion(func):
|
||||
def wrapper(*args, **kwargs):
|
||||
try:
|
||||
return func(*args, **kwargs)
|
||||
except (ObjectDoesNotExist, DatabaseError):
|
||||
except (ObjectDoesNotExist, DatabaseError, GraphDatabaseQueryException) as exc:
|
||||
tenant_id = kwargs.get("tenant_id")
|
||||
provider_id = kwargs.get("provider_id")
|
||||
database_alias = (
|
||||
DEFAULT_DB_ALIAS
|
||||
if isinstance(exc, GraphDatabaseQueryException)
|
||||
else READ_REPLICA_ALIAS
|
||||
)
|
||||
|
||||
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
|
||||
with rls_transaction(tenant_id, using=database_alias):
|
||||
if provider_id is None:
|
||||
scan_id = kwargs.get("scan_id")
|
||||
if scan_id is None:
|
||||
@@ -113,6 +121,13 @@ def handle_provider_deletion(func):
|
||||
raise ProviderDeletedException(
|
||||
f"Provider '{provider_id}' was deleted during the scan"
|
||||
) from None
|
||||
if isinstance(exc, GraphDatabaseQueryException) and (
|
||||
not Tenant.objects.filter(pk=tenant_id).exists()
|
||||
or not Membership.objects.filter(tenant_id=tenant_id).exists()
|
||||
):
|
||||
raise ProviderDeletedException(
|
||||
f"Tenant '{tenant_id}' was deleted during the scan"
|
||||
) from None
|
||||
raise
|
||||
|
||||
return wrapper
|
||||
|
||||
@@ -34,7 +34,7 @@ class HasPermissions(BasePermission):
|
||||
if not tenant_id:
|
||||
return False
|
||||
|
||||
user_roles = (
|
||||
user_roles = list(
|
||||
User.objects.using(MainRouter.admin_db)
|
||||
.get(id=request.user.id)
|
||||
.roles.using(MainRouter.admin_db)
|
||||
@@ -43,11 +43,10 @@ class HasPermissions(BasePermission):
|
||||
if not user_roles:
|
||||
return False
|
||||
|
||||
for perm in required_permissions:
|
||||
if not getattr(user_roles[0], perm.value, False):
|
||||
return False
|
||||
|
||||
return True
|
||||
return all(
|
||||
any(getattr(role, permission.value, False) for role in user_roles)
|
||||
for permission in required_permissions
|
||||
)
|
||||
|
||||
|
||||
def get_role(user: User, tenant_id: str) -> Role:
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
openapi: 3.0.3
|
||||
info:
|
||||
title: Prowler API
|
||||
version: 1.34.2
|
||||
version: 1.36.0
|
||||
description: |-
|
||||
Prowler API specification.
|
||||
|
||||
|
||||
@@ -14,6 +14,19 @@ from rest_framework_simplejwt.token_blacklist.models import (
|
||||
OutstandingToken,
|
||||
)
|
||||
|
||||
PASSWORD_CHANGE_PASSWORD = "InitialSecret123@"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def password_change_user(tenants_fixture):
|
||||
user = User.objects.create_user(
|
||||
name="password_change_user",
|
||||
email=f"password-change-{uuid4()}@prowler.com",
|
||||
password=PASSWORD_CHANGE_PASSWORD,
|
||||
)
|
||||
Membership.objects.create(user=user, tenant=tenants_fixture[0])
|
||||
return user
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_basic_authentication():
|
||||
@@ -109,16 +122,16 @@ def test_refresh_token(create_test_user, tenants_fixture):
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_password_change_invalidates_existing_tokens(create_test_user, tenants_fixture):
|
||||
def test_password_change_invalidates_existing_tokens(password_change_user):
|
||||
client = APIClient()
|
||||
new_password = "ChangedSecret123@"
|
||||
|
||||
access_token, refresh_token = get_api_tokens(
|
||||
client, create_test_user.email, TEST_PASSWORD
|
||||
client, password_change_user.email, PASSWORD_CHANGE_PASSWORD
|
||||
)
|
||||
auth_headers = get_authorization_header(access_token)
|
||||
outstanding_token_ids = list(
|
||||
OutstandingToken.objects.filter(user=create_test_user).values_list(
|
||||
OutstandingToken.objects.filter(user=password_change_user).values_list(
|
||||
"id", flat=True
|
||||
)
|
||||
)
|
||||
@@ -130,12 +143,12 @@ def test_password_change_invalidates_existing_tokens(create_test_user, tenants_f
|
||||
password_change_payload = {
|
||||
"data": {
|
||||
"type": "users",
|
||||
"id": str(create_test_user.id),
|
||||
"id": str(password_change_user.id),
|
||||
"attributes": {"password": new_password},
|
||||
}
|
||||
}
|
||||
password_change_response = client.patch(
|
||||
reverse("user-detail", kwargs={"pk": create_test_user.id}),
|
||||
reverse("user-detail", kwargs={"pk": password_change_user.id}),
|
||||
data=json.dumps(password_change_payload),
|
||||
headers=auth_headers,
|
||||
content_type="application/vnd.api+json",
|
||||
@@ -160,7 +173,9 @@ def test_password_change_invalidates_existing_tokens(create_test_user, tenants_f
|
||||
)
|
||||
assert old_refresh_response.status_code == 400
|
||||
|
||||
new_access_token, _ = get_api_tokens(client, create_test_user.email, new_password)
|
||||
new_access_token, _ = get_api_tokens(
|
||||
client, password_change_user.email, new_password
|
||||
)
|
||||
new_access_response = client.get(
|
||||
reverse("user-me"), headers=get_authorization_header(new_access_token)
|
||||
)
|
||||
@@ -169,13 +184,13 @@ def test_password_change_invalidates_existing_tokens(create_test_user, tenants_f
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_password_change_invalidates_rotated_refresh_token(
|
||||
create_test_user, tenants_fixture
|
||||
password_change_user,
|
||||
):
|
||||
client = APIClient()
|
||||
new_password = "ChangedSecret123@"
|
||||
|
||||
access_token, refresh_token = get_api_tokens(
|
||||
client, create_test_user.email, TEST_PASSWORD
|
||||
client, password_change_user.email, PASSWORD_CHANGE_PASSWORD
|
||||
)
|
||||
rotated_refresh_response = client.post(
|
||||
reverse("token-refresh"),
|
||||
@@ -195,12 +210,12 @@ def test_password_change_invalidates_rotated_refresh_token(
|
||||
password_change_payload = {
|
||||
"data": {
|
||||
"type": "users",
|
||||
"id": str(create_test_user.id),
|
||||
"id": str(password_change_user.id),
|
||||
"attributes": {"password": new_password},
|
||||
}
|
||||
}
|
||||
password_change_response = client.patch(
|
||||
reverse("user-detail", kwargs={"pk": create_test_user.id}),
|
||||
reverse("user-detail", kwargs={"pk": password_change_user.id}),
|
||||
data=json.dumps(password_change_payload),
|
||||
headers=get_authorization_header(access_token),
|
||||
content_type="application/vnd.api+json",
|
||||
@@ -304,9 +319,8 @@ def test_user_me_when_inviting_users(create_test_user, tenants_fixture, roles_fi
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestTokenSwitchTenant:
|
||||
def test_switch_tenant_with_valid_token(self, tenants_fixture, providers_fixture):
|
||||
def test_switch_tenant_with_valid_token(self, tenants_fixture, aws_provider):
|
||||
client = APIClient()
|
||||
aws_provider = providers_fixture[0]
|
||||
assert aws_provider
|
||||
|
||||
test_user = "test_email@prowler.com"
|
||||
@@ -1515,14 +1529,13 @@ class TestAPIKeyMultiTenantWorkflows:
|
||||
assert me_response2.json()["data"]["id"] == str(user.id)
|
||||
|
||||
def test_api_key_cannot_access_different_tenant_resources(
|
||||
self, tenants_fixture, providers_fixture
|
||||
self, tenants_fixture, aws_provider
|
||||
):
|
||||
"""API key from one tenant cannot access resources from another tenant.
|
||||
|
||||
Verifies RLS enforcement after authentication ensures tenant isolation.
|
||||
"""
|
||||
client = APIClient()
|
||||
aws_provider = providers_fixture[0]
|
||||
assert aws_provider
|
||||
|
||||
user1 = User.objects.create_user(
|
||||
|
||||
@@ -1,8 +1,12 @@
|
||||
"""Tests for rls_transaction retry and fallback logic."""
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from api.db_utils import rls_transaction
|
||||
from django.db import DEFAULT_DB_ALIAS
|
||||
from api.db_utils import POSTGRES_TENANT_VAR, rls_transaction
|
||||
from conftest import TEST_REPLICA_ALIAS
|
||||
from django.db import DEFAULT_DB_ALIAS, OperationalError, connections
|
||||
from psycopg2 import OperationalError as Psycopg2OperationalError
|
||||
from rest_framework_json_api.serializers import ValidationError
|
||||
|
||||
|
||||
@@ -36,3 +40,35 @@ class TestRLSTransaction:
|
||||
cursor.execute("SELECT current_setting(%s, true)", [custom_param])
|
||||
result = cursor.fetchone()
|
||||
assert result == (str(tenant.id),)
|
||||
|
||||
@pytest.mark.requires_test_replica_alias
|
||||
@pytest.mark.django_db(
|
||||
transaction=True, databases=[DEFAULT_DB_ALIAS, TEST_REPLICA_ALIAS]
|
||||
)
|
||||
def test_mid_query_replica_connection_loss_falls_back_to_primary(self, tenant):
|
||||
"""Real Django connection state: closed replica atomic falls back to primary."""
|
||||
replica = connections[TEST_REPLICA_ALIAS]
|
||||
sql = "SELECT current_setting(%s, true), %s"
|
||||
params = [POSTGRES_TENANT_VAR, 42]
|
||||
failed_once = {"value": False}
|
||||
|
||||
def close_replica_and_raise(execute, sql_arg, params_arg, many, context):
|
||||
if not failed_once["value"] and sql_arg == sql:
|
||||
failed_once["value"] = True
|
||||
replica.close()
|
||||
try:
|
||||
raise Psycopg2OperationalError("SSL SYSCALL error: EOF detected")
|
||||
except Psycopg2OperationalError as psycopg_error:
|
||||
raise OperationalError(
|
||||
"SSL SYSCALL error: EOF detected"
|
||||
) from psycopg_error
|
||||
return execute(sql_arg, params_arg, many, context)
|
||||
|
||||
with patch("api.db_utils.READ_REPLICA_ALIAS", TEST_REPLICA_ALIAS):
|
||||
with rls_transaction(str(tenant.id), using=TEST_REPLICA_ALIAS) as cursor:
|
||||
with replica.execute_wrapper(close_replica_and_raise):
|
||||
cursor.execute(sql, params)
|
||||
result = cursor.fetchone()
|
||||
|
||||
assert failed_once["value"]
|
||||
assert result == (str(tenant.id), 42)
|
||||
|
||||
@@ -2,11 +2,18 @@ from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from allauth.socialaccount.models import SocialLogin
|
||||
from allauth.account import app_settings as account_app_settings
|
||||
from allauth.account.models import EmailAddress
|
||||
from allauth.core import context
|
||||
from allauth.core.exceptions import ImmediateHttpResponse
|
||||
from allauth.socialaccount import app_settings as socialaccount_app_settings
|
||||
from allauth.socialaccount.internal.flows.login import complete_login
|
||||
from allauth.socialaccount.models import SocialAccount, SocialLogin
|
||||
from api.adapters import ProwlerSocialAccountAdapter
|
||||
from api.db_router import MainRouter
|
||||
from api.models import Invitation, Membership, SAMLConfiguration, Tenant
|
||||
from django.contrib.auth import get_user_model
|
||||
from django.core import mail
|
||||
|
||||
User = get_user_model()
|
||||
|
||||
@@ -40,6 +47,7 @@ def _saml_request(rf, organization_slug):
|
||||
def _saml_sociallogin(user):
|
||||
sociallogin = MagicMock(spec=SocialLogin)
|
||||
sociallogin.account = MagicMock()
|
||||
sociallogin.account.pk = None
|
||||
sociallogin.provider = MagicMock()
|
||||
sociallogin.provider.id = "saml"
|
||||
sociallogin.account.extra_data = {}
|
||||
@@ -48,6 +56,59 @@ def _saml_sociallogin(user):
|
||||
return sociallogin
|
||||
|
||||
|
||||
def _oauth_sociallogin(
|
||||
user,
|
||||
*,
|
||||
provider="google",
|
||||
provider_email_verified=True,
|
||||
include_extra_email=True,
|
||||
):
|
||||
sociallogin = MagicMock(spec=SocialLogin)
|
||||
sociallogin.account = MagicMock()
|
||||
sociallogin.account.pk = None
|
||||
sociallogin.provider = MagicMock()
|
||||
sociallogin.provider.id = provider
|
||||
sociallogin.account.extra_data = (
|
||||
{"email": user.email} if include_extra_email else {}
|
||||
)
|
||||
sociallogin.email_addresses = [
|
||||
EmailAddress(
|
||||
email=user.email,
|
||||
verified=provider_email_verified,
|
||||
primary=True,
|
||||
)
|
||||
]
|
||||
sociallogin.user = user
|
||||
sociallogin.connect = MagicMock()
|
||||
return sociallogin
|
||||
|
||||
|
||||
def _real_oauth_sociallogin(user, uid):
|
||||
provider = MagicMock()
|
||||
provider.id = "google"
|
||||
provider.app = None
|
||||
provider.get_settings.return_value = {}
|
||||
return SocialLogin(
|
||||
user=user,
|
||||
account=SocialAccount(
|
||||
provider="google",
|
||||
uid=uid,
|
||||
extra_data={"email": user.email},
|
||||
),
|
||||
email_addresses=[EmailAddress(email=user.email, verified=True, primary=True)],
|
||||
provider=provider,
|
||||
)
|
||||
|
||||
|
||||
def _verify_local_email(user):
|
||||
return EmailAddress.objects.create(
|
||||
user=user,
|
||||
email=user.email,
|
||||
verified=True,
|
||||
primary=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestProwlerSocialAccountAdapter:
|
||||
def test_get_user_by_email_returns_user(self, create_test_user):
|
||||
@@ -157,6 +218,7 @@ class TestProwlerSocialAccountAdapter:
|
||||
|
||||
sociallogin = MagicMock(spec=SocialLogin)
|
||||
sociallogin.account = MagicMock()
|
||||
sociallogin.account.pk = None
|
||||
sociallogin.provider = MagicMock()
|
||||
sociallogin.user = MagicMock()
|
||||
sociallogin.user.email = ""
|
||||
@@ -168,25 +230,119 @@ class TestProwlerSocialAccountAdapter:
|
||||
|
||||
sociallogin.connect.assert_not_called()
|
||||
|
||||
def test_pre_social_login_non_saml_links_by_email(self, create_test_user, rf):
|
||||
"""Non-SAML providers (e.g. Google/GitHub) still link to an existing
|
||||
local account by email; the tenant binding only applies to SAML."""
|
||||
def test_pre_social_login_blocks_unverified_local_email(self, create_test_user, rf):
|
||||
"""A verified OAuth email must not claim an unverified local account."""
|
||||
adapter = ProwlerSocialAccountAdapter()
|
||||
sociallogin = _oauth_sociallogin(create_test_user)
|
||||
|
||||
sociallogin = MagicMock(spec=SocialLogin)
|
||||
sociallogin.account = MagicMock()
|
||||
sociallogin.provider = MagicMock()
|
||||
sociallogin.provider.id = "google"
|
||||
sociallogin.account.extra_data = {"email": create_test_user.email}
|
||||
sociallogin.user = create_test_user
|
||||
sociallogin.connect = MagicMock()
|
||||
with pytest.raises(ImmediateHttpResponse) as exc_info:
|
||||
adapter.pre_social_login(rf.get("/"), sociallogin)
|
||||
|
||||
assert exc_info.value.response.status_code == 403
|
||||
sociallogin.connect.assert_not_called()
|
||||
|
||||
def test_complete_oauth_login_does_not_link_unverified_local_email(
|
||||
self, create_test_user, rf
|
||||
):
|
||||
"""Regression test for the complete pre-hijack account-linking flow."""
|
||||
incoming_user = User(email=create_test_user.email)
|
||||
incoming_user.set_unusable_password()
|
||||
sociallogin = _real_oauth_sociallogin(
|
||||
incoming_user,
|
||||
uid="victim-google-account",
|
||||
)
|
||||
request = rf.get("/")
|
||||
request.session = {}
|
||||
|
||||
with pytest.raises(ImmediateHttpResponse) as exc_info:
|
||||
complete_login(request, sociallogin, raises=True)
|
||||
|
||||
assert exc_info.value.response.status_code == 403
|
||||
assert not SocialAccount.objects.filter(
|
||||
provider="google", uid="victim-google-account"
|
||||
).exists()
|
||||
|
||||
def test_pre_social_login_allows_already_connected_account(
|
||||
self, create_test_user, rf
|
||||
):
|
||||
"""Existing provider bindings do not need to relink on every login."""
|
||||
adapter = ProwlerSocialAccountAdapter()
|
||||
sociallogin = _oauth_sociallogin(create_test_user)
|
||||
sociallogin.account.pk = "existing-social-account"
|
||||
|
||||
adapter.pre_social_login(rf.get("/"), sociallogin)
|
||||
|
||||
call_args = sociallogin.connect.call_args
|
||||
assert call_args is not None
|
||||
_, called_user = call_args[0]
|
||||
assert called_user.email == create_test_user.email
|
||||
sociallogin.connect.assert_not_called()
|
||||
|
||||
def test_pre_social_login_blocks_unverified_provider_email(
|
||||
self, create_test_user, rf
|
||||
):
|
||||
"""An OAuth provider must prove ownership of the matching email."""
|
||||
_verify_local_email(create_test_user)
|
||||
adapter = ProwlerSocialAccountAdapter()
|
||||
sociallogin = _oauth_sociallogin(
|
||||
create_test_user,
|
||||
provider="github",
|
||||
provider_email_verified=False,
|
||||
)
|
||||
|
||||
with pytest.raises(ImmediateHttpResponse) as exc_info:
|
||||
adapter.pre_social_login(rf.get("/"), sociallogin)
|
||||
|
||||
assert exc_info.value.response.status_code == 403
|
||||
sociallogin.connect.assert_not_called()
|
||||
|
||||
def test_pre_social_login_links_verified_emails(self, create_test_user, rf):
|
||||
_verify_local_email(create_test_user)
|
||||
adapter = ProwlerSocialAccountAdapter()
|
||||
sociallogin = _oauth_sociallogin(create_test_user)
|
||||
request = rf.get("/")
|
||||
|
||||
adapter.pre_social_login(request, sociallogin)
|
||||
|
||||
sociallogin.connect.assert_called_once_with(request, create_test_user)
|
||||
|
||||
def test_verified_social_account_link_does_not_send_notification(
|
||||
self, create_test_user, rf
|
||||
):
|
||||
_verify_local_email(create_test_user)
|
||||
sociallogin = _real_oauth_sociallogin(
|
||||
create_test_user,
|
||||
uid="verified-google-account",
|
||||
)
|
||||
|
||||
request = rf.get("/")
|
||||
with context.request_context(request):
|
||||
ProwlerSocialAccountAdapter().pre_social_login(request, sociallogin)
|
||||
|
||||
assert SocialAccount.objects.filter(
|
||||
provider="google",
|
||||
uid="verified-google-account",
|
||||
user=create_test_user,
|
||||
).exists()
|
||||
assert mail.outbox == []
|
||||
|
||||
def test_pre_social_login_uses_verified_email_missing_from_extra_data(
|
||||
self, create_test_user, rf
|
||||
):
|
||||
"""GitHub can return its verified primary email outside extra_data."""
|
||||
_verify_local_email(create_test_user)
|
||||
adapter = ProwlerSocialAccountAdapter()
|
||||
sociallogin = _oauth_sociallogin(
|
||||
create_test_user,
|
||||
provider="github",
|
||||
include_extra_email=False,
|
||||
)
|
||||
request = rf.get("/")
|
||||
|
||||
adapter.pre_social_login(request, sociallogin)
|
||||
|
||||
sociallogin.connect.assert_called_once_with(request, create_test_user)
|
||||
|
||||
def test_social_account_linking_settings_are_fail_closed(self):
|
||||
assert not socialaccount_app_settings.EMAIL_AUTHENTICATION
|
||||
assert not socialaccount_app_settings.EMAIL_AUTHENTICATION_AUTO_CONNECT
|
||||
assert not account_app_settings.EMAIL_NOTIFICATIONS
|
||||
|
||||
def test_save_user_social_with_invitation_joins_invited_tenant(
|
||||
self, rf, create_test_user, tenants_fixture
|
||||
|
||||
@@ -1,11 +1,16 @@
|
||||
from contextlib import contextmanager
|
||||
from datetime import UTC, datetime
|
||||
from enum import Enum
|
||||
from unittest.mock import MagicMock, patch
|
||||
from unittest.mock import MagicMock, call, patch
|
||||
|
||||
import pytest
|
||||
from api.db_utils import (
|
||||
POSTGRES_TENANT_VAR,
|
||||
SET_CONFIG_QUERY,
|
||||
SET_TRANSACTION_READ_ONLY_QUERY,
|
||||
PostgresEnumMigration,
|
||||
_is_replica_connection_failure,
|
||||
_is_safe_primary_replay,
|
||||
_should_create_index_on_partition,
|
||||
batch_delete,
|
||||
create_objects_in_batches,
|
||||
@@ -392,10 +397,23 @@ class TestRlsTransaction:
|
||||
|
||||
with patch("api.db_utils.get_read_db_alias", return_value=None):
|
||||
with patch("api.db_utils.connections") as mock_connections:
|
||||
mock_conn = MagicMock()
|
||||
mock_cursor = MagicMock()
|
||||
mock_conn.cursor.return_value.__enter__.return_value = mock_cursor
|
||||
mock_connections.__getitem__.return_value = mock_conn
|
||||
mock_replica_conn = MagicMock()
|
||||
mock_replica_cursor = MagicMock()
|
||||
mock_replica_conn.cursor.return_value.__enter__.return_value = (
|
||||
mock_replica_cursor
|
||||
)
|
||||
mock_primary_conn = MagicMock()
|
||||
mock_primary_cursor = MagicMock()
|
||||
mock_primary_conn.cursor.return_value.__enter__.return_value = (
|
||||
mock_primary_cursor
|
||||
)
|
||||
|
||||
def connections_getitem(alias):
|
||||
if alias == "replica":
|
||||
return mock_replica_conn
|
||||
return mock_primary_conn
|
||||
|
||||
mock_connections.__getitem__.side_effect = connections_getitem
|
||||
mock_connections.__contains__.return_value = True
|
||||
|
||||
with patch("api.db_utils.transaction.atomic"):
|
||||
@@ -525,7 +543,7 @@ class TestRlsTransaction:
|
||||
def atomic_side_effect(*args, **kwargs):
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
if call_count < 3:
|
||||
if call_count < 4:
|
||||
raise OperationalError("Connection error")
|
||||
return MagicMock(
|
||||
__enter__=MagicMock(return_value=None),
|
||||
@@ -544,10 +562,11 @@ class TestRlsTransaction:
|
||||
with rls_transaction(tenant_id):
|
||||
pass
|
||||
|
||||
assert mock_sleep.call_count == 2
|
||||
assert mock_sleep.call_count == 3
|
||||
mock_sleep.assert_any_call(0.5)
|
||||
mock_sleep.assert_any_call(1.0)
|
||||
assert mock_logger.info.call_count == 2
|
||||
mock_sleep.assert_any_call(2.0)
|
||||
assert mock_logger.info.call_count == 3
|
||||
|
||||
def test_rls_transaction_operational_error_inside_context_no_retry(
|
||||
self, tenants_fixture, enable_read_replica
|
||||
@@ -578,11 +597,12 @@ class TestRlsTransaction:
|
||||
raise OperationalError("Conflict with recovery")
|
||||
|
||||
mock_sleep.assert_not_called()
|
||||
mock_conn.close.assert_not_called()
|
||||
|
||||
def test_rls_transaction_max_three_attempts_for_replica(
|
||||
def test_rls_transaction_max_attempts_for_replica(
|
||||
self, tenants_fixture, enable_read_replica
|
||||
):
|
||||
"""Test maximum 3 attempts for replica database."""
|
||||
"""Test REPLICA_MAX_ATTEMPTS replica tries + 1 primary fallback."""
|
||||
tenant = tenants_fixture[0]
|
||||
tenant_id = str(tenant.id)
|
||||
|
||||
@@ -606,7 +626,11 @@ class TestRlsTransaction:
|
||||
with rls_transaction(tenant_id):
|
||||
pass
|
||||
|
||||
assert mock_atomic.call_count == 3
|
||||
assert mock_atomic.call_args_list[-1] == call(
|
||||
using=DEFAULT_DB_ALIAS
|
||||
)
|
||||
# 3 replica + 1 primary = 4 total
|
||||
assert mock_atomic.call_count == 4
|
||||
|
||||
def test_rls_transaction_replica_no_retry_when_disabled(
|
||||
self, tenants_fixture, enable_read_replica
|
||||
@@ -617,10 +641,23 @@ class TestRlsTransaction:
|
||||
|
||||
with patch("api.db_utils.get_read_db_alias", return_value=enable_read_replica):
|
||||
with patch("api.db_utils.connections") as mock_connections:
|
||||
mock_conn = MagicMock()
|
||||
mock_cursor = MagicMock()
|
||||
mock_conn.cursor.return_value.__enter__.return_value = mock_cursor
|
||||
mock_connections.__getitem__.return_value = mock_conn
|
||||
mock_replica_conn = MagicMock()
|
||||
mock_replica_cursor = MagicMock()
|
||||
mock_replica_conn.cursor.return_value.__enter__.return_value = (
|
||||
mock_replica_cursor
|
||||
)
|
||||
mock_primary_conn = MagicMock()
|
||||
mock_primary_cursor = MagicMock()
|
||||
mock_primary_conn.cursor.return_value.__enter__.return_value = (
|
||||
mock_primary_cursor
|
||||
)
|
||||
|
||||
def connections_getitem(alias):
|
||||
if alias == "replica":
|
||||
return mock_replica_conn
|
||||
return mock_primary_conn
|
||||
|
||||
mock_connections.__getitem__.side_effect = connections_getitem
|
||||
mock_connections.__contains__.return_value = True
|
||||
|
||||
with patch("api.db_utils.transaction.atomic") as mock_atomic:
|
||||
@@ -682,7 +719,7 @@ class TestRlsTransaction:
|
||||
def atomic_side_effect(*args, **kwargs):
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
if call_count < 3:
|
||||
if call_count < 4:
|
||||
raise OperationalError("Replica error")
|
||||
return MagicMock(
|
||||
__enter__=MagicMock(return_value=None),
|
||||
@@ -691,7 +728,7 @@ class TestRlsTransaction:
|
||||
|
||||
with patch(
|
||||
"api.db_utils.transaction.atomic", side_effect=atomic_side_effect
|
||||
):
|
||||
) as mock_atomic:
|
||||
with patch("api.db_utils.time.sleep"):
|
||||
with patch(
|
||||
"api.db_utils.set_read_db_alias", return_value="token"
|
||||
@@ -701,6 +738,9 @@ class TestRlsTransaction:
|
||||
with rls_transaction(tenant_id):
|
||||
pass
|
||||
|
||||
assert mock_atomic.call_args_list[-1] == call(
|
||||
using=DEFAULT_DB_ALIAS
|
||||
)
|
||||
mock_logger.warning.assert_called_once()
|
||||
warning_msg = mock_logger.warning.call_args[0][0]
|
||||
assert "falling back to primary DB" in warning_msg
|
||||
@@ -725,7 +765,7 @@ class TestRlsTransaction:
|
||||
def atomic_side_effect(*args, **kwargs):
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
if call_count < 3:
|
||||
if call_count < 4:
|
||||
raise OperationalError("Replica error")
|
||||
return MagicMock(
|
||||
__enter__=MagicMock(return_value=None),
|
||||
@@ -744,7 +784,7 @@ class TestRlsTransaction:
|
||||
with rls_transaction(tenant_id):
|
||||
pass
|
||||
|
||||
assert mock_logger.info.call_count == 2
|
||||
assert mock_logger.info.call_count == 3
|
||||
assert mock_logger.warning.call_count == 1
|
||||
|
||||
def test_rls_transaction_operational_error_raised_immediately_on_primary(
|
||||
@@ -910,6 +950,520 @@ class TestRlsTransaction:
|
||||
result = cursor.fetchone()
|
||||
assert result[0] == 1
|
||||
|
||||
# --- Mid-query failover tests ---
|
||||
|
||||
class _FakeDatabaseError(Exception):
|
||||
def __init__(self, message, pgcode=None):
|
||||
super().__init__(message)
|
||||
self.pgcode = pgcode
|
||||
|
||||
def _install_execute_wrapper(self, connection):
|
||||
connection.execute_wrappers = []
|
||||
|
||||
@contextmanager
|
||||
def _execute_wrapper(fn):
|
||||
connection.execute_wrappers.append(fn)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
connection.execute_wrappers.remove(fn)
|
||||
|
||||
connection.execute_wrapper = _execute_wrapper
|
||||
|
||||
def _mock_replica_and_primary_connections(self, mock_connections):
|
||||
mock_replica_conn = MagicMock()
|
||||
self._install_execute_wrapper(mock_replica_conn)
|
||||
mock_replica_cursor = MagicMock()
|
||||
mock_replica_conn.cursor.return_value.__enter__.return_value = (
|
||||
mock_replica_cursor
|
||||
)
|
||||
|
||||
mock_primary_conn = MagicMock()
|
||||
mock_primary_cursor = MagicMock()
|
||||
mock_primary_raw_cursor = MagicMock()
|
||||
mock_primary_cursor.cursor = mock_primary_raw_cursor
|
||||
mock_primary_conn.cursor.return_value = mock_primary_cursor
|
||||
|
||||
def connections_getitem(alias):
|
||||
if alias == "replica":
|
||||
return mock_replica_conn
|
||||
return mock_primary_conn
|
||||
|
||||
mock_connections.__getitem__.side_effect = connections_getitem
|
||||
mock_connections.__contains__.return_value = True
|
||||
|
||||
return mock_replica_conn, mock_primary_conn, mock_primary_cursor
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"error",
|
||||
[
|
||||
_FakeDatabaseError("connection lost", pgcode="08006"),
|
||||
_FakeDatabaseError("terminating connection", pgcode="57P01"),
|
||||
OperationalError("SSL SYSCALL error: EOF detected"),
|
||||
OperationalError("server closed the connection unexpectedly"),
|
||||
OperationalError("database system is starting up"),
|
||||
],
|
||||
)
|
||||
def test_replica_connection_failure_detection_allows_failover(self, error):
|
||||
assert _is_replica_connection_failure(error)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"error",
|
||||
[
|
||||
_FakeDatabaseError("canceling statement", pgcode="57014"),
|
||||
_FakeDatabaseError("could not serialize access", pgcode="40001"),
|
||||
_FakeDatabaseError("deadlock detected", pgcode="40P01"),
|
||||
OperationalError("deadlock detected"),
|
||||
],
|
||||
)
|
||||
def test_replica_connection_failure_detection_rejects_query_errors(self, error):
|
||||
assert not _is_replica_connection_failure(error)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("sql", "many", "expected"),
|
||||
[
|
||||
("SELECT 1", False, True),
|
||||
(" -- leading comment\nSELECT 1", False, True),
|
||||
("/* leading comment */ SELECT 1", False, True),
|
||||
("SELECT 1", True, False),
|
||||
("SELECTING 1", False, False),
|
||||
("INSERT INTO fake_table (name) VALUES (%s)", False, False),
|
||||
("WITH rows AS (SELECT 1) SELECT * FROM rows", False, False),
|
||||
("SELECT * INTO fake_table_copy FROM fake_table", False, False),
|
||||
("SELECT * FROM fake_table FOR UPDATE", False, False),
|
||||
("SELECT * FROM fake_table FOR SHARE", False, False),
|
||||
],
|
||||
)
|
||||
def test_primary_replay_safety_detection(self, sql, many, expected):
|
||||
assert _is_safe_primary_replay(sql, many) is expected
|
||||
|
||||
def test_mid_query_failure_falls_directly_back_to_primary(
|
||||
self, tenants_fixture, enable_read_replica
|
||||
):
|
||||
"""Mid-query replica connection loss is replayed once on primary."""
|
||||
tenant = tenants_fixture[0]
|
||||
tenant_id = str(tenant.id)
|
||||
|
||||
with patch("api.db_utils.get_read_db_alias", return_value=enable_read_replica):
|
||||
with patch("api.db_utils.connections") as mock_connections:
|
||||
(
|
||||
mock_replica_conn,
|
||||
mock_primary_conn,
|
||||
mock_primary_cursor,
|
||||
) = self._mock_replica_and_primary_connections(mock_connections)
|
||||
|
||||
outer_atomic = MagicMock()
|
||||
outer_atomic.__enter__ = MagicMock(return_value=None)
|
||||
outer_atomic.__exit__ = MagicMock(return_value=False)
|
||||
fallback_atomic = MagicMock()
|
||||
fallback_atomic.__enter__ = MagicMock(return_value=None)
|
||||
fallback_atomic.__exit__ = MagicMock(return_value=False)
|
||||
|
||||
with patch(
|
||||
"api.db_utils.transaction.atomic",
|
||||
side_effect=[outer_atomic, fallback_atomic],
|
||||
) as mock_atomic:
|
||||
with patch("api.db_utils.time.sleep") as mock_sleep:
|
||||
with patch(
|
||||
"api.db_utils.set_read_db_alias",
|
||||
side_effect=["replica-token", "primary-token"],
|
||||
) as mock_set_alias:
|
||||
with patch(
|
||||
"api.db_utils.reset_read_db_alias"
|
||||
) as mock_reset_alias:
|
||||
with rls_transaction(tenant_id):
|
||||
wrapper = mock_replica_conn.execute_wrappers[0]
|
||||
context_cursor = MagicMock()
|
||||
mock_execute = MagicMock(
|
||||
side_effect=OperationalError(
|
||||
"SSL SYSCALL error: EOF detected"
|
||||
)
|
||||
)
|
||||
|
||||
wrapper(
|
||||
mock_execute,
|
||||
"SELECT %s",
|
||||
["value"],
|
||||
False,
|
||||
{"cursor": context_cursor},
|
||||
)
|
||||
|
||||
mock_sleep.assert_not_called()
|
||||
(
|
||||
mock_replica_conn.ensure_connection.assert_not_called()
|
||||
)
|
||||
mock_replica_conn.close.assert_called_once()
|
||||
(
|
||||
mock_primary_conn.ensure_connection.assert_called_once()
|
||||
)
|
||||
mock_primary_conn.cursor.assert_called_once_with()
|
||||
mock_primary_cursor.execute.assert_has_calls(
|
||||
[
|
||||
call(SET_TRANSACTION_READ_ONLY_QUERY),
|
||||
call(
|
||||
SET_CONFIG_QUERY,
|
||||
[POSTGRES_TENANT_VAR, tenant_id],
|
||||
),
|
||||
call("SELECT %s", ["value"]),
|
||||
]
|
||||
)
|
||||
assert context_cursor.db == mock_primary_conn
|
||||
assert (
|
||||
context_cursor.cursor
|
||||
== mock_primary_cursor.cursor
|
||||
)
|
||||
|
||||
mock_set_alias.assert_has_calls(
|
||||
[
|
||||
call(enable_read_replica),
|
||||
call(DEFAULT_DB_ALIAS),
|
||||
]
|
||||
)
|
||||
mock_reset_alias.assert_has_calls(
|
||||
[call("primary-token"), call("replica-token")]
|
||||
)
|
||||
assert mock_atomic.call_args_list == [
|
||||
call(using=enable_read_replica),
|
||||
call(using=DEFAULT_DB_ALIAS),
|
||||
]
|
||||
assert mock_replica_conn.execute_wrappers == []
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("sql", "params", "many"),
|
||||
[
|
||||
("INSERT INTO fake_table (name) VALUES (%s)", [("one",), ("two",)], True),
|
||||
("INSERT INTO fake_table (name) VALUES (%s)", ["one"], False),
|
||||
("UPDATE fake_table SET name = %s", ["one"], False),
|
||||
("DELETE FROM fake_table WHERE id = %s", [1], False),
|
||||
(
|
||||
"WITH deleted AS (DELETE FROM fake_table RETURNING *) "
|
||||
"SELECT * FROM deleted",
|
||||
None,
|
||||
False,
|
||||
),
|
||||
("SELECT * INTO fake_table_copy FROM fake_table", None, False),
|
||||
],
|
||||
)
|
||||
def test_mid_query_fallback_rejects_unsafe_replay(
|
||||
self, tenants_fixture, enable_read_replica, sql, params, many
|
||||
):
|
||||
"""Only single SELECT statements are replayed on primary."""
|
||||
tenant = tenants_fixture[0]
|
||||
tenant_id = str(tenant.id)
|
||||
|
||||
with patch("api.db_utils.get_read_db_alias", return_value=enable_read_replica):
|
||||
with patch("api.db_utils.connections") as mock_connections:
|
||||
(
|
||||
mock_replica_conn,
|
||||
mock_primary_conn,
|
||||
mock_primary_cursor,
|
||||
) = self._mock_replica_and_primary_connections(mock_connections)
|
||||
|
||||
with patch("api.db_utils.transaction.atomic") as mock_atomic:
|
||||
mock_atomic.return_value.__enter__ = MagicMock(return_value=None)
|
||||
mock_atomic.return_value.__exit__ = MagicMock(return_value=False)
|
||||
with patch(
|
||||
"api.db_utils.set_read_db_alias",
|
||||
side_effect=["replica-token", "primary-token"],
|
||||
):
|
||||
with patch("api.db_utils.reset_read_db_alias"):
|
||||
with rls_transaction(tenant_id):
|
||||
wrapper = mock_replica_conn.execute_wrappers[0]
|
||||
mock_execute = MagicMock(
|
||||
side_effect=OperationalError(
|
||||
"server closed the connection"
|
||||
)
|
||||
)
|
||||
|
||||
with pytest.raises(OperationalError):
|
||||
wrapper(
|
||||
mock_execute,
|
||||
sql,
|
||||
params,
|
||||
many,
|
||||
{"cursor": MagicMock()},
|
||||
)
|
||||
|
||||
mock_primary_conn.ensure_connection.assert_not_called()
|
||||
mock_primary_conn.cursor.assert_not_called()
|
||||
mock_primary_cursor.execute.assert_not_called()
|
||||
mock_primary_cursor.executemany.assert_not_called()
|
||||
|
||||
def test_mid_query_non_connection_error_does_not_fall_back(
|
||||
self, tenants_fixture, enable_read_replica
|
||||
):
|
||||
"""Query/concurrency errors are not replayed on primary."""
|
||||
tenant = tenants_fixture[0]
|
||||
tenant_id = str(tenant.id)
|
||||
|
||||
with patch("api.db_utils.get_read_db_alias", return_value=enable_read_replica):
|
||||
with patch("api.db_utils.connections") as mock_connections:
|
||||
(
|
||||
mock_replica_conn,
|
||||
mock_primary_conn,
|
||||
_mock_primary_cursor,
|
||||
) = self._mock_replica_and_primary_connections(mock_connections)
|
||||
|
||||
with patch("api.db_utils.transaction.atomic") as mock_atomic:
|
||||
mock_atomic.return_value.__enter__ = MagicMock(return_value=None)
|
||||
mock_atomic.return_value.__exit__ = MagicMock(return_value=False)
|
||||
with patch(
|
||||
"api.db_utils.set_read_db_alias", return_value="replica-token"
|
||||
):
|
||||
with patch("api.db_utils.reset_read_db_alias"):
|
||||
with rls_transaction(tenant_id):
|
||||
wrapper = mock_replica_conn.execute_wrappers[0]
|
||||
mock_execute = MagicMock(
|
||||
side_effect=OperationalError("deadlock detected")
|
||||
)
|
||||
|
||||
with pytest.raises(OperationalError):
|
||||
wrapper(
|
||||
mock_execute,
|
||||
"SELECT 1",
|
||||
None,
|
||||
False,
|
||||
{"cursor": MagicMock()},
|
||||
)
|
||||
|
||||
mock_replica_conn.close.assert_not_called()
|
||||
(
|
||||
mock_primary_conn.ensure_connection.assert_not_called()
|
||||
)
|
||||
|
||||
def test_mid_query_primary_replay_failure_propagates(
|
||||
self, tenants_fixture, enable_read_replica
|
||||
):
|
||||
"""Primary fallback errors propagate as Django OperationalError."""
|
||||
tenant = tenants_fixture[0]
|
||||
tenant_id = str(tenant.id)
|
||||
|
||||
with patch("api.db_utils.get_read_db_alias", return_value=enable_read_replica):
|
||||
with patch("api.db_utils.connections") as mock_connections:
|
||||
(
|
||||
mock_replica_conn,
|
||||
_mock_primary_conn,
|
||||
mock_primary_cursor,
|
||||
) = self._mock_replica_and_primary_connections(mock_connections)
|
||||
mock_primary_cursor.execute.side_effect = [
|
||||
None,
|
||||
None,
|
||||
OperationalError("primary down"),
|
||||
]
|
||||
|
||||
with patch("api.db_utils.transaction.atomic") as mock_atomic:
|
||||
mock_atomic.return_value.__enter__ = MagicMock(return_value=None)
|
||||
mock_atomic.return_value.__exit__ = MagicMock(return_value=False)
|
||||
with patch(
|
||||
"api.db_utils.set_read_db_alias",
|
||||
side_effect=["replica-token", "primary-token"],
|
||||
):
|
||||
with patch("api.db_utils.reset_read_db_alias"):
|
||||
with pytest.raises(OperationalError, match="primary down"):
|
||||
with rls_transaction(tenant_id):
|
||||
wrapper = mock_replica_conn.execute_wrappers[0]
|
||||
mock_execute = MagicMock(
|
||||
side_effect=OperationalError(
|
||||
"server closed the connection"
|
||||
)
|
||||
)
|
||||
wrapper(
|
||||
mock_execute,
|
||||
"SELECT 1",
|
||||
None,
|
||||
False,
|
||||
{"cursor": MagicMock()},
|
||||
)
|
||||
|
||||
mock_primary_cursor.close.assert_called_once()
|
||||
|
||||
def test_mid_query_fallback_suppresses_cleanup_error(
|
||||
self, tenants_fixture, enable_read_replica
|
||||
):
|
||||
"""After successful primary fallback, replica cleanup error is suppressed."""
|
||||
tenant = tenants_fixture[0]
|
||||
tenant_id = str(tenant.id)
|
||||
|
||||
with patch("api.db_utils.get_read_db_alias", return_value=enable_read_replica):
|
||||
with patch("api.db_utils.connections") as mock_connections:
|
||||
(
|
||||
mock_replica_conn,
|
||||
_mock_primary_conn,
|
||||
mock_primary_cursor,
|
||||
) = self._mock_replica_and_primary_connections(mock_connections)
|
||||
|
||||
# Replica's atomic.__exit__ raises on dead replica cleanup;
|
||||
# primary's atomic.__exit__ returns False (healthy commit).
|
||||
mock_outer_atomic = MagicMock()
|
||||
mock_outer_atomic.__enter__ = MagicMock(return_value=None)
|
||||
mock_outer_atomic.__exit__ = MagicMock(
|
||||
side_effect=OperationalError("cleanup failed on dead replica")
|
||||
)
|
||||
|
||||
mock_fallback_atomic = MagicMock()
|
||||
mock_fallback_atomic.__enter__ = MagicMock(return_value=None)
|
||||
mock_fallback_atomic.__exit__ = MagicMock(return_value=False)
|
||||
|
||||
atomic_call_count = 0
|
||||
|
||||
def atomic_side_effect(*args, **kwargs):
|
||||
nonlocal atomic_call_count
|
||||
atomic_call_count += 1
|
||||
if atomic_call_count == 1:
|
||||
return mock_outer_atomic
|
||||
return mock_fallback_atomic
|
||||
|
||||
with patch(
|
||||
"api.db_utils.transaction.atomic",
|
||||
side_effect=atomic_side_effect,
|
||||
):
|
||||
with patch(
|
||||
"api.db_utils.set_read_db_alias",
|
||||
side_effect=["replica-token", "primary-token"],
|
||||
):
|
||||
with patch("api.db_utils.reset_read_db_alias"):
|
||||
with rls_transaction(tenant_id):
|
||||
wrapper = mock_replica_conn.execute_wrappers[0]
|
||||
mock_execute = MagicMock(
|
||||
side_effect=OperationalError(
|
||||
"server closed the connection"
|
||||
)
|
||||
)
|
||||
mock_context = {"cursor": MagicMock()}
|
||||
wrapper(
|
||||
mock_execute,
|
||||
"SELECT 1",
|
||||
None,
|
||||
False,
|
||||
mock_context,
|
||||
)
|
||||
|
||||
mock_primary_cursor.execute.assert_has_calls(
|
||||
[
|
||||
call(SET_TRANSACTION_READ_ONLY_QUERY),
|
||||
call(
|
||||
SET_CONFIG_QUERY,
|
||||
[POSTGRES_TENANT_VAR, tenant_id],
|
||||
),
|
||||
call("SELECT 1", None),
|
||||
]
|
||||
)
|
||||
|
||||
def test_wrapper_not_installed_on_primary(self, tenants_fixture):
|
||||
"""execute_wrapper is not installed when targeting primary DB."""
|
||||
tenant = tenants_fixture[0]
|
||||
tenant_id = str(tenant.id)
|
||||
|
||||
with patch("api.db_utils.get_read_db_alias", return_value=None):
|
||||
with patch("api.db_utils.connections") as mock_connections:
|
||||
mock_conn = MagicMock()
|
||||
mock_conn.execute_wrappers = []
|
||||
mock_cursor = MagicMock()
|
||||
mock_conn.cursor.return_value.__enter__.return_value = mock_cursor
|
||||
mock_connections.__getitem__.return_value = mock_conn
|
||||
mock_connections.__contains__.return_value = True
|
||||
|
||||
with patch("api.db_utils.transaction.atomic") as mock_atomic:
|
||||
mock_atomic.return_value.__enter__ = MagicMock(return_value=None)
|
||||
mock_atomic.return_value.__exit__ = MagicMock(return_value=False)
|
||||
|
||||
with rls_transaction(tenant_id):
|
||||
# No wrapper installed on primary
|
||||
assert len(mock_conn.execute_wrappers) == 0
|
||||
|
||||
def test_stale_connection_closed_on_pre_yield_retry(
|
||||
self, tenants_fixture, enable_read_replica
|
||||
):
|
||||
"""Stale connection is closed before each pre-yield retry."""
|
||||
tenant = tenants_fixture[0]
|
||||
tenant_id = str(tenant.id)
|
||||
|
||||
with patch("api.db_utils.get_read_db_alias", return_value=enable_read_replica):
|
||||
with patch("api.db_utils.connections") as mock_connections:
|
||||
mock_conn = MagicMock()
|
||||
mock_conn.execute_wrappers = []
|
||||
mock_cursor = MagicMock()
|
||||
mock_conn.cursor.return_value.__enter__.return_value = mock_cursor
|
||||
mock_connections.__getitem__.return_value = mock_conn
|
||||
mock_connections.__contains__.return_value = True
|
||||
|
||||
call_count = 0
|
||||
|
||||
def atomic_side_effect(*args, **kwargs):
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
if call_count < 3:
|
||||
raise OperationalError("Connection error")
|
||||
return MagicMock(
|
||||
__enter__=MagicMock(return_value=None),
|
||||
__exit__=MagicMock(return_value=False),
|
||||
)
|
||||
|
||||
with patch(
|
||||
"api.db_utils.transaction.atomic", side_effect=atomic_side_effect
|
||||
):
|
||||
with patch("api.db_utils.time.sleep"):
|
||||
with patch(
|
||||
"api.db_utils.set_read_db_alias", return_value="token"
|
||||
):
|
||||
with patch("api.db_utils.reset_read_db_alias"):
|
||||
with rls_transaction(tenant_id):
|
||||
pass
|
||||
|
||||
# close() called for each failed pre-yield attempt
|
||||
assert mock_conn.close.call_count == 2
|
||||
|
||||
def test_caller_error_propagates_after_successful_failover(
|
||||
self, tenants_fixture, enable_read_replica
|
||||
):
|
||||
"""OperationalError raised by caller after failover is NOT suppressed."""
|
||||
tenant = tenants_fixture[0]
|
||||
tenant_id = str(tenant.id)
|
||||
|
||||
with patch("api.db_utils.get_read_db_alias", return_value=enable_read_replica):
|
||||
with patch("api.db_utils.connections") as mock_connections:
|
||||
(
|
||||
mock_replica_conn,
|
||||
_mock_primary_conn,
|
||||
_mock_primary_cursor,
|
||||
) = self._mock_replica_and_primary_connections(mock_connections)
|
||||
|
||||
# Transaction cleanup succeeds so the caller error should surface.
|
||||
mock_atomic_cm = MagicMock()
|
||||
mock_atomic_cm.__enter__ = MagicMock(return_value=None)
|
||||
mock_atomic_cm.__exit__ = MagicMock(return_value=False)
|
||||
|
||||
with patch(
|
||||
"api.db_utils.transaction.atomic", return_value=mock_atomic_cm
|
||||
):
|
||||
with patch("api.db_utils.time.sleep"):
|
||||
with patch(
|
||||
"api.db_utils.set_read_db_alias", return_value="token"
|
||||
):
|
||||
with patch("api.db_utils.reset_read_db_alias"):
|
||||
with pytest.raises(
|
||||
OperationalError, match="caller error"
|
||||
):
|
||||
with rls_transaction(tenant_id):
|
||||
# Trigger failover (succeeds on primary)
|
||||
wrapper = mock_replica_conn.execute_wrappers[0]
|
||||
mock_execute = MagicMock(
|
||||
side_effect=OperationalError(
|
||||
"server closed the connection"
|
||||
)
|
||||
)
|
||||
mock_context = {"cursor": MagicMock()}
|
||||
wrapper(
|
||||
mock_execute,
|
||||
"SELECT 1",
|
||||
None,
|
||||
False,
|
||||
mock_context,
|
||||
)
|
||||
# Caller errors after successful failover
|
||||
# should still propagate.
|
||||
raise OperationalError("caller error")
|
||||
|
||||
|
||||
class TestPostgresEnumMigration:
|
||||
"""
|
||||
|
||||
@@ -2,11 +2,12 @@ import uuid
|
||||
from unittest.mock import call, patch
|
||||
|
||||
import pytest
|
||||
from api.attack_paths.database import GraphDatabaseQueryException
|
||||
from api.db_utils import POSTGRES_TENANT_VAR, SET_CONFIG_QUERY
|
||||
from api.decorators import handle_provider_deletion, set_tenant
|
||||
from api.exceptions import ProviderDeletedException
|
||||
from django.core.exceptions import ObjectDoesNotExist
|
||||
from django.db import DatabaseError, IntegrityError
|
||||
from django.db import DEFAULT_DB_ALIAS, DatabaseError, IntegrityError
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@@ -40,10 +41,10 @@ class TestSetTenantDecorator:
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestHandleProviderDeletionDecorator:
|
||||
def test_success_no_exception(self, tenants_fixture, providers_fixture):
|
||||
def test_success_no_exception(self, tenants_fixture, aws_provider):
|
||||
"""Decorated function runs normally when no exception is raised."""
|
||||
tenant = tenants_fixture[0]
|
||||
provider = providers_fixture[0]
|
||||
provider = aws_provider
|
||||
|
||||
@handle_provider_deletion
|
||||
def task_func(**kwargs):
|
||||
@@ -127,11 +128,11 @@ class TestHandleProviderDeletionDecorator:
|
||||
@patch("api.decorators.rls_transaction")
|
||||
@patch("api.decorators.Provider.objects.filter")
|
||||
def test_provider_exists_reraises_original(
|
||||
self, mock_filter, mock_rls, tenants_fixture, providers_fixture
|
||||
self, mock_filter, mock_rls, tenants_fixture, aws_provider
|
||||
):
|
||||
"""Re-raises original exception when provider still exists."""
|
||||
tenant = tenants_fixture[0]
|
||||
provider = providers_fixture[0]
|
||||
provider = aws_provider
|
||||
|
||||
mock_rls.return_value.__enter__ = lambda s: None
|
||||
mock_rls.return_value.__exit__ = lambda s, *args: None
|
||||
@@ -187,11 +188,11 @@ class TestHandleProviderDeletionDecorator:
|
||||
@patch("api.decorators.rls_transaction")
|
||||
@patch("api.decorators.Provider.objects.filter")
|
||||
def test_database_error_provider_exists_reraises(
|
||||
self, mock_filter, mock_rls, tenants_fixture, providers_fixture
|
||||
self, mock_filter, mock_rls, tenants_fixture, aws_provider
|
||||
):
|
||||
"""Re-raises original DatabaseError when provider still exists."""
|
||||
tenant = tenants_fixture[0]
|
||||
provider = providers_fixture[0]
|
||||
provider = aws_provider
|
||||
|
||||
mock_rls.return_value.__enter__ = lambda s: None
|
||||
mock_rls.return_value.__exit__ = lambda s, *args: None
|
||||
@@ -204,6 +205,106 @@ class TestHandleProviderDeletionDecorator:
|
||||
with pytest.raises(DatabaseError):
|
||||
task_func(tenant_id=str(tenant.id), provider_id=str(provider.id))
|
||||
|
||||
@patch("api.decorators.rls_transaction")
|
||||
@patch("api.decorators.Provider.objects.filter")
|
||||
def test_graph_database_error_provider_missing_or_soft_deleted(
|
||||
self, mock_provider_filter, mock_rls, tenants_fixture
|
||||
):
|
||||
tenant = tenants_fixture[0]
|
||||
provider_id = str(uuid.uuid4())
|
||||
|
||||
mock_rls.return_value.__enter__ = lambda s: None
|
||||
mock_rls.return_value.__exit__ = lambda s, *args: None
|
||||
mock_provider_filter.return_value.exists.return_value = False
|
||||
|
||||
@handle_provider_deletion
|
||||
def task_func(**kwargs):
|
||||
raise GraphDatabaseQueryException("Temporary database not found")
|
||||
|
||||
with pytest.raises(ProviderDeletedException):
|
||||
task_func(tenant_id=str(tenant.id), provider_id=provider_id)
|
||||
|
||||
@patch("api.decorators.rls_transaction")
|
||||
@patch("api.decorators.Tenant.objects.filter")
|
||||
@patch("api.decorators.Provider.objects.filter")
|
||||
def test_graph_database_error_tenant_missing(
|
||||
self, mock_provider_filter, mock_tenant_filter, mock_rls, tenants_fixture
|
||||
):
|
||||
tenant = tenants_fixture[0]
|
||||
provider_id = str(uuid.uuid4())
|
||||
|
||||
mock_rls.return_value.__enter__ = lambda s: None
|
||||
mock_rls.return_value.__exit__ = lambda s, *args: None
|
||||
mock_provider_filter.return_value.exists.return_value = True
|
||||
mock_tenant_filter.return_value.exists.return_value = False
|
||||
|
||||
@handle_provider_deletion
|
||||
def task_func(**kwargs):
|
||||
raise GraphDatabaseQueryException("Temporary database not found")
|
||||
|
||||
with pytest.raises(ProviderDeletedException):
|
||||
task_func(tenant_id=str(tenant.id), provider_id=provider_id)
|
||||
|
||||
@patch("api.decorators.rls_transaction")
|
||||
@patch("api.decorators.Membership.objects.filter")
|
||||
@patch("api.decorators.Tenant.objects.filter")
|
||||
@patch("api.decorators.Provider.objects.filter")
|
||||
def test_graph_database_error_tenant_without_memberships(
|
||||
self,
|
||||
mock_provider_filter,
|
||||
mock_tenant_filter,
|
||||
mock_membership_filter,
|
||||
mock_rls,
|
||||
tenants_fixture,
|
||||
):
|
||||
tenant = tenants_fixture[0]
|
||||
provider_id = str(uuid.uuid4())
|
||||
|
||||
mock_rls.return_value.__enter__ = lambda s: None
|
||||
mock_rls.return_value.__exit__ = lambda s, *args: None
|
||||
mock_provider_filter.return_value.exists.return_value = True
|
||||
mock_tenant_filter.return_value.exists.return_value = True
|
||||
mock_membership_filter.return_value.exists.return_value = False
|
||||
|
||||
@handle_provider_deletion
|
||||
def task_func(**kwargs):
|
||||
raise GraphDatabaseQueryException("Temporary database not found")
|
||||
|
||||
with pytest.raises(ProviderDeletedException):
|
||||
task_func(tenant_id=str(tenant.id), provider_id=provider_id)
|
||||
|
||||
@patch("api.decorators.rls_transaction")
|
||||
@patch("api.decorators.Membership.objects.filter")
|
||||
@patch("api.decorators.Tenant.objects.filter")
|
||||
@patch("api.decorators.Provider.objects.filter")
|
||||
def test_graph_database_error_active_provider_and_tenant_reraises(
|
||||
self,
|
||||
mock_provider_filter,
|
||||
mock_tenant_filter,
|
||||
mock_membership_filter,
|
||||
mock_rls,
|
||||
tenants_fixture,
|
||||
):
|
||||
tenant = tenants_fixture[0]
|
||||
provider_id = str(uuid.uuid4())
|
||||
graph_error = GraphDatabaseQueryException("Temporary database not found")
|
||||
|
||||
mock_rls.return_value.__enter__ = lambda s: None
|
||||
mock_rls.return_value.__exit__ = lambda s, *args: None
|
||||
mock_provider_filter.return_value.exists.return_value = True
|
||||
mock_tenant_filter.return_value.exists.return_value = True
|
||||
mock_membership_filter.return_value.exists.return_value = True
|
||||
|
||||
@handle_provider_deletion
|
||||
def task_func(**kwargs):
|
||||
raise graph_error
|
||||
|
||||
with pytest.raises(GraphDatabaseQueryException) as exc_info:
|
||||
task_func(tenant_id=str(tenant.id), provider_id=provider_id)
|
||||
|
||||
assert exc_info.value is graph_error
|
||||
mock_rls.assert_called_once_with(str(tenant.id), using=DEFAULT_DB_ALIAS)
|
||||
|
||||
def test_missing_provider_and_scan_raises_assertion(self, tenants_fixture):
|
||||
"""Raises AssertionError when neither provider_id nor scan_id in kwargs."""
|
||||
|
||||
|
||||
@@ -19,8 +19,8 @@ from django.db import IntegrityError
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestResourceModel:
|
||||
def test_setting_tags(self, providers_fixture):
|
||||
provider, *_ = providers_fixture
|
||||
def test_setting_tags(self, aws_provider):
|
||||
provider = aws_provider
|
||||
tenant_id = provider.tenant_id
|
||||
|
||||
resource = Resource.objects.create(
|
||||
@@ -111,9 +111,9 @@ class TestResourceModel:
|
||||
# @pytest.mark.django_db
|
||||
# class TestFindingModel:
|
||||
# def test_add_finding_with_long_uid(
|
||||
# self, providers_fixture, scans_fixture, resources_fixture
|
||||
# self, aws_provider, scans_fixture, resources_fixture
|
||||
# ):
|
||||
# provider, *_ = providers_fixture
|
||||
# provider = aws_provider
|
||||
# tenant_id = provider.tenant_id
|
||||
|
||||
# long_uid = "1" * 500
|
||||
@@ -372,8 +372,8 @@ class TestSAMLConfigurationModel:
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestProviderComplianceScoreModel:
|
||||
def test_create_provider_compliance_score(self, providers_fixture, scans_fixture):
|
||||
provider = providers_fixture[0]
|
||||
def test_create_provider_compliance_score(self, aws_provider, scans_fixture):
|
||||
provider = aws_provider
|
||||
scan = scans_fixture[0]
|
||||
scan.completed_at = datetime.now(UTC)
|
||||
scan.save()
|
||||
@@ -393,9 +393,9 @@ class TestProviderComplianceScoreModel:
|
||||
assert score.requirement_status == StatusChoices.PASS
|
||||
|
||||
def test_unique_constraint_per_provider_compliance_requirement(
|
||||
self, providers_fixture, scans_fixture
|
||||
self, aws_provider, scans_fixture
|
||||
):
|
||||
provider = providers_fixture[0]
|
||||
provider = aws_provider
|
||||
scan = scans_fixture[0]
|
||||
scan.completed_at = datetime.now(UTC)
|
||||
scan.save()
|
||||
@@ -422,9 +422,9 @@ class TestProviderComplianceScoreModel:
|
||||
)
|
||||
|
||||
def test_different_providers_same_requirement_allowed(
|
||||
self, providers_fixture, scans_fixture
|
||||
self, aws_provider_pair, scans_fixture
|
||||
):
|
||||
provider1, provider2, *_ = providers_fixture
|
||||
provider1, provider2 = aws_provider_pair
|
||||
scan1 = scans_fixture[0]
|
||||
scan1.completed_at = datetime.now(UTC)
|
||||
scan1.save()
|
||||
|
||||
@@ -11,6 +11,7 @@ from api.models import (
|
||||
User,
|
||||
UserRoleRelationship,
|
||||
)
|
||||
from api.rbac.permissions import HasPermissions, Permissions
|
||||
from api.v1.serializers import TokenSerializer
|
||||
from conftest import TEST_PASSWORD, TODAY
|
||||
from django.urls import reverse
|
||||
@@ -434,11 +435,11 @@ class TestUserViewSet:
|
||||
@pytest.mark.django_db
|
||||
class TestProviderViewSet:
|
||||
def test_list_providers_with_all_permissions(
|
||||
self, authenticated_client_rbac, providers_fixture
|
||||
self, authenticated_client_rbac, aws_provider
|
||||
):
|
||||
response = authenticated_client_rbac.get(reverse("provider-list"))
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
assert len(response.json()["data"]) == len(providers_fixture)
|
||||
assert len(response.json()["data"]) == 1
|
||||
|
||||
def test_list_providers_with_no_permissions(
|
||||
self, authenticated_client_no_permissions_rbac
|
||||
@@ -450,9 +451,9 @@ class TestProviderViewSet:
|
||||
assert len(response.json()["data"]) == 0
|
||||
|
||||
def test_retrieve_provider_with_all_permissions(
|
||||
self, authenticated_client_rbac, providers_fixture
|
||||
self, authenticated_client_rbac, aws_provider
|
||||
):
|
||||
provider = providers_fixture[0]
|
||||
provider = aws_provider
|
||||
response = authenticated_client_rbac.get(
|
||||
reverse("provider-detail", kwargs={"pk": provider.id})
|
||||
)
|
||||
@@ -460,9 +461,9 @@ class TestProviderViewSet:
|
||||
assert response.json()["data"]["attributes"]["alias"] == provider.alias
|
||||
|
||||
def test_retrieve_provider_with_no_permissions(
|
||||
self, authenticated_client_no_permissions_rbac, providers_fixture
|
||||
self, authenticated_client_no_permissions_rbac, aws_provider
|
||||
):
|
||||
provider = providers_fixture[0]
|
||||
provider = aws_provider
|
||||
response = authenticated_client_no_permissions_rbac.get(
|
||||
reverse("provider-detail", kwargs={"pk": provider.id})
|
||||
)
|
||||
@@ -486,9 +487,9 @@ class TestProviderViewSet:
|
||||
assert response.status_code == status.HTTP_403_FORBIDDEN
|
||||
|
||||
def test_partial_update_provider_with_all_permissions(
|
||||
self, authenticated_client_rbac, providers_fixture
|
||||
self, authenticated_client_rbac, aws_provider
|
||||
):
|
||||
provider = providers_fixture[0]
|
||||
provider = aws_provider
|
||||
payload = {
|
||||
"data": {
|
||||
"type": "providers",
|
||||
@@ -505,9 +506,9 @@ class TestProviderViewSet:
|
||||
assert response.json()["data"]["attributes"]["alias"] == "updated_alias"
|
||||
|
||||
def test_partial_update_provider_with_no_permissions(
|
||||
self, authenticated_client_no_permissions_rbac, providers_fixture
|
||||
self, authenticated_client_no_permissions_rbac, aws_provider
|
||||
):
|
||||
provider = providers_fixture[0]
|
||||
provider = aws_provider
|
||||
update_payload = {
|
||||
"data": {
|
||||
"type": "providers",
|
||||
@@ -528,7 +529,7 @@ class TestProviderViewSet:
|
||||
mock_delete_task,
|
||||
mock_task_get,
|
||||
authenticated_client_rbac,
|
||||
providers_fixture,
|
||||
aws_provider,
|
||||
tasks_fixture,
|
||||
):
|
||||
prowler_task = tasks_fixture[0]
|
||||
@@ -537,7 +538,7 @@ class TestProviderViewSet:
|
||||
mock_delete_task.return_value = task_mock
|
||||
mock_task_get.return_value = prowler_task
|
||||
|
||||
provider1, *_ = providers_fixture
|
||||
provider1 = aws_provider
|
||||
response = authenticated_client_rbac.delete(
|
||||
reverse("provider-detail", kwargs={"pk": provider1.id})
|
||||
)
|
||||
@@ -549,9 +550,9 @@ class TestProviderViewSet:
|
||||
assert response.headers["Content-Location"] == f"/api/v1/tasks/{task_mock.id}"
|
||||
|
||||
def test_delete_provider_with_no_permissions(
|
||||
self, authenticated_client_no_permissions_rbac, providers_fixture
|
||||
self, authenticated_client_no_permissions_rbac, aws_provider
|
||||
):
|
||||
provider = providers_fixture[0]
|
||||
provider = aws_provider
|
||||
response = authenticated_client_no_permissions_rbac.delete(
|
||||
reverse("provider-detail", kwargs={"pk": provider.id})
|
||||
)
|
||||
@@ -564,7 +565,7 @@ class TestProviderViewSet:
|
||||
mock_provider_connection,
|
||||
mock_task_get,
|
||||
authenticated_client_rbac,
|
||||
providers_fixture,
|
||||
aws_provider,
|
||||
tasks_fixture,
|
||||
):
|
||||
prowler_task = tasks_fixture[0]
|
||||
@@ -574,7 +575,7 @@ class TestProviderViewSet:
|
||||
mock_provider_connection.return_value = task_mock
|
||||
mock_task_get.return_value = prowler_task
|
||||
|
||||
provider1, *_ = providers_fixture
|
||||
provider1 = aws_provider
|
||||
assert provider1.connected is None
|
||||
assert provider1.connection_last_checked_at is None
|
||||
|
||||
@@ -589,9 +590,9 @@ class TestProviderViewSet:
|
||||
assert response.headers["Content-Location"] == f"/api/v1/tasks/{task_mock.id}"
|
||||
|
||||
def test_connection_with_no_permissions(
|
||||
self, authenticated_client_no_permissions_rbac, providers_fixture
|
||||
self, authenticated_client_no_permissions_rbac, aws_provider
|
||||
):
|
||||
provider = providers_fixture[0]
|
||||
provider = aws_provider
|
||||
response = authenticated_client_no_permissions_rbac.post(
|
||||
reverse("provider-connection", kwargs={"pk": provider.id})
|
||||
)
|
||||
@@ -604,10 +605,10 @@ class TestLimitedVisibility:
|
||||
TEST_PASSWORD = "Thisisapassword123@"
|
||||
|
||||
@pytest.fixture
|
||||
def limited_admin_user(self, django_db_blocker, tenants_fixture, providers_fixture):
|
||||
def limited_admin_user(self, django_db_blocker, tenants_fixture, aws_provider):
|
||||
with django_db_blocker.unblock():
|
||||
tenant = tenants_fixture[0]
|
||||
provider = providers_fixture[0]
|
||||
provider = aws_provider
|
||||
user = User.objects.create_user(
|
||||
name="testing",
|
||||
email=self.TEST_EMAIL,
|
||||
@@ -654,25 +655,17 @@ class TestLimitedVisibility:
|
||||
|
||||
@pytest.fixture
|
||||
def authenticated_client_rbac_limited(
|
||||
self, limited_admin_user, tenants_fixture, client
|
||||
self,
|
||||
limited_admin_user,
|
||||
tenants_fixture,
|
||||
authenticated_client_for_tenant_factory,
|
||||
):
|
||||
client.user = limited_admin_user
|
||||
tenant_id = tenants_fixture[0].id
|
||||
serializer = TokenSerializer(
|
||||
data={
|
||||
"type": "tokens",
|
||||
"email": self.TEST_EMAIL,
|
||||
"password": self.TEST_PASSWORD,
|
||||
"tenant_id": tenant_id,
|
||||
}
|
||||
return authenticated_client_for_tenant_factory(
|
||||
limited_admin_user, tenants_fixture[0]
|
||||
)
|
||||
serializer.is_valid(raise_exception=True)
|
||||
access_token = serializer.validated_data["access"]
|
||||
client.defaults["HTTP_AUTHORIZATION"] = f"Bearer {access_token}"
|
||||
return client
|
||||
|
||||
def test_integrations(
|
||||
self, authenticated_client_rbac_limited, integrations_fixture, providers_fixture
|
||||
self, authenticated_client_rbac_limited, integrations_fixture
|
||||
):
|
||||
# Integration 2 is related to provider1 and provider 2
|
||||
# This user cannot see provider 2
|
||||
@@ -692,7 +685,7 @@ class TestLimitedVisibility:
|
||||
def test_overviews_providers(
|
||||
self,
|
||||
authenticated_client_rbac_limited,
|
||||
providers_fixture,
|
||||
provider_factory,
|
||||
):
|
||||
# By default, the associated provider is the one which has the overview data
|
||||
response = authenticated_client_rbac_limited.get(reverse("overview-providers"))
|
||||
@@ -702,7 +695,7 @@ class TestLimitedVisibility:
|
||||
|
||||
# Changing the provider visibility, no data should be returned
|
||||
# Only the associated provider to that group is changed
|
||||
new_provider = providers_fixture[1]
|
||||
new_provider = provider_factory()
|
||||
ProviderGroupMembership.objects.all().update(provider=new_provider)
|
||||
|
||||
response = authenticated_client_rbac_limited.get(reverse("overview-providers"))
|
||||
@@ -722,7 +715,7 @@ class TestLimitedVisibility:
|
||||
self,
|
||||
endpoint_name,
|
||||
authenticated_client_rbac_limited,
|
||||
providers_fixture,
|
||||
provider_factory,
|
||||
):
|
||||
# By default, the associated provider is the one which has the overview data
|
||||
response = authenticated_client_rbac_limited.get(
|
||||
@@ -735,7 +728,7 @@ class TestLimitedVisibility:
|
||||
|
||||
# Changing the provider visibility, no data should be returned
|
||||
# Only the associated provider to that group is changed
|
||||
new_provider = providers_fixture[1]
|
||||
new_provider = provider_factory()
|
||||
ProviderGroupMembership.objects.all().update(provider=new_provider)
|
||||
|
||||
response = authenticated_client_rbac_limited.get(
|
||||
@@ -750,7 +743,7 @@ class TestLimitedVisibility:
|
||||
def test_overviews_services(
|
||||
self,
|
||||
authenticated_client_rbac_limited,
|
||||
providers_fixture,
|
||||
provider_factory,
|
||||
):
|
||||
# By default, the associated provider is the one which has the overview data
|
||||
response = authenticated_client_rbac_limited.get(
|
||||
@@ -762,7 +755,7 @@ class TestLimitedVisibility:
|
||||
|
||||
# Changing the provider visibility, no data should be returned
|
||||
# Only the associated provider to that group is changed
|
||||
new_provider = providers_fixture[1]
|
||||
new_provider = provider_factory()
|
||||
ProviderGroupMembership.objects.all().update(provider=new_provider)
|
||||
|
||||
response = authenticated_client_rbac_limited.get(
|
||||
@@ -824,6 +817,48 @@ class TestRolePermissions:
|
||||
assert response.status_code == status.HTTP_403_FORBIDDEN
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestHasPermissions:
|
||||
def test_permissions_are_combined_across_roles(
|
||||
self, create_test_user_rbac_no_roles
|
||||
):
|
||||
user = create_test_user_rbac_no_roles
|
||||
tenant = Membership.objects.get(user=user).tenant
|
||||
manage_users_role = Role.objects.create(
|
||||
name="manage_users_only",
|
||||
tenant=tenant,
|
||||
manage_users=True,
|
||||
)
|
||||
UserRoleRelationship.objects.create(
|
||||
user=user,
|
||||
role=manage_users_role,
|
||||
tenant=tenant,
|
||||
)
|
||||
request = Mock(user=user, tenant_id=tenant.id)
|
||||
view = Mock(
|
||||
required_permissions=[
|
||||
Permissions.MANAGE_USERS,
|
||||
Permissions.MANAGE_ACCOUNT,
|
||||
]
|
||||
)
|
||||
permission = HasPermissions()
|
||||
|
||||
assert not permission.has_permission(request, view)
|
||||
|
||||
manage_account_role = Role.objects.create(
|
||||
name="manage_account_only",
|
||||
tenant=tenant,
|
||||
manage_account=True,
|
||||
)
|
||||
UserRoleRelationship.objects.create(
|
||||
user=user,
|
||||
role=manage_account_role,
|
||||
tenant=tenant,
|
||||
)
|
||||
|
||||
assert permission.has_permission(request, view)
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestUserRoleLinkPermissions:
|
||||
def test_link_user_roles_with_manage_account_only_allowed(
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from api.attack_paths.retryable_session import RetryableSession
|
||||
from api.attack_paths.retryable_session import RetryableSession, RetryExhaustedError
|
||||
from neo4j.exceptions import ServiceUnavailable
|
||||
|
||||
|
||||
@@ -24,6 +24,7 @@ class TestRetryableSession:
|
||||
max_retries=3,
|
||||
retry_if=lambda exc: exc is retryable_error,
|
||||
initial_retry_delay_seconds=2,
|
||||
retry_context="Neptune write",
|
||||
)
|
||||
|
||||
assert session.execute_write(work) == "success"
|
||||
@@ -54,6 +55,7 @@ class TestRetryableSession:
|
||||
max_retries=3,
|
||||
retry_if=lambda _: False,
|
||||
initial_retry_delay_seconds=2,
|
||||
retry_context="Neptune write",
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError) as exc_info:
|
||||
@@ -83,3 +85,81 @@ class TestRetryableSession:
|
||||
driver_sessions[0].close.assert_called_once_with()
|
||||
driver_sessions[1].close.assert_called_once_with()
|
||||
driver_sessions[2].close.assert_not_called()
|
||||
|
||||
def test_retry_exhaustion_with_context_reports_attempts_and_elapsed_time(self):
|
||||
error = RuntimeError("still retryable")
|
||||
driver_sessions = [MagicMock() for _ in range(3)]
|
||||
for driver_session in driver_sessions:
|
||||
driver_session.execute_write.side_effect = error
|
||||
session = RetryableSession(
|
||||
session_factory=MagicMock(side_effect=driver_sessions),
|
||||
max_retries=2,
|
||||
retry_if=lambda _: True,
|
||||
retry_context="Neptune write",
|
||||
)
|
||||
|
||||
with (
|
||||
patch(
|
||||
"api.attack_paths.retryable_session.time.monotonic",
|
||||
side_effect=[100.0, 127.1234],
|
||||
),
|
||||
pytest.raises(RetryExhaustedError) as exc_info,
|
||||
):
|
||||
session.execute_write(MagicMock())
|
||||
|
||||
assert exc_info.value.method_name == "execute_write"
|
||||
assert exc_info.value.attempts == 3
|
||||
assert exc_info.value.elapsed_seconds == pytest.approx(27.1234)
|
||||
assert exc_info.value.last_error is error
|
||||
assert exc_info.value.__cause__ is error
|
||||
assert str(exc_info.value) == (
|
||||
"Neptune write execute_write failed after 3 attempts over 27.123s. "
|
||||
"Last error: still retryable"
|
||||
)
|
||||
|
||||
def test_retry_exhaustion_with_zero_retries_reports_one_attempt(self):
|
||||
error = ServiceUnavailable("still unavailable")
|
||||
driver_session = MagicMock()
|
||||
driver_session.execute_write.side_effect = error
|
||||
session = RetryableSession(
|
||||
session_factory=MagicMock(return_value=driver_session),
|
||||
max_retries=0,
|
||||
retry_context="Neptune write",
|
||||
)
|
||||
|
||||
with pytest.raises(RetryExhaustedError) as exc_info:
|
||||
session.execute_write(MagicMock())
|
||||
|
||||
assert exc_info.value.attempts == 1
|
||||
|
||||
@patch("api.attack_paths.retryable_session.time.sleep")
|
||||
@patch("api.attack_paths.retryable_session.random.uniform", return_value=3.0)
|
||||
def test_contextual_retry_warning_includes_original_error(
|
||||
self, _mock_uniform, _mock_sleep
|
||||
):
|
||||
error = RuntimeError("retryable detail")
|
||||
first_session = MagicMock()
|
||||
first_session.execute_write.side_effect = error
|
||||
second_session = MagicMock()
|
||||
second_session.execute_write.return_value = "success"
|
||||
session = RetryableSession(
|
||||
session_factory=MagicMock(side_effect=[first_session, second_session]),
|
||||
max_retries=1,
|
||||
retry_if=lambda _: True,
|
||||
initial_retry_delay_seconds=2,
|
||||
retry_context="Neptune write",
|
||||
)
|
||||
|
||||
with patch("api.attack_paths.retryable_session.logger.warning") as mock_warning:
|
||||
assert session.execute_write(MagicMock()) == "success"
|
||||
|
||||
mock_warning.assert_called_once_with(
|
||||
"%s %s failed with %s: %s; retry %s/%s in %.3fs",
|
||||
"Neptune write",
|
||||
"execute_write",
|
||||
"RuntimeError",
|
||||
"retryable detail",
|
||||
1,
|
||||
1,
|
||||
3.0,
|
||||
)
|
||||
|
||||
@@ -1,9 +1,34 @@
|
||||
import logging
|
||||
from unittest.mock import MagicMock
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from config.settings import sentry as sentry_settings
|
||||
from config.settings.sentry import before_send
|
||||
|
||||
|
||||
def test_initialize_sentry_skips_without_dsn():
|
||||
with (
|
||||
patch.object(sentry_settings.env, "str", return_value=""),
|
||||
patch.object(sentry_settings.sentry_sdk, "init") as mock_init,
|
||||
):
|
||||
sentry_settings.initialize_sentry()
|
||||
|
||||
mock_init.assert_not_called()
|
||||
|
||||
|
||||
def test_initialize_sentry_uses_configured_dsn():
|
||||
sentry_dsn = "https://fake-public-key@sentry.example.invalid/1"
|
||||
|
||||
with (
|
||||
patch.object(sentry_settings.env, "str", return_value=sentry_dsn),
|
||||
patch.object(sentry_settings.sentry_sdk, "init") as mock_init,
|
||||
):
|
||||
sentry_settings.initialize_sentry()
|
||||
|
||||
assert mock_init.call_args.kwargs["dsn"] == sentry_dsn
|
||||
assert mock_init.call_args.kwargs["before_send"] is sentry_settings.before_send
|
||||
|
||||
|
||||
def _make_log_record(msg, level=logging.ERROR, name="test", args=None):
|
||||
"""Build a real LogRecord so getMessage() works like in production."""
|
||||
record = logging.LogRecord(
|
||||
@@ -58,6 +83,45 @@ def test_before_send_passes_through_non_ignored_log():
|
||||
assert result == event
|
||||
|
||||
|
||||
def test_before_send_ignores_cartography_missing_temporary_database_log():
|
||||
log_record = _make_log_record(
|
||||
msg="Cartography job failed with %s for database %s",
|
||||
name="cartography.graph.job",
|
||||
args=(
|
||||
"Neo.ClientError.Database.DatabaseNotFound",
|
||||
"db-tmp-scan-12345678",
|
||||
),
|
||||
)
|
||||
|
||||
event = MagicMock()
|
||||
|
||||
assert before_send(event, {"log_record": log_record}) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("logger_name", "message"),
|
||||
[
|
||||
(
|
||||
"cartography.graph.job.worker",
|
||||
"Neo.ClientError.Database.DatabaseNotFound for db-tmp-scan-12345678",
|
||||
),
|
||||
(
|
||||
"cartography.graph.job",
|
||||
"DatabaseNotFound for db-tmp-scan-12345678",
|
||||
),
|
||||
(
|
||||
"cartography.graph.job",
|
||||
"Neo.ClientError.Database.DatabaseNotFound for db-tenant-12345678",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_before_send_passes_through_similar_cartography_logs(logger_name, message):
|
||||
log_record = _make_log_record(msg=message, name=logger_name)
|
||||
event = MagicMock()
|
||||
|
||||
assert before_send(event, {"log_record": log_record}) is event
|
||||
|
||||
|
||||
def test_before_send_passes_through_non_ignored_exception():
|
||||
"""Test that before_send passes through exceptions that don't contain ignored exceptions."""
|
||||
exc_info = (Exception, Exception("Some other error message"), None)
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
import pytest
|
||||
from api.v1.serializer_utils.integrations import S3ConfigSerializer
|
||||
from api.v1.serializer_utils.integrations import (
|
||||
JiraCredentialSerializer,
|
||||
S3ConfigSerializer,
|
||||
)
|
||||
from api.v1.serializers import ImageProviderSecret, KubernetesProviderSecret
|
||||
from rest_framework.exceptions import ValidationError
|
||||
|
||||
@@ -100,6 +103,59 @@ class TestS3ConfigSerializer:
|
||||
assert "output_directory" in serializer.errors
|
||||
|
||||
|
||||
class TestJiraCredentialSerializer:
|
||||
@pytest.mark.parametrize(
|
||||
"domain",
|
||||
(
|
||||
"a",
|
||||
"prowler",
|
||||
"prowler-domain",
|
||||
"A1-b2-C3",
|
||||
"a" * 63,
|
||||
),
|
||||
)
|
||||
def test_valid_site_name(self, domain):
|
||||
serializer = JiraCredentialSerializer(
|
||||
data={
|
||||
"user_mail": "testing@prowler.com",
|
||||
"api_token": "fake-api-token",
|
||||
"domain": domain,
|
||||
}
|
||||
)
|
||||
|
||||
assert serializer.is_valid(), serializer.errors
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"domain",
|
||||
(
|
||||
"169.254.169.254#",
|
||||
"internal/service",
|
||||
"internal?target",
|
||||
"internal\\target",
|
||||
"internal:8000",
|
||||
"user@internal",
|
||||
"example.atlassian.net",
|
||||
"-prowler",
|
||||
"prowler-",
|
||||
"a" * 64,
|
||||
" prowler",
|
||||
"prowler ",
|
||||
"prowler\n",
|
||||
),
|
||||
)
|
||||
def test_invalid_site_name(self, domain):
|
||||
serializer = JiraCredentialSerializer(
|
||||
data={
|
||||
"user_mail": "testing@prowler.com",
|
||||
"api_token": "fake-api-token",
|
||||
"domain": domain,
|
||||
}
|
||||
)
|
||||
|
||||
assert not serializer.is_valid()
|
||||
assert "domain" in serializer.errors
|
||||
|
||||
|
||||
class TestImageProviderSecret:
|
||||
"""Test cases for ImageProviderSecret validation."""
|
||||
|
||||
|
||||
@@ -11,7 +11,11 @@ from unittest.mock import MagicMock, patch
|
||||
import neo4j
|
||||
import pytest
|
||||
from api.attack_paths import sink as sink_module
|
||||
from api.attack_paths.database import GraphDatabaseQueryException
|
||||
from api.attack_paths.database import (
|
||||
GraphDatabaseQueryException,
|
||||
NeptuneWriteRetryExhaustedException,
|
||||
)
|
||||
from api.attack_paths.retryable_session import RetryExhaustedError
|
||||
from api.attack_paths.sink import factory
|
||||
from api.attack_paths.sink.neo4j import DATABASE_NOT_FOUND_CODE, Neo4jSink
|
||||
from api.attack_paths.sink.neptune import (
|
||||
@@ -123,6 +127,14 @@ class TestSinkFactory:
|
||||
assert mock_driver.call_count == 1
|
||||
|
||||
|
||||
def test_neo4j_sync_batch_size_defaults_to_1000():
|
||||
assert Neo4jSink.sync_batch_size == 1000
|
||||
|
||||
|
||||
def test_neptune_sync_batch_size_defaults_to_500():
|
||||
assert NeptuneSink.sync_batch_size == 500
|
||||
|
||||
|
||||
class TestGetBackendForScan:
|
||||
"""``get_backend_for_scan`` routes by the row's recorded sink backend."""
|
||||
|
||||
@@ -332,9 +344,10 @@ class TestNeptuneRetryPolicy:
|
||||
@pytest.mark.parametrize(
|
||||
"message",
|
||||
[
|
||||
"Operation failed due to conflicting concurrent operations "
|
||||
+ "(please retry), 0 transactions are currently rolling back.",
|
||||
"Operation terminated (deadline exceeded)",
|
||||
"Unexpected server exception 'Operation failed due to conflicting "
|
||||
"concurrent operations (please retry), 0 transactions are currently "
|
||||
"rolling back.'",
|
||||
"Unexpected server exception 'Operation terminated (deadline exceeded)'",
|
||||
],
|
||||
)
|
||||
def test_observed_transient_write_errors_are_retryable(self, message):
|
||||
@@ -345,7 +358,9 @@ class TestNeptuneRetryPolicy:
|
||||
|
||||
def test_unrelated_database_error_is_not_retryable(self):
|
||||
error = MagicMock(spec=neo4j.exceptions.Neo4jError)
|
||||
error.message = "Operation terminated (out of memory)"
|
||||
error.message = (
|
||||
"Unexpected server exception 'Operation terminated (out of memory)'"
|
||||
)
|
||||
|
||||
assert _is_retryable_write_error(error) is False
|
||||
|
||||
@@ -369,6 +384,7 @@ class TestNeptuneRetryPolicy:
|
||||
assert (
|
||||
kwargs["initial_retry_delay_seconds"] == NEPTUNE_WRITE_RETRY_DELAY_SECONDS
|
||||
)
|
||||
assert kwargs["retry_context"] == "Neptune write"
|
||||
|
||||
@patch("api.attack_paths.sink.neptune.RetryableSession")
|
||||
def test_reader_session_does_not_enable_write_retry_policy(self, retryable_session):
|
||||
@@ -381,6 +397,48 @@ class TestNeptuneRetryPolicy:
|
||||
kwargs = retryable_session.call_args.kwargs
|
||||
assert kwargs["retry_if"] is None
|
||||
assert kwargs["initial_retry_delay_seconds"] == 0
|
||||
assert kwargs["retry_context"] is None
|
||||
|
||||
def test_writer_retry_exhaustion_preserves_neptune_error_details(self):
|
||||
message = (
|
||||
"Unexpected server exception 'Operation failed due to conflicting "
|
||||
"concurrent operations (please retry), 0 transactions are currently "
|
||||
"rolling back.'"
|
||||
)
|
||||
error = neo4j.exceptions.Neo4jError._hydrate_neo4j(
|
||||
code="BoltProtocol.unexpectedException",
|
||||
message=message,
|
||||
)
|
||||
retry_error = RetryExhaustedError(
|
||||
retry_context="Neptune write",
|
||||
method_name="execute_write",
|
||||
attempts=4,
|
||||
elapsed_seconds=27.1234,
|
||||
last_error=error,
|
||||
)
|
||||
sink = NeptuneSink()
|
||||
driver = MagicMock()
|
||||
retryable_session = MagicMock()
|
||||
retryable_session.execute_write.side_effect = retry_error
|
||||
|
||||
with (
|
||||
patch.object(sink, "_get_writer", return_value=driver),
|
||||
patch(
|
||||
"api.attack_paths.sink.neptune.RetryableSession",
|
||||
return_value=retryable_session,
|
||||
),
|
||||
pytest.raises(NeptuneWriteRetryExhaustedException) as exc_info,
|
||||
):
|
||||
with sink.get_session() as session:
|
||||
session.execute_write(MagicMock())
|
||||
|
||||
assert exc_info.value.code == "BoltProtocol.unexpectedException"
|
||||
assert str(exc_info.value) == (
|
||||
"BoltProtocol.unexpectedException: Neptune write execute_write failed "
|
||||
"after 4 attempts over 27.123s. Last error: "
|
||||
f"{message}"
|
||||
)
|
||||
assert exc_info.value.__cause__ is error
|
||||
|
||||
|
||||
class TestNeptuneSinkDropSubgraph:
|
||||
|
||||
@@ -188,10 +188,10 @@ class TestProwlerProviderConnectionTest:
|
||||
@pytest.mark.django_db
|
||||
@patch("api.utils.return_prowler_provider")
|
||||
def test_prowler_provider_connection_test_without_secret(
|
||||
self, mock_return_prowler_provider, providers_fixture
|
||||
self, mock_return_prowler_provider, aws_provider
|
||||
):
|
||||
mock_return_prowler_provider.return_value = MagicMock()
|
||||
connection = prowler_provider_connection_test(providers_fixture[0])
|
||||
connection = prowler_provider_connection_test(aws_provider)
|
||||
|
||||
assert connection.is_connected is False
|
||||
assert isinstance(connection.error, Provider.secret.RelatedObjectDoesNotExist)
|
||||
@@ -856,7 +856,7 @@ class TestProwlerIntegrationConnectionTest:
|
||||
integration.credentials = {
|
||||
"user_mail": "test@example.com",
|
||||
"api_token": "test_api_token",
|
||||
"domain": "example.atlassian.net",
|
||||
"domain": "example",
|
||||
}
|
||||
integration.configuration = {}
|
||||
|
||||
@@ -884,7 +884,7 @@ class TestProwlerIntegrationConnectionTest:
|
||||
mock_jira_class.test_connection.assert_called_once_with(
|
||||
user_mail="test@example.com",
|
||||
api_token="test_api_token",
|
||||
domain="example.atlassian.net",
|
||||
domain="example",
|
||||
raise_on_exception=False,
|
||||
)
|
||||
|
||||
@@ -917,7 +917,7 @@ class TestProwlerIntegrationConnectionTest:
|
||||
integration.credentials = {
|
||||
"user_mail": "invalid@example.com",
|
||||
"api_token": "invalid_token",
|
||||
"domain": "invalid.atlassian.net",
|
||||
"domain": "invalid",
|
||||
}
|
||||
integration.configuration = {}
|
||||
|
||||
@@ -942,7 +942,7 @@ class TestProwlerIntegrationConnectionTest:
|
||||
mock_jira_class.test_connection.assert_called_once_with(
|
||||
user_mail="invalid@example.com",
|
||||
api_token="invalid_token",
|
||||
domain="invalid.atlassian.net",
|
||||
domain="invalid",
|
||||
raise_on_exception=False,
|
||||
)
|
||||
|
||||
@@ -970,7 +970,7 @@ class TestProwlerIntegrationConnectionTest:
|
||||
integration.credentials = {
|
||||
"user_mail": "test@example.com",
|
||||
"api_token": "test_api_token",
|
||||
"domain": "example.atlassian.net",
|
||||
"domain": "example",
|
||||
}
|
||||
integration.configuration = {
|
||||
"issue_types": {"OLD_PROJ": ["Task"]}, # Existing configuration
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -5,6 +5,10 @@ from api.v1.serializer_utils.base import BaseValidateSerializer
|
||||
from drf_spectacular.utils import extend_schema_field
|
||||
from rest_framework_json_api import serializers
|
||||
|
||||
ATLASSIAN_SITE_NAME_REGEX = re.compile(
|
||||
r"\A[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\Z"
|
||||
)
|
||||
|
||||
|
||||
class S3ConfigSerializer(BaseValidateSerializer):
|
||||
bucket_name = serializers.CharField()
|
||||
@@ -97,7 +101,17 @@ class AWSCredentialSerializer(BaseValidateSerializer):
|
||||
class JiraCredentialSerializer(BaseValidateSerializer):
|
||||
user_mail = serializers.EmailField(required=True)
|
||||
api_token = serializers.CharField(required=True)
|
||||
domain = serializers.CharField(required=True)
|
||||
domain = serializers.RegexField(
|
||||
regex=ATLASSIAN_SITE_NAME_REGEX,
|
||||
required=True,
|
||||
trim_whitespace=False,
|
||||
error_messages={
|
||||
"invalid": (
|
||||
"Domain must be a valid Atlassian site name containing only "
|
||||
"letters, numbers, and hyphens."
|
||||
)
|
||||
},
|
||||
)
|
||||
|
||||
class Meta:
|
||||
resource_name = "integrations"
|
||||
@@ -170,7 +184,10 @@ class JiraCredentialSerializer(BaseValidateSerializer):
|
||||
},
|
||||
"domain": {
|
||||
"type": "string",
|
||||
"description": "The JIRA domain/instance URL (e.g., 'your-domain.atlassian.net').",
|
||||
"description": "The Jira site name without the '.atlassian.net' suffix (e.g., 'your-domain').",
|
||||
"minLength": 1,
|
||||
"maxLength": 63,
|
||||
"pattern": "^[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?$",
|
||||
},
|
||||
},
|
||||
"required": ["user_mail", "api_token", "domain"],
|
||||
|
||||
@@ -481,8 +481,8 @@ class UserRoleRelationshipSerializer(RLSSerializer, BaseWriteSerializer):
|
||||
|
||||
def create(self, validated_data):
|
||||
role_ids = [item["id"] for item in validated_data["roles"]]
|
||||
roles = Role.objects.filter(id__in=role_ids)
|
||||
tenant_id = self.context.get("tenant_id")
|
||||
roles = Role.objects.filter(id__in=role_ids, tenant_id=tenant_id)
|
||||
|
||||
new_relationships = [
|
||||
UserRoleRelationship(
|
||||
@@ -496,8 +496,8 @@ class UserRoleRelationshipSerializer(RLSSerializer, BaseWriteSerializer):
|
||||
|
||||
def update(self, instance, validated_data):
|
||||
role_ids = [item["id"] for item in validated_data["roles"]]
|
||||
roles = Role.objects.filter(id__in=role_ids)
|
||||
tenant_id = self.context.get("tenant_id")
|
||||
roles = Role.objects.filter(id__in=role_ids, tenant_id=tenant_id)
|
||||
|
||||
# Safeguard: A tenant must always have at least one user with MANAGE_ACCOUNT.
|
||||
# If the target roles do NOT include MANAGE_ACCOUNT, and the current user is
|
||||
@@ -527,7 +527,7 @@ class UserRoleRelationshipSerializer(RLSSerializer, BaseWriteSerializer):
|
||||
}
|
||||
)
|
||||
|
||||
instance.roles.clear()
|
||||
UserRoleRelationship.objects.filter(user=instance, tenant_id=tenant_id).delete()
|
||||
new_relationships = [
|
||||
UserRoleRelationship(user=instance, role=r, tenant_id=tenant_id)
|
||||
for r in roles
|
||||
|
||||
@@ -74,6 +74,7 @@ celery_app.conf.task_annotations = {
|
||||
for name in (
|
||||
"scan-perform",
|
||||
"scan-perform-scheduled",
|
||||
"attack-paths-scan-perform",
|
||||
"provider-deletion",
|
||||
"tenant-deletion",
|
||||
)
|
||||
|
||||
@@ -308,9 +308,12 @@ CSRF_COOKIE_SECURE = True
|
||||
SESSION_COOKIE_SECURE = True
|
||||
|
||||
# Attack Paths
|
||||
ATTACK_PATHS_SCAN_INACTIVITY_THRESHOLD_MINUTES = env.int(
|
||||
"ATTACK_PATHS_SCAN_INACTIVITY_THRESHOLD_MINUTES", 30
|
||||
)
|
||||
ATTACK_PATHS_SCAN_STALE_THRESHOLD_MINUTES = env.int(
|
||||
"ATTACK_PATHS_SCAN_STALE_THRESHOLD_MINUTES", 2880
|
||||
) # 48h
|
||||
"ATTACK_PATHS_SCAN_STALE_THRESHOLD_MINUTES", 960
|
||||
) # 16h
|
||||
|
||||
# Selects where the persistent attack-paths graph is stored. The scan
|
||||
# temporary database is always Neo4j; only the sink is configurable.
|
||||
|
||||
@@ -91,6 +91,13 @@ def before_send(event, hint):
|
||||
log_msg = log_record.getMessage()
|
||||
log_lvl = log_record.levelno
|
||||
|
||||
if (
|
||||
getattr(log_record, "name", "") == "cartography.graph.job"
|
||||
and "Neo.ClientError.Database.DatabaseNotFound" in log_msg
|
||||
and "db-tmp-scan-" in log_msg
|
||||
):
|
||||
return None
|
||||
|
||||
# The Neo4j driver logs transient connection errors (defunct
|
||||
# connections, resets) at ERROR level via the `neo4j.io` logger.
|
||||
# `RetryableSession` handles these with retries. If all retries
|
||||
@@ -115,19 +122,27 @@ def before_send(event, hint):
|
||||
return event
|
||||
|
||||
|
||||
sentry_sdk.init(
|
||||
dsn=env.str("DJANGO_SENTRY_DSN", ""),
|
||||
# Add data like request headers and IP for users,
|
||||
# see https://docs.sentry.io/platforms/python/data-management/data-collected/ for more info
|
||||
before_send=before_send,
|
||||
send_default_pii=True,
|
||||
traces_sample_rate=env.float("DJANGO_SENTRY_TRACES_SAMPLE_RATE", default=0.02),
|
||||
_experiments={
|
||||
# Set continuous_profiling_auto_start to True
|
||||
# to automatically start the profiler on when
|
||||
# possible.
|
||||
"continuous_profiling_auto_start": True,
|
||||
},
|
||||
attach_stacktrace=True,
|
||||
ignore_errors=IGNORED_EXCEPTIONS,
|
||||
)
|
||||
def initialize_sentry():
|
||||
sentry_dsn = env.str("DJANGO_SENTRY_DSN", "")
|
||||
if not sentry_dsn:
|
||||
return
|
||||
|
||||
sentry_sdk.init(
|
||||
dsn=sentry_dsn,
|
||||
# Add data like request headers and IP for users,
|
||||
# see https://docs.sentry.io/platforms/python/data-management/data-collected/ for more info
|
||||
before_send=before_send,
|
||||
send_default_pii=True,
|
||||
traces_sample_rate=env.float("DJANGO_SENTRY_TRACES_SAMPLE_RATE", default=0.02),
|
||||
_experiments={
|
||||
# Set continuous_profiling_auto_start to True
|
||||
# to automatically start the profiler on when
|
||||
# possible.
|
||||
"continuous_profiling_auto_start": True,
|
||||
},
|
||||
attach_stacktrace=True,
|
||||
ignore_errors=IGNORED_EXCEPTIONS,
|
||||
)
|
||||
|
||||
|
||||
initialize_sentry()
|
||||
|
||||
@@ -13,16 +13,17 @@ GITHUB_OAUTH_CALLBACK_URL = env("SOCIAL_GITHUB_OAUTH_CALLBACK_URL", default="")
|
||||
ACCOUNT_LOGIN_METHODS = {"email"} # Use Email / Password authentication
|
||||
ACCOUNT_SIGNUP_FIELDS = ["email*", "password1*", "password2*"]
|
||||
ACCOUNT_EMAIL_VERIFICATION = "none" # Do not require email confirmation
|
||||
ACCOUNT_EMAIL_NOTIFICATIONS = False
|
||||
ACCOUNT_USER_MODEL_USERNAME_FIELD = None
|
||||
REST_AUTH = {
|
||||
"TOKEN_MODEL": None,
|
||||
"REST_USE_JWT": True,
|
||||
}
|
||||
# django-allauth (social)
|
||||
# Authenticate if local account with this email address already exists
|
||||
SOCIALACCOUNT_EMAIL_AUTHENTICATION = True
|
||||
# Connect local account and social account if local account with that email address already exists
|
||||
SOCIALACCOUNT_EMAIL_AUTHENTICATION_AUTO_CONNECT = True
|
||||
# Email-based account matching is handled by ProwlerSocialAccountAdapter, which
|
||||
# verifies both the provider email and the existing account email before linking.
|
||||
SOCIALACCOUNT_EMAIL_AUTHENTICATION = False
|
||||
SOCIALACCOUNT_EMAIL_AUTHENTICATION_AUTO_CONNECT = False
|
||||
SOCIALACCOUNT_ADAPTER = "api.adapters.ProwlerSocialAccountAdapter"
|
||||
|
||||
|
||||
|
||||
+278
-199
@@ -2,6 +2,7 @@ import logging
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from allauth.socialaccount.models import SocialLogin
|
||||
@@ -50,12 +51,14 @@ from api.v1.serializers import TokenSerializer
|
||||
from django.conf import settings
|
||||
from django.db import connection as django_connection
|
||||
from django.db import connections as django_connections
|
||||
from django.test import Client
|
||||
from django.urls import reverse
|
||||
from django_celery_results.models import TaskResult
|
||||
from prowler.lib.check.models import Severity
|
||||
from prowler.lib.outputs.finding import Status
|
||||
from rest_framework import status
|
||||
from rest_framework.test import APIClient
|
||||
from rest_framework_simplejwt.tokens import AccessToken
|
||||
from tasks.jobs.backfill import (
|
||||
aggregate_scan_category_summaries,
|
||||
aggregate_scan_resource_group_summaries,
|
||||
@@ -67,6 +70,7 @@ API_JSON_CONTENT_TYPE = "application/vnd.api+json"
|
||||
NO_TENANT_HTTP_STATUS = status.HTTP_401_UNAUTHORIZED
|
||||
TEST_USER = "dev@prowler.com"
|
||||
TEST_PASSWORD = "testing_psswd"
|
||||
TEST_REPLICA_ALIAS = "test_replica"
|
||||
|
||||
|
||||
def _install_compliance_catalog_test_cache() -> None:
|
||||
@@ -228,14 +232,15 @@ def create_test_user(_session_test_user, django_db_blocker):
|
||||
"""Re-create the session-scoped test user when a TransactionTestCase
|
||||
has truncated the users table."""
|
||||
with django_db_blocker.unblock():
|
||||
if not User.objects.filter(pk=_session_test_user.pk).exists():
|
||||
User.objects.create_user(
|
||||
user = User.objects.filter(pk=_session_test_user.pk).first()
|
||||
if user is None:
|
||||
user = User.objects.create_user(
|
||||
id=_session_test_user.pk,
|
||||
name="testing",
|
||||
email=TEST_USER,
|
||||
password=TEST_PASSWORD,
|
||||
)
|
||||
return _session_test_user
|
||||
return user
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
@@ -358,22 +363,42 @@ def create_test_user_rbac_manage_account(django_db_setup, django_db_blocker):
|
||||
return user
|
||||
|
||||
|
||||
def first_membership_tenant(user):
|
||||
return user.memberships.order_by("date_joined").first().tenant
|
||||
|
||||
|
||||
def access_token_for_tenant(user, tenant):
|
||||
access_token = AccessToken.for_user(user)
|
||||
access_token["tenant_id"] = str(tenant.id)
|
||||
access_token.payload["nbf"] = access_token["iat"]
|
||||
return str(access_token)
|
||||
|
||||
|
||||
def authenticate_client_for_tenant(client, user, tenant):
|
||||
client.user = user
|
||||
client.defaults["HTTP_AUTHORIZATION"] = (
|
||||
f"Bearer {access_token_for_tenant(user, tenant)}"
|
||||
)
|
||||
return client
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def authenticated_client_for_tenant_factory():
|
||||
def create_authenticated_client(user, tenant):
|
||||
return authenticate_client_for_tenant(Client(), user, tenant)
|
||||
|
||||
return create_authenticated_client
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def authenticated_client_rbac_manage_account(
|
||||
create_test_user_rbac_manage_account, tenants_fixture, client
|
||||
create_test_user_rbac_manage_account, client
|
||||
):
|
||||
client.user = create_test_user_rbac_manage_account
|
||||
serializer = TokenSerializer(
|
||||
data={
|
||||
"type": "tokens",
|
||||
"email": "rbac_manage_account@rbac.com",
|
||||
"password": TEST_PASSWORD,
|
||||
}
|
||||
return authenticate_client_for_tenant(
|
||||
client,
|
||||
create_test_user_rbac_manage_account,
|
||||
first_membership_tenant(create_test_user_rbac_manage_account),
|
||||
)
|
||||
serializer.is_valid()
|
||||
access_token = serializer.validated_data["access"]
|
||||
client.defaults["HTTP_AUTHORIZATION"] = f"Bearer {access_token}"
|
||||
return client
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
@@ -410,86 +435,43 @@ def create_test_user_rbac_manage_users_only(django_db_setup, django_db_blocker):
|
||||
def authenticated_client_rbac_manage_users_only(
|
||||
create_test_user_rbac_manage_users_only, client
|
||||
):
|
||||
client.user = create_test_user_rbac_manage_users_only
|
||||
serializer = TokenSerializer(
|
||||
data={
|
||||
"type": "tokens",
|
||||
"email": "rbac_manage_users_only@rbac.com",
|
||||
"password": TEST_PASSWORD,
|
||||
}
|
||||
return authenticate_client_for_tenant(
|
||||
client,
|
||||
create_test_user_rbac_manage_users_only,
|
||||
first_membership_tenant(create_test_user_rbac_manage_users_only),
|
||||
)
|
||||
serializer.is_valid()
|
||||
access_token = serializer.validated_data["access"]
|
||||
client.defaults["HTTP_AUTHORIZATION"] = f"Bearer {access_token}"
|
||||
return client
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def authenticated_client_rbac(create_test_user_rbac, tenants_fixture, client):
|
||||
client.user = create_test_user_rbac
|
||||
tenant_id = tenants_fixture[0].id
|
||||
serializer = TokenSerializer(
|
||||
data={
|
||||
"type": "tokens",
|
||||
"email": "rbac@rbac.com",
|
||||
"password": TEST_PASSWORD,
|
||||
"tenant_id": tenant_id,
|
||||
}
|
||||
return authenticate_client_for_tenant(
|
||||
client, create_test_user_rbac, tenants_fixture[0]
|
||||
)
|
||||
serializer.is_valid(raise_exception=True)
|
||||
access_token = serializer.validated_data["access"]
|
||||
client.defaults["HTTP_AUTHORIZATION"] = f"Bearer {access_token}"
|
||||
return client
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def authenticated_client_rbac_noroles(
|
||||
create_test_user_rbac_no_roles, tenants_fixture, client
|
||||
):
|
||||
client.user = create_test_user_rbac_no_roles
|
||||
serializer = TokenSerializer(
|
||||
data={
|
||||
"type": "tokens",
|
||||
"email": "rbac_noroles@rbac.com",
|
||||
"password": TEST_PASSWORD,
|
||||
}
|
||||
return authenticate_client_for_tenant(
|
||||
client, create_test_user_rbac_no_roles, tenants_fixture[0]
|
||||
)
|
||||
serializer.is_valid()
|
||||
access_token = serializer.validated_data["access"]
|
||||
client.defaults["HTTP_AUTHORIZATION"] = f"Bearer {access_token}"
|
||||
return client
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def authenticated_client_no_permissions_rbac(
|
||||
create_test_user_rbac_limited, tenants_fixture, client
|
||||
):
|
||||
client.user = create_test_user_rbac_limited
|
||||
serializer = TokenSerializer(
|
||||
data={
|
||||
"type": "tokens",
|
||||
"email": "rbac_limited@rbac.com",
|
||||
"password": TEST_PASSWORD,
|
||||
}
|
||||
return authenticate_client_for_tenant(
|
||||
client, create_test_user_rbac_limited, tenants_fixture[0]
|
||||
)
|
||||
serializer.is_valid()
|
||||
access_token = serializer.validated_data["access"]
|
||||
client.defaults["HTTP_AUTHORIZATION"] = f"Bearer {access_token}"
|
||||
return client
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def authenticated_client(
|
||||
create_test_user, tenants_fixture, set_user_admin_roles_fixture, client
|
||||
):
|
||||
client.user = create_test_user
|
||||
serializer = TokenSerializer(
|
||||
data={"type": "tokens", "email": TEST_USER, "password": TEST_PASSWORD}
|
||||
)
|
||||
serializer.is_valid()
|
||||
access_token = serializer.validated_data["access"]
|
||||
client.defaults["HTTP_AUTHORIZATION"] = f"Bearer {access_token}"
|
||||
return client
|
||||
return authenticate_client_for_tenant(client, create_test_user, tenants_fixture[0])
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -590,109 +572,191 @@ def users_fixture(django_user_model):
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def providers_fixture(tenants_fixture):
|
||||
tenant, *_ = tenants_fixture
|
||||
provider1 = Provider.objects.create(
|
||||
provider="aws",
|
||||
uid="123456789012",
|
||||
alias="aws_testing_1",
|
||||
tenant_id=tenant.id,
|
||||
)
|
||||
provider2 = Provider.objects.create(
|
||||
provider="aws",
|
||||
uid="123456789013",
|
||||
alias="aws_testing_2",
|
||||
tenant_id=tenant.id,
|
||||
)
|
||||
provider3 = Provider.objects.create(
|
||||
provider="gcp",
|
||||
uid="a12322-test321",
|
||||
alias="gcp_testing",
|
||||
tenant_id=tenant.id,
|
||||
)
|
||||
provider4 = Provider.objects.create(
|
||||
provider="kubernetes",
|
||||
uid="kubernetes-test-12345",
|
||||
alias="k8s_testing",
|
||||
tenant_id=tenant.id,
|
||||
)
|
||||
provider5 = Provider.objects.create(
|
||||
provider="azure",
|
||||
uid="37b065f8-26b0-4218-a665-0b23d07b27d9",
|
||||
alias="azure_testing",
|
||||
tenant_id=tenant.id,
|
||||
scanner_args={"key1": "value1", "key2": {"key21": "value21"}},
|
||||
)
|
||||
provider6 = Provider.objects.create(
|
||||
provider="m365",
|
||||
uid="m365.test.com",
|
||||
alias="m365_testing",
|
||||
tenant_id=tenant.id,
|
||||
)
|
||||
provider7 = Provider.objects.create(
|
||||
provider="oraclecloud",
|
||||
uid="ocid1.tenancy.oc1..aaaaaaaa3dwoazoox4q7wrvriywpokp5grlhgnkwtyt6dmwyou7no6mdmzda",
|
||||
alias="oci_testing",
|
||||
tenant_id=tenant.id,
|
||||
)
|
||||
provider8 = Provider.objects.create(
|
||||
provider="mongodbatlas",
|
||||
uid="64b1d3c0e4b03b1234567890",
|
||||
alias="mongodbatlas_testing",
|
||||
tenant_id=tenant.id,
|
||||
)
|
||||
provider9 = Provider.objects.create(
|
||||
provider="alibabacloud",
|
||||
uid="1234567890123456",
|
||||
alias="alibabacloud_testing",
|
||||
tenant_id=tenant.id,
|
||||
)
|
||||
provider10 = Provider.objects.create(
|
||||
provider="cloudflare",
|
||||
uid="a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4",
|
||||
alias="cloudflare_testing",
|
||||
tenant_id=tenant.id,
|
||||
)
|
||||
provider11 = Provider.objects.create(
|
||||
provider="openstack",
|
||||
uid="a1b2c3d4-e5f6-7890-abcd-ef1234567890",
|
||||
alias="openstack_testing",
|
||||
tenant_id=tenant.id,
|
||||
)
|
||||
provider12 = Provider.objects.create(
|
||||
provider="googleworkspace",
|
||||
uid="C12345678",
|
||||
alias="googleworkspace_testing",
|
||||
tenant_id=tenant.id,
|
||||
)
|
||||
provider13 = Provider.objects.create(
|
||||
provider="vercel",
|
||||
uid="team_abcdef1234567890ab",
|
||||
alias="vercel_testing",
|
||||
tenant_id=tenant.id,
|
||||
)
|
||||
provider14 = Provider.objects.create(
|
||||
provider="okta",
|
||||
uid="acme.okta.com",
|
||||
alias="okta_testing",
|
||||
tenant_id=tenant.id,
|
||||
def provider_factory(tenants_fixture):
|
||||
tenant = tenants_fixture[0]
|
||||
counters = {}
|
||||
|
||||
def next_counter(provider):
|
||||
counters[provider] = counters.get(provider, 0) + 1
|
||||
return counters[provider]
|
||||
|
||||
def defaults_for(provider, sequence):
|
||||
return {
|
||||
Provider.ProviderChoices.AWS.value: {
|
||||
"uid": f"{123456789011 + sequence:012d}",
|
||||
"alias": f"aws_testing_{sequence}",
|
||||
},
|
||||
Provider.ProviderChoices.AZURE.value: {
|
||||
"uid": str(uuid4()),
|
||||
"alias": f"azure_testing_{sequence}",
|
||||
"scanner_args": {"key1": "value1", "key2": {"key21": "value21"}},
|
||||
},
|
||||
Provider.ProviderChoices.GCP.value: {
|
||||
"uid": f"a12322-test{sequence:05d}",
|
||||
"alias": f"gcp_testing_{sequence}",
|
||||
},
|
||||
Provider.ProviderChoices.KUBERNETES.value: {
|
||||
"uid": f"kubernetes-test-{sequence}",
|
||||
"alias": f"k8s_testing_{sequence}",
|
||||
},
|
||||
Provider.ProviderChoices.M365.value: {
|
||||
"uid": f"m365-{sequence}.test.com",
|
||||
"alias": f"m365_testing_{sequence}",
|
||||
},
|
||||
Provider.ProviderChoices.GITHUB.value: {
|
||||
"uid": f"github-test-{sequence}",
|
||||
"alias": f"github_testing_{sequence}",
|
||||
},
|
||||
Provider.ProviderChoices.MONGODBATLAS.value: {
|
||||
"uid": f"64b1d3c0e4b03b{sequence:010x}",
|
||||
"alias": f"mongodbatlas_testing_{sequence}",
|
||||
},
|
||||
Provider.ProviderChoices.IAC.value: {
|
||||
"uid": f"https://github.com/prowler-cloud/test-{sequence}.git",
|
||||
"alias": f"iac_testing_{sequence}",
|
||||
},
|
||||
Provider.ProviderChoices.ORACLECLOUD.value: {
|
||||
"uid": f"ocid1.tenancy.oc1..aaaaaaaa{sequence:024d}",
|
||||
"alias": f"oci_testing_{sequence}",
|
||||
},
|
||||
Provider.ProviderChoices.ALIBABACLOUD.value: {
|
||||
"uid": f"{1234567890123455 + sequence:016d}",
|
||||
"alias": f"alibabacloud_testing_{sequence}",
|
||||
},
|
||||
Provider.ProviderChoices.CLOUDFLARE.value: {
|
||||
"uid": f"{0x1000000000000000000000000000000 + sequence:032x}",
|
||||
"alias": f"cloudflare_testing_{sequence}",
|
||||
},
|
||||
Provider.ProviderChoices.OPENSTACK.value: {
|
||||
"uid": f"openstack-project-{sequence}",
|
||||
"alias": f"openstack_testing_{sequence}",
|
||||
},
|
||||
Provider.ProviderChoices.IMAGE.value: {
|
||||
"uid": f"registry.example.com/prowler/test:{sequence}",
|
||||
"alias": f"image_testing_{sequence}",
|
||||
},
|
||||
Provider.ProviderChoices.GOOGLEWORKSPACE.value: {
|
||||
"uid": f"C{12345677 + sequence}",
|
||||
"alias": f"googleworkspace_testing_{sequence}",
|
||||
},
|
||||
Provider.ProviderChoices.VERCEL.value: {
|
||||
"uid": f"team_{sequence:016x}",
|
||||
"alias": f"vercel_testing_{sequence}",
|
||||
},
|
||||
Provider.ProviderChoices.OKTA.value: {
|
||||
"uid": f"acme-{sequence}.okta.com",
|
||||
"alias": f"okta_testing_{sequence}",
|
||||
},
|
||||
}[provider]
|
||||
|
||||
def create_provider(provider=Provider.ProviderChoices.AWS.value, **overrides):
|
||||
provider_value = getattr(provider, "value", provider)
|
||||
selected_tenant = overrides.pop("tenant", tenant)
|
||||
sequence = next_counter(provider_value)
|
||||
attributes = {
|
||||
"provider": provider_value,
|
||||
"tenant_id": selected_tenant.id,
|
||||
**defaults_for(provider_value, sequence),
|
||||
}
|
||||
attributes.update(overrides)
|
||||
return Provider.objects.create(**attributes)
|
||||
|
||||
return create_provider
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def aws_provider(provider_factory):
|
||||
return provider_factory(Provider.ProviderChoices.AWS.value)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def aws_provider_pair(aws_provider, provider_factory):
|
||||
return (
|
||||
aws_provider,
|
||||
provider_factory(Provider.ProviderChoices.AWS.value),
|
||||
)
|
||||
|
||||
return (
|
||||
provider1,
|
||||
provider2,
|
||||
provider3,
|
||||
provider4,
|
||||
provider5,
|
||||
provider6,
|
||||
provider7,
|
||||
provider8,
|
||||
provider9,
|
||||
provider10,
|
||||
provider11,
|
||||
provider12,
|
||||
provider13,
|
||||
provider14,
|
||||
|
||||
@pytest.fixture
|
||||
def azure_provider(provider_factory):
|
||||
return provider_factory(Provider.ProviderChoices.AZURE.value)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def gcp_provider(provider_factory):
|
||||
return provider_factory(Provider.ProviderChoices.GCP.value)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def kubernetes_provider(provider_factory):
|
||||
return provider_factory(Provider.ProviderChoices.KUBERNETES.value)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def m365_provider(provider_factory):
|
||||
return provider_factory(Provider.ProviderChoices.M365.value)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def github_provider(provider_factory):
|
||||
return provider_factory(Provider.ProviderChoices.GITHUB.value)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mongodbatlas_provider(provider_factory):
|
||||
return provider_factory(Provider.ProviderChoices.MONGODBATLAS.value)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def iac_provider(provider_factory):
|
||||
return provider_factory(Provider.ProviderChoices.IAC.value)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def oraclecloud_provider(provider_factory):
|
||||
return provider_factory(Provider.ProviderChoices.ORACLECLOUD.value)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def alibabacloud_provider(provider_factory):
|
||||
return provider_factory(Provider.ProviderChoices.ALIBABACLOUD.value)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def cloudflare_provider(provider_factory):
|
||||
return provider_factory(Provider.ProviderChoices.CLOUDFLARE.value)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def openstack_provider(provider_factory):
|
||||
return provider_factory(Provider.ProviderChoices.OPENSTACK.value)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def image_provider(provider_factory):
|
||||
return provider_factory(Provider.ProviderChoices.IMAGE.value)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def googleworkspace_provider(provider_factory):
|
||||
return provider_factory(Provider.ProviderChoices.GOOGLEWORKSPACE.value)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def vercel_provider(provider_factory):
|
||||
return provider_factory(Provider.ProviderChoices.VERCEL.value)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def okta_provider(provider_factory):
|
||||
return provider_factory(Provider.ProviderChoices.OKTA.value)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def all_provider_types_fixture(provider_factory):
|
||||
return tuple(
|
||||
provider_factory(provider_choice.value)
|
||||
for provider_choice in Provider.ProviderChoices
|
||||
)
|
||||
|
||||
|
||||
@@ -797,7 +861,7 @@ def roles_fixture(tenants_fixture):
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def provider_secret_fixture(providers_fixture):
|
||||
def provider_secret_fixture(all_provider_types_fixture):
|
||||
return tuple(
|
||||
ProviderSecret.objects.create(
|
||||
tenant_id=provider.tenant_id,
|
||||
@@ -806,14 +870,14 @@ def provider_secret_fixture(providers_fixture):
|
||||
secret={"key": "value"},
|
||||
name=provider.alias,
|
||||
)
|
||||
for provider in providers_fixture
|
||||
for provider in all_provider_types_fixture
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def scans_fixture(tenants_fixture, providers_fixture):
|
||||
def scans_fixture(tenants_fixture, aws_provider_pair):
|
||||
tenant, *_ = tenants_fixture
|
||||
provider, provider2, *_ = providers_fixture
|
||||
provider, provider2 = aws_provider_pair
|
||||
|
||||
now = datetime.now(UTC)
|
||||
|
||||
@@ -876,8 +940,8 @@ def tasks_fixture(tenants_fixture):
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def resources_fixture(providers_fixture):
|
||||
provider, *_ = providers_fixture
|
||||
def resources_fixture(aws_provider_pair):
|
||||
provider, provider2 = aws_provider_pair
|
||||
|
||||
tags = [
|
||||
ResourceTag.objects.create(
|
||||
@@ -918,8 +982,8 @@ def resources_fixture(providers_fixture):
|
||||
resource2.upsert_or_delete_tags(tags)
|
||||
|
||||
resource3 = Resource.objects.create(
|
||||
tenant_id=providers_fixture[1].tenant_id,
|
||||
provider=providers_fixture[1],
|
||||
tenant_id=provider2.tenant_id,
|
||||
provider=provider2,
|
||||
uid="arn:aws:ec2:us-east-1:123456789012:bucket/i-1234567890abcdef2",
|
||||
name="My Bucket 3",
|
||||
region="us-east-1",
|
||||
@@ -1267,9 +1331,9 @@ def get_api_tokens(
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def scan_summaries_fixture(tenants_fixture, providers_fixture):
|
||||
def scan_summaries_fixture(tenants_fixture, aws_provider):
|
||||
tenant = tenants_fixture[0]
|
||||
provider = providers_fixture[0]
|
||||
provider = aws_provider
|
||||
scan = Scan.objects.create(
|
||||
name="overview scan",
|
||||
provider=provider,
|
||||
@@ -1346,8 +1410,8 @@ def scan_summaries_fixture(tenants_fixture, providers_fixture):
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def integrations_fixture(providers_fixture):
|
||||
provider1, provider2, *_ = providers_fixture
|
||||
def integrations_fixture(aws_provider_pair):
|
||||
provider1, provider2 = aws_provider_pair
|
||||
tenant_id = provider1.tenant_id
|
||||
integration1 = Integration.objects.create(
|
||||
tenant_id=tenant_id,
|
||||
@@ -1408,9 +1472,9 @@ def lighthouse_config_fixture(authenticated_client, tenants_fixture):
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def latest_scan_finding(authenticated_client, providers_fixture, resources_fixture):
|
||||
provider = providers_fixture[0]
|
||||
tenant_id = str(providers_fixture[0].tenant_id)
|
||||
def latest_scan_finding(authenticated_client, aws_provider, resources_fixture):
|
||||
provider = aws_provider
|
||||
tenant_id = str(aws_provider.tenant_id)
|
||||
resource = resources_fixture[0]
|
||||
scan = Scan.objects.create(
|
||||
name="latest completed scan",
|
||||
@@ -1521,10 +1585,10 @@ def findings_with_multiple_categories(scans_fixture, resources_fixture):
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def latest_scan_finding_with_categories(
|
||||
authenticated_client, providers_fixture, resources_fixture
|
||||
authenticated_client, aws_provider, resources_fixture
|
||||
):
|
||||
provider = providers_fixture[0]
|
||||
tenant_id = str(providers_fixture[0].tenant_id)
|
||||
provider = aws_provider
|
||||
tenant_id = str(aws_provider.tenant_id)
|
||||
resource = resources_fixture[0]
|
||||
scan = Scan.objects.create(
|
||||
name="latest completed scan with categories",
|
||||
@@ -1558,9 +1622,9 @@ def latest_scan_finding_with_categories(
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def latest_scan_resource(authenticated_client, providers_fixture):
|
||||
provider = providers_fixture[0]
|
||||
tenant_id = str(providers_fixture[0].tenant_id)
|
||||
def latest_scan_resource(authenticated_client, aws_provider):
|
||||
provider = aws_provider
|
||||
tenant_id = str(aws_provider.tenant_id)
|
||||
scan = Scan.objects.create(
|
||||
name="latest completed scan for resource",
|
||||
provider=provider,
|
||||
@@ -2025,11 +2089,11 @@ def get_authorization_header(access_token: str) -> dict:
|
||||
|
||||
@pytest.fixture
|
||||
def provider_compliance_scores_fixture(
|
||||
tenants_fixture, providers_fixture, scans_fixture
|
||||
tenants_fixture, aws_provider_pair, scans_fixture
|
||||
):
|
||||
"""Create ProviderComplianceScore entries for compliance watchlist tests."""
|
||||
tenant = tenants_fixture[0]
|
||||
provider1, provider2, *_ = providers_fixture
|
||||
provider1, provider2 = aws_provider_pair
|
||||
scan1, _, scan3 = scans_fixture
|
||||
|
||||
scan1.completed_at = datetime.now(UTC) - timedelta(hours=1)
|
||||
@@ -2126,9 +2190,7 @@ def tenant_compliance_summary_fixture(tenants_fixture):
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def finding_groups_fixture(
|
||||
tenants_fixture, providers_fixture, scans_fixture, resources_fixture
|
||||
):
|
||||
def finding_groups_fixture(tenants_fixture, scans_fixture, resources_fixture):
|
||||
"""
|
||||
Create a comprehensive set of findings for testing Finding Groups aggregation.
|
||||
|
||||
@@ -2147,7 +2209,6 @@ def finding_groups_fixture(
|
||||
- Finding counts (pass, fail, muted, new, changed)
|
||||
"""
|
||||
tenant = tenants_fixture[0]
|
||||
provider1, provider2, *_ = providers_fixture
|
||||
scan1, scan2, *_ = scans_fixture
|
||||
resource1, resource2, *_ = resources_fixture
|
||||
|
||||
@@ -2398,7 +2459,7 @@ def finding_groups_fixture(
|
||||
|
||||
@pytest.fixture
|
||||
def finding_groups_title_variants_fixture(
|
||||
tenants_fixture, providers_fixture, scans_fixture, resources_fixture
|
||||
tenants_fixture, scans_fixture, resources_fixture
|
||||
):
|
||||
"""
|
||||
Two providers report the same check_id with different checktitle values.
|
||||
@@ -2409,7 +2470,6 @@ def finding_groups_title_variants_fixture(
|
||||
of which title variant matches the search term.
|
||||
"""
|
||||
tenant = tenants_fixture[0]
|
||||
provider1, provider2, *_ = providers_fixture
|
||||
scan1, scan2, *_ = scans_fixture
|
||||
resource1, resource2, *_ = resources_fixture
|
||||
|
||||
@@ -2483,8 +2543,27 @@ def pytest_collection_modifyitems(items):
|
||||
"""Ensure test_rbac.py is executed first."""
|
||||
items.sort(key=lambda item: 0 if "test_rbac.py" in item.nodeid else 1)
|
||||
|
||||
if any(item.get_closest_marker("requires_test_replica_alias") for item in items):
|
||||
default_database = settings.DATABASES["default"]
|
||||
if TEST_REPLICA_ALIAS not in settings.DATABASES:
|
||||
settings.DATABASES[TEST_REPLICA_ALIAS] = {
|
||||
**default_database,
|
||||
"TEST": {
|
||||
**default_database.get("TEST", {}),
|
||||
"MIRROR": "default",
|
||||
},
|
||||
}
|
||||
django_connections.databases[TEST_REPLICA_ALIAS] = settings.DATABASES[
|
||||
TEST_REPLICA_ALIAS
|
||||
]
|
||||
|
||||
|
||||
def pytest_configure(config):
|
||||
config.addinivalue_line(
|
||||
"markers",
|
||||
"requires_test_replica_alias: creates a test-only replica alias mirrored "
|
||||
"to default",
|
||||
)
|
||||
# Apply the mock before the test session starts. This is necessary to avoid admin error when running the
|
||||
# 0004_rbac_missing_admin_roles migration
|
||||
patch("api.db_router.MainRouter.admin_db", new="default").start()
|
||||
|
||||
@@ -8,6 +8,8 @@ import aioboto3
|
||||
import boto3
|
||||
import botocore
|
||||
import neo4j
|
||||
import neo4j.exceptions
|
||||
from api.attack_paths.database import DATABASE_NOT_FOUND_CODE
|
||||
from api.models import (
|
||||
AttackPathsScan as ProwlerAPIAttackPathsScan,
|
||||
)
|
||||
@@ -347,6 +349,12 @@ def sync_aws_account(
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
if (
|
||||
isinstance(e, neo4j.exceptions.Neo4jError)
|
||||
and e.code == DATABASE_NOT_FOUND_CODE
|
||||
):
|
||||
raise
|
||||
|
||||
logger.info(
|
||||
f"Synced function {func_name} for AWS account {prowler_api_provider.uid} in {time.perf_counter() - func_t0:.3f}s (FAILED)"
|
||||
)
|
||||
|
||||
@@ -1,40 +1,50 @@
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from functools import partial
|
||||
|
||||
from api.attack_paths import database as graph_database
|
||||
from api.db_router import MainRouter
|
||||
from api.db_utils import rls_transaction
|
||||
from api.models import AttackPathsScan, StateChoices
|
||||
from celery import states
|
||||
from celery import current_app, states
|
||||
from celery.utils.log import get_task_logger
|
||||
from config.django.base import ATTACK_PATHS_SCAN_STALE_THRESHOLD_MINUTES
|
||||
from config.django.base import (
|
||||
ATTACK_PATHS_SCAN_INACTIVITY_THRESHOLD_MINUTES,
|
||||
ATTACK_PATHS_SCAN_STALE_THRESHOLD_MINUTES,
|
||||
)
|
||||
from django.db import DatabaseError
|
||||
from django.db.transaction import on_commit
|
||||
from tasks.jobs.attack_paths.db_utils import (
|
||||
mark_scan_finished,
|
||||
recover_graph_data_ready,
|
||||
)
|
||||
from tasks.jobs.orphan_recovery import is_worker_alive as _is_worker_alive
|
||||
from tasks.jobs.orphan_recovery import revoke_task as _revoke_task
|
||||
|
||||
logger = get_task_logger(__name__)
|
||||
|
||||
WORKER_PING_BASE_TIMEOUT_SECONDS = 5
|
||||
WORKER_PING_MAX_ATTEMPTS = 3
|
||||
|
||||
|
||||
def cleanup_stale_attack_paths_scans() -> dict:
|
||||
"""
|
||||
Mark stale `AttackPathsScan` rows as `FAILED`.
|
||||
|
||||
Covers two stuck-state scenarios:
|
||||
1. `EXECUTING` scans whose workers are dead, or that have exceeded the
|
||||
stale threshold while alive.
|
||||
2. `SCHEDULED` scans that never made it to a worker — parent scan
|
||||
1. `EXECUTING` scans whose workers are unresponsive and whose rows have
|
||||
stopped receiving progress updates, or that exceeded the stale threshold.
|
||||
2. `SCHEDULED` scans that never made it to a worker - parent scan
|
||||
crashed before dispatch, broker lost the message, etc. Detected by
|
||||
age plus the parent `Scan` no longer being in flight.
|
||||
"""
|
||||
threshold = timedelta(minutes=ATTACK_PATHS_SCAN_STALE_THRESHOLD_MINUTES)
|
||||
now = datetime.now(tz=UTC)
|
||||
cutoff = now - threshold
|
||||
stale_cutoff = now - timedelta(minutes=ATTACK_PATHS_SCAN_STALE_THRESHOLD_MINUTES)
|
||||
inactivity_cutoff = now - timedelta(
|
||||
minutes=ATTACK_PATHS_SCAN_INACTIVITY_THRESHOLD_MINUTES
|
||||
)
|
||||
|
||||
cleaned_up: list[str] = []
|
||||
cleaned_up.extend(_cleanup_stale_executing_scans(cutoff))
|
||||
cleaned_up.extend(_cleanup_stale_scheduled_scans(cutoff))
|
||||
cleaned_up.extend(_cleanup_stale_executing_scans(stale_cutoff, inactivity_cutoff))
|
||||
cleaned_up.extend(_cleanup_stale_scheduled_scans(stale_cutoff))
|
||||
|
||||
logger.info(
|
||||
f"Stale `AttackPathsScan` cleanup: {len(cleaned_up)} scan(s) cleaned up"
|
||||
@@ -42,13 +52,57 @@ def cleanup_stale_attack_paths_scans() -> dict:
|
||||
return {"cleaned_up_count": len(cleaned_up), "scan_ids": cleaned_up}
|
||||
|
||||
|
||||
def _cleanup_stale_executing_scans(cutoff: datetime) -> list[str]:
|
||||
def _ping_workers(workers: set[str]) -> tuple[set[str], set[str] | None]:
|
||||
"""Ping worker destinations in parallel and retry only missing workers.
|
||||
|
||||
The second tuple item is `None` when the final ping attempt raises. In that
|
||||
case the pending workers have unknown liveness and their scans must be kept.
|
||||
"""
|
||||
pending = set(workers)
|
||||
responsive: set[str] = set()
|
||||
|
||||
for attempt in range(WORKER_PING_MAX_ATTEMPTS):
|
||||
if not pending:
|
||||
return responsive, set()
|
||||
|
||||
timeout = WORKER_PING_BASE_TIMEOUT_SECONDS * 2**attempt
|
||||
try:
|
||||
response = current_app.control.inspect(
|
||||
destination=sorted(pending), timeout=timeout
|
||||
).ping()
|
||||
except Exception:
|
||||
attempts_remaining = WORKER_PING_MAX_ATTEMPTS - attempt - 1
|
||||
if attempts_remaining:
|
||||
logger.warning(
|
||||
f"Attack Paths worker ping attempt {attempt + 1} failed; "
|
||||
f"retrying pending workers with {attempts_remaining} "
|
||||
"attempt(s) remaining",
|
||||
exc_info=True,
|
||||
)
|
||||
continue
|
||||
|
||||
logger.exception(
|
||||
"Attack Paths worker ping attempts exhausted; preserving scans "
|
||||
"for workers with unknown liveness"
|
||||
)
|
||||
return responsive, None
|
||||
|
||||
responded = pending.intersection((response or {}).keys())
|
||||
responsive.update(responded)
|
||||
pending.difference_update(responded)
|
||||
|
||||
return responsive, pending
|
||||
|
||||
|
||||
def _cleanup_stale_executing_scans(
|
||||
stale_cutoff: datetime, inactivity_cutoff: datetime
|
||||
) -> list[str]:
|
||||
"""
|
||||
Two-pass detection for `EXECUTING` scans:
|
||||
1. If `TaskResult.worker` exists, ping the worker.
|
||||
- Dead worker: cleanup immediately (any age).
|
||||
- Alive + past threshold: revoke the task, then cleanup.
|
||||
- Alive + within threshold: skip.
|
||||
1. Ping all recorded workers in parallel with bounded retries.
|
||||
- Responsive + past stale threshold: cleanup.
|
||||
- Unresponsive + past inactivity threshold: cleanup.
|
||||
- Unknown after a final ping exception: preserve.
|
||||
2. If no worker field: fall back to time-based heuristic only.
|
||||
"""
|
||||
executing_scans = list(
|
||||
@@ -57,14 +111,13 @@ def _cleanup_stale_executing_scans(cutoff: datetime) -> list[str]:
|
||||
.select_related("task__task_runner_task")
|
||||
)
|
||||
|
||||
# Cache worker liveness so each worker is pinged at most once
|
||||
workers = {
|
||||
tr.worker
|
||||
for scan in executing_scans
|
||||
if (tr := getattr(scan.task, "task_runner_task", None) if scan.task else None)
|
||||
and tr.worker
|
||||
}
|
||||
worker_alive = {w: _is_worker_alive(w) for w in workers}
|
||||
responsive_workers, unresponsive_workers = _ping_workers(workers)
|
||||
|
||||
cleaned_up: list[str] = []
|
||||
|
||||
@@ -75,27 +128,50 @@ def _cleanup_stale_executing_scans(cutoff: datetime) -> list[str]:
|
||||
worker = task_result.worker if task_result else None
|
||||
|
||||
if worker:
|
||||
alive = worker_alive.get(worker, True)
|
||||
|
||||
if alive:
|
||||
if scan.started_at and scan.started_at >= cutoff:
|
||||
if worker in responsive_workers:
|
||||
if scan.started_at is None or scan.started_at >= stale_cutoff:
|
||||
continue
|
||||
|
||||
# Alive but stale — revoke before cleanup
|
||||
_revoke_task(task_result)
|
||||
reason = "Scan exceeded stale threshold — cleaned up by periodic task"
|
||||
reason = "Scan exceeded stale threshold - cleaned up by periodic task"
|
||||
recheck_activity_cutoff = None
|
||||
elif unresponsive_workers is None or worker not in unresponsive_workers:
|
||||
logger.info(
|
||||
f"Preserving scan {scan.id}: worker {worker} liveness is "
|
||||
f"unknown (progress={scan.progress}, updated_at={scan.updated_at})"
|
||||
)
|
||||
continue
|
||||
else:
|
||||
reason = "Worker dead — cleaned up by periodic task"
|
||||
if scan.updated_at >= inactivity_cutoff:
|
||||
logger.info(
|
||||
f"Preserving scan {scan.id}: worker {worker} is unresponsive "
|
||||
f"but activity is recent (progress={scan.progress}, "
|
||||
f"updated_at={scan.updated_at})"
|
||||
)
|
||||
continue
|
||||
|
||||
reason = (
|
||||
"Worker unresponsive and scan inactive for "
|
||||
f"{ATTACK_PATHS_SCAN_INACTIVITY_THRESHOLD_MINUTES} minutes - "
|
||||
"cleaned up by periodic task"
|
||||
)
|
||||
recheck_activity_cutoff = inactivity_cutoff
|
||||
else:
|
||||
# No worker recorded, time-based heuristic only
|
||||
if scan.started_at and scan.started_at >= cutoff:
|
||||
if scan.started_at is None or scan.started_at >= stale_cutoff:
|
||||
continue
|
||||
reason = (
|
||||
"No worker recorded, scan exceeded stale threshold — "
|
||||
"No worker recorded, scan exceeded stale threshold - "
|
||||
"cleaned up by periodic task"
|
||||
)
|
||||
recheck_activity_cutoff = None
|
||||
|
||||
if _cleanup_scan(scan, task_result, reason):
|
||||
if _cleanup_scan(
|
||||
scan,
|
||||
task_result,
|
||||
reason,
|
||||
revoke=worker is not None,
|
||||
inactivity_cutoff=recheck_activity_cutoff,
|
||||
):
|
||||
cleaned_up.append(str(scan.id))
|
||||
|
||||
return cleaned_up
|
||||
@@ -112,10 +188,9 @@ def _cleanup_stale_scheduled_scans(cutoff: datetime) -> list[str]:
|
||||
avoids cleaning up rows whose parent Prowler scan is legitimately still
|
||||
running.
|
||||
|
||||
For each match: revoke the queued task (best-effort; harmless if already
|
||||
consumed), atomically flip to `FAILED`, and mark the `TaskResult`. The
|
||||
temp Neo4j database is never created while `SCHEDULED`, so no drop is
|
||||
needed.
|
||||
For each match: lock and recheck the row, mark the scan and `TaskResult` as
|
||||
failed, then revoke the queued task after the transaction commits. The temp
|
||||
Neo4j database is never created while `SCHEDULED`, so no drop is needed.
|
||||
"""
|
||||
scheduled_scans = list(
|
||||
AttackPathsScan.all_objects.using(MainRouter.admin_db)
|
||||
@@ -141,42 +216,54 @@ def _cleanup_stale_scheduled_scans(cutoff: datetime) -> list[str]:
|
||||
task_result = (
|
||||
getattr(scan.task, "task_runner_task", None) if scan.task else None
|
||||
)
|
||||
if task_result:
|
||||
_revoke_task(task_result, terminate=False)
|
||||
|
||||
reason = "Scan never started — cleaned up by periodic task"
|
||||
reason = "Scan never started - cleaned up by periodic task"
|
||||
if _cleanup_scheduled_scan(scan, task_result, reason):
|
||||
cleaned_up.append(str(scan.id))
|
||||
|
||||
return cleaned_up
|
||||
|
||||
|
||||
def _cleanup_scan(scan, task_result, reason: str) -> bool:
|
||||
def _cleanup_scan(
|
||||
scan,
|
||||
task_result,
|
||||
reason: str,
|
||||
*,
|
||||
revoke: bool = False,
|
||||
inactivity_cutoff: datetime | None = None,
|
||||
) -> bool:
|
||||
"""
|
||||
Clean up a single stale `AttackPathsScan`:
|
||||
drop temp DB, mark `FAILED`, update `TaskResult`, recover `graph_data_ready`.
|
||||
lock and recheck, mark `FAILED`, revoke after commit, drop the temp DB, and
|
||||
recover graph readiness.
|
||||
|
||||
Returns `True` if the scan was actually cleaned up, `False` if skipped.
|
||||
"""
|
||||
scan_id_str = str(scan.id)
|
||||
|
||||
# Drop temp Neo4j database
|
||||
try:
|
||||
fresh_scan = _finalize_failed_scan(
|
||||
scan,
|
||||
StateChoices.EXECUTING,
|
||||
reason,
|
||||
task_result=task_result,
|
||||
revoke=revoke,
|
||||
inactivity_cutoff=inactivity_cutoff,
|
||||
)
|
||||
except DatabaseError:
|
||||
logger.exception(
|
||||
f"Failed to mark stale Attack Paths scan {scan_id_str} as failed"
|
||||
)
|
||||
return False
|
||||
|
||||
if fresh_scan is None:
|
||||
return False
|
||||
|
||||
tmp_db_name = graph_database.get_database_name(scan.id, temporary=True)
|
||||
try:
|
||||
graph_database.drop_database(tmp_db_name)
|
||||
except Exception:
|
||||
logger.exception(f"Failed to drop temp database {tmp_db_name}")
|
||||
|
||||
fresh_scan = _finalize_failed_scan(scan, StateChoices.EXECUTING, reason)
|
||||
if fresh_scan is None:
|
||||
return False
|
||||
|
||||
# Mark `TaskResult` as `FAILURE` (not RLS-protected, outside lock)
|
||||
if task_result:
|
||||
task_result.status = states.FAILURE
|
||||
task_result.date_done = datetime.now(tz=UTC)
|
||||
task_result.save(update_fields=["status", "date_done"])
|
||||
|
||||
recover_graph_data_ready(fresh_scan)
|
||||
|
||||
logger.info(f"Cleaned up stale scan {scan_id_str}: {reason}")
|
||||
@@ -187,31 +274,49 @@ def _cleanup_scheduled_scan(scan, task_result, reason: str) -> bool:
|
||||
"""
|
||||
Clean up a `SCHEDULED` scan that never reached a worker.
|
||||
|
||||
Skips the temp Neo4j drop — the database is only created once the worker
|
||||
Skips the temp Neo4j drop - the database is only created once the worker
|
||||
enters `EXECUTING`, so dropping it here just produces noisy log output.
|
||||
|
||||
Returns `True` if the scan was actually cleaned up, `False` if skipped.
|
||||
"""
|
||||
scan_id_str = str(scan.id)
|
||||
|
||||
fresh_scan = _finalize_failed_scan(scan, StateChoices.SCHEDULED, reason)
|
||||
if fresh_scan is None:
|
||||
try:
|
||||
fresh_scan = _finalize_failed_scan(
|
||||
scan,
|
||||
StateChoices.SCHEDULED,
|
||||
reason,
|
||||
task_result=task_result,
|
||||
revoke=task_result is not None,
|
||||
terminate=False,
|
||||
)
|
||||
except DatabaseError:
|
||||
logger.exception(
|
||||
f"Failed to mark scheduled Attack Paths scan {scan_id_str} as failed"
|
||||
)
|
||||
return False
|
||||
|
||||
if task_result:
|
||||
task_result.status = states.FAILURE
|
||||
task_result.date_done = datetime.now(tz=UTC)
|
||||
task_result.save(update_fields=["status", "date_done"])
|
||||
if fresh_scan is None:
|
||||
return False
|
||||
|
||||
logger.info(f"Cleaned up scheduled scan {scan_id_str}: {reason}")
|
||||
return True
|
||||
|
||||
|
||||
def _finalize_failed_scan(scan, expected_state: str, reason: str):
|
||||
def _finalize_failed_scan(
|
||||
scan,
|
||||
expected_state: str,
|
||||
reason: str,
|
||||
*,
|
||||
task_result=None,
|
||||
revoke: bool = False,
|
||||
terminate: bool = True,
|
||||
inactivity_cutoff: datetime | None = None,
|
||||
):
|
||||
"""
|
||||
Atomically lock the row, verify it's still in `expected_state`, and
|
||||
mark it `FAILED`. Returns the locked row on success, `None` if the
|
||||
row is gone or has already moved on.
|
||||
Atomically lock the row, verify it's still eligible, and mark it `FAILED`.
|
||||
If requested, register revocation after commit. Returns the locked row on
|
||||
success, `None` if the row is gone or has already moved on.
|
||||
"""
|
||||
scan_id_str = str(scan.id)
|
||||
with rls_transaction(str(scan.tenant_id)):
|
||||
@@ -225,6 +330,23 @@ def _finalize_failed_scan(scan, expected_state: str, reason: str):
|
||||
logger.info(f"Scan {scan_id_str} is now {fresh_scan.state}, skipping")
|
||||
return None
|
||||
|
||||
if inactivity_cutoff is not None and fresh_scan.updated_at >= inactivity_cutoff:
|
||||
logger.info(
|
||||
f"Scan {scan_id_str} received activity during worker checks, skipping"
|
||||
)
|
||||
return None
|
||||
|
||||
mark_scan_finished(fresh_scan, StateChoices.FAILED, {"global_error": reason})
|
||||
|
||||
if task_result:
|
||||
task_result.status = states.FAILURE
|
||||
task_result.date_done = datetime.now(tz=UTC)
|
||||
task_result.save(update_fields=["status", "date_done"])
|
||||
|
||||
if revoke and task_result:
|
||||
on_commit(
|
||||
partial(_revoke_task, task_result, terminate=terminate),
|
||||
using=fresh_scan._state.db,
|
||||
)
|
||||
|
||||
return fresh_scan
|
||||
|
||||
@@ -10,13 +10,10 @@ NormalizedList = _provider_config.NormalizedList
|
||||
PROVIDER_CONFIGS = _provider_config.PROVIDER_CONFIGS
|
||||
ProviderConfig = _provider_config.ProviderConfig
|
||||
|
||||
# Batch size for Neo4j write operations (resource labeling, cleanup)
|
||||
BATCH_SIZE = env.int("ATTACK_PATHS_BATCH_SIZE", 1000)
|
||||
# Batch size for graph mutation operations (resource labeling and subgraph deletion)
|
||||
GRAPH_MUTATION_BATCH_SIZE = env.int("ATTACK_PATHS_GRAPH_MUTATION_BATCH_SIZE", 1000)
|
||||
# Batch size for Postgres findings fetch (keyset pagination page size)
|
||||
FINDINGS_BATCH_SIZE = env.int("ATTACK_PATHS_FINDINGS_BATCH_SIZE", 1000)
|
||||
# Batch size for temp-to-tenant graph sync (nodes and relationships per cursor page)
|
||||
SYNC_BATCH_SIZE = env.int("ATTACK_PATHS_SYNC_BATCH_SIZE", 1000)
|
||||
|
||||
# Neo4j internal labels (Prowler-specific, not provider-specific)
|
||||
# - `Internet`: Singleton node representing external internet access for exposed-resource queries
|
||||
# - `ProwlerFinding`: Label for finding nodes created by Prowler and linked to cloud resources
|
||||
|
||||
@@ -126,14 +126,17 @@ def starting_attack_paths_scan(
|
||||
if locked.state != StateChoices.SCHEDULED:
|
||||
return False
|
||||
|
||||
now = datetime.now(tz=UTC)
|
||||
locked.state = StateChoices.EXECUTING
|
||||
locked.started_at = datetime.now(tz=UTC)
|
||||
locked.started_at = now
|
||||
locked.updated_at = now
|
||||
locked.update_tag = cartography_config.update_tag
|
||||
locked.save(update_fields=["state", "started_at", "update_tag"])
|
||||
locked.save(update_fields=["state", "started_at", "updated_at", "update_tag"])
|
||||
|
||||
# Keep the in-memory object the caller is holding in sync.
|
||||
attack_paths_scan.state = locked.state
|
||||
attack_paths_scan.started_at = locked.started_at
|
||||
attack_paths_scan.updated_at = locked.updated_at
|
||||
attack_paths_scan.update_tag = locked.update_tag
|
||||
return True
|
||||
|
||||
@@ -181,7 +184,8 @@ def update_attack_paths_scan_progress(
|
||||
) -> None:
|
||||
with rls_transaction(attack_paths_scan.tenant_id):
|
||||
attack_paths_scan.progress = progress
|
||||
attack_paths_scan.save(update_fields=["progress"])
|
||||
attack_paths_scan.updated_at = datetime.now(tz=UTC)
|
||||
attack_paths_scan.save(update_fields=["progress", "updated_at"])
|
||||
|
||||
|
||||
def set_graph_data_ready(
|
||||
|
||||
@@ -21,8 +21,8 @@ from cartography.config import Config as CartographyConfig
|
||||
from celery.utils.log import get_task_logger
|
||||
from prowler.config import config as ProwlerConfig
|
||||
from tasks.jobs.attack_paths.config import (
|
||||
BATCH_SIZE,
|
||||
FINDINGS_BATCH_SIZE,
|
||||
GRAPH_MUTATION_BATCH_SIZE,
|
||||
get_node_uid_field,
|
||||
get_provider_resource_label,
|
||||
get_root_node_label,
|
||||
@@ -135,7 +135,7 @@ def add_resource_label(
|
||||
while labeled_count > 0:
|
||||
result = neo4j_session.run(
|
||||
query,
|
||||
{"provider_uid": provider_uid, "batch_size": BATCH_SIZE},
|
||||
{"provider_uid": provider_uid, "batch_size": GRAPH_MUTATION_BATCH_SIZE},
|
||||
)
|
||||
labeled_count = result.single().get("labeled_count", 0)
|
||||
total_labeled += labeled_count
|
||||
|
||||
@@ -372,7 +372,19 @@ def run(tenant_id: str, scan_id: str, task_id: str) -> dict[str, Any]:
|
||||
|
||||
except Exception as e:
|
||||
exception_message = utils.stringify_exception(e, "Attack Paths scan failed")
|
||||
logger.exception(exception_message)
|
||||
temporary_database_missing = (
|
||||
isinstance(e, graph_database.GraphDatabaseQueryException)
|
||||
and e.code == graph_database.DATABASE_NOT_FOUND_CODE
|
||||
and tmp_database_name in str(e)
|
||||
)
|
||||
if temporary_database_missing:
|
||||
logger.warning(exception_message)
|
||||
else:
|
||||
logger.exception(exception_message)
|
||||
cleanup_log_level = (
|
||||
logging.WARNING if temporary_database_missing else logging.ERROR
|
||||
)
|
||||
cleanup_exc_info = not temporary_database_missing
|
||||
ingestion_exceptions["global_error"] = exception_message
|
||||
|
||||
# Recover `graph_data_ready` based on how far the swap got
|
||||
@@ -387,19 +399,24 @@ def run(tenant_id: str, scan_id: str, task_id: str) -> dict[str, Any]:
|
||||
)
|
||||
|
||||
except Exception:
|
||||
logger.error(
|
||||
f"Failed to recover `graph_data_ready` for provider {attack_paths_scan.provider_id}",
|
||||
exc_info=True,
|
||||
logger.log(
|
||||
cleanup_log_level,
|
||||
"Failed to recover `graph_data_ready` for provider "
|
||||
f"{attack_paths_scan.provider_id}",
|
||||
exc_info=cleanup_exc_info,
|
||||
)
|
||||
|
||||
# Dropping the temporary database if it still exists
|
||||
try:
|
||||
graph_database.drop_database(tmp_cartography_config.neo4j_database)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Failed to drop temporary Neo4j database `{tmp_cartography_config.neo4j_database}` during cleanup: {e}",
|
||||
exc_info=True,
|
||||
except Exception as cleanup_error:
|
||||
logger.log(
|
||||
cleanup_log_level,
|
||||
"Failed to drop temporary Neo4j database "
|
||||
f"`{tmp_cartography_config.neo4j_database}` during cleanup: "
|
||||
f"{cleanup_error}",
|
||||
exc_info=cleanup_exc_info,
|
||||
)
|
||||
|
||||
# Set Attack Paths scan state to FAILED
|
||||
@@ -407,10 +424,12 @@ def run(tenant_id: str, scan_id: str, task_id: str) -> dict[str, Any]:
|
||||
db_utils.finish_attack_paths_scan(
|
||||
attack_paths_scan, StateChoices.FAILED, ingestion_exceptions
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Could not mark Attack Paths scan {attack_paths_scan.id} as `FAILED` (row may have been deleted): {e}",
|
||||
exc_info=True,
|
||||
except Exception as cleanup_error:
|
||||
logger.log(
|
||||
cleanup_log_level,
|
||||
f"Could not mark Attack Paths scan {attack_paths_scan.id} as `FAILED` "
|
||||
f"(row may have been deleted): {cleanup_error}",
|
||||
exc_info=cleanup_exc_info,
|
||||
)
|
||||
|
||||
raise
|
||||
|
||||
@@ -30,7 +30,6 @@ from tasks.jobs.attack_paths.config import (
|
||||
PROVIDER_CONFIGS,
|
||||
PROVIDER_ISOLATION_PROPERTIES,
|
||||
PROVIDER_RESOURCE_LABEL,
|
||||
SYNC_BATCH_SIZE,
|
||||
NormalizedList,
|
||||
get_provider_label,
|
||||
get_tenant_label,
|
||||
@@ -116,6 +115,7 @@ def sync_nodes(
|
||||
Source and target sessions are opened sequentially per batch to avoid
|
||||
holding two Bolt connections simultaneously for the entire sync duration.
|
||||
"""
|
||||
batch_size = sink.sync_batch_size
|
||||
t0 = time.perf_counter()
|
||||
last_id = -1
|
||||
parents_synced = 0
|
||||
@@ -137,7 +137,7 @@ def sync_nodes(
|
||||
with graph_database.get_session(source_database) as source_session:
|
||||
result = source_session.run(
|
||||
NODE_FETCH_QUERY,
|
||||
{"last_id": last_id, "batch_size": SYNC_BATCH_SIZE},
|
||||
{"last_id": last_id, "batch_size": batch_size},
|
||||
)
|
||||
for record in result:
|
||||
batch_count += 1
|
||||
@@ -156,17 +156,17 @@ def sync_nodes(
|
||||
|
||||
for labels, batch in parent_groups.items():
|
||||
rendered_labels = _render_labels(labels, extra_labels)
|
||||
for sink_batch in _iter_sink_batches(batch):
|
||||
for sink_batch in _iter_sink_batches(batch, batch_size):
|
||||
sink.write_nodes(target_database, rendered_labels, sink_batch)
|
||||
|
||||
for child_label, batch in child_groups.items():
|
||||
rendered_labels = _render_labels((child_label,), extra_labels)
|
||||
for sink_batch in _iter_sink_batches(batch):
|
||||
for sink_batch in _iter_sink_batches(batch, batch_size):
|
||||
sink.write_nodes(target_database, rendered_labels, sink_batch)
|
||||
children_synced += len(batch)
|
||||
|
||||
for rel_type, batch in rel_groups.items():
|
||||
for sink_batch in _iter_sink_batches(batch):
|
||||
for sink_batch in _iter_sink_batches(batch, batch_size):
|
||||
sink.write_relationships(
|
||||
target_database, rel_type, provider_id, sink_batch
|
||||
)
|
||||
@@ -205,6 +205,7 @@ def sync_relationships(
|
||||
Source and target sessions are opened sequentially per batch to avoid
|
||||
holding two Bolt connections simultaneously for the entire sync duration.
|
||||
"""
|
||||
batch_size = sink.sync_batch_size
|
||||
t0 = time.perf_counter()
|
||||
last_id = -1
|
||||
total_synced = 0
|
||||
@@ -217,7 +218,7 @@ def sync_relationships(
|
||||
with graph_database.get_session(source_database) as source_session:
|
||||
result = source_session.run(
|
||||
RELATIONSHIPS_FETCH_QUERY,
|
||||
{"last_id": last_id, "batch_size": SYNC_BATCH_SIZE},
|
||||
{"last_id": last_id, "batch_size": batch_size},
|
||||
)
|
||||
for record in result:
|
||||
batch_count += 1
|
||||
@@ -229,7 +230,7 @@ def sync_relationships(
|
||||
break
|
||||
|
||||
for rel_type, batch in grouped.items():
|
||||
for sink_batch in _iter_sink_batches(batch):
|
||||
for sink_batch in _iter_sink_batches(batch, batch_size):
|
||||
sink.write_relationships(
|
||||
target_database, rel_type, provider_id, sink_batch
|
||||
)
|
||||
@@ -247,10 +248,9 @@ def sync_relationships(
|
||||
|
||||
def _iter_sink_batches(
|
||||
rows: list[dict[str, Any]],
|
||||
batch_size: int | None = None,
|
||||
batch_size: int,
|
||||
) -> Iterator[list[dict[str, Any]]]:
|
||||
"""Yield final sink write batches after source rows have been transformed."""
|
||||
batch_size = SYNC_BATCH_SIZE if batch_size is None else batch_size
|
||||
if batch_size <= 0:
|
||||
raise ValueError("Sink batch size must be greater than zero")
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user