mirror of
https://github.com/prowler-cloud/prowler.git
synced 2026-06-10 21:42:29 +00:00
Compare commits
171 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 503b5bfe15 | |||
| 3afa98084f | |||
| b0ee914825 | |||
| eabe488437 | |||
| 8104382cc1 | |||
| 592c7bac81 | |||
| 3aefde14aa | |||
| 02f3e77eaf | |||
| bcd7b2d723 | |||
| 86946f3a84 | |||
| fce1e4f3d2 | |||
| 5d490fa185 | |||
| ea847d8824 | |||
| c5f7e80b20 | |||
| f5345a3982 | |||
| b539514d8d | |||
| 9acef41f96 | |||
| c40adce2ff | |||
| 378c2ff7f6 | |||
| d54095abde | |||
| a12cb5b6d6 | |||
| dde42b6a84 | |||
| 3316ec8d23 | |||
| 71220b2696 | |||
| dd730eec94 | |||
| afe2e0a09e | |||
| 507d163a50 | |||
| 530fef5106 | |||
| 5cbbceb3be | |||
| fa189e7eb9 | |||
| fb966213cc | |||
| 097a60ebc9 | |||
| db03556ef6 | |||
| ecc8eaf366 | |||
| 619d1ffc62 | |||
| 9e20cb2e5a | |||
| cb76e77851 | |||
| a24f818547 | |||
| e07687ce67 | |||
| d016039b18 | |||
| ac013ec6fc | |||
| 4ebded6ab1 | |||
| 770269772a | |||
| ab18ddb81a | |||
| cda7f89091 | |||
| 658ae755ae | |||
| 486719737b | |||
| cb9ab03778 | |||
| 96a2262730 | |||
| 69818abdd0 | |||
| d447bdfe54 | |||
| b5095f5dc7 | |||
| 9fe71d1046 | |||
| 547c53e07c | |||
| e1900fc776 | |||
| 3c0cb3cd58 | |||
| e66c9864f5 | |||
| b1f9971617 | |||
| d01f399cb2 | |||
| 2535b55951 | |||
| 0f55d6e21d | |||
| afb666e0da | |||
| 13cd882ed2 | |||
| f65879346b | |||
| 013f2e5d32 | |||
| bcaa95f973 | |||
| 625dd37fd4 | |||
| fee2f84b89 | |||
| 08730b4eb5 | |||
| c183a2a89a | |||
| e97e31c7ca | |||
| ad7be95dc3 | |||
| 04e2d15dd2 | |||
| 143d4b7c29 | |||
| 0c5778d4a1 | |||
| c77d9dd3a9 | |||
| 8783e963d3 | |||
| 5407f3c68e | |||
| 83ec3fa458 | |||
| ac32f03de3 | |||
| 7b11a716b9 | |||
| b2c18b69ee | |||
| 727fafb147 | |||
| 80c94faff9 | |||
| 065827cd38 | |||
| 6bb8dc6168 | |||
| 9e7ecb39fa | |||
| 255ce0e866 | |||
| dce406b39b | |||
| 28c36cc5fc | |||
| 8242b21f34 | |||
| 1897e38c6b | |||
| 3d6aa6c650 | |||
| ee93ad6cbc | |||
| 7f4c02c738 | |||
| d386730770 | |||
| 5784592437 | |||
| 35f263dea6 | |||
| a1637ec46b | |||
| 6c6a6c55cf | |||
| 31b53f091b | |||
| f7a16fff99 | |||
| cb5c9ea1c5 | |||
| cb367da97d | |||
| be2a58dc82 | |||
| 29133f2d7e | |||
| babf18ffea | |||
| b6a34d2220 | |||
| 77dc79df32 | |||
| 91e3c01f51 | |||
| 6cb0edf3e1 | |||
| 7dfafb9337 | |||
| dce05295ef | |||
| 03d4c19ed5 | |||
| 963ece9a0b | |||
| a32eff6946 | |||
| 3bb326133a | |||
| 799826758e | |||
| 1208005a94 | |||
| ecdece9f1e | |||
| 9c2c555628 | |||
| ca2f3ccc1c | |||
| 9ffa0043ab | |||
| e76ecfdd4d | |||
| f11f71bc42 | |||
| 607cfd61ef | |||
| 9c76dafaa4 | |||
| 7b839d9f9e | |||
| f39a82fdf4 | |||
| d1a7eed5fa | |||
| 5be4ec511f | |||
| a0166aede7 | |||
| 1a2a2ea3cc | |||
| e61d1401b9 | |||
| a2789b7fc6 | |||
| 34217492d0 | |||
| ed50ed1e6d | |||
| 186977f81c | |||
| c33f20ad72 | |||
| d0b0c66ef0 | |||
| e849959fd5 | |||
| 7c090a6a07 | |||
| bc4484f269 | |||
| 7601142e42 | |||
| f47310bceb | |||
| 032499c29a | |||
| d7af97b30a | |||
| aa24034ca7 | |||
| ec4eb70539 | |||
| 76a8610121 | |||
| d5e2c930a9 | |||
| 2c4f866e42 | |||
| 31845df1a7 | |||
| d8c1273a57 | |||
| 3317c0a5e0 | |||
| 847645543a | |||
| 76aa65cb61 | |||
| 484a1d1fef | |||
| c8bc0576ea | |||
| 76cda6d777 | |||
| 28978f6db6 | |||
| d4bc6d7531 | |||
| 1bf49747ad | |||
| 2cde4c939d | |||
| 9844379d30 | |||
| 211b1b67f9 | |||
| 864b2099c3 | |||
| 270266c906 | |||
| c8fab497fd | |||
| b0eea61468 | |||
| 463fc32fca |
@@ -48,6 +48,26 @@ POSTGRES_DB=prowler_db
|
||||
# POSTGRES_REPLICA_MAX_ATTEMPTS=3
|
||||
# POSTGRES_REPLICA_RETRY_BASE_DELAY=0.5
|
||||
|
||||
# Neo4j auth
|
||||
NEO4J_HOST=neo4j
|
||||
NEO4J_PORT=7687
|
||||
NEO4J_USER=neo4j
|
||||
NEO4J_PASSWORD=neo4j_password
|
||||
# Neo4j settings
|
||||
NEO4J_DBMS_MAX__DATABASES=1000
|
||||
NEO4J_SERVER_MEMORY_PAGECACHE_SIZE=1G
|
||||
NEO4J_SERVER_MEMORY_HEAP_INITIAL__SIZE=1G
|
||||
NEO4J_SERVER_MEMORY_HEAP_MAX__SIZE=1G
|
||||
NEO4J_POC_EXPORT_FILE_ENABLED=true
|
||||
NEO4J_APOC_IMPORT_FILE_ENABLED=true
|
||||
NEO4J_APOC_IMPORT_FILE_USE_NEO4J_CONFIG=true
|
||||
NEO4J_PLUGINS=["apoc"]
|
||||
NEO4J_DBMS_SECURITY_PROCEDURES_ALLOWLIST=apoc.*
|
||||
NEO4J_DBMS_SECURITY_PROCEDURES_UNRESTRICTED=apoc.*
|
||||
NEO4J_DBMS_CONNECTOR_BOLT_LISTEN_ADDRESS=0.0.0.0:7687
|
||||
# Neo4j Prowler settings
|
||||
ATTACK_PATHS_BATCH_SIZE=1000
|
||||
|
||||
# Celery-Prowler task settings
|
||||
TASK_RETRY_DELAY_SECONDS=0.1
|
||||
TASK_RETRY_ATTEMPTS=5
|
||||
@@ -117,7 +137,6 @@ SENTRY_ENVIRONMENT=local
|
||||
SENTRY_RELEASE=local
|
||||
NEXT_PUBLIC_SENTRY_ENVIRONMENT=${SENTRY_ENVIRONMENT}
|
||||
|
||||
|
||||
#### Prowler release version ####
|
||||
NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v5.16.0
|
||||
|
||||
|
||||
@@ -29,7 +29,7 @@ runs:
|
||||
run: |
|
||||
BRANCH_NAME="${GITHUB_HEAD_REF:-${GITHUB_REF_NAME}}"
|
||||
echo "Using branch: $BRANCH_NAME"
|
||||
sed -i "s|@master|@$BRANCH_NAME|g" pyproject.toml
|
||||
sed -i "s|\(git+https://github.com/prowler-cloud/prowler[^@]*\)@master|\1@$BRANCH_NAME|g" pyproject.toml
|
||||
|
||||
- name: Install poetry
|
||||
shell: bash
|
||||
|
||||
+20
-2
@@ -46,12 +46,22 @@ provider/oci:
|
||||
- changed-files:
|
||||
- any-glob-to-any-file: "prowler/providers/oraclecloud/**"
|
||||
- any-glob-to-any-file: "tests/providers/oraclecloud/**"
|
||||
|
||||
|
||||
provider/alibabacloud:
|
||||
- changed-files:
|
||||
- any-glob-to-any-file: "prowler/providers/alibabacloud/**"
|
||||
- any-glob-to-any-file: "tests/providers/alibabacloud/**"
|
||||
|
||||
provider/cloudflare:
|
||||
- changed-files:
|
||||
- any-glob-to-any-file: "prowler/providers/cloudflare/**"
|
||||
- any-glob-to-any-file: "tests/providers/cloudflare/**"
|
||||
|
||||
provider/openstack:
|
||||
- changed-files:
|
||||
- any-glob-to-any-file: "prowler/providers/openstack/**"
|
||||
- any-glob-to-any-file: "tests/providers/openstack/**"
|
||||
|
||||
github_actions:
|
||||
- changed-files:
|
||||
- any-glob-to-any-file: ".github/workflows/*"
|
||||
@@ -67,15 +77,23 @@ mutelist:
|
||||
- any-glob-to-any-file: "prowler/providers/azure/lib/mutelist/**"
|
||||
- any-glob-to-any-file: "prowler/providers/gcp/lib/mutelist/**"
|
||||
- any-glob-to-any-file: "prowler/providers/kubernetes/lib/mutelist/**"
|
||||
- any-glob-to-any-file: "prowler/providers/m365/lib/mutelist/**"
|
||||
- any-glob-to-any-file: "prowler/providers/mongodbatlas/lib/mutelist/**"
|
||||
- any-glob-to-any-file: "prowler/providers/oraclecloud/lib/mutelist/**"
|
||||
- any-glob-to-any-file: "prowler/providers/alibabacloud/lib/mutelist/**"
|
||||
- any-glob-to-any-file: "prowler/providers/cloudflare/lib/mutelist/**"
|
||||
- any-glob-to-any-file: "prowler/providers/openstack/lib/mutelist/**"
|
||||
- any-glob-to-any-file: "tests/lib/mutelist/**"
|
||||
- any-glob-to-any-file: "tests/providers/aws/lib/mutelist/**"
|
||||
- any-glob-to-any-file: "tests/providers/azure/lib/mutelist/**"
|
||||
- any-glob-to-any-file: "tests/providers/gcp/lib/mutelist/**"
|
||||
- any-glob-to-any-file: "tests/providers/kubernetes/lib/mutelist/**"
|
||||
- any-glob-to-any-file: "tests/providers/m365/lib/mutelist/**"
|
||||
- any-glob-to-any-file: "tests/providers/mongodbatlas/lib/mutelist/**"
|
||||
- any-glob-to-any-file: "tests/providers/oci/lib/mutelist/**"
|
||||
- any-glob-to-any-file: "tests/providers/oraclecloud/lib/mutelist/**"
|
||||
- any-glob-to-any-file: "tests/providers/alibabacloud/lib/mutelist/**"
|
||||
- any-glob-to-any-file: "tests/providers/cloudflare/lib/mutelist/**"
|
||||
- any-glob-to-any-file: "tests/providers/openstack/lib/mutelist/**"
|
||||
|
||||
integration/s3:
|
||||
- changed-files:
|
||||
|
||||
Executable
+257
@@ -0,0 +1,257 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Test Impact Analysis Script
|
||||
|
||||
Analyzes changed files and determines which tests need to run.
|
||||
Outputs GitHub Actions compatible outputs.
|
||||
|
||||
Usage:
|
||||
python test-impact.py <changed_files...>
|
||||
python test-impact.py --from-stdin # Read files from stdin (one per line)
|
||||
|
||||
Outputs (for GitHub Actions):
|
||||
- run-all: "true" if critical paths changed
|
||||
- sdk-tests: Space-separated list of SDK test paths
|
||||
- api-tests: Space-separated list of API test paths
|
||||
- ui-e2e: Space-separated list of UI E2E test paths
|
||||
- modules: Comma-separated list of affected module names
|
||||
"""
|
||||
|
||||
import fnmatch
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
def load_config() -> dict:
|
||||
"""Load test-impact.yml configuration."""
|
||||
config_path = Path(__file__).parent.parent / "test-impact.yml"
|
||||
with open(config_path) as f:
|
||||
return yaml.safe_load(f)
|
||||
|
||||
|
||||
def matches_pattern(file_path: str, pattern: str) -> bool:
|
||||
"""Check if file path matches a glob pattern."""
|
||||
# Normalize paths
|
||||
file_path = file_path.strip("/")
|
||||
pattern = pattern.strip("/")
|
||||
|
||||
# Handle ** patterns
|
||||
if "**" in pattern:
|
||||
# Convert glob pattern to work with fnmatch
|
||||
# e.g., "prowler/lib/**" matches "prowler/lib/check/foo.py"
|
||||
base = pattern.replace("/**", "")
|
||||
if file_path.startswith(base):
|
||||
return True
|
||||
# Also try standard fnmatch
|
||||
return fnmatch.fnmatch(file_path, pattern)
|
||||
|
||||
return fnmatch.fnmatch(file_path, pattern)
|
||||
|
||||
|
||||
def filter_ignored_files(
|
||||
changed_files: list[str], ignored_paths: list[str]
|
||||
) -> list[str]:
|
||||
"""Filter out files that match ignored patterns."""
|
||||
filtered = []
|
||||
for file_path in changed_files:
|
||||
is_ignored = False
|
||||
for pattern in ignored_paths:
|
||||
if matches_pattern(file_path, pattern):
|
||||
print(f" [IGNORED] {file_path} matches {pattern}", file=sys.stderr)
|
||||
is_ignored = True
|
||||
break
|
||||
if not is_ignored:
|
||||
filtered.append(file_path)
|
||||
return filtered
|
||||
|
||||
|
||||
def check_critical_paths(changed_files: list[str], critical_paths: list[str]) -> bool:
|
||||
"""Check if any changed file matches critical paths."""
|
||||
for file_path in changed_files:
|
||||
for pattern in critical_paths:
|
||||
if matches_pattern(file_path, pattern):
|
||||
print(f" [CRITICAL] {file_path} matches {pattern}", file=sys.stderr)
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def find_affected_modules(
|
||||
changed_files: list[str], modules: list[dict]
|
||||
) -> dict[str, dict]:
|
||||
"""Find which modules are affected by changed files."""
|
||||
affected = {}
|
||||
|
||||
for file_path in changed_files:
|
||||
for module in modules:
|
||||
module_name = module["name"]
|
||||
match_patterns = module.get("match", [])
|
||||
|
||||
for pattern in match_patterns:
|
||||
if matches_pattern(file_path, pattern):
|
||||
if module_name not in affected:
|
||||
affected[module_name] = {
|
||||
"tests": set(),
|
||||
"e2e": set(),
|
||||
"matched_files": [],
|
||||
}
|
||||
affected[module_name]["matched_files"].append(file_path)
|
||||
|
||||
# Add test patterns
|
||||
for test_pattern in module.get("tests", []):
|
||||
affected[module_name]["tests"].add(test_pattern)
|
||||
|
||||
# Add E2E patterns
|
||||
for e2e_pattern in module.get("e2e", []):
|
||||
affected[module_name]["e2e"].add(e2e_pattern)
|
||||
|
||||
break # File matched this module, move to next file
|
||||
|
||||
return affected
|
||||
|
||||
|
||||
def categorize_tests(
|
||||
affected_modules: dict[str, dict],
|
||||
) -> tuple[set[str], set[str], set[str]]:
|
||||
"""Categorize tests into SDK, API, and UI E2E."""
|
||||
sdk_tests = set()
|
||||
api_tests = set()
|
||||
ui_e2e = set()
|
||||
|
||||
for module_name, data in affected_modules.items():
|
||||
for test_path in data["tests"]:
|
||||
if test_path.startswith("tests/"):
|
||||
sdk_tests.add(test_path)
|
||||
elif test_path.startswith("api/"):
|
||||
api_tests.add(test_path)
|
||||
|
||||
for e2e_path in data["e2e"]:
|
||||
ui_e2e.add(e2e_path)
|
||||
|
||||
return sdk_tests, api_tests, ui_e2e
|
||||
|
||||
|
||||
def set_github_output(name: str, value: str):
|
||||
"""Set GitHub Actions output."""
|
||||
github_output = os.environ.get("GITHUB_OUTPUT")
|
||||
if github_output:
|
||||
with open(github_output, "a") as f:
|
||||
# Handle multiline values
|
||||
if "\n" in value:
|
||||
import uuid
|
||||
|
||||
delimiter = uuid.uuid4().hex
|
||||
f.write(f"{name}<<{delimiter}\n{value}\n{delimiter}\n")
|
||||
else:
|
||||
f.write(f"{name}={value}\n")
|
||||
# Print for debugging (without deprecated format)
|
||||
print(f" {name}={value}", file=sys.stderr)
|
||||
|
||||
|
||||
def main():
|
||||
# Parse arguments
|
||||
if "--from-stdin" in sys.argv:
|
||||
changed_files = [line.strip() for line in sys.stdin if line.strip()]
|
||||
else:
|
||||
changed_files = [f for f in sys.argv[1:] if f and not f.startswith("-")]
|
||||
|
||||
if not changed_files:
|
||||
print("No changed files provided", file=sys.stderr)
|
||||
set_github_output("run-all", "false")
|
||||
set_github_output("sdk-tests", "")
|
||||
set_github_output("api-tests", "")
|
||||
set_github_output("ui-e2e", "")
|
||||
set_github_output("modules", "")
|
||||
set_github_output("has-tests", "false")
|
||||
return
|
||||
|
||||
print(f"Analyzing {len(changed_files)} changed files...", file=sys.stderr)
|
||||
for f in changed_files[:10]: # Show first 10
|
||||
print(f" - {f}", file=sys.stderr)
|
||||
if len(changed_files) > 10:
|
||||
print(f" ... and {len(changed_files) - 10} more", file=sys.stderr)
|
||||
|
||||
# Load configuration
|
||||
config = load_config()
|
||||
|
||||
# Filter out ignored files (docs, configs, etc.)
|
||||
ignored_paths = config.get("ignored", {}).get("paths", [])
|
||||
changed_files = filter_ignored_files(changed_files, ignored_paths)
|
||||
|
||||
if not changed_files:
|
||||
print("\nAll changed files are ignored (docs, configs, etc.)", file=sys.stderr)
|
||||
print("No tests needed.", file=sys.stderr)
|
||||
set_github_output("run-all", "false")
|
||||
set_github_output("sdk-tests", "")
|
||||
set_github_output("api-tests", "")
|
||||
set_github_output("ui-e2e", "")
|
||||
set_github_output("modules", "none-ignored")
|
||||
set_github_output("has-tests", "false")
|
||||
return
|
||||
|
||||
print(
|
||||
f"\n{len(changed_files)} files remain after filtering ignored paths",
|
||||
file=sys.stderr,
|
||||
)
|
||||
|
||||
# Check critical paths
|
||||
critical_paths = config.get("critical", {}).get("paths", [])
|
||||
if check_critical_paths(changed_files, critical_paths):
|
||||
print("\nCritical path changed - running ALL tests", file=sys.stderr)
|
||||
set_github_output("run-all", "true")
|
||||
set_github_output("sdk-tests", "tests/")
|
||||
set_github_output("api-tests", "api/src/backend/")
|
||||
set_github_output("ui-e2e", "ui/tests/")
|
||||
set_github_output("modules", "all")
|
||||
set_github_output("has-tests", "true")
|
||||
return
|
||||
|
||||
# Find affected modules
|
||||
modules = config.get("modules", [])
|
||||
affected = find_affected_modules(changed_files, modules)
|
||||
|
||||
if not affected:
|
||||
print("\nNo test-mapped modules affected", file=sys.stderr)
|
||||
set_github_output("run-all", "false")
|
||||
set_github_output("sdk-tests", "")
|
||||
set_github_output("api-tests", "")
|
||||
set_github_output("ui-e2e", "")
|
||||
set_github_output("modules", "")
|
||||
set_github_output("has-tests", "false")
|
||||
return
|
||||
|
||||
# Report affected modules
|
||||
print(f"\nAffected modules: {len(affected)}", file=sys.stderr)
|
||||
for module_name, data in affected.items():
|
||||
print(f" [{module_name}]", file=sys.stderr)
|
||||
for f in data["matched_files"][:3]:
|
||||
print(f" - {f}", file=sys.stderr)
|
||||
if len(data["matched_files"]) > 3:
|
||||
print(
|
||||
f" ... and {len(data['matched_files']) - 3} more files",
|
||||
file=sys.stderr,
|
||||
)
|
||||
|
||||
# Categorize tests
|
||||
sdk_tests, api_tests, ui_e2e = categorize_tests(affected)
|
||||
|
||||
# Output results
|
||||
print("\nTest paths to run:", file=sys.stderr)
|
||||
print(f" SDK: {sdk_tests or 'none'}", file=sys.stderr)
|
||||
print(f" API: {api_tests or 'none'}", file=sys.stderr)
|
||||
print(f" E2E: {ui_e2e or 'none'}", file=sys.stderr)
|
||||
|
||||
set_github_output("run-all", "false")
|
||||
set_github_output("sdk-tests", " ".join(sorted(sdk_tests)))
|
||||
set_github_output("api-tests", " ".join(sorted(api_tests)))
|
||||
set_github_output("ui-e2e", " ".join(sorted(ui_e2e)))
|
||||
set_github_output("modules", ",".join(sorted(affected.keys())))
|
||||
set_github_output(
|
||||
"has-tests", "true" if (sdk_tests or api_tests or ui_e2e) else "false"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,402 @@
|
||||
# Test Impact Analysis Configuration
|
||||
# Defines which tests to run based on changed files
|
||||
#
|
||||
# Usage: Changes to paths in 'critical' always run all tests.
|
||||
# Changes to paths in 'modules' run only the mapped tests.
|
||||
# Changes to paths in 'ignored' don't trigger any tests.
|
||||
|
||||
# Ignored paths - changes here don't trigger any tests
|
||||
# Documentation, configs, and other non-code files
|
||||
ignored:
|
||||
paths:
|
||||
# Documentation
|
||||
- docs/**
|
||||
- "*.md"
|
||||
- "**/*.md"
|
||||
- mkdocs.yml
|
||||
|
||||
# Config files that don't affect runtime
|
||||
- .gitignore
|
||||
- .gitattributes
|
||||
- .editorconfig
|
||||
- .pre-commit-config.yaml
|
||||
- .backportrc.json
|
||||
- CODEOWNERS
|
||||
- LICENSE
|
||||
|
||||
# IDE/Editor configs
|
||||
- .vscode/**
|
||||
- .idea/**
|
||||
|
||||
# Examples and contrib (not production code)
|
||||
- examples/**
|
||||
- contrib/**
|
||||
|
||||
# Skills (AI agent configs, not runtime)
|
||||
- skills/**
|
||||
|
||||
# E2E setup helpers (not runnable tests)
|
||||
- ui/tests/setups/**
|
||||
|
||||
# Permissions docs
|
||||
- permissions/**
|
||||
|
||||
# Critical paths - changes here run ALL tests
|
||||
# These are foundational/shared code that can affect anything
|
||||
critical:
|
||||
paths:
|
||||
# SDK Core
|
||||
- prowler/lib/**
|
||||
- prowler/config/**
|
||||
- prowler/exceptions/**
|
||||
- prowler/providers/common/**
|
||||
|
||||
# API Core
|
||||
- api/src/backend/api/models.py
|
||||
- api/src/backend/config/**
|
||||
- api/src/backend/conftest.py
|
||||
|
||||
# UI Core
|
||||
- ui/lib/**
|
||||
- ui/types/**
|
||||
- ui/config/**
|
||||
- ui/middleware.ts
|
||||
|
||||
# CI/CD changes
|
||||
- .github/workflows/**
|
||||
- .github/test-impact.yml
|
||||
|
||||
# Module mappings - path patterns to test patterns
|
||||
modules:
|
||||
# ============================================
|
||||
# SDK - Providers (each provider is isolated)
|
||||
# ============================================
|
||||
- name: sdk-aws
|
||||
match:
|
||||
- prowler/providers/aws/**
|
||||
- prowler/compliance/aws/**
|
||||
tests:
|
||||
- tests/providers/aws/**
|
||||
e2e: []
|
||||
|
||||
- name: sdk-azure
|
||||
match:
|
||||
- prowler/providers/azure/**
|
||||
- prowler/compliance/azure/**
|
||||
tests:
|
||||
- tests/providers/azure/**
|
||||
e2e: []
|
||||
|
||||
- name: sdk-gcp
|
||||
match:
|
||||
- prowler/providers/gcp/**
|
||||
- prowler/compliance/gcp/**
|
||||
tests:
|
||||
- tests/providers/gcp/**
|
||||
e2e: []
|
||||
|
||||
- name: sdk-kubernetes
|
||||
match:
|
||||
- prowler/providers/kubernetes/**
|
||||
- prowler/compliance/kubernetes/**
|
||||
tests:
|
||||
- tests/providers/kubernetes/**
|
||||
e2e: []
|
||||
|
||||
- name: sdk-github
|
||||
match:
|
||||
- prowler/providers/github/**
|
||||
- prowler/compliance/github/**
|
||||
tests:
|
||||
- tests/providers/github/**
|
||||
e2e: []
|
||||
|
||||
- name: sdk-m365
|
||||
match:
|
||||
- prowler/providers/m365/**
|
||||
- prowler/compliance/m365/**
|
||||
tests:
|
||||
- tests/providers/m365/**
|
||||
e2e: []
|
||||
|
||||
- name: sdk-alibabacloud
|
||||
match:
|
||||
- prowler/providers/alibabacloud/**
|
||||
- prowler/compliance/alibabacloud/**
|
||||
tests:
|
||||
- tests/providers/alibabacloud/**
|
||||
e2e: []
|
||||
|
||||
- name: sdk-cloudflare
|
||||
match:
|
||||
- prowler/providers/cloudflare/**
|
||||
- prowler/compliance/cloudflare/**
|
||||
tests:
|
||||
- tests/providers/cloudflare/**
|
||||
e2e: []
|
||||
|
||||
- name: sdk-oraclecloud
|
||||
match:
|
||||
- prowler/providers/oraclecloud/**
|
||||
- prowler/compliance/oraclecloud/**
|
||||
tests:
|
||||
- tests/providers/oraclecloud/**
|
||||
e2e: []
|
||||
|
||||
- name: sdk-mongodbatlas
|
||||
match:
|
||||
- prowler/providers/mongodbatlas/**
|
||||
- prowler/compliance/mongodbatlas/**
|
||||
tests:
|
||||
- tests/providers/mongodbatlas/**
|
||||
e2e: []
|
||||
|
||||
- name: sdk-nhn
|
||||
match:
|
||||
- prowler/providers/nhn/**
|
||||
- prowler/compliance/nhn/**
|
||||
tests:
|
||||
- tests/providers/nhn/**
|
||||
e2e: []
|
||||
|
||||
- name: sdk-iac
|
||||
match:
|
||||
- prowler/providers/iac/**
|
||||
- prowler/compliance/iac/**
|
||||
tests:
|
||||
- tests/providers/iac/**
|
||||
e2e: []
|
||||
|
||||
- name: sdk-llm
|
||||
match:
|
||||
- prowler/providers/llm/**
|
||||
- prowler/compliance/llm/**
|
||||
tests:
|
||||
- tests/providers/llm/**
|
||||
e2e: []
|
||||
|
||||
# ============================================
|
||||
# SDK - Lib modules
|
||||
# ============================================
|
||||
- name: sdk-lib-check
|
||||
match:
|
||||
- prowler/lib/check/**
|
||||
tests:
|
||||
- tests/lib/check/**
|
||||
e2e: []
|
||||
|
||||
- name: sdk-lib-outputs
|
||||
match:
|
||||
- prowler/lib/outputs/**
|
||||
tests:
|
||||
- tests/lib/outputs/**
|
||||
e2e: []
|
||||
|
||||
- name: sdk-lib-scan
|
||||
match:
|
||||
- prowler/lib/scan/**
|
||||
tests:
|
||||
- tests/lib/scan/**
|
||||
e2e: []
|
||||
|
||||
- name: sdk-lib-cli
|
||||
match:
|
||||
- prowler/lib/cli/**
|
||||
tests:
|
||||
- tests/lib/cli/**
|
||||
e2e: []
|
||||
|
||||
- name: sdk-lib-mutelist
|
||||
match:
|
||||
- prowler/lib/mutelist/**
|
||||
tests:
|
||||
- tests/lib/mutelist/**
|
||||
e2e: []
|
||||
|
||||
# ============================================
|
||||
# API - Views, Serializers, Tasks
|
||||
# ============================================
|
||||
- name: api-views
|
||||
match:
|
||||
- api/src/backend/api/v1/views.py
|
||||
tests:
|
||||
- api/src/backend/api/tests/test_views.py
|
||||
e2e:
|
||||
# API view changes can break UI
|
||||
- ui/tests/**
|
||||
|
||||
- name: api-serializers
|
||||
match:
|
||||
- api/src/backend/api/v1/serializers.py
|
||||
- api/src/backend/api/v1/serializer_utils/**
|
||||
tests:
|
||||
- api/src/backend/api/tests/**
|
||||
e2e:
|
||||
# Serializer changes affect API responses → UI
|
||||
- ui/tests/**
|
||||
|
||||
- name: api-filters
|
||||
match:
|
||||
- api/src/backend/api/filters.py
|
||||
tests:
|
||||
- api/src/backend/api/tests/**
|
||||
e2e: []
|
||||
|
||||
- name: api-rbac
|
||||
match:
|
||||
- api/src/backend/api/rbac/**
|
||||
tests:
|
||||
- api/src/backend/api/tests/**
|
||||
e2e:
|
||||
- ui/tests/roles/**
|
||||
|
||||
- name: api-tasks
|
||||
match:
|
||||
- api/src/backend/tasks/**
|
||||
tests:
|
||||
- api/src/backend/tasks/tests/**
|
||||
e2e: []
|
||||
|
||||
- name: api-attack-paths
|
||||
match:
|
||||
- api/src/backend/api/attack_paths/**
|
||||
tests:
|
||||
- api/src/backend/api/tests/test_attack_paths.py
|
||||
e2e: []
|
||||
|
||||
# ============================================
|
||||
# UI - Components and Features
|
||||
# ============================================
|
||||
- name: ui-providers
|
||||
match:
|
||||
- ui/components/providers/**
|
||||
- ui/actions/providers/**
|
||||
- ui/app/**/providers/**
|
||||
tests: []
|
||||
e2e:
|
||||
- ui/tests/providers/**
|
||||
|
||||
- name: ui-findings
|
||||
match:
|
||||
- ui/components/findings/**
|
||||
- ui/actions/findings/**
|
||||
- ui/app/**/findings/**
|
||||
tests: []
|
||||
e2e:
|
||||
- ui/tests/findings/**
|
||||
|
||||
- name: ui-scans
|
||||
match:
|
||||
- ui/components/scans/**
|
||||
- ui/actions/scans/**
|
||||
- ui/app/**/scans/**
|
||||
tests: []
|
||||
e2e:
|
||||
- ui/tests/scans/**
|
||||
|
||||
- name: ui-compliance
|
||||
match:
|
||||
- ui/components/compliance/**
|
||||
- ui/actions/compliances/**
|
||||
- ui/app/**/compliance/**
|
||||
tests: []
|
||||
e2e:
|
||||
- ui/tests/compliance/**
|
||||
|
||||
- name: ui-auth
|
||||
match:
|
||||
- ui/components/auth/**
|
||||
- ui/actions/auth/**
|
||||
- ui/app/(auth)/**
|
||||
tests: []
|
||||
e2e:
|
||||
- ui/tests/sign-in/**
|
||||
- ui/tests/sign-up/**
|
||||
|
||||
- name: ui-invitations
|
||||
match:
|
||||
- ui/components/invitations/**
|
||||
- ui/actions/invitations/**
|
||||
- ui/app/**/invitations/**
|
||||
tests: []
|
||||
e2e:
|
||||
- ui/tests/invitations/**
|
||||
|
||||
- name: ui-roles
|
||||
match:
|
||||
- ui/components/roles/**
|
||||
- ui/actions/roles/**
|
||||
- ui/app/**/roles/**
|
||||
tests: []
|
||||
e2e:
|
||||
- ui/tests/roles/**
|
||||
|
||||
- name: ui-users
|
||||
match:
|
||||
- ui/components/users/**
|
||||
- ui/actions/users/**
|
||||
- ui/app/**/users/**
|
||||
tests: []
|
||||
e2e:
|
||||
- ui/tests/users/**
|
||||
|
||||
- name: ui-integrations
|
||||
match:
|
||||
- ui/components/integrations/**
|
||||
- ui/actions/integrations/**
|
||||
- ui/app/**/integrations/**
|
||||
tests: []
|
||||
e2e:
|
||||
- ui/tests/integrations/**
|
||||
|
||||
- name: ui-resources
|
||||
match:
|
||||
- ui/components/resources/**
|
||||
- ui/actions/resources/**
|
||||
- ui/app/**/resources/**
|
||||
tests: []
|
||||
e2e:
|
||||
- ui/tests/resources/**
|
||||
|
||||
- name: ui-profile
|
||||
match:
|
||||
- ui/app/**/profile/**
|
||||
tests: []
|
||||
e2e:
|
||||
- ui/tests/profile/**
|
||||
|
||||
- name: ui-lighthouse
|
||||
match:
|
||||
- ui/components/lighthouse/**
|
||||
- ui/actions/lighthouse/**
|
||||
- ui/app/**/lighthouse/**
|
||||
- ui/lib/lighthouse/**
|
||||
tests: []
|
||||
e2e:
|
||||
- ui/tests/lighthouse/**
|
||||
|
||||
- name: ui-overview
|
||||
match:
|
||||
- ui/components/overview/**
|
||||
- ui/actions/overview/**
|
||||
tests: []
|
||||
e2e:
|
||||
- ui/tests/home/**
|
||||
|
||||
- name: ui-shadcn
|
||||
match:
|
||||
- ui/components/shadcn/**
|
||||
- ui/components/ui/**
|
||||
tests: []
|
||||
e2e:
|
||||
# Shared components can affect any E2E
|
||||
- ui/tests/**
|
||||
|
||||
- name: ui-attack-paths
|
||||
match:
|
||||
- ui/components/attack-paths/**
|
||||
- ui/actions/attack-paths/**
|
||||
- ui/app/**/attack-paths/**
|
||||
tests: []
|
||||
e2e:
|
||||
- ui/tests/attack-paths/**
|
||||
@@ -46,6 +46,7 @@ jobs:
|
||||
api/docs/**
|
||||
api/README.md
|
||||
api/CHANGELOG.md
|
||||
api/AGENTS.md
|
||||
|
||||
- name: Setup Python with Poetry
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
|
||||
@@ -74,6 +74,7 @@ jobs:
|
||||
api/docs/**
|
||||
api/README.md
|
||||
api/CHANGELOG.md
|
||||
api/AGENTS.md
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
|
||||
@@ -46,6 +46,7 @@ jobs:
|
||||
api/docs/**
|
||||
api/README.md
|
||||
api/CHANGELOG.md
|
||||
api/AGENTS.md
|
||||
|
||||
- name: Setup Python with Poetry
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
@@ -60,7 +61,8 @@ jobs:
|
||||
|
||||
- name: Safety
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
run: poetry run safety check
|
||||
run: poetry run safety check --ignore 79023,79027
|
||||
# TODO: 79023 & 79027 knack ReDoS until `azure-cli-core` (via `cartography`) allows `knack` >=0.13.0
|
||||
|
||||
- name: Vulture
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
|
||||
@@ -86,6 +86,7 @@ jobs:
|
||||
api/docs/**
|
||||
api/README.md
|
||||
api/CHANGELOG.md
|
||||
api/AGENTS.md
|
||||
|
||||
- name: Setup Python with Poetry
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
|
||||
@@ -55,7 +55,7 @@ jobs:
|
||||
"danibarranqueroo"
|
||||
"HugoPBrito"
|
||||
"jfagoagas"
|
||||
"josemazo"
|
||||
"josema-xyz"
|
||||
"lydiavilchez"
|
||||
"mmuller88"
|
||||
"MrCloudSec"
|
||||
|
||||
@@ -42,14 +42,16 @@ jobs:
|
||||
ui/**
|
||||
prowler/**
|
||||
mcp_server/**
|
||||
poetry.lock
|
||||
pyproject.toml
|
||||
|
||||
- name: Check for folder changes and changelog presence
|
||||
id: check-folders
|
||||
run: |
|
||||
missing_changelogs=""
|
||||
|
||||
# Check api folder
|
||||
if [[ "${{ steps.changed-files.outputs.any_changed }}" == "true" ]]; then
|
||||
# Check monitored folders
|
||||
for folder in $MONITORED_FOLDERS; do
|
||||
# Get files changed in this folder
|
||||
changed_in_folder=$(echo "${{ steps.changed-files.outputs.all_changed_files }}" | tr ' ' '\n' | grep "^${folder}/" || true)
|
||||
@@ -64,6 +66,22 @@ jobs:
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
# Check root-level dependency files (poetry.lock, pyproject.toml)
|
||||
# These are associated with the prowler folder changelog
|
||||
root_deps_changed=$(echo "${{ steps.changed-files.outputs.all_changed_files }}" | tr ' ' '\n' | grep -E "^(poetry\.lock|pyproject\.toml)$" || true)
|
||||
if [ -n "$root_deps_changed" ]; then
|
||||
echo "Detected changes in root dependency files: $root_deps_changed"
|
||||
# Check if prowler/CHANGELOG.md was already updated (might have been caught above)
|
||||
prowler_changelog_updated=$(echo "${{ steps.changed-files.outputs.all_changed_files }}" | tr ' ' '\n' | grep "^prowler/CHANGELOG.md$" || true)
|
||||
if [ -z "$prowler_changelog_updated" ]; then
|
||||
# Only add if prowler wasn't already flagged
|
||||
if ! echo "$missing_changelogs" | grep -q "prowler"; then
|
||||
echo "No changelog update found for root dependency changes"
|
||||
missing_changelogs="${missing_changelogs}- \`prowler\` (root dependency files changed)"$'\n'
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
{
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
name: 'SDK: Check Duplicate Test Names'
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- 'master'
|
||||
- 'v5.*'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
check-duplicate-test-names:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Check for duplicate test names across providers
|
||||
run: |
|
||||
python3 << 'EOF'
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
def find_duplicate_test_names():
|
||||
"""Find test files with the same name across different providers."""
|
||||
tests_dir = Path("tests/providers")
|
||||
|
||||
if not tests_dir.exists():
|
||||
print("tests/providers directory not found")
|
||||
sys.exit(0)
|
||||
|
||||
# Dictionary: filename -> list of (provider, full_path)
|
||||
test_files = defaultdict(list)
|
||||
|
||||
# Find all *_test.py files
|
||||
for test_file in tests_dir.rglob("*_test.py"):
|
||||
relative_path = test_file.relative_to(tests_dir)
|
||||
provider = relative_path.parts[0]
|
||||
filename = test_file.name
|
||||
test_files[filename].append((provider, str(test_file)))
|
||||
|
||||
# Find duplicates (files appearing in multiple providers)
|
||||
duplicates = {
|
||||
filename: locations
|
||||
for filename, locations in test_files.items()
|
||||
if len(set(loc[0] for loc in locations)) > 1
|
||||
}
|
||||
|
||||
if not duplicates:
|
||||
print("No duplicate test file names found across providers.")
|
||||
print("All test names are unique within the repository.")
|
||||
sys.exit(0)
|
||||
|
||||
# Report duplicates
|
||||
print("::error::Duplicate test file names found across providers!")
|
||||
print()
|
||||
print("=" * 70)
|
||||
print("DUPLICATE TEST NAMES DETECTED")
|
||||
print("=" * 70)
|
||||
print()
|
||||
print("The following test files have the same name in multiple providers.")
|
||||
print("Please rename YOUR new test file by adding the provider prefix.")
|
||||
print()
|
||||
print("Example: 'kms_service_test.py' -> 'oraclecloud_kms_service_test.py'")
|
||||
print()
|
||||
|
||||
for filename, locations in sorted(duplicates.items()):
|
||||
print(f"### {filename}")
|
||||
print(f" Found in {len(locations)} providers:")
|
||||
for provider, path in sorted(locations):
|
||||
print(f" - {provider}: {path}")
|
||||
print()
|
||||
print(f" Suggested fix: Rename your new file to '<provider>_{filename}'")
|
||||
print()
|
||||
|
||||
print("=" * 70)
|
||||
print()
|
||||
print("See: tests/providers/TESTING.md for naming conventions.")
|
||||
sys.exit(1)
|
||||
|
||||
if __name__ == "__main__":
|
||||
find_duplicate_test_names()
|
||||
EOF
|
||||
@@ -47,6 +47,7 @@ jobs:
|
||||
ui/**
|
||||
dashboard/**
|
||||
mcp_server/**
|
||||
skills/**
|
||||
README.md
|
||||
mkdocs.yml
|
||||
.backportrc.json
|
||||
@@ -55,6 +56,7 @@ jobs:
|
||||
examples/**
|
||||
.gitignore
|
||||
contrib/**
|
||||
**/AGENTS.md
|
||||
|
||||
- name: Install Poetry
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
@@ -83,7 +85,7 @@ jobs:
|
||||
|
||||
- name: Check format with black
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
run: poetry run black --exclude api ui skills --check .
|
||||
run: poetry run black --exclude "api|ui|skills" --check .
|
||||
|
||||
- name: Lint with pylint
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
|
||||
@@ -78,6 +78,7 @@ jobs:
|
||||
ui/**
|
||||
dashboard/**
|
||||
mcp_server/**
|
||||
skills/**
|
||||
README.md
|
||||
mkdocs.yml
|
||||
.backportrc.json
|
||||
@@ -86,6 +87,7 @@ jobs:
|
||||
examples/**
|
||||
.gitignore
|
||||
contrib/**
|
||||
**/AGENTS.md
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
|
||||
@@ -42,6 +42,7 @@ jobs:
|
||||
ui/**
|
||||
dashboard/**
|
||||
mcp_server/**
|
||||
skills/**
|
||||
README.md
|
||||
mkdocs.yml
|
||||
.backportrc.json
|
||||
@@ -50,6 +51,7 @@ jobs:
|
||||
examples/**
|
||||
.gitignore
|
||||
contrib/**
|
||||
**/AGENTS.md
|
||||
|
||||
- name: Install Poetry
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
|
||||
@@ -47,6 +47,7 @@ jobs:
|
||||
ui/**
|
||||
dashboard/**
|
||||
mcp_server/**
|
||||
skills/**
|
||||
README.md
|
||||
mkdocs.yml
|
||||
.backportrc.json
|
||||
@@ -55,6 +56,7 @@ jobs:
|
||||
examples/**
|
||||
.gitignore
|
||||
contrib/**
|
||||
**/AGENTS.md
|
||||
|
||||
- name: Install Poetry
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
@@ -412,6 +414,30 @@ jobs:
|
||||
flags: prowler-py${{ matrix.python-version }}-oraclecloud
|
||||
files: ./oraclecloud_coverage.xml
|
||||
|
||||
# OpenStack Provider
|
||||
- name: Check if OpenStack files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: changed-openstack
|
||||
uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1
|
||||
with:
|
||||
files: |
|
||||
./prowler/**/openstack/**
|
||||
./tests/**/openstack/**
|
||||
./poetry.lock
|
||||
|
||||
- name: Run OpenStack tests
|
||||
if: steps.changed-openstack.outputs.any_changed == 'true'
|
||||
run: poetry run pytest -n auto --cov=./prowler/providers/openstack --cov-report=xml:openstack_coverage.xml tests/providers/openstack
|
||||
|
||||
- name: Upload OpenStack coverage to Codecov
|
||||
if: steps.changed-openstack.outputs.any_changed == 'true'
|
||||
uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2
|
||||
env:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
with:
|
||||
flags: prowler-py${{ matrix.python-version }}-openstack
|
||||
files: ./openstack_coverage.xml
|
||||
|
||||
# Lib
|
||||
- name: Check if Lib files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
name: Test Impact Analysis
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
outputs:
|
||||
run-all:
|
||||
description: "Whether to run all tests (critical path changed)"
|
||||
value: ${{ jobs.analyze.outputs.run-all }}
|
||||
sdk-tests:
|
||||
description: "SDK test paths to run"
|
||||
value: ${{ jobs.analyze.outputs.sdk-tests }}
|
||||
api-tests:
|
||||
description: "API test paths to run"
|
||||
value: ${{ jobs.analyze.outputs.api-tests }}
|
||||
ui-e2e:
|
||||
description: "UI E2E test paths to run"
|
||||
value: ${{ jobs.analyze.outputs.ui-e2e }}
|
||||
modules:
|
||||
description: "Comma-separated list of affected modules"
|
||||
value: ${{ jobs.analyze.outputs.modules }}
|
||||
has-tests:
|
||||
description: "Whether there are any tests to run"
|
||||
value: ${{ jobs.analyze.outputs.has-tests }}
|
||||
has-sdk-tests:
|
||||
description: "Whether there are SDK tests to run"
|
||||
value: ${{ jobs.analyze.outputs.has-sdk-tests }}
|
||||
has-api-tests:
|
||||
description: "Whether there are API tests to run"
|
||||
value: ${{ jobs.analyze.outputs.has-api-tests }}
|
||||
has-ui-e2e:
|
||||
description: "Whether there are UI E2E tests to run"
|
||||
value: ${{ jobs.analyze.outputs.has-ui-e2e }}
|
||||
|
||||
jobs:
|
||||
analyze:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
outputs:
|
||||
run-all: ${{ steps.impact.outputs.run-all }}
|
||||
sdk-tests: ${{ steps.impact.outputs.sdk-tests }}
|
||||
api-tests: ${{ steps.impact.outputs.api-tests }}
|
||||
ui-e2e: ${{ steps.impact.outputs.ui-e2e }}
|
||||
modules: ${{ steps.impact.outputs.modules }}
|
||||
has-tests: ${{ steps.impact.outputs.has-tests }}
|
||||
has-sdk-tests: ${{ steps.set-flags.outputs.has-sdk-tests }}
|
||||
has-api-tests: ${{ steps.set-flags.outputs.has-api-tests }}
|
||||
has-ui-e2e: ${{ steps.set-flags.outputs.has-ui-e2e }}
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Get changed files
|
||||
id: changed-files
|
||||
uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1
|
||||
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Install PyYAML
|
||||
run: pip install pyyaml
|
||||
|
||||
- name: Analyze test impact
|
||||
id: impact
|
||||
run: |
|
||||
echo "Changed files:"
|
||||
echo "${{ steps.changed-files.outputs.all_changed_files }}" | tr ' ' '\n'
|
||||
echo ""
|
||||
python .github/scripts/test-impact.py ${{ steps.changed-files.outputs.all_changed_files }}
|
||||
|
||||
- name: Set convenience flags
|
||||
id: set-flags
|
||||
run: |
|
||||
if [[ -n "${{ steps.impact.outputs.sdk-tests }}" ]]; then
|
||||
echo "has-sdk-tests=true" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "has-sdk-tests=false" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
if [[ -n "${{ steps.impact.outputs.api-tests }}" ]]; then
|
||||
echo "has-api-tests=true" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "has-api-tests=false" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
if [[ -n "${{ steps.impact.outputs.ui-e2e }}" ]]; then
|
||||
echo "has-ui-e2e=true" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "has-ui-e2e=false" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
- name: Summary
|
||||
run: |
|
||||
echo "## Test Impact Analysis" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
|
||||
if [[ "${{ steps.impact.outputs.run-all }}" == "true" ]]; then
|
||||
echo "🚨 **Critical path changed - running ALL tests**" >> $GITHUB_STEP_SUMMARY
|
||||
else
|
||||
echo "### Affected Modules" >> $GITHUB_STEP_SUMMARY
|
||||
echo "\`${{ steps.impact.outputs.modules }}\`" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
|
||||
echo "### Tests to Run" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| Category | Paths |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "|----------|-------|" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| SDK Tests | \`${{ steps.impact.outputs.sdk-tests || 'none' }}\` |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| API Tests | \`${{ steps.impact.outputs.api-tests || 'none' }}\` |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| UI E2E | \`${{ steps.impact.outputs.ui-e2e || 'none' }}\` |" >> $GITHUB_STEP_SUMMARY
|
||||
fi
|
||||
@@ -73,6 +73,7 @@ jobs:
|
||||
files_ignore: |
|
||||
ui/CHANGELOG.md
|
||||
ui/README.md
|
||||
ui/AGENTS.md
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
|
||||
@@ -1,4 +1,8 @@
|
||||
name: UI - E2E Tests
|
||||
name: UI - E2E Tests (Optimized)
|
||||
|
||||
# This is an optimized version that runs only relevant E2E tests
|
||||
# based on changed files. Falls back to running all tests if
|
||||
# critical paths are changed or if impact analysis fails.
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
@@ -6,13 +10,23 @@ on:
|
||||
- master
|
||||
- "v5.*"
|
||||
paths:
|
||||
- '.github/workflows/ui-e2e-tests.yml'
|
||||
- '.github/workflows/ui-e2e-tests-v2.yml'
|
||||
- '.github/test-impact.yml'
|
||||
- 'ui/**'
|
||||
- 'api/**' # API changes can affect UI E2E
|
||||
|
||||
jobs:
|
||||
|
||||
e2e-tests:
|
||||
# First, analyze which tests need to run
|
||||
impact-analysis:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
uses: ./.github/workflows/test-impact-analysis.yml
|
||||
|
||||
# Run E2E tests based on impact analysis
|
||||
e2e-tests:
|
||||
needs: impact-analysis
|
||||
if: |
|
||||
github.repository == 'prowler-cloud/prowler' &&
|
||||
(needs.impact-analysis.outputs.has-ui-e2e == 'true' || needs.impact-analysis.outputs.run-all == 'true')
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
AUTH_SECRET: 'fallback-ci-secret-for-testing'
|
||||
@@ -51,93 +65,119 @@ jobs:
|
||||
E2E_OCI_KEY_CONTENT: ${{ secrets.E2E_OCI_KEY_CONTENT }}
|
||||
E2E_OCI_REGION: ${{ secrets.E2E_OCI_REGION }}
|
||||
E2E_NEW_USER_PASSWORD: ${{ secrets.E2E_NEW_USER_PASSWORD }}
|
||||
E2E_ALIBABACLOUD_ACCOUNT_ID: ${{ secrets.E2E_ALIBABACLOUD_ACCOUNT_ID }}
|
||||
E2E_ALIBABACLOUD_ACCESS_KEY_ID: ${{ secrets.E2E_ALIBABACLOUD_ACCESS_KEY_ID }}
|
||||
E2E_ALIBABACLOUD_ACCESS_KEY_SECRET: ${{ secrets.E2E_ALIBABACLOUD_ACCESS_KEY_SECRET }}
|
||||
E2E_ALIBABACLOUD_ROLE_ARN: ${{ secrets.E2E_ALIBABACLOUD_ROLE_ARN }}
|
||||
# Pass E2E paths from impact analysis
|
||||
E2E_TEST_PATHS: ${{ needs.impact-analysis.outputs.ui-e2e }}
|
||||
RUN_ALL_TESTS: ${{ needs.impact-analysis.outputs.run-all }}
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Show test scope
|
||||
run: |
|
||||
echo "## E2E Test Scope" >> $GITHUB_STEP_SUMMARY
|
||||
if [[ "${{ env.RUN_ALL_TESTS }}" == "true" ]]; then
|
||||
echo "Running **ALL** E2E tests (critical path changed)" >> $GITHUB_STEP_SUMMARY
|
||||
else
|
||||
echo "Running tests matching: \`${{ env.E2E_TEST_PATHS }}\`" >> $GITHUB_STEP_SUMMARY
|
||||
fi
|
||||
echo ""
|
||||
echo "Affected modules: \`${{ needs.impact-analysis.outputs.modules }}\`" >> $GITHUB_STEP_SUMMARY
|
||||
|
||||
- name: Create k8s Kind Cluster
|
||||
uses: helm/kind-action@v1
|
||||
with:
|
||||
cluster_name: kind
|
||||
|
||||
- name: Modify kubeconfig
|
||||
run: |
|
||||
# Modify the kubeconfig to use the kind cluster server to https://kind-control-plane:6443
|
||||
# from worker service into docker-compose.yml
|
||||
kubectl config set-cluster kind-kind --server=https://kind-control-plane:6443
|
||||
kubectl config view
|
||||
kubectl config set-cluster kind-kind --server=https://kind-control-plane:6443
|
||||
kubectl config view
|
||||
|
||||
- name: Add network kind to docker compose
|
||||
run: |
|
||||
# Add the network kind to the docker compose to interconnect to kind cluster
|
||||
yq -i '.networks.kind.external = true' docker-compose.yml
|
||||
# Add network kind to worker service and default network too
|
||||
yq -i '.services.worker.networks = ["kind","default"]' docker-compose.yml
|
||||
|
||||
- name: Fix API data directory permissions
|
||||
run: docker run --rm -v $(pwd)/_data/api:/data alpine chown -R 1000:1000 /data
|
||||
- name: Add AWS credentials for testing AWS SDK Default Adding Provider
|
||||
|
||||
- name: Add AWS credentials for testing
|
||||
run: |
|
||||
echo "Adding AWS credentials for testing AWS SDK Default Adding Provider..."
|
||||
echo "AWS_ACCESS_KEY_ID=${{ secrets.E2E_AWS_PROVIDER_ACCESS_KEY }}" >> .env
|
||||
echo "AWS_SECRET_ACCESS_KEY=${{ secrets.E2E_AWS_PROVIDER_SECRET_KEY }}" >> .env
|
||||
|
||||
- name: Start API services
|
||||
run: |
|
||||
# Override docker-compose image tag to use latest instead of stable
|
||||
# This overrides any PROWLER_API_VERSION set in .env file
|
||||
export PROWLER_API_VERSION=latest
|
||||
echo "Using PROWLER_API_VERSION=${PROWLER_API_VERSION}"
|
||||
docker compose up -d api worker worker-beat
|
||||
|
||||
- name: Wait for API to be ready
|
||||
run: |
|
||||
echo "Waiting for prowler-api..."
|
||||
timeout=150 # 5 minutes max
|
||||
timeout=150
|
||||
elapsed=0
|
||||
while [ $elapsed -lt $timeout ]; do
|
||||
if curl -s ${NEXT_PUBLIC_API_BASE_URL}/docs >/dev/null 2>&1; then
|
||||
echo "Prowler API is ready!"
|
||||
exit 0
|
||||
fi
|
||||
echo "Waiting for prowler-api... (${elapsed}s elapsed)"
|
||||
echo "Waiting... (${elapsed}s elapsed)"
|
||||
sleep 5
|
||||
elapsed=$((elapsed + 5))
|
||||
done
|
||||
echo "Timeout waiting for prowler-api to start"
|
||||
echo "Timeout waiting for prowler-api"
|
||||
exit 1
|
||||
- name: Load database fixtures for E2E tests
|
||||
|
||||
- name: Load database fixtures
|
||||
run: |
|
||||
docker compose exec -T api sh -c '
|
||||
echo "Loading all fixtures from api/fixtures/dev/..."
|
||||
for fixture in api/fixtures/dev/*.json; do
|
||||
if [ -f "$fixture" ]; then
|
||||
echo "Loading $fixture"
|
||||
poetry run python manage.py loaddata "$fixture" --database admin
|
||||
fi
|
||||
done
|
||||
echo "All database fixtures loaded successfully!"
|
||||
'
|
||||
- name: Setup Node.js environment
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@395ad3262231945c25e8478fd5baf05154b1d79f # v6.1.0
|
||||
with:
|
||||
node-version: '20.x'
|
||||
node-version: '24.13.0'
|
||||
|
||||
- name: Setup pnpm
|
||||
uses: pnpm/action-setup@v4
|
||||
with:
|
||||
version: 10
|
||||
run_install: false
|
||||
|
||||
- name: Get pnpm store directory
|
||||
shell: bash
|
||||
run: echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV
|
||||
- name: Setup pnpm cache
|
||||
|
||||
- name: Setup pnpm and Next.js cache
|
||||
uses: actions/cache@9255dc7a253b0ccc959486e2bca901246202afeb # v5.0.1
|
||||
with:
|
||||
path: ${{ env.STORE_PATH }}
|
||||
key: ${{ runner.os }}-pnpm-store-${{ hashFiles('ui/pnpm-lock.yaml') }}
|
||||
path: |
|
||||
${{ env.STORE_PATH }}
|
||||
./ui/node_modules
|
||||
./ui/.next/cache
|
||||
key: ${{ runner.os }}-pnpm-nextjs-${{ hashFiles('ui/pnpm-lock.yaml') }}-${{ hashFiles('ui/**/*.ts', 'ui/**/*.tsx', 'ui/**/*.js', 'ui/**/*.jsx') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-pnpm-store-
|
||||
${{ runner.os }}-pnpm-nextjs-${{ hashFiles('ui/pnpm-lock.yaml') }}-
|
||||
${{ runner.os }}-pnpm-nextjs-
|
||||
|
||||
- name: Install UI dependencies
|
||||
working-directory: ./ui
|
||||
run: pnpm install --frozen-lockfile
|
||||
run: pnpm install --frozen-lockfile --prefer-offline
|
||||
|
||||
- name: Build UI application
|
||||
working-directory: ./ui
|
||||
run: pnpm run build
|
||||
|
||||
- name: Cache Playwright browsers
|
||||
uses: actions/cache@9255dc7a253b0ccc959486e2bca901246202afeb # v5.0.1
|
||||
id: playwright-cache
|
||||
@@ -146,13 +186,36 @@ jobs:
|
||||
key: ${{ runner.os }}-playwright-${{ hashFiles('ui/pnpm-lock.yaml') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-playwright-
|
||||
|
||||
- name: Install Playwright browsers
|
||||
working-directory: ./ui
|
||||
if: steps.playwright-cache.outputs.cache-hit != 'true'
|
||||
run: pnpm run test:e2e:install
|
||||
|
||||
- name: Run E2E tests
|
||||
working-directory: ./ui
|
||||
run: pnpm run test:e2e
|
||||
run: |
|
||||
if [[ "${{ env.RUN_ALL_TESTS }}" == "true" ]]; then
|
||||
echo "Running ALL E2E tests..."
|
||||
pnpm run test:e2e
|
||||
else
|
||||
echo "Running targeted E2E tests: ${{ env.E2E_TEST_PATHS }}"
|
||||
# Convert glob patterns to playwright test paths
|
||||
# e.g., "ui/tests/providers/**" -> "tests/providers"
|
||||
TEST_PATHS="${{ env.E2E_TEST_PATHS }}"
|
||||
# Remove ui/ prefix and convert ** to empty (playwright handles recursion)
|
||||
TEST_PATHS=$(echo "$TEST_PATHS" | sed 's|ui/||g' | sed 's|\*\*||g' | tr ' ' '\n' | sort -u)
|
||||
# Drop auth setup helpers (not runnable test suites)
|
||||
TEST_PATHS=$(echo "$TEST_PATHS" | grep -v '^tests/setups/')
|
||||
if [[ -z "$TEST_PATHS" ]]; then
|
||||
echo "No runnable E2E test paths after filtering setups"
|
||||
exit 0
|
||||
fi
|
||||
TEST_PATHS=$(echo "$TEST_PATHS" | tr '\n' ' ')
|
||||
echo "Resolved test paths: $TEST_PATHS"
|
||||
pnpm exec playwright test $TEST_PATHS
|
||||
fi
|
||||
|
||||
- name: Upload test reports
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
|
||||
if: failure()
|
||||
@@ -160,9 +223,27 @@ jobs:
|
||||
name: playwright-report
|
||||
path: ui/playwright-report/
|
||||
retention-days: 30
|
||||
|
||||
- name: Cleanup services
|
||||
if: always()
|
||||
run: |
|
||||
echo "Shutting down services..."
|
||||
docker compose down -v || true
|
||||
echo "Cleanup completed"
|
||||
|
||||
# Skip job - provides clear feedback when no E2E tests needed
|
||||
skip-e2e:
|
||||
needs: impact-analysis
|
||||
if: |
|
||||
github.repository == 'prowler-cloud/prowler' &&
|
||||
needs.impact-analysis.outputs.has-ui-e2e != 'true' &&
|
||||
needs.impact-analysis.outputs.run-all != 'true'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: No E2E tests needed
|
||||
run: |
|
||||
echo "## E2E Tests Skipped" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "No UI E2E tests needed for this change." >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "Affected modules: \`${{ needs.impact-analysis.outputs.modules }}\`" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "To run all tests, modify a file in a critical path (e.g., \`ui/lib/**\`)." >> $GITHUB_STEP_SUMMARY
|
||||
@@ -16,7 +16,7 @@ concurrency:
|
||||
|
||||
env:
|
||||
UI_WORKING_DIR: ./ui
|
||||
NODE_VERSION: '20.x'
|
||||
NODE_VERSION: '24.13.0'
|
||||
|
||||
jobs:
|
||||
ui-tests:
|
||||
@@ -42,6 +42,7 @@ jobs:
|
||||
files_ignore: |
|
||||
ui/CHANGELOG.md
|
||||
ui/README.md
|
||||
ui/AGENTS.md
|
||||
|
||||
- name: Setup Node.js ${{ env.NODE_VERSION }}
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
@@ -61,18 +62,22 @@ jobs:
|
||||
shell: bash
|
||||
run: echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV
|
||||
|
||||
- name: Setup pnpm cache
|
||||
- name: Setup pnpm and Next.js cache
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: actions/cache@9255dc7a253b0ccc959486e2bca901246202afeb # v5.0.1
|
||||
with:
|
||||
path: ${{ env.STORE_PATH }}
|
||||
key: ${{ runner.os }}-pnpm-store-${{ hashFiles('ui/pnpm-lock.yaml') }}
|
||||
path: |
|
||||
${{ env.STORE_PATH }}
|
||||
${{ env.UI_WORKING_DIR }}/node_modules
|
||||
${{ env.UI_WORKING_DIR }}/.next/cache
|
||||
key: ${{ runner.os }}-pnpm-nextjs-${{ hashFiles('ui/pnpm-lock.yaml') }}-${{ hashFiles('ui/**/*.ts', 'ui/**/*.tsx', 'ui/**/*.js', 'ui/**/*.jsx') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-pnpm-store-
|
||||
${{ runner.os }}-pnpm-nextjs-${{ hashFiles('ui/pnpm-lock.yaml') }}-
|
||||
${{ runner.os }}-pnpm-nextjs-
|
||||
|
||||
- name: Install dependencies
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
run: pnpm install --frozen-lockfile
|
||||
run: pnpm install --frozen-lockfile --prefer-offline
|
||||
|
||||
- name: Run healthcheck
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
|
||||
+3
-1
@@ -150,8 +150,10 @@ node_modules
|
||||
# Persistent data
|
||||
_data/
|
||||
|
||||
# Claude
|
||||
# AI Instructions (generated by skills/setup.sh from AGENTS.md)
|
||||
CLAUDE.md
|
||||
GEMINI.md
|
||||
.github/copilot-instructions.md
|
||||
|
||||
# Compliance report
|
||||
*.pdf
|
||||
|
||||
@@ -42,7 +42,7 @@ repos:
|
||||
"--remove-unused-variable",
|
||||
]
|
||||
|
||||
- repo: https://github.com/timothycrosley/isort
|
||||
- repo: https://github.com/pycqa/isort
|
||||
rev: 5.13.2
|
||||
hooks:
|
||||
- id: isort
|
||||
@@ -120,7 +120,8 @@ repos:
|
||||
name: safety
|
||||
description: "Safety is a tool that checks your installed dependencies for known security vulnerabilities"
|
||||
# TODO: Botocore needs urllib3 1.X so we need to ignore these vulnerabilities 77744,77745. Remove this once we upgrade to urllib3 2.X
|
||||
entry: bash -c 'safety check --ignore 70612,66963,74429,76352,76353,77744,77745'
|
||||
# TODO: 79023 & 79027 knack ReDoS until `azure-cli-core` (via `cartography`) allows `knack` >=0.13.0
|
||||
entry: bash -c 'safety check --ignore 70612,66963,74429,76352,76353,77744,77745,79023,79027'
|
||||
language: system
|
||||
|
||||
- id: vulture
|
||||
|
||||
@@ -20,6 +20,7 @@ Use these skills for detailed patterns on-demand:
|
||||
| `playwright` | Page Object Model, MCP workflow, selectors | [SKILL.md](skills/playwright/SKILL.md) |
|
||||
| `pytest` | Fixtures, mocking, markers, parametrize | [SKILL.md](skills/pytest/SKILL.md) |
|
||||
| `django-drf` | ViewSets, Serializers, Filters | [SKILL.md](skills/django-drf/SKILL.md) |
|
||||
| `jsonapi` | Strict JSON:API v1.1 spec compliance | [SKILL.md](skills/jsonapi/SKILL.md) |
|
||||
| `zod-4` | New API (z.email(), z.uuid()) | [SKILL.md](skills/zod-4/SKILL.md) |
|
||||
| `zustand-5` | Persist, selectors, slices | [SKILL.md](skills/zustand-5/SKILL.md) |
|
||||
| `ai-sdk-5` | UIMessage, streaming, LangChain | [SKILL.md](skills/ai-sdk-5/SKILL.md) |
|
||||
@@ -36,11 +37,82 @@ Use these skills for detailed patterns on-demand:
|
||||
| `prowler-test-api` | API testing (pytest-django + RLS) | [SKILL.md](skills/prowler-test-api/SKILL.md) |
|
||||
| `prowler-test-ui` | E2E testing (Playwright) | [SKILL.md](skills/prowler-test-ui/SKILL.md) |
|
||||
| `prowler-compliance` | Compliance framework structure | [SKILL.md](skills/prowler-compliance/SKILL.md) |
|
||||
| `prowler-compliance-review` | Review compliance framework PRs | [SKILL.md](skills/prowler-compliance-review/SKILL.md) |
|
||||
| `prowler-provider` | Add new cloud providers | [SKILL.md](skills/prowler-provider/SKILL.md) |
|
||||
| `prowler-changelog` | Changelog entries (keepachangelog.com) | [SKILL.md](skills/prowler-changelog/SKILL.md) |
|
||||
| `prowler-changelog-review` | Review changelog completeness between versions | [SKILL.md](skills/prowler-changelog-review/SKILL.md) |
|
||||
| `prowler-ci` | CI checks and PR gates (GitHub Actions) | [SKILL.md](skills/prowler-ci/SKILL.md) |
|
||||
| `prowler-commit` | Professional commits (conventional-commits) | [SKILL.md](skills/prowler-commit/SKILL.md) |
|
||||
| `prowler-pr` | Pull request conventions | [SKILL.md](skills/prowler-pr/SKILL.md) |
|
||||
| `prowler-docs` | Documentation style guide | [SKILL.md](skills/prowler-docs/SKILL.md) |
|
||||
| `prowler-attack-paths-query` | Create Attack Paths openCypher queries | [SKILL.md](skills/prowler-attack-paths-query/SKILL.md) |
|
||||
| `skill-creator` | Create new AI agent skills | [SKILL.md](skills/skill-creator/SKILL.md) |
|
||||
|
||||
### Auto-invoke Skills
|
||||
|
||||
When performing these actions, ALWAYS invoke the corresponding skill FIRST:
|
||||
|
||||
| Action | Skill |
|
||||
|--------|-------|
|
||||
| Add changelog entry for a PR or feature | `prowler-changelog` |
|
||||
| Adding DRF pagination or permissions | `django-drf` |
|
||||
| Adding new providers | `prowler-provider` |
|
||||
| Adding services to existing providers | `prowler-provider` |
|
||||
| Adding privilege escalation detection queries | `prowler-attack-paths-query` |
|
||||
| After creating/modifying a skill | `skill-sync` |
|
||||
| App Router / Server Actions | `nextjs-15` |
|
||||
| Building AI chat features | `ai-sdk-5` |
|
||||
| Committing changes | `prowler-commit` |
|
||||
| Create PR that requires changelog entry | `prowler-changelog` |
|
||||
| Create a PR with gh pr create | `prowler-pr` |
|
||||
| Creating API endpoints | `jsonapi` |
|
||||
| Creating Attack Paths queries | `prowler-attack-paths-query` |
|
||||
| Creating ViewSets, serializers, or filters in api/ | `django-drf` |
|
||||
| Creating Zod schemas | `zod-4` |
|
||||
| Creating a git commit | `prowler-commit` |
|
||||
| Creating new checks | `prowler-sdk-check` |
|
||||
| Creating new skills | `skill-creator` |
|
||||
| Creating/modifying Prowler UI components | `prowler-ui` |
|
||||
| Creating/modifying models, views, serializers | `prowler-api` |
|
||||
| Creating/updating compliance frameworks | `prowler-compliance` |
|
||||
| Debug why a GitHub Actions job is failing | `prowler-ci` |
|
||||
| Fill .github/pull_request_template.md (Context/Description/Steps to review/Checklist) | `prowler-pr` |
|
||||
| General Prowler development questions | `prowler` |
|
||||
| Implementing JSON:API endpoints | `django-drf` |
|
||||
| Inspect PR CI checks and gates (.github/workflows/*) | `prowler-ci` |
|
||||
| Inspect PR CI workflows (.github/workflows/*): conventional-commit, pr-check-changelog, pr-conflict-checker, labeler | `prowler-pr` |
|
||||
| Mapping checks to compliance controls | `prowler-compliance` |
|
||||
| Mocking AWS with moto in tests | `prowler-test-sdk` |
|
||||
| Modifying API responses | `jsonapi` |
|
||||
| Regenerate AGENTS.md Auto-invoke tables (sync.sh) | `skill-sync` |
|
||||
| Review PR requirements: template, title conventions, changelog gate | `prowler-pr` |
|
||||
| Review changelog completeness between versions | `prowler-changelog-review` |
|
||||
| Review changelog format and conventions | `prowler-changelog` |
|
||||
| Reviewing JSON:API compliance | `jsonapi` |
|
||||
| Reviewing compliance framework PRs | `prowler-compliance-review` |
|
||||
| Testing RLS tenant isolation | `prowler-test-api` |
|
||||
| Troubleshoot why a skill is missing from AGENTS.md auto-invoke | `skill-sync` |
|
||||
| Understand CODEOWNERS/labeler-based automation | `prowler-ci` |
|
||||
| Understand PR title conventional-commit validation | `prowler-ci` |
|
||||
| Understand changelog gate and no-changelog label behavior | `prowler-ci` |
|
||||
| Understand review ownership with CODEOWNERS | `prowler-pr` |
|
||||
| Update CHANGELOG.md in any component | `prowler-changelog` |
|
||||
| Updating existing Attack Paths queries | `prowler-attack-paths-query` |
|
||||
| Updating existing checks and metadata | `prowler-sdk-check` |
|
||||
| Using Zustand stores | `zustand-5` |
|
||||
| Working on MCP server tools | `prowler-mcp` |
|
||||
| Working on Prowler UI structure (actions/adapters/types/hooks) | `prowler-ui` |
|
||||
| Working with Prowler UI test helpers/pages | `prowler-test-ui` |
|
||||
| Working with Tailwind classes | `tailwind-4` |
|
||||
| Writing Playwright E2E tests | `playwright` |
|
||||
| Writing Prowler API tests | `prowler-test-api` |
|
||||
| Writing Prowler SDK tests | `prowler-test-sdk` |
|
||||
| Writing Prowler UI E2E tests | `prowler-test-ui` |
|
||||
| Writing Python tests with pytest | `pytest` |
|
||||
| Writing React components | `react-19` |
|
||||
| Writing TypeScript types/interfaces | `typescript` |
|
||||
| Writing documentation | `prowler-docs` |
|
||||
|
||||
---
|
||||
|
||||
## Project Overview
|
||||
|
||||
@@ -80,6 +80,23 @@ prowler dashboard
|
||||
```
|
||||

|
||||
|
||||
|
||||
## Attack Paths
|
||||
|
||||
Attack Paths automatically extends every completed AWS scan with a Neo4j graph that combines Cartography's cloud inventory with Prowler findings. The feature runs in the API worker after each scan and therefore requires:
|
||||
|
||||
- An accessible Neo4j instance (the Docker Compose files already ships a `neo4j` service).
|
||||
- The following environment variables so Django and Celery can connect:
|
||||
|
||||
| Variable | Description | Default |
|
||||
| --- | --- | --- |
|
||||
| `NEO4J_HOST` | Hostname used by the API containers. | `neo4j` |
|
||||
| `NEO4J_PORT` | Bolt port exposed by Neo4j. | `7687` |
|
||||
| `NEO4J_USER` / `NEO4J_PASSWORD` | Credentials with rights to create per-tenant databases. | `neo4j` / `neo4j_password` |
|
||||
|
||||
Every AWS provider scan will enqueue an Attack Paths ingestion job automatically. Other cloud providers will be added in future iterations.
|
||||
|
||||
|
||||
# Prowler at a Glance
|
||||
> [!Tip]
|
||||
> For the most accurate and up-to-date information about checks, services, frameworks, and categories, visit [**Prowler Hub**](https://hub.prowler.com).
|
||||
@@ -87,17 +104,19 @@ prowler dashboard
|
||||
|
||||
| Provider | Checks | Services | [Compliance Frameworks](https://docs.prowler.com/projects/prowler-open-source/en/latest/tutorials/compliance/) | [Categories](https://docs.prowler.com/projects/prowler-open-source/en/latest/tutorials/misc/#categories) | Support | Interface |
|
||||
|---|---|---|---|---|---|---|
|
||||
| AWS | 584 | 85 | 40 | 17 | Official | UI, API, CLI |
|
||||
| GCP | 89 | 17 | 14 | 5 | Official | UI, API, CLI |
|
||||
| Azure | 169 | 22 | 15 | 8 | Official | UI, API, CLI |
|
||||
| Kubernetes | 84 | 7 | 6 | 9 | Official | UI, API, CLI |
|
||||
| AWS | 585 | 84 | 40 | 17 | Official | UI, API, CLI |
|
||||
| Azure | 169 | 22 | 17 | 13 | Official | UI, API, CLI |
|
||||
| GCP | 100 | 17 | 14 | 7 | Official | UI, API, CLI |
|
||||
| Kubernetes | 84 | 7 | 7 | 9 | Official | UI, API, CLI |
|
||||
| GitHub | 20 | 2 | 1 | 2 | Official | UI, API, CLI |
|
||||
| M365 | 70 | 7 | 3 | 2 | Official | UI, API, CLI |
|
||||
| OCI | 52 | 15 | 1 | 12 | Official | UI, API, CLI |
|
||||
| Alibaba Cloud | 63 | 10 | 1 | 9 | Official | CLI |
|
||||
| M365 | 72 | 7 | 4 | 4 | Official | UI, API, CLI |
|
||||
| OCI | 52 | 14 | 1 | 12 | Official | UI, API, CLI |
|
||||
| Alibaba Cloud | 64 | 9 | 2 | 9 | Official | UI, API, CLI |
|
||||
| Cloudflare | 29 | 3 | 0 | 5 | Official | CLI |
|
||||
| IaC | [See `trivy` docs.](https://trivy.dev/latest/docs/coverage/iac/) | N/A | N/A | N/A | Official | UI, API, CLI |
|
||||
| MongoDB Atlas | 10 | 4 | 0 | 3 | Official | UI, API, CLI |
|
||||
| MongoDB Atlas | 10 | 3 | 0 | 3 | Official | UI, API, CLI |
|
||||
| LLM | [See `promptfoo` docs.](https://www.promptfoo.dev/docs/red-team/plugins/) | N/A | N/A | N/A | Official | CLI |
|
||||
| OpenStack | 1 | 1 | 0 | 2 | Official | CLI |
|
||||
| NHN | 6 | 2 | 1 | 0 | Unofficial | CLI |
|
||||
|
||||
> [!Note]
|
||||
|
||||
+1
-1
@@ -62,4 +62,4 @@ We strive to resolve all problems as quickly as possible, and we would like to p
|
||||
|
||||
---
|
||||
|
||||
For more information about our security policies, please refer to our [Security](https://docs.prowler.com/projects/prowler-open-source/en/latest/security/) section in our documentation.
|
||||
For more information about our security policies, please refer to our [Security](https://docs.prowler.com/security) section in our documentation.
|
||||
|
||||
@@ -3,9 +3,39 @@
|
||||
> **Skills Reference**: For detailed patterns, use these skills:
|
||||
> - [`prowler-api`](../skills/prowler-api/SKILL.md) - Models, Serializers, Views, RLS patterns
|
||||
> - [`prowler-test-api`](../skills/prowler-test-api/SKILL.md) - Testing patterns (pytest-django)
|
||||
> - [`prowler-attack-paths-query`](../skills/prowler-attack-paths-query/SKILL.md) - Attack Paths openCypher queries
|
||||
> - [`django-drf`](../skills/django-drf/SKILL.md) - Generic DRF patterns
|
||||
> - [`jsonapi`](../skills/jsonapi/SKILL.md) - Strict JSON:API v1.1 spec compliance
|
||||
> - [`pytest`](../skills/pytest/SKILL.md) - Generic pytest patterns
|
||||
|
||||
### Auto-invoke Skills
|
||||
|
||||
When performing these actions, ALWAYS invoke the corresponding skill FIRST:
|
||||
|
||||
| Action | Skill |
|
||||
|--------|-------|
|
||||
| Add changelog entry for a PR or feature | `prowler-changelog` |
|
||||
| Adding DRF pagination or permissions | `django-drf` |
|
||||
| Adding privilege escalation detection queries | `prowler-attack-paths-query` |
|
||||
| Committing changes | `prowler-commit` |
|
||||
| Create PR that requires changelog entry | `prowler-changelog` |
|
||||
| Creating API endpoints | `jsonapi` |
|
||||
| Creating Attack Paths queries | `prowler-attack-paths-query` |
|
||||
| Creating ViewSets, serializers, or filters in api/ | `django-drf` |
|
||||
| Creating a git commit | `prowler-commit` |
|
||||
| Creating/modifying models, views, serializers | `prowler-api` |
|
||||
| Implementing JSON:API endpoints | `django-drf` |
|
||||
| Modifying API responses | `jsonapi` |
|
||||
| Review changelog format and conventions | `prowler-changelog` |
|
||||
| Reviewing JSON:API compliance | `jsonapi` |
|
||||
| Testing RLS tenant isolation | `prowler-test-api` |
|
||||
| Update CHANGELOG.md in any component | `prowler-changelog` |
|
||||
| Updating existing Attack Paths queries | `prowler-attack-paths-query` |
|
||||
| Writing Prowler API tests | `prowler-test-api` |
|
||||
| Writing Python tests with pytest | `pytest` |
|
||||
|
||||
---
|
||||
|
||||
## CRITICAL RULES - NON-NEGOTIABLE
|
||||
|
||||
### Models
|
||||
|
||||
+182
-65
@@ -2,45 +2,112 @@
|
||||
|
||||
All notable changes to the **Prowler API** are documented in this file.
|
||||
|
||||
## [1.18.0] (Prowler UNRELEASED)
|
||||
## [1.20.0] (Prowler UNRELEASED)
|
||||
|
||||
### Added
|
||||
- `/api/v1/overviews/compliance-watchlist` to retrieve the compliance watchlist [(#9596)](https://github.com/prowler-cloud/prowler/pull/9596)
|
||||
- Support AlibabaCloud provider [(#9485)](https://github.com/prowler-cloud/prowler/pull/9485)
|
||||
- `provider_id` and `provider_id__in` filter aliases for findings endpoints to enable consistent frontend parameter naming [(#9701)](https://github.com/prowler-cloud/prowler/pull/9701)
|
||||
### 🔄 Changed
|
||||
|
||||
- Attack Paths: Queries definition now has short description and attribution [(#9983)](https://github.com/prowler-cloud/prowler/pull/9983)
|
||||
- Attack Paths: Internet node is created while scan [(#9992)](https://github.com/prowler-cloud/prowler/pull/9992)
|
||||
|
||||
---
|
||||
|
||||
## [1.17.2] (Prowler v5.16.2)
|
||||
## [1.19.2] (Prowler v5.18.2)
|
||||
|
||||
### Security
|
||||
- Updated dependencies to patch security vulnerabilities: Django 5.1.15 (CVE-2025-64460, CVE-2025-13372), Werkzeug 3.1.4 (CVE-2025-66221), sqlparse 0.5.5 (PVE-2025-82038), fonttools 4.60.2 (CVE-2025-66034) [(#9730)](https://github.com/prowler-cloud/prowler/pull/9730)
|
||||
### 🐞 Fixed
|
||||
|
||||
- SAML role mapping now prevents removing the last MANAGE_ACCOUNT user [(#10007)](https://github.com/prowler-cloud/prowler/pull/10007)
|
||||
|
||||
---
|
||||
|
||||
## [1.19.0] (Prowler v5.18.0)
|
||||
|
||||
### 🚀 Added
|
||||
|
||||
- Cloudflare provider support [(#9907)](https://github.com/prowler-cloud/prowler/pull/9907)
|
||||
- Attack Paths: Bedrock Code Interpreter and AttachRolePolicy privilege escalation queries [(#9885)](https://github.com/prowler-cloud/prowler/pull/9885)
|
||||
- `provider_id` and `provider_id__in` filters for resources endpoints (`GET /resources` and `GET /resources/metadata/latest`) [(#9864)](https://github.com/prowler-cloud/prowler/pull/9864)
|
||||
- Added memory optimizations for large compliance report generation [(#9444)](https://github.com/prowler-cloud/prowler/pull/9444)
|
||||
- `GET /api/v1/resources/{id}/events` endpoint to retrieve AWS resource modification history from CloudTrail [(#9101)](https://github.com/prowler-cloud/prowler/pull/9101)
|
||||
- Partial index on findings to speed up new failed findings queries [(#9904)](https://github.com/prowler-cloud/prowler/pull/9904)
|
||||
|
||||
### 🔄 Changed
|
||||
|
||||
- Lazy-load providers and compliance data to reduce API/worker startup memory and time [(#9857)](https://github.com/prowler-cloud/prowler/pull/9857)
|
||||
- Attack Paths: Pinned Cartography to version `0.126.1`, adding AWS scans for SageMaker, CloudFront and Bedrock [(#9893)](https://github.com/prowler-cloud/prowler/issues/9893)
|
||||
- Remove unused indexes [(#9904)](https://github.com/prowler-cloud/prowler/pull/9904)
|
||||
- Attack Paths: Modified the behaviour of the Cartography scans to use the same Neo4j database per tenant, instead of individual databases per scans [(#9955)](https://github.com/prowler-cloud/prowler/pull/9955)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- Attack Paths: `aws-security-groups-open-internet-facing` query returning no results due to incorrect relationship matching [(#9892)](https://github.com/prowler-cloud/prowler/pull/9892)
|
||||
|
||||
---
|
||||
|
||||
## [1.18.1] (Prowler v5.17.1)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- Improve API startup process by `manage.py` argument detection [(#9856)](https://github.com/prowler-cloud/prowler/pull/9856)
|
||||
- Deleting providers don't try to delete a `None` Neo4j database when an Attack Paths scan is scheduled [(#9858)](https://github.com/prowler-cloud/prowler/pull/9858)
|
||||
- Use replica database for reading Findings to add them to the Attack Paths graph [(#9861)](https://github.com/prowler-cloud/prowler/pull/9861)
|
||||
- Attack paths findings loading query to use streaming generator for O(batch_size) memory instead of O(total_findings) [(#9862)](https://github.com/prowler-cloud/prowler/pull/9862)
|
||||
- Lazy load Neo4j driver [(#9868)](https://github.com/prowler-cloud/prowler/pull/9868)
|
||||
- Use `Findings.all_objects` to avoid the `ActiveProviderPartitionedManager` [(#9869)](https://github.com/prowler-cloud/prowler/pull/9869)
|
||||
- Lazy load Neo4j driver for workers only [(#9872)](https://github.com/prowler-cloud/prowler/pull/9872)
|
||||
- Improve Cypher query for inserting Findings into Attack Paths scan graphs [(#9874)](https://github.com/prowler-cloud/prowler/pull/9874)
|
||||
- Clear Neo4j database cache after Attack Paths scan and each API query [(#9877)](https://github.com/prowler-cloud/prowler/pull/9877)
|
||||
- Deduplicated scheduled scans for long-running providers [(#9829)](https://github.com/prowler-cloud/prowler/pull/9829)
|
||||
|
||||
---
|
||||
|
||||
## [1.18.0] (Prowler v5.17.0)
|
||||
|
||||
### 🚀 Added
|
||||
|
||||
- `/api/v1/overviews/compliance-watchlist` endpoint to retrieve the compliance watchlist [(#9596)](https://github.com/prowler-cloud/prowler/pull/9596)
|
||||
- AlibabaCloud provider support [(#9485)](https://github.com/prowler-cloud/prowler/pull/9485)
|
||||
- `/api/v1/overviews/resource-groups` endpoint to retrieve an overview of resource groups based on finding severities [(#9694)](https://github.com/prowler-cloud/prowler/pull/9694)
|
||||
- `group` filter for `GET /findings` and `GET /findings/metadata/latest` endpoints [(#9694)](https://github.com/prowler-cloud/prowler/pull/9694)
|
||||
- `provider_id` and `provider_id__in` filter aliases for findings endpoints to enable consistent frontend parameter naming [(#9701)](https://github.com/prowler-cloud/prowler/pull/9701)
|
||||
- Attack Paths: `/api/v1/attack-paths-scans` for AWS providers backed by Neo4j [(#9805)](https://github.com/prowler-cloud/prowler/pull/9805)
|
||||
|
||||
### 🔐 Security
|
||||
|
||||
- Django 5.1.15 (CVE-2025-64460, CVE-2025-13372), Werkzeug 3.1.4 (CVE-2025-66221), sqlparse 0.5.5 (PVE-2025-82038), fonttools 4.60.2 (CVE-2025-66034) [(#9730)](https://github.com/prowler-cloud/prowler/pull/9730)
|
||||
- `safety` to `3.7.0` and `filelock` to `3.20.3` due to [Safety vulnerability 82754 (CVE-2025-68146)](https://data.safetycli.com/v/82754/97c/) [(#9816)](https://github.com/prowler-cloud/prowler/pull/9816)
|
||||
- `pyasn1` to v0.6.2 to address [CVE-2026-23490](https://nvd.nist.gov/vuln/detail/CVE-2026-23490) [(#9818)](https://github.com/prowler-cloud/prowler/pull/9818)
|
||||
- `django-allauth[saml]` to v65.13.0 to address [CVE-2025-65431](https://nvd.nist.gov/vuln/detail/CVE-2025-65431) [(#9575)](https://github.com/prowler-cloud/prowler/pull/9575)
|
||||
|
||||
---
|
||||
|
||||
## [1.17.1] (Prowler v5.16.1)
|
||||
|
||||
### Changed
|
||||
### 🔄 Changed
|
||||
|
||||
- Security Hub integration error when no regions [(#9635)](https://github.com/prowler-cloud/prowler/pull/9635)
|
||||
|
||||
### Fixed
|
||||
### 🐞 Fixed
|
||||
|
||||
- Orphan scheduled scans caused by transaction isolation during provider creation [(#9633)](https://github.com/prowler-cloud/prowler/pull/9633)
|
||||
|
||||
---
|
||||
|
||||
## [1.17.0] (Prowler v5.16.0)
|
||||
|
||||
### Added
|
||||
### 🚀 Added
|
||||
|
||||
- New endpoint to retrieve and overview of the categories based on finding severities [(#9529)](https://github.com/prowler-cloud/prowler/pull/9529)
|
||||
- Endpoints `GET /findings` and `GET /findings/latests` can now use the category filter [(#9529)](https://github.com/prowler-cloud/prowler/pull/9529)
|
||||
- Account id, alias and provider name to PDF reporting table [(#9574)](https://github.com/prowler-cloud/prowler/pull/9574)
|
||||
|
||||
### Changed
|
||||
### 🔄 Changed
|
||||
|
||||
- Endpoint `GET /overviews/attack-surfaces` no longer returns the related check IDs [(#9529)](https://github.com/prowler-cloud/prowler/pull/9529)
|
||||
- OpenAI provider to only load chat-compatible models with tool calling support [(#9523)](https://github.com/prowler-cloud/prowler/pull/9523)
|
||||
- Increased execution delay for the first scheduled scan tasks to 5 seconds[(#9558)](https://github.com/prowler-cloud/prowler/pull/9558)
|
||||
|
||||
### Fixed
|
||||
### 🐞 Fixed
|
||||
|
||||
- Made `scan_id` a required filter in the compliance overview endpoint [(#9560)](https://github.com/prowler-cloud/prowler/pull/9560)
|
||||
- Reduced unnecessary UPDATE resources operations by only saving when tag mappings change, lowering write load during scans [(#9569)](https://github.com/prowler-cloud/prowler/pull/9569)
|
||||
|
||||
@@ -48,19 +115,22 @@ All notable changes to the **Prowler API** are documented in this file.
|
||||
|
||||
## [1.16.1] (Prowler v5.15.1)
|
||||
|
||||
### Fixed
|
||||
### 🐞 Fixed
|
||||
|
||||
- Race condition in scheduled scan creation by adding countdown to task [(#9516)](https://github.com/prowler-cloud/prowler/pull/9516)
|
||||
|
||||
## [1.16.0] (Prowler v5.15.0)
|
||||
|
||||
### Added
|
||||
### 🚀 Added
|
||||
|
||||
- New endpoint to retrieve an overview of the attack surfaces [(#9309)](https://github.com/prowler-cloud/prowler/pull/9309)
|
||||
- New endpoint `GET /api/v1/overviews/findings_severity/timeseries` to retrieve daily aggregated findings by severity level [(#9363)](https://github.com/prowler-cloud/prowler/pull/9363)
|
||||
- Lighthouse AI support for Amazon Bedrock API key [(#9343)](https://github.com/prowler-cloud/prowler/pull/9343)
|
||||
- Exception handler for provider deletions during scans [(#9414)](https://github.com/prowler-cloud/prowler/pull/9414)
|
||||
- Support to use admin credentials through the read replica database [(#9440)](https://github.com/prowler-cloud/prowler/pull/9440)
|
||||
|
||||
### Changed
|
||||
### 🔄 Changed
|
||||
|
||||
- Error messages from Lighthouse celery tasks [(#9165)](https://github.com/prowler-cloud/prowler/pull/9165)
|
||||
- Restore the compliance overview endpoint's mandatory filters [(#9338)](https://github.com/prowler-cloud/prowler/pull/9338)
|
||||
|
||||
@@ -68,7 +138,8 @@ All notable changes to the **Prowler API** are documented in this file.
|
||||
|
||||
## [1.15.2] (Prowler v5.14.2)
|
||||
|
||||
### Fixed
|
||||
### 🐞 Fixed
|
||||
|
||||
- Unique constraint violation during compliance overviews task [(#9436)](https://github.com/prowler-cloud/prowler/pull/9436)
|
||||
- Division by zero error in ENS PDF report when all requirements are manual [(#9443)](https://github.com/prowler-cloud/prowler/pull/9443)
|
||||
|
||||
@@ -76,7 +147,8 @@ All notable changes to the **Prowler API** are documented in this file.
|
||||
|
||||
## [1.15.1] (Prowler v5.14.1)
|
||||
|
||||
### Fixed
|
||||
### 🐞 Fixed
|
||||
|
||||
- Fix typo in PDF reporting [(#9345)](https://github.com/prowler-cloud/prowler/pull/9345)
|
||||
- Fix IaC provider initialization failure when mutelist processor is configured [(#9331)](https://github.com/prowler-cloud/prowler/pull/9331)
|
||||
- Match logic for ThreatScore when counting findings [(#9348)](https://github.com/prowler-cloud/prowler/pull/9348)
|
||||
@@ -85,7 +157,8 @@ All notable changes to the **Prowler API** are documented in this file.
|
||||
|
||||
## [1.15.0] (Prowler v5.14.0)
|
||||
|
||||
### Added
|
||||
### 🚀 Added
|
||||
|
||||
- IaC (Infrastructure as Code) provider support for remote repositories [(#8751)](https://github.com/prowler-cloud/prowler/pull/8751)
|
||||
- Extend `GET /api/v1/providers` with provider-type filters and optional pagination disable to support the new Overview filters [(#8975)](https://github.com/prowler-cloud/prowler/pull/8975)
|
||||
- New endpoint to retrieve the number of providers grouped by provider type [(#8975)](https://github.com/prowler-cloud/prowler/pull/8975)
|
||||
@@ -104,11 +177,13 @@ All notable changes to the **Prowler API** are documented in this file.
|
||||
- Enhanced compliance overview endpoint with provider filtering and latest scan aggregation [(#9244)](https://github.com/prowler-cloud/prowler/pull/9244)
|
||||
- New endpoint `GET /api/v1/overview/regions` to retrieve aggregated findings data by region [(#9273)](https://github.com/prowler-cloud/prowler/pull/9273)
|
||||
|
||||
### Changed
|
||||
### 🔄 Changed
|
||||
|
||||
- Optimized database write queries for scan related tasks [(#9190)](https://github.com/prowler-cloud/prowler/pull/9190)
|
||||
- Date filters are now optional for `GET /api/v1/overviews/services` endpoint; returns latest scan data by default [(#9248)](https://github.com/prowler-cloud/prowler/pull/9248)
|
||||
|
||||
### Fixed
|
||||
### 🐞 Fixed
|
||||
|
||||
- Scans no longer fail when findings have UIDs exceeding 300 characters; such findings are now skipped with detailed logging [(#9246)](https://github.com/prowler-cloud/prowler/pull/9246)
|
||||
- Updated unique constraint for `Provider` model to exclude soft-deleted entries, resolving duplicate errors when re-deleting providers [(#9054)](https://github.com/prowler-cloud/prowler/pull/9054)
|
||||
- Removed compliance generation for providers without compliance frameworks [(#9208)](https://github.com/prowler-cloud/prowler/pull/9208)
|
||||
@@ -116,14 +191,16 @@ All notable changes to the **Prowler API** are documented in this file.
|
||||
- Severity overview endpoint now ignores muted findings as expected [(#9283)](https://github.com/prowler-cloud/prowler/pull/9283)
|
||||
- Fixed discrepancy between ThreatScore PDF report values and database calculations [(#9296)](https://github.com/prowler-cloud/prowler/pull/9296)
|
||||
|
||||
### Security
|
||||
### 🔐 Security
|
||||
|
||||
- Django updated to the latest 5.1 security release, 5.1.14, due to problems with potential [SQL injection](https://github.com/prowler-cloud/prowler/security/dependabot/113) and [denial-of-service vulnerability](https://github.com/prowler-cloud/prowler/security/dependabot/114) [(#9176)](https://github.com/prowler-cloud/prowler/pull/9176)
|
||||
|
||||
---
|
||||
|
||||
## [1.14.1] (Prowler v5.13.1)
|
||||
|
||||
### Fixed
|
||||
### 🐞 Fixed
|
||||
|
||||
- `/api/v1/overviews/providers` collapses data by provider type so the UI receives a single aggregated record per cloud family even when multiple accounts exist [(#9053)](https://github.com/prowler-cloud/prowler/pull/9053)
|
||||
- Added retry logic to database transactions to handle Aurora read replica connection failures during scale-down events [(#9064)](https://github.com/prowler-cloud/prowler/pull/9064)
|
||||
- Security Hub integrations stop failing when they read relationships via the replica by allowing replica relations and saving updates through the primary [(#9080)](https://github.com/prowler-cloud/prowler/pull/9080)
|
||||
@@ -132,7 +209,8 @@ All notable changes to the **Prowler API** are documented in this file.
|
||||
|
||||
## [1.14.0] (Prowler v5.13.0)
|
||||
|
||||
### Added
|
||||
### 🚀 Added
|
||||
|
||||
- Default JWT keys are generated and stored if they are missing from configuration [(#8655)](https://github.com/prowler-cloud/prowler/pull/8655)
|
||||
- `compliance_name` for each compliance [(#7920)](https://github.com/prowler-cloud/prowler/pull/7920)
|
||||
- Support C5 compliance framework for the AWS provider [(#8830)](https://github.com/prowler-cloud/prowler/pull/8830)
|
||||
@@ -145,35 +223,41 @@ All notable changes to the **Prowler API** are documented in this file.
|
||||
- Support Common Cloud Controls for AWS, Azure and GCP [(#8000)](https://github.com/prowler-cloud/prowler/pull/8000)
|
||||
- Add `provider_id__in` filter support to findings and findings severity overview endpoints [(#8951)](https://github.com/prowler-cloud/prowler/pull/8951)
|
||||
|
||||
### Changed
|
||||
### 🔄 Changed
|
||||
|
||||
- Now the MANAGE_ACCOUNT permission is required to modify or read user permissions instead of MANAGE_USERS [(#8281)](https://github.com/prowler-cloud/prowler/pull/8281)
|
||||
- Now at least one user with MANAGE_ACCOUNT permission is required in the tenant [(#8729)](https://github.com/prowler-cloud/prowler/pull/8729)
|
||||
|
||||
### Security
|
||||
### 🔐 Security
|
||||
|
||||
- Django updated to the latest 5.1 security release, 5.1.13, due to problems with potential [SQL injection](https://github.com/prowler-cloud/prowler/security/dependabot/104) and [directory traversals](https://github.com/prowler-cloud/prowler/security/dependabot/103) [(#8842)](https://github.com/prowler-cloud/prowler/pull/8842)
|
||||
|
||||
---
|
||||
|
||||
## [1.13.2] (Prowler v5.12.3)
|
||||
|
||||
### Fixed
|
||||
### 🐞 Fixed
|
||||
|
||||
- 500 error when deleting user [(#8731)](https://github.com/prowler-cloud/prowler/pull/8731)
|
||||
|
||||
---
|
||||
|
||||
## [1.13.1] (Prowler v5.12.2)
|
||||
|
||||
### Changed
|
||||
### 🔄 Changed
|
||||
|
||||
- Renamed compliance overview task queue to `compliance` [(#8755)](https://github.com/prowler-cloud/prowler/pull/8755)
|
||||
|
||||
### Security
|
||||
### 🔐 Security
|
||||
|
||||
- Django updated to the latest 5.1 security release, 5.1.12, due to [problems](https://www.djangoproject.com/weblog/2025/sep/03/security-releases/) with potential SQL injection in FilteredRelation column aliases [(#8693)](https://github.com/prowler-cloud/prowler/pull/8693)
|
||||
|
||||
---
|
||||
|
||||
## [1.13.0] (Prowler v5.12.0)
|
||||
|
||||
### Added
|
||||
### 🚀 Added
|
||||
|
||||
- Integration with JIRA, enabling sending findings to a JIRA project [(#8622)](https://github.com/prowler-cloud/prowler/pull/8622), [(#8637)](https://github.com/prowler-cloud/prowler/pull/8637)
|
||||
- `GET /overviews/findings_severity` now supports `filter[status]` and `filter[status__in]` to aggregate by specific statuses (`FAIL`, `PASS`)[(#8186)](https://github.com/prowler-cloud/prowler/pull/8186)
|
||||
- Throttling options for `/api/v1/tokens` using the `DJANGO_THROTTLE_TOKEN_OBTAIN` environment variable [(#8647)](https://github.com/prowler-cloud/prowler/pull/8647)
|
||||
@@ -182,101 +266,120 @@ All notable changes to the **Prowler API** are documented in this file.
|
||||
|
||||
## [1.12.0] (Prowler v5.11.0)
|
||||
|
||||
### Added
|
||||
### 🚀 Added
|
||||
|
||||
- Lighthouse support for OpenAI GPT-5 [(#8527)](https://github.com/prowler-cloud/prowler/pull/8527)
|
||||
- Integration with Amazon Security Hub, enabling sending findings to Security Hub [(#8365)](https://github.com/prowler-cloud/prowler/pull/8365)
|
||||
- Generate ASFF output for AWS providers with SecurityHub integration enabled [(#8569)](https://github.com/prowler-cloud/prowler/pull/8569)
|
||||
|
||||
### Fixed
|
||||
### 🐞 Fixed
|
||||
|
||||
- GitHub provider always scans user instead of organization when using provider UID [(#8587)](https://github.com/prowler-cloud/prowler/pull/8587)
|
||||
|
||||
---
|
||||
|
||||
## [1.11.0] (Prowler v5.10.0)
|
||||
|
||||
### Added
|
||||
### 🚀 Added
|
||||
|
||||
- Github provider support [(#8271)](https://github.com/prowler-cloud/prowler/pull/8271)
|
||||
- Integration with Amazon S3, enabling storage and retrieval of scan data via S3 buckets [(#8056)](https://github.com/prowler-cloud/prowler/pull/8056)
|
||||
|
||||
### Fixed
|
||||
### 🐞 Fixed
|
||||
|
||||
- Avoid sending errors to Sentry in M365 provider when user authentication fails [(#8420)](https://github.com/prowler-cloud/prowler/pull/8420)
|
||||
|
||||
---
|
||||
|
||||
## [1.10.2] (Prowler v5.9.2)
|
||||
|
||||
### Changed
|
||||
### 🔄 Changed
|
||||
|
||||
- Optimized queries for resources views [(#8336)](https://github.com/prowler-cloud/prowler/pull/8336)
|
||||
|
||||
---
|
||||
|
||||
## [v1.10.1] (Prowler v5.9.1)
|
||||
|
||||
### Fixed
|
||||
### 🐞 Fixed
|
||||
|
||||
- Calculate failed findings during scans to prevent heavy database queries [(#8322)](https://github.com/prowler-cloud/prowler/pull/8322)
|
||||
|
||||
---
|
||||
|
||||
## [v1.10.0] (Prowler v5.9.0)
|
||||
|
||||
### Added
|
||||
### 🚀 Added
|
||||
|
||||
- SSO with SAML support [(#8175)](https://github.com/prowler-cloud/prowler/pull/8175)
|
||||
- `GET /resources/metadata`, `GET /resources/metadata/latest` and `GET /resources/latest` to expose resource metadata and latest scan results [(#8112)](https://github.com/prowler-cloud/prowler/pull/8112)
|
||||
|
||||
### Changed
|
||||
### 🔄 Changed
|
||||
|
||||
- `/processors` endpoints to post-process findings. Currently, only the Mutelist processor is supported to allow to mute findings.
|
||||
- Optimized the underlying queries for resources endpoints [(#8112)](https://github.com/prowler-cloud/prowler/pull/8112)
|
||||
- Optimized include parameters for resources view [(#8229)](https://github.com/prowler-cloud/prowler/pull/8229)
|
||||
- Optimized overview background tasks [(#8300)](https://github.com/prowler-cloud/prowler/pull/8300)
|
||||
|
||||
### Fixed
|
||||
### 🐞 Fixed
|
||||
|
||||
- Search filter for findings and resources [(#8112)](https://github.com/prowler-cloud/prowler/pull/8112)
|
||||
- RBAC is now applied to `GET /overviews/providers` [(#8277)](https://github.com/prowler-cloud/prowler/pull/8277)
|
||||
|
||||
### Changed
|
||||
### 🔄 Changed
|
||||
|
||||
- `POST /schedules/daily` returns a `409 CONFLICT` if already created [(#8258)](https://github.com/prowler-cloud/prowler/pull/8258)
|
||||
|
||||
### Security
|
||||
### 🔐 Security
|
||||
|
||||
- Enhanced password validation to enforce 12+ character passwords with special characters, uppercase, lowercase, and numbers [(#8225)](https://github.com/prowler-cloud/prowler/pull/8225)
|
||||
|
||||
---
|
||||
|
||||
## [v1.9.1] (Prowler v5.8.1)
|
||||
|
||||
### Added
|
||||
### 🚀 Added
|
||||
|
||||
- Custom exception for provider connection errors during scans [(#8234)](https://github.com/prowler-cloud/prowler/pull/8234)
|
||||
|
||||
### Changed
|
||||
### 🔄 Changed
|
||||
|
||||
- Summary and overview tasks now use a dedicated queue and no longer propagate errors to compliance tasks [(#8214)](https://github.com/prowler-cloud/prowler/pull/8214)
|
||||
|
||||
### Fixed
|
||||
### 🐞 Fixed
|
||||
|
||||
- Scan with no resources will not trigger legacy code for findings metadata [(#8183)](https://github.com/prowler-cloud/prowler/pull/8183)
|
||||
- Invitation email comparison case-insensitive [(#8206)](https://github.com/prowler-cloud/prowler/pull/8206)
|
||||
|
||||
### Removed
|
||||
### ❌ Removed
|
||||
|
||||
- Validation of the provider's secret type during updates [(#8197)](https://github.com/prowler-cloud/prowler/pull/8197)
|
||||
|
||||
---
|
||||
|
||||
## [v1.9.0] (Prowler v5.8.0)
|
||||
|
||||
### Added
|
||||
### 🚀 Added
|
||||
|
||||
- Support GCP Service Account key [(#7824)](https://github.com/prowler-cloud/prowler/pull/7824)
|
||||
- `GET /compliance-overviews` endpoints to retrieve compliance metadata and specific requirements statuses [(#7877)](https://github.com/prowler-cloud/prowler/pull/7877)
|
||||
- Lighthouse configuration support [(#7848)](https://github.com/prowler-cloud/prowler/pull/7848)
|
||||
|
||||
### Changed
|
||||
### 🔄 Changed
|
||||
|
||||
- Reworked `GET /compliance-overviews` to return proper requirement metrics [(#7877)](https://github.com/prowler-cloud/prowler/pull/7877)
|
||||
- Optional `user` and `password` for M365 provider [(#7992)](https://github.com/prowler-cloud/prowler/pull/7992)
|
||||
|
||||
### Fixed
|
||||
### 🐞 Fixed
|
||||
|
||||
- Scheduled scans are no longer deleted when their daily schedule run is disabled [(#8082)](https://github.com/prowler-cloud/prowler/pull/8082)
|
||||
|
||||
---
|
||||
|
||||
## [v1.8.5] (Prowler v5.7.5)
|
||||
|
||||
### Fixed
|
||||
### 🐞 Fixed
|
||||
|
||||
- Normalize provider UID to ensure safe and unique export directory paths [(#8007)](https://github.com/prowler-cloud/prowler/pull/8007).
|
||||
- Blank resource types in `/metadata` endpoints [(#8027)](https://github.com/prowler-cloud/prowler/pull/8027)
|
||||
|
||||
@@ -284,20 +387,24 @@ All notable changes to the **Prowler API** are documented in this file.
|
||||
|
||||
## [v1.8.4] (Prowler v5.7.4)
|
||||
|
||||
### Removed
|
||||
### ❌ Removed
|
||||
|
||||
- Reverted RLS transaction handling and DB custom backend [(#7994)](https://github.com/prowler-cloud/prowler/pull/7994)
|
||||
|
||||
---
|
||||
|
||||
## [v1.8.3] (Prowler v5.7.3)
|
||||
|
||||
### Added
|
||||
### 🚀 Added
|
||||
|
||||
- Database backend to handle already closed connections [(#7935)](https://github.com/prowler-cloud/prowler/pull/7935)
|
||||
|
||||
### Changed
|
||||
### 🔄 Changed
|
||||
|
||||
- Renamed field encrypted_password to password for M365 provider [(#7784)](https://github.com/prowler-cloud/prowler/pull/7784)
|
||||
|
||||
### Fixed
|
||||
### 🐞 Fixed
|
||||
|
||||
- Transaction persistence with RLS operations [(#7916)](https://github.com/prowler-cloud/prowler/pull/7916)
|
||||
- Reverted the change `get_with_retry` to use the original `get` method for retrieving tasks [(#7932)](https://github.com/prowler-cloud/prowler/pull/7932)
|
||||
|
||||
@@ -305,7 +412,8 @@ All notable changes to the **Prowler API** are documented in this file.
|
||||
|
||||
## [v1.8.2] (Prowler v5.7.2)
|
||||
|
||||
### Fixed
|
||||
### 🐞 Fixed
|
||||
|
||||
- Task lookup to use task_kwargs instead of task_args for scan report resolution [(#7830)](https://github.com/prowler-cloud/prowler/pull/7830)
|
||||
- Kubernetes UID validation to allow valid context names [(#7871)](https://github.com/prowler-cloud/prowler/pull/7871)
|
||||
- Connection status verification before launching a scan [(#7831)](https://github.com/prowler-cloud/prowler/pull/7831)
|
||||
@@ -316,14 +424,16 @@ All notable changes to the **Prowler API** are documented in this file.
|
||||
|
||||
## [v1.8.1] (Prowler v5.7.1)
|
||||
|
||||
### Fixed
|
||||
### 🐞 Fixed
|
||||
|
||||
- Added database index to improve performance on finding lookup [(#7800)](https://github.com/prowler-cloud/prowler/pull/7800)
|
||||
|
||||
---
|
||||
|
||||
## [v1.8.0] (Prowler v5.7.0)
|
||||
|
||||
### Added
|
||||
### 🚀 Added
|
||||
|
||||
- Huge improvements to `/findings/metadata` and resource related filters for findings [(#7690)](https://github.com/prowler-cloud/prowler/pull/7690)
|
||||
- Improvements to `/overviews` endpoints [(#7690)](https://github.com/prowler-cloud/prowler/pull/7690)
|
||||
- Queue to perform backfill background tasks [(#7690)](https://github.com/prowler-cloud/prowler/pull/7690)
|
||||
@@ -334,7 +444,7 @@ All notable changes to the **Prowler API** are documented in this file.
|
||||
|
||||
## [v1.7.0] (Prowler v5.6.0)
|
||||
|
||||
### Added
|
||||
### 🚀 Added
|
||||
|
||||
- M365 as a new provider [(#7563)](https://github.com/prowler-cloud/prowler/pull/7563)
|
||||
- `compliance/` folder and ZIP‐export functionality for all compliance reports [(#7653)](https://github.com/prowler-cloud/prowler/pull/7653)
|
||||
@@ -344,7 +454,7 @@ All notable changes to the **Prowler API** are documented in this file.
|
||||
|
||||
## [v1.6.0] (Prowler v5.5.0)
|
||||
|
||||
### Added
|
||||
### 🚀 Added
|
||||
|
||||
- Support for developing new integrations [(#7167)](https://github.com/prowler-cloud/prowler/pull/7167)
|
||||
- HTTP Security Headers [(#7289)](https://github.com/prowler-cloud/prowler/pull/7289)
|
||||
@@ -356,14 +466,16 @@ All notable changes to the **Prowler API** are documented in this file.
|
||||
|
||||
## [v1.5.4] (Prowler v5.4.4)
|
||||
|
||||
### Fixed
|
||||
### 🐞 Fixed
|
||||
|
||||
- Bug with periodic tasks when trying to delete a provider [(#7466)](https://github.com/prowler-cloud/prowler/pull/7466)
|
||||
|
||||
---
|
||||
|
||||
## [v1.5.3] (Prowler v5.4.3)
|
||||
|
||||
### Fixed
|
||||
### 🐞 Fixed
|
||||
|
||||
- Duplicated scheduled scans handling [(#7401)](https://github.com/prowler-cloud/prowler/pull/7401)
|
||||
- Environment variable to configure the deletion task batch size [(#7423)](https://github.com/prowler-cloud/prowler/pull/7423)
|
||||
|
||||
@@ -371,14 +483,16 @@ All notable changes to the **Prowler API** are documented in this file.
|
||||
|
||||
## [v1.5.2] (Prowler v5.4.2)
|
||||
|
||||
### Changed
|
||||
### 🔄 Changed
|
||||
|
||||
- Refactored deletion logic and implemented retry mechanism for deletion tasks [(#7349)](https://github.com/prowler-cloud/prowler/pull/7349)
|
||||
|
||||
---
|
||||
|
||||
## [v1.5.1] (Prowler v5.4.1)
|
||||
|
||||
### Fixed
|
||||
### 🐞 Fixed
|
||||
|
||||
- Handle response in case local files are missing [(#7183)](https://github.com/prowler-cloud/prowler/pull/7183)
|
||||
- Race condition when deleting export files after the S3 upload [(#7172)](https://github.com/prowler-cloud/prowler/pull/7172)
|
||||
- Handle exception when a provider has no secret in test connection [(#7283)](https://github.com/prowler-cloud/prowler/pull/7283)
|
||||
@@ -387,19 +501,22 @@ All notable changes to the **Prowler API** are documented in this file.
|
||||
|
||||
## [v1.5.0] (Prowler v5.4.0)
|
||||
|
||||
### Added
|
||||
### 🚀 Added
|
||||
|
||||
- Social login integration with Google and GitHub [(#6906)](https://github.com/prowler-cloud/prowler/pull/6906)
|
||||
- API scan report system, now all scans launched from the API will generate a compressed file with the report in OCSF, CSV and HTML formats [(#6878)](https://github.com/prowler-cloud/prowler/pull/6878)
|
||||
- Configurable Sentry integration [(#6874)](https://github.com/prowler-cloud/prowler/pull/6874)
|
||||
|
||||
### Changed
|
||||
### 🔄 Changed
|
||||
|
||||
- Optimized `GET /findings` endpoint to improve response time and size [(#7019)](https://github.com/prowler-cloud/prowler/pull/7019)
|
||||
|
||||
---
|
||||
|
||||
## [v1.4.0] (Prowler v5.3.0)
|
||||
|
||||
### Changed
|
||||
### 🔄 Changed
|
||||
|
||||
- Daily scheduled scan instances are now created beforehand with `SCHEDULED` state [(#6700)](https://github.com/prowler-cloud/prowler/pull/6700)
|
||||
- Findings endpoints now require at least one date filter [(#6800)](https://github.com/prowler-cloud/prowler/pull/6800)
|
||||
- Findings metadata endpoint received a performance improvement [(#6863)](https://github.com/prowler-cloud/prowler/pull/6863)
|
||||
|
||||
@@ -32,7 +32,7 @@ start_prod_server() {
|
||||
|
||||
start_worker() {
|
||||
echo "Starting the worker..."
|
||||
poetry run python -m celery -A config.celery worker -l "${DJANGO_LOGGING_LEVEL:-info}" -Q celery,scans,scan-reports,deletion,backfill,overview,integrations,compliance -E --max-tasks-per-child 1
|
||||
poetry run python -m celery -A config.celery worker -l "${DJANGO_LOGGING_LEVEL:-info}" -Q celery,scans,scan-reports,deletion,backfill,overview,integrations,compliance,attack-paths-scans -E --max-tasks-per-child 1
|
||||
}
|
||||
|
||||
start_worker_beat() {
|
||||
|
||||
Generated
+3424
-1928
File diff suppressed because it is too large
Load Diff
+7
-4
@@ -5,10 +5,10 @@ requires = ["poetry-core"]
|
||||
[project]
|
||||
authors = [{name = "Prowler Engineering", email = "engineering@prowler.com"}]
|
||||
dependencies = [
|
||||
"celery[pytest] (>=5.4.0,<6.0.0)",
|
||||
"celery (>=5.4.0,<6.0.0)",
|
||||
"dj-rest-auth[with_social,jwt] (==7.0.1)",
|
||||
"django (==5.1.15)",
|
||||
"django-allauth[saml] (>=65.8.0,<66.0.0)",
|
||||
"django-allauth[saml] (>=65.13.0,<66.0.0)",
|
||||
"django-celery-beat (>=2.7.0,<3.0.0)",
|
||||
"django-celery-results (>=2.5.1,<3.0.0)",
|
||||
"django-cors-headers==4.4.0",
|
||||
@@ -36,6 +36,8 @@ dependencies = [
|
||||
"drf-simple-apikey (==2.2.1)",
|
||||
"matplotlib (>=3.10.6,<4.0.0)",
|
||||
"reportlab (>=4.4.4,<5.0.0)",
|
||||
"neo4j (<6.0.0)",
|
||||
"cartography @ git+https://github.com/prowler-cloud/cartography@0.126.1",
|
||||
"gevent (>=25.9.1,<26.0.0)",
|
||||
"werkzeug (>=3.1.4)",
|
||||
"sqlparse (>=0.5.4)",
|
||||
@@ -47,7 +49,7 @@ name = "prowler-api"
|
||||
package-mode = false
|
||||
# Needed for the SDK compatibility
|
||||
requires-python = ">=3.11,<3.13"
|
||||
version = "1.18.0"
|
||||
version = "1.20.0"
|
||||
|
||||
[project.scripts]
|
||||
celery = "src.backend.config.settings.celery"
|
||||
@@ -57,6 +59,7 @@ bandit = "1.7.9"
|
||||
coverage = "7.5.4"
|
||||
django-silk = "5.3.2"
|
||||
docker = "7.1.0"
|
||||
filelock = "3.20.3"
|
||||
freezegun = "1.5.1"
|
||||
marshmallow = ">=3.15.0,<4.0.0"
|
||||
mypy = "1.10.1"
|
||||
@@ -68,6 +71,6 @@ pytest-env = "1.1.3"
|
||||
pytest-randomly = "3.15.0"
|
||||
pytest-xdist = "3.6.1"
|
||||
ruff = "0.5.0"
|
||||
safety = "3.2.9"
|
||||
safety = "3.7.0"
|
||||
tqdm = "4.67.1"
|
||||
vulture = "2.14"
|
||||
|
||||
@@ -30,16 +30,48 @@ class ApiConfig(AppConfig):
|
||||
def ready(self):
|
||||
from api import schema_extensions # noqa: F401
|
||||
from api import signals # noqa: F401
|
||||
from api.compliance import load_prowler_compliance
|
||||
from api.attack_paths import database as graph_database
|
||||
|
||||
# Generate required cryptographic keys if not present, but only if:
|
||||
# `"manage.py" not in sys.argv`: If an external server (e.g., Gunicorn) is running the app
|
||||
# `"manage.py" not in sys.argv[0]`: If an external server (e.g., Gunicorn) is running the app
|
||||
# `os.environ.get("RUN_MAIN")`: If it's not a Django command or using `runserver`,
|
||||
# only the main process will do it
|
||||
if "manage.py" not in sys.argv or os.environ.get("RUN_MAIN"):
|
||||
if (len(sys.argv) >= 1 and "manage.py" not in sys.argv[0]) or os.environ.get(
|
||||
"RUN_MAIN"
|
||||
):
|
||||
self._ensure_crypto_keys()
|
||||
|
||||
load_prowler_compliance()
|
||||
# Commands that don't need Neo4j
|
||||
SKIP_NEO4J_DJANGO_COMMANDS = [
|
||||
"makemigrations",
|
||||
"migrate",
|
||||
"pgpartition",
|
||||
"check",
|
||||
"help",
|
||||
"showmigrations",
|
||||
"check_and_fix_socialaccount_sites_migration",
|
||||
]
|
||||
|
||||
# Skip Neo4j initialization during tests, some Django commands, and Celery
|
||||
if getattr(settings, "TESTING", False) or (
|
||||
len(sys.argv) > 1
|
||||
and (
|
||||
(
|
||||
"manage.py" in sys.argv[0]
|
||||
and sys.argv[1] in SKIP_NEO4J_DJANGO_COMMANDS
|
||||
)
|
||||
or "celery" in sys.argv[0]
|
||||
)
|
||||
):
|
||||
logger.info(
|
||||
"Skipping Neo4j initialization because tests, some Django commands or Celery"
|
||||
)
|
||||
|
||||
else:
|
||||
graph_database.init_driver()
|
||||
|
||||
# Neo4j driver is initialized at API startup (see api.attack_paths.database)
|
||||
# It remains lazy for Celery workers and selected Django commands
|
||||
|
||||
def _ensure_crypto_keys(self):
|
||||
"""
|
||||
@@ -54,7 +86,7 @@ class ApiConfig(AppConfig):
|
||||
global _keys_initialized
|
||||
|
||||
# Skip key generation if running tests
|
||||
if hasattr(settings, "TESTING") and settings.TESTING:
|
||||
if getattr(settings, "TESTING", False):
|
||||
return
|
||||
|
||||
# Skip if already initialized in this process
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
from api.attack_paths.queries import (
|
||||
AttackPathsQueryDefinition,
|
||||
AttackPathsQueryParameterDefinition,
|
||||
get_queries_for_provider,
|
||||
get_query_by_id,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"AttackPathsQueryDefinition",
|
||||
"AttackPathsQueryParameterDefinition",
|
||||
"get_queries_for_provider",
|
||||
"get_query_by_id",
|
||||
]
|
||||
@@ -0,0 +1,181 @@
|
||||
import atexit
|
||||
import logging
|
||||
import threading
|
||||
|
||||
from contextlib import contextmanager
|
||||
from typing import Iterator
|
||||
from uuid import UUID
|
||||
|
||||
import neo4j
|
||||
import neo4j.exceptions
|
||||
|
||||
from django.conf import settings
|
||||
|
||||
from api.attack_paths.retryable_session import RetryableSession
|
||||
from tasks.jobs.attack_paths.config import BATCH_SIZE, PROVIDER_RESOURCE_LABEL
|
||||
|
||||
# Without this Celery goes crazy with Neo4j logging
|
||||
logging.getLogger("neo4j").setLevel(logging.ERROR)
|
||||
logging.getLogger("neo4j").propagate = False
|
||||
|
||||
SERVICE_UNAVAILABLE_MAX_RETRIES = 3
|
||||
|
||||
# Module-level process-wide driver singleton
|
||||
_driver: neo4j.Driver | None = None
|
||||
_lock = threading.Lock()
|
||||
|
||||
# Base Neo4j functions
|
||||
|
||||
|
||||
def get_uri() -> str:
|
||||
host = settings.DATABASES["neo4j"]["HOST"]
|
||||
port = settings.DATABASES["neo4j"]["PORT"]
|
||||
return f"bolt://{host}:{port}"
|
||||
|
||||
|
||||
def init_driver() -> neo4j.Driver:
|
||||
global _driver
|
||||
if _driver is not None:
|
||||
return _driver
|
||||
|
||||
with _lock:
|
||||
if _driver is None:
|
||||
uri = get_uri()
|
||||
config = settings.DATABASES["neo4j"]
|
||||
|
||||
_driver = neo4j.GraphDatabase.driver(
|
||||
uri,
|
||||
auth=(config["USER"], config["PASSWORD"]),
|
||||
keep_alive=True,
|
||||
max_connection_lifetime=7200,
|
||||
connection_acquisition_timeout=120,
|
||||
max_connection_pool_size=50,
|
||||
)
|
||||
_driver.verify_connectivity()
|
||||
|
||||
# Register cleanup handler (only runs once since we're inside the _driver is None block)
|
||||
atexit.register(close_driver)
|
||||
|
||||
return _driver
|
||||
|
||||
|
||||
def get_driver() -> neo4j.Driver:
|
||||
return init_driver()
|
||||
|
||||
|
||||
def close_driver() -> None: # TODO: Use it
|
||||
global _driver
|
||||
with _lock:
|
||||
if _driver is not None:
|
||||
try:
|
||||
_driver.close()
|
||||
|
||||
finally:
|
||||
_driver = None
|
||||
|
||||
|
||||
@contextmanager
|
||||
def get_session(database: str | None = None) -> Iterator[RetryableSession]:
|
||||
session_wrapper: RetryableSession | None = None
|
||||
|
||||
try:
|
||||
session_wrapper = RetryableSession(
|
||||
session_factory=lambda: get_driver().session(database=database),
|
||||
max_retries=SERVICE_UNAVAILABLE_MAX_RETRIES,
|
||||
)
|
||||
yield session_wrapper
|
||||
|
||||
except neo4j.exceptions.Neo4jError as exc:
|
||||
message = exc.message if exc.message is not None else str(exc)
|
||||
raise GraphDatabaseQueryException(message=message, code=exc.code)
|
||||
|
||||
finally:
|
||||
if session_wrapper is not None:
|
||||
session_wrapper.close()
|
||||
|
||||
|
||||
def create_database(database: str) -> None:
|
||||
query = "CREATE DATABASE $database IF NOT EXISTS"
|
||||
parameters = {"database": database}
|
||||
|
||||
with get_session() as session:
|
||||
session.run(query, parameters)
|
||||
|
||||
|
||||
def drop_database(database: str) -> None:
|
||||
query = f"DROP DATABASE `{database}` IF EXISTS DESTROY DATA"
|
||||
|
||||
with get_session() as session:
|
||||
session.run(query)
|
||||
|
||||
|
||||
def drop_subgraph(database: str, provider_id: str) -> int:
|
||||
"""
|
||||
Delete all nodes for a provider from the tenant database.
|
||||
|
||||
Uses batched deletion to avoid memory issues with large graphs.
|
||||
Silently returns 0 if the database doesn't exist.
|
||||
"""
|
||||
deleted_nodes = 0
|
||||
parameters = {
|
||||
"provider_id": provider_id,
|
||||
"batch_size": BATCH_SIZE,
|
||||
}
|
||||
|
||||
try:
|
||||
with get_session(database) as session:
|
||||
deleted_count = 1
|
||||
while deleted_count > 0:
|
||||
result = session.run(
|
||||
f"""
|
||||
MATCH (n:{PROVIDER_RESOURCE_LABEL} {{provider_id: $provider_id}})
|
||||
WITH n LIMIT $batch_size
|
||||
DETACH DELETE n
|
||||
RETURN COUNT(n) AS deleted_nodes_count
|
||||
""",
|
||||
parameters,
|
||||
)
|
||||
deleted_count = result.single().get("deleted_nodes_count", 0)
|
||||
deleted_nodes += deleted_count
|
||||
|
||||
except GraphDatabaseQueryException as exc:
|
||||
if exc.code == "Neo.ClientError.Database.DatabaseNotFound":
|
||||
return 0
|
||||
raise
|
||||
|
||||
return deleted_nodes
|
||||
|
||||
|
||||
def clear_cache(database: str) -> None:
|
||||
query = "CALL db.clearQueryCaches()"
|
||||
|
||||
try:
|
||||
with get_session(database) as session:
|
||||
session.run(query)
|
||||
|
||||
except GraphDatabaseQueryException as exc:
|
||||
logging.warning(f"Failed to clear query cache for database `{database}`: {exc}")
|
||||
|
||||
|
||||
# Neo4j functions related to Prowler + Cartography
|
||||
|
||||
|
||||
def get_database_name(entity_id: str | UUID, temporary: bool = False) -> str:
|
||||
prefix = "tmp-scan" if temporary else "tenant"
|
||||
return f"db-{prefix}-{str(entity_id).lower()}"
|
||||
|
||||
|
||||
# Exceptions
|
||||
|
||||
|
||||
class GraphDatabaseQueryException(Exception):
|
||||
def __init__(self, message: str, code: str | None = None) -> None:
|
||||
super().__init__(message)
|
||||
self.message = message
|
||||
self.code = code
|
||||
|
||||
def __str__(self) -> str:
|
||||
if self.code:
|
||||
return f"{self.code}: {self.message}"
|
||||
|
||||
return self.message
|
||||
@@ -0,0 +1,16 @@
|
||||
from api.attack_paths.queries.types import (
|
||||
AttackPathsQueryDefinition,
|
||||
AttackPathsQueryParameterDefinition,
|
||||
)
|
||||
from api.attack_paths.queries.registry import (
|
||||
get_queries_for_provider,
|
||||
get_query_by_id,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"AttackPathsQueryDefinition",
|
||||
"AttackPathsQueryParameterDefinition",
|
||||
"get_queries_for_provider",
|
||||
"get_query_by_id",
|
||||
]
|
||||
@@ -0,0 +1,930 @@
|
||||
from api.attack_paths.queries.types import (
|
||||
AttackPathsQueryAttribution,
|
||||
AttackPathsQueryDefinition,
|
||||
AttackPathsQueryParameterDefinition,
|
||||
)
|
||||
from tasks.jobs.attack_paths.config import PROWLER_FINDING_LABEL
|
||||
|
||||
|
||||
# Custom Attack Path Queries
|
||||
# --------------------------
|
||||
|
||||
AWS_INTERNET_EXPOSED_EC2_SENSITIVE_S3_ACCESS = AttackPathsQueryDefinition(
|
||||
id="aws-internet-exposed-ec2-sensitive-s3-access",
|
||||
name="Internet-Exposed EC2 with Sensitive S3 Access",
|
||||
short_description="Find SSH-exposed EC2 instances that can assume roles to read tagged sensitive S3 buckets.",
|
||||
description="Detect EC2 instances with SSH exposed to the internet that can assume higher-privileged roles to read tagged sensitive S3 buckets despite bucket-level public access blocks.",
|
||||
provider="aws",
|
||||
cypher=f"""
|
||||
CALL apoc.create.vNode(['Internet'], {{id: 'Internet', name: 'Internet'}})
|
||||
YIELD node AS internet
|
||||
|
||||
MATCH path_s3 = (aws:AWSAccount {{id: $provider_uid}})--(s3:S3Bucket)--(t:AWSTag)
|
||||
WHERE toLower(t.key) = toLower($tag_key) AND toLower(t.value) = toLower($tag_value)
|
||||
|
||||
MATCH path_ec2 = (aws)--(ec2:EC2Instance)--(sg:EC2SecurityGroup)--(ipi:IpPermissionInbound)
|
||||
WHERE ec2.exposed_internet = true
|
||||
AND ipi.toport = 22
|
||||
|
||||
MATCH path_role = (r:AWSRole)--(pol:AWSPolicy)--(stmt:AWSPolicyStatement)
|
||||
WHERE ANY(x IN stmt.resource WHERE x CONTAINS s3.name)
|
||||
AND ANY(x IN stmt.action WHERE toLower(x) =~ 's3:(listbucket|getobject).*')
|
||||
|
||||
MATCH path_assume_role = (ec2)-[p:STS_ASSUMEROLE_ALLOW*1..9]-(r:AWSRole)
|
||||
|
||||
CALL apoc.create.vRelationship(internet, 'CAN_ACCESS', {{}}, ec2)
|
||||
YIELD rel AS can_access
|
||||
|
||||
UNWIND nodes(path_s3) + nodes(path_ec2) + nodes(path_role) + nodes(path_assume_role) as n
|
||||
OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}})
|
||||
|
||||
RETURN path_s3, path_ec2, path_role, path_assume_role, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr, internet, can_access
|
||||
""",
|
||||
parameters=[
|
||||
AttackPathsQueryParameterDefinition(
|
||||
name="tag_key",
|
||||
label="Tag key",
|
||||
description="Tag key to filter the S3 bucket, e.g. DataClassification.",
|
||||
placeholder="DataClassification",
|
||||
),
|
||||
AttackPathsQueryParameterDefinition(
|
||||
name="tag_value",
|
||||
label="Tag value",
|
||||
description="Tag value to filter the S3 bucket, e.g. Sensitive.",
|
||||
placeholder="Sensitive",
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
# Basic Resource Queries
|
||||
# ----------------------
|
||||
|
||||
AWS_RDS_INSTANCES = AttackPathsQueryDefinition(
|
||||
id="aws-rds-instances",
|
||||
name="RDS Instances Inventory",
|
||||
short_description="List all provisioned RDS database instances in the account.",
|
||||
description="List the selected AWS account alongside the RDS instances it owns.",
|
||||
provider="aws",
|
||||
cypher=f"""
|
||||
MATCH path = (aws:AWSAccount {{id: $provider_uid}})--(rds:RDSInstance)
|
||||
|
||||
UNWIND nodes(path) as n
|
||||
OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}})
|
||||
|
||||
RETURN path, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr
|
||||
""",
|
||||
parameters=[],
|
||||
)
|
||||
|
||||
AWS_RDS_UNENCRYPTED_STORAGE = AttackPathsQueryDefinition(
|
||||
id="aws-rds-unencrypted-storage",
|
||||
name="Unencrypted RDS Instances",
|
||||
short_description="Find RDS instances with storage encryption disabled.",
|
||||
description="Find RDS instances with storage encryption disabled within the selected account.",
|
||||
provider="aws",
|
||||
cypher=f"""
|
||||
MATCH path = (aws:AWSAccount {{id: $provider_uid}})--(rds:RDSInstance)
|
||||
WHERE rds.storage_encrypted = false
|
||||
|
||||
UNWIND nodes(path) as n
|
||||
OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}})
|
||||
|
||||
RETURN path, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr
|
||||
""",
|
||||
parameters=[],
|
||||
)
|
||||
|
||||
AWS_S3_ANONYMOUS_ACCESS_BUCKETS = AttackPathsQueryDefinition(
|
||||
id="aws-s3-anonymous-access-buckets",
|
||||
name="S3 Buckets with Anonymous Access",
|
||||
short_description="Find S3 buckets that allow anonymous access.",
|
||||
description="Find S3 buckets that allow anonymous access within the selected account.",
|
||||
provider="aws",
|
||||
cypher=f"""
|
||||
MATCH path = (aws:AWSAccount {{id: $provider_uid}})--(s3:S3Bucket)
|
||||
WHERE s3.anonymous_access = true
|
||||
|
||||
UNWIND nodes(path) as n
|
||||
OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}})
|
||||
|
||||
RETURN path, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr
|
||||
""",
|
||||
parameters=[],
|
||||
)
|
||||
|
||||
AWS_IAM_STATEMENTS_ALLOW_ALL_ACTIONS = AttackPathsQueryDefinition(
|
||||
id="aws-iam-statements-allow-all-actions",
|
||||
name="IAM Statements Allowing All Actions",
|
||||
short_description="Find IAM policy statements that allow all actions via wildcard (*).",
|
||||
description="Find IAM policy statements that allow all actions via '*' within the selected account.",
|
||||
provider="aws",
|
||||
cypher=f"""
|
||||
MATCH path = (aws:AWSAccount {{id: $provider_uid}})--(principal:AWSPrincipal)--(pol:AWSPolicy)--(stmt:AWSPolicyStatement)
|
||||
WHERE stmt.effect = 'Allow'
|
||||
AND any(x IN stmt.action WHERE x = '*')
|
||||
|
||||
UNWIND nodes(path) as n
|
||||
OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}})
|
||||
|
||||
RETURN path, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr
|
||||
""",
|
||||
parameters=[],
|
||||
)
|
||||
|
||||
AWS_IAM_STATEMENTS_ALLOW_DELETE_POLICY = AttackPathsQueryDefinition(
|
||||
id="aws-iam-statements-allow-delete-policy",
|
||||
name="IAM Statements Allowing Policy Deletion",
|
||||
short_description="Find IAM policy statements that allow iam:DeletePolicy.",
|
||||
description="Find IAM policy statements that allow the iam:DeletePolicy action within the selected account.",
|
||||
provider="aws",
|
||||
cypher=f"""
|
||||
MATCH path = (aws:AWSAccount {{id: $provider_uid}})--(principal:AWSPrincipal)--(pol:AWSPolicy)--(stmt:AWSPolicyStatement)
|
||||
WHERE stmt.effect = 'Allow'
|
||||
AND any(x IN stmt.action WHERE x = "iam:DeletePolicy")
|
||||
|
||||
UNWIND nodes(path) as n
|
||||
OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}})
|
||||
|
||||
RETURN path, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr
|
||||
""",
|
||||
parameters=[],
|
||||
)
|
||||
|
||||
AWS_IAM_STATEMENTS_ALLOW_CREATE_ACTIONS = AttackPathsQueryDefinition(
|
||||
id="aws-iam-statements-allow-create-actions",
|
||||
name="IAM Statements Allowing Create Actions",
|
||||
short_description="Find IAM policy statements that allow any create action.",
|
||||
description="Find IAM policy statements that allow actions containing 'create' within the selected account.",
|
||||
provider="aws",
|
||||
cypher=f"""
|
||||
MATCH path = (aws:AWSAccount {{id: $provider_uid}})--(principal:AWSPrincipal)--(pol:AWSPolicy)--(stmt:AWSPolicyStatement)
|
||||
WHERE stmt.effect = "Allow"
|
||||
AND any(x IN stmt.action WHERE toLower(x) CONTAINS "create")
|
||||
|
||||
UNWIND nodes(path) as n
|
||||
OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}})
|
||||
|
||||
RETURN path, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr
|
||||
""",
|
||||
parameters=[],
|
||||
)
|
||||
|
||||
|
||||
# Network Exposure Queries
|
||||
# ------------------------
|
||||
|
||||
AWS_EC2_INSTANCES_INTERNET_EXPOSED = AttackPathsQueryDefinition(
|
||||
id="aws-ec2-instances-internet-exposed",
|
||||
name="Internet-Exposed EC2 Instances",
|
||||
short_description="Find EC2 instances flagged as exposed to the internet.",
|
||||
description="Find EC2 instances flagged as exposed to the internet within the selected account.",
|
||||
provider="aws",
|
||||
cypher=f"""
|
||||
CALL apoc.create.vNode(['Internet'], {{id: 'Internet', name: 'Internet'}})
|
||||
YIELD node AS internet
|
||||
|
||||
MATCH path = (aws:AWSAccount {{id: $provider_uid}})--(ec2:EC2Instance)
|
||||
WHERE ec2.exposed_internet = true
|
||||
|
||||
CALL apoc.create.vRelationship(internet, 'CAN_ACCESS', {{}}, ec2)
|
||||
YIELD rel AS can_access
|
||||
|
||||
UNWIND nodes(path) as n
|
||||
OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}})
|
||||
|
||||
RETURN path, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr, internet, can_access
|
||||
""",
|
||||
parameters=[],
|
||||
)
|
||||
|
||||
AWS_SECURITY_GROUPS_OPEN_INTERNET_FACING = AttackPathsQueryDefinition(
|
||||
id="aws-security-groups-open-internet-facing",
|
||||
name="Open Security Groups on Internet-Facing Resources",
|
||||
short_description="Find internet-facing resources with security groups allowing inbound from 0.0.0.0/0.",
|
||||
description="Find internet-facing resources associated with security groups that allow inbound access from '0.0.0.0/0'.",
|
||||
provider="aws",
|
||||
cypher=f"""
|
||||
CALL apoc.create.vNode(['Internet'], {{id: 'Internet', name: 'Internet'}})
|
||||
YIELD node AS internet
|
||||
|
||||
// Match EC2 instances that are internet-exposed with open security groups (0.0.0.0/0)
|
||||
MATCH path_ec2 = (aws:AWSAccount {{id: $provider_uid}})--(ec2:EC2Instance)--(sg:EC2SecurityGroup)--(ipi:IpPermissionInbound)--(ir:IpRange)
|
||||
WHERE ec2.exposed_internet = true
|
||||
AND ir.range = "0.0.0.0/0"
|
||||
|
||||
CALL apoc.create.vRelationship(internet, 'CAN_ACCESS', {{}}, ec2)
|
||||
YIELD rel AS can_access
|
||||
|
||||
UNWIND nodes(path_ec2) as n
|
||||
OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}})
|
||||
|
||||
RETURN path_ec2, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr, internet, can_access
|
||||
""",
|
||||
parameters=[],
|
||||
)
|
||||
|
||||
AWS_CLASSIC_ELB_INTERNET_EXPOSED = AttackPathsQueryDefinition(
|
||||
id="aws-classic-elb-internet-exposed",
|
||||
name="Internet-Exposed Classic Load Balancers",
|
||||
short_description="Find Classic Load Balancers exposed to the internet with their listeners.",
|
||||
description="Find Classic Load Balancers exposed to the internet along with their listeners.",
|
||||
provider="aws",
|
||||
cypher=f"""
|
||||
CALL apoc.create.vNode(['Internet'], {{id: 'Internet', name: 'Internet'}})
|
||||
YIELD node AS internet
|
||||
|
||||
MATCH path = (aws:AWSAccount {{id: $provider_uid}})--(elb:LoadBalancer)--(listener:ELBListener)
|
||||
WHERE elb.exposed_internet = true
|
||||
|
||||
CALL apoc.create.vRelationship(internet, 'CAN_ACCESS', {{}}, elb)
|
||||
YIELD rel AS can_access
|
||||
|
||||
UNWIND nodes(path) as n
|
||||
OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}})
|
||||
|
||||
RETURN path, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr, internet, can_access
|
||||
""",
|
||||
parameters=[],
|
||||
)
|
||||
|
||||
AWS_ELBV2_INTERNET_EXPOSED = AttackPathsQueryDefinition(
|
||||
id="aws-elbv2-internet-exposed",
|
||||
name="Internet-Exposed ALB/NLB Load Balancers",
|
||||
short_description="Find ELBv2 (ALB/NLB) load balancers exposed to the internet with their listeners.",
|
||||
description="Find ELBv2 load balancers exposed to the internet along with their listeners.",
|
||||
provider="aws",
|
||||
cypher=f"""
|
||||
CALL apoc.create.vNode(['Internet'], {{id: 'Internet', name: 'Internet'}})
|
||||
YIELD node AS internet
|
||||
|
||||
MATCH path = (aws:AWSAccount {{id: $provider_uid}})--(elbv2:LoadBalancerV2)--(listener:ELBV2Listener)
|
||||
WHERE elbv2.exposed_internet = true
|
||||
|
||||
CALL apoc.create.vRelationship(internet, 'CAN_ACCESS', {{}}, elbv2)
|
||||
YIELD rel AS can_access
|
||||
|
||||
UNWIND nodes(path) as n
|
||||
OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}})
|
||||
|
||||
RETURN path, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr, internet, can_access
|
||||
""",
|
||||
parameters=[],
|
||||
)
|
||||
|
||||
AWS_PUBLIC_IP_RESOURCE_LOOKUP = AttackPathsQueryDefinition(
|
||||
id="aws-public-ip-resource-lookup",
|
||||
name="Resource Lookup by Public IP",
|
||||
short_description="Find the AWS resource associated with a given public IP address.",
|
||||
description="Given a public IP address, find the related AWS resource and its adjacent node within the selected account.",
|
||||
provider="aws",
|
||||
cypher=f"""
|
||||
CALL apoc.create.vNode(['Internet'], {{id: 'Internet', name: 'Internet'}})
|
||||
YIELD node AS internet
|
||||
|
||||
CALL () {{
|
||||
MATCH path = (aws:AWSAccount {{id: $provider_uid}})-[r]-(x:EC2PrivateIp)-[q]-(y)
|
||||
WHERE x.public_ip = $ip
|
||||
RETURN path, x
|
||||
|
||||
UNION MATCH path = (aws:AWSAccount {{id: $provider_uid}})-[r]-(x:EC2Instance)-[q]-(y)
|
||||
WHERE x.publicipaddress = $ip
|
||||
RETURN path, x
|
||||
|
||||
UNION MATCH path = (aws:AWSAccount {{id: $provider_uid}})-[r]-(x:NetworkInterface)-[q]-(y)
|
||||
WHERE x.public_ip = $ip
|
||||
RETURN path, x
|
||||
|
||||
UNION MATCH path = (aws:AWSAccount {{id: $provider_uid}})-[r]-(x:ElasticIPAddress)-[q]-(y)
|
||||
WHERE x.public_ip = $ip
|
||||
RETURN path, x
|
||||
}}
|
||||
|
||||
WITH path, x, internet
|
||||
|
||||
CALL apoc.create.vRelationship(internet, 'CAN_ACCESS', {{}}, x)
|
||||
YIELD rel AS can_access
|
||||
|
||||
UNWIND nodes(path) as n
|
||||
OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}})
|
||||
|
||||
RETURN path, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr, internet, can_access
|
||||
""",
|
||||
parameters=[
|
||||
AttackPathsQueryParameterDefinition(
|
||||
name="ip",
|
||||
label="IP address",
|
||||
description="Public IP address, e.g. 192.0.2.0.",
|
||||
placeholder="192.0.2.0",
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
# Privilege Escalation Queries (based on pathfinding.cloud research)
|
||||
# https://github.com/DataDog/pathfinding.cloud
|
||||
# -------------------------------------------------------------------
|
||||
|
||||
# BEDROCK-001
|
||||
AWS_BEDROCK_PRIVESC_PASSROLE_CODE_INTERPRETER = AttackPathsQueryDefinition(
|
||||
id="aws-bedrock-privesc-passrole-code-interpreter",
|
||||
name="Bedrock Code Interpreter with Privileged Role (BEDROCK-001)",
|
||||
short_description="Create a Bedrock AgentCore Code Interpreter with a privileged role attached.",
|
||||
description="Detect principals who can pass IAM roles and create Bedrock AgentCore Code Interpreters. This allows creating a code interpreter with a privileged role attached, gaining that role's permissions.",
|
||||
attribution=AttackPathsQueryAttribution(
|
||||
text="pathfinding.cloud - BEDROCK-001 - iam:PassRole + bedrock-agentcore:CreateCodeInterpreter",
|
||||
link="https://pathfinding.cloud/paths/bedrock-001",
|
||||
),
|
||||
provider="aws",
|
||||
cypher=f"""
|
||||
// Find principals with iam:PassRole permission
|
||||
MATCH path_principal = (aws:AWSAccount {{id: $provider_uid}})--(principal:AWSPrincipal)--(passrole_policy:AWSPolicy)--(stmt_passrole:AWSPolicyStatement)
|
||||
WHERE stmt_passrole.effect = 'Allow'
|
||||
AND any(action IN stmt_passrole.action WHERE
|
||||
toLower(action) = 'iam:passrole'
|
||||
OR toLower(action) = 'iam:*'
|
||||
OR action = '*'
|
||||
)
|
||||
|
||||
// Find bedrock-agentcore:CreateCodeInterpreter permission
|
||||
MATCH (principal)--(bedrock_policy:AWSPolicy)--(stmt_bedrock:AWSPolicyStatement)
|
||||
WHERE stmt_bedrock.effect = 'Allow'
|
||||
AND any(action IN stmt_bedrock.action WHERE
|
||||
toLower(action) = 'bedrock-agentcore:createcodeinterpreter'
|
||||
OR toLower(action) = 'bedrock-agentcore:*'
|
||||
OR action = '*'
|
||||
)
|
||||
|
||||
// Find roles that trust Bedrock service (can be passed to Bedrock)
|
||||
MATCH path_target = (aws)--(target_role:AWSRole)-[:TRUSTS_AWS_PRINCIPAL]->(:AWSPrincipal {{arn: 'bedrock.amazonaws.com'}})
|
||||
WHERE any(resource IN stmt_passrole.resource WHERE
|
||||
resource = '*'
|
||||
OR target_role.arn CONTAINS resource
|
||||
OR resource CONTAINS target_role.name
|
||||
)
|
||||
|
||||
UNWIND nodes(path_principal) + nodes(path_target) as n
|
||||
OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}})
|
||||
|
||||
RETURN path_principal, path_target,
|
||||
collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr
|
||||
""",
|
||||
parameters=[],
|
||||
)
|
||||
|
||||
# EC2-001
|
||||
AWS_EC2_PRIVESC_PASSROLE_IAM = AttackPathsQueryDefinition(
|
||||
id="aws-ec2-privesc-passrole-iam",
|
||||
name="EC2 Instance Launch with Privileged Role (EC2-001)",
|
||||
short_description="Launch EC2 instances with privileged IAM roles to gain their permissions via IMDS.",
|
||||
description="Detect principals who can launch EC2 instances with privileged IAM roles attached. This allows gaining the permissions of the passed role by accessing the EC2 instance metadata service.",
|
||||
attribution=AttackPathsQueryAttribution(
|
||||
text="pathfinding.cloud - EC2-001 - iam:PassRole + ec2:RunInstances",
|
||||
link="https://pathfinding.cloud/paths/ec2-001",
|
||||
),
|
||||
provider="aws",
|
||||
cypher=f"""
|
||||
// Find principals with iam:PassRole permission
|
||||
MATCH path_principal = (aws:AWSAccount {{id: $provider_uid}})--(principal:AWSPrincipal)--(passrole_policy:AWSPolicy)--(stmt_passrole:AWSPolicyStatement)
|
||||
WHERE stmt_passrole.effect = 'Allow'
|
||||
AND any(action IN stmt_passrole.action WHERE
|
||||
toLower(action) = 'iam:passrole'
|
||||
OR toLower(action) = 'iam:*'
|
||||
OR action = '*'
|
||||
)
|
||||
|
||||
// Find ec2:RunInstances permission
|
||||
MATCH (principal)--(ec2_policy:AWSPolicy)--(stmt_ec2:AWSPolicyStatement)
|
||||
WHERE stmt_ec2.effect = 'Allow'
|
||||
AND any(action IN stmt_ec2.action WHERE
|
||||
toLower(action) = 'ec2:runinstances'
|
||||
OR toLower(action) = 'ec2:*'
|
||||
OR action = '*'
|
||||
)
|
||||
|
||||
// Find roles that trust EC2 service (can be passed to EC2)
|
||||
MATCH path_target = (aws)--(target_role:AWSRole)-[:TRUSTS_AWS_PRINCIPAL]->(:AWSPrincipal {{arn: 'ec2.amazonaws.com'}})
|
||||
WHERE any(resource IN stmt_passrole.resource WHERE
|
||||
resource = '*'
|
||||
OR target_role.arn CONTAINS resource
|
||||
OR resource CONTAINS target_role.name
|
||||
)
|
||||
|
||||
UNWIND nodes(path_principal) + nodes(path_target) as n
|
||||
OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}})
|
||||
|
||||
RETURN path_principal, path_target,
|
||||
collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr
|
||||
""",
|
||||
parameters=[],
|
||||
)
|
||||
|
||||
# EC2-002
|
||||
AWS_EC2_PRIVESC_MODIFY_INSTANCE_ATTRIBUTE = AttackPathsQueryDefinition(
|
||||
id="aws-ec2-privesc-modify-instance-attribute",
|
||||
name="EC2 Role Hijacking via UserData Injection (EC2-002)",
|
||||
short_description="Inject malicious scripts into EC2 instance userData to gain the attached role's permissions.",
|
||||
description="Detect principals who can modify EC2 instance userData, stop, and start instances. This allows injecting malicious scripts that execute on instance restart, gaining the permissions of the instance's attached IAM role.",
|
||||
attribution=AttackPathsQueryAttribution(
|
||||
text="pathfinding.cloud - EC2-002 - ec2:ModifyInstanceAttribute + ec2:StopInstances + ec2:StartInstances",
|
||||
link="https://pathfinding.cloud/paths/ec2-002",
|
||||
),
|
||||
provider="aws",
|
||||
cypher=f"""
|
||||
// Find principals with ec2:ModifyInstanceAttribute permission
|
||||
MATCH path_principal = (aws:AWSAccount {{id: $provider_uid}})--(principal:AWSPrincipal)--(modify_policy:AWSPolicy)--(stmt_modify:AWSPolicyStatement)
|
||||
WHERE stmt_modify.effect = 'Allow'
|
||||
AND any(action IN stmt_modify.action WHERE
|
||||
toLower(action) = 'ec2:modifyinstanceattribute'
|
||||
OR toLower(action) = 'ec2:*'
|
||||
OR action = '*'
|
||||
)
|
||||
|
||||
// Find ec2:StopInstances permission (can be same or different policy)
|
||||
MATCH (principal)--(stop_policy:AWSPolicy)--(stmt_stop:AWSPolicyStatement)
|
||||
WHERE stmt_stop.effect = 'Allow'
|
||||
AND any(action IN stmt_stop.action WHERE
|
||||
toLower(action) = 'ec2:stopinstances'
|
||||
OR toLower(action) = 'ec2:*'
|
||||
OR action = '*'
|
||||
)
|
||||
|
||||
// Find ec2:StartInstances permission (can be same or different policy)
|
||||
MATCH (principal)--(start_policy:AWSPolicy)--(stmt_start:AWSPolicyStatement)
|
||||
WHERE stmt_start.effect = 'Allow'
|
||||
AND any(action IN stmt_start.action WHERE
|
||||
toLower(action) = 'ec2:startinstances'
|
||||
OR toLower(action) = 'ec2:*'
|
||||
OR action = '*'
|
||||
)
|
||||
|
||||
// Find EC2 instances with instance profiles (potential targets)
|
||||
MATCH path_target = (aws)--(ec2:EC2Instance)-[:STS_ASSUMEROLE_ALLOW]->(target_role:AWSRole)
|
||||
|
||||
UNWIND nodes(path_principal) + nodes(path_target) as n
|
||||
OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}})
|
||||
|
||||
RETURN path_principal, path_target,
|
||||
collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr
|
||||
""",
|
||||
parameters=[],
|
||||
)
|
||||
|
||||
# EC2-003
|
||||
AWS_EC2_PRIVESC_PASSROLE_SPOT_INSTANCES = AttackPathsQueryDefinition(
|
||||
id="aws-ec2-privesc-passrole-spot-instances",
|
||||
name="Spot Instance Launch with Privileged Role (EC2-003)",
|
||||
short_description="Launch EC2 Spot Instances with privileged IAM roles to gain their permissions via IMDS.",
|
||||
description="Detect principals who can pass IAM roles and request EC2 Spot Instances. This allows launching a spot instance with a privileged role attached, gaining that role's permissions via the instance metadata service.",
|
||||
attribution=AttackPathsQueryAttribution(
|
||||
text="pathfinding.cloud - EC2-003 - iam:PassRole + ec2:RequestSpotInstances",
|
||||
link="https://pathfinding.cloud/paths/ec2-003",
|
||||
),
|
||||
provider="aws",
|
||||
cypher=f"""
|
||||
// Find principals with iam:PassRole permission
|
||||
MATCH path_principal = (aws:AWSAccount {{id: $provider_uid}})--(principal:AWSPrincipal)--(passrole_policy:AWSPolicy)--(stmt_passrole:AWSPolicyStatement)
|
||||
WHERE stmt_passrole.effect = 'Allow'
|
||||
AND any(action IN stmt_passrole.action WHERE
|
||||
toLower(action) = 'iam:passrole'
|
||||
OR toLower(action) = 'iam:*'
|
||||
OR action = '*'
|
||||
)
|
||||
|
||||
// Find ec2:RequestSpotInstances permission
|
||||
MATCH (principal)--(spot_policy:AWSPolicy)--(stmt_spot:AWSPolicyStatement)
|
||||
WHERE stmt_spot.effect = 'Allow'
|
||||
AND any(action IN stmt_spot.action WHERE
|
||||
toLower(action) = 'ec2:requestspotinstances'
|
||||
OR toLower(action) = 'ec2:*'
|
||||
OR action = '*'
|
||||
)
|
||||
|
||||
// Find roles that trust EC2 service (can be passed to EC2 spot instances)
|
||||
MATCH path_target = (aws)--(target_role:AWSRole)-[:TRUSTS_AWS_PRINCIPAL]->(:AWSPrincipal {{arn: 'ec2.amazonaws.com'}})
|
||||
WHERE any(resource IN stmt_passrole.resource WHERE
|
||||
resource = '*'
|
||||
OR target_role.arn CONTAINS resource
|
||||
OR resource CONTAINS target_role.name
|
||||
)
|
||||
|
||||
UNWIND nodes(path_principal) + nodes(path_target) as n
|
||||
OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}})
|
||||
|
||||
RETURN path_principal, path_target,
|
||||
collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr
|
||||
""",
|
||||
parameters=[],
|
||||
)
|
||||
|
||||
# EC2-004
|
||||
AWS_EC2_PRIVESC_LAUNCH_TEMPLATE = AttackPathsQueryDefinition(
|
||||
id="aws-ec2-privesc-launch-template",
|
||||
name="Launch Template Poisoning for Role Access (EC2-004)",
|
||||
short_description="Inject malicious userData into launch templates that reference privileged roles, no PassRole needed.",
|
||||
description="Detect principals who can create new launch template versions and modify launch templates. This allows injecting malicious user data into existing templates that already reference privileged IAM roles, without requiring iam:PassRole permissions.",
|
||||
attribution=AttackPathsQueryAttribution(
|
||||
text="pathfinding.cloud - EC2-004 - ec2:CreateLaunchTemplateVersion + ec2:ModifyLaunchTemplate",
|
||||
link="https://pathfinding.cloud/paths/ec2-004",
|
||||
),
|
||||
provider="aws",
|
||||
cypher=f"""
|
||||
// Find principals with ec2:CreateLaunchTemplateVersion permission
|
||||
MATCH path_principal = (aws:AWSAccount {{id: $provider_uid}})--(principal:AWSPrincipal)--(create_policy:AWSPolicy)--(stmt_create:AWSPolicyStatement)
|
||||
WHERE stmt_create.effect = 'Allow'
|
||||
AND any(action IN stmt_create.action WHERE
|
||||
toLower(action) = 'ec2:createlaunchtemplateversion'
|
||||
OR toLower(action) = 'ec2:*'
|
||||
OR action = '*'
|
||||
)
|
||||
|
||||
// Find ec2:ModifyLaunchTemplate permission
|
||||
MATCH (principal)--(modify_policy:AWSPolicy)--(stmt_modify:AWSPolicyStatement)
|
||||
WHERE stmt_modify.effect = 'Allow'
|
||||
AND any(action IN stmt_modify.action WHERE
|
||||
toLower(action) = 'ec2:modifylaunchtemplate'
|
||||
OR toLower(action) = 'ec2:*'
|
||||
OR action = '*'
|
||||
)
|
||||
|
||||
// Find launch templates in the account (potential targets)
|
||||
MATCH path_target = (aws)--(template:LaunchTemplate)
|
||||
|
||||
UNWIND nodes(path_principal) + nodes(path_target) as n
|
||||
OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}})
|
||||
|
||||
RETURN path_principal, path_target,
|
||||
collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr
|
||||
""",
|
||||
parameters=[],
|
||||
)
|
||||
|
||||
# ECS-001
|
||||
AWS_ECS_PRIVESC_PASSROLE_CREATE_SERVICE = AttackPathsQueryDefinition(
|
||||
id="aws-ecs-privesc-passrole-create-service",
|
||||
name="ECS Service Creation with Privileged Role (ECS-001 - New Cluster)",
|
||||
short_description="Create an ECS cluster and service with a privileged Fargate task role to execute arbitrary code.",
|
||||
description="Detect principals who can pass IAM roles, create ECS clusters, register task definitions, and create services. This allows creating a Fargate task with a privileged role attached, gaining that role's permissions to execute arbitrary code via the container.",
|
||||
provider="aws",
|
||||
attribution=AttackPathsQueryAttribution(
|
||||
text="pathfinding.cloud - ECS-001 - iam:PassRole + ecs:CreateCluster + ecs:RegisterTaskDefinition + ecs:CreateService",
|
||||
link="https://pathfinding.cloud/paths/ecs-001",
|
||||
),
|
||||
cypher=f"""
|
||||
// Find principals with iam:PassRole permission
|
||||
MATCH path_principal = (aws:AWSAccount {{id: $provider_uid}})--(principal:AWSPrincipal)--(passrole_policy:AWSPolicy)--(stmt_passrole:AWSPolicyStatement)
|
||||
WHERE stmt_passrole.effect = 'Allow'
|
||||
AND any(action IN stmt_passrole.action WHERE
|
||||
toLower(action) = 'iam:passrole'
|
||||
OR toLower(action) = 'iam:*'
|
||||
OR action = '*'
|
||||
)
|
||||
|
||||
// Find ecs:CreateCluster permission
|
||||
MATCH (principal)--(cluster_policy:AWSPolicy)--(stmt_cluster:AWSPolicyStatement)
|
||||
WHERE stmt_cluster.effect = 'Allow'
|
||||
AND any(action IN stmt_cluster.action WHERE
|
||||
toLower(action) = 'ecs:createcluster'
|
||||
OR toLower(action) = 'ecs:*'
|
||||
OR action = '*'
|
||||
)
|
||||
|
||||
// Find ecs:RegisterTaskDefinition permission
|
||||
MATCH (principal)--(taskdef_policy:AWSPolicy)--(stmt_taskdef:AWSPolicyStatement)
|
||||
WHERE stmt_taskdef.effect = 'Allow'
|
||||
AND any(action IN stmt_taskdef.action WHERE
|
||||
toLower(action) = 'ecs:registertaskdefinition'
|
||||
OR toLower(action) = 'ecs:*'
|
||||
OR action = '*'
|
||||
)
|
||||
|
||||
// Find ecs:CreateService permission
|
||||
MATCH (principal)--(service_policy:AWSPolicy)--(stmt_service:AWSPolicyStatement)
|
||||
WHERE stmt_service.effect = 'Allow'
|
||||
AND any(action IN stmt_service.action WHERE
|
||||
toLower(action) = 'ecs:createservice'
|
||||
OR toLower(action) = 'ecs:*'
|
||||
OR action = '*'
|
||||
)
|
||||
|
||||
// Find roles that trust ECS tasks service (can be passed to ECS tasks)
|
||||
MATCH path_target = (aws)--(target_role:AWSRole)-[:TRUSTS_AWS_PRINCIPAL]->(:AWSPrincipal {{arn: 'ecs-tasks.amazonaws.com'}})
|
||||
WHERE any(resource IN stmt_passrole.resource WHERE
|
||||
resource = '*'
|
||||
OR target_role.arn CONTAINS resource
|
||||
OR resource CONTAINS target_role.name
|
||||
)
|
||||
|
||||
UNWIND nodes(path_principal) + nodes(path_target) as n
|
||||
OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}})
|
||||
|
||||
RETURN path_principal, path_target,
|
||||
collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr
|
||||
""",
|
||||
parameters=[],
|
||||
)
|
||||
|
||||
# ECS-002
|
||||
AWS_ECS_PRIVESC_PASSROLE_RUN_TASK = AttackPathsQueryDefinition(
|
||||
id="aws-ecs-privesc-passrole-run-task",
|
||||
name="ECS Task Execution with Privileged Role (ECS-002 - New Cluster)",
|
||||
short_description="Create an ECS cluster and run a one-off Fargate task with a privileged role to execute arbitrary code.",
|
||||
description="Detect principals who can pass IAM roles, create ECS clusters, register task definitions, and run tasks. This allows creating a Fargate task with a privileged role attached, gaining that role's permissions to execute arbitrary code via the container. Unlike ecs:CreateService, ecs:RunTask executes the task once without creating a persistent service.",
|
||||
attribution=AttackPathsQueryAttribution(
|
||||
text="pathfinding.cloud - ECS-002 - iam:PassRole + ecs:CreateCluster + ecs:RegisterTaskDefinition + ecs:RunTask",
|
||||
link="https://pathfinding.cloud/paths/ecs-002",
|
||||
),
|
||||
provider="aws",
|
||||
cypher=f"""
|
||||
// Find principals with iam:PassRole permission
|
||||
MATCH path_principal = (aws:AWSAccount {{id: $provider_uid}})--(principal:AWSPrincipal)--(passrole_policy:AWSPolicy)--(stmt_passrole:AWSPolicyStatement)
|
||||
WHERE stmt_passrole.effect = 'Allow'
|
||||
AND any(action IN stmt_passrole.action WHERE
|
||||
toLower(action) = 'iam:passrole'
|
||||
OR toLower(action) = 'iam:*'
|
||||
OR action = '*'
|
||||
)
|
||||
|
||||
// Find ecs:CreateCluster permission
|
||||
MATCH (principal)--(cluster_policy:AWSPolicy)--(stmt_cluster:AWSPolicyStatement)
|
||||
WHERE stmt_cluster.effect = 'Allow'
|
||||
AND any(action IN stmt_cluster.action WHERE
|
||||
toLower(action) = 'ecs:createcluster'
|
||||
OR toLower(action) = 'ecs:*'
|
||||
OR action = '*'
|
||||
)
|
||||
|
||||
// Find ecs:RegisterTaskDefinition permission
|
||||
MATCH (principal)--(taskdef_policy:AWSPolicy)--(stmt_taskdef:AWSPolicyStatement)
|
||||
WHERE stmt_taskdef.effect = 'Allow'
|
||||
AND any(action IN stmt_taskdef.action WHERE
|
||||
toLower(action) = 'ecs:registertaskdefinition'
|
||||
OR toLower(action) = 'ecs:*'
|
||||
OR action = '*'
|
||||
)
|
||||
|
||||
// Find ecs:RunTask permission
|
||||
MATCH (principal)--(runtask_policy:AWSPolicy)--(stmt_runtask:AWSPolicyStatement)
|
||||
WHERE stmt_runtask.effect = 'Allow'
|
||||
AND any(action IN stmt_runtask.action WHERE
|
||||
toLower(action) = 'ecs:runtask'
|
||||
OR toLower(action) = 'ecs:*'
|
||||
OR action = '*'
|
||||
)
|
||||
|
||||
// Find roles that trust ECS tasks service (can be passed to ECS tasks)
|
||||
MATCH path_target = (aws)--(target_role:AWSRole)-[:TRUSTS_AWS_PRINCIPAL]->(:AWSPrincipal {{arn: 'ecs-tasks.amazonaws.com'}})
|
||||
WHERE any(resource IN stmt_passrole.resource WHERE
|
||||
resource = '*'
|
||||
OR target_role.arn CONTAINS resource
|
||||
OR resource CONTAINS target_role.name
|
||||
)
|
||||
|
||||
UNWIND nodes(path_principal) + nodes(path_target) as n
|
||||
OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}})
|
||||
|
||||
RETURN path_principal, path_target,
|
||||
collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr
|
||||
""",
|
||||
parameters=[],
|
||||
)
|
||||
|
||||
# ECS-003
|
||||
AWS_ECS_PRIVESC_PASSROLE_CREATE_SERVICE_EXISTING_CLUSTER = AttackPathsQueryDefinition(
|
||||
id="aws-ecs-privesc-passrole-create-service-existing-cluster",
|
||||
name="ECS Service Creation with Privileged Role (ECS-003 - Existing Cluster)",
|
||||
short_description="Deploy a Fargate service with a privileged role on an existing ECS cluster.",
|
||||
description="Detect principals who can pass IAM roles, register ECS task definitions, and create services on existing clusters. Unlike ECS-001, this does not require ecs:CreateCluster since it targets clusters that already exist. The attacker registers a task definition with a privileged role and launches it as a Fargate service, gaining that role's permissions.",
|
||||
attribution=AttackPathsQueryAttribution(
|
||||
text="pathfinding.cloud - ECS-003 - iam:PassRole + ecs:RegisterTaskDefinition + ecs:CreateService",
|
||||
link="https://pathfinding.cloud/paths/ecs-003",
|
||||
),
|
||||
provider="aws",
|
||||
cypher=f"""
|
||||
// Find principals with iam:PassRole permission
|
||||
MATCH path_principal = (aws:AWSAccount {{id: $provider_uid}})--(principal:AWSPrincipal)--(passrole_policy:AWSPolicy)--(stmt_passrole:AWSPolicyStatement)
|
||||
WHERE stmt_passrole.effect = 'Allow'
|
||||
AND any(action IN stmt_passrole.action WHERE
|
||||
toLower(action) = 'iam:passrole'
|
||||
OR toLower(action) = 'iam:*'
|
||||
OR action = '*'
|
||||
)
|
||||
|
||||
// Find ecs:RegisterTaskDefinition permission
|
||||
MATCH (principal)--(taskdef_policy:AWSPolicy)--(stmt_taskdef:AWSPolicyStatement)
|
||||
WHERE stmt_taskdef.effect = 'Allow'
|
||||
AND any(action IN stmt_taskdef.action WHERE
|
||||
toLower(action) = 'ecs:registertaskdefinition'
|
||||
OR toLower(action) = 'ecs:*'
|
||||
OR action = '*'
|
||||
)
|
||||
|
||||
// Find ecs:CreateService permission
|
||||
MATCH (principal)--(service_policy:AWSPolicy)--(stmt_service:AWSPolicyStatement)
|
||||
WHERE stmt_service.effect = 'Allow'
|
||||
AND any(action IN stmt_service.action WHERE
|
||||
toLower(action) = 'ecs:createservice'
|
||||
OR toLower(action) = 'ecs:*'
|
||||
OR action = '*'
|
||||
)
|
||||
|
||||
// Find roles that trust ECS tasks service (can be passed to ECS tasks)
|
||||
MATCH path_target = (aws)--(target_role:AWSRole)-[:TRUSTS_AWS_PRINCIPAL]->(:AWSPrincipal {{arn: 'ecs-tasks.amazonaws.com'}})
|
||||
WHERE any(resource IN stmt_passrole.resource WHERE
|
||||
resource = '*'
|
||||
OR target_role.arn CONTAINS resource
|
||||
OR resource CONTAINS target_role.name
|
||||
)
|
||||
|
||||
UNWIND nodes(path_principal) + nodes(path_target) as n
|
||||
OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}})
|
||||
|
||||
RETURN path_principal, path_target,
|
||||
collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr
|
||||
""",
|
||||
parameters=[],
|
||||
)
|
||||
|
||||
# ECS-004
|
||||
AWS_ECS_PRIVESC_PASSROLE_RUN_TASK_EXISTING_CLUSTER = AttackPathsQueryDefinition(
|
||||
id="aws-ecs-privesc-passrole-run-task-existing-cluster",
|
||||
name="ECS Task Execution with Privileged Role (ECS-004 - Existing Cluster)",
|
||||
short_description="Run a one-off Fargate task with a privileged role on an existing ECS cluster.",
|
||||
description="Detect principals who can pass IAM roles, register ECS task definitions, and run tasks on existing clusters. Unlike ECS-002, this does not require ecs:CreateCluster since it targets clusters that already exist. The attacker registers a task definition with a privileged role and runs it as a one-off Fargate task, gaining that role's permissions.",
|
||||
attribution=AttackPathsQueryAttribution(
|
||||
text="pathfinding.cloud - ECS-004 - iam:PassRole + ecs:RegisterTaskDefinition + ecs:RunTask",
|
||||
link="https://pathfinding.cloud/paths/ecs-004",
|
||||
),
|
||||
provider="aws",
|
||||
cypher=f"""
|
||||
// Find principals with iam:PassRole permission
|
||||
MATCH path_principal = (aws:AWSAccount {{id: $provider_uid}})--(principal:AWSPrincipal)--(passrole_policy:AWSPolicy)--(stmt_passrole:AWSPolicyStatement)
|
||||
WHERE stmt_passrole.effect = 'Allow'
|
||||
AND any(action IN stmt_passrole.action WHERE
|
||||
toLower(action) = 'iam:passrole'
|
||||
OR toLower(action) = 'iam:*'
|
||||
OR action = '*'
|
||||
)
|
||||
|
||||
// Find ecs:RegisterTaskDefinition permission
|
||||
MATCH (principal)--(taskdef_policy:AWSPolicy)--(stmt_taskdef:AWSPolicyStatement)
|
||||
WHERE stmt_taskdef.effect = 'Allow'
|
||||
AND any(action IN stmt_taskdef.action WHERE
|
||||
toLower(action) = 'ecs:registertaskdefinition'
|
||||
OR toLower(action) = 'ecs:*'
|
||||
OR action = '*'
|
||||
)
|
||||
|
||||
// Find ecs:RunTask permission
|
||||
MATCH (principal)--(runtask_policy:AWSPolicy)--(stmt_runtask:AWSPolicyStatement)
|
||||
WHERE stmt_runtask.effect = 'Allow'
|
||||
AND any(action IN stmt_runtask.action WHERE
|
||||
toLower(action) = 'ecs:runtask'
|
||||
OR toLower(action) = 'ecs:*'
|
||||
OR action = '*'
|
||||
)
|
||||
|
||||
// Find roles that trust ECS tasks service (can be passed to ECS tasks)
|
||||
MATCH path_target = (aws)--(target_role:AWSRole)-[:TRUSTS_AWS_PRINCIPAL]->(:AWSPrincipal {{arn: 'ecs-tasks.amazonaws.com'}})
|
||||
WHERE any(resource IN stmt_passrole.resource WHERE
|
||||
resource = '*'
|
||||
OR target_role.arn CONTAINS resource
|
||||
OR resource CONTAINS target_role.name
|
||||
)
|
||||
|
||||
UNWIND nodes(path_principal) + nodes(path_target) as n
|
||||
OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}})
|
||||
|
||||
RETURN path_principal, path_target,
|
||||
collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr
|
||||
""",
|
||||
parameters=[],
|
||||
)
|
||||
|
||||
# GLUE-001
|
||||
AWS_GLUE_PRIVESC_PASSROLE_DEV_ENDPOINT = AttackPathsQueryDefinition(
|
||||
id="aws-glue-privesc-passrole-dev-endpoint",
|
||||
name="Glue Dev Endpoint with Privileged Role (GLUE-001)",
|
||||
short_description="Create a Glue development endpoint with a privileged role attached to gain its permissions.",
|
||||
description="Detect principals who can pass IAM roles and create Glue development endpoints. This allows creating a dev endpoint with a privileged role attached, gaining that role's permissions.",
|
||||
attribution=AttackPathsQueryAttribution(
|
||||
text="pathfinding.cloud - GLUE-001 - iam:PassRole + glue:CreateDevEndpoint",
|
||||
link="https://pathfinding.cloud/paths/glue-001",
|
||||
),
|
||||
provider="aws",
|
||||
cypher=f"""
|
||||
// Find principals with iam:PassRole permission
|
||||
MATCH path_principal = (aws:AWSAccount {{id: $provider_uid}})--(principal:AWSPrincipal)--(passrole_policy:AWSPolicy)--(stmt_passrole:AWSPolicyStatement)
|
||||
WHERE stmt_passrole.effect = 'Allow'
|
||||
AND any(action IN stmt_passrole.action WHERE
|
||||
toLower(action) = 'iam:passrole'
|
||||
OR toLower(action) = 'iam:*'
|
||||
OR action = '*'
|
||||
)
|
||||
|
||||
// Find glue:CreateDevEndpoint permission
|
||||
MATCH (principal)--(glue_policy:AWSPolicy)--(stmt_glue:AWSPolicyStatement)
|
||||
WHERE stmt_glue.effect = 'Allow'
|
||||
AND any(action IN stmt_glue.action WHERE
|
||||
toLower(action) = 'glue:createdevendpoint'
|
||||
OR toLower(action) = 'glue:*'
|
||||
OR action = '*'
|
||||
)
|
||||
|
||||
// Find roles that trust Glue service (can be passed to Glue)
|
||||
MATCH path_target = (aws)--(target_role:AWSRole)-[:TRUSTS_AWS_PRINCIPAL]->(:AWSPrincipal {{arn: 'glue.amazonaws.com'}})
|
||||
WHERE any(resource IN stmt_passrole.resource WHERE
|
||||
resource = '*'
|
||||
OR target_role.arn CONTAINS resource
|
||||
OR resource CONTAINS target_role.name
|
||||
)
|
||||
|
||||
UNWIND nodes(path_principal) + nodes(path_target) as n
|
||||
OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}})
|
||||
|
||||
RETURN path_principal, path_target,
|
||||
collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr
|
||||
""",
|
||||
parameters=[],
|
||||
)
|
||||
|
||||
# IAM-014
|
||||
AWS_IAM_PRIVESC_ATTACH_ROLE_POLICY_ASSUME_ROLE = AttackPathsQueryDefinition(
|
||||
id="aws-iam-privesc-attach-role-policy-assume-role",
|
||||
name="Role Policy Attachment and Assumption (IAM-014)",
|
||||
short_description="Attach policies to IAM roles and then assume them to gain elevated access.",
|
||||
description="Detect principals who can both attach policies to roles AND assume those roles. This allows modifying a role's permissions then assuming it to gain elevated access.",
|
||||
attribution=AttackPathsQueryAttribution(
|
||||
text="pathfinding.cloud - IAM-014 - iam:AttachRolePolicy + sts:AssumeRole",
|
||||
link="https://pathfinding.cloud/paths/iam-014",
|
||||
),
|
||||
provider="aws",
|
||||
cypher=f"""
|
||||
// Find principals with iam:AttachRolePolicy permission
|
||||
MATCH path_principal = (aws:AWSAccount {{id: $provider_uid}})--(principal:AWSPrincipal)--(attach_policy:AWSPolicy)--(stmt_attach:AWSPolicyStatement)
|
||||
WHERE stmt_attach.effect = 'Allow'
|
||||
AND any(action IN stmt_attach.action WHERE
|
||||
toLower(action) = 'iam:attachrolepolicy'
|
||||
OR toLower(action) = 'iam:*'
|
||||
OR action = '*'
|
||||
)
|
||||
|
||||
// Find sts:AssumeRole permission
|
||||
MATCH (principal)--(assume_policy:AWSPolicy)--(stmt_assume:AWSPolicyStatement)
|
||||
WHERE stmt_assume.effect = 'Allow'
|
||||
AND any(action IN stmt_assume.action WHERE
|
||||
toLower(action) = 'sts:assumerole'
|
||||
OR toLower(action) = 'sts:*'
|
||||
OR action = '*'
|
||||
)
|
||||
|
||||
// Find target roles that the principal can both modify AND assume
|
||||
MATCH path_target = (aws)--(target_role:AWSRole)
|
||||
WHERE target_role.arn CONTAINS $provider_uid
|
||||
AND any(resource IN stmt_attach.resource WHERE
|
||||
resource = '*'
|
||||
OR target_role.arn CONTAINS resource
|
||||
OR resource CONTAINS target_role.name
|
||||
)
|
||||
AND any(resource IN stmt_assume.resource WHERE
|
||||
resource = '*'
|
||||
OR target_role.arn CONTAINS resource
|
||||
OR resource CONTAINS target_role.name
|
||||
)
|
||||
|
||||
UNWIND nodes(path_principal) + nodes(path_target) as n
|
||||
OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}})
|
||||
|
||||
RETURN path_principal, path_target,
|
||||
collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr
|
||||
""",
|
||||
parameters=[],
|
||||
)
|
||||
|
||||
# AWS Queries List
|
||||
# ----------------
|
||||
|
||||
AWS_QUERIES: list[AttackPathsQueryDefinition] = [
|
||||
AWS_INTERNET_EXPOSED_EC2_SENSITIVE_S3_ACCESS,
|
||||
AWS_RDS_INSTANCES,
|
||||
AWS_RDS_UNENCRYPTED_STORAGE,
|
||||
AWS_S3_ANONYMOUS_ACCESS_BUCKETS,
|
||||
AWS_IAM_STATEMENTS_ALLOW_ALL_ACTIONS,
|
||||
AWS_IAM_STATEMENTS_ALLOW_DELETE_POLICY,
|
||||
AWS_IAM_STATEMENTS_ALLOW_CREATE_ACTIONS,
|
||||
AWS_EC2_INSTANCES_INTERNET_EXPOSED,
|
||||
AWS_SECURITY_GROUPS_OPEN_INTERNET_FACING,
|
||||
AWS_CLASSIC_ELB_INTERNET_EXPOSED,
|
||||
AWS_ELBV2_INTERNET_EXPOSED,
|
||||
AWS_PUBLIC_IP_RESOURCE_LOOKUP,
|
||||
AWS_BEDROCK_PRIVESC_PASSROLE_CODE_INTERPRETER,
|
||||
AWS_EC2_PRIVESC_PASSROLE_IAM,
|
||||
AWS_EC2_PRIVESC_MODIFY_INSTANCE_ATTRIBUTE,
|
||||
AWS_EC2_PRIVESC_PASSROLE_SPOT_INSTANCES,
|
||||
AWS_EC2_PRIVESC_LAUNCH_TEMPLATE,
|
||||
AWS_ECS_PRIVESC_PASSROLE_CREATE_SERVICE,
|
||||
AWS_ECS_PRIVESC_PASSROLE_RUN_TASK,
|
||||
AWS_ECS_PRIVESC_PASSROLE_CREATE_SERVICE_EXISTING_CLUSTER,
|
||||
AWS_ECS_PRIVESC_PASSROLE_RUN_TASK_EXISTING_CLUSTER,
|
||||
AWS_GLUE_PRIVESC_PASSROLE_DEV_ENDPOINT,
|
||||
AWS_IAM_PRIVESC_ATTACH_ROLE_POLICY_ASSUME_ROLE,
|
||||
]
|
||||
@@ -0,0 +1,25 @@
|
||||
from api.attack_paths.queries.types import AttackPathsQueryDefinition
|
||||
from api.attack_paths.queries.aws import AWS_QUERIES
|
||||
|
||||
|
||||
# Query definitions organized by provider
|
||||
_QUERY_DEFINITIONS: dict[str, list[AttackPathsQueryDefinition]] = {
|
||||
"aws": AWS_QUERIES,
|
||||
}
|
||||
|
||||
# Flat lookup by query ID for O(1) access
|
||||
_QUERIES_BY_ID: dict[str, AttackPathsQueryDefinition] = {
|
||||
definition.id: definition
|
||||
for definitions in _QUERY_DEFINITIONS.values()
|
||||
for definition in definitions
|
||||
}
|
||||
|
||||
|
||||
def get_queries_for_provider(provider: str) -> list[AttackPathsQueryDefinition]:
|
||||
"""Get all attack path queries for a specific provider."""
|
||||
return _QUERY_DEFINITIONS.get(provider, [])
|
||||
|
||||
|
||||
def get_query_by_id(query_id: str) -> AttackPathsQueryDefinition | None:
|
||||
"""Get a specific attack path query by its ID."""
|
||||
return _QUERIES_BY_ID.get(query_id)
|
||||
@@ -0,0 +1,39 @@
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
|
||||
@dataclass
|
||||
class AttackPathsQueryAttribution:
|
||||
"""Source attribution for an Attack Path query."""
|
||||
|
||||
text: str
|
||||
link: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class AttackPathsQueryParameterDefinition:
|
||||
"""
|
||||
Metadata describing a parameter that must be provided to an Attack Paths query.
|
||||
"""
|
||||
|
||||
name: str
|
||||
label: str
|
||||
data_type: str = "string"
|
||||
cast: type = str
|
||||
description: str | None = None
|
||||
placeholder: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class AttackPathsQueryDefinition:
|
||||
"""
|
||||
Immutable representation of an Attack Path query.
|
||||
"""
|
||||
|
||||
id: str
|
||||
name: str
|
||||
short_description: str
|
||||
description: str
|
||||
provider: str
|
||||
cypher: str
|
||||
attribution: AttackPathsQueryAttribution | None = None
|
||||
parameters: list[AttackPathsQueryParameterDefinition] = field(default_factory=list)
|
||||
@@ -0,0 +1,92 @@
|
||||
import logging
|
||||
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
import neo4j
|
||||
import neo4j.exceptions
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class RetryableSession:
|
||||
"""
|
||||
Wrapper around `neo4j.Session` that retries `neo4j.exceptions.ServiceUnavailable` errors.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
session_factory: Callable[[], neo4j.Session],
|
||||
max_retries: int,
|
||||
) -> None:
|
||||
self._session_factory = session_factory
|
||||
self._max_retries = max(0, max_retries)
|
||||
self._session = self._session_factory()
|
||||
|
||||
def close(self) -> None:
|
||||
if self._session is not None:
|
||||
self._session.close()
|
||||
self._session = None
|
||||
|
||||
def __enter__(self) -> "RetryableSession":
|
||||
return self
|
||||
|
||||
def __exit__(
|
||||
self, _: Any, __: Any, ___: Any
|
||||
) -> None: # Unused args: exc_type, exc, exc_tb
|
||||
self.close()
|
||||
|
||||
def run(self, *args: Any, **kwargs: Any) -> Any:
|
||||
return self._call_with_retry("run", *args, **kwargs)
|
||||
|
||||
def write_transaction(self, *args: Any, **kwargs: Any) -> Any:
|
||||
return self._call_with_retry("write_transaction", *args, **kwargs)
|
||||
|
||||
def read_transaction(self, *args: Any, **kwargs: Any) -> Any:
|
||||
return self._call_with_retry("read_transaction", *args, **kwargs)
|
||||
|
||||
def execute_write(self, *args: Any, **kwargs: Any) -> Any:
|
||||
return self._call_with_retry("execute_write", *args, **kwargs)
|
||||
|
||||
def execute_read(self, *args: Any, **kwargs: Any) -> Any:
|
||||
return self._call_with_retry("execute_read", *args, **kwargs)
|
||||
|
||||
def __getattr__(self, item: str) -> Any:
|
||||
return getattr(self._session, item)
|
||||
|
||||
def _call_with_retry(self, method_name: str, *args: Any, **kwargs: Any) -> Any:
|
||||
attempt = 0
|
||||
last_exc: Exception | None = None
|
||||
|
||||
while attempt <= self._max_retries:
|
||||
try:
|
||||
method = getattr(self._session, method_name)
|
||||
return method(*args, **kwargs)
|
||||
|
||||
except (
|
||||
BrokenPipeError,
|
||||
ConnectionResetError,
|
||||
neo4j.exceptions.ServiceUnavailable,
|
||||
) as exc: # pragma: no cover - depends on infra
|
||||
last_exc = exc
|
||||
attempt += 1
|
||||
|
||||
if attempt > self._max_retries:
|
||||
raise
|
||||
|
||||
logger.warning(
|
||||
f"Neo4j session {method_name} failed with {type(exc).__name__} ({attempt}/{self._max_retries} attempts). Retrying..."
|
||||
)
|
||||
self._refresh_session()
|
||||
|
||||
raise last_exc if last_exc else RuntimeError("Unexpected retry loop exit")
|
||||
|
||||
def _refresh_session(self) -> None:
|
||||
if self._session is not None:
|
||||
try:
|
||||
self._session.close()
|
||||
except Exception:
|
||||
# Best-effort close; failures just mean we open a new session below
|
||||
pass
|
||||
|
||||
self._session = self._session_factory()
|
||||
@@ -0,0 +1,148 @@
|
||||
import logging
|
||||
|
||||
from typing import Any, Iterable
|
||||
|
||||
from rest_framework.exceptions import APIException, ValidationError
|
||||
|
||||
from api.attack_paths import database as graph_database, AttackPathsQueryDefinition
|
||||
from api.models import AttackPathsScan
|
||||
from config.custom_logging import BackendLogger
|
||||
from tasks.jobs.attack_paths.config import INTERNAL_LABELS
|
||||
|
||||
logger = logging.getLogger(BackendLogger.API)
|
||||
|
||||
|
||||
def normalize_run_payload(raw_data):
|
||||
if not isinstance(raw_data, dict): # Let the serializer handle this
|
||||
return raw_data
|
||||
|
||||
if "data" in raw_data and isinstance(raw_data.get("data"), dict):
|
||||
data_section = raw_data.get("data") or {}
|
||||
attributes = data_section.get("attributes") or {}
|
||||
payload = {
|
||||
"id": attributes.get("id", data_section.get("id")),
|
||||
"parameters": attributes.get("parameters"),
|
||||
}
|
||||
|
||||
# Remove `None` parameters to allow defaults downstream
|
||||
if payload.get("parameters") is None:
|
||||
payload.pop("parameters")
|
||||
return payload
|
||||
|
||||
return raw_data
|
||||
|
||||
|
||||
def prepare_query_parameters(
|
||||
definition: AttackPathsQueryDefinition,
|
||||
provided_parameters: dict[str, Any],
|
||||
provider_uid: str,
|
||||
) -> dict[str, Any]:
|
||||
parameters = dict(provided_parameters or {})
|
||||
expected_names = {parameter.name for parameter in definition.parameters}
|
||||
provided_names = set(parameters.keys())
|
||||
|
||||
unexpected = provided_names - expected_names
|
||||
if unexpected:
|
||||
raise ValidationError(
|
||||
{"parameters": f"Unknown parameter(s): {', '.join(sorted(unexpected))}"}
|
||||
)
|
||||
|
||||
missing = expected_names - provided_names
|
||||
if missing:
|
||||
raise ValidationError(
|
||||
{
|
||||
"parameters": f"Missing required parameter(s): {', '.join(sorted(missing))}"
|
||||
}
|
||||
)
|
||||
|
||||
clean_parameters = {
|
||||
"provider_uid": str(provider_uid),
|
||||
}
|
||||
|
||||
for definition_parameter in definition.parameters:
|
||||
raw_value = provided_parameters[definition_parameter.name]
|
||||
|
||||
try:
|
||||
casted_value = definition_parameter.cast(raw_value)
|
||||
|
||||
except (ValueError, TypeError) as exc:
|
||||
raise ValidationError(
|
||||
{
|
||||
"parameters": (
|
||||
f"Invalid value for parameter `{definition_parameter.name}`: {str(exc)}"
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
clean_parameters[definition_parameter.name] = casted_value
|
||||
|
||||
return clean_parameters
|
||||
|
||||
|
||||
def execute_attack_paths_query(
|
||||
attack_paths_scan: AttackPathsScan,
|
||||
definition: AttackPathsQueryDefinition,
|
||||
parameters: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
with graph_database.get_session(attack_paths_scan.graph_database) as session:
|
||||
result = session.run(definition.cypher, parameters)
|
||||
return _serialize_graph(result.graph())
|
||||
|
||||
except graph_database.GraphDatabaseQueryException as exc:
|
||||
logger.error(f"Query failed for Attack Paths query `{definition.id}`: {exc}")
|
||||
raise APIException(
|
||||
"Attack Paths query execution failed due to a database error"
|
||||
)
|
||||
|
||||
|
||||
def _serialize_graph(graph):
|
||||
nodes = []
|
||||
for node in graph.nodes:
|
||||
nodes.append(
|
||||
{
|
||||
"id": node.element_id,
|
||||
"labels": _filter_labels(node.labels),
|
||||
"properties": _serialize_properties(node._properties),
|
||||
},
|
||||
)
|
||||
|
||||
relationships = []
|
||||
for relationship in graph.relationships:
|
||||
relationships.append(
|
||||
{
|
||||
"id": relationship.element_id,
|
||||
"label": relationship.type,
|
||||
"source": relationship.start_node.element_id,
|
||||
"target": relationship.end_node.element_id,
|
||||
"properties": _serialize_properties(relationship._properties),
|
||||
},
|
||||
)
|
||||
|
||||
return {
|
||||
"nodes": nodes,
|
||||
"relationships": relationships,
|
||||
}
|
||||
|
||||
|
||||
def _filter_labels(labels: Iterable[str]) -> list[str]:
|
||||
return [label for label in labels if label not in INTERNAL_LABELS]
|
||||
|
||||
|
||||
def _serialize_properties(properties: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Convert Neo4j property values into JSON-serializable primitives."""
|
||||
|
||||
def _serialize_value(value: Any) -> Any:
|
||||
# Neo4j temporal and spatial values expose `to_native` returning Python primitives
|
||||
if hasattr(value, "to_native") and callable(value.to_native):
|
||||
return _serialize_value(value.to_native())
|
||||
|
||||
if isinstance(value, (list, tuple)):
|
||||
return [_serialize_value(item) for item in value]
|
||||
|
||||
if isinstance(value, dict):
|
||||
return {key: _serialize_value(val) for key, val in value.items()}
|
||||
|
||||
return value
|
||||
|
||||
return {key: _serialize_value(val) for key, val in properties.items()}
|
||||
@@ -1,15 +1,99 @@
|
||||
from types import MappingProxyType
|
||||
from collections.abc import Iterable, Mapping
|
||||
|
||||
from api.models import Provider
|
||||
from prowler.config.config import get_available_compliance_frameworks
|
||||
from prowler.lib.check.compliance_models import Compliance
|
||||
from prowler.lib.check.models import CheckMetadata
|
||||
|
||||
PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE = {}
|
||||
PROWLER_CHECKS = {}
|
||||
AVAILABLE_COMPLIANCE_FRAMEWORKS = {}
|
||||
|
||||
|
||||
class LazyComplianceTemplate(Mapping):
|
||||
"""Lazy-load compliance templates per provider on first access."""
|
||||
|
||||
def __init__(self, provider_types: Iterable[str] | None = None) -> None:
|
||||
if provider_types is None:
|
||||
provider_types = Provider.ProviderChoices.values
|
||||
self._provider_types = tuple(provider_types)
|
||||
self._provider_types_set = set(self._provider_types)
|
||||
self._cache: dict[str, dict] = {}
|
||||
|
||||
def _load_provider(self, provider_type: str) -> dict:
|
||||
if provider_type not in self._provider_types_set:
|
||||
raise KeyError(provider_type)
|
||||
cached = self._cache.get(provider_type)
|
||||
if cached is not None:
|
||||
return cached
|
||||
_ensure_provider_loaded(provider_type)
|
||||
return self._cache[provider_type]
|
||||
|
||||
def __getitem__(self, key: str) -> dict:
|
||||
return self._load_provider(key)
|
||||
|
||||
def __iter__(self):
|
||||
return iter(self._provider_types)
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(self._provider_types)
|
||||
|
||||
def __contains__(self, key: object) -> bool:
|
||||
return key in self._provider_types_set
|
||||
|
||||
def get(self, key: str, default=None):
|
||||
if key not in self._provider_types_set:
|
||||
return default
|
||||
return self._load_provider(key)
|
||||
|
||||
def __repr__(self) -> str: # pragma: no cover - debugging helper
|
||||
loaded = ", ".join(sorted(self._cache))
|
||||
return f"{self.__class__.__name__}(loaded=[{loaded}])"
|
||||
|
||||
|
||||
class LazyChecksMapping(Mapping):
|
||||
"""Lazy-load checks mapping per provider on first access."""
|
||||
|
||||
def __init__(self, provider_types: Iterable[str] | None = None) -> None:
|
||||
if provider_types is None:
|
||||
provider_types = Provider.ProviderChoices.values
|
||||
self._provider_types = tuple(provider_types)
|
||||
self._provider_types_set = set(self._provider_types)
|
||||
self._cache: dict[str, dict] = {}
|
||||
|
||||
def _load_provider(self, provider_type: str) -> dict:
|
||||
if provider_type not in self._provider_types_set:
|
||||
raise KeyError(provider_type)
|
||||
cached = self._cache.get(provider_type)
|
||||
if cached is not None:
|
||||
return cached
|
||||
_ensure_provider_loaded(provider_type)
|
||||
return self._cache[provider_type]
|
||||
|
||||
def __getitem__(self, key: str) -> dict:
|
||||
return self._load_provider(key)
|
||||
|
||||
def __iter__(self):
|
||||
return iter(self._provider_types)
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(self._provider_types)
|
||||
|
||||
def __contains__(self, key: object) -> bool:
|
||||
return key in self._provider_types_set
|
||||
|
||||
def get(self, key: str, default=None):
|
||||
if key not in self._provider_types_set:
|
||||
return default
|
||||
return self._load_provider(key)
|
||||
|
||||
def __repr__(self) -> str: # pragma: no cover - debugging helper
|
||||
loaded = ", ".join(sorted(self._cache))
|
||||
return f"{self.__class__.__name__}(loaded=[{loaded}])"
|
||||
|
||||
|
||||
PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE = LazyComplianceTemplate()
|
||||
PROWLER_CHECKS = LazyChecksMapping()
|
||||
|
||||
|
||||
def get_compliance_frameworks(provider_type: Provider.ProviderChoices) -> list[str]:
|
||||
"""
|
||||
Retrieve and cache the list of available compliance frameworks for a specific cloud provider.
|
||||
@@ -70,28 +154,35 @@ def get_prowler_provider_compliance(provider_type: Provider.ProviderChoices) ->
|
||||
return Compliance.get_bulk(provider_type)
|
||||
|
||||
|
||||
def load_prowler_compliance():
|
||||
"""
|
||||
Load and initialize the Prowler compliance data and checks for all provider types.
|
||||
|
||||
This function retrieves compliance data for all supported provider types,
|
||||
generates a compliance overview template, and populates the global variables
|
||||
`PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE` and `PROWLER_CHECKS` with read-only mappings
|
||||
of the compliance templates and checks, respectively.
|
||||
"""
|
||||
global PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE
|
||||
global PROWLER_CHECKS
|
||||
|
||||
prowler_compliance = {
|
||||
provider_type: get_prowler_provider_compliance(provider_type)
|
||||
for provider_type in Provider.ProviderChoices.values
|
||||
}
|
||||
template = generate_compliance_overview_template(prowler_compliance)
|
||||
PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE = MappingProxyType(template)
|
||||
PROWLER_CHECKS = MappingProxyType(load_prowler_checks(prowler_compliance))
|
||||
def _load_provider_assets(provider_type: Provider.ProviderChoices) -> tuple[dict, dict]:
|
||||
prowler_compliance = {provider_type: get_prowler_provider_compliance(provider_type)}
|
||||
template = generate_compliance_overview_template(
|
||||
prowler_compliance, provider_types=[provider_type]
|
||||
)
|
||||
checks = load_prowler_checks(prowler_compliance, provider_types=[provider_type])
|
||||
return template.get(provider_type, {}), checks.get(provider_type, {})
|
||||
|
||||
|
||||
def load_prowler_checks(prowler_compliance):
|
||||
def _ensure_provider_loaded(provider_type: Provider.ProviderChoices) -> None:
|
||||
if (
|
||||
provider_type in PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE._cache
|
||||
and provider_type in PROWLER_CHECKS._cache
|
||||
):
|
||||
return
|
||||
template_cached = PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE._cache.get(provider_type)
|
||||
checks_cached = PROWLER_CHECKS._cache.get(provider_type)
|
||||
if template_cached is not None and checks_cached is not None:
|
||||
return
|
||||
template, checks = _load_provider_assets(provider_type)
|
||||
if template_cached is None:
|
||||
PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE._cache[provider_type] = template
|
||||
if checks_cached is None:
|
||||
PROWLER_CHECKS._cache[provider_type] = checks
|
||||
|
||||
|
||||
def load_prowler_checks(
|
||||
prowler_compliance, provider_types: Iterable[str] | None = None
|
||||
):
|
||||
"""
|
||||
Generate a mapping of checks to the compliance frameworks that include them.
|
||||
|
||||
@@ -100,21 +191,25 @@ def load_prowler_checks(prowler_compliance):
|
||||
of compliance names that include that check.
|
||||
|
||||
Args:
|
||||
prowler_compliance (dict): The compliance data for all provider types,
|
||||
prowler_compliance (dict): The compliance data for provider types,
|
||||
as returned by `get_prowler_provider_compliance`.
|
||||
provider_types (Iterable[str] | None): Optional subset of provider types to
|
||||
process. Defaults to all providers.
|
||||
|
||||
Returns:
|
||||
dict: A nested dictionary where the first-level keys are provider types,
|
||||
and the values are dictionaries mapping check IDs to sets of compliance names.
|
||||
"""
|
||||
checks = {}
|
||||
for provider_type in Provider.ProviderChoices.values:
|
||||
if provider_types is None:
|
||||
provider_types = Provider.ProviderChoices.values
|
||||
for provider_type in provider_types:
|
||||
checks[provider_type] = {
|
||||
check_id: set() for check_id in get_prowler_provider_checks(provider_type)
|
||||
}
|
||||
for compliance_name, compliance_data in prowler_compliance[
|
||||
provider_type
|
||||
].items():
|
||||
for compliance_name, compliance_data in prowler_compliance.get(
|
||||
provider_type, {}
|
||||
).items():
|
||||
for requirement in compliance_data.Requirements:
|
||||
for check in requirement.Checks:
|
||||
try:
|
||||
@@ -163,7 +258,9 @@ def generate_scan_compliance(
|
||||
] += 1
|
||||
|
||||
|
||||
def generate_compliance_overview_template(prowler_compliance: dict):
|
||||
def generate_compliance_overview_template(
|
||||
prowler_compliance: dict, provider_types: Iterable[str] | None = None
|
||||
):
|
||||
"""
|
||||
Generate a compliance overview template for all provider types.
|
||||
|
||||
@@ -173,17 +270,21 @@ def generate_compliance_overview_template(prowler_compliance: dict):
|
||||
counts for requirements status.
|
||||
|
||||
Args:
|
||||
prowler_compliance (dict): The compliance data for all provider types,
|
||||
prowler_compliance (dict): The compliance data for provider types,
|
||||
as returned by `get_prowler_provider_compliance`.
|
||||
provider_types (Iterable[str] | None): Optional subset of provider types to
|
||||
process. Defaults to all providers.
|
||||
|
||||
Returns:
|
||||
dict: A nested dictionary representing the compliance overview template,
|
||||
structured by provider type and compliance framework.
|
||||
"""
|
||||
template = {}
|
||||
for provider_type in Provider.ProviderChoices.values:
|
||||
if provider_types is None:
|
||||
provider_types = Provider.ProviderChoices.values
|
||||
for provider_type in provider_types:
|
||||
provider_compliance = template.setdefault(provider_type, {})
|
||||
compliance_data_dict = prowler_compliance[provider_type]
|
||||
compliance_data_dict = prowler_compliance.get(provider_type, {})
|
||||
|
||||
for compliance_name, compliance_data in compliance_data_dict.items():
|
||||
compliance_requirements = {}
|
||||
|
||||
@@ -12,7 +12,6 @@ from django.contrib.auth.models import BaseUserManager
|
||||
from django.db import (
|
||||
DEFAULT_DB_ALIAS,
|
||||
OperationalError,
|
||||
connection,
|
||||
connections,
|
||||
models,
|
||||
transaction,
|
||||
@@ -450,7 +449,7 @@ def create_index_on_partitions(
|
||||
all_partitions=True
|
||||
)
|
||||
"""
|
||||
with connection.cursor() as cursor:
|
||||
with schema_editor.connection.cursor() as cursor:
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT inhrelid::regclass::text
|
||||
@@ -462,6 +461,7 @@ def create_index_on_partitions(
|
||||
partitions = [row[0] for row in cursor.fetchall()]
|
||||
|
||||
where_sql = f" WHERE {where}" if where else ""
|
||||
conn = schema_editor.connection
|
||||
for partition in partitions:
|
||||
if _should_create_index_on_partition(partition, all_partitions):
|
||||
idx_name = f"{partition.replace('.', '_')}_{index_name}"
|
||||
@@ -470,7 +470,12 @@ def create_index_on_partitions(
|
||||
f"ON {partition} USING {method} ({columns})"
|
||||
f"{where_sql};"
|
||||
)
|
||||
schema_editor.execute(sql)
|
||||
old_autocommit = conn.connection.autocommit
|
||||
conn.connection.autocommit = True
|
||||
try:
|
||||
schema_editor.execute(sql)
|
||||
finally:
|
||||
conn.connection.autocommit = old_autocommit
|
||||
|
||||
|
||||
def drop_index_on_partitions(
|
||||
@@ -486,7 +491,8 @@ def drop_index_on_partitions(
|
||||
parent_table: The name of the root table (e.g. "findings").
|
||||
index_name: The same short name used when creating them.
|
||||
"""
|
||||
with connection.cursor() as cursor:
|
||||
conn = schema_editor.connection
|
||||
with conn.cursor() as cursor:
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT inhrelid::regclass::text
|
||||
@@ -500,7 +506,12 @@ def drop_index_on_partitions(
|
||||
for partition in partitions:
|
||||
idx_name = f"{partition.replace('.', '_')}_{index_name}"
|
||||
sql = f"DROP INDEX CONCURRENTLY IF EXISTS {idx_name};"
|
||||
schema_editor.execute(sql)
|
||||
old_autocommit = conn.connection.autocommit
|
||||
conn.connection.autocommit = True
|
||||
try:
|
||||
schema_editor.execute(sql)
|
||||
finally:
|
||||
conn.connection.autocommit = old_autocommit
|
||||
|
||||
|
||||
def generate_api_key_prefix():
|
||||
|
||||
@@ -107,3 +107,105 @@ class ConflictException(APIException):
|
||||
error_detail["source"] = {"pointer": pointer}
|
||||
|
||||
super().__init__(detail=[error_detail])
|
||||
|
||||
|
||||
# Upstream Provider Errors (for external API calls like CloudTrail)
|
||||
# These indicate issues with the provider, not with the user's API authentication
|
||||
|
||||
|
||||
class UpstreamAuthenticationError(APIException):
|
||||
"""Provider credentials are invalid or expired (502 Bad Gateway).
|
||||
|
||||
Used when AWS/Azure/GCP credentials fail to authenticate with the upstream
|
||||
provider. This is NOT the user's API authentication failing.
|
||||
"""
|
||||
|
||||
status_code = status.HTTP_502_BAD_GATEWAY
|
||||
default_detail = (
|
||||
"Provider credentials are invalid or expired. Please reconnect the provider."
|
||||
)
|
||||
default_code = "upstream_auth_failed"
|
||||
|
||||
def __init__(self, detail=None):
|
||||
super().__init__(
|
||||
detail=[
|
||||
{
|
||||
"detail": detail or self.default_detail,
|
||||
"status": str(self.status_code),
|
||||
"code": self.default_code,
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
class UpstreamAccessDeniedError(APIException):
|
||||
"""Provider credentials lack required permissions (502 Bad Gateway).
|
||||
|
||||
Used when credentials are valid but don't have the IAM permissions
|
||||
needed for the requested operation (e.g., cloudtrail:LookupEvents).
|
||||
This is 502 (not 403) because it's an upstream/gateway error - the USER
|
||||
authenticated fine, but the PROVIDER's credentials are misconfigured.
|
||||
"""
|
||||
|
||||
status_code = status.HTTP_502_BAD_GATEWAY
|
||||
default_detail = (
|
||||
"Access denied. The provider credentials do not have the required permissions."
|
||||
)
|
||||
default_code = "upstream_access_denied"
|
||||
|
||||
def __init__(self, detail=None):
|
||||
super().__init__(
|
||||
detail=[
|
||||
{
|
||||
"detail": detail or self.default_detail,
|
||||
"status": str(self.status_code),
|
||||
"code": self.default_code,
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
class UpstreamServiceUnavailableError(APIException):
|
||||
"""Provider service is unavailable (503 Service Unavailable).
|
||||
|
||||
Used when the upstream provider API returns an error or is unreachable.
|
||||
"""
|
||||
|
||||
status_code = status.HTTP_503_SERVICE_UNAVAILABLE
|
||||
default_detail = "Unable to communicate with the provider. Please try again later."
|
||||
default_code = "service_unavailable"
|
||||
|
||||
def __init__(self, detail=None):
|
||||
super().__init__(
|
||||
detail=[
|
||||
{
|
||||
"detail": detail or self.default_detail,
|
||||
"status": str(self.status_code),
|
||||
"code": self.default_code,
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
class UpstreamInternalError(APIException):
|
||||
"""Unexpected error communicating with provider (500 Internal Server Error).
|
||||
|
||||
Used as a catch-all for unexpected errors during provider communication.
|
||||
"""
|
||||
|
||||
status_code = status.HTTP_500_INTERNAL_SERVER_ERROR
|
||||
default_detail = (
|
||||
"An unexpected error occurred while communicating with the provider."
|
||||
)
|
||||
default_code = "internal_error"
|
||||
|
||||
def __init__(self, detail=None):
|
||||
super().__init__(
|
||||
detail=[
|
||||
{
|
||||
"detail": detail or self.default_detail,
|
||||
"status": str(self.status_code),
|
||||
"code": self.default_code,
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
@@ -29,6 +29,7 @@ from api.models import (
|
||||
Finding,
|
||||
Integration,
|
||||
Invitation,
|
||||
AttackPathsScan,
|
||||
LighthouseProviderConfiguration,
|
||||
LighthouseProviderModels,
|
||||
Membership,
|
||||
@@ -45,6 +46,7 @@ from api.models import (
|
||||
Role,
|
||||
Scan,
|
||||
ScanCategorySummary,
|
||||
ScanGroupSummary,
|
||||
ScanSummary,
|
||||
SeverityChoices,
|
||||
StateChoices,
|
||||
@@ -214,6 +216,9 @@ class CommonFindingFilters(FilterSet):
|
||||
category = CharFilter(method="filter_category")
|
||||
category__in = CharInFilter(field_name="categories", lookup_expr="overlap")
|
||||
|
||||
resource_groups = CharFilter(field_name="resource_groups", lookup_expr="exact")
|
||||
resource_groups__in = CharInFilter(field_name="resource_groups", lookup_expr="in")
|
||||
|
||||
# Temporarily disabled until we implement tag filtering in the UI
|
||||
# resource_tag_key = CharFilter(field_name="resources__tags__key")
|
||||
# resource_tag_key__in = CharInFilter(
|
||||
@@ -392,6 +397,23 @@ class ScanFilter(ProviderRelationshipFilterSet):
|
||||
}
|
||||
|
||||
|
||||
class AttackPathsScanFilter(ProviderRelationshipFilterSet):
|
||||
inserted_at = DateFilter(field_name="inserted_at", lookup_expr="date")
|
||||
completed_at = DateFilter(field_name="completed_at", lookup_expr="date")
|
||||
started_at = DateFilter(field_name="started_at", lookup_expr="date")
|
||||
state = ChoiceFilter(choices=StateChoices.choices)
|
||||
state__in = ChoiceInFilter(
|
||||
field_name="state", choices=StateChoices.choices, lookup_expr="in"
|
||||
)
|
||||
|
||||
class Meta:
|
||||
model = AttackPathsScan
|
||||
fields = {
|
||||
"provider": ["exact", "in"],
|
||||
"scan": ["exact", "in"],
|
||||
}
|
||||
|
||||
|
||||
class TaskFilter(FilterSet):
|
||||
name = CharFilter(field_name="task_runner_task__task_name", lookup_expr="exact")
|
||||
name__icontains = CharFilter(
|
||||
@@ -431,6 +453,8 @@ class ResourceTagFilter(FilterSet):
|
||||
|
||||
|
||||
class ResourceFilter(ProviderRelationshipFilterSet):
|
||||
provider_id = UUIDFilter(field_name="provider__id", lookup_expr="exact")
|
||||
provider_id__in = UUIDInFilter(field_name="provider__id", lookup_expr="in")
|
||||
tag_key = CharFilter(method="filter_tag_key")
|
||||
tag_value = CharFilter(method="filter_tag_value")
|
||||
tag = CharFilter(method="filter_tag")
|
||||
@@ -439,6 +463,8 @@ class ResourceFilter(ProviderRelationshipFilterSet):
|
||||
updated_at = DateFilter(field_name="updated_at", lookup_expr="date")
|
||||
scan = UUIDFilter(field_name="provider__scan", lookup_expr="exact")
|
||||
scan__in = UUIDInFilter(field_name="provider__scan", lookup_expr="in")
|
||||
groups = CharFilter(method="filter_groups")
|
||||
groups__in = CharInFilter(field_name="groups", lookup_expr="overlap")
|
||||
|
||||
class Meta:
|
||||
model = Resource
|
||||
@@ -453,6 +479,9 @@ class ResourceFilter(ProviderRelationshipFilterSet):
|
||||
"updated_at": ["gte", "lte"],
|
||||
}
|
||||
|
||||
def filter_groups(self, queryset, name, value):
|
||||
return queryset.filter(groups__contains=[value])
|
||||
|
||||
def filter_queryset(self, queryset):
|
||||
if not (self.data.get("scan") or self.data.get("scan__in")) and not (
|
||||
self.data.get("updated_at")
|
||||
@@ -513,10 +542,14 @@ class ResourceFilter(ProviderRelationshipFilterSet):
|
||||
|
||||
|
||||
class LatestResourceFilter(ProviderRelationshipFilterSet):
|
||||
provider_id = UUIDFilter(field_name="provider__id", lookup_expr="exact")
|
||||
provider_id__in = UUIDInFilter(field_name="provider__id", lookup_expr="in")
|
||||
tag_key = CharFilter(method="filter_tag_key")
|
||||
tag_value = CharFilter(method="filter_tag_value")
|
||||
tag = CharFilter(method="filter_tag")
|
||||
tags = CharFilter(method="filter_tag")
|
||||
groups = CharFilter(method="filter_groups")
|
||||
groups__in = CharInFilter(field_name="groups", lookup_expr="overlap")
|
||||
|
||||
class Meta:
|
||||
model = Resource
|
||||
@@ -529,6 +562,9 @@ class LatestResourceFilter(ProviderRelationshipFilterSet):
|
||||
"type": ["exact", "icontains", "in"],
|
||||
}
|
||||
|
||||
def filter_groups(self, queryset, name, value):
|
||||
return queryset.filter(groups__contains=[value])
|
||||
|
||||
def filter_tag_key(self, queryset, name, value):
|
||||
return queryset.filter(Q(tags__key=value) | Q(tags__key__icontains=value))
|
||||
|
||||
@@ -1154,6 +1190,26 @@ class CategoryOverviewFilter(BaseScanProviderFilter):
|
||||
|
||||
class Meta(BaseScanProviderFilter.Meta):
|
||||
model = ScanCategorySummary
|
||||
fields = {}
|
||||
|
||||
|
||||
class ResourceGroupOverviewFilter(FilterSet):
|
||||
provider_id = UUIDFilter(field_name="scan__provider__id", lookup_expr="exact")
|
||||
provider_id__in = UUIDInFilter(field_name="scan__provider__id", lookup_expr="in")
|
||||
provider_type = ChoiceFilter(
|
||||
field_name="scan__provider__provider", choices=Provider.ProviderChoices.choices
|
||||
)
|
||||
provider_type__in = ChoiceInFilter(
|
||||
field_name="scan__provider__provider",
|
||||
choices=Provider.ProviderChoices.choices,
|
||||
lookup_expr="in",
|
||||
)
|
||||
resource_group = CharFilter(field_name="resource_group", lookup_expr="exact")
|
||||
resource_group__in = CharInFilter(field_name="resource_group", lookup_expr="in")
|
||||
|
||||
class Meta:
|
||||
model = ScanGroupSummary
|
||||
fields = {}
|
||||
|
||||
|
||||
class ComplianceWatchlistFilter(BaseProviderFilter):
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
[
|
||||
{
|
||||
"model": "api.attackpathsscan",
|
||||
"pk": "a7f0f6de-6f8e-4b3a-8cbe-3f6dd9012345",
|
||||
"fields": {
|
||||
"tenant": "12646005-9067-4d2a-a098-8bb378604362",
|
||||
"provider": "b85601a8-4b45-4194-8135-03fb980ef428",
|
||||
"scan": "01920573-aa9c-73c9-bcda-f2e35c9b19d2",
|
||||
"state": "completed",
|
||||
"progress": 100,
|
||||
"update_tag": 1693586667,
|
||||
"graph_database": "db-a7f0f6de-6f8e-4b3a-8cbe-3f6dd9012345",
|
||||
"is_graph_database_deleted": false,
|
||||
"task": null,
|
||||
"inserted_at": "2024-09-01T17:24:37Z",
|
||||
"updated_at": "2024-09-01T17:44:37Z",
|
||||
"started_at": "2024-09-01T17:34:37Z",
|
||||
"completed_at": "2024-09-01T17:44:37Z",
|
||||
"duration": 269,
|
||||
"ingestion_exceptions": {}
|
||||
}
|
||||
},
|
||||
{
|
||||
"model": "api.attackpathsscan",
|
||||
"pk": "4a2fb2af-8a60-4d7d-9cae-4ca65e098765",
|
||||
"fields": {
|
||||
"tenant": "12646005-9067-4d2a-a098-8bb378604362",
|
||||
"provider": "15fce1fa-ecaa-433f-a9dc-62553f3a2555",
|
||||
"scan": "01929f3b-ed2e-7623-ad63-7c37cd37828f",
|
||||
"state": "executing",
|
||||
"progress": 48,
|
||||
"update_tag": 1697625000,
|
||||
"graph_database": "db-4a2fb2af-8a60-4d7d-9cae-4ca65e098765",
|
||||
"is_graph_database_deleted": false,
|
||||
"task": null,
|
||||
"inserted_at": "2024-10-18T10:55:57Z",
|
||||
"updated_at": "2024-10-18T10:56:15Z",
|
||||
"started_at": "2024-10-18T10:56:05Z"
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,126 @@
|
||||
import uuid
|
||||
|
||||
import django.db.models.deletion
|
||||
from django.db import migrations, models
|
||||
|
||||
import api.db_utils
|
||||
import api.rls
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("api", "0067_tenant_compliance_summary"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name="finding",
|
||||
name="resource_groups",
|
||||
field=models.TextField(
|
||||
blank=True,
|
||||
help_text="Resource group from check metadata for efficient filtering",
|
||||
null=True,
|
||||
),
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name="ScanGroupSummary",
|
||||
fields=[
|
||||
(
|
||||
"id",
|
||||
models.UUIDField(
|
||||
default=uuid.uuid4,
|
||||
editable=False,
|
||||
primary_key=True,
|
||||
serialize=False,
|
||||
),
|
||||
),
|
||||
(
|
||||
"tenant",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
to="api.tenant",
|
||||
),
|
||||
),
|
||||
(
|
||||
"inserted_at",
|
||||
models.DateTimeField(auto_now_add=True),
|
||||
),
|
||||
(
|
||||
"scan",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
related_name="resource_group_summaries",
|
||||
related_query_name="resource_group_summary",
|
||||
to="api.scan",
|
||||
),
|
||||
),
|
||||
(
|
||||
"resource_group",
|
||||
models.CharField(max_length=50),
|
||||
),
|
||||
(
|
||||
"severity",
|
||||
api.db_utils.SeverityEnumField(
|
||||
choices=[
|
||||
("critical", "Critical"),
|
||||
("high", "High"),
|
||||
("medium", "Medium"),
|
||||
("low", "Low"),
|
||||
("informational", "Informational"),
|
||||
],
|
||||
),
|
||||
),
|
||||
(
|
||||
"total_findings",
|
||||
models.IntegerField(
|
||||
default=0, help_text="Non-muted findings (PASS + FAIL)"
|
||||
),
|
||||
),
|
||||
(
|
||||
"failed_findings",
|
||||
models.IntegerField(
|
||||
default=0,
|
||||
help_text="Non-muted FAIL findings (subset of total_findings)",
|
||||
),
|
||||
),
|
||||
(
|
||||
"new_failed_findings",
|
||||
models.IntegerField(
|
||||
default=0,
|
||||
help_text="Non-muted FAIL with delta='new' (subset of failed_findings)",
|
||||
),
|
||||
),
|
||||
(
|
||||
"resources_count",
|
||||
models.IntegerField(
|
||||
default=0, help_text="Count of distinct resource_uid values"
|
||||
),
|
||||
),
|
||||
],
|
||||
options={
|
||||
"db_table": "scan_resource_group_summaries",
|
||||
"abstract": False,
|
||||
},
|
||||
),
|
||||
migrations.AddIndex(
|
||||
model_name="scangroupsummary",
|
||||
index=models.Index(
|
||||
fields=["tenant_id", "scan"], name="srgs_tenant_scan_idx"
|
||||
),
|
||||
),
|
||||
migrations.AddConstraint(
|
||||
model_name="scangroupsummary",
|
||||
constraint=models.UniqueConstraint(
|
||||
fields=("tenant_id", "scan_id", "resource_group", "severity"),
|
||||
name="unique_resource_group_severity_per_scan",
|
||||
),
|
||||
),
|
||||
migrations.AddConstraint(
|
||||
model_name="scangroupsummary",
|
||||
constraint=api.rls.RowLevelSecurityConstraint(
|
||||
field="tenant_id",
|
||||
name="rls_on_scangroupsummary",
|
||||
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
|
||||
),
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,21 @@
|
||||
from django.contrib.postgres.fields import ArrayField
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("api", "0068_finding_resource_group_scangroupsummary"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name="resource",
|
||||
name="groups",
|
||||
field=ArrayField(
|
||||
models.CharField(max_length=100),
|
||||
blank=True,
|
||||
help_text="Groups for categorization (e.g., compute, storage, IAM)",
|
||||
null=True,
|
||||
),
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,154 @@
|
||||
# Generated by Django 5.1.13 on 2025-11-06 16:20
|
||||
|
||||
import django.db.models.deletion
|
||||
|
||||
from django.db import migrations, models
|
||||
from uuid6 import uuid7
|
||||
|
||||
import api.rls
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("api", "0069_resource_resource_group"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name="AttackPathsScan",
|
||||
fields=[
|
||||
(
|
||||
"id",
|
||||
models.UUIDField(
|
||||
default=uuid7,
|
||||
editable=False,
|
||||
primary_key=True,
|
||||
serialize=False,
|
||||
),
|
||||
),
|
||||
("inserted_at", models.DateTimeField(auto_now_add=True)),
|
||||
("updated_at", models.DateTimeField(auto_now=True)),
|
||||
(
|
||||
"state",
|
||||
api.db_utils.StateEnumField(
|
||||
choices=[
|
||||
("available", "Available"),
|
||||
("scheduled", "Scheduled"),
|
||||
("executing", "Executing"),
|
||||
("completed", "Completed"),
|
||||
("failed", "Failed"),
|
||||
("cancelled", "Cancelled"),
|
||||
],
|
||||
default="available",
|
||||
),
|
||||
),
|
||||
("progress", models.IntegerField(default=0)),
|
||||
("started_at", models.DateTimeField(blank=True, null=True)),
|
||||
("completed_at", models.DateTimeField(blank=True, null=True)),
|
||||
(
|
||||
"duration",
|
||||
models.IntegerField(
|
||||
blank=True, help_text="Duration in seconds", null=True
|
||||
),
|
||||
),
|
||||
(
|
||||
"update_tag",
|
||||
models.BigIntegerField(
|
||||
blank=True,
|
||||
help_text="Cartography update tag (epoch)",
|
||||
null=True,
|
||||
),
|
||||
),
|
||||
(
|
||||
"graph_database",
|
||||
models.CharField(blank=True, max_length=63, null=True),
|
||||
),
|
||||
(
|
||||
"is_graph_database_deleted",
|
||||
models.BooleanField(default=False),
|
||||
),
|
||||
(
|
||||
"ingestion_exceptions",
|
||||
models.JSONField(blank=True, default=dict, null=True),
|
||||
),
|
||||
(
|
||||
"provider",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
related_name="attack_paths_scans",
|
||||
related_query_name="attack_paths_scan",
|
||||
to="api.provider",
|
||||
),
|
||||
),
|
||||
(
|
||||
"scan",
|
||||
models.ForeignKey(
|
||||
blank=True,
|
||||
null=True,
|
||||
on_delete=django.db.models.deletion.SET_NULL,
|
||||
related_name="attack_paths_scans",
|
||||
related_query_name="attack_paths_scan",
|
||||
to="api.scan",
|
||||
),
|
||||
),
|
||||
(
|
||||
"task",
|
||||
models.ForeignKey(
|
||||
blank=True,
|
||||
null=True,
|
||||
on_delete=django.db.models.deletion.SET_NULL,
|
||||
related_name="attack_paths_scans",
|
||||
related_query_name="attack_paths_scan",
|
||||
to="api.task",
|
||||
),
|
||||
),
|
||||
(
|
||||
"tenant",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE, to="api.tenant"
|
||||
),
|
||||
),
|
||||
],
|
||||
options={
|
||||
"db_table": "attack_paths_scans",
|
||||
"abstract": False,
|
||||
"indexes": [
|
||||
models.Index(
|
||||
fields=["tenant_id", "provider_id", "-inserted_at"],
|
||||
name="aps_prov_ins_desc_idx",
|
||||
),
|
||||
models.Index(
|
||||
fields=["tenant_id", "state", "-inserted_at"],
|
||||
name="aps_state_ins_desc_idx",
|
||||
),
|
||||
models.Index(
|
||||
fields=["tenant_id", "scan_id"],
|
||||
name="aps_scan_lookup_idx",
|
||||
),
|
||||
models.Index(
|
||||
fields=["tenant_id", "provider_id"],
|
||||
name="aps_active_graph_idx",
|
||||
include=["graph_database", "id"],
|
||||
condition=models.Q(("is_graph_database_deleted", False)),
|
||||
),
|
||||
models.Index(
|
||||
fields=["tenant_id", "provider_id", "-completed_at"],
|
||||
name="aps_completed_graph_idx",
|
||||
include=["graph_database", "id"],
|
||||
condition=models.Q(
|
||||
("state", "completed"),
|
||||
("is_graph_database_deleted", False),
|
||||
),
|
||||
),
|
||||
],
|
||||
},
|
||||
),
|
||||
migrations.AddConstraint(
|
||||
model_name="attackpathsscan",
|
||||
constraint=api.rls.RowLevelSecurityConstraint(
|
||||
"tenant_id",
|
||||
name="rls_on_attackpathsscan",
|
||||
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
|
||||
),
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,41 @@
|
||||
from django.db import migrations
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
"""
|
||||
Drop unused indexes on partitioned tables (findings, resource_finding_mappings).
|
||||
|
||||
NOTE: RemoveIndexConcurrently cannot be used on partitioned tables in PostgreSQL.
|
||||
Standard RemoveIndex drops the parent index, which cascades to all partitions.
|
||||
"""
|
||||
|
||||
dependencies = [
|
||||
("api", "0070_attack_paths_scan"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.RemoveIndex(
|
||||
model_name="finding",
|
||||
name="gin_findings_search_idx",
|
||||
),
|
||||
migrations.RemoveIndex(
|
||||
model_name="finding",
|
||||
name="gin_find_service_idx",
|
||||
),
|
||||
migrations.RemoveIndex(
|
||||
model_name="finding",
|
||||
name="gin_find_region_idx",
|
||||
),
|
||||
migrations.RemoveIndex(
|
||||
model_name="finding",
|
||||
name="gin_find_rtype_idx",
|
||||
),
|
||||
migrations.RemoveIndex(
|
||||
model_name="finding",
|
||||
name="find_delta_new_idx",
|
||||
),
|
||||
migrations.RemoveIndex(
|
||||
model_name="resourcefindingmapping",
|
||||
name="rfm_tenant_finding_idx",
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,91 @@
|
||||
"""
|
||||
Drop unused indexes on non-partitioned tables.
|
||||
|
||||
These tables are not partitioned, so RemoveIndexConcurrently can be used safely.
|
||||
"""
|
||||
|
||||
from uuid import uuid4
|
||||
|
||||
from django.contrib.postgres.operations import RemoveIndexConcurrently
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
def drop_resource_scan_summary_resource_id_index(apps, schema_editor):
|
||||
with schema_editor.connection.cursor() as cursor:
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT idx_ns.nspname, idx.relname
|
||||
FROM pg_class tbl
|
||||
JOIN pg_namespace tbl_ns ON tbl_ns.oid = tbl.relnamespace
|
||||
JOIN pg_index i ON i.indrelid = tbl.oid
|
||||
JOIN pg_class idx ON idx.oid = i.indexrelid
|
||||
JOIN pg_namespace idx_ns ON idx_ns.oid = idx.relnamespace
|
||||
JOIN pg_attribute a
|
||||
ON a.attrelid = tbl.oid
|
||||
AND a.attnum = (i.indkey::int[])[0]
|
||||
WHERE tbl_ns.nspname = ANY (current_schemas(false))
|
||||
AND tbl.relname = %s
|
||||
AND i.indnatts = 1
|
||||
AND a.attname = %s
|
||||
""",
|
||||
["resource_scan_summaries", "resource_id"],
|
||||
)
|
||||
row = cursor.fetchone()
|
||||
|
||||
if not row:
|
||||
return
|
||||
|
||||
schema_name, index_name = row
|
||||
quote_name = schema_editor.connection.ops.quote_name
|
||||
qualified_name = f"{quote_name(schema_name)}.{quote_name(index_name)}"
|
||||
schema_editor.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {qualified_name};")
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
atomic = False
|
||||
|
||||
dependencies = [
|
||||
("api", "0071_drop_partitioned_indexes"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
RemoveIndexConcurrently(
|
||||
model_name="resource",
|
||||
name="gin_resources_search_idx",
|
||||
),
|
||||
RemoveIndexConcurrently(
|
||||
model_name="resourcetag",
|
||||
name="gin_resource_tags_search_idx",
|
||||
),
|
||||
RemoveIndexConcurrently(
|
||||
model_name="scansummary",
|
||||
name="ss_tenant_scan_service_idx",
|
||||
),
|
||||
RemoveIndexConcurrently(
|
||||
model_name="complianceoverview",
|
||||
name="comp_ov_cp_id_idx",
|
||||
),
|
||||
RemoveIndexConcurrently(
|
||||
model_name="complianceoverview",
|
||||
name="comp_ov_req_fail_idx",
|
||||
),
|
||||
RemoveIndexConcurrently(
|
||||
model_name="complianceoverview",
|
||||
name="comp_ov_cp_id_req_fail_idx",
|
||||
),
|
||||
migrations.SeparateDatabaseAndState(
|
||||
database_operations=[
|
||||
migrations.RunPython(
|
||||
drop_resource_scan_summary_resource_id_index,
|
||||
reverse_code=migrations.RunPython.noop,
|
||||
),
|
||||
],
|
||||
state_operations=[
|
||||
migrations.AlterField(
|
||||
model_name="resourcescansummary",
|
||||
name="resource_id",
|
||||
field=models.UUIDField(default=uuid4),
|
||||
),
|
||||
],
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,31 @@
|
||||
from functools import partial
|
||||
|
||||
from django.db import migrations
|
||||
|
||||
from api.db_utils import create_index_on_partitions, drop_index_on_partitions
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
atomic = False
|
||||
|
||||
dependencies = [
|
||||
("api", "0072_drop_unused_indexes"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.RunPython(
|
||||
partial(
|
||||
create_index_on_partitions,
|
||||
parent_table="findings",
|
||||
index_name="find_tenant_scan_fail_new_idx",
|
||||
columns="tenant_id, scan_id",
|
||||
where="status = 'FAIL' AND delta = 'new'",
|
||||
all_partitions=True,
|
||||
),
|
||||
reverse_code=partial(
|
||||
drop_index_on_partitions,
|
||||
parent_table="findings",
|
||||
index_name="find_tenant_scan_fail_new_idx",
|
||||
),
|
||||
)
|
||||
]
|
||||
@@ -0,0 +1,54 @@
|
||||
from django.db import migrations, models
|
||||
|
||||
INDEX_NAME = "find_tenant_scan_fail_new_idx"
|
||||
PARENT_TABLE = "findings"
|
||||
|
||||
|
||||
def create_parent_and_attach(apps, schema_editor):
|
||||
with schema_editor.connection.cursor() as cursor:
|
||||
cursor.execute(
|
||||
f"CREATE INDEX {INDEX_NAME} ON ONLY {PARENT_TABLE} "
|
||||
f"USING btree (tenant_id, scan_id) "
|
||||
f"WHERE status = 'FAIL' AND delta = 'new'"
|
||||
)
|
||||
cursor.execute(
|
||||
"SELECT inhrelid::regclass::text "
|
||||
"FROM pg_inherits "
|
||||
"WHERE inhparent = %s::regclass",
|
||||
[PARENT_TABLE],
|
||||
)
|
||||
for (partition,) in cursor.fetchall():
|
||||
child_idx = f"{partition.replace('.', '_')}_{INDEX_NAME}"
|
||||
cursor.execute(f"ALTER INDEX {INDEX_NAME} ATTACH PARTITION {child_idx}")
|
||||
|
||||
|
||||
def drop_parent_index(apps, schema_editor):
|
||||
with schema_editor.connection.cursor() as cursor:
|
||||
cursor.execute(f"DROP INDEX IF EXISTS {INDEX_NAME}")
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("api", "0073_findings_fail_new_index_partitions"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.SeparateDatabaseAndState(
|
||||
state_operations=[
|
||||
migrations.AddIndex(
|
||||
model_name="finding",
|
||||
index=models.Index(
|
||||
condition=models.Q(status="FAIL", delta="new"),
|
||||
fields=["tenant_id", "scan_id"],
|
||||
name=INDEX_NAME,
|
||||
),
|
||||
),
|
||||
],
|
||||
database_operations=[
|
||||
migrations.RunPython(
|
||||
create_parent_and_attach,
|
||||
reverse_code=drop_parent_index,
|
||||
),
|
||||
],
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,38 @@
|
||||
# Generated by Django migration for Cloudflare provider support
|
||||
|
||||
from django.db import migrations
|
||||
|
||||
import api.db_utils
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("api", "0074_findings_fail_new_index_parent"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name="provider",
|
||||
name="provider",
|
||||
field=api.db_utils.ProviderEnumField(
|
||||
choices=[
|
||||
("aws", "AWS"),
|
||||
("azure", "Azure"),
|
||||
("gcp", "GCP"),
|
||||
("kubernetes", "Kubernetes"),
|
||||
("m365", "M365"),
|
||||
("github", "GitHub"),
|
||||
("mongodbatlas", "MongoDB Atlas"),
|
||||
("iac", "IaC"),
|
||||
("oraclecloud", "Oracle Cloud Infrastructure"),
|
||||
("alibabacloud", "Alibaba Cloud"),
|
||||
("cloudflare", "Cloudflare"),
|
||||
],
|
||||
default="aws",
|
||||
),
|
||||
),
|
||||
migrations.RunSQL(
|
||||
"ALTER TYPE provider ADD VALUE IF NOT EXISTS 'cloudflare';",
|
||||
reverse_sql=migrations.RunSQL.noop,
|
||||
),
|
||||
]
|
||||
+181
-30
@@ -12,7 +12,6 @@ from cryptography.fernet import Fernet, InvalidToken
|
||||
from django.conf import settings
|
||||
from django.contrib.auth.models import AbstractBaseUser
|
||||
from django.contrib.postgres.fields import ArrayField
|
||||
from django.contrib.postgres.indexes import GinIndex
|
||||
from django.contrib.postgres.search import SearchVector, SearchVectorField
|
||||
from django.contrib.sites.models import Site
|
||||
from django.core.exceptions import ValidationError
|
||||
@@ -288,6 +287,7 @@ class Provider(RowLevelSecurityProtectedModel):
|
||||
IAC = "iac", _("IaC")
|
||||
ORACLECLOUD = "oraclecloud", _("Oracle Cloud Infrastructure")
|
||||
ALIBABACLOUD = "alibabacloud", _("Alibaba Cloud")
|
||||
CLOUDFLARE = "cloudflare", _("Cloudflare")
|
||||
|
||||
@staticmethod
|
||||
def validate_aws_uid(value):
|
||||
@@ -401,6 +401,15 @@ class Provider(RowLevelSecurityProtectedModel):
|
||||
pointer="/data/attributes/uid",
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def validate_cloudflare_uid(value):
|
||||
if not re.match(r"^[a-f0-9]{32}$", value):
|
||||
raise ModelValidationError(
|
||||
detail="Cloudflare Account ID must be a 32-character hexadecimal string.",
|
||||
code="cloudflare-uid",
|
||||
pointer="/data/attributes/uid",
|
||||
)
|
||||
|
||||
id = models.UUIDField(primary_key=True, default=uuid4, editable=False)
|
||||
inserted_at = models.DateTimeField(auto_now_add=True, editable=False)
|
||||
updated_at = models.DateTimeField(auto_now=True, editable=False)
|
||||
@@ -626,6 +635,101 @@ class Scan(RowLevelSecurityProtectedModel):
|
||||
resource_name = "scans"
|
||||
|
||||
|
||||
class AttackPathsScan(RowLevelSecurityProtectedModel):
|
||||
objects = ActiveProviderManager()
|
||||
all_objects = models.Manager()
|
||||
|
||||
id = models.UUIDField(primary_key=True, default=uuid7, editable=False)
|
||||
inserted_at = models.DateTimeField(auto_now_add=True, editable=False)
|
||||
updated_at = models.DateTimeField(auto_now=True, editable=False)
|
||||
|
||||
state = StateEnumField(choices=StateChoices.choices, default=StateChoices.AVAILABLE)
|
||||
progress = models.IntegerField(default=0)
|
||||
|
||||
# Timing
|
||||
started_at = models.DateTimeField(null=True, blank=True)
|
||||
completed_at = models.DateTimeField(null=True, blank=True)
|
||||
duration = models.IntegerField(
|
||||
null=True, blank=True, help_text="Duration in seconds"
|
||||
)
|
||||
|
||||
# Relationship to the provider and optional prowler Scan and celery Task
|
||||
provider = models.ForeignKey(
|
||||
"Provider",
|
||||
on_delete=models.CASCADE,
|
||||
related_name="attack_paths_scans",
|
||||
related_query_name="attack_paths_scan",
|
||||
)
|
||||
scan = models.ForeignKey(
|
||||
"Scan",
|
||||
on_delete=models.SET_NULL,
|
||||
null=True,
|
||||
blank=True,
|
||||
related_name="attack_paths_scans",
|
||||
related_query_name="attack_paths_scan",
|
||||
)
|
||||
task = models.ForeignKey(
|
||||
"Task",
|
||||
on_delete=models.SET_NULL,
|
||||
null=True,
|
||||
blank=True,
|
||||
related_name="attack_paths_scans",
|
||||
related_query_name="attack_paths_scan",
|
||||
)
|
||||
|
||||
# Cartography specific metadata
|
||||
update_tag = models.BigIntegerField(
|
||||
null=True, blank=True, help_text="Cartography update tag (epoch)"
|
||||
)
|
||||
graph_database = models.CharField(max_length=63, null=True, blank=True)
|
||||
is_graph_database_deleted = models.BooleanField(default=False)
|
||||
ingestion_exceptions = models.JSONField(default=dict, null=True, blank=True)
|
||||
|
||||
class Meta(RowLevelSecurityProtectedModel.Meta):
|
||||
db_table = "attack_paths_scans"
|
||||
|
||||
constraints = [
|
||||
RowLevelSecurityConstraint(
|
||||
field="tenant_id",
|
||||
name="rls_on_%(class)s",
|
||||
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
|
||||
),
|
||||
]
|
||||
|
||||
indexes = [
|
||||
models.Index(
|
||||
fields=["tenant_id", "provider_id", "-inserted_at"],
|
||||
name="aps_prov_ins_desc_idx",
|
||||
),
|
||||
models.Index(
|
||||
fields=["tenant_id", "state", "-inserted_at"],
|
||||
name="aps_state_ins_desc_idx",
|
||||
),
|
||||
models.Index(
|
||||
fields=["tenant_id", "scan_id"],
|
||||
name="aps_scan_lookup_idx",
|
||||
),
|
||||
models.Index(
|
||||
fields=["tenant_id", "provider_id"],
|
||||
name="aps_active_graph_idx",
|
||||
include=["graph_database", "id"],
|
||||
condition=Q(is_graph_database_deleted=False),
|
||||
),
|
||||
models.Index(
|
||||
fields=["tenant_id", "provider_id", "-completed_at"],
|
||||
name="aps_completed_graph_idx",
|
||||
include=["graph_database", "id"],
|
||||
condition=Q(
|
||||
state=StateChoices.COMPLETED,
|
||||
is_graph_database_deleted=False,
|
||||
),
|
||||
),
|
||||
]
|
||||
|
||||
class JSONAPIMeta:
|
||||
resource_name = "attack-paths-scans"
|
||||
|
||||
|
||||
class ResourceTag(RowLevelSecurityProtectedModel):
|
||||
id = models.UUIDField(primary_key=True, default=uuid4, editable=False)
|
||||
inserted_at = models.DateTimeField(auto_now_add=True, editable=False)
|
||||
@@ -646,10 +750,6 @@ class ResourceTag(RowLevelSecurityProtectedModel):
|
||||
class Meta(RowLevelSecurityProtectedModel.Meta):
|
||||
db_table = "resource_tags"
|
||||
|
||||
indexes = [
|
||||
GinIndex(fields=["text_search"], name="gin_resource_tags_search_idx"),
|
||||
]
|
||||
|
||||
constraints = [
|
||||
models.UniqueConstraint(
|
||||
fields=("tenant_id", "key", "value"),
|
||||
@@ -704,6 +804,12 @@ class Resource(RowLevelSecurityProtectedModel):
|
||||
metadata = models.TextField(blank=True, null=True)
|
||||
details = models.TextField(blank=True, null=True)
|
||||
partition = models.TextField(blank=True, null=True)
|
||||
groups = ArrayField(
|
||||
models.CharField(max_length=100),
|
||||
blank=True,
|
||||
null=True,
|
||||
help_text="Groups for categorization (e.g., compute, storage, IAM)",
|
||||
)
|
||||
|
||||
failed_findings_count = models.IntegerField(default=0)
|
||||
|
||||
@@ -752,7 +858,6 @@ class Resource(RowLevelSecurityProtectedModel):
|
||||
fields=["tenant_id", "service", "region", "type"],
|
||||
name="resource_tenant_metadata_idx",
|
||||
),
|
||||
GinIndex(fields=["text_search"], name="gin_resources_search_idx"),
|
||||
models.Index(fields=["tenant_id", "id"], name="resources_tenant_id_idx"),
|
||||
models.Index(
|
||||
fields=["tenant_id", "provider_id"],
|
||||
@@ -890,6 +995,11 @@ class Finding(PostgresPartitionedModel, RowLevelSecurityProtectedModel):
|
||||
null=True,
|
||||
help_text="Categories from check metadata for efficient filtering",
|
||||
)
|
||||
resource_groups = models.TextField(
|
||||
blank=True,
|
||||
null=True,
|
||||
help_text="Resource group from check metadata for efficient filtering",
|
||||
)
|
||||
|
||||
# Relationships
|
||||
scan = models.ForeignKey(to=Scan, related_name="findings", on_delete=models.CASCADE)
|
||||
@@ -932,23 +1042,19 @@ class Finding(PostgresPartitionedModel, RowLevelSecurityProtectedModel):
|
||||
|
||||
indexes = [
|
||||
models.Index(fields=["tenant_id", "id"], name="findings_tenant_and_id_idx"),
|
||||
GinIndex(fields=["text_search"], name="gin_findings_search_idx"),
|
||||
models.Index(fields=["tenant_id", "scan_id"], name="find_tenant_scan_idx"),
|
||||
models.Index(
|
||||
fields=["tenant_id", "scan_id", "id"], name="find_tenant_scan_id_idx"
|
||||
),
|
||||
models.Index(
|
||||
fields=["tenant_id", "id"],
|
||||
condition=Q(delta="new"),
|
||||
name="find_delta_new_idx",
|
||||
condition=models.Q(status=StatusChoices.FAIL, delta="new"),
|
||||
fields=["tenant_id", "scan_id"],
|
||||
name="find_tenant_scan_fail_new_idx",
|
||||
),
|
||||
models.Index(
|
||||
fields=["tenant_id", "uid", "-inserted_at"],
|
||||
name="find_tenant_uid_inserted_idx",
|
||||
),
|
||||
GinIndex(fields=["resource_services"], name="gin_find_service_idx"),
|
||||
GinIndex(fields=["resource_regions"], name="gin_find_region_idx"),
|
||||
GinIndex(fields=["resource_types"], name="gin_find_rtype_idx"),
|
||||
models.Index(
|
||||
fields=["tenant_id", "scan_id", "check_id"],
|
||||
name="find_tenant_scan_check_idx",
|
||||
@@ -1016,10 +1122,6 @@ class ResourceFindingMapping(PostgresPartitionedModel, RowLevelSecurityProtected
|
||||
# - id
|
||||
|
||||
indexes = [
|
||||
models.Index(
|
||||
fields=["tenant_id", "finding_id"],
|
||||
name="rfm_tenant_finding_idx",
|
||||
),
|
||||
models.Index(
|
||||
fields=["tenant_id", "resource_id"],
|
||||
name="rfm_tenant_resource_idx",
|
||||
@@ -1336,14 +1438,6 @@ class ComplianceOverview(RowLevelSecurityProtectedModel):
|
||||
statements=["SELECT", "INSERT", "DELETE"],
|
||||
),
|
||||
]
|
||||
indexes = [
|
||||
models.Index(fields=["compliance_id"], name="comp_ov_cp_id_idx"),
|
||||
models.Index(fields=["requirements_failed"], name="comp_ov_req_fail_idx"),
|
||||
models.Index(
|
||||
fields=["compliance_id", "requirements_failed"],
|
||||
name="comp_ov_cp_id_req_fail_idx",
|
||||
),
|
||||
]
|
||||
|
||||
class JSONAPIMeta:
|
||||
resource_name = "compliance-overviews"
|
||||
@@ -1509,10 +1603,6 @@ class ScanSummary(RowLevelSecurityProtectedModel):
|
||||
fields=["tenant_id", "scan_id"],
|
||||
name="scan_summaries_tenant_scan_idx",
|
||||
),
|
||||
models.Index(
|
||||
fields=["tenant_id", "scan_id", "service"],
|
||||
name="ss_tenant_scan_service_idx",
|
||||
),
|
||||
models.Index(
|
||||
fields=["tenant_id", "scan_id", "severity"],
|
||||
name="ss_tenant_scan_severity_idx",
|
||||
@@ -1927,7 +2017,7 @@ class SAMLConfiguration(RowLevelSecurityProtectedModel):
|
||||
|
||||
class ResourceScanSummary(RowLevelSecurityProtectedModel):
|
||||
scan_id = models.UUIDField(default=uuid7, db_index=True)
|
||||
resource_id = models.UUIDField(default=uuid4, db_index=True)
|
||||
resource_id = models.UUIDField(default=uuid4)
|
||||
service = models.CharField(max_length=100)
|
||||
region = models.CharField(max_length=100)
|
||||
resource_type = models.CharField(max_length=100)
|
||||
@@ -2032,6 +2122,67 @@ class ScanCategorySummary(RowLevelSecurityProtectedModel):
|
||||
resource_name = "scan-category-summaries"
|
||||
|
||||
|
||||
class ScanGroupSummary(RowLevelSecurityProtectedModel):
|
||||
"""
|
||||
Pre-aggregated resource group metrics per scan by severity.
|
||||
|
||||
Stores one row per (resource_group, severity) combination per scan for efficient
|
||||
overview queries. Resource groups come from check_metadata.Group.
|
||||
|
||||
Count relationships (each is a subset of the previous):
|
||||
- total_findings >= failed_findings >= new_failed_findings
|
||||
"""
|
||||
|
||||
id = models.UUIDField(primary_key=True, default=uuid4, editable=False)
|
||||
inserted_at = models.DateTimeField(auto_now_add=True, editable=False)
|
||||
|
||||
scan = models.ForeignKey(
|
||||
Scan,
|
||||
on_delete=models.CASCADE,
|
||||
related_name="resource_group_summaries",
|
||||
related_query_name="resource_group_summary",
|
||||
)
|
||||
|
||||
resource_group = models.CharField(max_length=50)
|
||||
severity = SeverityEnumField(choices=SeverityChoices)
|
||||
|
||||
total_findings = models.IntegerField(
|
||||
default=0, help_text="Non-muted findings (PASS + FAIL)"
|
||||
)
|
||||
failed_findings = models.IntegerField(
|
||||
default=0, help_text="Non-muted FAIL findings (subset of total_findings)"
|
||||
)
|
||||
new_failed_findings = models.IntegerField(
|
||||
default=0,
|
||||
help_text="Non-muted FAIL with delta='new' (subset of failed_findings)",
|
||||
)
|
||||
resources_count = models.IntegerField(
|
||||
default=0, help_text="Count of distinct resource_uid values"
|
||||
)
|
||||
|
||||
class Meta(RowLevelSecurityProtectedModel.Meta):
|
||||
db_table = "scan_resource_group_summaries"
|
||||
|
||||
indexes = [
|
||||
models.Index(fields=["tenant_id", "scan"], name="srgs_tenant_scan_idx"),
|
||||
]
|
||||
|
||||
constraints = [
|
||||
models.UniqueConstraint(
|
||||
fields=("tenant_id", "scan_id", "resource_group", "severity"),
|
||||
name="unique_resource_group_severity_per_scan",
|
||||
),
|
||||
RowLevelSecurityConstraint(
|
||||
field="tenant_id",
|
||||
name="rls_on_%(class)s",
|
||||
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
|
||||
),
|
||||
]
|
||||
|
||||
class JSONAPIMeta:
|
||||
resource_name = "scan-resource-group-summaries"
|
||||
|
||||
|
||||
class LighthouseConfiguration(RowLevelSecurityProtectedModel):
|
||||
"""
|
||||
Stores configuration and API keys for LLM services.
|
||||
|
||||
+1745
-54
File diff suppressed because it is too large
Load Diff
@@ -1,10 +1,13 @@
|
||||
import os
|
||||
import sys
|
||||
import types
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from django.conf import settings
|
||||
|
||||
import api
|
||||
import api.apps as api_apps_module
|
||||
from api.apps import (
|
||||
ApiConfig,
|
||||
@@ -150,3 +153,82 @@ def test_ensure_crypto_keys_skips_when_env_vars(monkeypatch, tmp_path):
|
||||
|
||||
# Assert: orchestrator did not trigger generation when env present
|
||||
assert called["ensure"] is False
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def stub_api_modules():
|
||||
"""Provide dummy modules imported during ApiConfig.ready()."""
|
||||
created = []
|
||||
for name in ("api.schema_extensions", "api.signals"):
|
||||
if name not in sys.modules:
|
||||
sys.modules[name] = types.ModuleType(name)
|
||||
created.append(name)
|
||||
|
||||
yield
|
||||
|
||||
for name in created:
|
||||
sys.modules.pop(name, None)
|
||||
|
||||
|
||||
def _set_argv(monkeypatch, argv):
|
||||
monkeypatch.setattr(sys, "argv", argv, raising=False)
|
||||
|
||||
|
||||
def _set_testing(monkeypatch, value):
|
||||
monkeypatch.setattr(settings, "TESTING", value, raising=False)
|
||||
|
||||
|
||||
def _make_app():
|
||||
return ApiConfig("api", api)
|
||||
|
||||
|
||||
def test_ready_initializes_driver_for_api_process(monkeypatch):
|
||||
config = _make_app()
|
||||
_set_argv(monkeypatch, ["gunicorn"])
|
||||
_set_testing(monkeypatch, False)
|
||||
|
||||
with patch.object(ApiConfig, "_ensure_crypto_keys", return_value=None), patch(
|
||||
"api.attack_paths.database.init_driver"
|
||||
) as init_driver:
|
||||
config.ready()
|
||||
|
||||
init_driver.assert_called_once()
|
||||
|
||||
|
||||
def test_ready_skips_driver_for_celery(monkeypatch):
|
||||
config = _make_app()
|
||||
_set_argv(monkeypatch, ["celery", "-A", "api"])
|
||||
_set_testing(monkeypatch, False)
|
||||
|
||||
with patch.object(ApiConfig, "_ensure_crypto_keys", return_value=None), patch(
|
||||
"api.attack_paths.database.init_driver"
|
||||
) as init_driver:
|
||||
config.ready()
|
||||
|
||||
init_driver.assert_not_called()
|
||||
|
||||
|
||||
def test_ready_skips_driver_for_manage_py_skip_command(monkeypatch):
|
||||
config = _make_app()
|
||||
_set_argv(monkeypatch, ["manage.py", "migrate"])
|
||||
_set_testing(monkeypatch, False)
|
||||
|
||||
with patch.object(ApiConfig, "_ensure_crypto_keys", return_value=None), patch(
|
||||
"api.attack_paths.database.init_driver"
|
||||
) as init_driver:
|
||||
config.ready()
|
||||
|
||||
init_driver.assert_not_called()
|
||||
|
||||
|
||||
def test_ready_skips_driver_when_testing(monkeypatch):
|
||||
config = _make_app()
|
||||
_set_argv(monkeypatch, ["gunicorn"])
|
||||
_set_testing(monkeypatch, True)
|
||||
|
||||
with patch.object(ApiConfig, "_ensure_crypto_keys", return_value=None), patch(
|
||||
"api.attack_paths.database.init_driver"
|
||||
) as init_driver:
|
||||
config.ready()
|
||||
|
||||
init_driver.assert_not_called()
|
||||
|
||||
@@ -0,0 +1,174 @@
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from rest_framework.exceptions import APIException, ValidationError
|
||||
|
||||
from api.attack_paths import database as graph_database
|
||||
from api.attack_paths import views_helpers
|
||||
|
||||
|
||||
def test_normalize_run_payload_extracts_attributes_section():
|
||||
payload = {
|
||||
"data": {
|
||||
"id": "ignored",
|
||||
"attributes": {
|
||||
"id": "aws-rds",
|
||||
"parameters": {"ip": "192.0.2.0"},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
result = views_helpers.normalize_run_payload(payload)
|
||||
|
||||
assert result == {"id": "aws-rds", "parameters": {"ip": "192.0.2.0"}}
|
||||
|
||||
|
||||
def test_normalize_run_payload_passthrough_for_non_dict():
|
||||
sentinel = "not-a-dict"
|
||||
assert views_helpers.normalize_run_payload(sentinel) is sentinel
|
||||
|
||||
|
||||
def test_prepare_query_parameters_includes_provider_and_casts(
|
||||
attack_paths_query_definition_factory,
|
||||
):
|
||||
definition = attack_paths_query_definition_factory(cast_type=int)
|
||||
result = views_helpers.prepare_query_parameters(
|
||||
definition,
|
||||
{"limit": "5"},
|
||||
provider_uid="123456789012",
|
||||
)
|
||||
|
||||
assert result["provider_uid"] == "123456789012"
|
||||
assert result["limit"] == 5
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"provided,expected_message",
|
||||
[
|
||||
({}, "Missing required parameter"),
|
||||
({"limit": 10, "extra": True}, "Unknown parameter"),
|
||||
],
|
||||
)
|
||||
def test_prepare_query_parameters_validates_names(
|
||||
attack_paths_query_definition_factory, provided, expected_message
|
||||
):
|
||||
definition = attack_paths_query_definition_factory()
|
||||
|
||||
with pytest.raises(ValidationError) as exc:
|
||||
views_helpers.prepare_query_parameters(definition, provided, provider_uid="1")
|
||||
|
||||
assert expected_message in str(exc.value)
|
||||
|
||||
|
||||
def test_prepare_query_parameters_validates_cast(
|
||||
attack_paths_query_definition_factory,
|
||||
):
|
||||
definition = attack_paths_query_definition_factory(cast_type=int)
|
||||
|
||||
with pytest.raises(ValidationError) as exc:
|
||||
views_helpers.prepare_query_parameters(
|
||||
definition,
|
||||
{"limit": "not-an-int"},
|
||||
provider_uid="1",
|
||||
)
|
||||
|
||||
assert "Invalid value" in str(exc.value)
|
||||
|
||||
|
||||
def test_execute_attack_paths_query_serializes_graph(
|
||||
attack_paths_query_definition_factory, attack_paths_graph_stub_classes
|
||||
):
|
||||
definition = attack_paths_query_definition_factory(
|
||||
id="aws-rds",
|
||||
name="RDS",
|
||||
short_description="Short desc",
|
||||
description="",
|
||||
cypher="MATCH (n) RETURN n",
|
||||
parameters=[],
|
||||
)
|
||||
parameters = {"provider_uid": "123"}
|
||||
attack_paths_scan = SimpleNamespace(graph_database="tenant-db")
|
||||
|
||||
node = attack_paths_graph_stub_classes.Node(
|
||||
element_id="node-1",
|
||||
labels=["AWSAccount"],
|
||||
properties={
|
||||
"name": "account",
|
||||
"complex": {
|
||||
"items": [
|
||||
attack_paths_graph_stub_classes.NativeValue("value"),
|
||||
{"nested": 1},
|
||||
]
|
||||
},
|
||||
},
|
||||
)
|
||||
relationship = attack_paths_graph_stub_classes.Relationship(
|
||||
element_id="rel-1",
|
||||
rel_type="OWNS",
|
||||
start_node=node,
|
||||
end_node=attack_paths_graph_stub_classes.Node("node-2", ["RDSInstance"], {}),
|
||||
properties={"weight": 1},
|
||||
)
|
||||
graph = SimpleNamespace(nodes=[node], relationships=[relationship])
|
||||
|
||||
run_result = MagicMock()
|
||||
run_result.graph.return_value = graph
|
||||
|
||||
session = MagicMock()
|
||||
session.run.return_value = run_result
|
||||
|
||||
session_ctx = MagicMock()
|
||||
session_ctx.__enter__.return_value = session
|
||||
session_ctx.__exit__.return_value = False
|
||||
|
||||
with patch(
|
||||
"api.attack_paths.views_helpers.graph_database.get_session",
|
||||
return_value=session_ctx,
|
||||
) as mock_get_session:
|
||||
result = views_helpers.execute_attack_paths_query(
|
||||
attack_paths_scan, definition, parameters
|
||||
)
|
||||
|
||||
mock_get_session.assert_called_once_with("tenant-db")
|
||||
session.run.assert_called_once_with(definition.cypher, parameters)
|
||||
assert result["nodes"][0]["id"] == "node-1"
|
||||
assert result["nodes"][0]["properties"]["complex"]["items"][0] == "value"
|
||||
assert result["relationships"][0]["label"] == "OWNS"
|
||||
|
||||
|
||||
def test_execute_attack_paths_query_wraps_graph_errors(
|
||||
attack_paths_query_definition_factory,
|
||||
):
|
||||
definition = attack_paths_query_definition_factory(
|
||||
id="aws-rds",
|
||||
name="RDS",
|
||||
short_description="Short desc",
|
||||
description="",
|
||||
cypher="MATCH (n) RETURN n",
|
||||
parameters=[],
|
||||
)
|
||||
attack_paths_scan = SimpleNamespace(graph_database="tenant-db")
|
||||
parameters = {"provider_uid": "123"}
|
||||
|
||||
class ExplodingContext:
|
||||
def __enter__(self):
|
||||
raise graph_database.GraphDatabaseQueryException("boom")
|
||||
|
||||
def __exit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
with (
|
||||
patch(
|
||||
"api.attack_paths.views_helpers.graph_database.get_session",
|
||||
return_value=ExplodingContext(),
|
||||
),
|
||||
patch("api.attack_paths.views_helpers.logger") as mock_logger,
|
||||
):
|
||||
with pytest.raises(APIException):
|
||||
views_helpers.execute_attack_paths_query(
|
||||
attack_paths_scan, definition, parameters
|
||||
)
|
||||
|
||||
mock_logger.error.assert_called_once()
|
||||
@@ -0,0 +1,303 @@
|
||||
"""
|
||||
Tests for Neo4j database lazy initialization.
|
||||
|
||||
The Neo4j driver connects on first use by default. API processes may
|
||||
eagerly initialize the driver during app startup, while Celery workers
|
||||
remain lazy. These tests validate the database module behavior itself.
|
||||
"""
|
||||
|
||||
import threading
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
class TestLazyInitialization:
|
||||
"""Test that Neo4j driver is initialized lazily on first use."""
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def reset_module_state(self):
|
||||
"""Reset module-level singleton state before each test."""
|
||||
import api.attack_paths.database as db_module
|
||||
|
||||
original_driver = db_module._driver
|
||||
|
||||
db_module._driver = None
|
||||
|
||||
yield
|
||||
|
||||
db_module._driver = original_driver
|
||||
|
||||
def test_driver_not_initialized_at_import(self):
|
||||
"""Driver should be None after module import (no eager connection)."""
|
||||
import api.attack_paths.database as db_module
|
||||
|
||||
assert db_module._driver is None
|
||||
|
||||
@patch("api.attack_paths.database.settings")
|
||||
@patch("api.attack_paths.database.neo4j.GraphDatabase.driver")
|
||||
def test_init_driver_creates_connection_on_first_call(
|
||||
self, mock_driver_factory, mock_settings
|
||||
):
|
||||
"""init_driver() should create connection only when called."""
|
||||
import api.attack_paths.database as db_module
|
||||
|
||||
mock_driver = MagicMock()
|
||||
mock_driver_factory.return_value = mock_driver
|
||||
mock_settings.DATABASES = {
|
||||
"neo4j": {
|
||||
"HOST": "localhost",
|
||||
"PORT": 7687,
|
||||
"USER": "neo4j",
|
||||
"PASSWORD": "password",
|
||||
}
|
||||
}
|
||||
|
||||
assert db_module._driver is None
|
||||
|
||||
result = db_module.init_driver()
|
||||
|
||||
mock_driver_factory.assert_called_once()
|
||||
mock_driver.verify_connectivity.assert_called_once()
|
||||
assert result is mock_driver
|
||||
assert db_module._driver is mock_driver
|
||||
|
||||
@patch("api.attack_paths.database.settings")
|
||||
@patch("api.attack_paths.database.neo4j.GraphDatabase.driver")
|
||||
def test_init_driver_returns_cached_driver_on_subsequent_calls(
|
||||
self, mock_driver_factory, mock_settings
|
||||
):
|
||||
"""Subsequent calls should return cached driver without reconnecting."""
|
||||
import api.attack_paths.database as db_module
|
||||
|
||||
mock_driver = MagicMock()
|
||||
mock_driver_factory.return_value = mock_driver
|
||||
mock_settings.DATABASES = {
|
||||
"neo4j": {
|
||||
"HOST": "localhost",
|
||||
"PORT": 7687,
|
||||
"USER": "neo4j",
|
||||
"PASSWORD": "password",
|
||||
}
|
||||
}
|
||||
|
||||
first_result = db_module.init_driver()
|
||||
second_result = db_module.init_driver()
|
||||
third_result = db_module.init_driver()
|
||||
|
||||
# Only one connection attempt
|
||||
assert mock_driver_factory.call_count == 1
|
||||
assert mock_driver.verify_connectivity.call_count == 1
|
||||
|
||||
# All calls return same instance
|
||||
assert first_result is second_result is third_result
|
||||
|
||||
@patch("api.attack_paths.database.settings")
|
||||
@patch("api.attack_paths.database.neo4j.GraphDatabase.driver")
|
||||
def test_get_driver_delegates_to_init_driver(
|
||||
self, mock_driver_factory, mock_settings
|
||||
):
|
||||
"""get_driver() should use init_driver() for lazy initialization."""
|
||||
import api.attack_paths.database as db_module
|
||||
|
||||
mock_driver = MagicMock()
|
||||
mock_driver_factory.return_value = mock_driver
|
||||
mock_settings.DATABASES = {
|
||||
"neo4j": {
|
||||
"HOST": "localhost",
|
||||
"PORT": 7687,
|
||||
"USER": "neo4j",
|
||||
"PASSWORD": "password",
|
||||
}
|
||||
}
|
||||
|
||||
result = db_module.get_driver()
|
||||
|
||||
assert result is mock_driver
|
||||
mock_driver_factory.assert_called_once()
|
||||
|
||||
|
||||
class TestAtexitRegistration:
|
||||
"""Test that atexit cleanup handler is registered correctly."""
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def reset_module_state(self):
|
||||
"""Reset module-level singleton state before each test."""
|
||||
import api.attack_paths.database as db_module
|
||||
|
||||
original_driver = db_module._driver
|
||||
|
||||
db_module._driver = None
|
||||
|
||||
yield
|
||||
|
||||
db_module._driver = original_driver
|
||||
|
||||
@patch("api.attack_paths.database.settings")
|
||||
@patch("api.attack_paths.database.atexit.register")
|
||||
@patch("api.attack_paths.database.neo4j.GraphDatabase.driver")
|
||||
def test_atexit_registered_on_first_init(
|
||||
self, mock_driver_factory, mock_atexit_register, mock_settings
|
||||
):
|
||||
"""atexit.register should be called on first initialization."""
|
||||
import api.attack_paths.database as db_module
|
||||
|
||||
mock_driver_factory.return_value = MagicMock()
|
||||
mock_settings.DATABASES = {
|
||||
"neo4j": {
|
||||
"HOST": "localhost",
|
||||
"PORT": 7687,
|
||||
"USER": "neo4j",
|
||||
"PASSWORD": "password",
|
||||
}
|
||||
}
|
||||
|
||||
db_module.init_driver()
|
||||
|
||||
mock_atexit_register.assert_called_once_with(db_module.close_driver)
|
||||
|
||||
@patch("api.attack_paths.database.settings")
|
||||
@patch("api.attack_paths.database.atexit.register")
|
||||
@patch("api.attack_paths.database.neo4j.GraphDatabase.driver")
|
||||
def test_atexit_registered_only_once(
|
||||
self, mock_driver_factory, mock_atexit_register, mock_settings
|
||||
):
|
||||
"""atexit.register should only be called once across multiple inits.
|
||||
|
||||
The double-checked locking on _driver ensures the atexit registration
|
||||
block only executes once (when _driver is first created).
|
||||
"""
|
||||
import api.attack_paths.database as db_module
|
||||
|
||||
mock_driver_factory.return_value = MagicMock()
|
||||
mock_settings.DATABASES = {
|
||||
"neo4j": {
|
||||
"HOST": "localhost",
|
||||
"PORT": 7687,
|
||||
"USER": "neo4j",
|
||||
"PASSWORD": "password",
|
||||
}
|
||||
}
|
||||
|
||||
db_module.init_driver()
|
||||
db_module.init_driver()
|
||||
db_module.init_driver()
|
||||
|
||||
# Only registered once because subsequent calls hit the fast path
|
||||
assert mock_atexit_register.call_count == 1
|
||||
|
||||
|
||||
class TestCloseDriver:
|
||||
"""Test driver cleanup functionality."""
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def reset_module_state(self):
|
||||
"""Reset module-level singleton state before each test."""
|
||||
import api.attack_paths.database as db_module
|
||||
|
||||
original_driver = db_module._driver
|
||||
|
||||
db_module._driver = None
|
||||
|
||||
yield
|
||||
|
||||
db_module._driver = original_driver
|
||||
|
||||
def test_close_driver_closes_and_clears_driver(self):
|
||||
"""close_driver() should close the driver and set it to None."""
|
||||
import api.attack_paths.database as db_module
|
||||
|
||||
mock_driver = MagicMock()
|
||||
db_module._driver = mock_driver
|
||||
|
||||
db_module.close_driver()
|
||||
|
||||
mock_driver.close.assert_called_once()
|
||||
assert db_module._driver is None
|
||||
|
||||
def test_close_driver_handles_none_driver(self):
|
||||
"""close_driver() should handle case where driver is None."""
|
||||
import api.attack_paths.database as db_module
|
||||
|
||||
db_module._driver = None
|
||||
|
||||
# Should not raise
|
||||
db_module.close_driver()
|
||||
|
||||
assert db_module._driver is None
|
||||
|
||||
def test_close_driver_clears_driver_even_on_close_error(self):
|
||||
"""Driver should be cleared even if close() raises an exception."""
|
||||
import api.attack_paths.database as db_module
|
||||
|
||||
mock_driver = MagicMock()
|
||||
mock_driver.close.side_effect = Exception("Connection error")
|
||||
db_module._driver = mock_driver
|
||||
|
||||
with pytest.raises(Exception, match="Connection error"):
|
||||
db_module.close_driver()
|
||||
|
||||
# Driver should still be cleared
|
||||
assert db_module._driver is None
|
||||
|
||||
|
||||
class TestThreadSafety:
|
||||
"""Test thread-safe initialization."""
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def reset_module_state(self):
|
||||
"""Reset module-level singleton state before each test."""
|
||||
import api.attack_paths.database as db_module
|
||||
|
||||
original_driver = db_module._driver
|
||||
|
||||
db_module._driver = None
|
||||
|
||||
yield
|
||||
|
||||
db_module._driver = original_driver
|
||||
|
||||
@patch("api.attack_paths.database.settings")
|
||||
@patch("api.attack_paths.database.neo4j.GraphDatabase.driver")
|
||||
def test_concurrent_init_creates_single_driver(
|
||||
self, mock_driver_factory, mock_settings
|
||||
):
|
||||
"""Multiple threads calling init_driver() should create only one driver."""
|
||||
import api.attack_paths.database as db_module
|
||||
|
||||
mock_driver = MagicMock()
|
||||
mock_driver_factory.return_value = mock_driver
|
||||
mock_settings.DATABASES = {
|
||||
"neo4j": {
|
||||
"HOST": "localhost",
|
||||
"PORT": 7687,
|
||||
"USER": "neo4j",
|
||||
"PASSWORD": "password",
|
||||
}
|
||||
}
|
||||
|
||||
results = []
|
||||
errors = []
|
||||
|
||||
def call_init():
|
||||
try:
|
||||
result = db_module.init_driver()
|
||||
results.append(result)
|
||||
except Exception as e:
|
||||
errors.append(e)
|
||||
|
||||
threads = [threading.Thread(target=call_init) for _ in range(10)]
|
||||
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join()
|
||||
|
||||
assert not errors, f"Threads raised errors: {errors}"
|
||||
|
||||
# Only one driver created
|
||||
assert mock_driver_factory.call_count == 1
|
||||
|
||||
# All threads got the same driver instance
|
||||
assert all(r is mock_driver for r in results)
|
||||
assert len(results) == 10
|
||||
@@ -6,7 +6,6 @@ from api.compliance import (
|
||||
get_prowler_provider_checks,
|
||||
get_prowler_provider_compliance,
|
||||
load_prowler_checks,
|
||||
load_prowler_compliance,
|
||||
)
|
||||
from api.models import Provider
|
||||
|
||||
@@ -35,55 +34,6 @@ class TestCompliance:
|
||||
assert compliance_data == mock_compliance.get_bulk.return_value
|
||||
mock_compliance.get_bulk.assert_called_once_with(provider_type)
|
||||
|
||||
@patch("api.models.Provider.ProviderChoices")
|
||||
@patch("api.compliance.get_prowler_provider_compliance")
|
||||
@patch("api.compliance.generate_compliance_overview_template")
|
||||
@patch("api.compliance.load_prowler_checks")
|
||||
def test_load_prowler_compliance(
|
||||
self,
|
||||
mock_load_prowler_checks,
|
||||
mock_generate_compliance_overview_template,
|
||||
mock_get_prowler_provider_compliance,
|
||||
mock_provider_choices,
|
||||
):
|
||||
mock_provider_choices.values = ["aws", "azure"]
|
||||
|
||||
compliance_data_aws = {"compliance_aws": MagicMock()}
|
||||
compliance_data_azure = {"compliance_azure": MagicMock()}
|
||||
|
||||
compliance_data_dict = {
|
||||
"aws": compliance_data_aws,
|
||||
"azure": compliance_data_azure,
|
||||
}
|
||||
|
||||
def mock_get_compliance(provider_type):
|
||||
return compliance_data_dict[provider_type]
|
||||
|
||||
mock_get_prowler_provider_compliance.side_effect = mock_get_compliance
|
||||
|
||||
mock_generate_compliance_overview_template.return_value = {
|
||||
"template_key": "template_value"
|
||||
}
|
||||
|
||||
mock_load_prowler_checks.return_value = {"checks_key": "checks_value"}
|
||||
|
||||
load_prowler_compliance()
|
||||
|
||||
from api.compliance import PROWLER_CHECKS, PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE
|
||||
|
||||
assert PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE == {
|
||||
"template_key": "template_value"
|
||||
}
|
||||
assert PROWLER_CHECKS == {"checks_key": "checks_value"}
|
||||
|
||||
expected_prowler_compliance = compliance_data_dict
|
||||
mock_get_prowler_provider_compliance.assert_any_call("aws")
|
||||
mock_get_prowler_provider_compliance.assert_any_call("azure")
|
||||
mock_generate_compliance_overview_template.assert_called_once_with(
|
||||
expected_prowler_compliance
|
||||
)
|
||||
mock_load_prowler_checks.assert_called_once_with(expected_prowler_compliance)
|
||||
|
||||
@patch("api.compliance.get_prowler_provider_checks")
|
||||
@patch("api.models.Provider.ProviderChoices")
|
||||
def test_load_prowler_checks(
|
||||
|
||||
@@ -20,6 +20,7 @@ from prowler.providers.alibabacloud.alibabacloud_provider import AlibabacloudPro
|
||||
from prowler.providers.aws.aws_provider import AwsProvider
|
||||
from prowler.providers.aws.lib.security_hub.security_hub import SecurityHubConnection
|
||||
from prowler.providers.azure.azure_provider import AzureProvider
|
||||
from prowler.providers.cloudflare.cloudflare_provider import CloudflareProvider
|
||||
from prowler.providers.gcp.gcp_provider import GcpProvider
|
||||
from prowler.providers.github.github_provider import GithubProvider
|
||||
from prowler.providers.iac.iac_provider import IacProvider
|
||||
@@ -118,6 +119,7 @@ class TestReturnProwlerProvider:
|
||||
(Provider.ProviderChoices.ORACLECLOUD.value, OraclecloudProvider),
|
||||
(Provider.ProviderChoices.IAC.value, IacProvider),
|
||||
(Provider.ProviderChoices.ALIBABACLOUD.value, AlibabacloudProvider),
|
||||
(Provider.ProviderChoices.CLOUDFLARE.value, CloudflareProvider),
|
||||
],
|
||||
)
|
||||
def test_return_prowler_provider(self, provider_type, expected_provider):
|
||||
@@ -221,6 +223,10 @@ class TestGetProwlerProviderKwargs:
|
||||
Provider.ProviderChoices.MONGODBATLAS.value,
|
||||
{"atlas_organization_id": "provider_uid"},
|
||||
),
|
||||
(
|
||||
Provider.ProviderChoices.CLOUDFLARE.value,
|
||||
{"filter_accounts": ["provider_uid"]},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_get_prowler_provider_kwargs(self, provider_type, expected_extra_kwargs):
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,4 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from allauth.socialaccount.providers.oauth2.client import OAuth2Client
|
||||
from django.contrib.postgres.aggregates import ArrayAgg
|
||||
@@ -11,19 +14,26 @@ from api.exceptions import InvitationTokenExpiredException
|
||||
from api.models import Integration, Invitation, Processor, Provider, Resource
|
||||
from api.v1.serializers import FindingMetadataSerializer
|
||||
from prowler.lib.outputs.jira.jira import Jira, JiraBasicAuthError
|
||||
from prowler.providers.alibabacloud.alibabacloud_provider import AlibabacloudProvider
|
||||
from prowler.providers.aws.aws_provider import AwsProvider
|
||||
from prowler.providers.aws.lib.s3.s3 import S3
|
||||
from prowler.providers.aws.lib.security_hub.security_hub import SecurityHub
|
||||
from prowler.providers.azure.azure_provider import AzureProvider
|
||||
from prowler.providers.common.models import Connection
|
||||
from prowler.providers.gcp.gcp_provider import GcpProvider
|
||||
from prowler.providers.github.github_provider import GithubProvider
|
||||
from prowler.providers.iac.iac_provider import IacProvider
|
||||
from prowler.providers.kubernetes.kubernetes_provider import KubernetesProvider
|
||||
from prowler.providers.m365.m365_provider import M365Provider
|
||||
from prowler.providers.mongodbatlas.mongodbatlas_provider import MongodbatlasProvider
|
||||
from prowler.providers.oraclecloud.oraclecloud_provider import OraclecloudProvider
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from prowler.providers.alibabacloud.alibabacloud_provider import (
|
||||
AlibabacloudProvider,
|
||||
)
|
||||
from prowler.providers.aws.aws_provider import AwsProvider
|
||||
from prowler.providers.azure.azure_provider import AzureProvider
|
||||
from prowler.providers.cloudflare.cloudflare_provider import CloudflareProvider
|
||||
from prowler.providers.gcp.gcp_provider import GcpProvider
|
||||
from prowler.providers.github.github_provider import GithubProvider
|
||||
from prowler.providers.iac.iac_provider import IacProvider
|
||||
from prowler.providers.kubernetes.kubernetes_provider import KubernetesProvider
|
||||
from prowler.providers.m365.m365_provider import M365Provider
|
||||
from prowler.providers.mongodbatlas.mongodbatlas_provider import (
|
||||
MongodbatlasProvider,
|
||||
)
|
||||
from prowler.providers.oraclecloud.oraclecloud_provider import OraclecloudProvider
|
||||
|
||||
|
||||
class CustomOAuth2Client(OAuth2Client):
|
||||
@@ -82,32 +92,66 @@ def return_prowler_provider(
|
||||
provider (Provider): The provider object containing the provider type and associated secrets.
|
||||
|
||||
Returns:
|
||||
AlibabacloudProvider | AwsProvider | AzureProvider | GcpProvider | GithubProvider | IacProvider | KubernetesProvider | M365Provider | MongodbatlasProvider | OraclecloudProvider: The corresponding provider class.
|
||||
AlibabacloudProvider | AwsProvider | AzureProvider | CloudflareProvider | GcpProvider | GithubProvider | IacProvider | KubernetesProvider | M365Provider | MongodbatlasProvider | OraclecloudProvider: The corresponding provider class.
|
||||
|
||||
Raises:
|
||||
ValueError: If the provider type specified in `provider.provider` is not supported.
|
||||
"""
|
||||
match provider.provider:
|
||||
case Provider.ProviderChoices.AWS.value:
|
||||
from prowler.providers.aws.aws_provider import AwsProvider
|
||||
|
||||
prowler_provider = AwsProvider
|
||||
case Provider.ProviderChoices.GCP.value:
|
||||
from prowler.providers.gcp.gcp_provider import GcpProvider
|
||||
|
||||
prowler_provider = GcpProvider
|
||||
case Provider.ProviderChoices.AZURE.value:
|
||||
from prowler.providers.azure.azure_provider import AzureProvider
|
||||
|
||||
prowler_provider = AzureProvider
|
||||
case Provider.ProviderChoices.KUBERNETES.value:
|
||||
from prowler.providers.kubernetes.kubernetes_provider import (
|
||||
KubernetesProvider,
|
||||
)
|
||||
|
||||
prowler_provider = KubernetesProvider
|
||||
case Provider.ProviderChoices.M365.value:
|
||||
from prowler.providers.m365.m365_provider import M365Provider
|
||||
|
||||
prowler_provider = M365Provider
|
||||
case Provider.ProviderChoices.GITHUB.value:
|
||||
from prowler.providers.github.github_provider import GithubProvider
|
||||
|
||||
prowler_provider = GithubProvider
|
||||
case Provider.ProviderChoices.MONGODBATLAS.value:
|
||||
from prowler.providers.mongodbatlas.mongodbatlas_provider import (
|
||||
MongodbatlasProvider,
|
||||
)
|
||||
|
||||
prowler_provider = MongodbatlasProvider
|
||||
case Provider.ProviderChoices.IAC.value:
|
||||
from prowler.providers.iac.iac_provider import IacProvider
|
||||
|
||||
prowler_provider = IacProvider
|
||||
case Provider.ProviderChoices.ORACLECLOUD.value:
|
||||
from prowler.providers.oraclecloud.oraclecloud_provider import (
|
||||
OraclecloudProvider,
|
||||
)
|
||||
|
||||
prowler_provider = OraclecloudProvider
|
||||
case Provider.ProviderChoices.ALIBABACLOUD.value:
|
||||
from prowler.providers.alibabacloud.alibabacloud_provider import (
|
||||
AlibabacloudProvider,
|
||||
)
|
||||
|
||||
prowler_provider = AlibabacloudProvider
|
||||
case Provider.ProviderChoices.CLOUDFLARE.value:
|
||||
from prowler.providers.cloudflare.cloudflare_provider import (
|
||||
CloudflareProvider,
|
||||
)
|
||||
|
||||
prowler_provider = CloudflareProvider
|
||||
case _:
|
||||
raise ValueError(f"Provider type {provider.provider} not supported")
|
||||
return prowler_provider
|
||||
@@ -159,6 +203,11 @@ def get_prowler_provider_kwargs(
|
||||
**prowler_provider_kwargs,
|
||||
"atlas_organization_id": provider.uid,
|
||||
}
|
||||
elif provider.provider == Provider.ProviderChoices.CLOUDFLARE.value:
|
||||
prowler_provider_kwargs = {
|
||||
**prowler_provider_kwargs,
|
||||
"filter_accounts": [provider.uid],
|
||||
}
|
||||
|
||||
if mutelist_processor:
|
||||
mutelist_content = mutelist_processor.configuration.get("Mutelist", {})
|
||||
@@ -176,6 +225,7 @@ def initialize_prowler_provider(
|
||||
AlibabacloudProvider
|
||||
| AwsProvider
|
||||
| AzureProvider
|
||||
| CloudflareProvider
|
||||
| GcpProvider
|
||||
| GithubProvider
|
||||
| IacProvider
|
||||
@@ -191,7 +241,7 @@ def initialize_prowler_provider(
|
||||
mutelist_processor (Processor): The mutelist processor object containing the mutelist configuration.
|
||||
|
||||
Returns:
|
||||
AlibabacloudProvider | AwsProvider | AzureProvider | GcpProvider | GithubProvider | IacProvider | KubernetesProvider | M365Provider | MongodbatlasProvider | OraclecloudProvider: An instance of the corresponding provider class
|
||||
AlibabacloudProvider | AwsProvider | AzureProvider | CloudflareProvider | GcpProvider | GithubProvider | IacProvider | KubernetesProvider | M365Provider | MongodbatlasProvider | OraclecloudProvider: An instance of the corresponding provider class
|
||||
initialized with the provider's secrets.
|
||||
"""
|
||||
prowler_provider = return_prowler_provider(provider)
|
||||
@@ -393,11 +443,21 @@ def get_findings_metadata_no_aggregations(tenant_id: str, filtered_queryset):
|
||||
categories_set.update(categories_list)
|
||||
categories = sorted(categories_set)
|
||||
|
||||
# Aggregate groups from findings
|
||||
groups = list(
|
||||
filtered_queryset.exclude(resource_groups__isnull=True)
|
||||
.exclude(resource_groups__exact="")
|
||||
.values_list("resource_groups", flat=True)
|
||||
.distinct()
|
||||
.order_by("resource_groups")
|
||||
)
|
||||
|
||||
result = {
|
||||
"services": services,
|
||||
"regions": regions,
|
||||
"resource_types": resource_types,
|
||||
"categories": categories,
|
||||
"groups": groups,
|
||||
}
|
||||
|
||||
serializer = FindingMetadataSerializer(data=result)
|
||||
|
||||
@@ -346,6 +346,33 @@ from rest_framework_json_api import serializers
|
||||
},
|
||||
"required": ["role_arn", "access_key_id", "access_key_secret"],
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"title": "Cloudflare API Token",
|
||||
"properties": {
|
||||
"api_token": {
|
||||
"type": "string",
|
||||
"description": "Cloudflare API Token for authentication (recommended).",
|
||||
},
|
||||
},
|
||||
"required": ["api_token"],
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"title": "Cloudflare API Key + Email",
|
||||
"properties": {
|
||||
"api_key": {
|
||||
"type": "string",
|
||||
"description": "Cloudflare Global API Key for authentication (legacy).",
|
||||
},
|
||||
"api_email": {
|
||||
"type": "string",
|
||||
"format": "email",
|
||||
"description": "Email address associated with the Cloudflare account.",
|
||||
},
|
||||
},
|
||||
"required": ["api_key", "api_email"],
|
||||
},
|
||||
]
|
||||
}
|
||||
)
|
||||
|
||||
@@ -21,6 +21,7 @@ from rest_framework_simplejwt.tokens import RefreshToken
|
||||
from api.db_router import MainRouter
|
||||
from api.exceptions import ConflictException
|
||||
from api.models import (
|
||||
AttackPathsScan,
|
||||
Finding,
|
||||
Integration,
|
||||
IntegrationProviderRelationship,
|
||||
@@ -1132,6 +1133,119 @@ class ScanComplianceReportSerializer(BaseSerializerV1):
|
||||
fields = ["id", "name"]
|
||||
|
||||
|
||||
class AttackPathsScanSerializer(RLSSerializer):
|
||||
state = StateEnumSerializerField(read_only=True)
|
||||
provider_alias = serializers.SerializerMethodField(read_only=True)
|
||||
provider_type = serializers.SerializerMethodField(read_only=True)
|
||||
provider_uid = serializers.SerializerMethodField(read_only=True)
|
||||
|
||||
class Meta:
|
||||
model = AttackPathsScan
|
||||
fields = [
|
||||
"id",
|
||||
"state",
|
||||
"progress",
|
||||
"provider",
|
||||
"provider_alias",
|
||||
"provider_type",
|
||||
"provider_uid",
|
||||
"scan",
|
||||
"task",
|
||||
"inserted_at",
|
||||
"started_at",
|
||||
"completed_at",
|
||||
"duration",
|
||||
]
|
||||
|
||||
included_serializers = {
|
||||
"provider": "api.v1.serializers.ProviderIncludeSerializer",
|
||||
"scan": "api.v1.serializers.ScanIncludeSerializer",
|
||||
"task": "api.v1.serializers.TaskSerializer",
|
||||
}
|
||||
|
||||
def get_provider_alias(self, obj):
|
||||
provider = getattr(obj, "provider", None)
|
||||
return provider.alias if provider else None
|
||||
|
||||
def get_provider_type(self, obj):
|
||||
provider = getattr(obj, "provider", None)
|
||||
return provider.provider if provider else None
|
||||
|
||||
def get_provider_uid(self, obj):
|
||||
provider = getattr(obj, "provider", None)
|
||||
return provider.uid if provider else None
|
||||
|
||||
|
||||
class AttackPathsQueryAttributionSerializer(BaseSerializerV1):
|
||||
text = serializers.CharField()
|
||||
link = serializers.CharField()
|
||||
|
||||
class JSONAPIMeta:
|
||||
resource_name = "attack-paths-query-attributions"
|
||||
|
||||
|
||||
class AttackPathsQueryParameterSerializer(BaseSerializerV1):
|
||||
name = serializers.CharField()
|
||||
label = serializers.CharField()
|
||||
data_type = serializers.CharField(default="string")
|
||||
description = serializers.CharField(allow_null=True, required=False)
|
||||
placeholder = serializers.CharField(allow_null=True, required=False)
|
||||
|
||||
class JSONAPIMeta:
|
||||
resource_name = "attack-paths-query-parameters"
|
||||
|
||||
|
||||
class AttackPathsQuerySerializer(BaseSerializerV1):
|
||||
id = serializers.CharField()
|
||||
name = serializers.CharField()
|
||||
short_description = serializers.CharField()
|
||||
description = serializers.CharField()
|
||||
attribution = AttackPathsQueryAttributionSerializer(allow_null=True, required=False)
|
||||
provider = serializers.CharField()
|
||||
parameters = AttackPathsQueryParameterSerializer(many=True)
|
||||
|
||||
class JSONAPIMeta:
|
||||
resource_name = "attack-paths-queries"
|
||||
|
||||
|
||||
class AttackPathsQueryRunRequestSerializer(BaseSerializerV1):
|
||||
id = serializers.CharField()
|
||||
parameters = serializers.DictField(
|
||||
child=serializers.JSONField(), allow_empty=True, required=False
|
||||
)
|
||||
|
||||
class JSONAPIMeta:
|
||||
resource_name = "attack-paths-query-run-requests"
|
||||
|
||||
|
||||
class AttackPathsNodeSerializer(BaseSerializerV1):
|
||||
id = serializers.CharField()
|
||||
labels = serializers.ListField(child=serializers.CharField())
|
||||
properties = serializers.DictField(child=serializers.JSONField())
|
||||
|
||||
class JSONAPIMeta:
|
||||
resource_name = "attack-paths-query-result-nodes"
|
||||
|
||||
|
||||
class AttackPathsRelationshipSerializer(BaseSerializerV1):
|
||||
id = serializers.CharField()
|
||||
label = serializers.CharField()
|
||||
source = serializers.CharField()
|
||||
target = serializers.CharField()
|
||||
properties = serializers.DictField(child=serializers.JSONField())
|
||||
|
||||
class JSONAPIMeta:
|
||||
resource_name = "attack-paths-query-result-relationships"
|
||||
|
||||
|
||||
class AttackPathsQueryResultSerializer(BaseSerializerV1):
|
||||
nodes = AttackPathsNodeSerializer(many=True)
|
||||
relationships = AttackPathsRelationshipSerializer(many=True)
|
||||
|
||||
class JSONAPIMeta:
|
||||
resource_name = "attack-paths-query-results"
|
||||
|
||||
|
||||
class ResourceTagSerializer(RLSSerializer):
|
||||
"""
|
||||
Serializer for the ResourceTag model
|
||||
@@ -1175,6 +1289,7 @@ class ResourceSerializer(RLSSerializer):
|
||||
"metadata",
|
||||
"details",
|
||||
"partition",
|
||||
"groups",
|
||||
]
|
||||
extra_kwargs = {
|
||||
"id": {"read_only": True},
|
||||
@@ -1183,6 +1298,7 @@ class ResourceSerializer(RLSSerializer):
|
||||
"metadata": {"read_only": True},
|
||||
"details": {"read_only": True},
|
||||
"partition": {"read_only": True},
|
||||
"groups": {"read_only": True},
|
||||
}
|
||||
|
||||
included_serializers = {
|
||||
@@ -1276,6 +1392,7 @@ class ResourceMetadataSerializer(BaseSerializerV1):
|
||||
services = serializers.ListField(child=serializers.CharField(), allow_empty=True)
|
||||
regions = serializers.ListField(child=serializers.CharField(), allow_empty=True)
|
||||
types = serializers.ListField(child=serializers.CharField(), allow_empty=True)
|
||||
groups = serializers.ListField(child=serializers.CharField(), allow_empty=True)
|
||||
# Temporarily disabled until we implement tag filtering in the UI
|
||||
# tags = serializers.JSONField(help_text="Tags are described as key-value pairs.")
|
||||
|
||||
@@ -1302,6 +1419,7 @@ class FindingSerializer(RLSSerializer):
|
||||
"check_id",
|
||||
"check_metadata",
|
||||
"categories",
|
||||
"resource_groups",
|
||||
"raw_result",
|
||||
"inserted_at",
|
||||
"updated_at",
|
||||
@@ -1358,6 +1476,9 @@ class FindingMetadataSerializer(BaseSerializerV1):
|
||||
child=serializers.CharField(), allow_empty=True
|
||||
)
|
||||
categories = serializers.ListField(child=serializers.CharField(), allow_empty=True)
|
||||
groups = serializers.ListField(
|
||||
child=serializers.CharField(), allow_empty=True, required=False, default=list
|
||||
)
|
||||
# Temporarily disabled until we implement tag filtering in the UI
|
||||
# tags = serializers.JSONField(help_text="Tags are described as key-value pairs.")
|
||||
|
||||
@@ -1392,6 +1513,18 @@ class BaseWriteProviderSecretSerializer(BaseWriteSerializer):
|
||||
serializer = MongoDBAtlasProviderSecret(data=secret)
|
||||
elif provider_type == Provider.ProviderChoices.ALIBABACLOUD.value:
|
||||
serializer = AlibabaCloudProviderSecret(data=secret)
|
||||
elif provider_type == Provider.ProviderChoices.CLOUDFLARE.value:
|
||||
if "api_token" in secret:
|
||||
serializer = CloudflareTokenProviderSecret(data=secret)
|
||||
elif "api_key" in secret and "api_email" in secret:
|
||||
serializer = CloudflareApiKeyProviderSecret(data=secret)
|
||||
else:
|
||||
raise serializers.ValidationError(
|
||||
{
|
||||
"secret": "Cloudflare credentials must include either 'api_token' "
|
||||
"or both 'api_key' and 'api_email'."
|
||||
}
|
||||
)
|
||||
else:
|
||||
raise serializers.ValidationError(
|
||||
{"provider": f"Provider type not supported {provider_type}"}
|
||||
@@ -1543,6 +1676,21 @@ class OracleCloudProviderSecret(serializers.Serializer):
|
||||
resource_name = "provider-secrets"
|
||||
|
||||
|
||||
class CloudflareTokenProviderSecret(serializers.Serializer):
|
||||
api_token = serializers.CharField()
|
||||
|
||||
class Meta:
|
||||
resource_name = "provider-secrets"
|
||||
|
||||
|
||||
class CloudflareApiKeyProviderSecret(serializers.Serializer):
|
||||
api_key = serializers.CharField()
|
||||
api_email = serializers.EmailField()
|
||||
|
||||
class Meta:
|
||||
resource_name = "provider-secrets"
|
||||
|
||||
|
||||
class AlibabaCloudProviderSecret(serializers.Serializer):
|
||||
access_key_id = serializers.CharField()
|
||||
access_key_secret = serializers.CharField()
|
||||
@@ -2303,6 +2451,22 @@ class CategoryOverviewSerializer(BaseSerializerV1):
|
||||
resource_name = "category-overviews"
|
||||
|
||||
|
||||
class ResourceGroupOverviewSerializer(BaseSerializerV1):
|
||||
"""Serializer for resource group overview aggregations."""
|
||||
|
||||
id = serializers.CharField(source="resource_group")
|
||||
total_findings = serializers.IntegerField()
|
||||
failed_findings = serializers.IntegerField()
|
||||
new_failed_findings = serializers.IntegerField()
|
||||
resources_count = serializers.IntegerField()
|
||||
severity = serializers.JSONField(
|
||||
help_text="Severity breakdown: {informational, low, medium, high, critical}"
|
||||
)
|
||||
|
||||
class JSONAPIMeta:
|
||||
resource_name = "resource-group-overviews"
|
||||
|
||||
|
||||
class ComplianceWatchlistOverviewSerializer(BaseSerializerV1):
|
||||
"""Serializer for compliance watchlist overview with FAIL-dominant aggregation."""
|
||||
|
||||
@@ -3848,3 +4012,31 @@ class ThreatScoreSnapshotSerializer(RLSSerializer):
|
||||
if getattr(obj, "_aggregated", False):
|
||||
return "n/a"
|
||||
return str(obj.id)
|
||||
|
||||
|
||||
# Resource Events Serializers
|
||||
|
||||
|
||||
class ResourceEventSerializer(BaseSerializerV1):
|
||||
"""Serializer for resource events (CloudTrail modification history).
|
||||
|
||||
NOTE: drf-spectacular auto-generates fields[resource-events] sparse fieldsets
|
||||
parameter in the OpenAPI schema. This endpoint does not support sparse fieldsets.
|
||||
"""
|
||||
|
||||
id = serializers.CharField(source="event_id")
|
||||
event_time = serializers.DateTimeField()
|
||||
event_name = serializers.CharField()
|
||||
event_source = serializers.CharField()
|
||||
actor = serializers.CharField()
|
||||
actor_uid = serializers.CharField(allow_null=True, required=False)
|
||||
actor_type = serializers.CharField(allow_null=True, required=False)
|
||||
source_ip_address = serializers.CharField(allow_null=True, required=False)
|
||||
user_agent = serializers.CharField(allow_null=True, required=False)
|
||||
request_data = serializers.JSONField(allow_null=True, required=False)
|
||||
response_data = serializers.JSONField(allow_null=True, required=False)
|
||||
error_code = serializers.CharField(allow_null=True, required=False)
|
||||
error_message = serializers.CharField(allow_null=True, required=False)
|
||||
|
||||
class Meta:
|
||||
resource_name = "resource-events"
|
||||
|
||||
@@ -4,6 +4,7 @@ from drf_spectacular.views import SpectacularRedocView
|
||||
from rest_framework_nested import routers
|
||||
|
||||
from api.v1.views import (
|
||||
AttackPathsScanViewSet,
|
||||
ComplianceOverviewViewSet,
|
||||
CustomSAMLLoginView,
|
||||
CustomTokenObtainView,
|
||||
@@ -53,6 +54,9 @@ router.register(r"tenants", TenantViewSet, basename="tenant")
|
||||
router.register(r"providers", ProviderViewSet, basename="provider")
|
||||
router.register(r"provider-groups", ProviderGroupViewSet, basename="providergroup")
|
||||
router.register(r"scans", ScanViewSet, basename="scan")
|
||||
router.register(
|
||||
r"attack-paths-scans", AttackPathsScanViewSet, basename="attack-paths-scans"
|
||||
)
|
||||
router.register(r"tasks", TaskViewSet, basename="task")
|
||||
router.register(r"resources", ResourceViewSet, basename="resource")
|
||||
router.register(r"findings", FindingViewSet, basename="finding")
|
||||
|
||||
+637
-21
@@ -41,8 +41,9 @@ from django.db.models import (
|
||||
Sum,
|
||||
Value,
|
||||
When,
|
||||
Window,
|
||||
)
|
||||
from django.db.models.functions import Coalesce
|
||||
from django.db.models.functions import Coalesce, RowNumber
|
||||
from django.http import HttpResponse, QueryDict
|
||||
from django.shortcuts import redirect
|
||||
from django.urls import reverse
|
||||
@@ -73,6 +74,7 @@ from rest_framework.permissions import SAFE_METHODS
|
||||
from rest_framework_json_api.views import RelationshipView, Response
|
||||
from rest_framework_simplejwt.exceptions import InvalidToken, TokenError
|
||||
from tasks.beat import schedule_provider_scan
|
||||
from tasks.jobs.attack_paths import db_utils as attack_paths_db_utils
|
||||
from tasks.jobs.export import get_s3_client
|
||||
from tasks.tasks import (
|
||||
backfill_compliance_summaries_task,
|
||||
@@ -89,6 +91,9 @@ from tasks.tasks import (
|
||||
refresh_lighthouse_provider_models_task,
|
||||
)
|
||||
|
||||
from api.attack_paths import database as graph_database
|
||||
from api.attack_paths import get_queries_for_provider, get_query_by_id
|
||||
from api.attack_paths import views_helpers as attack_paths_views_helpers
|
||||
from api.base_views import BaseRLSViewSet, BaseTenantViewset, BaseUserViewset
|
||||
from api.compliance import (
|
||||
PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE,
|
||||
@@ -96,8 +101,15 @@ from api.compliance import (
|
||||
)
|
||||
from api.db_router import MainRouter
|
||||
from api.db_utils import rls_transaction
|
||||
from api.exceptions import TaskFailedException
|
||||
from api.exceptions import (
|
||||
TaskFailedException,
|
||||
UpstreamAccessDeniedError,
|
||||
UpstreamAuthenticationError,
|
||||
UpstreamInternalError,
|
||||
UpstreamServiceUnavailableError,
|
||||
)
|
||||
from api.filters import (
|
||||
AttackPathsScanFilter,
|
||||
AttackSurfaceOverviewFilter,
|
||||
CategoryOverviewFilter,
|
||||
ComplianceOverviewFilter,
|
||||
@@ -119,6 +131,7 @@ from api.filters import (
|
||||
ProviderGroupFilter,
|
||||
ProviderSecretFilter,
|
||||
ResourceFilter,
|
||||
ResourceGroupOverviewFilter,
|
||||
RoleFilter,
|
||||
ScanFilter,
|
||||
ScanSummaryFilter,
|
||||
@@ -130,6 +143,7 @@ from api.filters import (
|
||||
UserFilter,
|
||||
)
|
||||
from api.models import (
|
||||
AttackPathsScan,
|
||||
AttackSurfaceOverview,
|
||||
ComplianceOverviewSummary,
|
||||
ComplianceRequirementOverview,
|
||||
@@ -160,6 +174,7 @@ from api.models import (
|
||||
SAMLToken,
|
||||
Scan,
|
||||
ScanCategorySummary,
|
||||
ScanGroupSummary,
|
||||
ScanSummary,
|
||||
SeverityChoices,
|
||||
StateChoices,
|
||||
@@ -176,11 +191,16 @@ from api.rls import Tenant
|
||||
from api.utils import (
|
||||
CustomOAuth2Client,
|
||||
get_findings_metadata_no_aggregations,
|
||||
initialize_prowler_provider,
|
||||
validate_invitation,
|
||||
)
|
||||
from api.uuid_utils import datetime_to_uuid7, uuid7_start
|
||||
from api.v1.mixins import DisablePaginationMixin, PaginateByPkMixin, TaskManagementMixin
|
||||
from api.v1.serializers import (
|
||||
AttackPathsQueryResultSerializer,
|
||||
AttackPathsQueryRunRequestSerializer,
|
||||
AttackPathsQuerySerializer,
|
||||
AttackPathsScanSerializer,
|
||||
AttackSurfaceOverviewSerializer,
|
||||
CategoryOverviewSerializer,
|
||||
ComplianceOverviewAttributesSerializer,
|
||||
@@ -233,6 +253,8 @@ from api.v1.serializers import (
|
||||
ProviderSecretUpdateSerializer,
|
||||
ProviderSerializer,
|
||||
ProviderUpdateSerializer,
|
||||
ResourceEventSerializer,
|
||||
ResourceGroupOverviewSerializer,
|
||||
ResourceMetadataSerializer,
|
||||
ResourceSerializer,
|
||||
RoleCreateSerializer,
|
||||
@@ -262,6 +284,13 @@ from api.v1.serializers import (
|
||||
UserSerializer,
|
||||
UserUpdateSerializer,
|
||||
)
|
||||
from prowler.providers.aws.exceptions.exceptions import (
|
||||
AWSAssumeRoleError,
|
||||
AWSCredentialsError,
|
||||
)
|
||||
from prowler.providers.aws.lib.cloudtrail_timeline.cloudtrail_timeline import (
|
||||
CloudTrailTimeline,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(BackendLogger.API)
|
||||
|
||||
@@ -363,7 +392,7 @@ class SchemaView(SpectacularAPIView):
|
||||
|
||||
def get(self, request, *args, **kwargs):
|
||||
spectacular_settings.TITLE = "Prowler API"
|
||||
spectacular_settings.VERSION = "1.18.0"
|
||||
spectacular_settings.VERSION = "1.20.0"
|
||||
spectacular_settings.DESCRIPTION = (
|
||||
"Prowler API specification.\n\nThis file is auto-generated."
|
||||
)
|
||||
@@ -405,6 +434,10 @@ class SchemaView(SpectacularAPIView):
|
||||
"name": "Scan",
|
||||
"description": "Endpoints for triggering manual scans and viewing scan results.",
|
||||
},
|
||||
{
|
||||
"name": "Attack Paths",
|
||||
"description": "Endpoints for Attack Paths scan status and executing Attack Paths queries.",
|
||||
},
|
||||
{
|
||||
"name": "Schedule",
|
||||
"description": "Endpoints for managing scan schedules, allowing configuration of automated "
|
||||
@@ -730,27 +763,40 @@ class TenantFinishACSView(FinishACSView):
|
||||
.tenant
|
||||
)
|
||||
|
||||
# Check if tenant has only one user with MANAGE_ACCOUNT role
|
||||
users_with_manage_account = (
|
||||
role_name = (
|
||||
extra.get("userType", ["no_permissions"])[0].strip()
|
||||
if extra.get("userType")
|
||||
else "no_permissions"
|
||||
)
|
||||
role = (
|
||||
Role.objects.using(MainRouter.admin_db)
|
||||
.filter(name=role_name, tenant=tenant)
|
||||
.first()
|
||||
)
|
||||
|
||||
# Only skip mapping if it would remove the last MANAGE_ACCOUNT user
|
||||
remaining_manage_account_users = (
|
||||
UserRoleRelationship.objects.using(MainRouter.admin_db)
|
||||
.filter(role__manage_account=True, tenant_id=tenant.id)
|
||||
.exclude(user_id=user_id)
|
||||
.values("user")
|
||||
.distinct()
|
||||
.count()
|
||||
)
|
||||
user_has_manage_account = (
|
||||
UserRoleRelationship.objects.using(MainRouter.admin_db)
|
||||
.filter(role__manage_account=True, tenant_id=tenant.id, user_id=user_id)
|
||||
.exists()
|
||||
)
|
||||
role_manage_account = role.manage_account if role else False
|
||||
would_remove_last_manage_account = (
|
||||
user_has_manage_account
|
||||
and remaining_manage_account_users == 0
|
||||
and not role_manage_account
|
||||
)
|
||||
|
||||
# Only apply role mapping from userType if tenant does NOT have exactly one user with MANAGE_ACCOUNT
|
||||
if users_with_manage_account != 1:
|
||||
role_name = (
|
||||
extra.get("userType", ["no_permissions"])[0].strip()
|
||||
if extra.get("userType")
|
||||
else "no_permissions"
|
||||
)
|
||||
try:
|
||||
role = Role.objects.using(MainRouter.admin_db).get(
|
||||
name=role_name, tenant=tenant
|
||||
)
|
||||
except Role.DoesNotExist:
|
||||
if not would_remove_last_manage_account:
|
||||
if role is None:
|
||||
role = Role.objects.using(MainRouter.admin_db).create(
|
||||
name=role_name,
|
||||
tenant=tenant,
|
||||
@@ -2155,6 +2201,12 @@ class ScanViewSet(BaseRLSViewSet):
|
||||
},
|
||||
)
|
||||
|
||||
attack_paths_db_utils.create_attack_paths_scan(
|
||||
tenant_id=self.request.tenant_id,
|
||||
scan_id=str(scan.id),
|
||||
provider_id=str(scan.provider_id),
|
||||
)
|
||||
|
||||
prowler_task = Task.objects.get(id=task.id)
|
||||
scan.task_id = task.id
|
||||
scan.save(update_fields=["task_id"])
|
||||
@@ -2235,6 +2287,188 @@ class TaskViewSet(BaseRLSViewSet):
|
||||
)
|
||||
|
||||
|
||||
@extend_schema_view(
|
||||
list=extend_schema(
|
||||
tags=["Attack Paths"],
|
||||
summary="List Attack Paths scans",
|
||||
description="Retrieve Attack Paths scans for the tenant with support for filtering, ordering, and pagination.",
|
||||
),
|
||||
retrieve=extend_schema(
|
||||
tags=["Attack Paths"],
|
||||
summary="Retrieve Attack Paths scan details",
|
||||
description="Fetch full details for a specific Attack Paths scan.",
|
||||
),
|
||||
attack_paths_queries=extend_schema(
|
||||
tags=["Attack Paths"],
|
||||
summary="List Attack Paths queries",
|
||||
description="Retrieve the catalog of Attack Paths queries available for this Attack Paths scan.",
|
||||
responses={
|
||||
200: OpenApiResponse(AttackPathsQuerySerializer(many=True)),
|
||||
404: OpenApiResponse(
|
||||
description="No queries found for the selected provider"
|
||||
),
|
||||
},
|
||||
),
|
||||
run_attack_paths_query=extend_schema(
|
||||
tags=["Attack Paths"],
|
||||
summary="Execute an Attack Paths query",
|
||||
description="Execute the selected Attack Paths query against the Attack Paths graph and return the resulting subgraph.",
|
||||
request=AttackPathsQueryRunRequestSerializer,
|
||||
responses={
|
||||
200: OpenApiResponse(AttackPathsQueryResultSerializer),
|
||||
400: OpenApiResponse(
|
||||
description="Bad request (e.g., Unknown Attack Paths query for the selected provider)"
|
||||
),
|
||||
404: OpenApiResponse(
|
||||
description="No Attack Paths found for the given query and parameters"
|
||||
),
|
||||
500: OpenApiResponse(
|
||||
description="Attack Paths query execution failed due to a database error"
|
||||
),
|
||||
},
|
||||
),
|
||||
)
|
||||
class AttackPathsScanViewSet(BaseRLSViewSet):
|
||||
queryset = AttackPathsScan.objects.all()
|
||||
serializer_class = AttackPathsScanSerializer
|
||||
http_method_names = ["get", "post"]
|
||||
filterset_class = AttackPathsScanFilter
|
||||
ordering = ["-inserted_at"]
|
||||
ordering_fields = [
|
||||
"inserted_at",
|
||||
"started_at",
|
||||
]
|
||||
# RBAC required permissions
|
||||
required_permissions = [Permissions.MANAGE_SCANS]
|
||||
|
||||
def set_required_permissions(self):
|
||||
if self.request.method in SAFE_METHODS:
|
||||
self.required_permissions = []
|
||||
|
||||
else:
|
||||
self.required_permissions = [Permissions.MANAGE_SCANS]
|
||||
|
||||
def get_serializer_class(self):
|
||||
if self.action == "run_attack_paths_query":
|
||||
return AttackPathsQueryRunRequestSerializer
|
||||
|
||||
return super().get_serializer_class()
|
||||
|
||||
def get_queryset(self):
|
||||
user_roles = get_role(self.request.user)
|
||||
base_queryset = AttackPathsScan.objects.filter(tenant_id=self.request.tenant_id)
|
||||
|
||||
if user_roles.unlimited_visibility:
|
||||
queryset = base_queryset
|
||||
|
||||
else:
|
||||
queryset = base_queryset.filter(provider__in=get_providers(user_roles))
|
||||
|
||||
return queryset.select_related("provider", "scan", "task")
|
||||
|
||||
def list(self, request, *args, **kwargs):
|
||||
queryset = self.filter_queryset(self.get_queryset())
|
||||
|
||||
latest_per_provider = queryset.annotate(
|
||||
latest_scan_rank=Window(
|
||||
expression=RowNumber(),
|
||||
partition_by=[F("provider_id")],
|
||||
order_by=[F("inserted_at").desc()],
|
||||
)
|
||||
).filter(latest_scan_rank=1)
|
||||
|
||||
page = self.paginate_queryset(latest_per_provider)
|
||||
if page is not None:
|
||||
serializer = self.get_serializer(page, many=True)
|
||||
return self.get_paginated_response(serializer.data)
|
||||
|
||||
serializer = self.get_serializer(latest_per_provider, many=True)
|
||||
return Response(serializer.data)
|
||||
|
||||
@extend_schema(exclude=True)
|
||||
def create(self, request, *args, **kwargs):
|
||||
raise MethodNotAllowed(method="POST")
|
||||
|
||||
@extend_schema(exclude=True)
|
||||
def destroy(self, request, *args, **kwargs):
|
||||
raise MethodNotAllowed(method="DELETE")
|
||||
|
||||
@action(
|
||||
detail=True,
|
||||
methods=["get"],
|
||||
url_path="queries",
|
||||
url_name="queries",
|
||||
)
|
||||
def attack_paths_queries(self, request, pk=None):
|
||||
attack_paths_scan = self.get_object()
|
||||
queries = get_queries_for_provider(attack_paths_scan.provider.provider)
|
||||
|
||||
if not queries:
|
||||
return Response(
|
||||
{"detail": "No queries found for the selected provider"},
|
||||
status=status.HTTP_404_NOT_FOUND,
|
||||
)
|
||||
|
||||
serializer = AttackPathsQuerySerializer(queries, many=True)
|
||||
return Response(serializer.data, status=status.HTTP_200_OK)
|
||||
|
||||
@action(
|
||||
detail=True,
|
||||
methods=["post"],
|
||||
url_path="queries/run",
|
||||
url_name="queries-run",
|
||||
)
|
||||
def run_attack_paths_query(self, request, pk=None):
|
||||
attack_paths_scan = self.get_object()
|
||||
|
||||
if attack_paths_scan.state != StateChoices.COMPLETED:
|
||||
raise ValidationError(
|
||||
{
|
||||
"detail": "The Attack Paths scan must be completed before running Attack Paths queries"
|
||||
}
|
||||
)
|
||||
|
||||
if not attack_paths_scan.graph_database:
|
||||
logger.error(
|
||||
f"The Attack Paths Scan {attack_paths_scan.id} does not reference a graph database"
|
||||
)
|
||||
return Response(
|
||||
{"detail": "The Attack Paths scan does not reference a graph database"},
|
||||
status=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
)
|
||||
|
||||
payload = attack_paths_views_helpers.normalize_run_payload(request.data)
|
||||
serializer = AttackPathsQueryRunRequestSerializer(data=payload)
|
||||
serializer.is_valid(raise_exception=True)
|
||||
|
||||
query_definition = get_query_by_id(serializer.validated_data["id"])
|
||||
if (
|
||||
query_definition is None
|
||||
or query_definition.provider != attack_paths_scan.provider.provider
|
||||
):
|
||||
raise ValidationError(
|
||||
{"id": "Unknown Attack Paths query for the selected provider"}
|
||||
)
|
||||
|
||||
parameters = attack_paths_views_helpers.prepare_query_parameters(
|
||||
query_definition,
|
||||
serializer.validated_data.get("parameters", {}),
|
||||
attack_paths_scan.provider.uid,
|
||||
)
|
||||
|
||||
graph = attack_paths_views_helpers.execute_attack_paths_query(
|
||||
attack_paths_scan, query_definition, parameters
|
||||
)
|
||||
graph_database.clear_cache(attack_paths_scan.graph_database)
|
||||
|
||||
status_code = status.HTTP_200_OK
|
||||
if not graph.get("nodes"):
|
||||
status_code = status.HTTP_404_NOT_FOUND
|
||||
|
||||
response_serializer = AttackPathsQueryResultSerializer(graph)
|
||||
return Response(response_serializer.data, status=status_code)
|
||||
|
||||
|
||||
@extend_schema_view(
|
||||
list=extend_schema(
|
||||
tags=["Resource"],
|
||||
@@ -2293,6 +2527,20 @@ class ResourceViewSet(PaginateByPkMixin, BaseRLSViewSet):
|
||||
http_method_names = ["get"]
|
||||
filterset_class = ResourceFilter
|
||||
ordering = ["-failed_findings_count", "-updated_at"]
|
||||
|
||||
# Events endpoint constants (currently AWS-only, limited to 90 days by CloudTrail Event History)
|
||||
EVENTS_DEFAULT_LOOKBACK_DAYS = 90
|
||||
EVENTS_MIN_LOOKBACK_DAYS = 1
|
||||
EVENTS_MAX_LOOKBACK_DAYS = 90
|
||||
# Page size controls how many events CloudTrail returns (prepares for API pagination)
|
||||
EVENTS_DEFAULT_PAGE_SIZE = 50
|
||||
EVENTS_MIN_PAGE_SIZE = 1
|
||||
EVENTS_MAX_PAGE_SIZE = 50 # CloudTrail lookup_events max is 50
|
||||
# Allowed query parameters for the events endpoint
|
||||
EVENTS_ALLOWED_PARAMS = frozenset(
|
||||
{"lookback_days", "page[size]", "include_read_events"}
|
||||
)
|
||||
|
||||
ordering_fields = [
|
||||
"provider_uid",
|
||||
"uid",
|
||||
@@ -2368,6 +2616,8 @@ class ResourceViewSet(PaginateByPkMixin, BaseRLSViewSet):
|
||||
def get_serializer_class(self):
|
||||
if self.action in ["metadata", "metadata_latest"]:
|
||||
return ResourceMetadataSerializer
|
||||
if self.action == "events":
|
||||
return ResourceEventSerializer
|
||||
return super().get_serializer_class()
|
||||
|
||||
def get_filterset_class(self):
|
||||
@@ -2376,8 +2626,8 @@ class ResourceViewSet(PaginateByPkMixin, BaseRLSViewSet):
|
||||
return ResourceFilter
|
||||
|
||||
def filter_queryset(self, queryset):
|
||||
# Do not apply filters when retrieving specific resource
|
||||
if self.action == "retrieve":
|
||||
# Do not apply filters when retrieving specific resource or events
|
||||
if self.action in ["retrieve", "events"]:
|
||||
return queryset
|
||||
return super().filter_queryset(queryset)
|
||||
|
||||
@@ -2527,10 +2777,20 @@ class ResourceViewSet(PaginateByPkMixin, BaseRLSViewSet):
|
||||
.order_by("resource_type")
|
||||
)
|
||||
|
||||
# Get groups from Resource model (flatten ArrayField)
|
||||
all_groups = Resource.objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
groups__isnull=False,
|
||||
).values_list("groups", flat=True)
|
||||
groups = sorted(
|
||||
set(g for groups_list in all_groups if groups_list for g in groups_list)
|
||||
)
|
||||
|
||||
result = {
|
||||
"services": services,
|
||||
"regions": regions,
|
||||
"types": resource_types,
|
||||
"groups": groups,
|
||||
}
|
||||
|
||||
serializer = self.get_serializer(data=result)
|
||||
@@ -2587,16 +2847,243 @@ class ResourceViewSet(PaginateByPkMixin, BaseRLSViewSet):
|
||||
.order_by("resource_type")
|
||||
)
|
||||
|
||||
# Get groups from Resource model for resources in latest scans (flatten ArrayField)
|
||||
all_groups = Resource.objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
groups__isnull=False,
|
||||
).values_list("groups", flat=True)
|
||||
groups = sorted(
|
||||
set(g for groups_list in all_groups if groups_list for g in groups_list)
|
||||
)
|
||||
|
||||
result = {
|
||||
"services": services,
|
||||
"regions": regions,
|
||||
"types": resource_types,
|
||||
"groups": groups,
|
||||
}
|
||||
|
||||
serializer = self.get_serializer(data=result)
|
||||
serializer.is_valid(raise_exception=True)
|
||||
return Response(serializer.data)
|
||||
|
||||
@extend_schema(
|
||||
tags=["Resource"],
|
||||
summary="Get events for a resource",
|
||||
description=(
|
||||
"Retrieve events showing modification history for a resource. "
|
||||
"Returns who modified the resource and when. Currently only available for AWS resources.\n\n"
|
||||
"**Note:** Some events may not appear due to CloudTrail indexing limitations. "
|
||||
"Not all AWS API calls record the resource identifier in a searchable format."
|
||||
),
|
||||
parameters=[
|
||||
OpenApiParameter(
|
||||
name="lookback_days",
|
||||
type=OpenApiTypes.INT,
|
||||
location=OpenApiParameter.QUERY,
|
||||
description="Number of days to look back (default: 90, min: 1, max: 90).",
|
||||
required=False,
|
||||
),
|
||||
OpenApiParameter(
|
||||
name="page[size]",
|
||||
type=OpenApiTypes.INT,
|
||||
location=OpenApiParameter.QUERY,
|
||||
description="Maximum number of events to return (default: 50, min: 1, max: 50).",
|
||||
required=False,
|
||||
),
|
||||
OpenApiParameter(
|
||||
name="include_read_events",
|
||||
type=OpenApiTypes.BOOL,
|
||||
location=OpenApiParameter.QUERY,
|
||||
description=(
|
||||
"Include read-only events (Describe*, Get*, List*, etc.). "
|
||||
"Default: false. Set to true to include all events."
|
||||
),
|
||||
required=False,
|
||||
),
|
||||
# NOTE: drf-spectacular auto-generates page[number] and fields[resource-events]
|
||||
# parameters. This endpoint does not support pagination (results are limited by
|
||||
# page[size] only) nor sparse fieldsets.
|
||||
],
|
||||
responses={
|
||||
200: ResourceEventSerializer(many=True),
|
||||
400: OpenApiResponse(description="Invalid provider or parameters"),
|
||||
500: OpenApiResponse(description="Unexpected error retrieving events"),
|
||||
502: OpenApiResponse(
|
||||
description="Provider credentials invalid, expired, or lack required permissions"
|
||||
),
|
||||
503: OpenApiResponse(description="Provider service unavailable"),
|
||||
},
|
||||
)
|
||||
@action(
|
||||
detail=True,
|
||||
methods=["get"],
|
||||
url_name="events",
|
||||
filter_backends=[], # Disable filters - we're calling external API, not filtering queryset
|
||||
)
|
||||
def events(self, request, pk=None):
|
||||
"""Get events for a resource."""
|
||||
resource = self.get_object()
|
||||
|
||||
# Validate query parameters - reject unknown parameters
|
||||
for param in request.query_params.keys():
|
||||
if param not in self.EVENTS_ALLOWED_PARAMS:
|
||||
raise ValidationError(
|
||||
[
|
||||
{
|
||||
"detail": f"invalid parameter '{param}'",
|
||||
"status": "400",
|
||||
"source": {"parameter": param},
|
||||
"code": "invalid",
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
# Validate provider - currently only AWS CloudTrail is supported
|
||||
if resource.provider.provider != Provider.ProviderChoices.AWS:
|
||||
raise ValidationError(
|
||||
[
|
||||
{
|
||||
"detail": "Events are only available for AWS resources",
|
||||
"status": "400",
|
||||
"source": {"pointer": "/data/attributes/provider"},
|
||||
"code": "invalid_provider",
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
# Validate and parse lookback_days from query params
|
||||
lookback_days_str = request.query_params.get("lookback_days")
|
||||
if lookback_days_str is None:
|
||||
lookback_days = self.EVENTS_DEFAULT_LOOKBACK_DAYS
|
||||
else:
|
||||
try:
|
||||
lookback_days = int(lookback_days_str)
|
||||
except (ValueError, TypeError):
|
||||
raise ValidationError(
|
||||
[
|
||||
{
|
||||
"detail": "lookback_days must be a valid integer",
|
||||
"status": "400",
|
||||
"source": {"parameter": "lookback_days"},
|
||||
"code": "invalid",
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
if not (
|
||||
self.EVENTS_MIN_LOOKBACK_DAYS
|
||||
<= lookback_days
|
||||
<= self.EVENTS_MAX_LOOKBACK_DAYS
|
||||
):
|
||||
raise ValidationError(
|
||||
[
|
||||
{
|
||||
"detail": (
|
||||
f"lookback_days must be between {self.EVENTS_MIN_LOOKBACK_DAYS} "
|
||||
f"and {self.EVENTS_MAX_LOOKBACK_DAYS}"
|
||||
),
|
||||
"status": "400",
|
||||
"source": {"parameter": "lookback_days"},
|
||||
"code": "out_of_range",
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
# Validate and parse page[size] from query params (JSON:API pagination)
|
||||
page_size_str = request.query_params.get("page[size]")
|
||||
if page_size_str is None:
|
||||
page_size = self.EVENTS_DEFAULT_PAGE_SIZE
|
||||
else:
|
||||
try:
|
||||
page_size = int(page_size_str)
|
||||
except (ValueError, TypeError):
|
||||
raise ValidationError(
|
||||
[
|
||||
{
|
||||
"detail": "page[size] must be a valid integer",
|
||||
"status": "400",
|
||||
"source": {"parameter": "page[size]"},
|
||||
"code": "invalid",
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
if not (
|
||||
self.EVENTS_MIN_PAGE_SIZE <= page_size <= self.EVENTS_MAX_PAGE_SIZE
|
||||
):
|
||||
raise ValidationError(
|
||||
[
|
||||
{
|
||||
"detail": (
|
||||
f"page[size] must be between {self.EVENTS_MIN_PAGE_SIZE} "
|
||||
f"and {self.EVENTS_MAX_PAGE_SIZE}"
|
||||
),
|
||||
"status": "400",
|
||||
"source": {"parameter": "page[size]"},
|
||||
"code": "out_of_range",
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
# Parse include_read_events (default: false)
|
||||
include_read_events = (
|
||||
request.query_params.get("include_read_events", "").lower() == "true"
|
||||
)
|
||||
|
||||
try:
|
||||
# Initialize Prowler provider using existing utility
|
||||
prowler_provider = initialize_prowler_provider(resource.provider)
|
||||
|
||||
# Get the boto3 session from the Prowler provider
|
||||
session = prowler_provider._session.current_session
|
||||
|
||||
# Create timeline service (currently only AWS/CloudTrail is supported)
|
||||
timeline_service = CloudTrailTimeline(
|
||||
session=session,
|
||||
lookback_days=lookback_days,
|
||||
max_results=page_size,
|
||||
write_events_only=not include_read_events,
|
||||
)
|
||||
|
||||
# Get timeline events
|
||||
events = timeline_service.get_resource_timeline(
|
||||
region=resource.region,
|
||||
resource_uid=resource.uid,
|
||||
)
|
||||
|
||||
serializer = ResourceEventSerializer(events, many=True)
|
||||
return Response(serializer.data)
|
||||
|
||||
except NoCredentialsError:
|
||||
# 502 because this is an upstream auth failure, not API auth failure
|
||||
raise UpstreamAuthenticationError(
|
||||
detail="Credentials not found for this provider. Please reconnect the provider."
|
||||
)
|
||||
except AWSAssumeRoleError:
|
||||
# AssumeRole failed - usually IAM permission issue (not authorized to sts:AssumeRole)
|
||||
raise UpstreamAccessDeniedError(
|
||||
detail="Cannot assume role for this provider. Check IAM Role permissions and trust relationship."
|
||||
)
|
||||
except AWSCredentialsError:
|
||||
# Handles expired tokens, invalid keys, profile not found, etc.
|
||||
raise UpstreamAuthenticationError()
|
||||
except ClientError as e:
|
||||
error_code = e.response.get("Error", {}).get("Code", "")
|
||||
# AccessDenied is expected when credentials lack permissions - don't log as error
|
||||
if error_code in ("AccessDenied", "AccessDeniedException"):
|
||||
raise UpstreamAccessDeniedError()
|
||||
|
||||
# Unexpected ClientErrors should be logged for debugging
|
||||
logger.error(
|
||||
f"Provider API error retrieving events: {str(e)}",
|
||||
exc_info=True,
|
||||
)
|
||||
raise UpstreamServiceUnavailableError()
|
||||
except Exception as e:
|
||||
sentry_sdk.capture_exception(e)
|
||||
raise UpstreamInternalError(detail="Failed to retrieve events")
|
||||
|
||||
|
||||
@extend_schema_view(
|
||||
list=extend_schema(
|
||||
@@ -3019,11 +3506,23 @@ class FindingViewSet(PaginateByPkMixin, BaseRLSViewSet):
|
||||
categories_set.update(categories_list)
|
||||
categories = sorted(categories_set)
|
||||
|
||||
# Get groups from ScanGroupSummary for latest scans
|
||||
groups = list(
|
||||
ScanGroupSummary.objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
scan_id__in=latest_scans_queryset.values_list("id", flat=True),
|
||||
)
|
||||
.values_list("resource_group", flat=True)
|
||||
.distinct()
|
||||
.order_by("resource_group")
|
||||
)
|
||||
|
||||
result = {
|
||||
"services": services,
|
||||
"regions": regions,
|
||||
"resource_types": resource_types,
|
||||
"categories": categories,
|
||||
"groups": groups,
|
||||
}
|
||||
|
||||
serializer = self.get_serializer(data=result)
|
||||
@@ -3958,7 +4457,7 @@ class ComplianceOverviewViewSet(BaseRLSViewSet, TaskManagementMixin):
|
||||
# If we couldn't determine from database, try each provider type
|
||||
if not provider_type:
|
||||
for pt in Provider.ProviderChoices.values:
|
||||
if compliance_id in PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE.get(pt, {}):
|
||||
if compliance_id in get_compliance_frameworks(pt):
|
||||
provider_type = pt
|
||||
break
|
||||
|
||||
@@ -4097,6 +4596,30 @@ class ComplianceOverviewViewSet(BaseRLSViewSet, TaskManagementMixin):
|
||||
filters=True,
|
||||
responses={200: CategoryOverviewSerializer(many=True)},
|
||||
),
|
||||
resource_groups=extend_schema(
|
||||
summary="Get resource group overview",
|
||||
description=(
|
||||
"Retrieve aggregated resource group metrics from latest completed scans per provider. "
|
||||
"Returns one row per resource group with total, failed, and new failed findings counts, "
|
||||
"plus a severity breakdown showing failed findings per severity level, "
|
||||
"and a count of distinct resources evaluated per group."
|
||||
),
|
||||
tags=["Overview"],
|
||||
filters=True,
|
||||
responses={200: ResourceGroupOverviewSerializer(many=True)},
|
||||
),
|
||||
compliance_watchlist=extend_schema(
|
||||
summary="Get compliance watchlist overview",
|
||||
description=(
|
||||
"Retrieve compliance metrics with FAIL-dominant aggregation. "
|
||||
"Without filters: uses pre-aggregated TenantComplianceSummary. "
|
||||
"With provider filters: queries ProviderComplianceScore with FAIL-dominant logic "
|
||||
"where any FAIL in a requirement marks it as failed."
|
||||
),
|
||||
tags=["Overview"],
|
||||
filters=True,
|
||||
responses={200: ComplianceWatchlistOverviewSerializer(many=True)},
|
||||
),
|
||||
)
|
||||
@method_decorator(CACHE_DECORATOR, name="list")
|
||||
class OverviewViewSet(BaseRLSViewSet):
|
||||
@@ -4146,6 +4669,8 @@ class OverviewViewSet(BaseRLSViewSet):
|
||||
return AttackSurfaceOverviewSerializer
|
||||
elif self.action == "categories":
|
||||
return CategoryOverviewSerializer
|
||||
elif self.action == "resource_groups":
|
||||
return ResourceGroupOverviewSerializer
|
||||
elif self.action == "compliance_watchlist":
|
||||
return ComplianceWatchlistOverviewSerializer
|
||||
return super().get_serializer_class()
|
||||
@@ -4161,6 +4686,8 @@ class OverviewViewSet(BaseRLSViewSet):
|
||||
return DailySeveritySummaryFilter
|
||||
elif self.action == "categories":
|
||||
return CategoryOverviewFilter
|
||||
elif self.action == "resource_groups":
|
||||
return ResourceGroupOverviewFilter
|
||||
elif self.action == "attack_surface":
|
||||
return AttackSurfaceOverviewFilter
|
||||
elif self.action == "compliance_watchlist":
|
||||
@@ -5005,6 +5532,95 @@ class OverviewViewSet(BaseRLSViewSet):
|
||||
status=status.HTTP_200_OK,
|
||||
)
|
||||
|
||||
@action(
|
||||
detail=False,
|
||||
methods=["get"],
|
||||
url_name="resource-groups",
|
||||
url_path="resource-groups",
|
||||
)
|
||||
def resource_groups(self, request):
|
||||
tenant_id = request.tenant_id
|
||||
provider_filters = self._extract_provider_filters_from_params()
|
||||
latest_scan_ids = self._latest_scan_ids_for_allowed_providers(
|
||||
tenant_id, provider_filters
|
||||
)
|
||||
|
||||
base_queryset = ScanGroupSummary.objects.filter(
|
||||
tenant_id=tenant_id, scan_id__in=latest_scan_ids
|
||||
)
|
||||
provider_filter_keys = {
|
||||
"provider_id",
|
||||
"provider_id__in",
|
||||
"provider_type",
|
||||
"provider_type__in",
|
||||
}
|
||||
filtered_queryset = self._apply_filterset(
|
||||
base_queryset,
|
||||
ResourceGroupOverviewFilter,
|
||||
exclude_keys=provider_filter_keys,
|
||||
)
|
||||
|
||||
aggregation = (
|
||||
filtered_queryset.values("resource_group", "severity")
|
||||
.annotate(
|
||||
total=Coalesce(Sum("total_findings"), 0),
|
||||
failed=Coalesce(Sum("failed_findings"), 0),
|
||||
new_failed=Coalesce(Sum("new_failed_findings"), 0),
|
||||
)
|
||||
.order_by("resource_group", "severity")
|
||||
)
|
||||
|
||||
# Get resource_group-level resources_count:
|
||||
# 1. Max per (scan, resource_group) to deduplicate within-scan severity rows
|
||||
# 2. Sum across scans for cross-provider aggregation
|
||||
scan_resource_group_resources = filtered_queryset.values(
|
||||
"scan_id", "resource_group"
|
||||
).annotate(resources=Coalesce(Max("resources_count"), 0))
|
||||
resources_by_resource_group = defaultdict(int)
|
||||
for row in scan_resource_group_resources:
|
||||
resources_by_resource_group[row["resource_group"]] += row["resources"]
|
||||
|
||||
resource_group_data = defaultdict(
|
||||
lambda: {
|
||||
"total_findings": 0,
|
||||
"failed_findings": 0,
|
||||
"new_failed_findings": 0,
|
||||
"resources_count": 0,
|
||||
"severity": {
|
||||
"informational": 0,
|
||||
"low": 0,
|
||||
"medium": 0,
|
||||
"high": 0,
|
||||
"critical": 0,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
for row in aggregation:
|
||||
grp = row["resource_group"]
|
||||
sev = row["severity"]
|
||||
resource_group_data[grp]["total_findings"] += row["total"]
|
||||
resource_group_data[grp]["failed_findings"] += row["failed"]
|
||||
resource_group_data[grp]["new_failed_findings"] += row["new_failed"]
|
||||
if sev in resource_group_data[grp]["severity"]:
|
||||
resource_group_data[grp]["severity"][sev] = row["failed"]
|
||||
|
||||
# Set resources_count from resource_group-level aggregation
|
||||
for grp in resource_group_data:
|
||||
resource_group_data[grp]["resources_count"] = (
|
||||
resources_by_resource_group.get(grp, 0)
|
||||
)
|
||||
|
||||
response_data = [
|
||||
{"resource_group": grp, **data}
|
||||
for grp, data in sorted(resource_group_data.items())
|
||||
]
|
||||
|
||||
return Response(
|
||||
self.get_serializer(response_data, many=True).data,
|
||||
status=status.HTTP_200_OK,
|
||||
)
|
||||
|
||||
@action(
|
||||
detail=False,
|
||||
methods=["get"],
|
||||
@@ -5760,7 +6376,7 @@ class TenantApiKeyViewSet(BaseRLSViewSet):
|
||||
|
||||
@extend_schema(exclude=True)
|
||||
def destroy(self, request, *args, **kwargs):
|
||||
raise MethodNotAllowed(method="DESTROY")
|
||||
raise MethodNotAllowed(method="DELETE")
|
||||
|
||||
@action(detail=True, methods=["delete"])
|
||||
def revoke(self, request, *args, **kwargs):
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import warnings
|
||||
|
||||
from celery import Celery, Task
|
||||
|
||||
from config.env import env
|
||||
|
||||
# Suppress specific warnings from django-rest-auth: https://github.com/iMerica/dj-rest-auth/issues/684
|
||||
|
||||
@@ -276,7 +276,7 @@ FINDINGS_MAX_DAYS_IN_RANGE = env.int("DJANGO_FINDINGS_MAX_DAYS_IN_RANGE", 7)
|
||||
DJANGO_TMP_OUTPUT_DIRECTORY = env.str(
|
||||
"DJANGO_TMP_OUTPUT_DIRECTORY", "/tmp/prowler_api_output"
|
||||
)
|
||||
DJANGO_FINDINGS_BATCH_SIZE = env.str("DJANGO_FINDINGS_BATCH_SIZE", 1000)
|
||||
DJANGO_FINDINGS_BATCH_SIZE = env.int("DJANGO_FINDINGS_BATCH_SIZE", 1000)
|
||||
|
||||
DJANGO_OUTPUT_S3_AWS_OUTPUT_BUCKET = env.str("DJANGO_OUTPUT_S3_AWS_OUTPUT_BUCKET", "")
|
||||
DJANGO_OUTPUT_S3_AWS_ACCESS_KEY_ID = env.str("DJANGO_OUTPUT_S3_AWS_ACCESS_KEY_ID", "")
|
||||
|
||||
@@ -44,6 +44,12 @@ DATABASES = {
|
||||
"HOST": env("POSTGRES_REPLICA_HOST", default=default_db_host),
|
||||
"PORT": env("POSTGRES_REPLICA_PORT", default=default_db_port),
|
||||
},
|
||||
"neo4j": {
|
||||
"HOST": env.str("NEO4J_HOST", "neo4j"),
|
||||
"PORT": env.str("NEO4J_PORT", "7687"),
|
||||
"USER": env.str("NEO4J_USER", "neo4j"),
|
||||
"PASSWORD": env.str("NEO4J_PASSWORD", "neo4j_password"),
|
||||
},
|
||||
}
|
||||
|
||||
DATABASES["default"] = DATABASES["prowler_user"]
|
||||
|
||||
@@ -45,6 +45,12 @@ DATABASES = {
|
||||
"HOST": env("POSTGRES_REPLICA_HOST", default=default_db_host),
|
||||
"PORT": env("POSTGRES_REPLICA_PORT", default=default_db_port),
|
||||
},
|
||||
"neo4j": {
|
||||
"HOST": env.str("NEO4J_HOST"),
|
||||
"PORT": env.str("NEO4J_PORT"),
|
||||
"USER": env.str("NEO4J_USER"),
|
||||
"PASSWORD": env.str("NEO4J_PASSWORD"),
|
||||
},
|
||||
}
|
||||
|
||||
DATABASES["default"] = DATABASES["prowler_user"]
|
||||
|
||||
+221
-7
@@ -1,5 +1,6 @@
|
||||
import logging
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
@@ -14,10 +15,16 @@ from rest_framework.test import APIClient
|
||||
from tasks.jobs.backfill import (
|
||||
backfill_resource_scan_summaries,
|
||||
backfill_scan_category_summaries,
|
||||
backfill_scan_resource_group_summaries,
|
||||
)
|
||||
|
||||
from api.attack_paths import (
|
||||
AttackPathsQueryDefinition,
|
||||
AttackPathsQueryParameterDefinition,
|
||||
)
|
||||
from api.db_utils import rls_transaction
|
||||
from api.models import (
|
||||
AttackPathsScan,
|
||||
AttackSurfaceOverview,
|
||||
ComplianceOverview,
|
||||
ComplianceRequirementOverview,
|
||||
@@ -41,6 +48,7 @@ from api.models import (
|
||||
SAMLDomainIndex,
|
||||
Scan,
|
||||
ScanCategorySummary,
|
||||
ScanGroupSummary,
|
||||
ScanSummary,
|
||||
StateChoices,
|
||||
StatusChoices,
|
||||
@@ -166,22 +174,20 @@ def create_test_user_rbac_no_roles(django_db_setup, django_db_blocker, tenants_f
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def create_test_user_rbac_limited(django_db_setup, django_db_blocker):
|
||||
def create_test_user_rbac_limited(django_db_setup, django_db_blocker, tenants_fixture):
|
||||
with django_db_blocker.unblock():
|
||||
user = User.objects.create_user(
|
||||
name="testing_limited",
|
||||
email="rbac_limited@rbac.com",
|
||||
password=TEST_PASSWORD,
|
||||
)
|
||||
tenant = Tenant.objects.create(
|
||||
name="Tenant Test",
|
||||
)
|
||||
tenant = tenants_fixture[0]
|
||||
Membership.objects.create(
|
||||
user=user,
|
||||
tenant=tenant,
|
||||
role=Membership.RoleChoices.OWNER,
|
||||
)
|
||||
Role.objects.create(
|
||||
role = Role.objects.create(
|
||||
name="limited",
|
||||
tenant_id=tenant.id,
|
||||
manage_users=False,
|
||||
@@ -194,7 +200,7 @@ def create_test_user_rbac_limited(django_db_setup, django_db_blocker):
|
||||
)
|
||||
UserRoleRelationship.objects.create(
|
||||
user=user,
|
||||
role=Role.objects.get(name="limited"),
|
||||
role=role,
|
||||
tenant_id=tenant.id,
|
||||
)
|
||||
return user
|
||||
@@ -525,6 +531,12 @@ def providers_fixture(tenants_fixture):
|
||||
alias="alibabacloud_testing",
|
||||
tenant_id=tenant.id,
|
||||
)
|
||||
provider10 = Provider.objects.create(
|
||||
provider="cloudflare",
|
||||
uid="a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4",
|
||||
alias="cloudflare_testing",
|
||||
tenant_id=tenant.id,
|
||||
)
|
||||
|
||||
return (
|
||||
provider1,
|
||||
@@ -536,6 +548,7 @@ def providers_fixture(tenants_fixture):
|
||||
provider7,
|
||||
provider8,
|
||||
provider9,
|
||||
provider10,
|
||||
)
|
||||
|
||||
|
||||
@@ -739,6 +752,7 @@ def resources_fixture(providers_fixture):
|
||||
region="us-east-1",
|
||||
service="ec2",
|
||||
type="prowler-test",
|
||||
groups=["compute"],
|
||||
)
|
||||
|
||||
resource1.upsert_or_delete_tags(tags)
|
||||
@@ -751,6 +765,7 @@ def resources_fixture(providers_fixture):
|
||||
region="eu-west-1",
|
||||
service="s3",
|
||||
type="prowler-test",
|
||||
groups=["storage"],
|
||||
)
|
||||
resource2.upsert_or_delete_tags(tags)
|
||||
|
||||
@@ -762,6 +777,7 @@ def resources_fixture(providers_fixture):
|
||||
region="us-east-1",
|
||||
service="ec2",
|
||||
type="test",
|
||||
groups=["compute"],
|
||||
)
|
||||
|
||||
tags = [
|
||||
@@ -1234,7 +1250,7 @@ def lighthouse_config_fixture(authenticated_client, tenants_fixture):
|
||||
return LighthouseConfiguration.objects.create(
|
||||
tenant_id=tenants_fixture[0].id,
|
||||
name="OpenAI",
|
||||
api_key_decoded="sk-test1234567890T3BlbkFJtest1234567890",
|
||||
api_key_decoded="sk-fake-test-key-for-unit-testing-only",
|
||||
model="gpt-4o",
|
||||
temperature=0,
|
||||
max_tokens=4000,
|
||||
@@ -1383,11 +1399,13 @@ def latest_scan_finding_with_categories(
|
||||
check_id="genai_iam_check",
|
||||
check_metadata={"CheckId": "genai_iam_check"},
|
||||
categories=["gen-ai", "iam"],
|
||||
resource_groups="ai_ml",
|
||||
first_seen_at="2024-01-02T00:00:00Z",
|
||||
)
|
||||
finding.add_resources([resource])
|
||||
backfill_resource_scan_summaries(tenant_id, str(scan.id))
|
||||
backfill_scan_category_summaries(tenant_id, str(scan.id))
|
||||
backfill_scan_resource_group_summaries(tenant_id, str(scan.id))
|
||||
return finding
|
||||
|
||||
|
||||
@@ -1590,6 +1608,105 @@ def mute_rules_fixture(tenants_fixture, create_test_user, findings_fixture):
|
||||
return mute_rule1, mute_rule2
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def create_attack_paths_scan():
|
||||
"""Factory fixture to create Attack Paths scans for tests."""
|
||||
|
||||
def _create(
|
||||
provider,
|
||||
*,
|
||||
scan=None,
|
||||
state=StateChoices.COMPLETED,
|
||||
progress=0,
|
||||
graph_database="tenant-db",
|
||||
**extra_fields,
|
||||
):
|
||||
scan_instance = scan or Scan.objects.create(
|
||||
name=extra_fields.pop("scan_name", "Attack Paths Supporting Scan"),
|
||||
provider=provider,
|
||||
trigger=Scan.TriggerChoices.MANUAL,
|
||||
state=extra_fields.pop("scan_state", StateChoices.COMPLETED),
|
||||
tenant_id=provider.tenant_id,
|
||||
)
|
||||
|
||||
payload = {
|
||||
"tenant_id": provider.tenant_id,
|
||||
"provider": provider,
|
||||
"scan": scan_instance,
|
||||
"state": state,
|
||||
"progress": progress,
|
||||
"graph_database": graph_database,
|
||||
}
|
||||
payload.update(extra_fields)
|
||||
|
||||
return AttackPathsScan.objects.create(**payload)
|
||||
|
||||
return _create
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def attack_paths_query_definition_factory():
|
||||
"""Factory fixture for building Attack Paths query definitions."""
|
||||
|
||||
def _create(**overrides):
|
||||
cast_type = overrides.pop("cast_type", str)
|
||||
parameters = overrides.pop(
|
||||
"parameters",
|
||||
[
|
||||
AttackPathsQueryParameterDefinition(
|
||||
name="limit",
|
||||
label="Limit",
|
||||
cast=cast_type,
|
||||
)
|
||||
],
|
||||
)
|
||||
definition_payload = {
|
||||
"id": "aws-test",
|
||||
"name": "Attack Paths Test Query",
|
||||
"short_description": "Synthetic short description for tests.",
|
||||
"description": "Synthetic Attack Paths definition for tests.",
|
||||
"provider": "aws",
|
||||
"cypher": "RETURN 1",
|
||||
"parameters": parameters,
|
||||
}
|
||||
definition_payload.update(overrides)
|
||||
return AttackPathsQueryDefinition(**definition_payload)
|
||||
|
||||
return _create
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def attack_paths_graph_stub_classes():
|
||||
"""Provide lightweight graph element stubs for Attack Paths serialization tests."""
|
||||
|
||||
class AttackPathsNativeValue:
|
||||
def __init__(self, value):
|
||||
self._value = value
|
||||
|
||||
def to_native(self):
|
||||
return self._value
|
||||
|
||||
class AttackPathsNode:
|
||||
def __init__(self, element_id, labels, properties):
|
||||
self.element_id = element_id
|
||||
self.labels = labels
|
||||
self._properties = properties
|
||||
|
||||
class AttackPathsRelationship:
|
||||
def __init__(self, element_id, rel_type, start_node, end_node, properties):
|
||||
self.element_id = element_id
|
||||
self.type = rel_type
|
||||
self.start_node = start_node
|
||||
self.end_node = end_node
|
||||
self._properties = properties
|
||||
|
||||
return SimpleNamespace(
|
||||
NativeValue=AttackPathsNativeValue,
|
||||
Node=AttackPathsNode,
|
||||
Relationship=AttackPathsRelationship,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def create_attack_surface_overview():
|
||||
def _create(tenant, scan, attack_surface_type, total=10, failed=5, muted_failed=2):
|
||||
@@ -1629,6 +1746,103 @@ def create_scan_category_summary():
|
||||
return _create
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def findings_with_group(scans_fixture, resources_fixture):
|
||||
scan = scans_fixture[0]
|
||||
resource = resources_fixture[0]
|
||||
|
||||
finding = Finding.objects.create(
|
||||
tenant_id=scan.tenant_id,
|
||||
uid="finding_with_group_1",
|
||||
scan=scan,
|
||||
delta=None,
|
||||
status=Status.FAIL,
|
||||
status_extended="test status",
|
||||
impact=Severity.critical,
|
||||
impact_extended="test impact",
|
||||
severity=Severity.critical,
|
||||
raw_result={"status": Status.FAIL},
|
||||
check_id="storage_check",
|
||||
check_metadata={"CheckId": "storage_check"},
|
||||
resource_groups="storage",
|
||||
first_seen_at="2024-01-02T00:00:00Z",
|
||||
)
|
||||
finding.add_resources([resource])
|
||||
backfill_resource_scan_summaries(str(scan.tenant_id), str(scan.id))
|
||||
return finding
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def findings_with_multiple_groups(scans_fixture, resources_fixture):
|
||||
scan = scans_fixture[0]
|
||||
resource1, resource2 = resources_fixture[:2]
|
||||
|
||||
finding1 = Finding.objects.create(
|
||||
tenant_id=scan.tenant_id,
|
||||
uid="finding_multi_grp_1",
|
||||
scan=scan,
|
||||
delta=None,
|
||||
status=Status.FAIL,
|
||||
status_extended="test status",
|
||||
impact=Severity.critical,
|
||||
impact_extended="test impact",
|
||||
severity=Severity.critical,
|
||||
raw_result={"status": Status.FAIL},
|
||||
check_id="storage_check",
|
||||
check_metadata={"CheckId": "storage_check"},
|
||||
resource_groups="storage",
|
||||
first_seen_at="2024-01-02T00:00:00Z",
|
||||
)
|
||||
finding1.add_resources([resource1])
|
||||
|
||||
finding2 = Finding.objects.create(
|
||||
tenant_id=scan.tenant_id,
|
||||
uid="finding_multi_grp_2",
|
||||
scan=scan,
|
||||
delta=None,
|
||||
status=Status.FAIL,
|
||||
status_extended="test status 2",
|
||||
impact=Severity.high,
|
||||
impact_extended="test impact 2",
|
||||
severity=Severity.high,
|
||||
raw_result={"status": Status.FAIL},
|
||||
check_id="security_check",
|
||||
check_metadata={"CheckId": "security_check"},
|
||||
resource_groups="security",
|
||||
first_seen_at="2024-01-02T00:00:00Z",
|
||||
)
|
||||
finding2.add_resources([resource2])
|
||||
|
||||
backfill_resource_scan_summaries(str(scan.tenant_id), str(scan.id))
|
||||
return finding1, finding2
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def create_scan_resource_group_summary():
|
||||
def _create(
|
||||
tenant,
|
||||
scan,
|
||||
resource_group,
|
||||
severity,
|
||||
total_findings=10,
|
||||
failed_findings=5,
|
||||
new_failed_findings=2,
|
||||
resources_count=3,
|
||||
):
|
||||
return ScanGroupSummary.objects.create(
|
||||
tenant=tenant,
|
||||
scan=scan,
|
||||
resource_group=resource_group,
|
||||
severity=severity,
|
||||
total_findings=total_findings,
|
||||
failed_findings=failed_findings,
|
||||
new_failed_findings=new_failed_findings,
|
||||
resources_count=resources_count,
|
||||
)
|
||||
|
||||
return _create
|
||||
|
||||
|
||||
def get_authorization_header(access_token: str) -> dict:
|
||||
return {"Authorization": f"Bearer {access_token}"}
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ from tasks.tasks import perform_scheduled_scan_task
|
||||
from api.db_utils import rls_transaction
|
||||
from api.exceptions import ConflictException
|
||||
from api.models import Provider, Scan, StateChoices
|
||||
from tasks.jobs.attack_paths import db_utils as attack_paths_db_utils
|
||||
|
||||
|
||||
def schedule_provider_scan(provider_instance: Provider):
|
||||
@@ -39,6 +40,12 @@ def schedule_provider_scan(provider_instance: Provider):
|
||||
scheduled_at=datetime.now(timezone.utc),
|
||||
)
|
||||
|
||||
attack_paths_db_utils.create_attack_paths_scan(
|
||||
tenant_id=tenant_id,
|
||||
scan_id=str(scheduled_scan.id),
|
||||
provider_id=provider_id,
|
||||
)
|
||||
|
||||
# Schedule the task
|
||||
periodic_task_instance = PeriodicTask.objects.create(
|
||||
interval=schedule,
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
from tasks.jobs.attack_paths.db_utils import can_provider_run_attack_paths_scan
|
||||
from tasks.jobs.attack_paths.scan import run as attack_paths_scan
|
||||
|
||||
__all__ = [
|
||||
"attack_paths_scan",
|
||||
"can_provider_run_attack_paths_scan",
|
||||
]
|
||||
@@ -0,0 +1,253 @@
|
||||
# Portions of this file are based on code from the Cartography project
|
||||
# (https://github.com/cartography-cncf/cartography), which is licensed under the Apache 2.0 License.
|
||||
|
||||
from typing import Any
|
||||
|
||||
import aioboto3
|
||||
import boto3
|
||||
import neo4j
|
||||
|
||||
from cartography.config import Config as CartographyConfig
|
||||
from cartography.intel import aws as cartography_aws
|
||||
from celery.utils.log import get_task_logger
|
||||
|
||||
from api.models import (
|
||||
AttackPathsScan as ProwlerAPIAttackPathsScan,
|
||||
Provider as ProwlerAPIProvider,
|
||||
)
|
||||
from prowler.providers.common.provider import Provider as ProwlerSDKProvider
|
||||
from tasks.jobs.attack_paths import db_utils, utils
|
||||
|
||||
logger = get_task_logger(__name__)
|
||||
|
||||
|
||||
def start_aws_ingestion(
|
||||
neo4j_session: neo4j.Session,
|
||||
cartography_config: CartographyConfig,
|
||||
prowler_api_provider: ProwlerAPIProvider,
|
||||
prowler_sdk_provider: ProwlerSDKProvider,
|
||||
attack_paths_scan: ProwlerAPIAttackPathsScan,
|
||||
) -> dict[str, dict[str, str]]:
|
||||
"""
|
||||
Code based on Cartography, specifically on `cartography.intel.aws.__init__.py`.
|
||||
|
||||
For the scan progress updates:
|
||||
- The caller of this function (`tasks.jobs.attack_paths.scan.run`) has set it to 2.
|
||||
- When the control returns to the caller, it will be set to 95.
|
||||
"""
|
||||
|
||||
# Initialize variables common to all jobs
|
||||
common_job_parameters = {
|
||||
"UPDATE_TAG": cartography_config.update_tag,
|
||||
"permission_relationships_file": cartography_config.permission_relationships_file,
|
||||
"aws_guardduty_severity_threshold": cartography_config.aws_guardduty_severity_threshold,
|
||||
"aws_cloudtrail_management_events_lookback_hours": cartography_config.aws_cloudtrail_management_events_lookback_hours,
|
||||
"experimental_aws_inspector_batch": cartography_config.experimental_aws_inspector_batch,
|
||||
}
|
||||
|
||||
boto3_session = get_boto3_session(prowler_api_provider, prowler_sdk_provider)
|
||||
regions: list[str] = list(prowler_sdk_provider._enabled_regions)
|
||||
requested_syncs = list(cartography_aws.RESOURCE_FUNCTIONS.keys())
|
||||
|
||||
sync_args = cartography_aws._build_aws_sync_kwargs(
|
||||
neo4j_session,
|
||||
boto3_session,
|
||||
regions,
|
||||
prowler_api_provider.uid,
|
||||
cartography_config.update_tag,
|
||||
common_job_parameters,
|
||||
)
|
||||
|
||||
# Starting with sync functions
|
||||
logger.info(f"Syncing organizations for AWS account {prowler_api_provider.uid}")
|
||||
cartography_aws.organizations.sync(
|
||||
neo4j_session,
|
||||
{prowler_api_provider.alias: prowler_api_provider.uid},
|
||||
cartography_config.update_tag,
|
||||
common_job_parameters,
|
||||
)
|
||||
db_utils.update_attack_paths_scan_progress(attack_paths_scan, 3)
|
||||
|
||||
# Adding an extra field
|
||||
common_job_parameters["AWS_ID"] = prowler_api_provider.uid
|
||||
|
||||
cartography_aws._autodiscover_accounts(
|
||||
neo4j_session,
|
||||
boto3_session,
|
||||
prowler_api_provider.uid,
|
||||
cartography_config.update_tag,
|
||||
common_job_parameters,
|
||||
)
|
||||
db_utils.update_attack_paths_scan_progress(attack_paths_scan, 4)
|
||||
|
||||
failed_syncs = sync_aws_account(
|
||||
prowler_api_provider, requested_syncs, sync_args, attack_paths_scan
|
||||
)
|
||||
|
||||
if "permission_relationships" in requested_syncs:
|
||||
logger.info(
|
||||
f"Syncing function permission_relationships for AWS account {prowler_api_provider.uid}"
|
||||
)
|
||||
cartography_aws.RESOURCE_FUNCTIONS["permission_relationships"](**sync_args)
|
||||
db_utils.update_attack_paths_scan_progress(attack_paths_scan, 88)
|
||||
|
||||
if "resourcegroupstaggingapi" in requested_syncs:
|
||||
logger.info(
|
||||
f"Syncing function resourcegroupstaggingapi for AWS account {prowler_api_provider.uid}"
|
||||
)
|
||||
cartography_aws.RESOURCE_FUNCTIONS["resourcegroupstaggingapi"](**sync_args)
|
||||
db_utils.update_attack_paths_scan_progress(attack_paths_scan, 89)
|
||||
|
||||
logger.info(
|
||||
f"Syncing ec2_iaminstanceprofile scoped analysis for AWS account {prowler_api_provider.uid}"
|
||||
)
|
||||
cartography_aws.run_scoped_analysis_job(
|
||||
"aws_ec2_iaminstanceprofile.json",
|
||||
neo4j_session,
|
||||
common_job_parameters,
|
||||
)
|
||||
db_utils.update_attack_paths_scan_progress(attack_paths_scan, 90)
|
||||
|
||||
logger.info(
|
||||
f"Syncing lambda_ecr analysis for AWS account {prowler_api_provider.uid}"
|
||||
)
|
||||
cartography_aws.run_analysis_job(
|
||||
"aws_lambda_ecr.json",
|
||||
neo4j_session,
|
||||
common_job_parameters,
|
||||
)
|
||||
db_utils.update_attack_paths_scan_progress(attack_paths_scan, 91)
|
||||
|
||||
logger.info(f"Syncing metadata for AWS account {prowler_api_provider.uid}")
|
||||
cartography_aws.merge_module_sync_metadata(
|
||||
neo4j_session,
|
||||
group_type="AWSAccount",
|
||||
group_id=prowler_api_provider.uid,
|
||||
synced_type="AWSAccount",
|
||||
update_tag=cartography_config.update_tag,
|
||||
stat_handler=cartography_aws.stat_handler,
|
||||
)
|
||||
db_utils.update_attack_paths_scan_progress(attack_paths_scan, 92)
|
||||
|
||||
# Removing the added extra field
|
||||
del common_job_parameters["AWS_ID"]
|
||||
|
||||
logger.info(f"Syncing cleanup_job for AWS account {prowler_api_provider.uid}")
|
||||
cartography_aws.run_cleanup_job(
|
||||
"aws_post_ingestion_principals_cleanup.json",
|
||||
neo4j_session,
|
||||
common_job_parameters,
|
||||
)
|
||||
db_utils.update_attack_paths_scan_progress(attack_paths_scan, 93)
|
||||
|
||||
logger.info(f"Syncing analysis for AWS account {prowler_api_provider.uid}")
|
||||
cartography_aws._perform_aws_analysis(
|
||||
requested_syncs, neo4j_session, common_job_parameters
|
||||
)
|
||||
db_utils.update_attack_paths_scan_progress(attack_paths_scan, 94)
|
||||
|
||||
return failed_syncs
|
||||
|
||||
|
||||
def get_boto3_session(
|
||||
prowler_api_provider: ProwlerAPIProvider, prowler_sdk_provider: ProwlerSDKProvider
|
||||
) -> boto3.Session:
|
||||
boto3_session = prowler_sdk_provider.session.current_session
|
||||
|
||||
aws_accounts_from_session = cartography_aws.organizations.get_aws_account_default(
|
||||
boto3_session
|
||||
)
|
||||
if not aws_accounts_from_session:
|
||||
raise Exception(
|
||||
"No valid AWS credentials could be found. No AWS accounts can be synced."
|
||||
)
|
||||
|
||||
aws_account_id_from_session = list(aws_accounts_from_session.values())[0]
|
||||
if prowler_api_provider.uid != aws_account_id_from_session:
|
||||
raise Exception(
|
||||
f"Provider {prowler_api_provider.uid} doesn't match AWS account {aws_account_id_from_session}."
|
||||
)
|
||||
|
||||
if boto3_session.region_name is None:
|
||||
global_region = prowler_sdk_provider.get_global_region()
|
||||
boto3_session._session.set_config_variable("region", global_region)
|
||||
|
||||
return boto3_session
|
||||
|
||||
|
||||
def get_aioboto3_session(boto3_session: boto3.Session) -> aioboto3.Session:
|
||||
return aioboto3.Session(botocore_session=boto3_session._session)
|
||||
|
||||
|
||||
def sync_aws_account(
|
||||
prowler_api_provider: ProwlerAPIProvider,
|
||||
requested_syncs: list[str],
|
||||
sync_args: dict[str, Any],
|
||||
attack_paths_scan: ProwlerAPIAttackPathsScan,
|
||||
) -> dict[str, str]:
|
||||
current_progress = 4 # `cartography_aws._autodiscover_accounts`
|
||||
max_progress = (
|
||||
87 # `cartography_aws.RESOURCE_FUNCTIONS["permission_relationships"]` - 1
|
||||
)
|
||||
n_steps = (
|
||||
len(requested_syncs) - 2
|
||||
) # Excluding `permission_relationships` and `resourcegroupstaggingapi`
|
||||
progress_step = (max_progress - current_progress) / n_steps
|
||||
|
||||
failed_syncs = {}
|
||||
|
||||
for func_name in requested_syncs:
|
||||
if func_name in cartography_aws.RESOURCE_FUNCTIONS:
|
||||
logger.info(
|
||||
f"Syncing function {func_name} for AWS account {prowler_api_provider.uid}"
|
||||
)
|
||||
|
||||
# Updating progress, not really the right place but good enough
|
||||
current_progress += progress_step
|
||||
db_utils.update_attack_paths_scan_progress(
|
||||
attack_paths_scan, int(current_progress)
|
||||
)
|
||||
|
||||
try:
|
||||
# `ecr:image_layers` uses `aioboto3_session` instead of `boto3_session`
|
||||
if func_name == "ecr:image_layers":
|
||||
cartography_aws.RESOURCE_FUNCTIONS[func_name](
|
||||
neo4j_session=sync_args.get("neo4j_session"),
|
||||
aioboto3_session=get_aioboto3_session(
|
||||
sync_args.get("boto3_session")
|
||||
),
|
||||
regions=sync_args.get("regions"),
|
||||
current_aws_account_id=sync_args.get("current_aws_account_id"),
|
||||
update_tag=sync_args.get("update_tag"),
|
||||
common_job_parameters=sync_args.get("common_job_parameters"),
|
||||
)
|
||||
|
||||
# Skip permission relationships and tags for now because they rely on data already being in the graph
|
||||
elif func_name in [
|
||||
"permission_relationships",
|
||||
"resourcegroupstaggingapi",
|
||||
]:
|
||||
continue
|
||||
|
||||
else:
|
||||
cartography_aws.RESOURCE_FUNCTIONS[func_name](**sync_args)
|
||||
|
||||
except Exception as e:
|
||||
exception_message = utils.stringify_exception(
|
||||
e, f"Exception for AWS sync function: {func_name}"
|
||||
)
|
||||
failed_syncs[func_name] = exception_message
|
||||
|
||||
logger.warning(
|
||||
f"Caught exception syncing function {func_name} from AWS account {prowler_api_provider.uid}. We "
|
||||
"are continuing on to the next AWS sync function.",
|
||||
)
|
||||
|
||||
continue
|
||||
|
||||
else:
|
||||
raise ValueError(
|
||||
f'AWS sync function "{func_name}" was specified but does not exist. Did you misspell it?'
|
||||
)
|
||||
|
||||
return failed_syncs
|
||||
@@ -0,0 +1,88 @@
|
||||
from dataclasses import dataclass
|
||||
from typing import Callable
|
||||
|
||||
from config.env import env
|
||||
|
||||
from tasks.jobs.attack_paths import aws
|
||||
|
||||
|
||||
# Batch size for Neo4j operations
|
||||
BATCH_SIZE = env.int("ATTACK_PATHS_BATCH_SIZE", 1000)
|
||||
|
||||
# Neo4j internal labels (Prowler-specific, not provider-specific)
|
||||
# - `ProwlerFinding`: Label for finding nodes created by Prowler and linked to cloud resources.
|
||||
# - `ProviderResource`: Added to ALL synced nodes for provider isolation and drop/query ops.
|
||||
# - `Internet`: Singleton node representing external internet access for exposed-resource queries.
|
||||
PROWLER_FINDING_LABEL = "ProwlerFinding"
|
||||
PROVIDER_RESOURCE_LABEL = "ProviderResource"
|
||||
INTERNET_NODE_LABEL = "Internet"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ProviderConfig:
|
||||
"""Configuration for a cloud provider's Attack Paths integration."""
|
||||
|
||||
name: str
|
||||
root_node_label: str # e.g., "AWSAccount"
|
||||
uid_field: str # e.g., "arn"
|
||||
# Label for resources connected to the account node, enabling indexed finding lookups.
|
||||
resource_label: str # e.g., "AWSResource"
|
||||
ingestion_function: Callable
|
||||
|
||||
|
||||
# Provider Configurations
|
||||
# -----------------------
|
||||
|
||||
AWS_CONFIG = ProviderConfig(
|
||||
name="aws",
|
||||
root_node_label="AWSAccount",
|
||||
uid_field="arn",
|
||||
resource_label="AWSResource",
|
||||
ingestion_function=aws.start_aws_ingestion,
|
||||
)
|
||||
|
||||
PROVIDER_CONFIGS: dict[str, ProviderConfig] = {
|
||||
"aws": AWS_CONFIG,
|
||||
}
|
||||
|
||||
# Labels added by Prowler that should be filtered from API responses
|
||||
# Derived from provider configs + common internal labels
|
||||
INTERNAL_LABELS: list[str] = [
|
||||
"Tenant",
|
||||
PROVIDER_RESOURCE_LABEL,
|
||||
# Add all provider-specific resource labels
|
||||
*[config.resource_label for config in PROVIDER_CONFIGS.values()],
|
||||
]
|
||||
|
||||
|
||||
# Provider Config Accessors
|
||||
# -------------------------
|
||||
|
||||
|
||||
def is_provider_available(provider_type: str) -> bool:
|
||||
"""Check if a provider type is available for Attack Paths scans."""
|
||||
return provider_type in PROVIDER_CONFIGS
|
||||
|
||||
|
||||
def get_cartography_ingestion_function(provider_type: str) -> Callable | None:
|
||||
"""Get the Cartography ingestion function for a provider type."""
|
||||
config = PROVIDER_CONFIGS.get(provider_type)
|
||||
return config.ingestion_function if config else None
|
||||
|
||||
|
||||
def get_root_node_label(provider_type: str) -> str:
|
||||
"""Get the root node label for a provider type (e.g., AWSAccount)."""
|
||||
config = PROVIDER_CONFIGS.get(provider_type)
|
||||
return config.root_node_label if config else "UnknownProviderAccount"
|
||||
|
||||
|
||||
def get_node_uid_field(provider_type: str) -> str:
|
||||
"""Get the UID field for a provider type (e.g., arn for AWS)."""
|
||||
config = PROVIDER_CONFIGS.get(provider_type)
|
||||
return config.uid_field if config else "UnknownProviderUID"
|
||||
|
||||
|
||||
def get_provider_resource_label(provider_type: str) -> str:
|
||||
"""Get the resource label for a provider type (e.g., `AWSResource`)."""
|
||||
config = PROVIDER_CONFIGS.get(provider_type)
|
||||
return config.resource_label if config else "UnknownProviderResource"
|
||||
@@ -0,0 +1,146 @@
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
from cartography.config import Config as CartographyConfig
|
||||
|
||||
from api.db_utils import rls_transaction
|
||||
from api.models import (
|
||||
AttackPathsScan as ProwlerAPIAttackPathsScan,
|
||||
Provider as ProwlerAPIProvider,
|
||||
StateChoices,
|
||||
)
|
||||
from tasks.jobs.attack_paths.config import is_provider_available
|
||||
|
||||
|
||||
def can_provider_run_attack_paths_scan(tenant_id: str, provider_id: int) -> bool:
|
||||
with rls_transaction(tenant_id):
|
||||
prowler_api_provider = ProwlerAPIProvider.objects.get(id=provider_id)
|
||||
|
||||
return is_provider_available(prowler_api_provider.provider)
|
||||
|
||||
|
||||
def create_attack_paths_scan(
|
||||
tenant_id: str,
|
||||
scan_id: str,
|
||||
provider_id: int,
|
||||
) -> ProwlerAPIAttackPathsScan | None:
|
||||
if not can_provider_run_attack_paths_scan(tenant_id, provider_id):
|
||||
return None
|
||||
|
||||
with rls_transaction(tenant_id):
|
||||
attack_paths_scan = ProwlerAPIAttackPathsScan.objects.create(
|
||||
tenant_id=tenant_id,
|
||||
provider_id=provider_id,
|
||||
scan_id=scan_id,
|
||||
state=StateChoices.SCHEDULED,
|
||||
started_at=datetime.now(tz=timezone.utc),
|
||||
)
|
||||
attack_paths_scan.save()
|
||||
|
||||
return attack_paths_scan
|
||||
|
||||
|
||||
def retrieve_attack_paths_scan(
|
||||
tenant_id: str,
|
||||
scan_id: str,
|
||||
) -> ProwlerAPIAttackPathsScan | None:
|
||||
try:
|
||||
with rls_transaction(tenant_id):
|
||||
attack_paths_scan = ProwlerAPIAttackPathsScan.objects.get(
|
||||
scan_id=scan_id,
|
||||
)
|
||||
|
||||
return attack_paths_scan
|
||||
|
||||
except ProwlerAPIAttackPathsScan.DoesNotExist:
|
||||
return None
|
||||
|
||||
|
||||
def starting_attack_paths_scan(
|
||||
attack_paths_scan: ProwlerAPIAttackPathsScan,
|
||||
task_id: str,
|
||||
cartography_config: CartographyConfig,
|
||||
) -> None:
|
||||
with rls_transaction(attack_paths_scan.tenant_id):
|
||||
attack_paths_scan.task_id = task_id
|
||||
attack_paths_scan.state = StateChoices.EXECUTING
|
||||
attack_paths_scan.started_at = datetime.now(tz=timezone.utc)
|
||||
attack_paths_scan.update_tag = cartography_config.update_tag
|
||||
attack_paths_scan.graph_database = cartography_config.neo4j_database
|
||||
|
||||
attack_paths_scan.save(
|
||||
update_fields=[
|
||||
"task_id",
|
||||
"state",
|
||||
"started_at",
|
||||
"update_tag",
|
||||
"graph_database",
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def finish_attack_paths_scan(
|
||||
attack_paths_scan: ProwlerAPIAttackPathsScan,
|
||||
state: StateChoices,
|
||||
ingestion_exceptions: dict[str, Any],
|
||||
) -> None:
|
||||
with rls_transaction(attack_paths_scan.tenant_id):
|
||||
now = datetime.now(tz=timezone.utc)
|
||||
duration = int((now - attack_paths_scan.started_at).total_seconds())
|
||||
|
||||
attack_paths_scan.state = state
|
||||
attack_paths_scan.progress = 100
|
||||
attack_paths_scan.completed_at = now
|
||||
attack_paths_scan.duration = duration
|
||||
attack_paths_scan.ingestion_exceptions = ingestion_exceptions
|
||||
|
||||
attack_paths_scan.save(
|
||||
update_fields=[
|
||||
"state",
|
||||
"progress",
|
||||
"completed_at",
|
||||
"duration",
|
||||
"ingestion_exceptions",
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def update_attack_paths_scan_progress(
|
||||
attack_paths_scan: ProwlerAPIAttackPathsScan,
|
||||
progress: int,
|
||||
) -> None:
|
||||
with rls_transaction(attack_paths_scan.tenant_id):
|
||||
attack_paths_scan.progress = progress
|
||||
attack_paths_scan.save(update_fields=["progress"])
|
||||
|
||||
|
||||
def get_old_attack_paths_scans(
|
||||
tenant_id: str,
|
||||
provider_id: str,
|
||||
attack_paths_scan_id: str,
|
||||
) -> list[ProwlerAPIAttackPathsScan]:
|
||||
"""
|
||||
An `old_attack_paths_scan` is any `completed` Attack Paths scan for the same provider,
|
||||
with its graph database not deleted, excluding the current Attack Paths scan.
|
||||
"""
|
||||
|
||||
with rls_transaction(tenant_id):
|
||||
completed_scans_qs = (
|
||||
ProwlerAPIAttackPathsScan.objects.filter(
|
||||
provider_id=provider_id,
|
||||
state=StateChoices.COMPLETED,
|
||||
is_graph_database_deleted=False,
|
||||
)
|
||||
.exclude(id=attack_paths_scan_id)
|
||||
.all()
|
||||
)
|
||||
|
||||
return list(completed_scans_qs)
|
||||
|
||||
|
||||
def update_old_attack_paths_scan(
|
||||
old_attack_paths_scan: ProwlerAPIAttackPathsScan,
|
||||
) -> None:
|
||||
with rls_transaction(old_attack_paths_scan.tenant_id):
|
||||
old_attack_paths_scan.is_graph_database_deleted = True
|
||||
old_attack_paths_scan.save(update_fields=["is_graph_database_deleted"])
|
||||
@@ -0,0 +1,355 @@
|
||||
"""
|
||||
Prowler findings ingestion into Neo4j graph.
|
||||
|
||||
This module handles:
|
||||
- Adding resource labels to Cartography nodes for efficient lookups
|
||||
- Loading Prowler findings into the graph
|
||||
- Linking findings to resources
|
||||
- Cleaning up stale findings
|
||||
"""
|
||||
|
||||
from collections import defaultdict
|
||||
from dataclasses import asdict, dataclass, fields
|
||||
from typing import Any, Generator
|
||||
from uuid import UUID
|
||||
|
||||
import neo4j
|
||||
|
||||
from cartography.config import Config as CartographyConfig
|
||||
from celery.utils.log import get_task_logger
|
||||
|
||||
from api.db_router import READ_REPLICA_ALIAS
|
||||
from api.db_utils import rls_transaction
|
||||
from api.models import Finding as FindingModel
|
||||
from api.models import Provider, ResourceFindingMapping
|
||||
from prowler.config import config as ProwlerConfig
|
||||
from tasks.jobs.attack_paths.config import (
|
||||
BATCH_SIZE,
|
||||
get_node_uid_field,
|
||||
get_provider_resource_label,
|
||||
get_root_node_label,
|
||||
)
|
||||
from tasks.jobs.attack_paths.indexes import IndexType, create_indexes
|
||||
from tasks.jobs.attack_paths.queries import (
|
||||
ADD_RESOURCE_LABEL_TEMPLATE,
|
||||
CLEANUP_FINDINGS_TEMPLATE,
|
||||
INSERT_FINDING_TEMPLATE,
|
||||
render_cypher_template,
|
||||
)
|
||||
|
||||
logger = get_task_logger(__name__)
|
||||
|
||||
|
||||
# Type Definitions
|
||||
# -----------------
|
||||
|
||||
# Maps dataclass field names to Django ORM query field names
|
||||
_DB_FIELD_MAP: dict[str, str] = {
|
||||
"check_title": "check_metadata__checktitle",
|
||||
}
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class Finding:
|
||||
"""
|
||||
Finding data for Neo4j ingestion.
|
||||
|
||||
Can be created from a Django .values() query result using from_db_record().
|
||||
"""
|
||||
|
||||
id: str
|
||||
uid: str
|
||||
inserted_at: str
|
||||
updated_at: str
|
||||
first_seen_at: str
|
||||
scan_id: str
|
||||
delta: str
|
||||
status: str
|
||||
status_extended: str
|
||||
severity: str
|
||||
check_id: str
|
||||
check_title: str
|
||||
muted: bool
|
||||
muted_reason: str | None
|
||||
resource_uid: str | None = None
|
||||
|
||||
@classmethod
|
||||
def get_db_query_fields(cls) -> tuple[str, ...]:
|
||||
"""Get field names for Django .values() query."""
|
||||
return tuple(
|
||||
_DB_FIELD_MAP.get(f.name, f.name)
|
||||
for f in fields(cls)
|
||||
if f.name != "resource_uid"
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_db_record(cls, record: dict[str, Any], resource_uid: str) -> "Finding":
|
||||
"""Create a Finding from a Django .values() query result."""
|
||||
return cls(
|
||||
id=str(record["id"]),
|
||||
uid=record["uid"],
|
||||
inserted_at=record["inserted_at"],
|
||||
updated_at=record["updated_at"],
|
||||
first_seen_at=record["first_seen_at"],
|
||||
scan_id=str(record["scan_id"]),
|
||||
delta=record["delta"],
|
||||
status=record["status"],
|
||||
status_extended=record["status_extended"],
|
||||
severity=record["severity"],
|
||||
check_id=str(record["check_id"]),
|
||||
check_title=record["check_metadata__checktitle"],
|
||||
muted=record["muted"],
|
||||
muted_reason=record["muted_reason"],
|
||||
resource_uid=resource_uid,
|
||||
)
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""Convert to dict for Neo4j ingestion."""
|
||||
return asdict(self)
|
||||
|
||||
|
||||
# Public API
|
||||
# ----------
|
||||
|
||||
|
||||
def create_findings_indexes(neo4j_session: neo4j.Session) -> None:
|
||||
"""Create indexes for Prowler findings and resource lookups."""
|
||||
create_indexes(neo4j_session, IndexType.FINDINGS)
|
||||
|
||||
|
||||
def analysis(
|
||||
neo4j_session: neo4j.Session,
|
||||
prowler_api_provider: Provider,
|
||||
scan_id: str,
|
||||
config: CartographyConfig,
|
||||
) -> None:
|
||||
"""
|
||||
Main entry point for Prowler findings analysis.
|
||||
|
||||
Adds resource labels, loads findings, and cleans up stale data.
|
||||
"""
|
||||
add_resource_label(
|
||||
neo4j_session, prowler_api_provider.provider, str(prowler_api_provider.uid)
|
||||
)
|
||||
findings_data = stream_findings_with_resources(prowler_api_provider, scan_id)
|
||||
load_findings(neo4j_session, findings_data, prowler_api_provider, config)
|
||||
cleanup_findings(neo4j_session, prowler_api_provider, config)
|
||||
|
||||
|
||||
def add_resource_label(
|
||||
neo4j_session: neo4j.Session, provider_type: str, provider_uid: str
|
||||
) -> int:
|
||||
"""
|
||||
Add a common resource label to all nodes connected to the provider account.
|
||||
|
||||
This enables index usage for resource lookups in the findings query,
|
||||
since Cartography nodes don't have a common parent label.
|
||||
|
||||
Returns the total number of nodes labeled.
|
||||
"""
|
||||
query = render_cypher_template(
|
||||
ADD_RESOURCE_LABEL_TEMPLATE,
|
||||
{
|
||||
"__ROOT_LABEL__": get_root_node_label(provider_type),
|
||||
"__RESOURCE_LABEL__": get_provider_resource_label(provider_type),
|
||||
},
|
||||
)
|
||||
|
||||
logger.info(
|
||||
f"Adding {get_provider_resource_label(provider_type)} label to all resources for {provider_uid}"
|
||||
)
|
||||
|
||||
total_labeled = 0
|
||||
labeled_count = 1
|
||||
|
||||
while labeled_count > 0:
|
||||
result = neo4j_session.run(
|
||||
query,
|
||||
{"provider_uid": provider_uid, "batch_size": BATCH_SIZE},
|
||||
)
|
||||
labeled_count = result.single().get("labeled_count", 0)
|
||||
total_labeled += labeled_count
|
||||
|
||||
if labeled_count > 0:
|
||||
logger.info(
|
||||
f"Labeled {total_labeled} nodes with {get_provider_resource_label(provider_type)}"
|
||||
)
|
||||
|
||||
return total_labeled
|
||||
|
||||
|
||||
def load_findings(
|
||||
neo4j_session: neo4j.Session,
|
||||
findings_batches: Generator[list[Finding], None, None],
|
||||
prowler_api_provider: Provider,
|
||||
config: CartographyConfig,
|
||||
) -> None:
|
||||
"""Load Prowler findings into the graph, linking them to resources."""
|
||||
query = render_cypher_template(
|
||||
INSERT_FINDING_TEMPLATE,
|
||||
{
|
||||
"__ROOT_NODE_LABEL__": get_root_node_label(prowler_api_provider.provider),
|
||||
"__NODE_UID_FIELD__": get_node_uid_field(prowler_api_provider.provider),
|
||||
"__RESOURCE_LABEL__": get_provider_resource_label(
|
||||
prowler_api_provider.provider
|
||||
),
|
||||
},
|
||||
)
|
||||
|
||||
parameters = {
|
||||
"provider_uid": str(prowler_api_provider.uid),
|
||||
"last_updated": config.update_tag,
|
||||
"prowler_version": ProwlerConfig.prowler_version,
|
||||
}
|
||||
|
||||
batch_num = 0
|
||||
total_records = 0
|
||||
for batch in findings_batches:
|
||||
batch_num += 1
|
||||
batch_size = len(batch)
|
||||
total_records += batch_size
|
||||
|
||||
parameters["findings_data"] = [f.to_dict() for f in batch]
|
||||
|
||||
logger.info(f"Loading findings batch {batch_num} ({batch_size} records)")
|
||||
neo4j_session.run(query, parameters)
|
||||
|
||||
logger.info(f"Finished loading {total_records} records in {batch_num} batches")
|
||||
|
||||
|
||||
def cleanup_findings(
|
||||
neo4j_session: neo4j.Session,
|
||||
prowler_api_provider: Provider,
|
||||
config: CartographyConfig,
|
||||
) -> None:
|
||||
"""Remove stale findings (classic Cartography behaviour)."""
|
||||
parameters = {
|
||||
"provider_uid": str(prowler_api_provider.uid),
|
||||
"last_updated": config.update_tag,
|
||||
"batch_size": BATCH_SIZE,
|
||||
}
|
||||
|
||||
batch = 1
|
||||
deleted_count = 1
|
||||
while deleted_count > 0:
|
||||
logger.info(f"Cleaning findings batch {batch}")
|
||||
|
||||
result = neo4j_session.run(CLEANUP_FINDINGS_TEMPLATE, parameters)
|
||||
|
||||
deleted_count = result.single().get("deleted_findings_count", 0)
|
||||
batch += 1
|
||||
|
||||
|
||||
# Findings Streaming (Generator-based)
|
||||
# -------------------------------------
|
||||
|
||||
|
||||
def stream_findings_with_resources(
|
||||
prowler_api_provider: Provider,
|
||||
scan_id: str,
|
||||
) -> Generator[list[Finding], None, None]:
|
||||
"""
|
||||
Stream findings with their associated resources in batches.
|
||||
|
||||
Uses keyset pagination for efficient traversal of large datasets.
|
||||
Memory efficient: yields one batch at a time, never holds all findings in memory.
|
||||
"""
|
||||
logger.info(
|
||||
f"Starting findings stream for scan {scan_id} "
|
||||
f"(tenant {prowler_api_provider.tenant_id}) with batch size {BATCH_SIZE}"
|
||||
)
|
||||
|
||||
tenant_id = prowler_api_provider.tenant_id
|
||||
for batch in _paginate_findings(tenant_id, scan_id):
|
||||
enriched = _enrich_batch_with_resources(batch, tenant_id)
|
||||
if enriched:
|
||||
yield enriched
|
||||
|
||||
logger.info(f"Finished streaming findings for scan {scan_id}")
|
||||
|
||||
|
||||
def _paginate_findings(
|
||||
tenant_id: str,
|
||||
scan_id: str,
|
||||
) -> Generator[list[dict[str, Any]], None, None]:
|
||||
"""
|
||||
Paginate through findings using keyset pagination.
|
||||
|
||||
Each iteration fetches one batch within its own RLS transaction,
|
||||
preventing long-held database connections.
|
||||
"""
|
||||
last_id = None
|
||||
iteration = 0
|
||||
|
||||
while True:
|
||||
iteration += 1
|
||||
batch = _fetch_findings_batch(tenant_id, scan_id, last_id)
|
||||
|
||||
logger.info(f"Iteration #{iteration}: fetched {len(batch)} findings")
|
||||
|
||||
if not batch:
|
||||
break
|
||||
|
||||
last_id = batch[-1]["id"]
|
||||
yield batch
|
||||
|
||||
|
||||
def _fetch_findings_batch(
|
||||
tenant_id: str,
|
||||
scan_id: str,
|
||||
after_id: UUID | None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""
|
||||
Fetch a single batch of findings from the database.
|
||||
|
||||
Uses read replica and RLS-scoped transaction.
|
||||
"""
|
||||
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
|
||||
# Use all_objects to avoid the ActiveProviderManager's implicit JOIN
|
||||
# through Scan -> Provider (to check is_deleted=False).
|
||||
# The provider is already validated as active in this context.
|
||||
qs = FindingModel.all_objects.filter(scan_id=scan_id).order_by("id")
|
||||
|
||||
if after_id is not None:
|
||||
qs = qs.filter(id__gt=after_id)
|
||||
|
||||
return list(qs.values(*Finding.get_db_query_fields())[:BATCH_SIZE])
|
||||
|
||||
|
||||
# Batch Enrichment
|
||||
# -----------------
|
||||
|
||||
|
||||
def _enrich_batch_with_resources(
|
||||
findings_batch: list[dict[str, Any]],
|
||||
tenant_id: str,
|
||||
) -> list[Finding]:
|
||||
"""
|
||||
Enrich findings with their resource UIDs.
|
||||
|
||||
One finding with N resources becomes N output records.
|
||||
Findings without resources are skipped.
|
||||
"""
|
||||
finding_ids = [f["id"] for f in findings_batch]
|
||||
resource_map = _build_finding_resource_map(finding_ids, tenant_id)
|
||||
|
||||
return [
|
||||
Finding.from_db_record(finding, resource_uid)
|
||||
for finding in findings_batch
|
||||
for resource_uid in resource_map.get(finding["id"], [])
|
||||
]
|
||||
|
||||
|
||||
def _build_finding_resource_map(
|
||||
finding_ids: list[UUID], tenant_id: str
|
||||
) -> dict[UUID, list[str]]:
|
||||
"""Build mapping from finding_id to list of resource UIDs."""
|
||||
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
|
||||
resource_mappings = ResourceFindingMapping.objects.filter(
|
||||
finding_id__in=finding_ids
|
||||
).values_list("finding_id", "resource__uid")
|
||||
|
||||
result = defaultdict(list)
|
||||
for finding_id, resource_uid in resource_mappings:
|
||||
result[finding_id].append(resource_uid)
|
||||
return result
|
||||
@@ -0,0 +1,67 @@
|
||||
from enum import Enum
|
||||
|
||||
import neo4j
|
||||
|
||||
from cartography.client.core.tx import run_write_query
|
||||
from celery.utils.log import get_task_logger
|
||||
|
||||
from tasks.jobs.attack_paths.config import (
|
||||
INTERNET_NODE_LABEL,
|
||||
PROWLER_FINDING_LABEL,
|
||||
PROVIDER_RESOURCE_LABEL,
|
||||
)
|
||||
|
||||
logger = get_task_logger(__name__)
|
||||
|
||||
|
||||
class IndexType(Enum):
|
||||
"""Types of indexes that can be created."""
|
||||
|
||||
FINDINGS = "findings"
|
||||
SYNC = "sync"
|
||||
|
||||
|
||||
# Indexes for Prowler findings and resource lookups
|
||||
FINDINGS_INDEX_STATEMENTS = [
|
||||
# Resources indexes for quick Prowler Finding lookups
|
||||
"CREATE INDEX aws_resource_arn IF NOT EXISTS FOR (n:AWSResource) ON (n.arn);",
|
||||
"CREATE INDEX aws_resource_id IF NOT EXISTS FOR (n:AWSResource) ON (n.id);",
|
||||
# Prowler Finding indexes
|
||||
f"CREATE INDEX prowler_finding_id IF NOT EXISTS FOR (n:{PROWLER_FINDING_LABEL}) ON (n.id);",
|
||||
f"CREATE INDEX prowler_finding_provider_uid IF NOT EXISTS FOR (n:{PROWLER_FINDING_LABEL}) ON (n.provider_uid);",
|
||||
f"CREATE INDEX prowler_finding_lastupdated IF NOT EXISTS FOR (n:{PROWLER_FINDING_LABEL}) ON (n.lastupdated);",
|
||||
f"CREATE INDEX prowler_finding_status IF NOT EXISTS FOR (n:{PROWLER_FINDING_LABEL}) ON (n.status);",
|
||||
# Internet node index for MERGE lookups
|
||||
f"CREATE INDEX internet_id IF NOT EXISTS FOR (n:{INTERNET_NODE_LABEL}) ON (n.id);",
|
||||
]
|
||||
|
||||
# Indexes for provider resource sync operations
|
||||
SYNC_INDEX_STATEMENTS = [
|
||||
f"CREATE INDEX provider_element_id IF NOT EXISTS FOR (n:{PROVIDER_RESOURCE_LABEL}) ON (n.provider_element_id);",
|
||||
f"CREATE INDEX provider_resource_provider_id IF NOT EXISTS FOR (n:{PROVIDER_RESOURCE_LABEL}) ON (n.provider_id);",
|
||||
]
|
||||
|
||||
|
||||
def create_indexes(neo4j_session: neo4j.Session, index_type: IndexType) -> None:
|
||||
"""
|
||||
Create indexes for the specified type.
|
||||
|
||||
Args:
|
||||
`neo4j_session`: The Neo4j session to use
|
||||
`index_type`: The type of indexes to create (FINDINGS or SYNC)
|
||||
"""
|
||||
if index_type == IndexType.FINDINGS:
|
||||
logger.info("Creating indexes for Prowler Findings node types")
|
||||
for statement in FINDINGS_INDEX_STATEMENTS:
|
||||
run_write_query(neo4j_session, statement)
|
||||
|
||||
elif index_type == IndexType.SYNC:
|
||||
logger.info("Ensuring ProviderResource indexes exist")
|
||||
for statement in SYNC_INDEX_STATEMENTS:
|
||||
neo4j_session.run(statement)
|
||||
|
||||
|
||||
def create_all_indexes(neo4j_session: neo4j.Session) -> None:
|
||||
"""Create all indexes (both findings and sync)."""
|
||||
create_indexes(neo4j_session, IndexType.FINDINGS)
|
||||
create_indexes(neo4j_session, IndexType.SYNC)
|
||||
@@ -0,0 +1,67 @@
|
||||
"""
|
||||
Internet node enrichment for Attack Paths graph.
|
||||
|
||||
Creates a real Internet node and CAN_ACCESS relationships to
|
||||
internet-exposed resources (EC2Instance, LoadBalancer, LoadBalancerV2)
|
||||
in the temporary scan database before sync.
|
||||
"""
|
||||
|
||||
import neo4j
|
||||
|
||||
from cartography.config import Config as CartographyConfig
|
||||
from celery.utils.log import get_task_logger
|
||||
|
||||
from api.models import Provider
|
||||
from prowler.config import config as ProwlerConfig
|
||||
from tasks.jobs.attack_paths.config import get_root_node_label
|
||||
from tasks.jobs.attack_paths.queries import (
|
||||
CREATE_CAN_ACCESS_RELATIONSHIPS_TEMPLATE,
|
||||
CREATE_INTERNET_NODE,
|
||||
render_cypher_template,
|
||||
)
|
||||
|
||||
logger = get_task_logger(__name__)
|
||||
|
||||
|
||||
def analysis(
|
||||
neo4j_session: neo4j.Session,
|
||||
prowler_api_provider: Provider,
|
||||
config: CartographyConfig,
|
||||
) -> int:
|
||||
"""
|
||||
Create Internet node and CAN_ACCESS relationships to exposed resources.
|
||||
|
||||
Args:
|
||||
neo4j_session: Active Neo4j session (temp database).
|
||||
prowler_api_provider: The Prowler API provider instance.
|
||||
config: Cartography configuration with update_tag.
|
||||
|
||||
Returns:
|
||||
Number of CAN_ACCESS relationships created.
|
||||
"""
|
||||
provider_uid = str(prowler_api_provider.uid)
|
||||
|
||||
parameters = {
|
||||
"provider_uid": provider_uid,
|
||||
"last_updated": config.update_tag,
|
||||
"prowler_version": ProwlerConfig.prowler_version,
|
||||
}
|
||||
|
||||
logger.info(f"Creating Internet node for provider {provider_uid}")
|
||||
neo4j_session.run(CREATE_INTERNET_NODE, parameters)
|
||||
|
||||
query = render_cypher_template(
|
||||
CREATE_CAN_ACCESS_RELATIONSHIPS_TEMPLATE,
|
||||
{"__ROOT_LABEL__": get_root_node_label(prowler_api_provider.provider)},
|
||||
)
|
||||
|
||||
logger.info(
|
||||
f"Creating CAN_ACCESS relationships from Internet to exposed resources for {provider_uid}"
|
||||
)
|
||||
result = neo4j_session.run(query, parameters)
|
||||
relationships_merged = result.single().get("relationships_merged", 0)
|
||||
|
||||
logger.info(
|
||||
f"Created {relationships_merged} CAN_ACCESS relationships for provider {provider_uid}"
|
||||
)
|
||||
return relationships_merged
|
||||
@@ -0,0 +1,166 @@
|
||||
# Cypher query templates for Attack Paths operations
|
||||
from tasks.jobs.attack_paths.config import (
|
||||
INTERNET_NODE_LABEL,
|
||||
PROWLER_FINDING_LABEL,
|
||||
PROVIDER_RESOURCE_LABEL,
|
||||
)
|
||||
|
||||
|
||||
def render_cypher_template(template: str, replacements: dict[str, str]) -> str:
|
||||
"""
|
||||
Render a Cypher query template by replacing placeholders.
|
||||
|
||||
Placeholders use `__DOUBLE_UNDERSCORE__` format to avoid conflicts
|
||||
with Cypher syntax.
|
||||
"""
|
||||
query = template
|
||||
for placeholder, value in replacements.items():
|
||||
query = query.replace(placeholder, value)
|
||||
return query
|
||||
|
||||
|
||||
# Findings queries (used by findings.py)
|
||||
# ---------------------------------------
|
||||
|
||||
ADD_RESOURCE_LABEL_TEMPLATE = """
|
||||
MATCH (account:__ROOT_LABEL__ {id: $provider_uid})-->(r)
|
||||
WHERE NOT r:__ROOT_LABEL__ AND NOT r:__RESOURCE_LABEL__
|
||||
WITH r LIMIT $batch_size
|
||||
SET r:__RESOURCE_LABEL__
|
||||
RETURN COUNT(r) AS labeled_count
|
||||
"""
|
||||
|
||||
INSERT_FINDING_TEMPLATE = f"""
|
||||
MATCH (account:__ROOT_NODE_LABEL__ {{id: $provider_uid}})
|
||||
UNWIND $findings_data AS finding_data
|
||||
|
||||
OPTIONAL MATCH (account)-->(resource_by_uid:__RESOURCE_LABEL__)
|
||||
WHERE resource_by_uid.__NODE_UID_FIELD__ = finding_data.resource_uid
|
||||
WITH account, finding_data, resource_by_uid
|
||||
|
||||
OPTIONAL MATCH (account)-->(resource_by_id:__RESOURCE_LABEL__)
|
||||
WHERE resource_by_uid IS NULL
|
||||
AND resource_by_id.id = finding_data.resource_uid
|
||||
WITH account, finding_data, COALESCE(resource_by_uid, resource_by_id) AS resource
|
||||
WHERE resource IS NOT NULL
|
||||
|
||||
MERGE (finding:{PROWLER_FINDING_LABEL} {{id: finding_data.id}})
|
||||
ON CREATE SET
|
||||
finding.id = finding_data.id,
|
||||
finding.uid = finding_data.uid,
|
||||
finding.inserted_at = finding_data.inserted_at,
|
||||
finding.updated_at = finding_data.updated_at,
|
||||
finding.first_seen_at = finding_data.first_seen_at,
|
||||
finding.scan_id = finding_data.scan_id,
|
||||
finding.delta = finding_data.delta,
|
||||
finding.status = finding_data.status,
|
||||
finding.status_extended = finding_data.status_extended,
|
||||
finding.severity = finding_data.severity,
|
||||
finding.check_id = finding_data.check_id,
|
||||
finding.check_title = finding_data.check_title,
|
||||
finding.muted = finding_data.muted,
|
||||
finding.muted_reason = finding_data.muted_reason,
|
||||
finding.provider_uid = $provider_uid,
|
||||
finding.firstseen = timestamp(),
|
||||
finding.lastupdated = $last_updated,
|
||||
finding._module_name = 'cartography:prowler',
|
||||
finding._module_version = $prowler_version
|
||||
ON MATCH SET
|
||||
finding.status = finding_data.status,
|
||||
finding.status_extended = finding_data.status_extended,
|
||||
finding.lastupdated = $last_updated
|
||||
|
||||
MERGE (resource)-[rel:HAS_FINDING]->(finding)
|
||||
ON CREATE SET
|
||||
rel.provider_uid = $provider_uid,
|
||||
rel.firstseen = timestamp(),
|
||||
rel.lastupdated = $last_updated,
|
||||
rel._module_name = 'cartography:prowler',
|
||||
rel._module_version = $prowler_version
|
||||
ON MATCH SET
|
||||
rel.lastupdated = $last_updated
|
||||
"""
|
||||
|
||||
CLEANUP_FINDINGS_TEMPLATE = f"""
|
||||
MATCH (finding:{PROWLER_FINDING_LABEL} {{provider_uid: $provider_uid}})
|
||||
WHERE finding.lastupdated < $last_updated
|
||||
|
||||
WITH finding LIMIT $batch_size
|
||||
|
||||
DETACH DELETE finding
|
||||
|
||||
RETURN COUNT(finding) AS deleted_findings_count
|
||||
"""
|
||||
|
||||
# Internet queries (used by internet.py)
|
||||
# ---------------------------------------
|
||||
|
||||
CREATE_INTERNET_NODE = f"""
|
||||
MERGE (internet:{INTERNET_NODE_LABEL} {{id: 'Internet'}})
|
||||
ON CREATE SET
|
||||
internet.name = 'Internet',
|
||||
internet.firstseen = timestamp(),
|
||||
internet.lastupdated = $last_updated,
|
||||
internet._module_name = 'cartography:prowler',
|
||||
internet._module_version = $prowler_version
|
||||
ON MATCH SET
|
||||
internet.lastupdated = $last_updated
|
||||
"""
|
||||
|
||||
CREATE_CAN_ACCESS_RELATIONSHIPS_TEMPLATE = f"""
|
||||
MATCH (account:__ROOT_LABEL__ {{id: $provider_uid}})-->(resource)
|
||||
WHERE resource.exposed_internet = true
|
||||
WITH resource
|
||||
MATCH (internet:{INTERNET_NODE_LABEL} {{id: 'Internet'}})
|
||||
MERGE (internet)-[r:CAN_ACCESS]->(resource)
|
||||
ON CREATE SET
|
||||
r.firstseen = timestamp(),
|
||||
r.lastupdated = $last_updated,
|
||||
r._module_name = 'cartography:prowler',
|
||||
r._module_version = $prowler_version
|
||||
ON MATCH SET
|
||||
r.lastupdated = $last_updated
|
||||
RETURN COUNT(r) AS relationships_merged
|
||||
"""
|
||||
|
||||
# Sync queries (used by sync.py)
|
||||
# -------------------------------
|
||||
|
||||
NODE_FETCH_QUERY = """
|
||||
MATCH (n)
|
||||
WHERE id(n) > $last_id
|
||||
RETURN id(n) AS internal_id,
|
||||
elementId(n) AS element_id,
|
||||
labels(n) AS labels,
|
||||
properties(n) AS props
|
||||
ORDER BY internal_id
|
||||
LIMIT $batch_size
|
||||
"""
|
||||
|
||||
RELATIONSHIPS_FETCH_QUERY = """
|
||||
MATCH ()-[r]->()
|
||||
WHERE id(r) > $last_id
|
||||
RETURN id(r) AS internal_id,
|
||||
type(r) AS rel_type,
|
||||
elementId(startNode(r)) AS start_element_id,
|
||||
elementId(endNode(r)) AS end_element_id,
|
||||
properties(r) AS props
|
||||
ORDER BY internal_id
|
||||
LIMIT $batch_size
|
||||
"""
|
||||
|
||||
NODE_SYNC_TEMPLATE = """
|
||||
UNWIND $rows AS row
|
||||
MERGE (n:__NODE_LABELS__ {provider_element_id: row.provider_element_id})
|
||||
SET n += row.props
|
||||
SET n.provider_id = $provider_id
|
||||
"""
|
||||
|
||||
RELATIONSHIP_SYNC_TEMPLATE = f"""
|
||||
UNWIND $rows AS row
|
||||
MATCH (s:{PROVIDER_RESOURCE_LABEL} {{provider_element_id: row.start_element_id}})
|
||||
MATCH (t:{PROVIDER_RESOURCE_LABEL} {{provider_element_id: row.end_element_id}})
|
||||
MERGE (s)-[r:__REL_TYPE__ {{provider_element_id: row.provider_element_id}}]->(t)
|
||||
SET r += row.props
|
||||
SET r.provider_id = $provider_id
|
||||
"""
|
||||
@@ -0,0 +1,238 @@
|
||||
import logging
|
||||
import time
|
||||
|
||||
from typing import Any
|
||||
|
||||
from cartography.config import Config as CartographyConfig
|
||||
from cartography.intel import analysis as cartography_analysis
|
||||
from cartography.intel import create_indexes as cartography_create_indexes
|
||||
from cartography.intel import ontology as cartography_ontology
|
||||
from celery.utils.log import get_task_logger
|
||||
|
||||
from api.attack_paths import database as graph_database
|
||||
from api.db_utils import rls_transaction
|
||||
from api.models import (
|
||||
Provider as ProwlerAPIProvider,
|
||||
StateChoices,
|
||||
)
|
||||
from api.utils import initialize_prowler_provider
|
||||
from tasks.jobs.attack_paths import db_utils, findings, internet, sync, utils
|
||||
from tasks.jobs.attack_paths.config import get_cartography_ingestion_function
|
||||
|
||||
# Without this Celery goes crazy with Cartography logging
|
||||
logging.getLogger("cartography").setLevel(logging.ERROR)
|
||||
logging.getLogger("neo4j").propagate = False
|
||||
|
||||
logger = get_task_logger(__name__)
|
||||
|
||||
|
||||
def run(tenant_id: str, scan_id: str, task_id: str) -> dict[str, Any]:
|
||||
"""
|
||||
Code based on Cartography, specifically on `cartography.cli.main`, `cartography.cli.CLI.main`,
|
||||
`cartography.sync.run_with_config` and `cartography.sync.Sync.run`.
|
||||
"""
|
||||
ingestion_exceptions = {} # This will hold any exceptions raised during ingestion
|
||||
|
||||
# Prowler necessary objects
|
||||
with rls_transaction(tenant_id):
|
||||
prowler_api_provider = ProwlerAPIProvider.objects.get(scan__pk=scan_id)
|
||||
prowler_sdk_provider = initialize_prowler_provider(prowler_api_provider)
|
||||
|
||||
# Attack Paths Scan necessary objects
|
||||
cartography_ingestion_function = get_cartography_ingestion_function(
|
||||
prowler_api_provider.provider
|
||||
)
|
||||
attack_paths_scan = db_utils.retrieve_attack_paths_scan(tenant_id, scan_id)
|
||||
|
||||
# Checks before starting the scan
|
||||
if not cartography_ingestion_function:
|
||||
ingestion_exceptions = {
|
||||
"global_error": f"Provider {prowler_api_provider.provider} is not supported for Attack Paths scans"
|
||||
}
|
||||
if attack_paths_scan:
|
||||
db_utils.finish_attack_paths_scan(
|
||||
attack_paths_scan, StateChoices.COMPLETED, ingestion_exceptions
|
||||
)
|
||||
|
||||
logger.warning(
|
||||
f"Provider {prowler_api_provider.provider} is not supported for Attack Paths scans"
|
||||
)
|
||||
return ingestion_exceptions
|
||||
|
||||
else:
|
||||
if not attack_paths_scan:
|
||||
logger.warning(
|
||||
f"No Attack Paths Scan found for scan {scan_id} and tenant {tenant_id}, let's create it then"
|
||||
)
|
||||
attack_paths_scan = db_utils.create_attack_paths_scan(
|
||||
tenant_id, scan_id, prowler_api_provider.id
|
||||
)
|
||||
|
||||
tmp_database_name = graph_database.get_database_name(
|
||||
attack_paths_scan.id, temporary=True
|
||||
)
|
||||
tenant_database_name = graph_database.get_database_name(
|
||||
prowler_api_provider.tenant_id
|
||||
)
|
||||
|
||||
# While creating the Cartography configuration, attributes `neo4j_user` and `neo4j_password` are not really needed in this config object
|
||||
tmp_cartography_config = CartographyConfig(
|
||||
neo4j_uri=graph_database.get_uri(),
|
||||
neo4j_database=tmp_database_name,
|
||||
update_tag=int(time.time()),
|
||||
)
|
||||
tenant_cartography_config = CartographyConfig(
|
||||
neo4j_uri=tmp_cartography_config.neo4j_uri,
|
||||
neo4j_database=tenant_database_name,
|
||||
update_tag=tmp_cartography_config.update_tag,
|
||||
)
|
||||
|
||||
# Starting the Attack Paths scan
|
||||
db_utils.starting_attack_paths_scan(
|
||||
attack_paths_scan, task_id, tenant_cartography_config
|
||||
)
|
||||
|
||||
try:
|
||||
logger.info(
|
||||
f"Creating Neo4j database {tmp_cartography_config.neo4j_database} for tenant {prowler_api_provider.tenant_id}"
|
||||
)
|
||||
|
||||
graph_database.create_database(tmp_cartography_config.neo4j_database)
|
||||
db_utils.update_attack_paths_scan_progress(attack_paths_scan, 1)
|
||||
|
||||
logger.info(
|
||||
f"Starting Cartography ({attack_paths_scan.id}) for "
|
||||
f"{prowler_api_provider.provider.upper()} provider {prowler_api_provider.id}"
|
||||
)
|
||||
with graph_database.get_session(
|
||||
tmp_cartography_config.neo4j_database
|
||||
) as tmp_neo4j_session:
|
||||
# Indexes creation
|
||||
cartography_create_indexes.run(tmp_neo4j_session, tmp_cartography_config)
|
||||
findings.create_findings_indexes(tmp_neo4j_session)
|
||||
db_utils.update_attack_paths_scan_progress(attack_paths_scan, 2)
|
||||
|
||||
# The real scan, where iterates over cloud services
|
||||
ingestion_exceptions = utils.call_within_event_loop(
|
||||
cartography_ingestion_function,
|
||||
tmp_neo4j_session,
|
||||
tmp_cartography_config,
|
||||
prowler_api_provider,
|
||||
prowler_sdk_provider,
|
||||
attack_paths_scan,
|
||||
)
|
||||
|
||||
# Post-processing: Just keeping it to be more Cartography compliant
|
||||
logger.info(
|
||||
f"Syncing Cartography ontology for AWS account {prowler_api_provider.uid}"
|
||||
)
|
||||
cartography_ontology.run(tmp_neo4j_session, tmp_cartography_config)
|
||||
db_utils.update_attack_paths_scan_progress(attack_paths_scan, 95)
|
||||
|
||||
logger.info(
|
||||
f"Syncing Cartography analysis for AWS account {prowler_api_provider.uid}"
|
||||
)
|
||||
cartography_analysis.run(tmp_neo4j_session, tmp_cartography_config)
|
||||
db_utils.update_attack_paths_scan_progress(attack_paths_scan, 96)
|
||||
|
||||
# Creating Internet node and CAN_ACCESS relationships
|
||||
logger.info(
|
||||
f"Creating Internet graph for AWS account {prowler_api_provider.uid}"
|
||||
)
|
||||
internet.analysis(
|
||||
tmp_neo4j_session, prowler_api_provider, tmp_cartography_config
|
||||
)
|
||||
|
||||
# Adding Prowler Finding nodes and relationships
|
||||
logger.info(
|
||||
f"Syncing Prowler analysis for AWS account {prowler_api_provider.uid}"
|
||||
)
|
||||
findings.analysis(
|
||||
tmp_neo4j_session, prowler_api_provider, scan_id, tmp_cartography_config
|
||||
)
|
||||
db_utils.update_attack_paths_scan_progress(attack_paths_scan, 97)
|
||||
|
||||
logger.info(
|
||||
f"Clearing Neo4j cache for database {tmp_cartography_config.neo4j_database}"
|
||||
)
|
||||
graph_database.clear_cache(tmp_cartography_config.neo4j_database)
|
||||
|
||||
logger.info(
|
||||
f"Ensuring tenant database {tenant_database_name}, and its indexes, exists for tenant {prowler_api_provider.tenant_id}"
|
||||
)
|
||||
graph_database.create_database(tenant_database_name)
|
||||
with graph_database.get_session(tenant_database_name) as tenant_neo4j_session:
|
||||
cartography_create_indexes.run(
|
||||
tenant_neo4j_session, tenant_cartography_config
|
||||
)
|
||||
findings.create_findings_indexes(tenant_neo4j_session)
|
||||
sync.create_sync_indexes(tenant_neo4j_session)
|
||||
|
||||
logger.info(f"Deleting existing provider graph in {tenant_database_name}")
|
||||
graph_database.drop_subgraph(
|
||||
database=tenant_database_name,
|
||||
provider_id=str(prowler_api_provider.id),
|
||||
)
|
||||
db_utils.update_attack_paths_scan_progress(attack_paths_scan, 98)
|
||||
|
||||
logger.info(
|
||||
f"Syncing graph from {tmp_database_name} into {tenant_database_name}"
|
||||
)
|
||||
sync.sync_graph(
|
||||
source_database=tmp_database_name,
|
||||
target_database=tenant_database_name,
|
||||
provider_id=str(prowler_api_provider.id),
|
||||
)
|
||||
db_utils.update_attack_paths_scan_progress(attack_paths_scan, 99)
|
||||
|
||||
logger.info(f"Clearing Neo4j cache for database {tenant_database_name}")
|
||||
graph_database.clear_cache(tenant_database_name)
|
||||
|
||||
logger.info(
|
||||
f"Completed Cartography ({attack_paths_scan.id}) for "
|
||||
f"{prowler_api_provider.provider.upper()} provider {prowler_api_provider.id}"
|
||||
)
|
||||
|
||||
# TODO
|
||||
# This piece of code delete old Neo4j databases for this tenant's provider
|
||||
# When we clean all of these databases we need to:
|
||||
# - Delete this block
|
||||
# - Delete function from `db_utils` the functions get_old_attack_paths_scans` & `update_old_attack_paths_scan`
|
||||
# - Remove `graph_database` & `is_graph_database_deleted` from the AttackPathsScan model:
|
||||
# - Check indexes
|
||||
# - Create migration
|
||||
# - The use of `attack_paths_scan.graph_database` on `views` and `views_helpers`
|
||||
# - Tests
|
||||
old_attack_paths_scans = db_utils.get_old_attack_paths_scans(
|
||||
prowler_api_provider.tenant_id,
|
||||
prowler_api_provider.id,
|
||||
attack_paths_scan.id,
|
||||
)
|
||||
for old_attack_paths_scan in old_attack_paths_scans:
|
||||
old_graph_database = old_attack_paths_scan.graph_database
|
||||
if old_graph_database and old_graph_database != tenant_database_name:
|
||||
logger.info(
|
||||
f"Dropping old Neo4j database {old_graph_database} for provider {prowler_api_provider.id}"
|
||||
)
|
||||
graph_database.drop_database(old_graph_database)
|
||||
db_utils.update_old_attack_paths_scan(old_attack_paths_scan)
|
||||
|
||||
logger.info(f"Dropping temporary Neo4j database {tmp_database_name}")
|
||||
graph_database.drop_database(tmp_database_name)
|
||||
|
||||
db_utils.finish_attack_paths_scan(
|
||||
attack_paths_scan, StateChoices.COMPLETED, ingestion_exceptions
|
||||
)
|
||||
return ingestion_exceptions
|
||||
|
||||
except Exception as e:
|
||||
exception_message = utils.stringify_exception(e, "Cartography failed")
|
||||
logger.error(exception_message)
|
||||
ingestion_exceptions["global_cartography_error"] = exception_message
|
||||
|
||||
# Handling databases changes
|
||||
graph_database.drop_database(tmp_cartography_config.neo4j_database)
|
||||
db_utils.finish_attack_paths_scan(
|
||||
attack_paths_scan, StateChoices.FAILED, ingestion_exceptions
|
||||
)
|
||||
raise
|
||||
@@ -0,0 +1,202 @@
|
||||
"""
|
||||
Graph sync operations for Attack Paths.
|
||||
|
||||
This module handles syncing graph data from temporary scan databases
|
||||
to the tenant database, adding provider isolation labels and properties.
|
||||
"""
|
||||
|
||||
from collections import defaultdict
|
||||
from typing import Any
|
||||
|
||||
from celery.utils.log import get_task_logger
|
||||
|
||||
from api.attack_paths import database as graph_database
|
||||
from tasks.jobs.attack_paths.config import BATCH_SIZE, PROVIDER_RESOURCE_LABEL
|
||||
from tasks.jobs.attack_paths.indexes import IndexType, create_indexes
|
||||
from tasks.jobs.attack_paths.queries import (
|
||||
NODE_FETCH_QUERY,
|
||||
NODE_SYNC_TEMPLATE,
|
||||
RELATIONSHIP_SYNC_TEMPLATE,
|
||||
RELATIONSHIPS_FETCH_QUERY,
|
||||
render_cypher_template,
|
||||
)
|
||||
|
||||
logger = get_task_logger(__name__)
|
||||
|
||||
|
||||
def create_sync_indexes(neo4j_session) -> None:
|
||||
"""Create indexes for provider resource sync operations."""
|
||||
create_indexes(neo4j_session, IndexType.SYNC)
|
||||
|
||||
|
||||
def sync_graph(
|
||||
source_database: str,
|
||||
target_database: str,
|
||||
provider_id: str,
|
||||
) -> dict[str, int]:
|
||||
"""
|
||||
Sync all nodes and relationships from source to target database.
|
||||
|
||||
Args:
|
||||
`source_database`: The temporary scan database
|
||||
`target_database`: The tenant database
|
||||
`provider_id`: The provider ID for isolation
|
||||
|
||||
Returns:
|
||||
Dict with counts of synced nodes and relationships
|
||||
"""
|
||||
nodes_synced = sync_nodes(
|
||||
source_database,
|
||||
target_database,
|
||||
provider_id,
|
||||
)
|
||||
relationships_synced = sync_relationships(
|
||||
source_database,
|
||||
target_database,
|
||||
provider_id,
|
||||
)
|
||||
|
||||
return {
|
||||
"nodes": nodes_synced,
|
||||
"relationships": relationships_synced,
|
||||
}
|
||||
|
||||
|
||||
def sync_nodes(
|
||||
source_database: str,
|
||||
target_database: str,
|
||||
provider_id: str,
|
||||
) -> int:
|
||||
"""
|
||||
Sync nodes from source to target database.
|
||||
|
||||
Adds `ProviderResource` label and `provider_id` property to all nodes.
|
||||
"""
|
||||
last_id = -1
|
||||
total_synced = 0
|
||||
|
||||
with (
|
||||
graph_database.get_session(source_database) as source_session,
|
||||
graph_database.get_session(target_database) as target_session,
|
||||
):
|
||||
while True:
|
||||
rows = list(
|
||||
source_session.run(
|
||||
NODE_FETCH_QUERY,
|
||||
{"last_id": last_id, "batch_size": BATCH_SIZE},
|
||||
)
|
||||
)
|
||||
|
||||
if not rows:
|
||||
break
|
||||
|
||||
last_id = rows[-1]["internal_id"]
|
||||
|
||||
grouped: dict[tuple[str, ...], list[dict[str, Any]]] = defaultdict(list)
|
||||
for row in rows:
|
||||
labels = tuple(sorted(set(row["labels"] or [])))
|
||||
props = dict(row["props"] or {})
|
||||
_strip_internal_properties(props)
|
||||
provider_element_id = f"{provider_id}:{row['element_id']}"
|
||||
grouped[labels].append(
|
||||
{
|
||||
"provider_element_id": provider_element_id,
|
||||
"props": props,
|
||||
}
|
||||
)
|
||||
|
||||
for labels, batch in grouped.items():
|
||||
label_set = set(labels)
|
||||
label_set.add(PROVIDER_RESOURCE_LABEL)
|
||||
node_labels = ":".join(f"`{label}`" for label in sorted(label_set))
|
||||
|
||||
query = render_cypher_template(
|
||||
NODE_SYNC_TEMPLATE, {"__NODE_LABELS__": node_labels}
|
||||
)
|
||||
target_session.run(
|
||||
query,
|
||||
{
|
||||
"rows": batch,
|
||||
"provider_id": provider_id,
|
||||
},
|
||||
)
|
||||
|
||||
total_synced += len(rows)
|
||||
logger.info(
|
||||
f"Synced {total_synced} nodes from {source_database} to {target_database}"
|
||||
)
|
||||
|
||||
return total_synced
|
||||
|
||||
|
||||
def sync_relationships(
|
||||
source_database: str,
|
||||
target_database: str,
|
||||
provider_id: str,
|
||||
) -> int:
|
||||
"""
|
||||
Sync relationships from source to target database.
|
||||
|
||||
Adds `provider_id` property to all relationships.
|
||||
"""
|
||||
last_id = -1
|
||||
total_synced = 0
|
||||
|
||||
with (
|
||||
graph_database.get_session(source_database) as source_session,
|
||||
graph_database.get_session(target_database) as target_session,
|
||||
):
|
||||
while True:
|
||||
rows = list(
|
||||
source_session.run(
|
||||
RELATIONSHIPS_FETCH_QUERY,
|
||||
{"last_id": last_id, "batch_size": BATCH_SIZE},
|
||||
)
|
||||
)
|
||||
|
||||
if not rows:
|
||||
break
|
||||
|
||||
last_id = rows[-1]["internal_id"]
|
||||
|
||||
grouped: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
||||
for row in rows:
|
||||
props = dict(row["props"] or {})
|
||||
_strip_internal_properties(props)
|
||||
rel_type = row["rel_type"]
|
||||
grouped[rel_type].append(
|
||||
{
|
||||
"start_element_id": f"{provider_id}:{row['start_element_id']}",
|
||||
"end_element_id": f"{provider_id}:{row['end_element_id']}",
|
||||
"provider_element_id": f"{provider_id}:{rel_type}:{row['internal_id']}",
|
||||
"props": props,
|
||||
}
|
||||
)
|
||||
|
||||
for rel_type, batch in grouped.items():
|
||||
query = render_cypher_template(
|
||||
RELATIONSHIP_SYNC_TEMPLATE, {"__REL_TYPE__": rel_type}
|
||||
)
|
||||
target_session.run(
|
||||
query,
|
||||
{
|
||||
"rows": batch,
|
||||
"provider_id": provider_id,
|
||||
},
|
||||
)
|
||||
|
||||
total_synced += len(rows)
|
||||
logger.info(
|
||||
f"Synced {total_synced} relationships from {source_database} to {target_database}"
|
||||
)
|
||||
|
||||
return total_synced
|
||||
|
||||
|
||||
def _strip_internal_properties(props: dict[str, Any]) -> None:
|
||||
"""Remove internal properties that shouldn't be copied during sync."""
|
||||
for key in [
|
||||
"provider_element_id",
|
||||
"provider_id",
|
||||
]:
|
||||
props.pop(key, None)
|
||||
@@ -0,0 +1,40 @@
|
||||
import asyncio
|
||||
import traceback
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from celery.utils.log import get_task_logger
|
||||
|
||||
logger = get_task_logger(__name__)
|
||||
|
||||
|
||||
def stringify_exception(exception: Exception, context: str) -> str:
|
||||
"""Format an exception with timestamp and traceback for logging."""
|
||||
timestamp = datetime.now(tz=timezone.utc)
|
||||
exception_traceback = traceback.TracebackException.from_exception(exception)
|
||||
traceback_string = "".join(exception_traceback.format())
|
||||
return f"{timestamp} - {context}\n{traceback_string}"
|
||||
|
||||
|
||||
def call_within_event_loop(fn, *args, **kwargs):
|
||||
"""
|
||||
Execute a function within a new event loop.
|
||||
|
||||
Cartography needs a running event loop, so assuming there is none
|
||||
(Celery task or even regular DRF endpoint), this creates a new one
|
||||
and sets it as the current event loop for this thread.
|
||||
"""
|
||||
loop = asyncio.new_event_loop()
|
||||
try:
|
||||
asyncio.set_event_loop(loop)
|
||||
return fn(*args, **kwargs)
|
||||
|
||||
finally:
|
||||
try:
|
||||
loop.run_until_complete(loop.shutdown_asyncgens())
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to shutdown async generators cleanly: {e}")
|
||||
|
||||
loop.close()
|
||||
asyncio.set_event_loop(None)
|
||||
@@ -2,13 +2,13 @@ from collections import defaultdict
|
||||
from datetime import timedelta
|
||||
|
||||
from celery.utils.log import get_task_logger
|
||||
from django.db.models import Sum
|
||||
from django.db.models import OuterRef, Subquery, Sum
|
||||
from django.utils import timezone
|
||||
from tasks.jobs.queries import (
|
||||
COMPLIANCE_UPSERT_PROVIDER_SCORE_SQL,
|
||||
COMPLIANCE_UPSERT_TENANT_SUMMARY_ALL_SQL,
|
||||
)
|
||||
from tasks.jobs.scan import aggregate_category_counts
|
||||
from tasks.jobs.scan import aggregate_category_counts, aggregate_resource_group_counts
|
||||
|
||||
from api.db_router import READ_REPLICA_ALIAS, MainRouter
|
||||
from api.db_utils import (
|
||||
@@ -28,6 +28,7 @@ from api.models import (
|
||||
ResourceScanSummary,
|
||||
Scan,
|
||||
ScanCategorySummary,
|
||||
ScanGroupSummary,
|
||||
ScanSummary,
|
||||
StateChoices,
|
||||
)
|
||||
@@ -356,6 +357,92 @@ def backfill_scan_category_summaries(tenant_id: str, scan_id: str):
|
||||
return {"status": "backfilled", "categories_count": len(category_counts)}
|
||||
|
||||
|
||||
def backfill_scan_resource_group_summaries(tenant_id: str, scan_id: str):
|
||||
"""
|
||||
Backfill ScanGroupSummary for a completed scan.
|
||||
|
||||
Aggregates resource group counts from all findings in the scan and creates
|
||||
one ScanGroupSummary row per (resource_group, severity) combination.
|
||||
|
||||
Args:
|
||||
tenant_id: Target tenant UUID
|
||||
scan_id: Scan UUID to backfill
|
||||
|
||||
Returns:
|
||||
dict: Status indicating whether backfill was performed
|
||||
"""
|
||||
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
|
||||
if ScanGroupSummary.objects.filter(
|
||||
tenant_id=tenant_id, scan_id=scan_id
|
||||
).exists():
|
||||
return {"status": "already backfilled"}
|
||||
|
||||
if not Scan.objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
id=scan_id,
|
||||
state__in=(StateChoices.COMPLETED, StateChoices.FAILED),
|
||||
).exists():
|
||||
return {"status": "scan is not completed"}
|
||||
|
||||
resource_group_counts: dict[tuple[str, str], dict[str, int]] = {}
|
||||
group_resources_cache: dict[str, set] = {}
|
||||
# Get findings with their first resource UID via annotation
|
||||
resource_uid_subquery = ResourceFindingMapping.objects.filter(
|
||||
finding_id=OuterRef("id"), tenant_id=tenant_id
|
||||
).values("resource__uid")[:1]
|
||||
|
||||
for finding in (
|
||||
Finding.all_objects.filter(tenant_id=tenant_id, scan_id=scan_id)
|
||||
.annotate(resource_uid=Subquery(resource_uid_subquery))
|
||||
.values(
|
||||
"resource_groups",
|
||||
"severity",
|
||||
"status",
|
||||
"delta",
|
||||
"muted",
|
||||
"resource_uid",
|
||||
)
|
||||
):
|
||||
aggregate_resource_group_counts(
|
||||
resource_group=finding.get("resource_groups"),
|
||||
severity=finding.get("severity"),
|
||||
status=finding.get("status"),
|
||||
delta=finding.get("delta"),
|
||||
muted=finding.get("muted", False),
|
||||
resource_uid=finding.get("resource_uid") or "",
|
||||
cache=resource_group_counts,
|
||||
group_resources_cache=group_resources_cache,
|
||||
)
|
||||
|
||||
if not resource_group_counts:
|
||||
return {"status": "no resource groups to backfill"}
|
||||
|
||||
# Compute group-level resource counts (same value for all severity rows in a group)
|
||||
group_resource_counts = {
|
||||
grp: len(uids) for grp, uids in group_resources_cache.items()
|
||||
}
|
||||
resource_group_summaries = [
|
||||
ScanGroupSummary(
|
||||
tenant_id=tenant_id,
|
||||
scan_id=scan_id,
|
||||
resource_group=grp,
|
||||
severity=severity,
|
||||
total_findings=counts["total"],
|
||||
failed_findings=counts["failed"],
|
||||
new_failed_findings=counts["new_failed"],
|
||||
resources_count=group_resource_counts.get(grp, 0),
|
||||
)
|
||||
for (grp, severity), counts in resource_group_counts.items()
|
||||
]
|
||||
|
||||
with rls_transaction(tenant_id):
|
||||
ScanGroupSummary.objects.bulk_create(
|
||||
resource_group_summaries, batch_size=500, ignore_conflicts=True
|
||||
)
|
||||
|
||||
return {"status": "backfilled", "resource_groups_count": len(resource_group_counts)}
|
||||
|
||||
|
||||
def backfill_provider_compliance_scores(tenant_id: str) -> dict:
|
||||
"""
|
||||
Backfill ProviderComplianceScore from latest completed scan per provider.
|
||||
|
||||
@@ -1,9 +1,18 @@
|
||||
from celery.utils.log import get_task_logger
|
||||
from django.db import DatabaseError
|
||||
|
||||
from api.attack_paths import database as graph_database
|
||||
from api.db_router import MainRouter
|
||||
from api.db_utils import batch_delete, rls_transaction
|
||||
from api.models import Finding, Provider, Resource, Scan, ScanSummary, Tenant
|
||||
from api.models import (
|
||||
AttackPathsScan,
|
||||
Finding,
|
||||
Provider,
|
||||
Resource,
|
||||
Scan,
|
||||
ScanSummary,
|
||||
Tenant,
|
||||
)
|
||||
|
||||
logger = get_task_logger(__name__)
|
||||
|
||||
@@ -23,16 +32,27 @@ def delete_provider(tenant_id: str, pk: str):
|
||||
Raises:
|
||||
Provider.DoesNotExist: If no instance with the provided primary key exists.
|
||||
"""
|
||||
# Delete the Attack Paths' graph data related to the provider
|
||||
tenant_database_name = graph_database.get_database_name(tenant_id)
|
||||
try:
|
||||
graph_database.drop_subgraph(tenant_database_name, str(pk))
|
||||
|
||||
except graph_database.GraphDatabaseQueryException as gdb_error:
|
||||
logger.error(f"Error deleting Provider graph data: {gdb_error}")
|
||||
raise
|
||||
|
||||
# Get all provider related data and delete them in batches
|
||||
with rls_transaction(tenant_id):
|
||||
instance = Provider.all_objects.get(pk=pk)
|
||||
deletion_summary = {}
|
||||
deletion_steps = [
|
||||
("Scan Summaries", ScanSummary.all_objects.filter(scan__provider=instance)),
|
||||
("Findings", Finding.all_objects.filter(scan__provider=instance)),
|
||||
("Resources", Resource.all_objects.filter(provider=instance)),
|
||||
("Scans", Scan.all_objects.filter(provider=instance)),
|
||||
("AttackPathsScans", AttackPathsScan.all_objects.filter(provider=instance)),
|
||||
]
|
||||
|
||||
deletion_summary = {}
|
||||
for step_name, queryset in deletion_steps:
|
||||
try:
|
||||
_, step_summary = batch_delete(tenant_id, queryset)
|
||||
@@ -48,6 +68,7 @@ def delete_provider(tenant_id: str, pk: str):
|
||||
except DatabaseError as db_error:
|
||||
logger.error(f"Error deleting Provider: {db_error}")
|
||||
raise
|
||||
|
||||
return deletion_summary
|
||||
|
||||
|
||||
@@ -68,6 +89,13 @@ def delete_tenant(pk: str):
|
||||
summary = delete_provider(pk, provider.id)
|
||||
deletion_summary.update(summary)
|
||||
|
||||
try:
|
||||
tenant_database_name = graph_database.get_database_name(pk)
|
||||
graph_database.drop_database(tenant_database_name)
|
||||
except graph_database.GraphDatabaseQueryException as gdb_error:
|
||||
logger.error(f"Error dropping Tenant graph database: {gdb_error}")
|
||||
raise
|
||||
|
||||
Tenant.objects.using(MainRouter.admin_db).filter(id=pk).delete()
|
||||
|
||||
return deletion_summary
|
||||
|
||||
+155
-3536
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,186 @@
|
||||
# Base classes and data structures
|
||||
from .base import (
|
||||
BaseComplianceReportGenerator,
|
||||
ComplianceData,
|
||||
RequirementData,
|
||||
create_pdf_styles,
|
||||
get_requirement_metadata,
|
||||
)
|
||||
|
||||
# Chart functions
|
||||
from .charts import (
|
||||
create_horizontal_bar_chart,
|
||||
create_pie_chart,
|
||||
create_radar_chart,
|
||||
create_stacked_bar_chart,
|
||||
create_vertical_bar_chart,
|
||||
get_chart_color_for_percentage,
|
||||
)
|
||||
|
||||
# Reusable components
|
||||
# Reusable components: Color helpers, Badge components, Risk component,
|
||||
# Table components, Section components
|
||||
from .components import (
|
||||
ColumnConfig,
|
||||
create_badge,
|
||||
create_data_table,
|
||||
create_findings_table,
|
||||
create_info_table,
|
||||
create_multi_badge_row,
|
||||
create_risk_component,
|
||||
create_section_header,
|
||||
create_status_badge,
|
||||
create_summary_table,
|
||||
get_color_for_compliance,
|
||||
get_color_for_risk_level,
|
||||
get_color_for_weight,
|
||||
get_status_color,
|
||||
)
|
||||
|
||||
# Framework configuration: Main configuration, Color constants, ENS colors,
|
||||
# NIS2 colors, Chart colors, ENS constants, Section constants, Layout constants
|
||||
from .config import (
|
||||
CHART_COLOR_BLUE,
|
||||
CHART_COLOR_GREEN_1,
|
||||
CHART_COLOR_GREEN_2,
|
||||
CHART_COLOR_ORANGE,
|
||||
CHART_COLOR_RED,
|
||||
CHART_COLOR_YELLOW,
|
||||
COL_WIDTH_LARGE,
|
||||
COL_WIDTH_MEDIUM,
|
||||
COL_WIDTH_SMALL,
|
||||
COL_WIDTH_XLARGE,
|
||||
COL_WIDTH_XXLARGE,
|
||||
COLOR_BG_BLUE,
|
||||
COLOR_BG_LIGHT_BLUE,
|
||||
COLOR_BLUE,
|
||||
COLOR_DARK_GRAY,
|
||||
COLOR_ENS_ALTO,
|
||||
COLOR_ENS_BAJO,
|
||||
COLOR_ENS_MEDIO,
|
||||
COLOR_ENS_OPCIONAL,
|
||||
COLOR_GRAY,
|
||||
COLOR_HIGH_RISK,
|
||||
COLOR_LIGHT_BLUE,
|
||||
COLOR_LIGHT_GRAY,
|
||||
COLOR_LIGHTER_BLUE,
|
||||
COLOR_LOW_RISK,
|
||||
COLOR_MEDIUM_RISK,
|
||||
COLOR_NIS2_PRIMARY,
|
||||
COLOR_NIS2_SECONDARY,
|
||||
COLOR_PROWLER_DARK_GREEN,
|
||||
COLOR_SAFE,
|
||||
COLOR_WHITE,
|
||||
DIMENSION_KEYS,
|
||||
DIMENSION_MAPPING,
|
||||
DIMENSION_NAMES,
|
||||
ENS_NIVEL_ORDER,
|
||||
ENS_TIPO_ORDER,
|
||||
FRAMEWORK_REGISTRY,
|
||||
NIS2_SECTION_TITLES,
|
||||
NIS2_SECTIONS,
|
||||
PADDING_LARGE,
|
||||
PADDING_MEDIUM,
|
||||
PADDING_SMALL,
|
||||
PADDING_XLARGE,
|
||||
THREATSCORE_SECTIONS,
|
||||
TIPO_ICONS,
|
||||
FrameworkConfig,
|
||||
get_framework_config,
|
||||
)
|
||||
|
||||
# Framework-specific generators
|
||||
from .ens import ENSReportGenerator
|
||||
from .nis2 import NIS2ReportGenerator
|
||||
from .threatscore import ThreatScoreReportGenerator
|
||||
|
||||
__all__ = [
|
||||
# Base classes
|
||||
"BaseComplianceReportGenerator",
|
||||
"ComplianceData",
|
||||
"RequirementData",
|
||||
"create_pdf_styles",
|
||||
"get_requirement_metadata",
|
||||
# Framework-specific generators
|
||||
"ThreatScoreReportGenerator",
|
||||
"ENSReportGenerator",
|
||||
"NIS2ReportGenerator",
|
||||
# Configuration
|
||||
"FrameworkConfig",
|
||||
"FRAMEWORK_REGISTRY",
|
||||
"get_framework_config",
|
||||
# Color constants
|
||||
"COLOR_BLUE",
|
||||
"COLOR_LIGHT_BLUE",
|
||||
"COLOR_LIGHTER_BLUE",
|
||||
"COLOR_BG_BLUE",
|
||||
"COLOR_BG_LIGHT_BLUE",
|
||||
"COLOR_GRAY",
|
||||
"COLOR_LIGHT_GRAY",
|
||||
"COLOR_DARK_GRAY",
|
||||
"COLOR_WHITE",
|
||||
"COLOR_HIGH_RISK",
|
||||
"COLOR_MEDIUM_RISK",
|
||||
"COLOR_LOW_RISK",
|
||||
"COLOR_SAFE",
|
||||
"COLOR_PROWLER_DARK_GREEN",
|
||||
"COLOR_ENS_ALTO",
|
||||
"COLOR_ENS_MEDIO",
|
||||
"COLOR_ENS_BAJO",
|
||||
"COLOR_ENS_OPCIONAL",
|
||||
"COLOR_NIS2_PRIMARY",
|
||||
"COLOR_NIS2_SECONDARY",
|
||||
"CHART_COLOR_BLUE",
|
||||
"CHART_COLOR_GREEN_1",
|
||||
"CHART_COLOR_GREEN_2",
|
||||
"CHART_COLOR_YELLOW",
|
||||
"CHART_COLOR_ORANGE",
|
||||
"CHART_COLOR_RED",
|
||||
# ENS constants
|
||||
"DIMENSION_MAPPING",
|
||||
"DIMENSION_NAMES",
|
||||
"DIMENSION_KEYS",
|
||||
"ENS_NIVEL_ORDER",
|
||||
"ENS_TIPO_ORDER",
|
||||
"TIPO_ICONS",
|
||||
# Section constants
|
||||
"THREATSCORE_SECTIONS",
|
||||
"NIS2_SECTIONS",
|
||||
"NIS2_SECTION_TITLES",
|
||||
# Layout constants
|
||||
"COL_WIDTH_SMALL",
|
||||
"COL_WIDTH_MEDIUM",
|
||||
"COL_WIDTH_LARGE",
|
||||
"COL_WIDTH_XLARGE",
|
||||
"COL_WIDTH_XXLARGE",
|
||||
"PADDING_SMALL",
|
||||
"PADDING_MEDIUM",
|
||||
"PADDING_LARGE",
|
||||
"PADDING_XLARGE",
|
||||
# Color helpers
|
||||
"get_color_for_risk_level",
|
||||
"get_color_for_weight",
|
||||
"get_color_for_compliance",
|
||||
"get_status_color",
|
||||
# Badge components
|
||||
"create_badge",
|
||||
"create_status_badge",
|
||||
"create_multi_badge_row",
|
||||
# Risk component
|
||||
"create_risk_component",
|
||||
# Table components
|
||||
"create_info_table",
|
||||
"create_data_table",
|
||||
"create_findings_table",
|
||||
"ColumnConfig",
|
||||
# Section components
|
||||
"create_section_header",
|
||||
"create_summary_table",
|
||||
# Chart functions
|
||||
"get_chart_color_for_percentage",
|
||||
"create_vertical_bar_chart",
|
||||
"create_horizontal_bar_chart",
|
||||
"create_radar_chart",
|
||||
"create_pie_chart",
|
||||
"create_stacked_bar_chart",
|
||||
]
|
||||
@@ -0,0 +1,911 @@
|
||||
import gc
|
||||
import os
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from celery.utils.log import get_task_logger
|
||||
from reportlab.lib.enums import TA_CENTER
|
||||
from reportlab.lib.pagesizes import letter
|
||||
from reportlab.lib.styles import ParagraphStyle, getSampleStyleSheet
|
||||
from reportlab.lib.units import inch
|
||||
from reportlab.pdfbase import pdfmetrics
|
||||
from reportlab.pdfbase.ttfonts import TTFont
|
||||
from reportlab.pdfgen import canvas
|
||||
from reportlab.platypus import Image, PageBreak, Paragraph, SimpleDocTemplate, Spacer
|
||||
from tasks.jobs.threatscore_utils import (
|
||||
_aggregate_requirement_statistics_from_database,
|
||||
_calculate_requirements_data_from_statistics,
|
||||
_load_findings_for_requirement_checks,
|
||||
)
|
||||
|
||||
from api.db_router import READ_REPLICA_ALIAS
|
||||
from api.db_utils import rls_transaction
|
||||
from api.models import Provider, StatusChoices
|
||||
from api.utils import initialize_prowler_provider
|
||||
from prowler.lib.check.compliance_models import Compliance
|
||||
from prowler.lib.outputs.finding import Finding as FindingOutput
|
||||
|
||||
from .components import (
|
||||
ColumnConfig,
|
||||
create_data_table,
|
||||
create_info_table,
|
||||
create_status_badge,
|
||||
)
|
||||
from .config import (
|
||||
COLOR_BG_BLUE,
|
||||
COLOR_BG_LIGHT_BLUE,
|
||||
COLOR_BLUE,
|
||||
COLOR_BORDER_GRAY,
|
||||
COLOR_GRAY,
|
||||
COLOR_LIGHT_BLUE,
|
||||
COLOR_LIGHTER_BLUE,
|
||||
COLOR_PROWLER_DARK_GREEN,
|
||||
PADDING_LARGE,
|
||||
PADDING_SMALL,
|
||||
FrameworkConfig,
|
||||
)
|
||||
|
||||
logger = get_task_logger(__name__)
|
||||
|
||||
# Register fonts (done once at module load)
|
||||
_fonts_registered: bool = False
|
||||
|
||||
|
||||
def _register_fonts() -> None:
|
||||
"""Register custom fonts for PDF generation.
|
||||
|
||||
Uses a module-level flag to ensure fonts are only registered once,
|
||||
avoiding duplicate registration errors from reportlab.
|
||||
"""
|
||||
global _fonts_registered
|
||||
if _fonts_registered:
|
||||
return
|
||||
|
||||
fonts_dir = os.path.join(os.path.dirname(__file__), "../../assets/fonts")
|
||||
|
||||
pdfmetrics.registerFont(
|
||||
TTFont(
|
||||
"PlusJakartaSans",
|
||||
os.path.join(fonts_dir, "PlusJakartaSans-Regular.ttf"),
|
||||
)
|
||||
)
|
||||
|
||||
pdfmetrics.registerFont(
|
||||
TTFont(
|
||||
"FiraCode",
|
||||
os.path.join(fonts_dir, "FiraCode-Regular.ttf"),
|
||||
)
|
||||
)
|
||||
|
||||
_fonts_registered = True
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Data Classes
|
||||
# =============================================================================
|
||||
|
||||
|
||||
@dataclass
|
||||
class RequirementData:
|
||||
"""Data for a single compliance requirement.
|
||||
|
||||
Attributes:
|
||||
id: Requirement identifier
|
||||
description: Requirement description
|
||||
status: Compliance status (PASS, FAIL, MANUAL)
|
||||
passed_findings: Number of passed findings
|
||||
failed_findings: Number of failed findings
|
||||
total_findings: Total number of findings
|
||||
checks: List of check IDs associated with this requirement
|
||||
attributes: Framework-specific requirement attributes
|
||||
"""
|
||||
|
||||
id: str
|
||||
description: str
|
||||
status: str
|
||||
passed_findings: int = 0
|
||||
failed_findings: int = 0
|
||||
total_findings: int = 0
|
||||
checks: list[str] = field(default_factory=list)
|
||||
attributes: Any = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class ComplianceData:
|
||||
"""Aggregated compliance data for report generation.
|
||||
|
||||
This dataclass holds all the data needed to generate a compliance report,
|
||||
including compliance framework metadata, requirements, and findings.
|
||||
|
||||
Attributes:
|
||||
tenant_id: Tenant identifier
|
||||
scan_id: Scan identifier
|
||||
provider_id: Provider identifier
|
||||
compliance_id: Compliance framework identifier
|
||||
framework: Framework name (e.g., "CIS", "ENS")
|
||||
name: Full compliance framework name
|
||||
version: Framework version
|
||||
description: Framework description
|
||||
requirements: List of RequirementData objects
|
||||
attributes_by_requirement_id: Mapping of requirement IDs to their attributes
|
||||
findings_by_check_id: Mapping of check IDs to their findings
|
||||
provider_obj: Provider model object
|
||||
prowler_provider: Initialized Prowler provider
|
||||
"""
|
||||
|
||||
tenant_id: str
|
||||
scan_id: str
|
||||
provider_id: str
|
||||
compliance_id: str
|
||||
framework: str
|
||||
name: str
|
||||
version: str
|
||||
description: str
|
||||
requirements: list[RequirementData] = field(default_factory=list)
|
||||
attributes_by_requirement_id: dict[str, dict] = field(default_factory=dict)
|
||||
findings_by_check_id: dict[str, list[FindingOutput]] = field(default_factory=dict)
|
||||
provider_obj: Provider | None = None
|
||||
prowler_provider: Any = None
|
||||
|
||||
|
||||
def get_requirement_metadata(
|
||||
requirement_id: str,
|
||||
attributes_by_requirement_id: dict[str, dict],
|
||||
) -> Any | None:
|
||||
"""Get the first requirement metadata object from attributes.
|
||||
|
||||
This helper function extracts the requirement metadata (req_attributes)
|
||||
from the attributes dictionary. It's a common pattern used across all
|
||||
report generators.
|
||||
|
||||
Args:
|
||||
requirement_id: The requirement ID to look up.
|
||||
attributes_by_requirement_id: Mapping of requirement IDs to their attributes.
|
||||
|
||||
Returns:
|
||||
The first requirement attribute object, or None if not found.
|
||||
|
||||
Example:
|
||||
>>> meta = get_requirement_metadata(req.id, data.attributes_by_requirement_id)
|
||||
>>> if meta:
|
||||
... section = getattr(meta, "Section", "Unknown")
|
||||
"""
|
||||
req_attrs = attributes_by_requirement_id.get(requirement_id, {})
|
||||
meta_list = req_attrs.get("attributes", {}).get("req_attributes", [])
|
||||
if meta_list:
|
||||
return meta_list[0]
|
||||
return None
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# PDF Styles Cache
|
||||
# =============================================================================
|
||||
|
||||
_PDF_STYLES_CACHE: dict[str, ParagraphStyle] | None = None
|
||||
|
||||
|
||||
def create_pdf_styles() -> dict[str, ParagraphStyle]:
|
||||
"""Create and return PDF paragraph styles used throughout the report.
|
||||
|
||||
Styles are cached on first call to improve performance.
|
||||
|
||||
Returns:
|
||||
Dictionary containing the following styles:
|
||||
- 'title': Title style with prowler green color
|
||||
- 'h1': Heading 1 style with blue color and background
|
||||
- 'h2': Heading 2 style with light blue color
|
||||
- 'h3': Heading 3 style for sub-headings
|
||||
- 'normal': Normal text style with left indent
|
||||
- 'normal_center': Normal text style without indent
|
||||
"""
|
||||
global _PDF_STYLES_CACHE
|
||||
|
||||
if _PDF_STYLES_CACHE is not None:
|
||||
return _PDF_STYLES_CACHE
|
||||
|
||||
_register_fonts()
|
||||
styles = getSampleStyleSheet()
|
||||
|
||||
title_style = ParagraphStyle(
|
||||
"CustomTitle",
|
||||
parent=styles["Title"],
|
||||
fontSize=24,
|
||||
textColor=COLOR_PROWLER_DARK_GREEN,
|
||||
spaceAfter=20,
|
||||
fontName="PlusJakartaSans",
|
||||
alignment=TA_CENTER,
|
||||
)
|
||||
|
||||
h1 = ParagraphStyle(
|
||||
"CustomH1",
|
||||
parent=styles["Heading1"],
|
||||
fontSize=18,
|
||||
textColor=COLOR_BLUE,
|
||||
spaceBefore=20,
|
||||
spaceAfter=12,
|
||||
fontName="PlusJakartaSans",
|
||||
leftIndent=0,
|
||||
borderWidth=2,
|
||||
borderColor=COLOR_BLUE,
|
||||
borderPadding=PADDING_LARGE,
|
||||
backColor=COLOR_BG_BLUE,
|
||||
)
|
||||
|
||||
h2 = ParagraphStyle(
|
||||
"CustomH2",
|
||||
parent=styles["Heading2"],
|
||||
fontSize=14,
|
||||
textColor=COLOR_LIGHT_BLUE,
|
||||
spaceBefore=15,
|
||||
spaceAfter=8,
|
||||
fontName="PlusJakartaSans",
|
||||
leftIndent=10,
|
||||
borderWidth=1,
|
||||
borderColor=COLOR_BORDER_GRAY,
|
||||
borderPadding=5,
|
||||
backColor=COLOR_BG_LIGHT_BLUE,
|
||||
)
|
||||
|
||||
h3 = ParagraphStyle(
|
||||
"CustomH3",
|
||||
parent=styles["Heading3"],
|
||||
fontSize=12,
|
||||
textColor=COLOR_LIGHTER_BLUE,
|
||||
spaceBefore=10,
|
||||
spaceAfter=6,
|
||||
fontName="PlusJakartaSans",
|
||||
leftIndent=20,
|
||||
)
|
||||
|
||||
normal = ParagraphStyle(
|
||||
"CustomNormal",
|
||||
parent=styles["Normal"],
|
||||
fontSize=10,
|
||||
textColor=COLOR_GRAY,
|
||||
spaceBefore=PADDING_SMALL,
|
||||
spaceAfter=PADDING_SMALL,
|
||||
leftIndent=30,
|
||||
fontName="PlusJakartaSans",
|
||||
)
|
||||
|
||||
normal_center = ParagraphStyle(
|
||||
"CustomNormalCenter",
|
||||
parent=styles["Normal"],
|
||||
fontSize=10,
|
||||
textColor=COLOR_GRAY,
|
||||
fontName="PlusJakartaSans",
|
||||
)
|
||||
|
||||
_PDF_STYLES_CACHE = {
|
||||
"title": title_style,
|
||||
"h1": h1,
|
||||
"h2": h2,
|
||||
"h3": h3,
|
||||
"normal": normal,
|
||||
"normal_center": normal_center,
|
||||
}
|
||||
|
||||
return _PDF_STYLES_CACHE
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Base Report Generator
|
||||
# =============================================================================
|
||||
|
||||
|
||||
class BaseComplianceReportGenerator(ABC):
|
||||
"""Abstract base class for compliance PDF report generators.
|
||||
|
||||
This class implements the Template Method pattern, providing a common
|
||||
structure for all compliance reports while allowing subclasses to
|
||||
customize specific sections.
|
||||
|
||||
Subclasses must implement:
|
||||
- create_executive_summary()
|
||||
- create_charts_section()
|
||||
- create_requirements_index()
|
||||
|
||||
Optionally, subclasses can override:
|
||||
- create_cover_page()
|
||||
- create_detailed_findings()
|
||||
- get_footer_text()
|
||||
"""
|
||||
|
||||
def __init__(self, config: FrameworkConfig):
|
||||
"""Initialize the report generator.
|
||||
|
||||
Args:
|
||||
config: Framework configuration
|
||||
"""
|
||||
self.config = config
|
||||
self.styles = create_pdf_styles()
|
||||
|
||||
# =========================================================================
|
||||
# Template Method
|
||||
# =========================================================================
|
||||
|
||||
def generate(
|
||||
self,
|
||||
tenant_id: str,
|
||||
scan_id: str,
|
||||
compliance_id: str,
|
||||
output_path: str,
|
||||
provider_id: str,
|
||||
provider_obj: Provider | None = None,
|
||||
requirement_statistics: dict[str, dict[str, int]] | None = None,
|
||||
findings_cache: dict[str, list[FindingOutput]] | None = None,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
"""Generate the PDF compliance report.
|
||||
|
||||
This is the template method that orchestrates the report generation.
|
||||
It calls abstract methods that subclasses must implement.
|
||||
|
||||
Args:
|
||||
tenant_id: Tenant identifier for RLS context
|
||||
scan_id: Scan identifier
|
||||
compliance_id: Compliance framework identifier
|
||||
output_path: Path where the PDF will be saved
|
||||
provider_id: Provider identifier
|
||||
provider_obj: Optional pre-fetched Provider object
|
||||
requirement_statistics: Optional pre-aggregated statistics
|
||||
findings_cache: Optional pre-loaded findings cache
|
||||
**kwargs: Additional framework-specific arguments
|
||||
"""
|
||||
logger.info(
|
||||
"Generating %s report for scan %s", self.config.display_name, scan_id
|
||||
)
|
||||
|
||||
try:
|
||||
# 1. Load compliance data
|
||||
data = self._load_compliance_data(
|
||||
tenant_id=tenant_id,
|
||||
scan_id=scan_id,
|
||||
compliance_id=compliance_id,
|
||||
provider_id=provider_id,
|
||||
provider_obj=provider_obj,
|
||||
requirement_statistics=requirement_statistics,
|
||||
findings_cache=findings_cache,
|
||||
)
|
||||
|
||||
# 2. Create PDF document
|
||||
doc = self._create_document(output_path, data)
|
||||
|
||||
# 3. Build report elements incrementally to manage memory
|
||||
# We collect garbage after heavy sections to prevent OOM on large reports
|
||||
elements = []
|
||||
|
||||
# Cover page (lightweight)
|
||||
elements.extend(self.create_cover_page(data))
|
||||
elements.append(PageBreak())
|
||||
|
||||
# Executive summary (framework-specific)
|
||||
elements.extend(self.create_executive_summary(data))
|
||||
|
||||
# Body sections (charts + requirements index)
|
||||
# Override _build_body_sections() in subclasses to change section order
|
||||
elements.extend(self._build_body_sections(data))
|
||||
|
||||
# Detailed findings - heaviest section, loads findings on-demand
|
||||
logger.info("Building detailed findings section...")
|
||||
elements.extend(self.create_detailed_findings(data, **kwargs))
|
||||
gc.collect() # Free findings data after processing
|
||||
|
||||
# 4. Build the PDF
|
||||
logger.info("Building PDF document with %d elements...", len(elements))
|
||||
self._build_pdf(doc, elements, data)
|
||||
|
||||
# Final cleanup
|
||||
del elements
|
||||
gc.collect()
|
||||
|
||||
logger.info("Successfully generated report at %s", output_path)
|
||||
|
||||
except Exception as e:
|
||||
import traceback
|
||||
|
||||
tb_lineno = e.__traceback__.tb_lineno if e.__traceback__ else "unknown"
|
||||
logger.error("Error generating report, line %s -- %s", tb_lineno, e)
|
||||
logger.error("Full traceback:\n%s", traceback.format_exc())
|
||||
raise
|
||||
|
||||
def _build_body_sections(self, data: ComplianceData) -> list:
|
||||
"""Build the body sections between executive summary and detailed findings.
|
||||
|
||||
Override in subclasses to change section order.
|
||||
|
||||
Args:
|
||||
data: Aggregated compliance data.
|
||||
|
||||
Returns:
|
||||
List of ReportLab elements.
|
||||
"""
|
||||
elements = []
|
||||
|
||||
# Charts section (framework-specific) - heavy on memory due to matplotlib
|
||||
elements.extend(self.create_charts_section(data))
|
||||
elements.append(PageBreak())
|
||||
gc.collect() # Free matplotlib resources
|
||||
|
||||
# Requirements index (framework-specific)
|
||||
elements.extend(self.create_requirements_index(data))
|
||||
elements.append(PageBreak())
|
||||
|
||||
return elements
|
||||
|
||||
# =========================================================================
|
||||
# Abstract Methods (must be implemented by subclasses)
|
||||
# =========================================================================
|
||||
|
||||
@abstractmethod
|
||||
def create_executive_summary(self, data: ComplianceData) -> list:
|
||||
"""Create the executive summary section.
|
||||
|
||||
This section typically includes:
|
||||
- Overall compliance score/metrics
|
||||
- High-level statistics
|
||||
- Critical findings summary
|
||||
|
||||
Args:
|
||||
data: Aggregated compliance data
|
||||
|
||||
Returns:
|
||||
List of ReportLab elements
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def create_charts_section(self, data: ComplianceData) -> list:
|
||||
"""Create the charts and visualizations section.
|
||||
|
||||
This section typically includes:
|
||||
- Compliance score charts by section
|
||||
- Distribution charts
|
||||
- Trend visualizations
|
||||
|
||||
Args:
|
||||
data: Aggregated compliance data
|
||||
|
||||
Returns:
|
||||
List of ReportLab elements
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def create_requirements_index(self, data: ComplianceData) -> list:
|
||||
"""Create the requirements index/table of contents.
|
||||
|
||||
This section typically includes:
|
||||
- Hierarchical list of requirements
|
||||
- Status indicators
|
||||
- Section groupings
|
||||
|
||||
Args:
|
||||
data: Aggregated compliance data
|
||||
|
||||
Returns:
|
||||
List of ReportLab elements
|
||||
"""
|
||||
|
||||
# =========================================================================
|
||||
# Common Methods (can be overridden by subclasses)
|
||||
# =========================================================================
|
||||
|
||||
def create_cover_page(self, data: ComplianceData) -> list:
|
||||
"""Create the report cover page.
|
||||
|
||||
Args:
|
||||
data: Aggregated compliance data
|
||||
|
||||
Returns:
|
||||
List of ReportLab elements
|
||||
"""
|
||||
elements = []
|
||||
|
||||
# Prowler logo
|
||||
logo_path = os.path.join(
|
||||
os.path.dirname(__file__), "../../assets/img/prowler_logo.png"
|
||||
)
|
||||
if os.path.exists(logo_path):
|
||||
logo = Image(logo_path, width=5 * inch, height=1 * inch)
|
||||
elements.append(logo)
|
||||
|
||||
elements.append(Spacer(1, 0.5 * inch))
|
||||
|
||||
# Title
|
||||
title_text = f"{self.config.display_name} Report"
|
||||
elements.append(Paragraph(title_text, self.styles["title"]))
|
||||
elements.append(Spacer(1, 0.5 * inch))
|
||||
|
||||
# Compliance info table
|
||||
info_rows = self._build_info_rows(data, language=self.config.language)
|
||||
|
||||
info_table = create_info_table(
|
||||
rows=info_rows,
|
||||
label_width=2 * inch,
|
||||
value_width=4 * inch,
|
||||
normal_style=self.styles["normal_center"],
|
||||
)
|
||||
elements.append(info_table)
|
||||
|
||||
return elements
|
||||
|
||||
def _build_info_rows(
|
||||
self, data: ComplianceData, language: str = "en"
|
||||
) -> list[tuple[str, str]]:
|
||||
"""Build the standard info rows for the cover page table.
|
||||
|
||||
This helper method creates the common metadata rows used in all
|
||||
report cover pages. Subclasses can use this to maintain consistency
|
||||
while customizing other aspects of the cover page.
|
||||
|
||||
Args:
|
||||
data: Aggregated compliance data.
|
||||
language: Language for labels ("en" or "es").
|
||||
|
||||
Returns:
|
||||
List of (label, value) tuples for the info table.
|
||||
"""
|
||||
# Labels based on language
|
||||
labels = {
|
||||
"en": {
|
||||
"framework": "Framework:",
|
||||
"id": "ID:",
|
||||
"name": "Name:",
|
||||
"version": "Version:",
|
||||
"provider": "Provider:",
|
||||
"account_id": "Account ID:",
|
||||
"alias": "Alias:",
|
||||
"scan_id": "Scan ID:",
|
||||
"description": "Description:",
|
||||
},
|
||||
"es": {
|
||||
"framework": "Framework:",
|
||||
"id": "ID:",
|
||||
"name": "Nombre:",
|
||||
"version": "Versión:",
|
||||
"provider": "Proveedor:",
|
||||
"account_id": "Account ID:",
|
||||
"alias": "Alias:",
|
||||
"scan_id": "Scan ID:",
|
||||
"description": "Descripción:",
|
||||
},
|
||||
}
|
||||
lang_labels = labels.get(language, labels["en"])
|
||||
|
||||
info_rows = [
|
||||
(lang_labels["framework"], data.framework),
|
||||
(lang_labels["id"], data.compliance_id),
|
||||
(lang_labels["name"], data.name),
|
||||
(lang_labels["version"], data.version),
|
||||
]
|
||||
|
||||
# Add provider info if available
|
||||
if data.provider_obj:
|
||||
info_rows.append(
|
||||
(lang_labels["provider"], data.provider_obj.provider.upper())
|
||||
)
|
||||
info_rows.append(
|
||||
(lang_labels["account_id"], data.provider_obj.uid or "N/A")
|
||||
)
|
||||
info_rows.append((lang_labels["alias"], data.provider_obj.alias or "N/A"))
|
||||
|
||||
info_rows.append((lang_labels["scan_id"], data.scan_id))
|
||||
|
||||
if data.description:
|
||||
info_rows.append((lang_labels["description"], data.description))
|
||||
|
||||
return info_rows
|
||||
|
||||
def create_detailed_findings(self, data: ComplianceData, **kwargs) -> list:
|
||||
"""Create the detailed findings section.
|
||||
|
||||
This default implementation creates a requirement-by-requirement
|
||||
breakdown with findings tables. Subclasses can override for
|
||||
framework-specific presentation.
|
||||
|
||||
This method implements on-demand loading of findings using the shared
|
||||
findings cache to minimize database queries and memory usage.
|
||||
|
||||
Args:
|
||||
data: Aggregated compliance data
|
||||
**kwargs: Framework-specific options (e.g., only_failed)
|
||||
|
||||
Returns:
|
||||
List of ReportLab elements
|
||||
"""
|
||||
elements = []
|
||||
only_failed = kwargs.get("only_failed", True)
|
||||
include_manual = kwargs.get("include_manual", False)
|
||||
|
||||
# Filter requirements if needed
|
||||
requirements = data.requirements
|
||||
if only_failed:
|
||||
# Include FAIL requirements, and optionally MANUAL if include_manual is True
|
||||
if include_manual:
|
||||
requirements = [
|
||||
r
|
||||
for r in requirements
|
||||
if r.status in (StatusChoices.FAIL, StatusChoices.MANUAL)
|
||||
]
|
||||
else:
|
||||
requirements = [
|
||||
r for r in requirements if r.status == StatusChoices.FAIL
|
||||
]
|
||||
|
||||
# Collect all check IDs for requirements that will be displayed
|
||||
# This allows us to load only the findings we actually need (memory optimization)
|
||||
check_ids_to_load = []
|
||||
for req in requirements:
|
||||
check_ids_to_load.extend(req.checks)
|
||||
|
||||
# Load findings on-demand only for the checks that will be displayed
|
||||
# Uses the shared findings cache to avoid duplicate queries across reports
|
||||
logger.info("Loading findings on-demand for %d requirements", len(requirements))
|
||||
findings_by_check_id = _load_findings_for_requirement_checks(
|
||||
data.tenant_id,
|
||||
data.scan_id,
|
||||
check_ids_to_load,
|
||||
data.prowler_provider,
|
||||
data.findings_by_check_id, # Pass the cache to update it
|
||||
)
|
||||
|
||||
for req in requirements:
|
||||
# Requirement header
|
||||
elements.append(
|
||||
Paragraph(
|
||||
f"{req.id}: {req.description}",
|
||||
self.styles["h1"],
|
||||
)
|
||||
)
|
||||
|
||||
# Status badge
|
||||
elements.append(create_status_badge(req.status))
|
||||
elements.append(Spacer(1, 0.1 * inch))
|
||||
|
||||
# Findings for this requirement
|
||||
for check_id in req.checks:
|
||||
elements.append(Paragraph(f"Check: {check_id}", self.styles["h2"]))
|
||||
|
||||
findings = findings_by_check_id.get(check_id, [])
|
||||
if not findings:
|
||||
elements.append(
|
||||
Paragraph(
|
||||
"- No information for this finding currently",
|
||||
self.styles["normal"],
|
||||
)
|
||||
)
|
||||
else:
|
||||
# Create findings table
|
||||
findings_table = self._create_findings_table(findings)
|
||||
elements.append(findings_table)
|
||||
|
||||
elements.append(Spacer(1, 0.1 * inch))
|
||||
|
||||
elements.append(PageBreak())
|
||||
|
||||
return elements
|
||||
|
||||
def get_footer_text(self, page_num: int) -> tuple[str, str]:
|
||||
"""Get footer text for a page.
|
||||
|
||||
Args:
|
||||
page_num: Current page number
|
||||
|
||||
Returns:
|
||||
Tuple of (left_text, right_text) for the footer
|
||||
"""
|
||||
if self.config.language == "es":
|
||||
page_text = f"Página {page_num}"
|
||||
else:
|
||||
page_text = f"Page {page_num}"
|
||||
|
||||
return page_text, "Powered by Prowler"
|
||||
|
||||
# =========================================================================
|
||||
# Private Helper Methods
|
||||
# =========================================================================
|
||||
|
||||
def _load_compliance_data(
|
||||
self,
|
||||
tenant_id: str,
|
||||
scan_id: str,
|
||||
compliance_id: str,
|
||||
provider_id: str,
|
||||
provider_obj: Provider | None,
|
||||
requirement_statistics: dict | None,
|
||||
findings_cache: dict | None,
|
||||
) -> ComplianceData:
|
||||
"""Load and aggregate compliance data from the database.
|
||||
|
||||
Args:
|
||||
tenant_id: Tenant identifier
|
||||
scan_id: Scan identifier
|
||||
compliance_id: Compliance framework identifier
|
||||
provider_id: Provider identifier
|
||||
provider_obj: Optional pre-fetched Provider
|
||||
requirement_statistics: Optional pre-aggregated statistics
|
||||
findings_cache: Optional pre-loaded findings
|
||||
|
||||
Returns:
|
||||
Aggregated ComplianceData object
|
||||
"""
|
||||
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
|
||||
# Load provider
|
||||
if provider_obj is None:
|
||||
provider_obj = Provider.objects.get(id=provider_id)
|
||||
|
||||
prowler_provider = initialize_prowler_provider(provider_obj)
|
||||
provider_type = provider_obj.provider
|
||||
|
||||
# Load compliance framework
|
||||
frameworks_bulk = Compliance.get_bulk(provider_type)
|
||||
compliance_obj = frameworks_bulk.get(compliance_id)
|
||||
|
||||
if not compliance_obj:
|
||||
raise ValueError(f"Compliance framework not found: {compliance_id}")
|
||||
|
||||
framework = getattr(compliance_obj, "Framework", "N/A")
|
||||
name = getattr(compliance_obj, "Name", "N/A")
|
||||
version = getattr(compliance_obj, "Version", "N/A")
|
||||
description = getattr(compliance_obj, "Description", "")
|
||||
|
||||
# Aggregate requirement statistics
|
||||
if requirement_statistics is None:
|
||||
logger.info("Aggregating requirement statistics for scan %s", scan_id)
|
||||
requirement_statistics = _aggregate_requirement_statistics_from_database(
|
||||
tenant_id, scan_id
|
||||
)
|
||||
else:
|
||||
logger.info("Reusing pre-aggregated statistics for scan %s", scan_id)
|
||||
|
||||
# Calculate requirements data
|
||||
attributes_by_requirement_id, requirements_list = (
|
||||
_calculate_requirements_data_from_statistics(
|
||||
compliance_obj, requirement_statistics
|
||||
)
|
||||
)
|
||||
|
||||
# Convert to RequirementData objects
|
||||
requirements = []
|
||||
for req_dict in requirements_list:
|
||||
req = RequirementData(
|
||||
id=req_dict["id"],
|
||||
description=req_dict["attributes"].get("description", ""),
|
||||
status=req_dict["attributes"].get("status", StatusChoices.MANUAL),
|
||||
passed_findings=req_dict["attributes"].get("passed_findings", 0),
|
||||
failed_findings=req_dict["attributes"].get("failed_findings", 0),
|
||||
total_findings=req_dict["attributes"].get("total_findings", 0),
|
||||
checks=attributes_by_requirement_id.get(req_dict["id"], {})
|
||||
.get("attributes", {})
|
||||
.get("checks", []),
|
||||
)
|
||||
requirements.append(req)
|
||||
|
||||
return ComplianceData(
|
||||
tenant_id=tenant_id,
|
||||
scan_id=scan_id,
|
||||
provider_id=provider_id,
|
||||
compliance_id=compliance_id,
|
||||
framework=framework,
|
||||
name=name,
|
||||
version=version,
|
||||
description=description,
|
||||
requirements=requirements,
|
||||
attributes_by_requirement_id=attributes_by_requirement_id,
|
||||
findings_by_check_id=findings_cache if findings_cache is not None else {},
|
||||
provider_obj=provider_obj,
|
||||
prowler_provider=prowler_provider,
|
||||
)
|
||||
|
||||
def _create_document(
|
||||
self, output_path: str, data: ComplianceData
|
||||
) -> SimpleDocTemplate:
|
||||
"""Create the PDF document template.
|
||||
|
||||
Args:
|
||||
output_path: Path for the output PDF
|
||||
data: Compliance data for metadata
|
||||
|
||||
Returns:
|
||||
Configured SimpleDocTemplate
|
||||
"""
|
||||
return SimpleDocTemplate(
|
||||
output_path,
|
||||
pagesize=letter,
|
||||
title=f"{self.config.display_name} Report - {data.framework}",
|
||||
author="Prowler",
|
||||
subject=f"Compliance Report for {data.framework}",
|
||||
creator="Prowler Engineering Team",
|
||||
keywords=f"compliance,{data.framework},security,framework,prowler",
|
||||
)
|
||||
|
||||
def _build_pdf(
|
||||
self,
|
||||
doc: SimpleDocTemplate,
|
||||
elements: list,
|
||||
data: ComplianceData,
|
||||
) -> None:
|
||||
"""Build the final PDF with footers.
|
||||
|
||||
Args:
|
||||
doc: Document template
|
||||
elements: List of ReportLab elements
|
||||
data: Compliance data
|
||||
"""
|
||||
|
||||
def add_footer(
|
||||
canvas_obj: canvas.Canvas,
|
||||
doc_template: SimpleDocTemplate,
|
||||
) -> None:
|
||||
canvas_obj.saveState()
|
||||
width, _ = doc_template.pagesize
|
||||
left_text, right_text = self.get_footer_text(doc_template.page)
|
||||
|
||||
canvas_obj.setFont("PlusJakartaSans", 9)
|
||||
canvas_obj.setFillColorRGB(0.4, 0.4, 0.4)
|
||||
canvas_obj.drawString(30, 20, left_text)
|
||||
|
||||
text_width = canvas_obj.stringWidth(right_text, "PlusJakartaSans", 9)
|
||||
canvas_obj.drawString(width - text_width - 30, 20, right_text)
|
||||
canvas_obj.restoreState()
|
||||
|
||||
doc.build(
|
||||
elements,
|
||||
onFirstPage=add_footer,
|
||||
onLaterPages=add_footer,
|
||||
)
|
||||
|
||||
def _create_findings_table(self, findings: list[FindingOutput]) -> Any:
|
||||
"""Create a findings table.
|
||||
|
||||
Args:
|
||||
findings: List of finding objects
|
||||
|
||||
Returns:
|
||||
ReportLab Table element
|
||||
"""
|
||||
|
||||
def get_finding_title(f):
|
||||
metadata = getattr(f, "metadata", None)
|
||||
if metadata:
|
||||
return getattr(metadata, "CheckTitle", getattr(f, "check_id", ""))
|
||||
return getattr(f, "check_id", "")
|
||||
|
||||
def get_resource_name(f):
|
||||
name = getattr(f, "resource_name", "")
|
||||
if not name:
|
||||
name = getattr(f, "resource_uid", "")
|
||||
return name
|
||||
|
||||
def get_severity(f):
|
||||
metadata = getattr(f, "metadata", None)
|
||||
if metadata:
|
||||
return getattr(metadata, "Severity", "").capitalize()
|
||||
return ""
|
||||
|
||||
# Convert findings to dicts for the table
|
||||
data = []
|
||||
for f in findings:
|
||||
item = {
|
||||
"title": get_finding_title(f),
|
||||
"resource_name": get_resource_name(f),
|
||||
"severity": get_severity(f),
|
||||
"status": getattr(f, "status", "").upper(),
|
||||
"region": getattr(f, "region", "global"),
|
||||
}
|
||||
data.append(item)
|
||||
|
||||
columns = [
|
||||
ColumnConfig("Finding", 2.5 * inch, "title"),
|
||||
ColumnConfig("Resource", 3 * inch, "resource_name"),
|
||||
ColumnConfig("Severity", 0.9 * inch, "severity"),
|
||||
ColumnConfig("Status", 0.9 * inch, "status"),
|
||||
ColumnConfig("Region", 0.9 * inch, "region"),
|
||||
]
|
||||
|
||||
return create_data_table(
|
||||
data=data,
|
||||
columns=columns,
|
||||
header_color=self.config.primary_color,
|
||||
normal_style=self.styles["normal_center"],
|
||||
)
|
||||
@@ -0,0 +1,404 @@
|
||||
import gc
|
||||
import io
|
||||
import math
|
||||
from typing import Callable
|
||||
|
||||
import matplotlib
|
||||
|
||||
# Use non-interactive Agg backend for memory efficiency in server environments
|
||||
# This MUST be set before importing pyplot
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt # noqa: E402
|
||||
|
||||
from .config import ( # noqa: E402
|
||||
CHART_COLOR_BLUE,
|
||||
CHART_COLOR_GREEN_1,
|
||||
CHART_COLOR_GREEN_2,
|
||||
CHART_COLOR_ORANGE,
|
||||
CHART_COLOR_RED,
|
||||
CHART_COLOR_YELLOW,
|
||||
CHART_DPI_DEFAULT,
|
||||
)
|
||||
|
||||
# Use centralized DPI setting from config
|
||||
DEFAULT_CHART_DPI = CHART_DPI_DEFAULT
|
||||
|
||||
|
||||
def get_chart_color_for_percentage(percentage: float) -> str:
|
||||
"""Get chart color string based on percentage.
|
||||
|
||||
Args:
|
||||
percentage: Value between 0 and 100
|
||||
|
||||
Returns:
|
||||
Hex color string for matplotlib
|
||||
"""
|
||||
if percentage >= 80:
|
||||
return CHART_COLOR_GREEN_1
|
||||
if percentage >= 60:
|
||||
return CHART_COLOR_GREEN_2
|
||||
if percentage >= 40:
|
||||
return CHART_COLOR_YELLOW
|
||||
if percentage >= 20:
|
||||
return CHART_COLOR_ORANGE
|
||||
return CHART_COLOR_RED
|
||||
|
||||
|
||||
def create_vertical_bar_chart(
|
||||
labels: list[str],
|
||||
values: list[float],
|
||||
ylabel: str = "Compliance Score (%)",
|
||||
xlabel: str = "Section",
|
||||
title: str | None = None,
|
||||
color_func: Callable[[float], str] | None = None,
|
||||
colors: list[str] | None = None,
|
||||
figsize: tuple[int, int] = (10, 6),
|
||||
dpi: int = DEFAULT_CHART_DPI,
|
||||
y_limit: tuple[float, float] = (0, 100),
|
||||
show_labels: bool = True,
|
||||
rotation: int = 45,
|
||||
) -> io.BytesIO:
|
||||
"""Create a vertical bar chart.
|
||||
|
||||
Args:
|
||||
labels: X-axis labels
|
||||
values: Bar heights (numeric values)
|
||||
ylabel: Y-axis label
|
||||
xlabel: X-axis label
|
||||
title: Optional chart title
|
||||
color_func: Function to determine bar color based on value
|
||||
colors: Explicit list of colors (overrides color_func)
|
||||
figsize: Figure size (width, height) in inches
|
||||
dpi: Resolution for output image
|
||||
y_limit: Y-axis limits (min, max)
|
||||
show_labels: Whether to show value labels on bars
|
||||
rotation: X-axis label rotation angle
|
||||
|
||||
Returns:
|
||||
BytesIO buffer containing the PNG image
|
||||
"""
|
||||
if color_func is None:
|
||||
color_func = get_chart_color_for_percentage
|
||||
|
||||
fig, ax = plt.subplots(figsize=figsize)
|
||||
|
||||
# Determine colors
|
||||
if colors is None:
|
||||
colors_list = [color_func(v) for v in values]
|
||||
else:
|
||||
colors_list = colors
|
||||
|
||||
bars = ax.bar(labels, values, color=colors_list)
|
||||
|
||||
ax.set_ylabel(ylabel, fontsize=12)
|
||||
ax.set_xlabel(xlabel, fontsize=12)
|
||||
ax.set_ylim(*y_limit)
|
||||
|
||||
if title:
|
||||
ax.set_title(title, fontsize=14, fontweight="bold")
|
||||
|
||||
# Add value labels on bars
|
||||
if show_labels:
|
||||
for bar_item, value in zip(bars, values):
|
||||
height = bar_item.get_height()
|
||||
ax.text(
|
||||
bar_item.get_x() + bar_item.get_width() / 2.0,
|
||||
height + 1,
|
||||
f"{value:.1f}%",
|
||||
ha="center",
|
||||
va="bottom",
|
||||
fontweight="bold",
|
||||
)
|
||||
|
||||
plt.xticks(rotation=rotation, ha="right")
|
||||
ax.grid(True, alpha=0.3, axis="y")
|
||||
plt.tight_layout()
|
||||
|
||||
buffer = io.BytesIO()
|
||||
try:
|
||||
fig.savefig(buffer, format="png", dpi=dpi, bbox_inches="tight")
|
||||
buffer.seek(0)
|
||||
finally:
|
||||
plt.close(fig)
|
||||
gc.collect() # Force garbage collection after heavy matplotlib operation
|
||||
|
||||
return buffer
|
||||
|
||||
|
||||
def create_horizontal_bar_chart(
|
||||
labels: list[str],
|
||||
values: list[float],
|
||||
xlabel: str = "Compliance (%)",
|
||||
title: str | None = None,
|
||||
color_func: Callable[[float], str] | None = None,
|
||||
colors: list[str] | None = None,
|
||||
figsize: tuple[int, int] | None = None,
|
||||
dpi: int = DEFAULT_CHART_DPI,
|
||||
x_limit: tuple[float, float] = (0, 100),
|
||||
show_labels: bool = True,
|
||||
label_fontsize: int = 16,
|
||||
) -> io.BytesIO:
|
||||
"""Create a horizontal bar chart.
|
||||
|
||||
Args:
|
||||
labels: Y-axis labels (bar names)
|
||||
values: Bar widths (numeric values)
|
||||
xlabel: X-axis label
|
||||
title: Optional chart title
|
||||
color_func: Function to determine bar color based on value
|
||||
colors: Explicit list of colors (overrides color_func)
|
||||
figsize: Figure size (auto-calculated if None based on label count)
|
||||
dpi: Resolution for output image
|
||||
x_limit: X-axis limits (min, max)
|
||||
show_labels: Whether to show value labels on bars
|
||||
label_fontsize: Font size for y-axis labels
|
||||
|
||||
Returns:
|
||||
BytesIO buffer containing the PNG image
|
||||
"""
|
||||
if color_func is None:
|
||||
color_func = get_chart_color_for_percentage
|
||||
|
||||
# Auto-calculate figure size based on number of items
|
||||
if figsize is None:
|
||||
figsize = (10, max(6, int(len(labels) * 0.4)))
|
||||
|
||||
fig, ax = plt.subplots(figsize=figsize)
|
||||
|
||||
# Determine colors
|
||||
if colors is None:
|
||||
colors_list = [color_func(v) for v in values]
|
||||
else:
|
||||
colors_list = colors
|
||||
|
||||
y_pos = range(len(labels))
|
||||
bars = ax.barh(y_pos, values, color=colors_list)
|
||||
|
||||
ax.set_yticks(y_pos)
|
||||
ax.set_yticklabels(labels, fontsize=label_fontsize)
|
||||
ax.set_xlabel(xlabel, fontsize=14)
|
||||
ax.set_xlim(*x_limit)
|
||||
|
||||
if title:
|
||||
ax.set_title(title, fontsize=14, fontweight="bold")
|
||||
|
||||
# Add value labels
|
||||
if show_labels:
|
||||
for bar_item, value in zip(bars, values):
|
||||
width = bar_item.get_width()
|
||||
ax.text(
|
||||
width + 1,
|
||||
bar_item.get_y() + bar_item.get_height() / 2.0,
|
||||
f"{value:.1f}%",
|
||||
ha="left",
|
||||
va="center",
|
||||
fontweight="bold",
|
||||
fontsize=10,
|
||||
)
|
||||
|
||||
ax.grid(True, alpha=0.3, axis="x")
|
||||
plt.tight_layout()
|
||||
|
||||
buffer = io.BytesIO()
|
||||
try:
|
||||
fig.savefig(buffer, format="png", dpi=dpi, bbox_inches="tight")
|
||||
buffer.seek(0)
|
||||
finally:
|
||||
plt.close(fig)
|
||||
gc.collect() # Force garbage collection after heavy matplotlib operation
|
||||
|
||||
return buffer
|
||||
|
||||
|
||||
def create_radar_chart(
|
||||
labels: list[str],
|
||||
values: list[float],
|
||||
color: str = CHART_COLOR_BLUE,
|
||||
fill_alpha: float = 0.25,
|
||||
figsize: tuple[int, int] = (8, 8),
|
||||
dpi: int = DEFAULT_CHART_DPI,
|
||||
y_limit: tuple[float, float] = (0, 100),
|
||||
y_ticks: list[int] | None = None,
|
||||
label_fontsize: int = 14,
|
||||
title: str | None = None,
|
||||
) -> io.BytesIO:
|
||||
"""Create a radar/spider chart.
|
||||
|
||||
Args:
|
||||
labels: Category names around the chart
|
||||
values: Values for each category (should have same length as labels)
|
||||
color: Line and fill color
|
||||
fill_alpha: Transparency of the fill (0-1)
|
||||
figsize: Figure size (width, height) in inches
|
||||
dpi: Resolution for output image
|
||||
y_limit: Radial axis limits (min, max)
|
||||
y_ticks: Custom tick values for radial axis
|
||||
label_fontsize: Font size for category labels
|
||||
title: Optional chart title
|
||||
|
||||
Returns:
|
||||
BytesIO buffer containing the PNG image
|
||||
"""
|
||||
num_vars = len(labels)
|
||||
angles = [n / float(num_vars) * 2 * math.pi for n in range(num_vars)]
|
||||
|
||||
# Close the polygon
|
||||
values_closed = list(values) + [values[0]]
|
||||
angles_closed = angles + [angles[0]]
|
||||
|
||||
fig, ax = plt.subplots(figsize=figsize, subplot_kw={"projection": "polar"})
|
||||
|
||||
ax.plot(angles_closed, values_closed, "o-", linewidth=2, color=color)
|
||||
ax.fill(angles_closed, values_closed, alpha=fill_alpha, color=color)
|
||||
|
||||
ax.set_xticks(angles)
|
||||
ax.set_xticklabels(labels, fontsize=label_fontsize)
|
||||
ax.set_ylim(*y_limit)
|
||||
|
||||
if y_ticks is None:
|
||||
y_ticks = [20, 40, 60, 80, 100]
|
||||
ax.set_yticks(y_ticks)
|
||||
ax.set_yticklabels([f"{t}%" for t in y_ticks], fontsize=12)
|
||||
|
||||
ax.grid(True, alpha=0.3)
|
||||
|
||||
if title:
|
||||
ax.set_title(title, fontsize=14, fontweight="bold", y=1.08)
|
||||
|
||||
plt.tight_layout()
|
||||
|
||||
buffer = io.BytesIO()
|
||||
try:
|
||||
fig.savefig(buffer, format="png", dpi=dpi, bbox_inches="tight")
|
||||
buffer.seek(0)
|
||||
finally:
|
||||
plt.close(fig)
|
||||
gc.collect() # Force garbage collection after heavy matplotlib operation
|
||||
|
||||
return buffer
|
||||
|
||||
|
||||
def create_pie_chart(
|
||||
labels: list[str],
|
||||
values: list[float],
|
||||
colors: list[str] | None = None,
|
||||
figsize: tuple[int, int] = (6, 6),
|
||||
dpi: int = DEFAULT_CHART_DPI,
|
||||
autopct: str = "%1.1f%%",
|
||||
startangle: int = 90,
|
||||
title: str | None = None,
|
||||
) -> io.BytesIO:
|
||||
"""Create a pie chart.
|
||||
|
||||
Args:
|
||||
labels: Slice labels
|
||||
values: Slice values
|
||||
colors: Optional list of colors for slices
|
||||
figsize: Figure size (width, height) in inches
|
||||
dpi: Resolution for output image
|
||||
autopct: Format string for percentage labels
|
||||
startangle: Starting angle for first slice
|
||||
title: Optional chart title
|
||||
|
||||
Returns:
|
||||
BytesIO buffer containing the PNG image
|
||||
"""
|
||||
fig, ax = plt.subplots(figsize=figsize)
|
||||
|
||||
_, _, autotexts = ax.pie(
|
||||
values,
|
||||
labels=labels,
|
||||
colors=colors,
|
||||
autopct=autopct,
|
||||
startangle=startangle,
|
||||
)
|
||||
|
||||
# Style the text
|
||||
for autotext in autotexts:
|
||||
autotext.set_fontweight("bold")
|
||||
|
||||
if title:
|
||||
ax.set_title(title, fontsize=14, fontweight="bold")
|
||||
|
||||
plt.tight_layout()
|
||||
|
||||
buffer = io.BytesIO()
|
||||
try:
|
||||
fig.savefig(buffer, format="png", dpi=dpi, bbox_inches="tight")
|
||||
buffer.seek(0)
|
||||
finally:
|
||||
plt.close(fig)
|
||||
gc.collect() # Force garbage collection after heavy matplotlib operation
|
||||
|
||||
return buffer
|
||||
|
||||
|
||||
def create_stacked_bar_chart(
|
||||
labels: list[str],
|
||||
data_series: dict[str, list[float]],
|
||||
colors: dict[str, str] | None = None,
|
||||
xlabel: str = "",
|
||||
ylabel: str = "Count",
|
||||
title: str | None = None,
|
||||
figsize: tuple[int, int] = (10, 6),
|
||||
dpi: int = DEFAULT_CHART_DPI,
|
||||
rotation: int = 45,
|
||||
show_legend: bool = True,
|
||||
) -> io.BytesIO:
|
||||
"""Create a stacked bar chart.
|
||||
|
||||
Args:
|
||||
labels: X-axis labels
|
||||
data_series: Dictionary mapping series name to list of values
|
||||
colors: Dictionary mapping series name to color
|
||||
xlabel: X-axis label
|
||||
ylabel: Y-axis label
|
||||
title: Optional chart title
|
||||
figsize: Figure size (width, height) in inches
|
||||
dpi: Resolution for output image
|
||||
rotation: X-axis label rotation angle
|
||||
show_legend: Whether to show the legend
|
||||
|
||||
Returns:
|
||||
BytesIO buffer containing the PNG image
|
||||
"""
|
||||
fig, ax = plt.subplots(figsize=figsize)
|
||||
|
||||
# Default colors if not provided
|
||||
default_colors = {
|
||||
"Pass": CHART_COLOR_GREEN_1,
|
||||
"Fail": CHART_COLOR_RED,
|
||||
"Manual": CHART_COLOR_YELLOW,
|
||||
}
|
||||
if colors is None:
|
||||
colors = default_colors
|
||||
|
||||
bottom = [0] * len(labels)
|
||||
for series_name, values in data_series.items():
|
||||
color = colors.get(series_name, CHART_COLOR_BLUE)
|
||||
ax.bar(labels, values, bottom=bottom, label=series_name, color=color)
|
||||
bottom = [b + v for b, v in zip(bottom, values)]
|
||||
|
||||
ax.set_xlabel(xlabel, fontsize=12)
|
||||
ax.set_ylabel(ylabel, fontsize=12)
|
||||
|
||||
if title:
|
||||
ax.set_title(title, fontsize=14, fontweight="bold")
|
||||
|
||||
plt.xticks(rotation=rotation, ha="right")
|
||||
|
||||
if show_legend:
|
||||
ax.legend()
|
||||
|
||||
ax.grid(True, alpha=0.3, axis="y")
|
||||
plt.tight_layout()
|
||||
|
||||
buffer = io.BytesIO()
|
||||
try:
|
||||
fig.savefig(buffer, format="png", dpi=dpi, bbox_inches="tight")
|
||||
buffer.seek(0)
|
||||
finally:
|
||||
plt.close(fig)
|
||||
gc.collect() # Force garbage collection after heavy matplotlib operation
|
||||
|
||||
return buffer
|
||||
@@ -0,0 +1,599 @@
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Callable
|
||||
|
||||
from reportlab.lib import colors
|
||||
from reportlab.lib.styles import ParagraphStyle
|
||||
from reportlab.lib.units import inch
|
||||
from reportlab.platypus import LongTable, Paragraph, Spacer, Table, TableStyle
|
||||
|
||||
from .config import (
|
||||
ALTERNATE_ROWS_MAX_SIZE,
|
||||
COLOR_BLUE,
|
||||
COLOR_BORDER_GRAY,
|
||||
COLOR_DARK_GRAY,
|
||||
COLOR_GRID_GRAY,
|
||||
COLOR_HIGH_RISK,
|
||||
COLOR_LIGHT_GRAY,
|
||||
COLOR_LOW_RISK,
|
||||
COLOR_MEDIUM_RISK,
|
||||
COLOR_SAFE,
|
||||
COLOR_WHITE,
|
||||
LONG_TABLE_THRESHOLD,
|
||||
PADDING_LARGE,
|
||||
PADDING_MEDIUM,
|
||||
PADDING_SMALL,
|
||||
PADDING_XLARGE,
|
||||
)
|
||||
|
||||
|
||||
def get_color_for_risk_level(risk_level: int) -> colors.Color:
|
||||
"""
|
||||
Get color based on risk level.
|
||||
|
||||
Args:
|
||||
risk_level (int): Numeric risk level (0-5).
|
||||
|
||||
Returns:
|
||||
colors.Color: Appropriate color for the risk level.
|
||||
"""
|
||||
if risk_level >= 4:
|
||||
return COLOR_HIGH_RISK
|
||||
if risk_level >= 3:
|
||||
return COLOR_MEDIUM_RISK
|
||||
if risk_level >= 2:
|
||||
return COLOR_LOW_RISK
|
||||
return COLOR_SAFE
|
||||
|
||||
|
||||
def get_color_for_weight(weight: int) -> colors.Color:
|
||||
"""
|
||||
Get color based on weight value.
|
||||
|
||||
Args:
|
||||
weight (int): Numeric weight value.
|
||||
|
||||
Returns:
|
||||
colors.Color: Appropriate color for the weight.
|
||||
"""
|
||||
if weight > 100:
|
||||
return COLOR_HIGH_RISK
|
||||
if weight > 50:
|
||||
return COLOR_LOW_RISK
|
||||
return COLOR_SAFE
|
||||
|
||||
|
||||
def get_color_for_compliance(percentage: float) -> colors.Color:
|
||||
"""
|
||||
Get color based on compliance percentage.
|
||||
|
||||
Args:
|
||||
percentage (float): Compliance percentage (0-100).
|
||||
|
||||
Returns:
|
||||
colors.Color: Appropriate color for the compliance level.
|
||||
"""
|
||||
if percentage >= 80:
|
||||
return COLOR_SAFE
|
||||
if percentage >= 60:
|
||||
return COLOR_LOW_RISK
|
||||
return COLOR_HIGH_RISK
|
||||
|
||||
|
||||
def get_status_color(status: str) -> colors.Color:
|
||||
"""
|
||||
Get color for a status value.
|
||||
|
||||
Args:
|
||||
status (str): Status string (PASS, FAIL, MANUAL, etc.).
|
||||
|
||||
Returns:
|
||||
colors.Color: Appropriate color for the status.
|
||||
"""
|
||||
status_upper = status.upper()
|
||||
if status_upper == "PASS":
|
||||
return COLOR_SAFE
|
||||
if status_upper == "FAIL":
|
||||
return COLOR_HIGH_RISK
|
||||
return COLOR_DARK_GRAY
|
||||
|
||||
|
||||
def create_badge(
|
||||
text: str,
|
||||
bg_color: colors.Color,
|
||||
text_color: colors.Color = COLOR_WHITE,
|
||||
width: float = 1.4 * inch,
|
||||
font: str = "FiraCode",
|
||||
font_size: int = 11,
|
||||
) -> Table:
|
||||
"""
|
||||
Create a generic colored badge component.
|
||||
|
||||
Args:
|
||||
text (str): Text to display in the badge.
|
||||
bg_color (colors.Color): Background color.
|
||||
text_color (colors.Color): Text color (default white).
|
||||
width (float): Badge width in inches.
|
||||
font (str): Font name to use.
|
||||
font_size (int): Font size.
|
||||
|
||||
Returns:
|
||||
Table: A Table object styled as a badge.
|
||||
"""
|
||||
data = [[text]]
|
||||
table = Table(data, colWidths=[width])
|
||||
|
||||
table.setStyle(
|
||||
TableStyle(
|
||||
[
|
||||
("BACKGROUND", (0, 0), (0, 0), bg_color),
|
||||
("TEXTCOLOR", (0, 0), (0, 0), text_color),
|
||||
("FONTNAME", (0, 0), (0, 0), font),
|
||||
("ALIGN", (0, 0), (-1, -1), "CENTER"),
|
||||
("VALIGN", (0, 0), (-1, -1), "MIDDLE"),
|
||||
("FONTSIZE", (0, 0), (-1, -1), font_size),
|
||||
("GRID", (0, 0), (-1, -1), 0.5, colors.black),
|
||||
("LEFTPADDING", (0, 0), (-1, -1), PADDING_LARGE),
|
||||
("RIGHTPADDING", (0, 0), (-1, -1), PADDING_LARGE),
|
||||
("TOPPADDING", (0, 0), (-1, -1), PADDING_LARGE),
|
||||
("BOTTOMPADDING", (0, 0), (-1, -1), PADDING_LARGE),
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
return table
|
||||
|
||||
|
||||
def create_status_badge(status: str) -> Table:
|
||||
"""
|
||||
Create a PASS/FAIL/MANUAL status badge.
|
||||
|
||||
Args:
|
||||
status (str): Status value (e.g., "PASS", "FAIL", "MANUAL").
|
||||
|
||||
Returns:
|
||||
Table: A styled Table badge for the status.
|
||||
"""
|
||||
status_upper = status.upper()
|
||||
status_color = get_status_color(status_upper)
|
||||
|
||||
data = [["State:", status_upper]]
|
||||
table = Table(data, colWidths=[0.6 * inch, 0.8 * inch])
|
||||
|
||||
table.setStyle(
|
||||
TableStyle(
|
||||
[
|
||||
("BACKGROUND", (0, 0), (0, 0), COLOR_LIGHT_GRAY),
|
||||
("FONTNAME", (0, 0), (0, 0), "PlusJakartaSans"),
|
||||
("BACKGROUND", (1, 0), (1, 0), status_color),
|
||||
("TEXTCOLOR", (1, 0), (1, 0), COLOR_WHITE),
|
||||
("FONTNAME", (1, 0), (1, 0), "FiraCode"),
|
||||
("ALIGN", (0, 0), (-1, -1), "CENTER"),
|
||||
("VALIGN", (0, 0), (-1, -1), "MIDDLE"),
|
||||
("FONTSIZE", (0, 0), (-1, -1), 12),
|
||||
("GRID", (0, 0), (-1, -1), 0.5, colors.black),
|
||||
("LEFTPADDING", (0, 0), (-1, -1), PADDING_LARGE),
|
||||
("RIGHTPADDING", (0, 0), (-1, -1), PADDING_LARGE),
|
||||
("TOPPADDING", (0, 0), (-1, -1), PADDING_XLARGE),
|
||||
("BOTTOMPADDING", (0, 0), (-1, -1), PADDING_XLARGE),
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
return table
|
||||
|
||||
|
||||
def create_multi_badge_row(
|
||||
badges: list[tuple[str, colors.Color]],
|
||||
badge_width: float = 0.4 * inch,
|
||||
font: str = "FiraCode",
|
||||
) -> Table:
|
||||
"""
|
||||
Create a row of multiple small badges.
|
||||
|
||||
Args:
|
||||
badges (list[tuple[str, colors.Color]]): List of (text, color) tuples for each badge.
|
||||
badge_width (float): Width of each badge.
|
||||
font (str): Font name to use.
|
||||
|
||||
Returns:
|
||||
Table: A Table with multiple colored badges in a row.
|
||||
"""
|
||||
if not badges:
|
||||
data = [["N/A"]]
|
||||
table = Table(data, colWidths=[1 * inch])
|
||||
table.setStyle(
|
||||
TableStyle(
|
||||
[
|
||||
("BACKGROUND", (0, 0), (0, 0), COLOR_LIGHT_GRAY),
|
||||
("ALIGN", (0, 0), (-1, -1), "CENTER"),
|
||||
("FONTSIZE", (0, 0), (-1, -1), 10),
|
||||
]
|
||||
)
|
||||
)
|
||||
return table
|
||||
|
||||
data = [[text for text, _ in badges]]
|
||||
col_widths = [badge_width] * len(badges)
|
||||
table = Table(data, colWidths=col_widths)
|
||||
|
||||
styles = [
|
||||
("ALIGN", (0, 0), (-1, -1), "CENTER"),
|
||||
("VALIGN", (0, 0), (-1, -1), "MIDDLE"),
|
||||
("FONTNAME", (0, 0), (-1, -1), font),
|
||||
("FONTSIZE", (0, 0), (-1, -1), 10),
|
||||
("TEXTCOLOR", (0, 0), (-1, -1), COLOR_WHITE),
|
||||
("GRID", (0, 0), (-1, -1), 0.5, colors.black),
|
||||
("LEFTPADDING", (0, 0), (-1, -1), PADDING_SMALL),
|
||||
("RIGHTPADDING", (0, 0), (-1, -1), PADDING_SMALL),
|
||||
("TOPPADDING", (0, 0), (-1, -1), PADDING_MEDIUM),
|
||||
("BOTTOMPADDING", (0, 0), (-1, -1), PADDING_MEDIUM),
|
||||
]
|
||||
|
||||
for idx, (_, badge_color) in enumerate(badges):
|
||||
styles.append(("BACKGROUND", (idx, 0), (idx, 0), badge_color))
|
||||
|
||||
table.setStyle(TableStyle(styles))
|
||||
return table
|
||||
|
||||
|
||||
def create_risk_component(
|
||||
risk_level: int,
|
||||
weight: int,
|
||||
score: int = 0,
|
||||
) -> Table:
|
||||
"""
|
||||
Create a visual risk component showing risk level, weight, and score.
|
||||
|
||||
Args:
|
||||
risk_level (int): The risk level (0-5).
|
||||
weight (int): The weight value.
|
||||
score (int): The calculated score (default 0).
|
||||
|
||||
Returns:
|
||||
Table: A styled Table showing risk metrics.
|
||||
"""
|
||||
risk_color = get_color_for_risk_level(risk_level)
|
||||
weight_color = get_color_for_weight(weight)
|
||||
|
||||
data = [
|
||||
[
|
||||
"Risk Level:",
|
||||
str(risk_level),
|
||||
"Weight:",
|
||||
str(weight),
|
||||
"Score:",
|
||||
str(score),
|
||||
]
|
||||
]
|
||||
|
||||
table = Table(
|
||||
data,
|
||||
colWidths=[
|
||||
0.8 * inch,
|
||||
0.4 * inch,
|
||||
0.6 * inch,
|
||||
0.4 * inch,
|
||||
0.5 * inch,
|
||||
0.4 * inch,
|
||||
],
|
||||
)
|
||||
|
||||
table.setStyle(
|
||||
TableStyle(
|
||||
[
|
||||
("BACKGROUND", (0, 0), (0, 0), COLOR_LIGHT_GRAY),
|
||||
("BACKGROUND", (1, 0), (1, 0), risk_color),
|
||||
("TEXTCOLOR", (1, 0), (1, 0), COLOR_WHITE),
|
||||
("FONTNAME", (1, 0), (1, 0), "FiraCode"),
|
||||
("BACKGROUND", (2, 0), (2, 0), COLOR_LIGHT_GRAY),
|
||||
("BACKGROUND", (3, 0), (3, 0), weight_color),
|
||||
("TEXTCOLOR", (3, 0), (3, 0), COLOR_WHITE),
|
||||
("FONTNAME", (3, 0), (3, 0), "FiraCode"),
|
||||
("BACKGROUND", (4, 0), (4, 0), COLOR_LIGHT_GRAY),
|
||||
("BACKGROUND", (5, 0), (5, 0), COLOR_DARK_GRAY),
|
||||
("TEXTCOLOR", (5, 0), (5, 0), COLOR_WHITE),
|
||||
("FONTNAME", (5, 0), (5, 0), "FiraCode"),
|
||||
("ALIGN", (0, 0), (-1, -1), "CENTER"),
|
||||
("VALIGN", (0, 0), (-1, -1), "MIDDLE"),
|
||||
("FONTSIZE", (0, 0), (-1, -1), 10),
|
||||
("GRID", (0, 0), (-1, -1), 0.5, colors.black),
|
||||
("LEFTPADDING", (0, 0), (-1, -1), PADDING_MEDIUM),
|
||||
("RIGHTPADDING", (0, 0), (-1, -1), PADDING_MEDIUM),
|
||||
("TOPPADDING", (0, 0), (-1, -1), PADDING_LARGE),
|
||||
("BOTTOMPADDING", (0, 0), (-1, -1), PADDING_LARGE),
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
return table
|
||||
|
||||
|
||||
def create_info_table(
|
||||
rows: list[tuple[str, Any]],
|
||||
label_width: float = 2 * inch,
|
||||
value_width: float = 4 * inch,
|
||||
label_color: colors.Color = COLOR_BLUE,
|
||||
value_bg_color: colors.Color | None = None,
|
||||
normal_style: ParagraphStyle | None = None,
|
||||
) -> Table:
|
||||
"""
|
||||
Create a key-value information table.
|
||||
|
||||
Args:
|
||||
rows (list[tuple[str, Any]]): List of (label, value) tuples.
|
||||
label_width (float): Width of the label column.
|
||||
value_width (float): Width of the value column.
|
||||
label_color (colors.Color): Background color for labels.
|
||||
value_bg_color (colors.Color | None): Background color for values (optional).
|
||||
normal_style (ParagraphStyle | None): ParagraphStyle for wrapping long values.
|
||||
|
||||
Returns:
|
||||
Table: A styled Table with key-value pairs.
|
||||
"""
|
||||
from .config import COLOR_BG_BLUE
|
||||
|
||||
if value_bg_color is None:
|
||||
value_bg_color = COLOR_BG_BLUE
|
||||
|
||||
# Handle empty rows case - Table requires at least one row
|
||||
if not rows:
|
||||
table = Table([["", ""]], colWidths=[label_width, value_width])
|
||||
table.setStyle(TableStyle([("FONTSIZE", (0, 0), (-1, -1), 0)]))
|
||||
return table
|
||||
|
||||
# Process rows - wrap long values in Paragraph if style provided
|
||||
table_data = []
|
||||
for label, value in rows:
|
||||
if normal_style and isinstance(value, str) and len(value) > 50:
|
||||
value = Paragraph(value, normal_style)
|
||||
table_data.append([label, value])
|
||||
|
||||
table = Table(table_data, colWidths=[label_width, value_width])
|
||||
|
||||
table.setStyle(
|
||||
TableStyle(
|
||||
[
|
||||
("BACKGROUND", (0, 0), (0, -1), label_color),
|
||||
("TEXTCOLOR", (0, 0), (0, -1), COLOR_WHITE),
|
||||
("FONTNAME", (0, 0), (0, -1), "FiraCode"),
|
||||
("BACKGROUND", (1, 0), (1, -1), value_bg_color),
|
||||
("TEXTCOLOR", (1, 0), (1, -1), COLOR_DARK_GRAY),
|
||||
("FONTNAME", (1, 0), (1, -1), "PlusJakartaSans"),
|
||||
("ALIGN", (0, 0), (-1, -1), "LEFT"),
|
||||
("VALIGN", (0, 0), (-1, -1), "TOP"),
|
||||
("FONTSIZE", (0, 0), (-1, -1), 11),
|
||||
("GRID", (0, 0), (-1, -1), 1, COLOR_BORDER_GRAY),
|
||||
("LEFTPADDING", (0, 0), (-1, -1), PADDING_XLARGE),
|
||||
("RIGHTPADDING", (0, 0), (-1, -1), PADDING_XLARGE),
|
||||
("TOPPADDING", (0, 0), (-1, -1), PADDING_LARGE),
|
||||
("BOTTOMPADDING", (0, 0), (-1, -1), PADDING_LARGE),
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
return table
|
||||
|
||||
|
||||
@dataclass
|
||||
class ColumnConfig:
|
||||
"""
|
||||
Configuration for a table column.
|
||||
|
||||
Attributes:
|
||||
header (str): Column header text.
|
||||
width (float): Column width in inches.
|
||||
field (str | Callable[[Any], str]): Field name or callable to extract value from data.
|
||||
align (str): Text alignment (LEFT, CENTER, RIGHT).
|
||||
"""
|
||||
|
||||
header: str
|
||||
width: float
|
||||
field: str | Callable[[Any], str]
|
||||
align: str = "CENTER"
|
||||
|
||||
|
||||
def create_data_table(
|
||||
data: list[dict[str, Any]],
|
||||
columns: list[ColumnConfig],
|
||||
header_color: colors.Color = COLOR_BLUE,
|
||||
alternate_rows: bool = True,
|
||||
normal_style: ParagraphStyle | None = None,
|
||||
) -> Table | LongTable:
|
||||
"""
|
||||
Create a data table with configurable columns.
|
||||
|
||||
Uses LongTable for large datasets (>50 rows) for better memory efficiency
|
||||
and page splitting. LongTable repeats headers on each page and has
|
||||
optimized memory handling for large tables.
|
||||
|
||||
Args:
|
||||
data (list[dict[str, Any]]): List of data dictionaries.
|
||||
columns (list[ColumnConfig]): Column configuration list.
|
||||
header_color (colors.Color): Background color for header row.
|
||||
alternate_rows (bool): Whether to alternate row backgrounds.
|
||||
normal_style (ParagraphStyle | None): ParagraphStyle for cell values.
|
||||
|
||||
Returns:
|
||||
Table or LongTable: A styled table with data.
|
||||
"""
|
||||
# Build header row
|
||||
header_row = [col.header for col in columns]
|
||||
table_data = [header_row]
|
||||
|
||||
# Build data rows
|
||||
for item in data:
|
||||
row = []
|
||||
for col in columns:
|
||||
if callable(col.field):
|
||||
value = col.field(item)
|
||||
else:
|
||||
value = item.get(col.field, "")
|
||||
|
||||
if normal_style and isinstance(value, str):
|
||||
value = Paragraph(value, normal_style)
|
||||
row.append(value)
|
||||
table_data.append(row)
|
||||
|
||||
col_widths = [col.width for col in columns]
|
||||
|
||||
# Use LongTable for large datasets - it handles page breaks better
|
||||
# and has optimized memory handling for tables with many rows
|
||||
use_long_table = len(data) > LONG_TABLE_THRESHOLD
|
||||
if use_long_table:
|
||||
table = LongTable(table_data, colWidths=col_widths, repeatRows=1)
|
||||
else:
|
||||
table = Table(table_data, colWidths=col_widths)
|
||||
|
||||
styles = [
|
||||
("BACKGROUND", (0, 0), (-1, 0), header_color),
|
||||
("TEXTCOLOR", (0, 0), (-1, 0), COLOR_WHITE),
|
||||
("FONTNAME", (0, 0), (-1, 0), "FiraCode"),
|
||||
("FONTSIZE", (0, 0), (-1, 0), 10),
|
||||
("FONTSIZE", (0, 1), (-1, -1), 9),
|
||||
("VALIGN", (0, 0), (-1, -1), "MIDDLE"),
|
||||
("GRID", (0, 0), (-1, -1), 1, COLOR_GRID_GRAY),
|
||||
("LEFTPADDING", (0, 0), (-1, -1), PADDING_MEDIUM),
|
||||
("RIGHTPADDING", (0, 0), (-1, -1), PADDING_MEDIUM),
|
||||
("TOPPADDING", (0, 0), (-1, -1), PADDING_MEDIUM),
|
||||
("BOTTOMPADDING", (0, 0), (-1, -1), PADDING_MEDIUM),
|
||||
]
|
||||
|
||||
# Apply column alignments
|
||||
for idx, col in enumerate(columns):
|
||||
styles.append(("ALIGN", (idx, 0), (idx, -1), col.align))
|
||||
|
||||
# Alternate row backgrounds - skip for very large tables as it adds memory overhead
|
||||
if (
|
||||
alternate_rows
|
||||
and len(table_data) > 1
|
||||
and len(table_data) <= ALTERNATE_ROWS_MAX_SIZE
|
||||
):
|
||||
for i in range(1, len(table_data)):
|
||||
if i % 2 == 0:
|
||||
styles.append(
|
||||
("BACKGROUND", (0, i), (-1, i), colors.Color(0.98, 0.98, 0.98))
|
||||
)
|
||||
|
||||
table.setStyle(TableStyle(styles))
|
||||
return table
|
||||
|
||||
|
||||
def create_findings_table(
|
||||
findings: list[Any],
|
||||
columns: list[ColumnConfig] | None = None,
|
||||
header_color: colors.Color = COLOR_BLUE,
|
||||
normal_style: ParagraphStyle | None = None,
|
||||
) -> Table:
|
||||
"""
|
||||
Create a findings table with default or custom columns.
|
||||
|
||||
Args:
|
||||
findings (list[Any]): List of finding objects.
|
||||
columns (list[ColumnConfig] | None): Optional column configuration (defaults to standard columns).
|
||||
header_color (colors.Color): Background color for header row.
|
||||
normal_style (ParagraphStyle | None): ParagraphStyle for cell values.
|
||||
|
||||
Returns:
|
||||
Table: A styled Table with findings data.
|
||||
"""
|
||||
if columns is None:
|
||||
columns = [
|
||||
ColumnConfig("Finding", 2.5 * inch, "title"),
|
||||
ColumnConfig("Resource", 3 * inch, "resource_name"),
|
||||
ColumnConfig("Severity", 0.9 * inch, "severity"),
|
||||
ColumnConfig("Status", 0.9 * inch, "status"),
|
||||
ColumnConfig("Region", 0.9 * inch, "region"),
|
||||
]
|
||||
|
||||
# Convert findings to dicts
|
||||
data = []
|
||||
for finding in findings:
|
||||
item = {}
|
||||
for col in columns:
|
||||
if callable(col.field):
|
||||
item[col.header.lower()] = col.field(finding)
|
||||
elif hasattr(finding, col.field):
|
||||
item[col.field] = getattr(finding, col.field, "")
|
||||
elif isinstance(finding, dict):
|
||||
item[col.field] = finding.get(col.field, "")
|
||||
data.append(item)
|
||||
|
||||
return create_data_table(
|
||||
data=data,
|
||||
columns=columns,
|
||||
header_color=header_color,
|
||||
alternate_rows=True,
|
||||
normal_style=normal_style,
|
||||
)
|
||||
|
||||
|
||||
def create_section_header(
|
||||
text: str,
|
||||
style: ParagraphStyle,
|
||||
add_spacer: bool = True,
|
||||
spacer_height: float = 0.2,
|
||||
) -> list:
|
||||
"""
|
||||
Create a section header with optional spacer.
|
||||
|
||||
Args:
|
||||
text (str): Header text.
|
||||
style (ParagraphStyle): ParagraphStyle to apply.
|
||||
add_spacer (bool): Whether to add a spacer after the header.
|
||||
spacer_height (float): Height of the spacer in inches.
|
||||
|
||||
Returns:
|
||||
list: List of elements (Paragraph and optional Spacer).
|
||||
"""
|
||||
elements = [Paragraph(text, style)]
|
||||
if add_spacer:
|
||||
elements.append(Spacer(1, spacer_height * inch))
|
||||
return elements
|
||||
|
||||
|
||||
def create_summary_table(
|
||||
label: str,
|
||||
value: str,
|
||||
value_color: colors.Color,
|
||||
label_width: float = 2.5 * inch,
|
||||
value_width: float = 2 * inch,
|
||||
) -> Table:
|
||||
"""
|
||||
Create a summary metric table (e.g., for ThreatScore display).
|
||||
|
||||
Args:
|
||||
label (str): Label text (e.g., "ThreatScore:").
|
||||
value (str): Value text (e.g., "85.5%").
|
||||
value_color (colors.Color): Background color for the value cell.
|
||||
label_width (float): Width of the label column.
|
||||
value_width (float): Width of the value column.
|
||||
|
||||
Returns:
|
||||
Table: A styled summary Table.
|
||||
"""
|
||||
data = [[label, value]]
|
||||
table = Table(data, colWidths=[label_width, value_width])
|
||||
|
||||
table.setStyle(
|
||||
TableStyle(
|
||||
[
|
||||
("BACKGROUND", (0, 0), (0, 0), colors.Color(0.1, 0.3, 0.5)),
|
||||
("TEXTCOLOR", (0, 0), (0, 0), COLOR_WHITE),
|
||||
("FONTNAME", (0, 0), (0, 0), "FiraCode"),
|
||||
("FONTSIZE", (0, 0), (0, 0), 12),
|
||||
("BACKGROUND", (1, 0), (1, 0), value_color),
|
||||
("TEXTCOLOR", (1, 0), (1, 0), COLOR_WHITE),
|
||||
("FONTNAME", (1, 0), (1, 0), "FiraCode"),
|
||||
("FONTSIZE", (1, 0), (1, 0), 16),
|
||||
("ALIGN", (0, 0), (-1, -1), "CENTER"),
|
||||
("VALIGN", (0, 0), (-1, -1), "MIDDLE"),
|
||||
("GRID", (0, 0), (-1, -1), 1.5, colors.Color(0.5, 0.6, 0.7)),
|
||||
("LEFTPADDING", (0, 0), (-1, -1), 12),
|
||||
("RIGHTPADDING", (0, 0), (-1, -1), 12),
|
||||
("TOPPADDING", (0, 0), (-1, -1), 10),
|
||||
("BOTTOMPADDING", (0, 0), (-1, -1), 10),
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
return table
|
||||
@@ -0,0 +1,286 @@
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from reportlab.lib import colors
|
||||
from reportlab.lib.units import inch
|
||||
|
||||
# =============================================================================
|
||||
# Performance & Memory Optimization Settings
|
||||
# =============================================================================
|
||||
# These settings control memory usage and performance for large reports.
|
||||
# Adjust these values if workers are running out of memory.
|
||||
|
||||
# Chart settings - lower DPI = less memory, 150 is good quality for PDF
|
||||
CHART_DPI_DEFAULT = 150
|
||||
|
||||
# LongTable threshold - use LongTable for tables with more rows than this
|
||||
# LongTable handles page breaks better and has optimized memory for large tables
|
||||
LONG_TABLE_THRESHOLD = 50
|
||||
|
||||
# Skip alternating row colors for tables larger than this (reduces memory)
|
||||
ALTERNATE_ROWS_MAX_SIZE = 200
|
||||
|
||||
# Database query batch size for findings (matches Django settings)
|
||||
# Larger = fewer queries but more memory per batch
|
||||
FINDINGS_BATCH_SIZE = 2000
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Base colors
|
||||
# =============================================================================
|
||||
COLOR_PROWLER_DARK_GREEN = colors.Color(0.1, 0.5, 0.2)
|
||||
COLOR_BLUE = colors.Color(0.2, 0.4, 0.6)
|
||||
COLOR_LIGHT_BLUE = colors.Color(0.3, 0.5, 0.7)
|
||||
COLOR_LIGHTER_BLUE = colors.Color(0.4, 0.6, 0.8)
|
||||
COLOR_BG_BLUE = colors.Color(0.95, 0.97, 1.0)
|
||||
COLOR_BG_LIGHT_BLUE = colors.Color(0.98, 0.99, 1.0)
|
||||
COLOR_GRAY = colors.Color(0.2, 0.2, 0.2)
|
||||
COLOR_LIGHT_GRAY = colors.Color(0.9, 0.9, 0.9)
|
||||
COLOR_BORDER_GRAY = colors.Color(0.7, 0.8, 0.9)
|
||||
COLOR_GRID_GRAY = colors.Color(0.7, 0.7, 0.7)
|
||||
COLOR_DARK_GRAY = colors.Color(0.4, 0.4, 0.4)
|
||||
COLOR_HEADER_DARK = colors.Color(0.1, 0.3, 0.5)
|
||||
COLOR_HEADER_MEDIUM = colors.Color(0.15, 0.35, 0.55)
|
||||
COLOR_WHITE = colors.white
|
||||
|
||||
# Risk and status colors
|
||||
COLOR_HIGH_RISK = colors.Color(0.8, 0.2, 0.2)
|
||||
COLOR_MEDIUM_RISK = colors.Color(0.9, 0.6, 0.2)
|
||||
COLOR_LOW_RISK = colors.Color(0.9, 0.9, 0.2)
|
||||
COLOR_SAFE = colors.Color(0.2, 0.8, 0.2)
|
||||
|
||||
# ENS specific colors
|
||||
COLOR_ENS_ALTO = colors.Color(0.8, 0.2, 0.2)
|
||||
COLOR_ENS_MEDIO = colors.Color(0.98, 0.75, 0.13)
|
||||
COLOR_ENS_BAJO = colors.Color(0.06, 0.72, 0.51)
|
||||
COLOR_ENS_OPCIONAL = colors.Color(0.42, 0.45, 0.50)
|
||||
COLOR_ENS_TIPO = colors.Color(0.2, 0.4, 0.6)
|
||||
COLOR_ENS_AUTO = colors.Color(0.30, 0.69, 0.31)
|
||||
COLOR_ENS_MANUAL = colors.Color(0.96, 0.60, 0.0)
|
||||
|
||||
# NIS2 specific colors
|
||||
COLOR_NIS2_PRIMARY = colors.Color(0.12, 0.23, 0.54)
|
||||
COLOR_NIS2_SECONDARY = colors.Color(0.23, 0.51, 0.96)
|
||||
COLOR_NIS2_BG_BLUE = colors.Color(0.96, 0.97, 0.99)
|
||||
|
||||
# Chart colors (hex strings for matplotlib)
|
||||
CHART_COLOR_GREEN_1 = "#4CAF50"
|
||||
CHART_COLOR_GREEN_2 = "#8BC34A"
|
||||
CHART_COLOR_YELLOW = "#FFEB3B"
|
||||
CHART_COLOR_ORANGE = "#FF9800"
|
||||
CHART_COLOR_RED = "#F44336"
|
||||
CHART_COLOR_BLUE = "#2196F3"
|
||||
|
||||
# ENS dimension mappings: dimension name -> (abbreviation, color)
|
||||
DIMENSION_MAPPING = {
|
||||
"trazabilidad": ("T", colors.Color(0.26, 0.52, 0.96)),
|
||||
"autenticidad": ("A", colors.Color(0.30, 0.69, 0.31)),
|
||||
"integridad": ("I", colors.Color(0.61, 0.15, 0.69)),
|
||||
"confidencialidad": ("C", colors.Color(0.96, 0.26, 0.21)),
|
||||
"disponibilidad": ("D", colors.Color(1.0, 0.60, 0.0)),
|
||||
}
|
||||
|
||||
# ENS tipo icons
|
||||
TIPO_ICONS = {
|
||||
"requisito": "\u26a0\ufe0f",
|
||||
"refuerzo": "\U0001f6e1\ufe0f",
|
||||
"recomendacion": "\U0001f4a1",
|
||||
"medida": "\U0001f4cb",
|
||||
}
|
||||
|
||||
# Dimension names for charts (Spanish)
|
||||
DIMENSION_NAMES = [
|
||||
"Trazabilidad",
|
||||
"Autenticidad",
|
||||
"Integridad",
|
||||
"Confidencialidad",
|
||||
"Disponibilidad",
|
||||
]
|
||||
|
||||
DIMENSION_KEYS = [
|
||||
"trazabilidad",
|
||||
"autenticidad",
|
||||
"integridad",
|
||||
"confidencialidad",
|
||||
"disponibilidad",
|
||||
]
|
||||
|
||||
# ENS nivel and tipo order
|
||||
ENS_NIVEL_ORDER = ["alto", "medio", "bajo", "opcional"]
|
||||
ENS_TIPO_ORDER = ["requisito", "refuerzo", "recomendacion", "medida"]
|
||||
|
||||
# ThreatScore sections
|
||||
THREATSCORE_SECTIONS = [
|
||||
"1. IAM",
|
||||
"2. Attack Surface",
|
||||
"3. Logging and Monitoring",
|
||||
"4. Encryption",
|
||||
]
|
||||
|
||||
# NIS2 sections
|
||||
NIS2_SECTIONS = [
|
||||
"1",
|
||||
"2",
|
||||
"3",
|
||||
"4",
|
||||
"5",
|
||||
"6",
|
||||
"7",
|
||||
"9",
|
||||
"11",
|
||||
"12",
|
||||
]
|
||||
|
||||
NIS2_SECTION_TITLES = {
|
||||
"1": "1. Policy on Security",
|
||||
"2": "2. Risk Management",
|
||||
"3": "3. Incident Handling",
|
||||
"4": "4. Business Continuity",
|
||||
"5": "5. Supply Chain",
|
||||
"6": "6. Acquisition & Dev",
|
||||
"7": "7. Effectiveness",
|
||||
"9": "9. Cryptography",
|
||||
"11": "11. Access Control",
|
||||
"12": "12. Asset Management",
|
||||
}
|
||||
|
||||
# Table column widths
|
||||
COL_WIDTH_SMALL = 0.4 * inch
|
||||
COL_WIDTH_MEDIUM = 0.9 * inch
|
||||
COL_WIDTH_LARGE = 1.5 * inch
|
||||
COL_WIDTH_XLARGE = 2 * inch
|
||||
COL_WIDTH_XXLARGE = 3 * inch
|
||||
|
||||
# Common padding values
|
||||
PADDING_SMALL = 4
|
||||
PADDING_MEDIUM = 6
|
||||
PADDING_LARGE = 8
|
||||
PADDING_XLARGE = 10
|
||||
|
||||
|
||||
@dataclass
|
||||
class FrameworkConfig:
|
||||
"""
|
||||
Configuration for a compliance framework PDF report.
|
||||
|
||||
This dataclass defines all the configurable aspects of a compliance framework
|
||||
report, including visual styling, metadata fields, and feature flags.
|
||||
|
||||
Attributes:
|
||||
name (str): Internal framework identifier (e.g., "prowler_threatscore").
|
||||
display_name (str): Human-readable framework name for the report title.
|
||||
logo_filename (str | None): Optional filename of the framework logo in assets/img/.
|
||||
primary_color (colors.Color): Main color used for headers and important elements.
|
||||
secondary_color (colors.Color): Secondary color for sub-headers and accents.
|
||||
bg_color (colors.Color): Background color for highlighted sections.
|
||||
attribute_fields (list[str]): List of metadata field names to extract from requirements.
|
||||
sections (list[str] | None): Optional ordered list of section names for grouping.
|
||||
language (str): Report language ("en" for English, "es" for Spanish).
|
||||
has_risk_levels (bool): Whether the framework uses numeric risk levels.
|
||||
has_dimensions (bool): Whether the framework uses security dimensions (ENS).
|
||||
has_niveles (bool): Whether the framework uses nivel classification (ENS).
|
||||
has_weight (bool): Whether requirements have weight values.
|
||||
"""
|
||||
|
||||
name: str
|
||||
display_name: str
|
||||
logo_filename: str | None = None
|
||||
primary_color: colors.Color = field(default_factory=lambda: COLOR_BLUE)
|
||||
secondary_color: colors.Color = field(default_factory=lambda: COLOR_LIGHT_BLUE)
|
||||
bg_color: colors.Color = field(default_factory=lambda: COLOR_BG_BLUE)
|
||||
attribute_fields: list[str] = field(default_factory=list)
|
||||
sections: list[str] | None = None
|
||||
language: str = "en"
|
||||
has_risk_levels: bool = False
|
||||
has_dimensions: bool = False
|
||||
has_niveles: bool = False
|
||||
has_weight: bool = False
|
||||
|
||||
|
||||
FRAMEWORK_REGISTRY: dict[str, FrameworkConfig] = {
|
||||
"prowler_threatscore": FrameworkConfig(
|
||||
name="prowler_threatscore",
|
||||
display_name="Prowler ThreatScore",
|
||||
logo_filename=None,
|
||||
primary_color=COLOR_BLUE,
|
||||
secondary_color=COLOR_LIGHT_BLUE,
|
||||
bg_color=COLOR_BG_BLUE,
|
||||
attribute_fields=[
|
||||
"Title",
|
||||
"Section",
|
||||
"SubSection",
|
||||
"LevelOfRisk",
|
||||
"Weight",
|
||||
"AttributeDescription",
|
||||
"AdditionalInformation",
|
||||
],
|
||||
sections=THREATSCORE_SECTIONS,
|
||||
language="en",
|
||||
has_risk_levels=True,
|
||||
has_weight=True,
|
||||
),
|
||||
"ens": FrameworkConfig(
|
||||
name="ens",
|
||||
display_name="ENS RD2022",
|
||||
logo_filename="ens_logo.png",
|
||||
primary_color=COLOR_ENS_ALTO,
|
||||
secondary_color=COLOR_ENS_MEDIO,
|
||||
bg_color=COLOR_BG_BLUE,
|
||||
attribute_fields=[
|
||||
"IdGrupoControl",
|
||||
"Marco",
|
||||
"Categoria",
|
||||
"DescripcionControl",
|
||||
"Tipo",
|
||||
"Nivel",
|
||||
"Dimensiones",
|
||||
"ModoEjecucion",
|
||||
],
|
||||
sections=None,
|
||||
language="es",
|
||||
has_risk_levels=False,
|
||||
has_dimensions=True,
|
||||
has_niveles=True,
|
||||
has_weight=False,
|
||||
),
|
||||
"nis2": FrameworkConfig(
|
||||
name="nis2",
|
||||
display_name="NIS2 Directive",
|
||||
logo_filename="nis2_logo.png",
|
||||
primary_color=COLOR_NIS2_PRIMARY,
|
||||
secondary_color=COLOR_NIS2_SECONDARY,
|
||||
bg_color=COLOR_NIS2_BG_BLUE,
|
||||
attribute_fields=[
|
||||
"Section",
|
||||
"SubSection",
|
||||
"Description",
|
||||
],
|
||||
sections=NIS2_SECTIONS,
|
||||
language="en",
|
||||
has_risk_levels=False,
|
||||
has_dimensions=False,
|
||||
has_niveles=False,
|
||||
has_weight=False,
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def get_framework_config(compliance_id: str) -> FrameworkConfig | None:
|
||||
"""
|
||||
Get framework configuration based on compliance ID.
|
||||
|
||||
Args:
|
||||
compliance_id (str): The compliance framework identifier (e.g., "prowler_threatscore_aws").
|
||||
|
||||
Returns:
|
||||
FrameworkConfig | None: The framework configuration if found, None otherwise.
|
||||
"""
|
||||
compliance_lower = compliance_id.lower()
|
||||
|
||||
if "threatscore" in compliance_lower:
|
||||
return FRAMEWORK_REGISTRY["prowler_threatscore"]
|
||||
if "ens" in compliance_lower:
|
||||
return FRAMEWORK_REGISTRY["ens"]
|
||||
if "nis2" in compliance_lower:
|
||||
return FRAMEWORK_REGISTRY["nis2"]
|
||||
|
||||
return None
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,471 @@
|
||||
import os
|
||||
from collections import defaultdict
|
||||
|
||||
from reportlab.lib.units import inch
|
||||
from reportlab.platypus import Image, PageBreak, Paragraph, Spacer, Table, TableStyle
|
||||
|
||||
from api.models import StatusChoices
|
||||
|
||||
from .base import (
|
||||
BaseComplianceReportGenerator,
|
||||
ComplianceData,
|
||||
get_requirement_metadata,
|
||||
)
|
||||
from .charts import create_horizontal_bar_chart, get_chart_color_for_percentage
|
||||
from .config import (
|
||||
COLOR_BORDER_GRAY,
|
||||
COLOR_DARK_GRAY,
|
||||
COLOR_GRAY,
|
||||
COLOR_GRID_GRAY,
|
||||
COLOR_HIGH_RISK,
|
||||
COLOR_NIS2_BG_BLUE,
|
||||
COLOR_NIS2_PRIMARY,
|
||||
COLOR_SAFE,
|
||||
COLOR_WHITE,
|
||||
NIS2_SECTION_TITLES,
|
||||
NIS2_SECTIONS,
|
||||
)
|
||||
|
||||
|
||||
def _extract_section_number(section_string: str) -> str:
|
||||
"""Extract the section number from a full NIS2 section title.
|
||||
|
||||
NIS2 section strings are formatted like:
|
||||
"1 POLICY ON THE SECURITY OF NETWORK AND INFORMATION SYSTEMS..."
|
||||
|
||||
This function extracts just the leading number.
|
||||
|
||||
Args:
|
||||
section_string: Full section title string.
|
||||
|
||||
Returns:
|
||||
Section number as string (e.g., "1", "2", "11").
|
||||
"""
|
||||
if not section_string:
|
||||
return "Other"
|
||||
parts = section_string.split()
|
||||
if parts and parts[0].isdigit():
|
||||
return parts[0]
|
||||
return "Other"
|
||||
|
||||
|
||||
class NIS2ReportGenerator(BaseComplianceReportGenerator):
|
||||
"""
|
||||
PDF report generator for NIS2 Directive (EU) 2022/2555.
|
||||
|
||||
This generator creates comprehensive PDF reports containing:
|
||||
- Cover page with both Prowler and NIS2 logos
|
||||
- Executive summary with overall compliance score
|
||||
- Section analysis with horizontal bar chart
|
||||
- SubSection breakdown table
|
||||
- Critical failed requirements
|
||||
- Requirements index organized by section and subsection
|
||||
- Detailed findings for failed requirements
|
||||
"""
|
||||
|
||||
def create_cover_page(self, data: ComplianceData) -> list:
|
||||
"""
|
||||
Create the NIS2 report cover page with both logos.
|
||||
|
||||
Args:
|
||||
data: Aggregated compliance data.
|
||||
|
||||
Returns:
|
||||
List of ReportLab elements.
|
||||
"""
|
||||
elements = []
|
||||
|
||||
# Create logos side by side
|
||||
prowler_logo_path = os.path.join(
|
||||
os.path.dirname(__file__), "../../assets/img/prowler_logo.png"
|
||||
)
|
||||
nis2_logo_path = os.path.join(
|
||||
os.path.dirname(__file__), "../../assets/img/nis2_logo.png"
|
||||
)
|
||||
|
||||
prowler_logo = Image(prowler_logo_path, width=3.5 * inch, height=0.7 * inch)
|
||||
nis2_logo = Image(nis2_logo_path, width=2.3 * inch, height=1.5 * inch)
|
||||
|
||||
logos_table = Table(
|
||||
[[prowler_logo, nis2_logo]], colWidths=[4 * inch, 2.5 * inch]
|
||||
)
|
||||
logos_table.setStyle(
|
||||
TableStyle(
|
||||
[
|
||||
("ALIGN", (0, 0), (0, 0), "LEFT"),
|
||||
("ALIGN", (1, 0), (1, 0), "RIGHT"),
|
||||
("VALIGN", (0, 0), (0, 0), "MIDDLE"),
|
||||
("VALIGN", (1, 0), (1, 0), "MIDDLE"),
|
||||
]
|
||||
)
|
||||
)
|
||||
elements.append(logos_table)
|
||||
elements.append(Spacer(1, 0.3 * inch))
|
||||
|
||||
# Title
|
||||
title = Paragraph(
|
||||
"NIS2 Compliance Report<br/>Directive (EU) 2022/2555",
|
||||
self.styles["title"],
|
||||
)
|
||||
elements.append(title)
|
||||
elements.append(Spacer(1, 0.3 * inch))
|
||||
|
||||
# Compliance metadata table - use base class helper for consistency
|
||||
info_rows = self._build_info_rows(data, language="en")
|
||||
# Convert tuples to lists and wrap long text in Paragraphs
|
||||
metadata_data = []
|
||||
for label, value in info_rows:
|
||||
if label in ("Name:", "Description:") and value:
|
||||
metadata_data.append(
|
||||
[label, Paragraph(value, self.styles["normal_center"])]
|
||||
)
|
||||
else:
|
||||
metadata_data.append([label, value])
|
||||
|
||||
metadata_table = Table(metadata_data, colWidths=[2 * inch, 4 * inch])
|
||||
metadata_table.setStyle(
|
||||
TableStyle(
|
||||
[
|
||||
("BACKGROUND", (0, 0), (0, -1), COLOR_NIS2_PRIMARY),
|
||||
("TEXTCOLOR", (0, 0), (0, -1), COLOR_WHITE),
|
||||
("FONTNAME", (0, 0), (0, -1), "FiraCode"),
|
||||
("BACKGROUND", (1, 0), (1, -1), COLOR_NIS2_BG_BLUE),
|
||||
("TEXTCOLOR", (1, 0), (1, -1), COLOR_GRAY),
|
||||
("FONTNAME", (1, 0), (1, -1), "PlusJakartaSans"),
|
||||
("ALIGN", (0, 0), (-1, -1), "LEFT"),
|
||||
("VALIGN", (0, 0), (-1, -1), "TOP"),
|
||||
("FONTSIZE", (0, 0), (-1, -1), 11),
|
||||
("GRID", (0, 0), (-1, -1), 1, COLOR_BORDER_GRAY),
|
||||
("LEFTPADDING", (0, 0), (-1, -1), 10),
|
||||
("RIGHTPADDING", (0, 0), (-1, -1), 10),
|
||||
("TOPPADDING", (0, 0), (-1, -1), 8),
|
||||
("BOTTOMPADDING", (0, 0), (-1, -1), 8),
|
||||
]
|
||||
)
|
||||
)
|
||||
elements.append(metadata_table)
|
||||
|
||||
return elements
|
||||
|
||||
def create_executive_summary(self, data: ComplianceData) -> list:
|
||||
"""
|
||||
Create the executive summary with compliance metrics.
|
||||
|
||||
Args:
|
||||
data: Aggregated compliance data.
|
||||
|
||||
Returns:
|
||||
List of ReportLab elements.
|
||||
"""
|
||||
elements = []
|
||||
|
||||
elements.append(Paragraph("Executive Summary", self.styles["h1"]))
|
||||
elements.append(Spacer(1, 0.1 * inch))
|
||||
|
||||
# Calculate statistics
|
||||
total = len(data.requirements)
|
||||
passed = sum(1 for r in data.requirements if r.status == StatusChoices.PASS)
|
||||
failed = sum(1 for r in data.requirements if r.status == StatusChoices.FAIL)
|
||||
manual = sum(1 for r in data.requirements if r.status == StatusChoices.MANUAL)
|
||||
|
||||
# Calculate compliance excluding manual
|
||||
evaluated = passed + failed
|
||||
overall_compliance = (passed / evaluated * 100) if evaluated > 0 else 100
|
||||
|
||||
# Summary statistics table
|
||||
summary_data = [
|
||||
["Metric", "Value"],
|
||||
["Total Requirements", str(total)],
|
||||
["Passed ✓", str(passed)],
|
||||
["Failed ✗", str(failed)],
|
||||
["Manual ⊙", str(manual)],
|
||||
["Overall Compliance", f"{overall_compliance:.1f}%"],
|
||||
]
|
||||
|
||||
summary_table = Table(summary_data, colWidths=[3 * inch, 2 * inch])
|
||||
summary_table.setStyle(
|
||||
TableStyle(
|
||||
[
|
||||
("BACKGROUND", (0, 0), (-1, 0), COLOR_NIS2_PRIMARY),
|
||||
("TEXTCOLOR", (0, 0), (-1, 0), COLOR_WHITE),
|
||||
("BACKGROUND", (0, 2), (0, 2), COLOR_SAFE),
|
||||
("TEXTCOLOR", (0, 2), (0, 2), COLOR_WHITE),
|
||||
("BACKGROUND", (0, 3), (0, 3), COLOR_HIGH_RISK),
|
||||
("TEXTCOLOR", (0, 3), (0, 3), COLOR_WHITE),
|
||||
("BACKGROUND", (0, 4), (0, 4), COLOR_DARK_GRAY),
|
||||
("TEXTCOLOR", (0, 4), (0, 4), COLOR_WHITE),
|
||||
("ALIGN", (0, 0), (-1, -1), "CENTER"),
|
||||
("FONTNAME", (0, 0), (-1, 0), "PlusJakartaSans"),
|
||||
("FONTSIZE", (0, 0), (-1, 0), 12),
|
||||
("FONTSIZE", (0, 1), (-1, -1), 10),
|
||||
("BOTTOMPADDING", (0, 0), (-1, 0), 10),
|
||||
("GRID", (0, 0), (-1, -1), 0.5, COLOR_BORDER_GRAY),
|
||||
(
|
||||
"ROWBACKGROUNDS",
|
||||
(1, 1),
|
||||
(1, -1),
|
||||
[COLOR_WHITE, COLOR_NIS2_BG_BLUE],
|
||||
),
|
||||
]
|
||||
)
|
||||
)
|
||||
elements.append(summary_table)
|
||||
|
||||
return elements
|
||||
|
||||
def create_charts_section(self, data: ComplianceData) -> list:
|
||||
"""
|
||||
Create the charts section with section analysis.
|
||||
|
||||
Args:
|
||||
data: Aggregated compliance data.
|
||||
|
||||
Returns:
|
||||
List of ReportLab elements.
|
||||
"""
|
||||
elements = []
|
||||
|
||||
# Section chart
|
||||
elements.append(Paragraph("Compliance by Section", self.styles["h1"]))
|
||||
elements.append(Spacer(1, 0.1 * inch))
|
||||
elements.append(
|
||||
Paragraph(
|
||||
"The following chart shows compliance percentage for each main section "
|
||||
"of the NIS2 directive:",
|
||||
self.styles["normal_center"],
|
||||
)
|
||||
)
|
||||
elements.append(Spacer(1, 0.1 * inch))
|
||||
|
||||
chart_buffer = self._create_section_chart(data)
|
||||
chart_buffer.seek(0)
|
||||
chart_image = Image(chart_buffer, width=6.5 * inch, height=5 * inch)
|
||||
elements.append(chart_image)
|
||||
elements.append(PageBreak())
|
||||
|
||||
# SubSection breakdown table
|
||||
elements.append(Paragraph("SubSection Breakdown", self.styles["h1"]))
|
||||
elements.append(Spacer(1, 0.1 * inch))
|
||||
|
||||
subsection_table = self._create_subsection_table(data)
|
||||
elements.append(subsection_table)
|
||||
|
||||
return elements
|
||||
|
||||
def create_requirements_index(self, data: ComplianceData) -> list:
|
||||
"""
|
||||
Create the requirements index organized by section and subsection.
|
||||
|
||||
Args:
|
||||
data: Aggregated compliance data.
|
||||
|
||||
Returns:
|
||||
List of ReportLab elements.
|
||||
"""
|
||||
elements = []
|
||||
|
||||
elements.append(Paragraph("Requirements Index", self.styles["h1"]))
|
||||
elements.append(Spacer(1, 0.1 * inch))
|
||||
|
||||
# Organize by section number and subsection
|
||||
sections = {}
|
||||
for req in data.requirements:
|
||||
m = get_requirement_metadata(req.id, data.attributes_by_requirement_id)
|
||||
if m:
|
||||
full_section = getattr(m, "Section", "Other")
|
||||
# Extract section number from full title (e.g., "1 POLICY..." -> "1")
|
||||
section_num = _extract_section_number(full_section)
|
||||
subsection = getattr(m, "SubSection", "")
|
||||
description = getattr(m, "Description", req.description)
|
||||
|
||||
if section_num not in sections:
|
||||
sections[section_num] = {}
|
||||
if subsection not in sections[section_num]:
|
||||
sections[section_num][subsection] = []
|
||||
|
||||
sections[section_num][subsection].append(
|
||||
{
|
||||
"id": req.id,
|
||||
"description": description,
|
||||
"status": req.status,
|
||||
}
|
||||
)
|
||||
|
||||
# Sort by NIS2 section order
|
||||
for section in NIS2_SECTIONS:
|
||||
if section not in sections:
|
||||
continue
|
||||
|
||||
section_title = NIS2_SECTION_TITLES.get(section, f"Section {section}")
|
||||
elements.append(Paragraph(section_title, self.styles["h2"]))
|
||||
|
||||
for subsection_name, reqs in sections[section].items():
|
||||
if subsection_name:
|
||||
# Truncate long subsection names for display
|
||||
display_subsection = (
|
||||
subsection_name[:80] + "..."
|
||||
if len(subsection_name) > 80
|
||||
else subsection_name
|
||||
)
|
||||
elements.append(Paragraph(display_subsection, self.styles["h3"]))
|
||||
|
||||
for req in reqs:
|
||||
status_indicator = (
|
||||
"✓" if req["status"] == StatusChoices.PASS else "✗"
|
||||
)
|
||||
if req["status"] == StatusChoices.MANUAL:
|
||||
status_indicator = "⊙"
|
||||
|
||||
desc = (
|
||||
req["description"][:60] + "..."
|
||||
if len(req["description"]) > 60
|
||||
else req["description"]
|
||||
)
|
||||
elements.append(
|
||||
Paragraph(
|
||||
f"{status_indicator} {req['id']}: {desc}",
|
||||
self.styles["normal"],
|
||||
)
|
||||
)
|
||||
|
||||
elements.append(Spacer(1, 0.1 * inch))
|
||||
|
||||
return elements
|
||||
|
||||
def _create_section_chart(self, data: ComplianceData):
|
||||
"""
|
||||
Create the section compliance chart.
|
||||
|
||||
Args:
|
||||
data: Aggregated compliance data.
|
||||
|
||||
Returns:
|
||||
BytesIO buffer containing the chart image.
|
||||
"""
|
||||
section_scores = defaultdict(lambda: {"passed": 0, "total": 0})
|
||||
|
||||
for req in data.requirements:
|
||||
if req.status == StatusChoices.MANUAL:
|
||||
continue
|
||||
|
||||
m = get_requirement_metadata(req.id, data.attributes_by_requirement_id)
|
||||
if m:
|
||||
full_section = getattr(m, "Section", "Other")
|
||||
# Extract section number from full title (e.g., "1 POLICY..." -> "1")
|
||||
section_num = _extract_section_number(full_section)
|
||||
section_scores[section_num]["total"] += 1
|
||||
if req.status == StatusChoices.PASS:
|
||||
section_scores[section_num]["passed"] += 1
|
||||
|
||||
# Build labels and values in NIS2 section order
|
||||
labels = []
|
||||
values = []
|
||||
for section in NIS2_SECTIONS:
|
||||
if section in section_scores and section_scores[section]["total"] > 0:
|
||||
scores = section_scores[section]
|
||||
pct = (scores["passed"] / scores["total"]) * 100
|
||||
section_title = NIS2_SECTION_TITLES.get(section, f"Section {section}")
|
||||
labels.append(section_title)
|
||||
values.append(pct)
|
||||
|
||||
return create_horizontal_bar_chart(
|
||||
labels=labels,
|
||||
values=values,
|
||||
xlabel="Compliance (%)",
|
||||
color_func=get_chart_color_for_percentage,
|
||||
)
|
||||
|
||||
def _create_subsection_table(self, data: ComplianceData) -> Table:
|
||||
"""
|
||||
Create the subsection breakdown table.
|
||||
|
||||
Args:
|
||||
data: Aggregated compliance data.
|
||||
|
||||
Returns:
|
||||
ReportLab Table element.
|
||||
"""
|
||||
subsection_scores = defaultdict(lambda: {"passed": 0, "failed": 0, "manual": 0})
|
||||
|
||||
for req in data.requirements:
|
||||
m = get_requirement_metadata(req.id, data.attributes_by_requirement_id)
|
||||
if m:
|
||||
full_section = getattr(m, "Section", "")
|
||||
subsection = getattr(m, "SubSection", "")
|
||||
# Use section number + subsection for grouping
|
||||
section_num = _extract_section_number(full_section)
|
||||
# Create a shorter key using section number
|
||||
if subsection:
|
||||
# Extract subsection number if present (e.g., "1.1 Policy..." -> "1.1")
|
||||
subsection_parts = subsection.split()
|
||||
if subsection_parts:
|
||||
key = subsection_parts[0] # Just the number like "1.1"
|
||||
else:
|
||||
key = f"{section_num}"
|
||||
else:
|
||||
key = section_num
|
||||
|
||||
if req.status == StatusChoices.PASS:
|
||||
subsection_scores[key]["passed"] += 1
|
||||
elif req.status == StatusChoices.FAIL:
|
||||
subsection_scores[key]["failed"] += 1
|
||||
else:
|
||||
subsection_scores[key]["manual"] += 1
|
||||
|
||||
table_data = [["Section", "Passed", "Failed", "Manual", "Compliance"]]
|
||||
for key, scores in sorted(
|
||||
subsection_scores.items(), key=lambda x: self._sort_section_key(x[0])
|
||||
):
|
||||
total = scores["passed"] + scores["failed"]
|
||||
pct = (scores["passed"] / total * 100) if total > 0 else 100
|
||||
table_data.append(
|
||||
[
|
||||
key,
|
||||
str(scores["passed"]),
|
||||
str(scores["failed"]),
|
||||
str(scores["manual"]),
|
||||
f"{pct:.1f}%",
|
||||
]
|
||||
)
|
||||
|
||||
table = Table(
|
||||
table_data,
|
||||
colWidths=[1.2 * inch, 0.9 * inch, 0.9 * inch, 0.9 * inch, 1.2 * inch],
|
||||
)
|
||||
table.setStyle(
|
||||
TableStyle(
|
||||
[
|
||||
("BACKGROUND", (0, 0), (-1, 0), COLOR_NIS2_PRIMARY),
|
||||
("TEXTCOLOR", (0, 0), (-1, 0), COLOR_WHITE),
|
||||
("FONTNAME", (0, 0), (-1, 0), "FiraCode"),
|
||||
("FONTSIZE", (0, 0), (-1, 0), 10),
|
||||
("ALIGN", (0, 0), (-1, -1), "CENTER"),
|
||||
("VALIGN", (0, 0), (-1, -1), "MIDDLE"),
|
||||
("FONTSIZE", (0, 1), (-1, -1), 9),
|
||||
("GRID", (0, 0), (-1, -1), 0.5, COLOR_GRID_GRAY),
|
||||
("LEFTPADDING", (0, 0), (-1, -1), 6),
|
||||
("RIGHTPADDING", (0, 0), (-1, -1), 6),
|
||||
("TOPPADDING", (0, 0), (-1, -1), 4),
|
||||
("BOTTOMPADDING", (0, 0), (-1, -1), 4),
|
||||
(
|
||||
"ROWBACKGROUNDS",
|
||||
(0, 1),
|
||||
(-1, -1),
|
||||
[COLOR_WHITE, COLOR_NIS2_BG_BLUE],
|
||||
),
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
return table
|
||||
|
||||
def _sort_section_key(self, key: str) -> tuple:
|
||||
"""Sort section keys numerically (e.g., 1, 1.1, 1.2, 2, 11)."""
|
||||
parts = key.split(".")
|
||||
result = []
|
||||
for part in parts:
|
||||
try:
|
||||
result.append(int(part))
|
||||
except ValueError:
|
||||
result.append(float("inf"))
|
||||
return tuple(result)
|
||||
@@ -0,0 +1,509 @@
|
||||
import gc
|
||||
|
||||
from reportlab.lib import colors
|
||||
from reportlab.lib.styles import ParagraphStyle
|
||||
from reportlab.lib.units import inch
|
||||
from reportlab.platypus import Image, PageBreak, Paragraph, Spacer, Table, TableStyle
|
||||
|
||||
from api.models import StatusChoices
|
||||
|
||||
from .base import (
|
||||
BaseComplianceReportGenerator,
|
||||
ComplianceData,
|
||||
get_requirement_metadata,
|
||||
)
|
||||
from .charts import create_vertical_bar_chart, get_chart_color_for_percentage
|
||||
from .components import get_color_for_compliance, get_color_for_weight
|
||||
from .config import COLOR_HIGH_RISK, COLOR_WHITE
|
||||
|
||||
|
||||
class ThreatScoreReportGenerator(BaseComplianceReportGenerator):
|
||||
"""
|
||||
PDF report generator for Prowler ThreatScore framework.
|
||||
|
||||
This generator creates comprehensive PDF reports containing:
|
||||
- Compliance overview and metadata
|
||||
- Section-by-section compliance scores with charts
|
||||
- Overall ThreatScore calculation
|
||||
- Critical failed requirements
|
||||
- Detailed findings for each requirement
|
||||
"""
|
||||
|
||||
def create_executive_summary(self, data: ComplianceData) -> list:
|
||||
"""
|
||||
Create the executive summary section with ThreatScore calculation.
|
||||
|
||||
Args:
|
||||
data: Aggregated compliance data.
|
||||
|
||||
Returns:
|
||||
List of ReportLab elements.
|
||||
"""
|
||||
elements = []
|
||||
|
||||
elements.append(Paragraph("Compliance Score by Sections", self.styles["h1"]))
|
||||
elements.append(Spacer(1, 0.2 * inch))
|
||||
|
||||
# Create section score chart
|
||||
chart_buffer = self._create_section_score_chart(data)
|
||||
chart_image = Image(chart_buffer, width=7 * inch, height=5.5 * inch)
|
||||
elements.append(chart_image)
|
||||
|
||||
# Calculate overall ThreatScore
|
||||
overall_compliance = self._calculate_threatscore(data)
|
||||
|
||||
elements.append(Spacer(1, 0.3 * inch))
|
||||
|
||||
# Summary table
|
||||
summary_data = [["ThreatScore:", f"{overall_compliance:.2f}%"]]
|
||||
compliance_color = get_color_for_compliance(overall_compliance)
|
||||
|
||||
summary_table = Table(summary_data, colWidths=[2.5 * inch, 2 * inch])
|
||||
summary_table.setStyle(
|
||||
TableStyle(
|
||||
[
|
||||
("BACKGROUND", (0, 0), (0, 0), colors.Color(0.1, 0.3, 0.5)),
|
||||
("TEXTCOLOR", (0, 0), (0, 0), colors.white),
|
||||
("FONTNAME", (0, 0), (0, 0), "FiraCode"),
|
||||
("FONTSIZE", (0, 0), (0, 0), 12),
|
||||
("BACKGROUND", (1, 0), (1, 0), compliance_color),
|
||||
("TEXTCOLOR", (1, 0), (1, 0), colors.white),
|
||||
("FONTNAME", (1, 0), (1, 0), "FiraCode"),
|
||||
("FONTSIZE", (1, 0), (1, 0), 16),
|
||||
("ALIGN", (0, 0), (-1, -1), "CENTER"),
|
||||
("VALIGN", (0, 0), (-1, -1), "MIDDLE"),
|
||||
("GRID", (0, 0), (-1, -1), 1.5, colors.Color(0.5, 0.6, 0.7)),
|
||||
("LEFTPADDING", (0, 0), (-1, -1), 12),
|
||||
("RIGHTPADDING", (0, 0), (-1, -1), 12),
|
||||
("TOPPADDING", (0, 0), (-1, -1), 10),
|
||||
("BOTTOMPADDING", (0, 0), (-1, -1), 10),
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
elements.append(summary_table)
|
||||
|
||||
return elements
|
||||
|
||||
def _build_body_sections(self, data: ComplianceData) -> list:
|
||||
"""Override section order: Requirements Index before Critical Requirements."""
|
||||
elements = []
|
||||
|
||||
# Page break to separate from executive summary
|
||||
elements.append(PageBreak())
|
||||
|
||||
# Requirements index first
|
||||
elements.extend(self.create_requirements_index(data))
|
||||
|
||||
# Critical requirements section (already starts with PageBreak internally)
|
||||
elements.extend(self.create_charts_section(data))
|
||||
elements.append(PageBreak())
|
||||
gc.collect()
|
||||
|
||||
return elements
|
||||
|
||||
def create_charts_section(self, data: ComplianceData) -> list:
|
||||
"""
|
||||
Create the critical failed requirements section.
|
||||
|
||||
Args:
|
||||
data: Aggregated compliance data.
|
||||
|
||||
Returns:
|
||||
List of ReportLab elements.
|
||||
"""
|
||||
elements = []
|
||||
min_risk_level = getattr(self, "_min_risk_level", 4)
|
||||
|
||||
# Start on a new page
|
||||
elements.append(PageBreak())
|
||||
elements.append(
|
||||
Paragraph("Top Requirements by Level of Risk", self.styles["h1"])
|
||||
)
|
||||
elements.append(Spacer(1, 0.1 * inch))
|
||||
elements.append(
|
||||
Paragraph(
|
||||
f"Critical Failed Requirements (Risk Level ≥ {min_risk_level})",
|
||||
self.styles["h2"],
|
||||
)
|
||||
)
|
||||
elements.append(Spacer(1, 0.2 * inch))
|
||||
|
||||
critical_failed = self._get_critical_failed_requirements(data, min_risk_level)
|
||||
|
||||
if not critical_failed:
|
||||
elements.append(
|
||||
Paragraph(
|
||||
"✅ No critical failed requirements found. Great job!",
|
||||
self.styles["normal"],
|
||||
)
|
||||
)
|
||||
else:
|
||||
elements.append(
|
||||
Paragraph(
|
||||
f"Found {len(critical_failed)} critical failed requirements "
|
||||
"that require immediate attention:",
|
||||
self.styles["normal"],
|
||||
)
|
||||
)
|
||||
elements.append(Spacer(1, 0.5 * inch))
|
||||
|
||||
table = self._create_critical_requirements_table(critical_failed)
|
||||
elements.append(table)
|
||||
|
||||
# Immediate action required banner
|
||||
elements.append(Spacer(1, 0.3 * inch))
|
||||
elements.append(self._create_action_required_banner())
|
||||
|
||||
return elements
|
||||
|
||||
def create_requirements_index(self, data: ComplianceData) -> list:
|
||||
"""
|
||||
Create the requirements index organized by section and subsection.
|
||||
|
||||
Args:
|
||||
data: Aggregated compliance data.
|
||||
|
||||
Returns:
|
||||
List of ReportLab elements.
|
||||
"""
|
||||
elements = []
|
||||
|
||||
elements.append(Paragraph("Requirements Index", self.styles["h1"]))
|
||||
|
||||
# Organize requirements by section and subsection
|
||||
sections = {}
|
||||
for req_id in data.attributes_by_requirement_id:
|
||||
m = get_requirement_metadata(req_id, data.attributes_by_requirement_id)
|
||||
if m:
|
||||
section = getattr(m, "Section", "N/A")
|
||||
subsection = getattr(m, "SubSection", "N/A")
|
||||
title = getattr(m, "Title", "N/A")
|
||||
|
||||
if section not in sections:
|
||||
sections[section] = {}
|
||||
if subsection not in sections[section]:
|
||||
sections[section][subsection] = []
|
||||
|
||||
sections[section][subsection].append({"id": req_id, "title": title})
|
||||
|
||||
section_num = 1
|
||||
for section_name, subsections in sections.items():
|
||||
elements.append(
|
||||
Paragraph(f"{section_num}. {section_name}", self.styles["h2"])
|
||||
)
|
||||
|
||||
for subsection_name, requirements in subsections.items():
|
||||
elements.append(Paragraph(f"{subsection_name}", self.styles["h3"]))
|
||||
|
||||
for req in requirements:
|
||||
elements.append(
|
||||
Paragraph(
|
||||
f"{req['id']} - {req['title']}", self.styles["normal"]
|
||||
)
|
||||
)
|
||||
|
||||
section_num += 1
|
||||
elements.append(Spacer(1, 0.1 * inch))
|
||||
|
||||
return elements
|
||||
|
||||
def _create_section_score_chart(self, data: ComplianceData):
|
||||
"""
|
||||
Create the section compliance score chart using weighted ThreatScore formula.
|
||||
|
||||
The section score uses the same weighted formula as the overall ThreatScore:
|
||||
Score = Σ(rate_i * total_findings_i * weight_i * rfac_i) / Σ(total_findings_i * weight_i * rfac_i)
|
||||
Where rfac_i = 1 + 0.25 * risk_level
|
||||
|
||||
Sections without findings are shown with 100% score.
|
||||
|
||||
Args:
|
||||
data: Aggregated compliance data.
|
||||
|
||||
Returns:
|
||||
BytesIO buffer containing the chart image.
|
||||
"""
|
||||
# First, collect ALL sections from requirements (including those without findings)
|
||||
all_sections = set()
|
||||
sections_data = {}
|
||||
|
||||
for req in data.requirements:
|
||||
m = get_requirement_metadata(req.id, data.attributes_by_requirement_id)
|
||||
if m:
|
||||
section = getattr(m, "Section", "Other")
|
||||
all_sections.add(section)
|
||||
|
||||
# Only calculate scores for requirements with findings
|
||||
if req.total_findings == 0:
|
||||
continue
|
||||
|
||||
risk_level_raw = getattr(m, "LevelOfRisk", 0)
|
||||
weight_raw = getattr(m, "Weight", 0)
|
||||
# Ensure numeric types for calculations (compliance data may have str)
|
||||
try:
|
||||
risk_level = int(risk_level_raw) if risk_level_raw else 0
|
||||
except (ValueError, TypeError):
|
||||
risk_level = 0
|
||||
try:
|
||||
weight = int(weight_raw) if weight_raw else 0
|
||||
except (ValueError, TypeError):
|
||||
weight = 0
|
||||
|
||||
# ThreatScore formula components
|
||||
rate_i = req.passed_findings / req.total_findings
|
||||
rfac_i = 1 + 0.25 * risk_level
|
||||
|
||||
if section not in sections_data:
|
||||
sections_data[section] = {
|
||||
"numerator": 0,
|
||||
"denominator": 0,
|
||||
}
|
||||
|
||||
sections_data[section]["numerator"] += (
|
||||
rate_i * req.total_findings * weight * rfac_i
|
||||
)
|
||||
sections_data[section]["denominator"] += (
|
||||
req.total_findings * weight * rfac_i
|
||||
)
|
||||
|
||||
# Calculate percentages for all sections
|
||||
labels = []
|
||||
values = []
|
||||
for section in sorted(all_sections):
|
||||
if section in sections_data and sections_data[section]["denominator"] > 0:
|
||||
pct = (
|
||||
sections_data[section]["numerator"]
|
||||
/ sections_data[section]["denominator"]
|
||||
) * 100
|
||||
else:
|
||||
# Sections without findings get 100%
|
||||
pct = 100.0
|
||||
labels.append(section)
|
||||
values.append(pct)
|
||||
|
||||
return create_vertical_bar_chart(
|
||||
labels=labels,
|
||||
values=values,
|
||||
ylabel="Compliance Score (%)",
|
||||
xlabel="",
|
||||
color_func=get_chart_color_for_percentage,
|
||||
rotation=0,
|
||||
)
|
||||
|
||||
def _calculate_threatscore(self, data: ComplianceData) -> float:
|
||||
"""
|
||||
Calculate the overall ThreatScore using the weighted formula.
|
||||
|
||||
Args:
|
||||
data: Aggregated compliance data.
|
||||
|
||||
Returns:
|
||||
Overall ThreatScore percentage.
|
||||
"""
|
||||
numerator = 0
|
||||
denominator = 0
|
||||
has_findings = False
|
||||
|
||||
for req in data.requirements:
|
||||
if req.total_findings == 0:
|
||||
continue
|
||||
|
||||
has_findings = True
|
||||
m = get_requirement_metadata(req.id, data.attributes_by_requirement_id)
|
||||
|
||||
if m:
|
||||
risk_level_raw = getattr(m, "LevelOfRisk", 0)
|
||||
weight_raw = getattr(m, "Weight", 0)
|
||||
# Ensure numeric types for calculations (compliance data may have str)
|
||||
try:
|
||||
risk_level = int(risk_level_raw) if risk_level_raw else 0
|
||||
except (ValueError, TypeError):
|
||||
risk_level = 0
|
||||
try:
|
||||
weight = int(weight_raw) if weight_raw else 0
|
||||
except (ValueError, TypeError):
|
||||
weight = 0
|
||||
|
||||
rate_i = req.passed_findings / req.total_findings
|
||||
rfac_i = 1 + 0.25 * risk_level
|
||||
|
||||
numerator += rate_i * req.total_findings * weight * rfac_i
|
||||
denominator += req.total_findings * weight * rfac_i
|
||||
|
||||
if not has_findings:
|
||||
return 100.0
|
||||
if denominator > 0:
|
||||
return (numerator / denominator) * 100
|
||||
return 0.0
|
||||
|
||||
def _get_critical_failed_requirements(
|
||||
self, data: ComplianceData, min_risk_level: int
|
||||
) -> list[dict]:
|
||||
"""
|
||||
Get critical failed requirements sorted by risk level and weight.
|
||||
|
||||
Args:
|
||||
data: Aggregated compliance data.
|
||||
min_risk_level: Minimum risk level threshold.
|
||||
|
||||
Returns:
|
||||
List of critical failed requirement dictionaries.
|
||||
"""
|
||||
critical = []
|
||||
|
||||
for req in data.requirements:
|
||||
if req.status != StatusChoices.FAIL:
|
||||
continue
|
||||
|
||||
m = get_requirement_metadata(req.id, data.attributes_by_requirement_id)
|
||||
|
||||
if m:
|
||||
risk_level_raw = getattr(m, "LevelOfRisk", 0)
|
||||
weight_raw = getattr(m, "Weight", 0)
|
||||
# Ensure numeric types for calculations (compliance data may have str)
|
||||
try:
|
||||
risk_level = int(risk_level_raw) if risk_level_raw else 0
|
||||
except (ValueError, TypeError):
|
||||
risk_level = 0
|
||||
try:
|
||||
weight = int(weight_raw) if weight_raw else 0
|
||||
except (ValueError, TypeError):
|
||||
weight = 0
|
||||
|
||||
if risk_level >= min_risk_level:
|
||||
critical.append(
|
||||
{
|
||||
"id": req.id,
|
||||
"risk_level": risk_level,
|
||||
"weight": weight,
|
||||
"title": getattr(m, "Title", "N/A"),
|
||||
"section": getattr(m, "Section", "N/A"),
|
||||
}
|
||||
)
|
||||
|
||||
critical.sort(key=lambda x: (x["risk_level"], x["weight"]), reverse=True)
|
||||
return critical
|
||||
|
||||
def _create_critical_requirements_table(self, critical_requirements: list) -> Table:
|
||||
"""
|
||||
Create the critical requirements table.
|
||||
|
||||
Args:
|
||||
critical_requirements: List of critical requirement dictionaries.
|
||||
|
||||
Returns:
|
||||
ReportLab Table element.
|
||||
"""
|
||||
table_data = [["Risk", "Weight", "Requirement ID", "Title", "Section"]]
|
||||
|
||||
for req in critical_requirements:
|
||||
title = req["title"]
|
||||
if len(title) > 50:
|
||||
title = title[:47] + "..."
|
||||
|
||||
table_data.append(
|
||||
[
|
||||
str(req["risk_level"]),
|
||||
str(req["weight"]),
|
||||
req["id"],
|
||||
title,
|
||||
req["section"],
|
||||
]
|
||||
)
|
||||
|
||||
table = Table(
|
||||
table_data,
|
||||
colWidths=[0.7 * inch, 0.9 * inch, 1.3 * inch, 3.1 * inch, 1.5 * inch],
|
||||
)
|
||||
|
||||
table.setStyle(
|
||||
TableStyle(
|
||||
[
|
||||
("BACKGROUND", (0, 0), (-1, 0), COLOR_HIGH_RISK),
|
||||
("TEXTCOLOR", (0, 0), (-1, 0), COLOR_WHITE),
|
||||
("FONTNAME", (0, 0), (-1, 0), "FiraCode"),
|
||||
("FONTSIZE", (0, 0), (-1, 0), 10),
|
||||
("BACKGROUND", (0, 1), (0, -1), COLOR_HIGH_RISK),
|
||||
("TEXTCOLOR", (0, 1), (0, -1), COLOR_WHITE),
|
||||
("FONTNAME", (0, 1), (0, -1), "FiraCode"),
|
||||
("ALIGN", (0, 1), (0, -1), "CENTER"),
|
||||
("FONTSIZE", (0, 1), (0, -1), 12),
|
||||
("ALIGN", (1, 1), (1, -1), "CENTER"),
|
||||
("FONTNAME", (1, 1), (1, -1), "FiraCode"),
|
||||
("FONTNAME", (2, 1), (2, -1), "FiraCode"),
|
||||
("FONTSIZE", (2, 1), (2, -1), 9),
|
||||
("FONTNAME", (3, 1), (-1, -1), "PlusJakartaSans"),
|
||||
("FONTSIZE", (3, 1), (-1, -1), 8),
|
||||
("VALIGN", (0, 0), (-1, -1), "MIDDLE"),
|
||||
("GRID", (0, 0), (-1, -1), 1, colors.Color(0.7, 0.7, 0.7)),
|
||||
("LEFTPADDING", (0, 0), (-1, -1), 6),
|
||||
("RIGHTPADDING", (0, 0), (-1, -1), 6),
|
||||
("TOPPADDING", (0, 0), (-1, -1), 8),
|
||||
("BOTTOMPADDING", (0, 0), (-1, -1), 8),
|
||||
("BACKGROUND", (1, 1), (-1, -1), colors.Color(0.98, 0.98, 0.98)),
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
# Color weight column based on value
|
||||
for idx, req in enumerate(critical_requirements):
|
||||
row_idx = idx + 1
|
||||
weight_color = get_color_for_weight(req["weight"])
|
||||
table.setStyle(
|
||||
TableStyle(
|
||||
[
|
||||
("BACKGROUND", (1, row_idx), (1, row_idx), weight_color),
|
||||
("TEXTCOLOR", (1, row_idx), (1, row_idx), COLOR_WHITE),
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
return table
|
||||
|
||||
def _create_action_required_banner(self) -> Table:
|
||||
"""
|
||||
Create the 'Immediate Action Required' banner for critical requirements.
|
||||
|
||||
Returns:
|
||||
ReportLab Table element styled as a red-bordered alert banner.
|
||||
"""
|
||||
banner_style = ParagraphStyle(
|
||||
"ActionRequired",
|
||||
fontName="PlusJakartaSans",
|
||||
fontSize=11,
|
||||
textColor=COLOR_HIGH_RISK,
|
||||
leading=16,
|
||||
)
|
||||
|
||||
banner_content = Paragraph(
|
||||
"<b>IMMEDIATE ACTION REQUIRED:</b><br/>"
|
||||
"These requirements have the highest risk levels and have failed "
|
||||
"compliance checks. Please prioritize addressing these issues to "
|
||||
"improve your security posture.",
|
||||
banner_style,
|
||||
)
|
||||
|
||||
banner_table = Table(
|
||||
[[banner_content]],
|
||||
colWidths=[6.5 * inch],
|
||||
)
|
||||
banner_table.setStyle(
|
||||
TableStyle(
|
||||
[
|
||||
(
|
||||
"BACKGROUND",
|
||||
(0, 0),
|
||||
(0, 0),
|
||||
colors.Color(0.98, 0.92, 0.92),
|
||||
),
|
||||
("BOX", (0, 0), (0, 0), 2, COLOR_HIGH_RISK),
|
||||
("LEFTPADDING", (0, 0), (0, 0), 20),
|
||||
("RIGHTPADDING", (0, 0), (0, 0), 20),
|
||||
("TOPPADDING", (0, 0), (0, 0), 15),
|
||||
("BOTTOMPADDING", (0, 0), (0, 0), 15),
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
return banner_table
|
||||
@@ -45,6 +45,7 @@ from api.models import (
|
||||
ResourceTag,
|
||||
Scan,
|
||||
ScanCategorySummary,
|
||||
ScanGroupSummary,
|
||||
ScanSummary,
|
||||
StateChoices,
|
||||
)
|
||||
@@ -127,6 +128,50 @@ def aggregate_category_counts(
|
||||
cache[key]["new_failed"] += 1
|
||||
|
||||
|
||||
def aggregate_resource_group_counts(
|
||||
resource_group: str | None,
|
||||
severity: str,
|
||||
status: str,
|
||||
delta: str | None,
|
||||
muted: bool,
|
||||
resource_uid: str,
|
||||
cache: dict[tuple[str, str], dict[str, int]],
|
||||
group_resources_cache: dict[str, set],
|
||||
) -> None:
|
||||
"""
|
||||
Increment resource group counters in-place for a finding.
|
||||
|
||||
Args:
|
||||
resource_group: Resource group from check metadata (e.g., "database", "compute").
|
||||
severity: Severity level (e.g., "high", "medium").
|
||||
status: Finding status as string ("FAIL", "PASS").
|
||||
delta: Delta value as string ("new", "changed") or None.
|
||||
muted: Whether the finding is muted.
|
||||
resource_uid: Unique identifier for the resource to count distinct resources.
|
||||
cache: Dict {(resource_group, severity): {"total", "failed", "new_failed"}} to update.
|
||||
group_resources_cache: Dict {resource_group: set(resource_uids)} for group-level resource tracking.
|
||||
"""
|
||||
if not resource_group:
|
||||
return
|
||||
|
||||
is_failed = status == "FAIL" and not muted
|
||||
is_new_failed = is_failed and delta == "new"
|
||||
|
||||
key = (resource_group, severity)
|
||||
if key not in cache:
|
||||
cache[key] = {"total": 0, "failed": 0, "new_failed": 0}
|
||||
if not muted:
|
||||
cache[key]["total"] += 1
|
||||
if is_failed:
|
||||
cache[key]["failed"] += 1
|
||||
if is_new_failed:
|
||||
cache[key]["new_failed"] += 1
|
||||
|
||||
# Track resources at GROUP level (not per-severity) to avoid over-counting
|
||||
if resource_uid and not muted:
|
||||
group_resources_cache.setdefault(resource_group, set()).add(resource_uid)
|
||||
|
||||
|
||||
def _get_attack_surface_mapping_from_provider(provider_type: str) -> dict:
|
||||
global _ATTACK_SURFACE_MAPPING_CACHE
|
||||
|
||||
@@ -438,6 +483,8 @@ def _process_finding_micro_batch(
|
||||
scan_resource_cache: set,
|
||||
mute_rules_cache: dict,
|
||||
scan_categories_cache: dict[tuple[str, str], dict[str, int]],
|
||||
scan_resource_groups_cache: dict[tuple[str, str], dict[str, int]],
|
||||
group_resources_cache: dict[str, set],
|
||||
) -> None:
|
||||
"""
|
||||
Process a micro-batch of findings and persist them using bulk operations.
|
||||
@@ -459,6 +506,8 @@ def _process_finding_micro_batch(
|
||||
scan_resource_cache: Set of tuples used to create `ResourceScanSummary` rows.
|
||||
mute_rules_cache: Map of finding UID -> mute reason gathered before the scan.
|
||||
scan_categories_cache: Dict tracking category counts {(category, severity): {"total", "failed", "new_failed"}}.
|
||||
scan_resource_groups_cache: Dict tracking resource group counts {(resource_group, severity): {"total", "failed", "new_failed"}}.
|
||||
group_resources_cache: Dict tracking unique resources per group {resource_group: set(resource_uids)}.
|
||||
"""
|
||||
# Accumulate objects for bulk operations
|
||||
findings_to_create = []
|
||||
@@ -499,6 +548,8 @@ def _process_finding_micro_batch(
|
||||
with rls_transaction(tenant_id):
|
||||
resource_uid = finding.resource_uid
|
||||
if resource_uid not in resource_cache:
|
||||
check_metadata = finding.get_metadata()
|
||||
group = check_metadata.get("resourcegroup") or None
|
||||
resource_instance, _ = Resource.objects.get_or_create(
|
||||
tenant_id=tenant_id,
|
||||
provider=provider_instance,
|
||||
@@ -508,6 +559,7 @@ def _process_finding_micro_batch(
|
||||
"service": finding.service_name,
|
||||
"type": finding.resource_type,
|
||||
"name": finding.resource_name,
|
||||
"groups": [group] if group else None,
|
||||
},
|
||||
)
|
||||
resource_cache[resource_uid] = resource_instance
|
||||
@@ -528,6 +580,8 @@ def _process_finding_micro_batch(
|
||||
|
||||
# Track resource field changes (defer save)
|
||||
updated = False
|
||||
check_metadata = finding.get_metadata()
|
||||
group = check_metadata.get("resourcegroup") or None
|
||||
if finding.region and resource_instance.region != finding.region:
|
||||
resource_instance.region = finding.region
|
||||
updated = True
|
||||
@@ -548,6 +602,11 @@ def _process_finding_micro_batch(
|
||||
if resource_instance.partition != finding.partition:
|
||||
resource_instance.partition = finding.partition
|
||||
updated = True
|
||||
if group and (
|
||||
not resource_instance.groups or group not in resource_instance.groups
|
||||
):
|
||||
resource_instance.groups = (resource_instance.groups or []) + [group]
|
||||
updated = True
|
||||
|
||||
if updated:
|
||||
dirty_resources[resource_uid] = resource_instance
|
||||
@@ -633,6 +692,7 @@ def _process_finding_micro_batch(
|
||||
muted_reason=muted_reason,
|
||||
compliance=finding.compliance,
|
||||
categories=check_metadata.get("categories", []) or [],
|
||||
resource_groups=check_metadata.get("resourcegroup") or None,
|
||||
)
|
||||
findings_to_create.append(finding_instance)
|
||||
resource_denormalized_data.append((finding_instance, resource_instance))
|
||||
@@ -657,6 +717,18 @@ def _process_finding_micro_batch(
|
||||
cache=scan_categories_cache,
|
||||
)
|
||||
|
||||
# Track resource groups with counts for ScanGroupSummary
|
||||
aggregate_resource_group_counts(
|
||||
resource_group=check_metadata.get("resourcegroup") or None,
|
||||
severity=finding.severity.value,
|
||||
status=status.value,
|
||||
delta=delta.value if delta else None,
|
||||
muted=is_muted,
|
||||
resource_uid=resource_instance.uid if resource_instance else "",
|
||||
cache=scan_resource_groups_cache,
|
||||
group_resources_cache=group_resources_cache,
|
||||
)
|
||||
|
||||
# Bulk operations within single transaction
|
||||
with rls_transaction(tenant_id):
|
||||
# Bulk create findings
|
||||
@@ -714,7 +786,15 @@ def _process_finding_micro_batch(
|
||||
tenant_id=tenant_id,
|
||||
model=Resource,
|
||||
objects=list(dirty_resources.values()),
|
||||
fields=["metadata", "details", "partition", "region", "service", "type"],
|
||||
fields=[
|
||||
"metadata",
|
||||
"details",
|
||||
"partition",
|
||||
"region",
|
||||
"service",
|
||||
"type",
|
||||
"groups",
|
||||
],
|
||||
batch_size=1000,
|
||||
)
|
||||
|
||||
@@ -757,6 +837,8 @@ def perform_prowler_scan(
|
||||
unique_resources = set()
|
||||
scan_resource_cache: set[tuple[str, str, str, str]] = set()
|
||||
scan_categories_cache: dict[tuple[str, str], dict[str, int]] = {}
|
||||
scan_resource_groups_cache: dict[tuple[str, str], dict[str, int]] = {}
|
||||
group_resources_cache: dict[str, set] = {}
|
||||
start_time = time.time()
|
||||
exc = None
|
||||
|
||||
@@ -847,6 +929,8 @@ def perform_prowler_scan(
|
||||
scan_resource_cache=scan_resource_cache,
|
||||
mute_rules_cache=mute_rules_cache,
|
||||
scan_categories_cache=scan_categories_cache,
|
||||
scan_resource_groups_cache=scan_resource_groups_cache,
|
||||
group_resources_cache=group_resources_cache,
|
||||
)
|
||||
|
||||
# Update scan progress
|
||||
@@ -933,6 +1017,38 @@ def perform_prowler_scan(
|
||||
sentry_sdk.capture_exception(cat_exception)
|
||||
logger.error(f"Error storing categories for scan {scan_id}: {cat_exception}")
|
||||
|
||||
try:
|
||||
if scan_resource_groups_cache:
|
||||
# Compute group-level resource counts (same value for all severity rows in a group)
|
||||
group_resource_counts = {
|
||||
grp: len(uids) for grp, uids in group_resources_cache.items()
|
||||
}
|
||||
resource_group_summaries = [
|
||||
ScanGroupSummary(
|
||||
tenant_id=tenant_id,
|
||||
scan_id=scan_id,
|
||||
resource_group=grp,
|
||||
severity=severity,
|
||||
total_findings=counts["total"],
|
||||
failed_findings=counts["failed"],
|
||||
new_failed_findings=counts["new_failed"],
|
||||
resources_count=group_resource_counts.get(grp, 0),
|
||||
)
|
||||
for (
|
||||
grp,
|
||||
severity,
|
||||
), counts in scan_resource_groups_cache.items()
|
||||
]
|
||||
with rls_transaction(tenant_id):
|
||||
ScanGroupSummary.objects.bulk_create(
|
||||
resource_group_summaries, batch_size=500, ignore_conflicts=True
|
||||
)
|
||||
except Exception as rg_exception:
|
||||
sentry_sdk.capture_exception(rg_exception)
|
||||
logger.error(
|
||||
f"Error storing resource groups for scan {scan_id}: {rg_exception}"
|
||||
)
|
||||
|
||||
serializer = ScanTaskSerializer(instance=scan_instance)
|
||||
return serializer.data
|
||||
|
||||
|
||||
@@ -131,9 +131,11 @@ def compute_threatscore_metrics(
|
||||
continue
|
||||
|
||||
m = metadata[0]
|
||||
risk_level = getattr(m, "LevelOfRisk", 0)
|
||||
weight = getattr(m, "Weight", 0)
|
||||
risk_level_raw = getattr(m, "LevelOfRisk", 0)
|
||||
weight_raw = getattr(m, "Weight", 0)
|
||||
section = getattr(m, "Section", "Unknown")
|
||||
risk_level = int(risk_level_raw) if risk_level_raw else 0
|
||||
weight = int(weight_raw) if weight_raw else 0
|
||||
|
||||
# Calculate ThreatScore components using formula from UI
|
||||
rate_i = req_passed_findings / req_total_findings
|
||||
|
||||
@@ -1,9 +1,6 @@
|
||||
from collections import defaultdict
|
||||
|
||||
from celery.utils.log import get_task_logger
|
||||
from config.django.base import DJANGO_FINDINGS_BATCH_SIZE
|
||||
from django.db.models import Count, Q
|
||||
from tasks.utils import batched
|
||||
|
||||
from api.db_router import READ_REPLICA_ALIAS
|
||||
from api.db_utils import rls_transaction
|
||||
@@ -154,6 +151,12 @@ def _load_findings_for_requirement_checks(
|
||||
Supports optional caching to avoid duplicate queries when generating multiple
|
||||
reports for the same scan.
|
||||
|
||||
Memory optimizations:
|
||||
- Uses database iterator with chunk_size for streaming large result sets
|
||||
- Shares references between cache and return dict (no duplication)
|
||||
- Only selects required fields from database
|
||||
- Processes findings in batches to reduce memory pressure
|
||||
|
||||
Args:
|
||||
tenant_id (str): The tenant ID for Row-Level Security context.
|
||||
scan_id (str): The ID of the scan to retrieve findings for.
|
||||
@@ -171,69 +174,73 @@ def _load_findings_for_requirement_checks(
|
||||
'aws_s3_bucket_public_access': [FindingOutput(...)]
|
||||
}
|
||||
"""
|
||||
findings_by_check_id = defaultdict(list)
|
||||
|
||||
if not check_ids:
|
||||
return dict(findings_by_check_id)
|
||||
return {}
|
||||
|
||||
# Initialize cache if not provided
|
||||
if findings_cache is None:
|
||||
findings_cache = {}
|
||||
|
||||
# Deduplicate check_ids to avoid redundant processing
|
||||
unique_check_ids = list(set(check_ids))
|
||||
|
||||
# Separate cached and non-cached check_ids
|
||||
check_ids_to_load = []
|
||||
cache_hits = 0
|
||||
cache_misses = 0
|
||||
|
||||
for check_id in check_ids:
|
||||
for check_id in unique_check_ids:
|
||||
if check_id in findings_cache:
|
||||
# Reuse from cache
|
||||
findings_by_check_id[check_id] = findings_cache[check_id]
|
||||
cache_hits += 1
|
||||
else:
|
||||
# Need to load from database
|
||||
check_ids_to_load.append(check_id)
|
||||
cache_misses += 1
|
||||
|
||||
if cache_hits > 0:
|
||||
total_checks = len(unique_check_ids)
|
||||
logger.info(
|
||||
f"Findings cache: {cache_hits} hits, {cache_misses} misses "
|
||||
f"({cache_hits / (cache_hits + cache_misses) * 100:.1f}% hit rate)"
|
||||
f"Findings cache: {cache_hits}/{total_checks} hits "
|
||||
f"({cache_hits / total_checks * 100:.1f}% hit rate)"
|
||||
)
|
||||
|
||||
# If all check_ids were in cache, return early
|
||||
if not check_ids_to_load:
|
||||
return dict(findings_by_check_id)
|
||||
|
||||
logger.info(f"Loading findings for {len(check_ids_to_load)} checks on-demand")
|
||||
|
||||
findings_queryset = (
|
||||
Finding.all_objects.filter(
|
||||
tenant_id=tenant_id, scan_id=scan_id, check_id__in=check_ids_to_load
|
||||
# Load missing check_ids from database
|
||||
if check_ids_to_load:
|
||||
logger.info(
|
||||
f"Loading findings for {len(check_ids_to_load)} checks from database"
|
||||
)
|
||||
.order_by("uid")
|
||||
.iterator()
|
||||
)
|
||||
|
||||
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
|
||||
for batch, is_last_batch in batched(
|
||||
findings_queryset, DJANGO_FINDINGS_BATCH_SIZE
|
||||
):
|
||||
for finding_model in batch:
|
||||
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
|
||||
# Use iterator with chunk_size for memory-efficient streaming
|
||||
# chunk_size controls how many rows Django fetches from DB at once
|
||||
findings_queryset = (
|
||||
Finding.all_objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
scan_id=scan_id,
|
||||
check_id__in=check_ids_to_load,
|
||||
)
|
||||
.order_by("check_id", "uid")
|
||||
.iterator(chunk_size=DJANGO_FINDINGS_BATCH_SIZE)
|
||||
)
|
||||
|
||||
# Pre-initialize empty lists for all check_ids to load
|
||||
# This avoids repeated dict lookups and 'if not in' checks
|
||||
for check_id in check_ids_to_load:
|
||||
findings_cache[check_id] = []
|
||||
|
||||
findings_count = 0
|
||||
for finding_model in findings_queryset:
|
||||
finding_output = FindingOutput.transform_api_finding(
|
||||
finding_model, prowler_provider
|
||||
)
|
||||
findings_by_check_id[finding_output.check_id].append(finding_output)
|
||||
# Update cache with newly loaded findings
|
||||
if finding_output.check_id not in findings_cache:
|
||||
findings_cache[finding_output.check_id] = []
|
||||
findings_cache[finding_output.check_id].append(finding_output)
|
||||
findings_count += 1
|
||||
|
||||
total_findings_loaded = sum(
|
||||
len(findings) for findings in findings_by_check_id.values()
|
||||
)
|
||||
logger.info(
|
||||
f"Loaded {total_findings_loaded} findings for {len(findings_by_check_id)} checks"
|
||||
)
|
||||
logger.info(
|
||||
f"Loaded {findings_count} findings for {len(check_ids_to_load)} checks"
|
||||
)
|
||||
|
||||
return dict(findings_by_check_id)
|
||||
# Build result dict using cache references (no data duplication)
|
||||
# This shares the same list objects between cache and result
|
||||
result = {
|
||||
check_id: findings_cache.get(check_id, []) for check_id in unique_check_ids
|
||||
}
|
||||
|
||||
return result
|
||||
|
||||
@@ -8,12 +8,17 @@ from celery.utils.log import get_task_logger
|
||||
from config.celery import RLSTask
|
||||
from config.django.base import DJANGO_FINDINGS_BATCH_SIZE, DJANGO_TMP_OUTPUT_DIRECTORY
|
||||
from django_celery_beat.models import PeriodicTask
|
||||
from tasks.jobs.attack_paths import (
|
||||
attack_paths_scan,
|
||||
can_provider_run_attack_paths_scan,
|
||||
)
|
||||
from tasks.jobs.backfill import (
|
||||
backfill_compliance_summaries,
|
||||
backfill_daily_severity_summaries,
|
||||
backfill_provider_compliance_scores,
|
||||
backfill_resource_scan_summaries,
|
||||
backfill_scan_category_summaries,
|
||||
backfill_scan_resource_group_summaries,
|
||||
)
|
||||
from tasks.jobs.connection import (
|
||||
check_integration_connection,
|
||||
@@ -47,7 +52,11 @@ from tasks.jobs.scan import (
|
||||
perform_prowler_scan,
|
||||
update_provider_compliance_scores,
|
||||
)
|
||||
from tasks.utils import batched, get_next_execution_datetime
|
||||
from tasks.utils import (
|
||||
_get_or_create_scheduled_scan,
|
||||
batched,
|
||||
get_next_execution_datetime,
|
||||
)
|
||||
|
||||
from api.compliance import get_compliance_frameworks
|
||||
from api.db_router import READ_REPLICA_ALIAS
|
||||
@@ -152,6 +161,11 @@ def _perform_scan_complete_tasks(tenant_id: str, scan_id: str, provider_id: str)
|
||||
),
|
||||
).apply_async()
|
||||
|
||||
if can_provider_run_attack_paths_scan(tenant_id, provider_id):
|
||||
perform_attack_paths_scan_task.apply_async(
|
||||
kwargs={"tenant_id": tenant_id, "scan_id": scan_id}
|
||||
)
|
||||
|
||||
|
||||
@shared_task(base=RLSTask, name="provider-connection-check")
|
||||
@set_tenant
|
||||
@@ -264,44 +278,38 @@ def perform_scheduled_scan_task(self, tenant_id: str, provider_id: str):
|
||||
periodic_task_instance = PeriodicTask.objects.get(
|
||||
name=f"scan-perform-scheduled-{provider_id}"
|
||||
)
|
||||
|
||||
executed_scan = Scan.objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
provider_id=provider_id,
|
||||
task__task_runner_task__task_id=task_id,
|
||||
).order_by("completed_at")
|
||||
|
||||
if (
|
||||
executing_scan = (
|
||||
Scan.objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
provider_id=provider_id,
|
||||
trigger=Scan.TriggerChoices.SCHEDULED,
|
||||
state=StateChoices.EXECUTING,
|
||||
scheduler_task_id=periodic_task_instance.id,
|
||||
scheduled_at__date=datetime.now(timezone.utc).date(),
|
||||
).exists()
|
||||
or executed_scan.exists()
|
||||
):
|
||||
# Duplicated task execution due to visibility timeout or scan is already running
|
||||
logger.warning(f"Duplicated scheduled scan for provider {provider_id}.")
|
||||
try:
|
||||
affected_scan = executed_scan.first()
|
||||
if not affected_scan:
|
||||
raise ValueError(
|
||||
"Error retrieving affected scan details after detecting duplicated scheduled "
|
||||
"scan."
|
||||
)
|
||||
# Return the affected scan details to avoid losing data
|
||||
serializer = ScanTaskSerializer(instance=affected_scan)
|
||||
except Exception as duplicated_scan_exception:
|
||||
logger.error(
|
||||
f"Duplicated scheduled scan for provider {provider_id}. Error retrieving affected scan details: "
|
||||
f"{str(duplicated_scan_exception)}"
|
||||
)
|
||||
raise duplicated_scan_exception
|
||||
return serializer.data
|
||||
)
|
||||
.order_by("-started_at")
|
||||
.first()
|
||||
)
|
||||
if executing_scan:
|
||||
logger.warning(
|
||||
f"Scheduled scan already executing for provider {provider_id}. Skipping."
|
||||
)
|
||||
return ScanTaskSerializer(instance=executing_scan).data
|
||||
|
||||
executed_scan = Scan.objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
provider_id=provider_id,
|
||||
task__task_runner_task__task_id=task_id,
|
||||
).first()
|
||||
|
||||
if executed_scan:
|
||||
# Duplicated task execution due to visibility timeout
|
||||
logger.warning(f"Duplicated scheduled scan for provider {provider_id}.")
|
||||
return ScanTaskSerializer(instance=executed_scan).data
|
||||
|
||||
interval = periodic_task_instance.interval
|
||||
next_scan_datetime = get_next_execution_datetime(task_id, provider_id)
|
||||
current_scan_datetime = next_scan_datetime - timedelta(
|
||||
**{interval.period: interval.every}
|
||||
)
|
||||
|
||||
# TEMPORARY WORKAROUND: Clean up orphan scans from transaction isolation issue
|
||||
_cleanup_orphan_scheduled_scans(
|
||||
@@ -310,19 +318,12 @@ def perform_scheduled_scan_task(self, tenant_id: str, provider_id: str):
|
||||
scheduler_task_id=periodic_task_instance.id,
|
||||
)
|
||||
|
||||
scan_instance, _ = Scan.objects.get_or_create(
|
||||
scan_instance = _get_or_create_scheduled_scan(
|
||||
tenant_id=tenant_id,
|
||||
provider_id=provider_id,
|
||||
trigger=Scan.TriggerChoices.SCHEDULED,
|
||||
state__in=(StateChoices.SCHEDULED, StateChoices.AVAILABLE),
|
||||
scheduler_task_id=periodic_task_instance.id,
|
||||
defaults={
|
||||
"state": StateChoices.SCHEDULED,
|
||||
"name": "Daily scheduled scan",
|
||||
"scheduled_at": next_scan_datetime - timedelta(days=1),
|
||||
},
|
||||
scheduled_at=current_scan_datetime,
|
||||
)
|
||||
|
||||
scan_instance.task_id = task_id
|
||||
scan_instance.save()
|
||||
|
||||
@@ -332,18 +333,19 @@ def perform_scheduled_scan_task(self, tenant_id: str, provider_id: str):
|
||||
scan_id=str(scan_instance.id),
|
||||
provider_id=provider_id,
|
||||
)
|
||||
except Exception as e:
|
||||
raise e
|
||||
finally:
|
||||
with rls_transaction(tenant_id):
|
||||
Scan.objects.get_or_create(
|
||||
now = datetime.now(timezone.utc)
|
||||
if next_scan_datetime <= now:
|
||||
interval_delta = timedelta(**{interval.period: interval.every})
|
||||
while next_scan_datetime <= now:
|
||||
next_scan_datetime += interval_delta
|
||||
_get_or_create_scheduled_scan(
|
||||
tenant_id=tenant_id,
|
||||
name="Daily scheduled scan",
|
||||
provider_id=provider_id,
|
||||
trigger=Scan.TriggerChoices.SCHEDULED,
|
||||
state=StateChoices.SCHEDULED,
|
||||
scheduled_at=next_scan_datetime,
|
||||
scheduler_task_id=periodic_task_instance.id,
|
||||
scheduled_at=next_scan_datetime,
|
||||
update_state=True,
|
||||
)
|
||||
|
||||
_perform_scan_complete_tasks(tenant_id, str(scan_instance.id), provider_id)
|
||||
@@ -357,6 +359,29 @@ def perform_scan_summary_task(tenant_id: str, scan_id: str):
|
||||
return aggregate_findings(tenant_id=tenant_id, scan_id=scan_id)
|
||||
|
||||
|
||||
@shared_task(
|
||||
base=RLSTask,
|
||||
bind=True,
|
||||
name="attack-paths-scan-perform",
|
||||
queue="attack-paths-scans",
|
||||
)
|
||||
def perform_attack_paths_scan_task(self, tenant_id: str, scan_id: str):
|
||||
"""
|
||||
Execute an Attack Paths scan for the given provider within the current tenant RLS context.
|
||||
|
||||
Args:
|
||||
self: The task instance (automatically passed when bind=True).
|
||||
tenant_id (str): The tenant identifier for RLS context.
|
||||
scan_id (str): The Prowler scan identifier for obtaining the tenant and provider context.
|
||||
|
||||
Returns:
|
||||
Any: The result from `attack_paths_scan`, including any per-scan failure details.
|
||||
"""
|
||||
return attack_paths_scan(
|
||||
tenant_id=tenant_id, scan_id=scan_id, task_id=self.request.id
|
||||
)
|
||||
|
||||
|
||||
@shared_task(name="tenant-deletion", queue="deletion", autoretry_for=(Exception,))
|
||||
def delete_tenant_task(tenant_id: str):
|
||||
return delete_tenant(pk=tenant_id)
|
||||
@@ -613,6 +638,21 @@ def backfill_scan_category_summaries_task(tenant_id: str, scan_id: str):
|
||||
return backfill_scan_category_summaries(tenant_id=tenant_id, scan_id=scan_id)
|
||||
|
||||
|
||||
@shared_task(name="backfill-scan-resource-group-summaries", queue="backfill")
|
||||
@handle_provider_deletion
|
||||
def backfill_scan_resource_group_summaries_task(tenant_id: str, scan_id: str):
|
||||
"""
|
||||
Backfill ScanGroupSummary for a completed scan.
|
||||
|
||||
Aggregates unique resource groups from findings and creates a summary row.
|
||||
|
||||
Args:
|
||||
tenant_id (str): The tenant identifier.
|
||||
scan_id (str): The scan identifier.
|
||||
"""
|
||||
return backfill_scan_resource_group_summaries(tenant_id=tenant_id, scan_id=scan_id)
|
||||
|
||||
|
||||
@shared_task(name="backfill-provider-compliance-scores", queue="backfill")
|
||||
def backfill_provider_compliance_scores_task(tenant_id: str):
|
||||
"""
|
||||
|
||||
@@ -0,0 +1,806 @@
|
||||
from contextlib import nullcontext
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, call, patch
|
||||
|
||||
import pytest
|
||||
from tasks.jobs.attack_paths import findings as findings_module
|
||||
from tasks.jobs.attack_paths import internet as internet_module
|
||||
from tasks.jobs.attack_paths.scan import run as attack_paths_run
|
||||
|
||||
from api.models import (
|
||||
AttackPathsScan,
|
||||
Finding,
|
||||
Provider,
|
||||
Resource,
|
||||
ResourceFindingMapping,
|
||||
Scan,
|
||||
StateChoices,
|
||||
StatusChoices,
|
||||
)
|
||||
from prowler.lib.check.models import Severity
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestAttackPathsRun:
|
||||
# Patching with decorators as we got a `SyntaxError: too many statically nested blocks` error if we use context managers
|
||||
@patch("tasks.jobs.attack_paths.scan.graph_database.drop_database")
|
||||
@patch(
|
||||
"tasks.jobs.attack_paths.scan.utils.call_within_event_loop",
|
||||
side_effect=lambda fn, *a, **kw: fn(*a, **kw),
|
||||
)
|
||||
@patch(
|
||||
"tasks.jobs.attack_paths.scan.db_utils.get_old_attack_paths_scans",
|
||||
return_value=[],
|
||||
)
|
||||
@patch("tasks.jobs.attack_paths.scan.db_utils.finish_attack_paths_scan")
|
||||
@patch("tasks.jobs.attack_paths.scan.db_utils.update_attack_paths_scan_progress")
|
||||
@patch("tasks.jobs.attack_paths.scan.db_utils.starting_attack_paths_scan")
|
||||
@patch("tasks.jobs.attack_paths.scan.sync.sync_graph")
|
||||
@patch("tasks.jobs.attack_paths.scan.graph_database.drop_subgraph")
|
||||
@patch("tasks.jobs.attack_paths.scan.sync.create_sync_indexes")
|
||||
@patch("tasks.jobs.attack_paths.scan.internet.analysis")
|
||||
@patch("tasks.jobs.attack_paths.scan.findings.analysis")
|
||||
@patch("tasks.jobs.attack_paths.scan.findings.create_findings_indexes")
|
||||
@patch("tasks.jobs.attack_paths.scan.cartography_ontology.run")
|
||||
@patch("tasks.jobs.attack_paths.scan.cartography_analysis.run")
|
||||
@patch("tasks.jobs.attack_paths.scan.cartography_create_indexes.run")
|
||||
@patch("tasks.jobs.attack_paths.scan.graph_database.clear_cache")
|
||||
@patch("tasks.jobs.attack_paths.scan.graph_database.create_database")
|
||||
@patch(
|
||||
"tasks.jobs.attack_paths.scan.graph_database.get_uri",
|
||||
return_value="bolt://neo4j",
|
||||
)
|
||||
@patch(
|
||||
"tasks.jobs.attack_paths.scan.initialize_prowler_provider",
|
||||
return_value=MagicMock(_enabled_regions=["us-east-1"]),
|
||||
)
|
||||
@patch(
|
||||
"tasks.jobs.attack_paths.scan.rls_transaction",
|
||||
new=lambda *args, **kwargs: nullcontext(),
|
||||
)
|
||||
def test_run_success_flow(
|
||||
self,
|
||||
mock_init_provider,
|
||||
mock_get_uri,
|
||||
mock_create_db,
|
||||
mock_clear_cache,
|
||||
mock_cartography_indexes,
|
||||
mock_cartography_analysis,
|
||||
mock_cartography_ontology,
|
||||
mock_findings_indexes,
|
||||
mock_findings_analysis,
|
||||
mock_internet_analysis,
|
||||
mock_sync_indexes,
|
||||
mock_drop_subgraph,
|
||||
mock_sync,
|
||||
mock_starting,
|
||||
mock_update_progress,
|
||||
mock_finish,
|
||||
mock_get_old_scans,
|
||||
mock_event_loop,
|
||||
mock_drop_db,
|
||||
tenants_fixture,
|
||||
providers_fixture,
|
||||
scans_fixture,
|
||||
):
|
||||
tenant = tenants_fixture[0]
|
||||
provider = providers_fixture[0]
|
||||
provider.provider = Provider.ProviderChoices.AWS
|
||||
provider.save()
|
||||
scan = scans_fixture[0]
|
||||
scan.provider = provider
|
||||
scan.save()
|
||||
|
||||
attack_paths_scan = AttackPathsScan.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider,
|
||||
scan=scan,
|
||||
state=StateChoices.SCHEDULED,
|
||||
)
|
||||
|
||||
mock_session = MagicMock()
|
||||
session_ctx = MagicMock()
|
||||
session_ctx.__enter__.return_value = mock_session
|
||||
session_ctx.__exit__.return_value = False
|
||||
ingestion_result = {"organizations": "warning"}
|
||||
ingestion_fn = MagicMock(return_value=ingestion_result)
|
||||
|
||||
with (
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.scan.graph_database.get_database_name",
|
||||
side_effect=["db-scan-id", "tenant-db"],
|
||||
) as mock_get_db_name,
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.scan.graph_database.get_session",
|
||||
return_value=session_ctx,
|
||||
) as mock_get_session,
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.scan.db_utils.retrieve_attack_paths_scan",
|
||||
return_value=attack_paths_scan,
|
||||
) as mock_retrieve_scan,
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.scan.get_cartography_ingestion_function",
|
||||
return_value=ingestion_fn,
|
||||
) as mock_get_ingestion,
|
||||
):
|
||||
result = attack_paths_run(str(tenant.id), str(scan.id), "task-123")
|
||||
|
||||
assert result == ingestion_result
|
||||
mock_retrieve_scan.assert_called_once_with(str(tenant.id), str(scan.id))
|
||||
mock_starting.assert_called_once()
|
||||
config = mock_starting.call_args[0][2]
|
||||
assert config.neo4j_database == "tenant-db"
|
||||
mock_get_db_name.assert_has_calls(
|
||||
[call(attack_paths_scan.id, temporary=True), call(provider.tenant_id)]
|
||||
)
|
||||
|
||||
mock_create_db.assert_has_calls([call("db-scan-id"), call("tenant-db")])
|
||||
mock_get_session.assert_has_calls([call("db-scan-id"), call("tenant-db")])
|
||||
assert mock_cartography_indexes.call_count == 2
|
||||
mock_findings_indexes.assert_has_calls([call(mock_session), call(mock_session)])
|
||||
mock_sync_indexes.assert_called_once_with(mock_session)
|
||||
# These use tmp_cartography_config (neo4j_database="db-scan-id")
|
||||
mock_cartography_analysis.assert_called_once()
|
||||
mock_cartography_ontology.assert_called_once()
|
||||
mock_internet_analysis.assert_called_once()
|
||||
mock_findings_analysis.assert_called_once()
|
||||
mock_drop_subgraph.assert_called_once_with(
|
||||
database="tenant-db",
|
||||
provider_id=str(provider.id),
|
||||
)
|
||||
mock_sync.assert_called_once_with(
|
||||
source_database="db-scan-id",
|
||||
target_database="tenant-db",
|
||||
provider_id=str(provider.id),
|
||||
)
|
||||
mock_get_ingestion.assert_called_once_with(provider.provider)
|
||||
mock_event_loop.assert_called_once()
|
||||
mock_update_progress.assert_any_call(attack_paths_scan, 1)
|
||||
mock_update_progress.assert_any_call(attack_paths_scan, 2)
|
||||
mock_update_progress.assert_any_call(attack_paths_scan, 95)
|
||||
mock_update_progress.assert_any_call(attack_paths_scan, 97)
|
||||
mock_update_progress.assert_any_call(attack_paths_scan, 98)
|
||||
mock_update_progress.assert_any_call(attack_paths_scan, 99)
|
||||
mock_finish.assert_called_once_with(
|
||||
attack_paths_scan, StateChoices.COMPLETED, ingestion_result
|
||||
)
|
||||
|
||||
def test_run_failure_marks_scan_failed(
|
||||
self, tenants_fixture, providers_fixture, scans_fixture
|
||||
):
|
||||
tenant = tenants_fixture[0]
|
||||
provider = providers_fixture[0]
|
||||
provider.provider = Provider.ProviderChoices.AWS
|
||||
provider.save()
|
||||
scan = scans_fixture[0]
|
||||
scan.provider = provider
|
||||
scan.save()
|
||||
|
||||
attack_paths_scan = AttackPathsScan.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider,
|
||||
scan=scan,
|
||||
state=StateChoices.SCHEDULED,
|
||||
)
|
||||
|
||||
mock_session = MagicMock()
|
||||
session_ctx = MagicMock()
|
||||
session_ctx.__enter__.return_value = mock_session
|
||||
session_ctx.__exit__.return_value = False
|
||||
ingestion_fn = MagicMock(side_effect=RuntimeError("ingestion boom"))
|
||||
|
||||
with (
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.scan.rls_transaction",
|
||||
new=lambda *args, **kwargs: nullcontext(),
|
||||
),
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.scan.initialize_prowler_provider",
|
||||
return_value=MagicMock(_enabled_regions=["us-east-1"]),
|
||||
),
|
||||
patch("tasks.jobs.attack_paths.scan.graph_database.get_uri"),
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.scan.graph_database.get_database_name",
|
||||
return_value="db-scan-id",
|
||||
),
|
||||
patch("tasks.jobs.attack_paths.scan.graph_database.create_database"),
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.scan.graph_database.get_session",
|
||||
return_value=session_ctx,
|
||||
),
|
||||
patch("tasks.jobs.attack_paths.scan.cartography_create_indexes.run"),
|
||||
patch("tasks.jobs.attack_paths.scan.cartography_analysis.run"),
|
||||
patch("tasks.jobs.attack_paths.scan.findings.create_findings_indexes"),
|
||||
patch("tasks.jobs.attack_paths.scan.internet.analysis"),
|
||||
patch("tasks.jobs.attack_paths.scan.findings.analysis"),
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.scan.db_utils.retrieve_attack_paths_scan",
|
||||
return_value=attack_paths_scan,
|
||||
),
|
||||
patch("tasks.jobs.attack_paths.scan.db_utils.starting_attack_paths_scan"),
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.scan.db_utils.update_attack_paths_scan_progress"
|
||||
),
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.scan.db_utils.finish_attack_paths_scan"
|
||||
) as mock_finish,
|
||||
patch("tasks.jobs.attack_paths.scan.graph_database.drop_database"),
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.scan.get_cartography_ingestion_function",
|
||||
return_value=ingestion_fn,
|
||||
),
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.scan.utils.call_within_event_loop",
|
||||
side_effect=lambda fn, *a, **kw: fn(*a, **kw),
|
||||
),
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.scan.utils.stringify_exception",
|
||||
return_value="Cartography failed: ingestion boom",
|
||||
),
|
||||
):
|
||||
with pytest.raises(RuntimeError, match="ingestion boom"):
|
||||
attack_paths_run(str(tenant.id), str(scan.id), "task-456")
|
||||
|
||||
failure_args = mock_finish.call_args[0]
|
||||
assert failure_args[0] is attack_paths_scan
|
||||
assert failure_args[1] == StateChoices.FAILED
|
||||
assert failure_args[2] == {
|
||||
"global_cartography_error": "Cartography failed: ingestion boom"
|
||||
}
|
||||
|
||||
def test_run_returns_early_for_unsupported_provider(self, tenants_fixture):
|
||||
tenant = tenants_fixture[0]
|
||||
provider = Provider.objects.create(
|
||||
provider=Provider.ProviderChoices.GCP,
|
||||
uid="gcp-account",
|
||||
alias="gcp",
|
||||
tenant_id=tenant.id,
|
||||
)
|
||||
scan = Scan.objects.create(
|
||||
name="GCP Scan",
|
||||
provider=provider,
|
||||
trigger=Scan.TriggerChoices.MANUAL,
|
||||
state=StateChoices.AVAILABLE,
|
||||
tenant_id=tenant.id,
|
||||
)
|
||||
|
||||
with (
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.scan.rls_transaction",
|
||||
new=lambda *args, **kwargs: nullcontext(),
|
||||
),
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.scan.initialize_prowler_provider",
|
||||
return_value=MagicMock(),
|
||||
),
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.scan.get_cartography_ingestion_function",
|
||||
return_value=None,
|
||||
) as mock_get_ingestion,
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.scan.db_utils.retrieve_attack_paths_scan"
|
||||
) as mock_retrieve,
|
||||
):
|
||||
mock_retrieve.return_value = None
|
||||
result = attack_paths_run(str(tenant.id), str(scan.id), "task-789")
|
||||
|
||||
assert result == {
|
||||
"global_error": "Provider gcp is not supported for Attack Paths scans"
|
||||
}
|
||||
mock_get_ingestion.assert_called_once_with(provider.provider)
|
||||
mock_retrieve.assert_called_once_with(str(tenant.id), str(scan.id))
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestAttackPathsFindingsHelpers:
|
||||
def test_create_findings_indexes_executes_all_statements(self):
|
||||
mock_session = MagicMock()
|
||||
with patch("tasks.jobs.attack_paths.indexes.run_write_query") as mock_run_write:
|
||||
findings_module.create_findings_indexes(mock_session)
|
||||
|
||||
from tasks.jobs.attack_paths.indexes import FINDINGS_INDEX_STATEMENTS
|
||||
|
||||
assert mock_run_write.call_count == len(FINDINGS_INDEX_STATEMENTS)
|
||||
mock_run_write.assert_has_calls(
|
||||
[call(mock_session, stmt) for stmt in FINDINGS_INDEX_STATEMENTS]
|
||||
)
|
||||
|
||||
def test_load_findings_batches_requests(self, providers_fixture):
|
||||
provider = providers_fixture[0]
|
||||
provider.provider = Provider.ProviderChoices.AWS
|
||||
provider.save()
|
||||
|
||||
# Create mock Finding objects with to_dict() method
|
||||
mock_finding_1 = MagicMock()
|
||||
mock_finding_1.to_dict.return_value = {"id": "1", "resource_uid": "r-1"}
|
||||
mock_finding_2 = MagicMock()
|
||||
mock_finding_2.to_dict.return_value = {"id": "2", "resource_uid": "r-2"}
|
||||
|
||||
# Create a generator that yields two batches of Finding instances
|
||||
def findings_generator():
|
||||
yield [mock_finding_1]
|
||||
yield [mock_finding_2]
|
||||
|
||||
config = SimpleNamespace(update_tag=12345)
|
||||
mock_session = MagicMock()
|
||||
|
||||
with (
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.findings.get_root_node_label",
|
||||
return_value="AWSAccount",
|
||||
),
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.findings.get_node_uid_field",
|
||||
return_value="arn",
|
||||
),
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.findings.get_provider_resource_label",
|
||||
return_value="AWSResource",
|
||||
),
|
||||
):
|
||||
findings_module.load_findings(
|
||||
mock_session, findings_generator(), provider, config
|
||||
)
|
||||
|
||||
assert mock_session.run.call_count == 2
|
||||
for call_args in mock_session.run.call_args_list:
|
||||
params = call_args.args[1]
|
||||
assert params["provider_uid"] == str(provider.uid)
|
||||
assert params["last_updated"] == config.update_tag
|
||||
assert "findings_data" in params
|
||||
|
||||
def test_cleanup_findings_runs_batches(self, providers_fixture):
|
||||
provider = providers_fixture[0]
|
||||
config = SimpleNamespace(update_tag=1024)
|
||||
mock_session = MagicMock()
|
||||
|
||||
first_batch = MagicMock()
|
||||
first_batch.single.return_value = {"deleted_findings_count": 3}
|
||||
second_batch = MagicMock()
|
||||
second_batch.single.return_value = {"deleted_findings_count": 0}
|
||||
mock_session.run.side_effect = [first_batch, second_batch]
|
||||
|
||||
findings_module.cleanup_findings(mock_session, provider, config)
|
||||
|
||||
assert mock_session.run.call_count == 2
|
||||
params = mock_session.run.call_args.args[1]
|
||||
assert params["provider_uid"] == str(provider.uid)
|
||||
assert params["last_updated"] == config.update_tag
|
||||
|
||||
def test_stream_findings_with_resources_returns_latest_scan_data(
|
||||
self,
|
||||
tenants_fixture,
|
||||
providers_fixture,
|
||||
):
|
||||
tenant = tenants_fixture[0]
|
||||
provider = providers_fixture[0]
|
||||
provider.provider = Provider.ProviderChoices.AWS
|
||||
provider.save()
|
||||
|
||||
resource = Resource.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider,
|
||||
uid="resource-uid",
|
||||
name="Resource",
|
||||
region="us-east-1",
|
||||
service="ec2",
|
||||
type="instance",
|
||||
)
|
||||
|
||||
older_scan = Scan.objects.create(
|
||||
name="Older",
|
||||
provider=provider,
|
||||
trigger=Scan.TriggerChoices.MANUAL,
|
||||
state=StateChoices.COMPLETED,
|
||||
tenant_id=tenant.id,
|
||||
)
|
||||
old_finding = Finding.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
uid="older-finding",
|
||||
scan=older_scan,
|
||||
delta=Finding.DeltaChoices.NEW,
|
||||
status=StatusChoices.PASS,
|
||||
status_extended="ok",
|
||||
severity=Severity.low,
|
||||
impact=Severity.low,
|
||||
impact_extended="",
|
||||
raw_result={},
|
||||
check_id="check-old",
|
||||
check_metadata={"checktitle": "Old"},
|
||||
first_seen_at=older_scan.inserted_at,
|
||||
)
|
||||
ResourceFindingMapping.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
resource=resource,
|
||||
finding=old_finding,
|
||||
)
|
||||
|
||||
latest_scan = Scan.objects.create(
|
||||
name="Latest",
|
||||
provider=provider,
|
||||
trigger=Scan.TriggerChoices.MANUAL,
|
||||
state=StateChoices.COMPLETED,
|
||||
tenant_id=tenant.id,
|
||||
)
|
||||
finding = Finding.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
uid="finding-uid",
|
||||
scan=latest_scan,
|
||||
delta=Finding.DeltaChoices.NEW,
|
||||
status=StatusChoices.FAIL,
|
||||
status_extended="failed",
|
||||
severity=Severity.high,
|
||||
impact=Severity.high,
|
||||
impact_extended="",
|
||||
raw_result={},
|
||||
check_id="check-1",
|
||||
check_metadata={"checktitle": "Check title"},
|
||||
first_seen_at=latest_scan.inserted_at,
|
||||
)
|
||||
ResourceFindingMapping.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
resource=resource,
|
||||
finding=finding,
|
||||
)
|
||||
|
||||
latest_scan.refresh_from_db()
|
||||
|
||||
with (
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.findings.rls_transaction",
|
||||
new=lambda *args, **kwargs: nullcontext(),
|
||||
),
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.findings.READ_REPLICA_ALIAS",
|
||||
"default",
|
||||
),
|
||||
):
|
||||
# Generator yields batches, collect all findings from all batches
|
||||
findings_batches = findings_module.stream_findings_with_resources(
|
||||
provider,
|
||||
str(latest_scan.id),
|
||||
)
|
||||
findings_data = []
|
||||
for batch in findings_batches:
|
||||
findings_data.extend(batch)
|
||||
|
||||
assert len(findings_data) == 1
|
||||
finding_result = findings_data[0]
|
||||
assert finding_result.id == str(finding.id)
|
||||
assert finding_result.resource_uid == resource.uid
|
||||
assert finding_result.check_title == "Check title"
|
||||
assert finding_result.scan_id == str(latest_scan.id)
|
||||
|
||||
def test_enrich_batch_with_resources_single_resource(
|
||||
self,
|
||||
tenants_fixture,
|
||||
providers_fixture,
|
||||
):
|
||||
"""One finding + one resource = one output Finding instance"""
|
||||
tenant = tenants_fixture[0]
|
||||
provider = providers_fixture[0]
|
||||
provider.provider = Provider.ProviderChoices.AWS
|
||||
provider.save()
|
||||
|
||||
resource = Resource.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider,
|
||||
uid="resource-uid-1",
|
||||
name="Resource 1",
|
||||
region="us-east-1",
|
||||
service="ec2",
|
||||
type="instance",
|
||||
)
|
||||
|
||||
scan = Scan.objects.create(
|
||||
name="Test Scan",
|
||||
provider=provider,
|
||||
trigger=Scan.TriggerChoices.MANUAL,
|
||||
state=StateChoices.COMPLETED,
|
||||
tenant_id=tenant.id,
|
||||
)
|
||||
|
||||
finding = Finding.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
uid="finding-uid",
|
||||
scan=scan,
|
||||
delta=Finding.DeltaChoices.NEW,
|
||||
status=StatusChoices.FAIL,
|
||||
status_extended="failed",
|
||||
severity=Severity.high,
|
||||
impact=Severity.high,
|
||||
impact_extended="",
|
||||
raw_result={},
|
||||
check_id="check-1",
|
||||
check_metadata={"checktitle": "Check title"},
|
||||
first_seen_at=scan.inserted_at,
|
||||
)
|
||||
ResourceFindingMapping.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
resource=resource,
|
||||
finding=finding,
|
||||
)
|
||||
|
||||
# Simulate the dict returned by .values()
|
||||
finding_dict = {
|
||||
"id": finding.id,
|
||||
"uid": finding.uid,
|
||||
"inserted_at": finding.inserted_at,
|
||||
"updated_at": finding.updated_at,
|
||||
"first_seen_at": finding.first_seen_at,
|
||||
"scan_id": scan.id,
|
||||
"delta": finding.delta,
|
||||
"status": finding.status,
|
||||
"status_extended": finding.status_extended,
|
||||
"severity": finding.severity,
|
||||
"check_id": finding.check_id,
|
||||
"check_metadata__checktitle": finding.check_metadata["checktitle"],
|
||||
"muted": finding.muted,
|
||||
"muted_reason": finding.muted_reason,
|
||||
}
|
||||
|
||||
# _enrich_batch_with_resources queries ResourceFindingMapping directly
|
||||
# No RLS mock needed - test DB doesn't enforce RLS policies
|
||||
with patch(
|
||||
"tasks.jobs.attack_paths.findings.READ_REPLICA_ALIAS",
|
||||
"default",
|
||||
):
|
||||
result = findings_module._enrich_batch_with_resources(
|
||||
[finding_dict], str(tenant.id)
|
||||
)
|
||||
|
||||
assert len(result) == 1
|
||||
assert result[0].resource_uid == resource.uid
|
||||
assert result[0].id == str(finding.id)
|
||||
assert result[0].status == "FAIL"
|
||||
|
||||
def test_enrich_batch_with_resources_multiple_resources(
|
||||
self,
|
||||
tenants_fixture,
|
||||
providers_fixture,
|
||||
):
|
||||
"""One finding + three resources = three output Finding instances"""
|
||||
tenant = tenants_fixture[0]
|
||||
provider = providers_fixture[0]
|
||||
provider.provider = Provider.ProviderChoices.AWS
|
||||
provider.save()
|
||||
|
||||
resources = []
|
||||
for i in range(3):
|
||||
resource = Resource.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider,
|
||||
uid=f"resource-uid-{i}",
|
||||
name=f"Resource {i}",
|
||||
region="us-east-1",
|
||||
service="ec2",
|
||||
type="instance",
|
||||
)
|
||||
resources.append(resource)
|
||||
|
||||
scan = Scan.objects.create(
|
||||
name="Test Scan",
|
||||
provider=provider,
|
||||
trigger=Scan.TriggerChoices.MANUAL,
|
||||
state=StateChoices.COMPLETED,
|
||||
tenant_id=tenant.id,
|
||||
)
|
||||
|
||||
finding = Finding.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
uid="finding-uid",
|
||||
scan=scan,
|
||||
delta=Finding.DeltaChoices.NEW,
|
||||
status=StatusChoices.FAIL,
|
||||
status_extended="failed",
|
||||
severity=Severity.high,
|
||||
impact=Severity.high,
|
||||
impact_extended="",
|
||||
raw_result={},
|
||||
check_id="check-1",
|
||||
check_metadata={"checktitle": "Check title"},
|
||||
first_seen_at=scan.inserted_at,
|
||||
)
|
||||
|
||||
# Map finding to all 3 resources
|
||||
for resource in resources:
|
||||
ResourceFindingMapping.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
resource=resource,
|
||||
finding=finding,
|
||||
)
|
||||
|
||||
finding_dict = {
|
||||
"id": finding.id,
|
||||
"uid": finding.uid,
|
||||
"inserted_at": finding.inserted_at,
|
||||
"updated_at": finding.updated_at,
|
||||
"first_seen_at": finding.first_seen_at,
|
||||
"scan_id": scan.id,
|
||||
"delta": finding.delta,
|
||||
"status": finding.status,
|
||||
"status_extended": finding.status_extended,
|
||||
"severity": finding.severity,
|
||||
"check_id": finding.check_id,
|
||||
"check_metadata__checktitle": finding.check_metadata["checktitle"],
|
||||
"muted": finding.muted,
|
||||
"muted_reason": finding.muted_reason,
|
||||
}
|
||||
|
||||
# _enrich_batch_with_resources queries ResourceFindingMapping directly
|
||||
# No RLS mock needed - test DB doesn't enforce RLS policies
|
||||
with patch(
|
||||
"tasks.jobs.attack_paths.findings.READ_REPLICA_ALIAS",
|
||||
"default",
|
||||
):
|
||||
result = findings_module._enrich_batch_with_resources(
|
||||
[finding_dict], str(tenant.id)
|
||||
)
|
||||
|
||||
assert len(result) == 3
|
||||
result_resource_uids = {r.resource_uid for r in result}
|
||||
assert result_resource_uids == {r.uid for r in resources}
|
||||
|
||||
# All should have same finding data
|
||||
for r in result:
|
||||
assert r.id == str(finding.id)
|
||||
assert r.status == "FAIL"
|
||||
|
||||
def test_enrich_batch_with_resources_no_resources_skips(
|
||||
self,
|
||||
tenants_fixture,
|
||||
providers_fixture,
|
||||
):
|
||||
"""Finding without resources should be skipped"""
|
||||
tenant = tenants_fixture[0]
|
||||
provider = providers_fixture[0]
|
||||
provider.provider = Provider.ProviderChoices.AWS
|
||||
provider.save()
|
||||
|
||||
scan = Scan.objects.create(
|
||||
name="Test Scan",
|
||||
provider=provider,
|
||||
trigger=Scan.TriggerChoices.MANUAL,
|
||||
state=StateChoices.COMPLETED,
|
||||
tenant_id=tenant.id,
|
||||
)
|
||||
|
||||
finding = Finding.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
uid="orphan-finding",
|
||||
scan=scan,
|
||||
delta=Finding.DeltaChoices.NEW,
|
||||
status=StatusChoices.FAIL,
|
||||
status_extended="failed",
|
||||
severity=Severity.high,
|
||||
impact=Severity.high,
|
||||
impact_extended="",
|
||||
raw_result={},
|
||||
check_id="check-1",
|
||||
check_metadata={"checktitle": "Check title"},
|
||||
first_seen_at=scan.inserted_at,
|
||||
)
|
||||
# Note: No ResourceFindingMapping created
|
||||
|
||||
finding_dict = {
|
||||
"id": finding.id,
|
||||
"uid": finding.uid,
|
||||
"inserted_at": finding.inserted_at,
|
||||
"updated_at": finding.updated_at,
|
||||
"first_seen_at": finding.first_seen_at,
|
||||
"scan_id": scan.id,
|
||||
"delta": finding.delta,
|
||||
"status": finding.status,
|
||||
"status_extended": finding.status_extended,
|
||||
"severity": finding.severity,
|
||||
"check_id": finding.check_id,
|
||||
"check_metadata__checktitle": finding.check_metadata["checktitle"],
|
||||
"muted": finding.muted,
|
||||
"muted_reason": finding.muted_reason,
|
||||
}
|
||||
|
||||
# Mock logger to verify no warning is emitted
|
||||
with (
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.findings.READ_REPLICA_ALIAS",
|
||||
"default",
|
||||
),
|
||||
patch("tasks.jobs.attack_paths.findings.logger") as mock_logger,
|
||||
):
|
||||
result = findings_module._enrich_batch_with_resources(
|
||||
[finding_dict], str(tenant.id)
|
||||
)
|
||||
|
||||
assert len(result) == 0
|
||||
mock_logger.warning.assert_not_called()
|
||||
|
||||
def test_generator_is_lazy(self, providers_fixture):
|
||||
"""Generator should not execute queries until iterated"""
|
||||
provider = providers_fixture[0]
|
||||
provider.provider = Provider.ProviderChoices.AWS
|
||||
provider.save()
|
||||
scan_id = "some-scan-id"
|
||||
|
||||
with (
|
||||
patch("tasks.jobs.attack_paths.findings.rls_transaction") as mock_rls,
|
||||
patch("tasks.jobs.attack_paths.findings.Finding") as mock_finding,
|
||||
):
|
||||
# Create generator but don't iterate
|
||||
findings_module.stream_findings_with_resources(provider, scan_id)
|
||||
|
||||
# Nothing should be called yet
|
||||
mock_rls.assert_not_called()
|
||||
mock_finding.objects.filter.assert_not_called()
|
||||
|
||||
def test_load_findings_empty_generator(self, providers_fixture):
|
||||
"""Empty generator should not call neo4j"""
|
||||
provider = providers_fixture[0]
|
||||
provider.provider = Provider.ProviderChoices.AWS
|
||||
provider.save()
|
||||
|
||||
mock_session = MagicMock()
|
||||
config = SimpleNamespace(update_tag=12345)
|
||||
|
||||
def empty_gen():
|
||||
return
|
||||
yield # Make it a generator
|
||||
|
||||
with (
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.findings.get_root_node_label",
|
||||
return_value="AWSAccount",
|
||||
),
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.findings.get_node_uid_field",
|
||||
return_value="arn",
|
||||
),
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.findings.get_provider_resource_label",
|
||||
return_value="AWSResource",
|
||||
),
|
||||
):
|
||||
findings_module.load_findings(mock_session, empty_gen(), provider, config)
|
||||
|
||||
mock_session.run.assert_not_called()
|
||||
|
||||
|
||||
class TestInternetAnalysis:
|
||||
def _make_provider_and_config(self):
|
||||
provider = MagicMock()
|
||||
provider.provider = "aws"
|
||||
provider.uid = "123456789012"
|
||||
config = SimpleNamespace(update_tag=1234567890)
|
||||
return provider, config
|
||||
|
||||
def test_analysis_creates_node_and_relationships(self):
|
||||
"""Verify both Cypher statements are executed and relationship count returned."""
|
||||
mock_session = MagicMock()
|
||||
mock_result = MagicMock()
|
||||
mock_result.single.return_value = {"relationships_merged": 3}
|
||||
mock_session.run.side_effect = [None, mock_result]
|
||||
provider, config = self._make_provider_and_config()
|
||||
|
||||
with patch(
|
||||
"tasks.jobs.attack_paths.internet.get_root_node_label",
|
||||
return_value="AWSAccount",
|
||||
):
|
||||
result = internet_module.analysis(mock_session, provider, config)
|
||||
|
||||
assert mock_session.run.call_count == 2
|
||||
assert result == 3
|
||||
|
||||
def test_analysis_zero_exposed_resources(self):
|
||||
"""When no resources are exposed, zero relationships are created."""
|
||||
mock_session = MagicMock()
|
||||
mock_result = MagicMock()
|
||||
mock_result.single.return_value = {"relationships_merged": 0}
|
||||
mock_session.run.side_effect = [None, mock_result]
|
||||
provider, config = self._make_provider_and_config()
|
||||
|
||||
with patch(
|
||||
"tasks.jobs.attack_paths.internet.get_root_node_label",
|
||||
return_value="AWSAccount",
|
||||
):
|
||||
result = internet_module.analysis(mock_session, provider, config)
|
||||
|
||||
assert result == 0
|
||||
@@ -8,6 +8,7 @@ from tasks.jobs.backfill import (
|
||||
backfill_provider_compliance_scores,
|
||||
backfill_resource_scan_summaries,
|
||||
backfill_scan_category_summaries,
|
||||
backfill_scan_resource_group_summaries,
|
||||
)
|
||||
|
||||
from api.models import (
|
||||
@@ -16,6 +17,7 @@ from api.models import (
|
||||
ResourceScanSummary,
|
||||
Scan,
|
||||
ScanCategorySummary,
|
||||
ScanGroupSummary,
|
||||
StateChoices,
|
||||
)
|
||||
from prowler.lib.check.models import Severity
|
||||
@@ -265,6 +267,94 @@ class TestBackfillScanCategorySummaries:
|
||||
assert summary.new_failed_findings == 1
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def findings_with_group_fixture(scans_fixture, resources_fixture):
|
||||
scan = scans_fixture[0]
|
||||
resource = resources_fixture[0]
|
||||
|
||||
finding = Finding.objects.create(
|
||||
tenant_id=scan.tenant_id,
|
||||
uid="finding_with_group",
|
||||
scan=scan,
|
||||
delta="new",
|
||||
status=Status.FAIL,
|
||||
status_extended="test status",
|
||||
impact=Severity.high,
|
||||
impact_extended="test impact",
|
||||
severity=Severity.high,
|
||||
raw_result={"status": Status.FAIL},
|
||||
check_id="test_check",
|
||||
check_metadata={"CheckId": "test_check"},
|
||||
resource_groups="ai_ml",
|
||||
first_seen_at="2024-01-02T00:00:00Z",
|
||||
)
|
||||
finding.add_resources([resource])
|
||||
return finding
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def scan_resource_group_summary_fixture(scans_fixture):
|
||||
scan = scans_fixture[0]
|
||||
return ScanGroupSummary.objects.create(
|
||||
tenant_id=scan.tenant_id,
|
||||
scan=scan,
|
||||
resource_group="existing-group",
|
||||
severity=Severity.high,
|
||||
total_findings=1,
|
||||
failed_findings=0,
|
||||
new_failed_findings=0,
|
||||
resources_count=1,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestBackfillScanGroupSummaries:
|
||||
def test_already_backfilled(self, scan_resource_group_summary_fixture):
|
||||
tenant_id = scan_resource_group_summary_fixture.tenant_id
|
||||
scan_id = scan_resource_group_summary_fixture.scan_id
|
||||
|
||||
result = backfill_scan_resource_group_summaries(str(tenant_id), str(scan_id))
|
||||
|
||||
assert result == {"status": "already backfilled"}
|
||||
|
||||
def test_not_completed_scan(self, get_not_completed_scans):
|
||||
for scan in get_not_completed_scans:
|
||||
result = backfill_scan_resource_group_summaries(
|
||||
str(scan.tenant_id), str(scan.id)
|
||||
)
|
||||
assert result == {"status": "scan is not completed"}
|
||||
|
||||
def test_no_resource_groups_to_backfill(self, scans_fixture):
|
||||
scan = scans_fixture[1] # Failed scan with no findings
|
||||
result = backfill_scan_resource_group_summaries(
|
||||
str(scan.tenant_id), str(scan.id)
|
||||
)
|
||||
assert result == {"status": "no resource groups to backfill"}
|
||||
|
||||
def test_successful_backfill(self, findings_with_group_fixture):
|
||||
finding = findings_with_group_fixture
|
||||
tenant_id = str(finding.tenant_id)
|
||||
scan_id = str(finding.scan_id)
|
||||
|
||||
result = backfill_scan_resource_group_summaries(tenant_id, scan_id)
|
||||
|
||||
# 1 resource group × 1 severity = 1 row
|
||||
assert result == {"status": "backfilled", "resource_groups_count": 1}
|
||||
|
||||
summaries = ScanGroupSummary.objects.filter(
|
||||
tenant_id=tenant_id, scan_id=scan_id
|
||||
)
|
||||
assert summaries.count() == 1
|
||||
|
||||
summary = summaries.first()
|
||||
assert summary.resource_group == "ai_ml"
|
||||
assert summary.severity == Severity.high
|
||||
assert summary.total_findings == 1
|
||||
assert summary.failed_findings == 1
|
||||
assert summary.new_failed_findings == 1
|
||||
assert summary.resources_count == 1
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestBackfillProviderComplianceScores:
|
||||
def test_no_completed_scans(self, tenants_fixture):
|
||||
|
||||
@@ -82,7 +82,7 @@ def test_check_provider_connection_exception(
|
||||
[
|
||||
{
|
||||
"name": "OpenAI",
|
||||
"api_key_decoded": "sk-test1234567890T3BlbkFJtest1234567890",
|
||||
"api_key_decoded": "sk-fake-test-key-for-unit-testing-only",
|
||||
"model": "gpt-4o",
|
||||
"temperature": 0,
|
||||
"max_tokens": 4000,
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user