diff --git a/.env b/.env index c761dd0fd0..f5c9802afe 100644 --- a/.env +++ b/.env @@ -145,7 +145,7 @@ SENTRY_RELEASE=local NEXT_PUBLIC_SENTRY_ENVIRONMENT=${SENTRY_ENVIRONMENT} #### Prowler release version #### -NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v5.24.0 +NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v5.25.0 # Social login credentials SOCIAL_GOOGLE_OAUTH_CALLBACK_URL="${AUTH_URL}/api/auth/callback/google" diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index b953610fa1..3300394d83 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,14 +1,15 @@ # SDK -/* @prowler-cloud/sdk -/prowler/ @prowler-cloud/sdk @prowler-cloud/detection-and-remediation -/tests/ @prowler-cloud/sdk @prowler-cloud/detection-and-remediation -/dashboard/ @prowler-cloud/sdk -/docs/ @prowler-cloud/sdk -/examples/ @prowler-cloud/sdk -/util/ @prowler-cloud/sdk -/contrib/ @prowler-cloud/sdk -/permissions/ @prowler-cloud/sdk -/codecov.yml @prowler-cloud/sdk @prowler-cloud/api +/* @prowler-cloud/detection-remediation +/prowler/ @prowler-cloud/detection-remediation +/prowler/compliance/ @prowler-cloud/compliance +/tests/ @prowler-cloud/detection-remediation +/dashboard/ @prowler-cloud/detection-remediation +/docs/ @prowler-cloud/detection-remediation +/examples/ @prowler-cloud/detection-remediation +/util/ @prowler-cloud/detection-remediation +/contrib/ @prowler-cloud/detection-remediation +/permissions/ @prowler-cloud/detection-remediation +/codecov.yml @prowler-cloud/detection-remediation @prowler-cloud/api # API /api/ @prowler-cloud/api @@ -17,7 +18,7 @@ /ui/ @prowler-cloud/ui # AI -/mcp_server/ @prowler-cloud/ai +/mcp_server/ @prowler-cloud/detection-remediation # Platform /.github/ @prowler-cloud/platform diff --git a/.github/actions/setup-python-poetry/action.yml b/.github/actions/setup-python-poetry/action.yml index fd96796b9b..dd5be4b3a2 100644 --- a/.github/actions/setup-python-poetry/action.yml +++ b/.github/actions/setup-python-poetry/action.yml @@ -64,19 +64,6 @@ runs: echo "Updated resolved_reference:" grep -A2 -B2 "resolved_reference" poetry.lock - - name: Update SDK resolved_reference to latest commit (prowler repo on push) - if: github.event_name == 'push' && github.ref == 'refs/heads/master' && github.repository == 'prowler-cloud/prowler' - shell: bash - working-directory: ${{ inputs.working-directory }} - run: | - LATEST_COMMIT=$(curl -s "https://api.github.com/repos/prowler-cloud/prowler/commits/master" | jq -r '.sha') - echo "Latest commit hash: $LATEST_COMMIT" - sed -i '/url = "https:\/\/github\.com\/prowler-cloud\/prowler\.git"/,/resolved_reference = / { - s/resolved_reference = "[a-f0-9]\{40\}"/resolved_reference = "'"$LATEST_COMMIT"'"/ - }' poetry.lock - echo "Updated resolved_reference:" - grep -A2 -B2 "resolved_reference" poetry.lock - - name: Update poetry.lock (prowler repo only) if: github.repository == 'prowler-cloud/prowler' && inputs.update-lock == 'true' shell: bash diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 28eff02ff6..953c0742f6 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -66,6 +66,18 @@ updates: cooldown: default-days: 7 + - package-ecosystem: "pre-commit" + directory: "/" + schedule: + interval: "monthly" + open-pull-requests-limit: 25 + target-branch: master + labels: + - "dependencies" + - "pre-commit" + cooldown: + default-days: 7 + # Dependabot Updates are temporary disabled - 2025/04/15 # v4.6 # - package-ecosystem: "pip" diff --git a/.github/workflows/find-secrets.yml b/.github/workflows/find-secrets.yml index 0aa955413f..88f84d6729 100644 --- a/.github/workflows/find-secrets.yml +++ b/.github/workflows/find-secrets.yml @@ -27,11 +27,12 @@ jobs: - name: Harden Runner uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 with: - egress-policy: block - allowed-endpoints: > - github.com:443 - ghcr.io:443 - pkg-containers.githubusercontent.com:443 + # We can't block as Trufflehog needs to verify secrets against vendors + egress-policy: audit + # allowed-endpoints: > + # github.com:443 + # ghcr.io:443 + # pkg-containers.githubusercontent.com:443 - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 diff --git a/.gitignore b/.gitignore index 8686f4f143..d959c0e524 100644 --- a/.gitignore +++ b/.gitignore @@ -84,6 +84,7 @@ continue.json .continuerc.json # AI Coding Assistants - OpenCode +.opencode/ opencode.json # AI Coding Assistants - GitHub Copilot diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 633c10a80f..c7d3f86f3b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ repos: ## GENERAL - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.6.0 + rev: v6.0.0 hooks: - id: check-merge-conflict - id: check-yaml @@ -16,7 +16,7 @@ repos: ## TOML - repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks - rev: v2.13.0 + rev: v2.16.0 hooks: - id: pretty-format-toml args: [--autofix] @@ -24,21 +24,21 @@ repos: ## GITHUB ACTIONS - repo: https://github.com/zizmorcore/zizmor-pre-commit - rev: v1.6.0 + rev: v1.24.1 hooks: - id: zizmor files: ^\.github/ ## BASH - repo: https://github.com/koalaman/shellcheck-precommit - rev: v0.10.0 + rev: v0.11.0 hooks: - id: shellcheck exclude: contrib ## PYTHON - repo: https://github.com/myint/autoflake - rev: v2.3.1 + rev: v2.3.3 hooks: - id: autoflake exclude: ^skills/ @@ -50,20 +50,20 @@ repos: ] - repo: https://github.com/pycqa/isort - rev: 5.13.2 + rev: 8.0.1 hooks: - id: isort exclude: ^skills/ args: ["--profile", "black"] - repo: https://github.com/psf/black - rev: 24.4.2 + rev: 26.3.1 hooks: - id: black exclude: ^skills/ - repo: https://github.com/pycqa/flake8 - rev: 7.0.0 + rev: 7.3.0 hooks: - id: flake8 exclude: (contrib|^skills/) @@ -93,7 +93,7 @@ repos: pass_filenames: false - repo: https://github.com/hadolint/hadolint - rev: v2.13.0-beta + rev: v2.14.0 hooks: - id: hadolint args: ["--ignore=DL3013"] diff --git a/api/CHANGELOG.md b/api/CHANGELOG.md index ff44ef24f0..a1f73e0700 100644 --- a/api/CHANGELOG.md +++ b/api/CHANGELOG.md @@ -2,7 +2,34 @@ All notable changes to the **Prowler API** are documented in this file. -## [1.25.0] (Prowler UNRELEASED) +## [1.25.2] (Prowler v5.24.2) + +### 🔄 Changed + +- Finding groups `/resources` endpoints now materialize the filtered finding IDs into a Python list before filtering `ResourceFindingMapping`, so PostgreSQL switches from a Merge Semi Join that read hundreds of thousands of RFM index entries to a Nested Loop Index Scan over `finding_id`. The `has_mappings.exists()` pre-check is removed, and a request-scoped cache deduplicates the finding-id round-trip across the helpers that build different RFM querysets [(#10816)](https://github.com/prowler-cloud/prowler/pull/10816) + +### 🐞 Fixed + +- `/finding-groups/latest//resources` now selects the latest completed scan per provider by `-completed_at` (then `-inserted_at`) instead of `-inserted_at`, matching the `/finding-groups/latest` summary path and the daily-summary upsert so overlapping scans no longer produce diverging `delta`/`new_count` between the two endpoints [(#10802)](https://github.com/prowler-cloud/prowler/pull/10802) + +--- + +## [1.25.1] (Prowler v5.24.1) + +### 🔄 Changed + +- Attack Paths: Restore `SYNC_BATCH_SIZE` and `FINDINGS_BATCH_SIZE` defaults to 1000, upgrade Cartography to 0.135.0, enable Celery queue priority for cleanup task, rewrite Finding insertion, remove AWS graph cleanup and add timing logs [(#10729)](https://github.com/prowler-cloud/prowler/pull/10729) + +### 🐞 Fixed + +- Finding group resources endpoints now include findings without associated resources (orphaned IaC findings) as simulated resource rows, and return one row per finding when multiple findings share a resource [(#10708)](https://github.com/prowler-cloud/prowler/pull/10708) +- Attack Paths: Missing `tenant_id` filter while getting related findings after scan completes [(#10722)](https://github.com/prowler-cloud/prowler/pull/10722) +- Finding group counters `pass_count`, `fail_count` and `manual_count` now exclude muted findings [(#10753)](https://github.com/prowler-cloud/prowler/pull/10753) +- Silent data loss in `ResourceFindingMapping` bulk insert that left findings orphaned when `INSERT ... ON CONFLICT DO NOTHING` dropped rows without raising; added explicit `unique_fields` [(#10724)](https://github.com/prowler-cloud/prowler/pull/10724) + +--- + +## [1.25.0] (Prowler v5.24.0) ### 🔄 Changed diff --git a/api/poetry.lock b/api/poetry.lock index 5e0ca3cb09..6bad57a758 100644 --- a/api/poetry.lock +++ b/api/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.3.4 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.2.1 and should not be changed by hand. [[package]] name = "about-time" @@ -682,21 +682,21 @@ requests = ">=2.21.0,<3.0.0" [[package]] name = "alibabacloud-tea-openapi" -version = "0.4.1" +version = "0.4.4" description = "Alibaba Cloud openapi SDK Library for Python" optional = false python-versions = ">=3.7" groups = ["main"] files = [ - {file = "alibabacloud_tea_openapi-0.4.1-py3-none-any.whl", hash = "sha256:e46bfa3ca34086d2c357d217a0b7284ecbd4b3bab5c88e075e73aec637b0e4a0"}, - {file = "alibabacloud_tea_openapi-0.4.1.tar.gz", hash = "sha256:2384b090870fdb089c3c40f3fb8cf0145b8c7d6c14abbac521f86a01abb5edaf"}, + {file = "alibabacloud_tea_openapi-0.4.4-py3-none-any.whl", hash = "sha256:cea6bc1fe35b0319a8752cb99eb0ecb0dab7ca1a71b99c12970ba0867410995f"}, + {file = "alibabacloud_tea_openapi-0.4.4.tar.gz", hash = "sha256:1b0917bc03cd49417da64945e92731716d53e2eb8707b235f54e45b7473221ce"}, ] [package.dependencies] alibabacloud-credentials = ">=1.0.2,<2.0.0" alibabacloud-gateway-spi = ">=0.0.2,<1.0.0" alibabacloud-tea-util = ">=0.3.13,<1.0.0" -cryptography = ">=3.0.0,<45.0.0" +cryptography = {version = ">=3.0.0,<47.0.0", markers = "python_version >= \"3.8\""} darabonba-core = ">=1.0.3,<2.0.0" [[package]] @@ -1526,19 +1526,19 @@ typing-extensions = ">=4.6.0" [[package]] name = "azure-mgmt-resource" -version = "23.3.0" +version = "24.0.0" description = "Microsoft Azure Resource Management Client Library for Python" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["main"] files = [ - {file = "azure_mgmt_resource-23.3.0-py3-none-any.whl", hash = "sha256:ab216ee28e29db6654b989746e0c85a1181f66653929d2cb6e48fba66d9af323"}, - {file = "azure_mgmt_resource-23.3.0.tar.gz", hash = "sha256:fc4f1fd8b6aad23f8af4ed1f913df5f5c92df117449dc354fea6802a2829fea4"}, + {file = "azure_mgmt_resource-24.0.0-py3-none-any.whl", hash = "sha256:27b32cd223e2784269f5a0db3c282042886ee4072d79cedc638438ece7cd0df4"}, + {file = "azure_mgmt_resource-24.0.0.tar.gz", hash = "sha256:cf6b8995fcdd407ac9ff1dd474087129429a1d90dbb1ac77f97c19b96237b265"}, ] [package.dependencies] azure-common = ">=1.1" -azure-mgmt-core = ">=1.3.2" +azure-mgmt-core = ">=1.5.0" isodate = ">=0.6.1" typing-extensions = ">=4.6.0" @@ -1822,19 +1822,19 @@ crt = ["awscrt (==0.27.6)"] [[package]] name = "cartography" -version = "0.132.0" +version = "0.135.0" description = "Explore assets and their relationships across your technical infrastructure." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "cartography-0.132.0-py3-none-any.whl", hash = "sha256:c070aa51d0ab4479cb043cae70b35e7df49f2fb5f1fa95ccf10000bbeb952262"}, - {file = "cartography-0.132.0.tar.gz", hash = "sha256:7c6332bc57fd2629d7b83aee7bd95a7b2edb0d51ef746efa0461399e0b66625c"}, + {file = "cartography-0.135.0-py3-none-any.whl", hash = "sha256:c62c32a6917b8f23a8b98fe2b6c7c4a918b50f55918482966c4dae1cf5f538e1"}, + {file = "cartography-0.135.0.tar.gz", hash = "sha256:3f500cd22c3b392d00e8b49f62acc95fd4dcd559ce514aafe2eb8101133c7a49"}, ] [package.dependencies] adal = ">=1.2.4" -aioboto3 = ">=13.0.0" +aioboto3 = ">=15.0.0" azure-cli-core = ">=2.26.0" azure-identity = ">=1.5.0" azure-keyvault-certificates = ">=4.0.0" @@ -1852,9 +1852,9 @@ azure-mgmt-keyvault = ">=10.0.0" azure-mgmt-logic = ">=10.0.0" azure-mgmt-monitor = ">=3.0.0" azure-mgmt-network = ">=25.0.0" -azure-mgmt-resource = ">=10.2.0,<25.0.0" +azure-mgmt-resource = ">=24.0.0,<25" azure-mgmt-security = ">=5.0.0" -azure-mgmt-sql = ">=3.0.1,<4" +azure-mgmt-sql = ">=3.0.1" azure-mgmt-storage = ">=16.0.0" azure-mgmt-synapse = ">=2.0.0" azure-mgmt-web = ">=7.0.0" @@ -1862,38 +1862,39 @@ azure-synapse-artifacts = ">=0.17.0" backoff = ">=2.1.2" boto3 = ">=1.15.1" botocore = ">=1.18.1" -cloudflare = ">=4.1.0,<5.0.0" +cloudflare = ">=4.1.0" crowdstrike-falconpy = ">=0.5.1" -cryptography = "*" -dnspython = ">=1.15.0" -duo-client = "*" -google-api-python-client = ">=1.7.8" +cryptography = ">=45.0.0" +dnspython = ">=2.0.0" +duo-client = ">=5.5.0" +google-api-python-client = ">=2.0.0" google-auth = ">=2.37.0" google-cloud-asset = ">=1.0.0" google-cloud-resource-manager = ">=1.14.2" httpx = ">=0.24.0" kubernetes = ">=22.6.0" -marshmallow = ">=3.0.0rc7" -msgraph-sdk = "*" +marshmallow = ">=4.0.0" +msgraph-sdk = ">=1.53.0" msrestazure = ">=0.6.4" neo4j = ">=6.0.0" oci = ">=2.71.0" okta = "<1.0.0" -packageurl-python = "*" -packaging = "*" +packageurl-python = ">=0.17.0" +packaging = ">=26.0.0" pagerduty = ">=4.0.1" policyuniverse = ">=1.1.0.0" PyJWT = {version = ">=2.0.0", extras = ["crypto"]} -python-dateutil = "*" +python-dateutil = ">=2.9.0" python-digitalocean = ">=1.16.0" pyyaml = ">=5.3.1" requests = ">=2.22.0" scaleway = ">=2.10.0" slack-sdk = ">=3.37.0" -statsd = "*" +statsd = ">=4.0.0" typer = ">=0.9.0" -types-aiobotocore-ecr = "*" -xmltodict = "*" +types-aiobotocore-ecr = ">=3.1.0" +workos = ">=5.44.0" +xmltodict = ">=1.0.0" [[package]] name = "celery" @@ -2503,62 +2504,74 @@ dev = ["bandit", "coverage", "flake8", "pydocstyle", "pylint", "pytest", "pytest [[package]] name = "cryptography" -version = "44.0.3" +version = "46.0.6" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." optional = false -python-versions = "!=3.9.0,!=3.9.1,>=3.7" +python-versions = "!=3.9.0,!=3.9.1,>=3.8" groups = ["main", "dev"] files = [ - {file = "cryptography-44.0.3-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:962bc30480a08d133e631e8dfd4783ab71cc9e33d5d7c1e192f0b7c06397bb88"}, - {file = "cryptography-44.0.3-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ffc61e8f3bf5b60346d89cd3d37231019c17a081208dfbbd6e1605ba03fa137"}, - {file = "cryptography-44.0.3-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58968d331425a6f9eedcee087f77fd3c927c88f55368f43ff7e0a19891f2642c"}, - {file = "cryptography-44.0.3-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:e28d62e59a4dbd1d22e747f57d4f00c459af22181f0b2f787ea83f5a876d7c76"}, - {file = "cryptography-44.0.3-cp37-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:af653022a0c25ef2e3ffb2c673a50e5a0d02fecc41608f4954176f1933b12359"}, - {file = "cryptography-44.0.3-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:157f1f3b8d941c2bd8f3ffee0af9b049c9665c39d3da9db2dc338feca5e98a43"}, - {file = "cryptography-44.0.3-cp37-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:c6cd67722619e4d55fdb42ead64ed8843d64638e9c07f4011163e46bc512cf01"}, - {file = "cryptography-44.0.3-cp37-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:b424563394c369a804ecbee9b06dfb34997f19d00b3518e39f83a5642618397d"}, - {file = "cryptography-44.0.3-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:c91fc8e8fd78af553f98bc7f2a1d8db977334e4eea302a4bfd75b9461c2d8904"}, - {file = "cryptography-44.0.3-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:25cd194c39fa5a0aa4169125ee27d1172097857b27109a45fadc59653ec06f44"}, - {file = "cryptography-44.0.3-cp37-abi3-win32.whl", hash = "sha256:3be3f649d91cb182c3a6bd336de8b61a0a71965bd13d1a04a0e15b39c3d5809d"}, - {file = "cryptography-44.0.3-cp37-abi3-win_amd64.whl", hash = "sha256:3883076d5c4cc56dbef0b898a74eb6992fdac29a7b9013870b34efe4ddb39a0d"}, - {file = "cryptography-44.0.3-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:5639c2b16764c6f76eedf722dbad9a0914960d3489c0cc38694ddf9464f1bb2f"}, - {file = "cryptography-44.0.3-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3ffef566ac88f75967d7abd852ed5f182da252d23fac11b4766da3957766759"}, - {file = "cryptography-44.0.3-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:192ed30fac1728f7587c6f4613c29c584abdc565d7417c13904708db10206645"}, - {file = "cryptography-44.0.3-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:7d5fe7195c27c32a64955740b949070f21cba664604291c298518d2e255931d2"}, - {file = "cryptography-44.0.3-cp39-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3f07943aa4d7dad689e3bb1638ddc4944cc5e0921e3c227486daae0e31a05e54"}, - {file = "cryptography-44.0.3-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:cb90f60e03d563ca2445099edf605c16ed1d5b15182d21831f58460c48bffb93"}, - {file = "cryptography-44.0.3-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:ab0b005721cc0039e885ac3503825661bd9810b15d4f374e473f8c89b7d5460c"}, - {file = "cryptography-44.0.3-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:3bb0847e6363c037df8f6ede57d88eaf3410ca2267fb12275370a76f85786a6f"}, - {file = "cryptography-44.0.3-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:b0cc66c74c797e1db750aaa842ad5b8b78e14805a9b5d1348dc603612d3e3ff5"}, - {file = "cryptography-44.0.3-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6866df152b581f9429020320e5eb9794c8780e90f7ccb021940d7f50ee00ae0b"}, - {file = "cryptography-44.0.3-cp39-abi3-win32.whl", hash = "sha256:c138abae3a12a94c75c10499f1cbae81294a6f983b3af066390adee73f433028"}, - {file = "cryptography-44.0.3-cp39-abi3-win_amd64.whl", hash = "sha256:5d186f32e52e66994dce4f766884bcb9c68b8da62d61d9d215bfe5fb56d21334"}, - {file = "cryptography-44.0.3-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:cad399780053fb383dc067475135e41c9fe7d901a97dd5d9c5dfb5611afc0d7d"}, - {file = "cryptography-44.0.3-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:21a83f6f35b9cc656d71b5de8d519f566df01e660ac2578805ab245ffd8523f8"}, - {file = "cryptography-44.0.3-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:fc3c9babc1e1faefd62704bb46a69f359a9819eb0292e40df3fb6e3574715cd4"}, - {file = "cryptography-44.0.3-pp310-pypy310_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:e909df4053064a97f1e6565153ff8bb389af12c5c8d29c343308760890560aff"}, - {file = "cryptography-44.0.3-pp310-pypy310_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:dad80b45c22e05b259e33ddd458e9e2ba099c86ccf4e88db7bbab4b747b18d06"}, - {file = "cryptography-44.0.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:479d92908277bed6e1a1c69b277734a7771c2b78633c224445b5c60a9f4bc1d9"}, - {file = "cryptography-44.0.3-pp311-pypy311_pp73-macosx_10_9_x86_64.whl", hash = "sha256:896530bc9107b226f265effa7ef3f21270f18a2026bc09fed1ebd7b66ddf6375"}, - {file = "cryptography-44.0.3-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:9b4d4a5dbee05a2c390bf212e78b99434efec37b17a4bff42f50285c5c8c9647"}, - {file = "cryptography-44.0.3-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:02f55fb4f8b79c1221b0961488eaae21015b69b210e18c386b69de182ebb1259"}, - {file = "cryptography-44.0.3-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:dd3db61b8fe5be220eee484a17233287d0be6932d056cf5738225b9c05ef4fff"}, - {file = "cryptography-44.0.3-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:978631ec51a6bbc0b7e58f23b68a8ce9e5f09721940933e9c217068388789fe5"}, - {file = "cryptography-44.0.3-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:5d20cc348cca3a8aa7312f42ab953a56e15323800ca3ab0706b8cd452a3a056c"}, - {file = "cryptography-44.0.3.tar.gz", hash = "sha256:fe19d8bc5536a91a24a8133328880a41831b6c5df54599a8417b62fe015d3053"}, + {file = "cryptography-46.0.6-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:64235194bad039a10bb6d2d930ab3323baaec67e2ce36215fd0952fad0930ca8"}, + {file = "cryptography-46.0.6-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:26031f1e5ca62fcb9d1fcb34b2b60b390d1aacaa15dc8b895a9ed00968b97b30"}, + {file = "cryptography-46.0.6-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9a693028b9cbe51b5a1136232ee8f2bc242e4e19d456ded3fa7c86e43c713b4a"}, + {file = "cryptography-46.0.6-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:67177e8a9f421aa2d3a170c3e56eca4e0128883cf52a071a7cbf53297f18b175"}, + {file = "cryptography-46.0.6-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:d9528b535a6c4f8ff37847144b8986a9a143585f0540fbcb1a98115b543aa463"}, + {file = "cryptography-46.0.6-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:22259338084d6ae497a19bae5d4c66b7ca1387d3264d1c2c0e72d9e9b6a77b97"}, + {file = "cryptography-46.0.6-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:760997a4b950ff00d418398ad73fbc91aa2894b5c1db7ccb45b4f68b42a63b3c"}, + {file = "cryptography-46.0.6-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:3dfa6567f2e9e4c5dceb8ccb5a708158a2a871052fa75c8b78cb0977063f1507"}, + {file = "cryptography-46.0.6-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:cdcd3edcbc5d55757e5f5f3d330dd00007ae463a7e7aa5bf132d1f22a4b62b19"}, + {file = "cryptography-46.0.6-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:d4e4aadb7fc1f88687f47ca20bb7227981b03afaae69287029da08096853b738"}, + {file = "cryptography-46.0.6-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2b417edbe8877cda9022dde3a008e2deb50be9c407eef034aeeb3a8b11d9db3c"}, + {file = "cryptography-46.0.6-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:380343e0653b1c9d7e1f55b52aaa2dbb2fdf2730088d48c43ca1c7c0abb7cc2f"}, + {file = "cryptography-46.0.6-cp311-abi3-win32.whl", hash = "sha256:bcb87663e1f7b075e48c3be3ecb5f0b46c8fc50b50a97cf264e7f60242dca3f2"}, + {file = "cryptography-46.0.6-cp311-abi3-win_amd64.whl", hash = "sha256:6739d56300662c468fddb0e5e291f9b4d084bead381667b9e654c7dd81705124"}, + {file = "cryptography-46.0.6-cp314-cp314t-macosx_10_9_universal2.whl", hash = "sha256:2ef9e69886cbb137c2aef9772c2e7138dc581fad4fcbcf13cc181eb5a3ab6275"}, + {file = "cryptography-46.0.6-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7f417f034f91dcec1cb6c5c35b07cdbb2ef262557f701b4ecd803ee8cefed4f4"}, + {file = "cryptography-46.0.6-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d24c13369e856b94892a89ddf70b332e0b70ad4a5c43cf3e9cb71d6d7ffa1f7b"}, + {file = "cryptography-46.0.6-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:aad75154a7ac9039936d50cf431719a2f8d4ed3d3c277ac03f3339ded1a5e707"}, + {file = "cryptography-46.0.6-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:3c21d92ed15e9cfc6eb64c1f5a0326db22ca9c2566ca46d845119b45b4400361"}, + {file = "cryptography-46.0.6-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:4668298aef7cddeaf5c6ecc244c2302a2b8e40f384255505c22875eebb47888b"}, + {file = "cryptography-46.0.6-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:8ce35b77aaf02f3b59c90b2c8a05c73bac12cea5b4e8f3fbece1f5fddea5f0ca"}, + {file = "cryptography-46.0.6-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:c89eb37fae9216985d8734c1afd172ba4927f5a05cfd9bf0e4863c6d5465b013"}, + {file = "cryptography-46.0.6-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:ed418c37d095aeddf5336898a132fba01091f0ac5844e3e8018506f014b6d2c4"}, + {file = "cryptography-46.0.6-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:69cf0056d6947edc6e6760e5f17afe4bea06b56a9ac8a06de9d2bd6b532d4f3a"}, + {file = "cryptography-46.0.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8e7304c4f4e9490e11efe56af6713983460ee0780f16c63f219984dab3af9d2d"}, + {file = "cryptography-46.0.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:b928a3ca837c77a10e81a814a693f2295200adb3352395fad024559b7be7a736"}, + {file = "cryptography-46.0.6-cp314-cp314t-win32.whl", hash = "sha256:97c8115b27e19e592a05c45d0dd89c57f81f841cc9880e353e0d3bf25b2139ed"}, + {file = "cryptography-46.0.6-cp314-cp314t-win_amd64.whl", hash = "sha256:c797e2517cb7880f8297e2c0f43bb910e91381339336f75d2c1c2cbf811b70b4"}, + {file = "cryptography-46.0.6-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:12cae594e9473bca1a7aceb90536060643128bb274fcea0fc459ab90f7d1ae7a"}, + {file = "cryptography-46.0.6-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:639301950939d844a9e1c4464d7e07f902fe9a7f6b215bb0d4f28584729935d8"}, + {file = "cryptography-46.0.6-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ed3775295fb91f70b4027aeba878d79b3e55c0b3e97eaa4de71f8f23a9f2eb77"}, + {file = "cryptography-46.0.6-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:8927ccfbe967c7df312ade694f987e7e9e22b2425976ddbf28271d7e58845290"}, + {file = "cryptography-46.0.6-cp38-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:b12c6b1e1651e42ab5de8b1e00dc3b6354fdfd778e7fa60541ddacc27cd21410"}, + {file = "cryptography-46.0.6-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:063b67749f338ca9c5a0b7fe438a52c25f9526b851e24e6c9310e7195aad3b4d"}, + {file = "cryptography-46.0.6-cp38-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:02fad249cb0e090b574e30b276a3da6a149e04ee2f049725b1f69e7b8351ec70"}, + {file = "cryptography-46.0.6-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:7e6142674f2a9291463e5e150090b95a8519b2fb6e6aaec8917dd8d094ce750d"}, + {file = "cryptography-46.0.6-cp38-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:456b3215172aeefb9284550b162801d62f5f264a081049a3e94307fe20792cfa"}, + {file = "cryptography-46.0.6-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:341359d6c9e68834e204ceaf25936dffeafea3829ab80e9503860dcc4f4dac58"}, + {file = "cryptography-46.0.6-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:9a9c42a2723999a710445bc0d974e345c32adfd8d2fac6d8a251fa829ad31cfb"}, + {file = "cryptography-46.0.6-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6617f67b1606dfd9fe4dbfa354a9508d4a6d37afe30306fe6c101b7ce3274b72"}, + {file = "cryptography-46.0.6-cp38-abi3-win32.whl", hash = "sha256:7f6690b6c55e9c5332c0b59b9c8a3fb232ebf059094c17f9019a51e9827df91c"}, + {file = "cryptography-46.0.6-cp38-abi3-win_amd64.whl", hash = "sha256:79e865c642cfc5c0b3eb12af83c35c5aeff4fa5c672dc28c43721c2c9fdd2f0f"}, + {file = "cryptography-46.0.6-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:2ea0f37e9a9cf0df2952893ad145fd9627d326a59daec9b0802480fa3bcd2ead"}, + {file = "cryptography-46.0.6-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:a3e84d5ec9ba01f8fd03802b2147ba77f0c8f2617b2aff254cedd551844209c8"}, + {file = "cryptography-46.0.6-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:12f0fa16cc247b13c43d56d7b35287ff1569b5b1f4c5e87e92cc4fcc00cd10c0"}, + {file = "cryptography-46.0.6-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:50575a76e2951fe7dbd1f56d181f8c5ceeeb075e9ff88e7ad997d2f42af06e7b"}, + {file = "cryptography-46.0.6-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:90e5f0a7b3be5f40c3a0a0eafb32c681d8d2c181fc2a1bdabe9b3f611d9f6b1a"}, + {file = "cryptography-46.0.6-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:6728c49e3b2c180ef26f8e9f0a883a2c585638db64cf265b49c9ba10652d430e"}, + {file = "cryptography-46.0.6.tar.gz", hash = "sha256:27550628a518c5c6c903d84f637fbecf287f6cb9ced3804838a1295dc1fd0759"}, ] [package.dependencies] -cffi = {version = ">=1.12", markers = "platform_python_implementation != \"PyPy\""} +cffi = {version = ">=2.0.0", markers = "python_full_version >= \"3.9.0\" and platform_python_implementation != \"PyPy\""} [package.extras] -docs = ["sphinx (>=5.3.0)", "sphinx-rtd-theme (>=3.0.0) ; python_version >= \"3.8\""] +docs = ["sphinx (>=5.3.0)", "sphinx-inline-tabs", "sphinx-rtd-theme (>=3.0.0)"] docstest = ["pyenchant (>=3)", "readme-renderer (>=30.0)", "sphinxcontrib-spelling (>=7.3.1)"] -nox = ["nox (>=2024.4.15)", "nox[uv] (>=2024.3.2) ; python_version >= \"3.8\""] -pep8test = ["check-sdist ; python_version >= \"3.8\"", "click (>=8.0.1)", "mypy (>=1.4)", "ruff (>=0.3.6)"] +nox = ["nox[uv] (>=2024.4.15)"] +pep8test = ["check-sdist", "click (>=8.0.1)", "mypy (>=1.14)", "ruff (>=0.11.11)"] sdist = ["build (>=1.0.0)"] ssh = ["bcrypt (>=3.1.5)"] -test = ["certifi (>=2024)", "cryptography-vectors (==44.0.3)", "pretend (>=0.7)", "pytest (>=7.4.0)", "pytest-benchmark (>=4.0)", "pytest-cov (>=2.10.1)", "pytest-xdist (>=3.5.0)"] +test = ["certifi (>=2024)", "cryptography-vectors (==46.0.6)", "pretend (>=0.7)", "pytest (>=7.4.0)", "pytest-benchmark (>=4.0)", "pytest-cov (>=2.10.1)", "pytest-xdist (>=3.5.0)"] test-randomorder = ["pytest-randomly"] [[package]] @@ -2961,7 +2974,7 @@ files = [ [package.dependencies] autopep8 = "*" Django = ">=4.2" -gprof2dot = ">=2017.9.19" +gprof2dot = ">=2017.09.19" sqlparse = "*" [[package]] @@ -3740,19 +3753,19 @@ urllib3 = ["packaging", "urllib3"] [[package]] name = "google-auth-httplib2" -version = "0.2.1" +version = "0.2.0" description = "Google Authentication Library: httplib2 transport" optional = false -python-versions = ">=3.7" +python-versions = "*" groups = ["main"] files = [ - {file = "google_auth_httplib2-0.2.1-py3-none-any.whl", hash = "sha256:1be94c611db91c01f9703e7f62b0a59bbd5587a95571c7b6fade510d648bc08b"}, - {file = "google_auth_httplib2-0.2.1.tar.gz", hash = "sha256:5ef03be3927423c87fb69607b42df23a444e434ddb2555b73b3679793187b7de"}, + {file = "google-auth-httplib2-0.2.0.tar.gz", hash = "sha256:38aa7badf48f974f1eb9861794e9c0cb2a0511a4ec0679b1f886d108f5640e05"}, + {file = "google_auth_httplib2-0.2.0-py2.py3-none-any.whl", hash = "sha256:b65a0a2123300dd71281a7bf6e64d65a0759287df52729bdd1ae2e47dc311a3d"}, ] [package.dependencies] -google-auth = ">=1.32.0,<3.0.0" -httplib2 = ">=0.19.0,<1.0.0" +google-auth = "*" +httplib2 = ">=0.19.0" [[package]] name = "google-cloud-access-context-manager" @@ -4569,7 +4582,7 @@ files = [ [package.dependencies] attrs = ">=22.2.0" -jsonschema-specifications = ">=2023.3.6" +jsonschema-specifications = ">=2023.03.6" referencing = ">=0.28.4" rpds-py = ">=0.7.1" @@ -4777,7 +4790,7 @@ librabbitmq = ["librabbitmq (>=2.0.0) ; python_version < \"3.11\""] mongodb = ["pymongo (==4.15.3)"] msgpack = ["msgpack (==1.1.2)"] pyro = ["pyro4 (==4.82)"] -qpid = ["qpid-python (==1.36.0.post1)", "qpid-tools (==1.36.0.post1)"] +qpid = ["qpid-python (==1.36.0-1)", "qpid-tools (==1.36.0-1)"] redis = ["redis (>=4.5.2,!=4.5.5,!=5.0.2,<6.5)"] slmq = ["softlayer_messaging (>=1.0.3)"] sqlalchemy = ["sqlalchemy (>=1.4.48,<2.1)"] @@ -4798,7 +4811,7 @@ files = [ ] [package.dependencies] -certifi = ">=14.5.14" +certifi = ">=14.05.14" durationpy = ">=0.7" google-auth = ">=1.0.1" oauthlib = ">=3.2.2" @@ -5181,24 +5194,16 @@ files = [ [[package]] name = "marshmallow" -version = "3.26.2" +version = "4.3.0" description = "A lightweight library for converting complex datatypes to and from native Python datatypes." optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main", "dev"] files = [ - {file = "marshmallow-3.26.2-py3-none-any.whl", hash = "sha256:013fa8a3c4c276c24d26d84ce934dc964e2aa794345a0f8c7e5a7191482c8a73"}, - {file = "marshmallow-3.26.2.tar.gz", hash = "sha256:bbe2adb5a03e6e3571b573f42527c6fe926e17467833660bebd11593ab8dfd57"}, + {file = "marshmallow-4.3.0-py3-none-any.whl", hash = "sha256:46c4fe6984707e3cbd485dfebbf0a59874f58d695aad05c1668d15e8c6e13b46"}, + {file = "marshmallow-4.3.0.tar.gz", hash = "sha256:fb43c53b3fe240b8f6af37223d6ef1636f927ad9bea8ab323afad95dff090880"}, ] -[package.dependencies] -packaging = ">=17.0" - -[package.extras] -dev = ["marshmallow[tests]", "pre-commit (>=3.5,<5.0)", "tox"] -docs = ["autodocsumm (==0.2.14)", "furo (==2024.8.6)", "sphinx (==8.1.3)", "sphinx-copybutton (==0.5.2)", "sphinx-issues (==5.0.0)", "sphinxext-opengraph (==0.9.1)"] -tests = ["pytest", "simplejson"] - [[package]] name = "matplotlib" version = "3.10.8" @@ -5492,14 +5497,14 @@ dev = ["bumpver", "isort", "mypy", "pylint", "pytest", "yapf"] [[package]] name = "msgraph-sdk" -version = "1.23.0" +version = "1.55.0" description = "The Microsoft Graph Python SDK" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "msgraph_sdk-1.23.0-py3-none-any.whl", hash = "sha256:58e0047b4ca59fd82022c02cd73fec0170a3d84f3b76721e3db2a0314df9a58a"}, - {file = "msgraph_sdk-1.23.0.tar.gz", hash = "sha256:6dd1ba9a46f5f0ce8599fd9610133adbd9d1493941438b5d3632fce9e55ed607"}, + {file = "msgraph_sdk-1.55.0-py3-none-any.whl", hash = "sha256:c8e68ebc4b88af5111de312e7fa910a4e76ddf48a4534feadb1fb8a411c48cfc"}, + {file = "msgraph_sdk-1.55.0.tar.gz", hash = "sha256:6df691a31954a050d26b8a678968017e157d940fb377f2a8a4e17a9741b98756"}, ] [package.dependencies] @@ -5925,23 +5930,24 @@ signedtoken = ["cryptography (>=3.0.0)", "pyjwt (>=2.0.0,<3)"] [[package]] name = "oci" -version = "2.160.3" +version = "2.169.0" description = "Oracle Cloud Infrastructure Python SDK" optional = false python-versions = "*" groups = ["main"] files = [ - {file = "oci-2.160.3-py3-none-any.whl", hash = "sha256:858bff3e697098bdda44833d2476bfb4632126f0182178e7dbde4dbd156d71f0"}, - {file = "oci-2.160.3.tar.gz", hash = "sha256:57514889be3b713a8385d86e3ba8a33cf46e3563c2a7e29a93027fb30b8a2537"}, + {file = "oci-2.169.0-py3-none-any.whl", hash = "sha256:c71bb5143f307791082b3e33cc1545c2490a518cfed85ab1948ef5107c36d30b"}, + {file = "oci-2.169.0.tar.gz", hash = "sha256:f3c5fff00b01783b5325ea7b13bf140053ec1e9f41da20bfb9c8a349ee7662fa"}, ] [package.dependencies] certifi = "*" circuitbreaker = {version = ">=1.3.1,<3.0.0", markers = "python_version >= \"3.7\""} -cryptography = ">=3.2.1,<46.0.0" -pyOpenSSL = ">=17.5.0,<25.0.0" +cryptography = ">=3.2.1,<47.0.0" +pyOpenSSL = ">=17.5.0,<27.0.0" python-dateutil = ">=2.5.3,<3.0.0" pytz = ">=2016.10" +urllib3 = {version = ">=2.6.3", markers = "python_version >= \"3.10.0\""} [package.extras] adk = ["docstring-parser (>=0.16) ; python_version >= \"3.10\" and python_version < \"4\"", "mcp (>=1.6.0) ; python_version >= \"3.10\" and python_version < \"4\"", "pydantic (>=2.10.6) ; python_version >= \"3.10\" and python_version < \"4\"", "rich (>=13.9.4) ; python_version >= \"3.10\" and python_version < \"4\""] @@ -6659,7 +6665,7 @@ files = [ [[package]] name = "prowler" -version = "5.23.0" +version = "5.25.0" description = "Prowler is an Open Source security tool to perform AWS, GCP and Azure security best practices assessments, audits, incident response, continuous monitoring, hardening and forensics readiness. It contains hundreds of controls covering CIS, NIST 800, NIST CSF, CISA, RBI, FedRAMP, PCI-DSS, GDPR, HIPAA, FFIEC, SOC2, GXP, AWS Well-Architected Framework Security Pillar, AWS Foundational Technical Review (FTR), ENS (Spanish National Security Scheme) and your custom security frameworks." optional = false python-versions = ">=3.10,<3.13" @@ -6679,7 +6685,7 @@ alibabacloud-rds20140815 = "12.0.0" alibabacloud_sas20181203 = "6.1.0" alibabacloud-sls20201230 = "5.9.0" alibabacloud_sts20150401 = "1.1.6" -alibabacloud_tea_openapi = "0.4.1" +alibabacloud_tea_openapi = "0.4.4" alibabacloud_vpc20160428 = "6.13.0" alive-progress = "3.3.0" awsipranges = "0.3.3" @@ -6701,7 +6707,7 @@ azure-mgmt-postgresqlflexibleservers = "1.1.0" azure-mgmt-rdbms = "10.1.0" azure-mgmt-recoveryservices = "3.1.0" azure-mgmt-recoveryservicesbackup = "9.2.0" -azure-mgmt-resource = "23.3.0" +azure-mgmt-resource = "24.0.0" azure-mgmt-search = "9.1.0" azure-mgmt-security = "7.0.0" azure-mgmt-sql = "3.0.1" @@ -6714,29 +6720,29 @@ boto3 = "1.40.61" botocore = "1.40.61" cloudflare = "4.3.1" colorama = "0.4.6" -cryptography = "44.0.3" +cryptography = "46.0.6" dash = "3.1.1" dash-bootstrap-components = "2.0.3" -defusedxml = ">=0.7.1" +defusedxml = "0.7.1" detect-secrets = "1.5.0" dulwich = "0.23.0" google-api-python-client = "2.163.0" -google-auth-httplib2 = ">=0.1,<0.3" +google-auth-httplib2 = "0.2.0" h2 = "4.3.0" jsonschema = "4.23.0" kubernetes = "32.0.1" markdown = "3.10.2" microsoft-kiota-abstractions = "1.9.2" -msgraph-sdk = "1.23.0" +msgraph-sdk = "1.55.0" numpy = "2.0.2" -oci = "2.160.3" +oci = "2.169.0" openstacksdk = "4.2.0" pandas = "2.2.3" py-iam-expand = "0.1.0" py-ocsf-models = "0.8.1" -pydantic = ">=2.0,<3.0" +pydantic = "2.12.5" pygithub = "2.8.0" -python-dateutil = ">=2.9.0.post0,<3.0.0" +python-dateutil = "2.9.0.post0" pytz = "2025.1" schema = "0.7.5" shodan = "1.31.0" @@ -6749,7 +6755,7 @@ uuid6 = "2024.7.10" type = "git" url = "https://github.com/prowler-cloud/prowler.git" reference = "master" -resolved_reference = "6ac90eb1b58590b6f2f51645dbef17b9231053f4" +resolved_reference = "ca29e354b622198ff6a70e2ea5eb04e4a44a0903" [[package]] name = "psutil" @@ -6914,14 +6920,14 @@ pydantic = ">=2.12.0,<3.0.0" [[package]] name = "pyasn1" -version = "0.6.2" +version = "0.6.3" description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "pyasn1-0.6.2-py3-none-any.whl", hash = "sha256:1eb26d860996a18e9b6ed05e7aae0e9fc21619fcee6af91cca9bad4fbea224bf"}, - {file = "pyasn1-0.6.2.tar.gz", hash = "sha256:9b59a2b25ba7e4f8197db7686c09fb33e658b98339fadb826e9512629017833b"}, + {file = "pyasn1-0.6.3-py3-none-any.whl", hash = "sha256:a80184d120f0864a52a073acc6fc642847d0be408e7c7252f31390c0f4eadcde"}, + {file = "pyasn1-0.6.3.tar.gz", hash = "sha256:697a8ecd6d98891189184ca1fa05d1bb00e2f84b5977c481452050549c8a72cf"}, ] [[package]] @@ -6958,11 +6964,11 @@ description = "C parser in Python" optional = false python-versions = ">=3.10" groups = ["main", "dev"] -markers = "platform_python_implementation != \"PyPy\" and implementation_name != \"PyPy\"" files = [ {file = "pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992"}, {file = "pycparser-3.0.tar.gz", hash = "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29"}, ] +markers = {main = "implementation_name != \"PyPy\" and platform_python_implementation != \"PyPy\"", dev = "platform_python_implementation != \"PyPy\" and implementation_name != \"PyPy\""} [[package]] name = "pydantic" @@ -7188,7 +7194,7 @@ files = [ ] [package.dependencies] -astroid = ">=3.2.2,<=3.3.0.dev0" +astroid = ">=3.2.2,<=3.3.0-dev0" colorama = {version = ">=0.4.5", markers = "sys_platform == \"win32\""} dill = [ {version = ">=0.3.7", markers = "python_version >= \"3.12\""}, @@ -7288,18 +7294,19 @@ tests = ["hypothesis (>=3.27.0)", "pytest (>=7.4.0)", "pytest-cov (>=2.10.1)", " [[package]] name = "pyopenssl" -version = "24.3.0" +version = "26.0.0" description = "Python wrapper module around the OpenSSL library" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" groups = ["main"] files = [ - {file = "pyOpenSSL-24.3.0-py3-none-any.whl", hash = "sha256:e474f5a473cd7f92221cc04976e48f4d11502804657a08a989fb3be5514c904a"}, - {file = "pyopenssl-24.3.0.tar.gz", hash = "sha256:49f7a019577d834746bc55c5fce6ecbcec0f2b4ec5ce1cf43a9a173b8138bb36"}, + {file = "pyopenssl-26.0.0-py3-none-any.whl", hash = "sha256:df94d28498848b98cc1c0ffb8ef1e71e40210d3b0a8064c9d29571ed2904bf81"}, + {file = "pyopenssl-26.0.0.tar.gz", hash = "sha256:f293934e52936f2e3413b89c6ce36df66a0b34ae1ea3a053b8c5020ff2f513fc"}, ] [package.dependencies] -cryptography = ">=41.0.5,<45" +cryptography = ">=46.0.0,<47" +typing-extensions = {version = ">=4.9", markers = "python_version < \"3.13\" and python_version >= \"3.8\""} [package.extras] docs = ["sphinx (!=5.2.0,!=5.2.0.post0,!=7.2.5)", "sphinx_rtd_theme"] @@ -8202,10 +8209,10 @@ files = [ ] [package.dependencies] -botocore = ">=1.37.4,<2.0a0" +botocore = ">=1.37.4,<2.0a.0" [package.extras] -crt = ["botocore[crt] (>=1.37.4,<2.0a0)"] +crt = ["botocore[crt] (>=1.37.4,<2.0a.0)"] [[package]] name = "safety" @@ -8807,6 +8814,23 @@ markupsafe = ">=2.1.1" [package.extras] watchdog = ["watchdog (>=2.3)"] +[[package]] +name = "workos" +version = "6.0.4" +description = "WorkOS Python Client" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "workos-6.0.4-py3-none-any.whl", hash = "sha256:548668b3702673536f853ba72a7b5bbbc269e467aaf9ac4f477b6e0177df5e21"}, + {file = "workos-6.0.4.tar.gz", hash = "sha256:b0bfe8fd212b8567422c4ea3732eb33608794033eb3a69900c6b04db183c32d6"}, +] + +[package.dependencies] +cryptography = ">=46.0,<47.0" +httpx = ">=0.28,<1.0" +pyjwt = ">=2.12,<3.0" + [[package]] name = "wrapt" version = "1.17.3" @@ -9400,4 +9424,4 @@ files = [ [metadata] lock-version = "2.1" python-versions = ">=3.11,<3.13" -content-hash = "077e89853cfe3a6d934841488cfa5a98ff6c92b71f74b817b71387d11559f143" +content-hash = "a3ab982d11a87d951ff15694d2ca7fd51f1f51a451abb0baa067ccf6966367a8" diff --git a/api/pyproject.toml b/api/pyproject.toml index 19827f2480..398936d62c 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -38,7 +38,7 @@ dependencies = [ "matplotlib (==3.10.8)", "reportlab (==4.4.10)", "neo4j (==6.1.0)", - "cartography (==0.132.0)", + "cartography (==0.135.0)", "gevent (==25.9.1)", "werkzeug (==3.1.7)", "sqlparse (==0.5.5)", @@ -50,7 +50,7 @@ name = "prowler-api" package-mode = false # Needed for the SDK compatibility requires-python = ">=3.11,<3.13" -version = "1.25.0" +version = "1.26.0" [project.scripts] celery = "src.backend.config.settings.celery" @@ -62,7 +62,6 @@ django-silk = "5.3.2" docker = "7.1.0" filelock = "3.20.3" freezegun = "1.5.1" -marshmallow = "==3.26.2" mypy = "1.10.1" pylint = "3.2.5" pytest = "9.0.3" diff --git a/api/src/backend/api/migrations/0090_attack_paths_cleanup_priority.py b/api/src/backend/api/migrations/0090_attack_paths_cleanup_priority.py new file mode 100644 index 0000000000..5ef8529b08 --- /dev/null +++ b/api/src/backend/api/migrations/0090_attack_paths_cleanup_priority.py @@ -0,0 +1,23 @@ +from django.db import migrations + +TASK_NAME = "attack-paths-cleanup-stale-scans" + + +def set_cleanup_priority(apps, schema_editor): + PeriodicTask = apps.get_model("django_celery_beat", "PeriodicTask") + PeriodicTask.objects.filter(name=TASK_NAME).update(priority=0) + + +def unset_cleanup_priority(apps, schema_editor): + PeriodicTask = apps.get_model("django_celery_beat", "PeriodicTask") + PeriodicTask.objects.filter(name=TASK_NAME).update(priority=None) + + +class Migration(migrations.Migration): + dependencies = [ + ("api", "0089_backfill_finding_group_status_muted"), + ] + + operations = [ + migrations.RunPython(set_cleanup_priority, unset_cleanup_priority), + ] diff --git a/api/src/backend/api/specs/v1.yaml b/api/src/backend/api/specs/v1.yaml index 816790de07..c7725e61df 100644 --- a/api/src/backend/api/specs/v1.yaml +++ b/api/src/backend/api/specs/v1.yaml @@ -1,7 +1,7 @@ openapi: 3.0.3 info: title: Prowler API - version: 1.25.0 + version: 1.26.0 description: |- Prowler API specification. diff --git a/api/src/backend/api/tests/test_views.py b/api/src/backend/api/tests/test_views.py index 2249a041ab..3d2107a03e 100644 --- a/api/src/backend/api/tests/test_views.py +++ b/api/src/backend/api/tests/test_views.py @@ -57,6 +57,7 @@ from api.models import ( ProviderGroupMembership, ProviderSecret, Resource, + ResourceFindingMapping, Role, RoleProviderGroupRelationship, SAMLConfiguration, @@ -15465,7 +15466,7 @@ class TestFindingGroupViewSet: attrs = data[0]["attributes"] assert attrs["status"] == "FAIL" assert attrs["muted"] is True - assert attrs["fail_count"] == 2 + assert attrs["fail_count"] == 0 assert attrs["fail_muted_count"] == 2 assert attrs["pass_muted_count"] == 0 assert attrs["manual_muted_count"] == 0 @@ -16030,6 +16031,36 @@ class TestFindingGroupViewSet: # s3_bucket_public_access has 2 findings with 2 different resources assert len(data) == 2 + def test_resources_id_matches_resource_id_for_mapped_findings( + self, authenticated_client, finding_groups_fixture + ): + """Findings with a resource expose the resource id as row id (hot path contract).""" + response = authenticated_client.get( + reverse( + "finding-group-resources", kwargs={"pk": "s3_bucket_public_access"} + ), + {"filter[inserted_at]": TODAY}, + ) + + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert data, "expected resources in response" + + resource_ids = set( + ResourceFindingMapping.objects.filter( + finding__check_id="s3_bucket_public_access", + ).values_list("resource_id", flat=True) + ) + finding_ids = set( + Finding.objects.filter( + check_id="s3_bucket_public_access", + ).values_list("id", flat=True) + ) + + returned_ids = {item["id"] for item in data} + assert returned_ids <= {str(rid) for rid in resource_ids} + assert returned_ids.isdisjoint({str(fid) for fid in finding_ids}) + def test_resources_fields(self, authenticated_client, finding_groups_fixture): """Test resource fields (uid, name, service, region, type) have valid values.""" response = authenticated_client.get( @@ -17240,3 +17271,111 @@ class TestFindingGroupViewSet: attrs = item["attributes"] assert "finding_id" in attrs assert attrs["finding_id"] in rds_finding_ids + + def test_latest_resources_picks_scan_by_completed_at_when_overlap( + self, + authenticated_client, + tenants_fixture, + providers_fixture, + resources_fixture, + ): + """Overlapping scans on the same provider must resolve to the scan + with the latest completed_at, matching the /latest summary path and + the daily-summary upsert (keyed on midnight(completed_at)). Picking + by inserted_at here produced /resources and /latest reading from + different scans and reporting diverging delta/new counts. + """ + tenant = tenants_fixture[0] + provider = providers_fixture[0] + resource = resources_fixture[0] + check_id = "overlap_regression_check" + + t0 = datetime.now(timezone.utc) - timedelta(hours=5) + t1 = t0 + timedelta(hours=1) + t1_end = t1 + timedelta(minutes=30) + t2 = t0 + timedelta(hours=4) + + scan_long = Scan.objects.create( + name="long overlap scan", + provider=provider, + trigger=Scan.TriggerChoices.MANUAL, + state=StateChoices.COMPLETED, + tenant_id=tenant.id, + started_at=t0, + completed_at=t2, + ) + scan_short = Scan.objects.create( + name="short overlap scan", + provider=provider, + trigger=Scan.TriggerChoices.MANUAL, + state=StateChoices.COMPLETED, + tenant_id=tenant.id, + started_at=t1, + completed_at=t1_end, + ) + # inserted_at is auto_now_add so override with .update() to recreate + # the overlap shape: short scan inserted later but completed earlier. + Scan.all_objects.filter(pk=scan_long.pk).update(inserted_at=t0) + Scan.all_objects.filter(pk=scan_short.pk).update(inserted_at=t1) + scan_long.refresh_from_db() + scan_short.refresh_from_db() + + assert scan_short.inserted_at > scan_long.inserted_at + assert scan_long.completed_at > scan_short.completed_at + + long_finding = Finding.objects.create( + tenant_id=tenant.id, + uid=f"{check_id}_long", + scan=scan_long, + delta=None, + status=Status.FAIL, + status_extended="long scan finding", + impact=Severity.high, + impact_extended="high", + severity=Severity.high, + raw_result={"status": Status.FAIL, "severity": Severity.high}, + check_id=check_id, + check_metadata={ + "CheckId": check_id, + "checktitle": "Overlap regression", + "Description": "Overlapping scan regression.", + }, + first_seen_at=t0, + muted=False, + ) + long_finding.add_resources([resource]) + + short_finding = Finding.objects.create( + tenant_id=tenant.id, + uid=f"{check_id}_short", + scan=scan_short, + delta="new", + status=Status.FAIL, + status_extended="short scan finding", + impact=Severity.high, + impact_extended="high", + severity=Severity.high, + raw_result={"status": Status.FAIL, "severity": Severity.high}, + check_id=check_id, + check_metadata={ + "CheckId": check_id, + "checktitle": "Overlap regression", + "Description": "Overlapping scan regression.", + }, + first_seen_at=t1, + muted=False, + ) + short_finding.add_resources([resource]) + + response = authenticated_client.get( + reverse( + "finding-group-latest_resources", + kwargs={"check_id": check_id}, + ), + ) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == 1 + attrs = data[0]["attributes"] + assert attrs["finding_id"] == str(long_finding.id) + assert attrs["delta"] is None diff --git a/api/src/backend/api/v1/serializers.py b/api/src/backend/api/v1/serializers.py index 52ec47f4f5..51cb95fbff 100644 --- a/api/src/backend/api/v1/serializers.py +++ b/api/src/backend/api/v1/serializers.py @@ -4225,10 +4225,11 @@ class FindingGroupResourceSerializer(BaseSerializerV1): Serializer for Finding Group Resources - resources within a finding group. Returns individual resources with their current status, severity, - and timing information. + and timing information. Orphan findings (without any resource) expose the + finding id as `id` so the row stays identifiable in the UI. """ - id = serializers.UUIDField(source="resource_id") + id = serializers.UUIDField(source="row_id") resource = serializers.SerializerMethodField() provider = serializers.SerializerMethodField() finding_id = serializers.UUIDField() diff --git a/api/src/backend/api/v1/views.py b/api/src/backend/api/v1/views.py index 0d085349cc..5a2c790655 100644 --- a/api/src/backend/api/v1/views.py +++ b/api/src/backend/api/v1/views.py @@ -35,11 +35,13 @@ from django.db.models import ( CharField, Count, DecimalField, + Exists, ExpressionWrapper, F, IntegerField, Max, Min, + OuterRef, Prefetch, Q, QuerySet, @@ -415,7 +417,7 @@ class SchemaView(SpectacularAPIView): def get(self, request, *args, **kwargs): spectacular_settings.TITLE = "Prowler API" - spectacular_settings.VERSION = "1.25.0" + spectacular_settings.VERSION = "1.26.0" spectacular_settings.DESCRIPTION = ( "Prowler API specification.\n\nThis file is auto-generated." ) @@ -7125,17 +7127,16 @@ class FindingGroupViewSet(BaseRLSViewSet): output_field=IntegerField(), ) - # `pass_count`, `fail_count` and `manual_count` count *every* finding - # for the check (muted or not) so the aggregated `status` reflects the - # underlying check outcome regardless of mute state. Whether the group - # is actionable is signalled by the orthogonal `muted` flag below. + # `pass_count`, `fail_count` and `manual_count` only count non-muted + # findings. Muted findings are tracked separately via the + # `*_muted_count` fields. return ( queryset.values("check_id") .annotate( severity_order=Max(severity_case), - pass_count=Count("id", filter=Q(status="PASS")), - fail_count=Count("id", filter=Q(status="FAIL")), - manual_count=Count("id", filter=Q(status="MANUAL")), + pass_count=Count("id", filter=Q(status="PASS", muted=False)), + fail_count=Count("id", filter=Q(status="FAIL", muted=False)), + manual_count=Count("id", filter=Q(status="MANUAL", muted=False)), pass_muted_count=Count("id", filter=Q(status="PASS", muted=True)), fail_muted_count=Count("id", filter=Q(status="FAIL", muted=True)), manual_muted_count=Count("id", filter=Q(status="MANUAL", muted=True)), @@ -7280,12 +7281,14 @@ class FindingGroupViewSet(BaseRLSViewSet): # finding-level aggregation path. row.pop("nonmuted_count", None) - # Compute aggregated status. Counts are inclusive of muted findings, - # so the underlying check outcome surfaces even when the group is - # fully muted. - if row.get("fail_count", 0) > 0: + # Compute aggregated status from non-muted counts first, then + # fall back to muted counts so fully-muted groups still reflect + # the underlying check outcome. + total_fail = row.get("fail_count", 0) + row.get("fail_muted_count", 0) + total_pass = row.get("pass_count", 0) + row.get("pass_muted_count", 0) + if total_fail > 0: row["status"] = "FAIL" - elif row.get("pass_count", 0) > 0: + elif total_pass > 0: row["status"] = "PASS" else: row["status"] = "MANUAL" @@ -7385,9 +7388,12 @@ class FindingGroupViewSet(BaseRLSViewSet): if computed_params.get("status") or computed_params.getlist("status__in"): queryset = queryset.annotate( + total_fail=F("fail_count") + F("fail_muted_count"), + total_pass=F("pass_count") + F("pass_muted_count"), + ).annotate( aggregated_status=Case( - When(fail_count__gt=0, then=Value("FAIL")), - When(pass_count__gt=0, then=Value("PASS")), + When(total_fail__gt=0, then=Value("FAIL")), + When(total_pass__gt=0, then=Value("PASS")), default=Value("MANUAL"), output_field=CharField(), ) @@ -7407,6 +7413,25 @@ class FindingGroupViewSet(BaseRLSViewSet): return filterset.qs + def _resolve_finding_ids(self, filtered_queryset): + """ + Materialize and request-cache the finding_ids list used to anchor + RFM lookups. + + Turning `finding_id__in=Subquery(findings_qs)` into `finding_id__in= + [uuid, ...]` nudges PostgreSQL out of a Merge Semi Join that ends up + reading hundreds of thousands of RFM index entries just to post- + filter tenant_id. Caching on the ViewSet instance (one instance per + request) avoids duplicating the findings round-trip when several + helpers build different RFM querysets from the same filtered set. + """ + cached = getattr(self, "_finding_ids_cache", None) + if cached is not None and cached[0] is filtered_queryset: + return cached[1] + finding_ids = list(filtered_queryset.order_by().values_list("id", flat=True)) + self._finding_ids_cache = (filtered_queryset, finding_ids) + return finding_ids + def _build_resource_mapping_queryset( self, filtered_queryset, resource_ids=None, tenant_id: str | None = None ): @@ -7416,10 +7441,10 @@ class FindingGroupViewSet(BaseRLSViewSet): Starting from ResourceFindingMapping avoids scanning all mappings before applying check_id/date filters on findings. """ - finding_ids = filtered_queryset.order_by().values("id") + finding_ids = self._resolve_finding_ids(filtered_queryset) mapping_queryset = ResourceFindingMapping.objects.filter( - finding_id__in=Subquery(finding_ids) + finding_id__in=finding_ids ) if tenant_id: mapping_queryset = mapping_queryset.filter(tenant_id=tenant_id) @@ -7578,6 +7603,53 @@ class FindingGroupViewSet(BaseRLSViewSet): .order_by(*ordering) ) + def _orphan_findings_queryset(self, filtered_queryset, finding_ids=None): + """Findings in the filtered set with no ResourceFindingMapping entries.""" + orphan_qs = filtered_queryset.filter( + ~Exists(ResourceFindingMapping.objects.filter(finding_id=OuterRef("pk"))) + ) + if finding_ids is not None: + orphan_qs = orphan_qs.filter(id__in=finding_ids) + return orphan_qs + + def _has_orphan_findings(self, filtered_queryset) -> bool: + """Return True if any finding in the filtered set has no resource mapping.""" + return self._orphan_findings_queryset(filtered_queryset).exists() + + def _orphan_aggregation_values(self, orphan_queryset): + """Raw rows for orphan findings; resource payload synthesized from metadata. + + check_metadata is stored with lowercase keys (see + `prowler.lib.outputs.finding.Finding.get_metadata`) and + `Finding.resource_groups` is already denormalized at ingest time. + """ + return orphan_queryset.annotate( + _provider_type=F("scan__provider__provider"), + _provider_uid=F("scan__provider__uid"), + _provider_alias=F("scan__provider__alias"), + _svc=KeyTextTransform("servicename", "check_metadata"), + _region=KeyTextTransform("region", "check_metadata"), + _rtype=KeyTextTransform("resourcetype", "check_metadata"), + _rgroup=F("resource_groups"), + ).values( + "id", + "uid", + "status", + "severity", + "delta", + "muted", + "muted_reason", + "first_seen_at", + "inserted_at", + "_provider_type", + "_provider_uid", + "_provider_alias", + "_svc", + "_region", + "_rtype", + "_rgroup", + ) + def _post_process_resources(self, resource_data): """Convert resource aggregation rows to API output.""" results = [] @@ -7599,9 +7671,13 @@ class FindingGroupViewSet(BaseRLSViewSet): else: delta = None + resource_id = row["resource_id"] + finding_id = str(row["finding_id"]) if row.get("finding_id") else None + results.append( { - "resource_id": row["resource_id"], + "row_id": resource_id, + "resource_id": resource_id, "resource_uid": row["resource_uid"], "resource_name": row["resource_name"], "resource_service": row["resource_service"], @@ -7620,9 +7696,46 @@ class FindingGroupViewSet(BaseRLSViewSet): "muted": bool(row.get("muted", False)), "muted_reason": row.get("muted_reason"), "resource_group": row.get("resource_group", ""), - "finding_id": ( - str(row["finding_id"]) if row.get("finding_id") else None - ), + "finding_id": finding_id, + } + ) + + return results + + def _post_process_orphans(self, orphan_rows): + """Convert orphan finding rows into the same API shape as mapping rows.""" + results = [] + for row in orphan_rows: + status_val = row["status"] + status = status_val if status_val in ("FAIL", "PASS") else "MANUAL" + + muted = bool(row["muted"]) + delta_val = row.get("delta") + delta = delta_val if delta_val in ("new", "changed") and not muted else None + + finding_id = str(row["id"]) + + results.append( + { + "row_id": finding_id, + "resource_id": None, + "resource_uid": row["uid"], + "resource_name": row["uid"], + "resource_service": row["_svc"] or "", + "resource_region": row["_region"] or "", + "resource_type": row["_rtype"] or "", + "provider_type": row["_provider_type"], + "provider_uid": row["_provider_uid"], + "provider_alias": row["_provider_alias"], + "status": status, + "severity": row["severity"], + "delta": delta, + "first_seen_at": row["first_seen_at"], + "last_seen_at": row["inserted_at"], + "muted": muted, + "muted_reason": row.get("muted_reason"), + "resource_group": row["_rgroup"] or "", + "finding_id": finding_id, } ) @@ -7683,16 +7796,14 @@ class FindingGroupViewSet(BaseRLSViewSet): sort_param, self._FINDING_GROUP_SORT_MAP ) if ordering: - # status_order is annotated on demand so groups can be sorted by - # their aggregated status (FAIL > PASS > MANUAL), mirroring the - # priority used in _post_process_aggregation. Counts are - # inclusive of muted findings, so the underlying check outcome - # surfaces even for fully muted groups. if any(field.lstrip("-") == "status_order" for field in ordering): aggregated_queryset = aggregated_queryset.annotate( + total_fail_for_sort=F("fail_count") + F("fail_muted_count"), + total_pass_for_sort=F("pass_count") + F("pass_muted_count"), + ).annotate( status_order=Case( - When(fail_count__gt=0, then=Value(3)), - When(pass_count__gt=0, then=Value(2)), + When(total_fail_for_sort__gt=0, then=Value(3)), + When(total_pass_for_sort__gt=0, then=Value(2)), default=Value(1), output_field=IntegerField(), ) @@ -7731,41 +7842,65 @@ class FindingGroupViewSet(BaseRLSViewSet): def _paginated_resource_response( self, request, filtered_queryset, resource_ids, tenant_id ): - """Paginate and return resources. + """Paginate and return resources, appending orphan findings when present. - Without sort: paginate lightweight resource IDs first, aggregate only the page. - With sort: build a lightweight ordering subquery (resource_id + sort keys), - paginate that, then aggregate full details only for the page. + Hot path (no orphans, or resource filter applied): resources come from + ResourceFindingMapping aggregation. Untouched pre-existing behaviour. + + Orphan fallback: findings without a mapping (e.g. IaC) are appended + after mapping rows as synthesised resource-like rows so they remain + visible in the UI without paying the aggregation cost on the hot path. """ sort_param = request.query_params.get("sort") - + ordering = None if sort_param: - ordering = self._validate_sort_fields(sort_param, self._RESOURCE_SORT_MAP) - if ordering: - if "resource_id" not in {field.lstrip("-") for field in ordering}: - ordering.append("resource_id") + validated = self._validate_sort_fields(sort_param, self._RESOURCE_SORT_MAP) + ordering = validated if validated else None - # Phase 1: lightweight aggregation with only sort keys, paginate - ordering_qs = self._build_resource_ordering_queryset( - filtered_queryset, - resource_ids=resource_ids, - tenant_id=tenant_id, - ordering=ordering, - ) - page = self.paginate_queryset(ordering_qs) - if page is not None: - page_ids = [row["resource_id"] for row in page] - resource_data = self._build_resource_aggregation( - filtered_queryset, resource_ids=page_ids, tenant_id=tenant_id - ) - # Re-sort to match the page ordering - id_order = {rid: idx for idx, rid in enumerate(page_ids)} - results = self._post_process_resources(resource_data) - results.sort(key=lambda r: id_order.get(r["resource_id"], 0)) - serializer = FindingGroupResourceSerializer(results, many=True) - return self.get_paginated_response(serializer.data) + # Resource filters can only match findings with resources; skip orphan + # detection entirely when they are present. + if resource_ids is not None: + return self._mapping_paginated_response( + request, filtered_queryset, resource_ids, tenant_id, ordering + ) - page_ids = [row["resource_id"] for row in ordering_qs] + # Serve the mapping response directly and piggyback on the paginator + # count to detect orphan-only groups, instead of paying a separate + # has_mappings.exists() semi-join over ResourceFindingMapping on + # every non-IaC request. TODO: once the ephemeral resources strategy + # is decided, mixed groups should route to _combined_paginated_response. + response = self._mapping_paginated_response( + request, filtered_queryset, resource_ids, tenant_id, ordering + ) + + page = getattr(self.paginator, "page", None) + mapping_total = page.paginator.count if page is not None else None + if mapping_total == 0: + # Pure orphan group (e.g. IaC): synthesize resource-like rows. + return self._combined_paginated_response( + request, filtered_queryset, tenant_id, ordering + ) + + return response + + def _mapping_paginated_response( + self, request, filtered_queryset, resource_ids, tenant_id, ordering + ): + """Mapping-only paginated response (original fast path).""" + if ordering: + if "resource_id" not in {field.lstrip("-") for field in ordering}: + ordering.append("resource_id") + + # Phase 1: lightweight aggregation with only sort keys, paginate + ordering_qs = self._build_resource_ordering_queryset( + filtered_queryset, + resource_ids=resource_ids, + tenant_id=tenant_id, + ordering=ordering, + ) + page = self.paginate_queryset(ordering_qs) + if page is not None: + page_ids = [row["resource_id"] for row in page] resource_data = self._build_resource_aggregation( filtered_queryset, resource_ids=page_ids, tenant_id=tenant_id ) @@ -7773,10 +7908,18 @@ class FindingGroupViewSet(BaseRLSViewSet): results = self._post_process_resources(resource_data) results.sort(key=lambda r: id_order.get(r["resource_id"], 0)) serializer = FindingGroupResourceSerializer(results, many=True) - return Response(serializer.data) + return self.get_paginated_response(serializer.data) + + page_ids = [row["resource_id"] for row in ordering_qs] + resource_data = self._build_resource_aggregation( + filtered_queryset, resource_ids=page_ids, tenant_id=tenant_id + ) + id_order = {rid: idx for idx, rid in enumerate(page_ids)} + results = self._post_process_resources(resource_data) + results.sort(key=lambda r: id_order.get(r["resource_id"], 0)) + serializer = FindingGroupResourceSerializer(results, many=True) + return Response(serializer.data) - # No sort (or only empty sort fragments): paginate lightweight resource IDs - # first, aggregate only the page. mapping_qs = self._build_resource_mapping_queryset( filtered_queryset, resource_ids=resource_ids, tenant_id=tenant_id ) @@ -7804,6 +7947,95 @@ class FindingGroupViewSet(BaseRLSViewSet): serializer = FindingGroupResourceSerializer(results, many=True) return Response(serializer.data) + def _combined_paginated_response( + self, request, filtered_queryset, tenant_id, ordering + ): + """Mapping rows + orphan findings appended at end. + + Orphans sit after mapping rows regardless of sort. This keeps the + mapping-only code path intact for checks that have no orphans (the + common case) and avoids paying UNION/coalesce costs there. + """ + mapping_qs = self._build_resource_mapping_queryset( + filtered_queryset, resource_ids=None, tenant_id=tenant_id + ) + mapping_count = mapping_qs.values("resource_id").distinct().count() + + orphan_ids = list( + self._orphan_findings_queryset(filtered_queryset) + .order_by("id") + .values_list("id", flat=True) + ) + orphan_count = len(orphan_ids) + total = mapping_count + orphan_count + + # Paginate a simple [0..total) index sequence so DRF produces proper + # links/meta; then slice mapping / orphan sources accordingly. + page = self.paginate_queryset(range(total)) + page_indices = list(page) if page is not None else list(range(total)) + + mapping_indices = [i for i in page_indices if i < mapping_count] + orphan_positions = [ + i - mapping_count for i in page_indices if i >= mapping_count + ] + + mapping_results = [] + if mapping_indices: + start = mapping_indices[0] + stop = mapping_indices[-1] + 1 + if ordering: + ordering_fields = list(ordering) + if "resource_id" not in { + field.lstrip("-") for field in ordering_fields + }: + ordering_fields.append("resource_id") + ordered_qs = self._build_resource_ordering_queryset( + filtered_queryset, + resource_ids=None, + tenant_id=tenant_id, + ordering=ordering_fields, + ) + slice_rids = [row["resource_id"] for row in ordered_qs[start:stop]] + else: + slice_rids = list( + mapping_qs.values_list("resource_id", flat=True) + .distinct() + .order_by("resource_id")[start:stop] + ) + if slice_rids: + resource_data = self._build_resource_aggregation( + filtered_queryset, + resource_ids=slice_rids, + tenant_id=tenant_id, + ) + rows_by_rid = {row["resource_id"]: row for row in resource_data} + ordered_rows = [ + rows_by_rid[rid] for rid in slice_rids if rid in rows_by_rid + ] + mapping_results = self._post_process_resources(ordered_rows) + + orphan_results = [] + if orphan_positions: + slice_fids = [orphan_ids[pos] for pos in orphan_positions] + raw_rows = list( + self._orphan_aggregation_values( + self._orphan_findings_queryset( + filtered_queryset, finding_ids=slice_fids + ) + ) + ) + rows_by_fid = {row["id"]: row for row in raw_rows} + ordered_rows = [ + rows_by_fid[fid] for fid in slice_fids if fid in rows_by_fid + ] + orphan_results = self._post_process_orphans(ordered_rows) + + results = mapping_results + orphan_results + serializer = FindingGroupResourceSerializer(results, many=True) + if page is not None: + return self.get_paginated_response(serializer.data) + return Response(serializer.data) + def list(self, request, *args, **kwargs): """ List finding groups with aggregation and filtering. @@ -7933,10 +8165,13 @@ class FindingGroupViewSet(BaseRLSViewSet): tenant_id = request.tenant_id queryset = self._get_finding_queryset() - # Get latest completed scan for each provider + # Order by -completed_at (matching the /latest summary path and the + # daily summary upsert keyed on midnight(completed_at)) so that + # overlapping scans do not make /resources and /latest read from + # different scans and report diverging counts. latest_scan_ids = ( Scan.objects.filter(tenant_id=tenant_id, state=StateChoices.COMPLETED) - .order_by("provider_id", "-inserted_at") + .order_by("provider_id", "-completed_at", "-inserted_at") .distinct("provider_id") .values_list("id", flat=True) ) diff --git a/api/src/backend/config/celery.py b/api/src/backend/config/celery.py index aaa1b1c386..c46c8a426c 100644 --- a/api/src/backend/config/celery.py +++ b/api/src/backend/config/celery.py @@ -17,8 +17,10 @@ celery_app.config_from_object("django.conf:settings", namespace="CELERY") celery_app.conf.update(result_extended=True, result_expires=None) celery_app.conf.broker_transport_options = { - "visibility_timeout": BROKER_VISIBILITY_TIMEOUT + "visibility_timeout": BROKER_VISIBILITY_TIMEOUT, + "queue_order_strategy": "priority", } +celery_app.conf.task_default_priority = 6 celery_app.conf.result_backend_transport_options = { "visibility_timeout": BROKER_VISIBILITY_TIMEOUT } diff --git a/api/src/backend/tasks/jobs/attack_paths/aws.py b/api/src/backend/tasks/jobs/attack_paths/aws.py index e7f26b5173..7248cc39e8 100644 --- a/api/src/backend/tasks/jobs/attack_paths/aws.py +++ b/api/src/backend/tasks/jobs/attack_paths/aws.py @@ -1,6 +1,8 @@ # Portions of this file are based on code from the Cartography project # (https://github.com/cartography-cncf/cartography), which is licensed under the Apache 2.0 License. +import time + from typing import Any import aioboto3 @@ -33,7 +35,7 @@ def start_aws_ingestion( For the scan progress updates: - The caller of this function (`tasks.jobs.attack_paths.scan.run`) has set it to 2. - - When the control returns to the caller, it will be set to 95. + - When the control returns to the caller, it will be set to 93. """ # Initialize variables common to all jobs @@ -89,34 +91,50 @@ def start_aws_ingestion( logger.info( f"Syncing function permission_relationships for AWS account {prowler_api_provider.uid}" ) + t0 = time.perf_counter() cartography_aws.RESOURCE_FUNCTIONS["permission_relationships"](**sync_args) + logger.info( + f"Synced function permission_relationships for AWS account {prowler_api_provider.uid} in {time.perf_counter() - t0:.3f}s" + ) db_utils.update_attack_paths_scan_progress(attack_paths_scan, 88) if "resourcegroupstaggingapi" in requested_syncs: logger.info( f"Syncing function resourcegroupstaggingapi for AWS account {prowler_api_provider.uid}" ) + t0 = time.perf_counter() cartography_aws.RESOURCE_FUNCTIONS["resourcegroupstaggingapi"](**sync_args) + logger.info( + f"Synced function resourcegroupstaggingapi for AWS account {prowler_api_provider.uid} in {time.perf_counter() - t0:.3f}s" + ) db_utils.update_attack_paths_scan_progress(attack_paths_scan, 89) logger.info( f"Syncing ec2_iaminstanceprofile scoped analysis for AWS account {prowler_api_provider.uid}" ) + t0 = time.perf_counter() cartography_aws.run_scoped_analysis_job( "aws_ec2_iaminstanceprofile.json", neo4j_session, common_job_parameters, ) + logger.info( + f"Synced ec2_iaminstanceprofile scoped analysis for AWS account {prowler_api_provider.uid} in {time.perf_counter() - t0:.3f}s" + ) db_utils.update_attack_paths_scan_progress(attack_paths_scan, 90) logger.info( f"Syncing lambda_ecr analysis for AWS account {prowler_api_provider.uid}" ) + t0 = time.perf_counter() cartography_aws.run_analysis_job( "aws_lambda_ecr.json", neo4j_session, common_job_parameters, ) + logger.info( + f"Synced lambda_ecr analysis for AWS account {prowler_api_provider.uid} in {time.perf_counter() - t0:.3f}s" + ) if all( s in requested_syncs @@ -125,25 +143,34 @@ def start_aws_ingestion( logger.info( f"Syncing lb_container_exposure scoped analysis for AWS account {prowler_api_provider.uid}" ) + t0 = time.perf_counter() cartography_aws.run_scoped_analysis_job( "aws_lb_container_exposure.json", neo4j_session, common_job_parameters, ) + logger.info( + f"Synced lb_container_exposure scoped analysis for AWS account {prowler_api_provider.uid} in {time.perf_counter() - t0:.3f}s" + ) if all(s in requested_syncs for s in ["ec2:network_acls", "ec2:load_balancer_v2"]): logger.info( f"Syncing lb_nacl_direct scoped analysis for AWS account {prowler_api_provider.uid}" ) + t0 = time.perf_counter() cartography_aws.run_scoped_analysis_job( "aws_lb_nacl_direct.json", neo4j_session, common_job_parameters, ) + logger.info( + f"Synced lb_nacl_direct scoped analysis for AWS account {prowler_api_provider.uid} in {time.perf_counter() - t0:.3f}s" + ) db_utils.update_attack_paths_scan_progress(attack_paths_scan, 91) logger.info(f"Syncing metadata for AWS account {prowler_api_provider.uid}") + t0 = time.perf_counter() cartography_aws.merge_module_sync_metadata( neo4j_session, group_type="AWSAccount", @@ -152,24 +179,23 @@ def start_aws_ingestion( update_tag=cartography_config.update_tag, stat_handler=cartography_aws.stat_handler, ) + logger.info( + f"Synced metadata for AWS account {prowler_api_provider.uid} in {time.perf_counter() - t0:.3f}s" + ) db_utils.update_attack_paths_scan_progress(attack_paths_scan, 92) # Removing the added extra field del common_job_parameters["AWS_ID"] - logger.info(f"Syncing cleanup_job for AWS account {prowler_api_provider.uid}") - cartography_aws.run_cleanup_job( - "aws_post_ingestion_principals_cleanup.json", - neo4j_session, - common_job_parameters, - ) - db_utils.update_attack_paths_scan_progress(attack_paths_scan, 93) - logger.info(f"Syncing analysis for AWS account {prowler_api_provider.uid}") + t0 = time.perf_counter() cartography_aws._perform_aws_analysis( requested_syncs, neo4j_session, common_job_parameters ) - db_utils.update_attack_paths_scan_progress(attack_paths_scan, 94) + logger.info( + f"Synced analysis for AWS account {prowler_api_provider.uid} in {time.perf_counter() - t0:.3f}s" + ) + db_utils.update_attack_paths_scan_progress(attack_paths_scan, 93) return failed_syncs @@ -234,6 +260,8 @@ def sync_aws_account( ) try: + func_t0 = time.perf_counter() + # `ecr:image_layers` uses `aioboto3_session` instead of `boto3_session` if func_name == "ecr:image_layers": cartography_aws.RESOURCE_FUNCTIONS[func_name]( @@ -257,7 +285,15 @@ def sync_aws_account( else: cartography_aws.RESOURCE_FUNCTIONS[func_name](**sync_args) + logger.info( + f"Synced function {func_name} for AWS account {prowler_api_provider.uid} in {time.perf_counter() - func_t0:.3f}s" + ) + except Exception as e: + logger.info( + f"Synced function {func_name} for AWS account {prowler_api_provider.uid} in {time.perf_counter() - func_t0:.3f}s (FAILED)" + ) + exception_message = utils.stringify_exception( e, f"Exception for AWS sync function: {func_name}" ) diff --git a/api/src/backend/tasks/jobs/attack_paths/config.py b/api/src/backend/tasks/jobs/attack_paths/config.py index 76dbdc1dc5..5f5c523ceb 100644 --- a/api/src/backend/tasks/jobs/attack_paths/config.py +++ b/api/src/backend/tasks/jobs/attack_paths/config.py @@ -8,9 +8,9 @@ from tasks.jobs.attack_paths import aws # Batch size for Neo4j write operations (resource labeling, cleanup) BATCH_SIZE = env.int("ATTACK_PATHS_BATCH_SIZE", 1000) # Batch size for Postgres findings fetch (keyset pagination page size) -FINDINGS_BATCH_SIZE = env.int("ATTACK_PATHS_FINDINGS_BATCH_SIZE", 500) +FINDINGS_BATCH_SIZE = env.int("ATTACK_PATHS_FINDINGS_BATCH_SIZE", 1000) # Batch size for temp-to-tenant graph sync (nodes and relationships per cursor page) -SYNC_BATCH_SIZE = env.int("ATTACK_PATHS_SYNC_BATCH_SIZE", 250) +SYNC_BATCH_SIZE = env.int("ATTACK_PATHS_SYNC_BATCH_SIZE", 1000) # Neo4j internal labels (Prowler-specific, not provider-specific) # - `Internet`: Singleton node representing external internet access for exposed-resource queries diff --git a/api/src/backend/tasks/jobs/attack_paths/findings.py b/api/src/backend/tasks/jobs/attack_paths/findings.py index 56df42bf9d..0b2ecb4c45 100644 --- a/api/src/backend/tasks/jobs/attack_paths/findings.py +++ b/api/src/backend/tasks/jobs/attack_paths/findings.py @@ -12,6 +12,7 @@ from typing import Any, Generator from uuid import UUID import neo4j + from cartography.config import Config as CartographyConfig from celery.utils.log import get_task_logger from tasks.jobs.attack_paths.config import ( @@ -86,17 +87,21 @@ def analysis( prowler_api_provider: Provider, scan_id: str, config: CartographyConfig, -) -> None: +) -> tuple[int, int]: """ Main entry point for Prowler findings analysis. Adds resource labels and loads findings. + Returns (labeled_nodes, findings_loaded). """ - add_resource_label( + total_labeled = add_resource_label( neo4j_session, prowler_api_provider.provider, str(prowler_api_provider.uid) ) findings_data = stream_findings_with_resources(prowler_api_provider, scan_id) - load_findings(neo4j_session, findings_data, prowler_api_provider, config) + total_loaded = load_findings( + neo4j_session, findings_data, prowler_api_provider, config + ) + return total_labeled, total_loaded def add_resource_label( @@ -146,12 +151,11 @@ def load_findings( findings_batches: Generator[list[dict[str, Any]], None, None], prowler_api_provider: Provider, config: CartographyConfig, -) -> None: +) -> int: """Load Prowler findings into the graph, linking them to resources.""" query = render_cypher_template( INSERT_FINDING_TEMPLATE, { - "__ROOT_NODE_LABEL__": get_root_node_label(prowler_api_provider.provider), "__NODE_UID_FIELD__": get_node_uid_field(prowler_api_provider.provider), "__RESOURCE_LABEL__": get_provider_resource_label( prowler_api_provider.provider @@ -160,7 +164,6 @@ def load_findings( ) parameters = { - "provider_uid": str(prowler_api_provider.uid), "last_updated": config.update_tag, "prowler_version": ProwlerConfig.prowler_version, } @@ -178,6 +181,7 @@ def load_findings( neo4j_session.run(query, parameters) logger.info(f"Finished loading {total_records} records in {batch_num} batches") + return total_records # Findings Streaming (Generator-based) @@ -248,7 +252,9 @@ def _fetch_findings_batch( with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS): # Use `all_objects` to get `Findings` even on soft-deleted `Providers` # But even the provider is already validated as active in this context - qs = FindingModel.all_objects.filter(scan_id=scan_id).order_by("id") + qs = FindingModel.all_objects.filter( + tenant_id=tenant_id, scan_id=scan_id + ).order_by("id") if after_id is not None: qs = qs.filter(id__gt=after_id) diff --git a/api/src/backend/tasks/jobs/attack_paths/queries.py b/api/src/backend/tasks/jobs/attack_paths/queries.py index ab13150b56..26ffa32f92 100644 --- a/api/src/backend/tasks/jobs/attack_paths/queries.py +++ b/api/src/backend/tasks/jobs/attack_paths/queries.py @@ -32,17 +32,14 @@ ADD_RESOURCE_LABEL_TEMPLATE = """ """ INSERT_FINDING_TEMPLATE = f""" - MATCH (account:__ROOT_NODE_LABEL__ {{id: $provider_uid}}) UNWIND $findings_data AS finding_data - OPTIONAL MATCH (account)-->(resource_by_uid:__RESOURCE_LABEL__) - WHERE resource_by_uid.__NODE_UID_FIELD__ = finding_data.resource_uid - WITH account, finding_data, resource_by_uid + OPTIONAL MATCH (resource_by_uid:__RESOURCE_LABEL__ {{__NODE_UID_FIELD__: finding_data.resource_uid}}) + WITH finding_data, resource_by_uid - OPTIONAL MATCH (account)-->(resource_by_id:__RESOURCE_LABEL__) + OPTIONAL MATCH (resource_by_id:__RESOURCE_LABEL__ {{id: finding_data.resource_uid}}) WHERE resource_by_uid IS NULL - AND resource_by_id.id = finding_data.resource_uid - WITH account, finding_data, COALESCE(resource_by_uid, resource_by_id) AS resource + WITH finding_data, COALESCE(resource_by_uid, resource_by_id) AS resource WHERE resource IS NOT NULL MERGE (finding:{PROWLER_FINDING_LABEL} {{id: finding_data.id}}) diff --git a/api/src/backend/tasks/jobs/attack_paths/scan.py b/api/src/backend/tasks/jobs/attack_paths/scan.py index a53a6a530f..382c231eeb 100644 --- a/api/src/backend/tasks/jobs/attack_paths/scan.py +++ b/api/src/backend/tasks/jobs/attack_paths/scan.py @@ -55,6 +55,7 @@ exception propagates to Celery. import logging import time + from typing import Any from cartography.config import Config as CartographyConfig @@ -144,6 +145,12 @@ def run(tenant_id: str, scan_id: str, task_id: str) -> dict[str, Any]: attack_paths_scan, task_id, tenant_cartography_config ) + scan_t0 = time.perf_counter() + logger.info( + f"Starting Attack Paths scan ({attack_paths_scan.id}) for " + f"{prowler_api_provider.provider.upper()} provider {prowler_api_provider.id}" + ) + subgraph_dropped = False sync_completed = False provider_gated = False @@ -169,6 +176,7 @@ def run(tenant_id: str, scan_id: str, task_id: str) -> dict[str, Any]: db_utils.update_attack_paths_scan_progress(attack_paths_scan, 2) # The real scan, where iterates over cloud services + t0 = time.perf_counter() ingestion_exceptions = utils.call_within_event_loop( cartography_ingestion_function, tmp_neo4j_session, @@ -177,19 +185,23 @@ def run(tenant_id: str, scan_id: str, task_id: str) -> dict[str, Any]: prowler_sdk_provider, attack_paths_scan, ) + logger.info( + f"Cartography ingestion completed in {time.perf_counter() - t0:.3f}s " + f"(failed_syncs={len(ingestion_exceptions)})" + ) # Post-processing: Just keeping it to be more Cartography compliant logger.info( f"Syncing Cartography ontology for AWS account {prowler_api_provider.uid}" ) cartography_ontology.run(tmp_neo4j_session, tmp_cartography_config) - db_utils.update_attack_paths_scan_progress(attack_paths_scan, 95) + db_utils.update_attack_paths_scan_progress(attack_paths_scan, 94) logger.info( f"Syncing Cartography analysis for AWS account {prowler_api_provider.uid}" ) cartography_analysis.run(tmp_neo4j_session, tmp_cartography_config) - db_utils.update_attack_paths_scan_progress(attack_paths_scan, 96) + db_utils.update_attack_paths_scan_progress(attack_paths_scan, 95) # Creating Internet node and CAN_ACCESS relationships logger.info( @@ -198,14 +210,20 @@ def run(tenant_id: str, scan_id: str, task_id: str) -> dict[str, Any]: internet.analysis( tmp_neo4j_session, prowler_api_provider, tmp_cartography_config ) + db_utils.update_attack_paths_scan_progress(attack_paths_scan, 96) # Adding Prowler Finding nodes and relationships logger.info( f"Syncing Prowler analysis for AWS account {prowler_api_provider.uid}" ) - findings.analysis( + t0 = time.perf_counter() + labeled_nodes, findings_loaded = findings.analysis( tmp_neo4j_session, prowler_api_provider, scan_id, tmp_cartography_config ) + logger.info( + f"Prowler analysis completed in {time.perf_counter() - t0:.3f}s " + f"(findings={findings_loaded}, labeled_nodes={labeled_nodes})" + ) db_utils.update_attack_paths_scan_progress(attack_paths_scan, 97) logger.info( @@ -227,22 +245,33 @@ def run(tenant_id: str, scan_id: str, task_id: str) -> dict[str, Any]: logger.info(f"Deleting existing provider graph in {tenant_database_name}") db_utils.set_provider_graph_data_ready(attack_paths_scan, False) provider_gated = True - graph_database.drop_subgraph( + + t0 = time.perf_counter() + deleted_nodes = graph_database.drop_subgraph( database=tenant_database_name, provider_id=str(prowler_api_provider.id), ) + logger.info( + f"Deleted existing provider graph in {time.perf_counter() - t0:.3f}s " + f"(deleted_nodes={deleted_nodes})" + ) subgraph_dropped = True db_utils.update_attack_paths_scan_progress(attack_paths_scan, 98) logger.info( f"Syncing graph from {tmp_database_name} into {tenant_database_name}" ) - sync.sync_graph( + t0 = time.perf_counter() + sync_result = sync.sync_graph( source_database=tmp_database_name, target_database=tenant_database_name, tenant_id=str(prowler_api_provider.tenant_id), provider_id=str(prowler_api_provider.id), ) + logger.info( + f"Synced graph in {time.perf_counter() - t0:.3f}s " + f"(nodes={sync_result['nodes']}, relationships={sync_result['relationships']})" + ) sync_completed = True db_utils.set_graph_data_ready(attack_paths_scan, True) db_utils.update_attack_paths_scan_progress(attack_paths_scan, 99) @@ -250,17 +279,16 @@ def run(tenant_id: str, scan_id: str, task_id: str) -> dict[str, Any]: logger.info(f"Clearing Neo4j cache for database {tenant_database_name}") graph_database.clear_cache(tenant_database_name) - logger.info( - f"Completed Cartography ({attack_paths_scan.id}) for " - f"{prowler_api_provider.provider.upper()} provider {prowler_api_provider.id}" - ) - logger.info(f"Dropping temporary Neo4j database {tmp_database_name}") graph_database.drop_database(tmp_database_name) db_utils.finish_attack_paths_scan( attack_paths_scan, StateChoices.COMPLETED, ingestion_exceptions ) + logger.info( + f"Attack Paths scan completed in {time.perf_counter() - scan_t0:.3f}s " + f"(state=completed, failed_syncs={len(ingestion_exceptions)})" + ) return ingestion_exceptions except Exception as e: diff --git a/api/src/backend/tasks/jobs/attack_paths/sync.py b/api/src/backend/tasks/jobs/attack_paths/sync.py index 24ffa6cf48..f720a12e82 100644 --- a/api/src/backend/tasks/jobs/attack_paths/sync.py +++ b/api/src/backend/tasks/jobs/attack_paths/sync.py @@ -5,6 +5,8 @@ This module handles syncing graph data from temporary scan databases to the tenant database, adding provider isolation labels and properties. """ +import time + from collections import defaultdict from typing import Any @@ -81,6 +83,7 @@ def sync_nodes( Source and target sessions are opened sequentially per batch to avoid holding two Bolt connections simultaneously for the entire sync duration. """ + t0 = time.perf_counter() last_id = -1 total_synced = 0 @@ -117,7 +120,7 @@ def sync_nodes( total_synced += batch_count logger.info( - f"Synced {total_synced} nodes from {source_database} to {target_database}" + f"Synced {total_synced} nodes from {source_database} to {target_database} in {time.perf_counter() - t0:.3f}s" ) return total_synced @@ -136,6 +139,7 @@ def sync_relationships( Source and target sessions are opened sequentially per batch to avoid holding two Bolt connections simultaneously for the entire sync duration. """ + t0 = time.perf_counter() last_id = -1 total_synced = 0 @@ -166,7 +170,7 @@ def sync_relationships( total_synced += batch_count logger.info( - f"Synced {total_synced} relationships from {source_database} to {target_database}" + f"Synced {total_synced} relationships from {source_database} to {target_database} in {time.perf_counter() - t0:.3f}s" ) return total_synced diff --git a/api/src/backend/tasks/jobs/scan.py b/api/src/backend/tasks/jobs/scan.py index 2c73c97f2f..456fd786cb 100644 --- a/api/src/backend/tasks/jobs/scan.py +++ b/api/src/backend/tasks/jobs/scan.py @@ -752,11 +752,19 @@ def _process_finding_micro_batch( ) if mappings_to_create: - ResourceFindingMapping.objects.bulk_create( + created_mappings = ResourceFindingMapping.objects.bulk_create( mappings_to_create, batch_size=SCAN_DB_BATCH_SIZE, ignore_conflicts=True, + unique_fields=["tenant_id", "resource_id", "finding_id"], ) + inserted = sum(1 for m in created_mappings if m.pk) + if inserted != len(mappings_to_create): + logger.error( + f"scan {scan_instance.id}: expected " + f"{len(mappings_to_create)} ResourceFindingMapping rows, " + f"inserted {inserted}. Rolling back micro-batch." + ) # Update finding denormalized arrays findings_to_update = [] @@ -1804,11 +1812,9 @@ def aggregate_finding_group_summaries(tenant_id: str, scan_id: str): ) # Aggregate findings by check_id for this scan. - # `pass_count`, `fail_count` and `manual_count` count *every* finding - # in this group, regardless of mute state, so the aggregated `status` - # always reflects the underlying check outcome (FAIL > PASS > MANUAL) - # even when the group is fully muted. The orthogonal `muted` flag is - # what tells whether the group has any actionable (non-muted) findings. + # `pass_count`, `fail_count` and `manual_count` only count non-muted + # findings. Muted findings are tracked separately via the + # `*_muted_count` fields. aggregated = ( Finding.objects.filter( tenant_id=tenant_id, @@ -1817,9 +1823,9 @@ def aggregate_finding_group_summaries(tenant_id: str, scan_id: str): .values("check_id") .annotate( severity_order=Max(severity_case), - pass_count=Count("id", filter=Q(status="PASS")), - fail_count=Count("id", filter=Q(status="FAIL")), - manual_count=Count("id", filter=Q(status="MANUAL")), + pass_count=Count("id", filter=Q(status="PASS", muted=False)), + fail_count=Count("id", filter=Q(status="FAIL", muted=False)), + manual_count=Count("id", filter=Q(status="MANUAL", muted=False)), pass_muted_count=Count("id", filter=Q(status="PASS", muted=True)), fail_muted_count=Count("id", filter=Q(status="FAIL", muted=True)), manual_muted_count=Count("id", filter=Q(status="MANUAL", muted=True)), diff --git a/api/src/backend/tasks/tests/test_attack_paths_scan.py b/api/src/backend/tasks/tests/test_attack_paths_scan.py index 1b1beb11a9..283c0650e1 100644 --- a/api/src/backend/tasks/tests/test_attack_paths_scan.py +++ b/api/src/backend/tasks/tests/test_attack_paths_scan.py @@ -38,11 +38,14 @@ class TestAttackPathsRun: @patch("tasks.jobs.attack_paths.scan.db_utils.finish_attack_paths_scan") @patch("tasks.jobs.attack_paths.scan.db_utils.update_attack_paths_scan_progress") @patch("tasks.jobs.attack_paths.scan.db_utils.starting_attack_paths_scan") - @patch("tasks.jobs.attack_paths.scan.sync.sync_graph") - @patch("tasks.jobs.attack_paths.scan.graph_database.drop_subgraph") + @patch( + "tasks.jobs.attack_paths.scan.sync.sync_graph", + return_value={"nodes": 0, "relationships": 0}, + ) + @patch("tasks.jobs.attack_paths.scan.graph_database.drop_subgraph", return_value=0) @patch("tasks.jobs.attack_paths.scan.indexes.create_sync_indexes") @patch("tasks.jobs.attack_paths.scan.internet.analysis") - @patch("tasks.jobs.attack_paths.scan.findings.analysis") + @patch("tasks.jobs.attack_paths.scan.findings.analysis", return_value=(0, 0)) @patch("tasks.jobs.attack_paths.scan.indexes.create_findings_indexes") @patch("tasks.jobs.attack_paths.scan.cartography_ontology.run") @patch("tasks.jobs.attack_paths.scan.cartography_analysis.run") @@ -188,7 +191,7 @@ class TestAttackPathsRun: @patch("tasks.jobs.attack_paths.scan.db_utils.set_provider_graph_data_ready") @patch("tasks.jobs.attack_paths.scan.db_utils.update_attack_paths_scan_progress") @patch("tasks.jobs.attack_paths.scan.db_utils.starting_attack_paths_scan") - @patch("tasks.jobs.attack_paths.scan.findings.analysis") + @patch("tasks.jobs.attack_paths.scan.findings.analysis", return_value=(0, 0)) @patch("tasks.jobs.attack_paths.scan.internet.analysis") @patch("tasks.jobs.attack_paths.scan.indexes.create_findings_indexes") @patch("tasks.jobs.attack_paths.scan.cartography_analysis.run") @@ -287,7 +290,7 @@ class TestAttackPathsRun: @patch("tasks.jobs.attack_paths.scan.db_utils.set_provider_graph_data_ready") @patch("tasks.jobs.attack_paths.scan.db_utils.update_attack_paths_scan_progress") @patch("tasks.jobs.attack_paths.scan.db_utils.starting_attack_paths_scan") - @patch("tasks.jobs.attack_paths.scan.findings.analysis") + @patch("tasks.jobs.attack_paths.scan.findings.analysis", return_value=(0, 0)) @patch("tasks.jobs.attack_paths.scan.internet.analysis") @patch("tasks.jobs.attack_paths.scan.indexes.create_findings_indexes") @patch("tasks.jobs.attack_paths.scan.cartography_analysis.run") @@ -390,7 +393,7 @@ class TestAttackPathsRun: @patch("tasks.jobs.attack_paths.scan.db_utils.set_provider_graph_data_ready") @patch("tasks.jobs.attack_paths.scan.db_utils.update_attack_paths_scan_progress") @patch("tasks.jobs.attack_paths.scan.db_utils.starting_attack_paths_scan") - @patch("tasks.jobs.attack_paths.scan.findings.analysis") + @patch("tasks.jobs.attack_paths.scan.findings.analysis", return_value=(0, 0)) @patch("tasks.jobs.attack_paths.scan.internet.analysis") @patch("tasks.jobs.attack_paths.scan.indexes.create_findings_indexes") @patch("tasks.jobs.attack_paths.scan.cartography_analysis.run") @@ -489,14 +492,17 @@ class TestAttackPathsRun: @patch("tasks.jobs.attack_paths.scan.db_utils.set_provider_graph_data_ready") @patch("tasks.jobs.attack_paths.scan.db_utils.update_attack_paths_scan_progress") @patch("tasks.jobs.attack_paths.scan.db_utils.starting_attack_paths_scan") - @patch("tasks.jobs.attack_paths.scan.sync.sync_graph") + @patch( + "tasks.jobs.attack_paths.scan.sync.sync_graph", + return_value={"nodes": 0, "relationships": 0}, + ) @patch( "tasks.jobs.attack_paths.scan.graph_database.drop_subgraph", side_effect=RuntimeError("drop failed"), ) @patch("tasks.jobs.attack_paths.scan.indexes.create_sync_indexes") @patch("tasks.jobs.attack_paths.scan.internet.analysis") - @patch("tasks.jobs.attack_paths.scan.findings.analysis") + @patch("tasks.jobs.attack_paths.scan.findings.analysis", return_value=(0, 0)) @patch("tasks.jobs.attack_paths.scan.indexes.create_findings_indexes") @patch("tasks.jobs.attack_paths.scan.cartography_ontology.run") @patch("tasks.jobs.attack_paths.scan.cartography_analysis.run") @@ -609,7 +615,7 @@ class TestAttackPathsRun: @patch("tasks.jobs.attack_paths.scan.graph_database.drop_subgraph") @patch("tasks.jobs.attack_paths.scan.indexes.create_sync_indexes") @patch("tasks.jobs.attack_paths.scan.internet.analysis") - @patch("tasks.jobs.attack_paths.scan.findings.analysis") + @patch("tasks.jobs.attack_paths.scan.findings.analysis", return_value=(0, 0)) @patch("tasks.jobs.attack_paths.scan.indexes.create_findings_indexes") @patch("tasks.jobs.attack_paths.scan.cartography_ontology.run") @patch("tasks.jobs.attack_paths.scan.cartography_analysis.run") @@ -718,11 +724,14 @@ class TestAttackPathsRun: @patch("tasks.jobs.attack_paths.scan.db_utils.set_provider_graph_data_ready") @patch("tasks.jobs.attack_paths.scan.db_utils.update_attack_paths_scan_progress") @patch("tasks.jobs.attack_paths.scan.db_utils.starting_attack_paths_scan") - @patch("tasks.jobs.attack_paths.scan.sync.sync_graph") + @patch( + "tasks.jobs.attack_paths.scan.sync.sync_graph", + return_value={"nodes": 0, "relationships": 0}, + ) @patch("tasks.jobs.attack_paths.scan.graph_database.drop_subgraph") @patch("tasks.jobs.attack_paths.scan.indexes.create_sync_indexes") @patch("tasks.jobs.attack_paths.scan.internet.analysis") - @patch("tasks.jobs.attack_paths.scan.findings.analysis") + @patch("tasks.jobs.attack_paths.scan.findings.analysis", return_value=(0, 0)) @patch("tasks.jobs.attack_paths.scan.indexes.create_findings_indexes") @patch("tasks.jobs.attack_paths.scan.cartography_ontology.run") @patch("tasks.jobs.attack_paths.scan.cartography_analysis.run") @@ -833,14 +842,17 @@ class TestAttackPathsRun: @patch("tasks.jobs.attack_paths.scan.db_utils.set_provider_graph_data_ready") @patch("tasks.jobs.attack_paths.scan.db_utils.update_attack_paths_scan_progress") @patch("tasks.jobs.attack_paths.scan.db_utils.starting_attack_paths_scan") - @patch("tasks.jobs.attack_paths.scan.sync.sync_graph") + @patch( + "tasks.jobs.attack_paths.scan.sync.sync_graph", + return_value={"nodes": 0, "relationships": 0}, + ) @patch( "tasks.jobs.attack_paths.scan.graph_database.drop_subgraph", side_effect=RuntimeError("drop failed"), ) @patch("tasks.jobs.attack_paths.scan.indexes.create_sync_indexes") @patch("tasks.jobs.attack_paths.scan.internet.analysis") - @patch("tasks.jobs.attack_paths.scan.findings.analysis") + @patch("tasks.jobs.attack_paths.scan.findings.analysis", return_value=(0, 0)) @patch("tasks.jobs.attack_paths.scan.indexes.create_findings_indexes") @patch("tasks.jobs.attack_paths.scan.cartography_ontology.run") @patch("tasks.jobs.attack_paths.scan.cartography_analysis.run") @@ -1274,10 +1286,6 @@ class TestAttackPathsFindingsHelpers: mock_session = MagicMock() with ( - patch( - "tasks.jobs.attack_paths.findings.get_root_node_label", - return_value="AWSAccount", - ), patch( "tasks.jobs.attack_paths.findings.get_node_uid_field", return_value="arn", @@ -1294,7 +1302,6 @@ class TestAttackPathsFindingsHelpers: assert mock_session.run.call_count == 2 for call_args in mock_session.run.call_args_list: params = call_args.args[1] - assert params["provider_uid"] == str(provider.uid) assert params["last_updated"] == config.update_tag assert "findings_data" in params @@ -1673,10 +1680,6 @@ class TestAttackPathsFindingsHelpers: yield # Make it a generator with ( - patch( - "tasks.jobs.attack_paths.findings.get_root_node_label", - return_value="AWSAccount", - ), patch( "tasks.jobs.attack_paths.findings.get_node_uid_field", return_value="arn", diff --git a/contrib/reverse-proxy/README.md b/contrib/reverse-proxy/README.md new file mode 100644 index 0000000000..a6387e689a --- /dev/null +++ b/contrib/reverse-proxy/README.md @@ -0,0 +1,64 @@ +# Prowler Reverse Proxy Configuration + +Ready-to-use nginx configuration for running Prowler behind a reverse proxy. + +## Problem + +Prowler's default Docker setup exposes two separate services: +- **UI** on port 3000 +- **API** on port 8080 + +This causes CORS issues and authentication failures (especially SAML SSO) when accessed through an external reverse proxy, since the proxy typically exposes a single domain. + +## Solution + +This adds an nginx container that unifies both services behind a single port, correctly forwarding headers so that Django generates proper URLs for SAML ACS callbacks and API responses. + +## Quick Start + +From the prowler root directory: + + docker compose -f docker-compose.yml \ + -f contrib/reverse-proxy/docker-compose.reverse-proxy.yml \ + up -d + +Access Prowler at http://localhost (port 80). + +## With an External Reverse Proxy + +Point your external reverse proxy to the prowler-nginx container on port 80. + +### Environment Variables + +| Variable | Default | Description | +|----------|---------|-------------| +| PROWLER_PROXY_PORT | 80 | Port exposed by the nginx proxy | + +### Example: Traefik + + services: + nginx: + labels: + - "traefik.enable=true" + - "traefik.http.routers.prowler.rule=Host(`prowler.example.com`)" + - "traefik.http.routers.prowler.tls.certresolver=letsencrypt" + - "traefik.http.services.prowler.loadbalancer.server.port=80" + +### Example: Caddy + + prowler.example.com { + reverse_proxy prowler-nginx:80 + } + +## SAML SSO + +If using SAML SSO behind a reverse proxy, also set the SAML_ACS_BASE_URL environment variable: + + SAML_ACS_BASE_URL=https://prowler.example.com + +## Architecture + + Internet -> External Reverse Proxy -> prowler-nginx:80 + |-- /api/* -> prowler-api:8080 + |-- /accounts/saml/ -> prowler-api:8080 + +-- /* -> prowler-ui:3000 diff --git a/contrib/reverse-proxy/docker-compose.reverse-proxy.yml b/contrib/reverse-proxy/docker-compose.reverse-proxy.yml new file mode 100644 index 0000000000..08c52f3558 --- /dev/null +++ b/contrib/reverse-proxy/docker-compose.reverse-proxy.yml @@ -0,0 +1,42 @@ +# Prowler Reverse Proxy - Docker Compose Override +# +# Use this alongside the main docker-compose.yml to add an nginx +# reverse proxy that unifies UI and API behind a single port. +# +# Usage: +# docker compose -f docker-compose.yml -f contrib/reverse-proxy/docker-compose.reverse-proxy.yml up -d +# +# Then access Prowler at http://localhost (port 80) or configure +# your external reverse proxy (Traefik, Caddy, Cloudflare Tunnel, +# Pangolin, etc.) to point to this container on port 80. +# +# For HTTPS with your own certs, see the README in this directory. +# +# Fixes: https://github.com/prowler-cloud/prowler/issues/8516 + +services: + nginx: + image: nginx:alpine + container_name: prowler-nginx + restart: unless-stopped + ports: + - "${PROWLER_PROXY_PORT:-80}:80" + volumes: + - ./contrib/reverse-proxy/nginx.conf:/etc/nginx/conf.d/default.conf:ro + depends_on: + - prowler-ui + - prowler-api + networks: + - prowler-network + + # Override UI to not expose port externally (nginx handles it) + prowler-ui: + ports: !reset [] + + # Override API to not expose port externally (nginx handles it) + prowler-api: + ports: !reset [] + +networks: + prowler-network: + driver: bridge diff --git a/contrib/reverse-proxy/nginx.conf b/contrib/reverse-proxy/nginx.conf new file mode 100644 index 0000000000..58520295bc --- /dev/null +++ b/contrib/reverse-proxy/nginx.conf @@ -0,0 +1,70 @@ +# Prowler Reverse Proxy Configuration +# Routes both UI and API through a single endpoint +# +# Usage: See docker-compose.reverse-proxy.yml +# Fixes: https://github.com/prowler-cloud/prowler/issues/8516 + +upstream prowler-ui { + server prowler-ui:3000; +} + +upstream prowler-api { + server prowler-api:8080; +} + +server { + listen 80; + server_name _; + + # Security headers + add_header X-Content-Type-Options "nosniff" always; + add_header X-Frame-Options "SAMEORIGIN" always; + add_header Referrer-Policy "strict-origin-when-cross-origin" always; + + # API requests — proxy to prowler-api + location /api/ { + proxy_pass http://prowler-api/api/; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Host $host; + proxy_read_timeout 300s; + proxy_connect_timeout 10s; + + # Handle large scan payloads + client_max_body_size 50m; + } + + # SAML endpoints — proxy to prowler-api + location /accounts/saml/ { + proxy_pass http://prowler-api/accounts/saml/; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Host $host; + } + + # Everything else — proxy to prowler-ui + location / { + proxy_pass http://prowler-ui/; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Host $host; + + # WebSocket support for Next.js HMR (dev) and live updates + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } + + # Health check endpoint + location /health { + access_log off; + return 200 "ok\n"; + add_header Content-Type text/plain; + } +} diff --git a/docs/developer-guide/lighthouse-architecture.mdx b/docs/developer-guide/lighthouse-architecture.mdx index 38ee50c5b8..e8acc6278c 100644 --- a/docs/developer-guide/lighthouse-architecture.mdx +++ b/docs/developer-guide/lighthouse-architecture.mdx @@ -15,8 +15,7 @@ This document describes the internal architecture of Prowler Lighthouse AI, enab Lighthouse AI operates as a Langchain-based agent that connects Large Language Models (LLMs) with Prowler security data through the Model Context Protocol (MCP). -Prowler Lighthouse Architecture -Prowler Lighthouse Architecture +![Prowler Lighthouse Architecture](/images/lighthouse-architecture.png) ### Three-Tier Architecture diff --git a/docs/docs.json b/docs/docs.json index c76b7174ef..82c23ba42f 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -12,6 +12,24 @@ "dark": "/images/prowler-logo-white.png", "light": "/images/prowler-logo-black.png" }, + "contextual": { + "options": [ + "copy", + "view", + { + "title": "Request a feature", + "description": "Open a feature request on GitHub", + "icon": "plus", + "href": "https://github.com/prowler-cloud/prowler/issues/new?template=feature-request.yml" + }, + { + "title": "Report an issue", + "description": "Open a bug report on GitHub", + "icon": "bug", + "href": "https://github.com/prowler-cloud/prowler/issues/new?template=bug_report.yml" + } + ] + }, "navigation": { "tabs": [ { @@ -133,6 +151,7 @@ ] }, "user-guide/tutorials/prowler-app-attack-paths", + "user-guide/tutorials/prowler-app-finding-groups", "user-guide/tutorials/prowler-cloud-public-ips", { "group": "Tutorials", diff --git a/docs/getting-started/installation/prowler-app.mdx b/docs/getting-started/installation/prowler-app.mdx index ea4029c234..c27eb2d35a 100644 --- a/docs/getting-started/installation/prowler-app.mdx +++ b/docs/getting-started/installation/prowler-app.mdx @@ -121,8 +121,8 @@ To update the environment file: Edit the `.env` file and change version values: ```env -PROWLER_UI_VERSION="5.23.0" -PROWLER_API_VERSION="5.23.0" +PROWLER_UI_VERSION="5.24.0" +PROWLER_API_VERSION="5.24.0" ``` diff --git a/docs/getting-started/products/prowler-lighthouse-ai.mdx b/docs/getting-started/products/prowler-lighthouse-ai.mdx index 6f197ae546..4e2eed8538 100644 --- a/docs/getting-started/products/prowler-lighthouse-ai.mdx +++ b/docs/getting-started/products/prowler-lighthouse-ai.mdx @@ -59,6 +59,10 @@ Prowler Lighthouse AI is powerful, but there are limitations: - **NextJS session dependence**: If your Prowler application session expires or logs out, Lighthouse AI will error out. Refresh and log back in to continue. - **Response quality**: The response quality depends on the selected LLM provider and model. Choose models with strong tool-calling capabilities for best results. We recommend `gpt-5` model from OpenAI. +## Architecture + +![Prowler Lighthouse Architecture](/images/lighthouse-architecture.png) + ## Extending Lighthouse AI Lighthouse AI retrieves data through Prowler MCP. To add new capabilities, extend the Prowler MCP Server with additional tools and Lighthouse AI discovers them automatically. diff --git a/docs/getting-started/products/prowler-mcp.mdx b/docs/getting-started/products/prowler-mcp.mdx index ea6fe8fcc0..762b088326 100644 --- a/docs/getting-started/products/prowler-mcp.mdx +++ b/docs/getting-started/products/prowler-mcp.mdx @@ -46,8 +46,7 @@ Search and retrieve official Prowler documentation: The following diagram illustrates the Prowler MCP Server architecture and its integration points: -Prowler MCP Server Schema -Prowler MCP Server Schema +![Prowler MCP Server Schema](/images/prowler_mcp_schema.png) The architecture shows how AI assistants connect through the MCP protocol to access Prowler's three main components: - Prowler Cloud/App for security operations diff --git a/docs/images/finding-groups-drawer.png b/docs/images/finding-groups-drawer.png new file mode 100644 index 0000000000..2c07f63249 Binary files /dev/null and b/docs/images/finding-groups-drawer.png differ diff --git a/docs/images/finding-groups-expanded.png b/docs/images/finding-groups-expanded.png new file mode 100644 index 0000000000..677df0020f Binary files /dev/null and b/docs/images/finding-groups-expanded.png differ diff --git a/docs/images/finding-groups-list.png b/docs/images/finding-groups-list.png new file mode 100644 index 0000000000..e70d2bb969 Binary files /dev/null and b/docs/images/finding-groups-list.png differ diff --git a/docs/images/finding-groups-other-findings.png b/docs/images/finding-groups-other-findings.png new file mode 100644 index 0000000000..35605a7710 Binary files /dev/null and b/docs/images/finding-groups-other-findings.png differ diff --git a/docs/images/lighthouse-architecture-dark.png b/docs/images/lighthouse-architecture-dark.png deleted file mode 100644 index 0ed77712c5..0000000000 Binary files a/docs/images/lighthouse-architecture-dark.png and /dev/null differ diff --git a/docs/images/lighthouse-architecture-light.png b/docs/images/lighthouse-architecture-light.png deleted file mode 100644 index 076240ccb2..0000000000 Binary files a/docs/images/lighthouse-architecture-light.png and /dev/null differ diff --git a/docs/images/lighthouse-architecture.mmd b/docs/images/lighthouse-architecture.mmd new file mode 100644 index 0000000000..47407544e9 --- /dev/null +++ b/docs/images/lighthouse-architecture.mmd @@ -0,0 +1,37 @@ +flowchart TB + browser([Browser]) + + subgraph NEXTJS["Next.js Server"] + route["API Route
(auth + context assembly)"] + agent["LangChain Agent"] + + subgraph TOOLS["Agent Tools"] + metatools["Meta-tools
describe_tool / execute_tool / load_skill"] + end + + mcpclient["MCP Client
(HTTP transport)"] + end + + llm["LLM Provider
(OpenAI / Bedrock / OpenAI-compatible)"] + + subgraph MCP["Prowler MCP Server"] + app_tools["prowler_app_* tools
(auth required)"] + hub_tools["prowler_hub_* tools
(no auth)"] + docs_tools["prowler_docs_* tools
(no auth)"] + end + + api["Prowler API"] + hub["hub.prowler.com"] + docs["docs.prowler.com
(Mintlify)"] + + browser <-->|SSE stream| route + route --> agent + agent <-->|LLM API| llm + agent --> metatools + metatools --> mcpclient + mcpclient -->|MCP HTTP · Bearer token
for prowler_app_* only| app_tools + mcpclient -->|MCP HTTP| hub_tools + mcpclient -->|MCP HTTP| docs_tools + app_tools -->|REST| api + hub_tools -->|REST| hub + docs_tools -->|REST| docs diff --git a/docs/images/lighthouse-architecture.png b/docs/images/lighthouse-architecture.png new file mode 100644 index 0000000000..b2e8266154 Binary files /dev/null and b/docs/images/lighthouse-architecture.png differ diff --git a/docs/images/products/prowler-app-architecture.mmd b/docs/images/products/prowler-app-architecture.mmd index bfc687b82e..0c13d580c3 100644 --- a/docs/images/products/prowler-app-architecture.mmd +++ b/docs/images/products/prowler-app-architecture.mmd @@ -23,6 +23,8 @@ flowchart TB user --> ui user --> cli ui -->|REST| api + ui -->|MCP HTTP| mcp + mcp -->|REST| api api --> pg api --> valkey beat -->|enqueue jobs| valkey @@ -31,7 +33,5 @@ flowchart TB worker -->|Attack Paths| neo4j worker -->|invokes| sdk cli --> sdk - api -. AI tools .-> mcp - mcp -. context .-> api sdk --> providers diff --git a/docs/images/products/prowler-app-architecture.png b/docs/images/products/prowler-app-architecture.png index 19bbfab0ee..6b0dacb776 100644 Binary files a/docs/images/products/prowler-app-architecture.png and b/docs/images/products/prowler-app-architecture.png differ diff --git a/docs/images/prowler_mcp_schema.mmd b/docs/images/prowler_mcp_schema.mmd new file mode 100644 index 0000000000..96973546f6 --- /dev/null +++ b/docs/images/prowler_mcp_schema.mmd @@ -0,0 +1,29 @@ +flowchart LR + subgraph HOSTS["MCP Hosts"] + chat["Chat Interfaces
(Claude Desktop, LobeChat)"] + ide["IDEs and Code Editors
(Claude Code, Cursor)"] + apps["Other AI Applications
(5ire, custom agents)"] + end + + subgraph MCP["Prowler MCP Server"] + app_tools["prowler_app_* tools
(JWT or API key auth)
Findings · Providers · Scans
Resources · Muting · Compliance
Attack Paths"] + hub_tools["prowler_hub_* tools
(no auth)
Checks Catalog · Check Code
Fixers · Compliance Frameworks"] + docs_tools["prowler_docs_* tools
(no auth)
Search · Document Retrieval"] + end + + api["Prowler API
(REST)"] + hub["hub.prowler.com
(REST)"] + docs["docs.prowler.com
(Mintlify)"] + + chat -->|STDIO or HTTP| app_tools + chat -->|STDIO or HTTP| hub_tools + chat -->|STDIO or HTTP| docs_tools + ide -->|STDIO or HTTP| app_tools + ide -->|STDIO or HTTP| hub_tools + ide -->|STDIO or HTTP| docs_tools + apps -->|STDIO or HTTP| app_tools + apps -->|STDIO or HTTP| hub_tools + apps -->|STDIO or HTTP| docs_tools + app_tools -->|REST| api + hub_tools -->|REST| hub + docs_tools -->|REST| docs diff --git a/docs/images/prowler_mcp_schema.png b/docs/images/prowler_mcp_schema.png new file mode 100644 index 0000000000..8a8884fa5e Binary files /dev/null and b/docs/images/prowler_mcp_schema.png differ diff --git a/docs/images/prowler_mcp_schema_dark.png b/docs/images/prowler_mcp_schema_dark.png deleted file mode 100644 index 7771557601..0000000000 Binary files a/docs/images/prowler_mcp_schema_dark.png and /dev/null differ diff --git a/docs/images/prowler_mcp_schema_light.png b/docs/images/prowler_mcp_schema_light.png deleted file mode 100644 index c542d84ed2..0000000000 Binary files a/docs/images/prowler_mcp_schema_light.png and /dev/null differ diff --git a/docs/user-guide/providers/aws/regions-and-partitions.mdx b/docs/user-guide/providers/aws/regions-and-partitions.mdx index 02ae8fe6ab..377612013e 100644 --- a/docs/user-guide/providers/aws/regions-and-partitions.mdx +++ b/docs/user-guide/providers/aws/regions-and-partitions.mdx @@ -33,6 +33,41 @@ To scan a particular AWS region with Prowler, use: prowler aws -f/--region eu-west-1 us-east-1 ``` +### Excluding Specific Regions + +To scan all supported AWS regions except a specific subset, use the `--excluded-region` flag: + +```console +prowler aws --excluded-region eu-west-1 me-south-1 +``` + +You can also configure the exclusion list with the `PROWLER_AWS_DISALLOWED_REGIONS` environment variable as a comma-separated list: + +```console +export PROWLER_AWS_DISALLOWED_REGIONS="eu-west-1,me-south-1" +prowler aws +``` + +Or with the AWS provider configuration in `config.yaml`: + +```yaml +aws: + disallowed_regions: + - eu-west-1 + - me-south-1 +``` + +When more than one source is set, precedence is: + +1. `--excluded-region` +2. `PROWLER_AWS_DISALLOWED_REGIONS` +3. `aws.disallowed_regions` in `config.yaml` + + +For self-hosted App or API-triggered scans, set `PROWLER_AWS_DISALLOWED_REGIONS` in the runtime environment of the backend scan containers such as `api` and `worker`. The `ui` container does not enforce AWS region selection. + + + ### AWS Credentials Configuration For details on configuring AWS credentials, refer to the following [Botocore](https://github.com/boto/botocore) [file](https://github.com/boto/botocore/blob/22a19ea7c4c2c4dd7df4ab8c32733cba0c7597a4/botocore/data/partitions.json). diff --git a/docs/user-guide/tutorials/prowler-app-finding-groups.mdx b/docs/user-guide/tutorials/prowler-app-finding-groups.mdx new file mode 100644 index 0000000000..e1bf83a50f --- /dev/null +++ b/docs/user-guide/tutorials/prowler-app-finding-groups.mdx @@ -0,0 +1,119 @@ +--- +title: 'Finding Groups' +description: 'Organize and triage security findings by check to reduce noise and prioritize remediation effectively.' +--- + +import { VersionBadge } from "/snippets/version-badge.mdx" + + + +Finding Groups transforms security findings triage by grouping them by check instead of displaying a flat list. This dramatically reduces noise and enables faster, more effective prioritization. + +## Triage Challenges with Flat Finding Lists + +A real cloud environment produces thousands of findings per scan. A flat list makes it impossible to triage effectively: + +- **Signal buried in noise**: the same misconfiguration repeated across 200 resources shows up as 200 rows, burying the signal in repetitive data +- **Prioritization guesswork**: without grouping, understanding which issues affect the most resources requires manual counting and correlation +- **Tedious muting**: muting a false positive globally requires manually acting on each individual finding across the list +- **Lost context**: when investigating a single resource, related findings are scattered across the same flat list, making it hard to see the full picture + +## How Finding Groups Addresses These Challenges + +Finding Groups addresses these challenges by intelligently grouping findings by check. + +### Grouped View at a Glance + +Each row represents a single check title with key information immediately visible: + +- **Severity** indicator for quick risk assessment +- **Impacted providers** showing which cloud platforms are affected +- **X of Y impacted resources** counter displaying how many resources fail this check + +For example, `Vercel project has the Web Application Firewall enabled` across every affected project collapses to a single row — not one per project. Sort or filter by severity, provider, or status at the group level to triage top-down instead of drowning in per-resource rows. + +![Finding Groups list view](/images/finding-groups-list.png) + +### Expanding Groups for Details + +Expand any group inline to see the failing resources with detailed information: + +| Column | Description | +|--------|-------------| +| **UID** | Unique identifier for the resource | +| **Service** | The cloud service the resource belongs to | +| **Region** | Geographic region where the resource is deployed | +| **Severity** | Risk level of the finding | +| **Provider** | Cloud provider (AWS, Azure, GCP, Kubernetes, etc.) | +| **Last Seen** | When the finding was last detected | +| **Failing For** | Duration the resource has been in a failing state | + +![Finding Groups expanded view](/images/finding-groups-expanded.png) + +### Resource Detail Drawer + +Select any resource to open the detail drawer with full finding context: + +- **Risk**: the security risk associated with this finding +- **Description**: detailed explanation of what was detected +- **Status Extended**: additional status information and context +- **Remediation**: step-by-step guidance to resolve the issue +- **View in Prowler Hub**: direct link to explore the check in Prowler Hub +- **Analyze This Finding With Lighthouse AI**: one-click AI-powered analysis for deeper insights + +![Finding Groups resource detail drawer](/images/finding-groups-drawer.png) + +### Bulk Actions + +Bulk-mute an entire group instead of chasing duplicates across the list. This is especially useful for: + +- Known false positives that appear across many resources +- Findings in development or test environments +- Accepted risks that have been documented and approved + + +Muting findings does not resolve underlying security issues. Review each finding carefully before muting to ensure it represents an acceptable risk or has been properly addressed. + + +## Other Findings for This Resource + +Inside the resource detail drawer, the **Other Findings For This Resource** tab lists every finding that hits the same resource — passing, failing, and muted — alongside the one currently being reviewed. + +![Other Findings For This Resource tab](/images/finding-groups-other-findings.png) + +### Why This Matters + +When reviewing "WAF not enabled" on a Vercel project, the tab immediately shows: + +- Skew protection status +- Rate limiting configuration +- IP blocking settings +- Custom firewall rules +- Password protection findings + +All for that same project, without navigating back to the main list and filtering by resource UID. + +### Complete Context Within the Drawer + +Pair the Other Findings tab with: + +- **Scans tab**: scan history for this resource +- **Events tab**: changes and events over time + +This provides full context without leaving the drawer. + +## Best Practices + +1. **Start with high severity groups**: focus on critical and high severity groups first for maximum impact. +2. **Use filters strategically**: filter by provider or status at the group level to narrow the triage scope. +3. **Leverage bulk mute**: when a finding represents a confirmed false positive, mute the entire group at once. +4. **Check related findings**: review the Other Findings tab to understand the full security posture of a resource. +5. **Track failure duration**: use the "Failing For" column to prioritize long-standing issues that may indicate systemic problems. + +## Getting Started + +1. Navigate to the **Findings** section in Prowler Cloud/App. +2. Toggle to the **Grouped View** to see findings organized by check. +3. Select any group row to expand and see affected resources. +4. Select a resource to open the detail drawer with full context. +5. Use the **Other Findings For This Resource** tab to see all findings for that resource. diff --git a/docs/user-guide/tutorials/prowler-app-lighthouse.mdx b/docs/user-guide/tutorials/prowler-app-lighthouse.mdx index cf4886d2ee..b119893cb5 100644 --- a/docs/user-guide/tutorials/prowler-app-lighthouse.mdx +++ b/docs/user-guide/tutorials/prowler-app-lighthouse.mdx @@ -25,8 +25,7 @@ Behind the scenes, Lighthouse AI works as follows: Lighthouse AI supports multiple LLM providers including OpenAI, Amazon Bedrock, and OpenAI-compatible services. For configuration details, see [Using Multiple LLM Providers with Lighthouse](/user-guide/tutorials/prowler-app-lighthouse-multi-llm). -Prowler Lighthouse Architecture -Prowler Lighthouse Architecture +![Prowler Lighthouse Architecture](/images/lighthouse-architecture.png) diff --git a/poetry.lock b/poetry.lock index 7d5ccdf355..3b2ba9c4a5 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.3.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.3.4 and should not be changed by hand. [[package]] name = "about-time" @@ -1267,19 +1267,19 @@ typing-extensions = ">=4.6.0" [[package]] name = "azure-mgmt-resource" -version = "23.3.0" +version = "24.0.0" description = "Microsoft Azure Resource Management Client Library for Python" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["main"] files = [ - {file = "azure_mgmt_resource-23.3.0-py3-none-any.whl", hash = "sha256:ab216ee28e29db6654b989746e0c85a1181f66653929d2cb6e48fba66d9af323"}, - {file = "azure_mgmt_resource-23.3.0.tar.gz", hash = "sha256:fc4f1fd8b6aad23f8af4ed1f913df5f5c92df117449dc354fea6802a2829fea4"}, + {file = "azure_mgmt_resource-24.0.0-py3-none-any.whl", hash = "sha256:27b32cd223e2784269f5a0db3c282042886ee4072d79cedc638438ece7cd0df4"}, + {file = "azure_mgmt_resource-24.0.0.tar.gz", hash = "sha256:cf6b8995fcdd407ac9ff1dd474087129429a1d90dbb1ac77f97c19b96237b265"}, ] [package.dependencies] azure-common = ">=1.1" -azure-mgmt-core = ">=1.3.2" +azure-mgmt-core = ">=1.5.0" isodate = ">=0.6.1" typing-extensions = ">=4.6.0" @@ -1425,6 +1425,64 @@ typing-extensions = ">=4.6.0" [package.extras] aio = ["azure-core[aio] (>=1.30.0)"] +[[package]] +name = "backports-datetime-fromisoformat" +version = "2.0.3" +description = "Backport of Python 3.11's datetime.fromisoformat" +optional = false +python-versions = ">3" +groups = ["dev"] +markers = "python_version == \"3.10\"" +files = [ + {file = "backports_datetime_fromisoformat-2.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5f681f638f10588fa3c101ee9ae2b63d3734713202ddfcfb6ec6cea0778a29d4"}, + {file = "backports_datetime_fromisoformat-2.0.3-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:cd681460e9142f1249408e5aee6d178c6d89b49e06d44913c8fdfb6defda8d1c"}, + {file = "backports_datetime_fromisoformat-2.0.3-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:ee68bc8735ae5058695b76d3bb2aee1d137c052a11c8303f1e966aa23b72b65b"}, + {file = "backports_datetime_fromisoformat-2.0.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8273fe7932db65d952a43e238318966eab9e49e8dd546550a41df12175cc2be4"}, + {file = "backports_datetime_fromisoformat-2.0.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:39d57ea50aa5a524bb239688adc1d1d824c31b6094ebd39aa164d6cadb85de22"}, + {file = "backports_datetime_fromisoformat-2.0.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:ac6272f87693e78209dc72e84cf9ab58052027733cd0721c55356d3c881791cf"}, + {file = "backports_datetime_fromisoformat-2.0.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:44c497a71f80cd2bcfc26faae8857cf8e79388e3d5fbf79d2354b8c360547d58"}, + {file = "backports_datetime_fromisoformat-2.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:6335a4c9e8af329cb1ded5ab41a666e1448116161905a94e054f205aa6d263bc"}, + {file = "backports_datetime_fromisoformat-2.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e2e4b66e017253cdbe5a1de49e0eecff3f66cd72bcb1229d7db6e6b1832c0443"}, + {file = "backports_datetime_fromisoformat-2.0.3-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:43e2d648e150777e13bbc2549cc960373e37bf65bd8a5d2e0cef40e16e5d8dd0"}, + {file = "backports_datetime_fromisoformat-2.0.3-cp311-cp311-macosx_11_0_x86_64.whl", hash = "sha256:4ce6326fd86d5bae37813c7bf1543bae9e4c215ec6f5afe4c518be2635e2e005"}, + {file = "backports_datetime_fromisoformat-2.0.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7c8fac333bf860208fd522a5394369ee3c790d0aa4311f515fcc4b6c5ef8d75"}, + {file = "backports_datetime_fromisoformat-2.0.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:24a4da5ab3aa0cc293dc0662a0c6d1da1a011dc1edcbc3122a288cfed13a0b45"}, + {file = "backports_datetime_fromisoformat-2.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:58ea11e3bf912bd0a36b0519eae2c5b560b3cb972ea756e66b73fb9be460af01"}, + {file = "backports_datetime_fromisoformat-2.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:8a375c7dbee4734318714a799b6c697223e4bbb57232af37fbfff88fb48a14c6"}, + {file = "backports_datetime_fromisoformat-2.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:ac677b1664c4585c2e014739f6678137c8336815406052349c85898206ec7061"}, + {file = "backports_datetime_fromisoformat-2.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:66ce47ee1ba91e146149cf40565c3d750ea1be94faf660ca733d8601e0848147"}, + {file = "backports_datetime_fromisoformat-2.0.3-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:8b7e069910a66b3bba61df35b5f879e5253ff0821a70375b9daf06444d046fa4"}, + {file = "backports_datetime_fromisoformat-2.0.3-cp312-cp312-macosx_11_0_x86_64.whl", hash = "sha256:a3b5d1d04a9e0f7b15aa1e647c750631a873b298cdd1255687bb68779fe8eb35"}, + {file = "backports_datetime_fromisoformat-2.0.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec1b95986430e789c076610aea704db20874f0781b8624f648ca9fb6ef67c6e1"}, + {file = "backports_datetime_fromisoformat-2.0.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ffe5f793db59e2f1d45ec35a1cf51404fdd69df9f6952a0c87c3060af4c00e32"}, + {file = "backports_datetime_fromisoformat-2.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:620e8e73bd2595dfff1b4d256a12b67fce90ece3de87b38e1dde46b910f46f4d"}, + {file = "backports_datetime_fromisoformat-2.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:4cf9c0a985d68476c1cabd6385c691201dda2337d7453fb4da9679ce9f23f4e7"}, + {file = "backports_datetime_fromisoformat-2.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:d144868a73002e6e2e6fef72333e7b0129cecdd121aa8f1edba7107fd067255d"}, + {file = "backports_datetime_fromisoformat-2.0.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:e81b26497a17c29595bc7df20bc6a872ceea5f8c9d6537283945d4b6396aec10"}, + {file = "backports_datetime_fromisoformat-2.0.3-cp38-cp38-macosx_11_0_universal2.whl", hash = "sha256:5ba00ead8d9d82fd6123eb4891c566d30a293454e54e32ff7ead7644f5f7e575"}, + {file = "backports_datetime_fromisoformat-2.0.3-cp38-cp38-macosx_11_0_x86_64.whl", hash = "sha256:24d574cb4072e1640b00864e94c4c89858033936ece3fc0e1c6f7179f120d0a8"}, + {file = "backports_datetime_fromisoformat-2.0.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9735695a66aad654500b0193525e590c693ab3368478ce07b34b443a1ea5e824"}, + {file = "backports_datetime_fromisoformat-2.0.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:63d39709e17eb72685d052ac82acf0763e047f57c86af1b791505b1fec96915d"}, + {file = "backports_datetime_fromisoformat-2.0.3-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:1ea2cc84224937d6b9b4c07f5cb7c667f2bde28c255645ba27f8a675a7af8234"}, + {file = "backports_datetime_fromisoformat-2.0.3-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:4024e6d35a9fdc1b3fd6ac7a673bd16cb176c7e0b952af6428b7129a70f72cce"}, + {file = "backports_datetime_fromisoformat-2.0.3-cp38-cp38-win_amd64.whl", hash = "sha256:5e2dcc94dc9c9ab8704409d86fcb5236316e9dcef6feed8162287634e3568f4c"}, + {file = "backports_datetime_fromisoformat-2.0.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:fa2de871801d824c255fac7e5e7e50f2be6c9c376fd9268b40c54b5e9da91f42"}, + {file = "backports_datetime_fromisoformat-2.0.3-cp39-cp39-macosx_11_0_universal2.whl", hash = "sha256:1314d4923c1509aa9696712a7bc0c7160d3b7acf72adafbbe6c558d523f5d491"}, + {file = "backports_datetime_fromisoformat-2.0.3-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:b750ecba3a8815ad8bc48311552f3f8ab99dd2326d29df7ff670d9c49321f48f"}, + {file = "backports_datetime_fromisoformat-2.0.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b2d5117dce805d8a2f78baeddc8c6127281fa0a5e2c40c6dd992ba6b2b367876"}, + {file = "backports_datetime_fromisoformat-2.0.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fb35f607bd1cbe37b896379d5f5ed4dc298b536f4b959cb63180e05cacc0539d"}, + {file = "backports_datetime_fromisoformat-2.0.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:61c74710900602637d2d145dda9720c94e303380803bf68811b2a151deec75c2"}, + {file = "backports_datetime_fromisoformat-2.0.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:ece59af54ebf67ecbfbbf3ca9066f5687879e36527ad69d8b6e3ac565d565a62"}, + {file = "backports_datetime_fromisoformat-2.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:d0a7c5f875068efe106f62233bc712d50db4d07c13c7db570175c7857a7b5dbd"}, + {file = "backports_datetime_fromisoformat-2.0.3-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:90e202e72a3d5aae673fcc8c9a4267d56b2f532beeb9173361293625fe4d2039"}, + {file = "backports_datetime_fromisoformat-2.0.3-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2df98ef1b76f5a58bb493dda552259ba60c3a37557d848e039524203951c9f06"}, + {file = "backports_datetime_fromisoformat-2.0.3-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7100adcda5e818b5a894ad0626e38118bb896a347f40ebed8981155675b9ba7b"}, + {file = "backports_datetime_fromisoformat-2.0.3-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5e410383f5d6a449a529d074e88af8bc80020bb42b402265f9c02c8358c11da5"}, + {file = "backports_datetime_fromisoformat-2.0.3-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f2797593760da6bcc32c4a13fa825af183cd4bfd333c60b3dbf84711afca26ef"}, + {file = "backports_datetime_fromisoformat-2.0.3-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:35a144fd681a0bea1013ccc4cd3fd4dc758ea17ee23dca019c02b82ec46fc0c4"}, + {file = "backports_datetime_fromisoformat-2.0.3.tar.gz", hash = "sha256:b58edc8f517b66b397abc250ecc737969486703a66eb97e01e6d51291b1a139d"}, +] + [[package]] name = "bandit" version = "1.8.3" @@ -3350,23 +3408,19 @@ files = [ [[package]] name = "marshmallow" -version = "3.26.2" +version = "4.3.0" description = "A lightweight library for converting complex datatypes to and from native Python datatypes." optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["dev"] files = [ - {file = "marshmallow-3.26.2-py3-none-any.whl", hash = "sha256:013fa8a3c4c276c24d26d84ce934dc964e2aa794345a0f8c7e5a7191482c8a73"}, - {file = "marshmallow-3.26.2.tar.gz", hash = "sha256:bbe2adb5a03e6e3571b573f42527c6fe926e17467833660bebd11593ab8dfd57"}, + {file = "marshmallow-4.3.0-py3-none-any.whl", hash = "sha256:46c4fe6984707e3cbd485dfebbf0a59874f58d695aad05c1668d15e8c6e13b46"}, + {file = "marshmallow-4.3.0.tar.gz", hash = "sha256:fb43c53b3fe240b8f6af37223d6ef1636f927ad9bea8ab323afad95dff090880"}, ] [package.dependencies] -packaging = ">=17.0" - -[package.extras] -dev = ["marshmallow[tests]", "pre-commit (>=3.5,<5.0)", "tox"] -docs = ["autodocsumm (==0.2.14)", "furo (==2024.8.6)", "sphinx (==8.1.3)", "sphinx-copybutton (==0.5.2)", "sphinx-issues (==5.0.0)", "sphinxext-opengraph (==0.9.1)"] -tests = ["pytest", "simplejson"] +backports-datetime-fromisoformat = {version = "*", markers = "python_version < \"3.11\""} +typing-extensions = {version = "*", markers = "python_version < \"3.11\""} [[package]] name = "mccabe" @@ -3662,14 +3716,14 @@ dev = ["bumpver", "isort", "mypy", "pylint", "pytest", "yapf"] [[package]] name = "msgraph-sdk" -version = "1.23.0" +version = "1.55.0" description = "The Microsoft Graph Python SDK" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "msgraph_sdk-1.23.0-py3-none-any.whl", hash = "sha256:58e0047b4ca59fd82022c02cd73fec0170a3d84f3b76721e3db2a0314df9a58a"}, - {file = "msgraph_sdk-1.23.0.tar.gz", hash = "sha256:6dd1ba9a46f5f0ce8599fd9610133adbd9d1493941438b5d3632fce9e55ed607"}, + {file = "msgraph_sdk-1.55.0-py3-none-any.whl", hash = "sha256:c8e68ebc4b88af5111de312e7fa910a4e76ddf48a4534feadb1fb8a411c48cfc"}, + {file = "msgraph_sdk-1.55.0.tar.gz", hash = "sha256:6df691a31954a050d26b8a678968017e157d940fb377f2a8a4e17a9741b98756"}, ] [package.dependencies] @@ -6681,4 +6735,4 @@ files = [ [metadata] lock-version = "2.1" python-versions = ">=3.10,<3.13" -content-hash = "786921163bb46716defae1d9de1df001af2abf17edd3061165638707bcd28ce4" +content-hash = "09ce4507a464b318702ed8c6a738f3bb1bc4cc6ff5a50a9c2884f560af9ab034" diff --git a/prowler/CHANGELOG.md b/prowler/CHANGELOG.md index 44c30bb3c5..eff2506c04 100644 --- a/prowler/CHANGELOG.md +++ b/prowler/CHANGELOG.md @@ -2,7 +2,23 @@ All notable changes to the **Prowler SDK** are documented in this file. -## [5.24.0] (Prowler UNRELEASED) +## [5.24.1] (Prowler v5.24.1) + +### 🔄 Changed + +- `msgraph-sdk` from 1.23.0 to 1.55.0 and `azure-mgmt-resource` from 23.3.0 to 24.0.0, removing `marshmallow` as is a transitively dev dependency [(#10733)](https://github.com/prowler-cloud/prowler/pull/10733) + +### 🐞 Fixed + +- Cloudflare account-scoped API tokens failing connection test in the App with `CloudflareUserTokenRequiredError` [(#10723)](https://github.com/prowler-cloud/prowler/pull/10723) +- `prowler image --registry-list` crashes with `AttributeError` because `ImageProvider.__init__` returns early before registering the global provider [(#10691)](https://github.com/prowler-cloud/prowler/pull/10691) +- Google Workspace Calendar checks false FAIL on unconfigured settings with secure Google defaults [(#10726)](https://github.com/prowler-cloud/prowler/pull/10726) +- Google Workspace Drive checks false FAIL on unconfigured settings with secure Google defaults [(#10727)](https://github.com/prowler-cloud/prowler/pull/10727) +- Cloudflare `validate_credentials` can hang in an infinite pagination loop when the SDK repeats accounts, blocking connection tests [(#10771)](https://github.com/prowler-cloud/prowler/pull/10771) + +--- + +## [5.24.0] (Prowler v5.24.0) ### 🚀 Added @@ -13,17 +29,22 @@ All notable changes to the **Prowler SDK** are documented in this file. - `iam_role_access_not_stale_to_bedrock` and `iam_user_access_not_stale_to_bedrock` checks for AWS provider [(#10536)](https://github.com/prowler-cloud/prowler/pull/10536) - `iam_policy_no_wildcard_marketplace_subscribe` and `iam_inline_policy_no_wildcard_marketplace_subscribe` checks for AWS provider [(#10525)](https://github.com/prowler-cloud/prowler/pull/10525) - `bedrock_vpc_endpoints_configured` check for AWS provider [(#10591)](https://github.com/prowler-cloud/prowler/pull/10591) -- `exchange_organization_delicensing_resiliency_enabled` check for m365 provider [(#10608)](https://github.com/prowler-cloud/prowler/pull/10608) +- `exchange_organization_delicensing_resiliency_enabled` check for M365 provider [(#10608)](https://github.com/prowler-cloud/prowler/pull/10608) - `entra_conditional_access_policy_mfa_enforced_for_guest_users` check for M365 provider [(#10616)](https://github.com/prowler-cloud/prowler/pull/10616) -- `entra_conditional_access_policy_corporate_device_sign_in_frequency_enforced` check for m365 provider [(#10618)](https://github.com/prowler-cloud/prowler/pull/10618) -- `entra_conditional_access_policy_block_unknown_device_platforms` check for m365 provider [(#10615)](https://github.com/prowler-cloud/prowler/pull/10615) -- Support for external/custom providers, checks, and compliance frameworks without modifying core code [(#10700)](https://github.com/prowler-cloud/prowler/pull/10700) +- `entra_conditional_access_policy_corporate_device_sign_in_frequency_enforced` check for M365 provider [(#10618)](https://github.com/prowler-cloud/prowler/pull/10618) +- `entra_conditional_access_policy_block_unknown_device_platforms` check for M365 provider [(#10615)](https://github.com/prowler-cloud/prowler/pull/10615) +- `--excluded-region` CLI flag, `PROWLER_AWS_DISALLOWED_REGIONS` environment variable, and `aws.disallowed_regions` config entry to skip specific AWS regions during scans [(#10688)](https://github.com/prowler-cloud/prowler/pull/10688) ### 🔄 Changed - Bump Poetry to `2.3.4` and consolidate SDK workflows onto the `setup-python-poetry` composite action with opt-in lockfile regeneration [(#10681)](https://github.com/prowler-cloud/prowler/pull/10681) - Normalize Conditional Access platform values in Entra models and simplify platform-based checks [(#10635)](https://github.com/prowler-cloud/prowler/pull/10635) +### 🐞 Fixed + +- `prowler image --registry-list` crashes with `AttributeError` because `ImageProvider.__init__` returns early before registering the global provider [(#10691)](https://github.com/prowler-cloud/prowler/pull/10691) +- Vercel firewall config handling for team-scoped projects and current API response shapes [(#10695)](https://github.com/prowler-cloud/prowler/pull/10695) + --- ## [5.23.0] (Prowler v5.23.0) @@ -788,7 +809,7 @@ All notable changes to the **Prowler SDK** are documented in this file. - S3 `test_connection` uses AWS S3 API `HeadBucket` instead of `GetBucketLocation` [(#8456)](https://github.com/prowler-cloud/prowler/pull/8456) - Add more validations to Azure Storage models when some values are None to avoid serialization issues [(#8325)](https://github.com/prowler-cloud/prowler/pull/8325) - `sns_topics_not_publicly_accessible` false positive with `aws:SourceArn` conditions [(#8326)](https://github.com/prowler-cloud/prowler/issues/8326) -- Remove typo from description req 1.2.3 - Prowler ThreatScore m365 [(#8384)](https://github.com/prowler-cloud/prowler/pull/8384) +- Remove typo from description req 1.2.3 - Prowler ThreatScore M365 [(#8384)](https://github.com/prowler-cloud/prowler/pull/8384) - Way of counting FAILED/PASS reqs from `kisa_isms_p_2023_aws` table [(#8382)](https://github.com/prowler-cloud/prowler/pull/8382) - Use default tenant domain instead of first domain in list for Azure and M365 providers [(#8402)](https://github.com/prowler-cloud/prowler/pull/8402) - Avoid multiple module error calls in M365 provider [(#8353)](https://github.com/prowler-cloud/prowler/pull/8353) @@ -829,7 +850,7 @@ All notable changes to the **Prowler SDK** are documented in this file. - Title & description wording for `iam_user_accesskey_unused` check for AWS provider [(#8233)](https://github.com/prowler-cloud/prowler/pull/8233) - Add GitHub provider to lateral panel in documentation and change -h environment variable output [(#8246)](https://github.com/prowler-cloud/prowler/pull/8246) -- Show `m365_identity_type` and `m365_identity_id` in cloud reports [(#8247)](https://github.com/prowler-cloud/prowler/pull/8247) +- Show `M365_identity_type` and `M365_identity_id` in cloud reports [(#8247)](https://github.com/prowler-cloud/prowler/pull/8247) - Ensure `is_service_role` only returns `True` for service roles [(#8274)](https://github.com/prowler-cloud/prowler/pull/8274) - Update DynamoDB check metadata to fix broken link [(#8273)](https://github.com/prowler-cloud/prowler/pull/8273) - Show correct count of findings in Dashboard Security Posture page [(#8270)](https://github.com/prowler-cloud/prowler/pull/8270) @@ -951,9 +972,9 @@ All notable changes to the **Prowler SDK** are documented in this file. ### Fixed -- `m365_powershell test_credentials` to use sanitized credentials [(#7761)](https://github.com/prowler-cloud/prowler/pull/7761) +- `M365_powershell test_credentials` to use sanitized credentials [(#7761)](https://github.com/prowler-cloud/prowler/pull/7761) - `admincenter_users_admins_reduced_license_footprint` check logic to pass when admin user has no license [(#7779)](https://github.com/prowler-cloud/prowler/pull/7779) -- `m365_powershell` to close the PowerShell sessions in msgraph services [(#7816)](https://github.com/prowler-cloud/prowler/pull/7816) +- `M365_powershell` to close the PowerShell sessions in msgraph services [(#7816)](https://github.com/prowler-cloud/prowler/pull/7816) - `defender_ensure_notify_alerts_severity_is_high`check to accept high or lower severity [(#7862)](https://github.com/prowler-cloud/prowler/pull/7862) - Replace `Directory.Read.All` permission with `Domain.Read.All` which is more restrictive [(#7888)](https://github.com/prowler-cloud/prowler/pull/7888) - Split calls to list Azure Functions attributes [(#7778)](https://github.com/prowler-cloud/prowler/pull/7778) @@ -1027,7 +1048,7 @@ All notable changes to the **Prowler SDK** are documented in this file. - New check `teams_meeting_chat_anonymous_users_disabled` [(#7579)](https://github.com/prowler-cloud/prowler/pull/7579) - Prowler Threat Score Compliance Framework [(#7603)](https://github.com/prowler-cloud/prowler/pull/7603) - Documentation for M365 provider [(#7622)](https://github.com/prowler-cloud/prowler/pull/7622) -- Support for m365 provider in Prowler Dashboard [(#7633)](https://github.com/prowler-cloud/prowler/pull/7633) +- Support for M365 provider in Prowler Dashboard [(#7633)](https://github.com/prowler-cloud/prowler/pull/7633) - New check for Modern Authentication enabled for Exchange Online in M365 [(#7636)](https://github.com/prowler-cloud/prowler/pull/7636) - New check `sharepoint_onedrive_sync_restricted_unmanaged_devices` [(#7589)](https://github.com/prowler-cloud/prowler/pull/7589) - New check for Additional Storage restricted for Exchange in M365 [(#7638)](https://github.com/prowler-cloud/prowler/pull/7638) @@ -1037,7 +1058,7 @@ All notable changes to the **Prowler SDK** are documented in this file. - New check for MailTips full enabled for Exchange in M365 [(#7637)](https://github.com/prowler-cloud/prowler/pull/7637) - New check for Comprehensive Attachments Filter Applied for Defender in M365 [(#7661)](https://github.com/prowler-cloud/prowler/pull/7661) - Modified check `exchange_mailbox_properties_auditing_enabled` to make it configurable [(#7662)](https://github.com/prowler-cloud/prowler/pull/7662) -- snapshots to m365 documentation [(#7673)](https://github.com/prowler-cloud/prowler/pull/7673) +- snapshots to M365 documentation [(#7673)](https://github.com/prowler-cloud/prowler/pull/7673) - support for static credentials for sending findings to Amazon S3 and AWS Security Hub [(#7322)](https://github.com/prowler-cloud/prowler/pull/7322) - Prowler ThreatScore for M365 provider [(#7692)](https://github.com/prowler-cloud/prowler/pull/7692) - Microsoft User and User Credential auth to reports [(#7681)](https://github.com/prowler-cloud/prowler/pull/7681) diff --git a/prowler/__main__.py b/prowler/__main__.py index a0e94a885f..5768be8884 100644 --- a/prowler/__main__.py +++ b/prowler/__main__.py @@ -293,6 +293,10 @@ def prowler(): if not args.only_logs: global_provider.print_credentials() + # --registry-list: listing already printed during provider init, exit + if getattr(global_provider, "_listing_only", False): + sys.exit() + # Skip service and check loading for external-tool providers if provider not in EXTERNAL_TOOL_PROVIDERS: # Import custom checks from folder @@ -1338,8 +1342,12 @@ def prowler(): global_provider.identity.audited_regions, ) if not global_provider.identity.audited_regions - else global_provider.identity.audited_regions + else set(global_provider.identity.audited_regions) ) + if global_provider._enabled_regions is not None: + security_hub_regions = security_hub_regions.intersection( + global_provider._enabled_regions + ) security_hub = SecurityHub( aws_account_id=global_provider.identity.account, diff --git a/prowler/config/config.py b/prowler/config/config.py index 1d3bf78303..1530f10e72 100644 --- a/prowler/config/config.py +++ b/prowler/config/config.py @@ -39,7 +39,7 @@ class _MutableTimestamp: timestamp = _MutableTimestamp(datetime.today()) timestamp_utc = _MutableTimestamp(datetime.now(timezone.utc)) -prowler_version = "5.24.0" +prowler_version = "5.25.0" html_logo_url = "https://github.com/prowler-cloud/prowler/" square_logo_img = "https://raw.githubusercontent.com/prowler-cloud/prowler/dc7d2d5aeb92fdf12e8604f42ef6472cd3e8e889/docs/img/prowler-logo-black.png" aws_logo = "https://user-images.githubusercontent.com/38561120/235953920-3e3fba08-0795-41dc-b480-9bea57db9f2e.png" diff --git a/prowler/config/config.yaml b/prowler/config/config.yaml index c645d6e25b..024cd974f1 100644 --- a/prowler/config/config.yaml +++ b/prowler/config/config.yaml @@ -3,6 +3,10 @@ aws: # AWS Global Configuration # aws.mute_non_default_regions --> Set to True to muted failed findings in non-default regions for AccessAnalyzer, GuardDuty, SecurityHub, DRS and Config mute_non_default_regions: False + # aws.disallowed_regions --> List of AWS regions to exclude from the scan. + # Also settable via the PROWLER_AWS_DISALLOWED_REGIONS environment variable or + # the --excluded-region CLI flag. Precedence: CLI > env var > config file. + # disallowed_regions: [] # If you want to mute failed findings only in specific regions, create a file with the following syntax and run it with `prowler aws -w mutelist.yaml`: # Mutelist: # Accounts: diff --git a/prowler/lib/check/compliance_models.py b/prowler/lib/check/compliance_models.py index 71e53c5592..0027092a16 100644 --- a/prowler/lib/check/compliance_models.py +++ b/prowler/lib/check/compliance_models.py @@ -1,10 +1,11 @@ import importlib.metadata +import json import os import sys from enum import Enum from typing import Optional, Union -from pydantic.v1 import BaseModel, ValidationError, root_validator +from pydantic.v1 import BaseModel, Field, ValidationError, root_validator from prowler.lib.check.utils import list_compliance_modules from prowler.lib.logger import logger @@ -460,3 +461,482 @@ def load_compliance_framework( sys.exit(1) else: return compliance_framework + + +# ─── Universal Compliance Schema Models (Phase 1-3) ───────────────────────── + + +class OutputFormats(BaseModel): + """Flags indicating in which output formats an attribute should be included.""" + + csv: bool = True + ocsf: bool = True + + +class AttributeMetadata(BaseModel): + """Schema descriptor for a single attribute field in a universal compliance framework.""" + + key: str + label: Optional[str] = None + type: str = "str" # str, int, float, list_str, list_dict, bool + enum: Optional[list] = None + required: bool = False + enum_display: Optional[dict] = None # enum_value -> EnumValueDisplay dict + enum_order: Optional[list] = None # explicit ordering of enum values + chart_label: Optional[str] = None # axis label when used in charts + output_formats: OutputFormats = Field(default_factory=OutputFormats) + + +class SplitByConfig(BaseModel): + """Column-splitting configuration (e.g. CIS Level 1/Level 2).""" + + field: str + values: list + + +class ScoringConfig(BaseModel): + """Weighted scoring configuration (e.g. ThreatScore).""" + + risk_field: str + weight_field: str + + +class TableLabels(BaseModel): + """Custom pass/fail labels for console table rendering.""" + + pass_label: str = "PASS" + fail_label: str = "FAIL" + provider_header: str = "Provider" + group_header: Optional[str] = None + status_header: str = "Status" + title: Optional[str] = None + results_title: Optional[str] = None + footer_note: Optional[str] = None + + +class TableConfig(BaseModel): + """Declarative rendering instructions for the console compliance table.""" + + group_by: str + split_by: Optional[SplitByConfig] = None + scoring: Optional[ScoringConfig] = None + labels: Optional[TableLabels] = None + + +class EnumValueDisplay(BaseModel): + """Per-enum-value visual metadata for PDF rendering. + + Replaces hardcoded DIMENSION_MAPPING, TIPO_ICONS, nivel colors. + """ + + label: Optional[str] = None # "Trazabilidad" + abbreviation: Optional[str] = None # "T" + color: Optional[str] = None # "#4286F4" + icon: Optional[str] = None # emoji + + +class ChartConfig(BaseModel): + """Declarative chart description for PDF reports.""" + + id: str + type: str # vertical_bar | horizontal_bar | radar + group_by: str # attribute key to group by + title: Optional[str] = None + x_label: Optional[str] = None + y_label: Optional[str] = None + value_source: str = "compliance_percent" + color_mode: str = "by_value" # by_value | fixed | by_group + fixed_color: Optional[str] = None + + +class ScoringFormula(BaseModel): + """Weighted scoring formula (e.g. ThreatScore).""" + + risk_field: str # "LevelOfRisk" + weight_field: str # "Weight" + risk_boost_factor: float = 0.25 # rfac = 1 + factor * risk_level + + +class CriticalRequirementsFilter(BaseModel): + """Filter for critical requirements section in PDF reports.""" + + filter_field: str # "LevelOfRisk" + min_value: Optional[int] = None # 4 (int-based filter) + filter_value: Optional[str] = None # "alto" (string-based filter) + status_filter: str = "FAIL" + title: Optional[str] = None # "Critical Failed Requirements" + + +class ReportFilter(BaseModel): + """Default report filtering for PDF generation.""" + + only_failed: bool = True + include_manual: bool = False + + +class I18nLabels(BaseModel): + """Localized labels for PDF report rendering.""" + + report_title: Optional[str] = None + page_label: str = "Page" + powered_by: str = "Powered by Prowler" + framework_label: str = "Framework:" + version_label: str = "Version:" + provider_label: str = "Provider:" + description_label: str = "Description:" + compliance_score_label: str = "Compliance Score by Sections" + requirements_index_label: str = "Requirements Index" + detailed_findings_label: str = "Detailed Findings" + + +class PDFConfig(BaseModel): + """Declarative PDF report configuration. + + Drives the API report generator from JSON data instead of hardcoded + Python config. Colors are hex strings (e.g. '#336699'). + """ + + language: str = "en" + logo_filename: Optional[str] = None + primary_color: Optional[str] = None + secondary_color: Optional[str] = None + bg_color: Optional[str] = None + sections: Optional[list] = None + section_short_names: Optional[dict] = None + group_by_field: Optional[str] = None + sub_group_by_field: Optional[str] = None + section_titles: Optional[dict] = None + charts: Optional[list] = None + scoring: Optional[ScoringFormula] = None + critical_filter: Optional[CriticalRequirementsFilter] = None + filter: Optional[ReportFilter] = None + labels: Optional[I18nLabels] = None + + +class UniversalComplianceRequirement(BaseModel): + """Universal requirement with flat dict-based attributes.""" + + id: str + description: str + name: Optional[str] = None + attributes: dict = Field(default_factory=dict) + checks: dict[str, list[str]] = Field(default_factory=dict) + tactics: Optional[list] = None + sub_techniques: Optional[list] = None + platforms: Optional[list] = None + technique_url: Optional[str] = None + + +class OutputsConfig(BaseModel): + """Container for output-related configuration (table, PDF, etc.).""" + + table_config: Optional[TableConfig] = None + pdf_config: Optional[PDFConfig] = None + + +class ComplianceFramework(BaseModel): + """Universal top-level container for any compliance framework. + + Provider may be explicit (single-provider JSON) or derived from checks + keys across all requirements. + """ + + framework: str + name: str + provider: Optional[str] = None + version: Optional[str] = None + description: str + icon: Optional[str] = None + requirements: list[UniversalComplianceRequirement] + attributes_metadata: Optional[list[AttributeMetadata]] = None + outputs: Optional[OutputsConfig] = None + + @root_validator + # noqa: F841 - since vulture raises unused variable 'cls' + def validate_attributes_against_metadata(cls, values): # noqa: F841 + """Validate every Requirement's attributes dict against attributes_metadata. + + Checks: + - Required keys (required=True) must be present in each Requirement. + - Enum-constrained keys must have a value within the declared enum list. + - Basic type validation (int, float, bool) for non-None values. + """ + metadata = values.get("attributes_metadata") + requirements = values.get("requirements", []) + if not metadata: + return values + + required_keys = {m.key for m in metadata if m.required} + valid_keys = {m.key for m in metadata} + enum_map = {m.key: m.enum for m in metadata if m.enum} + type_map = {m.key: m.type for m in metadata} + + type_checks = { + "int": int, + "float": (int, float), + "bool": bool, + } + + errors = [] + for req in requirements: + attrs = req.attributes + + # Required keys + for key in required_keys: + if key not in attrs or attrs[key] is None: + errors.append( + f"Requirement '{req.id}': missing required attribute '{key}'" + ) + + # Unknown keys — anything outside the declared schema is a typo or drift + unknown_keys = set(attrs) - valid_keys + for key in sorted(unknown_keys): + errors.append( + f"Requirement '{req.id}': unknown attribute '{key}' " + f"(not declared in attributes_metadata)" + ) + + # Enum validation + for key, allowed in enum_map.items(): + if key in attrs and attrs[key] is not None: + if attrs[key] not in allowed: + errors.append( + f"Requirement '{req.id}': attribute '{key}' value " + f"'{attrs[key]}' not in {allowed}" + ) + + # Type validation for non-string types + for key in attrs: + if key not in valid_keys or attrs[key] is None: + continue + expected_type = type_map.get(key, "str") + py_type = type_checks.get(expected_type) + if py_type and not isinstance(attrs[key], py_type): + errors.append( + f"Requirement '{req.id}': attribute '{key}' expected " + f"type {expected_type}, got {type(attrs[key]).__name__}" + ) + + if errors: + detail = "\n ".join(errors) + raise ValueError(f"attributes_metadata validation failed:\n {detail}") + + return values + + def get_providers(self) -> list: + """Derive the set of providers this framework supports. + + Inspects checks keys across all requirements. Falls back to the + explicit provider field for single-provider frameworks with no + requirement-level checks. + """ + providers = set() + for req in self.requirements: + providers.update(k.lower() for k in req.checks.keys()) + if self.provider and not providers: + providers.add(self.provider.lower()) + return sorted(providers) + + def supports_provider(self, provider: str) -> bool: + """Return True if this framework has checks for the given provider.""" + provider_lower = provider.lower() + for req in self.requirements: + if any(k.lower() == provider_lower for k in req.checks.keys()): + return True + return self.provider is not None and self.provider.lower() == provider_lower + + +# ─── Legacy-to-Universal Adapter (Phase 2) ────────────────────────────────── + + +def _infer_attribute_metadata(legacy: Compliance) -> Optional[list[AttributeMetadata]]: + """Introspect the first requirement's attribute model to build attributes_metadata.""" + try: + if not legacy.Requirements: + return None + + first_req = legacy.Requirements[0] + + # MITRE requirements have Tactics at top level, not in Attributes + if isinstance(first_req, Mitre_Requirement): + return None + + if not first_req.Attributes: + return None + + sample_attr = first_req.Attributes[0] + metadata = [] + + for field_name, field_obj in sample_attr.__fields__.items(): + field_type = field_obj.outer_type_ + type_str = "str" + enum_values = None + + origin = getattr(field_type, "__origin__", None) + if field_type is int: + type_str = "int" + elif field_type is float: + type_str = "float" + elif field_type is bool: + type_str = "bool" + elif origin is list: + args = getattr(field_type, "__args__", ()) + if args and args[0] is dict: + type_str = "list_dict" + else: + type_str = "list_str" + elif isinstance(field_type, type) and issubclass(field_type, Enum): + type_str = "str" + enum_values = [e.value for e in field_type] + + metadata.append( + AttributeMetadata( + key=field_name, + type=type_str, + enum=enum_values, + required=field_obj.required, + ) + ) + + return metadata + except Exception: + return None + + +def adapt_legacy_to_universal(legacy: Compliance) -> ComplianceFramework: + """Convert a legacy Compliance object to a ComplianceFramework.""" + universal_requirements = [] + legacy_provider_key = legacy.Provider.lower() + + for req in legacy.Requirements: + req_checks = {legacy_provider_key: list(req.Checks)} if req.Checks else {} + if isinstance(req, Mitre_Requirement): + # For MITRE, promote special fields and store raw attributes + raw_attrs = [attr.dict() for attr in req.Attributes] + attrs = {"_raw_attributes": raw_attrs} + universal_requirements.append( + UniversalComplianceRequirement( + id=req.Id, + description=req.Description, + name=req.Name, + attributes=attrs, + checks=req_checks, + tactics=req.Tactics, + sub_techniques=req.SubTechniques, + platforms=req.Platforms, + technique_url=req.TechniqueURL, + ) + ) + else: + # Standard requirement: flatten first attribute to dict + if req.Attributes: + attrs = req.Attributes[0].dict() + else: + attrs = {} + universal_requirements.append( + UniversalComplianceRequirement( + id=req.Id, + description=req.Description, + name=req.Name, + attributes=attrs, + checks=req_checks, + ) + ) + + inferred_metadata = _infer_attribute_metadata(legacy) + + return ComplianceFramework( + framework=legacy.Framework, + name=legacy.Name, + provider=legacy.Provider, + version=legacy.Version, + description=legacy.Description, + requirements=universal_requirements, + attributes_metadata=inferred_metadata, + ) + + +def load_compliance_framework_universal(path: str) -> ComplianceFramework: + """Load a compliance JSON as a ComplianceFramework, handling both new and legacy formats.""" + try: + with open(path, "r") as f: + data = json.load(f) + + if "attributes_metadata" in data or "requirements" in data: + # New universal format — parse directly + return ComplianceFramework(**data) + else: + # Legacy format — parse as Compliance, then adapt + legacy = Compliance(**data) + return adapt_legacy_to_universal(legacy) + except Exception as e: + logger.error( + f"Failed to load universal compliance framework from {path}: " + f"{e.__class__.__name__}[{e.__traceback__.tb_lineno}] -- {e}" + ) + return None + + +def _load_jsons_from_dir(dir_path: str, provider: str, bulk: dict) -> None: + """Scan *dir_path* for JSON files and add matching frameworks to *bulk*.""" + for filename in os.listdir(dir_path): + file_path = os.path.join(dir_path, filename) + if not ( + os.path.isfile(file_path) + and filename.endswith(".json") + and os.stat(file_path).st_size > 0 + ): + continue + framework_name = filename.split(".json")[0] + if framework_name in bulk: + continue + fw = load_compliance_framework_universal(file_path) + if fw is None: + continue + if fw.provider and fw.provider.lower() == provider.lower(): + bulk[framework_name] = fw + elif fw.supports_provider(provider): + bulk[framework_name] = fw + + +def get_bulk_compliance_frameworks_universal(provider: str) -> dict: + """Bulk load all compliance frameworks relevant to the given provider. + + Scans: + + 1. The **top-level** ``prowler/compliance/`` directory for multi-provider + JSONs (``Checks`` keyed by provider, no ``Provider`` field). + 2. Every **provider sub-directory** (``prowler/compliance/{p}/``) so that + single-provider JSONs are also picked up. + + A framework is included when its explicit ``Provider`` matches + (case-insensitive) **or** any requirement has dict-style ``Checks`` + with a key for *provider*. + """ + bulk = {} + try: + available_modules = list_compliance_modules() + + # Resolve the compliance root once (parent of provider sub-dirs). + compliance_root = None + seen_paths = set() + + for module in available_modules: + dir_path = f"{module.module_finder.path}/{module.name.split('.')[-1]}" + if not os.path.isdir(dir_path) or dir_path in seen_paths: + continue + seen_paths.add(dir_path) + + # Remember the root the first time we see a valid sub-dir. + if compliance_root is None: + compliance_root = module.module_finder.path + + _load_jsons_from_dir(dir_path, provider, bulk) + + # Also scan top-level compliance/ for provider-agnostic JSONs. + if compliance_root and os.path.isdir(compliance_root): + _load_jsons_from_dir(compliance_root, provider, bulk) + + except Exception as e: + logger.error(f"{e.__class__.__name__}[{e.__traceback__.tb_lineno}] -- {e}") + return bulk diff --git a/prowler/providers/aws/aws_provider.py b/prowler/providers/aws/aws_provider.py index 9a248e826f..cf9356f2fb 100644 --- a/prowler/providers/aws/aws_provider.py +++ b/prowler/providers/aws/aws_provider.py @@ -111,6 +111,7 @@ class AwsProvider(Provider): mfa: bool = False, profile: str = None, regions: set = set(), + excluded_regions: set = None, organizations_role_arn: str = None, scan_unused_services: bool = False, resource_tags: list[str] = [], @@ -136,6 +137,10 @@ class AwsProvider(Provider): - mfa: A boolean indicating whether MFA is enabled. - profile: The name of the AWS CLI profile to use. - regions: A set of regions to audit. + - excluded_regions: A set of regions to skip during the scan. Applied + on top of `regions` and of the account's enabled regions. Also + settable via the PROWLER_AWS_DISALLOWED_REGIONS environment variable + or the `disallowed_regions` key in the provider config file. - organizations_role_arn: The ARN of the AWS Organizations IAM role to assume. - scan_unused_services: A boolean indicating whether to scan unused services. False by default. - resource_tags: A list of tags to filter the resources to audit. @@ -190,6 +195,33 @@ class AwsProvider(Provider): logger.info("Initializing AWS provider ...") + # Load provider config early because provider-level settings can affect + # bootstrap region selection before the scan starts. + if config_content is not None: + self._audit_config = config_content + else: + if not config_path: + config_path = default_config_file_path + self._audit_config = load_and_validate_config_file(self._type, config_path) + + excluded_regions = self.resolve_excluded_regions( + excluded_regions, self._audit_config + ) + + # Normalize excluded_regions and prune the include-list up front so + # every downstream consumer (identity, STS region, service/region + # enumeration) sees an already-filtered view. + if excluded_regions and regions: + regions = set(regions) - excluded_regions + if not regions: + raise AWSArgumentTypeValidationError( + message=( + "All requested AWS regions are excluded by the " + "disallowed regions configuration." + ), + file=pathlib.Path(__file__).name, + ) + ######## AWS Session logger.info("Generating original session ...") @@ -215,7 +247,7 @@ class AwsProvider(Provider): # After the session is created, validate it logger.info("Validating credentials ...") sts_region = get_aws_region_for_sts( - self.session.current_session.region_name, regions + self.session.current_session.region_name, regions, excluded_regions ) # Validate the credentials @@ -229,7 +261,9 @@ class AwsProvider(Provider): ######## AWS Provider Identity # Get profile region - profile_region = self.get_profile_region(self._session.current_session) + profile_region = self.get_profile_region( + self._session.current_session, excluded_regions + ) # Set identity self._identity = self.set_identity( @@ -332,7 +366,26 @@ class AwsProvider(Provider): ) ######## - # Parse Scan Tags + # Get Enabled Regions + self._enabled_regions = self.get_aws_enabled_regions( + self._session.current_session + ) + + # Apply the exclusion to the account's enabled regions. This is the + # gate used by generate_regional_clients, so skipped regions never get + # a boto3 client created for them and cannot stall the scan. + if excluded_regions: + if self._enabled_regions is not None: + self._enabled_regions = self._enabled_regions - excluded_regions + if self._identity.audited_regions: + self._identity.audited_regions = ( + set(self._identity.audited_regions) - excluded_regions + ) + logger.info(f"Excluding AWS regions from scan: {sorted(excluded_regions)}") + self._excluded_regions = excluded_regions + + # Parse Scan Tags after region exclusions are applied so tag discovery + # also skips disallowed regions. if resource_tags: self._audit_resources = self.get_tagged_resources(resource_tags) @@ -340,22 +393,9 @@ class AwsProvider(Provider): if resource_arn: self._audit_resources = resource_arn - # Get Enabled Regions - self._enabled_regions = self.get_aws_enabled_regions( - self._session.current_session - ) - # Set ignore unused services self._scan_unused_services = scan_unused_services - # Audit Config - if config_content: - self._audit_config = config_content - else: - if not config_path: - config_path = default_config_file_path - self._audit_config = load_and_validate_config_file(self._type, config_path) - # Fixer Config self._fixer_config = fixer_config @@ -468,12 +508,53 @@ class AwsProvider(Provider): ) @staticmethod - def get_profile_region(session: Session): - profile_region = AWS_REGION_US_EAST_1 - if session.region_name: - profile_region = session.region_name + def resolve_excluded_regions( + excluded_regions: set | list | tuple | None, + audit_config: dict | None, + ) -> set[str]: + """Resolve AWS region exclusions with precedence arg > env > config.""" + if excluded_regions is not None: + raw_regions = excluded_regions + else: + raw_regions = Provider.get_excluded_regions_from_env() + if not raw_regions and isinstance(audit_config, dict): + raw_regions = audit_config.get("disallowed_regions") or [] - return profile_region + return {str(region).strip() for region in raw_regions if str(region).strip()} + + @staticmethod + def get_bootstrap_region_candidates(session_region: str | None) -> tuple[str, ...]: + """Return safe fallback regions for bootstrap AWS calls.""" + if session_region: + if session_region.startswith("cn-"): + return ("cn-north-1", "cn-northwest-1") + if session_region.startswith("us-gov-"): + return ("us-gov-east-1", "us-gov-west-1") + if session_region.startswith("eusc-"): + return ("eusc-de-east-1",) + if session_region.startswith("us-iso"): + return (session_region,) + + return (AWS_STS_GLOBAL_ENDPOINT_REGION, "us-east-2", "us-west-2", "eu-west-1") + + @staticmethod + def get_profile_region( + session: Session, excluded_regions: set[str] | None = None + ) -> str: + excluded_regions = set(excluded_regions or ()) + session_region = session.region_name + if session_region and session_region not in excluded_regions: + return session_region + + for region in AwsProvider.get_bootstrap_region_candidates(session_region): + if region not in excluded_regions: + if session_region and session_region != region: + logger.info( + f"Configured AWS profile region {session_region} is excluded; using {region} for bootstrap clients." + ) + return region + + return session_region or AWS_REGION_US_EAST_1 @staticmethod def set_identity( @@ -701,12 +782,15 @@ class AwsProvider(Provider): Caller Identity ARN: arn:aws:iam::123456789012:user/prowler ``` """ - # Beautify audited regions, set "all" if there is no filter region - regions = ( - ", ".join(self._identity.audited_regions) - if self._identity.audited_regions is not None - else "all" - ) + # Beautify audited regions. If the scan includes all regions but some + # are explicitly excluded, reflect that in the banner instead of + # showing the misleading "all" label. + if self._identity.audited_regions: + regions = ", ".join(sorted(self._identity.audited_regions)) + elif getattr(self, "_excluded_regions", None): + regions = f"all except {', '.join(sorted(self._excluded_regions))}" + else: + regions = "all" # Beautify audited profile, set "default" if there is no profile set profile = ( self._identity.profile if self._identity.profile is not None else "default" @@ -745,6 +829,8 @@ class AwsProvider(Provider): service_regions = AwsProvider.get_available_aws_service_regions( service, self._identity.partition, self._identity.audited_regions ) + if getattr(self, "_excluded_regions", None): + service_regions = service_regions - self._excluded_regions # Get the regions enabled for the account and get the intersection with the service available regions if self._enabled_regions is not None: @@ -962,6 +1048,8 @@ class AwsProvider(Provider): service_regions = AwsProvider.get_available_aws_service_regions( service, self._identity.partition, self._identity.audited_regions ) + if getattr(self, "_excluded_regions", None): + service_regions = service_regions - self._excluded_regions default_region = self.get_global_region() # global region of the partition when all regions are audited and there is no profile region if self._identity.profile_region in service_regions: @@ -1565,13 +1653,19 @@ def read_aws_regions_file() -> dict: # TODO: This can be moved to another class since it doesn't need self -def get_aws_region_for_sts(session_region: str, regions: set[str]) -> str: +def get_aws_region_for_sts( + session_region: str, + regions: set[str], + excluded_regions: set[str] | None = None, +) -> str: """ Get the AWS region for the STS Assume Role operation. Args: - session_region (str): The region configured in the AWS session. - regions (set[str]): The regions passed with the -f/--region/--filter-region option. + - excluded_regions (set[str] | None): Regions that should be avoided for + bootstrap calls when possible. Returns: str: The AWS region for the STS Assume Role operation @@ -1579,20 +1673,21 @@ def get_aws_region_for_sts(session_region: str, regions: set[str]) -> str: Example: aws_region = get_aws_region_for_sts(session_region, regions) """ - # If there is no region passed with -f/--region/--filter-region - if regions is None or len(regions) == 0: - # If you have a region configured in your AWS config or credentials file - if session_region is not None: - aws_region = session_region - else: - # If there is no region set passed with -f/--region - # we use the Global STS Endpoint Region, us-east-1 - aws_region = AWS_STS_GLOBAL_ENDPOINT_REGION - else: - # Get the first region passed to the -f/--region - aws_region = list(regions)[0] + excluded_regions = set(excluded_regions or ()) - return aws_region + if regions: + for region in regions: + if region not in excluded_regions: + return region + + if session_region and session_region not in excluded_regions: + return session_region + + for region in AwsProvider.get_bootstrap_region_candidates(session_region): + if region not in excluded_regions: + return region + + return session_region or AWS_STS_GLOBAL_ENDPOINT_REGION # TODO: this duplicates the provider arguments validation library diff --git a/prowler/providers/aws/lib/arguments/arguments.py b/prowler/providers/aws/lib/arguments/arguments.py index eb611c0d04..50f4665b2d 100644 --- a/prowler/providers/aws/lib/arguments/arguments.py +++ b/prowler/providers/aws/lib/arguments/arguments.py @@ -66,6 +66,16 @@ def init_parser(self): help="AWS region names to run Prowler against", choices=AwsProvider.get_regions(partition=None), ) + aws_regions_subparser.add_argument( + "--excluded-region", + "--excluded-regions", + nargs="+", + help=( + "AWS region names to exclude from the scan. Overrides the " + "PROWLER_AWS_DISALLOWED_REGIONS environment variable when set." + ), + choices=AwsProvider.get_regions(partition=None), + ) # AWS Organizations aws_orgs_subparser = aws_parser.add_argument_group("AWS Organizations") aws_orgs_subparser.add_argument( diff --git a/prowler/providers/aws/lib/quick_inventory/quick_inventory.py b/prowler/providers/aws/lib/quick_inventory/quick_inventory.py index 8e7bc6e450..95e9aea7bb 100644 --- a/prowler/providers/aws/lib/quick_inventory/quick_inventory.py +++ b/prowler/providers/aws/lib/quick_inventory/quick_inventory.py @@ -30,10 +30,12 @@ def quick_inventory(provider: AwsProvider, args): ec2_client = provider.session.current_session.client( "ec2", region_name=provider.identity.profile_region ) + excluded_regions = getattr(provider, "_excluded_regions", set()) # Get all the available regions provider.identity.audited_regions = [ region["RegionName"] for region in ec2_client.describe_regions()["Regions"] + if region["RegionName"] not in excluded_regions ] with alive_bar( diff --git a/prowler/providers/cloudflare/cloudflare_provider.py b/prowler/providers/cloudflare/cloudflare_provider.py index 35dd88ef76..48763df395 100644 --- a/prowler/providers/cloudflare/cloudflare_provider.py +++ b/prowler/providers/cloudflare/cloudflare_provider.py @@ -274,8 +274,12 @@ class CloudflareProvider(Provider): for account in client.accounts.list(): account_id = getattr(account, "id", None) - # Prevent infinite loop - skip if we've seen this account + # Prevent infinite loop on repeated pages from the SDK paginator if account_id in seen_account_ids: + logger.warning( + "Detected repeated Cloudflare account ID while listing accounts. " + "Stopping pagination to avoid an infinite loop." + ) break seen_account_ids.add(account_id) @@ -332,19 +336,16 @@ class CloudflareProvider(Provider): return except PermissionDeniedError as error: error_str = str(error) - # Check for user-level authentication required (code 9109) - if "9109" in error_str: - logger.error(f"CloudflareUserTokenRequiredError: {error}") - raise CloudflareUserTokenRequiredError( - file=os.path.basename(__file__), - ) # Check for invalid API key or email (code 9103) - comes as 403 if "9103" in error_str or "Unknown X-Auth-Key" in error_str: logger.error(f"CloudflareInvalidAPIKeyError: {error}") raise CloudflareInvalidAPIKeyError( file=os.path.basename(__file__), ) - # For other permission errors, try accounts.list() as fallback + # For permission errors (including 9109 account-scoped tokens), + # try accounts.list() as fallback before failing. + # Error 9109 means the token is account-scoped, not user-level, + # which is valid for scanning — only fail if accounts.list() also fails. logger.warning( f"Unable to retrieve Cloudflare user info: {error}. " "Trying accounts.list() as fallback." @@ -398,7 +399,20 @@ class CloudflareProvider(Provider): # Fallback: try accounts.list() try: - accounts = list(client.accounts.list()) + accounts: list = [] + seen_account_ids: set = set() + for account in client.accounts.list(): + account_id = getattr(account, "id", None) + # Prevent infinite loop on repeated pages from the SDK paginator + if account_id in seen_account_ids: + logger.warning( + "Detected repeated Cloudflare account ID while validating credentials. " + "Stopping pagination to avoid an infinite loop." + ) + break + seen_account_ids.add(account_id) + accounts.append(account) + if not accounts: logger.error("CloudflareNoAccountsError: No accounts found") raise CloudflareNoAccountsError( diff --git a/prowler/providers/common/provider.py b/prowler/providers/common/provider.py index bc00607f10..8613957a68 100644 --- a/prowler/providers/common/provider.py +++ b/prowler/providers/common/provider.py @@ -1,5 +1,6 @@ import importlib import importlib.metadata +import os import pkgutil import sys from abc import ABC, abstractmethod @@ -199,6 +200,18 @@ class Provider(ABC): # --- End dynamic provider contract methods --- + @staticmethod + def get_excluded_regions_from_env() -> set: + """Parse the PROWLER_AWS_DISALLOWED_REGIONS environment variable. + + The variable is a comma-separated list of region identifiers to skip + during scans (e.g. "me-south-1, ap-east-1"). Whitespace around entries + is tolerated and empty entries are dropped. Returns an empty set when + the variable is unset or contains no usable values. + """ + raw = os.environ.get("PROWLER_AWS_DISALLOWED_REGIONS", "") + return {region.strip() for region in raw.split(",") if region.strip()} + @staticmethod def get_global_provider() -> "Provider": return Provider._global @@ -232,6 +245,11 @@ class Provider(ABC): if not isinstance(Provider._global, provider_class): if "aws" in provider_class_name.lower(): + excluded_regions = ( + set(arguments.excluded_region) + if getattr(arguments, "excluded_region", None) + else None + ) provider_class( retries_max_attempts=arguments.aws_retries_max_attempts, role_arn=arguments.role, @@ -241,6 +259,7 @@ class Provider(ABC): mfa=arguments.mfa, profile=arguments.profile, regions=set(arguments.region) if arguments.region else None, + excluded_regions=excluded_regions, organizations_role_arn=arguments.organizations_role, scan_unused_services=arguments.scan_unused_services, resource_tags=arguments.resource_tag, diff --git a/prowler/providers/googleworkspace/services/calendar/calendar_external_invitations_warning/calendar_external_invitations_warning.py b/prowler/providers/googleworkspace/services/calendar/calendar_external_invitations_warning/calendar_external_invitations_warning.py index 7af51cbfba..1da8769b06 100644 --- a/prowler/providers/googleworkspace/services/calendar/calendar_external_invitations_warning/calendar_external_invitations_warning.py +++ b/prowler/providers/googleworkspace/services/calendar/calendar_external_invitations_warning/calendar_external_invitations_warning.py @@ -35,21 +35,20 @@ class calendar_external_invitations_warning(Check): f"External invitation warnings for Google Calendar are enabled " f"in domain {calendar_client.provider.identity.domain}." ) + elif warning_enabled is None: + report.status = "PASS" + report.status_extended = ( + f"External invitation warnings for Google Calendar use Google's " + f"secure default configuration (enabled) " + f"in domain {calendar_client.provider.identity.domain}." + ) else: report.status = "FAIL" - if warning_enabled is None: - report.status_extended = ( - f"External invitation warnings for Google Calendar are not " - f"explicitly configured in domain " - f"{calendar_client.provider.identity.domain}. " - f"Users should be warned when inviting guests outside the organization." - ) - else: - report.status_extended = ( - f"External invitation warnings for Google Calendar are disabled " - f"in domain {calendar_client.provider.identity.domain}. " - f"Users should be warned when inviting guests outside the organization." - ) + report.status_extended = ( + f"External invitation warnings for Google Calendar are disabled " + f"in domain {calendar_client.provider.identity.domain}. " + f"Users should be warned when inviting guests outside the organization." + ) findings.append(report) diff --git a/prowler/providers/googleworkspace/services/calendar/calendar_external_sharing_primary_calendar/calendar_external_sharing_primary_calendar.py b/prowler/providers/googleworkspace/services/calendar/calendar_external_sharing_primary_calendar/calendar_external_sharing_primary_calendar.py index b019acf4de..eaf9f90911 100644 --- a/prowler/providers/googleworkspace/services/calendar/calendar_external_sharing_primary_calendar/calendar_external_sharing_primary_calendar.py +++ b/prowler/providers/googleworkspace/services/calendar/calendar_external_sharing_primary_calendar/calendar_external_sharing_primary_calendar.py @@ -36,20 +36,20 @@ class calendar_external_sharing_primary_calendar(Check): f"{calendar_client.provider.identity.domain} is restricted to " f"free/busy information only." ) + elif sharing is None: + report.status = "PASS" + report.status_extended = ( + f"Primary calendar external sharing uses Google's secure default " + f"configuration (free/busy only) " + f"in domain {calendar_client.provider.identity.domain}." + ) else: report.status = "FAIL" - if sharing is None: - report.status_extended = ( - f"Primary calendar external sharing is not explicitly configured " - f"in domain {calendar_client.provider.identity.domain}. " - f"External sharing should be restricted to free/busy information only." - ) - else: - report.status_extended = ( - f"Primary calendar external sharing in domain " - f"{calendar_client.provider.identity.domain} is set to {sharing}. " - f"External sharing should be restricted to free/busy information only." - ) + report.status_extended = ( + f"Primary calendar external sharing in domain " + f"{calendar_client.provider.identity.domain} is set to {sharing}. " + f"External sharing should be restricted to free/busy information only." + ) findings.append(report) diff --git a/prowler/providers/googleworkspace/services/drive/drive_external_sharing_warn_users/drive_external_sharing_warn_users.py b/prowler/providers/googleworkspace/services/drive/drive_external_sharing_warn_users/drive_external_sharing_warn_users.py index 973a672f82..15f6187949 100644 --- a/prowler/providers/googleworkspace/services/drive/drive_external_sharing_warn_users/drive_external_sharing_warn_users.py +++ b/prowler/providers/googleworkspace/services/drive/drive_external_sharing_warn_users/drive_external_sharing_warn_users.py @@ -33,21 +33,20 @@ class drive_external_sharing_warn_users(Check): f"External sharing warnings for Drive and Docs are enabled " f"in domain {drive_client.provider.identity.domain}." ) + elif warning_enabled is None: + report.status = "PASS" + report.status_extended = ( + f"External sharing warnings for Drive and Docs use Google's " + f"secure default configuration (enabled) " + f"in domain {drive_client.provider.identity.domain}." + ) else: report.status = "FAIL" - if warning_enabled is None: - report.status_extended = ( - f"External sharing warnings for Drive and Docs are not " - f"explicitly configured in domain " - f"{drive_client.provider.identity.domain}. " - f"Users should be warned when sharing files outside the organization." - ) - else: - report.status_extended = ( - f"External sharing warnings for Drive and Docs are disabled " - f"in domain {drive_client.provider.identity.domain}. " - f"Users should be warned when sharing files outside the organization." - ) + report.status_extended = ( + f"External sharing warnings for Drive and Docs are disabled " + f"in domain {drive_client.provider.identity.domain}. " + f"Users should be warned when sharing files outside the organization." + ) findings.append(report) diff --git a/prowler/providers/googleworkspace/services/drive/drive_shared_drive_creation_allowed/drive_shared_drive_creation_allowed.py b/prowler/providers/googleworkspace/services/drive/drive_shared_drive_creation_allowed/drive_shared_drive_creation_allowed.py index c36611326f..f976891147 100644 --- a/prowler/providers/googleworkspace/services/drive/drive_shared_drive_creation_allowed/drive_shared_drive_creation_allowed.py +++ b/prowler/providers/googleworkspace/services/drive/drive_shared_drive_creation_allowed/drive_shared_drive_creation_allowed.py @@ -35,22 +35,21 @@ class drive_shared_drive_creation_allowed(Check): f"Users in domain {drive_client.provider.identity.domain} " f"are allowed to create new shared drives." ) + elif allow_creation is None: + report.status = "PASS" + report.status_extended = ( + f"Shared drive creation uses Google's secure default " + f"configuration (allowed) " + f"in domain {drive_client.provider.identity.domain}." + ) else: report.status = "FAIL" - if allow_creation is None: - report.status_extended = ( - f"Shared drive creation is not explicitly configured in " - f"domain {drive_client.provider.identity.domain}. " - f"Users should be allowed to create new shared drives to avoid " - f"data loss when accounts are deleted." - ) - else: - report.status_extended = ( - f"Users in domain {drive_client.provider.identity.domain} " - f"are prevented from creating new shared drives. " - f"Users should be allowed to create new shared drives to avoid " - f"data loss when accounts are deleted." - ) + report.status_extended = ( + f"Users in domain {drive_client.provider.identity.domain} " + f"are prevented from creating new shared drives. " + f"Users should be allowed to create new shared drives to avoid " + f"data loss when accounts are deleted." + ) findings.append(report) diff --git a/prowler/providers/googleworkspace/services/drive/drive_shared_drive_disable_download_print_copy/drive_shared_drive_disable_download_print_copy.py b/prowler/providers/googleworkspace/services/drive/drive_shared_drive_disable_download_print_copy/drive_shared_drive_disable_download_print_copy.py index b112cd70b5..45edd64884 100644 --- a/prowler/providers/googleworkspace/services/drive/drive_shared_drive_disable_download_print_copy/drive_shared_drive_disable_download_print_copy.py +++ b/prowler/providers/googleworkspace/services/drive/drive_shared_drive_disable_download_print_copy/drive_shared_drive_disable_download_print_copy.py @@ -35,21 +35,21 @@ class drive_shared_drive_disable_download_print_copy(Check): f"{drive_client.provider.identity.domain} is restricted to " f"{allowed}." ) + elif allowed is None: + report.status = "PASS" + report.status_extended = ( + f"Download, print, and copy restrictions for shared drives use " + f"Google's secure default configuration (disabled for viewers " + f"and commenters) " + f"in domain {drive_client.provider.identity.domain}." + ) else: report.status = "FAIL" - if allowed is None: - report.status_extended = ( - f"Download, print, and copy restrictions for shared drive " - f"viewers and commenters are not explicitly configured in " - f"domain {drive_client.provider.identity.domain}. " - f"These actions should be restricted to editors or managers only." - ) - else: - report.status_extended = ( - f"Download, print, and copy in shared drives in domain " - f"{drive_client.provider.identity.domain} is set to {allowed}. " - f"These actions should be restricted to editors or managers only." - ) + report.status_extended = ( + f"Download, print, and copy in shared drives in domain " + f"{drive_client.provider.identity.domain} is set to {allowed}. " + f"These actions should be restricted to editors or managers only." + ) findings.append(report) diff --git a/prowler/providers/googleworkspace/services/drive/drive_warn_sharing_with_allowlisted_domains/drive_warn_sharing_with_allowlisted_domains.py b/prowler/providers/googleworkspace/services/drive/drive_warn_sharing_with_allowlisted_domains/drive_warn_sharing_with_allowlisted_domains.py index e5f243ae36..f8d8f4b586 100644 --- a/prowler/providers/googleworkspace/services/drive/drive_warn_sharing_with_allowlisted_domains/drive_warn_sharing_with_allowlisted_domains.py +++ b/prowler/providers/googleworkspace/services/drive/drive_warn_sharing_with_allowlisted_domains/drive_warn_sharing_with_allowlisted_domains.py @@ -36,21 +36,20 @@ class drive_warn_sharing_with_allowlisted_domains(Check): f"Users are warned when sharing files with allowlisted " f"domains in domain {drive_client.provider.identity.domain}." ) + elif warn_enabled is None: + report.status = "PASS" + report.status_extended = ( + f"Warning when sharing with allowlisted domains uses Google's " + f"secure default configuration (enabled) " + f"in domain {drive_client.provider.identity.domain}." + ) else: report.status = "FAIL" - if warn_enabled is None: - report.status_extended = ( - f"Warning when sharing with allowlisted domains is not " - f"explicitly configured in domain " - f"{drive_client.provider.identity.domain}. " - f"Users should be warned when sharing files with users in allowlisted domains." - ) - else: - report.status_extended = ( - f"Warning when sharing with allowlisted domains is disabled " - f"in domain {drive_client.provider.identity.domain}. " - f"Users should be warned when sharing files with users in allowlisted domains." - ) + report.status_extended = ( + f"Warning when sharing with allowlisted domains is disabled " + f"in domain {drive_client.provider.identity.domain}. " + f"Users should be warned when sharing files with users in allowlisted domains." + ) findings.append(report) diff --git a/prowler/providers/image/image_provider.py b/prowler/providers/image/image_provider.py index d724542e28..7c9a9cbb08 100644 --- a/prowler/providers/image/image_provider.py +++ b/prowler/providers/image/image_provider.py @@ -163,42 +163,50 @@ class ImageProvider(Provider): # Registry scan mode: enumerate images from registry if self.registry: self._enumerate_registry() - if self._listing_only: - return - for image in self.images: - self._validate_image_name(image) - - if not self.images: - raise ImageNoImagesProvidedError( - file=__file__, - message="No images provided for scanning.", - ) - - # Audit Config - if config_content: - self._audit_config = config_content - else: - if not config_path: - config_path = default_config_file_path - self._audit_config = load_and_validate_config_file(self._type, config_path) - - # Fixer Config - self._fixer_config = fixer_config if fixer_config is not None else {} - - # Mutelist (not needed for Image provider since Trivy has its own logic) + # Safe defaults for listing-only mode (overwritten below in scan mode) + self._audit_config = {} + self._fixer_config = {} self._mutelist = None + self.audit_metadata = None - self.audit_metadata = Audit_Metadata( - provider=self._type, - account_id=self.audited_account, - account_name="image", - region=self.region, - services_scanned=0, - expected_checks=[], - completed_checks=0, - audit_progress=0, - ) + # Skip scan setup for listing-only mode + if not self._listing_only: + for image in self.images: + self._validate_image_name(image) + + if not self.images: + raise ImageNoImagesProvidedError( + file=__file__, + message="No images provided for scanning.", + ) + + # Audit Config + if config_content: + self._audit_config = config_content + else: + if not config_path: + config_path = default_config_file_path + self._audit_config = load_and_validate_config_file( + self._type, config_path + ) + + # Fixer Config + self._fixer_config = fixer_config if fixer_config is not None else {} + + # Mutelist (not needed for Image provider since Trivy has its own logic) + self._mutelist = None + + self.audit_metadata = Audit_Metadata( + provider=self._type, + account_id=self.audited_account, + account_name="image", + region=self.region, + services_scanned=0, + expected_checks=[], + completed_checks=0, + audit_progress=0, + ) Provider.set_global_provider(self) diff --git a/prowler/providers/vercel/services/project/project_service.py b/prowler/providers/vercel/services/project/project_service.py index 9b3b24859d..e139a2a939 100644 --- a/prowler/providers/vercel/services/project/project_service.py +++ b/prowler/providers/vercel/services/project/project_service.py @@ -55,6 +55,7 @@ class Project(VercelService): # Parse password protection pwd_protection = proj.get("passwordProtection") + security = proj.get("security", {}) or {} self.projects[project_id] = VercelProject( id=project_id, @@ -75,6 +76,16 @@ class Project(VercelService): git_fork_protection=proj.get("gitForkProtection", True), git_repository=proj.get("link"), secure_compute=proj.get("secureCompute"), + firewall_enabled=security.get("firewallEnabled"), + firewall_config_version=( + str(security.get("firewallConfigVersion")) + if security.get("firewallConfigVersion") is not None + else None + ), + managed_rules=security.get( + "managedRules", security.get("managedRulesets") + ), + bot_id_enabled=security.get("botIdEnabled"), ) logger.info(f"Project - Found {len(self.projects)} project(s)") @@ -160,4 +171,8 @@ class VercelProject(BaseModel): git_fork_protection: bool = True git_repository: Optional[dict] = None secure_compute: Optional[dict] = None + firewall_enabled: Optional[bool] = None + firewall_config_version: Optional[str] = None + managed_rules: Optional[dict] = None + bot_id_enabled: Optional[bool] = None environment_variables: list[VercelEnvironmentVariable] = Field(default_factory=list) diff --git a/prowler/providers/vercel/services/security/security_service.py b/prowler/providers/vercel/services/security/security_service.py index eadac856b9..3c79f5c2f5 100644 --- a/prowler/providers/vercel/services/security/security_service.py +++ b/prowler/providers/vercel/services/security/security_service.py @@ -26,10 +26,7 @@ class Security(VercelService): def _fetch_firewall_config(self, project): """Fetch WAF/Firewall config for a single project.""" try: - data = self._get( - "/v1/security/firewall/config", - params={"projectId": project.id}, - ) + data = self._read_firewall_config(project) if data is None: # 403 — plan limitation, store with managed_rulesets=None @@ -44,39 +41,60 @@ class Security(VercelService): ) return - # Parse firewall config - fw = data.get("firewallConfig", data) if isinstance(data, dict) else {} + fw = self._normalize_firewall_config(data) - # Determine if firewall is enabled - rules = fw.get("rules", []) or [] - managed = fw.get("managedRules", fw.get("managedRulesets")) + if not fw: + fallback_firewall_enabled = self._fallback_firewall_enabled(project) + self.firewall_configs[project.id] = VercelFirewallConfig( + project_id=project.id, + project_name=project.name, + team_id=project.team_id, + firewall_enabled=( + fallback_firewall_enabled + if fallback_firewall_enabled is not None + else False + ), + managed_rulesets=self._fallback_managed_rulesets(project), + name=project.name, + id=project.id, + ) + return + + rules = [ + rule for rule in (fw.get("rules", []) or []) if self._is_active(rule) + ] + managed = self._active_managed_rulesets( + fw.get("managedRules", fw.get("managedRulesets", fw.get("crs"))) + ) custom_rules = [] - ip_blocking = [] + ip_blocking = list(fw.get("ips", []) or []) rate_limiting = [] for rule in rules: - rule_action = rule.get("action", {}) - action_type = ( - rule_action.get("type", "") - if isinstance(rule_action, dict) - else str(rule_action) - ) + mitigate_action = self._mitigate_action(rule) - if action_type == "rate_limit" or rule.get("rateLimit"): + if self._is_rate_limiting_rule(rule, mitigate_action): rate_limiting.append(rule) - elif action_type in ("deny", "block") and self._is_ip_rule(rule): + elif self._is_ip_rule(rule): ip_blocking.append(rule) else: custom_rules.append(rule) - firewall_enabled = bool(rules) or bool(managed) + firewall_enabled = fw.get("firewallEnabled") + if firewall_enabled is None: + firewall_enabled = self._fallback_firewall_enabled(project) + if firewall_enabled is None: + firewall_enabled = bool(rules) or bool(ip_blocking) or bool(managed) + + if not managed: + managed = self._fallback_managed_rulesets(project) self.firewall_configs[project.id] = VercelFirewallConfig( project_id=project.id, project_name=project.name, team_id=project.team_id, firewall_enabled=firewall_enabled, - managed_rulesets=managed if managed is not None else {}, + managed_rulesets=managed, custom_rules=custom_rules, ip_blocking_rules=ip_blocking, rate_limiting_rules=rate_limiting, @@ -95,6 +113,117 @@ class Security(VercelService): f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" ) + def _read_firewall_config(self, project): + """Read the deployed firewall config via the documented endpoint. + + See: https://vercel.com/docs/rest-api/security/read-firewall-configuration + """ + params = self._firewall_params(project) + config_version = getattr(project, "firewall_config_version", None) + + endpoints = [] + if config_version: + endpoints.append(f"/v1/security/firewall/config/{config_version}") + endpoints.append("/v1/security/firewall/config/active") + + last_error = None + for endpoint in endpoints: + try: + return self._get(endpoint, params=params) + except Exception as error: + last_error = error + logger.warning( + f"Security - Firewall config read failed for project " + f"{project.id} (team={getattr(project, 'team_id', None)}) " + f"on {endpoint} with params={params}: " + f"{error.__class__.__name__}: {error}" + ) + + if last_error is not None: + logger.debug( + f"Security - Falling back to firewall config wrapper for " + f"{project.id} after {last_error.__class__.__name__}: {last_error}" + ) + + return self._get("/v1/security/firewall/config", params=params) + + @staticmethod + def _firewall_params(project) -> dict: + """Build firewall request params, preserving team scope for team projects.""" + params = {"projectId": project.id} + team_id = getattr(project, "team_id", None) + + if isinstance(team_id, str) and team_id: + params["teamId"] = team_id + + return params + + @staticmethod + def _normalize_firewall_config(data: dict) -> dict: + """Normalize firewall responses across Vercel endpoint variants.""" + if not isinstance(data, dict): + return {} + + if "firewallConfig" in data and isinstance(data["firewallConfig"], dict): + return data["firewallConfig"] + + if any(key in data for key in ("active", "draft", "versions")): + return data.get("active") or {} + + return data + + @staticmethod + def _active_managed_rulesets(managed_rules: dict | None) -> dict: + """Return only active managed rulesets.""" + if not isinstance(managed_rules, dict): + return {} + + return { + ruleset: config + for ruleset, config in managed_rules.items() + if not isinstance(config, dict) or config.get("active", False) + } + + @classmethod + def _fallback_managed_rulesets(cls, project) -> dict: + """Return active managed rulesets from project metadata.""" + return cls._active_managed_rulesets(getattr(project, "managed_rules", None)) + + @staticmethod + def _fallback_firewall_enabled(project) -> bool | None: + """Return firewall enabled state from project metadata when available.""" + return getattr(project, "firewall_enabled", None) + + @staticmethod + def _mitigate_action(rule: dict) -> dict: + """Extract the nested Vercel mitigation action payload for a rule.""" + action = rule.get("action", {}) + if not isinstance(action, dict): + return {} + + mitigate = action.get("mitigate") + return mitigate if isinstance(mitigate, dict) else action + + @staticmethod + def _is_active(rule: dict) -> bool: + """Treat missing active flags as enabled for backwards compatibility.""" + return rule.get("active", True) is not False + + @classmethod + def _is_rate_limiting_rule( + cls, rule: dict, mitigate_action: dict | None = None + ) -> bool: + """Check if a firewall rule enforces rate limiting.""" + if rule.get("rateLimit"): + return True + + mitigate = ( + mitigate_action + if isinstance(mitigate_action, dict) + else cls._mitigate_action(rule) + ) + return bool(mitigate.get("rateLimit")) or mitigate.get("action") == "rate_limit" + @staticmethod def _is_ip_rule(rule: dict) -> bool: """Check if a rule is an IP blocking rule based on conditions.""" diff --git a/pyproject.toml b/pyproject.toml index 0b4294a547..717fb338a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,7 @@ dependencies = [ "azure-mgmt-postgresqlflexibleservers==1.1.0", "azure-mgmt-recoveryservices==3.1.0", "azure-mgmt-recoveryservicesbackup==9.2.0", - "azure-mgmt-resource==23.3.0", + "azure-mgmt-resource==24.0.0", "azure-mgmt-search==9.1.0", "azure-mgmt-security==7.0.0", "azure-mgmt-sql==3.0.1", @@ -57,7 +57,7 @@ dependencies = [ "kubernetes==32.0.1", "markdown==3.10.2", "microsoft-kiota-abstractions==1.9.2", - "msgraph-sdk==1.23.0", + "msgraph-sdk==1.55.0", "numpy==2.0.2", "openstacksdk==4.2.0", "pandas==2.2.3", @@ -95,7 +95,7 @@ maintainers = [{name = "Prowler Engineering", email = "engineering@prowler.com"} name = "prowler" readme = "README.md" requires-python = ">=3.10,<3.13" -version = "5.24.0" +version = "5.25.0" [project.scripts] prowler = "prowler.__main__:prowler" @@ -121,7 +121,6 @@ docker = "7.1.0" filelock = "3.20.3" flake8 = "7.1.2" freezegun = "1.5.1" -marshmallow = "==3.26.2" mock = "5.2.0" moto = {extras = ["all"], version = "5.1.11"} openapi-schema-validator = "0.6.3" diff --git a/skills/prowler-compliance/SKILL.md b/skills/prowler-compliance/SKILL.md index 1853d23d8b..51c68eb05f 100644 --- a/skills/prowler-compliance/SKILL.md +++ b/skills/prowler-compliance/SKILL.md @@ -1,16 +1,28 @@ --- name: prowler-compliance description: > - Creates and manages Prowler compliance frameworks. - Trigger: When working with compliance frameworks (CIS, NIST, PCI-DSS, SOC2, GDPR, ISO27001, ENS, MITRE ATT&CK). + Creates, syncs, audits and manages Prowler compliance frameworks end-to-end. + Covers the four-layer architecture (SDK models → JSON catalogs → output + formatters → API/UI), upstream sync workflows, cloud-auditor check-mapping + reviews, output formatter creation, and framework-specific attribute models. + Trigger: When working with compliance frameworks (CIS, NIST, PCI-DSS, SOC2, + GDPR, ISO27001, ENS, MITRE ATT&CK, CCC, C5, CSA CCM, KISA ISMS-P, + Prowler ThreatScore, FedRAMP, HIPAA), syncing with upstream catalogs, + auditing check-to-requirement mappings, adding output formatters, or fixing + compliance JSON bugs (duplicate IDs, empty Version, wrong Section, stale + check refs). license: Apache-2.0 metadata: author: prowler-cloud - version: "1.1" + version: "1.2" scope: [root, sdk] auto_invoke: - "Creating/updating compliance frameworks" - "Mapping checks to compliance controls" + - "Syncing compliance framework with upstream catalog" + - "Auditing check-to-requirement mappings as a cloud auditor" + - "Adding a compliance output formatter (per-provider class + table dispatcher)" + - "Fixing compliance JSON bugs (duplicate IDs, empty Section, stale refs)" allowed-tools: Read, Edit, Write, Glob, Grep, Bash, WebFetch, WebSearch, Task --- @@ -18,10 +30,82 @@ allowed-tools: Read, Edit, Write, Glob, Grep, Bash, WebFetch, WebSearch, Task Use this skill when: - Creating a new compliance framework for any provider +- **Syncing an existing framework with an upstream source of truth** (CIS, FINOS CCC, CSA CCM, NIST, ENS, etc.) - Adding requirements to existing frameworks - Mapping checks to compliance controls +- **Auditing existing check mappings as a cloud auditor** (user asks "are these mappings correct?", "which checks apply to this requirement?", "review the mappings") +- **Adding a new output formatter** (new framework needs a table dispatcher + per-provider classes + CSV models) +- **Fixing JSON bugs**: duplicate IDs, empty Version, wrong Section, stale check refs, inconsistent FamilyName, padded tangential check mappings +- **Registering a framework in the CLI table dispatcher or API export map** +- Investigating why a finding/check isn't showing under the expected compliance framework in the UI - Understanding compliance framework structures and attributes +## Four-Layer Architecture (Mental Model) + +Prowler compliance is a **four-layer system** hanging off one Pydantic model tree. Bugs usually happen where one layer doesn't match another, so know all four before touching anything. + +### Layer 1: SDK / Core Models — `prowler/lib/check/` + +- **`compliance_models.py`** — Pydantic **v1** model tree (`from pydantic.v1 import`). One `*_Requirement_Attribute` class per framework type + `Generic_Compliance_Requirement_Attribute` as fallback. +- `Compliance_Requirement.Attributes: list[Union[...]]` — **`Generic_Compliance_Requirement_Attribute` MUST be LAST** in the Union or every framework-specific attribute falls through to Generic (Pydantic v1 tries union members in order). +- **`compliance.py`** — runtime linker. `get_check_compliance()` builds the key as `f"{Framework}-{Version}"` **only if `Version` is non-empty**. An empty Version makes the key just `"{Framework}"` — this breaks downstream filters and tests that expect the versioned key. +- `Compliance.get_bulk(provider)` walks `prowler/compliance/{provider}/` and parses every `.json` file. No central index — just directory scan. + +### Layer 2: JSON Frameworks — `prowler/compliance/{provider}/` + +See "Compliance Framework Location" and "Framework-Specific Attribute Structures" sections below. + +### Layer 3: Output Formatters — `prowler/lib/outputs/compliance/{framework}/` + +**Every framework directory follows this exact convention** — do not deviate: + +``` +{framework}/ +├── __init__.py +├── {framework}.py # ONLY get_{framework}_table() — NO function docstring +├── {framework}_{provider}.py # One class per provider (e.g., CCC_AWS, CCC_Azure, CCC_GCP) +└── models.py # One Pydantic v2 BaseModel per provider (CSV columns) +``` + +- **`{framework}.py`** holds the **table dispatcher function** `get_{framework}_table()`. It prints the pass/fail/muted summary table. **Must NOT import `Finding` or `ComplianceOutput`** — doing so creates a circular import with `prowler/lib/outputs/compliance/compliance.py`. Only imports: `colorama`, `tabulate`, `prowler.config.config.orange_color`. +- **`{framework}_{provider}.py`** holds a per-provider class like `CCC_AWS(ComplianceOutput)` with a `transform()` method that walks findings and emits rows. This file IS allowed to import `Finding` because it's not on the dispatcher import chain. +- **`models.py`** holds one Pydantic v2 `BaseModel` per provider. Field names become CSV column headers (**public API** — renaming breaks downstream consumers). +- **Never collapse per-provider files into a unified parameterized class**, even when DRY-tempting. Every framework in Prowler follows the per-provider file pattern and reviewers will reject the refactor. CSV columns differ per provider (`AccountId`/`Region` vs `SubscriptionId`/`Location` vs `ProjectId`/`Location`) — three classes is the convention. +- **No function docstring on `get_{framework}_table()`** — no other framework has one; stay consistent. +- Register in `prowler/lib/outputs/compliance/compliance.py` → `display_compliance_table()` with an `elif compliance_framework.startswith("{framework}_"):` branch. Import the table function at the top of the file. + +### Layer 4: API / UI + +- **API table dispatcher**: `api/src/backend/tasks/jobs/export.py` → `COMPLIANCE_CLASS_MAP` keyed by provider. Uses `startswith` predicates: `(lambda name: name.startswith("ccc_"), CCC_AWS)`. **Never use exact match** (`name == "ccc_aws"`) — it's inconsistent and breaks versioning. +- **API lazy loader**: `api/src/backend/api/compliance.py` — `LazyComplianceTemplate` and `LazyChecksMapping` load compliance per provider on first access. +- **UI mapper routing**: `ui/lib/compliance/compliance-mapper.ts` routes framework names → per-framework mapper. +- **UI per-framework mapper**: `ui/lib/compliance/{framework}.tsx` flattens `Requirements` into a 3-level tree (Framework → Category → Control → Requirement) for the accordion view. Groups by `Attributes[0].FamilyName` and `Attributes[0].Section`. +- **UI detail panel**: `ui/components/compliance/compliance-custom-details/{framework}-details.tsx`. +- **UI types**: `ui/types/compliance.ts` — TypeScript mirrors of the attribute metadata. + +### The CLI Pipeline (end-to-end) + +``` +prowler aws --compliance ccc_aws + ↓ +Compliance.get_bulk("aws") → parses prowler/compliance/aws/*.json + ↓ +update_checks_metadata_with_compliance() → attaches compliance info to CheckMetadata + ↓ +execute_checks() → runs checks, produces Finding objects + ↓ +get_check_compliance(finding, "aws", bulk_checks_metadata) + → dict "{Framework}-{Version}" → [requirement_ids] + ↓ +CCC_AWS(findings, compliance).transform() → per-provider class builds CSV rows + ↓ +batch_write_data_to_file() → writes {output_filename}_ccc_aws.csv + ↓ +display_compliance_table() → get_ccc_table() → prints stdout summary +``` + +--- + ## Compliance Framework Location Frameworks are JSON files located in: `prowler/compliance/{provider}/{framework_name}_{provider}.json` @@ -455,14 +539,453 @@ Prowler ThreatScore is a custom security scoring framework developed by Prowler - **M365:** `cis_4.0_m365.json`, `iso27001_2022_m365.json` - **NHN:** `iso27001_2022_nhn.json` +## Workflow A: Sync a Framework With an Upstream Catalog + +Use when the framework is maintained upstream (CIS Benchmarks, FINOS CCC, CSA CCM, NIST, ENS, etc.) and Prowler needs to catch up. + +### Step 1 — Cache the upstream source + +Download every upstream file to a local cache so subsequent iterations don't hit the network. For FINOS CCC: + +```bash +mkdir -p /tmp/ccc_upstream +catalogs="core/ccc storage/object management/auditlog management/logging ..." +for p in $catalogs; do + safe=$(echo "$p" | tr '/' '_') + gh api "repos/finos/common-cloud-controls/contents/catalogs/$p/controls.yaml" \ + -H "Accept: application/vnd.github.raw" > "/tmp/ccc_upstream/${safe}.yaml" +done +``` + +### Step 2 — Run the generic sync runner against a framework config + +The sync tooling is split into three layers so adding a new framework only takes a YAML config (and optionally a new parser module for an unfamiliar upstream format): + +``` +skills/prowler-compliance/assets/ +├── sync_framework.py # generic runner — works for any framework +├── configs/ +│ └── ccc.yaml # per-framework config (canonical example) +└── parsers/ + ├── __init__.py + └── finos_ccc.py # parser module for FINOS CCC YAML +``` + +**For frameworks that already have a config + parser** (today: FINOS CCC), run: + +```bash +python skills/prowler-compliance/assets/sync_framework.py \ + skills/prowler-compliance/assets/configs/ccc.yaml +``` + +The runner loads the config, validates it, dynamically imports the parser declared in `parser.module`, calls `parser.parse_upstream(config) -> list[dict]`, then applies generic post-processing (id uniqueness safety net, `FamilyName` normalization, legacy check-mapping preservation) and writes the provider JSONs. + +**To add a new framework sync**: + +1. **Write a config file** at `skills/prowler-compliance/assets/configs/{framework}.yaml`. See `configs/ccc.yaml` as the canonical example. Required top-level sections: + - `framework` — `name`, `display_name`, `version` (**never empty** — empty Version silently breaks `get_check_compliance()` key construction, so the runner refuses to start), `description_template` (accepts `{provider_display}`, `{provider_key}`, `{framework_name}`, `{framework_display}`, `{version}` placeholders). + - `providers` — list of `{key, display}` pairs, one per Prowler provider the framework targets. + - `output.path_template` — supports `{provider}`, `{framework}`, `{version}` placeholders. Examples: `"prowler/compliance/{provider}/ccc_{provider}.json"` for unversioned file names, `"prowler/compliance/{provider}/cis_{version}_{provider}.json"` for versioned ones. + - `upstream.dir` — local cache directory (populate via Step 1). + - `parser.module` — name of the module under `parsers/` to load (without `.py`). Everything else under `parser.` is opaque to the runner and passed to the parser as config. + - `post_processing.check_preservation.primary_key` — top-level field name for the primary legacy-mapping lookup (almost always `Id`). + - `post_processing.check_preservation.fallback_keys` — **config-driven fallback keys** for preserving check mappings when ids change. Each entry is a list of `Attributes[0]` field names composed into a tuple. Examples: + - CCC: `- [Section, Applicability]` (because `Applicability` is a CCC-only attribute, verified in `compliance_models.py:213`). + - CIS would use `- [Section, Profile]`. + - NIST would use `- [ItemId]`. + - List-valued fields (like `Applicability`) are automatically frozen to `frozenset` so the tuple is hashable. + - `post_processing.family_name_normalization` (optional) — map of raw → canonical `FamilyName` values. The UI groups by `Attributes[0].FamilyName` exactly, so inconsistent upstream variants otherwise become separate tree branches. + +2. **Reuse an existing parser** if the upstream format matches one (currently only `finos_ccc` exists). Otherwise, **write a new parser** at `parsers/{name}.py` implementing: + + ```python + def parse_upstream(config: dict) -> list[dict]: + """Return Prowler-format requirements {Id, Description, Attributes: [...], Checks: []}. + + Ids MUST be unique in the returned list. The runner raises ValueError + on duplicates — it does NOT silently renumber, because mutating a + canonical upstream id (e.g. CIS '1.1.1' or NIST 'AC-2(1)') would be + catastrophic. The parser owns all upstream-format quirks: foreign-prefix + rewriting, genuine collision renumbering, shape handling. + """ + ``` + + The parser reads its own settings from `config['upstream']` and `config['parser']`. It does NOT load existing Prowler JSONs (the runner does that for check preservation) and does NOT write output (the runner does that too). + +**Gotchas the runner already handles for you** (learned from the FINOS CCC v2025.10 sync — they're documented here so you don't re-discover them): + +- **Multiple upstream YAML shapes**. Most FINOS CCC catalogs use `control-families: [...]`, but `storage/object` uses a top-level `controls: [...]` with a `family: "CCC.X.Y"` reference id and no human-readable family name. A parser that only handles shape 1 silently drops the shape-2 catalog — this exact bug dropped ObjStor from Prowler for a full iteration. `parsers/finos_ccc.py` handles both shapes; if you write a new parser for a similar format, test with at least one file of each shape. +- **Whitespace collapse**. Upstream YAML multi-line block scalars (`|`) preserve newlines. Prowler stores descriptions single-line. Collapse with `" ".join(value.split())` before emitting (see `parsers/finos_ccc.py::clean()`). +- **Foreign-prefix AR id rewriting**. Upstream sometimes aliases requirements across catalogs by keeping the original prefix (e.g., `CCC.AuditLog.CN08.AR01` appears nested under `CCC.Logging.CN03`). Rewrite the foreign id to fit its parent control: `CCC.Logging.CN03.AR01`. This logic is parser-specific because the id structure varies per framework (CCC uses 3-dot depth; CIS uses numeric dots; NIST uses `AC-2(1)`). +- **Genuine upstream collision renumbering**. Sometimes upstream has a real typo where two different requirements share the same id (e.g., `CCC.Core.CN14.AR02` defined twice for 30-day and 14-day backup variants). Renumber the second copy to the next free AR number (`.AR03`). The parser handles this; the runner asserts the final list has unique ids as a safety net. +- **Existing check mapping preservation**. The runner uses the `primary_key` + `fallback_keys` declared in config to look up the old `Checks` list for each requirement. For CCC this means primary index by `Id` plus fallback index by `(Section, frozenset(Applicability))` — the fallback recovers mappings for requirements whose ids were rewritten or renumbered by the parser. +- **FamilyName normalization**. Configured via `post_processing.family_name_normalization` — no code changes needed to collapse upstream variants like `"Logging & Monitoring"` → `"Logging and Monitoring"`. +- **Populate `Version`**. The runner refuses to start on empty `framework.version` — fail-fast replaces the silent bug where `get_check_compliance()` would build the key as just `"{Framework}"`. + +### Step 3 — Validate before committing + +```python +from prowler.lib.check.compliance_models import Compliance +for prov in ['aws', 'azure', 'gcp']: + c = Compliance.parse_file(f"prowler/compliance/{prov}/ccc_{prov}.json") + print(f"{prov}: {len(c.Requirements)} reqs, version={c.Version}") +``` + +Any `ValidationError` means the Attribute fields don't match the `*_Requirement_Attribute` model. Either fix the JSON or extend the model in `compliance_models.py` (remember: Generic stays last). + +### Step 4 — Verify every check id exists + +```python +import json +from pathlib import Path +for prov in ['aws', 'azure', 'gcp']: + existing = {p.stem.replace('.metadata','') + for p in Path(f'prowler/providers/{prov}/services').rglob('*.metadata.json')} + with open(f'prowler/compliance/{prov}/ccc_{prov}.json') as f: + data = json.load(f) + refs = {c for r in data['Requirements'] for c in r['Checks']} + missing = refs - existing + assert not missing, f"{prov} missing: {missing}" +``` + +A stale check id silently becomes dead weight — no finding will ever map to it. This pre-validation **must run on every write**; bake it into the generator script. + +### Step 5 — Add an attribute model if needed + +Only if the framework has fields beyond `Generic_Compliance_Requirement_Attribute`. Add the class to `prowler/lib/check/compliance_models.py` and register it in `Compliance_Requirement.Attributes: list[Union[...]]`. **Generic stays last.** + +--- + +## Workflow B: Audit Check Mappings as a Cloud Auditor + +Use when the user asks to review existing mappings ("are these correct?", "verify that the checks apply", "audit the CCC mappings"). This is the highest-value compliance task — it surfaces padded mappings with zero actual coverage and missing mappings for legitimate coverage. + +### The golden rule + +> A Prowler check's title/risk MUST **literally describe what the requirement text says**. "Related" is not enough. If no check actually addresses the requirement, leave `Checks: []` (MANUAL) — **honest MANUAL is worth more than padded coverage**. + +### Audit process + +**Step 1 — Build a per-provider check inventory** (cache in `/tmp/`): + +```python +import json +from pathlib import Path +for provider in ['aws', 'azure', 'gcp']: + inv = {} + for meta in Path(f'prowler/providers/{provider}/services').rglob('*.metadata.json'): + with open(meta) as f: + d = json.load(f) + cid = d.get('CheckID') or meta.stem.replace('.metadata','') + inv[cid] = { + 'service': d.get('ServiceName', ''), + 'title': d.get('CheckTitle', ''), + 'risk': d.get('Risk', ''), + 'description': d.get('Description', ''), + } + with open(f'/tmp/checks_{provider}.json', 'w') as f: + json.dump(inv, f, indent=2) +``` + +**Step 2 — Keyword/service query helper** — see [assets/query_checks.py](assets/query_checks.py): + +```bash +python assets/query_checks.py aws encryption transit # keyword AND-search +python assets/query_checks.py aws --service iam # all iam checks +python assets/query_checks.py aws --id kms_cmk_rotation_enabled # full metadata +``` + +**Step 3 — Dump a framework section with current mappings** — see [assets/dump_section.py](assets/dump_section.py): + +```bash +python assets/dump_section.py ccc "CCC.Core." # all Core ARs across 3 providers +python assets/dump_section.py ccc "CCC.AuditLog." # all AuditLog ARs +``` + +**Step 4 — Encode explicit REPLACE decisions** — see [assets/audit_framework_template.py](assets/audit_framework_template.py). Structure: + +```python +DECISIONS = {} + +DECISIONS["CCC.Core.CN01.AR01"] = { + "aws": [ + "cloudfront_distributions_https_enabled", + "cloudfront_distributions_origin_traffic_encrypted", + # ... + ], + "azure": [ + "storage_secure_transfer_required_is_enabled", + "app_minimum_tls_version_12", + # ... + ], + "gcp": [ + "cloudsql_instance_ssl_connections", + ], + # Missing provider key = leave the legacy mapping untouched +} + +# Empty list = EXPLICITLY MANUAL (overwrites legacy) +DECISIONS["CCC.Core.CN01.AR07"] = { + "aws": [], # Prowler has no IANA port/protocol check + "azure": [], + "gcp": [], +} +``` + +**REPLACE, not PATCH.** Encoding every mapping as a full list (not add/remove delta) makes the audit reproducible and surfaces hidden assumptions from the legacy data. + +**Step 5 — Pre-validation**. The audit script MUST validate every check id against the inventory and **abort with stderr listing typos**. Common typos caught during a real audit: + +- `fsx_file_system_encryption_at_rest_using_kms` (doesn't exist) +- `cosmosdb_account_encryption_at_rest_with_cmk` (doesn't exist) +- `sqlserver_geo_replication` (doesn't exist) +- `redshift_cluster_audit_logging` (should be `redshift_cluster_encrypted_at_rest`) +- `postgresql_flexible_server_require_secure_transport` (should be `postgresql_flexible_server_enforce_ssl_enabled`) +- `storage_secure_transfer_required_enabled` (should be `storage_secure_transfer_required_is_enabled`) +- `sqlserver_minimum_tls_version_12` (should be `sqlserver_recommended_minimal_tls_version`) + +**Step 6 — Apply + validate + test**: + +```bash +python /path/to/audit_script.py # applies decisions, pre-validates +python -m pytest tests/lib/outputs/compliance/ tests/lib/check/ -q +``` + +### Audit Reference Table: Requirement Text → Prowler Checks + +Use this table to map CCC-style / NIST-style / ISO-style requirements to the checks that actually verify them. Built from a real audit of 172 CCC ARs × 3 providers. + +| Requirement text | AWS checks | Azure checks | GCP checks | +|---|---|---|---| +| **TLS in transit enforced** | `cloudfront_distributions_https_enabled`, `s3_bucket_secure_transport_policy`, `elbv2_ssl_listeners`, `elbv2_insecure_ssl_ciphers`, `elb_ssl_listeners`, `elb_insecure_ssl_ciphers`, `opensearch_service_domains_https_communications_enforced`, `rds_instance_transport_encrypted`, `redshift_cluster_in_transit_encryption_enabled`, `elasticache_redis_cluster_in_transit_encryption_enabled`, `dynamodb_accelerator_cluster_in_transit_encryption_enabled`, `dms_endpoint_ssl_enabled`, `kafka_cluster_in_transit_encryption_enabled`, `transfer_server_in_transit_encryption_enabled`, `glue_database_connections_ssl_enabled`, `sns_subscription_not_using_http_endpoints` | `storage_secure_transfer_required_is_enabled`, `storage_ensure_minimum_tls_version_12`, `postgresql_flexible_server_enforce_ssl_enabled`, `mysql_flexible_server_ssl_connection_enabled`, `mysql_flexible_server_minimum_tls_version_12`, `sqlserver_recommended_minimal_tls_version`, `app_minimum_tls_version_12`, `app_ensure_http_is_redirected_to_https`, `app_ftp_deployment_disabled` | `cloudsql_instance_ssl_connections` (almost only option) | +| **TLS 1.3 specifically** | Partial: `cloudfront_distributions_using_deprecated_ssl_protocols`, `elb*_insecure_ssl_ciphers`, `*_minimum_tls_version_12` | Partial: `*_minimum_tls_version_12` checks | None — accept as MANUAL | +| **SSH / port 22 hardening** | `ec2_instance_port_ssh_exposed_to_internet`, `ec2_securitygroup_allow_ingress_from_internet_to_tcp_port_22`, `ec2_networkacl_allow_ingress_tcp_port_22` | `network_ssh_internet_access_restricted`, `vm_linux_enforce_ssh_authentication` | `compute_firewall_ssh_access_from_the_internet_allowed`, `compute_instance_block_project_wide_ssh_keys_disabled`, `compute_project_os_login_enabled`, `compute_project_os_login_2fa_enabled` | +| **mTLS (mutual TLS)** | `kafka_cluster_mutual_tls_authentication_enabled`, `apigateway_restapi_client_certificate_enabled` | `app_client_certificates_on` | None — MANUAL | +| **Data at rest encrypted** | `s3_bucket_default_encryption`, `s3_bucket_kms_encryption`, `ec2_ebs_default_encryption`, `ec2_ebs_volume_encryption`, `rds_instance_storage_encrypted`, `rds_cluster_storage_encrypted`, `rds_snapshots_encrypted`, `dynamodb_tables_kms_cmk_encryption_enabled`, `redshift_cluster_encrypted_at_rest`, `neptune_cluster_storage_encrypted`, `documentdb_cluster_storage_encrypted`, `opensearch_service_domains_encryption_at_rest_enabled`, `kinesis_stream_encrypted_at_rest`, `firehose_stream_encrypted_at_rest`, `sns_topics_kms_encryption_at_rest_enabled`, `sqs_queues_server_side_encryption_enabled`, `efs_encryption_at_rest_enabled`, `athena_workgroup_encryption`, `glue_data_catalogs_metadata_encryption_enabled`, `backup_vaults_encrypted`, `backup_recovery_point_encrypted`, `cloudtrail_kms_encryption_enabled`, `cloudwatch_log_group_kms_encryption_enabled`, `eks_cluster_kms_cmk_encryption_in_secrets_enabled`, `sagemaker_notebook_instance_encryption_enabled`, `apigateway_restapi_cache_encrypted`, `kafka_cluster_encryption_at_rest_uses_cmk`, `dynamodb_accelerator_cluster_encryption_enabled`, `storagegateway_fileshare_encryption_enabled` | `storage_infrastructure_encryption_is_enabled`, `storage_ensure_encryption_with_customer_managed_keys`, `vm_ensure_attached_disks_encrypted_with_cmk`, `vm_ensure_unattached_disks_encrypted_with_cmk`, `sqlserver_tde_encryption_enabled`, `sqlserver_tde_encrypted_with_cmk`, `databricks_workspace_cmk_encryption_enabled`, `monitor_storage_account_with_activity_logs_cmk_encrypted` | `compute_instance_encryption_with_csek_enabled`, `dataproc_encrypted_with_cmks_disabled`, `bigquery_dataset_cmk_encryption`, `bigquery_table_cmk_encryption` | +| **CMEK required (customer-managed keys)** | `kms_cmk_are_used` | `storage_ensure_encryption_with_customer_managed_keys`, `vm_ensure_attached_disks_encrypted_with_cmk`, `vm_ensure_unattached_disks_encrypted_with_cmk`, `sqlserver_tde_encrypted_with_cmk`, `databricks_workspace_cmk_encryption_enabled` | `bigquery_dataset_cmk_encryption`, `bigquery_table_cmk_encryption`, `dataproc_encrypted_with_cmks_disabled`, `compute_instance_encryption_with_csek_enabled` | +| **Key rotation enabled** | `kms_cmk_rotation_enabled` | `keyvault_key_rotation_enabled`, `storage_key_rotation_90_days` | `kms_key_rotation_enabled` | +| **MFA for UI access** | `iam_root_mfa_enabled`, `iam_root_hardware_mfa_enabled`, `iam_user_mfa_enabled_console_access`, `iam_user_hardware_mfa_enabled`, `iam_administrator_access_with_mfa`, `cognito_user_pool_mfa_enabled` | `entra_privileged_user_has_mfa`, `entra_non_privileged_user_has_mfa`, `entra_user_with_vm_access_has_mfa`, `entra_security_defaults_enabled` | `compute_project_os_login_2fa_enabled` | +| **API access / credentials** | `iam_no_root_access_key`, `iam_user_no_setup_initial_access_key`, `apigateway_restapi_authorizers_enabled`, `apigateway_restapi_public_with_authorizer`, `apigatewayv2_api_authorizers_enabled` | `entra_conditional_access_policy_require_mfa_for_management_api`, `app_function_access_keys_configured`, `app_function_identity_is_configured` | `apikeys_api_restrictions_configured`, `apikeys_key_exists`, `apikeys_key_rotated_in_90_days` | +| **Log all admin/config changes** | `cloudtrail_multi_region_enabled`, `cloudtrail_multi_region_enabled_logging_management_events`, `cloudtrail_cloudwatch_logging_enabled`, `cloudtrail_log_file_validation_enabled`, `cloudwatch_log_metric_filter_*`, `cloudwatch_changes_to_*_alarm_configured`, `config_recorder_all_regions_enabled` | `monitor_diagnostic_settings_exists`, `monitor_diagnostic_setting_with_appropriate_categories`, `monitor_alert_*` | `iam_audit_logs_enabled`, `logging_log_metric_filter_and_alert_for_*`, `logging_sink_created` | +| **Log integrity (digital signatures)** | `cloudtrail_log_file_validation_enabled` (exact) | None | None | +| **Public access denied** | `s3_bucket_public_access`, `s3_bucket_public_list_acl`, `s3_bucket_public_write_acl`, `s3_account_level_public_access_blocks`, `apigateway_restapi_public`, `awslambda_function_url_public`, `awslambda_function_not_publicly_accessible`, `rds_instance_no_public_access`, `rds_snapshots_public_access`, `ec2_securitygroup_allow_ingress_from_internet_to_all_ports`, `sns_topics_not_publicly_accessible`, `sqs_queues_not_publicly_accessible` | `storage_blob_public_access_level_is_disabled`, `storage_ensure_private_endpoints_in_storage_accounts`, `containerregistry_not_publicly_accessible`, `keyvault_private_endpoints`, `app_function_not_publicly_accessible`, `aks_clusters_public_access_disabled`, `network_http_internet_access_restricted` | `cloudstorage_bucket_public_access`, `compute_instance_public_ip`, `cloudsql_instance_public_ip`, `compute_firewall_*_access_from_the_internet_allowed` | +| **IAM least privilege** | `iam_*_no_administrative_privileges`, `iam_policy_allows_privilege_escalation`, `iam_inline_policy_allows_privilege_escalation`, `iam_role_administratoraccess_policy`, `iam_group_administrator_access_policy`, `iam_user_administrator_access_policy`, `iam_policy_attached_only_to_group_or_roles`, `iam_role_cross_service_confused_deputy_prevention` | `iam_role_user_access_admin_restricted`, `iam_subscription_roles_owner_custom_not_created`, `iam_custom_role_has_permissions_to_administer_resource_locks` | `iam_sa_no_administrative_privileges`, `iam_no_service_roles_at_project_level`, `iam_role_kms_enforce_separation_of_duties`, `iam_role_sa_enforce_separation_of_duties` | +| **Password policy** | `iam_password_policy_minimum_length_14`, `iam_password_policy_uppercase`, `iam_password_policy_lowercase`, `iam_password_policy_symbol`, `iam_password_policy_number`, `iam_password_policy_expires_passwords_within_90_days_or_less`, `iam_password_policy_reuse_24` | None | None | +| **Credential rotation / unused** | `iam_rotate_access_key_90_days`, `iam_user_accesskey_unused`, `iam_user_console_access_unused` | None | `iam_sa_user_managed_key_rotate_90_days`, `iam_sa_user_managed_key_unused`, `iam_service_account_unused` | +| **VPC / flow logs** | `vpc_flow_logs_enabled` | `network_flow_log_captured_sent`, `network_watcher_enabled`, `network_flow_log_more_than_90_days` | `compute_subnet_flow_logs_enabled` | +| **Backup / DR / Multi-AZ** | `backup_vaults_exist`, `backup_plans_exist`, `backup_reportplans_exist`, `rds_instance_backup_enabled`, `rds_*_protected_by_backup_plan`, `rds_cluster_multi_az`, `neptune_cluster_backup_enabled`, `documentdb_cluster_backup_enabled`, `efs_have_backup_enabled`, `s3_bucket_cross_region_replication`, `dynamodb_table_protected_by_backup_plan` | `vm_backup_enabled`, `vm_sufficient_daily_backup_retention_period`, `storage_geo_redundant_enabled` | `cloudsql_instance_automated_backups`, `cloudstorage_bucket_log_retention_policy_lock`, `cloudstorage_bucket_sufficient_retention_period` | +| **Access analysis / discovery** | `accessanalyzer_enabled`, `accessanalyzer_enabled_without_findings` | None specific | `iam_account_access_approval_enabled`, `iam_cloud_asset_inventory_enabled` | +| **Object lock / retention** | `s3_bucket_object_lock`, `s3_bucket_object_versioning`, `s3_bucket_lifecycle_enabled`, `cloudtrail_bucket_requires_mfa_delete`, `s3_bucket_no_mfa_delete` | `storage_ensure_soft_delete_is_enabled`, `storage_blob_versioning_is_enabled`, `storage_ensure_file_shares_soft_delete_is_enabled` | `cloudstorage_bucket_log_retention_policy_lock`, `cloudstorage_bucket_soft_delete_enabled`, `cloudstorage_bucket_versioning_enabled`, `cloudstorage_bucket_sufficient_retention_period` | +| **Uniform bucket-level access** | `s3_bucket_acl_prohibited` | `storage_account_key_access_disabled`, `storage_default_to_entra_authorization_enabled` | `cloudstorage_bucket_uniform_bucket_level_access` | +| **Container vulnerability scanning** | `ecr_registry_scan_images_on_push_enabled`, `ecr_repositories_scan_vulnerabilities_in_latest_image` | `defender_container_images_scan_enabled`, `defender_container_images_resolved_vulnerabilities` | `artifacts_container_analysis_enabled`, `gcr_container_scanning_enabled` | +| **WAF / rate limiting** | `wafv2_webacl_with_rules`, `waf_*_webacl_with_rules`, `wafv2_webacl_logging_enabled`, `waf_global_webacl_logging_enabled` | None | None | +| **Deployment region restriction** | `organizations_scp_check_deny_regions` | None | None | +| **Secrets automatic rotation** | `secretsmanager_automatic_rotation_enabled`, `secretsmanager_secret_rotated_periodically` | `keyvault_rbac_secret_expiration_set`, `keyvault_non_rbac_secret_expiration_set` | None | +| **Certificate management** | `acm_certificates_expiration_check`, `acm_certificates_with_secure_key_algorithms`, `acm_certificates_transparency_logs_enabled` | `keyvault_key_expiration_set_in_non_rbac`, `keyvault_rbac_key_expiration_set`, `keyvault_non_rbac_secret_expiration_set` | None | +| **GenAI guardrails / input/output filtering** | `bedrock_guardrail_prompt_attack_filter_enabled`, `bedrock_guardrail_sensitive_information_filter_enabled`, `bedrock_agent_guardrail_enabled`, `bedrock_model_invocation_logging_enabled`, `bedrock_api_key_no_administrative_privileges`, `bedrock_api_key_no_long_term_credentials` | None | None | +| **ML dev environment security** | `sagemaker_notebook_instance_root_access_disabled`, `sagemaker_notebook_instance_without_direct_internet_access_configured`, `sagemaker_notebook_instance_vpc_settings_configured`, `sagemaker_models_vpc_settings_configured`, `sagemaker_training_jobs_vpc_settings_configured`, `sagemaker_training_jobs_network_isolation_enabled`, `sagemaker_training_jobs_volume_and_output_encryption_enabled` | None | None | +| **Threat detection / anomalous behavior** | `cloudtrail_threat_detection_enumeration`, `cloudtrail_threat_detection_privilege_escalation`, `cloudtrail_threat_detection_llm_jacking`, `guardduty_is_enabled`, `guardduty_no_high_severity_findings` | None | None | +| **Serverless private access** | `awslambda_function_inside_vpc`, `awslambda_function_not_publicly_accessible`, `awslambda_function_url_public` | `app_function_not_publicly_accessible` | None | + +### What Prowler Does NOT Cover (accept MANUAL honestly) + +Don't pad mappings for these — mark `Checks: []` and move on: + +- **TLS 1.3 version specifically** — Prowler verifies TLS is enforced, not always the exact version +- **IANA port-protocol consistency** — no check for "protocol running on its assigned port" +- **mTLS on most Azure/GCP services** — limited to App Service client certs on Azure, nothing on GCP +- **Rate limiting** on monitoring endpoints, load balancers, serverless invocations, vector ingestion +- **Session cookie expiry** (LB stickiness) +- **HTTP header scrubbing** (Server, X-Powered-By) +- **Certificate transparency verification for imports** +- **Model version pinning, red teaming, AI quality review** +- **Vector embedding validation, dimensional constraints, ANN vs exact search** +- **Secret region replication** (cross-region residency) +- **Lifecycle cleanup policies on container registries** +- **Row-level / column-level security in data warehouses** +- **Deployment region restriction on Azure/GCP** (AWS has `organizations_scp_check_deny_regions`, others don't) +- **Cross-tenant alert silencing permissions** +- **Field-level masking in logs** +- **Managed view enforcement for database access** +- **Automatic MFA delete on all S3 buckets** (only CloudTrail bucket variant exists for some frameworks — AWS has the generic `s3_bucket_no_mfa_delete` though) + +--- + +## Workflow C: Add a New Output Formatter + +Use when a new framework needs its own CSV columns or terminal table. Follow the c5/csa/ens layout exactly: + +```bash +mkdir -p prowler/lib/outputs/compliance/{framework} +touch prowler/lib/outputs/compliance/{framework}/__init__.py +``` + +### Step 1 — Create `{framework}.py` (table dispatcher ONLY) + +Copy from `prowler/lib/outputs/compliance/c5/c5.py` and change the function name + framework string. The `diff` between your file and `c5.py` should be just those two lines. **No function docstring** — other frameworks don't have one, stay consistent. + +### Step 2 — Create `models.py` + +One Pydantic v2 `BaseModel` per provider. Field names become CSV column headers (public API — don't rename later without a migration). + +```python +from typing import Optional +from pydantic import BaseModel + +class {Framework}_AWSModel(BaseModel): + Provider: str + Description: str + AccountId: str + Region: str + AssessmentDate: str + Requirements_Id: str + Requirements_Description: str + # ... provider-specific columns + Status: str + StatusExtended: str + ResourceId: str + ResourceName: str + CheckId: str + Muted: bool +``` + +### Step 3 — Create `{framework}_{provider}.py` for each provider + +Copy from `prowler/lib/outputs/compliance/c5/c5_aws.py` etc. Contains the `{Framework}_AWS(ComplianceOutput)` class with `transform()` that walks findings and emits model rows. This file IS allowed to import `Finding`. + +### Step 4 — Register everywhere + +**`prowler/lib/outputs/compliance/compliance.py`** (CLI table dispatcher): +```python +from prowler.lib.outputs.compliance.{framework}.{framework} import get_{framework}_table + +def display_compliance_table(...): + ... + elif compliance_framework.startswith("{framework}_"): + get_{framework}_table(findings, bulk_checks_metadata, + compliance_framework, output_filename, + output_directory, compliance_overview) +``` + +**`prowler/__main__.py`** (CLI output writer per provider): +Add imports at the top: +```python +from prowler.lib.outputs.compliance.{framework}.{framework}_aws import {Framework}_AWS +from prowler.lib.outputs.compliance.{framework}.{framework}_azure import {Framework}_Azure +from prowler.lib.outputs.compliance.{framework}.{framework}_gcp import {Framework}_GCP +``` +Add provider-specific `elif compliance_name.startswith("{framework}_"):` branches that instantiate the class and call `batch_write_data_to_file()`. + +**`api/src/backend/tasks/jobs/export.py`** (API export dispatcher): +```python +from prowler.lib.outputs.compliance.{framework}.{framework}_aws import {Framework}_AWS +# ... azure, gcp + +COMPLIANCE_CLASS_MAP = { + "aws": [ + # ... + (lambda name: name.startswith("{framework}_"), {Framework}_AWS), + ], + # ... azure, gcp +} +``` + +**Always use `startswith`**, never `name == "framework_aws"`. Exact match is a regression. + +### Step 5 — Add tests + +Create `tests/lib/outputs/compliance/{framework}/` with `{framework}_aws_test.py`, `{framework}_azure_test.py`, `{framework}_gcp_test.py`. See the test template in [references/test_template.md](references/test_template.md). + +Add fixtures to `tests/lib/outputs/compliance/fixtures.py`: one `Compliance` object per provider with 1 evaluated + 1 manual requirement to exercise both code paths in `transform()`. + +### Circular import warning + +**The table dispatcher file (`{framework}.py`) MUST NOT import `Finding`** (directly or transitively). The cycle is: + +``` +compliance.compliance imports get_{framework}_table + → {framework}.py imports ComplianceOutput + → compliance_output imports Finding + → finding imports get_check_compliance from compliance.compliance + → CIRCULAR +``` + +Keep `{framework}.py` bare — only `colorama`, `tabulate`, `prowler.config.config`. Put anything that imports `Finding` in the per-provider `{framework}_{provider}.py` files. + +--- + +## Conventions and Hard-Won Gotchas + +These are lessons from the FINOS CCC v2025.10 sync + 172-AR audit pass (April 2026). Learn them once; save days of debugging. + +1. **Per-provider files are non-negotiable.** Never collapse `{framework}_aws.py`, `{framework}_azure.py`, `{framework}_gcp.py` into a single parameterized class, no matter how DRY-tempting. Every other framework in the codebase follows the per-provider pattern and reviewers will reject the refactor. The CSV column names differ per provider — three classes is the convention. +2. **`{framework}.py` has NO function docstring.** Other frameworks don't have them. Don't add one to be "helpful". +3. **Circular import protection**: the table dispatcher file MUST NOT import `Finding` (directly or transitively). Split the code so `{framework}.py` only has `get_{framework}_table()` with bare imports, and `{framework}_{provider}.py` holds the class that needs `Finding`. +4. **`Generic_Compliance_Requirement_Attribute` is the fallback** — in the `Compliance_Requirement.Attributes` Union in `compliance_models.py`, Generic MUST be LAST because Pydantic v1 tries union members in order. Putting Generic first means every framework-specific attribute falls through to Generic and the specific model is never used. +5. **Pydantic v1 imports.** `from pydantic.v1 import BaseModel` in `compliance_models.py` — not v2. Mixing causes validation errors. Pydantic v2 is used in the CSV models (`models.py`) — that's fine because they're separate trees. +6. **`get_check_compliance()` key format** is `f"{Framework}-{Version}"` ONLY if Version is set. Empty Version → key is `"{Framework}"` (no version suffix). Tests that mock compliance dicts must match this exact format — when a framework ships with `Version: ""`, downstream code and tests break silently. +7. **CSV column names from `models.py` are public API.** Don't rename a field without migrating downstream consumers — CSV headers change. +8. **Upstream YAML multi-line scalars** (`|` block scalars) preserve newlines. Collapse to single-line with `" ".join(value.split())` before writing to JSON. +9. **Upstream catalogs can use multiple shapes.** FINOS CCC uses `control-families: [...]` in most catalogs but `controls: [...]` at the top level in `storage/object`. Any sync script must handle both or silently drop entire catalogs. +10. **Foreign-prefix AR ids.** Upstream sometimes "imports" requirements from one catalog into another by keeping the original id prefix (e.g., `CCC.AuditLog.CN08.AR01` appearing under `CCC.Logging.CN03`). Prowler's compliance model requires unique ids within a catalog — rewrite the foreign id to fit the parent control: `CCC.AuditLog.CN08.AR01` (inside `CCC.Logging.CN03`) → `CCC.Logging.CN03.AR01`. +11. **Genuine upstream id collisions.** Sometimes upstream has a real typo where two different requirements share the same id (e.g., `CCC.Core.CN14.AR02` defined twice for 30-day and 14-day backup variants). Renumber the second copy to the next free AR number. Preserve check mappings by matching on `(Section, frozenset(Applicability))` since the renumbered id won't match by id. +12. **`COMPLIANCE_CLASS_MAP` in `export.py` uses `startswith` predicates** for all modern frameworks. Exact match (`name == "ccc_aws"`) is an anti-pattern — it was present for CCC until April 2026 and was the reason CCC couldn't have versioned variants. +13. **Pre-validate every check id** against the per-provider inventory before writing the JSON. A typo silently creates an unreferenced check that will fail when findings try to map to it. The audit script MUST abort with stderr listing typos, not swallow them. +14. **REPLACE is better than PATCH** for audit decisions. Encoding every mapping explicitly makes the audit reproducible and surfaces hidden assumptions from the legacy data. A PATCH system that adds/removes is too easy to forget. +15. **When no check applies, MANUAL is correct.** Do not pad mappings with tangential checks "just in case". Prowler's compliance reports are meant to be actionable — padding them with noise breaks that. Honest manual reqs can be mapped later when new checks land. +16. **UI groups by `Attributes[0].FamilyName` and `Attributes[0].Section`.** If FamilyName has inconsistent variants within the same JSON (e.g., "Logging & Monitoring" vs "Logging and Monitoring"), the UI renders them as separate categories. Section empty → the requirement falls into an orphan control with label "". Normalize before shipping. +17. **Provider coverage is asymmetric.** AWS has dense coverage (~586 checks across 80+ services): in-transit encryption, IAM, database encryption, backup. Azure (~167 checks) and GCP (~102 checks) are thinner especially for in-transit encryption, mTLS, and ML/AI. Accept the asymmetry in mappings — don't force GCP parity where Prowler genuinely can't verify. + +--- + +## Useful One-Liners + +```bash +# Count requirements per service prefix (CCC, CIS sections, etc.) +jq -r '.Requirements[].Id | split(".")[1]' prowler/compliance/aws/ccc_aws.json | sort | uniq -c + +# Find duplicate requirement IDs +jq -r '.Requirements[].Id' file.json | sort | uniq -d + +# Count manual requirements (no checks) +jq '[.Requirements[] | select((.Checks | length) == 0)] | length' file.json + +# List all unique check references in a framework +jq -r '.Requirements[].Checks[]' file.json | sort -u + +# List all unique Sections (to spot inconsistency) +jq '[.Requirements[].Attributes[0].Section] | unique' file.json + +# List all unique FamilyNames (to spot inconsistency) +jq '[.Requirements[].Attributes[0].FamilyName] | unique' file.json + +# Diff requirement ids between two versions of the same framework +diff <(jq -r '.Requirements[].Id' a.json | sort) <(jq -r '.Requirements[].Id' b.json | sort) + +# Find where a check id is used across all frameworks +grep -rl "my_check_name" prowler/compliance/ + +# Check if a Prowler check exists +find prowler/providers/aws/services -name "{check_id}.metadata.json" + +# Validate a JSON with Pydantic +python -c "from prowler.lib.check.compliance_models import Compliance; print(Compliance.parse_file('prowler/compliance/aws/ccc_aws.json').Framework)" +``` + +--- + ## Best Practices 1. **Requirement IDs**: Follow the original framework numbering exactly (e.g., "1.1", "A.5.1", "T1190", "ac_2_1") -2. **Check Mapping**: Map to existing checks when possible. Use `Checks: []` for manual-only requirements +2. **Check Mapping**: Map to existing checks when possible. Use `Checks: []` for manual-only requirements — honest MANUAL beats padded coverage 3. **Completeness**: Include all framework requirements, even those without automated checks -4. **Version Control**: Include framework version in `Name` and `Version` fields +4. **Version Control**: Include framework version in `Name` and `Version` fields. **Never leave `Version: ""`** — it breaks `get_check_compliance()` key format 5. **File Naming**: Use format `{framework}_{version}_{provider}.json` -6. **Validation**: Prowler validates JSON against Pydantic models at startup - invalid JSON will cause errors +6. **Validation**: Prowler validates JSON against Pydantic models at startup — invalid JSON will cause errors +7. **Pre-validate check ids** against the provider's `*.metadata.json` inventory before every commit +8. **Normalize FamilyName and Section** to avoid inconsistent UI tree branches +9. **Register everywhere**: SDK model (if needed) → `compliance.py` dispatcher → `__main__.py` CLI writer → `export.py` API map → UI mapper. Skipping any layer results in silent failures +10. **Audit, don't pad**: when reviewing mappings, apply the golden rule — the check's title/risk MUST literally describe what the requirement text says. Tangential relation doesn't count ## Commands @@ -482,11 +1005,46 @@ prowler aws --compliance cis_5.0_aws -M csv json html ## Code References -- **Compliance Models:** `prowler/lib/check/compliance_models.py` -- **Compliance Processing:** `prowler/lib/check/compliance.py` -- **Compliance Output:** `prowler/lib/outputs/compliance/` +### Layer 1 — SDK / Core +- **Compliance Models:** `prowler/lib/check/compliance_models.py` (Pydantic v1 model tree) +- **Compliance Processing / Linker:** `prowler/lib/check/compliance.py` (`get_check_compliance`, `update_checks_metadata_with_compliance`) +- **Check Utils:** `prowler/lib/check/utils.py` (`list_compliance_modules`) + +### Layer 2 — JSON Catalogs +- **Framework JSONs:** `prowler/compliance/{provider}/` (auto-discovered via directory walk) + +### Layer 3 — Output Formatters +- **Per-framework folders:** `prowler/lib/outputs/compliance/{framework}/` +- **Shared base class:** `prowler/lib/outputs/compliance/compliance_output.py` (`ComplianceOutput` + `batch_write_data_to_file`) +- **CLI table dispatcher:** `prowler/lib/outputs/compliance/compliance.py` (`display_compliance_table`) +- **Finding model:** `prowler/lib/outputs/finding.py` (**do not import transitively from table dispatcher files — circular import**) +- **CLI writer:** `prowler/__main__.py` (per-provider `elif compliance_name.startswith(...)` branches that instantiate per-provider classes) + +### Layer 4 — API / UI +- **API lazy loader:** `api/src/backend/api/compliance.py` (`LazyComplianceTemplate`, `LazyChecksMapping`) +- **API export dispatcher:** `api/src/backend/tasks/jobs/export.py` (`COMPLIANCE_CLASS_MAP` with `startswith` predicates) +- **UI framework router:** `ui/lib/compliance/compliance-mapper.ts` +- **UI per-framework mapper:** `ui/lib/compliance/{framework}.tsx` +- **UI detail panel:** `ui/components/compliance/compliance-custom-details/{framework}-details.tsx` +- **UI types:** `ui/types/compliance.ts` +- **UI icon:** `ui/components/icons/compliance/{framework}.svg` + registration in `IconCompliance.tsx` + +### Tests +- **Output formatter tests:** `tests/lib/outputs/compliance/{framework}/{framework}_{provider}_test.py` +- **Shared fixtures:** `tests/lib/outputs/compliance/fixtures.py` ## Resources -- **Templates:** See [assets/](assets/) for framework JSON templates +- **JSON Templates:** See [assets/](assets/) for framework JSON templates (cis, ens, iso27001, mitre_attack, prowler_threatscore, generic) +- **Config-driven compliance sync** (any upstream-backed framework): + - [assets/sync_framework.py](assets/sync_framework.py) — generic runner. Loads a YAML config, dynamically imports the declared parser, applies generic post-processing (id uniqueness safety net, `FamilyName` normalization, legacy check-mapping preservation with config-driven fallback keys), and writes the provider JSONs with Pydantic post-validation. Framework-agnostic — works for any compliance framework. + - [assets/configs/ccc.yaml](assets/configs/ccc.yaml) — canonical config example (FINOS CCC v2025.10). Copy and adapt for new frameworks. + - [assets/parsers/finos_ccc.py](assets/parsers/finos_ccc.py) — FINOS CCC YAML parser. Handles both upstream shapes (`control-families` and top-level `controls`), foreign-prefix AR rewriting, and genuine collision renumbering. Exposes `parse_upstream(config) -> list[dict]`. + - [assets/parsers/](assets/parsers/) — add new parser modules here for unfamiliar upstream formats (NIST OSCAL JSON, MITRE STIX, CIS Benchmarks, etc.). Each parser is a `{name}.py` file implementing `parse_upstream(config) -> list[dict]` with guaranteed-unique ids. +- **Reusable audit tooling** (added April 2026 after the FINOS CCC v2025.10 sync): + - [assets/audit_framework_template.py](assets/audit_framework_template.py) — explicit REPLACE decision ledger with pre-validation against the per-provider inventory. Drop-in template for auditing any framework. + - [assets/query_checks.py](assets/query_checks.py) — keyword/service/id query helper over `/tmp/checks_{provider}.json`. + - [assets/dump_section.py](assets/dump_section.py) — dumps every AR for a given id prefix across all 3 providers with current check mappings. + - [assets/build_inventory.py](assets/build_inventory.py) — generates `/tmp/checks_{provider}.json` from `*.metadata.json` files. - **Documentation:** See [references/compliance-docs.md](references/compliance-docs.md) for additional resources +- **Related skill:** [prowler-compliance-review](../prowler-compliance-review/SKILL.md) — PR review checklist and validator script for compliance framework PRs diff --git a/skills/prowler-compliance/assets/audit_framework_template.py b/skills/prowler-compliance/assets/audit_framework_template.py new file mode 100644 index 0000000000..f2d58603d7 --- /dev/null +++ b/skills/prowler-compliance/assets/audit_framework_template.py @@ -0,0 +1,207 @@ +#!/usr/bin/env python3 +""" +Cloud-auditor pass template for any Prowler compliance framework. + +Encode explicit REPLACE decisions per (requirement_id, provider) pair below. +Each decision FULLY overwrites the legacy Checks list for that requirement. + +Workflow: + 1. Run build_inventory.py first to cache per-provider check metadata. + 2. Run dump_section.py to see current mappings for the catalog you're auditing. + 3. Fill in DECISIONS below with explicit check lists. + 4. Run this script — it pre-validates every check id against the inventory + and aborts with stderr listing typos before writing. + +Decision rules (apply as a hostile cloud auditor): + - The Prowler check's title/risk MUST literally describe what the AR text says. + "Related" is not enough. + - If no check actually addresses the requirement, leave `[]` (= MANUAL). + HONEST MANUAL is worth more than padded coverage. + - Missing provider key = leave the legacy mapping untouched. + - Empty list `[]` = explicitly MANUAL (overwrites legacy). + +Usage: + # 1. Copy this file to /tmp/audit_.py and fill in DECISIONS + # 2. Edit FRAMEWORK_KEY below to match your framework file naming + # 3. Run: + python /tmp/audit_.py +""" +from __future__ import annotations + +import json +import sys +from pathlib import Path + +# --------------------------------------------------------------------------- +# Configure for your framework +# --------------------------------------------------------------------------- + +# Framework file basename inside prowler/compliance/{provider}/. +# If your framework is called "cis_5.0_aws.json", FRAMEWORK_KEY is "cis_5.0". +# If the file is "ccc_aws.json", FRAMEWORK_KEY is "ccc". +FRAMEWORK_KEY = "ccc" + +# Which providers to apply decisions to. +PROVIDERS = ["aws", "azure", "gcp"] + +PROWLER_DIR = Path("prowler/compliance") +CHECK_INV = {prov: Path(f"/tmp/checks_{prov}.json") for prov in PROVIDERS} + + +# --------------------------------------------------------------------------- +# DECISIONS — encode one entry per requirement you want to audit +# --------------------------------------------------------------------------- + +# DECISIONS[requirement_id][provider] = list[str] of check ids +# See SKILL.md → "Audit Reference Table: Requirement Text → Prowler Checks" +# for a comprehensive mapping cheat sheet built from a 172-AR CCC audit. + +DECISIONS: dict[str, dict[str, list[str]]] = {} + +# ---- Example entries (delete and replace with your own) ---- + +# Example 1: TLS in transit enforced (non-SSH traffic) +# DECISIONS["CCC.Core.CN01.AR01"] = { +# "aws": [ +# "cloudfront_distributions_https_enabled", +# "cloudfront_distributions_origin_traffic_encrypted", +# "s3_bucket_secure_transport_policy", +# "elbv2_ssl_listeners", +# "rds_instance_transport_encrypted", +# "kafka_cluster_in_transit_encryption_enabled", +# "redshift_cluster_in_transit_encryption_enabled", +# "opensearch_service_domains_https_communications_enforced", +# ], +# "azure": [ +# "storage_secure_transfer_required_is_enabled", +# "app_minimum_tls_version_12", +# "postgresql_flexible_server_enforce_ssl_enabled", +# "sqlserver_recommended_minimal_tls_version", +# ], +# "gcp": [ +# "cloudsql_instance_ssl_connections", +# ], +# } + +# Example 2: MANUAL — no Prowler check exists +# DECISIONS["CCC.Core.CN01.AR07"] = { +# "aws": [], # no IANA port/protocol check exists in Prowler +# "azure": [], +# "gcp": [], +# } + +# Example 3: Reuse a decision for multiple sibling ARs +# DECISIONS["CCC.ObjStor.CN05.AR02"] = DECISIONS["CCC.ObjStor.CN05.AR01"] + + +# --------------------------------------------------------------------------- +# Driver — do not edit below +# --------------------------------------------------------------------------- + +def load_inventory(provider: str) -> dict: + path = CHECK_INV[provider] + if not path.exists(): + raise SystemExit( + f"Check inventory missing: {path}\n" + f"Run: python skills/prowler-compliance/assets/build_inventory.py {provider}" + ) + with open(path) as f: + return json.load(f) + + +def resolve_json_path(provider: str) -> Path: + """Resolve the JSON file path for a given provider. + + Handles both shapes: {FRAMEWORK_KEY}_{provider}.json (ccc_aws.json) and + cases where FRAMEWORK_KEY already contains the provider suffix. + """ + candidates = [ + PROWLER_DIR / provider / f"{FRAMEWORK_KEY}_{provider}.json", + PROWLER_DIR / provider / f"{FRAMEWORK_KEY}.json", + ] + for c in candidates: + if c.exists(): + return c + raise SystemExit( + f"Could not find framework JSON for provider={provider} " + f"with FRAMEWORK_KEY={FRAMEWORK_KEY}. Tried: {candidates}" + ) + + +def plan_for_provider( + provider: str, +) -> tuple[Path, dict, tuple[int, int, int], list[tuple[str, str]]]: + """Build the updated JSON for one provider without writing it. + + Returns (path, mutated_data, (touched, added, removed), unknowns). + Writing is deferred to a second pass so that a typo in any provider + aborts the whole run before any file on disk changes. + """ + path = resolve_json_path(provider) + with open(path) as f: + data = json.load(f) + inv = load_inventory(provider) + + touched = 0 + add_count = 0 + rm_count = 0 + unknown: list[tuple[str, str]] = [] + + for req in data["Requirements"]: + rid = req["Id"] + if rid not in DECISIONS or provider not in DECISIONS[rid]: + continue + new_checks = list(dict.fromkeys(DECISIONS[rid][provider])) + for c in new_checks: + if c not in inv: + unknown.append((rid, c)) + before = set(req.get("Checks") or []) + after = set(new_checks) + rm_count += len(before - after) + add_count += len(after - before) + req["Checks"] = new_checks + touched += 1 + + return path, data, (touched, add_count, rm_count), unknown + + +def main() -> int: + if not DECISIONS: + print("No DECISIONS encoded. Fill in the DECISIONS dict and re-run.") + return 1 + print(f"Applying {len(DECISIONS)} decisions to framework '{FRAMEWORK_KEY}'...") + + # Pass 1: validate every provider before touching disk. A typo in any + # provider must abort the run before ANY file has been rewritten. + plans: list[tuple[str, Path, dict, tuple[int, int, int]]] = [] + all_unknown: list[tuple[str, str, str]] = [] + for provider in PROVIDERS: + path, data, counts, unknown = plan_for_provider(provider) + for rid, c in unknown: + all_unknown.append((provider, rid, c)) + plans.append((provider, path, data, counts)) + + if all_unknown: + print("\n!! UNKNOWN CHECK IDS (typos?):", file=sys.stderr) + for provider, rid, c in all_unknown: + print(f" {provider} {rid} -> {c}", file=sys.stderr) + print( + "\nAborting: fix the check ids above and re-run. " + "No files were modified.", + file=sys.stderr, + ) + return 2 + + # Pass 2: all providers validated cleanly — write. + for provider, path, data, (touched, added, removed) in plans: + with open(path, "w") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + f.write("\n") + print( + f" {provider}: touched={touched} added={added} removed={removed}" + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/skills/prowler-compliance/assets/build_inventory.py b/skills/prowler-compliance/assets/build_inventory.py new file mode 100644 index 0000000000..f743aa75a5 --- /dev/null +++ b/skills/prowler-compliance/assets/build_inventory.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +""" +Build a per-provider check inventory by scanning Prowler's check metadata files. + +Outputs one JSON per provider at /tmp/checks_{provider}.json with the shape: + { + "check_id": { + "service": "...", + "subservice": "...", + "resource": "...", + "severity": "...", + "title": "...", + "description": "...", + "risk": "..." + }, + ... + } + +This is the reference used by audit_framework_template.py for pre-validation +(every check id in the audit ledger must exist in the inventory) and by +query_checks.py for keyword/service lookup. + +Usage: + python skills/prowler-compliance/assets/build_inventory.py + # Or for a specific provider: + python skills/prowler-compliance/assets/build_inventory.py aws + +Output: + /tmp/checks_{provider}.json for every provider discovered under + prowler/providers/ with a services/ directory. +""" +from __future__ import annotations + +import json +import sys +from pathlib import Path + +PROVIDERS_ROOT = Path("prowler/providers") + + +def discover_providers() -> list[str]: + """Return every provider that currently has a services/ directory. + + Derived from the filesystem so new providers are picked up automatically + and stale hard-coded lists cannot drift from the repo. + """ + if not PROVIDERS_ROOT.exists(): + return [] + return sorted( + p.name + for p in PROVIDERS_ROOT.iterdir() + if p.is_dir() and (p / "services").is_dir() + ) + + +def build_for_provider(provider: str) -> dict: + inventory: dict[str, dict] = {} + base = Path(f"prowler/providers/{provider}/services") + if not base.exists(): + print(f" skip {provider}: no services directory", file=sys.stderr) + return inventory + for meta_path in base.rglob("*.metadata.json"): + try: + with open(meta_path) as f: + data = json.load(f) + except Exception as exc: + print(f" warn: cannot parse {meta_path}: {exc}", file=sys.stderr) + continue + cid = data.get("CheckID") or meta_path.stem.replace(".metadata", "") + inventory[cid] = { + "service": data.get("ServiceName", ""), + "subservice": data.get("SubServiceName", ""), + "resource": data.get("ResourceType", ""), + "severity": data.get("Severity", ""), + "title": data.get("CheckTitle", ""), + "description": data.get("Description", ""), + "risk": data.get("Risk", ""), + } + return inventory + + +def main() -> int: + providers = sys.argv[1:] or discover_providers() + if not providers: + print( + f"error: no providers found under {PROVIDERS_ROOT}/", + file=sys.stderr, + ) + return 1 + for provider in providers: + inv = build_for_provider(provider) + out_path = Path(f"/tmp/checks_{provider}.json") + with open(out_path, "w") as f: + json.dump(inv, f, indent=2) + print(f" {provider}: {len(inv)} checks → {out_path}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/skills/prowler-compliance/assets/configs/ccc.yaml b/skills/prowler-compliance/assets/configs/ccc.yaml new file mode 100644 index 0000000000..deb757ffc9 --- /dev/null +++ b/skills/prowler-compliance/assets/configs/ccc.yaml @@ -0,0 +1,120 @@ +# FINOS Common Cloud Controls (CCC) sync config for sync_framework.py. +# +# Usage: +# python skills/prowler-compliance/assets/sync_framework.py \ +# skills/prowler-compliance/assets/configs/ccc.yaml +# +# Prerequisite: run the upstream fetch step from SKILL.md Workflow A Step 1 to +# populate upstream.dir with the raw FINOS catalog YAML files. + +framework: + name: CCC + display_name: Common Cloud Controls Catalog (CCC) + version: v2025.10 + # The {provider_display} placeholder is replaced at output time with the + # per-provider display string from the providers list below. + description_template: "Common Cloud Controls Catalog (CCC) for {provider_display}" + +providers: + - key: aws + display: AWS + - key: azure + display: Azure + - key: gcp + display: GCP + +output: + # Supported placeholders: {provider}, {framework}, {version}. + # For versioned frameworks like CIS the template would be + # "prowler/compliance/{provider}/cis_{version}_{provider}.json". + path_template: "prowler/compliance/{provider}/ccc_{provider}.json" + +upstream: + # Directory containing the cached FINOS catalog YAMLs. Populate via + # SKILL.md Workflow A Step 1 (gh api raw download commands). + dir: /tmp/ccc_upstream + fetch_docs: "See SKILL.md Workflow A Step 1 for gh api fetch commands" + +parser: + # Name of the parser module under parsers/ (loaded dynamically by the + # runner). For FINOS CCC YAML this is always finos_ccc. + module: finos_ccc + + # FINOS CCC catalog files in load order. Core first so its ARs render + # first in the output JSON. + catalog_files: + - core_ccc.yaml + - management_auditlog.yaml + - management_logging.yaml + - management_monitoring.yaml + - storage_object.yaml + - networking_loadbalancer.yaml + - networking_vpc.yaml + - crypto_key.yaml + - crypto_secrets.yaml + - database_warehouse.yaml + - database_vector.yaml + - database_relational.yaml + - devtools_build.yaml + - devtools_container-registry.yaml + - identity_iam.yaml + - ai-ml_gen-ai.yaml + - ai-ml_mlde.yaml + - app-integration_message.yaml + - compute_serverless-computing.yaml + + # Shape-2 catalogs (storage/object) reference the family via id only + # (e.g. "CCC.ObjStor.Data") with no human-readable title or description + # in the YAML. Map the suffix (after the last dot) to a canonical title + # and description so the generated JSON has consistent FamilyName fields + # regardless of upstream shape. + family_id_title: + Data: Data + IAM: Identity and Access Management + Identity: Identity and Access Management + Encryption: Encryption + Logging: Logging and Monitoring + Network: Network Security + Availability: Availability + Integrity: Integrity + Confidentiality: Confidentiality + family_id_description: + Data: "The Data control family ensures the confidentiality, integrity, availability, and sovereignty of data across its lifecycle." + IAM: "The Identity and Access Management control family ensures that only trusted and authenticated entities can access resources." + +post_processing: + # Collapse FamilyName variants that appear inconsistently across upstream + # catalogs. The Prowler UI groups by Attributes[0].FamilyName exactly, + # so each variant would otherwise become a separate tree branch. + family_name_normalization: + "Logging & Monitoring": "Logging and Monitoring" + "Logging and Metrics Publication": "Logging and Monitoring" + + # Preserve existing Checks lists from the legacy Prowler JSON when + # regenerating. The runner builds two lookup tables from the legacy + # output: a primary index by Id, and fallback indexes composed of + # attribute field names. + # + # primary_key: the top-level requirement field to use as the primary + # lookup key (almost always "Id") + # fallback_keys: a list of composite keys. Each composite key is a list + # of Attributes[0] field names to join into a tuple. List-valued fields + # (like Applicability) are frozen to frozenset so the tuple is hashable. + # + # CCC uses (Section, Applicability) because Applicability is a CCC-only + # top-level attribute field. CIS would use (Section, Profile). NIST would + # use (ItemId,). The fallback is how renumbered or rewritten ids still + # recover their check mappings. + # + # legacy_path_template (optional): path to read legacy Checks FROM. + # Defaults to output.path_template, which is correct for unversioned + # frameworks (like CCC) where regeneration overwrites the same file. + # For versioned frameworks that write to a new file on each version + # bump (e.g. cis_5.1_aws.json while the legacy mappings live in + # cis_5.0_aws.json), set this to the previous-version path so Checks + # are preserved instead of lost: + # legacy_path_template: "prowler/compliance/{provider}/cis_5.0_{provider}.json" + check_preservation: + primary_key: Id + fallback_keys: + - [Section, Applicability] diff --git a/skills/prowler-compliance/assets/dump_section.py b/skills/prowler-compliance/assets/dump_section.py new file mode 100644 index 0000000000..ca2fff0e1b --- /dev/null +++ b/skills/prowler-compliance/assets/dump_section.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +""" +Dump every requirement of a compliance framework for a given id prefix across +providers, with their current Check mappings. + +Useful for reviewing a whole control family in one pass before encoding audit +decisions in audit_framework_template.py. + +Usage: + # Dump all CCC.Core requirements across aws/azure/gcp + python skills/prowler-compliance/assets/dump_section.py ccc "CCC.Core." + + # Dump all CIS 5.0 section 1 requirements for AWS only + python skills/prowler-compliance/assets/dump_section.py cis_5.0_aws "1." + +Arguments: + framework_key: file prefix inside prowler/compliance/{provider}/ without + the provider suffix. Examples: + - "ccc" → loads ccc_aws.json / ccc_azure.json / ccc_gcp.json + - "cis_5.0_aws" → loads only that one file + - "iso27001_2022" → loads all providers + id_prefix: Requirement id prefix to filter by (e.g. "CCC.Core.", + "1.1.", "A.5."). +""" +from __future__ import annotations + +import json +import sys +from collections import defaultdict +from pathlib import Path + +PROWLER_COMPLIANCE_DIR = Path("prowler/compliance") + + +def main() -> int: + if len(sys.argv) < 3: + print(__doc__) + return 1 + + framework_key = sys.argv[1] + id_prefix = sys.argv[2] + + # Find matching JSON files across all providers + candidates: list[tuple[str, Path]] = [] + for prov_dir in sorted(PROWLER_COMPLIANCE_DIR.iterdir()): + if not prov_dir.is_dir(): + continue + for json_path in prov_dir.glob("*.json"): + stem = json_path.stem + if stem == framework_key or stem.startswith(f"{framework_key}_") \ + or stem == f"{framework_key}_{prov_dir.name}": + candidates.append((prov_dir.name, json_path)) + + if not candidates: + print(f"No files matching '{framework_key}'", file=sys.stderr) + return 2 + + discovered_providers = sorted({prov for prov, _ in candidates}) + + by_id: dict[str, dict] = defaultdict(dict) + for prov, path in candidates: + with open(path) as f: + data = json.load(f) + for req in data["Requirements"]: + if req["Id"].startswith(id_prefix): + by_id[req["Id"]][prov] = { + "desc": req.get("Description", ""), + "sec": (req.get("Attributes") or [{}])[0].get("Section", ""), + "obj": (req.get("Attributes") or [{}])[0].get( + "SubSectionObjective", "" + ), + "checks": req.get("Checks") or [], + } + + for ar_id in sorted(by_id): + rows = by_id[ar_id] + sample = next(iter(rows.values())) + print(f"\n### {ar_id}") + print(f" desc: {sample['desc']}") + if sample["sec"]: + print(f" sec : {sample['sec']}") + if sample["obj"]: + print(f" obj : {sample['obj']}") + for prov in discovered_providers: + if prov in rows: + checks = rows[prov]["checks"] + print(f" {prov}: ({len(checks)}) {checks}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/skills/prowler-compliance/assets/parsers/__init__.py b/skills/prowler-compliance/assets/parsers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/skills/prowler-compliance/assets/parsers/finos_ccc.py b/skills/prowler-compliance/assets/parsers/finos_ccc.py new file mode 100644 index 0000000000..a613b15857 --- /dev/null +++ b/skills/prowler-compliance/assets/parsers/finos_ccc.py @@ -0,0 +1,223 @@ +""" +FINOS Common Cloud Controls (CCC) YAML parser. + +Reads cached upstream YAML files and emits Prowler-format requirements +(``{Id, Description, Attributes: [...], Checks: []}``). This module is +agnostic to providers, JSON output paths, framework metadata and legacy +check-mapping preservation — those are handled by ``sync_framework.py``. + +Contract +-------- +``parse_upstream(config: dict) -> list[dict]`` + Returns a list of Prowler-format requirement dicts with **guaranteed + unique ids**. Foreign-prefix AR rewriting and genuine collision + renumbering both happen inside this module — the runner treats id + uniqueness as a contract violation, not as something to fix. + +Config keys consumed +-------------------- +This parser reads the following config entries (the rest of the config is +opaque to it): + +- ``upstream.dir`` — directory containing the cached YAMLs +- ``parser.catalog_files`` — ordered list of YAML filenames to load +- ``parser.family_id_title`` — suffix → canonical family title (shape 2) +- ``parser.family_id_description`` — suffix → family description (shape 2) + +Upstream shapes +--------------- +FINOS CCC catalogs come in two shapes: + +1. ``control-families: [{title, description, controls: [...]}]`` + (used by most catalogs) +2. ``controls: [{id, family: "CCC.X.Y", ...}]`` (no families wrapper; used + by ``storage/object``). The ``family`` field references a family id with + no human-readable title in the file — the title/description come from + ``config.parser.family_id_title`` / ``family_id_description``. + +Id rewriting rules +------------------ +- **Foreign-prefix rewriting**: upstream intentionally aliases requirements + across catalogs by keeping the original prefix (e.g. ``CCC.AuditLog.CN08.AR01`` + appears nested under ``CCC.Logging.CN03``). Prowler requires unique ids + within a catalog file, so we rename the AR to fit its parent control: + ``CCC.Logging.CN03.AR01``. See ``rewrite_ar_id()``. +- **Genuine collision renumbering**: sometimes upstream has a real typo + where two distinct requirements share the same id (e.g. + ``CCC.Core.CN14.AR02`` appears twice for 30-day and 14-day backup variants). + The second copy is renumbered to the next free AR number within the + control. See the ``seen_ids`` logic in ``emit_requirement()``. +""" +from __future__ import annotations + +from pathlib import Path + +import yaml + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def clean(value: str | None) -> str: + """Trim and collapse internal whitespace/newlines into single spaces. + + Upstream YAML uses ``|`` block scalars that preserve newlines; Prowler + stores descriptions as single-line text. + """ + if not value: + return "" + return " ".join(value.split()) + + +def flatten_mappings(mappings): + """Convert upstream ``{reference-id, entries: [{reference-id, ...}]}`` to + Prowler's ``{ReferenceId, Identifiers: [...]}``. + """ + if not mappings: + return [] + out = [] + for m in mappings: + ids = [] + for entry in m.get("entries") or []: + eid = entry.get("reference-id") + if eid: + ids.append(eid) + out.append({"ReferenceId": m.get("reference-id", ""), "Identifiers": ids}) + return out + + +def ar_prefix(ar_id: str) -> str: + """Return the first three dot-segments of an AR id (the parent control). + + e.g. ``CCC.Core.CN01.AR01`` -> ``CCC.Core.CN01``. + """ + return ".".join(ar_id.split(".")[:3]) + + +def rewrite_ar_id(parent_control_id: str, original_ar_id: str, ar_index: int) -> str: + """If an AR's id doesn't share its parent control's prefix, rename it. + + Example + ------- + parent ``CCC.Logging.CN03`` + AR id ``CCC.AuditLog.CN08.AR01`` with + index 0 -> ``CCC.Logging.CN03.AR01``. + """ + if ar_prefix(original_ar_id) == parent_control_id: + return original_ar_id + return f"{parent_control_id}.AR{ar_index + 1:02d}" + + +def emit_requirement( + control: dict, + family_name: str, + family_desc: str, + seen_ids: set[str], + requirements: list[dict], +) -> None: + """Translate one FINOS control + its assessment-requirements into + Prowler-format requirement dicts and append them to ``requirements``. + + Applies foreign-prefix rewriting and genuine-collision renumbering so + the final list is guaranteed to have unique ids. + """ + control_id = clean(control.get("id")) + control_title = clean(control.get("title")) + section = f"{control_id} {control_title}".strip() + objective = clean(control.get("objective")) + threat_mappings = flatten_mappings(control.get("threat-mappings")) + guideline_mappings = flatten_mappings(control.get("guideline-mappings")) + ars = control.get("assessment-requirements") or [] + for idx, ar in enumerate(ars): + raw_id = clean(ar.get("id")) + if not raw_id: + continue + new_id = rewrite_ar_id(control_id, raw_id, idx) + # Renumber on genuine upstream collision (find next free AR number) + if new_id in seen_ids: + base = ".".join(new_id.split(".")[:-1]) + n = 1 + while f"{base}.AR{n:02d}" in seen_ids: + n += 1 + new_id = f"{base}.AR{n:02d}" + seen_ids.add(new_id) + + requirements.append( + { + "Id": new_id, + "Description": clean(ar.get("text")), + "Attributes": [ + { + "FamilyName": family_name, + "FamilyDescription": family_desc, + "Section": section, + "SubSection": "", + "SubSectionObjective": objective, + "Applicability": list(ar.get("applicability") or []), + "Recommendation": clean(ar.get("recommendation")), + "SectionThreatMappings": threat_mappings, + "SectionGuidelineMappings": guideline_mappings, + } + ], + "Checks": [], + } + ) + + +# --------------------------------------------------------------------------- +# Public entry point +# --------------------------------------------------------------------------- + + +def parse_upstream(config: dict) -> list[dict]: + """Walk upstream YAMLs and emit Prowler-format requirements. + + Handles both top-level shapes (``control-families`` and ``controls``). + Ids are guaranteed unique in the returned list. + """ + upstream_dir = Path(config["upstream"]["dir"]) + parser_cfg = config.get("parser") or {} + catalog_files = parser_cfg.get("catalog_files") or [] + family_id_title = parser_cfg.get("family_id_title") or {} + family_id_description = parser_cfg.get("family_id_description") or {} + + requirements: list[dict] = [] + seen_ids: set[str] = set() + + for filename in catalog_files: + path = upstream_dir / filename + if not path.exists(): + # parser.catalog_files is the closed set of upstream catalogs + # that define the framework. Silently skipping a missing file + # would emit valid-looking JSON with part of the framework + # dropped, defeating the whole point of a canonical sync. + raise FileNotFoundError( + f"upstream catalog file not found: {path}\n" + f" hint: refresh the upstream cache (see SKILL.md Workflow A " + f"Step 1), or remove {filename!r} from parser.catalog_files " + f"if it has been retired upstream." + ) + with open(path) as f: + doc = yaml.safe_load(f) or {} + + # Shape 1: control-families wrapper + for family in doc.get("control-families") or []: + family_name = clean(family.get("title")) + family_desc = clean(family.get("description")) + for control in family.get("controls") or []: + emit_requirement( + control, family_name, family_desc, seen_ids, requirements + ) + + # Shape 2: top-level controls with family reference id + for control in doc.get("controls") or []: + family_ref = clean(control.get("family")) + suffix = family_ref.split(".")[-1] if family_ref else "" + family_name = family_id_title.get(suffix, suffix or "Data") + family_desc = family_id_description.get(suffix, "") + emit_requirement( + control, family_name, family_desc, seen_ids, requirements + ) + + return requirements diff --git a/skills/prowler-compliance/assets/query_checks.py b/skills/prowler-compliance/assets/query_checks.py new file mode 100644 index 0000000000..46405be982 --- /dev/null +++ b/skills/prowler-compliance/assets/query_checks.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +""" +Keyword/service/id lookup over a Prowler check inventory produced by +build_inventory.py. + +Usage: + # Keyword AND-search across id + title + risk + description + python skills/prowler-compliance/assets/query_checks.py aws encryption transit + + # Show all checks for a service + python skills/prowler-compliance/assets/query_checks.py aws --service iam + + # Show full metadata for one check id + python skills/prowler-compliance/assets/query_checks.py aws --id kms_cmk_rotation_enabled +""" +from __future__ import annotations + +import json +import sys + + +def main() -> int: + if len(sys.argv) < 3: + print(__doc__) + return 1 + + provider = sys.argv[1] + try: + with open(f"/tmp/checks_{provider}.json") as f: + inv = json.load(f) + except FileNotFoundError: + print( + f"No inventory for {provider}. Run build_inventory.py first.", + file=sys.stderr, + ) + return 2 + + if sys.argv[2] == "--service": + if len(sys.argv) < 4: + print("usage: --service ") + return 1 + svc = sys.argv[3] + hits = [cid for cid in sorted(inv) if inv[cid].get("service") == svc] + for cid in hits: + print(f" {cid}") + print(f" {inv[cid].get('title', '')}") + print(f"\n{len(hits)} checks in service '{svc}'") + elif sys.argv[2] == "--id": + if len(sys.argv) < 4: + print("usage: --id ") + return 1 + cid = sys.argv[3] + if cid not in inv: + print(f"NOT FOUND: {cid}") + return 3 + m = inv[cid] + print(f"== {cid} ==") + print(f"service : {m.get('service')}") + print(f"severity: {m.get('severity')}") + print(f"resource: {m.get('resource')}") + print(f"title : {m.get('title')}") + print(f"desc : {m.get('description', '')[:500]}") + print(f"risk : {m.get('risk', '')[:500]}") + else: + keywords = [k.lower() for k in sys.argv[2:]] + hits = 0 + for cid in sorted(inv): + m = inv[cid] + blob = " ".join( + [ + cid, + m.get("title", ""), + m.get("risk", ""), + m.get("description", ""), + ] + ).lower() + if all(k in blob for k in keywords): + hits += 1 + print(f" {cid} [{m.get('service', '')}]") + print(f" {m.get('title', '')[:120]}") + print(f"\n{hits} matches for {' + '.join(keywords)}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/skills/prowler-compliance/assets/sync_framework.py b/skills/prowler-compliance/assets/sync_framework.py new file mode 100644 index 0000000000..9e070f2691 --- /dev/null +++ b/skills/prowler-compliance/assets/sync_framework.py @@ -0,0 +1,536 @@ +#!/usr/bin/env python3 +""" +Generic, config-driven compliance framework sync runner. + +Usage: + python skills/prowler-compliance/assets/sync_framework.py \ + skills/prowler-compliance/assets/configs/ccc.yaml + +Pipeline: + 1. Load and validate the YAML config (fail fast on missing or empty + required fields — notably ``framework.version``, which silently + breaks ``get_check_compliance()`` key construction if empty). + 2. Dynamically import the parser module declared in ``parser.module`` + (resolved as ``parsers.{name}`` under this script's directory). + 3. Call ``parser.parse_upstream(config) -> list[dict]`` to get raw + Prowler-format requirements. The parser owns all upstream-format + quirks (foreign-prefix AR rewriting, collision renumbering, shape + handling) and MUST return ids that are unique within the returned + list. + 4. **Safety net**: assert id uniqueness. The runner raises + ``ValueError`` on any duplicate — it does NOT silently renumber, + because mutating a canonical upstream id (e.g. CIS ``1.1.1`` or + NIST ``AC-2(1)``) would be catastrophic. + 5. Apply generic ``FamilyName`` normalization from + ``post_processing.family_name_normalization`` (optional). + 6. Preserve legacy ``Checks`` lists from the existing Prowler JSON + using a config-driven primary key + fallback key chain. CCC uses + ``(Section, Applicability)`` as fallback; CIS would use + ``(Section, Profile)``; NIST would use ``(ItemId,)``. + For versioned frameworks (e.g. ``cis__.json``) + where a version bump writes to a brand-new file, set + ``post_processing.check_preservation.legacy_path_template`` to + point at the previous version's file so its Checks are preserved + instead of silently lost. Defaults to ``output.path_template`` + when omitted, which is correct for unversioned frameworks. + 7. Wrap each provider's requirements in the framework metadata dict + built from the config templates. + 8. Write each provider's JSON to the path resolved from + ``output.path_template`` (supports ``{framework}``, ``{version}`` + and ``{provider}`` placeholders). + 9. Pydantic-validate the written JSON via ``Compliance.parse_file()`` + and report the load counts per provider. + +The runner is strictly generic — it never mentions CCC, knows nothing +about YAML shapes, and can handle any upstream-backed framework given a +parser module and a config file. +""" +from __future__ import annotations + +import importlib +import json +import sys +from pathlib import Path +from typing import Any + +import yaml + +# Make sibling `parsers/` package importable regardless of the runner's +# invocation directory. +_SCRIPT_DIR = Path(__file__).resolve().parent +if str(_SCRIPT_DIR) not in sys.path: + sys.path.insert(0, str(_SCRIPT_DIR)) + + +# --------------------------------------------------------------------------- +# Config loading and validation +# --------------------------------------------------------------------------- + + +class ConfigError(ValueError): + """Raised when the sync config is malformed or missing required fields.""" + + +def _require(cfg: dict, dotted_path: str) -> Any: + """Fetch a dotted-path key from nested dicts. Raises ConfigError on + missing or empty values (empty-string, empty-list, None).""" + current: Any = cfg + parts = dotted_path.split(".") + for i, part in enumerate(parts): + if not isinstance(current, dict) or part not in current: + raise ConfigError(f"config: missing required field '{dotted_path}'") + current = current[part] + if current in ("", None, [], {}): + raise ConfigError(f"config: field '{dotted_path}' must not be empty") + return current + + +def load_config(path: Path) -> dict: + if not path.exists(): + raise ConfigError(f"config file not found: {path}") + with open(path) as f: + cfg = yaml.safe_load(f) or {} + if not isinstance(cfg, dict): + raise ConfigError(f"config root must be a mapping, got {type(cfg).__name__}") + + # Required fields — fail fast. Empty Version in particular silently + # breaks get_check_compliance() key construction. + _require(cfg, "framework.name") + _require(cfg, "framework.display_name") + _require(cfg, "framework.version") + _require(cfg, "framework.description_template") + _require(cfg, "providers") + _require(cfg, "output.path_template") + _require(cfg, "upstream.dir") + _require(cfg, "parser.module") + _require(cfg, "post_processing.check_preservation.primary_key") + + providers = cfg["providers"] + if not isinstance(providers, list) or not providers: + raise ConfigError("config: 'providers' must be a non-empty list") + for idx, p in enumerate(providers): + if not isinstance(p, dict) or "key" not in p or "display" not in p: + raise ConfigError( + f"config: providers[{idx}] must have 'key' and 'display' fields" + ) + + return cfg + + +# --------------------------------------------------------------------------- +# Parser loading +# --------------------------------------------------------------------------- + + +def load_parser(parser_module_name: str): + try: + return importlib.import_module(f"parsers.{parser_module_name}") + except ImportError as exc: + raise ConfigError( + f"cannot import parser 'parsers.{parser_module_name}': {exc}" + ) from exc + + +# --------------------------------------------------------------------------- +# Post-processing: id uniqueness safety net +# --------------------------------------------------------------------------- + + +def assert_unique_ids(requirements: list[dict]) -> None: + """Enforce the parser contract: every requirement must have a unique Id. + + The runner never renumbers silently — a duplicate is a parser bug. + """ + seen: set[str] = set() + dups: list[str] = [] + for req in requirements: + rid = req.get("Id") + if not rid: + raise ValueError(f"requirement missing Id: {req}") + if rid in seen: + dups.append(rid) + seen.add(rid) + if dups: + raise ValueError( + f"parser returned duplicate requirement ids: {sorted(set(dups))}" + ) + + +# --------------------------------------------------------------------------- +# Post-processing: FamilyName normalization +# --------------------------------------------------------------------------- + + +def normalize_family_names(requirements: list[dict], norm_map: dict[str, str]) -> None: + """Apply ``Attributes[0].FamilyName`` normalization in place.""" + if not norm_map: + return + for req in requirements: + for attr in req.get("Attributes") or []: + name = attr.get("FamilyName") + if name in norm_map: + attr["FamilyName"] = norm_map[name] + + +# --------------------------------------------------------------------------- +# Post-processing: legacy check-mapping preservation +# --------------------------------------------------------------------------- + + +def _freeze(value: Any) -> Any: + """Make a value hashable for use in composite lookup keys. + + Lists become frozensets (order-insensitive match). Scalars pass through. + """ + if isinstance(value, list): + return frozenset(value) + return value + + +def _build_fallback_key(attrs: dict, field_names: list[str]) -> tuple | None: + """Build a composite tuple key from the given attribute field names. + + Returns None if any field is missing or falsy — that key will be + skipped (the lookup table just won't have an entry for it). + """ + parts = [] + for name in field_names: + if name not in attrs: + return None + value = attrs[name] + if value in ("", None, [], {}): + return None + parts.append(_freeze(value)) + return tuple(parts) + + +def load_legacy_check_maps( + legacy_path: Path, + primary_key: str, + fallback_keys: list[list[str]], +) -> tuple[dict[str, list[str]], list[dict[tuple, list[str]]]]: + """Read the existing Prowler JSON and build lookup tables for check + preservation. + + Fails fast on ambiguous preservation keys. If two distinct legacy + requirements share the same primary value or the same fallback tuple, + merging their ``Checks`` silently would corrupt the preserved mapping + for unrelated requirements. Raises ``ValueError`` listing every + conflict so the user can either dedupe the legacy data or strengthen + ``check_preservation`` in the sync config. + + Returns + ------- + by_primary : dict + ``{primary_value: [checks]}`` — e.g. ``{ar_id: [checks]}``. + by_fallback : list[dict] + One lookup dict per entry in ``fallback_keys``. Each maps a + composite tuple key to its preserved checks list. + """ + by_primary: dict[str, list[str]] = {} + by_fallback: list[dict[tuple, list[str]]] = [{} for _ in fallback_keys] + + if not legacy_path.exists(): + return by_primary, by_fallback + + with open(legacy_path) as f: + data = json.load(f) + + # Track which legacy requirement Ids contributed to each bucket so we + # can surface ambiguity after the scan completes. + primary_sources: dict[str, list[str]] = {} + fallback_sources: list[dict[tuple, list[str]]] = [{} for _ in fallback_keys] + + for req in data.get("Requirements") or []: + legacy_id = req.get("Id") or "" + checks = req.get("Checks") or [] + + pv = req.get(primary_key) + if pv: + primary_sources.setdefault(pv, []).append(legacy_id) + bucket = by_primary.setdefault(pv, []) + for c in checks: + if c not in bucket: + bucket.append(c) + + attributes = req.get("Attributes") or [] + if not attributes: + continue + attrs = attributes[0] + for i, field_names in enumerate(fallback_keys): + key = _build_fallback_key(attrs, field_names) + if key is None: + continue + fallback_sources[i].setdefault(key, []).append(legacy_id) + bucket = by_fallback[i].setdefault(key, []) + for c in checks: + if c not in bucket: + bucket.append(c) + + conflicts: list[str] = [] + for pv, ids in primary_sources.items(): + if len(ids) > 1: + conflicts.append( + f"primary_key={primary_key!r} value={pv!r} shared by {ids}" + ) + for i, field_names in enumerate(fallback_keys): + for key, ids in fallback_sources[i].items(): + if len(ids) > 1: + conflicts.append( + f"fallback_key={field_names} value={key!r} shared by {ids}" + ) + if conflicts: + details = "\n - ".join(conflicts) + raise ValueError( + f"ambiguous preservation keys in {legacy_path} — cannot " + f"faithfully preserve Checks across distinct requirements:\n" + f" - {details}\n" + f"Fix: dedupe the legacy JSON, or strengthen " + f"'post_processing.check_preservation' in the sync config " + f"(e.g. add a more discriminating field to fallback_keys)." + ) + + return by_primary, by_fallback + + +def lookup_preserved_checks( + req: dict, + by_primary: dict, + by_fallback: list[dict], + primary_key: str, + fallback_keys: list[list[str]], +) -> list[str]: + """Return preserved check ids for a requirement, trying the primary + key first then each fallback in order.""" + pv = req.get(primary_key) + if pv and pv in by_primary: + return list(by_primary[pv]) + attributes = req.get("Attributes") or [] + if not attributes: + return [] + attrs = attributes[0] + for i, field_names in enumerate(fallback_keys): + key = _build_fallback_key(attrs, field_names) + if key and key in by_fallback[i]: + return list(by_fallback[i][key]) + return [] + + +# --------------------------------------------------------------------------- +# Provider output assembly +# --------------------------------------------------------------------------- + + +def resolve_output_path(template: str, framework: dict, provider_key: str) -> Path: + return Path( + template.format( + provider=provider_key, + framework=framework["name"].lower(), + version=framework["version"], + ) + ) + + +def build_provider_json( + config: dict, + provider: dict, + base_requirements: list[dict], +) -> tuple[dict, dict[str, int]]: + """Produce the provider-specific JSON dict ready to dump. + + Returns ``(json_dict, counts)`` where ``counts`` tracks how each + requirement's checks were resolved (primary, fallback, or none). + """ + framework = config["framework"] + preservation = config["post_processing"]["check_preservation"] + primary_key = preservation["primary_key"] + fallback_keys = preservation.get("fallback_keys") or [] + + # For versioned frameworks, the file we WRITE (output.path_template + # resolved at the new version) is not the file we want to READ legacy + # Checks from. Allow the config to override the legacy source path so + # a version bump can still preserve mappings from the previous file. + legacy_template = ( + preservation.get("legacy_path_template") + or config["output"]["path_template"] + ) + legacy_path = resolve_output_path( + legacy_template, framework, provider["key"] + ) + by_primary, by_fallback = load_legacy_check_maps( + legacy_path, primary_key, fallback_keys + ) + + counts = {"primary": 0, "fallback": 0, "none": 0} + enriched: list[dict] = [] + for req in base_requirements: + # Try primary key first + pv = req.get(primary_key) + checks: list[str] = [] + source = "none" + if pv and pv in by_primary: + checks = list(by_primary[pv]) + source = "primary" + else: + attributes = req.get("Attributes") or [] + if attributes: + attrs = attributes[0] + for i, field_names in enumerate(fallback_keys): + key = _build_fallback_key(attrs, field_names) + if key and key in by_fallback[i]: + checks = list(by_fallback[i][key]) + source = "fallback" + break + counts[source] += 1 + enriched.append( + { + "Id": req["Id"], + "Description": req["Description"], + # Shallow-copy attribute dicts so providers don't share refs + "Attributes": [dict(a) for a in req.get("Attributes") or []], + "Checks": checks, + } + ) + + description = framework["description_template"].format( + provider_display=provider["display"], + provider_key=provider["key"], + framework_name=framework["name"], + framework_display=framework["display_name"], + version=framework["version"], + ) + out = { + "Framework": framework["name"], + "Version": framework["version"], + "Provider": provider["display"], + "Name": framework["display_name"], + "Description": description, + "Requirements": enriched, + } + return out, counts + + +# --------------------------------------------------------------------------- +# Pydantic post-validation +# --------------------------------------------------------------------------- + + +def pydantic_validate(json_path: Path) -> int: + """Import Prowler lazily so the runner still works without Prowler + installed (validation step is skipped in that case).""" + try: + from prowler.lib.check.compliance_models import Compliance + except ImportError: + print( + " note: prowler package not importable — skipping Pydantic validation", + file=sys.stderr, + ) + return -1 + try: + parsed = Compliance.parse_file(str(json_path)) + except Exception as exc: + raise RuntimeError( + f"Pydantic validation failed for {json_path}: {exc}" + ) from exc + return len(parsed.Requirements) + + +# --------------------------------------------------------------------------- +# Driver +# --------------------------------------------------------------------------- + + +def main() -> int: + if len(sys.argv) != 2: + print("usage: sync_framework.py ", file=sys.stderr) + return 1 + + config_path = Path(sys.argv[1]) + try: + config = load_config(config_path) + except ConfigError as exc: + print(f"config error: {exc}", file=sys.stderr) + return 2 + + framework_name = config["framework"]["name"] + upstream_dir = Path(config["upstream"]["dir"]) + if not upstream_dir.exists(): + print( + f"error: upstream cache dir {upstream_dir} not found\n" + f" hint: {config['upstream'].get('fetch_docs', '(see SKILL.md Workflow A Step 1)')}", + file=sys.stderr, + ) + return 3 + + parser_module_name = config["parser"]["module"] + print( + f"Sync: framework={framework_name} version={config['framework']['version']} " + f"parser={parser_module_name}" + ) + + try: + parser = load_parser(parser_module_name) + except ConfigError as exc: + print(f"parser error: {exc}", file=sys.stderr) + return 4 + + print(f"Parsing upstream from {upstream_dir}...") + try: + base_requirements = parser.parse_upstream(config) + except FileNotFoundError as exc: + # A missing catalog declared in parser.catalog_files is a hard + # failure: emitting JSON with part of the framework silently + # dropped would violate the canonical-sync contract. + print(f"upstream error: {exc}", file=sys.stderr) + return 6 + print(f" parser returned {len(base_requirements)} requirements") + + # Safety-net: parser contract + try: + assert_unique_ids(base_requirements) + except ValueError as exc: + print(f"parser contract violation: {exc}", file=sys.stderr) + return 5 + + # Post-processing: family name normalization + norm_map = ( + config.get("post_processing", {}) + .get("family_name_normalization") + or {} + ) + normalize_family_names(base_requirements, norm_map) + + # Per-provider output + print() + for provider in config["providers"]: + provider_json, counts = build_provider_json( + config, provider, base_requirements + ) + out_path = resolve_output_path( + config["output"]["path_template"], + config["framework"], + provider["key"], + ) + out_path.parent.mkdir(parents=True, exist_ok=True) + with open(out_path, "w") as f: + json.dump(provider_json, f, indent=2, ensure_ascii=False) + f.write("\n") + + validated = pydantic_validate(out_path) + validated_msg = ( + f" pydantic_reqs={validated}" if validated >= 0 else " pydantic=skipped" + ) + print( + f" {provider['key']}: total={len(provider_json['Requirements'])} " + f"matched_primary={counts['primary']} " + f"matched_fallback={counts['fallback']} " + f"new_or_unmatched={counts['none']}{validated_msg}" + ) + print(f" wrote {out_path}") + + print("\nDone.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/skills/setup.sh b/skills/setup.sh index ec5512e8c5..c24706d718 100755 --- a/skills/setup.sh +++ b/skills/setup.sh @@ -1,10 +1,10 @@ #!/bin/bash # Setup AI Skills for Prowler development # Configures AI coding assistants that follow agentskills.io standard: -# - Claude Code: .claude/skills/ symlink + CLAUDE.md copies -# - Gemini CLI: .gemini/skills/ symlink + GEMINI.md copies +# - Claude Code: .claude/skills/ symlink + CLAUDE.md symlink +# - Gemini CLI: .gemini/skills/ symlink + GEMINI.md symlink # - Codex (OpenAI): .codex/skills/ symlink + AGENTS.md (native) -# - GitHub Copilot: .github/copilot-instructions.md copy +# - GitHub Copilot: .github/copilot-instructions.md symlink # # Usage: # ./setup.sh # Interactive mode (select AI assistants) @@ -37,6 +37,28 @@ SETUP_COPILOT=false # HELPER FUNCTIONS # ============================================================================= +add_to_gitignore() { + local pattern="$1" + local gitignore_file="$REPO_ROOT/.gitignore" + local header="# AI Coding assistants assets" + + # Create .gitignore if it doesn't exist + if [ ! -f "$gitignore_file" ]; then + touch "$gitignore_file" + fi + + # Check if pattern exists (exact match or at end of file) + if ! grep -qxF "$pattern" "$gitignore_file"; then + # Check if header exists + if ! grep -qxF "$header" "$gitignore_file"; then + echo -e "\n\n$header" >> "$gitignore_file" + fi + + echo "$pattern" >> "$gitignore_file" + echo -e "${GREEN} ✓ Added $pattern to .gitignore${NC}" + fi +} + show_help() { echo "Usage: $0 [OPTIONS]" echo "" @@ -109,6 +131,7 @@ setup_claude() { if [ ! -d "$REPO_ROOT/.claude" ]; then mkdir -p "$REPO_ROOT/.claude" fi + add_to_gitignore ".claude/skills" if [ -L "$target" ]; then rm "$target" @@ -119,8 +142,9 @@ setup_claude() { ln -s "$SKILLS_SOURCE" "$target" echo -e "${GREEN} ✓ .claude/skills -> skills/${NC}" - # Copy AGENTS.md to CLAUDE.md - copy_agents_md "CLAUDE.md" + # Link AGENTS.md to CLAUDE.md + link_agents_md "CLAUDE.md" + add_to_gitignore "CLAUDE.md" } setup_gemini() { @@ -129,6 +153,7 @@ setup_gemini() { if [ ! -d "$REPO_ROOT/.gemini" ]; then mkdir -p "$REPO_ROOT/.gemini" fi + add_to_gitignore ".gemini/skills" if [ -L "$target" ]; then rm "$target" @@ -139,8 +164,9 @@ setup_gemini() { ln -s "$SKILLS_SOURCE" "$target" echo -e "${GREEN} ✓ .gemini/skills -> skills/${NC}" - # Copy AGENTS.md to GEMINI.md - copy_agents_md "GEMINI.md" + # Link AGENTS.md to GEMINI.md + link_agents_md "GEMINI.md" + add_to_gitignore "GEMINI.md" } setup_codex() { @@ -149,6 +175,7 @@ setup_codex() { if [ ! -d "$REPO_ROOT/.codex" ]; then mkdir -p "$REPO_ROOT/.codex" fi + add_to_gitignore ".codex/skills" if [ -L "$target" ]; then rm "$target" @@ -164,12 +191,19 @@ setup_codex() { setup_copilot() { if [ -f "$REPO_ROOT/AGENTS.md" ]; then mkdir -p "$REPO_ROOT/.github" - cp "$REPO_ROOT/AGENTS.md" "$REPO_ROOT/.github/copilot-instructions.md" + + # Link AGENTS.md -> .github/copilot-instructions.md + local target="$REPO_ROOT/.github/copilot-instructions.md" + ln -sf "../AGENTS.md" "$target" + echo -e "${GREEN} ✓ AGENTS.md -> .github/copilot-instructions.md${NC}" + + # Add specifically the file, NOT the .github folder + add_to_gitignore ".github/copilot-instructions.md" fi } -copy_agents_md() { +link_agents_md() { local target_name="$1" local agents_files local count=0 @@ -179,11 +213,15 @@ copy_agents_md() { for agents_file in $agents_files; do local agents_dir agents_dir=$(dirname "$agents_file") - cp "$agents_file" "$agents_dir/$target_name" + + # Create relative symlink + # Since files are in same dir, we can just link to basename + (cd "$agents_dir" && ln -sf "$(basename "$agents_file")" "$target_name") + count=$((count + 1)) done - echo -e "${GREEN} ✓ Copied $count AGENTS.md -> $target_name${NC}" + echo -e "${GREEN} ✓ Linked $count AGENTS.md -> $target_name${NC}" } # ============================================================================= @@ -302,4 +340,4 @@ echo "Configured:" [ "$SETUP_COPILOT" = true ] && echo " • GitHub Copilot: .github/copilot-instructions.md" echo "" echo -e "${BLUE}Note: Restart your AI assistant to load the skills.${NC}" -echo -e "${BLUE} AGENTS.md is the source of truth - edit it, then re-run this script.${NC}" +echo -e "${BLUE} AGENTS.md is the source of truth - changes are reflected automatically via symlinks.${NC}" diff --git a/skills/setup_test.sh b/skills/setup_test.sh index c0e80afe99..db4749ed3f 100755 --- a/skills/setup_test.sh +++ b/skills/setup_test.sh @@ -201,40 +201,40 @@ test_symlink_not_created_without_flag() { } # ============================================================================= -# TESTS: AGENTS.md COPYING +# TESTS: AGENTS.md LINKING # ============================================================================= -test_copy_claude_agents_md() { +test_link_claude_agents_md() { run_setup --claude > /dev/null - assert_file_exists "$TEST_DIR/CLAUDE.md" "Root CLAUDE.md should exist" && \ - assert_file_exists "$TEST_DIR/api/CLAUDE.md" "api/CLAUDE.md should exist" && \ - assert_file_exists "$TEST_DIR/ui/CLAUDE.md" "ui/CLAUDE.md should exist" + assert_symlink_exists "$TEST_DIR/CLAUDE.md" "Root CLAUDE.md should be a symlink" && \ + assert_symlink_exists "$TEST_DIR/api/CLAUDE.md" "api/CLAUDE.md should be a symlink" && \ + assert_symlink_exists "$TEST_DIR/ui/CLAUDE.md" "ui/CLAUDE.md should be a symlink" } -test_copy_gemini_agents_md() { +test_link_gemini_agents_md() { run_setup --gemini > /dev/null - assert_file_exists "$TEST_DIR/GEMINI.md" "Root GEMINI.md should exist" && \ - assert_file_exists "$TEST_DIR/api/GEMINI.md" "api/GEMINI.md should exist" && \ - assert_file_exists "$TEST_DIR/ui/GEMINI.md" "ui/GEMINI.md should exist" + assert_symlink_exists "$TEST_DIR/GEMINI.md" "Root GEMINI.md should be a symlink" && \ + assert_symlink_exists "$TEST_DIR/api/GEMINI.md" "api/GEMINI.md should be a symlink" && \ + assert_symlink_exists "$TEST_DIR/ui/GEMINI.md" "ui/GEMINI.md should be a symlink" } -test_copy_copilot_to_github() { +test_link_copilot_to_github() { run_setup --copilot > /dev/null - assert_file_exists "$TEST_DIR/.github/copilot-instructions.md" "Copilot instructions should exist" + assert_symlink_exists "$TEST_DIR/.github/copilot-instructions.md" "Copilot instructions should be a symlink" } -test_copy_codex_no_extra_files() { +test_link_codex_no_extra_files() { run_setup --codex > /dev/null assert_file_not_exists "$TEST_DIR/CODEX.md" "CODEX.md should not be created" } -test_copy_not_created_without_flag() { +test_link_not_created_without_flag() { run_setup --codex > /dev/null - assert_file_not_exists "$TEST_DIR/CLAUDE.md" "CLAUDE.md should not exist" && \ - assert_file_not_exists "$TEST_DIR/GEMINI.md" "GEMINI.md should not exist" + assert_symlink_not_exists "$TEST_DIR/CLAUDE.md" "CLAUDE.md should not exist" && \ + assert_symlink_not_exists "$TEST_DIR/GEMINI.md" "GEMINI.md should not exist" } -test_copy_content_matches_source() { +test_link_content_matches_source() { run_setup --claude > /dev/null local source_content target_content source_content=$(cat "$TEST_DIR/AGENTS.md") @@ -272,7 +272,7 @@ test_idempotent_multiple_runs() { run_setup --claude > /dev/null run_setup --claude > /dev/null assert_symlink_exists "$TEST_DIR/.claude/skills" "Symlink should still exist after second run" && \ - assert_file_exists "$TEST_DIR/CLAUDE.md" "CLAUDE.md should still exist after second run" + assert_symlink_exists "$TEST_DIR/CLAUDE.md" "CLAUDE.md should still be a symlink after second run" } # ============================================================================= diff --git a/tests/lib/check/universal_compliance_models_test.py b/tests/lib/check/universal_compliance_models_test.py new file mode 100644 index 0000000000..5a3e4aae56 --- /dev/null +++ b/tests/lib/check/universal_compliance_models_test.py @@ -0,0 +1,1118 @@ +import json +import os + +import pytest +from pydantic.v1 import ValidationError + +from prowler.lib.check.compliance_models import ( + AttributeMetadata, + ChartConfig, + Compliance, + ComplianceFramework, + CriticalRequirementsFilter, + EnumValueDisplay, + I18nLabels, + OutputFormats, + OutputsConfig, + PDFConfig, + ReportFilter, + ScoringConfig, + ScoringFormula, + SplitByConfig, + TableConfig, + TableLabels, + UniversalComplianceRequirement, + adapt_legacy_to_universal, + load_compliance_framework_universal, +) +from tests.lib.outputs.compliance.fixtures import ( + CIS_1_4_AWS, + ENS_RD2022_AWS, + KISA_ISMSP_AWS, + MITRE_ATTACK_AWS, + NIST_800_53_REVISION_4_AWS, + PROWLER_THREATSCORE_AWS, +) + + +class TestOutputFormats: + def test_defaults(self): + of = OutputFormats() + assert of.csv is True + assert of.ocsf is True + + def test_explicit_false(self): + of = OutputFormats(csv=False, ocsf=False) + assert of.csv is False + assert of.ocsf is False + + +class TestAttributeMetadata: + def test_basic(self): + meta = AttributeMetadata(key="Section", type="str") + assert meta.key == "Section" + assert meta.type == "str" + assert meta.output_formats.csv is True + assert meta.required is False + + def test_with_enum(self): + meta = AttributeMetadata( + key="Profile", + type="str", + enum=["Level 1", "Level 2"], + ) + assert meta.enum == ["Level 1", "Level 2"] + + def test_int_type(self): + meta = AttributeMetadata(key="LevelOfRisk", type="int", required=True) + assert meta.type == "int" + assert meta.required is True + + def test_enum_display_field(self): + meta = AttributeMetadata( + key="Dimensiones", + type="str", + enum=["confidencialidad", "integridad", "trazabilidad"], + enum_display={ + "confidencialidad": { + "label": "Confidencialidad", + "abbreviation": "C", + "color": "#FF6347", + }, + "integridad": { + "label": "Integridad", + "abbreviation": "I", + "color": "#4286F4", + }, + "trazabilidad": { + "label": "Trazabilidad", + "abbreviation": "T", + "color": "#32CD32", + }, + }, + ) + assert meta.enum_display is not None + assert meta.enum_display["confidencialidad"]["abbreviation"] == "C" + assert meta.enum_display["integridad"]["color"] == "#4286F4" + + def test_enum_order_field(self): + meta = AttributeMetadata( + key="Nivel", + type="str", + enum=["opcional", "bajo", "medio", "alto"], + enum_order=["alto", "medio", "bajo", "opcional"], + ) + assert meta.enum_order == ["alto", "medio", "bajo", "opcional"] + + def test_chart_label_field(self): + meta = AttributeMetadata( + key="Section", + type="str", + chart_label="Security Domain", + ) + assert meta.chart_label == "Security Domain" + + def test_output_formats_default_true(self): + meta = AttributeMetadata(key="Section") + assert meta.output_formats.csv is True + assert meta.output_formats.ocsf is True + + def test_output_formats_explicit_false(self): + meta = AttributeMetadata( + key="InternalNote", + output_formats=OutputFormats(csv=False, ocsf=False), + ) + assert meta.output_formats.csv is False + assert meta.output_formats.ocsf is False + + def test_new_fields_default_none(self): + meta = AttributeMetadata(key="Section") + assert meta.enum_display is None + assert meta.enum_order is None + assert meta.chart_label is None + + +class TestEnumValueDisplay: + def test_basic(self): + evd = EnumValueDisplay(label="Test") + assert evd.label == "Test" + assert evd.abbreviation is None + assert evd.color is None + assert evd.icon is None + + def test_dimension_style(self): + evd = EnumValueDisplay( + label="Trazabilidad", + abbreviation="T", + color="#4286F4", + ) + assert evd.label == "Trazabilidad" + assert evd.abbreviation == "T" + assert evd.color == "#4286F4" + + def test_tipo_style(self): + evd = EnumValueDisplay( + label="Requisito", + icon="⚠️", + ) + assert evd.icon == "⚠️" + assert evd.abbreviation is None + + +class TestChartConfig: + def test_horizontal_bar(self): + chart = ChartConfig( + id="section_compliance", + type="horizontal_bar", + group_by="Section", + title="Compliance Score by Domain", + y_label="Domain", + x_label="Compliance %", + ) + assert chart.type == "horizontal_bar" + assert chart.group_by == "Section" + assert chart.value_source == "compliance_percent" + assert chart.color_mode == "by_value" + + def test_vertical_bar(self): + chart = ChartConfig( + id="risk_distribution", + type="vertical_bar", + group_by="LevelOfRisk", + color_mode="fixed", + fixed_color="#336699", + ) + assert chart.type == "vertical_bar" + assert chart.fixed_color == "#336699" + + def test_radar(self): + chart = ChartConfig( + id="dimension_radar", + type="radar", + group_by="Dimensiones", + ) + assert chart.type == "radar" + + def test_defaults(self): + chart = ChartConfig(id="test", type="vertical_bar", group_by="Section") + assert chart.title is None + assert chart.x_label is None + assert chart.y_label is None + assert chart.value_source == "compliance_percent" + assert chart.color_mode == "by_value" + assert chart.fixed_color is None + + +class TestScoringFormula: + def test_threatscore_style(self): + formula = ScoringFormula( + risk_field="LevelOfRisk", + weight_field="Weight", + risk_boost_factor=0.25, + ) + assert formula.risk_field == "LevelOfRisk" + assert formula.weight_field == "Weight" + assert formula.risk_boost_factor == 0.25 + + def test_custom_boost_factor(self): + formula = ScoringFormula( + risk_field="Risk", + weight_field="Impact", + risk_boost_factor=0.5, + ) + assert formula.risk_boost_factor == 0.5 + + def test_default_boost_factor(self): + formula = ScoringFormula(risk_field="LevelOfRisk", weight_field="Weight") + assert formula.risk_boost_factor == 0.25 + + +class TestCriticalRequirementsFilter: + def test_int_based(self): + crf = CriticalRequirementsFilter( + filter_field="LevelOfRisk", + min_value=4, + title="Critical Failed Requirements", + ) + assert crf.filter_field == "LevelOfRisk" + assert crf.min_value == 4 + assert crf.filter_value is None + assert crf.status_filter == "FAIL" + assert crf.title == "Critical Failed Requirements" + + def test_string_based(self): + crf = CriticalRequirementsFilter( + filter_field="Nivel", + filter_value="alto", + ) + assert crf.filter_value == "alto" + assert crf.min_value is None + + def test_defaults(self): + crf = CriticalRequirementsFilter(filter_field="LevelOfRisk") + assert crf.status_filter == "FAIL" + assert crf.title is None + assert crf.min_value is None + assert crf.filter_value is None + + +class TestReportFilter: + def test_defaults(self): + rf = ReportFilter() + assert rf.only_failed is True + assert rf.include_manual is False + + def test_custom(self): + rf = ReportFilter(only_failed=False, include_manual=True) + assert rf.only_failed is False + assert rf.include_manual is True + + +class TestI18nLabels: + def test_english_defaults(self): + labels = I18nLabels() + assert labels.page_label == "Page" + assert labels.powered_by == "Powered by Prowler" + assert labels.framework_label == "Framework:" + assert labels.provider_label == "Provider:" + assert labels.report_title is None + + def test_spanish_override(self): + labels = I18nLabels( + report_title="Informe de Cumplimiento ENS", + page_label="Página", + powered_by="Generado por Prowler", + framework_label="Marco:", + version_label="Versión:", + provider_label="Proveedor:", + description_label="Descripción:", + compliance_score_label="Puntuación de Cumplimiento por Secciones", + requirements_index_label="Índice de Requisitos", + detailed_findings_label="Hallazgos Detallados", + ) + assert labels.page_label == "Página" + assert labels.provider_label == "Proveedor:" + assert labels.report_title == "Informe de Cumplimiento ENS" + + +class TestSplitByConfig: + def test_cis_style(self): + config = SplitByConfig(field="Profile", values=["Level 1", "Level 2"]) + assert config.field == "Profile" + assert len(config.values) == 2 + + def test_ens_style(self): + config = SplitByConfig( + field="Nivel", + values=["alto", "medio", "bajo", "opcional"], + ) + assert len(config.values) == 4 + + +class TestScoringConfig: + def test_threatscore_style(self): + config = ScoringConfig(risk_field="LevelOfRisk", weight_field="Weight") + assert config.risk_field == "LevelOfRisk" + assert config.weight_field == "Weight" + + +class TestTableLabels: + def test_defaults(self): + labels = TableLabels() + assert labels.pass_label == "PASS" + assert labels.fail_label == "FAIL" + assert labels.provider_header == "Provider" + + def test_ens_spanish(self): + labels = TableLabels( + pass_label="CUMPLE", + fail_label="NO CUMPLE", + provider_header="Proveedor", + ) + assert labels.pass_label == "CUMPLE" + + +class TestTableConfig: + def test_grouped_mode(self): + tc = TableConfig(group_by="Section") + assert tc.group_by == "Section" + assert tc.split_by is None + assert tc.scoring is None + + def test_split_mode(self): + tc = TableConfig( + group_by="Section", + split_by=SplitByConfig(field="Profile", values=["Level 1", "Level 2"]), + ) + assert tc.split_by is not None + assert tc.split_by.field == "Profile" + + def test_scored_mode(self): + tc = TableConfig( + group_by="Section", + scoring=ScoringConfig(risk_field="LevelOfRisk", weight_field="Weight"), + ) + assert tc.scoring is not None + + +class TestPDFConfig: + def test_defaults(self): + pdf = PDFConfig() + assert pdf.language == "en" + assert pdf.logo_filename is None + assert pdf.primary_color is None + assert pdf.sections is None + assert pdf.section_short_names is None + assert pdf.group_by_field is None + assert pdf.sub_group_by_field is None + assert pdf.section_titles is None + assert pdf.charts is None + assert pdf.scoring is None + assert pdf.critical_filter is None + assert pdf.filter is None + assert pdf.labels is None + + def test_csa_ccm_style(self): + pdf = PDFConfig( + primary_color="#336699", + secondary_color="#4D80B3", + bg_color="#F2F8FF", + group_by_field="Section", + sections=["Audit & Assurance", "Identity & Access Management"], + section_short_names={"Identity & Access Management": "IAM"}, + charts=[ + ChartConfig( + id="section_compliance", + type="horizontal_bar", + group_by="Section", + title="Compliance Score by Domain", + ).dict() + ], + filter=ReportFilter(only_failed=True, include_manual=False), + ) + assert pdf.primary_color == "#336699" + assert len(pdf.sections) == 2 + assert pdf.section_short_names["Identity & Access Management"] == "IAM" + assert pdf.group_by_field == "Section" + assert pdf.charts is not None + assert len(pdf.charts) == 1 + assert pdf.filter.only_failed is True + + def test_ens_style(self): + pdf = PDFConfig( + language="es", + logo_filename="ens_logo.png", + primary_color="#CC3333", + group_by_field="Marco", + sub_group_by_field="Categoria", + labels=I18nLabels( + page_label="Página", + provider_label="Proveedor:", + ), + ) + assert pdf.language == "es" + assert pdf.logo_filename == "ens_logo.png" + assert pdf.group_by_field == "Marco" + assert pdf.sub_group_by_field == "Categoria" + assert pdf.labels.page_label == "Página" + + def test_threatscore_style(self): + pdf = PDFConfig( + primary_color="#336699", + sections=["1. IAM", "2. Attack Surface"], + scoring=ScoringFormula( + risk_field="LevelOfRisk", + weight_field="Weight", + risk_boost_factor=0.25, + ), + critical_filter=CriticalRequirementsFilter( + filter_field="LevelOfRisk", + min_value=4, + title="Critical Failed Requirements", + ), + ) + assert pdf.scoring is not None + assert pdf.scoring.risk_field == "LevelOfRisk" + assert pdf.critical_filter.min_value == 4 + + def test_section_titles(self): + pdf = PDFConfig( + section_titles={ + "1": "1. Policy on Security", + "2": "2. Risk Management", + }, + ) + assert pdf.section_titles["1"] == "1. Policy on Security" + + def test_in_framework(self): + fw = ComplianceFramework( + framework="Test", + name="Test Framework", + description="Test", + requirements=[], + outputs=OutputsConfig( + pdf_config=PDFConfig( + primary_color="#336699", + sections=["Section A"], + charts=[ + ChartConfig( + id="test_chart", + type="vertical_bar", + group_by="Section", + ).dict() + ], + ), + ), + ) + assert fw.outputs is not None + assert fw.outputs.pdf_config is not None + assert fw.outputs.pdf_config.primary_color == "#336699" + assert fw.outputs.pdf_config.sections == ["Section A"] + assert fw.outputs.pdf_config.charts is not None + assert len(fw.outputs.pdf_config.charts) == 1 + assert fw.outputs.pdf_config.charts[0]["id"] == "test_chart" + assert fw.outputs.pdf_config.charts[0]["type"] == "vertical_bar" + + def test_framework_without_pdf_config(self): + fw = ComplianceFramework( + framework="Test", + name="Test Framework", + description="Test", + requirements=[], + ) + assert fw.outputs is None + + +class TestUniversalComplianceRequirement: + def test_flat_dict_attributes(self): + req = UniversalComplianceRequirement( + id="1.1", + description="Test requirement", + attributes={"Section": "IAM", "Profile": "Level 1"}, + checks={"aws": ["check_a", "check_b"]}, + ) + assert req.attributes["Section"] == "IAM" + assert len(req.checks["aws"]) == 2 + + def test_mitre_optional_fields(self): + req = UniversalComplianceRequirement( + id="T1190", + description="Exploit Public-Facing Application", + attributes={}, + checks={"aws": ["drs_job_exist"]}, + tactics=["Initial Access"], + sub_techniques=[], + platforms=["IaaS", "Linux"], + technique_url="https://attack.mitre.org/techniques/T1190/", + ) + assert req.tactics == ["Initial Access"] + assert req.technique_url == "https://attack.mitre.org/techniques/T1190/" + + def test_dict_checks_multi_provider(self): + req = UniversalComplianceRequirement( + id="1.1", + description="Multi-provider", + attributes={}, + checks={"aws": ["check_a"], "azure": ["check_b"]}, + ) + assert isinstance(req.checks, dict) + assert "aws" in req.checks + + def test_empty_checks(self): + req = UniversalComplianceRequirement( + id="manual-1", + description="Manual requirement", + attributes={"Section": "Governance"}, + checks={}, + ) + assert req.checks == {} + + def test_checks_default_is_empty_dict(self): + req = UniversalComplianceRequirement( + id="1.1", + description="No checks provided", + ) + assert req.checks == {} + + +class TestComplianceFramework: + def test_basic_framework(self): + fw = ComplianceFramework( + framework="TestFW", + name="Test Framework", + provider="AWS", + version="1.0", + description="A test framework", + requirements=[ + UniversalComplianceRequirement( + id="1.1", + description="Test", + attributes={"Section": "IAM"}, + checks={"aws": ["check_a"]}, + ) + ], + attributes_metadata=[ + AttributeMetadata(key="Section", type="str"), + ], + outputs=OutputsConfig(table_config=TableConfig(group_by="Section")), + ) + assert fw.framework == "TestFW" + assert fw.outputs.table_config.group_by == "Section" + assert len(fw.attributes_metadata) == 1 + assert len(fw.requirements) == 1 + + def test_optional_provider(self): + fw = ComplianceFramework( + framework="MultiCloud", + name="Multi-cloud framework", + description="A multi-provider framework", + requirements=[], + ) + assert fw.provider is None + + def test_get_providers_from_dict_checks(self): + fw = ComplianceFramework( + framework="MultiCloud", + name="Multi-cloud", + description="test", + requirements=[ + UniversalComplianceRequirement( + id="1.1", + description="test", + attributes={}, + checks={ + "aws": ["check_a"], + "azure": ["check_b"], + "gcp": ["check_c"], + }, + ), + UniversalComplianceRequirement( + id="1.2", + description="test2", + attributes={}, + checks={"aws": ["check_d"]}, + ), + ], + ) + providers = fw.get_providers() + assert providers == ["aws", "azure", "gcp"] + + def test_get_providers_fallback_to_explicit(self): + fw = ComplianceFramework( + framework="SingleCloud", + name="Single-cloud", + provider="AWS", + description="test", + requirements=[ + UniversalComplianceRequirement( + id="1.1", + description="test", + attributes={}, + checks={}, + ), + ], + ) + providers = fw.get_providers() + assert providers == ["aws"] + + def test_supports_provider_dict_checks(self): + fw = ComplianceFramework( + framework="MultiCloud", + name="Multi-cloud", + description="test", + requirements=[ + UniversalComplianceRequirement( + id="1.1", + description="test", + attributes={}, + checks={"aws": ["check_a"], "azure": ["check_b"]}, + ), + ], + ) + assert fw.supports_provider("aws") is True + assert fw.supports_provider("azure") is True + assert fw.supports_provider("gcp") is False + + def test_supports_provider_explicit_only(self): + """Framework with explicit provider but no per-requirement checks still supports the provider.""" + fw = ComplianceFramework( + framework="SingleCloud", + name="Single-cloud", + provider="AWS", + description="test", + requirements=[ + UniversalComplianceRequirement( + id="1.1", + description="Manual requirement", + attributes={}, + checks={}, + ), + ], + ) + assert fw.supports_provider("aws") is True + assert fw.supports_provider("azure") is False + + def test_no_provider_field_with_dict_checks(self): + """Multi-provider JSON has no Provider field — providers derived from checks.""" + fw = ComplianceFramework( + framework="CSA_CCM", + name="CSA CCM 4.0", + description="Cloud Controls Matrix", + requirements=[ + UniversalComplianceRequirement( + id="A&A-01", + description="Audit & Assurance", + attributes={"Domain": "A&A"}, + checks={ + "aws": ["check_a"], + "azure": ["check_b"], + "gcp": ["check_c"], + }, + ), + ], + ) + assert fw.provider is None + assert fw.get_providers() == ["aws", "azure", "gcp"] + assert fw.supports_provider("aws") + assert fw.supports_provider("azure") + assert fw.supports_provider("gcp") + assert not fw.supports_provider("kubernetes") + + def test_icon_field(self): + fw = ComplianceFramework( + framework="CSA_CCM", + name="CSA CCM 4.0", + description="Cloud Controls Matrix", + icon="csa", + requirements=[], + ) + assert fw.icon == "csa" + + def test_icon_defaults_to_none(self): + fw = ComplianceFramework( + framework="Test", + name="Test", + description="d", + requirements=[], + ) + assert fw.icon is None + + +class TestAdaptLegacyToUniversal: + def test_adapt_cis(self): + fw = adapt_legacy_to_universal(CIS_1_4_AWS) + assert fw.framework == "CIS" + assert fw.provider == "AWS" + assert len(fw.requirements) == 2 + # First requirement should have flat attributes + req = fw.requirements[0] + assert "Section" in req.attributes + assert req.attributes["Section"] == "2. Storage" + assert req.tactics is None + # Checks must be wrapped in dict keyed by provider + assert isinstance(req.checks, dict) + assert "aws" in req.checks + + def test_adapt_ens(self): + fw = adapt_legacy_to_universal(ENS_RD2022_AWS) + assert fw.framework == "ENS" + req = fw.requirements[0] + assert "Marco" in req.attributes + assert req.attributes["Marco"] == "operacional" + + def test_adapt_mitre(self): + fw = adapt_legacy_to_universal(MITRE_ATTACK_AWS) + assert fw.framework == "MITRE-ATTACK" + req = fw.requirements[0] + assert req.tactics == ["Initial Access"] + assert req.technique_url == "https://attack.mitre.org/techniques/T1190/" + assert "_raw_attributes" in req.attributes + assert isinstance(req.checks, dict) + assert "aws" in req.checks + + def test_adapt_threatscore(self): + fw = adapt_legacy_to_universal(PROWLER_THREATSCORE_AWS) + req = fw.requirements[0] + assert req.attributes["LevelOfRisk"] == 5 + assert req.attributes["Weight"] == 1000 + + def test_adapt_generic(self): + fw = adapt_legacy_to_universal(NIST_800_53_REVISION_4_AWS) + req = fw.requirements[0] + assert "Section" in req.attributes + + def test_adapt_kisa(self): + fw = adapt_legacy_to_universal(KISA_ISMSP_AWS) + req = fw.requirements[0] + assert "Domain" in req.attributes + + def test_inferred_metadata_cis(self): + fw = adapt_legacy_to_universal(CIS_1_4_AWS) + assert fw.attributes_metadata is not None + keys = [m.key for m in fw.attributes_metadata] + assert "Section" in keys + assert "Profile" in keys + + def test_inferred_metadata_mitre_is_none(self): + fw = adapt_legacy_to_universal(MITRE_ATTACK_AWS) + assert fw.attributes_metadata is None + + def test_table_config_is_none(self): + fw = adapt_legacy_to_universal(CIS_1_4_AWS) + assert fw.outputs is None + + +class TestLoadComplianceFrameworkUniversal: + def test_load_universal_format(self, tmp_path): + data = { + "framework": "TestFW", + "name": "Test", + "provider": "AWS", + "version": "1.0", + "description": "desc", + "icon": "prowlerthreatscore", + "attributes_metadata": [{"key": "Section", "type": "str"}], + "outputs": {"table_config": {"group_by": "Section"}}, + "requirements": [ + { + "id": "1.1", + "description": "test", + "attributes": {"Section": "IAM"}, + "checks": {"aws": ["check_a"]}, + } + ], + } + path = tmp_path / "test.json" + path.write_text(json.dumps(data)) + fw = load_compliance_framework_universal(str(path)) + assert fw is not None + assert fw.framework == "TestFW" + assert fw.icon == "prowlerthreatscore" + assert fw.outputs.table_config.group_by == "Section" + + def test_load_universal_multi_provider(self, tmp_path): + data = { + "framework": "CSA_CCM", + "name": "CSA CCM 4.0", + "version": "4.0", + "description": "Cloud Controls Matrix", + "attributes_metadata": [{"key": "Domain", "type": "str"}], + "outputs": {"table_config": {"group_by": "Domain"}}, + "requirements": [ + { + "id": "A&A-01", + "description": "Audit", + "attributes": {"Domain": "Audit"}, + "checks": { + "aws": ["check_a"], + "azure": ["check_b"], + "gcp": ["check_c"], + }, + } + ], + } + path = tmp_path / "csa_ccm_4.0.json" + path.write_text(json.dumps(data)) + fw = load_compliance_framework_universal(str(path)) + assert fw is not None + assert fw.provider is None + assert fw.get_providers() == ["aws", "azure", "gcp"] + assert fw.supports_provider("aws") + assert not fw.supports_provider("kubernetes") + + def test_load_legacy_format(self, tmp_path): + data = { + "Framework": "SOC2", + "Name": "SOC2", + "Provider": "AWS", + "Version": "", + "Description": "desc", + "Requirements": [ + { + "Id": "1.1", + "Description": "test", + "Attributes": [{"Section": "Access Control"}], + "Checks": ["check_a"], + } + ], + } + path = tmp_path / "legacy.json" + path.write_text(json.dumps(data)) + fw = load_compliance_framework_universal(str(path)) + assert fw is not None + assert fw.framework == "SOC2" + assert fw.outputs is None + assert fw.requirements[0].attributes["Section"] == "Access Control" + assert fw.requirements[0].checks == {"aws": ["check_a"]} + + +class TestSmokeLoadAllJSONs: + """Parametrized smoke test: every existing compliance JSON must load as ComplianceFramework.""" + + @staticmethod + def _find_all_compliance_jsons(): + base = os.path.join( + os.path.dirname(__file__), + "..", + "..", + "..", + "prowler", + "compliance", + ) + base = os.path.normpath(base) + jsons = [] + if os.path.isdir(base): + # Top-level JSONs (multi-provider) + for filename in os.listdir(base): + if filename.endswith(".json"): + jsons.append(os.path.join(base, filename)) + # Provider sub-directory JSONs + for provider_dir in os.listdir(base): + provider_path = os.path.join(base, provider_dir) + if os.path.isdir(provider_path): + for filename in os.listdir(provider_path): + if filename.endswith(".json"): + jsons.append(os.path.join(provider_path, filename)) + return jsons + + @pytest.mark.parametrize( + "json_path", + _find_all_compliance_jsons.__func__(), + ids=lambda p: os.path.basename(p), + ) + def test_loads_as_universal(self, json_path): + fw = load_compliance_framework_universal(json_path) + assert fw is not None, f"Failed to load {json_path}" + assert fw.framework + assert fw.name + assert len(fw.requirements) >= 0 + + +class TestBackwardCompat: + """Ensure Compliance.get_bulk still returns Compliance objects.""" + + def test_get_bulk_still_works(self): + # This test just validates the legacy path still returns Compliance objects + # We test with a constructed Compliance object + legacy = CIS_1_4_AWS + assert isinstance(legacy, Compliance) + assert legacy.Framework == "CIS" + + +class TestAttributesMetadataValidation: + """Validate that Requirement attributes match their attributes_metadata schema.""" + + def _metadata(self, required=False, enum=None, type_str="str"): + return [ + AttributeMetadata(key="Section", type="str", required=True), + AttributeMetadata(key="Level", type=type_str, required=required, enum=enum), + ] + + def test_valid_attributes_pass(self): + fw = ComplianceFramework( + framework="Test", + name="Test", + description="d", + requirements=[ + UniversalComplianceRequirement( + id="1.1", + description="d", + attributes={"Section": "IAM", "Level": "high"}, + checks={}, + ), + ], + attributes_metadata=self._metadata(), + ) + assert len(fw.requirements) == 1 + + def test_missing_required_key_raises(self): + with pytest.raises( + ValidationError, match="missing required attribute 'Section'" + ): + ComplianceFramework( + framework="Test", + name="Test", + description="d", + requirements=[ + UniversalComplianceRequirement( + id="1.1", + description="d", + attributes={"Level": "high"}, + checks={}, + ), + ], + attributes_metadata=self._metadata(), + ) + + def test_invalid_enum_value_raises(self): + with pytest.raises(ValidationError, match="not in"): + ComplianceFramework( + framework="Test", + name="Test", + description="d", + requirements=[ + UniversalComplianceRequirement( + id="1.1", + description="d", + attributes={"Section": "IAM", "Level": "invalid"}, + checks={}, + ), + ], + attributes_metadata=self._metadata(enum=["high", "low"]), + ) + + def test_valid_enum_value_passes(self): + fw = ComplianceFramework( + framework="Test", + name="Test", + description="d", + requirements=[ + UniversalComplianceRequirement( + id="1.1", + description="d", + attributes={"Section": "IAM", "Level": "high"}, + checks={}, + ), + ], + attributes_metadata=self._metadata(enum=["high", "low"]), + ) + assert len(fw.requirements) == 1 + + def test_wrong_type_int_raises(self): + with pytest.raises(ValidationError, match="expected type int"): + ComplianceFramework( + framework="Test", + name="Test", + description="d", + requirements=[ + UniversalComplianceRequirement( + id="1.1", + description="d", + attributes={"Section": "IAM", "Level": "not_a_number"}, + checks={}, + ), + ], + attributes_metadata=self._metadata(type_str="int"), + ) + + def test_correct_type_int_passes(self): + fw = ComplianceFramework( + framework="Test", + name="Test", + description="d", + requirements=[ + UniversalComplianceRequirement( + id="1.1", + description="d", + attributes={"Section": "IAM", "Level": 5}, + checks={}, + ), + ], + attributes_metadata=self._metadata(type_str="int"), + ) + assert fw.requirements[0].attributes["Level"] == 5 + + def test_none_optional_value_skips_validation(self): + """None values for non-required keys should not trigger type/enum errors.""" + fw = ComplianceFramework( + framework="Test", + name="Test", + description="d", + requirements=[ + UniversalComplianceRequirement( + id="1.1", + description="d", + attributes={"Section": "IAM", "Level": None}, + checks={}, + ), + ], + attributes_metadata=self._metadata(enum=["high", "low"]), + ) + assert len(fw.requirements) == 1 + + def test_no_metadata_skips_validation(self): + """Frameworks without attributes_metadata should not be validated.""" + fw = ComplianceFramework( + framework="Test", + name="Test", + description="d", + requirements=[ + UniversalComplianceRequirement( + id="1.1", + description="d", + attributes={"anything": "goes"}, + checks={}, + ), + ], + ) + assert len(fw.requirements) == 1 + + def test_unknown_attribute_key_raises(self): + """Typos like 'Sectoin' must be rejected by the schema validator.""" + with pytest.raises(ValidationError, match="unknown attribute 'Sectoin'"): + ComplianceFramework( + framework="Test", + name="Test", + description="d", + requirements=[ + UniversalComplianceRequirement( + id="1.1", + description="d", + attributes={"Sectoin": "IAM", "Level": "high"}, + checks={}, + ), + ], + attributes_metadata=self._metadata(enum=["high", "low"]), + ) + + def test_multiple_unknown_keys_all_reported(self): + """Every unknown key must appear in the validation error (deterministic order).""" + with pytest.raises( + ValidationError, + match=r"unknown attribute 'Bogus1'[\s\S]*unknown attribute 'Bogus2'", + ): + ComplianceFramework( + framework="Test", + name="Test", + description="d", + requirements=[ + UniversalComplianceRequirement( + id="1.1", + description="d", + attributes={ + "Section": "IAM", + "Level": "high", + "Bogus1": "x", + "Bogus2": "y", + }, + checks={}, + ), + ], + attributes_metadata=self._metadata(enum=["high", "low"]), + ) + + def test_multiple_errors_reported(self): + """All validation errors should be collected and reported together.""" + with pytest.raises( + ValidationError, match="missing required attribute 'Section'" + ): + ComplianceFramework( + framework="Test", + name="Test", + description="d", + requirements=[ + UniversalComplianceRequirement( + id="1.1", + description="d", + attributes={"Level": "bad"}, + checks={}, + ), + UniversalComplianceRequirement( + id="1.2", + description="d", + attributes={"Level": "also_bad"}, + checks={}, + ), + ], + attributes_metadata=self._metadata(enum=["high", "low"]), + ) diff --git a/tests/providers/aws/aws_provider_test.py b/tests/providers/aws/aws_provider_test.py index 68342a2735..1a07ab13c9 100644 --- a/tests/providers/aws/aws_provider_test.py +++ b/tests/providers/aws/aws_provider_test.py @@ -839,6 +839,132 @@ aws: assert isinstance(aws_provider, AwsProvider) + @mock_aws + def test_excluded_regions_removed_from_enabled_regions(self): + aws_provider = AwsProvider(excluded_regions={AWS_REGION_EU_WEST_1}) + + assert AWS_REGION_EU_WEST_1 not in aws_provider._enabled_regions + assert AWS_REGION_EU_WEST_1 not in aws_provider.generate_regional_clients("ec2") + + @mock_aws + def test_excluded_regions_pruned_from_input_regions(self): + aws_provider = AwsProvider( + regions={AWS_REGION_EU_WEST_1, AWS_REGION_US_EAST_1}, + excluded_regions={AWS_REGION_EU_WEST_1}, + ) + + assert AWS_REGION_EU_WEST_1 not in aws_provider._identity.audited_regions + assert AWS_REGION_US_EAST_1 in aws_provider._identity.audited_regions + + @mock_aws + def test_excluded_regions_from_config_file(self): + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as tmp: + tmp.write(f"aws:\n disallowed_regions:\n - {AWS_REGION_EU_WEST_1}\n") + config_path = tmp.name + try: + aws_provider = AwsProvider(config_path=config_path) + assert AWS_REGION_EU_WEST_1 not in aws_provider._enabled_regions + assert aws_provider._excluded_regions == {AWS_REGION_EU_WEST_1} + finally: + os.remove(config_path) + + @mock_aws + def test_excluded_regions_from_env_on_direct_provider_init(self): + with mock.patch.dict( + os.environ, + {"PROWLER_AWS_DISALLOWED_REGIONS": AWS_REGION_EU_WEST_1}, + clear=False, + ): + aws_provider = AwsProvider() + + assert aws_provider._excluded_regions == {AWS_REGION_EU_WEST_1} + assert AWS_REGION_EU_WEST_1 not in aws_provider._enabled_regions + + @mock_aws + def test_excluded_regions_precedence_explicit_over_env_and_config(self): + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as tmp: + tmp.write(f"aws:\n disallowed_regions:\n - {AWS_REGION_EU_WEST_1}\n") + config_path = tmp.name + try: + with mock.patch.dict( + os.environ, + {"PROWLER_AWS_DISALLOWED_REGIONS": AWS_REGION_US_EAST_1}, + clear=False, + ): + aws_provider = AwsProvider( + config_path=config_path, + excluded_regions={AWS_REGION_US_EAST_2}, + ) + + assert aws_provider._excluded_regions == {AWS_REGION_US_EAST_2} + assert AWS_REGION_US_EAST_2 not in aws_provider._enabled_regions + assert AWS_REGION_EU_WEST_1 in aws_provider._enabled_regions + assert AWS_REGION_US_EAST_1 in aws_provider._enabled_regions + finally: + os.remove(config_path) + + @mock_aws + def test_excluded_regions_from_config_avoid_excluded_profile_region( + self, monkeypatch + ): + monkeypatch.setenv("AWS_DEFAULT_REGION", AWS_REGION_EU_WEST_1) + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as tmp: + tmp.write(f"aws:\n disallowed_regions:\n - {AWS_REGION_EU_WEST_1}\n") + config_path = tmp.name + try: + aws_provider = AwsProvider(config_path=config_path) + + assert aws_provider.identity.profile_region == AWS_REGION_US_EAST_1 + finally: + os.remove(config_path) + + @mock_aws + def test_aws_provider_raises_when_all_input_regions_are_excluded(self): + with raises(AWSArgumentTypeValidationError): + AwsProvider( + regions={AWS_REGION_EU_WEST_1}, + excluded_regions={AWS_REGION_EU_WEST_1}, + ) + + def test_get_excluded_regions_from_env_parses_comma_list(self): + with mock.patch.dict( + os.environ, + {"PROWLER_AWS_DISALLOWED_REGIONS": " me-south-1 , ap-east-1 ,, "}, + ): + assert Provider.get_excluded_regions_from_env() == { + "me-south-1", + "ap-east-1", + } + + def test_get_excluded_regions_from_env_ignores_legacy_generic_name(self): + with mock.patch.dict( + os.environ, + {"PROWLER_DISALLOWED_REGIONS": "me-south-1"}, + clear=True, + ): + assert Provider.get_excluded_regions_from_env() == set() + + def test_get_excluded_regions_from_env_unset(self): + with mock.patch.dict(os.environ, {}, clear=True): + assert Provider.get_excluded_regions_from_env() == set() + + @mock_aws + def test_print_credentials_shows_all_except_excluded_regions(self): + aws_provider = AwsProvider( + excluded_regions={AWS_REGION_EU_WEST_1, AWS_REGION_US_EAST_1} + ) + + with patch( + "prowler.providers.aws.aws_provider.print_boxes" + ) as mock_print_boxes: + aws_provider.print_credentials() + + report_lines = mock_print_boxes.call_args.args[0] + assert any( + "AWS Regions:" in line and "all except eu-west-1, us-east-1" in line + for line in report_lines + ) + @mock_aws def test_generate_regional_clients_all_enabled_regions(self): aws_provider = AwsProvider() @@ -2033,6 +2159,24 @@ aws: == AWS_REGION_EU_WEST_1 ) + def test_get_aws_region_for_sts_avoids_excluded_session_region(self): + input_regions = None + session_region = AWS_REGION_EU_WEST_1 + assert ( + get_aws_region_for_sts( + session_region, input_regions, {AWS_REGION_EU_WEST_1} + ) + == AWS_REGION_US_EAST_1 + ) + + def test_get_profile_region_avoids_excluded_session_region(self): + mocked_session = mock.Mock(region_name=AWS_REGION_EU_WEST_1) + + assert ( + AwsProvider.get_profile_region(mocked_session, {AWS_REGION_EU_WEST_1}) + == AWS_REGION_US_EAST_1 + ) + @mock_aws def test_set_session_config_default(self): aws_provider = AwsProvider() diff --git a/tests/providers/cloudflare/cloudflare_provider_test.py b/tests/providers/cloudflare/cloudflare_provider_test.py index 58b7eb8086..c3ba6d75e2 100644 --- a/tests/providers/cloudflare/cloudflare_provider_test.py +++ b/tests/providers/cloudflare/cloudflare_provider_test.py @@ -433,6 +433,29 @@ class TestCloudflareValidateCredentials: with pytest.raises(CloudflareNoAccountsError): CloudflareProvider.validate_credentials(session) + def test_validate_credentials_breaks_on_repeated_account_ids(self): + """Pagination must stop when the SDK repeats account IDs to avoid infinite loops.""" + + def repeating_accounts(): + account = MagicMock() + account.id = ACCOUNT_ID + while True: + yield account + + mock_client = MagicMock() + mock_client.user.get.side_effect = Exception("Some other error") + mock_client.accounts.list.return_value = repeating_accounts() + + session = CloudflareSession( + client=mock_client, + api_token=API_TOKEN, + api_key=None, + api_email=None, + ) + + # Must return without hanging; repeated IDs break the loop. + CloudflareProvider.validate_credentials(session) + class TestCloudflareTestConnection: """Tests for test_connection method.""" diff --git a/tests/providers/googleworkspace/services/calendar/calendar_external_invitations_warning/calendar_external_invitations_warning_test.py b/tests/providers/googleworkspace/services/calendar/calendar_external_invitations_warning/calendar_external_invitations_warning_test.py index f367d5938d..0320ea4a82 100644 --- a/tests/providers/googleworkspace/services/calendar/calendar_external_invitations_warning/calendar_external_invitations_warning_test.py +++ b/tests/providers/googleworkspace/services/calendar/calendar_external_invitations_warning/calendar_external_invitations_warning_test.py @@ -73,8 +73,8 @@ class TestCalendarExternalInvitationsWarning: assert findings[0].status == "FAIL" assert "disabled" in findings[0].status_extended - def test_fail_no_policy_set(self): - """Test FAIL when no explicit policy is set (None) but fetch succeeded""" + def test_pass_using_default(self): + """Test PASS when no explicit policy is set (None) — Google default is secure (enabled)""" mock_provider = set_mocked_googleworkspace_provider() with ( @@ -100,8 +100,8 @@ class TestCalendarExternalInvitationsWarning: findings = check.execute() assert len(findings) == 1 - assert findings[0].status == "FAIL" - assert "not explicitly configured" in findings[0].status_extended + assert findings[0].status == "PASS" + assert "secure default" in findings[0].status_extended def test_no_findings_when_fetch_failed(self): """Test no findings returned when the API fetch failed""" diff --git a/tests/providers/googleworkspace/services/calendar/calendar_external_sharing_primary_calendar/calendar_external_sharing_primary_calendar_test.py b/tests/providers/googleworkspace/services/calendar/calendar_external_sharing_primary_calendar/calendar_external_sharing_primary_calendar_test.py index 32056e9fcb..cb8713630f 100644 --- a/tests/providers/googleworkspace/services/calendar/calendar_external_sharing_primary_calendar/calendar_external_sharing_primary_calendar_test.py +++ b/tests/providers/googleworkspace/services/calendar/calendar_external_sharing_primary_calendar/calendar_external_sharing_primary_calendar_test.py @@ -104,8 +104,8 @@ class TestCalendarExternalSharingPrimaryCalendar: assert findings[0].status == "FAIL" assert "EXTERNAL_ALL_INFO_READ_WRITE" in findings[0].status_extended - def test_fail_no_policy_set(self): - """Test FAIL when no explicit policy is set (None) but fetch succeeded""" + def test_pass_using_default(self): + """Test PASS when no explicit policy is set (None) — Google default is secure (free/busy only)""" mock_provider = set_mocked_googleworkspace_provider() with ( @@ -131,8 +131,8 @@ class TestCalendarExternalSharingPrimaryCalendar: findings = check.execute() assert len(findings) == 1 - assert findings[0].status == "FAIL" - assert "not explicitly configured" in findings[0].status_extended + assert findings[0].status == "PASS" + assert "secure default" in findings[0].status_extended def test_no_findings_when_fetch_failed(self): """Test no findings returned when the API fetch failed""" diff --git a/tests/providers/googleworkspace/services/drive/drive_external_sharing_warn_users/drive_external_sharing_warn_users_test.py b/tests/providers/googleworkspace/services/drive/drive_external_sharing_warn_users/drive_external_sharing_warn_users_test.py index c9c7317f2b..2ad66b3d2f 100644 --- a/tests/providers/googleworkspace/services/drive/drive_external_sharing_warn_users/drive_external_sharing_warn_users_test.py +++ b/tests/providers/googleworkspace/services/drive/drive_external_sharing_warn_users/drive_external_sharing_warn_users_test.py @@ -67,8 +67,8 @@ class TestDriveExternalSharingWarnUsers: assert findings[0].status == "FAIL" assert "disabled" in findings[0].status_extended - def test_fail_no_policy_set(self): - """Test FAIL when no explicit policy is set (None) but fetch succeeded""" + def test_pass_using_default(self): + """Test PASS when no explicit policy is set (None) — Google default is secure""" mock_provider = set_mocked_googleworkspace_provider() with ( @@ -92,8 +92,8 @@ class TestDriveExternalSharingWarnUsers: findings = check.execute() assert len(findings) == 1 - assert findings[0].status == "FAIL" - assert "not explicitly configured" in findings[0].status_extended + assert findings[0].status == "PASS" + assert "secure default" in findings[0].status_extended def test_no_findings_when_fetch_failed(self): """Test no findings returned when the API fetch failed""" diff --git a/tests/providers/googleworkspace/services/drive/drive_shared_drive_creation_allowed/drive_shared_drive_creation_allowed_test.py b/tests/providers/googleworkspace/services/drive/drive_shared_drive_creation_allowed/drive_shared_drive_creation_allowed_test.py index f221e1ec75..016e643f2d 100644 --- a/tests/providers/googleworkspace/services/drive/drive_shared_drive_creation_allowed/drive_shared_drive_creation_allowed_test.py +++ b/tests/providers/googleworkspace/services/drive/drive_shared_drive_creation_allowed/drive_shared_drive_creation_allowed_test.py @@ -69,8 +69,8 @@ class TestDriveSharedDriveCreationAllowed: assert findings[0].status == "FAIL" assert "prevented" in findings[0].status_extended - def test_fail_no_policy_set(self): - """Test FAIL when no explicit policy is set (None) but fetch succeeded""" + def test_pass_using_default(self): + """Test PASS when no explicit policy is set (None) — Google default is secure""" mock_provider = set_mocked_googleworkspace_provider() with ( @@ -94,8 +94,8 @@ class TestDriveSharedDriveCreationAllowed: findings = check.execute() assert len(findings) == 1 - assert findings[0].status == "FAIL" - assert "not explicitly configured" in findings[0].status_extended + assert findings[0].status == "PASS" + assert "secure default" in findings[0].status_extended def test_no_findings_when_fetch_failed(self): """Test no findings returned when the API fetch failed""" diff --git a/tests/providers/googleworkspace/services/drive/drive_shared_drive_disable_download_print_copy/drive_shared_drive_disable_download_print_copy_test.py b/tests/providers/googleworkspace/services/drive/drive_shared_drive_disable_download_print_copy/drive_shared_drive_disable_download_print_copy_test.py index 35ee7b1651..05edcb57a2 100644 --- a/tests/providers/googleworkspace/services/drive/drive_shared_drive_disable_download_print_copy/drive_shared_drive_disable_download_print_copy_test.py +++ b/tests/providers/googleworkspace/services/drive/drive_shared_drive_disable_download_print_copy/drive_shared_drive_disable_download_print_copy_test.py @@ -101,8 +101,8 @@ class TestDriveSharedDriveDisableDownloadPrintCopy: assert findings[0].status == "FAIL" assert "ALL" in findings[0].status_extended - def test_fail_no_policy_set(self): - """Test FAIL when no explicit policy is set (None) but fetch succeeded""" + def test_pass_using_default(self): + """Test PASS when no explicit policy is set (None) — Google default is secure""" mock_provider = set_mocked_googleworkspace_provider() with ( @@ -128,8 +128,8 @@ class TestDriveSharedDriveDisableDownloadPrintCopy: findings = check.execute() assert len(findings) == 1 - assert findings[0].status == "FAIL" - assert "not explicitly configured" in findings[0].status_extended + assert findings[0].status == "PASS" + assert "secure default" in findings[0].status_extended def test_no_findings_when_fetch_failed(self): """Test no findings returned when the API fetch failed""" diff --git a/tests/providers/googleworkspace/services/drive/drive_warn_sharing_with_allowlisted_domains/drive_warn_sharing_with_allowlisted_domains_test.py b/tests/providers/googleworkspace/services/drive/drive_warn_sharing_with_allowlisted_domains/drive_warn_sharing_with_allowlisted_domains_test.py index a494bbe2eb..1731b81fcc 100644 --- a/tests/providers/googleworkspace/services/drive/drive_warn_sharing_with_allowlisted_domains/drive_warn_sharing_with_allowlisted_domains_test.py +++ b/tests/providers/googleworkspace/services/drive/drive_warn_sharing_with_allowlisted_domains/drive_warn_sharing_with_allowlisted_domains_test.py @@ -71,8 +71,8 @@ class TestDriveWarnSharingWithAllowlistedDomains: assert findings[0].status == "FAIL" assert "disabled" in findings[0].status_extended - def test_fail_no_policy_set(self): - """Test FAIL when no explicit policy is set (None) but fetch succeeded""" + def test_pass_using_default(self): + """Test PASS when no explicit policy is set (None) — Google default is secure""" mock_provider = set_mocked_googleworkspace_provider() with ( @@ -98,8 +98,8 @@ class TestDriveWarnSharingWithAllowlistedDomains: findings = check.execute() assert len(findings) == 1 - assert findings[0].status == "FAIL" - assert "not explicitly configured" in findings[0].status_extended + assert findings[0].status == "PASS" + assert "secure default" in findings[0].status_extended def test_no_findings_when_fetch_failed(self): """Test no findings returned when the API fetch failed""" diff --git a/tests/providers/image/image_provider_test.py b/tests/providers/image/image_provider_test.py index 941685f662..68b0c5d7a2 100644 --- a/tests/providers/image/image_provider_test.py +++ b/tests/providers/image/image_provider_test.py @@ -1185,3 +1185,58 @@ class TestInitGlobalProviderRegistryEnumeration: # The "other/lib" repo should be filtered out by --image-filter assert not any("other/lib" in img for img in provider.images) assert len(provider.images) == 3 + + +class TestRegistryListMode: + """Regression test: `prowler image --registry --registry-list` crashes. + + When --registry-list is passed, ImageProvider._enumerate_registry sets + _listing_only = True and __init__ returns early — before calling + Provider.set_global_provider(self). The caller in __main__.py then calls + global_provider.print_credentials() on a None reference, raising + AttributeError: 'NoneType' object has no attribute 'print_credentials'. + """ + + @patch("prowler.providers.image.image_provider.create_registry_adapter") + @patch("prowler.providers.common.provider.load_and_validate_config_file") + def test_registry_list_does_not_crash(self, mock_load_config, mock_adapter_factory): + """Reproduce the --registry-list crash by running the same sequence + as __main__.py: init_global_provider, get_global_provider, + then print_credentials.""" + mock_load_config.return_value = {} + + adapter = MagicMock() + adapter.list_repositories.return_value = ["myorg/app"] + adapter.list_tags.return_value = ["v1.0", "latest"] + mock_adapter_factory.return_value = adapter + + arguments = Namespace( + provider="image", + config_file=None, + fixer_config=None, + images=None, + image_list_file=None, + scanners=["vuln"], + image_config_scanners=None, + trivy_severity=None, + ignore_unfixed=False, + timeout="5m", + registry="myregistry.io", + image_filter=None, + tag_filter=None, + max_images=0, + registry_insecure=False, + registry_list_images=True, + ) + + # Reproduce the exact crash sequence from __main__.py lines 289-294: + # Provider.init_global_provider(args) + # global_provider = Provider.get_global_provider() + # global_provider.print_credentials() + with mock.patch.object(Provider, "_global", None): + Provider.init_global_provider(arguments) + global_provider = Provider.get_global_provider() + + # This is the line that crashes: global_provider is None so + # .print_credentials() raises AttributeError. + global_provider.print_credentials() diff --git a/tests/providers/vercel/services/project/project_service_test.py b/tests/providers/vercel/services/project/project_service_test.py new file mode 100644 index 0000000000..b82588ddd9 --- /dev/null +++ b/tests/providers/vercel/services/project/project_service_test.py @@ -0,0 +1,45 @@ +from unittest import mock + +from prowler.providers.vercel.services.project.project_service import Project +from tests.providers.vercel.vercel_fixtures import ( + PROJECT_ID, + PROJECT_NAME, + TEAM_ID, + set_mocked_vercel_provider, +) + + +class TestProjectService: + def test_list_projects_parses_security_metadata(self): + service = Project.__new__(Project) + service.provider = set_mocked_vercel_provider() + service.projects = {} + service._paginate = mock.MagicMock( + return_value=[ + { + "id": PROJECT_ID, + "name": PROJECT_NAME, + "accountId": TEAM_ID, + "security": { + "firewallEnabled": True, + "firewallConfigVersion": 42, + "managedRules": { + "owasp": {"active": True, "action": "log"}, + "ai_bots": {"active": False, "action": "deny"}, + }, + "botIdEnabled": True, + }, + } + ] + ) + + service._list_projects() + + project = service.projects[PROJECT_ID] + assert project.firewall_enabled is True + assert project.firewall_config_version == "42" + assert project.managed_rules == { + "owasp": {"active": True, "action": "log"}, + "ai_bots": {"active": False, "action": "deny"}, + } + assert project.bot_id_enabled is True diff --git a/tests/providers/vercel/services/security/security_service_test.py b/tests/providers/vercel/services/security/security_service_test.py new file mode 100644 index 0000000000..d0a690b1db --- /dev/null +++ b/tests/providers/vercel/services/security/security_service_test.py @@ -0,0 +1,199 @@ +from unittest import mock + +from prowler.providers.vercel.services.project.project_service import VercelProject +from prowler.providers.vercel.services.security.security_service import Security +from tests.providers.vercel.vercel_fixtures import PROJECT_ID, PROJECT_NAME, TEAM_ID + + +class TestSecurityService: + def test_fetch_firewall_config_reads_active_version_and_normalizes_response(self): + project = VercelProject(id=PROJECT_ID, name=PROJECT_NAME, team_id=TEAM_ID) + service = Security.__new__(Security) + service.firewall_configs = {} + + service._get = mock.MagicMock( + return_value={ + "active": { + "firewallEnabled": True, + "managedRules": { + "owasp": {"active": True, "action": "deny"}, + "ai_bots": {"active": False, "action": "deny"}, + }, + "rules": [ + { + "id": "rule-custom", + "name": "Block admin access", + "active": True, + "conditionGroup": [ + { + "conditions": [ + { + "type": "path", + "op": "pre", + "value": "/admin", + } + ] + } + ], + "action": { + "mitigate": { + "action": "deny", + } + }, + }, + { + "id": "rule-rate-limit", + "name": "Rate limit login", + "active": True, + "conditionGroup": [ + { + "conditions": [ + { + "type": "path", + "op": "eq", + "value": "/login", + } + ] + } + ], + "action": { + "mitigate": { + "action": "deny", + "rateLimit": { + "algo": "fixed_window", + "window": 60, + "limit": 10, + }, + } + }, + }, + ], + "ips": [ + { + "id": "ip-rule", + "ip": "203.0.113.7", + "action": "deny", + } + ], + }, + "draft": None, + "versions": [1], + } + ) + + service._fetch_firewall_config(project) + + service._get.assert_called_once_with( + "/v1/security/firewall/config/active", + params={"projectId": PROJECT_ID, "teamId": TEAM_ID}, + ) + + config = service.firewall_configs[PROJECT_ID] + assert config.firewall_enabled is True + assert config.managed_rulesets == {"owasp": {"active": True, "action": "deny"}} + assert [rule["id"] for rule in config.custom_rules] == ["rule-custom"] + assert [rule["id"] for rule in config.rate_limiting_rules] == [ + "rule-rate-limit" + ] + assert [rule["id"] for rule in config.ip_blocking_rules] == ["ip-rule"] + + def test_fetch_firewall_config_parses_crs_managed_rulesets(self): + project = VercelProject( + id=PROJECT_ID, + name=PROJECT_NAME, + team_id=TEAM_ID, + firewall_config_version="1", + ) + service = Security.__new__(Security) + service.firewall_configs = {} + + service._get = mock.MagicMock( + return_value={ + "id": "waf_test", + "version": 1, + "firewallEnabled": True, + "crs": { + "gen": {"active": True, "action": "log"}, + "xss": {"active": True, "action": "deny"}, + "php": {"active": False, "action": "log"}, + }, + "rules": [], + "ips": [], + } + ) + + service._fetch_firewall_config(project) + + config = service.firewall_configs[PROJECT_ID] + assert config.firewall_enabled is True + assert config.managed_rulesets == { + "gen": {"active": True, "action": "log"}, + "xss": {"active": True, "action": "deny"}, + } + + def test_fetch_firewall_config_falls_back_to_wrapper_when_active_missing(self): + project = VercelProject(id=PROJECT_ID, name=PROJECT_NAME, team_id=TEAM_ID) + service = Security.__new__(Security) + service.firewall_configs = {} + + service._get = mock.MagicMock( + side_effect=[ + Exception("404 active config not found"), + {"active": None, "draft": None, "versions": []}, + ] + ) + + service._fetch_firewall_config(project) + + assert service._get.call_args_list == [ + mock.call( + "/v1/security/firewall/config/active", + params={"projectId": PROJECT_ID, "teamId": TEAM_ID}, + ), + mock.call( + "/v1/security/firewall/config", + params={"projectId": PROJECT_ID, "teamId": TEAM_ID}, + ), + ] + + config = service.firewall_configs[PROJECT_ID] + assert config.firewall_enabled is False + assert config.managed_rulesets == {} + assert config.custom_rules == [] + assert config.rate_limiting_rules == [] + assert config.ip_blocking_rules == [] + + def test_fetch_firewall_config_uses_project_security_metadata_when_config_empty( + self, + ): + project = VercelProject( + id=PROJECT_ID, + name=PROJECT_NAME, + team_id=TEAM_ID, + firewall_enabled=True, + firewall_config_version="42", + managed_rules={ + "owasp": {"active": True, "action": "log"}, + "ai_bots": {"active": False, "action": "deny"}, + }, + ) + service = Security.__new__(Security) + service.firewall_configs = {} + + service._get = mock.MagicMock( + return_value={"active": None, "draft": None, "versions": []} + ) + + service._fetch_firewall_config(project) + + service._get.assert_called_once_with( + "/v1/security/firewall/config/42", + params={"projectId": PROJECT_ID, "teamId": TEAM_ID}, + ) + + config = service.firewall_configs[PROJECT_ID] + assert config.firewall_enabled is True + assert config.managed_rulesets == {"owasp": {"active": True, "action": "log"}} + assert config.custom_rules == [] + assert config.rate_limiting_rules == [] + assert config.ip_blocking_rules == [] diff --git a/ui/.dockerignore b/ui/.dockerignore index 54ed258b8c..9049298c54 100644 --- a/ui/.dockerignore +++ b/ui/.dockerignore @@ -13,5 +13,5 @@ README.md !.next/static !.next/standalone .git -.husky +.pre-commit-config.yaml scripts/setup-git-hooks.js diff --git a/ui/.pre-commit-config.yaml b/ui/.pre-commit-config.yaml new file mode 100644 index 0000000000..c92ea6cfd3 --- /dev/null +++ b/ui/.pre-commit-config.yaml @@ -0,0 +1,37 @@ +orphan: true + +repos: + - repo: local + hooks: + - id: ui-typecheck + name: UI - TypeScript Check + entry: pnpm run typecheck + language: system + files: '\.(ts|tsx|js|jsx)$' + pass_filenames: false + priority: 0 + + - id: ui-lint + name: UI - ESLint + entry: pnpm run lint:check + language: system + files: '\.(ts|tsx|js|jsx)$' + pass_filenames: false + priority: 0 + + - id: ui-tests + name: UI - Unit Tests + entry: pnpm exec vitest related --run + language: system + files: '\.(ts|tsx|js|jsx)$' + exclude: '\.test\.|\.spec\.|vitest\.config|vitest\.setup' + pass_filenames: true + priority: 1 + + - id: ui-build + name: UI - Build + entry: pnpm run build + language: system + files: '\.(ts|tsx|js|jsx|json|css)$' + pass_filenames: false + priority: 2 diff --git a/ui/CHANGELOG.md b/ui/CHANGELOG.md index ded143758e..d4f378adf8 100644 --- a/ui/CHANGELOG.md +++ b/ui/CHANGELOG.md @@ -2,16 +2,51 @@ All notable changes to the **Prowler UI** are documented in this file. -## [1.24.0] (Prowler UNRELEASED) +## [1.25.0] (Prowler UNRELEASED) + +### 🔄 Changed + +- Redesign compliance page with a horizontal ThreatScore card (always-visible pillar breakdown + ActionDropdown), client-side search for compliance frameworks, compact scan selector trigger, responsive mobile filters, download-started toasts for CSV/PDF exports, enhanced compliance cards with truncated titles, and Alert-based empty/error states; migrate Progress component from HeroUI to shadcn [(#10767)](https://github.com/prowler-cloud/prowler/pull/10767) +- Backward-compatibility middleware redirect from `/sign-up?invitation_token=…` to `/invitation/accept?invitation_token=…`; new invitation emails use `/invitation/accept` directly [(#10797)](https://github.com/prowler-cloud/prowler/pull/10797) + +--- + +## [1.24.1] (Prowler v5.24.1) + +### 🐞 Fixed + +- Findings and filter UX fixes: exclude muted findings by default in the resource detail drawer and finding group resource views, show category context label (for example `Status: FAIL`) on MultiSelect triggers instead of hiding the placeholder, and add a `wide` width option for filter dropdowns applied to the findings Scan filter to prevent label truncation [(#10734)](https://github.com/prowler-cloud/prowler/pull/10734) +- Findings grouped view now handles zero-resource IaC counters, refines drawer loading states, and adds provider indicators to finding groups [(#10736)](https://github.com/prowler-cloud/prowler/pull/10736) +- Other Findings for this resource: ordering by `severity` [(#10778)](https://github.com/prowler-cloud/prowler/pull/10778) +- Other Findings for this resource: show `delta` indicator [(#10778)](https://github.com/prowler-cloud/prowler/pull/10778) +- Compliance: requirement findings do not show muted findings [(#10778)](https://github.com/prowler-cloud/prowler/pull/10778) +- Latest new findings: link to finding groups order by `-severity,-last_seen_at` [(#10778)](https://github.com/prowler-cloud/prowler/pull/10778) + +### 🔒 Security + +- Upgrade React to 19.2.5 and Next.js to 16.2.3 to mitigate CVE-2026-23869 (React2DoS), a high-severity unauthenticated remote DoS vulnerability in the React Flight Protocol's Server Function deserialization [(#10754)](https://github.com/prowler-cloud/prowler/pull/10754) + +--- + +## [1.24.0] (Prowler v5.24.0) ### 🚀 Added - Resources side drawer with redesigned detail panel [(#10673)](https://github.com/prowler-cloud/prowler/pull/10673) +- Syntax highlighting for remediation code blocks in finding groups drawer with provider-aware auto-detection (Shell, HCL, YAML, Bicep) [(#10698)](https://github.com/prowler-cloud/prowler/pull/10698) + +### 🔄 Changed + +- Attack Paths scan selection: contextual button labels based on graph availability, tooltips on disabled actions, green dot indicator for selectable scans, and a warning banner when viewing data from a previous scan cycle [(#10685)](https://github.com/prowler-cloud/prowler/pull/10685) +- Remove legacy finding detail sheet, row-details wrapper, and resource detail panel; unify findings and resources around new side drawers [(#10692)](https://github.com/prowler-cloud/prowler/pull/10692) +- Attack Paths "View Finding" now opens the finding drawer inline over the graph instead of navigating to `/findings` in a new tab, preserving graph zoom, selection, and filter state +- Attack Paths scan table: replace action buttons with radio buttons, add dedicated Graph column, use info-colored In Progress badge, remove redundant Progress column, and fix info banner variant [(#10704)](https://github.com/prowler-cloud/prowler/pull/10704) ### 🐞 Fixed - Findings group resource filters now strip unsupported scan parameters, display scan name instead of provider alias in filter badges, migrate mute modal from HeroUI to shadcn, and add searchable accounts/provider type selectors [(#10662)](https://github.com/prowler-cloud/prowler/pull/10662) - Compliance detail page header now reflects the actual provider, alias and UID of the selected scan instead of always defaulting to AWS [(#10674)](https://github.com/prowler-cloud/prowler/pull/10674) +- Provider wizard modal moved to a stable page-level host so the providers table refreshes after link, authenticate, and connection check without closing the modal [(#10675)](https://github.com/prowler-cloud/prowler/pull/10675) --- @@ -40,7 +75,6 @@ All notable changes to the **Prowler UI** are documented in this file. ### 🐞 Fixed - Preserve query parameters in callbackUrl during invitation flow [(#10571)](https://github.com/prowler-cloud/prowler/pull/10571) -- Deleting the active organization now switches to the target org before deleting, preventing JWT rejection from the backend [(#10491)](https://github.com/prowler-cloud/prowler/pull/10491) - Clear Filters now resets all filters including muted findings and auto-applies, Clear all in pills only removes pill-visible sub-filters, and the discard icon is now an Undo text button [(#10446)](https://github.com/prowler-cloud/prowler/pull/10446) - Send to Jira modal now dynamically fetches and displays available issue types per project instead of hardcoding `"Task"`, fixing failures on non-English Jira instances [(#10534)](https://github.com/prowler-cloud/prowler/pull/10534) - Exclude service filter from finding group resources endpoint to prevent empty results when a service filter is active [(#10652)](https://github.com/prowler-cloud/prowler/pull/10652) diff --git a/ui/README.md b/ui/README.md index 148ef61d5b..b1af73ffe3 100644 --- a/ui/README.md +++ b/ui/README.md @@ -85,10 +85,10 @@ git clone git@github.com:prowler-cloud/ui.git pnpm install ``` -**Note:** The `pnpm install` command will automatically configure Git hooks for code quality checks. If you experience issues, you can manually configure them: +**Note:** The `pnpm install` command will automatically configure prek Git hooks for code quality checks. If hooks are not installed, run from the repo root: ```bash -git config core.hooksPath "ui/.husky" +prek install ``` #### Run the development server diff --git a/ui/actions/compliances/compliances.ts b/ui/actions/compliances/compliances.ts index b23723f670..d5f4fd4954 100644 --- a/ui/actions/compliances/compliances.ts +++ b/ui/actions/compliances/compliances.ts @@ -6,12 +6,10 @@ import { handleApiResponse } from "@/lib/server-actions-helper"; export const getCompliancesOverview = async ({ scanId, region, - query, filters = {}, }: { scanId?: string; region?: string | string[]; - query?: string; filters?: Record; } = {}) => { const headers = await getAuthHeaders({ contentType: false }); @@ -31,8 +29,6 @@ export const getCompliancesOverview = async ({ setParam("filter[scan_id]", scanId); setParam("filter[region__in]", region); - if (query) url.searchParams.set("filter[search]", query); - try { const response = await fetch(url.toString(), { headers, @@ -46,15 +42,16 @@ export const getCompliancesOverview = async ({ }; export const getComplianceOverviewMetadataInfo = async ({ - query = "", sort = "", filters = {}, -}) => { +}: { + sort?: string; + filters?: Record; +} = {}) => { const headers = await getAuthHeaders({ contentType: false }); const url = new URL(`${apiBaseUrl}/compliance-overviews/metadata`); - if (query) url.searchParams.append("filter[search]", query); if (sort) url.searchParams.append("sort", sort); Object.entries(filters).forEach(([key, value]) => { diff --git a/ui/actions/finding-groups/finding-groups.test.ts b/ui/actions/finding-groups/finding-groups.test.ts index f71fe989fb..709469b416 100644 --- a/ui/actions/finding-groups/finding-groups.test.ts +++ b/ui/actions/finding-groups/finding-groups.test.ts @@ -70,7 +70,7 @@ describe("getFindingGroups — default sort for muted and non-muted rows", () => const calledUrl = fetchMock.mock.calls[0][0] as string; const url = new URL(calledUrl); expect(url.searchParams.get("sort")).toBe( - "-status,-new_fail_count,-changed_fail_count,-severity,-fail_count,-last_seen_at", + "-status,-severity,-new_fail_count,-changed_fail_count,-fail_count,-last_seen_at", ); }); @@ -84,7 +84,7 @@ describe("getFindingGroups — default sort for muted and non-muted rows", () => const calledUrl = fetchMock.mock.calls[0][0] as string; const url = new URL(calledUrl); expect(url.searchParams.get("sort")).toBe( - "-status,-new_fail_count,-changed_fail_count,-severity,-new_fail_muted_count,-changed_fail_muted_count,-fail_count,-fail_muted_count,-last_seen_at", + "-status,-severity,-new_fail_count,-changed_fail_count,-new_fail_muted_count,-changed_fail_muted_count,-fail_count,-fail_muted_count,-last_seen_at", ); }); }); @@ -106,7 +106,7 @@ describe("getLatestFindingGroups — default sort for muted and non-muted rows", const calledUrl = fetchMock.mock.calls[0][0] as string; const url = new URL(calledUrl); expect(url.searchParams.get("sort")).toBe( - "-status,-new_fail_count,-changed_fail_count,-severity,-fail_count,-last_seen_at", + "-status,-severity,-new_fail_count,-changed_fail_count,-fail_count,-last_seen_at", ); }); @@ -120,7 +120,7 @@ describe("getLatestFindingGroups — default sort for muted and non-muted rows", const calledUrl = fetchMock.mock.calls[0][0] as string; const url = new URL(calledUrl); expect(url.searchParams.get("sort")).toBe( - "-status,-new_fail_count,-changed_fail_count,-severity,-new_fail_muted_count,-changed_fail_muted_count,-fail_count,-fail_muted_count,-last_seen_at", + "-status,-severity,-new_fail_count,-changed_fail_count,-new_fail_muted_count,-changed_fail_muted_count,-fail_count,-fail_muted_count,-last_seen_at", ); }); }); @@ -262,7 +262,7 @@ describe("getFindingGroupResources — Blocker 1: FAIL-first sort", () => { const calledUrl = fetchMock.mock.calls[0][0] as string; const url = new URL(calledUrl); expect(url.searchParams.get("sort")).toBe( - "-status,-delta,-severity,-last_seen_at", + "-status,-severity,-delta,-last_seen_at", ); }); @@ -300,7 +300,7 @@ describe("getLatestFindingGroupResources — Blocker 1: FAIL-first sort", () => const calledUrl = fetchMock.mock.calls[0][0] as string; const url = new URL(calledUrl); expect(url.searchParams.get("sort")).toBe( - "-status,-delta,-severity,-last_seen_at", + "-status,-severity,-delta,-last_seen_at", ); }); @@ -344,7 +344,7 @@ describe("getFindingGroupResources — triangulation: params coexist", () => { expect(url.searchParams.get("page[number]")).toBe("2"); expect(url.searchParams.get("page[size]")).toBe("50"); expect(url.searchParams.get("sort")).toBe( - "-status,-delta,-severity,-last_seen_at", + "-status,-severity,-delta,-last_seen_at", ); expect(url.searchParams.get("filter[status]")).toBeNull(); }); @@ -372,7 +372,7 @@ describe("getLatestFindingGroupResources — triangulation: params coexist", () expect(url.searchParams.get("page[number]")).toBe("3"); expect(url.searchParams.get("page[size]")).toBe("20"); expect(url.searchParams.get("sort")).toBe( - "-status,-delta,-severity,-last_seen_at", + "-status,-severity,-delta,-last_seen_at", ); expect(url.searchParams.get("filter[status]")).toBeNull(); }); @@ -443,7 +443,7 @@ describe("getFindingGroupResources — caller filters are preserved", () => { const calledUrl = fetchMock.mock.calls[0][0] as string; const url = new URL(calledUrl); expect(url.searchParams.get("sort")).toBe( - "-status,-delta,-severity,-last_seen_at", + "-status,-severity,-delta,-last_seen_at", ); expect(url.searchParams.get("filter[name__icontains]")).toBe("bucket-prod"); expect(url.searchParams.get("filter[severity__in]")).toBe("high"); @@ -533,7 +533,7 @@ describe("getLatestFindingGroupResources — caller filters are preserved", () = const calledUrl = fetchMock.mock.calls[0][0] as string; const url = new URL(calledUrl); expect(url.searchParams.get("sort")).toBe( - "-status,-delta,-severity,-last_seen_at", + "-status,-severity,-delta,-last_seen_at", ); expect(url.searchParams.get("filter[name__icontains]")).toBe( "instance-prod", diff --git a/ui/actions/finding-groups/finding-groups.ts b/ui/actions/finding-groups/finding-groups.ts index c409d804b7..30798ea3e6 100644 --- a/ui/actions/finding-groups/finding-groups.ts +++ b/ui/actions/finding-groups/finding-groups.ts @@ -83,13 +83,13 @@ function normalizeFindingGroupResourceFilters( } const DEFAULT_FINDING_GROUPS_SORT = - "-status,-new_fail_count,-changed_fail_count,-severity,-fail_count,-last_seen_at"; + "-status,-severity,-new_fail_count,-changed_fail_count,-fail_count,-last_seen_at"; const DEFAULT_FINDING_GROUPS_SORT_WITH_MUTED = - "-status,-new_fail_count,-changed_fail_count,-severity,-new_fail_muted_count,-changed_fail_muted_count,-fail_count,-fail_muted_count,-last_seen_at"; + "-status,-severity,-new_fail_count,-changed_fail_count,-new_fail_muted_count,-changed_fail_muted_count,-fail_count,-fail_muted_count,-last_seen_at"; const DEFAULT_FINDING_GROUP_RESOURCES_SORT = - "-status,-delta,-severity,-last_seen_at"; + "-status,-severity,-delta,-last_seen_at"; interface FetchFindingGroupsParams { page?: number; diff --git a/ui/actions/findings/findings-by-resource.adapter.test.ts b/ui/actions/findings/findings-by-resource.adapter.test.ts index b8781f0036..87d37c192c 100644 --- a/ui/actions/findings/findings-by-resource.adapter.test.ts +++ b/ui/actions/findings/findings-by-resource.adapter.test.ts @@ -115,4 +115,35 @@ describe("adaptFindingsByResourceResponse — malformed input", () => { expect(result[0].id).toBe("finding-1"); expect(result[0].checkId).toBe("s3_check"); }); + + it("should normalize a single finding response into a one-item drawer array", () => { + // Given — getFindingById returns a single JSON:API resource object + const input = { + data: { + id: "finding-1", + attributes: { + uid: "uid-1", + check_id: "s3_check", + status: "FAIL", + severity: "critical", + check_metadata: { + checktitle: "S3 Check", + }, + }, + relationships: { + resources: { data: [] }, + scan: { data: null }, + }, + }, + included: [], + }; + + // When + const result = adaptFindingsByResourceResponse(input); + + // Then + expect(result).toHaveLength(1); + expect(result[0].id).toBe("finding-1"); + expect(result[0].checkTitle).toBe("S3 Check"); + }); }); diff --git a/ui/actions/findings/findings-by-resource.adapter.ts b/ui/actions/findings/findings-by-resource.adapter.ts index 75e8f5a81b..d1685eaa9f 100644 --- a/ui/actions/findings/findings-by-resource.adapter.ts +++ b/ui/actions/findings/findings-by-resource.adapter.ts @@ -165,16 +165,18 @@ type IncludedDict = Record; * then resolves each finding's resource and provider relationships. */ interface JsonApiResponse { - data: FindingApiItem[]; + data: FindingApiItem | FindingApiItem[]; included?: Record[]; } function isJsonApiResponse(value: unknown): value is JsonApiResponse { + const data = (value as { data?: unknown })?.data; + return ( value !== null && typeof value === "object" && "data" in value && - Array.isArray((value as { data: unknown }).data) + (Array.isArray(data) || (data !== null && typeof data === "object")) ); } @@ -188,8 +190,11 @@ export function adaptFindingsByResourceResponse( const resourcesDict = createDict("resources", apiResponse) as IncludedDict; const scansDict = createDict("scans", apiResponse) as IncludedDict; const providersDict = createDict("providers", apiResponse) as IncludedDict; + const findings = Array.isArray(apiResponse.data) + ? apiResponse.data + : [apiResponse.data]; - return apiResponse.data.map((item) => { + return findings.map((item) => { const attrs = item.attributes; const meta = (attrs.check_metadata || {}) as Record; const remediationRaw = meta.remediation as diff --git a/ui/actions/findings/findings-by-resource.test.ts b/ui/actions/findings/findings-by-resource.test.ts index 4aff44a2cf..7bc2793195 100644 --- a/ui/actions/findings/findings-by-resource.test.ts +++ b/ui/actions/findings/findings-by-resource.test.ts @@ -43,6 +43,7 @@ vi.mock("@/actions/finding-groups", () => ({ })); import { + getLatestFindingsByResourceUid, resolveFindingIdsByCheckIds, resolveFindingIdsByVisibleGroupResources, } from "./findings-by-resource"; @@ -262,3 +263,46 @@ describe("resolveFindingIdsByVisibleGroupResources", () => { expect(fetchMock).not.toHaveBeenCalled(); }); }); + +describe("getLatestFindingsByResourceUid", () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.stubGlobal("fetch", fetchMock); + getAuthHeadersMock.mockResolvedValue({ Authorization: "Bearer token" }); + handleApiResponseMock.mockResolvedValue({ data: [] }); + }); + + it("should restrict to FAIL, exclude muted findings, and apply severity/time sorting by default", async () => { + fetchMock.mockResolvedValue(new Response("", { status: 200 })); + + await getLatestFindingsByResourceUid({ + resourceUid: "resource-1", + }); + + const calledUrl = new URL(fetchMock.mock.calls[0][0]); + expect(calledUrl.pathname).toBe("/api/v1/findings/latest"); + expect(calledUrl.searchParams.get("filter[resource_uid]")).toBe( + "resource-1", + ); + // Status filter is applied server-side so the page[size]=50 window + // always holds FAIL rows — guards against PASS-heavy resources + // starving FAILs out of the result. + expect(calledUrl.searchParams.get("filter[status]")).toBe("FAIL"); + expect(calledUrl.searchParams.get("filter[muted]")).toBe("false"); + expect(calledUrl.searchParams.get("sort")).toBe("severity,-updated_at"); + }); + + it("should include muted findings only when explicitly requested", async () => { + fetchMock.mockResolvedValue(new Response("", { status: 200 })); + + await getLatestFindingsByResourceUid({ + resourceUid: "resource-1", + includeMuted: true, + }); + + const calledUrl = new URL(fetchMock.mock.calls[0][0]); + expect(calledUrl.searchParams.get("filter[status]")).toBe("FAIL"); + expect(calledUrl.searchParams.get("filter[muted]")).toBe("include"); + expect(calledUrl.searchParams.get("sort")).toBe("severity,-updated_at"); + }); +}); diff --git a/ui/actions/findings/findings-by-resource.ts b/ui/actions/findings/findings-by-resource.ts index 344c5607a0..74a0bcb6de 100644 --- a/ui/actions/findings/findings-by-resource.ts +++ b/ui/actions/findings/findings-by-resource.ts @@ -250,10 +250,12 @@ export const getLatestFindingsByResourceUid = async ({ resourceUid, page = 1, pageSize = 50, + includeMuted = false, }: { resourceUid: string; page?: number; pageSize?: number; + includeMuted?: boolean; }) => { const headers = await getAuthHeaders({ contentType: false }); @@ -263,8 +265,8 @@ export const getLatestFindingsByResourceUid = async ({ url.searchParams.append("filter[resource_uid]", resourceUid); url.searchParams.append("filter[status]", "FAIL"); - url.searchParams.append("filter[muted]", "include"); - url.searchParams.append("sort", "-severity,-updated_at"); + url.searchParams.append("filter[muted]", includeMuted ? "include" : "false"); + url.searchParams.append("sort", "severity,-updated_at"); if (page) url.searchParams.append("page[number]", page.toString()); if (pageSize) url.searchParams.append("page[size]", pageSize.toString()); diff --git a/ui/actions/findings/findings.ts b/ui/actions/findings/findings.ts index 7ce7f931ad..242bd007a8 100644 --- a/ui/actions/findings/findings.ts +++ b/ui/actions/findings/findings.ts @@ -141,7 +141,15 @@ export const getLatestMetadataInfo = async ({ } }; -export const getFindingById = async (findingId: string, include = "") => { +interface GetFindingByIdOptions { + source?: "resource-detail-drawer"; +} + +export const getFindingById = async ( + findingId: string, + include = "", + _options?: GetFindingByIdOptions, +) => { const headers = await getAuthHeaders({ contentType: false }); const url = new URL(`${apiBaseUrl}/findings/${findingId}`); diff --git a/ui/actions/providers/providers.test.ts b/ui/actions/providers/providers.test.ts new file mode 100644 index 0000000000..ab3037db8b --- /dev/null +++ b/ui/actions/providers/providers.test.ts @@ -0,0 +1,135 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const { + fetchMock, + getAuthHeadersMock, + getFormValueMock, + handleApiErrorMock, + handleApiResponseMock, +} = vi.hoisted(() => ({ + fetchMock: vi.fn(), + getAuthHeadersMock: vi.fn(), + getFormValueMock: vi.fn(), + handleApiErrorMock: vi.fn(), + handleApiResponseMock: vi.fn(), +})); + +vi.mock("next/cache", () => ({ + revalidatePath: vi.fn(), +})); + +vi.mock("next/navigation", () => ({ + redirect: vi.fn(), +})); + +vi.mock("@/lib", () => ({ + apiBaseUrl: "https://api.example.com/api/v1", + getAuthHeaders: getAuthHeadersMock, + getFormValue: getFormValueMock, + wait: vi.fn(), +})); + +vi.mock("@/lib/provider-credentials/build-crendentials", () => ({ + buildSecretConfig: vi.fn(() => ({ + secretType: "access-secret-key", + secret: { key: "value" }, + })), +})); + +vi.mock("@/lib/provider-filters", () => ({ + appendSanitizedProviderInFilters: vi.fn(), +})); + +vi.mock("@/lib/server-actions-helper", () => ({ + handleApiError: handleApiErrorMock, + handleApiResponse: handleApiResponseMock, +})); + +import { + addCredentialsProvider, + addProvider, + checkConnectionProvider, + updateCredentialsProvider, +} from "./providers"; + +describe("providers actions", () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.stubGlobal("fetch", fetchMock); + getAuthHeadersMock.mockResolvedValue({ Authorization: "Bearer token" }); + getFormValueMock.mockImplementation((formData: FormData, field: string) => + formData.get(field), + ); + handleApiErrorMock.mockReturnValue({ error: "Unexpected error" }); + handleApiResponseMock.mockResolvedValue({ data: { id: "secret-1" } }); + fetchMock.mockResolvedValue( + new Response(JSON.stringify({ data: { id: "secret-1" } }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }), + ); + }); + + it("should revalidate providers after linking a cloud provider", async () => { + // Given + const formData = new FormData(); + formData.set("providerType", "aws"); + formData.set("providerUid", "111111111111"); + + // When + await addProvider(formData); + + // Then + expect(handleApiResponseMock).toHaveBeenCalledWith( + expect.any(Response), + "/providers", + ); + }); + + it("should revalidate providers after adding credentials in the wizard", async () => { + // Given + const formData = new FormData(); + formData.set("providerId", "provider-1"); + formData.set("providerType", "aws"); + + // When + await addCredentialsProvider(formData); + + // Then + expect(handleApiResponseMock).toHaveBeenCalledWith( + expect.any(Response), + "/providers", + ); + }); + + it("should revalidate providers after updating credentials in the wizard", async () => { + // Given + const formData = new FormData(); + formData.set("providerId", "provider-1"); + formData.set("providerType", "oraclecloud"); + + // When + await updateCredentialsProvider("secret-1", formData); + + // Then + expect(handleApiResponseMock).toHaveBeenCalledWith( + expect.any(Response), + "/providers", + ); + }); + + it("should revalidate providers when checking connection from the wizard", async () => { + // Given + const formData = new FormData(); + formData.set("providerId", "provider-1"); + + // When + await checkConnectionProvider(formData); + + // Then + expect(handleApiResponseMock).toHaveBeenCalledWith( + expect.any(Response), + "/providers", + ); + }); +}); diff --git a/ui/actions/resources/index.ts b/ui/actions/resources/index.ts index afaf91a381..600346e6c6 100644 --- a/ui/actions/resources/index.ts +++ b/ui/actions/resources/index.ts @@ -3,6 +3,7 @@ export { getLatestResources, getMetadataInfo, getResourceById, + getResourceDrawerData, getResourceEvents, getResources, } from "./resources"; diff --git a/ui/actions/resources/resources.ts b/ui/actions/resources/resources.ts index 36f9761959..b0831bdb26 100644 --- a/ui/actions/resources/resources.ts +++ b/ui/actions/resources/resources.ts @@ -2,9 +2,12 @@ import { redirect } from "next/navigation"; +import { getLatestFindings } from "@/actions/findings"; +import { listOrganizationsSafe } from "@/actions/organizations/organizations"; import { apiBaseUrl, getAuthHeaders } from "@/lib"; import { appendSanitizedProviderTypeFilters } from "@/lib/provider-filters"; import { handleApiResponse } from "@/lib/server-actions-helper"; +import { OrganizationResource } from "@/types/organizations"; export const getResources = async ({ page = 1, @@ -255,3 +258,57 @@ export const getResourceById = async ( return undefined; } }; + +export const getResourceDrawerData = async ({ + resourceId, + resourceUid, + providerId, + providerType, + page = 1, + pageSize = 10, + query = "", +}: { + resourceId: string; + resourceUid: string; + providerId: string; + providerType: string; + page?: number; + pageSize?: number; + query?: string; +}) => { + const isCloudEnv = process.env.NEXT_PUBLIC_IS_CLOUD_ENV === "true"; + + const [resourceData, findingsResponse, organizationsResponse] = + await Promise.all([ + getResourceById(resourceId, { fields: ["tags"] }), + getLatestFindings({ + page, + pageSize, + query, + sort: "severity,-inserted_at", + filters: { + "filter[resource_uid]": resourceUid, + "filter[status]": "FAIL", + }, + }), + isCloudEnv && providerType === "aws" + ? listOrganizationsSafe() + : Promise.resolve({ data: [] }), + ]); + + const providerOrg = + providerType === "aws" + ? (organizationsResponse.data.find((organization: OrganizationResource) => + organization.relationships?.providers?.data?.some( + (provider: { id: string }) => provider.id === providerId, + ), + ) ?? null) + : null; + + return { + findings: findingsResponse?.data ?? [], + findingsMeta: findingsResponse?.meta ?? null, + providerOrg, + resourceTags: resourceData?.data?.attributes.tags ?? {}, + }; +}; diff --git a/ui/app/(auth)/invitation/accept/accept-invitation-client.tsx b/ui/app/(auth)/invitation/accept/accept-invitation-client.tsx index 73e6dbe8fd..f334053b48 100644 --- a/ui/app/(auth)/invitation/accept/accept-invitation-client.tsx +++ b/ui/app/(auth)/invitation/accept/accept-invitation-client.tsx @@ -8,10 +8,6 @@ import { useEffect, useRef, useState } from "react"; import { acceptInvitation } from "@/actions/invitations"; import { Button } from "@/components/shadcn"; -import { - INVITATION_ACTION_PARAM, - INVITATION_SIGNUP_ACTION, -} from "@/lib/invitation-routing"; type AcceptState = | { kind: "no-token" } @@ -204,7 +200,7 @@ export function AcceptInvitationClient({ className="w-full" onClick={() => { router.push( - `/sign-up?invitation_token=${encodeURIComponent(token!)}&${INVITATION_ACTION_PARAM}=${INVITATION_SIGNUP_ACTION}`, + `/sign-up?invitation_token=${encodeURIComponent(token!)}`, ); }} > diff --git a/ui/app/(prowler)/_overview/graphs-tabs/findings-view/findings-view.ssr.test.ts b/ui/app/(prowler)/_overview/graphs-tabs/findings-view/findings-view.ssr.test.ts new file mode 100644 index 0000000000..03b990c1d5 --- /dev/null +++ b/ui/app/(prowler)/_overview/graphs-tabs/findings-view/findings-view.ssr.test.ts @@ -0,0 +1,16 @@ +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { describe, expect, it } from "vitest"; + +describe("findings view overview SSR", () => { + const currentDir = path.dirname(fileURLToPath(import.meta.url)); + const filePath = path.join(currentDir, "findings-view.ssr.tsx"); + const source = readFileSync(filePath, "utf8"); + + it("uses the non-legacy latest findings columns", () => { + expect(source).toContain("ColumnLatestFindings"); + expect(source).not.toContain("ColumnNewFindingsToDate"); + }); +}); diff --git a/ui/app/(prowler)/_overview/graphs-tabs/findings-view/findings-view.ssr.tsx b/ui/app/(prowler)/_overview/graphs-tabs/findings-view/findings-view.ssr.tsx index be2aec7770..cca971578c 100644 --- a/ui/app/(prowler)/_overview/graphs-tabs/findings-view/findings-view.ssr.tsx +++ b/ui/app/(prowler)/_overview/graphs-tabs/findings-view/findings-view.ssr.tsx @@ -3,7 +3,8 @@ import { getLatestFindings } from "@/actions/findings/findings"; import { LighthouseBanner } from "@/components/lighthouse/banner"; import { LinkToFindings } from "@/components/overview"; -import { ColumnNewFindingsToDate } from "@/components/overview/new-findings-table/table/column-new-findings-to-date"; +import { ColumnLatestFindings } from "@/components/overview/new-findings-table/table"; +import { CardTitle } from "@/components/shadcn"; import { DataTable } from "@/components/ui/table"; import { createDict } from "@/lib/helper"; import { FindingProps, SearchParamsProps } from "@/types"; @@ -57,24 +58,23 @@ export async function FindingsViewSSR({ searchParams }: FindingsViewSSRProps) { }; return ( -
+
-
-
-

- Latest new failing findings -

-

- Showing the latest 10 new failing findings by severity. -

- -
-
- +
+ Latest New Failed Findings +

+ Showing the latest 10 sorted by severity +

+
+ +
+ } />
); diff --git a/ui/app/(prowler)/_overview/graphs-tabs/graphs-tabs-wrapper.tsx b/ui/app/(prowler)/_overview/graphs-tabs/graphs-tabs-wrapper.tsx index c21491e37f..f9741dc75e 100644 --- a/ui/app/(prowler)/_overview/graphs-tabs/graphs-tabs-wrapper.tsx +++ b/ui/app/(prowler)/_overview/graphs-tabs/graphs-tabs-wrapper.tsx @@ -1,6 +1,7 @@ import { Skeleton } from "@heroui/skeleton"; import { Suspense } from "react"; +import { SkeletonTableNewFindings } from "@/components/overview/new-findings-table/table"; import { SearchParamsProps } from "@/types"; import { GraphsTabsClient } from "./_components/graphs-tabs-client"; @@ -18,6 +19,10 @@ const LoadingFallback = () => ( ); +const TAB_FALLBACKS: Partial> = { + findings: , +}; + type GraphComponent = React.ComponentType<{ searchParams: SearchParamsProps }>; const GRAPH_COMPONENTS: Record = { @@ -38,9 +43,10 @@ export const GraphsTabsWrapper = async ({ const tabsContent = Object.fromEntries( GRAPH_TABS.map((tab) => { const Component = GRAPH_COMPONENTS[tab.id]; + const fallback = TAB_FALLBACKS[tab.id] ?? ; return [ tab.id, - }> + , ]; diff --git a/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/node-detail/node-detail-panel.test.tsx b/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/node-detail/node-detail-panel.test.tsx new file mode 100644 index 0000000000..d7209c68c8 --- /dev/null +++ b/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/node-detail/node-detail-panel.test.tsx @@ -0,0 +1,93 @@ +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import type { ReactNode } from "react"; +import { describe, expect, it, vi } from "vitest"; + +import type { GraphNode } from "@/types/attack-paths"; + +import { NodeDetailPanel } from "./node-detail-panel"; + +vi.mock("@/components/ui/sheet/sheet", () => ({ + Sheet: ({ children }: { children: ReactNode }) =>
{children}
, + SheetContent: ({ children }: { children: ReactNode }) => ( +
{children}
+ ), + SheetDescription: ({ children }: { children: ReactNode }) => ( +
{children}
+ ), + SheetHeader: ({ children }: { children: ReactNode }) =>
{children}
, + SheetTitle: ({ children }: { children: ReactNode }) =>
{children}
, +})); + +vi.mock("./node-overview", () => ({ + NodeOverview: () =>
Node overview
, +})); + +vi.mock("./node-findings", () => ({ + NodeFindings: () =>
Node findings
, +})); + +vi.mock("./node-resources", () => ({ + NodeResources: () =>
Node resources
, +})); + +const findingNode: GraphNode = { + id: "graph-node-id", + labels: ["ProwlerFinding"], + properties: { + id: "finding-123", + check_title: "Open S3 bucket", + name: "Open S3 bucket", + }, +}; + +const resourceNode: GraphNode = { + id: "resource-node-id", + labels: ["S3Bucket"], + properties: { + id: "bucket-123", + name: "bucket-123", + }, +}; + +describe("NodeDetailPanel", () => { + it("renders the view finding button only for finding nodes", () => { + const { rerender } = render(); + + expect( + screen.getByRole("button", { name: /view finding finding-123/i }), + ).toBeInTheDocument(); + + rerender(); + + expect( + screen.queryByRole("button", { name: /view finding/i }), + ).not.toBeInTheDocument(); + }); + + it("calls onViewFinding with the node finding id", async () => { + const user = userEvent.setup(); + const onViewFinding = vi.fn(); + + render( + , + ); + + await user.click( + screen.getByRole("button", { name: /view finding finding-123/i }), + ); + + expect(onViewFinding).toHaveBeenCalledWith("finding-123"); + }); + + it("disables the button and shows the spinner while loading", () => { + render(); + + const button = screen.getByRole("button", { + name: /view finding finding-123/i, + }); + + expect(button).toBeDisabled(); + expect(screen.getByLabelText("Loading")).toHaveClass("size-4"); + }); +}); diff --git a/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/node-detail/node-detail-panel.tsx b/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/node-detail/node-detail-panel.tsx index 4d8885e65c..97d8657bfd 100644 --- a/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/node-detail/node-detail-panel.tsx +++ b/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/node-detail/node-detail-panel.tsx @@ -1,6 +1,7 @@ "use client"; import { Button, Card, CardContent } from "@/components/shadcn"; +import { Spinner } from "@/components/shadcn/spinner/spinner"; import { Sheet, SheetContent, @@ -18,6 +19,8 @@ interface NodeDetailPanelProps { node: GraphNode | null; allNodes?: GraphNode[]; onClose?: () => void; + onViewFinding?: (findingId: string) => void; + viewFindingLoading?: boolean; } /** @@ -26,9 +29,13 @@ interface NodeDetailPanelProps { export const NodeDetailContent = ({ node, allNodes = [], + onViewFinding, + viewFindingLoading = false, }: { node: GraphNode; allNodes?: GraphNode[]; + onViewFinding?: (findingId: string) => void; + viewFindingLoading?: boolean; }) => { const isProwlerFinding = node?.labels.some((label) => label.toLowerCase().includes("finding"), @@ -56,7 +63,12 @@ export const NodeDetailContent = ({
Findings connected to this node
- + )} @@ -88,12 +100,15 @@ export const NodeDetailPanel = ({ node, allNodes = [], onClose, + onViewFinding, + viewFindingLoading = false, }: NodeDetailPanelProps) => { const isOpen = node !== null; const isProwlerFinding = node?.labels.some((label) => label.toLowerCase().includes("finding"), ); + const findingId = node ? String(node.properties?.id || node.id) : ""; return ( !open && onClose?.()}> @@ -107,15 +122,19 @@ export const NodeDetailPanel = ({ {node && isProwlerFinding && ( - )} @@ -123,7 +142,12 @@ export const NodeDetailPanel = ({ {node && (
- +
)} diff --git a/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/node-detail/node-findings.tsx b/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/node-detail/node-findings.tsx index bb424a818c..253401fa97 100644 --- a/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/node-detail/node-findings.tsx +++ b/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/node-detail/node-findings.tsx @@ -1,5 +1,7 @@ "use client"; +import { Button } from "@/components/shadcn"; +import { Spinner } from "@/components/shadcn/spinner/spinner"; import { SeverityBadge } from "@/components/ui/table/severity-badge"; import type { GraphNode } from "@/types/attack-paths"; @@ -16,13 +18,20 @@ type Severity = (typeof SEVERITY_LEVELS)[keyof typeof SEVERITY_LEVELS]; interface NodeFindingsProps { node: GraphNode; allNodes?: GraphNode[]; + onViewFinding?: (findingId: string) => void; + viewFindingLoading?: boolean; } /** * Node findings section showing related findings for the selected node * Displays findings that are connected to the node via HAS_FINDING edges */ -export const NodeFindings = ({ node, allNodes = [] }: NodeFindingsProps) => { +export const NodeFindings = ({ + node, + allNodes = [], + onViewFinding, + viewFindingLoading = false, +}: NodeFindingsProps) => { // Get finding IDs from the node's findings array (populated by adapter) const findingIds = node.findings || []; @@ -79,15 +88,20 @@ export const NodeFindings = ({ node, allNodes = [] }: NodeFindingsProps) => { ID: {findingId}

- onViewFinding?.(findingId)} + disabled={viewFindingLoading} aria-label={`View full finding for ${findingName}`} className="text-text-info dark:text-text-info h-auto shrink-0 p-0 text-xs font-medium hover:underline" > - View Full Finding → - + {viewFindingLoading ? ( + + ) : ( + "View Full Finding →" + )} + {finding.properties?.description && (
diff --git a/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/node-detail/node-remediation.tsx b/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/node-detail/node-remediation.tsx index e3421e5a43..3ece3184f0 100644 --- a/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/node-detail/node-remediation.tsx +++ b/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/node-detail/node-remediation.tsx @@ -1,8 +1,8 @@ "use client"; -import Link from "next/link"; - import { Badge } from "@/components/shadcn/badge/badge"; +import { Button } from "@/components/shadcn/button/button"; +import { Spinner } from "@/components/shadcn/spinner/spinner"; interface Finding { id: string; @@ -13,12 +13,18 @@ interface Finding { interface NodeRemediationProps { findings: Finding[]; + onViewFinding?: (findingId: string) => void; + viewFindingLoading?: boolean; } /** * Node remediation section showing related Prowler findings */ -export const NodeRemediation = ({ findings }: NodeRemediationProps) => { +export const NodeRemediation = ({ + findings, + onViewFinding, + viewFindingLoading = false, +}: NodeRemediationProps) => { const getSeverityVariant = (severity: string) => { switch (severity) { case "critical": @@ -66,15 +72,20 @@ export const NodeRemediation = ({ findings }: NodeRemediationProps) => {
- onViewFinding?.(finding.id)} + disabled={viewFindingLoading} aria-label={`View full finding for ${finding.title}`} - className="text-text-info dark:text-text-info text-sm transition-all hover:opacity-80 dark:hover:opacity-80" + className="text-text-info dark:text-text-info h-auto p-0 text-sm transition-all hover:opacity-80 dark:hover:opacity-80" > - View Full Finding → - + {viewFindingLoading ? ( + + ) : ( + "View Full Finding →" + )} +
))} diff --git a/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/scan-list-table.test.tsx b/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/scan-list-table.test.tsx index a029481e7a..4aec4b97d3 100644 --- a/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/scan-list-table.test.tsx +++ b/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/scan-list-table.test.tsx @@ -24,6 +24,19 @@ vi.mock("next/navigation", () => ({ useSearchParams: () => navigationState.searchParams, })); +vi.mock("@/components/shadcn/tooltip", () => ({ + Tooltip: ({ children }: { children: ReactNode }) => <>{children}, + TooltipTrigger: ({ + children, + }: { + children: ReactNode; + asChild?: boolean; + }) => <>{children}, + TooltipContent: ({ children }: { children: ReactNode }) => ( + {children} + ), +})); + vi.mock("@/components/ui/entities/entity-info", () => ({ EntityInfo: ({ entityAlias, @@ -156,14 +169,14 @@ describe("ScanListTable", () => { expect(screen.getByText("12 Total Entries")).toBeInTheDocument(); expect(screen.getByText("Page 1 of 3")).toBeInTheDocument(); - await user.click(screen.getAllByRole("button", { name: "Select scan" })[0]); + await user.click(screen.getAllByRole("radio", { name: "Select scan" })[0]); expect(pushMock).toHaveBeenCalledWith( "/attack-paths?scanPage=1&scanPageSize=5&scanId=scan-1", ); }); - it("enables the select button for a failed scan when graph data is ready", async () => { + it("enables the radio button for a failed scan when graph data is ready", async () => { const user = userEvent.setup(); const failedScan: AttackPathScan = { ...createScan(1), @@ -176,18 +189,18 @@ describe("ScanListTable", () => { render(); - const button = screen.getByRole("button", { name: "Select scan" }); - expect(button).toBeEnabled(); - expect(button).toHaveTextContent("Select"); + const radio = screen.getByRole("radio", { name: "Select scan" }); + expect(radio).toBeEnabled(); + expect(radio).toHaveAttribute("aria-checked", "false"); - await user.click(button); + await user.click(radio); expect(pushMock).toHaveBeenCalledWith( "/attack-paths?scanPage=1&scanPageSize=5&scanId=scan-1", ); }); - it("disables the select button for a failed scan when graph data is not ready", () => { + it("disables the radio button for a failed scan when graph data is not ready", () => { const failedScan: AttackPathScan = { ...createScan(1), attributes: { @@ -199,8 +212,114 @@ describe("ScanListTable", () => { render(); - const button = screen.getByRole("button", { name: "Select scan" }); - expect(button).toBeDisabled(); - expect(button).toHaveTextContent("Failed"); + const radio = screen.getByRole("radio", { name: "Scan not available" }); + expect(radio).toBeDisabled(); + }); + + it("shows a disabled radio button for a scheduled scan without graph data", () => { + const scheduledScan: AttackPathScan = { + ...createScan(1), + attributes: { + ...createScan(1).attributes, + state: "scheduled", + progress: 0, + graph_data_ready: false, + completed_at: null, + duration: null, + }, + }; + + render(); + + const radio = screen.getByRole("radio", { name: "Scan not available" }); + expect(radio).toBeDisabled(); + }); + + it("shows a disabled radio button for an executing scan without graph data", () => { + const executingScan: AttackPathScan = { + ...createScan(1), + attributes: { + ...createScan(1).attributes, + state: "executing", + progress: 45, + graph_data_ready: false, + completed_at: null, + duration: null, + }, + }; + + render(); + + const radio = screen.getByRole("radio", { name: "Scan not available" }); + expect(radio).toBeDisabled(); + }); + + it("enables the radio button for a scheduled scan when graph data is ready from a previous cycle", async () => { + const user = userEvent.setup(); + const scheduledWithGraph: AttackPathScan = { + ...createScan(1), + attributes: { + ...createScan(1).attributes, + state: "scheduled", + progress: 0, + graph_data_ready: true, + }, + }; + + render(); + + const radio = screen.getByRole("radio", { name: "Select scan" }); + expect(radio).toBeEnabled(); + expect(radio).toHaveAttribute("aria-checked", "false"); + + await user.click(radio); + + expect(pushMock).toHaveBeenCalledWith( + "/attack-paths?scanPage=1&scanPageSize=5&scanId=scan-1", + ); + }); + + it("exposes an accessible label in the Graph column when graph data is ready", () => { + render(); + + expect(screen.getByLabelText("Graph available")).toHaveClass( + "text-text-success-primary", + ); + }); + + it("exposes an accessible label in the Graph column when graph data is not ready", () => { + const noGraphScan: AttackPathScan = { + ...createScan(1), + attributes: { + ...createScan(1).attributes, + graph_data_ready: false, + }, + }; + + render(); + + expect(screen.getByLabelText("Graph not available")).toHaveClass( + "text-text-neutral-secondary", + ); + }); + + it("renders a tooltip explaining a completed scan without graph data", () => { + const completedNoGraph: AttackPathScan = { + ...createScan(1), + attributes: { + ...createScan(1).attributes, + state: "completed", + graph_data_ready: false, + }, + }; + + render(); + + expect( + screen.getByRole("radio", { name: "Scan not available" }), + ).toBeDisabled(); + expect( + screen.getByText("This scan completed without producing graph data."), + ).toBeInTheDocument(); }); }); diff --git a/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/scan-list-table.tsx b/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/scan-list-table.tsx index b5bd240cf0..aa686be15c 100644 --- a/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/scan-list-table.tsx +++ b/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/scan-list-table.tsx @@ -1,15 +1,25 @@ "use client"; import { ColumnDef } from "@tanstack/react-table"; +import { Check, Minus } from "lucide-react"; import { usePathname, useRouter, useSearchParams } from "next/navigation"; -import { Button } from "@/components/shadcn/button/button"; +import { + RadioGroup, + RadioGroupItem, +} from "@/components/shadcn/radio-group/radio-group"; +import { + Tooltip, + TooltipContent, + TooltipTrigger, +} from "@/components/shadcn/tooltip"; import { DateWithTime } from "@/components/ui/entities/date-with-time"; import { EntityInfo } from "@/components/ui/entities/entity-info"; import { DataTable, DataTableColumnHeader } from "@/components/ui/table"; import { formatDuration } from "@/lib/date-utils"; +import { cn } from "@/lib/utils"; import type { MetaDataProps, ProviderType } from "@/types"; -import type { AttackPathScan, ScanState } from "@/types/attack-paths"; +import type { AttackPathScan } from "@/types/attack-paths"; import { SCAN_STATES } from "@/types/attack-paths"; import { ScanStatusBadge } from "./scan-status-badge"; @@ -20,12 +30,6 @@ interface ScanListTableProps { const DEFAULT_PAGE_SIZE = 5; const PAGE_SIZE_OPTIONS = [2, 5, 10, 15]; -const WAITING_STATES: readonly ScanState[] = [ - SCAN_STATES.SCHEDULED, - SCAN_STATES.AVAILABLE, - SCAN_STATES.EXECUTING, -]; - const parsePageParam = (value: string | null, fallback: number) => { if (!value) return fallback; @@ -38,34 +42,32 @@ const formatNullableDuration = (duration: number | null) => { return formatDuration(duration); }; -const isSelectDisabled = ( - scan: AttackPathScan, - selectedScanId: string | null, -) => { - return !scan.attributes.graph_data_ready || selectedScanId === scan.id; -}; - -const getSelectButtonLabel = ( - scan: AttackPathScan, - selectedScanId: string | null, -) => { - if (selectedScanId === scan.id) { - return "Selected"; - } - +const getDisabledTooltip = (scan: AttackPathScan): string | null => { if (scan.attributes.graph_data_ready) { - return "Select"; + return null; } - if (WAITING_STATES.includes(scan.attributes.state)) { - return "Waiting..."; + if (scan.attributes.state === SCAN_STATES.SCHEDULED) { + return "Graph will be available once this scan runs and completes."; + } + + if (scan.attributes.state === SCAN_STATES.AVAILABLE) { + return "This scan is queued. Graph will be available once it completes."; + } + + if (scan.attributes.state === SCAN_STATES.EXECUTING) { + return "Scan is running. Graph will be available once it completes."; } if (scan.attributes.state === SCAN_STATES.FAILED) { - return "Failed"; + return "This scan failed. No graph data is available."; } - return "Select"; + if (scan.attributes.state === SCAN_STATES.COMPLETED) { + return "This scan completed without producing graph data."; + } + + return "Graph data is not available for this scan."; }; const getSelectedRowSelection = ( @@ -97,11 +99,54 @@ const buildMetadata = ( const getColumns = ({ selectedScanId, - onSelectScan, }: { selectedScanId: string | null; - onSelectScan: (scanId: string) => void; }): ColumnDef[] => [ + { + id: "select", + header: () => Select, + cell: ({ row }) => { + const isSelected = selectedScanId === row.original.id; + const canSelect = row.original.attributes.graph_data_ready; + const tooltip = getDisabledTooltip(row.original); + + const radio = ( + + ); + + if (!canSelect && !isSelected && tooltip) { + return ( + + + {radio} + + {tooltip} + + ); + } + + return radio; + }, + enableSorting: false, + }, { accessorKey: "provider", header: ({ column }) => ( @@ -135,22 +180,32 @@ const getColumns = ({ ), cell: ({ row }) => ( - +
+ +
), enableSorting: false, }, { - accessorKey: "progress", - header: ({ column }) => ( - - ), - cell: ({ row }) => ( - {row.original.attributes.progress}% - ), + accessorKey: "graph_data_ready", + header: () => Graph, + cell: ({ row }) => + row.original.attributes.graph_data_ready ? ( + + ) : ( + + ), enableSorting: false, }, { @@ -165,29 +220,6 @@ const getColumns = ({ ), enableSorting: false, }, - { - id: "actions", - header: () => Actions, - cell: ({ row }) => { - const isDisabled = isSelectDisabled(row.original, selectedScanId); - - return ( -
- -
- ); - }, - enableSorting: false, - }, ]; /** @@ -237,19 +269,27 @@ export const ScanListTable = ({ scans }: ScanListTableProps) => { }; return ( - + + { + if (row.original.attributes.graph_data_ready) { + handleSelectScan(row.original.id); + } + }} + enableRowSelection + rowSelection={getSelectedRowSelection(paginatedScans, selectedScanId)} + /> + ); }; diff --git a/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/scan-status-badge.tsx b/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/scan-status-badge.tsx index de8833749e..0c04f95257 100644 --- a/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/scan-status-badge.tsx +++ b/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/scan-status-badge.tsx @@ -3,94 +3,55 @@ import { Loader2 } from "lucide-react"; import { Badge } from "@/components/shadcn/badge/badge"; -import { - Tooltip, - TooltipContent, - TooltipTrigger, -} from "@/components/shadcn/tooltip"; +import { cn } from "@/lib/utils"; import type { ScanState } from "@/types/attack-paths"; import { SCAN_STATES } from "@/types/attack-paths"; -const BADGE_CONFIG: Record< - ScanState, - { className: string; label: string; showGraphDot: boolean } -> = { +const BADGE_CONFIG: Record = { [SCAN_STATES.SCHEDULED]: { className: "bg-bg-neutral-tertiary text-text-neutral-primary", label: "Scheduled", - showGraphDot: true, }, [SCAN_STATES.AVAILABLE]: { className: "bg-bg-neutral-tertiary text-text-neutral-primary", label: "Queued", - showGraphDot: true, }, [SCAN_STATES.EXECUTING]: { - className: "bg-bg-warning-secondary text-text-neutral-primary", + className: "bg-bg-info-secondary text-text-info", label: "In Progress", - showGraphDot: false, }, [SCAN_STATES.COMPLETED]: { className: "bg-bg-pass-secondary text-text-success-primary", label: "Completed", - showGraphDot: false, }, [SCAN_STATES.FAILED]: { className: "bg-bg-fail-secondary text-text-error-primary", label: "Failed", - showGraphDot: true, }, }; interface ScanStatusBadgeProps { status: ScanState; progress?: number; - graphDataReady?: boolean; } export const ScanStatusBadge = ({ status, progress = 0, - graphDataReady = false, }: ScanStatusBadgeProps) => { const config = BADGE_CONFIG[status]; - const graphDot = graphDataReady && config.showGraphDot && ( - - ); - - const tooltipText = graphDataReady - ? "Graph available" - : status === SCAN_STATES.FAILED || status === SCAN_STATES.COMPLETED - ? "Graph not available" - : "Graph not available yet"; - - const icon = - status === SCAN_STATES.EXECUTING ? ( - - ) : ( - graphDot - ); - const label = status === SCAN_STATES.EXECUTING ? `${config.label} (${progress}%)` : config.label; return ( - - - - {icon} - {label} - - - {tooltipText} - + + {status === SCAN_STATES.EXECUTING && ( + + )} + {label} + ); }; diff --git a/ui/app/(prowler)/attack-paths/(workflow)/query-builder/attack-paths-page.tsx b/ui/app/(prowler)/attack-paths/(workflow)/query-builder/attack-paths-page.tsx index f6b1007f5a..4fbd48c793 100644 --- a/ui/app/(prowler)/attack-paths/(workflow)/query-builder/attack-paths-page.tsx +++ b/ui/app/(prowler)/attack-paths/(workflow)/query-builder/attack-paths-page.tsx @@ -14,6 +14,8 @@ import { getAvailableQueries, } from "@/actions/attack-paths"; import { adaptQueryResultToGraphData } from "@/actions/attack-paths/query-result.adapter"; +import { FindingDetailDrawer } from "@/components/findings/table"; +import { useFindingDetails } from "@/components/resources/table/use-finding-details"; import { AutoRefresh } from "@/components/scans"; import { Alert, @@ -30,6 +32,7 @@ import { DialogTitle, DialogTrigger, } from "@/components/shadcn/dialog"; +import { Spinner } from "@/components/shadcn/spinner/spinner"; import { useToast } from "@/components/ui"; import type { AttackPathQuery, @@ -37,7 +40,7 @@ import type { AttackPathScan, GraphNode, } from "@/types/attack-paths"; -import { ATTACK_PATH_QUERY_IDS } from "@/types/attack-paths"; +import { ATTACK_PATH_QUERY_IDS, SCAN_STATES } from "@/types/attack-paths"; import { AttackPathGraph, @@ -65,6 +68,7 @@ export default function AttackPathsPage() { const searchParams = useSearchParams(); const scanId = searchParams.get("scanId"); const graphState = useGraphState(); + const finding = useFindingDetails(); const { toast } = useToast(); const [scansLoading, setScansLoading] = useState(true); @@ -116,10 +120,17 @@ export default function AttackPathsPage() { // Check if there's an executing scan for auto-refresh const hasExecutingScan = scans.some( (scan) => - scan.attributes.state === "executing" || - scan.attributes.state === "scheduled", + scan.attributes.state === SCAN_STATES.EXECUTING || + scan.attributes.state === SCAN_STATES.SCHEDULED, ); + // Detect if the selected scan is showing data from a previous cycle + const selectedScan = scans.find((scan) => scan.id === scanId); + const isViewingPreviousCycleData = + selectedScan && + selectedScan.attributes.graph_data_ready && + selectedScan.attributes.state !== SCAN_STATES.COMPLETED; + // Callback to refresh scans (used by AutoRefresh component) const refreshScans = async () => { try { @@ -304,6 +315,14 @@ export default function AttackPathsPage() { graphState.selectNode(null); }; + const getFindingId = (node: GraphNode | null) => + node ? String(node.properties?.id || node.id) : ""; + + const handleViewFinding = (findingId: string) => { + if (!findingId) return; + void finding.navigateToFinding(findingId); + }; + const handleGraphExport = (svgElement: SVGSVGElement | null) => { try { if (svgElement) { @@ -339,11 +358,11 @@ export default function AttackPathsPage() {

Attack Paths

-

+

Select a scan, build a query, and visualize Attack Paths in your infrastructure.

-

+

Scans can be selected when data is available. A new scan does not interrupt access to existing data.

@@ -373,6 +392,21 @@ export default function AttackPathsPage() { + {/* Banner: viewing data from a previous scan cycle */} + {isViewingPreviousCycleData && ( + + + Viewing data from a previous scan + + This scan is currently{" "} + {selectedScan.attributes.state === SCAN_STATES.EXECUTING + ? `running (${selectedScan.attributes.progress}%)` + : selectedScan.attributes.state} + . The graph data shown is from the last completed cycle. + + + )} + {/* Query Builder Section - shown only after selecting a scan */} {scanId && (
@@ -568,7 +602,7 @@ export default function AttackPathsPage() {
-

+

{graphState.selectedNode?.labels.some( (label) => label @@ -591,7 +625,7 @@ export default function AttackPathsPage() {

Type

-

+

{graphState.selectedNode?.labels .map(formatNodeLabel) .join(", ")} @@ -641,7 +675,7 @@ export default function AttackPathsPage() {

Node Details

-

+

{String( graphState.selectedNode.labels.some((label) => label.toLowerCase().includes("finding"), @@ -659,15 +693,20 @@ export default function AttackPathsPage() { {graphState.selectedNode.labels.some((label) => label.toLowerCase().includes("finding"), ) && ( - )}

)} + + {finding.findingDetails && ( + { + if (!open) finding.resetFindingDetails(); + }} + /> + )} )}
diff --git a/ui/app/(prowler)/compliance/[compliancetitle]/page.tsx b/ui/app/(prowler)/compliance/[compliancetitle]/page.tsx index dd86be1f8a..069826c3ae 100644 --- a/ui/app/(prowler)/compliance/[compliancetitle]/page.tsx +++ b/ui/app/(prowler)/compliance/[compliancetitle]/page.tsx @@ -78,7 +78,7 @@ export default async function ComplianceDetail({ await Promise.all([ getComplianceOverviewMetadataInfo({ filters: { - "filter[scan_id]": selectedScanId, + "filter[scan_id]": selectedScanId ?? undefined, }, }), getComplianceAttributes(complianceId), diff --git a/ui/app/(prowler)/compliance/page.test.tsx b/ui/app/(prowler)/compliance/page.test.tsx new file mode 100644 index 0000000000..42bbbe672f --- /dev/null +++ b/ui/app/(prowler)/compliance/page.test.tsx @@ -0,0 +1,16 @@ +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { describe, expect, it } from "vitest"; + +describe("Compliance overview page", () => { + const currentDir = path.dirname(fileURLToPath(import.meta.url)); + const filePath = path.join(currentDir, "page.tsx"); + const source = readFileSync(filePath, "utf8"); + + it("delegates client-side search to ComplianceOverviewGrid", () => { + expect(source).toContain("ComplianceOverviewGrid"); + expect(source).not.toContain("filter[search]"); + }); +}); diff --git a/ui/app/(prowler)/compliance/page.tsx b/ui/app/(prowler)/compliance/page.tsx index e3e0a63705..92015cd142 100644 --- a/ui/app/(prowler)/compliance/page.tsx +++ b/ui/app/(prowler)/compliance/page.tsx @@ -1,3 +1,4 @@ +import { Info } from "lucide-react"; import { Suspense } from "react"; import { @@ -7,12 +8,14 @@ import { import { getThreatScore } from "@/actions/overview"; import { getScans } from "@/actions/scans"; import { - ComplianceCard, ComplianceSkeletonGrid, NoScansAvailable, ThreatScoreBadge, } from "@/components/compliance"; -import { ComplianceHeader } from "@/components/compliance/compliance-header/compliance-header"; +import { ComplianceFilters } from "@/components/compliance/compliance-header/compliance-filters"; +import { ComplianceOverviewGrid } from "@/components/compliance/compliance-overview-grid"; +import { Alert, AlertDescription } from "@/components/shadcn/alert"; +import { Card, CardContent } from "@/components/shadcn/card/card"; import { ContentLayout } from "@/components/ui"; import { ExpandedScanData, @@ -30,12 +33,6 @@ export default async function Compliance({ const resolvedSearchParams = await searchParams; const searchParamsKey = JSON.stringify(resolvedSearchParams || {}); - const filters = Object.fromEntries( - Object.entries(resolvedSearchParams).filter(([key]) => - key.startsWith("filter["), - ), - ); - const scansData = await getScans({ filters: { "filter[state]": "completed", @@ -79,9 +76,12 @@ export default async function Compliance({ .filter(Boolean) as ExpandedScanData[]; // Use scanId from URL, or select the first scan if not provided - const selectedScanId = - resolvedSearchParams.scanId || expandedScansData[0]?.id || null; - const query = (filters["filter[search]"] as string) || ""; + const scanIdParam = resolvedSearchParams.scanId; + const scanIdFromUrl = Array.isArray(scanIdParam) + ? scanIdParam[0] + : scanIdParam; + const selectedScanId: string | null = + scanIdFromUrl || expandedScansData[0]?.id || null; // Find the selected scan const selectedScan = expandedScansData.find( @@ -102,7 +102,6 @@ export default async function Compliance({ // Fetch metadata if we have a selected scan const metadataInfoData = selectedScanId ? await getComplianceOverviewMetadataInfo({ - query, filters: { "filter[scan_id]": selectedScanId, }, @@ -131,28 +130,39 @@ export default async function Compliance({ {selectedScanId ? ( <> -
-
- -
- {threatScoreData && - typeof selectedScanId === "string" && - selectedScan && ( -
- -
- )} + {/* Row 1: Filters */} +
+
- }> + + {/* Row 2: ThreatScore card — full width, horizontal */} + {threatScoreData && + typeof selectedScanId === "string" && + selectedScan && ( +
+ +
+ )} + + {/* Row 3: Compliance grid with client-side search */} + + + + } + > key.startsWith("filter[")), - ); - - // Extract query from filters - const query = (filters["filter[search]"] as string) || ""; - // Only fetch compliance data if we have a valid scanId const compliancesData = scanId && scanId.trim() !== "" ? await getCompliancesOverview({ scanId, region: regionFilter, - query, }) : { data: [], errors: [] }; const type = compliancesData?.data?.type; + const frameworks = compliancesData?.data + ?.filter((compliance: ComplianceOverviewData) => { + return compliance.attributes.framework !== "ProwlerThreatScore"; + }) + .sort((a: ComplianceOverviewData, b: ComplianceOverviewData) => + a.attributes.framework.localeCompare(b.attributes.framework), + ); // Check if the response contains no data if ( @@ -204,58 +212,49 @@ const SSRComplianceGrid = async ({ type === "tasks" ) { return ( -
-
- No compliance data available for the selected scan. -
-
+ + + + This scan has no compliance data available yet, please select a + different one. + + ); } // Handle errors returned by the API if (compliancesData?.errors?.length > 0) { return ( -
-
Provide a valid scan ID.
-
+ + + Provide a valid scan ID. + ); } return ( -
- {compliancesData.data - .filter((compliance: ComplianceOverviewData) => { - // Filter out ProwlerThreatScore from the grid - return compliance.attributes.framework !== "ProwlerThreatScore"; - }) - .sort((a: ComplianceOverviewData, b: ComplianceOverviewData) => - a.attributes.framework.localeCompare(b.attributes.framework), - ) - .map((compliance: ComplianceOverviewData) => { - const { attributes, id } = compliance; - const { - framework, - version, - requirements_passed, - total_requirements, - } = attributes; - - return ( - - ); - })} -
+ + + + ); +}; + +const ComplianceOverviewPanel = ({ + children, +}: { + children: React.ReactNode; +}) => { + return ( + + {children} + ); }; diff --git a/ui/app/(prowler)/findings/page.test.ts b/ui/app/(prowler)/findings/page.test.ts index 76462dff99..444ed47f9e 100644 --- a/ui/app/(prowler)/findings/page.test.ts +++ b/ui/app/(prowler)/findings/page.test.ts @@ -25,8 +25,10 @@ describe("findings page", () => { expect(source).toContain("resolveFindingScanDateFilters"); }); - it("uses getLatestFindingGroups for non-date/scan queries and getFindingGroups for historical", () => { - expect(source).toContain("hasDateOrScan"); + it("uses resolved filters to choose getFindingGroups for historical queries and getLatestFindingGroups otherwise", () => { + expect(source).toContain("hasHistoricalData"); + expect(source).toContain("hasDateOrScanFilter(filtersWithScanDates)"); + expect(source).toContain("hasDateOrScanFilter(filters)"); expect(source).toContain("getFindingGroups"); expect(source).toContain("getLatestFindingGroups"); }); diff --git a/ui/app/(prowler)/findings/page.tsx b/ui/app/(prowler)/findings/page.tsx index 2211800451..22c90330bb 100644 --- a/ui/app/(prowler)/findings/page.tsx +++ b/ui/app/(prowler)/findings/page.tsx @@ -34,12 +34,6 @@ export default async function Findings({ const { encodedSort } = extractSortAndKey(resolvedSearchParams); const { filters, query } = extractFiltersAndQuery(resolvedSearchParams); - // Check if the searchParams contain any date or scan filter - const hasDateOrScan = hasDateOrScanFilter(resolvedSearchParams); - - // TODO: Re-implement deep link support (/findings?id=) using the grouped view's resource detail drawer - // once the legacy FindingDetailsSheet is fully deprecated (still used by /resources and overview dashboard). - const [providersData, scansData] = await Promise.all([ getProviders({ pageSize: 50 }), getScans({ pageSize: 50 }), @@ -54,8 +48,10 @@ export default async function Findings({ }, }); + const hasHistoricalData = hasDateOrScanFilter(filtersWithScanDates); + const metadataInfoData = await ( - hasDateOrScan ? getMetadataInfo : getLatestMetadataInfo + hasHistoricalData ? getMetadataInfo : getLatestMetadataInfo )({ query, sort: encodedSort, @@ -122,10 +118,9 @@ const SSRDataTable = async ({ const pageSize = parseInt(searchParams.pageSize?.toString() || "10", 10); const { encodedSort } = extractSortAndKey(searchParams); - // Check if the searchParams contain any date or scan filter - const hasDateOrScan = hasDateOrScanFilter(searchParams); + const hasHistoricalData = hasDateOrScanFilter(filters); - const fetchFindingGroups = hasDateOrScan + const fetchFindingGroups = hasHistoricalData ? getFindingGroups : getLatestFindingGroups; @@ -154,7 +149,7 @@ const SSRDataTable = async ({ data={groups} metadata={findingGroupsData?.meta} resolvedFilters={filters} - hasHistoricalData={hasDateOrScan} + hasHistoricalData={hasHistoricalData} /> ); diff --git a/ui/app/(prowler)/providers/page.tsx b/ui/app/(prowler)/providers/page.tsx index 3520670c32..8cbdfb3459 100644 --- a/ui/app/(prowler)/providers/page.tsx +++ b/ui/app/(prowler)/providers/page.tsx @@ -1,11 +1,6 @@ import { Suspense } from "react"; -import { - AddProviderButton, - MutedFindingsConfigButton, - ProvidersAccountsTable, - ProvidersFilters, -} from "@/components/providers"; +import { ProvidersAccountsView } from "@/components/providers"; import { SkeletonTableProviders } from "@/components/providers/table"; import { Skeleton } from "@/components/shadcn/skeleton/skeleton"; import { ContentLayout } from "@/components/ui"; @@ -56,15 +51,6 @@ export default async function Providers({ ); } -const ProvidersActions = () => { - return ( -
- - -
- ); -}; - const ProvidersTableFallback = () => { return (
@@ -120,17 +106,12 @@ const ProvidersAccountsContent = async ({ }); return ( -
- } - /> - -
+ ); }; diff --git a/ui/components/compliance/compliance-accordion/client-accordion-content.test.ts b/ui/components/compliance/compliance-accordion/client-accordion-content.test.ts new file mode 100644 index 0000000000..7f724e20b7 --- /dev/null +++ b/ui/components/compliance/compliance-accordion/client-accordion-content.test.ts @@ -0,0 +1,16 @@ +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { describe, expect, it } from "vitest"; + +describe("client accordion content", () => { + const currentDir = path.dirname(fileURLToPath(import.meta.url)); + const filePath = path.join(currentDir, "client-accordion-content.tsx"); + const source = readFileSync(filePath, "utf8"); + + it("uses the shared standalone finding columns instead of the legacy findings columns", () => { + expect(source).toContain("getStandaloneFindingColumns"); + expect(source).not.toContain("getColumnFindings"); + }); +}); diff --git a/ui/components/compliance/compliance-accordion/client-accordion-content.tsx b/ui/components/compliance/compliance-accordion/client-accordion-content.tsx index 965df073a1..659d3a83fa 100644 --- a/ui/components/compliance/compliance-accordion/client-accordion-content.tsx +++ b/ui/components/compliance/compliance-accordion/client-accordion-content.tsx @@ -5,7 +5,7 @@ import { useEffect, useRef, useState } from "react"; import { getFindings } from "@/actions/findings/findings"; import { - getColumnFindings, + getStandaloneFindingColumns, SkeletonTableFindings, } from "@/components/findings/table"; import { Accordion } from "@/components/ui/accordion/Accordion"; @@ -33,6 +33,7 @@ export const ClientAccordionContent = ({ const searchParams = useSearchParams(); const pageNumber = searchParams.get("page") || "1"; const complianceId = searchParams.get("complianceId"); + const openFindingId = searchParams.get("id"); const defaultSort = "severity,status,-inserted_at"; const sort = searchParams.get("sort") || defaultSort; const loadedPageRef = useRef(null); @@ -61,6 +62,7 @@ export const ClientAccordionContent = ({ filters: { "filter[check_id__in]": checkIds.join(","), "filter[scan]": scanId, + "filter[muted]": "false", ...(region && { "filter[region__in]": region }), }, page: parseInt(pageNumber, 10), @@ -159,12 +161,7 @@ export const ClientAccordionContent = ({

Findings

- col.id !== "select" && - !("accessorKey" in col && col.accessorKey === "updated_at"), - )} + columns={getStandaloneFindingColumns({ openFindingId })} data={expandedFindings || []} metadata={findings?.meta} disableScroll={true} diff --git a/ui/components/compliance/compliance-card.test.tsx b/ui/components/compliance/compliance-card.test.tsx new file mode 100644 index 0000000000..c7a199a7fc --- /dev/null +++ b/ui/components/compliance/compliance-card.test.tsx @@ -0,0 +1,30 @@ +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { describe, expect, it } from "vitest"; + +describe("ComplianceCard", () => { + const currentDir = path.dirname(fileURLToPath(import.meta.url)); + const filePath = path.join(currentDir, "compliance-card.tsx"); + const source = readFileSync(filePath, "utf8"); + + it("keeps the shadcn Card base variant", () => { + expect(source).toContain('variant="base"'); + }); + + it("uses a responsive stacked layout for narrow screens", () => { + expect(source).toContain("flex-col"); + expect(source).toContain("sm:flex-row"); + }); + + it("uses the shadcn progress component instead of Hero UI", () => { + expect(source).toContain('from "@/components/shadcn/progress"'); + expect(source).not.toContain("@heroui/progress"); + }); + + it("places compact actions in the icon column on larger screens", () => { + expect(source).toContain('orientation="column"'); + expect(source).toContain('buttonWidth="icon"'); + }); +}); diff --git a/ui/components/compliance/compliance-card.tsx b/ui/components/compliance/compliance-card.tsx index 5d1b0425ba..2c9f383f2b 100644 --- a/ui/components/compliance/compliance-card.tsx +++ b/ui/components/compliance/compliance-card.tsx @@ -1,11 +1,20 @@ "use client"; -import { Progress } from "@heroui/progress"; import Image from "next/image"; import { useRouter, useSearchParams } from "next/navigation"; import { Card, CardContent } from "@/components/shadcn/card/card"; +import { Progress } from "@/components/shadcn/progress"; +import { + Tooltip, + TooltipContent, + TooltipTrigger, +} from "@/components/shadcn/tooltip"; import { getReportTypeForFramework } from "@/lib/compliance/compliance-report-types"; +import { + getScoreIndicatorClass, + type ScoreColorVariant, +} from "@/lib/compliance/score-utils"; import { ScanEntity } from "@/types/scans"; import { getComplianceIcon } from "../icons"; @@ -45,13 +54,9 @@ export const ComplianceCard: React.FC = ({ (passingRequirements / totalRequirements) * 100, ); - const getRatingColor = (ratingPercentage: number) => { - if (ratingPercentage <= 10) { - return "danger"; - } - if (ratingPercentage <= 40) { - return "warning"; - } + const getRatingVariant = (value: number): ScoreColorVariant => { + if (value <= 10) return "danger"; + if (value <= 40) return "warning"; return "success"; }; @@ -80,58 +85,76 @@ export const ComplianceCard: React.FC = ({ onClick={navigateToDetail} > -
- {getComplianceIcon(title) && ( - {`${title} - )} -
-

- {formatTitle(title)} - {version ? ` - ${version}` : ""} -

- +
+ {getComplianceIcon(title) && ( + {`${title} + )} +
e.stopPropagation()} + onKeyDown={(e) => { + if (e.key === "Enter" || e.key === " ") { + e.stopPropagation(); + } }} - color={getRatingColor(ratingPercentage)} - /> -
- + role="group" + tabIndex={0} + > + +
+
+
+ + +

+ {formatTitle(title)} + {version ? ` - ${version}` : ""} +

+ + + {formatTitle(title)} + {version ? ` - ${version}` : ""} + + +
+
+ + Score: + + + {ratingPercentage}% + +
+ +
+
+ {passingRequirements} / {totalRequirements} Passing Requirements - -
e.stopPropagation()} - onKeyDown={(e) => { - if (e.key === "Enter" || e.key === " ") { - e.stopPropagation(); - } - }} - role="group" - tabIndex={0} - > - -
diff --git a/ui/components/compliance/compliance-download-container.test.tsx b/ui/components/compliance/compliance-download-container.test.tsx new file mode 100644 index 0000000000..4a13c35fbc --- /dev/null +++ b/ui/components/compliance/compliance-download-container.test.tsx @@ -0,0 +1,133 @@ +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const { downloadComplianceCsvMock, downloadComplianceReportPdfMock } = + vi.hoisted(() => ({ + downloadComplianceCsvMock: vi.fn(), + downloadComplianceReportPdfMock: vi.fn(), + })); + +vi.mock("@/lib/helper", () => ({ + downloadComplianceCsv: downloadComplianceCsvMock, + downloadComplianceReportPdf: downloadComplianceReportPdfMock, +})); + +vi.mock("@/components/ui", () => ({ + toast: {}, +})); + +import { ComplianceDownloadContainer } from "./compliance-download-container"; + +describe("ComplianceDownloadContainer", () => { + const currentDir = path.dirname(fileURLToPath(import.meta.url)); + const filePath = path.join(currentDir, "compliance-download-container.tsx"); + const source = readFileSync(filePath, "utf8"); + + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("uses the shared action dropdown for the card actions mode", () => { + expect(source).toContain("ActionDropdown"); + expect(source).not.toContain("@heroui/button"); + }); + + it("should expose an accessible actions menu trigger", () => { + render( + , + ); + + expect( + screen.getByRole("button", { name: "Open compliance export actions" }), + ).toBeInTheDocument(); + }); + + it("should support fixed icon-sized dropdown trigger in column mode", () => { + render( + , + ); + + const trigger = screen.getByRole("button", { + name: "Open compliance export actions", + }); + expect(trigger.className).toContain("border-text-neutral-secondary"); + }); + + it("should open export actions from the compact trigger", async () => { + const user = userEvent.setup(); + + render( + , + ); + + await user.click( + screen.getByRole("button", { name: "Open compliance export actions" }), + ); + + expect(screen.getByText("Download CSV report")).toBeInTheDocument(); + expect(screen.getByText("Download PDF report")).toBeInTheDocument(); + }); + + it("should trigger both downloads from the actions menu", async () => { + const user = userEvent.setup(); + + render( + , + ); + + await user.click( + screen.getByRole("button", { name: "Open compliance export actions" }), + ); + await user.click( + screen.getByRole("menuitem", { name: /Download CSV report/i }), + ); + await user.click( + screen.getByRole("button", { name: "Open compliance export actions" }), + ); + await user.click( + screen.getByRole("menuitem", { name: /Download PDF report/i }), + ); + + expect(downloadComplianceCsvMock).toHaveBeenCalledWith( + "scan-1", + "compliance-1", + {}, + ); + expect(downloadComplianceReportPdfMock).toHaveBeenCalledWith( + "scan-1", + "threatscore", + {}, + ); + }); +}); diff --git a/ui/components/compliance/compliance-download-container.tsx b/ui/components/compliance/compliance-download-container.tsx index 526057ea5c..415da1d85a 100644 --- a/ui/components/compliance/compliance-download-container.tsx +++ b/ui/components/compliance/compliance-download-container.tsx @@ -4,6 +4,15 @@ import { DownloadIcon, FileTextIcon } from "lucide-react"; import { useState } from "react"; import { Button } from "@/components/shadcn/button/button"; +import { + ActionDropdown, + ActionDropdownItem, +} from "@/components/shadcn/dropdown"; +import { + Tooltip, + TooltipContent, + TooltipTrigger, +} from "@/components/shadcn/tooltip"; import { toast } from "@/components/ui"; import type { ComplianceReportType } from "@/lib/compliance/compliance-report-types"; import { @@ -18,6 +27,9 @@ interface ComplianceDownloadContainerProps { reportType?: ComplianceReportType; compact?: boolean; disabled?: boolean; + orientation?: "row" | "column"; + buttonWidth?: "auto" | "icon"; + presentation?: "buttons" | "dropdown"; } export const ComplianceDownloadContainer = ({ @@ -26,9 +38,14 @@ export const ComplianceDownloadContainer = ({ reportType, compact = false, disabled = false, + orientation = "row", + buttonWidth = "auto", + presentation = "buttons", }: ComplianceDownloadContainerProps) => { const [isDownloadingCsv, setIsDownloadingCsv] = useState(false); const [isDownloadingPdf, setIsDownloadingPdf] = useState(false); + const isIconWidth = buttonWidth === "icon"; + const isDropdown = presentation === "dropdown"; const handleDownloadCsv = async () => { if (isDownloadingCsv) return; @@ -52,40 +69,116 @@ export const ComplianceDownloadContainer = ({ const buttonClassName = cn( "border-button-primary text-button-primary hover:bg-button-primary/10", - compact && "h-7 px-2 text-xs", + compact && + !isIconWidth && + "h-7 px-2 text-xs sm:w-full sm:justify-center sm:px-2.5", + orientation === "column" && !isIconWidth && "w-full", + isIconWidth && "size-10 rounded-lg p-0", ); + const labelClassName = isIconWidth + ? "sr-only" + : compact + ? "sr-only sm:not-sr-only" + : undefined; + const showTooltip = compact || isIconWidth; return ( -
- - {reportType && ( - + {reportType && ( + + } + label="Download PDF report" + onSelect={handleDownloadPdf} + disabled={disabled || isDownloadingPdf} + /> + )} + + ) : ( +
+ + + + + {showTooltip && ( + Download CSV report + )} + + {reportType && ( + + + + + {showTooltip && ( + Download PDF report + )} + + )} +
)}
); diff --git a/ui/components/compliance/compliance-header/compliance-filters.tsx b/ui/components/compliance/compliance-header/compliance-filters.tsx new file mode 100644 index 0000000000..474b61d4e7 --- /dev/null +++ b/ui/components/compliance/compliance-header/compliance-filters.tsx @@ -0,0 +1,76 @@ +"use client"; + +import { useRouter, useSearchParams } from "next/navigation"; + +import { ClearFiltersButton } from "@/components/filters/clear-filters-button"; +import { + MultiSelect, + MultiSelectContent, + MultiSelectItem, + MultiSelectSelectAll, + MultiSelectSeparator, + MultiSelectTrigger, + MultiSelectValue, +} from "@/components/shadcn/select/multiselect"; +import { useUrlFilters } from "@/hooks/use-url-filters"; + +import { ScanSelector, SelectScanComplianceDataProps } from "./scan-selector"; + +interface ComplianceFiltersProps { + scans: SelectScanComplianceDataProps["scans"]; + uniqueRegions: string[]; + selectedScanId: string; +} + +export const ComplianceFilters = ({ + scans, + uniqueRegions, + selectedScanId, +}: ComplianceFiltersProps) => { + const router = useRouter(); + const searchParams = useSearchParams(); + const { updateFilter } = useUrlFilters(); + + const handleScanChange = (selectedKey: string) => { + const params = new URLSearchParams(searchParams); + params.set("scanId", selectedKey); + router.push(`?${params.toString()}`, { scroll: false }); + }; + + const regionValues = + searchParams.get("filter[region__in]")?.split(",").filter(Boolean) ?? []; + + return ( +
+
+ +
+ {uniqueRegions.length > 0 && ( +
+ updateFilter("region__in", values)} + > + + + + + Select All + + {uniqueRegions.map((region) => ( + + {region} + + ))} + + +
+ )} + +
+ ); +}; diff --git a/ui/components/compliance/compliance-header/compliance-header.test.tsx b/ui/components/compliance/compliance-header/compliance-header.test.tsx new file mode 100644 index 0000000000..b646199f7f --- /dev/null +++ b/ui/components/compliance/compliance-header/compliance-header.test.tsx @@ -0,0 +1,18 @@ +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { describe, expect, it } from "vitest"; + +describe("ComplianceHeader", () => { + const currentDir = path.dirname(fileURLToPath(import.meta.url)); + const filePath = path.join(currentDir, "compliance-header.tsx"); + const source = readFileSync(filePath, "utf8"); + + it("renders the scan selector inside the shared filters grid using default layout", () => { + expect(source).toContain("prependElement"); + expect(source).toContain(" { const frameworkFilters = []; + const prependElement = showProviders ? ( + + ) : undefined; // Add CIS Profile Level filter if framework is CIS if (framework === "CIS") { @@ -42,6 +45,7 @@ export const ComplianceHeader = ({ key: "cis_profile_level", labelCheckboxGroup: "Level", values: ["Level 1", "Level 2"], + width: "wide" as const, index: 0, // Show first showSelectAll: false, // No "Select All" option since Level 2 includes Level 1 defaultValues: ["Level 2"], // Default to Level 2 selected (which includes Level 1) @@ -55,6 +59,7 @@ export const ComplianceHeader = ({ key: "region__in", labelCheckboxGroup: "Regions", values: uniqueRegions, + width: "wide" as const, index: 1, // Show after framework filters }, ] @@ -77,9 +82,11 @@ export const ComplianceHeader = ({ {selectedScan && } {/* Showed in the compliance page */} - {showProviders && } - {!hideFilters && allFilters.length > 0 && ( - + {!hideFilters && (allFilters.length > 0 || showProviders) && ( + )}
{logoPath && complianceTitle && ( diff --git a/ui/components/compliance/compliance-header/data-compliance.tsx b/ui/components/compliance/compliance-header/data-compliance.tsx index 992dc3d683..4787d00921 100644 --- a/ui/components/compliance/compliance-header/data-compliance.tsx +++ b/ui/components/compliance/compliance-header/data-compliance.tsx @@ -7,11 +7,13 @@ import { ScanSelector, SelectScanComplianceDataProps, } from "@/components/compliance/compliance-header/index"; +import { cn } from "@/lib/utils"; interface DataComplianceProps { scans: SelectScanComplianceDataProps["scans"]; + className?: string; } -export const DataCompliance = ({ scans }: DataComplianceProps) => { +export const DataCompliance = ({ scans, className }: DataComplianceProps) => { const router = useRouter(); const searchParams = useSearchParams(); @@ -36,7 +38,7 @@ export const DataCompliance = ({ scans }: DataComplianceProps) => { }; return ( -
+
{ const selectedScan = scans.find((item) => item.id === selectedScanId); + const triggerLabel = selectedScan ? getScanEntityLabel(selectedScan) : ""; return (