diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index b953610fa1..3300394d83 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,14 +1,15 @@ # SDK -/* @prowler-cloud/sdk -/prowler/ @prowler-cloud/sdk @prowler-cloud/detection-and-remediation -/tests/ @prowler-cloud/sdk @prowler-cloud/detection-and-remediation -/dashboard/ @prowler-cloud/sdk -/docs/ @prowler-cloud/sdk -/examples/ @prowler-cloud/sdk -/util/ @prowler-cloud/sdk -/contrib/ @prowler-cloud/sdk -/permissions/ @prowler-cloud/sdk -/codecov.yml @prowler-cloud/sdk @prowler-cloud/api +/* @prowler-cloud/detection-remediation +/prowler/ @prowler-cloud/detection-remediation +/prowler/compliance/ @prowler-cloud/compliance +/tests/ @prowler-cloud/detection-remediation +/dashboard/ @prowler-cloud/detection-remediation +/docs/ @prowler-cloud/detection-remediation +/examples/ @prowler-cloud/detection-remediation +/util/ @prowler-cloud/detection-remediation +/contrib/ @prowler-cloud/detection-remediation +/permissions/ @prowler-cloud/detection-remediation +/codecov.yml @prowler-cloud/detection-remediation @prowler-cloud/api # API /api/ @prowler-cloud/api @@ -17,7 +18,7 @@ /ui/ @prowler-cloud/ui # AI -/mcp_server/ @prowler-cloud/ai +/mcp_server/ @prowler-cloud/detection-remediation # Platform /.github/ @prowler-cloud/platform diff --git a/.github/actions/setup-python-poetry/action.yml b/.github/actions/setup-python-poetry/action.yml index fd96796b9b..dd5be4b3a2 100644 --- a/.github/actions/setup-python-poetry/action.yml +++ b/.github/actions/setup-python-poetry/action.yml @@ -64,19 +64,6 @@ runs: echo "Updated resolved_reference:" grep -A2 -B2 "resolved_reference" poetry.lock - - name: Update SDK resolved_reference to latest commit (prowler repo on push) - if: github.event_name == 'push' && github.ref == 'refs/heads/master' && github.repository == 'prowler-cloud/prowler' - shell: bash - working-directory: ${{ inputs.working-directory }} - run: | - LATEST_COMMIT=$(curl -s "https://api.github.com/repos/prowler-cloud/prowler/commits/master" | jq -r '.sha') - echo "Latest commit hash: $LATEST_COMMIT" - sed -i '/url = "https:\/\/github\.com\/prowler-cloud\/prowler\.git"/,/resolved_reference = / { - s/resolved_reference = "[a-f0-9]\{40\}"/resolved_reference = "'"$LATEST_COMMIT"'"/ - }' poetry.lock - echo "Updated resolved_reference:" - grep -A2 -B2 "resolved_reference" poetry.lock - - name: Update poetry.lock (prowler repo only) if: github.repository == 'prowler-cloud/prowler' && inputs.update-lock == 'true' shell: bash diff --git a/.github/workflows/find-secrets.yml b/.github/workflows/find-secrets.yml index 0aa955413f..88f84d6729 100644 --- a/.github/workflows/find-secrets.yml +++ b/.github/workflows/find-secrets.yml @@ -27,11 +27,12 @@ jobs: - name: Harden Runner uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 with: - egress-policy: block - allowed-endpoints: > - github.com:443 - ghcr.io:443 - pkg-containers.githubusercontent.com:443 + # We can't block as Trufflehog needs to verify secrets against vendors + egress-policy: audit + # allowed-endpoints: > + # github.com:443 + # ghcr.io:443 + # pkg-containers.githubusercontent.com:443 - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 diff --git a/api/CHANGELOG.md b/api/CHANGELOG.md index 0d798fb29f..213ee215de 100644 --- a/api/CHANGELOG.md +++ b/api/CHANGELOG.md @@ -2,6 +2,26 @@ All notable changes to the **Prowler API** are documented in this file. +## [1.25.3] (Prowler v5.24.3) + +### 🐞 Fixed + +- Finding groups aggregated `status` now treats muted findings as resolved: a group is `FAIL` only while at least one non-muted FAIL remains, otherwise it is `PASS` (including fully-muted groups). The `filter[status]` filter and the `sort=status` ordering share the same semantics, keeping `status` consistent with `fail_count` and the orthogonal `muted` flag [(#10825)](https://github.com/prowler-cloud/prowler/pull/10825) + +--- + +## [1.25.2] (Prowler v5.24.2) + +### πŸ”„ Changed + +- Finding groups `/resources` endpoints now materialize the filtered finding IDs into a Python list before filtering `ResourceFindingMapping`, so PostgreSQL switches from a Merge Semi Join that read hundreds of thousands of RFM index entries to a Nested Loop Index Scan over `finding_id`. The `has_mappings.exists()` pre-check is removed, and a request-scoped cache deduplicates the finding-id round-trip across the helpers that build different RFM querysets [(#10816)](https://github.com/prowler-cloud/prowler/pull/10816) + +### 🐞 Fixed + +- `/finding-groups/latest//resources` now selects the latest completed scan per provider by `-completed_at` (then `-inserted_at`) instead of `-inserted_at`, matching the `/finding-groups/latest` summary path and the daily-summary upsert so overlapping scans no longer produce diverging `delta`/`new_count` between the two endpoints [(#10802)](https://github.com/prowler-cloud/prowler/pull/10802) + +--- + ## [1.25.1] (Prowler v5.24.1) ### πŸ”„ Changed @@ -10,8 +30,10 @@ All notable changes to the **Prowler API** are documented in this file. ### 🐞 Fixed +- Finding group resources endpoints now include findings without associated resources (orphaned IaC findings) as simulated resource rows, and return one row per finding when multiple findings share a resource [(#10708)](https://github.com/prowler-cloud/prowler/pull/10708) - Attack Paths: Missing `tenant_id` filter while getting related findings after scan completes [(#10722)](https://github.com/prowler-cloud/prowler/pull/10722) - Finding group counters `pass_count`, `fail_count` and `manual_count` now exclude muted findings [(#10753)](https://github.com/prowler-cloud/prowler/pull/10753) +- Silent data loss in `ResourceFindingMapping` bulk insert that left findings orphaned when `INSERT ... ON CONFLICT DO NOTHING` dropped rows without raising; added explicit `unique_fields` [(#10724)](https://github.com/prowler-cloud/prowler/pull/10724) --- @@ -26,7 +48,6 @@ All notable changes to the **Prowler API** are documented in this file. - Worker-beat race condition on cold start: replaced `sleep 15` with API service healthcheck dependency (Docker Compose) and init containers (Helm), aligned Gunicorn default port to `8080` [(#10603)](https://github.com/prowler-cloud/prowler/pull/10603) - API container startup crash on Linux due to root-owned bind-mount preventing JWT key generation [(#10646)](https://github.com/prowler-cloud/prowler/pull/10646) -- Finding group resources endpoints now include findings without associated resources (orphan IaC findings) as simulated resource rows, and return one row per finding when multiple findings share a resource [(#10708)](https://github.com/prowler-cloud/prowler/pull/10708) ### πŸ” Security diff --git a/api/poetry.lock b/api/poetry.lock index e066685a9b..6bad57a758 100644 --- a/api/poetry.lock +++ b/api/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.3.4 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.2.1 and should not be changed by hand. [[package]] name = "about-time" @@ -2974,7 +2974,7 @@ files = [ [package.dependencies] autopep8 = "*" Django = ">=4.2" -gprof2dot = ">=2017.9.19" +gprof2dot = ">=2017.09.19" sqlparse = "*" [[package]] @@ -4582,7 +4582,7 @@ files = [ [package.dependencies] attrs = ">=22.2.0" -jsonschema-specifications = ">=2023.3.6" +jsonschema-specifications = ">=2023.03.6" referencing = ">=0.28.4" rpds-py = ">=0.7.1" @@ -4790,7 +4790,7 @@ librabbitmq = ["librabbitmq (>=2.0.0) ; python_version < \"3.11\""] mongodb = ["pymongo (==4.15.3)"] msgpack = ["msgpack (==1.1.2)"] pyro = ["pyro4 (==4.82)"] -qpid = ["qpid-python (==1.36.0.post1)", "qpid-tools (==1.36.0.post1)"] +qpid = ["qpid-python (==1.36.0-1)", "qpid-tools (==1.36.0-1)"] redis = ["redis (>=4.5.2,!=4.5.5,!=5.0.2,<6.5)"] slmq = ["softlayer_messaging (>=1.0.3)"] sqlalchemy = ["sqlalchemy (>=1.4.48,<2.1)"] @@ -4811,7 +4811,7 @@ files = [ ] [package.dependencies] -certifi = ">=14.5.14" +certifi = ">=14.05.14" durationpy = ">=0.7" google-auth = ">=1.0.1" oauthlib = ">=3.2.2" @@ -6920,14 +6920,14 @@ pydantic = ">=2.12.0,<3.0.0" [[package]] name = "pyasn1" -version = "0.6.2" +version = "0.6.3" description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "pyasn1-0.6.2-py3-none-any.whl", hash = "sha256:1eb26d860996a18e9b6ed05e7aae0e9fc21619fcee6af91cca9bad4fbea224bf"}, - {file = "pyasn1-0.6.2.tar.gz", hash = "sha256:9b59a2b25ba7e4f8197db7686c09fb33e658b98339fadb826e9512629017833b"}, + {file = "pyasn1-0.6.3-py3-none-any.whl", hash = "sha256:a80184d120f0864a52a073acc6fc642847d0be408e7c7252f31390c0f4eadcde"}, + {file = "pyasn1-0.6.3.tar.gz", hash = "sha256:697a8ecd6d98891189184ca1fa05d1bb00e2f84b5977c481452050549c8a72cf"}, ] [[package]] @@ -7194,7 +7194,7 @@ files = [ ] [package.dependencies] -astroid = ">=3.2.2,<=3.3.0.dev0" +astroid = ">=3.2.2,<=3.3.0-dev0" colorama = {version = ">=0.4.5", markers = "sys_platform == \"win32\""} dill = [ {version = ">=0.3.7", markers = "python_version >= \"3.12\""}, @@ -8209,10 +8209,10 @@ files = [ ] [package.dependencies] -botocore = ">=1.37.4,<2.0a0" +botocore = ">=1.37.4,<2.0a.0" [package.extras] -crt = ["botocore[crt] (>=1.37.4,<2.0a0)"] +crt = ["botocore[crt] (>=1.37.4,<2.0a.0)"] [[package]] name = "safety" diff --git a/api/src/backend/api/tests/test_views.py b/api/src/backend/api/tests/test_views.py index c519aa104b..b2e5271bda 100644 --- a/api/src/backend/api/tests/test_views.py +++ b/api/src/backend/api/tests/test_views.py @@ -15446,15 +15446,15 @@ class TestFindingGroupViewSet: # iam_password_policy has only PASS findings assert data[0]["attributes"]["status"] == "PASS" - def test_finding_groups_fully_muted_group_reflects_underlying_status( + def test_finding_groups_fully_muted_group_is_pass( self, authenticated_client, finding_groups_fixture ): - """A fully-muted group still surfaces its underlying status (no MUTED). + """A fully-muted group reports status=PASS and muted=True. - rds_encryption has 2 muted FAIL findings, so the group must report - status=FAIL (the orthogonal `muted` boolean signals it isn't actionable). - The statusΓ—muted breakdown lets clients answer 'how many failing - findings are muted in this group'. + rds_encryption has 2 muted FAIL findings. Muted findings are treated + as resolved/accepted, so the group is no longer actionable and its + status must be PASS. The `muted` flag is True because every finding + in the group is muted. """ response = authenticated_client.get( reverse("finding-group-list"), @@ -15464,7 +15464,7 @@ class TestFindingGroupViewSet: data = response.json()["data"] assert len(data) == 1 attrs = data[0]["attributes"] - assert attrs["status"] == "FAIL" + assert attrs["status"] == "PASS" assert attrs["muted"] is True assert attrs["fail_count"] == 0 assert attrs["fail_muted_count"] == 2 @@ -15479,6 +15479,83 @@ class TestFindingGroupViewSet: == attrs["muted_count"] ) + def test_finding_groups_status_ignores_muted_failures( + self, + authenticated_client, + tenants_fixture, + scans_fixture, + resources_fixture, + ): + """Muted FAIL findings must not drive the aggregated status. + + When a group mixes one non-muted PASS with one muted FAIL, the + actionable outcome is PASS: there are no unmuted failures left. The + aggregated `status` must reflect that (not FAIL), while `muted` + stays False because the group still has a non-muted finding. + """ + tenant = tenants_fixture[0] + scan1, *_ = scans_fixture + resource1, *_ = resources_fixture + + pass_finding = Finding.objects.create( + tenant_id=tenant.id, + uid="fg_mixed_muted_pass", + scan=scan1, + delta=None, + status=Status.PASS, + severity=Severity.low, + impact=Severity.low, + check_id="mixed_muted_check", + check_metadata={ + "CheckId": "mixed_muted_check", + "checktitle": "Mixed muted check", + "Description": "Fixture for muted status aggregation.", + }, + first_seen_at="2024-01-11T00:00:00Z", + muted=False, + ) + pass_finding.add_resources([resource1]) + + fail_muted_finding = Finding.objects.create( + tenant_id=tenant.id, + uid="fg_mixed_muted_fail", + scan=scan1, + delta=None, + status=Status.FAIL, + severity=Severity.high, + impact=Severity.high, + check_id="mixed_muted_check", + check_metadata={ + "CheckId": "mixed_muted_check", + "checktitle": "Mixed muted check", + "Description": "Fixture for muted status aggregation.", + }, + first_seen_at="2024-01-12T00:00:00Z", + muted=True, + ) + fail_muted_finding.add_resources([resource1]) + + # filter[region] forces finding-level aggregation so we exercise the + # raw-findings path without touching the daily summary fixture. + response = authenticated_client.get( + reverse("finding-group-list"), + { + "filter[inserted_at]": TODAY, + "filter[check_id]": "mixed_muted_check", + "filter[region]": "us-east-1", + }, + ) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == 1 + attrs = data[0]["attributes"] + assert attrs["status"] == "PASS" + assert attrs["muted"] is False + assert attrs["pass_count"] == 1 + assert attrs["fail_count"] == 0 + assert attrs["fail_muted_count"] == 1 + assert attrs["muted_count"] == 1 + def test_finding_groups_status_filter( self, authenticated_client, finding_groups_fixture ): @@ -17271,3 +17348,111 @@ class TestFindingGroupViewSet: attrs = item["attributes"] assert "finding_id" in attrs assert attrs["finding_id"] in rds_finding_ids + + def test_latest_resources_picks_scan_by_completed_at_when_overlap( + self, + authenticated_client, + tenants_fixture, + providers_fixture, + resources_fixture, + ): + """Overlapping scans on the same provider must resolve to the scan + with the latest completed_at, matching the /latest summary path and + the daily-summary upsert (keyed on midnight(completed_at)). Picking + by inserted_at here produced /resources and /latest reading from + different scans and reporting diverging delta/new counts. + """ + tenant = tenants_fixture[0] + provider = providers_fixture[0] + resource = resources_fixture[0] + check_id = "overlap_regression_check" + + t0 = datetime.now(timezone.utc) - timedelta(hours=5) + t1 = t0 + timedelta(hours=1) + t1_end = t1 + timedelta(minutes=30) + t2 = t0 + timedelta(hours=4) + + scan_long = Scan.objects.create( + name="long overlap scan", + provider=provider, + trigger=Scan.TriggerChoices.MANUAL, + state=StateChoices.COMPLETED, + tenant_id=tenant.id, + started_at=t0, + completed_at=t2, + ) + scan_short = Scan.objects.create( + name="short overlap scan", + provider=provider, + trigger=Scan.TriggerChoices.MANUAL, + state=StateChoices.COMPLETED, + tenant_id=tenant.id, + started_at=t1, + completed_at=t1_end, + ) + # inserted_at is auto_now_add so override with .update() to recreate + # the overlap shape: short scan inserted later but completed earlier. + Scan.all_objects.filter(pk=scan_long.pk).update(inserted_at=t0) + Scan.all_objects.filter(pk=scan_short.pk).update(inserted_at=t1) + scan_long.refresh_from_db() + scan_short.refresh_from_db() + + assert scan_short.inserted_at > scan_long.inserted_at + assert scan_long.completed_at > scan_short.completed_at + + long_finding = Finding.objects.create( + tenant_id=tenant.id, + uid=f"{check_id}_long", + scan=scan_long, + delta=None, + status=Status.FAIL, + status_extended="long scan finding", + impact=Severity.high, + impact_extended="high", + severity=Severity.high, + raw_result={"status": Status.FAIL, "severity": Severity.high}, + check_id=check_id, + check_metadata={ + "CheckId": check_id, + "checktitle": "Overlap regression", + "Description": "Overlapping scan regression.", + }, + first_seen_at=t0, + muted=False, + ) + long_finding.add_resources([resource]) + + short_finding = Finding.objects.create( + tenant_id=tenant.id, + uid=f"{check_id}_short", + scan=scan_short, + delta="new", + status=Status.FAIL, + status_extended="short scan finding", + impact=Severity.high, + impact_extended="high", + severity=Severity.high, + raw_result={"status": Status.FAIL, "severity": Severity.high}, + check_id=check_id, + check_metadata={ + "CheckId": check_id, + "checktitle": "Overlap regression", + "Description": "Overlapping scan regression.", + }, + first_seen_at=t1, + muted=False, + ) + short_finding.add_resources([resource]) + + response = authenticated_client.get( + reverse( + "finding-group-latest_resources", + kwargs={"check_id": check_id}, + ), + ) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == 1 + attrs = data[0]["attributes"] + assert attrs["finding_id"] == str(long_finding.id) + assert attrs["delta"] is None diff --git a/api/src/backend/api/v1/views.py b/api/src/backend/api/v1/views.py index d9b81e5156..2f74d0283d 100644 --- a/api/src/backend/api/v1/views.py +++ b/api/src/backend/api/v1/views.py @@ -7281,14 +7281,18 @@ class FindingGroupViewSet(BaseRLSViewSet): # finding-level aggregation path. row.pop("nonmuted_count", None) - # Compute aggregated status from non-muted counts first, then - # fall back to muted counts so fully-muted groups still reflect - # the underlying check outcome. - total_fail = row.get("fail_count", 0) + row.get("fail_muted_count", 0) - total_pass = row.get("pass_count", 0) + row.get("pass_muted_count", 0) - if total_fail > 0: + # Muted findings are treated as resolved/accepted, so they do not + # contribute to a failing status. A group is FAIL only when there + # is at least one non-muted FAIL; otherwise any pass (muted or + # not) or any muted fail makes the group PASS. Only groups whose + # findings are exclusively MANUAL fall through to MANUAL. + if row.get("fail_count", 0) > 0: row["status"] = "FAIL" - elif total_pass > 0: + elif ( + row.get("pass_count", 0) > 0 + or row.get("pass_muted_count", 0) > 0 + or row.get("fail_muted_count", 0) > 0 + ): row["status"] = "PASS" else: row["status"] = "MANUAL" @@ -7388,12 +7392,11 @@ class FindingGroupViewSet(BaseRLSViewSet): if computed_params.get("status") or computed_params.getlist("status__in"): queryset = queryset.annotate( - total_fail=F("fail_count") + F("fail_muted_count"), - total_pass=F("pass_count") + F("pass_muted_count"), - ).annotate( aggregated_status=Case( - When(total_fail__gt=0, then=Value("FAIL")), - When(total_pass__gt=0, then=Value("PASS")), + When(fail_count__gt=0, then=Value("FAIL")), + When(pass_count__gt=0, then=Value("PASS")), + When(pass_muted_count__gt=0, then=Value("PASS")), + When(fail_muted_count__gt=0, then=Value("PASS")), default=Value("MANUAL"), output_field=CharField(), ) @@ -7413,6 +7416,25 @@ class FindingGroupViewSet(BaseRLSViewSet): return filterset.qs + def _resolve_finding_ids(self, filtered_queryset): + """ + Materialize and request-cache the finding_ids list used to anchor + RFM lookups. + + Turning `finding_id__in=Subquery(findings_qs)` into `finding_id__in= + [uuid, ...]` nudges PostgreSQL out of a Merge Semi Join that ends up + reading hundreds of thousands of RFM index entries just to post- + filter tenant_id. Caching on the ViewSet instance (one instance per + request) avoids duplicating the findings round-trip when several + helpers build different RFM querysets from the same filtered set. + """ + cached = getattr(self, "_finding_ids_cache", None) + if cached is not None and cached[0] is filtered_queryset: + return cached[1] + finding_ids = list(filtered_queryset.order_by().values_list("id", flat=True)) + self._finding_ids_cache = (filtered_queryset, finding_ids) + return finding_ids + def _build_resource_mapping_queryset( self, filtered_queryset, resource_ids=None, tenant_id: str | None = None ): @@ -7422,10 +7444,10 @@ class FindingGroupViewSet(BaseRLSViewSet): Starting from ResourceFindingMapping avoids scanning all mappings before applying check_id/date filters on findings. """ - finding_ids = filtered_queryset.order_by().values("id") + finding_ids = self._resolve_finding_ids(filtered_queryset) mapping_queryset = ResourceFindingMapping.objects.filter( - finding_id__in=Subquery(finding_ids) + finding_id__in=finding_ids ) if tenant_id: mapping_queryset = mapping_queryset.filter(tenant_id=tenant_id) @@ -7779,12 +7801,11 @@ class FindingGroupViewSet(BaseRLSViewSet): if ordering: if any(field.lstrip("-") == "status_order" for field in ordering): aggregated_queryset = aggregated_queryset.annotate( - total_fail_for_sort=F("fail_count") + F("fail_muted_count"), - total_pass_for_sort=F("pass_count") + F("pass_muted_count"), - ).annotate( status_order=Case( - When(total_fail_for_sort__gt=0, then=Value(3)), - When(total_pass_for_sort__gt=0, then=Value(2)), + When(fail_count__gt=0, then=Value(3)), + When(pass_count__gt=0, then=Value(2)), + When(pass_muted_count__gt=0, then=Value(2)), + When(fail_muted_count__gt=0, then=Value(2)), default=Value(1), output_field=IntegerField(), ) @@ -7845,23 +7866,24 @@ class FindingGroupViewSet(BaseRLSViewSet): request, filtered_queryset, resource_ids, tenant_id, ordering ) - has_mappings = self._build_resource_mapping_queryset( - filtered_queryset, resource_ids=None, tenant_id=tenant_id - ).exists() + # Serve the mapping response directly and piggyback on the paginator + # count to detect orphan-only groups, instead of paying a separate + # has_mappings.exists() semi-join over ResourceFindingMapping on + # every non-IaC request. TODO: once the ephemeral resources strategy + # is decided, mixed groups should route to _combined_paginated_response. + response = self._mapping_paginated_response( + request, filtered_queryset, resource_ids, tenant_id, ordering + ) - if has_mappings: - # Normal or mixed group: serve only resource-mapped rows. - # TODO: Orphan findings in mixed groups are intentionally excluded - # until the ephemeral resources strategy is decided. When resolved, - # route mixed groups to _combined_paginated_response instead. - return self._mapping_paginated_response( - request, filtered_queryset, resource_ids, tenant_id, ordering + page = getattr(self.paginator, "page", None) + mapping_total = page.paginator.count if page is not None else None + if mapping_total == 0: + # Pure orphan group (e.g. IaC): synthesize resource-like rows. + return self._combined_paginated_response( + request, filtered_queryset, tenant_id, ordering ) - # Pure orphan group (e.g. IaC): synthesize resource-like rows. - return self._combined_paginated_response( - request, filtered_queryset, tenant_id, ordering - ) + return response def _mapping_paginated_response( self, request, filtered_queryset, resource_ids, tenant_id, ordering @@ -8145,10 +8167,13 @@ class FindingGroupViewSet(BaseRLSViewSet): tenant_id = request.tenant_id queryset = self._get_finding_queryset() - # Get latest completed scan for each provider + # Order by -completed_at (matching the /latest summary path and the + # daily summary upsert keyed on midnight(completed_at)) so that + # overlapping scans do not make /resources and /latest read from + # different scans and report diverging counts. latest_scan_ids = ( Scan.objects.filter(tenant_id=tenant_id, state=StateChoices.COMPLETED) - .order_by("provider_id", "-inserted_at") + .order_by("provider_id", "-completed_at", "-inserted_at") .distinct("provider_id") .values_list("id", flat=True) ) diff --git a/api/src/backend/tasks/jobs/scan.py b/api/src/backend/tasks/jobs/scan.py index 97eba7cd4f..456fd786cb 100644 --- a/api/src/backend/tasks/jobs/scan.py +++ b/api/src/backend/tasks/jobs/scan.py @@ -752,11 +752,19 @@ def _process_finding_micro_batch( ) if mappings_to_create: - ResourceFindingMapping.objects.bulk_create( + created_mappings = ResourceFindingMapping.objects.bulk_create( mappings_to_create, batch_size=SCAN_DB_BATCH_SIZE, ignore_conflicts=True, + unique_fields=["tenant_id", "resource_id", "finding_id"], ) + inserted = sum(1 for m in created_mappings if m.pk) + if inserted != len(mappings_to_create): + logger.error( + f"scan {scan_instance.id}: expected " + f"{len(mappings_to_create)} ResourceFindingMapping rows, " + f"inserted {inserted}. Rolling back micro-batch." + ) # Update finding denormalized arrays findings_to_update = [] diff --git a/contrib/reverse-proxy/README.md b/contrib/reverse-proxy/README.md new file mode 100644 index 0000000000..a6387e689a --- /dev/null +++ b/contrib/reverse-proxy/README.md @@ -0,0 +1,64 @@ +# Prowler Reverse Proxy Configuration + +Ready-to-use nginx configuration for running Prowler behind a reverse proxy. + +## Problem + +Prowler's default Docker setup exposes two separate services: +- **UI** on port 3000 +- **API** on port 8080 + +This causes CORS issues and authentication failures (especially SAML SSO) when accessed through an external reverse proxy, since the proxy typically exposes a single domain. + +## Solution + +This adds an nginx container that unifies both services behind a single port, correctly forwarding headers so that Django generates proper URLs for SAML ACS callbacks and API responses. + +## Quick Start + +From the prowler root directory: + + docker compose -f docker-compose.yml \ + -f contrib/reverse-proxy/docker-compose.reverse-proxy.yml \ + up -d + +Access Prowler at http://localhost (port 80). + +## With an External Reverse Proxy + +Point your external reverse proxy to the prowler-nginx container on port 80. + +### Environment Variables + +| Variable | Default | Description | +|----------|---------|-------------| +| PROWLER_PROXY_PORT | 80 | Port exposed by the nginx proxy | + +### Example: Traefik + + services: + nginx: + labels: + - "traefik.enable=true" + - "traefik.http.routers.prowler.rule=Host(`prowler.example.com`)" + - "traefik.http.routers.prowler.tls.certresolver=letsencrypt" + - "traefik.http.services.prowler.loadbalancer.server.port=80" + +### Example: Caddy + + prowler.example.com { + reverse_proxy prowler-nginx:80 + } + +## SAML SSO + +If using SAML SSO behind a reverse proxy, also set the SAML_ACS_BASE_URL environment variable: + + SAML_ACS_BASE_URL=https://prowler.example.com + +## Architecture + + Internet -> External Reverse Proxy -> prowler-nginx:80 + |-- /api/* -> prowler-api:8080 + |-- /accounts/saml/ -> prowler-api:8080 + +-- /* -> prowler-ui:3000 diff --git a/contrib/reverse-proxy/docker-compose.reverse-proxy.yml b/contrib/reverse-proxy/docker-compose.reverse-proxy.yml new file mode 100644 index 0000000000..08c52f3558 --- /dev/null +++ b/contrib/reverse-proxy/docker-compose.reverse-proxy.yml @@ -0,0 +1,42 @@ +# Prowler Reverse Proxy - Docker Compose Override +# +# Use this alongside the main docker-compose.yml to add an nginx +# reverse proxy that unifies UI and API behind a single port. +# +# Usage: +# docker compose -f docker-compose.yml -f contrib/reverse-proxy/docker-compose.reverse-proxy.yml up -d +# +# Then access Prowler at http://localhost (port 80) or configure +# your external reverse proxy (Traefik, Caddy, Cloudflare Tunnel, +# Pangolin, etc.) to point to this container on port 80. +# +# For HTTPS with your own certs, see the README in this directory. +# +# Fixes: https://github.com/prowler-cloud/prowler/issues/8516 + +services: + nginx: + image: nginx:alpine + container_name: prowler-nginx + restart: unless-stopped + ports: + - "${PROWLER_PROXY_PORT:-80}:80" + volumes: + - ./contrib/reverse-proxy/nginx.conf:/etc/nginx/conf.d/default.conf:ro + depends_on: + - prowler-ui + - prowler-api + networks: + - prowler-network + + # Override UI to not expose port externally (nginx handles it) + prowler-ui: + ports: !reset [] + + # Override API to not expose port externally (nginx handles it) + prowler-api: + ports: !reset [] + +networks: + prowler-network: + driver: bridge diff --git a/contrib/reverse-proxy/nginx.conf b/contrib/reverse-proxy/nginx.conf new file mode 100644 index 0000000000..58520295bc --- /dev/null +++ b/contrib/reverse-proxy/nginx.conf @@ -0,0 +1,70 @@ +# Prowler Reverse Proxy Configuration +# Routes both UI and API through a single endpoint +# +# Usage: See docker-compose.reverse-proxy.yml +# Fixes: https://github.com/prowler-cloud/prowler/issues/8516 + +upstream prowler-ui { + server prowler-ui:3000; +} + +upstream prowler-api { + server prowler-api:8080; +} + +server { + listen 80; + server_name _; + + # Security headers + add_header X-Content-Type-Options "nosniff" always; + add_header X-Frame-Options "SAMEORIGIN" always; + add_header Referrer-Policy "strict-origin-when-cross-origin" always; + + # API requests β€” proxy to prowler-api + location /api/ { + proxy_pass http://prowler-api/api/; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Host $host; + proxy_read_timeout 300s; + proxy_connect_timeout 10s; + + # Handle large scan payloads + client_max_body_size 50m; + } + + # SAML endpoints β€” proxy to prowler-api + location /accounts/saml/ { + proxy_pass http://prowler-api/accounts/saml/; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Host $host; + } + + # Everything else β€” proxy to prowler-ui + location / { + proxy_pass http://prowler-ui/; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Host $host; + + # WebSocket support for Next.js HMR (dev) and live updates + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } + + # Health check endpoint + location /health { + access_log off; + return 200 "ok\n"; + add_header Content-Type text/plain; + } +} diff --git a/docs/user-guide/providers/aws/authentication.mdx b/docs/user-guide/providers/aws/authentication.mdx index b919cde8ec..22deb6ab99 100644 --- a/docs/user-guide/providers/aws/authentication.mdx +++ b/docs/user-guide/providers/aws/authentication.mdx @@ -7,6 +7,11 @@ Prowler requires AWS credentials to function properly. Authentication is availab - Static Credentials - Assumed Role +When using **Assumed Role**, the Prowler UI exposes two credential sources for calling `sts:AssumeRole`. The labels differ between Prowler Cloud and self-hosted Prowler App, but both map to the same underlying credential types: + +- **AWS SDK Default** (shown as *"Prowler Cloud will assume your IAM role"* in Prowler Cloud and *"AWS SDK Default"* in self-hosted Prowler App): Prowler uses the credentials already available to the API and worker containers through the [AWS SDK default credential chain](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html). This is the default in Prowler Cloud and requires extra configuration in self-hosted Prowler App (see [Configuring AWS SDK Default for Self-Hosted Prowler App](#configuring-aws-sdk-default-for-self-hosted-prowler-app)). +- **Access & Secret Key**: You paste an IAM user's `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and optionally `AWS_SESSION_TOKEN` into the form. Prowler uses those keys to call `sts:AssumeRole`. + ## Required Permissions To ensure full functionality, attach the following AWS managed policies to the designated user or role: @@ -76,6 +81,68 @@ This method grants permanent access and is the recommended setup for production --- +## Configuring AWS SDK Default for Self-Hosted Prowler App + +When self-hosting Prowler App with Docker Compose, the API and worker containers do not have AWS credentials by default. Selecting **AWS SDK Default** without configuring those credentials produces: + +``` +AWSAssumeRoleError[1012]: AWS assume role error - An error occurred (InvalidClientTokenId) when calling the AssumeRole operation: The security token included in the request is invalid. +``` + +To fix this, expose an IAM identity with `sts:AssumeRole` permission on the target role to both the `api` and `worker` services. + +### Option 1: Environment Variables in `.env` + +Add the following keys to the `.env` file used by `docker-compose.yml`: + +```bash +AWS_ACCESS_KEY_ID="" +AWS_SECRET_ACCESS_KEY="" +AWS_SESSION_TOKEN="" +AWS_DEFAULT_REGION="us-east-1" +``` + +The existing `docker-compose.yml` already loads `.env` into the `api`, `worker`, and `worker-beat` services, so `boto3` will pick them up through the default credential chain. + + +Treat the `.env` file as a secret. Do not commit it to version control, scope the IAM identity to the minimum permissions required (`sts:AssumeRole` on the target `ProwlerScan` role only), prefer short-lived credentials over long-lived access keys, and rotate the keys immediately if you suspect exposure. + + +Recreate the containers to apply the change. A plain `docker compose restart` will **not** reload values from a modified `.env` file β€” you must force-recreate: + +```bash +docker compose up -d --force-recreate api worker worker-beat +``` + +### Option 2: IAM Role (Host with Instance Metadata) + +If you run Prowler App on an EC2 instance, ECS task, or EKS pod with an attached IAM role that can assume the scan role, no extra configuration is needed β€” `boto3` resolves credentials through instance or task metadata automatically. + +### Trust Policy: Align `IAMPrincipal` With Your Identity + +The [Prowler scan role CloudFormation template](https://github.com/prowler-cloud/prowler/blob/master/permissions/templates/cloudformation/prowler-scan-role.yml) restricts the trust policy with: + +``` +aws:PrincipalArn StringLike arn:aws:iam::: +``` + +`IAMPrincipal` defaults to `role/prowler*`, which only allows IAM roles whose name starts with `prowler`. If the identity hosting the API and worker containers is anything else, the `sts:AssumeRole` call fails with `AccessDenied` even when the credentials themselves are valid. + +Redeploy (or update) the CloudFormation stack with an `IAMPrincipal` that matches your identity: + +| Your identity on the API/worker containers | `IAMPrincipal` value | +| --- | --- | +| IAM user (for example `prowler-app`) | `user/prowler-app` | +| IAM role whose name doesn't start with `prowler` | `role/` | + +`AccountId` must also point to the account where that identity lives β€” the default is Prowler Cloud's account and only applies when assuming from Prowler Cloud. + + +The same `External ID` entered in the Prowler UI must match the `ExternalId` parameter used when deploying the CloudFormation stack. A mismatch produces `AccessDenied` on `sts:AssumeRole`, not `InvalidClientTokenId`. + + +--- + ## Credentials diff --git a/docs/user-guide/providers/aws/getting-started-aws.mdx b/docs/user-guide/providers/aws/getting-started-aws.mdx index 2c94700b15..8127e873e7 100644 --- a/docs/user-guide/providers/aws/getting-started-aws.mdx +++ b/docs/user-guide/providers/aws/getting-started-aws.mdx @@ -46,15 +46,15 @@ Before proceeding, choose the preferred authentication mode: **Credentials** -* Quick scan as current user -* No extra setup -* Credentials time out +* Quick scan using an IAM user's access keys +* No extra setup in AWS +* Static keys can be rotated or revoked at any time **Assumed Role** -* Preferred Setup -* Permanent Credentials -* Requires access to create role +* Recommended for production +* With AWS SDK Default as the credential source, no long-lived keys are stored in Prowler (Access & Secret Key still requires pasted keys) +* Requires permission to create an IAM role in the target account --- @@ -67,18 +67,23 @@ This method grants permanent access and is the recommended setup for production For detailed instructions on how to create the role, see [Authentication > Assume Role](/user-guide/providers/aws/authentication#assume-role-recommended). -8. Once the role is created, go to the **IAM Console**, click on the "ProwlerScan" role to open its details: +7. Once the role is created, go to the **IAM Console**, click on the "ProwlerScan" role to open its details: ![ProwlerScan role info](/images/providers/prowler-scan-pre-info.png) -9. Copy the **Role ARN** +8. Copy the **Role ARN** ![New Role Info](/images/providers/get-role-arn.png) -10. Paste the ARN into the corresponding field in Prowler Cloud or Prowler App +9. Paste the ARN into the corresponding field in Prowler Cloud or Prowler App ![Input the Role ARN](/images/providers/paste-role-arn-prowler.png) +10. Select the credential source Prowler should use to call `sts:AssumeRole`. The option label differs between deployments but both map to the same `aws-sdk-default` credential type: + + - **"Prowler Cloud will assume your IAM role"** (default in Prowler Cloud) / **"AWS SDK Default"** (in self-hosted Prowler App): Prowler uses the credentials available in the API and worker environment through the [AWS SDK default credential chain](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html). In self-hosted Prowler App, these containers have no AWS credentials by default β€” see [Configuring AWS SDK Default for Self-Hosted Prowler App](/user-guide/providers/aws/authentication#configuring-aws-sdk-default-for-self-hosted-prowler-app) before choosing this option, or the connection test will fail with `InvalidClientTokenId`. + - **Access & Secret Key**: Paste an IAM user's `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` (and optional `AWS_SESSION_TOKEN`) into the form. The IAM principal must be allowed to assume the target role and must match the `IAMPrincipal` parameter of the scan role template (default: `role/prowler*`). + 11. Click "Next", then "Launch Scan" ![Next button in Prowler Cloud](/images/providers/next-button-prowler-cloud.png) diff --git a/docs/user-guide/tutorials/prowler-ctf-breach-investigation.mdx b/docs/user-guide/tutorials/prowler-ctf-breach-investigation.mdx deleted file mode 100644 index b4124f5325..0000000000 --- a/docs/user-guide/tutorials/prowler-ctf-breach-investigation.mdx +++ /dev/null @@ -1,59 +0,0 @@ ---- -title: 'Prowler CTF - Breach Investigation' -description: 'Investigate a simulated AWS data exfiltration case with Prowler Cloud and recover three flags.' ---- - -## Background - -Astoneta Corp's security team received an alert: someone has been exfiltrating data from an AWS account. Initial triage suggests that a threat actor found an entry point through a misconfigured storage resource, used leaked credentials to escalate privileges, and launched compute infrastructure to extract data. - -Use **Prowler** to scan the AWS account, investigate the findings, and uncover three flags hidden across the attack chain. - -## Getting Started - -You have been given a set of AWS credentials with read-only access. Use **Prowler Cloud** to connect the target account and run a scan: - -1. Log in to [Prowler Cloud](https://cloud.prowler.com). -2. Add a new **AWS provider** using **Static Access Keys**. Follow the [setup guide](https://docs.prowler.com/user-guide/providers/aws/getting-started-aws#credentials-static-access-keys). -3. Enter the provided `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`. -4. Launch a scan and wait for it to complete. - -Once the scan finishes, examine the findings to trace the threat actor's steps. - -## Challenge 1 - The Entry Point - -> "Every breach starts somewhere. The attacker walked in through a door that should never have been open." - -An S3 bucket has been left publicly accessible. Anyone on the internet can read its contents, and the threat actor did exactly that. - -Find the misconfigured bucket in the Prowler findings and inspect its tags. The flag is the **base64 encoding of the value of the `CaseId` tag** on the vulnerable bucket. - -**Hint:** Look for findings related to S3 public access. Every resource in this environment carries a `CaseId` tag, but only the vulnerable bucket's value is the correct flag. Do not take the first `CaseId` you find. Use Prowler's findings to identify which bucket is misconfigured, then review its tags. In Prowler Cloud, resource tags are visible in the metadata view. - -## Challenge 2 - Compromised Identity - -> "Inside the bucket, the attacker found what every intruder dreams of: admin credentials with no second factor." - -The leaked credentials belong to an IAM user with console access and **no MFA enabled**, holding administrator-level permissions. - -Find the user flagged by Prowler for having console access without MFA. The flag is the **MD5 hash of the username**. - -**Hint:** Not every IAM user in the account is vulnerable. Some exist only for programmatic access and will not trigger this finding. - -## Challenge 3 - The Exfiltration Node - -> "With admin access secured, the attacker launched an EC2 instance wide open to the internet - their staging ground for data exfiltration." - -A security group allows inbound traffic from `0.0.0.0/0` on multiple sensitive ports. An EC2 instance uses this security group to run an exfiltration node. - -The flag is hidden in one of the instance's **tags**, but the instance has many tags and only one holds the flag. The `CaseId` tag is **not** the answer this time. - -To determine which tag matters, think through the exposed services. The flag is the **base64 encoding of the value of the correct tag**. - -**Hint:** The security group allows HTTP traffic. Try accessing the instance. It may point to the tag that matters. The page itself is not the flag. In Prowler Cloud, resource tags are visible in the metadata view. - -## Submission - -Submit all three flags. - -Good luck, investigator. diff --git a/prowler/CHANGELOG.md b/prowler/CHANGELOG.md index a4b292b9dd..eb7905727e 100644 --- a/prowler/CHANGELOG.md +++ b/prowler/CHANGELOG.md @@ -2,17 +2,31 @@ All notable changes to the **Prowler SDK** are documented in this file. -## [5.24.1] (Prowler UNRELEASED) +## [5.24.3] (Prowler v5.24.3) + +### 🐞 Fixed + +- CloudTrail resource timeline uses resource name as fallback in `LookupEvents` [(#10828)](https://github.com/prowler-cloud/prowler/pull/10828) + +--- + +## [5.24.1] (Prowler v5.24.1) + +### πŸš€ Added + +- `--repo-list-file` CLI flag for GitHub provider to load repositories from a file [(#10501)](https://github.com/prowler-cloud/prowler/pull/10501) ### πŸ”„ Changed -- bumped `msgraph-sdk` from 1.23.0 to 1.55.0 and `azure-mgmt-resource` from 23.3.0 to 24.0.0, removing `marshmallow` as is a transitively dev dependency [(#10733)](https://github.com/prowler-cloud/prowler/pull/10733) +- `msgraph-sdk` from 1.23.0 to 1.55.0 and `azure-mgmt-resource` from 23.3.0 to 24.0.0, removing `marshmallow` as is a transitively dev dependency [(#10733)](https://github.com/prowler-cloud/prowler/pull/10733) ### 🐞 Fixed - Cloudflare account-scoped API tokens failing connection test in the App with `CloudflareUserTokenRequiredError` [(#10723)](https://github.com/prowler-cloud/prowler/pull/10723) -- `prowler image --registry` failing with `ImageNoImagesProvidedError` due to registry arguments not being forwarded to `ImageProvider` in `init_global_provider` [(#10470)](https://github.com/prowler-cloud/prowler/pull/10470) +- `prowler image --registry-list` crashes with `AttributeError` because `ImageProvider.__init__` returns early before registering the global provider [(#10691)](https://github.com/prowler-cloud/prowler/pull/10691) - Google Workspace Calendar checks false FAIL on unconfigured settings with secure Google defaults [(#10726)](https://github.com/prowler-cloud/prowler/pull/10726) +- Google Workspace Drive checks false FAIL on unconfigured settings with secure Google defaults [(#10727)](https://github.com/prowler-cloud/prowler/pull/10727) +- Cloudflare `validate_credentials` can hang in an infinite pagination loop when the SDK repeats accounts, blocking connection tests [(#10771)](https://github.com/prowler-cloud/prowler/pull/10771) --- @@ -42,7 +56,6 @@ All notable changes to the **Prowler SDK** are documented in this file. - `prowler image --registry-list` crashes with `AttributeError` because `ImageProvider.__init__` returns early before registering the global provider [(#10691)](https://github.com/prowler-cloud/prowler/pull/10691) - Vercel firewall config handling for team-scoped projects and current API response shapes [(#10695)](https://github.com/prowler-cloud/prowler/pull/10695) -- Google Workspace Drive checks false FAIL on unconfigured settings with secure Google defaults [(#10727)](https://github.com/prowler-cloud/prowler/pull/10727) --- @@ -93,6 +106,7 @@ All notable changes to the **Prowler SDK** are documented in this file. - Oracle Cloud `kms_key_rotation_enabled` now checks current key version age to avoid false positives on vaults without auto-rotation support [(#10450)](https://github.com/prowler-cloud/prowler/pull/10450) - OCI filestorage, blockstorage, KMS, and compute services now honor `--region` for scanning outside the tenancy home region [(#10472)](https://github.com/prowler-cloud/prowler/pull/10472) - OCI provider now supports multi-region filtering via `--region` [(#10473)](https://github.com/prowler-cloud/prowler/pull/10473) +- `prowler image --registry` failing with `ImageNoImagesProvidedError` due to registry arguments not being forwarded to `ImageProvider` in `init_global_provider` [(#10470)](https://github.com/prowler-cloud/prowler/pull/10470) - OCI multi-region support for identity client configuration in blockstorage, identity, and filestorage services [(#10520)](https://github.com/prowler-cloud/prowler/pull/10520) - Google Workspace Calendar checks now filter for customer-level policies only, skipping OU and group overrides that could produce incorrect audit results [(#10658)](https://github.com/prowler-cloud/prowler/pull/10658) diff --git a/prowler/lib/check/compliance_models.py b/prowler/lib/check/compliance_models.py index d1f3b8c35d..136b49f83b 100644 --- a/prowler/lib/check/compliance_models.py +++ b/prowler/lib/check/compliance_models.py @@ -1,9 +1,10 @@ +import json import os import sys from enum import Enum from typing import Optional, Union -from pydantic.v1 import BaseModel, ValidationError, root_validator +from pydantic.v1 import BaseModel, Field, ValidationError, root_validator from prowler.lib.check.utils import list_compliance_modules from prowler.lib.logger import logger @@ -430,3 +431,482 @@ def load_compliance_framework( sys.exit(1) else: return compliance_framework + + +# ─── Universal Compliance Schema Models (Phase 1-3) ───────────────────────── + + +class OutputFormats(BaseModel): + """Flags indicating in which output formats an attribute should be included.""" + + csv: bool = True + ocsf: bool = True + + +class AttributeMetadata(BaseModel): + """Schema descriptor for a single attribute field in a universal compliance framework.""" + + key: str + label: Optional[str] = None + type: str = "str" # str, int, float, list_str, list_dict, bool + enum: Optional[list] = None + required: bool = False + enum_display: Optional[dict] = None # enum_value -> EnumValueDisplay dict + enum_order: Optional[list] = None # explicit ordering of enum values + chart_label: Optional[str] = None # axis label when used in charts + output_formats: OutputFormats = Field(default_factory=OutputFormats) + + +class SplitByConfig(BaseModel): + """Column-splitting configuration (e.g. CIS Level 1/Level 2).""" + + field: str + values: list + + +class ScoringConfig(BaseModel): + """Weighted scoring configuration (e.g. ThreatScore).""" + + risk_field: str + weight_field: str + + +class TableLabels(BaseModel): + """Custom pass/fail labels for console table rendering.""" + + pass_label: str = "PASS" + fail_label: str = "FAIL" + provider_header: str = "Provider" + group_header: Optional[str] = None + status_header: str = "Status" + title: Optional[str] = None + results_title: Optional[str] = None + footer_note: Optional[str] = None + + +class TableConfig(BaseModel): + """Declarative rendering instructions for the console compliance table.""" + + group_by: str + split_by: Optional[SplitByConfig] = None + scoring: Optional[ScoringConfig] = None + labels: Optional[TableLabels] = None + + +class EnumValueDisplay(BaseModel): + """Per-enum-value visual metadata for PDF rendering. + + Replaces hardcoded DIMENSION_MAPPING, TIPO_ICONS, nivel colors. + """ + + label: Optional[str] = None # "Trazabilidad" + abbreviation: Optional[str] = None # "T" + color: Optional[str] = None # "#4286F4" + icon: Optional[str] = None # emoji + + +class ChartConfig(BaseModel): + """Declarative chart description for PDF reports.""" + + id: str + type: str # vertical_bar | horizontal_bar | radar + group_by: str # attribute key to group by + title: Optional[str] = None + x_label: Optional[str] = None + y_label: Optional[str] = None + value_source: str = "compliance_percent" + color_mode: str = "by_value" # by_value | fixed | by_group + fixed_color: Optional[str] = None + + +class ScoringFormula(BaseModel): + """Weighted scoring formula (e.g. ThreatScore).""" + + risk_field: str # "LevelOfRisk" + weight_field: str # "Weight" + risk_boost_factor: float = 0.25 # rfac = 1 + factor * risk_level + + +class CriticalRequirementsFilter(BaseModel): + """Filter for critical requirements section in PDF reports.""" + + filter_field: str # "LevelOfRisk" + min_value: Optional[int] = None # 4 (int-based filter) + filter_value: Optional[str] = None # "alto" (string-based filter) + status_filter: str = "FAIL" + title: Optional[str] = None # "Critical Failed Requirements" + + +class ReportFilter(BaseModel): + """Default report filtering for PDF generation.""" + + only_failed: bool = True + include_manual: bool = False + + +class I18nLabels(BaseModel): + """Localized labels for PDF report rendering.""" + + report_title: Optional[str] = None + page_label: str = "Page" + powered_by: str = "Powered by Prowler" + framework_label: str = "Framework:" + version_label: str = "Version:" + provider_label: str = "Provider:" + description_label: str = "Description:" + compliance_score_label: str = "Compliance Score by Sections" + requirements_index_label: str = "Requirements Index" + detailed_findings_label: str = "Detailed Findings" + + +class PDFConfig(BaseModel): + """Declarative PDF report configuration. + + Drives the API report generator from JSON data instead of hardcoded + Python config. Colors are hex strings (e.g. '#336699'). + """ + + language: str = "en" + logo_filename: Optional[str] = None + primary_color: Optional[str] = None + secondary_color: Optional[str] = None + bg_color: Optional[str] = None + sections: Optional[list] = None + section_short_names: Optional[dict] = None + group_by_field: Optional[str] = None + sub_group_by_field: Optional[str] = None + section_titles: Optional[dict] = None + charts: Optional[list] = None + scoring: Optional[ScoringFormula] = None + critical_filter: Optional[CriticalRequirementsFilter] = None + filter: Optional[ReportFilter] = None + labels: Optional[I18nLabels] = None + + +class UniversalComplianceRequirement(BaseModel): + """Universal requirement with flat dict-based attributes.""" + + id: str + description: str + name: Optional[str] = None + attributes: dict = Field(default_factory=dict) + checks: dict[str, list[str]] = Field(default_factory=dict) + tactics: Optional[list] = None + sub_techniques: Optional[list] = None + platforms: Optional[list] = None + technique_url: Optional[str] = None + + +class OutputsConfig(BaseModel): + """Container for output-related configuration (table, PDF, etc.).""" + + table_config: Optional[TableConfig] = None + pdf_config: Optional[PDFConfig] = None + + +class ComplianceFramework(BaseModel): + """Universal top-level container for any compliance framework. + + Provider may be explicit (single-provider JSON) or derived from checks + keys across all requirements. + """ + + framework: str + name: str + provider: Optional[str] = None + version: Optional[str] = None + description: str + icon: Optional[str] = None + requirements: list[UniversalComplianceRequirement] + attributes_metadata: Optional[list[AttributeMetadata]] = None + outputs: Optional[OutputsConfig] = None + + @root_validator + # noqa: F841 - since vulture raises unused variable 'cls' + def validate_attributes_against_metadata(cls, values): # noqa: F841 + """Validate every Requirement's attributes dict against attributes_metadata. + + Checks: + - Required keys (required=True) must be present in each Requirement. + - Enum-constrained keys must have a value within the declared enum list. + - Basic type validation (int, float, bool) for non-None values. + """ + metadata = values.get("attributes_metadata") + requirements = values.get("requirements", []) + if not metadata: + return values + + required_keys = {m.key for m in metadata if m.required} + valid_keys = {m.key for m in metadata} + enum_map = {m.key: m.enum for m in metadata if m.enum} + type_map = {m.key: m.type for m in metadata} + + type_checks = { + "int": int, + "float": (int, float), + "bool": bool, + } + + errors = [] + for req in requirements: + attrs = req.attributes + + # Required keys + for key in required_keys: + if key not in attrs or attrs[key] is None: + errors.append( + f"Requirement '{req.id}': missing required attribute '{key}'" + ) + + # Unknown keys β€” anything outside the declared schema is a typo or drift + unknown_keys = set(attrs) - valid_keys + for key in sorted(unknown_keys): + errors.append( + f"Requirement '{req.id}': unknown attribute '{key}' " + f"(not declared in attributes_metadata)" + ) + + # Enum validation + for key, allowed in enum_map.items(): + if key in attrs and attrs[key] is not None: + if attrs[key] not in allowed: + errors.append( + f"Requirement '{req.id}': attribute '{key}' value " + f"'{attrs[key]}' not in {allowed}" + ) + + # Type validation for non-string types + for key in attrs: + if key not in valid_keys or attrs[key] is None: + continue + expected_type = type_map.get(key, "str") + py_type = type_checks.get(expected_type) + if py_type and not isinstance(attrs[key], py_type): + errors.append( + f"Requirement '{req.id}': attribute '{key}' expected " + f"type {expected_type}, got {type(attrs[key]).__name__}" + ) + + if errors: + detail = "\n ".join(errors) + raise ValueError(f"attributes_metadata validation failed:\n {detail}") + + return values + + def get_providers(self) -> list: + """Derive the set of providers this framework supports. + + Inspects checks keys across all requirements. Falls back to the + explicit provider field for single-provider frameworks with no + requirement-level checks. + """ + providers = set() + for req in self.requirements: + providers.update(k.lower() for k in req.checks.keys()) + if self.provider and not providers: + providers.add(self.provider.lower()) + return sorted(providers) + + def supports_provider(self, provider: str) -> bool: + """Return True if this framework has checks for the given provider.""" + provider_lower = provider.lower() + for req in self.requirements: + if any(k.lower() == provider_lower for k in req.checks.keys()): + return True + return self.provider is not None and self.provider.lower() == provider_lower + + +# ─── Legacy-to-Universal Adapter (Phase 2) ────────────────────────────────── + + +def _infer_attribute_metadata(legacy: Compliance) -> Optional[list[AttributeMetadata]]: + """Introspect the first requirement's attribute model to build attributes_metadata.""" + try: + if not legacy.Requirements: + return None + + first_req = legacy.Requirements[0] + + # MITRE requirements have Tactics at top level, not in Attributes + if isinstance(first_req, Mitre_Requirement): + return None + + if not first_req.Attributes: + return None + + sample_attr = first_req.Attributes[0] + metadata = [] + + for field_name, field_obj in sample_attr.__fields__.items(): + field_type = field_obj.outer_type_ + type_str = "str" + enum_values = None + + origin = getattr(field_type, "__origin__", None) + if field_type is int: + type_str = "int" + elif field_type is float: + type_str = "float" + elif field_type is bool: + type_str = "bool" + elif origin is list: + args = getattr(field_type, "__args__", ()) + if args and args[0] is dict: + type_str = "list_dict" + else: + type_str = "list_str" + elif isinstance(field_type, type) and issubclass(field_type, Enum): + type_str = "str" + enum_values = [e.value for e in field_type] + + metadata.append( + AttributeMetadata( + key=field_name, + type=type_str, + enum=enum_values, + required=field_obj.required, + ) + ) + + return metadata + except Exception: + return None + + +def adapt_legacy_to_universal(legacy: Compliance) -> ComplianceFramework: + """Convert a legacy Compliance object to a ComplianceFramework.""" + universal_requirements = [] + legacy_provider_key = legacy.Provider.lower() + + for req in legacy.Requirements: + req_checks = {legacy_provider_key: list(req.Checks)} if req.Checks else {} + if isinstance(req, Mitre_Requirement): + # For MITRE, promote special fields and store raw attributes + raw_attrs = [attr.dict() for attr in req.Attributes] + attrs = {"_raw_attributes": raw_attrs} + universal_requirements.append( + UniversalComplianceRequirement( + id=req.Id, + description=req.Description, + name=req.Name, + attributes=attrs, + checks=req_checks, + tactics=req.Tactics, + sub_techniques=req.SubTechniques, + platforms=req.Platforms, + technique_url=req.TechniqueURL, + ) + ) + else: + # Standard requirement: flatten first attribute to dict + if req.Attributes: + attrs = req.Attributes[0].dict() + else: + attrs = {} + universal_requirements.append( + UniversalComplianceRequirement( + id=req.Id, + description=req.Description, + name=req.Name, + attributes=attrs, + checks=req_checks, + ) + ) + + inferred_metadata = _infer_attribute_metadata(legacy) + + return ComplianceFramework( + framework=legacy.Framework, + name=legacy.Name, + provider=legacy.Provider, + version=legacy.Version, + description=legacy.Description, + requirements=universal_requirements, + attributes_metadata=inferred_metadata, + ) + + +def load_compliance_framework_universal(path: str) -> ComplianceFramework: + """Load a compliance JSON as a ComplianceFramework, handling both new and legacy formats.""" + try: + with open(path, "r") as f: + data = json.load(f) + + if "attributes_metadata" in data or "requirements" in data: + # New universal format β€” parse directly + return ComplianceFramework(**data) + else: + # Legacy format β€” parse as Compliance, then adapt + legacy = Compliance(**data) + return adapt_legacy_to_universal(legacy) + except Exception as e: + logger.error( + f"Failed to load universal compliance framework from {path}: " + f"{e.__class__.__name__}[{e.__traceback__.tb_lineno}] -- {e}" + ) + return None + + +def _load_jsons_from_dir(dir_path: str, provider: str, bulk: dict) -> None: + """Scan *dir_path* for JSON files and add matching frameworks to *bulk*.""" + for filename in os.listdir(dir_path): + file_path = os.path.join(dir_path, filename) + if not ( + os.path.isfile(file_path) + and filename.endswith(".json") + and os.stat(file_path).st_size > 0 + ): + continue + framework_name = filename.split(".json")[0] + if framework_name in bulk: + continue + fw = load_compliance_framework_universal(file_path) + if fw is None: + continue + if fw.provider and fw.provider.lower() == provider.lower(): + bulk[framework_name] = fw + elif fw.supports_provider(provider): + bulk[framework_name] = fw + + +def get_bulk_compliance_frameworks_universal(provider: str) -> dict: + """Bulk load all compliance frameworks relevant to the given provider. + + Scans: + + 1. The **top-level** ``prowler/compliance/`` directory for multi-provider + JSONs (``Checks`` keyed by provider, no ``Provider`` field). + 2. Every **provider sub-directory** (``prowler/compliance/{p}/``) so that + single-provider JSONs are also picked up. + + A framework is included when its explicit ``Provider`` matches + (case-insensitive) **or** any requirement has dict-style ``Checks`` + with a key for *provider*. + """ + bulk = {} + try: + available_modules = list_compliance_modules() + + # Resolve the compliance root once (parent of provider sub-dirs). + compliance_root = None + seen_paths = set() + + for module in available_modules: + dir_path = f"{module.module_finder.path}/{module.name.split('.')[-1]}" + if not os.path.isdir(dir_path) or dir_path in seen_paths: + continue + seen_paths.add(dir_path) + + # Remember the root the first time we see a valid sub-dir. + if compliance_root is None: + compliance_root = module.module_finder.path + + _load_jsons_from_dir(dir_path, provider, bulk) + + # Also scan top-level compliance/ for provider-agnostic JSONs. + if compliance_root and os.path.isdir(compliance_root): + _load_jsons_from_dir(compliance_root, provider, bulk) + + except Exception as e: + logger.error(f"{e.__class__.__name__}[{e.__traceback__.tb_lineno}] -- {e}") + return bulk diff --git a/prowler/providers/aws/lib/cloudtrail_timeline/cloudtrail_timeline.py b/prowler/providers/aws/lib/cloudtrail_timeline/cloudtrail_timeline.py index 45dff270d0..b73d070078 100644 --- a/prowler/providers/aws/lib/cloudtrail_timeline/cloudtrail_timeline.py +++ b/prowler/providers/aws/lib/cloudtrail_timeline/cloudtrail_timeline.py @@ -135,25 +135,54 @@ class CloudTrailTimeline(TimelineService): ) -> List[Dict[str, Any]]: """Query CloudTrail for events related to a specific resource. - Uses MaxResults to limit the number of events returned, preparing - for API-level pagination. Currently returns up to max_results events - from the first page only. + CloudTrail's ResourceName attribute is populated per-service by AWS + and is not consistent: KMS and SNS store full ARNs, while S3, IAM, + EC2, Lambda, RDS and others store only the resource name or ID. We + first look up using the identifier as-is, and if no events come back + we retry with the last segment extracted from the ARN. """ client = self._get_client(region) start_time = datetime.now(timezone.utc) - timedelta(days=self._lookback_days) - # Use direct API call with MaxResults instead of paginator - # This limits CloudTrail to return only max_results events + events = self._lookup_events_by_name(client, resource_identifier, start_time) + + if not events and resource_identifier.startswith("arn:"): + short_name = self._extract_short_name(resource_identifier) + if short_name and short_name != resource_identifier: + logger.debug( + f"CloudTrail timeline: no events for '{resource_identifier}', " + f"retrying lookup with short name '{short_name}'" + ) + events = self._lookup_events_by_name(client, short_name, start_time) + + return events + + def _lookup_events_by_name( + self, client, resource_name: str, start_time: datetime + ) -> List[Dict[str, Any]]: response = client.lookup_events( LookupAttributes=[ - {"AttributeKey": "ResourceName", "AttributeValue": resource_identifier} + {"AttributeKey": "ResourceName", "AttributeValue": resource_name} ], StartTime=start_time, MaxResults=self._max_results, ) - return response.get("Events", []) + @staticmethod + def _extract_short_name(identifier: str) -> str: + """Return the last segment of an ARN or identifier. + + ARNs take the form `arn:partition:service:region:account:resource-info` + where resource-info is one of `name`, `type/name`, or `type:name`. + Splitting on the final `/` and then the final `:` yields the value + CloudTrail stores for most services: S3 bucket name, IAM user/role + name, EC2 resource ID, Lambda function name, RDS DB identifier, etc. + """ + if not identifier: + return identifier + return identifier.rsplit("/", 1)[-1].rsplit(":", 1)[-1] + def _parse_event(self, raw_event: Dict[str, Any]) -> Optional[Dict[str, Any]]: """Parse a raw CloudTrail event into a TimelineEvent dictionary.""" try: diff --git a/prowler/providers/cloudflare/cloudflare_provider.py b/prowler/providers/cloudflare/cloudflare_provider.py index 35067c30d8..48763df395 100644 --- a/prowler/providers/cloudflare/cloudflare_provider.py +++ b/prowler/providers/cloudflare/cloudflare_provider.py @@ -274,8 +274,12 @@ class CloudflareProvider(Provider): for account in client.accounts.list(): account_id = getattr(account, "id", None) - # Prevent infinite loop - skip if we've seen this account + # Prevent infinite loop on repeated pages from the SDK paginator if account_id in seen_account_ids: + logger.warning( + "Detected repeated Cloudflare account ID while listing accounts. " + "Stopping pagination to avoid an infinite loop." + ) break seen_account_ids.add(account_id) @@ -395,7 +399,20 @@ class CloudflareProvider(Provider): # Fallback: try accounts.list() try: - accounts = list(client.accounts.list()) + accounts: list = [] + seen_account_ids: set = set() + for account in client.accounts.list(): + account_id = getattr(account, "id", None) + # Prevent infinite loop on repeated pages from the SDK paginator + if account_id in seen_account_ids: + logger.warning( + "Detected repeated Cloudflare account ID while validating credentials. " + "Stopping pagination to avoid an infinite loop." + ) + break + seen_account_ids.add(account_id) + accounts.append(account) + if not accounts: logger.error("CloudflareNoAccountsError: No accounts found") raise CloudflareNoAccountsError( diff --git a/prowler/providers/common/provider.py b/prowler/providers/common/provider.py index f2c77bdc79..610cd10f04 100644 --- a/prowler/providers/common/provider.py +++ b/prowler/providers/common/provider.py @@ -280,6 +280,7 @@ class Provider(ABC): mutelist_path=arguments.mutelist_file, config_path=arguments.config_file, repositories=repos, + repo_list_file=getattr(arguments, "repo_list_file", None), organizations=orgs, ) elif "googleworkspace" in provider_class_name.lower(): diff --git a/prowler/providers/github/exceptions/exceptions.py b/prowler/providers/github/exceptions/exceptions.py index b49cce8ebe..b0f472a76c 100644 --- a/prowler/providers/github/exceptions/exceptions.py +++ b/prowler/providers/github/exceptions/exceptions.py @@ -34,6 +34,14 @@ class GithubBaseException(ProwlerException): "message": "The provided provider ID does not match with the authenticated user or accessible organizations", "remediation": "Check the provider ID and ensure it matches the authenticated user or an organization you have access to.", }, + (5007, "GithubRepoListFileNotFoundError"): { + "message": "The repo list file was not found", + "remediation": "Check the file path and ensure it exists.", + }, + (5008, "GithubRepoListFileReadError"): { + "message": "Error reading the repo list file", + "remediation": "Check the file permissions and format.", + }, } def __init__(self, code, file=None, original_exception=None, message=None): @@ -104,3 +112,21 @@ class GithubInvalidProviderIdError(GithubCredentialsError): super().__init__( 5006, file=file, original_exception=original_exception, message=message ) + + +class GithubRepoListFileNotFoundError(GithubBaseException): + """Exception raised when the repo list file is not found.""" + + def __init__(self, file=None, original_exception=None, message=None): + super().__init__( + 5007, file=file, original_exception=original_exception, message=message + ) + + +class GithubRepoListFileReadError(GithubBaseException): + """Exception raised when the repo list file cannot be read.""" + + def __init__(self, file=None, original_exception=None, message=None): + super().__init__( + 5008, file=file, original_exception=original_exception, message=message + ) diff --git a/prowler/providers/github/github_provider.py b/prowler/providers/github/github_provider.py index 16d13f7434..ab3441b81c 100644 --- a/prowler/providers/github/github_provider.py +++ b/prowler/providers/github/github_provider.py @@ -22,6 +22,8 @@ from prowler.providers.github.exceptions.exceptions import ( GithubInvalidCredentialsError, GithubInvalidProviderIdError, GithubInvalidTokenError, + GithubRepoListFileNotFoundError, + GithubRepoListFileReadError, GithubSetUpIdentityError, GithubSetUpSessionError, ) @@ -90,6 +92,8 @@ class GithubProvider(Provider): _type: str = "github" _auth_method: str = None + MAX_REPO_LIST_LINES: int = 10_000 + MAX_REPO_NAME_LENGTH: int = 500 _session: GithubSession _identity: GithubIdentityInfo _audit_config: dict @@ -113,6 +117,7 @@ class GithubProvider(Provider): mutelist_path: str = None, mutelist_content: dict = None, repositories: list = None, + repo_list_file: str = None, organizations: list = None, ): """ @@ -130,6 +135,7 @@ class GithubProvider(Provider): mutelist_path (str): Path to the mutelist file. mutelist_content (dict): Mutelist content. repositories (list): List of repository names to scan in 'owner/repo-name' format. + repo_list_file (str): Path to a file containing repository names (one per line). organizations (list): List of organization or user names to scan repositories for. """ logger.info("Instantiating GitHub Provider...") @@ -147,6 +153,10 @@ class GithubProvider(Provider): else: self._repositories = list(repositories) + # Load repos from file if provided + if repo_list_file: + self._load_repos_from_file(repo_list_file) + if organizations is None: self._organizations = [] elif isinstance(organizations, str): @@ -256,6 +266,46 @@ class GithubProvider(Provider): """ return self._organizations + def _load_repos_from_file(self, file_path: str) -> None: + """Load repository names from a file (one per line).""" + try: + repo_count = 0 + before = len(self._repositories) + with open(file_path, "r") as f: + for line in f: + line = line.strip() + if not line or line.startswith("#"): + continue + repo_count += 1 + if repo_count > self.MAX_REPO_LIST_LINES: + raise GithubRepoListFileReadError( + file=file_path, + message=f"Repo list file exceeds maximum of {self.MAX_REPO_LIST_LINES} lines.", + ) + if len(line) > self.MAX_REPO_NAME_LENGTH: + logger.warning( + f"Skipping repo name exceeding {self.MAX_REPO_NAME_LENGTH} chars at line {repo_count} in {file_path}" + ) + continue + self._repositories.append(line) + self._repositories = list(dict.fromkeys(self._repositories)) + logger.info( + f"Loaded {len(self._repositories) - before} repositories from {file_path}" + ) + except FileNotFoundError: + raise GithubRepoListFileNotFoundError( + file=file_path, + message=f"Repo list file not found: {file_path}", + ) + except (GithubRepoListFileReadError, GithubRepoListFileNotFoundError): + raise + except Exception as error: + raise GithubRepoListFileReadError( + file=file_path, + original_exception=error, + message=f"Error reading repo list file: {error}", + ) + @staticmethod def setup_session( personal_access_token: str = None, diff --git a/prowler/providers/github/lib/arguments/arguments.py b/prowler/providers/github/lib/arguments/arguments.py index 748d77a927..946029ab43 100644 --- a/prowler/providers/github/lib/arguments/arguments.py +++ b/prowler/providers/github/lib/arguments/arguments.py @@ -50,6 +50,12 @@ def init_parser(self): default=None, metavar="REPOSITORY", ) + github_scoping_subparser.add_argument( + "--repo-list-file", + dest="repo_list_file", + default=None, + help="Path to a file containing a list of repositories to scan (one per line in 'owner/repo-name' format). Lines starting with # are treated as comments.", + ) github_scoping_subparser.add_argument( "--organization", "--organizations", diff --git a/skills/prowler-compliance/SKILL.md b/skills/prowler-compliance/SKILL.md index 1853d23d8b..51c68eb05f 100644 --- a/skills/prowler-compliance/SKILL.md +++ b/skills/prowler-compliance/SKILL.md @@ -1,16 +1,28 @@ --- name: prowler-compliance description: > - Creates and manages Prowler compliance frameworks. - Trigger: When working with compliance frameworks (CIS, NIST, PCI-DSS, SOC2, GDPR, ISO27001, ENS, MITRE ATT&CK). + Creates, syncs, audits and manages Prowler compliance frameworks end-to-end. + Covers the four-layer architecture (SDK models β†’ JSON catalogs β†’ output + formatters β†’ API/UI), upstream sync workflows, cloud-auditor check-mapping + reviews, output formatter creation, and framework-specific attribute models. + Trigger: When working with compliance frameworks (CIS, NIST, PCI-DSS, SOC2, + GDPR, ISO27001, ENS, MITRE ATT&CK, CCC, C5, CSA CCM, KISA ISMS-P, + Prowler ThreatScore, FedRAMP, HIPAA), syncing with upstream catalogs, + auditing check-to-requirement mappings, adding output formatters, or fixing + compliance JSON bugs (duplicate IDs, empty Version, wrong Section, stale + check refs). license: Apache-2.0 metadata: author: prowler-cloud - version: "1.1" + version: "1.2" scope: [root, sdk] auto_invoke: - "Creating/updating compliance frameworks" - "Mapping checks to compliance controls" + - "Syncing compliance framework with upstream catalog" + - "Auditing check-to-requirement mappings as a cloud auditor" + - "Adding a compliance output formatter (per-provider class + table dispatcher)" + - "Fixing compliance JSON bugs (duplicate IDs, empty Section, stale refs)" allowed-tools: Read, Edit, Write, Glob, Grep, Bash, WebFetch, WebSearch, Task --- @@ -18,10 +30,82 @@ allowed-tools: Read, Edit, Write, Glob, Grep, Bash, WebFetch, WebSearch, Task Use this skill when: - Creating a new compliance framework for any provider +- **Syncing an existing framework with an upstream source of truth** (CIS, FINOS CCC, CSA CCM, NIST, ENS, etc.) - Adding requirements to existing frameworks - Mapping checks to compliance controls +- **Auditing existing check mappings as a cloud auditor** (user asks "are these mappings correct?", "which checks apply to this requirement?", "review the mappings") +- **Adding a new output formatter** (new framework needs a table dispatcher + per-provider classes + CSV models) +- **Fixing JSON bugs**: duplicate IDs, empty Version, wrong Section, stale check refs, inconsistent FamilyName, padded tangential check mappings +- **Registering a framework in the CLI table dispatcher or API export map** +- Investigating why a finding/check isn't showing under the expected compliance framework in the UI - Understanding compliance framework structures and attributes +## Four-Layer Architecture (Mental Model) + +Prowler compliance is a **four-layer system** hanging off one Pydantic model tree. Bugs usually happen where one layer doesn't match another, so know all four before touching anything. + +### Layer 1: SDK / Core Models β€” `prowler/lib/check/` + +- **`compliance_models.py`** β€” Pydantic **v1** model tree (`from pydantic.v1 import`). One `*_Requirement_Attribute` class per framework type + `Generic_Compliance_Requirement_Attribute` as fallback. +- `Compliance_Requirement.Attributes: list[Union[...]]` β€” **`Generic_Compliance_Requirement_Attribute` MUST be LAST** in the Union or every framework-specific attribute falls through to Generic (Pydantic v1 tries union members in order). +- **`compliance.py`** β€” runtime linker. `get_check_compliance()` builds the key as `f"{Framework}-{Version}"` **only if `Version` is non-empty**. An empty Version makes the key just `"{Framework}"` β€” this breaks downstream filters and tests that expect the versioned key. +- `Compliance.get_bulk(provider)` walks `prowler/compliance/{provider}/` and parses every `.json` file. No central index β€” just directory scan. + +### Layer 2: JSON Frameworks β€” `prowler/compliance/{provider}/` + +See "Compliance Framework Location" and "Framework-Specific Attribute Structures" sections below. + +### Layer 3: Output Formatters β€” `prowler/lib/outputs/compliance/{framework}/` + +**Every framework directory follows this exact convention** β€” do not deviate: + +``` +{framework}/ +β”œβ”€β”€ __init__.py +β”œβ”€β”€ {framework}.py # ONLY get_{framework}_table() β€” NO function docstring +β”œβ”€β”€ {framework}_{provider}.py # One class per provider (e.g., CCC_AWS, CCC_Azure, CCC_GCP) +└── models.py # One Pydantic v2 BaseModel per provider (CSV columns) +``` + +- **`{framework}.py`** holds the **table dispatcher function** `get_{framework}_table()`. It prints the pass/fail/muted summary table. **Must NOT import `Finding` or `ComplianceOutput`** β€” doing so creates a circular import with `prowler/lib/outputs/compliance/compliance.py`. Only imports: `colorama`, `tabulate`, `prowler.config.config.orange_color`. +- **`{framework}_{provider}.py`** holds a per-provider class like `CCC_AWS(ComplianceOutput)` with a `transform()` method that walks findings and emits rows. This file IS allowed to import `Finding` because it's not on the dispatcher import chain. +- **`models.py`** holds one Pydantic v2 `BaseModel` per provider. Field names become CSV column headers (**public API** β€” renaming breaks downstream consumers). +- **Never collapse per-provider files into a unified parameterized class**, even when DRY-tempting. Every framework in Prowler follows the per-provider file pattern and reviewers will reject the refactor. CSV columns differ per provider (`AccountId`/`Region` vs `SubscriptionId`/`Location` vs `ProjectId`/`Location`) β€” three classes is the convention. +- **No function docstring on `get_{framework}_table()`** β€” no other framework has one; stay consistent. +- Register in `prowler/lib/outputs/compliance/compliance.py` β†’ `display_compliance_table()` with an `elif compliance_framework.startswith("{framework}_"):` branch. Import the table function at the top of the file. + +### Layer 4: API / UI + +- **API table dispatcher**: `api/src/backend/tasks/jobs/export.py` β†’ `COMPLIANCE_CLASS_MAP` keyed by provider. Uses `startswith` predicates: `(lambda name: name.startswith("ccc_"), CCC_AWS)`. **Never use exact match** (`name == "ccc_aws"`) β€” it's inconsistent and breaks versioning. +- **API lazy loader**: `api/src/backend/api/compliance.py` β€” `LazyComplianceTemplate` and `LazyChecksMapping` load compliance per provider on first access. +- **UI mapper routing**: `ui/lib/compliance/compliance-mapper.ts` routes framework names β†’ per-framework mapper. +- **UI per-framework mapper**: `ui/lib/compliance/{framework}.tsx` flattens `Requirements` into a 3-level tree (Framework β†’ Category β†’ Control β†’ Requirement) for the accordion view. Groups by `Attributes[0].FamilyName` and `Attributes[0].Section`. +- **UI detail panel**: `ui/components/compliance/compliance-custom-details/{framework}-details.tsx`. +- **UI types**: `ui/types/compliance.ts` β€” TypeScript mirrors of the attribute metadata. + +### The CLI Pipeline (end-to-end) + +``` +prowler aws --compliance ccc_aws + ↓ +Compliance.get_bulk("aws") β†’ parses prowler/compliance/aws/*.json + ↓ +update_checks_metadata_with_compliance() β†’ attaches compliance info to CheckMetadata + ↓ +execute_checks() β†’ runs checks, produces Finding objects + ↓ +get_check_compliance(finding, "aws", bulk_checks_metadata) + β†’ dict "{Framework}-{Version}" β†’ [requirement_ids] + ↓ +CCC_AWS(findings, compliance).transform() β†’ per-provider class builds CSV rows + ↓ +batch_write_data_to_file() β†’ writes {output_filename}_ccc_aws.csv + ↓ +display_compliance_table() β†’ get_ccc_table() β†’ prints stdout summary +``` + +--- + ## Compliance Framework Location Frameworks are JSON files located in: `prowler/compliance/{provider}/{framework_name}_{provider}.json` @@ -455,14 +539,453 @@ Prowler ThreatScore is a custom security scoring framework developed by Prowler - **M365:** `cis_4.0_m365.json`, `iso27001_2022_m365.json` - **NHN:** `iso27001_2022_nhn.json` +## Workflow A: Sync a Framework With an Upstream Catalog + +Use when the framework is maintained upstream (CIS Benchmarks, FINOS CCC, CSA CCM, NIST, ENS, etc.) and Prowler needs to catch up. + +### Step 1 β€” Cache the upstream source + +Download every upstream file to a local cache so subsequent iterations don't hit the network. For FINOS CCC: + +```bash +mkdir -p /tmp/ccc_upstream +catalogs="core/ccc storage/object management/auditlog management/logging ..." +for p in $catalogs; do + safe=$(echo "$p" | tr '/' '_') + gh api "repos/finos/common-cloud-controls/contents/catalogs/$p/controls.yaml" \ + -H "Accept: application/vnd.github.raw" > "/tmp/ccc_upstream/${safe}.yaml" +done +``` + +### Step 2 β€” Run the generic sync runner against a framework config + +The sync tooling is split into three layers so adding a new framework only takes a YAML config (and optionally a new parser module for an unfamiliar upstream format): + +``` +skills/prowler-compliance/assets/ +β”œβ”€β”€ sync_framework.py # generic runner β€” works for any framework +β”œβ”€β”€ configs/ +β”‚ └── ccc.yaml # per-framework config (canonical example) +└── parsers/ + β”œβ”€β”€ __init__.py + └── finos_ccc.py # parser module for FINOS CCC YAML +``` + +**For frameworks that already have a config + parser** (today: FINOS CCC), run: + +```bash +python skills/prowler-compliance/assets/sync_framework.py \ + skills/prowler-compliance/assets/configs/ccc.yaml +``` + +The runner loads the config, validates it, dynamically imports the parser declared in `parser.module`, calls `parser.parse_upstream(config) -> list[dict]`, then applies generic post-processing (id uniqueness safety net, `FamilyName` normalization, legacy check-mapping preservation) and writes the provider JSONs. + +**To add a new framework sync**: + +1. **Write a config file** at `skills/prowler-compliance/assets/configs/{framework}.yaml`. See `configs/ccc.yaml` as the canonical example. Required top-level sections: + - `framework` β€” `name`, `display_name`, `version` (**never empty** β€” empty Version silently breaks `get_check_compliance()` key construction, so the runner refuses to start), `description_template` (accepts `{provider_display}`, `{provider_key}`, `{framework_name}`, `{framework_display}`, `{version}` placeholders). + - `providers` β€” list of `{key, display}` pairs, one per Prowler provider the framework targets. + - `output.path_template` β€” supports `{provider}`, `{framework}`, `{version}` placeholders. Examples: `"prowler/compliance/{provider}/ccc_{provider}.json"` for unversioned file names, `"prowler/compliance/{provider}/cis_{version}_{provider}.json"` for versioned ones. + - `upstream.dir` β€” local cache directory (populate via Step 1). + - `parser.module` β€” name of the module under `parsers/` to load (without `.py`). Everything else under `parser.` is opaque to the runner and passed to the parser as config. + - `post_processing.check_preservation.primary_key` β€” top-level field name for the primary legacy-mapping lookup (almost always `Id`). + - `post_processing.check_preservation.fallback_keys` β€” **config-driven fallback keys** for preserving check mappings when ids change. Each entry is a list of `Attributes[0]` field names composed into a tuple. Examples: + - CCC: `- [Section, Applicability]` (because `Applicability` is a CCC-only attribute, verified in `compliance_models.py:213`). + - CIS would use `- [Section, Profile]`. + - NIST would use `- [ItemId]`. + - List-valued fields (like `Applicability`) are automatically frozen to `frozenset` so the tuple is hashable. + - `post_processing.family_name_normalization` (optional) β€” map of raw β†’ canonical `FamilyName` values. The UI groups by `Attributes[0].FamilyName` exactly, so inconsistent upstream variants otherwise become separate tree branches. + +2. **Reuse an existing parser** if the upstream format matches one (currently only `finos_ccc` exists). Otherwise, **write a new parser** at `parsers/{name}.py` implementing: + + ```python + def parse_upstream(config: dict) -> list[dict]: + """Return Prowler-format requirements {Id, Description, Attributes: [...], Checks: []}. + + Ids MUST be unique in the returned list. The runner raises ValueError + on duplicates β€” it does NOT silently renumber, because mutating a + canonical upstream id (e.g. CIS '1.1.1' or NIST 'AC-2(1)') would be + catastrophic. The parser owns all upstream-format quirks: foreign-prefix + rewriting, genuine collision renumbering, shape handling. + """ + ``` + + The parser reads its own settings from `config['upstream']` and `config['parser']`. It does NOT load existing Prowler JSONs (the runner does that for check preservation) and does NOT write output (the runner does that too). + +**Gotchas the runner already handles for you** (learned from the FINOS CCC v2025.10 sync β€” they're documented here so you don't re-discover them): + +- **Multiple upstream YAML shapes**. Most FINOS CCC catalogs use `control-families: [...]`, but `storage/object` uses a top-level `controls: [...]` with a `family: "CCC.X.Y"` reference id and no human-readable family name. A parser that only handles shape 1 silently drops the shape-2 catalog β€” this exact bug dropped ObjStor from Prowler for a full iteration. `parsers/finos_ccc.py` handles both shapes; if you write a new parser for a similar format, test with at least one file of each shape. +- **Whitespace collapse**. Upstream YAML multi-line block scalars (`|`) preserve newlines. Prowler stores descriptions single-line. Collapse with `" ".join(value.split())` before emitting (see `parsers/finos_ccc.py::clean()`). +- **Foreign-prefix AR id rewriting**. Upstream sometimes aliases requirements across catalogs by keeping the original prefix (e.g., `CCC.AuditLog.CN08.AR01` appears nested under `CCC.Logging.CN03`). Rewrite the foreign id to fit its parent control: `CCC.Logging.CN03.AR01`. This logic is parser-specific because the id structure varies per framework (CCC uses 3-dot depth; CIS uses numeric dots; NIST uses `AC-2(1)`). +- **Genuine upstream collision renumbering**. Sometimes upstream has a real typo where two different requirements share the same id (e.g., `CCC.Core.CN14.AR02` defined twice for 30-day and 14-day backup variants). Renumber the second copy to the next free AR number (`.AR03`). The parser handles this; the runner asserts the final list has unique ids as a safety net. +- **Existing check mapping preservation**. The runner uses the `primary_key` + `fallback_keys` declared in config to look up the old `Checks` list for each requirement. For CCC this means primary index by `Id` plus fallback index by `(Section, frozenset(Applicability))` β€” the fallback recovers mappings for requirements whose ids were rewritten or renumbered by the parser. +- **FamilyName normalization**. Configured via `post_processing.family_name_normalization` β€” no code changes needed to collapse upstream variants like `"Logging & Monitoring"` β†’ `"Logging and Monitoring"`. +- **Populate `Version`**. The runner refuses to start on empty `framework.version` β€” fail-fast replaces the silent bug where `get_check_compliance()` would build the key as just `"{Framework}"`. + +### Step 3 β€” Validate before committing + +```python +from prowler.lib.check.compliance_models import Compliance +for prov in ['aws', 'azure', 'gcp']: + c = Compliance.parse_file(f"prowler/compliance/{prov}/ccc_{prov}.json") + print(f"{prov}: {len(c.Requirements)} reqs, version={c.Version}") +``` + +Any `ValidationError` means the Attribute fields don't match the `*_Requirement_Attribute` model. Either fix the JSON or extend the model in `compliance_models.py` (remember: Generic stays last). + +### Step 4 β€” Verify every check id exists + +```python +import json +from pathlib import Path +for prov in ['aws', 'azure', 'gcp']: + existing = {p.stem.replace('.metadata','') + for p in Path(f'prowler/providers/{prov}/services').rglob('*.metadata.json')} + with open(f'prowler/compliance/{prov}/ccc_{prov}.json') as f: + data = json.load(f) + refs = {c for r in data['Requirements'] for c in r['Checks']} + missing = refs - existing + assert not missing, f"{prov} missing: {missing}" +``` + +A stale check id silently becomes dead weight β€” no finding will ever map to it. This pre-validation **must run on every write**; bake it into the generator script. + +### Step 5 β€” Add an attribute model if needed + +Only if the framework has fields beyond `Generic_Compliance_Requirement_Attribute`. Add the class to `prowler/lib/check/compliance_models.py` and register it in `Compliance_Requirement.Attributes: list[Union[...]]`. **Generic stays last.** + +--- + +## Workflow B: Audit Check Mappings as a Cloud Auditor + +Use when the user asks to review existing mappings ("are these correct?", "verify that the checks apply", "audit the CCC mappings"). This is the highest-value compliance task β€” it surfaces padded mappings with zero actual coverage and missing mappings for legitimate coverage. + +### The golden rule + +> A Prowler check's title/risk MUST **literally describe what the requirement text says**. "Related" is not enough. If no check actually addresses the requirement, leave `Checks: []` (MANUAL) β€” **honest MANUAL is worth more than padded coverage**. + +### Audit process + +**Step 1 β€” Build a per-provider check inventory** (cache in `/tmp/`): + +```python +import json +from pathlib import Path +for provider in ['aws', 'azure', 'gcp']: + inv = {} + for meta in Path(f'prowler/providers/{provider}/services').rglob('*.metadata.json'): + with open(meta) as f: + d = json.load(f) + cid = d.get('CheckID') or meta.stem.replace('.metadata','') + inv[cid] = { + 'service': d.get('ServiceName', ''), + 'title': d.get('CheckTitle', ''), + 'risk': d.get('Risk', ''), + 'description': d.get('Description', ''), + } + with open(f'/tmp/checks_{provider}.json', 'w') as f: + json.dump(inv, f, indent=2) +``` + +**Step 2 β€” Keyword/service query helper** β€” see [assets/query_checks.py](assets/query_checks.py): + +```bash +python assets/query_checks.py aws encryption transit # keyword AND-search +python assets/query_checks.py aws --service iam # all iam checks +python assets/query_checks.py aws --id kms_cmk_rotation_enabled # full metadata +``` + +**Step 3 β€” Dump a framework section with current mappings** β€” see [assets/dump_section.py](assets/dump_section.py): + +```bash +python assets/dump_section.py ccc "CCC.Core." # all Core ARs across 3 providers +python assets/dump_section.py ccc "CCC.AuditLog." # all AuditLog ARs +``` + +**Step 4 β€” Encode explicit REPLACE decisions** β€” see [assets/audit_framework_template.py](assets/audit_framework_template.py). Structure: + +```python +DECISIONS = {} + +DECISIONS["CCC.Core.CN01.AR01"] = { + "aws": [ + "cloudfront_distributions_https_enabled", + "cloudfront_distributions_origin_traffic_encrypted", + # ... + ], + "azure": [ + "storage_secure_transfer_required_is_enabled", + "app_minimum_tls_version_12", + # ... + ], + "gcp": [ + "cloudsql_instance_ssl_connections", + ], + # Missing provider key = leave the legacy mapping untouched +} + +# Empty list = EXPLICITLY MANUAL (overwrites legacy) +DECISIONS["CCC.Core.CN01.AR07"] = { + "aws": [], # Prowler has no IANA port/protocol check + "azure": [], + "gcp": [], +} +``` + +**REPLACE, not PATCH.** Encoding every mapping as a full list (not add/remove delta) makes the audit reproducible and surfaces hidden assumptions from the legacy data. + +**Step 5 β€” Pre-validation**. The audit script MUST validate every check id against the inventory and **abort with stderr listing typos**. Common typos caught during a real audit: + +- `fsx_file_system_encryption_at_rest_using_kms` (doesn't exist) +- `cosmosdb_account_encryption_at_rest_with_cmk` (doesn't exist) +- `sqlserver_geo_replication` (doesn't exist) +- `redshift_cluster_audit_logging` (should be `redshift_cluster_encrypted_at_rest`) +- `postgresql_flexible_server_require_secure_transport` (should be `postgresql_flexible_server_enforce_ssl_enabled`) +- `storage_secure_transfer_required_enabled` (should be `storage_secure_transfer_required_is_enabled`) +- `sqlserver_minimum_tls_version_12` (should be `sqlserver_recommended_minimal_tls_version`) + +**Step 6 β€” Apply + validate + test**: + +```bash +python /path/to/audit_script.py # applies decisions, pre-validates +python -m pytest tests/lib/outputs/compliance/ tests/lib/check/ -q +``` + +### Audit Reference Table: Requirement Text β†’ Prowler Checks + +Use this table to map CCC-style / NIST-style / ISO-style requirements to the checks that actually verify them. Built from a real audit of 172 CCC ARs Γ— 3 providers. + +| Requirement text | AWS checks | Azure checks | GCP checks | +|---|---|---|---| +| **TLS in transit enforced** | `cloudfront_distributions_https_enabled`, `s3_bucket_secure_transport_policy`, `elbv2_ssl_listeners`, `elbv2_insecure_ssl_ciphers`, `elb_ssl_listeners`, `elb_insecure_ssl_ciphers`, `opensearch_service_domains_https_communications_enforced`, `rds_instance_transport_encrypted`, `redshift_cluster_in_transit_encryption_enabled`, `elasticache_redis_cluster_in_transit_encryption_enabled`, `dynamodb_accelerator_cluster_in_transit_encryption_enabled`, `dms_endpoint_ssl_enabled`, `kafka_cluster_in_transit_encryption_enabled`, `transfer_server_in_transit_encryption_enabled`, `glue_database_connections_ssl_enabled`, `sns_subscription_not_using_http_endpoints` | `storage_secure_transfer_required_is_enabled`, `storage_ensure_minimum_tls_version_12`, `postgresql_flexible_server_enforce_ssl_enabled`, `mysql_flexible_server_ssl_connection_enabled`, `mysql_flexible_server_minimum_tls_version_12`, `sqlserver_recommended_minimal_tls_version`, `app_minimum_tls_version_12`, `app_ensure_http_is_redirected_to_https`, `app_ftp_deployment_disabled` | `cloudsql_instance_ssl_connections` (almost only option) | +| **TLS 1.3 specifically** | Partial: `cloudfront_distributions_using_deprecated_ssl_protocols`, `elb*_insecure_ssl_ciphers`, `*_minimum_tls_version_12` | Partial: `*_minimum_tls_version_12` checks | None β€” accept as MANUAL | +| **SSH / port 22 hardening** | `ec2_instance_port_ssh_exposed_to_internet`, `ec2_securitygroup_allow_ingress_from_internet_to_tcp_port_22`, `ec2_networkacl_allow_ingress_tcp_port_22` | `network_ssh_internet_access_restricted`, `vm_linux_enforce_ssh_authentication` | `compute_firewall_ssh_access_from_the_internet_allowed`, `compute_instance_block_project_wide_ssh_keys_disabled`, `compute_project_os_login_enabled`, `compute_project_os_login_2fa_enabled` | +| **mTLS (mutual TLS)** | `kafka_cluster_mutual_tls_authentication_enabled`, `apigateway_restapi_client_certificate_enabled` | `app_client_certificates_on` | None β€” MANUAL | +| **Data at rest encrypted** | `s3_bucket_default_encryption`, `s3_bucket_kms_encryption`, `ec2_ebs_default_encryption`, `ec2_ebs_volume_encryption`, `rds_instance_storage_encrypted`, `rds_cluster_storage_encrypted`, `rds_snapshots_encrypted`, `dynamodb_tables_kms_cmk_encryption_enabled`, `redshift_cluster_encrypted_at_rest`, `neptune_cluster_storage_encrypted`, `documentdb_cluster_storage_encrypted`, `opensearch_service_domains_encryption_at_rest_enabled`, `kinesis_stream_encrypted_at_rest`, `firehose_stream_encrypted_at_rest`, `sns_topics_kms_encryption_at_rest_enabled`, `sqs_queues_server_side_encryption_enabled`, `efs_encryption_at_rest_enabled`, `athena_workgroup_encryption`, `glue_data_catalogs_metadata_encryption_enabled`, `backup_vaults_encrypted`, `backup_recovery_point_encrypted`, `cloudtrail_kms_encryption_enabled`, `cloudwatch_log_group_kms_encryption_enabled`, `eks_cluster_kms_cmk_encryption_in_secrets_enabled`, `sagemaker_notebook_instance_encryption_enabled`, `apigateway_restapi_cache_encrypted`, `kafka_cluster_encryption_at_rest_uses_cmk`, `dynamodb_accelerator_cluster_encryption_enabled`, `storagegateway_fileshare_encryption_enabled` | `storage_infrastructure_encryption_is_enabled`, `storage_ensure_encryption_with_customer_managed_keys`, `vm_ensure_attached_disks_encrypted_with_cmk`, `vm_ensure_unattached_disks_encrypted_with_cmk`, `sqlserver_tde_encryption_enabled`, `sqlserver_tde_encrypted_with_cmk`, `databricks_workspace_cmk_encryption_enabled`, `monitor_storage_account_with_activity_logs_cmk_encrypted` | `compute_instance_encryption_with_csek_enabled`, `dataproc_encrypted_with_cmks_disabled`, `bigquery_dataset_cmk_encryption`, `bigquery_table_cmk_encryption` | +| **CMEK required (customer-managed keys)** | `kms_cmk_are_used` | `storage_ensure_encryption_with_customer_managed_keys`, `vm_ensure_attached_disks_encrypted_with_cmk`, `vm_ensure_unattached_disks_encrypted_with_cmk`, `sqlserver_tde_encrypted_with_cmk`, `databricks_workspace_cmk_encryption_enabled` | `bigquery_dataset_cmk_encryption`, `bigquery_table_cmk_encryption`, `dataproc_encrypted_with_cmks_disabled`, `compute_instance_encryption_with_csek_enabled` | +| **Key rotation enabled** | `kms_cmk_rotation_enabled` | `keyvault_key_rotation_enabled`, `storage_key_rotation_90_days` | `kms_key_rotation_enabled` | +| **MFA for UI access** | `iam_root_mfa_enabled`, `iam_root_hardware_mfa_enabled`, `iam_user_mfa_enabled_console_access`, `iam_user_hardware_mfa_enabled`, `iam_administrator_access_with_mfa`, `cognito_user_pool_mfa_enabled` | `entra_privileged_user_has_mfa`, `entra_non_privileged_user_has_mfa`, `entra_user_with_vm_access_has_mfa`, `entra_security_defaults_enabled` | `compute_project_os_login_2fa_enabled` | +| **API access / credentials** | `iam_no_root_access_key`, `iam_user_no_setup_initial_access_key`, `apigateway_restapi_authorizers_enabled`, `apigateway_restapi_public_with_authorizer`, `apigatewayv2_api_authorizers_enabled` | `entra_conditional_access_policy_require_mfa_for_management_api`, `app_function_access_keys_configured`, `app_function_identity_is_configured` | `apikeys_api_restrictions_configured`, `apikeys_key_exists`, `apikeys_key_rotated_in_90_days` | +| **Log all admin/config changes** | `cloudtrail_multi_region_enabled`, `cloudtrail_multi_region_enabled_logging_management_events`, `cloudtrail_cloudwatch_logging_enabled`, `cloudtrail_log_file_validation_enabled`, `cloudwatch_log_metric_filter_*`, `cloudwatch_changes_to_*_alarm_configured`, `config_recorder_all_regions_enabled` | `monitor_diagnostic_settings_exists`, `monitor_diagnostic_setting_with_appropriate_categories`, `monitor_alert_*` | `iam_audit_logs_enabled`, `logging_log_metric_filter_and_alert_for_*`, `logging_sink_created` | +| **Log integrity (digital signatures)** | `cloudtrail_log_file_validation_enabled` (exact) | None | None | +| **Public access denied** | `s3_bucket_public_access`, `s3_bucket_public_list_acl`, `s3_bucket_public_write_acl`, `s3_account_level_public_access_blocks`, `apigateway_restapi_public`, `awslambda_function_url_public`, `awslambda_function_not_publicly_accessible`, `rds_instance_no_public_access`, `rds_snapshots_public_access`, `ec2_securitygroup_allow_ingress_from_internet_to_all_ports`, `sns_topics_not_publicly_accessible`, `sqs_queues_not_publicly_accessible` | `storage_blob_public_access_level_is_disabled`, `storage_ensure_private_endpoints_in_storage_accounts`, `containerregistry_not_publicly_accessible`, `keyvault_private_endpoints`, `app_function_not_publicly_accessible`, `aks_clusters_public_access_disabled`, `network_http_internet_access_restricted` | `cloudstorage_bucket_public_access`, `compute_instance_public_ip`, `cloudsql_instance_public_ip`, `compute_firewall_*_access_from_the_internet_allowed` | +| **IAM least privilege** | `iam_*_no_administrative_privileges`, `iam_policy_allows_privilege_escalation`, `iam_inline_policy_allows_privilege_escalation`, `iam_role_administratoraccess_policy`, `iam_group_administrator_access_policy`, `iam_user_administrator_access_policy`, `iam_policy_attached_only_to_group_or_roles`, `iam_role_cross_service_confused_deputy_prevention` | `iam_role_user_access_admin_restricted`, `iam_subscription_roles_owner_custom_not_created`, `iam_custom_role_has_permissions_to_administer_resource_locks` | `iam_sa_no_administrative_privileges`, `iam_no_service_roles_at_project_level`, `iam_role_kms_enforce_separation_of_duties`, `iam_role_sa_enforce_separation_of_duties` | +| **Password policy** | `iam_password_policy_minimum_length_14`, `iam_password_policy_uppercase`, `iam_password_policy_lowercase`, `iam_password_policy_symbol`, `iam_password_policy_number`, `iam_password_policy_expires_passwords_within_90_days_or_less`, `iam_password_policy_reuse_24` | None | None | +| **Credential rotation / unused** | `iam_rotate_access_key_90_days`, `iam_user_accesskey_unused`, `iam_user_console_access_unused` | None | `iam_sa_user_managed_key_rotate_90_days`, `iam_sa_user_managed_key_unused`, `iam_service_account_unused` | +| **VPC / flow logs** | `vpc_flow_logs_enabled` | `network_flow_log_captured_sent`, `network_watcher_enabled`, `network_flow_log_more_than_90_days` | `compute_subnet_flow_logs_enabled` | +| **Backup / DR / Multi-AZ** | `backup_vaults_exist`, `backup_plans_exist`, `backup_reportplans_exist`, `rds_instance_backup_enabled`, `rds_*_protected_by_backup_plan`, `rds_cluster_multi_az`, `neptune_cluster_backup_enabled`, `documentdb_cluster_backup_enabled`, `efs_have_backup_enabled`, `s3_bucket_cross_region_replication`, `dynamodb_table_protected_by_backup_plan` | `vm_backup_enabled`, `vm_sufficient_daily_backup_retention_period`, `storage_geo_redundant_enabled` | `cloudsql_instance_automated_backups`, `cloudstorage_bucket_log_retention_policy_lock`, `cloudstorage_bucket_sufficient_retention_period` | +| **Access analysis / discovery** | `accessanalyzer_enabled`, `accessanalyzer_enabled_without_findings` | None specific | `iam_account_access_approval_enabled`, `iam_cloud_asset_inventory_enabled` | +| **Object lock / retention** | `s3_bucket_object_lock`, `s3_bucket_object_versioning`, `s3_bucket_lifecycle_enabled`, `cloudtrail_bucket_requires_mfa_delete`, `s3_bucket_no_mfa_delete` | `storage_ensure_soft_delete_is_enabled`, `storage_blob_versioning_is_enabled`, `storage_ensure_file_shares_soft_delete_is_enabled` | `cloudstorage_bucket_log_retention_policy_lock`, `cloudstorage_bucket_soft_delete_enabled`, `cloudstorage_bucket_versioning_enabled`, `cloudstorage_bucket_sufficient_retention_period` | +| **Uniform bucket-level access** | `s3_bucket_acl_prohibited` | `storage_account_key_access_disabled`, `storage_default_to_entra_authorization_enabled` | `cloudstorage_bucket_uniform_bucket_level_access` | +| **Container vulnerability scanning** | `ecr_registry_scan_images_on_push_enabled`, `ecr_repositories_scan_vulnerabilities_in_latest_image` | `defender_container_images_scan_enabled`, `defender_container_images_resolved_vulnerabilities` | `artifacts_container_analysis_enabled`, `gcr_container_scanning_enabled` | +| **WAF / rate limiting** | `wafv2_webacl_with_rules`, `waf_*_webacl_with_rules`, `wafv2_webacl_logging_enabled`, `waf_global_webacl_logging_enabled` | None | None | +| **Deployment region restriction** | `organizations_scp_check_deny_regions` | None | None | +| **Secrets automatic rotation** | `secretsmanager_automatic_rotation_enabled`, `secretsmanager_secret_rotated_periodically` | `keyvault_rbac_secret_expiration_set`, `keyvault_non_rbac_secret_expiration_set` | None | +| **Certificate management** | `acm_certificates_expiration_check`, `acm_certificates_with_secure_key_algorithms`, `acm_certificates_transparency_logs_enabled` | `keyvault_key_expiration_set_in_non_rbac`, `keyvault_rbac_key_expiration_set`, `keyvault_non_rbac_secret_expiration_set` | None | +| **GenAI guardrails / input/output filtering** | `bedrock_guardrail_prompt_attack_filter_enabled`, `bedrock_guardrail_sensitive_information_filter_enabled`, `bedrock_agent_guardrail_enabled`, `bedrock_model_invocation_logging_enabled`, `bedrock_api_key_no_administrative_privileges`, `bedrock_api_key_no_long_term_credentials` | None | None | +| **ML dev environment security** | `sagemaker_notebook_instance_root_access_disabled`, `sagemaker_notebook_instance_without_direct_internet_access_configured`, `sagemaker_notebook_instance_vpc_settings_configured`, `sagemaker_models_vpc_settings_configured`, `sagemaker_training_jobs_vpc_settings_configured`, `sagemaker_training_jobs_network_isolation_enabled`, `sagemaker_training_jobs_volume_and_output_encryption_enabled` | None | None | +| **Threat detection / anomalous behavior** | `cloudtrail_threat_detection_enumeration`, `cloudtrail_threat_detection_privilege_escalation`, `cloudtrail_threat_detection_llm_jacking`, `guardduty_is_enabled`, `guardduty_no_high_severity_findings` | None | None | +| **Serverless private access** | `awslambda_function_inside_vpc`, `awslambda_function_not_publicly_accessible`, `awslambda_function_url_public` | `app_function_not_publicly_accessible` | None | + +### What Prowler Does NOT Cover (accept MANUAL honestly) + +Don't pad mappings for these β€” mark `Checks: []` and move on: + +- **TLS 1.3 version specifically** β€” Prowler verifies TLS is enforced, not always the exact version +- **IANA port-protocol consistency** β€” no check for "protocol running on its assigned port" +- **mTLS on most Azure/GCP services** β€” limited to App Service client certs on Azure, nothing on GCP +- **Rate limiting** on monitoring endpoints, load balancers, serverless invocations, vector ingestion +- **Session cookie expiry** (LB stickiness) +- **HTTP header scrubbing** (Server, X-Powered-By) +- **Certificate transparency verification for imports** +- **Model version pinning, red teaming, AI quality review** +- **Vector embedding validation, dimensional constraints, ANN vs exact search** +- **Secret region replication** (cross-region residency) +- **Lifecycle cleanup policies on container registries** +- **Row-level / column-level security in data warehouses** +- **Deployment region restriction on Azure/GCP** (AWS has `organizations_scp_check_deny_regions`, others don't) +- **Cross-tenant alert silencing permissions** +- **Field-level masking in logs** +- **Managed view enforcement for database access** +- **Automatic MFA delete on all S3 buckets** (only CloudTrail bucket variant exists for some frameworks β€” AWS has the generic `s3_bucket_no_mfa_delete` though) + +--- + +## Workflow C: Add a New Output Formatter + +Use when a new framework needs its own CSV columns or terminal table. Follow the c5/csa/ens layout exactly: + +```bash +mkdir -p prowler/lib/outputs/compliance/{framework} +touch prowler/lib/outputs/compliance/{framework}/__init__.py +``` + +### Step 1 β€” Create `{framework}.py` (table dispatcher ONLY) + +Copy from `prowler/lib/outputs/compliance/c5/c5.py` and change the function name + framework string. The `diff` between your file and `c5.py` should be just those two lines. **No function docstring** β€” other frameworks don't have one, stay consistent. + +### Step 2 β€” Create `models.py` + +One Pydantic v2 `BaseModel` per provider. Field names become CSV column headers (public API β€” don't rename later without a migration). + +```python +from typing import Optional +from pydantic import BaseModel + +class {Framework}_AWSModel(BaseModel): + Provider: str + Description: str + AccountId: str + Region: str + AssessmentDate: str + Requirements_Id: str + Requirements_Description: str + # ... provider-specific columns + Status: str + StatusExtended: str + ResourceId: str + ResourceName: str + CheckId: str + Muted: bool +``` + +### Step 3 β€” Create `{framework}_{provider}.py` for each provider + +Copy from `prowler/lib/outputs/compliance/c5/c5_aws.py` etc. Contains the `{Framework}_AWS(ComplianceOutput)` class with `transform()` that walks findings and emits model rows. This file IS allowed to import `Finding`. + +### Step 4 β€” Register everywhere + +**`prowler/lib/outputs/compliance/compliance.py`** (CLI table dispatcher): +```python +from prowler.lib.outputs.compliance.{framework}.{framework} import get_{framework}_table + +def display_compliance_table(...): + ... + elif compliance_framework.startswith("{framework}_"): + get_{framework}_table(findings, bulk_checks_metadata, + compliance_framework, output_filename, + output_directory, compliance_overview) +``` + +**`prowler/__main__.py`** (CLI output writer per provider): +Add imports at the top: +```python +from prowler.lib.outputs.compliance.{framework}.{framework}_aws import {Framework}_AWS +from prowler.lib.outputs.compliance.{framework}.{framework}_azure import {Framework}_Azure +from prowler.lib.outputs.compliance.{framework}.{framework}_gcp import {Framework}_GCP +``` +Add provider-specific `elif compliance_name.startswith("{framework}_"):` branches that instantiate the class and call `batch_write_data_to_file()`. + +**`api/src/backend/tasks/jobs/export.py`** (API export dispatcher): +```python +from prowler.lib.outputs.compliance.{framework}.{framework}_aws import {Framework}_AWS +# ... azure, gcp + +COMPLIANCE_CLASS_MAP = { + "aws": [ + # ... + (lambda name: name.startswith("{framework}_"), {Framework}_AWS), + ], + # ... azure, gcp +} +``` + +**Always use `startswith`**, never `name == "framework_aws"`. Exact match is a regression. + +### Step 5 β€” Add tests + +Create `tests/lib/outputs/compliance/{framework}/` with `{framework}_aws_test.py`, `{framework}_azure_test.py`, `{framework}_gcp_test.py`. See the test template in [references/test_template.md](references/test_template.md). + +Add fixtures to `tests/lib/outputs/compliance/fixtures.py`: one `Compliance` object per provider with 1 evaluated + 1 manual requirement to exercise both code paths in `transform()`. + +### Circular import warning + +**The table dispatcher file (`{framework}.py`) MUST NOT import `Finding`** (directly or transitively). The cycle is: + +``` +compliance.compliance imports get_{framework}_table + β†’ {framework}.py imports ComplianceOutput + β†’ compliance_output imports Finding + β†’ finding imports get_check_compliance from compliance.compliance + β†’ CIRCULAR +``` + +Keep `{framework}.py` bare β€” only `colorama`, `tabulate`, `prowler.config.config`. Put anything that imports `Finding` in the per-provider `{framework}_{provider}.py` files. + +--- + +## Conventions and Hard-Won Gotchas + +These are lessons from the FINOS CCC v2025.10 sync + 172-AR audit pass (April 2026). Learn them once; save days of debugging. + +1. **Per-provider files are non-negotiable.** Never collapse `{framework}_aws.py`, `{framework}_azure.py`, `{framework}_gcp.py` into a single parameterized class, no matter how DRY-tempting. Every other framework in the codebase follows the per-provider pattern and reviewers will reject the refactor. The CSV column names differ per provider β€” three classes is the convention. +2. **`{framework}.py` has NO function docstring.** Other frameworks don't have them. Don't add one to be "helpful". +3. **Circular import protection**: the table dispatcher file MUST NOT import `Finding` (directly or transitively). Split the code so `{framework}.py` only has `get_{framework}_table()` with bare imports, and `{framework}_{provider}.py` holds the class that needs `Finding`. +4. **`Generic_Compliance_Requirement_Attribute` is the fallback** β€” in the `Compliance_Requirement.Attributes` Union in `compliance_models.py`, Generic MUST be LAST because Pydantic v1 tries union members in order. Putting Generic first means every framework-specific attribute falls through to Generic and the specific model is never used. +5. **Pydantic v1 imports.** `from pydantic.v1 import BaseModel` in `compliance_models.py` β€” not v2. Mixing causes validation errors. Pydantic v2 is used in the CSV models (`models.py`) β€” that's fine because they're separate trees. +6. **`get_check_compliance()` key format** is `f"{Framework}-{Version}"` ONLY if Version is set. Empty Version β†’ key is `"{Framework}"` (no version suffix). Tests that mock compliance dicts must match this exact format β€” when a framework ships with `Version: ""`, downstream code and tests break silently. +7. **CSV column names from `models.py` are public API.** Don't rename a field without migrating downstream consumers β€” CSV headers change. +8. **Upstream YAML multi-line scalars** (`|` block scalars) preserve newlines. Collapse to single-line with `" ".join(value.split())` before writing to JSON. +9. **Upstream catalogs can use multiple shapes.** FINOS CCC uses `control-families: [...]` in most catalogs but `controls: [...]` at the top level in `storage/object`. Any sync script must handle both or silently drop entire catalogs. +10. **Foreign-prefix AR ids.** Upstream sometimes "imports" requirements from one catalog into another by keeping the original id prefix (e.g., `CCC.AuditLog.CN08.AR01` appearing under `CCC.Logging.CN03`). Prowler's compliance model requires unique ids within a catalog β€” rewrite the foreign id to fit the parent control: `CCC.AuditLog.CN08.AR01` (inside `CCC.Logging.CN03`) β†’ `CCC.Logging.CN03.AR01`. +11. **Genuine upstream id collisions.** Sometimes upstream has a real typo where two different requirements share the same id (e.g., `CCC.Core.CN14.AR02` defined twice for 30-day and 14-day backup variants). Renumber the second copy to the next free AR number. Preserve check mappings by matching on `(Section, frozenset(Applicability))` since the renumbered id won't match by id. +12. **`COMPLIANCE_CLASS_MAP` in `export.py` uses `startswith` predicates** for all modern frameworks. Exact match (`name == "ccc_aws"`) is an anti-pattern β€” it was present for CCC until April 2026 and was the reason CCC couldn't have versioned variants. +13. **Pre-validate every check id** against the per-provider inventory before writing the JSON. A typo silently creates an unreferenced check that will fail when findings try to map to it. The audit script MUST abort with stderr listing typos, not swallow them. +14. **REPLACE is better than PATCH** for audit decisions. Encoding every mapping explicitly makes the audit reproducible and surfaces hidden assumptions from the legacy data. A PATCH system that adds/removes is too easy to forget. +15. **When no check applies, MANUAL is correct.** Do not pad mappings with tangential checks "just in case". Prowler's compliance reports are meant to be actionable β€” padding them with noise breaks that. Honest manual reqs can be mapped later when new checks land. +16. **UI groups by `Attributes[0].FamilyName` and `Attributes[0].Section`.** If FamilyName has inconsistent variants within the same JSON (e.g., "Logging & Monitoring" vs "Logging and Monitoring"), the UI renders them as separate categories. Section empty β†’ the requirement falls into an orphan control with label "". Normalize before shipping. +17. **Provider coverage is asymmetric.** AWS has dense coverage (~586 checks across 80+ services): in-transit encryption, IAM, database encryption, backup. Azure (~167 checks) and GCP (~102 checks) are thinner especially for in-transit encryption, mTLS, and ML/AI. Accept the asymmetry in mappings β€” don't force GCP parity where Prowler genuinely can't verify. + +--- + +## Useful One-Liners + +```bash +# Count requirements per service prefix (CCC, CIS sections, etc.) +jq -r '.Requirements[].Id | split(".")[1]' prowler/compliance/aws/ccc_aws.json | sort | uniq -c + +# Find duplicate requirement IDs +jq -r '.Requirements[].Id' file.json | sort | uniq -d + +# Count manual requirements (no checks) +jq '[.Requirements[] | select((.Checks | length) == 0)] | length' file.json + +# List all unique check references in a framework +jq -r '.Requirements[].Checks[]' file.json | sort -u + +# List all unique Sections (to spot inconsistency) +jq '[.Requirements[].Attributes[0].Section] | unique' file.json + +# List all unique FamilyNames (to spot inconsistency) +jq '[.Requirements[].Attributes[0].FamilyName] | unique' file.json + +# Diff requirement ids between two versions of the same framework +diff <(jq -r '.Requirements[].Id' a.json | sort) <(jq -r '.Requirements[].Id' b.json | sort) + +# Find where a check id is used across all frameworks +grep -rl "my_check_name" prowler/compliance/ + +# Check if a Prowler check exists +find prowler/providers/aws/services -name "{check_id}.metadata.json" + +# Validate a JSON with Pydantic +python -c "from prowler.lib.check.compliance_models import Compliance; print(Compliance.parse_file('prowler/compliance/aws/ccc_aws.json').Framework)" +``` + +--- + ## Best Practices 1. **Requirement IDs**: Follow the original framework numbering exactly (e.g., "1.1", "A.5.1", "T1190", "ac_2_1") -2. **Check Mapping**: Map to existing checks when possible. Use `Checks: []` for manual-only requirements +2. **Check Mapping**: Map to existing checks when possible. Use `Checks: []` for manual-only requirements β€” honest MANUAL beats padded coverage 3. **Completeness**: Include all framework requirements, even those without automated checks -4. **Version Control**: Include framework version in `Name` and `Version` fields +4. **Version Control**: Include framework version in `Name` and `Version` fields. **Never leave `Version: ""`** β€” it breaks `get_check_compliance()` key format 5. **File Naming**: Use format `{framework}_{version}_{provider}.json` -6. **Validation**: Prowler validates JSON against Pydantic models at startup - invalid JSON will cause errors +6. **Validation**: Prowler validates JSON against Pydantic models at startup β€” invalid JSON will cause errors +7. **Pre-validate check ids** against the provider's `*.metadata.json` inventory before every commit +8. **Normalize FamilyName and Section** to avoid inconsistent UI tree branches +9. **Register everywhere**: SDK model (if needed) β†’ `compliance.py` dispatcher β†’ `__main__.py` CLI writer β†’ `export.py` API map β†’ UI mapper. Skipping any layer results in silent failures +10. **Audit, don't pad**: when reviewing mappings, apply the golden rule β€” the check's title/risk MUST literally describe what the requirement text says. Tangential relation doesn't count ## Commands @@ -482,11 +1005,46 @@ prowler aws --compliance cis_5.0_aws -M csv json html ## Code References -- **Compliance Models:** `prowler/lib/check/compliance_models.py` -- **Compliance Processing:** `prowler/lib/check/compliance.py` -- **Compliance Output:** `prowler/lib/outputs/compliance/` +### Layer 1 β€” SDK / Core +- **Compliance Models:** `prowler/lib/check/compliance_models.py` (Pydantic v1 model tree) +- **Compliance Processing / Linker:** `prowler/lib/check/compliance.py` (`get_check_compliance`, `update_checks_metadata_with_compliance`) +- **Check Utils:** `prowler/lib/check/utils.py` (`list_compliance_modules`) + +### Layer 2 β€” JSON Catalogs +- **Framework JSONs:** `prowler/compliance/{provider}/` (auto-discovered via directory walk) + +### Layer 3 β€” Output Formatters +- **Per-framework folders:** `prowler/lib/outputs/compliance/{framework}/` +- **Shared base class:** `prowler/lib/outputs/compliance/compliance_output.py` (`ComplianceOutput` + `batch_write_data_to_file`) +- **CLI table dispatcher:** `prowler/lib/outputs/compliance/compliance.py` (`display_compliance_table`) +- **Finding model:** `prowler/lib/outputs/finding.py` (**do not import transitively from table dispatcher files β€” circular import**) +- **CLI writer:** `prowler/__main__.py` (per-provider `elif compliance_name.startswith(...)` branches that instantiate per-provider classes) + +### Layer 4 β€” API / UI +- **API lazy loader:** `api/src/backend/api/compliance.py` (`LazyComplianceTemplate`, `LazyChecksMapping`) +- **API export dispatcher:** `api/src/backend/tasks/jobs/export.py` (`COMPLIANCE_CLASS_MAP` with `startswith` predicates) +- **UI framework router:** `ui/lib/compliance/compliance-mapper.ts` +- **UI per-framework mapper:** `ui/lib/compliance/{framework}.tsx` +- **UI detail panel:** `ui/components/compliance/compliance-custom-details/{framework}-details.tsx` +- **UI types:** `ui/types/compliance.ts` +- **UI icon:** `ui/components/icons/compliance/{framework}.svg` + registration in `IconCompliance.tsx` + +### Tests +- **Output formatter tests:** `tests/lib/outputs/compliance/{framework}/{framework}_{provider}_test.py` +- **Shared fixtures:** `tests/lib/outputs/compliance/fixtures.py` ## Resources -- **Templates:** See [assets/](assets/) for framework JSON templates +- **JSON Templates:** See [assets/](assets/) for framework JSON templates (cis, ens, iso27001, mitre_attack, prowler_threatscore, generic) +- **Config-driven compliance sync** (any upstream-backed framework): + - [assets/sync_framework.py](assets/sync_framework.py) β€” generic runner. Loads a YAML config, dynamically imports the declared parser, applies generic post-processing (id uniqueness safety net, `FamilyName` normalization, legacy check-mapping preservation with config-driven fallback keys), and writes the provider JSONs with Pydantic post-validation. Framework-agnostic β€” works for any compliance framework. + - [assets/configs/ccc.yaml](assets/configs/ccc.yaml) β€” canonical config example (FINOS CCC v2025.10). Copy and adapt for new frameworks. + - [assets/parsers/finos_ccc.py](assets/parsers/finos_ccc.py) β€” FINOS CCC YAML parser. Handles both upstream shapes (`control-families` and top-level `controls`), foreign-prefix AR rewriting, and genuine collision renumbering. Exposes `parse_upstream(config) -> list[dict]`. + - [assets/parsers/](assets/parsers/) β€” add new parser modules here for unfamiliar upstream formats (NIST OSCAL JSON, MITRE STIX, CIS Benchmarks, etc.). Each parser is a `{name}.py` file implementing `parse_upstream(config) -> list[dict]` with guaranteed-unique ids. +- **Reusable audit tooling** (added April 2026 after the FINOS CCC v2025.10 sync): + - [assets/audit_framework_template.py](assets/audit_framework_template.py) β€” explicit REPLACE decision ledger with pre-validation against the per-provider inventory. Drop-in template for auditing any framework. + - [assets/query_checks.py](assets/query_checks.py) β€” keyword/service/id query helper over `/tmp/checks_{provider}.json`. + - [assets/dump_section.py](assets/dump_section.py) β€” dumps every AR for a given id prefix across all 3 providers with current check mappings. + - [assets/build_inventory.py](assets/build_inventory.py) β€” generates `/tmp/checks_{provider}.json` from `*.metadata.json` files. - **Documentation:** See [references/compliance-docs.md](references/compliance-docs.md) for additional resources +- **Related skill:** [prowler-compliance-review](../prowler-compliance-review/SKILL.md) β€” PR review checklist and validator script for compliance framework PRs diff --git a/skills/prowler-compliance/assets/audit_framework_template.py b/skills/prowler-compliance/assets/audit_framework_template.py new file mode 100644 index 0000000000..f2d58603d7 --- /dev/null +++ b/skills/prowler-compliance/assets/audit_framework_template.py @@ -0,0 +1,207 @@ +#!/usr/bin/env python3 +""" +Cloud-auditor pass template for any Prowler compliance framework. + +Encode explicit REPLACE decisions per (requirement_id, provider) pair below. +Each decision FULLY overwrites the legacy Checks list for that requirement. + +Workflow: + 1. Run build_inventory.py first to cache per-provider check metadata. + 2. Run dump_section.py to see current mappings for the catalog you're auditing. + 3. Fill in DECISIONS below with explicit check lists. + 4. Run this script β€” it pre-validates every check id against the inventory + and aborts with stderr listing typos before writing. + +Decision rules (apply as a hostile cloud auditor): + - The Prowler check's title/risk MUST literally describe what the AR text says. + "Related" is not enough. + - If no check actually addresses the requirement, leave `[]` (= MANUAL). + HONEST MANUAL is worth more than padded coverage. + - Missing provider key = leave the legacy mapping untouched. + - Empty list `[]` = explicitly MANUAL (overwrites legacy). + +Usage: + # 1. Copy this file to /tmp/audit_.py and fill in DECISIONS + # 2. Edit FRAMEWORK_KEY below to match your framework file naming + # 3. Run: + python /tmp/audit_.py +""" +from __future__ import annotations + +import json +import sys +from pathlib import Path + +# --------------------------------------------------------------------------- +# Configure for your framework +# --------------------------------------------------------------------------- + +# Framework file basename inside prowler/compliance/{provider}/. +# If your framework is called "cis_5.0_aws.json", FRAMEWORK_KEY is "cis_5.0". +# If the file is "ccc_aws.json", FRAMEWORK_KEY is "ccc". +FRAMEWORK_KEY = "ccc" + +# Which providers to apply decisions to. +PROVIDERS = ["aws", "azure", "gcp"] + +PROWLER_DIR = Path("prowler/compliance") +CHECK_INV = {prov: Path(f"/tmp/checks_{prov}.json") for prov in PROVIDERS} + + +# --------------------------------------------------------------------------- +# DECISIONS β€” encode one entry per requirement you want to audit +# --------------------------------------------------------------------------- + +# DECISIONS[requirement_id][provider] = list[str] of check ids +# See SKILL.md β†’ "Audit Reference Table: Requirement Text β†’ Prowler Checks" +# for a comprehensive mapping cheat sheet built from a 172-AR CCC audit. + +DECISIONS: dict[str, dict[str, list[str]]] = {} + +# ---- Example entries (delete and replace with your own) ---- + +# Example 1: TLS in transit enforced (non-SSH traffic) +# DECISIONS["CCC.Core.CN01.AR01"] = { +# "aws": [ +# "cloudfront_distributions_https_enabled", +# "cloudfront_distributions_origin_traffic_encrypted", +# "s3_bucket_secure_transport_policy", +# "elbv2_ssl_listeners", +# "rds_instance_transport_encrypted", +# "kafka_cluster_in_transit_encryption_enabled", +# "redshift_cluster_in_transit_encryption_enabled", +# "opensearch_service_domains_https_communications_enforced", +# ], +# "azure": [ +# "storage_secure_transfer_required_is_enabled", +# "app_minimum_tls_version_12", +# "postgresql_flexible_server_enforce_ssl_enabled", +# "sqlserver_recommended_minimal_tls_version", +# ], +# "gcp": [ +# "cloudsql_instance_ssl_connections", +# ], +# } + +# Example 2: MANUAL β€” no Prowler check exists +# DECISIONS["CCC.Core.CN01.AR07"] = { +# "aws": [], # no IANA port/protocol check exists in Prowler +# "azure": [], +# "gcp": [], +# } + +# Example 3: Reuse a decision for multiple sibling ARs +# DECISIONS["CCC.ObjStor.CN05.AR02"] = DECISIONS["CCC.ObjStor.CN05.AR01"] + + +# --------------------------------------------------------------------------- +# Driver β€” do not edit below +# --------------------------------------------------------------------------- + +def load_inventory(provider: str) -> dict: + path = CHECK_INV[provider] + if not path.exists(): + raise SystemExit( + f"Check inventory missing: {path}\n" + f"Run: python skills/prowler-compliance/assets/build_inventory.py {provider}" + ) + with open(path) as f: + return json.load(f) + + +def resolve_json_path(provider: str) -> Path: + """Resolve the JSON file path for a given provider. + + Handles both shapes: {FRAMEWORK_KEY}_{provider}.json (ccc_aws.json) and + cases where FRAMEWORK_KEY already contains the provider suffix. + """ + candidates = [ + PROWLER_DIR / provider / f"{FRAMEWORK_KEY}_{provider}.json", + PROWLER_DIR / provider / f"{FRAMEWORK_KEY}.json", + ] + for c in candidates: + if c.exists(): + return c + raise SystemExit( + f"Could not find framework JSON for provider={provider} " + f"with FRAMEWORK_KEY={FRAMEWORK_KEY}. Tried: {candidates}" + ) + + +def plan_for_provider( + provider: str, +) -> tuple[Path, dict, tuple[int, int, int], list[tuple[str, str]]]: + """Build the updated JSON for one provider without writing it. + + Returns (path, mutated_data, (touched, added, removed), unknowns). + Writing is deferred to a second pass so that a typo in any provider + aborts the whole run before any file on disk changes. + """ + path = resolve_json_path(provider) + with open(path) as f: + data = json.load(f) + inv = load_inventory(provider) + + touched = 0 + add_count = 0 + rm_count = 0 + unknown: list[tuple[str, str]] = [] + + for req in data["Requirements"]: + rid = req["Id"] + if rid not in DECISIONS or provider not in DECISIONS[rid]: + continue + new_checks = list(dict.fromkeys(DECISIONS[rid][provider])) + for c in new_checks: + if c not in inv: + unknown.append((rid, c)) + before = set(req.get("Checks") or []) + after = set(new_checks) + rm_count += len(before - after) + add_count += len(after - before) + req["Checks"] = new_checks + touched += 1 + + return path, data, (touched, add_count, rm_count), unknown + + +def main() -> int: + if not DECISIONS: + print("No DECISIONS encoded. Fill in the DECISIONS dict and re-run.") + return 1 + print(f"Applying {len(DECISIONS)} decisions to framework '{FRAMEWORK_KEY}'...") + + # Pass 1: validate every provider before touching disk. A typo in any + # provider must abort the run before ANY file has been rewritten. + plans: list[tuple[str, Path, dict, tuple[int, int, int]]] = [] + all_unknown: list[tuple[str, str, str]] = [] + for provider in PROVIDERS: + path, data, counts, unknown = plan_for_provider(provider) + for rid, c in unknown: + all_unknown.append((provider, rid, c)) + plans.append((provider, path, data, counts)) + + if all_unknown: + print("\n!! UNKNOWN CHECK IDS (typos?):", file=sys.stderr) + for provider, rid, c in all_unknown: + print(f" {provider} {rid} -> {c}", file=sys.stderr) + print( + "\nAborting: fix the check ids above and re-run. " + "No files were modified.", + file=sys.stderr, + ) + return 2 + + # Pass 2: all providers validated cleanly β€” write. + for provider, path, data, (touched, added, removed) in plans: + with open(path, "w") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + f.write("\n") + print( + f" {provider}: touched={touched} added={added} removed={removed}" + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/skills/prowler-compliance/assets/build_inventory.py b/skills/prowler-compliance/assets/build_inventory.py new file mode 100644 index 0000000000..f743aa75a5 --- /dev/null +++ b/skills/prowler-compliance/assets/build_inventory.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +""" +Build a per-provider check inventory by scanning Prowler's check metadata files. + +Outputs one JSON per provider at /tmp/checks_{provider}.json with the shape: + { + "check_id": { + "service": "...", + "subservice": "...", + "resource": "...", + "severity": "...", + "title": "...", + "description": "...", + "risk": "..." + }, + ... + } + +This is the reference used by audit_framework_template.py for pre-validation +(every check id in the audit ledger must exist in the inventory) and by +query_checks.py for keyword/service lookup. + +Usage: + python skills/prowler-compliance/assets/build_inventory.py + # Or for a specific provider: + python skills/prowler-compliance/assets/build_inventory.py aws + +Output: + /tmp/checks_{provider}.json for every provider discovered under + prowler/providers/ with a services/ directory. +""" +from __future__ import annotations + +import json +import sys +from pathlib import Path + +PROVIDERS_ROOT = Path("prowler/providers") + + +def discover_providers() -> list[str]: + """Return every provider that currently has a services/ directory. + + Derived from the filesystem so new providers are picked up automatically + and stale hard-coded lists cannot drift from the repo. + """ + if not PROVIDERS_ROOT.exists(): + return [] + return sorted( + p.name + for p in PROVIDERS_ROOT.iterdir() + if p.is_dir() and (p / "services").is_dir() + ) + + +def build_for_provider(provider: str) -> dict: + inventory: dict[str, dict] = {} + base = Path(f"prowler/providers/{provider}/services") + if not base.exists(): + print(f" skip {provider}: no services directory", file=sys.stderr) + return inventory + for meta_path in base.rglob("*.metadata.json"): + try: + with open(meta_path) as f: + data = json.load(f) + except Exception as exc: + print(f" warn: cannot parse {meta_path}: {exc}", file=sys.stderr) + continue + cid = data.get("CheckID") or meta_path.stem.replace(".metadata", "") + inventory[cid] = { + "service": data.get("ServiceName", ""), + "subservice": data.get("SubServiceName", ""), + "resource": data.get("ResourceType", ""), + "severity": data.get("Severity", ""), + "title": data.get("CheckTitle", ""), + "description": data.get("Description", ""), + "risk": data.get("Risk", ""), + } + return inventory + + +def main() -> int: + providers = sys.argv[1:] or discover_providers() + if not providers: + print( + f"error: no providers found under {PROVIDERS_ROOT}/", + file=sys.stderr, + ) + return 1 + for provider in providers: + inv = build_for_provider(provider) + out_path = Path(f"/tmp/checks_{provider}.json") + with open(out_path, "w") as f: + json.dump(inv, f, indent=2) + print(f" {provider}: {len(inv)} checks β†’ {out_path}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/skills/prowler-compliance/assets/configs/ccc.yaml b/skills/prowler-compliance/assets/configs/ccc.yaml new file mode 100644 index 0000000000..deb757ffc9 --- /dev/null +++ b/skills/prowler-compliance/assets/configs/ccc.yaml @@ -0,0 +1,120 @@ +# FINOS Common Cloud Controls (CCC) sync config for sync_framework.py. +# +# Usage: +# python skills/prowler-compliance/assets/sync_framework.py \ +# skills/prowler-compliance/assets/configs/ccc.yaml +# +# Prerequisite: run the upstream fetch step from SKILL.md Workflow A Step 1 to +# populate upstream.dir with the raw FINOS catalog YAML files. + +framework: + name: CCC + display_name: Common Cloud Controls Catalog (CCC) + version: v2025.10 + # The {provider_display} placeholder is replaced at output time with the + # per-provider display string from the providers list below. + description_template: "Common Cloud Controls Catalog (CCC) for {provider_display}" + +providers: + - key: aws + display: AWS + - key: azure + display: Azure + - key: gcp + display: GCP + +output: + # Supported placeholders: {provider}, {framework}, {version}. + # For versioned frameworks like CIS the template would be + # "prowler/compliance/{provider}/cis_{version}_{provider}.json". + path_template: "prowler/compliance/{provider}/ccc_{provider}.json" + +upstream: + # Directory containing the cached FINOS catalog YAMLs. Populate via + # SKILL.md Workflow A Step 1 (gh api raw download commands). + dir: /tmp/ccc_upstream + fetch_docs: "See SKILL.md Workflow A Step 1 for gh api fetch commands" + +parser: + # Name of the parser module under parsers/ (loaded dynamically by the + # runner). For FINOS CCC YAML this is always finos_ccc. + module: finos_ccc + + # FINOS CCC catalog files in load order. Core first so its ARs render + # first in the output JSON. + catalog_files: + - core_ccc.yaml + - management_auditlog.yaml + - management_logging.yaml + - management_monitoring.yaml + - storage_object.yaml + - networking_loadbalancer.yaml + - networking_vpc.yaml + - crypto_key.yaml + - crypto_secrets.yaml + - database_warehouse.yaml + - database_vector.yaml + - database_relational.yaml + - devtools_build.yaml + - devtools_container-registry.yaml + - identity_iam.yaml + - ai-ml_gen-ai.yaml + - ai-ml_mlde.yaml + - app-integration_message.yaml + - compute_serverless-computing.yaml + + # Shape-2 catalogs (storage/object) reference the family via id only + # (e.g. "CCC.ObjStor.Data") with no human-readable title or description + # in the YAML. Map the suffix (after the last dot) to a canonical title + # and description so the generated JSON has consistent FamilyName fields + # regardless of upstream shape. + family_id_title: + Data: Data + IAM: Identity and Access Management + Identity: Identity and Access Management + Encryption: Encryption + Logging: Logging and Monitoring + Network: Network Security + Availability: Availability + Integrity: Integrity + Confidentiality: Confidentiality + family_id_description: + Data: "The Data control family ensures the confidentiality, integrity, availability, and sovereignty of data across its lifecycle." + IAM: "The Identity and Access Management control family ensures that only trusted and authenticated entities can access resources." + +post_processing: + # Collapse FamilyName variants that appear inconsistently across upstream + # catalogs. The Prowler UI groups by Attributes[0].FamilyName exactly, + # so each variant would otherwise become a separate tree branch. + family_name_normalization: + "Logging & Monitoring": "Logging and Monitoring" + "Logging and Metrics Publication": "Logging and Monitoring" + + # Preserve existing Checks lists from the legacy Prowler JSON when + # regenerating. The runner builds two lookup tables from the legacy + # output: a primary index by Id, and fallback indexes composed of + # attribute field names. + # + # primary_key: the top-level requirement field to use as the primary + # lookup key (almost always "Id") + # fallback_keys: a list of composite keys. Each composite key is a list + # of Attributes[0] field names to join into a tuple. List-valued fields + # (like Applicability) are frozen to frozenset so the tuple is hashable. + # + # CCC uses (Section, Applicability) because Applicability is a CCC-only + # top-level attribute field. CIS would use (Section, Profile). NIST would + # use (ItemId,). The fallback is how renumbered or rewritten ids still + # recover their check mappings. + # + # legacy_path_template (optional): path to read legacy Checks FROM. + # Defaults to output.path_template, which is correct for unversioned + # frameworks (like CCC) where regeneration overwrites the same file. + # For versioned frameworks that write to a new file on each version + # bump (e.g. cis_5.1_aws.json while the legacy mappings live in + # cis_5.0_aws.json), set this to the previous-version path so Checks + # are preserved instead of lost: + # legacy_path_template: "prowler/compliance/{provider}/cis_5.0_{provider}.json" + check_preservation: + primary_key: Id + fallback_keys: + - [Section, Applicability] diff --git a/skills/prowler-compliance/assets/dump_section.py b/skills/prowler-compliance/assets/dump_section.py new file mode 100644 index 0000000000..ca2fff0e1b --- /dev/null +++ b/skills/prowler-compliance/assets/dump_section.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +""" +Dump every requirement of a compliance framework for a given id prefix across +providers, with their current Check mappings. + +Useful for reviewing a whole control family in one pass before encoding audit +decisions in audit_framework_template.py. + +Usage: + # Dump all CCC.Core requirements across aws/azure/gcp + python skills/prowler-compliance/assets/dump_section.py ccc "CCC.Core." + + # Dump all CIS 5.0 section 1 requirements for AWS only + python skills/prowler-compliance/assets/dump_section.py cis_5.0_aws "1." + +Arguments: + framework_key: file prefix inside prowler/compliance/{provider}/ without + the provider suffix. Examples: + - "ccc" β†’ loads ccc_aws.json / ccc_azure.json / ccc_gcp.json + - "cis_5.0_aws" β†’ loads only that one file + - "iso27001_2022" β†’ loads all providers + id_prefix: Requirement id prefix to filter by (e.g. "CCC.Core.", + "1.1.", "A.5."). +""" +from __future__ import annotations + +import json +import sys +from collections import defaultdict +from pathlib import Path + +PROWLER_COMPLIANCE_DIR = Path("prowler/compliance") + + +def main() -> int: + if len(sys.argv) < 3: + print(__doc__) + return 1 + + framework_key = sys.argv[1] + id_prefix = sys.argv[2] + + # Find matching JSON files across all providers + candidates: list[tuple[str, Path]] = [] + for prov_dir in sorted(PROWLER_COMPLIANCE_DIR.iterdir()): + if not prov_dir.is_dir(): + continue + for json_path in prov_dir.glob("*.json"): + stem = json_path.stem + if stem == framework_key or stem.startswith(f"{framework_key}_") \ + or stem == f"{framework_key}_{prov_dir.name}": + candidates.append((prov_dir.name, json_path)) + + if not candidates: + print(f"No files matching '{framework_key}'", file=sys.stderr) + return 2 + + discovered_providers = sorted({prov for prov, _ in candidates}) + + by_id: dict[str, dict] = defaultdict(dict) + for prov, path in candidates: + with open(path) as f: + data = json.load(f) + for req in data["Requirements"]: + if req["Id"].startswith(id_prefix): + by_id[req["Id"]][prov] = { + "desc": req.get("Description", ""), + "sec": (req.get("Attributes") or [{}])[0].get("Section", ""), + "obj": (req.get("Attributes") or [{}])[0].get( + "SubSectionObjective", "" + ), + "checks": req.get("Checks") or [], + } + + for ar_id in sorted(by_id): + rows = by_id[ar_id] + sample = next(iter(rows.values())) + print(f"\n### {ar_id}") + print(f" desc: {sample['desc']}") + if sample["sec"]: + print(f" sec : {sample['sec']}") + if sample["obj"]: + print(f" obj : {sample['obj']}") + for prov in discovered_providers: + if prov in rows: + checks = rows[prov]["checks"] + print(f" {prov}: ({len(checks)}) {checks}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/skills/prowler-compliance/assets/parsers/__init__.py b/skills/prowler-compliance/assets/parsers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/skills/prowler-compliance/assets/parsers/finos_ccc.py b/skills/prowler-compliance/assets/parsers/finos_ccc.py new file mode 100644 index 0000000000..a613b15857 --- /dev/null +++ b/skills/prowler-compliance/assets/parsers/finos_ccc.py @@ -0,0 +1,223 @@ +""" +FINOS Common Cloud Controls (CCC) YAML parser. + +Reads cached upstream YAML files and emits Prowler-format requirements +(``{Id, Description, Attributes: [...], Checks: []}``). This module is +agnostic to providers, JSON output paths, framework metadata and legacy +check-mapping preservation β€” those are handled by ``sync_framework.py``. + +Contract +-------- +``parse_upstream(config: dict) -> list[dict]`` + Returns a list of Prowler-format requirement dicts with **guaranteed + unique ids**. Foreign-prefix AR rewriting and genuine collision + renumbering both happen inside this module β€” the runner treats id + uniqueness as a contract violation, not as something to fix. + +Config keys consumed +-------------------- +This parser reads the following config entries (the rest of the config is +opaque to it): + +- ``upstream.dir`` β€” directory containing the cached YAMLs +- ``parser.catalog_files`` β€” ordered list of YAML filenames to load +- ``parser.family_id_title`` β€” suffix β†’ canonical family title (shape 2) +- ``parser.family_id_description`` β€” suffix β†’ family description (shape 2) + +Upstream shapes +--------------- +FINOS CCC catalogs come in two shapes: + +1. ``control-families: [{title, description, controls: [...]}]`` + (used by most catalogs) +2. ``controls: [{id, family: "CCC.X.Y", ...}]`` (no families wrapper; used + by ``storage/object``). The ``family`` field references a family id with + no human-readable title in the file β€” the title/description come from + ``config.parser.family_id_title`` / ``family_id_description``. + +Id rewriting rules +------------------ +- **Foreign-prefix rewriting**: upstream intentionally aliases requirements + across catalogs by keeping the original prefix (e.g. ``CCC.AuditLog.CN08.AR01`` + appears nested under ``CCC.Logging.CN03``). Prowler requires unique ids + within a catalog file, so we rename the AR to fit its parent control: + ``CCC.Logging.CN03.AR01``. See ``rewrite_ar_id()``. +- **Genuine collision renumbering**: sometimes upstream has a real typo + where two distinct requirements share the same id (e.g. + ``CCC.Core.CN14.AR02`` appears twice for 30-day and 14-day backup variants). + The second copy is renumbered to the next free AR number within the + control. See the ``seen_ids`` logic in ``emit_requirement()``. +""" +from __future__ import annotations + +from pathlib import Path + +import yaml + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def clean(value: str | None) -> str: + """Trim and collapse internal whitespace/newlines into single spaces. + + Upstream YAML uses ``|`` block scalars that preserve newlines; Prowler + stores descriptions as single-line text. + """ + if not value: + return "" + return " ".join(value.split()) + + +def flatten_mappings(mappings): + """Convert upstream ``{reference-id, entries: [{reference-id, ...}]}`` to + Prowler's ``{ReferenceId, Identifiers: [...]}``. + """ + if not mappings: + return [] + out = [] + for m in mappings: + ids = [] + for entry in m.get("entries") or []: + eid = entry.get("reference-id") + if eid: + ids.append(eid) + out.append({"ReferenceId": m.get("reference-id", ""), "Identifiers": ids}) + return out + + +def ar_prefix(ar_id: str) -> str: + """Return the first three dot-segments of an AR id (the parent control). + + e.g. ``CCC.Core.CN01.AR01`` -> ``CCC.Core.CN01``. + """ + return ".".join(ar_id.split(".")[:3]) + + +def rewrite_ar_id(parent_control_id: str, original_ar_id: str, ar_index: int) -> str: + """If an AR's id doesn't share its parent control's prefix, rename it. + + Example + ------- + parent ``CCC.Logging.CN03`` + AR id ``CCC.AuditLog.CN08.AR01`` with + index 0 -> ``CCC.Logging.CN03.AR01``. + """ + if ar_prefix(original_ar_id) == parent_control_id: + return original_ar_id + return f"{parent_control_id}.AR{ar_index + 1:02d}" + + +def emit_requirement( + control: dict, + family_name: str, + family_desc: str, + seen_ids: set[str], + requirements: list[dict], +) -> None: + """Translate one FINOS control + its assessment-requirements into + Prowler-format requirement dicts and append them to ``requirements``. + + Applies foreign-prefix rewriting and genuine-collision renumbering so + the final list is guaranteed to have unique ids. + """ + control_id = clean(control.get("id")) + control_title = clean(control.get("title")) + section = f"{control_id} {control_title}".strip() + objective = clean(control.get("objective")) + threat_mappings = flatten_mappings(control.get("threat-mappings")) + guideline_mappings = flatten_mappings(control.get("guideline-mappings")) + ars = control.get("assessment-requirements") or [] + for idx, ar in enumerate(ars): + raw_id = clean(ar.get("id")) + if not raw_id: + continue + new_id = rewrite_ar_id(control_id, raw_id, idx) + # Renumber on genuine upstream collision (find next free AR number) + if new_id in seen_ids: + base = ".".join(new_id.split(".")[:-1]) + n = 1 + while f"{base}.AR{n:02d}" in seen_ids: + n += 1 + new_id = f"{base}.AR{n:02d}" + seen_ids.add(new_id) + + requirements.append( + { + "Id": new_id, + "Description": clean(ar.get("text")), + "Attributes": [ + { + "FamilyName": family_name, + "FamilyDescription": family_desc, + "Section": section, + "SubSection": "", + "SubSectionObjective": objective, + "Applicability": list(ar.get("applicability") or []), + "Recommendation": clean(ar.get("recommendation")), + "SectionThreatMappings": threat_mappings, + "SectionGuidelineMappings": guideline_mappings, + } + ], + "Checks": [], + } + ) + + +# --------------------------------------------------------------------------- +# Public entry point +# --------------------------------------------------------------------------- + + +def parse_upstream(config: dict) -> list[dict]: + """Walk upstream YAMLs and emit Prowler-format requirements. + + Handles both top-level shapes (``control-families`` and ``controls``). + Ids are guaranteed unique in the returned list. + """ + upstream_dir = Path(config["upstream"]["dir"]) + parser_cfg = config.get("parser") or {} + catalog_files = parser_cfg.get("catalog_files") or [] + family_id_title = parser_cfg.get("family_id_title") or {} + family_id_description = parser_cfg.get("family_id_description") or {} + + requirements: list[dict] = [] + seen_ids: set[str] = set() + + for filename in catalog_files: + path = upstream_dir / filename + if not path.exists(): + # parser.catalog_files is the closed set of upstream catalogs + # that define the framework. Silently skipping a missing file + # would emit valid-looking JSON with part of the framework + # dropped, defeating the whole point of a canonical sync. + raise FileNotFoundError( + f"upstream catalog file not found: {path}\n" + f" hint: refresh the upstream cache (see SKILL.md Workflow A " + f"Step 1), or remove {filename!r} from parser.catalog_files " + f"if it has been retired upstream." + ) + with open(path) as f: + doc = yaml.safe_load(f) or {} + + # Shape 1: control-families wrapper + for family in doc.get("control-families") or []: + family_name = clean(family.get("title")) + family_desc = clean(family.get("description")) + for control in family.get("controls") or []: + emit_requirement( + control, family_name, family_desc, seen_ids, requirements + ) + + # Shape 2: top-level controls with family reference id + for control in doc.get("controls") or []: + family_ref = clean(control.get("family")) + suffix = family_ref.split(".")[-1] if family_ref else "" + family_name = family_id_title.get(suffix, suffix or "Data") + family_desc = family_id_description.get(suffix, "") + emit_requirement( + control, family_name, family_desc, seen_ids, requirements + ) + + return requirements diff --git a/skills/prowler-compliance/assets/query_checks.py b/skills/prowler-compliance/assets/query_checks.py new file mode 100644 index 0000000000..46405be982 --- /dev/null +++ b/skills/prowler-compliance/assets/query_checks.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +""" +Keyword/service/id lookup over a Prowler check inventory produced by +build_inventory.py. + +Usage: + # Keyword AND-search across id + title + risk + description + python skills/prowler-compliance/assets/query_checks.py aws encryption transit + + # Show all checks for a service + python skills/prowler-compliance/assets/query_checks.py aws --service iam + + # Show full metadata for one check id + python skills/prowler-compliance/assets/query_checks.py aws --id kms_cmk_rotation_enabled +""" +from __future__ import annotations + +import json +import sys + + +def main() -> int: + if len(sys.argv) < 3: + print(__doc__) + return 1 + + provider = sys.argv[1] + try: + with open(f"/tmp/checks_{provider}.json") as f: + inv = json.load(f) + except FileNotFoundError: + print( + f"No inventory for {provider}. Run build_inventory.py first.", + file=sys.stderr, + ) + return 2 + + if sys.argv[2] == "--service": + if len(sys.argv) < 4: + print("usage: --service ") + return 1 + svc = sys.argv[3] + hits = [cid for cid in sorted(inv) if inv[cid].get("service") == svc] + for cid in hits: + print(f" {cid}") + print(f" {inv[cid].get('title', '')}") + print(f"\n{len(hits)} checks in service '{svc}'") + elif sys.argv[2] == "--id": + if len(sys.argv) < 4: + print("usage: --id ") + return 1 + cid = sys.argv[3] + if cid not in inv: + print(f"NOT FOUND: {cid}") + return 3 + m = inv[cid] + print(f"== {cid} ==") + print(f"service : {m.get('service')}") + print(f"severity: {m.get('severity')}") + print(f"resource: {m.get('resource')}") + print(f"title : {m.get('title')}") + print(f"desc : {m.get('description', '')[:500]}") + print(f"risk : {m.get('risk', '')[:500]}") + else: + keywords = [k.lower() for k in sys.argv[2:]] + hits = 0 + for cid in sorted(inv): + m = inv[cid] + blob = " ".join( + [ + cid, + m.get("title", ""), + m.get("risk", ""), + m.get("description", ""), + ] + ).lower() + if all(k in blob for k in keywords): + hits += 1 + print(f" {cid} [{m.get('service', '')}]") + print(f" {m.get('title', '')[:120]}") + print(f"\n{hits} matches for {' + '.join(keywords)}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/skills/prowler-compliance/assets/sync_framework.py b/skills/prowler-compliance/assets/sync_framework.py new file mode 100644 index 0000000000..9e070f2691 --- /dev/null +++ b/skills/prowler-compliance/assets/sync_framework.py @@ -0,0 +1,536 @@ +#!/usr/bin/env python3 +""" +Generic, config-driven compliance framework sync runner. + +Usage: + python skills/prowler-compliance/assets/sync_framework.py \ + skills/prowler-compliance/assets/configs/ccc.yaml + +Pipeline: + 1. Load and validate the YAML config (fail fast on missing or empty + required fields β€” notably ``framework.version``, which silently + breaks ``get_check_compliance()`` key construction if empty). + 2. Dynamically import the parser module declared in ``parser.module`` + (resolved as ``parsers.{name}`` under this script's directory). + 3. Call ``parser.parse_upstream(config) -> list[dict]`` to get raw + Prowler-format requirements. The parser owns all upstream-format + quirks (foreign-prefix AR rewriting, collision renumbering, shape + handling) and MUST return ids that are unique within the returned + list. + 4. **Safety net**: assert id uniqueness. The runner raises + ``ValueError`` on any duplicate β€” it does NOT silently renumber, + because mutating a canonical upstream id (e.g. CIS ``1.1.1`` or + NIST ``AC-2(1)``) would be catastrophic. + 5. Apply generic ``FamilyName`` normalization from + ``post_processing.family_name_normalization`` (optional). + 6. Preserve legacy ``Checks`` lists from the existing Prowler JSON + using a config-driven primary key + fallback key chain. CCC uses + ``(Section, Applicability)`` as fallback; CIS would use + ``(Section, Profile)``; NIST would use ``(ItemId,)``. + For versioned frameworks (e.g. ``cis__.json``) + where a version bump writes to a brand-new file, set + ``post_processing.check_preservation.legacy_path_template`` to + point at the previous version's file so its Checks are preserved + instead of silently lost. Defaults to ``output.path_template`` + when omitted, which is correct for unversioned frameworks. + 7. Wrap each provider's requirements in the framework metadata dict + built from the config templates. + 8. Write each provider's JSON to the path resolved from + ``output.path_template`` (supports ``{framework}``, ``{version}`` + and ``{provider}`` placeholders). + 9. Pydantic-validate the written JSON via ``Compliance.parse_file()`` + and report the load counts per provider. + +The runner is strictly generic β€” it never mentions CCC, knows nothing +about YAML shapes, and can handle any upstream-backed framework given a +parser module and a config file. +""" +from __future__ import annotations + +import importlib +import json +import sys +from pathlib import Path +from typing import Any + +import yaml + +# Make sibling `parsers/` package importable regardless of the runner's +# invocation directory. +_SCRIPT_DIR = Path(__file__).resolve().parent +if str(_SCRIPT_DIR) not in sys.path: + sys.path.insert(0, str(_SCRIPT_DIR)) + + +# --------------------------------------------------------------------------- +# Config loading and validation +# --------------------------------------------------------------------------- + + +class ConfigError(ValueError): + """Raised when the sync config is malformed or missing required fields.""" + + +def _require(cfg: dict, dotted_path: str) -> Any: + """Fetch a dotted-path key from nested dicts. Raises ConfigError on + missing or empty values (empty-string, empty-list, None).""" + current: Any = cfg + parts = dotted_path.split(".") + for i, part in enumerate(parts): + if not isinstance(current, dict) or part not in current: + raise ConfigError(f"config: missing required field '{dotted_path}'") + current = current[part] + if current in ("", None, [], {}): + raise ConfigError(f"config: field '{dotted_path}' must not be empty") + return current + + +def load_config(path: Path) -> dict: + if not path.exists(): + raise ConfigError(f"config file not found: {path}") + with open(path) as f: + cfg = yaml.safe_load(f) or {} + if not isinstance(cfg, dict): + raise ConfigError(f"config root must be a mapping, got {type(cfg).__name__}") + + # Required fields β€” fail fast. Empty Version in particular silently + # breaks get_check_compliance() key construction. + _require(cfg, "framework.name") + _require(cfg, "framework.display_name") + _require(cfg, "framework.version") + _require(cfg, "framework.description_template") + _require(cfg, "providers") + _require(cfg, "output.path_template") + _require(cfg, "upstream.dir") + _require(cfg, "parser.module") + _require(cfg, "post_processing.check_preservation.primary_key") + + providers = cfg["providers"] + if not isinstance(providers, list) or not providers: + raise ConfigError("config: 'providers' must be a non-empty list") + for idx, p in enumerate(providers): + if not isinstance(p, dict) or "key" not in p or "display" not in p: + raise ConfigError( + f"config: providers[{idx}] must have 'key' and 'display' fields" + ) + + return cfg + + +# --------------------------------------------------------------------------- +# Parser loading +# --------------------------------------------------------------------------- + + +def load_parser(parser_module_name: str): + try: + return importlib.import_module(f"parsers.{parser_module_name}") + except ImportError as exc: + raise ConfigError( + f"cannot import parser 'parsers.{parser_module_name}': {exc}" + ) from exc + + +# --------------------------------------------------------------------------- +# Post-processing: id uniqueness safety net +# --------------------------------------------------------------------------- + + +def assert_unique_ids(requirements: list[dict]) -> None: + """Enforce the parser contract: every requirement must have a unique Id. + + The runner never renumbers silently β€” a duplicate is a parser bug. + """ + seen: set[str] = set() + dups: list[str] = [] + for req in requirements: + rid = req.get("Id") + if not rid: + raise ValueError(f"requirement missing Id: {req}") + if rid in seen: + dups.append(rid) + seen.add(rid) + if dups: + raise ValueError( + f"parser returned duplicate requirement ids: {sorted(set(dups))}" + ) + + +# --------------------------------------------------------------------------- +# Post-processing: FamilyName normalization +# --------------------------------------------------------------------------- + + +def normalize_family_names(requirements: list[dict], norm_map: dict[str, str]) -> None: + """Apply ``Attributes[0].FamilyName`` normalization in place.""" + if not norm_map: + return + for req in requirements: + for attr in req.get("Attributes") or []: + name = attr.get("FamilyName") + if name in norm_map: + attr["FamilyName"] = norm_map[name] + + +# --------------------------------------------------------------------------- +# Post-processing: legacy check-mapping preservation +# --------------------------------------------------------------------------- + + +def _freeze(value: Any) -> Any: + """Make a value hashable for use in composite lookup keys. + + Lists become frozensets (order-insensitive match). Scalars pass through. + """ + if isinstance(value, list): + return frozenset(value) + return value + + +def _build_fallback_key(attrs: dict, field_names: list[str]) -> tuple | None: + """Build a composite tuple key from the given attribute field names. + + Returns None if any field is missing or falsy β€” that key will be + skipped (the lookup table just won't have an entry for it). + """ + parts = [] + for name in field_names: + if name not in attrs: + return None + value = attrs[name] + if value in ("", None, [], {}): + return None + parts.append(_freeze(value)) + return tuple(parts) + + +def load_legacy_check_maps( + legacy_path: Path, + primary_key: str, + fallback_keys: list[list[str]], +) -> tuple[dict[str, list[str]], list[dict[tuple, list[str]]]]: + """Read the existing Prowler JSON and build lookup tables for check + preservation. + + Fails fast on ambiguous preservation keys. If two distinct legacy + requirements share the same primary value or the same fallback tuple, + merging their ``Checks`` silently would corrupt the preserved mapping + for unrelated requirements. Raises ``ValueError`` listing every + conflict so the user can either dedupe the legacy data or strengthen + ``check_preservation`` in the sync config. + + Returns + ------- + by_primary : dict + ``{primary_value: [checks]}`` β€” e.g. ``{ar_id: [checks]}``. + by_fallback : list[dict] + One lookup dict per entry in ``fallback_keys``. Each maps a + composite tuple key to its preserved checks list. + """ + by_primary: dict[str, list[str]] = {} + by_fallback: list[dict[tuple, list[str]]] = [{} for _ in fallback_keys] + + if not legacy_path.exists(): + return by_primary, by_fallback + + with open(legacy_path) as f: + data = json.load(f) + + # Track which legacy requirement Ids contributed to each bucket so we + # can surface ambiguity after the scan completes. + primary_sources: dict[str, list[str]] = {} + fallback_sources: list[dict[tuple, list[str]]] = [{} for _ in fallback_keys] + + for req in data.get("Requirements") or []: + legacy_id = req.get("Id") or "" + checks = req.get("Checks") or [] + + pv = req.get(primary_key) + if pv: + primary_sources.setdefault(pv, []).append(legacy_id) + bucket = by_primary.setdefault(pv, []) + for c in checks: + if c not in bucket: + bucket.append(c) + + attributes = req.get("Attributes") or [] + if not attributes: + continue + attrs = attributes[0] + for i, field_names in enumerate(fallback_keys): + key = _build_fallback_key(attrs, field_names) + if key is None: + continue + fallback_sources[i].setdefault(key, []).append(legacy_id) + bucket = by_fallback[i].setdefault(key, []) + for c in checks: + if c not in bucket: + bucket.append(c) + + conflicts: list[str] = [] + for pv, ids in primary_sources.items(): + if len(ids) > 1: + conflicts.append( + f"primary_key={primary_key!r} value={pv!r} shared by {ids}" + ) + for i, field_names in enumerate(fallback_keys): + for key, ids in fallback_sources[i].items(): + if len(ids) > 1: + conflicts.append( + f"fallback_key={field_names} value={key!r} shared by {ids}" + ) + if conflicts: + details = "\n - ".join(conflicts) + raise ValueError( + f"ambiguous preservation keys in {legacy_path} β€” cannot " + f"faithfully preserve Checks across distinct requirements:\n" + f" - {details}\n" + f"Fix: dedupe the legacy JSON, or strengthen " + f"'post_processing.check_preservation' in the sync config " + f"(e.g. add a more discriminating field to fallback_keys)." + ) + + return by_primary, by_fallback + + +def lookup_preserved_checks( + req: dict, + by_primary: dict, + by_fallback: list[dict], + primary_key: str, + fallback_keys: list[list[str]], +) -> list[str]: + """Return preserved check ids for a requirement, trying the primary + key first then each fallback in order.""" + pv = req.get(primary_key) + if pv and pv in by_primary: + return list(by_primary[pv]) + attributes = req.get("Attributes") or [] + if not attributes: + return [] + attrs = attributes[0] + for i, field_names in enumerate(fallback_keys): + key = _build_fallback_key(attrs, field_names) + if key and key in by_fallback[i]: + return list(by_fallback[i][key]) + return [] + + +# --------------------------------------------------------------------------- +# Provider output assembly +# --------------------------------------------------------------------------- + + +def resolve_output_path(template: str, framework: dict, provider_key: str) -> Path: + return Path( + template.format( + provider=provider_key, + framework=framework["name"].lower(), + version=framework["version"], + ) + ) + + +def build_provider_json( + config: dict, + provider: dict, + base_requirements: list[dict], +) -> tuple[dict, dict[str, int]]: + """Produce the provider-specific JSON dict ready to dump. + + Returns ``(json_dict, counts)`` where ``counts`` tracks how each + requirement's checks were resolved (primary, fallback, or none). + """ + framework = config["framework"] + preservation = config["post_processing"]["check_preservation"] + primary_key = preservation["primary_key"] + fallback_keys = preservation.get("fallback_keys") or [] + + # For versioned frameworks, the file we WRITE (output.path_template + # resolved at the new version) is not the file we want to READ legacy + # Checks from. Allow the config to override the legacy source path so + # a version bump can still preserve mappings from the previous file. + legacy_template = ( + preservation.get("legacy_path_template") + or config["output"]["path_template"] + ) + legacy_path = resolve_output_path( + legacy_template, framework, provider["key"] + ) + by_primary, by_fallback = load_legacy_check_maps( + legacy_path, primary_key, fallback_keys + ) + + counts = {"primary": 0, "fallback": 0, "none": 0} + enriched: list[dict] = [] + for req in base_requirements: + # Try primary key first + pv = req.get(primary_key) + checks: list[str] = [] + source = "none" + if pv and pv in by_primary: + checks = list(by_primary[pv]) + source = "primary" + else: + attributes = req.get("Attributes") or [] + if attributes: + attrs = attributes[0] + for i, field_names in enumerate(fallback_keys): + key = _build_fallback_key(attrs, field_names) + if key and key in by_fallback[i]: + checks = list(by_fallback[i][key]) + source = "fallback" + break + counts[source] += 1 + enriched.append( + { + "Id": req["Id"], + "Description": req["Description"], + # Shallow-copy attribute dicts so providers don't share refs + "Attributes": [dict(a) for a in req.get("Attributes") or []], + "Checks": checks, + } + ) + + description = framework["description_template"].format( + provider_display=provider["display"], + provider_key=provider["key"], + framework_name=framework["name"], + framework_display=framework["display_name"], + version=framework["version"], + ) + out = { + "Framework": framework["name"], + "Version": framework["version"], + "Provider": provider["display"], + "Name": framework["display_name"], + "Description": description, + "Requirements": enriched, + } + return out, counts + + +# --------------------------------------------------------------------------- +# Pydantic post-validation +# --------------------------------------------------------------------------- + + +def pydantic_validate(json_path: Path) -> int: + """Import Prowler lazily so the runner still works without Prowler + installed (validation step is skipped in that case).""" + try: + from prowler.lib.check.compliance_models import Compliance + except ImportError: + print( + " note: prowler package not importable β€” skipping Pydantic validation", + file=sys.stderr, + ) + return -1 + try: + parsed = Compliance.parse_file(str(json_path)) + except Exception as exc: + raise RuntimeError( + f"Pydantic validation failed for {json_path}: {exc}" + ) from exc + return len(parsed.Requirements) + + +# --------------------------------------------------------------------------- +# Driver +# --------------------------------------------------------------------------- + + +def main() -> int: + if len(sys.argv) != 2: + print("usage: sync_framework.py ", file=sys.stderr) + return 1 + + config_path = Path(sys.argv[1]) + try: + config = load_config(config_path) + except ConfigError as exc: + print(f"config error: {exc}", file=sys.stderr) + return 2 + + framework_name = config["framework"]["name"] + upstream_dir = Path(config["upstream"]["dir"]) + if not upstream_dir.exists(): + print( + f"error: upstream cache dir {upstream_dir} not found\n" + f" hint: {config['upstream'].get('fetch_docs', '(see SKILL.md Workflow A Step 1)')}", + file=sys.stderr, + ) + return 3 + + parser_module_name = config["parser"]["module"] + print( + f"Sync: framework={framework_name} version={config['framework']['version']} " + f"parser={parser_module_name}" + ) + + try: + parser = load_parser(parser_module_name) + except ConfigError as exc: + print(f"parser error: {exc}", file=sys.stderr) + return 4 + + print(f"Parsing upstream from {upstream_dir}...") + try: + base_requirements = parser.parse_upstream(config) + except FileNotFoundError as exc: + # A missing catalog declared in parser.catalog_files is a hard + # failure: emitting JSON with part of the framework silently + # dropped would violate the canonical-sync contract. + print(f"upstream error: {exc}", file=sys.stderr) + return 6 + print(f" parser returned {len(base_requirements)} requirements") + + # Safety-net: parser contract + try: + assert_unique_ids(base_requirements) + except ValueError as exc: + print(f"parser contract violation: {exc}", file=sys.stderr) + return 5 + + # Post-processing: family name normalization + norm_map = ( + config.get("post_processing", {}) + .get("family_name_normalization") + or {} + ) + normalize_family_names(base_requirements, norm_map) + + # Per-provider output + print() + for provider in config["providers"]: + provider_json, counts = build_provider_json( + config, provider, base_requirements + ) + out_path = resolve_output_path( + config["output"]["path_template"], + config["framework"], + provider["key"], + ) + out_path.parent.mkdir(parents=True, exist_ok=True) + with open(out_path, "w") as f: + json.dump(provider_json, f, indent=2, ensure_ascii=False) + f.write("\n") + + validated = pydantic_validate(out_path) + validated_msg = ( + f" pydantic_reqs={validated}" if validated >= 0 else " pydantic=skipped" + ) + print( + f" {provider['key']}: total={len(provider_json['Requirements'])} " + f"matched_primary={counts['primary']} " + f"matched_fallback={counts['fallback']} " + f"new_or_unmatched={counts['none']}{validated_msg}" + ) + print(f" wrote {out_path}") + + print("\nDone.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/skills/setup.sh b/skills/setup.sh index ec5512e8c5..c24706d718 100755 --- a/skills/setup.sh +++ b/skills/setup.sh @@ -1,10 +1,10 @@ #!/bin/bash # Setup AI Skills for Prowler development # Configures AI coding assistants that follow agentskills.io standard: -# - Claude Code: .claude/skills/ symlink + CLAUDE.md copies -# - Gemini CLI: .gemini/skills/ symlink + GEMINI.md copies +# - Claude Code: .claude/skills/ symlink + CLAUDE.md symlink +# - Gemini CLI: .gemini/skills/ symlink + GEMINI.md symlink # - Codex (OpenAI): .codex/skills/ symlink + AGENTS.md (native) -# - GitHub Copilot: .github/copilot-instructions.md copy +# - GitHub Copilot: .github/copilot-instructions.md symlink # # Usage: # ./setup.sh # Interactive mode (select AI assistants) @@ -37,6 +37,28 @@ SETUP_COPILOT=false # HELPER FUNCTIONS # ============================================================================= +add_to_gitignore() { + local pattern="$1" + local gitignore_file="$REPO_ROOT/.gitignore" + local header="# AI Coding assistants assets" + + # Create .gitignore if it doesn't exist + if [ ! -f "$gitignore_file" ]; then + touch "$gitignore_file" + fi + + # Check if pattern exists (exact match or at end of file) + if ! grep -qxF "$pattern" "$gitignore_file"; then + # Check if header exists + if ! grep -qxF "$header" "$gitignore_file"; then + echo -e "\n\n$header" >> "$gitignore_file" + fi + + echo "$pattern" >> "$gitignore_file" + echo -e "${GREEN} βœ“ Added $pattern to .gitignore${NC}" + fi +} + show_help() { echo "Usage: $0 [OPTIONS]" echo "" @@ -109,6 +131,7 @@ setup_claude() { if [ ! -d "$REPO_ROOT/.claude" ]; then mkdir -p "$REPO_ROOT/.claude" fi + add_to_gitignore ".claude/skills" if [ -L "$target" ]; then rm "$target" @@ -119,8 +142,9 @@ setup_claude() { ln -s "$SKILLS_SOURCE" "$target" echo -e "${GREEN} βœ“ .claude/skills -> skills/${NC}" - # Copy AGENTS.md to CLAUDE.md - copy_agents_md "CLAUDE.md" + # Link AGENTS.md to CLAUDE.md + link_agents_md "CLAUDE.md" + add_to_gitignore "CLAUDE.md" } setup_gemini() { @@ -129,6 +153,7 @@ setup_gemini() { if [ ! -d "$REPO_ROOT/.gemini" ]; then mkdir -p "$REPO_ROOT/.gemini" fi + add_to_gitignore ".gemini/skills" if [ -L "$target" ]; then rm "$target" @@ -139,8 +164,9 @@ setup_gemini() { ln -s "$SKILLS_SOURCE" "$target" echo -e "${GREEN} βœ“ .gemini/skills -> skills/${NC}" - # Copy AGENTS.md to GEMINI.md - copy_agents_md "GEMINI.md" + # Link AGENTS.md to GEMINI.md + link_agents_md "GEMINI.md" + add_to_gitignore "GEMINI.md" } setup_codex() { @@ -149,6 +175,7 @@ setup_codex() { if [ ! -d "$REPO_ROOT/.codex" ]; then mkdir -p "$REPO_ROOT/.codex" fi + add_to_gitignore ".codex/skills" if [ -L "$target" ]; then rm "$target" @@ -164,12 +191,19 @@ setup_codex() { setup_copilot() { if [ -f "$REPO_ROOT/AGENTS.md" ]; then mkdir -p "$REPO_ROOT/.github" - cp "$REPO_ROOT/AGENTS.md" "$REPO_ROOT/.github/copilot-instructions.md" + + # Link AGENTS.md -> .github/copilot-instructions.md + local target="$REPO_ROOT/.github/copilot-instructions.md" + ln -sf "../AGENTS.md" "$target" + echo -e "${GREEN} βœ“ AGENTS.md -> .github/copilot-instructions.md${NC}" + + # Add specifically the file, NOT the .github folder + add_to_gitignore ".github/copilot-instructions.md" fi } -copy_agents_md() { +link_agents_md() { local target_name="$1" local agents_files local count=0 @@ -179,11 +213,15 @@ copy_agents_md() { for agents_file in $agents_files; do local agents_dir agents_dir=$(dirname "$agents_file") - cp "$agents_file" "$agents_dir/$target_name" + + # Create relative symlink + # Since files are in same dir, we can just link to basename + (cd "$agents_dir" && ln -sf "$(basename "$agents_file")" "$target_name") + count=$((count + 1)) done - echo -e "${GREEN} βœ“ Copied $count AGENTS.md -> $target_name${NC}" + echo -e "${GREEN} βœ“ Linked $count AGENTS.md -> $target_name${NC}" } # ============================================================================= @@ -302,4 +340,4 @@ echo "Configured:" [ "$SETUP_COPILOT" = true ] && echo " β€’ GitHub Copilot: .github/copilot-instructions.md" echo "" echo -e "${BLUE}Note: Restart your AI assistant to load the skills.${NC}" -echo -e "${BLUE} AGENTS.md is the source of truth - edit it, then re-run this script.${NC}" +echo -e "${BLUE} AGENTS.md is the source of truth - changes are reflected automatically via symlinks.${NC}" diff --git a/skills/setup_test.sh b/skills/setup_test.sh index c0e80afe99..db4749ed3f 100755 --- a/skills/setup_test.sh +++ b/skills/setup_test.sh @@ -201,40 +201,40 @@ test_symlink_not_created_without_flag() { } # ============================================================================= -# TESTS: AGENTS.md COPYING +# TESTS: AGENTS.md LINKING # ============================================================================= -test_copy_claude_agents_md() { +test_link_claude_agents_md() { run_setup --claude > /dev/null - assert_file_exists "$TEST_DIR/CLAUDE.md" "Root CLAUDE.md should exist" && \ - assert_file_exists "$TEST_DIR/api/CLAUDE.md" "api/CLAUDE.md should exist" && \ - assert_file_exists "$TEST_DIR/ui/CLAUDE.md" "ui/CLAUDE.md should exist" + assert_symlink_exists "$TEST_DIR/CLAUDE.md" "Root CLAUDE.md should be a symlink" && \ + assert_symlink_exists "$TEST_DIR/api/CLAUDE.md" "api/CLAUDE.md should be a symlink" && \ + assert_symlink_exists "$TEST_DIR/ui/CLAUDE.md" "ui/CLAUDE.md should be a symlink" } -test_copy_gemini_agents_md() { +test_link_gemini_agents_md() { run_setup --gemini > /dev/null - assert_file_exists "$TEST_DIR/GEMINI.md" "Root GEMINI.md should exist" && \ - assert_file_exists "$TEST_DIR/api/GEMINI.md" "api/GEMINI.md should exist" && \ - assert_file_exists "$TEST_DIR/ui/GEMINI.md" "ui/GEMINI.md should exist" + assert_symlink_exists "$TEST_DIR/GEMINI.md" "Root GEMINI.md should be a symlink" && \ + assert_symlink_exists "$TEST_DIR/api/GEMINI.md" "api/GEMINI.md should be a symlink" && \ + assert_symlink_exists "$TEST_DIR/ui/GEMINI.md" "ui/GEMINI.md should be a symlink" } -test_copy_copilot_to_github() { +test_link_copilot_to_github() { run_setup --copilot > /dev/null - assert_file_exists "$TEST_DIR/.github/copilot-instructions.md" "Copilot instructions should exist" + assert_symlink_exists "$TEST_DIR/.github/copilot-instructions.md" "Copilot instructions should be a symlink" } -test_copy_codex_no_extra_files() { +test_link_codex_no_extra_files() { run_setup --codex > /dev/null assert_file_not_exists "$TEST_DIR/CODEX.md" "CODEX.md should not be created" } -test_copy_not_created_without_flag() { +test_link_not_created_without_flag() { run_setup --codex > /dev/null - assert_file_not_exists "$TEST_DIR/CLAUDE.md" "CLAUDE.md should not exist" && \ - assert_file_not_exists "$TEST_DIR/GEMINI.md" "GEMINI.md should not exist" + assert_symlink_not_exists "$TEST_DIR/CLAUDE.md" "CLAUDE.md should not exist" && \ + assert_symlink_not_exists "$TEST_DIR/GEMINI.md" "GEMINI.md should not exist" } -test_copy_content_matches_source() { +test_link_content_matches_source() { run_setup --claude > /dev/null local source_content target_content source_content=$(cat "$TEST_DIR/AGENTS.md") @@ -272,7 +272,7 @@ test_idempotent_multiple_runs() { run_setup --claude > /dev/null run_setup --claude > /dev/null assert_symlink_exists "$TEST_DIR/.claude/skills" "Symlink should still exist after second run" && \ - assert_file_exists "$TEST_DIR/CLAUDE.md" "CLAUDE.md should still exist after second run" + assert_symlink_exists "$TEST_DIR/CLAUDE.md" "CLAUDE.md should still be a symlink after second run" } # ============================================================================= diff --git a/tests/lib/check/universal_compliance_models_test.py b/tests/lib/check/universal_compliance_models_test.py new file mode 100644 index 0000000000..5a3e4aae56 --- /dev/null +++ b/tests/lib/check/universal_compliance_models_test.py @@ -0,0 +1,1118 @@ +import json +import os + +import pytest +from pydantic.v1 import ValidationError + +from prowler.lib.check.compliance_models import ( + AttributeMetadata, + ChartConfig, + Compliance, + ComplianceFramework, + CriticalRequirementsFilter, + EnumValueDisplay, + I18nLabels, + OutputFormats, + OutputsConfig, + PDFConfig, + ReportFilter, + ScoringConfig, + ScoringFormula, + SplitByConfig, + TableConfig, + TableLabels, + UniversalComplianceRequirement, + adapt_legacy_to_universal, + load_compliance_framework_universal, +) +from tests.lib.outputs.compliance.fixtures import ( + CIS_1_4_AWS, + ENS_RD2022_AWS, + KISA_ISMSP_AWS, + MITRE_ATTACK_AWS, + NIST_800_53_REVISION_4_AWS, + PROWLER_THREATSCORE_AWS, +) + + +class TestOutputFormats: + def test_defaults(self): + of = OutputFormats() + assert of.csv is True + assert of.ocsf is True + + def test_explicit_false(self): + of = OutputFormats(csv=False, ocsf=False) + assert of.csv is False + assert of.ocsf is False + + +class TestAttributeMetadata: + def test_basic(self): + meta = AttributeMetadata(key="Section", type="str") + assert meta.key == "Section" + assert meta.type == "str" + assert meta.output_formats.csv is True + assert meta.required is False + + def test_with_enum(self): + meta = AttributeMetadata( + key="Profile", + type="str", + enum=["Level 1", "Level 2"], + ) + assert meta.enum == ["Level 1", "Level 2"] + + def test_int_type(self): + meta = AttributeMetadata(key="LevelOfRisk", type="int", required=True) + assert meta.type == "int" + assert meta.required is True + + def test_enum_display_field(self): + meta = AttributeMetadata( + key="Dimensiones", + type="str", + enum=["confidencialidad", "integridad", "trazabilidad"], + enum_display={ + "confidencialidad": { + "label": "Confidencialidad", + "abbreviation": "C", + "color": "#FF6347", + }, + "integridad": { + "label": "Integridad", + "abbreviation": "I", + "color": "#4286F4", + }, + "trazabilidad": { + "label": "Trazabilidad", + "abbreviation": "T", + "color": "#32CD32", + }, + }, + ) + assert meta.enum_display is not None + assert meta.enum_display["confidencialidad"]["abbreviation"] == "C" + assert meta.enum_display["integridad"]["color"] == "#4286F4" + + def test_enum_order_field(self): + meta = AttributeMetadata( + key="Nivel", + type="str", + enum=["opcional", "bajo", "medio", "alto"], + enum_order=["alto", "medio", "bajo", "opcional"], + ) + assert meta.enum_order == ["alto", "medio", "bajo", "opcional"] + + def test_chart_label_field(self): + meta = AttributeMetadata( + key="Section", + type="str", + chart_label="Security Domain", + ) + assert meta.chart_label == "Security Domain" + + def test_output_formats_default_true(self): + meta = AttributeMetadata(key="Section") + assert meta.output_formats.csv is True + assert meta.output_formats.ocsf is True + + def test_output_formats_explicit_false(self): + meta = AttributeMetadata( + key="InternalNote", + output_formats=OutputFormats(csv=False, ocsf=False), + ) + assert meta.output_formats.csv is False + assert meta.output_formats.ocsf is False + + def test_new_fields_default_none(self): + meta = AttributeMetadata(key="Section") + assert meta.enum_display is None + assert meta.enum_order is None + assert meta.chart_label is None + + +class TestEnumValueDisplay: + def test_basic(self): + evd = EnumValueDisplay(label="Test") + assert evd.label == "Test" + assert evd.abbreviation is None + assert evd.color is None + assert evd.icon is None + + def test_dimension_style(self): + evd = EnumValueDisplay( + label="Trazabilidad", + abbreviation="T", + color="#4286F4", + ) + assert evd.label == "Trazabilidad" + assert evd.abbreviation == "T" + assert evd.color == "#4286F4" + + def test_tipo_style(self): + evd = EnumValueDisplay( + label="Requisito", + icon="⚠️", + ) + assert evd.icon == "⚠️" + assert evd.abbreviation is None + + +class TestChartConfig: + def test_horizontal_bar(self): + chart = ChartConfig( + id="section_compliance", + type="horizontal_bar", + group_by="Section", + title="Compliance Score by Domain", + y_label="Domain", + x_label="Compliance %", + ) + assert chart.type == "horizontal_bar" + assert chart.group_by == "Section" + assert chart.value_source == "compliance_percent" + assert chart.color_mode == "by_value" + + def test_vertical_bar(self): + chart = ChartConfig( + id="risk_distribution", + type="vertical_bar", + group_by="LevelOfRisk", + color_mode="fixed", + fixed_color="#336699", + ) + assert chart.type == "vertical_bar" + assert chart.fixed_color == "#336699" + + def test_radar(self): + chart = ChartConfig( + id="dimension_radar", + type="radar", + group_by="Dimensiones", + ) + assert chart.type == "radar" + + def test_defaults(self): + chart = ChartConfig(id="test", type="vertical_bar", group_by="Section") + assert chart.title is None + assert chart.x_label is None + assert chart.y_label is None + assert chart.value_source == "compliance_percent" + assert chart.color_mode == "by_value" + assert chart.fixed_color is None + + +class TestScoringFormula: + def test_threatscore_style(self): + formula = ScoringFormula( + risk_field="LevelOfRisk", + weight_field="Weight", + risk_boost_factor=0.25, + ) + assert formula.risk_field == "LevelOfRisk" + assert formula.weight_field == "Weight" + assert formula.risk_boost_factor == 0.25 + + def test_custom_boost_factor(self): + formula = ScoringFormula( + risk_field="Risk", + weight_field="Impact", + risk_boost_factor=0.5, + ) + assert formula.risk_boost_factor == 0.5 + + def test_default_boost_factor(self): + formula = ScoringFormula(risk_field="LevelOfRisk", weight_field="Weight") + assert formula.risk_boost_factor == 0.25 + + +class TestCriticalRequirementsFilter: + def test_int_based(self): + crf = CriticalRequirementsFilter( + filter_field="LevelOfRisk", + min_value=4, + title="Critical Failed Requirements", + ) + assert crf.filter_field == "LevelOfRisk" + assert crf.min_value == 4 + assert crf.filter_value is None + assert crf.status_filter == "FAIL" + assert crf.title == "Critical Failed Requirements" + + def test_string_based(self): + crf = CriticalRequirementsFilter( + filter_field="Nivel", + filter_value="alto", + ) + assert crf.filter_value == "alto" + assert crf.min_value is None + + def test_defaults(self): + crf = CriticalRequirementsFilter(filter_field="LevelOfRisk") + assert crf.status_filter == "FAIL" + assert crf.title is None + assert crf.min_value is None + assert crf.filter_value is None + + +class TestReportFilter: + def test_defaults(self): + rf = ReportFilter() + assert rf.only_failed is True + assert rf.include_manual is False + + def test_custom(self): + rf = ReportFilter(only_failed=False, include_manual=True) + assert rf.only_failed is False + assert rf.include_manual is True + + +class TestI18nLabels: + def test_english_defaults(self): + labels = I18nLabels() + assert labels.page_label == "Page" + assert labels.powered_by == "Powered by Prowler" + assert labels.framework_label == "Framework:" + assert labels.provider_label == "Provider:" + assert labels.report_title is None + + def test_spanish_override(self): + labels = I18nLabels( + report_title="Informe de Cumplimiento ENS", + page_label="PΓ‘gina", + powered_by="Generado por Prowler", + framework_label="Marco:", + version_label="VersiΓ³n:", + provider_label="Proveedor:", + description_label="DescripciΓ³n:", + compliance_score_label="PuntuaciΓ³n de Cumplimiento por Secciones", + requirements_index_label="Índice de Requisitos", + detailed_findings_label="Hallazgos Detallados", + ) + assert labels.page_label == "PΓ‘gina" + assert labels.provider_label == "Proveedor:" + assert labels.report_title == "Informe de Cumplimiento ENS" + + +class TestSplitByConfig: + def test_cis_style(self): + config = SplitByConfig(field="Profile", values=["Level 1", "Level 2"]) + assert config.field == "Profile" + assert len(config.values) == 2 + + def test_ens_style(self): + config = SplitByConfig( + field="Nivel", + values=["alto", "medio", "bajo", "opcional"], + ) + assert len(config.values) == 4 + + +class TestScoringConfig: + def test_threatscore_style(self): + config = ScoringConfig(risk_field="LevelOfRisk", weight_field="Weight") + assert config.risk_field == "LevelOfRisk" + assert config.weight_field == "Weight" + + +class TestTableLabels: + def test_defaults(self): + labels = TableLabels() + assert labels.pass_label == "PASS" + assert labels.fail_label == "FAIL" + assert labels.provider_header == "Provider" + + def test_ens_spanish(self): + labels = TableLabels( + pass_label="CUMPLE", + fail_label="NO CUMPLE", + provider_header="Proveedor", + ) + assert labels.pass_label == "CUMPLE" + + +class TestTableConfig: + def test_grouped_mode(self): + tc = TableConfig(group_by="Section") + assert tc.group_by == "Section" + assert tc.split_by is None + assert tc.scoring is None + + def test_split_mode(self): + tc = TableConfig( + group_by="Section", + split_by=SplitByConfig(field="Profile", values=["Level 1", "Level 2"]), + ) + assert tc.split_by is not None + assert tc.split_by.field == "Profile" + + def test_scored_mode(self): + tc = TableConfig( + group_by="Section", + scoring=ScoringConfig(risk_field="LevelOfRisk", weight_field="Weight"), + ) + assert tc.scoring is not None + + +class TestPDFConfig: + def test_defaults(self): + pdf = PDFConfig() + assert pdf.language == "en" + assert pdf.logo_filename is None + assert pdf.primary_color is None + assert pdf.sections is None + assert pdf.section_short_names is None + assert pdf.group_by_field is None + assert pdf.sub_group_by_field is None + assert pdf.section_titles is None + assert pdf.charts is None + assert pdf.scoring is None + assert pdf.critical_filter is None + assert pdf.filter is None + assert pdf.labels is None + + def test_csa_ccm_style(self): + pdf = PDFConfig( + primary_color="#336699", + secondary_color="#4D80B3", + bg_color="#F2F8FF", + group_by_field="Section", + sections=["Audit & Assurance", "Identity & Access Management"], + section_short_names={"Identity & Access Management": "IAM"}, + charts=[ + ChartConfig( + id="section_compliance", + type="horizontal_bar", + group_by="Section", + title="Compliance Score by Domain", + ).dict() + ], + filter=ReportFilter(only_failed=True, include_manual=False), + ) + assert pdf.primary_color == "#336699" + assert len(pdf.sections) == 2 + assert pdf.section_short_names["Identity & Access Management"] == "IAM" + assert pdf.group_by_field == "Section" + assert pdf.charts is not None + assert len(pdf.charts) == 1 + assert pdf.filter.only_failed is True + + def test_ens_style(self): + pdf = PDFConfig( + language="es", + logo_filename="ens_logo.png", + primary_color="#CC3333", + group_by_field="Marco", + sub_group_by_field="Categoria", + labels=I18nLabels( + page_label="PΓ‘gina", + provider_label="Proveedor:", + ), + ) + assert pdf.language == "es" + assert pdf.logo_filename == "ens_logo.png" + assert pdf.group_by_field == "Marco" + assert pdf.sub_group_by_field == "Categoria" + assert pdf.labels.page_label == "PΓ‘gina" + + def test_threatscore_style(self): + pdf = PDFConfig( + primary_color="#336699", + sections=["1. IAM", "2. Attack Surface"], + scoring=ScoringFormula( + risk_field="LevelOfRisk", + weight_field="Weight", + risk_boost_factor=0.25, + ), + critical_filter=CriticalRequirementsFilter( + filter_field="LevelOfRisk", + min_value=4, + title="Critical Failed Requirements", + ), + ) + assert pdf.scoring is not None + assert pdf.scoring.risk_field == "LevelOfRisk" + assert pdf.critical_filter.min_value == 4 + + def test_section_titles(self): + pdf = PDFConfig( + section_titles={ + "1": "1. Policy on Security", + "2": "2. Risk Management", + }, + ) + assert pdf.section_titles["1"] == "1. Policy on Security" + + def test_in_framework(self): + fw = ComplianceFramework( + framework="Test", + name="Test Framework", + description="Test", + requirements=[], + outputs=OutputsConfig( + pdf_config=PDFConfig( + primary_color="#336699", + sections=["Section A"], + charts=[ + ChartConfig( + id="test_chart", + type="vertical_bar", + group_by="Section", + ).dict() + ], + ), + ), + ) + assert fw.outputs is not None + assert fw.outputs.pdf_config is not None + assert fw.outputs.pdf_config.primary_color == "#336699" + assert fw.outputs.pdf_config.sections == ["Section A"] + assert fw.outputs.pdf_config.charts is not None + assert len(fw.outputs.pdf_config.charts) == 1 + assert fw.outputs.pdf_config.charts[0]["id"] == "test_chart" + assert fw.outputs.pdf_config.charts[0]["type"] == "vertical_bar" + + def test_framework_without_pdf_config(self): + fw = ComplianceFramework( + framework="Test", + name="Test Framework", + description="Test", + requirements=[], + ) + assert fw.outputs is None + + +class TestUniversalComplianceRequirement: + def test_flat_dict_attributes(self): + req = UniversalComplianceRequirement( + id="1.1", + description="Test requirement", + attributes={"Section": "IAM", "Profile": "Level 1"}, + checks={"aws": ["check_a", "check_b"]}, + ) + assert req.attributes["Section"] == "IAM" + assert len(req.checks["aws"]) == 2 + + def test_mitre_optional_fields(self): + req = UniversalComplianceRequirement( + id="T1190", + description="Exploit Public-Facing Application", + attributes={}, + checks={"aws": ["drs_job_exist"]}, + tactics=["Initial Access"], + sub_techniques=[], + platforms=["IaaS", "Linux"], + technique_url="https://attack.mitre.org/techniques/T1190/", + ) + assert req.tactics == ["Initial Access"] + assert req.technique_url == "https://attack.mitre.org/techniques/T1190/" + + def test_dict_checks_multi_provider(self): + req = UniversalComplianceRequirement( + id="1.1", + description="Multi-provider", + attributes={}, + checks={"aws": ["check_a"], "azure": ["check_b"]}, + ) + assert isinstance(req.checks, dict) + assert "aws" in req.checks + + def test_empty_checks(self): + req = UniversalComplianceRequirement( + id="manual-1", + description="Manual requirement", + attributes={"Section": "Governance"}, + checks={}, + ) + assert req.checks == {} + + def test_checks_default_is_empty_dict(self): + req = UniversalComplianceRequirement( + id="1.1", + description="No checks provided", + ) + assert req.checks == {} + + +class TestComplianceFramework: + def test_basic_framework(self): + fw = ComplianceFramework( + framework="TestFW", + name="Test Framework", + provider="AWS", + version="1.0", + description="A test framework", + requirements=[ + UniversalComplianceRequirement( + id="1.1", + description="Test", + attributes={"Section": "IAM"}, + checks={"aws": ["check_a"]}, + ) + ], + attributes_metadata=[ + AttributeMetadata(key="Section", type="str"), + ], + outputs=OutputsConfig(table_config=TableConfig(group_by="Section")), + ) + assert fw.framework == "TestFW" + assert fw.outputs.table_config.group_by == "Section" + assert len(fw.attributes_metadata) == 1 + assert len(fw.requirements) == 1 + + def test_optional_provider(self): + fw = ComplianceFramework( + framework="MultiCloud", + name="Multi-cloud framework", + description="A multi-provider framework", + requirements=[], + ) + assert fw.provider is None + + def test_get_providers_from_dict_checks(self): + fw = ComplianceFramework( + framework="MultiCloud", + name="Multi-cloud", + description="test", + requirements=[ + UniversalComplianceRequirement( + id="1.1", + description="test", + attributes={}, + checks={ + "aws": ["check_a"], + "azure": ["check_b"], + "gcp": ["check_c"], + }, + ), + UniversalComplianceRequirement( + id="1.2", + description="test2", + attributes={}, + checks={"aws": ["check_d"]}, + ), + ], + ) + providers = fw.get_providers() + assert providers == ["aws", "azure", "gcp"] + + def test_get_providers_fallback_to_explicit(self): + fw = ComplianceFramework( + framework="SingleCloud", + name="Single-cloud", + provider="AWS", + description="test", + requirements=[ + UniversalComplianceRequirement( + id="1.1", + description="test", + attributes={}, + checks={}, + ), + ], + ) + providers = fw.get_providers() + assert providers == ["aws"] + + def test_supports_provider_dict_checks(self): + fw = ComplianceFramework( + framework="MultiCloud", + name="Multi-cloud", + description="test", + requirements=[ + UniversalComplianceRequirement( + id="1.1", + description="test", + attributes={}, + checks={"aws": ["check_a"], "azure": ["check_b"]}, + ), + ], + ) + assert fw.supports_provider("aws") is True + assert fw.supports_provider("azure") is True + assert fw.supports_provider("gcp") is False + + def test_supports_provider_explicit_only(self): + """Framework with explicit provider but no per-requirement checks still supports the provider.""" + fw = ComplianceFramework( + framework="SingleCloud", + name="Single-cloud", + provider="AWS", + description="test", + requirements=[ + UniversalComplianceRequirement( + id="1.1", + description="Manual requirement", + attributes={}, + checks={}, + ), + ], + ) + assert fw.supports_provider("aws") is True + assert fw.supports_provider("azure") is False + + def test_no_provider_field_with_dict_checks(self): + """Multi-provider JSON has no Provider field β€” providers derived from checks.""" + fw = ComplianceFramework( + framework="CSA_CCM", + name="CSA CCM 4.0", + description="Cloud Controls Matrix", + requirements=[ + UniversalComplianceRequirement( + id="A&A-01", + description="Audit & Assurance", + attributes={"Domain": "A&A"}, + checks={ + "aws": ["check_a"], + "azure": ["check_b"], + "gcp": ["check_c"], + }, + ), + ], + ) + assert fw.provider is None + assert fw.get_providers() == ["aws", "azure", "gcp"] + assert fw.supports_provider("aws") + assert fw.supports_provider("azure") + assert fw.supports_provider("gcp") + assert not fw.supports_provider("kubernetes") + + def test_icon_field(self): + fw = ComplianceFramework( + framework="CSA_CCM", + name="CSA CCM 4.0", + description="Cloud Controls Matrix", + icon="csa", + requirements=[], + ) + assert fw.icon == "csa" + + def test_icon_defaults_to_none(self): + fw = ComplianceFramework( + framework="Test", + name="Test", + description="d", + requirements=[], + ) + assert fw.icon is None + + +class TestAdaptLegacyToUniversal: + def test_adapt_cis(self): + fw = adapt_legacy_to_universal(CIS_1_4_AWS) + assert fw.framework == "CIS" + assert fw.provider == "AWS" + assert len(fw.requirements) == 2 + # First requirement should have flat attributes + req = fw.requirements[0] + assert "Section" in req.attributes + assert req.attributes["Section"] == "2. Storage" + assert req.tactics is None + # Checks must be wrapped in dict keyed by provider + assert isinstance(req.checks, dict) + assert "aws" in req.checks + + def test_adapt_ens(self): + fw = adapt_legacy_to_universal(ENS_RD2022_AWS) + assert fw.framework == "ENS" + req = fw.requirements[0] + assert "Marco" in req.attributes + assert req.attributes["Marco"] == "operacional" + + def test_adapt_mitre(self): + fw = adapt_legacy_to_universal(MITRE_ATTACK_AWS) + assert fw.framework == "MITRE-ATTACK" + req = fw.requirements[0] + assert req.tactics == ["Initial Access"] + assert req.technique_url == "https://attack.mitre.org/techniques/T1190/" + assert "_raw_attributes" in req.attributes + assert isinstance(req.checks, dict) + assert "aws" in req.checks + + def test_adapt_threatscore(self): + fw = adapt_legacy_to_universal(PROWLER_THREATSCORE_AWS) + req = fw.requirements[0] + assert req.attributes["LevelOfRisk"] == 5 + assert req.attributes["Weight"] == 1000 + + def test_adapt_generic(self): + fw = adapt_legacy_to_universal(NIST_800_53_REVISION_4_AWS) + req = fw.requirements[0] + assert "Section" in req.attributes + + def test_adapt_kisa(self): + fw = adapt_legacy_to_universal(KISA_ISMSP_AWS) + req = fw.requirements[0] + assert "Domain" in req.attributes + + def test_inferred_metadata_cis(self): + fw = adapt_legacy_to_universal(CIS_1_4_AWS) + assert fw.attributes_metadata is not None + keys = [m.key for m in fw.attributes_metadata] + assert "Section" in keys + assert "Profile" in keys + + def test_inferred_metadata_mitre_is_none(self): + fw = adapt_legacy_to_universal(MITRE_ATTACK_AWS) + assert fw.attributes_metadata is None + + def test_table_config_is_none(self): + fw = adapt_legacy_to_universal(CIS_1_4_AWS) + assert fw.outputs is None + + +class TestLoadComplianceFrameworkUniversal: + def test_load_universal_format(self, tmp_path): + data = { + "framework": "TestFW", + "name": "Test", + "provider": "AWS", + "version": "1.0", + "description": "desc", + "icon": "prowlerthreatscore", + "attributes_metadata": [{"key": "Section", "type": "str"}], + "outputs": {"table_config": {"group_by": "Section"}}, + "requirements": [ + { + "id": "1.1", + "description": "test", + "attributes": {"Section": "IAM"}, + "checks": {"aws": ["check_a"]}, + } + ], + } + path = tmp_path / "test.json" + path.write_text(json.dumps(data)) + fw = load_compliance_framework_universal(str(path)) + assert fw is not None + assert fw.framework == "TestFW" + assert fw.icon == "prowlerthreatscore" + assert fw.outputs.table_config.group_by == "Section" + + def test_load_universal_multi_provider(self, tmp_path): + data = { + "framework": "CSA_CCM", + "name": "CSA CCM 4.0", + "version": "4.0", + "description": "Cloud Controls Matrix", + "attributes_metadata": [{"key": "Domain", "type": "str"}], + "outputs": {"table_config": {"group_by": "Domain"}}, + "requirements": [ + { + "id": "A&A-01", + "description": "Audit", + "attributes": {"Domain": "Audit"}, + "checks": { + "aws": ["check_a"], + "azure": ["check_b"], + "gcp": ["check_c"], + }, + } + ], + } + path = tmp_path / "csa_ccm_4.0.json" + path.write_text(json.dumps(data)) + fw = load_compliance_framework_universal(str(path)) + assert fw is not None + assert fw.provider is None + assert fw.get_providers() == ["aws", "azure", "gcp"] + assert fw.supports_provider("aws") + assert not fw.supports_provider("kubernetes") + + def test_load_legacy_format(self, tmp_path): + data = { + "Framework": "SOC2", + "Name": "SOC2", + "Provider": "AWS", + "Version": "", + "Description": "desc", + "Requirements": [ + { + "Id": "1.1", + "Description": "test", + "Attributes": [{"Section": "Access Control"}], + "Checks": ["check_a"], + } + ], + } + path = tmp_path / "legacy.json" + path.write_text(json.dumps(data)) + fw = load_compliance_framework_universal(str(path)) + assert fw is not None + assert fw.framework == "SOC2" + assert fw.outputs is None + assert fw.requirements[0].attributes["Section"] == "Access Control" + assert fw.requirements[0].checks == {"aws": ["check_a"]} + + +class TestSmokeLoadAllJSONs: + """Parametrized smoke test: every existing compliance JSON must load as ComplianceFramework.""" + + @staticmethod + def _find_all_compliance_jsons(): + base = os.path.join( + os.path.dirname(__file__), + "..", + "..", + "..", + "prowler", + "compliance", + ) + base = os.path.normpath(base) + jsons = [] + if os.path.isdir(base): + # Top-level JSONs (multi-provider) + for filename in os.listdir(base): + if filename.endswith(".json"): + jsons.append(os.path.join(base, filename)) + # Provider sub-directory JSONs + for provider_dir in os.listdir(base): + provider_path = os.path.join(base, provider_dir) + if os.path.isdir(provider_path): + for filename in os.listdir(provider_path): + if filename.endswith(".json"): + jsons.append(os.path.join(provider_path, filename)) + return jsons + + @pytest.mark.parametrize( + "json_path", + _find_all_compliance_jsons.__func__(), + ids=lambda p: os.path.basename(p), + ) + def test_loads_as_universal(self, json_path): + fw = load_compliance_framework_universal(json_path) + assert fw is not None, f"Failed to load {json_path}" + assert fw.framework + assert fw.name + assert len(fw.requirements) >= 0 + + +class TestBackwardCompat: + """Ensure Compliance.get_bulk still returns Compliance objects.""" + + def test_get_bulk_still_works(self): + # This test just validates the legacy path still returns Compliance objects + # We test with a constructed Compliance object + legacy = CIS_1_4_AWS + assert isinstance(legacy, Compliance) + assert legacy.Framework == "CIS" + + +class TestAttributesMetadataValidation: + """Validate that Requirement attributes match their attributes_metadata schema.""" + + def _metadata(self, required=False, enum=None, type_str="str"): + return [ + AttributeMetadata(key="Section", type="str", required=True), + AttributeMetadata(key="Level", type=type_str, required=required, enum=enum), + ] + + def test_valid_attributes_pass(self): + fw = ComplianceFramework( + framework="Test", + name="Test", + description="d", + requirements=[ + UniversalComplianceRequirement( + id="1.1", + description="d", + attributes={"Section": "IAM", "Level": "high"}, + checks={}, + ), + ], + attributes_metadata=self._metadata(), + ) + assert len(fw.requirements) == 1 + + def test_missing_required_key_raises(self): + with pytest.raises( + ValidationError, match="missing required attribute 'Section'" + ): + ComplianceFramework( + framework="Test", + name="Test", + description="d", + requirements=[ + UniversalComplianceRequirement( + id="1.1", + description="d", + attributes={"Level": "high"}, + checks={}, + ), + ], + attributes_metadata=self._metadata(), + ) + + def test_invalid_enum_value_raises(self): + with pytest.raises(ValidationError, match="not in"): + ComplianceFramework( + framework="Test", + name="Test", + description="d", + requirements=[ + UniversalComplianceRequirement( + id="1.1", + description="d", + attributes={"Section": "IAM", "Level": "invalid"}, + checks={}, + ), + ], + attributes_metadata=self._metadata(enum=["high", "low"]), + ) + + def test_valid_enum_value_passes(self): + fw = ComplianceFramework( + framework="Test", + name="Test", + description="d", + requirements=[ + UniversalComplianceRequirement( + id="1.1", + description="d", + attributes={"Section": "IAM", "Level": "high"}, + checks={}, + ), + ], + attributes_metadata=self._metadata(enum=["high", "low"]), + ) + assert len(fw.requirements) == 1 + + def test_wrong_type_int_raises(self): + with pytest.raises(ValidationError, match="expected type int"): + ComplianceFramework( + framework="Test", + name="Test", + description="d", + requirements=[ + UniversalComplianceRequirement( + id="1.1", + description="d", + attributes={"Section": "IAM", "Level": "not_a_number"}, + checks={}, + ), + ], + attributes_metadata=self._metadata(type_str="int"), + ) + + def test_correct_type_int_passes(self): + fw = ComplianceFramework( + framework="Test", + name="Test", + description="d", + requirements=[ + UniversalComplianceRequirement( + id="1.1", + description="d", + attributes={"Section": "IAM", "Level": 5}, + checks={}, + ), + ], + attributes_metadata=self._metadata(type_str="int"), + ) + assert fw.requirements[0].attributes["Level"] == 5 + + def test_none_optional_value_skips_validation(self): + """None values for non-required keys should not trigger type/enum errors.""" + fw = ComplianceFramework( + framework="Test", + name="Test", + description="d", + requirements=[ + UniversalComplianceRequirement( + id="1.1", + description="d", + attributes={"Section": "IAM", "Level": None}, + checks={}, + ), + ], + attributes_metadata=self._metadata(enum=["high", "low"]), + ) + assert len(fw.requirements) == 1 + + def test_no_metadata_skips_validation(self): + """Frameworks without attributes_metadata should not be validated.""" + fw = ComplianceFramework( + framework="Test", + name="Test", + description="d", + requirements=[ + UniversalComplianceRequirement( + id="1.1", + description="d", + attributes={"anything": "goes"}, + checks={}, + ), + ], + ) + assert len(fw.requirements) == 1 + + def test_unknown_attribute_key_raises(self): + """Typos like 'Sectoin' must be rejected by the schema validator.""" + with pytest.raises(ValidationError, match="unknown attribute 'Sectoin'"): + ComplianceFramework( + framework="Test", + name="Test", + description="d", + requirements=[ + UniversalComplianceRequirement( + id="1.1", + description="d", + attributes={"Sectoin": "IAM", "Level": "high"}, + checks={}, + ), + ], + attributes_metadata=self._metadata(enum=["high", "low"]), + ) + + def test_multiple_unknown_keys_all_reported(self): + """Every unknown key must appear in the validation error (deterministic order).""" + with pytest.raises( + ValidationError, + match=r"unknown attribute 'Bogus1'[\s\S]*unknown attribute 'Bogus2'", + ): + ComplianceFramework( + framework="Test", + name="Test", + description="d", + requirements=[ + UniversalComplianceRequirement( + id="1.1", + description="d", + attributes={ + "Section": "IAM", + "Level": "high", + "Bogus1": "x", + "Bogus2": "y", + }, + checks={}, + ), + ], + attributes_metadata=self._metadata(enum=["high", "low"]), + ) + + def test_multiple_errors_reported(self): + """All validation errors should be collected and reported together.""" + with pytest.raises( + ValidationError, match="missing required attribute 'Section'" + ): + ComplianceFramework( + framework="Test", + name="Test", + description="d", + requirements=[ + UniversalComplianceRequirement( + id="1.1", + description="d", + attributes={"Level": "bad"}, + checks={}, + ), + UniversalComplianceRequirement( + id="1.2", + description="d", + attributes={"Level": "also_bad"}, + checks={}, + ), + ], + attributes_metadata=self._metadata(enum=["high", "low"]), + ) diff --git a/tests/providers/aws/lib/cloudtrail_timeline/cloudtrail_timeline_test.py b/tests/providers/aws/lib/cloudtrail_timeline/cloudtrail_timeline_test.py index 1c1c10bfd3..aeca0f7c1d 100644 --- a/tests/providers/aws/lib/cloudtrail_timeline/cloudtrail_timeline_test.py +++ b/tests/providers/aws/lib/cloudtrail_timeline/cloudtrail_timeline_test.py @@ -120,7 +120,7 @@ class TestCloudTrailTimeline: assert result[0]["event_name"] == "RunInstances" def test_get_resource_timeline_prefers_uid_over_id(self, mock_session): - """When both resource_id and resource_uid are provided, UID should be used.""" + """When both resource_id and resource_uid are provided, UID is tried first.""" mock_client = MagicMock() mock_client.lookup_events.return_value = {"Events": []} mock_session.client.return_value = mock_client @@ -132,9 +132,9 @@ class TestCloudTrailTimeline: resource_uid="arn:aws:ec2:us-east-1:123:instance/i-1234", ) - # Verify UID was used in the lookup - call_args = mock_client.lookup_events.call_args - lookup_attrs = call_args.kwargs["LookupAttributes"] + # Verify UID was used on the first lookup call + first_call = mock_client.lookup_events.call_args_list[0] + lookup_attrs = first_call.kwargs["LookupAttributes"] assert ( lookup_attrs[0]["AttributeValue"] == "arn:aws:ec2:us-east-1:123:instance/i-1234" @@ -606,3 +606,159 @@ class TestIsReadOnlyEvent: """Verify write events are not marked as read-only.""" timeline = CloudTrailTimeline(session=mock_session) assert timeline._is_read_only_event(event_name) is False + + +class TestExtractShortName: + """Tests for _extract_short_name static method.""" + + @pytest.mark.parametrize( + "identifier,expected", + [ + ("arn:aws:s3:::my-bucket", "my-bucket"), + ("arn:aws:iam::123456789012:user/alice", "alice"), + ("arn:aws:iam::123456789012:role/MyRole", "MyRole"), + ( + "arn:aws:ec2:us-east-1:123456789012:instance/i-0abc1234", + "i-0abc1234", + ), + ( + "arn:aws:lambda:us-east-1:123456789012:function:my-func", + "my-func", + ), + ("arn:aws:rds:us-east-1:123456789012:db:mydb", "mydb"), + ("arn:aws:dynamodb:us-east-1:123456789012:table/MyTable", "MyTable"), + ( + "arn:aws:kms:us-east-1:123456789012:key/abcd-efgh", + "abcd-efgh", + ), + ("i-0abc1234", "i-0abc1234"), + ("my-bucket", "my-bucket"), + ("", ""), + ], + ) + def test_extract_short_name(self, identifier, expected): + assert CloudTrailTimeline._extract_short_name(identifier) == expected + + +class TestLookupEventsFallback: + """Tests for the ARN-to-short-name fallback in _lookup_events.""" + + @pytest.fixture + def mock_session(self): + return MagicMock() + + @pytest.fixture + def sample_event(self): + return { + "EventId": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + "EventTime": datetime(2024, 1, 15, 10, 30, 0, tzinfo=timezone.utc), + "EventName": "CreateBucket", + "EventSource": "s3.amazonaws.com", + "CloudTrailEvent": json.dumps( + { + "userIdentity": { + "type": "IAMUser", + "arn": "arn:aws:iam::123456789012:user/admin", + "userName": "admin", + } + } + ), + } + + def test_no_fallback_when_arn_returns_events(self, mock_session, sample_event): + """When the ARN lookup returns events, we do not retry with the short name.""" + mock_client = MagicMock() + mock_client.lookup_events.return_value = {"Events": [sample_event]} + mock_session.client.return_value = mock_client + + timeline = CloudTrailTimeline(session=mock_session) + result = timeline.get_resource_timeline( + region="us-east-1", + resource_uid="arn:aws:kms:us-east-1:123456789012:key/abcd-efgh", + ) + + assert len(result) == 1 + assert mock_client.lookup_events.call_count == 1 + call = mock_client.lookup_events.call_args + assert ( + call.kwargs["LookupAttributes"][0]["AttributeValue"] + == "arn:aws:kms:us-east-1:123456789012:key/abcd-efgh" + ) + + def test_fallback_to_short_name_when_arn_returns_empty( + self, mock_session, sample_event + ): + """When the ARN lookup returns nothing, we retry with the short name.""" + mock_client = MagicMock() + mock_client.lookup_events.side_effect = [ + {"Events": []}, + {"Events": [sample_event]}, + ] + mock_session.client.return_value = mock_client + + timeline = CloudTrailTimeline(session=mock_session) + result = timeline.get_resource_timeline( + region="us-east-1", resource_uid="arn:aws:s3:::my-bucket" + ) + + assert len(result) == 1 + assert mock_client.lookup_events.call_count == 2 + first_call, second_call = mock_client.lookup_events.call_args_list + assert ( + first_call.kwargs["LookupAttributes"][0]["AttributeValue"] + == "arn:aws:s3:::my-bucket" + ) + assert ( + second_call.kwargs["LookupAttributes"][0]["AttributeValue"] == "my-bucket" + ) + + def test_no_fallback_when_identifier_has_no_short_name(self, mock_session): + """A non-ARN identifier collapses to itself; no retry should fire.""" + mock_client = MagicMock() + mock_client.lookup_events.return_value = {"Events": []} + mock_session.client.return_value = mock_client + + timeline = CloudTrailTimeline(session=mock_session) + result = timeline.get_resource_timeline( + region="us-east-1", resource_id="i-0abc1234" + ) + + assert result == [] + assert mock_client.lookup_events.call_count == 1 + + def test_no_fallback_when_identifier_is_not_arn(self, mock_session): + """A non-ARN identifier with / or : must not trigger the retry.""" + mock_client = MagicMock() + mock_client.lookup_events.return_value = {"Events": []} + mock_session.client.return_value = mock_client + + timeline = CloudTrailTimeline(session=mock_session) + result = timeline.get_resource_timeline( + region="us-east-1", resource_id="some-prefix/weird:value" + ) + + assert result == [] + assert mock_client.lookup_events.call_count == 1 + + def test_both_lookups_empty_returns_empty_list(self, mock_session): + """If both the ARN and short-name lookups return empty, we return [].""" + mock_client = MagicMock() + mock_client.lookup_events.return_value = {"Events": []} + mock_session.client.return_value = mock_client + + timeline = CloudTrailTimeline(session=mock_session) + result = timeline.get_resource_timeline( + region="us-east-1", + resource_uid="arn:aws:ec2:us-east-1:123456789012:instance/i-0abc1234", + ) + + assert result == [] + assert mock_client.lookup_events.call_count == 2 + first_call, second_call = mock_client.lookup_events.call_args_list + assert ( + first_call.kwargs["LookupAttributes"][0]["AttributeValue"] + == "arn:aws:ec2:us-east-1:123456789012:instance/i-0abc1234" + ) + assert ( + second_call.kwargs["LookupAttributes"][0]["AttributeValue"] == "i-0abc1234" + ) diff --git a/tests/providers/cloudflare/cloudflare_provider_test.py b/tests/providers/cloudflare/cloudflare_provider_test.py index 58b7eb8086..c3ba6d75e2 100644 --- a/tests/providers/cloudflare/cloudflare_provider_test.py +++ b/tests/providers/cloudflare/cloudflare_provider_test.py @@ -433,6 +433,29 @@ class TestCloudflareValidateCredentials: with pytest.raises(CloudflareNoAccountsError): CloudflareProvider.validate_credentials(session) + def test_validate_credentials_breaks_on_repeated_account_ids(self): + """Pagination must stop when the SDK repeats account IDs to avoid infinite loops.""" + + def repeating_accounts(): + account = MagicMock() + account.id = ACCOUNT_ID + while True: + yield account + + mock_client = MagicMock() + mock_client.user.get.side_effect = Exception("Some other error") + mock_client.accounts.list.return_value = repeating_accounts() + + session = CloudflareSession( + client=mock_client, + api_token=API_TOKEN, + api_key=None, + api_email=None, + ) + + # Must return without hanging; repeated IDs break the loop. + CloudflareProvider.validate_credentials(session) + class TestCloudflareTestConnection: """Tests for test_connection method.""" diff --git a/tests/providers/github/github_provider_test.py b/tests/providers/github/github_provider_test.py index a87a35e038..00a3c33cbc 100644 --- a/tests/providers/github/github_provider_test.py +++ b/tests/providers/github/github_provider_test.py @@ -12,6 +12,8 @@ from prowler.providers.github.exceptions.exceptions import ( GithubInvalidCredentialsError, GithubInvalidProviderIdError, GithubInvalidTokenError, + GithubRepoListFileNotFoundError, + GithubRepoListFileReadError, GithubSetUpIdentityError, GithubSetUpSessionError, ) @@ -708,3 +710,81 @@ class Test_GithubProvider_Scoping: assert provider_none.repositories == [] assert provider_none.organizations == [] + + +class TestGitHubProviderLoadReposFromFile: + """Tests for GithubProvider._load_repos_from_file""" + + def _make_provider(self): + """Create a GithubProvider instance with mocked session/identity.""" + with ( + patch( + "prowler.providers.github.github_provider.GithubProvider.setup_session", + return_value=GithubSession(token=PAT_TOKEN, id="", key=""), + ), + patch( + "prowler.providers.github.github_provider.GithubProvider.setup_identity", + return_value=GithubIdentityInfo( + account_id=ACCOUNT_ID, + account_name=ACCOUNT_NAME, + account_url=ACCOUNT_URL, + ), + ), + ): + provider = GithubProvider( + personal_access_token=PAT_TOKEN, + ) + return provider + + def test_load_repos_from_file_happy_path(self, tmp_path): + provider = self._make_provider() + repo_file = tmp_path / "repos.txt" + repo_file.write_text("owner/repo-a\nowner/repo-b\nowner/repo-c\n") + + provider._load_repos_from_file(str(repo_file)) + + assert "owner/repo-a" in provider.repositories + assert "owner/repo-b" in provider.repositories + assert "owner/repo-c" in provider.repositories + + def test_load_repos_from_file_comments_and_blanks(self, tmp_path): + provider = self._make_provider() + repo_file = tmp_path / "repos.txt" + repo_file.write_text( + "# This is a comment\n" + "\n" + "owner/repo-a\n" + " # Another comment\n" + " \n" + "owner/repo-b\n" + ) + + provider._load_repos_from_file(str(repo_file)) + + assert provider.repositories == ["owner/repo-a", "owner/repo-b"] + + def test_load_repos_from_file_not_found(self): + provider = self._make_provider() + + with pytest.raises(GithubRepoListFileNotFoundError): + provider._load_repos_from_file("/nonexistent/path/repos.txt") + + def test_load_repos_from_file_exceeds_max_lines(self, tmp_path): + provider = self._make_provider() + repo_file = tmp_path / "repos.txt" + # Write MAX_REPO_LIST_LINES + 1 lines to trigger the guard + lines = [f"owner/repo-{i}" for i in range(provider.MAX_REPO_LIST_LINES + 1)] + repo_file.write_text("\n".join(lines) + "\n") + + with pytest.raises(GithubRepoListFileReadError): + provider._load_repos_from_file(str(repo_file)) + + def test_load_repos_from_file_skips_long_names(self, tmp_path): + provider = self._make_provider() + repo_file = tmp_path / "repos.txt" + long_name = "a" * (provider.MAX_REPO_NAME_LENGTH + 1) + repo_file.write_text(f"owner/valid-repo\n{long_name}\nowner/also-valid\n") + + provider._load_repos_from_file(str(repo_file)) + + assert provider.repositories == ["owner/valid-repo", "owner/also-valid"] diff --git a/tests/providers/github/lib/arguments/github_arguments_test.py b/tests/providers/github/lib/arguments/github_arguments_test.py index fd2f813b6e..20fd3f18be 100644 --- a/tests/providers/github/lib/arguments/github_arguments_test.py +++ b/tests/providers/github/lib/arguments/github_arguments_test.py @@ -82,13 +82,14 @@ class Test_GitHubArguments: arguments.init_parser(mock_github_args) # Verify scoping arguments were added - assert self.mock_scoping_group.add_argument.call_count == 2 + assert self.mock_scoping_group.add_argument.call_count == 3 # Check that all scoping arguments are present calls = self.mock_scoping_group.add_argument.call_args_list scoping_args = [call[0][0] for call in calls] assert "--repository" in scoping_args + assert "--repo-list-file" in scoping_args assert "--organization" in scoping_args def test_repository_argument_configuration(self): @@ -277,6 +278,33 @@ class Test_GitHubArguments_Integration: assert args.repository == ["owner1/repo1"] assert args.organization == ["org1"] + def test_real_argument_parsing_with_repo_list_file(self): + """Test parsing arguments with repo-list-file scoping""" + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers() + common_parser = argparse.ArgumentParser(add_help=False) + + mock_github_args = MagicMock() + mock_github_args.subparsers = subparsers + mock_github_args.common_providers_parser = common_parser + + arguments.init_parser(mock_github_args) + + # Parse arguments with repo-list-file + args = parser.parse_args( + [ + "github", + "--personal-access-token", + "test-token", + "--repo-list-file", + "/path/to/repos.txt", + ] + ) + + assert args.personal_access_token == "test-token" + assert args.repo_list_file == "/path/to/repos.txt" + assert args.repository is None + def test_real_argument_parsing_empty_scoping(self): """Test parsing arguments with empty scoping values""" parser = argparse.ArgumentParser() diff --git a/ui/.dockerignore b/ui/.dockerignore index 54ed258b8c..9049298c54 100644 --- a/ui/.dockerignore +++ b/ui/.dockerignore @@ -13,5 +13,5 @@ README.md !.next/static !.next/standalone .git -.husky +.pre-commit-config.yaml scripts/setup-git-hooks.js diff --git a/ui/.pre-commit-config.yaml b/ui/.pre-commit-config.yaml new file mode 100644 index 0000000000..c92ea6cfd3 --- /dev/null +++ b/ui/.pre-commit-config.yaml @@ -0,0 +1,37 @@ +orphan: true + +repos: + - repo: local + hooks: + - id: ui-typecheck + name: UI - TypeScript Check + entry: pnpm run typecheck + language: system + files: '\.(ts|tsx|js|jsx)$' + pass_filenames: false + priority: 0 + + - id: ui-lint + name: UI - ESLint + entry: pnpm run lint:check + language: system + files: '\.(ts|tsx|js|jsx)$' + pass_filenames: false + priority: 0 + + - id: ui-tests + name: UI - Unit Tests + entry: pnpm exec vitest related --run + language: system + files: '\.(ts|tsx|js|jsx)$' + exclude: '\.test\.|\.spec\.|vitest\.config|vitest\.setup' + pass_filenames: true + priority: 1 + + - id: ui-build + name: UI - Build + entry: pnpm run build + language: system + files: '\.(ts|tsx|js|jsx|json|css)$' + pass_filenames: false + priority: 2 diff --git a/ui/CHANGELOG.md b/ui/CHANGELOG.md index 3f1916af71..39abcd20b4 100644 --- a/ui/CHANGELOG.md +++ b/ui/CHANGELOG.md @@ -8,10 +8,32 @@ All notable changes to the **Prowler UI** are documented in this file. - Sign-in and sign-up redesigned with an animated background and a live release highlights panel from GitHub, with adaptive summaries for patch releases [(#10774)](https://github.com/prowler-cloud/prowler/pull/10774) +### πŸ”„ Changed + +- Redesign compliance page with a horizontal ThreatScore card (always-visible pillar breakdown + ActionDropdown), client-side search for compliance frameworks, compact scan selector trigger, responsive mobile filters, download-started toasts for CSV/PDF exports, enhanced compliance cards with truncated titles, and Alert-based empty/error states; migrate Progress component from HeroUI to shadcn [(#10767)](https://github.com/prowler-cloud/prowler/pull/10767) +- Backward-compatibility middleware redirect from `/sign-up?invitation_token=…` to `/invitation/accept?invitation_token=…`; new invitation emails use `/invitation/accept` directly [(#10797)](https://github.com/prowler-cloud/prowler/pull/10797) + +--- + +## [1.24.2] (Prowler v5.24.2) + +### 🐞 Fixed + +- Default muted filter now applied consistently on the findings page and the finding-group resource drill-down, keeping muted findings hidden unless the "include muted findings" checkbox is opted in [(#10818)](https://github.com/prowler-cloud/prowler/pull/10818) + --- ## [1.24.1] (Prowler v5.24.1) +### 🐞 Fixed + +- Findings and filter UX fixes: exclude muted findings by default in the resource detail drawer and finding group resource views, show category context label (for example `Status: FAIL`) on MultiSelect triggers instead of hiding the placeholder, and add a `wide` width option for filter dropdowns applied to the findings Scan filter to prevent label truncation [(#10734)](https://github.com/prowler-cloud/prowler/pull/10734) +- Findings grouped view now handles zero-resource IaC counters, refines drawer loading states, and adds provider indicators to finding groups [(#10736)](https://github.com/prowler-cloud/prowler/pull/10736) +- Other Findings for this resource: ordering by `severity` [(#10778)](https://github.com/prowler-cloud/prowler/pull/10778) +- Other Findings for this resource: show `delta` indicator [(#10778)](https://github.com/prowler-cloud/prowler/pull/10778) +- Compliance: requirement findings do not show muted findings [(#10778)](https://github.com/prowler-cloud/prowler/pull/10778) +- Latest new findings: link to finding groups order by `-severity,-last_seen_at` [(#10778)](https://github.com/prowler-cloud/prowler/pull/10778) + ### πŸ”’ Security - Upgrade React to 19.2.5 and Next.js to 16.2.3 to mitigate CVE-2026-23869 (React2DoS), a high-severity unauthenticated remote DoS vulnerability in the React Flight Protocol's Server Function deserialization [(#10754)](https://github.com/prowler-cloud/prowler/pull/10754) diff --git a/ui/README.md b/ui/README.md index 148ef61d5b..b1af73ffe3 100644 --- a/ui/README.md +++ b/ui/README.md @@ -85,10 +85,10 @@ git clone git@github.com:prowler-cloud/ui.git pnpm install ``` -**Note:** The `pnpm install` command will automatically configure Git hooks for code quality checks. If you experience issues, you can manually configure them: +**Note:** The `pnpm install` command will automatically configure prek Git hooks for code quality checks. If hooks are not installed, run from the repo root: ```bash -git config core.hooksPath "ui/.husky" +prek install ``` #### Run the development server diff --git a/ui/actions/compliances/compliances.ts b/ui/actions/compliances/compliances.ts index b23723f670..d5f4fd4954 100644 --- a/ui/actions/compliances/compliances.ts +++ b/ui/actions/compliances/compliances.ts @@ -6,12 +6,10 @@ import { handleApiResponse } from "@/lib/server-actions-helper"; export const getCompliancesOverview = async ({ scanId, region, - query, filters = {}, }: { scanId?: string; region?: string | string[]; - query?: string; filters?: Record; } = {}) => { const headers = await getAuthHeaders({ contentType: false }); @@ -31,8 +29,6 @@ export const getCompliancesOverview = async ({ setParam("filter[scan_id]", scanId); setParam("filter[region__in]", region); - if (query) url.searchParams.set("filter[search]", query); - try { const response = await fetch(url.toString(), { headers, @@ -46,15 +42,16 @@ export const getCompliancesOverview = async ({ }; export const getComplianceOverviewMetadataInfo = async ({ - query = "", sort = "", filters = {}, -}) => { +}: { + sort?: string; + filters?: Record; +} = {}) => { const headers = await getAuthHeaders({ contentType: false }); const url = new URL(`${apiBaseUrl}/compliance-overviews/metadata`); - if (query) url.searchParams.append("filter[search]", query); if (sort) url.searchParams.append("sort", sort); Object.entries(filters).forEach(([key, value]) => { diff --git a/ui/actions/findings/findings-by-resource.adapter.test.ts b/ui/actions/findings/findings-by-resource.adapter.test.ts index b8781f0036..87d37c192c 100644 --- a/ui/actions/findings/findings-by-resource.adapter.test.ts +++ b/ui/actions/findings/findings-by-resource.adapter.test.ts @@ -115,4 +115,35 @@ describe("adaptFindingsByResourceResponse β€” malformed input", () => { expect(result[0].id).toBe("finding-1"); expect(result[0].checkId).toBe("s3_check"); }); + + it("should normalize a single finding response into a one-item drawer array", () => { + // Given β€” getFindingById returns a single JSON:API resource object + const input = { + data: { + id: "finding-1", + attributes: { + uid: "uid-1", + check_id: "s3_check", + status: "FAIL", + severity: "critical", + check_metadata: { + checktitle: "S3 Check", + }, + }, + relationships: { + resources: { data: [] }, + scan: { data: null }, + }, + }, + included: [], + }; + + // When + const result = adaptFindingsByResourceResponse(input); + + // Then + expect(result).toHaveLength(1); + expect(result[0].id).toBe("finding-1"); + expect(result[0].checkTitle).toBe("S3 Check"); + }); }); diff --git a/ui/actions/findings/findings-by-resource.adapter.ts b/ui/actions/findings/findings-by-resource.adapter.ts index 75e8f5a81b..d1685eaa9f 100644 --- a/ui/actions/findings/findings-by-resource.adapter.ts +++ b/ui/actions/findings/findings-by-resource.adapter.ts @@ -165,16 +165,18 @@ type IncludedDict = Record; * then resolves each finding's resource and provider relationships. */ interface JsonApiResponse { - data: FindingApiItem[]; + data: FindingApiItem | FindingApiItem[]; included?: Record[]; } function isJsonApiResponse(value: unknown): value is JsonApiResponse { + const data = (value as { data?: unknown })?.data; + return ( value !== null && typeof value === "object" && "data" in value && - Array.isArray((value as { data: unknown }).data) + (Array.isArray(data) || (data !== null && typeof data === "object")) ); } @@ -188,8 +190,11 @@ export function adaptFindingsByResourceResponse( const resourcesDict = createDict("resources", apiResponse) as IncludedDict; const scansDict = createDict("scans", apiResponse) as IncludedDict; const providersDict = createDict("providers", apiResponse) as IncludedDict; + const findings = Array.isArray(apiResponse.data) + ? apiResponse.data + : [apiResponse.data]; - return apiResponse.data.map((item) => { + return findings.map((item) => { const attrs = item.attributes; const meta = (attrs.check_metadata || {}) as Record; const remediationRaw = meta.remediation as diff --git a/ui/actions/findings/findings-by-resource.test.ts b/ui/actions/findings/findings-by-resource.test.ts index 4aff44a2cf..7bc2793195 100644 --- a/ui/actions/findings/findings-by-resource.test.ts +++ b/ui/actions/findings/findings-by-resource.test.ts @@ -43,6 +43,7 @@ vi.mock("@/actions/finding-groups", () => ({ })); import { + getLatestFindingsByResourceUid, resolveFindingIdsByCheckIds, resolveFindingIdsByVisibleGroupResources, } from "./findings-by-resource"; @@ -262,3 +263,46 @@ describe("resolveFindingIdsByVisibleGroupResources", () => { expect(fetchMock).not.toHaveBeenCalled(); }); }); + +describe("getLatestFindingsByResourceUid", () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.stubGlobal("fetch", fetchMock); + getAuthHeadersMock.mockResolvedValue({ Authorization: "Bearer token" }); + handleApiResponseMock.mockResolvedValue({ data: [] }); + }); + + it("should restrict to FAIL, exclude muted findings, and apply severity/time sorting by default", async () => { + fetchMock.mockResolvedValue(new Response("", { status: 200 })); + + await getLatestFindingsByResourceUid({ + resourceUid: "resource-1", + }); + + const calledUrl = new URL(fetchMock.mock.calls[0][0]); + expect(calledUrl.pathname).toBe("/api/v1/findings/latest"); + expect(calledUrl.searchParams.get("filter[resource_uid]")).toBe( + "resource-1", + ); + // Status filter is applied server-side so the page[size]=50 window + // always holds FAIL rows β€” guards against PASS-heavy resources + // starving FAILs out of the result. + expect(calledUrl.searchParams.get("filter[status]")).toBe("FAIL"); + expect(calledUrl.searchParams.get("filter[muted]")).toBe("false"); + expect(calledUrl.searchParams.get("sort")).toBe("severity,-updated_at"); + }); + + it("should include muted findings only when explicitly requested", async () => { + fetchMock.mockResolvedValue(new Response("", { status: 200 })); + + await getLatestFindingsByResourceUid({ + resourceUid: "resource-1", + includeMuted: true, + }); + + const calledUrl = new URL(fetchMock.mock.calls[0][0]); + expect(calledUrl.searchParams.get("filter[status]")).toBe("FAIL"); + expect(calledUrl.searchParams.get("filter[muted]")).toBe("include"); + expect(calledUrl.searchParams.get("sort")).toBe("severity,-updated_at"); + }); +}); diff --git a/ui/actions/findings/findings-by-resource.ts b/ui/actions/findings/findings-by-resource.ts index 5a848fff4b..74a0bcb6de 100644 --- a/ui/actions/findings/findings-by-resource.ts +++ b/ui/actions/findings/findings-by-resource.ts @@ -250,10 +250,12 @@ export const getLatestFindingsByResourceUid = async ({ resourceUid, page = 1, pageSize = 50, + includeMuted = false, }: { resourceUid: string; page?: number; pageSize?: number; + includeMuted?: boolean; }) => { const headers = await getAuthHeaders({ contentType: false }); @@ -262,8 +264,9 @@ export const getLatestFindingsByResourceUid = async ({ ); url.searchParams.append("filter[resource_uid]", resourceUid); - url.searchParams.append("filter[muted]", "include"); - url.searchParams.append("sort", "-severity,-updated_at"); + url.searchParams.append("filter[status]", "FAIL"); + url.searchParams.append("filter[muted]", includeMuted ? "include" : "false"); + url.searchParams.append("sort", "severity,-updated_at"); if (page) url.searchParams.append("page[number]", page.toString()); if (pageSize) url.searchParams.append("page[size]", pageSize.toString()); diff --git a/ui/actions/findings/findings.ts b/ui/actions/findings/findings.ts index 7ce7f931ad..242bd007a8 100644 --- a/ui/actions/findings/findings.ts +++ b/ui/actions/findings/findings.ts @@ -141,7 +141,15 @@ export const getLatestMetadataInfo = async ({ } }; -export const getFindingById = async (findingId: string, include = "") => { +interface GetFindingByIdOptions { + source?: "resource-detail-drawer"; +} + +export const getFindingById = async ( + findingId: string, + include = "", + _options?: GetFindingByIdOptions, +) => { const headers = await getAuthHeaders({ contentType: false }); const url = new URL(`${apiBaseUrl}/findings/${findingId}`); diff --git a/ui/app/(auth)/invitation/accept/accept-invitation-client.tsx b/ui/app/(auth)/invitation/accept/accept-invitation-client.tsx index 73e6dbe8fd..f334053b48 100644 --- a/ui/app/(auth)/invitation/accept/accept-invitation-client.tsx +++ b/ui/app/(auth)/invitation/accept/accept-invitation-client.tsx @@ -8,10 +8,6 @@ import { useEffect, useRef, useState } from "react"; import { acceptInvitation } from "@/actions/invitations"; import { Button } from "@/components/shadcn"; -import { - INVITATION_ACTION_PARAM, - INVITATION_SIGNUP_ACTION, -} from "@/lib/invitation-routing"; type AcceptState = | { kind: "no-token" } @@ -204,7 +200,7 @@ export function AcceptInvitationClient({ className="w-full" onClick={() => { router.push( - `/sign-up?invitation_token=${encodeURIComponent(token!)}&${INVITATION_ACTION_PARAM}=${INVITATION_SIGNUP_ACTION}`, + `/sign-up?invitation_token=${encodeURIComponent(token!)}`, ); }} > diff --git a/ui/app/(prowler)/_overview/graphs-tabs/findings-view/findings-view.ssr.tsx b/ui/app/(prowler)/_overview/graphs-tabs/findings-view/findings-view.ssr.tsx index 9554869dce..cca971578c 100644 --- a/ui/app/(prowler)/_overview/graphs-tabs/findings-view/findings-view.ssr.tsx +++ b/ui/app/(prowler)/_overview/graphs-tabs/findings-view/findings-view.ssr.tsx @@ -4,6 +4,7 @@ import { getLatestFindings } from "@/actions/findings/findings"; import { LighthouseBanner } from "@/components/lighthouse/banner"; import { LinkToFindings } from "@/components/overview"; import { ColumnLatestFindings } from "@/components/overview/new-findings-table/table"; +import { CardTitle } from "@/components/shadcn"; import { DataTable } from "@/components/ui/table"; import { createDict } from "@/lib/helper"; import { FindingProps, SearchParamsProps } from "@/types"; @@ -57,24 +58,23 @@ export async function FindingsViewSSR({ searchParams }: FindingsViewSSRProps) { }; return ( -
+
-
-
-

- Latest new failing findings -

-

- Showing the latest 10 new failing findings by severity. -

- -
-
- +
+ Latest New Failed Findings +

+ Showing the latest 10 sorted by severity +

+
+ +
+ } />
); diff --git a/ui/app/(prowler)/_overview/graphs-tabs/graphs-tabs-wrapper.tsx b/ui/app/(prowler)/_overview/graphs-tabs/graphs-tabs-wrapper.tsx index c21491e37f..f9741dc75e 100644 --- a/ui/app/(prowler)/_overview/graphs-tabs/graphs-tabs-wrapper.tsx +++ b/ui/app/(prowler)/_overview/graphs-tabs/graphs-tabs-wrapper.tsx @@ -1,6 +1,7 @@ import { Skeleton } from "@heroui/skeleton"; import { Suspense } from "react"; +import { SkeletonTableNewFindings } from "@/components/overview/new-findings-table/table"; import { SearchParamsProps } from "@/types"; import { GraphsTabsClient } from "./_components/graphs-tabs-client"; @@ -18,6 +19,10 @@ const LoadingFallback = () => ( ); +const TAB_FALLBACKS: Partial> = { + findings: , +}; + type GraphComponent = React.ComponentType<{ searchParams: SearchParamsProps }>; const GRAPH_COMPONENTS: Record = { @@ -38,9 +43,10 @@ export const GraphsTabsWrapper = async ({ const tabsContent = Object.fromEntries( GRAPH_TABS.map((tab) => { const Component = GRAPH_COMPONENTS[tab.id]; + const fallback = TAB_FALLBACKS[tab.id] ?? ; return [ tab.id, - }> + , ]; diff --git a/ui/app/(prowler)/compliance/[compliancetitle]/page.tsx b/ui/app/(prowler)/compliance/[compliancetitle]/page.tsx index dd86be1f8a..069826c3ae 100644 --- a/ui/app/(prowler)/compliance/[compliancetitle]/page.tsx +++ b/ui/app/(prowler)/compliance/[compliancetitle]/page.tsx @@ -78,7 +78,7 @@ export default async function ComplianceDetail({ await Promise.all([ getComplianceOverviewMetadataInfo({ filters: { - "filter[scan_id]": selectedScanId, + "filter[scan_id]": selectedScanId ?? undefined, }, }), getComplianceAttributes(complianceId), diff --git a/ui/app/(prowler)/compliance/page.test.tsx b/ui/app/(prowler)/compliance/page.test.tsx new file mode 100644 index 0000000000..42bbbe672f --- /dev/null +++ b/ui/app/(prowler)/compliance/page.test.tsx @@ -0,0 +1,16 @@ +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { describe, expect, it } from "vitest"; + +describe("Compliance overview page", () => { + const currentDir = path.dirname(fileURLToPath(import.meta.url)); + const filePath = path.join(currentDir, "page.tsx"); + const source = readFileSync(filePath, "utf8"); + + it("delegates client-side search to ComplianceOverviewGrid", () => { + expect(source).toContain("ComplianceOverviewGrid"); + expect(source).not.toContain("filter[search]"); + }); +}); diff --git a/ui/app/(prowler)/compliance/page.tsx b/ui/app/(prowler)/compliance/page.tsx index e3e0a63705..92015cd142 100644 --- a/ui/app/(prowler)/compliance/page.tsx +++ b/ui/app/(prowler)/compliance/page.tsx @@ -1,3 +1,4 @@ +import { Info } from "lucide-react"; import { Suspense } from "react"; import { @@ -7,12 +8,14 @@ import { import { getThreatScore } from "@/actions/overview"; import { getScans } from "@/actions/scans"; import { - ComplianceCard, ComplianceSkeletonGrid, NoScansAvailable, ThreatScoreBadge, } from "@/components/compliance"; -import { ComplianceHeader } from "@/components/compliance/compliance-header/compliance-header"; +import { ComplianceFilters } from "@/components/compliance/compliance-header/compliance-filters"; +import { ComplianceOverviewGrid } from "@/components/compliance/compliance-overview-grid"; +import { Alert, AlertDescription } from "@/components/shadcn/alert"; +import { Card, CardContent } from "@/components/shadcn/card/card"; import { ContentLayout } from "@/components/ui"; import { ExpandedScanData, @@ -30,12 +33,6 @@ export default async function Compliance({ const resolvedSearchParams = await searchParams; const searchParamsKey = JSON.stringify(resolvedSearchParams || {}); - const filters = Object.fromEntries( - Object.entries(resolvedSearchParams).filter(([key]) => - key.startsWith("filter["), - ), - ); - const scansData = await getScans({ filters: { "filter[state]": "completed", @@ -79,9 +76,12 @@ export default async function Compliance({ .filter(Boolean) as ExpandedScanData[]; // Use scanId from URL, or select the first scan if not provided - const selectedScanId = - resolvedSearchParams.scanId || expandedScansData[0]?.id || null; - const query = (filters["filter[search]"] as string) || ""; + const scanIdParam = resolvedSearchParams.scanId; + const scanIdFromUrl = Array.isArray(scanIdParam) + ? scanIdParam[0] + : scanIdParam; + const selectedScanId: string | null = + scanIdFromUrl || expandedScansData[0]?.id || null; // Find the selected scan const selectedScan = expandedScansData.find( @@ -102,7 +102,6 @@ export default async function Compliance({ // Fetch metadata if we have a selected scan const metadataInfoData = selectedScanId ? await getComplianceOverviewMetadataInfo({ - query, filters: { "filter[scan_id]": selectedScanId, }, @@ -131,28 +130,39 @@ export default async function Compliance({ {selectedScanId ? ( <> -
-
- -
- {threatScoreData && - typeof selectedScanId === "string" && - selectedScan && ( -
- -
- )} + {/* Row 1: Filters */} +
+
- }> + + {/* Row 2: ThreatScore card β€” full width, horizontal */} + {threatScoreData && + typeof selectedScanId === "string" && + selectedScan && ( +
+ +
+ )} + + {/* Row 3: Compliance grid with client-side search */} + + + + } + > key.startsWith("filter[")), - ); - - // Extract query from filters - const query = (filters["filter[search]"] as string) || ""; - // Only fetch compliance data if we have a valid scanId const compliancesData = scanId && scanId.trim() !== "" ? await getCompliancesOverview({ scanId, region: regionFilter, - query, }) : { data: [], errors: [] }; const type = compliancesData?.data?.type; + const frameworks = compliancesData?.data + ?.filter((compliance: ComplianceOverviewData) => { + return compliance.attributes.framework !== "ProwlerThreatScore"; + }) + .sort((a: ComplianceOverviewData, b: ComplianceOverviewData) => + a.attributes.framework.localeCompare(b.attributes.framework), + ); // Check if the response contains no data if ( @@ -204,58 +212,49 @@ const SSRComplianceGrid = async ({ type === "tasks" ) { return ( -
-
- No compliance data available for the selected scan. -
-
+ + + + This scan has no compliance data available yet, please select a + different one. + + ); } // Handle errors returned by the API if (compliancesData?.errors?.length > 0) { return ( -
-
Provide a valid scan ID.
-
+ + + Provide a valid scan ID. + ); } return ( -
- {compliancesData.data - .filter((compliance: ComplianceOverviewData) => { - // Filter out ProwlerThreatScore from the grid - return compliance.attributes.framework !== "ProwlerThreatScore"; - }) - .sort((a: ComplianceOverviewData, b: ComplianceOverviewData) => - a.attributes.framework.localeCompare(b.attributes.framework), - ) - .map((compliance: ComplianceOverviewData) => { - const { attributes, id } = compliance; - const { - framework, - version, - requirements_passed, - total_requirements, - } = attributes; - - return ( - - ); - })} -
+ + + + ); +}; + +const ComplianceOverviewPanel = ({ + children, +}: { + children: React.ReactNode; +}) => { + return ( + + {children} + ); }; diff --git a/ui/app/(prowler)/findings/page.test.ts b/ui/app/(prowler)/findings/page.test.ts index 76462dff99..2038b92291 100644 --- a/ui/app/(prowler)/findings/page.test.ts +++ b/ui/app/(prowler)/findings/page.test.ts @@ -25,8 +25,10 @@ describe("findings page", () => { expect(source).toContain("resolveFindingScanDateFilters"); }); - it("uses getLatestFindingGroups for non-date/scan queries and getFindingGroups for historical", () => { - expect(source).toContain("hasDateOrScan"); + it("uses resolved filters to choose getFindingGroups for historical queries and getLatestFindingGroups otherwise", () => { + expect(source).toContain("hasHistoricalData"); + expect(source).toContain("hasDateOrScanFilter(filtersWithScanDates)"); + expect(source).toContain("hasDateOrScanFilter(filters)"); expect(source).toContain("getFindingGroups"); expect(source).toContain("getLatestFindingGroups"); }); @@ -34,4 +36,8 @@ describe("findings page", () => { it("guards errors array access with a length check", () => { expect(source).toContain("errors?.length > 0"); }); + + it("applies the shared default muted filter so muted findings are hidden unless the caller opts in", () => { + expect(source).toContain("applyDefaultMutedFilter"); + }); }); diff --git a/ui/app/(prowler)/findings/page.tsx b/ui/app/(prowler)/findings/page.tsx index 948d264602..dea6e9b4fb 100644 --- a/ui/app/(prowler)/findings/page.tsx +++ b/ui/app/(prowler)/findings/page.tsx @@ -16,6 +16,7 @@ import { import { ContentLayout } from "@/components/ui"; import { FilterTransitionWrapper } from "@/contexts"; import { + applyDefaultMutedFilter, createScanDetailsMapping, extractFiltersAndQuery, extractSortAndKey, @@ -34,25 +35,26 @@ export default async function Findings({ const { encodedSort } = extractSortAndKey(resolvedSearchParams); const { filters, query } = extractFiltersAndQuery(resolvedSearchParams); - // Check if the searchParams contain any date or scan filter - const hasDateOrScan = hasDateOrScanFilter(resolvedSearchParams); - const [providersData, scansData] = await Promise.all([ getProviders({ pageSize: 50 }), getScans({ pageSize: 50 }), ]); - const filtersWithScanDates = await resolveFindingScanDateFilters({ - filters, - scans: scansData?.data || [], - loadScan: async (scanId: string) => { - const response = await getScan(scanId); - return response?.data; - }, - }); + const filtersWithScanDates = applyDefaultMutedFilter( + await resolveFindingScanDateFilters({ + filters, + scans: scansData?.data || [], + loadScan: async (scanId: string) => { + const response = await getScan(scanId); + return response?.data; + }, + }), + ); + + const hasHistoricalData = hasDateOrScanFilter(filtersWithScanDates); const metadataInfoData = await ( - hasDateOrScan ? getMetadataInfo : getLatestMetadataInfo + hasHistoricalData ? getMetadataInfo : getLatestMetadataInfo )({ query, sort: encodedSort, @@ -119,10 +121,9 @@ const SSRDataTable = async ({ const pageSize = parseInt(searchParams.pageSize?.toString() || "10", 10); const { encodedSort } = extractSortAndKey(searchParams); - // Check if the searchParams contain any date or scan filter - const hasDateOrScan = hasDateOrScanFilter(searchParams); + const hasHistoricalData = hasDateOrScanFilter(filters); - const fetchFindingGroups = hasDateOrScan + const fetchFindingGroups = hasHistoricalData ? getFindingGroups : getLatestFindingGroups; @@ -151,7 +152,7 @@ const SSRDataTable = async ({ data={groups} metadata={findingGroupsData?.meta} resolvedFilters={filters} - hasHistoricalData={hasDateOrScan} + hasHistoricalData={hasHistoricalData} /> ); diff --git a/ui/components/compliance/compliance-accordion/client-accordion-content.tsx b/ui/components/compliance/compliance-accordion/client-accordion-content.tsx index f582f2b3c2..659d3a83fa 100644 --- a/ui/components/compliance/compliance-accordion/client-accordion-content.tsx +++ b/ui/components/compliance/compliance-accordion/client-accordion-content.tsx @@ -62,6 +62,7 @@ export const ClientAccordionContent = ({ filters: { "filter[check_id__in]": checkIds.join(","), "filter[scan]": scanId, + "filter[muted]": "false", ...(region && { "filter[region__in]": region }), }, page: parseInt(pageNumber, 10), diff --git a/ui/components/compliance/compliance-card.test.tsx b/ui/components/compliance/compliance-card.test.tsx new file mode 100644 index 0000000000..c7a199a7fc --- /dev/null +++ b/ui/components/compliance/compliance-card.test.tsx @@ -0,0 +1,30 @@ +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { describe, expect, it } from "vitest"; + +describe("ComplianceCard", () => { + const currentDir = path.dirname(fileURLToPath(import.meta.url)); + const filePath = path.join(currentDir, "compliance-card.tsx"); + const source = readFileSync(filePath, "utf8"); + + it("keeps the shadcn Card base variant", () => { + expect(source).toContain('variant="base"'); + }); + + it("uses a responsive stacked layout for narrow screens", () => { + expect(source).toContain("flex-col"); + expect(source).toContain("sm:flex-row"); + }); + + it("uses the shadcn progress component instead of Hero UI", () => { + expect(source).toContain('from "@/components/shadcn/progress"'); + expect(source).not.toContain("@heroui/progress"); + }); + + it("places compact actions in the icon column on larger screens", () => { + expect(source).toContain('orientation="column"'); + expect(source).toContain('buttonWidth="icon"'); + }); +}); diff --git a/ui/components/compliance/compliance-card.tsx b/ui/components/compliance/compliance-card.tsx index 5d1b0425ba..2c9f383f2b 100644 --- a/ui/components/compliance/compliance-card.tsx +++ b/ui/components/compliance/compliance-card.tsx @@ -1,11 +1,20 @@ "use client"; -import { Progress } from "@heroui/progress"; import Image from "next/image"; import { useRouter, useSearchParams } from "next/navigation"; import { Card, CardContent } from "@/components/shadcn/card/card"; +import { Progress } from "@/components/shadcn/progress"; +import { + Tooltip, + TooltipContent, + TooltipTrigger, +} from "@/components/shadcn/tooltip"; import { getReportTypeForFramework } from "@/lib/compliance/compliance-report-types"; +import { + getScoreIndicatorClass, + type ScoreColorVariant, +} from "@/lib/compliance/score-utils"; import { ScanEntity } from "@/types/scans"; import { getComplianceIcon } from "../icons"; @@ -45,13 +54,9 @@ export const ComplianceCard: React.FC = ({ (passingRequirements / totalRequirements) * 100, ); - const getRatingColor = (ratingPercentage: number) => { - if (ratingPercentage <= 10) { - return "danger"; - } - if (ratingPercentage <= 40) { - return "warning"; - } + const getRatingVariant = (value: number): ScoreColorVariant => { + if (value <= 10) return "danger"; + if (value <= 40) return "warning"; return "success"; }; @@ -80,58 +85,76 @@ export const ComplianceCard: React.FC = ({ onClick={navigateToDetail} > -
- {getComplianceIcon(title) && ( - {`${title} - )} -
-

- {formatTitle(title)} - {version ? ` - ${version}` : ""} -

- +
+ {getComplianceIcon(title) && ( + {`${title} + )} +
e.stopPropagation()} + onKeyDown={(e) => { + if (e.key === "Enter" || e.key === " ") { + e.stopPropagation(); + } }} - color={getRatingColor(ratingPercentage)} - /> -
- + role="group" + tabIndex={0} + > + +
+
+
+ + +

+ {formatTitle(title)} + {version ? ` - ${version}` : ""} +

+ + + {formatTitle(title)} + {version ? ` - ${version}` : ""} + + +
+
+ + Score: + + + {ratingPercentage}% + +
+ +
+
+ {passingRequirements} / {totalRequirements} Passing Requirements - -
e.stopPropagation()} - onKeyDown={(e) => { - if (e.key === "Enter" || e.key === " ") { - e.stopPropagation(); - } - }} - role="group" - tabIndex={0} - > - -
diff --git a/ui/components/compliance/compliance-download-container.test.tsx b/ui/components/compliance/compliance-download-container.test.tsx new file mode 100644 index 0000000000..4a13c35fbc --- /dev/null +++ b/ui/components/compliance/compliance-download-container.test.tsx @@ -0,0 +1,133 @@ +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const { downloadComplianceCsvMock, downloadComplianceReportPdfMock } = + vi.hoisted(() => ({ + downloadComplianceCsvMock: vi.fn(), + downloadComplianceReportPdfMock: vi.fn(), + })); + +vi.mock("@/lib/helper", () => ({ + downloadComplianceCsv: downloadComplianceCsvMock, + downloadComplianceReportPdf: downloadComplianceReportPdfMock, +})); + +vi.mock("@/components/ui", () => ({ + toast: {}, +})); + +import { ComplianceDownloadContainer } from "./compliance-download-container"; + +describe("ComplianceDownloadContainer", () => { + const currentDir = path.dirname(fileURLToPath(import.meta.url)); + const filePath = path.join(currentDir, "compliance-download-container.tsx"); + const source = readFileSync(filePath, "utf8"); + + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("uses the shared action dropdown for the card actions mode", () => { + expect(source).toContain("ActionDropdown"); + expect(source).not.toContain("@heroui/button"); + }); + + it("should expose an accessible actions menu trigger", () => { + render( + , + ); + + expect( + screen.getByRole("button", { name: "Open compliance export actions" }), + ).toBeInTheDocument(); + }); + + it("should support fixed icon-sized dropdown trigger in column mode", () => { + render( + , + ); + + const trigger = screen.getByRole("button", { + name: "Open compliance export actions", + }); + expect(trigger.className).toContain("border-text-neutral-secondary"); + }); + + it("should open export actions from the compact trigger", async () => { + const user = userEvent.setup(); + + render( + , + ); + + await user.click( + screen.getByRole("button", { name: "Open compliance export actions" }), + ); + + expect(screen.getByText("Download CSV report")).toBeInTheDocument(); + expect(screen.getByText("Download PDF report")).toBeInTheDocument(); + }); + + it("should trigger both downloads from the actions menu", async () => { + const user = userEvent.setup(); + + render( + , + ); + + await user.click( + screen.getByRole("button", { name: "Open compliance export actions" }), + ); + await user.click( + screen.getByRole("menuitem", { name: /Download CSV report/i }), + ); + await user.click( + screen.getByRole("button", { name: "Open compliance export actions" }), + ); + await user.click( + screen.getByRole("menuitem", { name: /Download PDF report/i }), + ); + + expect(downloadComplianceCsvMock).toHaveBeenCalledWith( + "scan-1", + "compliance-1", + {}, + ); + expect(downloadComplianceReportPdfMock).toHaveBeenCalledWith( + "scan-1", + "threatscore", + {}, + ); + }); +}); diff --git a/ui/components/compliance/compliance-download-container.tsx b/ui/components/compliance/compliance-download-container.tsx index 526057ea5c..415da1d85a 100644 --- a/ui/components/compliance/compliance-download-container.tsx +++ b/ui/components/compliance/compliance-download-container.tsx @@ -4,6 +4,15 @@ import { DownloadIcon, FileTextIcon } from "lucide-react"; import { useState } from "react"; import { Button } from "@/components/shadcn/button/button"; +import { + ActionDropdown, + ActionDropdownItem, +} from "@/components/shadcn/dropdown"; +import { + Tooltip, + TooltipContent, + TooltipTrigger, +} from "@/components/shadcn/tooltip"; import { toast } from "@/components/ui"; import type { ComplianceReportType } from "@/lib/compliance/compliance-report-types"; import { @@ -18,6 +27,9 @@ interface ComplianceDownloadContainerProps { reportType?: ComplianceReportType; compact?: boolean; disabled?: boolean; + orientation?: "row" | "column"; + buttonWidth?: "auto" | "icon"; + presentation?: "buttons" | "dropdown"; } export const ComplianceDownloadContainer = ({ @@ -26,9 +38,14 @@ export const ComplianceDownloadContainer = ({ reportType, compact = false, disabled = false, + orientation = "row", + buttonWidth = "auto", + presentation = "buttons", }: ComplianceDownloadContainerProps) => { const [isDownloadingCsv, setIsDownloadingCsv] = useState(false); const [isDownloadingPdf, setIsDownloadingPdf] = useState(false); + const isIconWidth = buttonWidth === "icon"; + const isDropdown = presentation === "dropdown"; const handleDownloadCsv = async () => { if (isDownloadingCsv) return; @@ -52,40 +69,116 @@ export const ComplianceDownloadContainer = ({ const buttonClassName = cn( "border-button-primary text-button-primary hover:bg-button-primary/10", - compact && "h-7 px-2 text-xs", + compact && + !isIconWidth && + "h-7 px-2 text-xs sm:w-full sm:justify-center sm:px-2.5", + orientation === "column" && !isIconWidth && "w-full", + isIconWidth && "size-10 rounded-lg p-0", ); + const labelClassName = isIconWidth + ? "sr-only" + : compact + ? "sr-only sm:not-sr-only" + : undefined; + const showTooltip = compact || isIconWidth; return ( -
- - {reportType && ( - + {reportType && ( + + } + label="Download PDF report" + onSelect={handleDownloadPdf} + disabled={disabled || isDownloadingPdf} + /> + )} + + ) : ( +
+ + + + + {showTooltip && ( + Download CSV report + )} + + {reportType && ( + + + + + {showTooltip && ( + Download PDF report + )} + + )} +
)}
); diff --git a/ui/components/compliance/compliance-header/compliance-filters.tsx b/ui/components/compliance/compliance-header/compliance-filters.tsx new file mode 100644 index 0000000000..474b61d4e7 --- /dev/null +++ b/ui/components/compliance/compliance-header/compliance-filters.tsx @@ -0,0 +1,76 @@ +"use client"; + +import { useRouter, useSearchParams } from "next/navigation"; + +import { ClearFiltersButton } from "@/components/filters/clear-filters-button"; +import { + MultiSelect, + MultiSelectContent, + MultiSelectItem, + MultiSelectSelectAll, + MultiSelectSeparator, + MultiSelectTrigger, + MultiSelectValue, +} from "@/components/shadcn/select/multiselect"; +import { useUrlFilters } from "@/hooks/use-url-filters"; + +import { ScanSelector, SelectScanComplianceDataProps } from "./scan-selector"; + +interface ComplianceFiltersProps { + scans: SelectScanComplianceDataProps["scans"]; + uniqueRegions: string[]; + selectedScanId: string; +} + +export const ComplianceFilters = ({ + scans, + uniqueRegions, + selectedScanId, +}: ComplianceFiltersProps) => { + const router = useRouter(); + const searchParams = useSearchParams(); + const { updateFilter } = useUrlFilters(); + + const handleScanChange = (selectedKey: string) => { + const params = new URLSearchParams(searchParams); + params.set("scanId", selectedKey); + router.push(`?${params.toString()}`, { scroll: false }); + }; + + const regionValues = + searchParams.get("filter[region__in]")?.split(",").filter(Boolean) ?? []; + + return ( +
+
+ +
+ {uniqueRegions.length > 0 && ( +
+ updateFilter("region__in", values)} + > + + + + + Select All + + {uniqueRegions.map((region) => ( + + {region} + + ))} + + +
+ )} + +
+ ); +}; diff --git a/ui/components/compliance/compliance-header/compliance-header.test.tsx b/ui/components/compliance/compliance-header/compliance-header.test.tsx new file mode 100644 index 0000000000..b646199f7f --- /dev/null +++ b/ui/components/compliance/compliance-header/compliance-header.test.tsx @@ -0,0 +1,18 @@ +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { describe, expect, it } from "vitest"; + +describe("ComplianceHeader", () => { + const currentDir = path.dirname(fileURLToPath(import.meta.url)); + const filePath = path.join(currentDir, "compliance-header.tsx"); + const source = readFileSync(filePath, "utf8"); + + it("renders the scan selector inside the shared filters grid using default layout", () => { + expect(source).toContain("prependElement"); + expect(source).toContain(" { const frameworkFilters = []; + const prependElement = showProviders ? ( + + ) : undefined; // Add CIS Profile Level filter if framework is CIS if (framework === "CIS") { @@ -42,6 +45,7 @@ export const ComplianceHeader = ({ key: "cis_profile_level", labelCheckboxGroup: "Level", values: ["Level 1", "Level 2"], + width: "wide" as const, index: 0, // Show first showSelectAll: false, // No "Select All" option since Level 2 includes Level 1 defaultValues: ["Level 2"], // Default to Level 2 selected (which includes Level 1) @@ -55,6 +59,7 @@ export const ComplianceHeader = ({ key: "region__in", labelCheckboxGroup: "Regions", values: uniqueRegions, + width: "wide" as const, index: 1, // Show after framework filters }, ] @@ -77,9 +82,11 @@ export const ComplianceHeader = ({ {selectedScan && } {/* Showed in the compliance page */} - {showProviders && } - {!hideFilters && allFilters.length > 0 && ( - + {!hideFilters && (allFilters.length > 0 || showProviders) && ( + )}
{logoPath && complianceTitle && ( diff --git a/ui/components/compliance/compliance-header/data-compliance.tsx b/ui/components/compliance/compliance-header/data-compliance.tsx index 992dc3d683..4787d00921 100644 --- a/ui/components/compliance/compliance-header/data-compliance.tsx +++ b/ui/components/compliance/compliance-header/data-compliance.tsx @@ -7,11 +7,13 @@ import { ScanSelector, SelectScanComplianceDataProps, } from "@/components/compliance/compliance-header/index"; +import { cn } from "@/lib/utils"; interface DataComplianceProps { scans: SelectScanComplianceDataProps["scans"]; + className?: string; } -export const DataCompliance = ({ scans }: DataComplianceProps) => { +export const DataCompliance = ({ scans, className }: DataComplianceProps) => { const router = useRouter(); const searchParams = useSearchParams(); @@ -36,7 +38,7 @@ export const DataCompliance = ({ scans }: DataComplianceProps) => { }; return ( -
+
{ const selectedScan = scans.find((item) => item.id === selectedScanId); + const triggerLabel = selectedScan ? getScanEntityLabel(selectedScan) : ""; return (