diff --git a/api/CHANGELOG.md b/api/CHANGELOG.md index 1385a9c0f5..f2103ea4a4 100644 --- a/api/CHANGELOG.md +++ b/api/CHANGELOG.md @@ -15,6 +15,7 @@ All notable changes to the **Prowler API** are documented in this file. - 403 error for admin users listing tenants due to roles query not using the admin database connection [(#10460)](https://github.com/prowler-cloud/prowler/pull/10460) - Filter transient Neo4j defunct connection logs in Sentry `before_send` to suppress false-positive alerts handled by `RetryableSession` retries [(#10452)](https://github.com/prowler-cloud/prowler/pull/10452) - `MANAGE_ACCOUNT` permission no longer required for listing and creating tenants [(#10468)](https://github.com/prowler-cloud/prowler/pull/10468) +- Finding groups muted filter, counters, metadata extraction and mute reaggregation [(#10477)](https://github.com/prowler-cloud/prowler/pull/10477) ## [1.23.0] (Prowler v5.22.0) diff --git a/api/src/backend/api/filters.py b/api/src/backend/api/filters.py index 84596a9773..ab6df84834 100644 --- a/api/src/backend/api/filters.py +++ b/api/src/backend/api/filters.py @@ -1065,7 +1065,7 @@ class LatestFindingGroupSummaryFilter(FilterSet): class FindingGroupAggregatedComputedFilter(FilterSet): - """Filter aggregated finding-group rows by computed status/severity.""" + """Filter aggregated finding-group rows by computed status/severity/muted.""" STATUS_CHOICES = ( ("FAIL", "Fail"), @@ -1077,6 +1077,7 @@ class FindingGroupAggregatedComputedFilter(FilterSet): status__in = CharInFilter(method="filter_status_in", lookup_expr="in") severity = ChoiceFilter(method="filter_severity", choices=SeverityChoices) severity__in = CharInFilter(method="filter_severity_in", lookup_expr="in") + include_muted = BooleanFilter(method="filter_include_muted") def filter_status(self, queryset, name, value): return queryset.filter(aggregated_status=value) @@ -1148,6 +1149,12 @@ class FindingGroupAggregatedComputedFilter(FilterSet): return queryset.filter(severity_order__in=orders) + def filter_include_muted(self, queryset, name, value): + if value is True: + return queryset + # include_muted=false: exclude fully-muted groups + return queryset.exclude(fail_count=0, pass_count=0, muted_count__gt=0) + class ProviderSecretFilter(FilterSet): inserted_at = DateFilter( diff --git a/api/src/backend/api/tests/test_views.py b/api/src/backend/api/tests/test_views.py index e50a0c84bb..ec907b149b 100644 --- a/api/src/backend/api/tests/test_views.py +++ b/api/src/backend/api/tests/test_views.py @@ -14747,14 +14747,14 @@ class TestMuteRuleViewSet: assert data[0]["id"] == str(mute_rules_fixture[first_index].id) @patch("api.v1.views.chain") - @patch("api.v1.views.aggregate_finding_group_summaries_task.si") + @patch("api.v1.views.reaggregate_all_finding_group_summaries_task.si") @patch("api.v1.views.mute_historical_findings_task.si") @patch("api.v1.views.transaction.on_commit", side_effect=lambda fn: fn()) def test_mute_rules_create_valid( self, _mock_on_commit, mock_mute_signature, - mock_aggregate_signature, + mock_reaggregate_signature, mock_chain, authenticated_client, findings_fixture, @@ -14793,12 +14793,12 @@ class TestMuteRuleViewSet: assert finding.muted_at is not None assert finding.muted_reason == "Security exception approved" - # Verify background task chain was called + # Verify background task chain was called: mute → reaggregate all mock_mute_signature.assert_called_once() - mock_aggregate_signature.assert_called_once() + mock_reaggregate_signature.assert_called_once() mock_chain.assert_called_once_with( mock_mute_signature.return_value, - mock_aggregate_signature.return_value, + mock_reaggregate_signature.return_value, ) mock_chain.return_value.apply_async.assert_called_once() diff --git a/api/src/backend/api/v1/views.py b/api/src/backend/api/v1/views.py index 5d2e574630..9d4d311b46 100644 --- a/api/src/backend/api/v1/views.py +++ b/api/src/backend/api/v1/views.py @@ -48,7 +48,8 @@ from django.db.models import ( When, Window, ) -from django.db.models.functions import Cast, Coalesce, DenseRank, RowNumber +from django.db.models.fields.json import KeyTextTransform +from django.db.models.functions import Cast, Coalesce, RowNumber from django.http import HttpResponse, QueryDict from django.shortcuts import redirect from django.urls import reverse @@ -83,7 +84,6 @@ from tasks.beat import schedule_provider_scan from tasks.jobs.attack_paths import db_utils as attack_paths_db_utils from tasks.jobs.export import get_s3_client from tasks.tasks import ( - aggregate_finding_group_summaries_task, backfill_compliance_summaries_task, backfill_scan_resource_summaries_task, check_integration_connection_task, @@ -95,6 +95,7 @@ from tasks.tasks import ( jira_integration_task, mute_historical_findings_task, perform_scan_task, + reaggregate_all_finding_group_summaries_task, refresh_lighthouse_provider_models_task, ) @@ -6739,23 +6740,15 @@ class MuteRuleViewSet(BaseRLSViewSet): muted_reason=mute_rule.reason, ) - # Launch background task for historical muting - latest_scan_id = ( - Scan.objects.filter(tenant_id=tenant_id, state=StateChoices.COMPLETED) - .order_by("-completed_at", "-inserted_at") - .values_list("id", flat=True) - .first() - ) - + # Launch background task for historical muting + reaggregation transaction.on_commit( lambda: chain( mute_historical_findings_task.si( tenant_id=tenant_id, mute_rule_id=str(mute_rule.id), ), - aggregate_finding_group_summaries_task.si( + reaggregate_all_finding_group_summaries_task.si( tenant_id=tenant_id, - scan_id=str(latest_scan_id), ), ).apply_async() ) @@ -7012,13 +7005,13 @@ class FindingGroupViewSet(BaseRLSViewSet): "first_seen_at", filter=Q(status="FAIL", muted=False) ), check_title=Coalesce( - Max(Cast("check_metadata__CheckTitle", CharField())), - Max(Cast("check_metadata__checktitle", CharField())), - Max(Cast("check_metadata__Checktitle", CharField())), + Max(KeyTextTransform("checktitle", "check_metadata")), + Max(KeyTextTransform("CheckTitle", "check_metadata")), + Max(KeyTextTransform("Checktitle", "check_metadata")), ), check_description=Coalesce( - Max(Cast("check_metadata__Description", CharField())), - Max(Cast("check_metadata__description", CharField())), + Max(KeyTextTransform("description", "check_metadata")), + Max(KeyTextTransform("Description", "check_metadata")), ), ) @@ -7026,7 +7019,13 @@ class FindingGroupViewSet(BaseRLSViewSet): self, params: QueryDict ) -> tuple[QueryDict, QueryDict]: """Split finding filters from computed aggregate filters.""" - computed_keys = {"status", "status__in", "severity", "severity__in"} + computed_keys = { + "status", + "status__in", + "severity", + "severity__in", + "include_muted", + } finding_params = QueryDict(mutable=True) computed_params = QueryDict(mutable=True) @@ -7038,24 +7037,18 @@ class FindingGroupViewSet(BaseRLSViewSet): return finding_params, computed_params - def _get_latest_findings_per_check_provider(self, filtered_queryset): - """Keep all findings from the latest scan per (check_id, provider).""" - latest_ids = ( - filtered_queryset.annotate( - scan_rank=Window( - expression=DenseRank(), - partition_by=[F("check_id"), F("scan__provider_id")], - order_by=[ - F("scan__completed_at").desc(nulls_last=True), - F("scan_id").desc(), - ], - ) + def _get_latest_findings_per_provider(self, filtered_queryset): + """Keep only findings from each provider's most recent completed scan.""" + latest_scan_ids = ( + Scan.objects.filter( + tenant_id=self.request.tenant_id, + state=StateChoices.COMPLETED, ) - .filter(scan_rank=1) + .order_by("provider_id", "-completed_at", "-inserted_at") + .distinct("provider_id") .values("id") ) - - return filtered_queryset.filter(id__in=Subquery(latest_ids)) + return filtered_queryset.filter(scan_id__in=latest_scan_ids) def _post_process_aggregation(self, aggregated_data): """ @@ -7155,6 +7148,10 @@ class FindingGroupViewSet(BaseRLSViewSet): ) ) + # Exclude fully-muted groups by default unless include_muted is set + if "include_muted" not in computed_params: + queryset = queryset.exclude(fail_count=0, pass_count=0, muted_count__gt=0) + filterset = FindingGroupAggregatedComputedFilter( computed_params, queryset=queryset ) @@ -7299,7 +7296,7 @@ class FindingGroupViewSet(BaseRLSViewSet): raise ValidationError(filterset.errors) filtered_queryset = filterset.qs if latest: - filtered_queryset = self._get_latest_findings_per_check_provider( + filtered_queryset = self._get_latest_findings_per_provider( filtered_queryset ) return self._aggregate_findings(filtered_queryset) @@ -7309,15 +7306,14 @@ class FindingGroupViewSet(BaseRLSViewSet): if not filterset.is_valid(): raise ValidationError(filterset.errors) filtered_queryset = filterset.qs - if latest: - latest_per_check_ids = ( - filtered_queryset.order_by("check_id", "provider_id", "-inserted_at") - .distinct("check_id", "provider_id") - .values("id") - ) - filtered_queryset = filtered_queryset.filter( - id__in=Subquery(latest_per_check_ids) - ) + # Only include summaries from each provider's most recent date + # (within the filtered range) + filtered_queryset = filtered_queryset.annotate( + _max_provider_date=Window( + expression=Max("inserted_at"), + partition_by=[F("provider_id")], + ), + ).filter(inserted_at=F("_max_provider_date")) return self._aggregate_daily_summaries(filtered_queryset) def _sorted_paginated_response(self, request, aggregated_queryset): diff --git a/api/src/backend/tasks/jobs/scan.py b/api/src/backend/tasks/jobs/scan.py index 47ee29f8d9..3a96e892e1 100644 --- a/api/src/backend/tasks/jobs/scan.py +++ b/api/src/backend/tasks/jobs/scan.py @@ -1888,7 +1888,8 @@ def aggregate_finding_group_summaries(tenant_id: str, scan_id: str): inserted_at=summary_timestamp, updated_at=updated_at, check_title=metadata.get("checktitle", ""), - check_description=metadata.get("Description", ""), + check_description=metadata.get("description", "") + or metadata.get("Description", ""), severity_order=row["severity_order"] or 1, pass_count=row["pass_count"], fail_count=row["fail_count"], diff --git a/api/src/backend/tasks/tasks.py b/api/src/backend/tasks/tasks.py index 2e31ebc0f0..a32ffcd8d4 100644 --- a/api/src/backend/tasks/tasks.py +++ b/api/src/backend/tasks/tasks.py @@ -11,8 +11,8 @@ from django_celery_beat.models import PeriodicTask from tasks.jobs.attack_paths import ( attack_paths_scan, can_provider_run_attack_paths_scan, - db_utils as attack_paths_db_utils, ) +from tasks.jobs.attack_paths import db_utils as attack_paths_db_utils from tasks.jobs.backfill import ( backfill_compliance_summaries, backfill_daily_severity_summaries, @@ -760,6 +760,33 @@ def aggregate_finding_group_summaries_task(tenant_id: str, scan_id: str): return aggregate_finding_group_summaries(tenant_id=tenant_id, scan_id=scan_id) +@shared_task( + base=RLSTask, name="reaggregate-all-finding-group-summaries", queue="overview" +) +@set_tenant(keep_tenant=True) +def reaggregate_all_finding_group_summaries_task(tenant_id: str): + """Reaggregate finding group summaries for all providers' latest completed scans.""" + latest_scan_ids = list( + Scan.objects.filter(tenant_id=tenant_id, state=StateChoices.COMPLETED) + .order_by("provider_id", "-completed_at", "-inserted_at") + .distinct("provider_id") + .values_list("id", flat=True) + ) + if latest_scan_ids: + logger.info( + "Reaggregating finding group summaries for %d scans: %s", + len(latest_scan_ids), + latest_scan_ids, + ) + group( + aggregate_finding_group_summaries_task.si( + tenant_id=tenant_id, scan_id=str(scan_id) + ) + for scan_id in latest_scan_ids + ).apply_async() + return {"scans_reaggregated": len(latest_scan_ids)} + + @shared_task(base=RLSTask, name="lighthouse-connection-check") @set_tenant def check_lighthouse_connection_task(lighthouse_config_id: str, tenant_id: str = None): diff --git a/api/src/backend/tasks/tests/test_tasks.py b/api/src/backend/tasks/tests/test_tasks.py index 3a58118c62..8469b7db09 100644 --- a/api/src/backend/tasks/tests/test_tasks.py +++ b/api/src/backend/tasks/tests/test_tasks.py @@ -20,6 +20,7 @@ from tasks.tasks import ( generate_outputs_task, perform_attack_paths_scan_task, perform_scheduled_scan_task, + reaggregate_all_finding_group_summaries_task, refresh_lighthouse_provider_models_task, s3_integration_task, security_hub_integration_task, @@ -2351,3 +2352,47 @@ class TestPerformScheduledScanTask: ).count() == 1 ) + + +@pytest.mark.django_db +class TestReaggregateAllFindingGroupSummaries: + def setup_method(self): + self.tenant_id = str(uuid.uuid4()) + + @patch("tasks.tasks.group") + @patch("tasks.tasks.aggregate_finding_group_summaries_task") + @patch("tasks.tasks.Scan.objects.filter") + def test_dispatches_subtasks_for_each_provider( + self, mock_scan_filter, mock_agg_task, mock_group + ): + scan_id_1 = uuid.uuid4() + scan_id_2 = uuid.uuid4() + mock_group_result = MagicMock() + mock_group.side_effect = lambda gen: (list(gen), mock_group_result)[1] + + mock_scan_filter.return_value.order_by.return_value.distinct.return_value.values_list.return_value = [ + scan_id_1, + scan_id_2, + ] + + result = reaggregate_all_finding_group_summaries_task(tenant_id=self.tenant_id) + + assert result == {"scans_reaggregated": 2} + assert mock_agg_task.si.call_count == 2 + mock_agg_task.si.assert_any_call( + tenant_id=self.tenant_id, scan_id=str(scan_id_1) + ) + mock_agg_task.si.assert_any_call( + tenant_id=self.tenant_id, scan_id=str(scan_id_2) + ) + mock_group_result.apply_async.assert_called_once() + + @patch("tasks.tasks.group") + @patch("tasks.tasks.Scan.objects.filter") + def test_no_completed_scans_skips_dispatch(self, mock_scan_filter, mock_group): + mock_scan_filter.return_value.order_by.return_value.distinct.return_value.values_list.return_value = [] + + result = reaggregate_all_finding_group_summaries_task(tenant_id=self.tenant_id) + + assert result == {"scans_reaggregated": 0} + mock_group.assert_not_called()