fix(api): fix finding groups muted filter, counters and reaggregation (#10477)

api/CHANGELOG.md

@@ -15,6 +15,7 @@ All notable changes to the **Prowler API** are documented in this file.
 - 403 error for admin users listing tenants due to roles query not using the admin database connection [(#10460)](https://github.com/prowler-cloud/prowler/pull/10460)
 - Filter transient Neo4j defunct connection logs in Sentry `before_send` to suppress false-positive alerts handled by `RetryableSession` retries [(#10452)](https://github.com/prowler-cloud/prowler/pull/10452)
 - `MANAGE_ACCOUNT` permission no longer required for listing and creating tenants [(#10468)](https://github.com/prowler-cloud/prowler/pull/10468)
+- Finding groups muted filter, counters, metadata extraction and mute reaggregation [(#10477)](https://github.com/prowler-cloud/prowler/pull/10477)
 
 ## [1.23.0] (Prowler v5.22.0)
 

api/src/backend/api/filters.py

@@ -1065,7 +1065,7 @@ class LatestFindingGroupSummaryFilter(FilterSet):
 
 
 class FindingGroupAggregatedComputedFilter(FilterSet):
-    """Filter aggregated finding-group rows by computed status/severity."""
+    """Filter aggregated finding-group rows by computed status/severity/muted."""
 
     STATUS_CHOICES = (
         ("FAIL", "Fail"),
@@ -1077,6 +1077,7 @@ class FindingGroupAggregatedComputedFilter(FilterSet):
     status__in = CharInFilter(method="filter_status_in", lookup_expr="in")
     severity = ChoiceFilter(method="filter_severity", choices=SeverityChoices)
     severity__in = CharInFilter(method="filter_severity_in", lookup_expr="in")
+    include_muted = BooleanFilter(method="filter_include_muted")
 
     def filter_status(self, queryset, name, value):
         return queryset.filter(aggregated_status=value)
@@ -1148,6 +1149,12 @@ class FindingGroupAggregatedComputedFilter(FilterSet):
 
         return queryset.filter(severity_order__in=orders)
 
+    def filter_include_muted(self, queryset, name, value):
+        if value is True:
+            return queryset
+        # include_muted=false: exclude fully-muted groups
+        return queryset.exclude(fail_count=0, pass_count=0, muted_count__gt=0)
+
 
 class ProviderSecretFilter(FilterSet):
     inserted_at = DateFilter(

api/src/backend/api/tests/test_views.py

@@ -14747,14 +14747,14 @@ class TestMuteRuleViewSet:
         assert data[0]["id"] == str(mute_rules_fixture[first_index].id)
 
     @patch("api.v1.views.chain")
-    @patch("api.v1.views.aggregate_finding_group_summaries_task.si")
+    @patch("api.v1.views.reaggregate_all_finding_group_summaries_task.si")
     @patch("api.v1.views.mute_historical_findings_task.si")
     @patch("api.v1.views.transaction.on_commit", side_effect=lambda fn: fn())
     def test_mute_rules_create_valid(
         self,
         _mock_on_commit,
         mock_mute_signature,
-        mock_aggregate_signature,
+        mock_reaggregate_signature,
         mock_chain,
         authenticated_client,
         findings_fixture,
@@ -14793,12 +14793,12 @@ class TestMuteRuleViewSet:
         assert finding.muted_at is not None
         assert finding.muted_reason == "Security exception approved"
 
-        # Verify background task chain was called
+        # Verify background task chain was called: mute → reaggregate all
         mock_mute_signature.assert_called_once()
-        mock_aggregate_signature.assert_called_once()
+        mock_reaggregate_signature.assert_called_once()
         mock_chain.assert_called_once_with(
             mock_mute_signature.return_value,
-            mock_aggregate_signature.return_value,
+            mock_reaggregate_signature.return_value,
         )
         mock_chain.return_value.apply_async.assert_called_once()
 

api/src/backend/api/v1/views.py

@@ -48,7 +48,8 @@ from django.db.models import (
     When,
     Window,
 )
-from django.db.models.functions import Cast, Coalesce, DenseRank, RowNumber
+from django.db.models.fields.json import KeyTextTransform
+from django.db.models.functions import Cast, Coalesce, RowNumber
 from django.http import HttpResponse, QueryDict
 from django.shortcuts import redirect
 from django.urls import reverse
@@ -83,7 +84,6 @@ from tasks.beat import schedule_provider_scan
 from tasks.jobs.attack_paths import db_utils as attack_paths_db_utils
 from tasks.jobs.export import get_s3_client
 from tasks.tasks import (
-    aggregate_finding_group_summaries_task,
     backfill_compliance_summaries_task,
     backfill_scan_resource_summaries_task,
     check_integration_connection_task,
@@ -95,6 +95,7 @@ from tasks.tasks import (
     jira_integration_task,
     mute_historical_findings_task,
     perform_scan_task,
+    reaggregate_all_finding_group_summaries_task,
     refresh_lighthouse_provider_models_task,
 )
 
@@ -6739,23 +6740,15 @@ class MuteRuleViewSet(BaseRLSViewSet):
             muted_reason=mute_rule.reason,
         )
 
-        # Launch background task for historical muting
-        latest_scan_id = (
-            Scan.objects.filter(tenant_id=tenant_id, state=StateChoices.COMPLETED)
-            .order_by("-completed_at", "-inserted_at")
-            .values_list("id", flat=True)
-            .first()
-        )
-
+        # Launch background task for historical muting + reaggregation
         transaction.on_commit(
             lambda: chain(
                 mute_historical_findings_task.si(
                     tenant_id=tenant_id,
                     mute_rule_id=str(mute_rule.id),
                 ),
-                aggregate_finding_group_summaries_task.si(
+                reaggregate_all_finding_group_summaries_task.si(
                     tenant_id=tenant_id,
-                    scan_id=str(latest_scan_id),
                 ),
             ).apply_async()
         )
@@ -7012,13 +7005,13 @@ class FindingGroupViewSet(BaseRLSViewSet):
                 "first_seen_at", filter=Q(status="FAIL", muted=False)
             ),
             check_title=Coalesce(
-                Max(Cast("check_metadata__CheckTitle", CharField())),
-                Max(Cast("check_metadata__checktitle", CharField())),
-                Max(Cast("check_metadata__Checktitle", CharField())),
+                Max(KeyTextTransform("checktitle", "check_metadata")),
+                Max(KeyTextTransform("CheckTitle", "check_metadata")),
+                Max(KeyTextTransform("Checktitle", "check_metadata")),
             ),
             check_description=Coalesce(
-                Max(Cast("check_metadata__Description", CharField())),
-                Max(Cast("check_metadata__description", CharField())),
+                Max(KeyTextTransform("description", "check_metadata")),
+                Max(KeyTextTransform("Description", "check_metadata")),
             ),
         )
 
@@ -7026,7 +7019,13 @@ class FindingGroupViewSet(BaseRLSViewSet):
         self, params: QueryDict
     ) -> tuple[QueryDict, QueryDict]:
         """Split finding filters from computed aggregate filters."""
-        computed_keys = {"status", "status__in", "severity", "severity__in"}
+        computed_keys = {
+            "status",
+            "status__in",
+            "severity",
+            "severity__in",
+            "include_muted",
+        }
         finding_params = QueryDict(mutable=True)
         computed_params = QueryDict(mutable=True)
 
@@ -7038,24 +7037,18 @@ class FindingGroupViewSet(BaseRLSViewSet):
 
         return finding_params, computed_params
 
-    def _get_latest_findings_per_check_provider(self, filtered_queryset):
-        """Keep all findings from the latest scan per (check_id, provider)."""
-        latest_ids = (
-            filtered_queryset.annotate(
-                scan_rank=Window(
-                    expression=DenseRank(),
-                    partition_by=[F("check_id"), F("scan__provider_id")],
-                    order_by=[
-                        F("scan__completed_at").desc(nulls_last=True),
-                        F("scan_id").desc(),
-                    ],
-                )
+    def _get_latest_findings_per_provider(self, filtered_queryset):
+        """Keep only findings from each provider's most recent completed scan."""
+        latest_scan_ids = (
+            Scan.objects.filter(
+                tenant_id=self.request.tenant_id,
+                state=StateChoices.COMPLETED,
             )
-            .filter(scan_rank=1)
+            .order_by("provider_id", "-completed_at", "-inserted_at")
+            .distinct("provider_id")
             .values("id")
         )
-
-        return filtered_queryset.filter(id__in=Subquery(latest_ids))
+        return filtered_queryset.filter(scan_id__in=latest_scan_ids)
 
     def _post_process_aggregation(self, aggregated_data):
         """
@@ -7155,6 +7148,10 @@ class FindingGroupViewSet(BaseRLSViewSet):
                 )
             )
 
+        # Exclude fully-muted groups by default unless include_muted is set
+        if "include_muted" not in computed_params:
+            queryset = queryset.exclude(fail_count=0, pass_count=0, muted_count__gt=0)
+
         filterset = FindingGroupAggregatedComputedFilter(
             computed_params, queryset=queryset
         )
@@ -7299,7 +7296,7 @@ class FindingGroupViewSet(BaseRLSViewSet):
                 raise ValidationError(filterset.errors)
             filtered_queryset = filterset.qs
             if latest:
-                filtered_queryset = self._get_latest_findings_per_check_provider(
+                filtered_queryset = self._get_latest_findings_per_provider(
                     filtered_queryset
                 )
             return self._aggregate_findings(filtered_queryset)
@@ -7309,15 +7306,14 @@ class FindingGroupViewSet(BaseRLSViewSet):
         if not filterset.is_valid():
             raise ValidationError(filterset.errors)
         filtered_queryset = filterset.qs
-        if latest:
-            latest_per_check_ids = (
-                filtered_queryset.order_by("check_id", "provider_id", "-inserted_at")
-                .distinct("check_id", "provider_id")
-                .values("id")
-            )
-            filtered_queryset = filtered_queryset.filter(
-                id__in=Subquery(latest_per_check_ids)
-            )
+        # Only include summaries from each provider's most recent date
+        # (within the filtered range)
+        filtered_queryset = filtered_queryset.annotate(
+            _max_provider_date=Window(
+                expression=Max("inserted_at"),
+                partition_by=[F("provider_id")],
+            ),
+        ).filter(inserted_at=F("_max_provider_date"))
         return self._aggregate_daily_summaries(filtered_queryset)
 
     def _sorted_paginated_response(self, request, aggregated_queryset):

api/src/backend/tasks/jobs/scan.py

@@ -1888,7 +1888,8 @@ def aggregate_finding_group_summaries(tenant_id: str, scan_id: str):
                     inserted_at=summary_timestamp,
                     updated_at=updated_at,
                     check_title=metadata.get("checktitle", ""),
-                    check_description=metadata.get("Description", ""),
+                    check_description=metadata.get("description", "")
+                    or metadata.get("Description", ""),
                     severity_order=row["severity_order"] or 1,
                     pass_count=row["pass_count"],
                     fail_count=row["fail_count"],

api/src/backend/tasks/tasks.py

@@ -11,8 +11,8 @@ from django_celery_beat.models import PeriodicTask
 from tasks.jobs.attack_paths import (
     attack_paths_scan,
     can_provider_run_attack_paths_scan,
-    db_utils as attack_paths_db_utils,
 )
+from tasks.jobs.attack_paths import db_utils as attack_paths_db_utils
 from tasks.jobs.backfill import (
     backfill_compliance_summaries,
     backfill_daily_severity_summaries,
@@ -760,6 +760,33 @@ def aggregate_finding_group_summaries_task(tenant_id: str, scan_id: str):
     return aggregate_finding_group_summaries(tenant_id=tenant_id, scan_id=scan_id)
 
 
+@shared_task(
+    base=RLSTask, name="reaggregate-all-finding-group-summaries", queue="overview"
+)
+@set_tenant(keep_tenant=True)
+def reaggregate_all_finding_group_summaries_task(tenant_id: str):
+    """Reaggregate finding group summaries for all providers' latest completed scans."""
+    latest_scan_ids = list(
+        Scan.objects.filter(tenant_id=tenant_id, state=StateChoices.COMPLETED)
+        .order_by("provider_id", "-completed_at", "-inserted_at")
+        .distinct("provider_id")
+        .values_list("id", flat=True)
+    )
+    if latest_scan_ids:
+        logger.info(
+            "Reaggregating finding group summaries for %d scans: %s",
+            len(latest_scan_ids),
+            latest_scan_ids,
+        )
+        group(
+            aggregate_finding_group_summaries_task.si(
+                tenant_id=tenant_id, scan_id=str(scan_id)
+            )
+            for scan_id in latest_scan_ids
+        ).apply_async()
+    return {"scans_reaggregated": len(latest_scan_ids)}
+
+
 @shared_task(base=RLSTask, name="lighthouse-connection-check")
 @set_tenant
 def check_lighthouse_connection_task(lighthouse_config_id: str, tenant_id: str = None):

api/src/backend/tasks/tests/test_tasks.py

@@ -20,6 +20,7 @@ from tasks.tasks import (
     generate_outputs_task,
     perform_attack_paths_scan_task,
     perform_scheduled_scan_task,
+    reaggregate_all_finding_group_summaries_task,
     refresh_lighthouse_provider_models_task,
     s3_integration_task,
     security_hub_integration_task,
@@ -2351,3 +2352,47 @@ class TestPerformScheduledScanTask:
             ).count()
             == 1
         )
+
+
+@pytest.mark.django_db
+class TestReaggregateAllFindingGroupSummaries:
+    def setup_method(self):
+        self.tenant_id = str(uuid.uuid4())
+
+    @patch("tasks.tasks.group")
+    @patch("tasks.tasks.aggregate_finding_group_summaries_task")
+    @patch("tasks.tasks.Scan.objects.filter")
+    def test_dispatches_subtasks_for_each_provider(
+        self, mock_scan_filter, mock_agg_task, mock_group
+    ):
+        scan_id_1 = uuid.uuid4()
+        scan_id_2 = uuid.uuid4()
+        mock_group_result = MagicMock()
+        mock_group.side_effect = lambda gen: (list(gen), mock_group_result)[1]
+
+        mock_scan_filter.return_value.order_by.return_value.distinct.return_value.values_list.return_value = [
+            scan_id_1,
+            scan_id_2,
+        ]
+
+        result = reaggregate_all_finding_group_summaries_task(tenant_id=self.tenant_id)
+
+        assert result == {"scans_reaggregated": 2}
+        assert mock_agg_task.si.call_count == 2
+        mock_agg_task.si.assert_any_call(
+            tenant_id=self.tenant_id, scan_id=str(scan_id_1)
+        )
+        mock_agg_task.si.assert_any_call(
+            tenant_id=self.tenant_id, scan_id=str(scan_id_2)
+        )
+        mock_group_result.apply_async.assert_called_once()
+
+    @patch("tasks.tasks.group")
+    @patch("tasks.tasks.Scan.objects.filter")
+    def test_no_completed_scans_skips_dispatch(self, mock_scan_filter, mock_group):
+        mock_scan_filter.return_value.order_by.return_value.distinct.return_value.values_list.return_value = []
+
+        result = reaggregate_all_finding_group_summaries_task(tenant_id=self.tenant_id)
+
+        assert result == {"scans_reaggregated": 0}
+        mock_group.assert_not_called()