feat(api): add resource group overview endpoint and filtering (#9694)

Co-authored-by: Alan Buscaglia <gentlemanprogramming@gmail.com>
Co-authored-by: Víctor Fernández Poyatos <victor@prowler.com>
This commit is contained in:
Adrián Peña
2026-01-15 13:05:25 +01:00
committed by GitHub
parent 3317c0a5e0
commit d8c1273a57
16 changed files with 1645 additions and 17 deletions
+2
View File
@@ -7,6 +7,8 @@ All notable changes to the **Prowler API** are documented in this file.
### Added
- `/api/v1/overviews/compliance-watchlist` to retrieve the compliance watchlist [(#9596)](https://github.com/prowler-cloud/prowler/pull/9596)
- Support AlibabaCloud provider [(#9485)](https://github.com/prowler-cloud/prowler/pull/9485)
- `/api/v1/overviews/resource-groups` to retrieve an overview of the resource groups based on finding severities [(#9694)](https://github.com/prowler-cloud/prowler/pull/9694)
- Endpoints `GET /findings` and `GET /findings/metadata/latest` now support the `group` filter [(#9694)](https://github.com/prowler-cloud/prowler/pull/9694)
- `provider_id` and `provider_id__in` filter aliases for findings endpoints to enable consistent frontend parameter naming [(#9701)](https://github.com/prowler-cloud/prowler/pull/9701)
---
+34
View File
@@ -45,6 +45,7 @@ from api.models import (
Role,
Scan,
ScanCategorySummary,
ScanGroupSummary,
ScanSummary,
SeverityChoices,
StateChoices,
@@ -214,6 +215,9 @@ class CommonFindingFilters(FilterSet):
category = CharFilter(method="filter_category")
category__in = CharInFilter(field_name="categories", lookup_expr="overlap")
resource_groups = CharFilter(field_name="resource_groups", lookup_expr="exact")
resource_groups__in = CharInFilter(field_name="resource_groups", lookup_expr="in")
# Temporarily disabled until we implement tag filtering in the UI
# resource_tag_key = CharFilter(field_name="resources__tags__key")
# resource_tag_key__in = CharInFilter(
@@ -439,6 +443,8 @@ class ResourceFilter(ProviderRelationshipFilterSet):
updated_at = DateFilter(field_name="updated_at", lookup_expr="date")
scan = UUIDFilter(field_name="provider__scan", lookup_expr="exact")
scan__in = UUIDInFilter(field_name="provider__scan", lookup_expr="in")
groups = CharFilter(method="filter_groups")
groups__in = CharInFilter(field_name="groups", lookup_expr="overlap")
class Meta:
model = Resource
@@ -453,6 +459,9 @@ class ResourceFilter(ProviderRelationshipFilterSet):
"updated_at": ["gte", "lte"],
}
def filter_groups(self, queryset, name, value):
return queryset.filter(groups__contains=[value])
def filter_queryset(self, queryset):
if not (self.data.get("scan") or self.data.get("scan__in")) and not (
self.data.get("updated_at")
@@ -517,6 +526,8 @@ class LatestResourceFilter(ProviderRelationshipFilterSet):
tag_value = CharFilter(method="filter_tag_value")
tag = CharFilter(method="filter_tag")
tags = CharFilter(method="filter_tag")
groups = CharFilter(method="filter_groups")
groups__in = CharInFilter(field_name="groups", lookup_expr="overlap")
class Meta:
model = Resource
@@ -529,6 +540,9 @@ class LatestResourceFilter(ProviderRelationshipFilterSet):
"type": ["exact", "icontains", "in"],
}
def filter_groups(self, queryset, name, value):
return queryset.filter(groups__contains=[value])
def filter_tag_key(self, queryset, name, value):
return queryset.filter(Q(tags__key=value) | Q(tags__key__icontains=value))
@@ -1154,6 +1168,26 @@ class CategoryOverviewFilter(BaseScanProviderFilter):
class Meta(BaseScanProviderFilter.Meta):
model = ScanCategorySummary
fields = {}
class ResourceGroupOverviewFilter(FilterSet):
provider_id = UUIDFilter(field_name="scan__provider__id", lookup_expr="exact")
provider_id__in = UUIDInFilter(field_name="scan__provider__id", lookup_expr="in")
provider_type = ChoiceFilter(
field_name="scan__provider__provider", choices=Provider.ProviderChoices.choices
)
provider_type__in = ChoiceInFilter(
field_name="scan__provider__provider",
choices=Provider.ProviderChoices.choices,
lookup_expr="in",
)
resource_group = CharFilter(field_name="resource_group", lookup_expr="exact")
resource_group__in = CharInFilter(field_name="resource_group", lookup_expr="in")
class Meta:
model = ScanGroupSummary
fields = {}
class ComplianceWatchlistFilter(BaseProviderFilter):
@@ -0,0 +1,126 @@
import uuid
import django.db.models.deletion
from django.db import migrations, models
import api.db_utils
import api.rls
class Migration(migrations.Migration):
dependencies = [
("api", "0067_tenant_compliance_summary"),
]
operations = [
migrations.AddField(
model_name="finding",
name="resource_groups",
field=models.TextField(
blank=True,
help_text="Resource group from check metadata for efficient filtering",
null=True,
),
),
migrations.CreateModel(
name="ScanGroupSummary",
fields=[
(
"id",
models.UUIDField(
default=uuid.uuid4,
editable=False,
primary_key=True,
serialize=False,
),
),
(
"tenant",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
to="api.tenant",
),
),
(
"inserted_at",
models.DateTimeField(auto_now_add=True),
),
(
"scan",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="resource_group_summaries",
related_query_name="resource_group_summary",
to="api.scan",
),
),
(
"resource_group",
models.CharField(max_length=50),
),
(
"severity",
api.db_utils.SeverityEnumField(
choices=[
("critical", "Critical"),
("high", "High"),
("medium", "Medium"),
("low", "Low"),
("informational", "Informational"),
],
),
),
(
"total_findings",
models.IntegerField(
default=0, help_text="Non-muted findings (PASS + FAIL)"
),
),
(
"failed_findings",
models.IntegerField(
default=0,
help_text="Non-muted FAIL findings (subset of total_findings)",
),
),
(
"new_failed_findings",
models.IntegerField(
default=0,
help_text="Non-muted FAIL with delta='new' (subset of failed_findings)",
),
),
(
"resources_count",
models.IntegerField(
default=0, help_text="Count of distinct resource_uid values"
),
),
],
options={
"db_table": "scan_resource_group_summaries",
"abstract": False,
},
),
migrations.AddIndex(
model_name="scangroupsummary",
index=models.Index(
fields=["tenant_id", "scan"], name="srgs_tenant_scan_idx"
),
),
migrations.AddConstraint(
model_name="scangroupsummary",
constraint=models.UniqueConstraint(
fields=("tenant_id", "scan_id", "resource_group", "severity"),
name="unique_resource_group_severity_per_scan",
),
),
migrations.AddConstraint(
model_name="scangroupsummary",
constraint=api.rls.RowLevelSecurityConstraint(
field="tenant_id",
name="rls_on_scangroupsummary",
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
),
),
]
@@ -0,0 +1,21 @@
from django.contrib.postgres.fields import ArrayField
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("api", "0068_finding_resource_group_scangroupsummary"),
]
operations = [
migrations.AddField(
model_name="resource",
name="groups",
field=ArrayField(
models.CharField(max_length=100),
blank=True,
help_text="Groups for categorization (e.g., compute, storage, IAM)",
null=True,
),
),
]
+72
View File
@@ -704,6 +704,12 @@ class Resource(RowLevelSecurityProtectedModel):
metadata = models.TextField(blank=True, null=True)
details = models.TextField(blank=True, null=True)
partition = models.TextField(blank=True, null=True)
groups = ArrayField(
models.CharField(max_length=100),
blank=True,
null=True,
help_text="Groups for categorization (e.g., compute, storage, IAM)",
)
failed_findings_count = models.IntegerField(default=0)
@@ -890,6 +896,11 @@ class Finding(PostgresPartitionedModel, RowLevelSecurityProtectedModel):
null=True,
help_text="Categories from check metadata for efficient filtering",
)
resource_groups = models.TextField(
blank=True,
null=True,
help_text="Resource group from check metadata for efficient filtering",
)
# Relationships
scan = models.ForeignKey(to=Scan, related_name="findings", on_delete=models.CASCADE)
@@ -2032,6 +2043,67 @@ class ScanCategorySummary(RowLevelSecurityProtectedModel):
resource_name = "scan-category-summaries"
class ScanGroupSummary(RowLevelSecurityProtectedModel):
"""
Pre-aggregated resource group metrics per scan by severity.
Stores one row per (resource_group, severity) combination per scan for efficient
overview queries. Resource groups come from check_metadata.Group.
Count relationships (each is a subset of the previous):
- total_findings >= failed_findings >= new_failed_findings
"""
id = models.UUIDField(primary_key=True, default=uuid4, editable=False)
inserted_at = models.DateTimeField(auto_now_add=True, editable=False)
scan = models.ForeignKey(
Scan,
on_delete=models.CASCADE,
related_name="resource_group_summaries",
related_query_name="resource_group_summary",
)
resource_group = models.CharField(max_length=50)
severity = SeverityEnumField(choices=SeverityChoices)
total_findings = models.IntegerField(
default=0, help_text="Non-muted findings (PASS + FAIL)"
)
failed_findings = models.IntegerField(
default=0, help_text="Non-muted FAIL findings (subset of total_findings)"
)
new_failed_findings = models.IntegerField(
default=0,
help_text="Non-muted FAIL with delta='new' (subset of failed_findings)",
)
resources_count = models.IntegerField(
default=0, help_text="Count of distinct resource_uid values"
)
class Meta(RowLevelSecurityProtectedModel.Meta):
db_table = "scan_resource_group_summaries"
indexes = [
models.Index(fields=["tenant_id", "scan"], name="srgs_tenant_scan_idx"),
]
constraints = [
models.UniqueConstraint(
fields=("tenant_id", "scan_id", "resource_group", "severity"),
name="unique_resource_group_severity_per_scan",
),
RowLevelSecurityConstraint(
field="tenant_id",
name="rls_on_%(class)s",
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
),
]
class JSONAPIMeta:
resource_name = "scan-resource-group-summaries"
class LighthouseConfiguration(RowLevelSecurityProtectedModel):
"""
Stores configuration and API keys for LLM services.
+488 -14
View File
@@ -712,6 +712,7 @@ paths:
- check_id
- check_metadata
- categories
- resource_groups
- raw_result
- inserted_at
- updated_at
@@ -988,6 +989,19 @@ paths:
description: Multiple values may be separated by commas.
explode: false
style: form
- in: query
name: filter[resource_groups]
schema:
type: string
- in: query
name: filter[resource_groups__in]
schema:
type: array
items:
type: string
description: Multiple values may be separated by commas.
explode: false
style: form
- in: query
name: filter[resource_name]
schema:
@@ -1239,6 +1253,7 @@ paths:
- check_id
- check_metadata
- categories
- resource_groups
- raw_result
- inserted_at
- updated_at
@@ -1560,6 +1575,19 @@ paths:
description: Multiple values may be separated by commas.
explode: false
style: form
- in: query
name: filter[resource_groups]
schema:
type: string
- in: query
name: filter[resource_groups__in]
schema:
type: array
items:
type: string
description: Multiple values may be separated by commas.
explode: false
style: form
- in: query
name: filter[resource_name]
schema:
@@ -1789,6 +1817,7 @@ paths:
- check_id
- check_metadata
- categories
- resource_groups
- raw_result
- inserted_at
- updated_at
@@ -2040,6 +2069,19 @@ paths:
description: Multiple values may be separated by commas.
explode: false
style: form
- in: query
name: filter[resource_groups]
schema:
type: string
- in: query
name: filter[resource_groups__in]
schema:
type: array
items:
type: string
description: Multiple values may be separated by commas.
explode: false
style: form
- in: query
name: filter[resource_name]
schema:
@@ -2251,6 +2293,7 @@ paths:
- regions
- resource_types
- categories
- groups
description: endpoint return only specific fields in the response on a per-type
basis by including a fields[TYPE] query parameter.
explode: false
@@ -2518,6 +2561,19 @@ paths:
description: Multiple values may be separated by commas.
explode: false
style: form
- in: query
name: filter[resource_groups]
schema:
type: string
- in: query
name: filter[resource_groups__in]
schema:
type: array
items:
type: string
description: Multiple values may be separated by commas.
explode: false
style: form
- in: query
name: filter[resource_name]
schema:
@@ -2742,6 +2798,7 @@ paths:
- regions
- resource_types
- categories
- groups
description: endpoint return only specific fields in the response on a per-type
basis by including a fields[TYPE] query parameter.
explode: false
@@ -2984,6 +3041,19 @@ paths:
description: Multiple values may be separated by commas.
explode: false
style: form
- in: query
name: filter[resource_groups]
schema:
type: string
- in: query
name: filter[resource_groups__in]
schema:
type: array
items:
type: string
description: Multiple values may be separated by commas.
explode: false
style: form
- in: query
name: filter[resource_name]
schema:
@@ -4968,12 +5038,12 @@ paths:
description: ''
/api/v1/overviews/compliance-watchlist:
get:
operationId: overviews_compliance_watchlist_retrieve
description: |-
Get compliance watchlist overview with FAIL-dominant aggregation.
Without filters: uses pre-aggregated TenantComplianceSummary (~70 rows).
With provider filters: queries ProviderComplianceScore with FAIL-dominant logic.
operationId: overviews_compliance_watchlist_list
description: 'Retrieve compliance metrics with FAIL-dominant aggregation. Without
filters: uses pre-aggregated TenantComplianceSummary. With provider filters:
queries ProviderComplianceScore with FAIL-dominant logic where any FAIL in
a requirement marks it as failed.'
summary: Get compliance watchlist overview
parameters:
- in: query
name: fields[compliance-watchlist-overviews]
@@ -4991,6 +5061,119 @@ paths:
description: endpoint return only specific fields in the response on a per-type
basis by including a fields[TYPE] query parameter.
explode: false
- in: query
name: filter[provider_id]
schema:
type: string
format: uuid
- in: query
name: filter[provider_id__in]
schema:
type: array
items:
type: string
format: uuid
description: Multiple values may be separated by commas.
explode: false
style: form
- in: query
name: filter[provider_type]
schema:
type: string
enum:
- alibabacloud
- aws
- azure
- gcp
- github
- iac
- kubernetes
- m365
- mongodbatlas
- oraclecloud
description: |-
* `aws` - AWS
* `azure` - Azure
* `gcp` - GCP
* `kubernetes` - Kubernetes
* `m365` - M365
* `github` - GitHub
* `mongodbatlas` - MongoDB Atlas
* `iac` - IaC
* `oraclecloud` - Oracle Cloud Infrastructure
* `alibabacloud` - Alibaba Cloud
- in: query
name: filter[provider_type__in]
schema:
type: array
items:
type: string
enum:
- alibabacloud
- aws
- azure
- gcp
- github
- iac
- kubernetes
- m365
- mongodbatlas
- oraclecloud
description: |-
Multiple values may be separated by commas.
* `aws` - AWS
* `azure` - Azure
* `gcp` - GCP
* `kubernetes` - Kubernetes
* `m365` - M365
* `github` - GitHub
* `mongodbatlas` - MongoDB Atlas
* `iac` - IaC
* `oraclecloud` - Oracle Cloud Infrastructure
* `alibabacloud` - Alibaba Cloud
explode: false
style: form
- name: filter[search]
required: false
in: query
description: A search term.
schema:
type: string
- name: page[number]
required: false
in: query
description: A page number within the paginated result set.
schema:
type: integer
- name: page[size]
required: false
in: query
description: Number of results to return per page.
schema:
type: integer
- name: sort
required: false
in: query
description: '[list of fields to sort by](https://jsonapi.org/format/#fetching-sorting)'
schema:
type: array
items:
type: string
enum:
- id
- -id
- compliance_id
- -compliance_id
- requirements_passed
- -requirements_passed
- requirements_failed
- -requirements_failed
- requirements_manual
- -requirements_manual
- total_requirements
- -total_requirements
explode: false
tags:
- Overview
security:
@@ -5000,7 +5183,7 @@ paths:
content:
application/vnd.api+json:
schema:
$ref: '#/components/schemas/ComplianceWatchlistOverviewResponse'
$ref: '#/components/schemas/PaginatedComplianceWatchlistOverviewList'
description: ''
/api/v1/overviews/findings:
get:
@@ -5797,6 +5980,170 @@ paths:
schema:
$ref: '#/components/schemas/OverviewRegionResponse'
description: ''
/api/v1/overviews/resource-groups:
get:
operationId: overviews_resource_groups_list
description: Retrieve aggregated resource group metrics from latest completed
scans per provider. Returns one row per resource group with total, failed,
and new failed findings counts, plus a severity breakdown showing failed findings
per severity level, and a count of distinct resources evaluated per group.
summary: Get resource group overview
parameters:
- in: query
name: fields[resource-group-overviews]
schema:
type: array
items:
type: string
enum:
- id
- total_findings
- failed_findings
- new_failed_findings
- resources_count
- severity
description: endpoint return only specific fields in the response on a per-type
basis by including a fields[TYPE] query parameter.
explode: false
- in: query
name: filter[provider_id]
schema:
type: string
format: uuid
- in: query
name: filter[provider_id__in]
schema:
type: array
items:
type: string
format: uuid
description: Multiple values may be separated by commas.
explode: false
style: form
- in: query
name: filter[provider_type]
schema:
type: string
x-spec-enum-id: 684bf4173d2b754f
enum:
- alibabacloud
- aws
- azure
- gcp
- github
- iac
- kubernetes
- m365
- mongodbatlas
- oraclecloud
description: |-
* `aws` - AWS
* `azure` - Azure
* `gcp` - GCP
* `kubernetes` - Kubernetes
* `m365` - M365
* `github` - GitHub
* `mongodbatlas` - MongoDB Atlas
* `iac` - IaC
* `oraclecloud` - Oracle Cloud Infrastructure
* `alibabacloud` - Alibaba Cloud
- in: query
name: filter[provider_type__in]
schema:
type: array
items:
type: string
x-spec-enum-id: 684bf4173d2b754f
enum:
- alibabacloud
- aws
- azure
- gcp
- github
- iac
- kubernetes
- m365
- mongodbatlas
- oraclecloud
description: |-
Multiple values may be separated by commas.
* `aws` - AWS
* `azure` - Azure
* `gcp` - GCP
* `kubernetes` - Kubernetes
* `m365` - M365
* `github` - GitHub
* `mongodbatlas` - MongoDB Atlas
* `iac` - IaC
* `oraclecloud` - Oracle Cloud Infrastructure
* `alibabacloud` - Alibaba Cloud
explode: false
style: form
- in: query
name: filter[resource_group]
schema:
type: string
- in: query
name: filter[resource_group__in]
schema:
type: array
items:
type: string
description: Multiple values may be separated by commas.
explode: false
style: form
- name: filter[search]
required: false
in: query
description: A search term.
schema:
type: string
- name: page[number]
required: false
in: query
description: A page number within the paginated result set.
schema:
type: integer
- name: page[size]
required: false
in: query
description: Number of results to return per page.
schema:
type: integer
- name: sort
required: false
in: query
description: '[list of fields to sort by](https://jsonapi.org/format/#fetching-sorting)'
schema:
type: array
items:
type: string
enum:
- id
- -id
- total_findings
- -total_findings
- failed_findings
- -failed_findings
- new_failed_findings
- -new_failed_findings
- resources_count
- -resources_count
- severity
- -severity
explode: false
tags:
- Overview
security:
- JWT or API Key: []
responses:
'200':
content:
application/vnd.api+json:
schema:
$ref: '#/components/schemas/PaginatedResourceGroupOverviewList'
description: ''
/api/v1/overviews/services:
get:
operationId: overviews_services_retrieve
@@ -7333,10 +7680,24 @@ paths:
- metadata
- details
- partition
- groups
- type
description: endpoint return only specific fields in the response on a per-type
basis by including a fields[TYPE] query parameter.
explode: false
- in: query
name: filter[groups]
schema:
type: string
- in: query
name: filter[groups__in]
schema:
type: array
items:
type: string
description: Multiple values may be separated by commas.
explode: false
style: form
- in: query
name: filter[inserted_at]
schema:
@@ -7673,6 +8034,7 @@ paths:
- metadata
- details
- partition
- groups
- type
description: endpoint return only specific fields in the response on a per-type
basis by including a fields[TYPE] query parameter.
@@ -7735,10 +8097,24 @@ paths:
- metadata
- details
- partition
- groups
- type
description: endpoint return only specific fields in the response on a per-type
basis by including a fields[TYPE] query parameter.
explode: false
- in: query
name: filter[groups]
schema:
type: string
- in: query
name: filter[groups__in]
schema:
type: array
items:
type: string
description: Multiple values may be separated by commas.
explode: false
style: form
- in: query
name: filter[name]
schema:
@@ -8003,9 +8379,23 @@ paths:
- services
- regions
- types
- groups
description: endpoint return only specific fields in the response on a per-type
basis by including a fields[TYPE] query parameter.
explode: false
- in: query
name: filter[groups]
schema:
type: string
- in: query
name: filter[groups__in]
schema:
type: array
items:
type: string
description: Multiple values may be separated by commas.
explode: false
style: form
- in: query
name: filter[inserted_at]
schema:
@@ -8306,9 +8696,23 @@ paths:
- services
- regions
- types
- groups
description: endpoint return only specific fields in the response on a per-type
basis by including a fields[TYPE] query parameter.
explode: false
- in: query
name: filter[groups]
schema:
type: string
- in: query
name: filter[groups__in]
schema:
type: array
items:
type: string
description: Multiple values may be separated by commas.
explode: false
style: form
- in: query
name: filter[name]
schema:
@@ -11573,13 +11977,6 @@ components:
- requirements_failed
- requirements_manual
- total_requirements
ComplianceWatchlistOverviewResponse:
type: object
properties:
data:
$ref: '#/components/schemas/ComplianceWatchlistOverview'
required:
- data
Finding:
type: object
required:
@@ -11654,6 +12051,10 @@ components:
maxLength: 100
nullable: true
description: Categories from check metadata for efficient filtering
resource_groups:
type: string
nullable: true
description: Resource group from check metadata for efficient filtering
raw_result: {}
inserted_at:
type: string
@@ -11808,6 +12209,10 @@ components:
type: array
items:
type: string
groups:
type: array
items:
type: string
required:
- services
- regions
@@ -14560,6 +14965,15 @@ components:
$ref: '#/components/schemas/ComplianceOverview'
required:
- data
PaginatedComplianceWatchlistOverviewList:
type: object
properties:
data:
type: array
items:
$ref: '#/components/schemas/ComplianceWatchlistOverview'
required:
- data
PaginatedFindingList:
type: object
properties:
@@ -14677,6 +15091,15 @@ components:
$ref: '#/components/schemas/ProviderSecret'
required:
- data
PaginatedResourceGroupOverviewList:
type: object
properties:
data:
type: array
items:
$ref: '#/components/schemas/ResourceGroupOverview'
required:
- data
PaginatedResourceList:
type: object
properties:
@@ -18681,6 +19104,14 @@ components:
type: string
readOnly: true
nullable: true
groups:
type: array
items:
type: string
maxLength: 100
readOnly: true
nullable: true
description: Groups for categorization (e.g., compute, storage, IAM)
type:
type: string
readOnly: true
@@ -18742,6 +19173,44 @@ components:
readOnly: true
required:
- provider
ResourceGroupOverview:
type: object
required:
- type
- id
additionalProperties: false
properties:
type:
type: string
description: The [type](https://jsonapi.org/format/#document-resource-object-identification)
member is used to describe resource objects that share common attributes
and relationships.
enum:
- resource-group-overviews
id: {}
attributes:
type: object
properties:
id:
type: string
total_findings:
type: integer
failed_findings:
type: integer
new_failed_findings:
type: integer
resources_count:
type: integer
severity:
description: 'Severity breakdown: {informational, low, medium, high,
critical}'
required:
- id
- total_findings
- failed_findings
- new_failed_findings
- resources_count
- severity
ResourceMetadata:
type: object
required:
@@ -18772,10 +19241,15 @@ components:
type: array
items:
type: string
groups:
type: array
items:
type: string
required:
- services
- regions
- types
- groups
ResourceMetadataResponse:
type: object
properties:
+285
View File
@@ -3625,6 +3625,7 @@ class TestResourceViewSet:
assert "metadata" in response.json()["data"][0]["attributes"]
assert "details" in response.json()["data"][0]["attributes"]
assert "partition" in response.json()["data"][0]["attributes"]
assert "groups" in response.json()["data"][0]["attributes"]
@pytest.mark.parametrize(
"include_values, expected_resources",
@@ -3699,6 +3700,10 @@ class TestResourceViewSet:
# full text search on resource tags
("search", "multi word", 1),
("search", "key2", 2),
# groups filter (ArrayField)
("groups", "compute", 2),
("groups", "storage", 1),
("groups.in", "compute,storage", 3),
]
),
)
@@ -3845,12 +3850,14 @@ class TestResourceViewSet:
expected_services = {"ec2", "s3"}
expected_regions = {"us-east-1", "eu-west-1"}
expected_resource_types = {"prowler-test"}
expected_groups = {"compute", "storage"}
assert data["data"]["type"] == "resources-metadata"
assert data["data"]["id"] is None
assert set(data["data"]["attributes"]["services"]) == expected_services
assert set(data["data"]["attributes"]["regions"]) == expected_regions
assert set(data["data"]["attributes"]["types"]) == expected_resource_types
assert set(data["data"]["attributes"]["groups"]) == expected_groups
def test_resources_metadata_resource_filter_retrieve(
self, authenticated_client, resources_fixture, backfill_scan_metadata_fixture
@@ -3886,6 +3893,7 @@ class TestResourceViewSet:
assert data["data"]["attributes"]["services"] == []
assert data["data"]["attributes"]["regions"] == []
assert data["data"]["attributes"]["types"] == []
assert data["data"]["attributes"]["groups"] == []
def test_resources_metadata_invalid_date(self, authenticated_client):
response = authenticated_client.get(
@@ -3925,6 +3933,7 @@ class TestResourceViewSet:
assert attributes["services"] == [latest_scan_resource.service]
assert attributes["regions"] == [latest_scan_resource.region]
assert attributes["types"] == [latest_scan_resource.type]
assert "groups" in attributes
@pytest.mark.django_db
@@ -4417,6 +4426,17 @@ class TestFindingViewSet:
attributes = response.json()["data"]["attributes"]
assert set(attributes["categories"]) == {"gen-ai", "iam"}
def test_findings_metadata_latest_groups(
self, authenticated_client, latest_scan_finding_with_categories
):
response = authenticated_client.get(
reverse("finding-metadata_latest"),
)
assert response.status_code == status.HTTP_200_OK
attributes = response.json()["data"]["attributes"]
assert "groups" in attributes
assert "ai_ml" in attributes["groups"]
def test_findings_filter_by_category(
self, authenticated_client, findings_with_categories
):
@@ -4463,6 +4483,49 @@ class TestFindingViewSet:
assert response.status_code == status.HTTP_200_OK
assert len(response.json()["data"]) == 0
def test_findings_filter_by_resource_groups(
self, authenticated_client, findings_with_group
):
finding = findings_with_group
response = authenticated_client.get(
reverse("finding-list"),
{
"filter[resource_groups]": "storage",
"filter[inserted_at]": finding.inserted_at.strftime("%Y-%m-%d"),
},
)
assert response.status_code == status.HTTP_200_OK
assert len(response.json()["data"]) == 1
assert response.json()["data"][0]["attributes"]["resource_groups"] == "storage"
def test_findings_filter_by_resource_groups_in(
self, authenticated_client, findings_with_multiple_groups
):
finding1, _ = findings_with_multiple_groups
response = authenticated_client.get(
reverse("finding-list"),
{
"filter[resource_groups__in]": "storage,security",
"filter[inserted_at]": finding1.inserted_at.strftime("%Y-%m-%d"),
},
)
assert response.status_code == status.HTTP_200_OK
assert len(response.json()["data"]) == 2
def test_findings_filter_by_resource_groups_no_match(
self, authenticated_client, findings_with_group
):
finding = findings_with_group
response = authenticated_client.get(
reverse("finding-list"),
{
"filter[resource_groups]": "nonexistent",
"filter[inserted_at]": finding.inserted_at.strftime("%Y-%m-%d"),
},
)
assert response.status_code == status.HTTP_200_OK
assert len(response.json()["data"]) == 0
@pytest.mark.django_db
class TestJWTFields:
@@ -8009,6 +8072,228 @@ class TestOverviewViewSet:
assert data[0]["attributes"]["failed_findings"] == 13
assert data[0]["attributes"]["new_failed_findings"] == 5
def test_overview_groups_no_data(self, authenticated_client):
response = authenticated_client.get(reverse("overview-resource-groups"))
assert response.status_code == status.HTTP_200_OK
assert response.json()["data"] == []
def test_overview_groups_aggregates_by_group_with_severity(
self,
authenticated_client,
tenants_fixture,
providers_fixture,
create_scan_resource_group_summary,
):
tenant = tenants_fixture[0]
provider = providers_fixture[0]
scan = Scan.objects.create(
name="resource-groups-scan",
provider=provider,
trigger=Scan.TriggerChoices.MANUAL,
state=StateChoices.COMPLETED,
tenant=tenant,
)
# resources_count is group-level (same for all severities within a group)
create_scan_resource_group_summary(
tenant,
scan,
"storage",
"high",
total_findings=20,
failed_findings=10,
new_failed_findings=5,
resources_count=8,
)
create_scan_resource_group_summary(
tenant,
scan,
"storage",
"medium",
total_findings=15,
failed_findings=7,
new_failed_findings=3,
resources_count=8, # Same as high - group-level count
)
create_scan_resource_group_summary(
tenant,
scan,
"security",
"critical",
total_findings=10,
failed_findings=8,
new_failed_findings=2,
resources_count=4,
)
response = authenticated_client.get(reverse("overview-resource-groups"))
assert response.status_code == status.HTTP_200_OK
data = response.json()["data"]
assert len(data) == 2
storage_data = next(d for d in data if d["id"] == "storage")
security_data = next(d for d in data if d["id"] == "security")
assert storage_data["attributes"]["total_findings"] == 35
assert storage_data["attributes"]["failed_findings"] == 17
assert storage_data["attributes"]["new_failed_findings"] == 8
assert (
storage_data["attributes"]["resources_count"] == 8
) # Group-level, not sum
assert security_data["attributes"]["total_findings"] == 10
assert security_data["attributes"]["failed_findings"] == 8
assert security_data["attributes"]["resources_count"] == 4
@pytest.mark.parametrize(
"filter_key,filter_value_fn,expected_total,expected_failed",
[
("filter[provider_id]", lambda p1, p2: str(p1.id), 10, 5),
("filter[provider_id__in]", lambda p1, p2: f"{p1.id},{p2.id}", 25, 12),
("filter[provider_type]", lambda p1, p2: "aws", 10, 5),
("filter[provider_type__in]", lambda p1, p2: "aws,gcp", 25, 12),
],
)
def test_overview_groups_provider_filters(
self,
authenticated_client,
tenants_fixture,
providers_fixture,
create_scan_resource_group_summary,
filter_key,
filter_value_fn,
expected_total,
expected_failed,
):
tenant = tenants_fixture[0]
provider1 = providers_fixture[0] # AWS
gcp_provider = providers_fixture[2] # GCP
scan1 = Scan.objects.create(
name="aws-rg-scan",
provider=provider1,
trigger=Scan.TriggerChoices.MANUAL,
state=StateChoices.COMPLETED,
tenant=tenant,
)
scan2 = Scan.objects.create(
name="gcp-rg-scan",
provider=gcp_provider,
trigger=Scan.TriggerChoices.MANUAL,
state=StateChoices.COMPLETED,
tenant=tenant,
)
create_scan_resource_group_summary(
tenant, scan1, "storage", "high", total_findings=10, failed_findings=5
)
create_scan_resource_group_summary(
tenant, scan2, "storage", "high", total_findings=15, failed_findings=7
)
response = authenticated_client.get(
reverse("overview-resource-groups"),
{filter_key: filter_value_fn(provider1, gcp_provider)},
)
assert response.status_code == status.HTTP_200_OK
data = response.json()["data"]
assert len(data) == 1
assert data[0]["attributes"]["total_findings"] == expected_total
assert data[0]["attributes"]["failed_findings"] == expected_failed
def test_overview_groups_group_filter(
self,
authenticated_client,
tenants_fixture,
providers_fixture,
create_scan_resource_group_summary,
):
tenant = tenants_fixture[0]
provider = providers_fixture[0]
scan = Scan.objects.create(
name="rg-filter-scan",
provider=provider,
trigger=Scan.TriggerChoices.MANUAL,
state=StateChoices.COMPLETED,
tenant=tenant,
)
create_scan_resource_group_summary(
tenant, scan, "storage", "high", total_findings=10, failed_findings=5
)
create_scan_resource_group_summary(
tenant, scan, "compute", "medium", total_findings=20, failed_findings=8
)
create_scan_resource_group_summary(
tenant, scan, "security", "low", total_findings=15, failed_findings=3
)
response = authenticated_client.get(
reverse("overview-resource-groups"),
{"filter[resource_group__in]": "storage,compute"},
)
assert response.status_code == status.HTTP_200_OK
data = response.json()["data"]
group_ids = {item["id"] for item in data}
assert group_ids == {"storage", "compute"}
def test_overview_groups_aggregates_multiple_providers(
self,
authenticated_client,
tenants_fixture,
providers_fixture,
create_scan_resource_group_summary,
):
tenant = tenants_fixture[0]
provider1, provider2, *_ = providers_fixture
scan1 = Scan.objects.create(
name="multi-provider-rg-scan-1",
provider=provider1,
trigger=Scan.TriggerChoices.MANUAL,
state=StateChoices.COMPLETED,
tenant=tenant,
)
scan2 = Scan.objects.create(
name="multi-provider-rg-scan-2",
provider=provider2,
trigger=Scan.TriggerChoices.MANUAL,
state=StateChoices.COMPLETED,
tenant=tenant,
)
create_scan_resource_group_summary(
tenant,
scan1,
"storage",
"high",
total_findings=10,
failed_findings=5,
new_failed_findings=2,
resources_count=4,
)
create_scan_resource_group_summary(
tenant,
scan2,
"storage",
"high",
total_findings=15,
failed_findings=8,
new_failed_findings=3,
resources_count=6,
)
response = authenticated_client.get(reverse("overview-resource-groups"))
assert response.status_code == status.HTTP_200_OK
data = response.json()["data"]
assert len(data) == 1
assert data[0]["id"] == "storage"
assert data[0]["attributes"]["total_findings"] == 25
assert data[0]["attributes"]["failed_findings"] == 13
assert data[0]["attributes"]["new_failed_findings"] == 5
assert data[0]["attributes"]["resources_count"] == 10
def test_compliance_watchlist_no_filters_uses_tenant_summary(
self, authenticated_client, tenant_compliance_summary_fixture
):
+10
View File
@@ -393,11 +393,21 @@ def get_findings_metadata_no_aggregations(tenant_id: str, filtered_queryset):
categories_set.update(categories_list)
categories = sorted(categories_set)
# Aggregate groups from findings
groups = list(
filtered_queryset.exclude(resource_groups__isnull=True)
.exclude(resource_groups__exact="")
.values_list("resource_groups", flat=True)
.distinct()
.order_by("resource_groups")
)
result = {
"services": services,
"regions": regions,
"resource_types": resource_types,
"categories": categories,
"groups": groups,
}
serializer = FindingMetadataSerializer(data=result)
+23
View File
@@ -1175,6 +1175,7 @@ class ResourceSerializer(RLSSerializer):
"metadata",
"details",
"partition",
"groups",
]
extra_kwargs = {
"id": {"read_only": True},
@@ -1183,6 +1184,7 @@ class ResourceSerializer(RLSSerializer):
"metadata": {"read_only": True},
"details": {"read_only": True},
"partition": {"read_only": True},
"groups": {"read_only": True},
}
included_serializers = {
@@ -1276,6 +1278,7 @@ class ResourceMetadataSerializer(BaseSerializerV1):
services = serializers.ListField(child=serializers.CharField(), allow_empty=True)
regions = serializers.ListField(child=serializers.CharField(), allow_empty=True)
types = serializers.ListField(child=serializers.CharField(), allow_empty=True)
groups = serializers.ListField(child=serializers.CharField(), allow_empty=True)
# Temporarily disabled until we implement tag filtering in the UI
# tags = serializers.JSONField(help_text="Tags are described as key-value pairs.")
@@ -1302,6 +1305,7 @@ class FindingSerializer(RLSSerializer):
"check_id",
"check_metadata",
"categories",
"resource_groups",
"raw_result",
"inserted_at",
"updated_at",
@@ -1358,6 +1362,9 @@ class FindingMetadataSerializer(BaseSerializerV1):
child=serializers.CharField(), allow_empty=True
)
categories = serializers.ListField(child=serializers.CharField(), allow_empty=True)
groups = serializers.ListField(
child=serializers.CharField(), allow_empty=True, required=False, default=list
)
# Temporarily disabled until we implement tag filtering in the UI
# tags = serializers.JSONField(help_text="Tags are described as key-value pairs.")
@@ -2303,6 +2310,22 @@ class CategoryOverviewSerializer(BaseSerializerV1):
resource_name = "category-overviews"
class ResourceGroupOverviewSerializer(BaseSerializerV1):
"""Serializer for resource group overview aggregations."""
id = serializers.CharField(source="resource_group")
total_findings = serializers.IntegerField()
failed_findings = serializers.IntegerField()
new_failed_findings = serializers.IntegerField()
resources_count = serializers.IntegerField()
severity = serializers.JSONField(
help_text="Severity breakdown: {informational, low, medium, high, critical}"
)
class JSONAPIMeta:
resource_name = "resource-group-overviews"
class ComplianceWatchlistOverviewSerializer(BaseSerializerV1):
"""Serializer for compliance watchlist overview with FAIL-dominant aggregation."""
+152
View File
@@ -119,6 +119,7 @@ from api.filters import (
ProviderGroupFilter,
ProviderSecretFilter,
ResourceFilter,
ResourceGroupOverviewFilter,
RoleFilter,
ScanFilter,
ScanSummaryFilter,
@@ -160,6 +161,7 @@ from api.models import (
SAMLToken,
Scan,
ScanCategorySummary,
ScanGroupSummary,
ScanSummary,
SeverityChoices,
StateChoices,
@@ -233,6 +235,7 @@ from api.v1.serializers import (
ProviderSecretUpdateSerializer,
ProviderSerializer,
ProviderUpdateSerializer,
ResourceGroupOverviewSerializer,
ResourceMetadataSerializer,
ResourceSerializer,
RoleCreateSerializer,
@@ -2527,10 +2530,20 @@ class ResourceViewSet(PaginateByPkMixin, BaseRLSViewSet):
.order_by("resource_type")
)
# Get groups from Resource model (flatten ArrayField)
all_groups = Resource.objects.filter(
tenant_id=tenant_id,
groups__isnull=False,
).values_list("groups", flat=True)
groups = sorted(
set(g for groups_list in all_groups if groups_list for g in groups_list)
)
result = {
"services": services,
"regions": regions,
"types": resource_types,
"groups": groups,
}
serializer = self.get_serializer(data=result)
@@ -2587,10 +2600,20 @@ class ResourceViewSet(PaginateByPkMixin, BaseRLSViewSet):
.order_by("resource_type")
)
# Get groups from Resource model for resources in latest scans (flatten ArrayField)
all_groups = Resource.objects.filter(
tenant_id=tenant_id,
groups__isnull=False,
).values_list("groups", flat=True)
groups = sorted(
set(g for groups_list in all_groups if groups_list for g in groups_list)
)
result = {
"services": services,
"regions": regions,
"types": resource_types,
"groups": groups,
}
serializer = self.get_serializer(data=result)
@@ -3019,11 +3042,23 @@ class FindingViewSet(PaginateByPkMixin, BaseRLSViewSet):
categories_set.update(categories_list)
categories = sorted(categories_set)
# Get groups from ScanGroupSummary for latest scans
groups = list(
ScanGroupSummary.objects.filter(
tenant_id=tenant_id,
scan_id__in=latest_scans_queryset.values_list("id", flat=True),
)
.values_list("resource_group", flat=True)
.distinct()
.order_by("resource_group")
)
result = {
"services": services,
"regions": regions,
"resource_types": resource_types,
"categories": categories,
"groups": groups,
}
serializer = self.get_serializer(data=result)
@@ -4097,6 +4132,30 @@ class ComplianceOverviewViewSet(BaseRLSViewSet, TaskManagementMixin):
filters=True,
responses={200: CategoryOverviewSerializer(many=True)},
),
resource_groups=extend_schema(
summary="Get resource group overview",
description=(
"Retrieve aggregated resource group metrics from latest completed scans per provider. "
"Returns one row per resource group with total, failed, and new failed findings counts, "
"plus a severity breakdown showing failed findings per severity level, "
"and a count of distinct resources evaluated per group."
),
tags=["Overview"],
filters=True,
responses={200: ResourceGroupOverviewSerializer(many=True)},
),
compliance_watchlist=extend_schema(
summary="Get compliance watchlist overview",
description=(
"Retrieve compliance metrics with FAIL-dominant aggregation. "
"Without filters: uses pre-aggregated TenantComplianceSummary. "
"With provider filters: queries ProviderComplianceScore with FAIL-dominant logic "
"where any FAIL in a requirement marks it as failed."
),
tags=["Overview"],
filters=True,
responses={200: ComplianceWatchlistOverviewSerializer(many=True)},
),
)
@method_decorator(CACHE_DECORATOR, name="list")
class OverviewViewSet(BaseRLSViewSet):
@@ -4146,6 +4205,8 @@ class OverviewViewSet(BaseRLSViewSet):
return AttackSurfaceOverviewSerializer
elif self.action == "categories":
return CategoryOverviewSerializer
elif self.action == "resource_groups":
return ResourceGroupOverviewSerializer
elif self.action == "compliance_watchlist":
return ComplianceWatchlistOverviewSerializer
return super().get_serializer_class()
@@ -4161,6 +4222,8 @@ class OverviewViewSet(BaseRLSViewSet):
return DailySeveritySummaryFilter
elif self.action == "categories":
return CategoryOverviewFilter
elif self.action == "resource_groups":
return ResourceGroupOverviewFilter
elif self.action == "attack_surface":
return AttackSurfaceOverviewFilter
elif self.action == "compliance_watchlist":
@@ -5005,6 +5068,95 @@ class OverviewViewSet(BaseRLSViewSet):
status=status.HTTP_200_OK,
)
@action(
detail=False,
methods=["get"],
url_name="resource-groups",
url_path="resource-groups",
)
def resource_groups(self, request):
tenant_id = request.tenant_id
provider_filters = self._extract_provider_filters_from_params()
latest_scan_ids = self._latest_scan_ids_for_allowed_providers(
tenant_id, provider_filters
)
base_queryset = ScanGroupSummary.objects.filter(
tenant_id=tenant_id, scan_id__in=latest_scan_ids
)
provider_filter_keys = {
"provider_id",
"provider_id__in",
"provider_type",
"provider_type__in",
}
filtered_queryset = self._apply_filterset(
base_queryset,
ResourceGroupOverviewFilter,
exclude_keys=provider_filter_keys,
)
aggregation = (
filtered_queryset.values("resource_group", "severity")
.annotate(
total=Coalesce(Sum("total_findings"), 0),
failed=Coalesce(Sum("failed_findings"), 0),
new_failed=Coalesce(Sum("new_failed_findings"), 0),
)
.order_by("resource_group", "severity")
)
# Get resource_group-level resources_count:
# 1. Max per (scan, resource_group) to deduplicate within-scan severity rows
# 2. Sum across scans for cross-provider aggregation
scan_resource_group_resources = filtered_queryset.values(
"scan_id", "resource_group"
).annotate(resources=Coalesce(Max("resources_count"), 0))
resources_by_resource_group = defaultdict(int)
for row in scan_resource_group_resources:
resources_by_resource_group[row["resource_group"]] += row["resources"]
resource_group_data = defaultdict(
lambda: {
"total_findings": 0,
"failed_findings": 0,
"new_failed_findings": 0,
"resources_count": 0,
"severity": {
"informational": 0,
"low": 0,
"medium": 0,
"high": 0,
"critical": 0,
},
}
)
for row in aggregation:
grp = row["resource_group"]
sev = row["severity"]
resource_group_data[grp]["total_findings"] += row["total"]
resource_group_data[grp]["failed_findings"] += row["failed"]
resource_group_data[grp]["new_failed_findings"] += row["new_failed"]
if sev in resource_group_data[grp]["severity"]:
resource_group_data[grp]["severity"][sev] = row["failed"]
# Set resources_count from resource_group-level aggregation
for grp in resource_group_data:
resource_group_data[grp]["resources_count"] = (
resources_by_resource_group.get(grp, 0)
)
response_data = [
{"resource_group": grp, **data}
for grp, data in sorted(resource_group_data.items())
]
return Response(
self.get_serializer(response_data, many=True).data,
status=status.HTTP_200_OK,
)
@action(
detail=False,
methods=["get"],
+104
View File
@@ -14,6 +14,7 @@ from rest_framework.test import APIClient
from tasks.jobs.backfill import (
backfill_resource_scan_summaries,
backfill_scan_category_summaries,
backfill_scan_resource_group_summaries,
)
from api.db_utils import rls_transaction
@@ -41,6 +42,7 @@ from api.models import (
SAMLDomainIndex,
Scan,
ScanCategorySummary,
ScanGroupSummary,
ScanSummary,
StateChoices,
StatusChoices,
@@ -739,6 +741,7 @@ def resources_fixture(providers_fixture):
region="us-east-1",
service="ec2",
type="prowler-test",
groups=["compute"],
)
resource1.upsert_or_delete_tags(tags)
@@ -751,6 +754,7 @@ def resources_fixture(providers_fixture):
region="eu-west-1",
service="s3",
type="prowler-test",
groups=["storage"],
)
resource2.upsert_or_delete_tags(tags)
@@ -762,6 +766,7 @@ def resources_fixture(providers_fixture):
region="us-east-1",
service="ec2",
type="test",
groups=["compute"],
)
tags = [
@@ -1383,11 +1388,13 @@ def latest_scan_finding_with_categories(
check_id="genai_iam_check",
check_metadata={"CheckId": "genai_iam_check"},
categories=["gen-ai", "iam"],
resource_groups="ai_ml",
first_seen_at="2024-01-02T00:00:00Z",
)
finding.add_resources([resource])
backfill_resource_scan_summaries(tenant_id, str(scan.id))
backfill_scan_category_summaries(tenant_id, str(scan.id))
backfill_scan_resource_group_summaries(tenant_id, str(scan.id))
return finding
@@ -1629,6 +1636,103 @@ def create_scan_category_summary():
return _create
@pytest.fixture(scope="function")
def findings_with_group(scans_fixture, resources_fixture):
scan = scans_fixture[0]
resource = resources_fixture[0]
finding = Finding.objects.create(
tenant_id=scan.tenant_id,
uid="finding_with_group_1",
scan=scan,
delta=None,
status=Status.FAIL,
status_extended="test status",
impact=Severity.critical,
impact_extended="test impact",
severity=Severity.critical,
raw_result={"status": Status.FAIL},
check_id="storage_check",
check_metadata={"CheckId": "storage_check"},
resource_groups="storage",
first_seen_at="2024-01-02T00:00:00Z",
)
finding.add_resources([resource])
backfill_resource_scan_summaries(str(scan.tenant_id), str(scan.id))
return finding
@pytest.fixture(scope="function")
def findings_with_multiple_groups(scans_fixture, resources_fixture):
scan = scans_fixture[0]
resource1, resource2 = resources_fixture[:2]
finding1 = Finding.objects.create(
tenant_id=scan.tenant_id,
uid="finding_multi_grp_1",
scan=scan,
delta=None,
status=Status.FAIL,
status_extended="test status",
impact=Severity.critical,
impact_extended="test impact",
severity=Severity.critical,
raw_result={"status": Status.FAIL},
check_id="storage_check",
check_metadata={"CheckId": "storage_check"},
resource_groups="storage",
first_seen_at="2024-01-02T00:00:00Z",
)
finding1.add_resources([resource1])
finding2 = Finding.objects.create(
tenant_id=scan.tenant_id,
uid="finding_multi_grp_2",
scan=scan,
delta=None,
status=Status.FAIL,
status_extended="test status 2",
impact=Severity.high,
impact_extended="test impact 2",
severity=Severity.high,
raw_result={"status": Status.FAIL},
check_id="security_check",
check_metadata={"CheckId": "security_check"},
resource_groups="security",
first_seen_at="2024-01-02T00:00:00Z",
)
finding2.add_resources([resource2])
backfill_resource_scan_summaries(str(scan.tenant_id), str(scan.id))
return finding1, finding2
@pytest.fixture
def create_scan_resource_group_summary():
def _create(
tenant,
scan,
resource_group,
severity,
total_findings=10,
failed_findings=5,
new_failed_findings=2,
resources_count=3,
):
return ScanGroupSummary.objects.create(
tenant=tenant,
scan=scan,
resource_group=resource_group,
severity=severity,
total_findings=total_findings,
failed_findings=failed_findings,
new_failed_findings=new_failed_findings,
resources_count=resources_count,
)
return _create
def get_authorization_header(access_token: str) -> dict:
return {"Authorization": f"Bearer {access_token}"}
+89 -2
View File
@@ -2,13 +2,13 @@ from collections import defaultdict
from datetime import timedelta
from celery.utils.log import get_task_logger
from django.db.models import Sum
from django.db.models import OuterRef, Subquery, Sum
from django.utils import timezone
from tasks.jobs.queries import (
COMPLIANCE_UPSERT_PROVIDER_SCORE_SQL,
COMPLIANCE_UPSERT_TENANT_SUMMARY_ALL_SQL,
)
from tasks.jobs.scan import aggregate_category_counts
from tasks.jobs.scan import aggregate_category_counts, aggregate_resource_group_counts
from api.db_router import READ_REPLICA_ALIAS, MainRouter
from api.db_utils import (
@@ -28,6 +28,7 @@ from api.models import (
ResourceScanSummary,
Scan,
ScanCategorySummary,
ScanGroupSummary,
ScanSummary,
StateChoices,
)
@@ -356,6 +357,92 @@ def backfill_scan_category_summaries(tenant_id: str, scan_id: str):
return {"status": "backfilled", "categories_count": len(category_counts)}
def backfill_scan_resource_group_summaries(tenant_id: str, scan_id: str):
"""
Backfill ScanGroupSummary for a completed scan.
Aggregates resource group counts from all findings in the scan and creates
one ScanGroupSummary row per (resource_group, severity) combination.
Args:
tenant_id: Target tenant UUID
scan_id: Scan UUID to backfill
Returns:
dict: Status indicating whether backfill was performed
"""
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
if ScanGroupSummary.objects.filter(
tenant_id=tenant_id, scan_id=scan_id
).exists():
return {"status": "already backfilled"}
if not Scan.objects.filter(
tenant_id=tenant_id,
id=scan_id,
state__in=(StateChoices.COMPLETED, StateChoices.FAILED),
).exists():
return {"status": "scan is not completed"}
resource_group_counts: dict[tuple[str, str], dict[str, int]] = {}
group_resources_cache: dict[str, set] = {}
# Get findings with their first resource UID via annotation
resource_uid_subquery = ResourceFindingMapping.objects.filter(
finding_id=OuterRef("id"), tenant_id=tenant_id
).values("resource__uid")[:1]
for finding in (
Finding.all_objects.filter(tenant_id=tenant_id, scan_id=scan_id)
.annotate(resource_uid=Subquery(resource_uid_subquery))
.values(
"resource_groups",
"severity",
"status",
"delta",
"muted",
"resource_uid",
)
):
aggregate_resource_group_counts(
resource_group=finding.get("resource_groups"),
severity=finding.get("severity"),
status=finding.get("status"),
delta=finding.get("delta"),
muted=finding.get("muted", False),
resource_uid=finding.get("resource_uid") or "",
cache=resource_group_counts,
group_resources_cache=group_resources_cache,
)
if not resource_group_counts:
return {"status": "no resource groups to backfill"}
# Compute group-level resource counts (same value for all severity rows in a group)
group_resource_counts = {
grp: len(uids) for grp, uids in group_resources_cache.items()
}
resource_group_summaries = [
ScanGroupSummary(
tenant_id=tenant_id,
scan_id=scan_id,
resource_group=grp,
severity=severity,
total_findings=counts["total"],
failed_findings=counts["failed"],
new_failed_findings=counts["new_failed"],
resources_count=group_resource_counts.get(grp, 0),
)
for (grp, severity), counts in resource_group_counts.items()
]
with rls_transaction(tenant_id):
ScanGroupSummary.objects.bulk_create(
resource_group_summaries, batch_size=500, ignore_conflicts=True
)
return {"status": "backfilled", "resource_groups_count": len(resource_group_counts)}
def backfill_provider_compliance_scores(tenant_id: str) -> dict:
"""
Backfill ProviderComplianceScore from latest completed scan per provider.
+117 -1
View File
@@ -45,6 +45,7 @@ from api.models import (
ResourceTag,
Scan,
ScanCategorySummary,
ScanGroupSummary,
ScanSummary,
StateChoices,
)
@@ -127,6 +128,50 @@ def aggregate_category_counts(
cache[key]["new_failed"] += 1
def aggregate_resource_group_counts(
resource_group: str | None,
severity: str,
status: str,
delta: str | None,
muted: bool,
resource_uid: str,
cache: dict[tuple[str, str], dict[str, int]],
group_resources_cache: dict[str, set],
) -> None:
"""
Increment resource group counters in-place for a finding.
Args:
resource_group: Resource group from check metadata (e.g., "database", "compute").
severity: Severity level (e.g., "high", "medium").
status: Finding status as string ("FAIL", "PASS").
delta: Delta value as string ("new", "changed") or None.
muted: Whether the finding is muted.
resource_uid: Unique identifier for the resource to count distinct resources.
cache: Dict {(resource_group, severity): {"total", "failed", "new_failed"}} to update.
group_resources_cache: Dict {resource_group: set(resource_uids)} for group-level resource tracking.
"""
if not resource_group:
return
is_failed = status == "FAIL" and not muted
is_new_failed = is_failed and delta == "new"
key = (resource_group, severity)
if key not in cache:
cache[key] = {"total": 0, "failed": 0, "new_failed": 0}
if not muted:
cache[key]["total"] += 1
if is_failed:
cache[key]["failed"] += 1
if is_new_failed:
cache[key]["new_failed"] += 1
# Track resources at GROUP level (not per-severity) to avoid over-counting
if resource_uid and not muted:
group_resources_cache.setdefault(resource_group, set()).add(resource_uid)
def _get_attack_surface_mapping_from_provider(provider_type: str) -> dict:
global _ATTACK_SURFACE_MAPPING_CACHE
@@ -438,6 +483,8 @@ def _process_finding_micro_batch(
scan_resource_cache: set,
mute_rules_cache: dict,
scan_categories_cache: dict[tuple[str, str], dict[str, int]],
scan_resource_groups_cache: dict[tuple[str, str], dict[str, int]],
group_resources_cache: dict[str, set],
) -> None:
"""
Process a micro-batch of findings and persist them using bulk operations.
@@ -459,6 +506,8 @@ def _process_finding_micro_batch(
scan_resource_cache: Set of tuples used to create `ResourceScanSummary` rows.
mute_rules_cache: Map of finding UID -> mute reason gathered before the scan.
scan_categories_cache: Dict tracking category counts {(category, severity): {"total", "failed", "new_failed"}}.
scan_resource_groups_cache: Dict tracking resource group counts {(resource_group, severity): {"total", "failed", "new_failed"}}.
group_resources_cache: Dict tracking unique resources per group {resource_group: set(resource_uids)}.
"""
# Accumulate objects for bulk operations
findings_to_create = []
@@ -499,6 +548,8 @@ def _process_finding_micro_batch(
with rls_transaction(tenant_id):
resource_uid = finding.resource_uid
if resource_uid not in resource_cache:
check_metadata = finding.get_metadata()
group = check_metadata.get("resourcegroup") or None
resource_instance, _ = Resource.objects.get_or_create(
tenant_id=tenant_id,
provider=provider_instance,
@@ -508,6 +559,7 @@ def _process_finding_micro_batch(
"service": finding.service_name,
"type": finding.resource_type,
"name": finding.resource_name,
"groups": [group] if group else None,
},
)
resource_cache[resource_uid] = resource_instance
@@ -528,6 +580,8 @@ def _process_finding_micro_batch(
# Track resource field changes (defer save)
updated = False
check_metadata = finding.get_metadata()
group = check_metadata.get("resourcegroup") or None
if finding.region and resource_instance.region != finding.region:
resource_instance.region = finding.region
updated = True
@@ -548,6 +602,11 @@ def _process_finding_micro_batch(
if resource_instance.partition != finding.partition:
resource_instance.partition = finding.partition
updated = True
if group and (
not resource_instance.groups or group not in resource_instance.groups
):
resource_instance.groups = (resource_instance.groups or []) + [group]
updated = True
if updated:
dirty_resources[resource_uid] = resource_instance
@@ -633,6 +692,7 @@ def _process_finding_micro_batch(
muted_reason=muted_reason,
compliance=finding.compliance,
categories=check_metadata.get("categories", []) or [],
resource_groups=check_metadata.get("resourcegroup") or None,
)
findings_to_create.append(finding_instance)
resource_denormalized_data.append((finding_instance, resource_instance))
@@ -657,6 +717,18 @@ def _process_finding_micro_batch(
cache=scan_categories_cache,
)
# Track resource groups with counts for ScanGroupSummary
aggregate_resource_group_counts(
resource_group=check_metadata.get("resourcegroup") or None,
severity=finding.severity.value,
status=status.value,
delta=delta.value if delta else None,
muted=is_muted,
resource_uid=resource_instance.uid if resource_instance else "",
cache=scan_resource_groups_cache,
group_resources_cache=group_resources_cache,
)
# Bulk operations within single transaction
with rls_transaction(tenant_id):
# Bulk create findings
@@ -714,7 +786,15 @@ def _process_finding_micro_batch(
tenant_id=tenant_id,
model=Resource,
objects=list(dirty_resources.values()),
fields=["metadata", "details", "partition", "region", "service", "type"],
fields=[
"metadata",
"details",
"partition",
"region",
"service",
"type",
"groups",
],
batch_size=1000,
)
@@ -757,6 +837,8 @@ def perform_prowler_scan(
unique_resources = set()
scan_resource_cache: set[tuple[str, str, str, str]] = set()
scan_categories_cache: dict[tuple[str, str], dict[str, int]] = {}
scan_resource_groups_cache: dict[tuple[str, str], dict[str, int]] = {}
group_resources_cache: dict[str, set] = {}
start_time = time.time()
exc = None
@@ -847,6 +929,8 @@ def perform_prowler_scan(
scan_resource_cache=scan_resource_cache,
mute_rules_cache=mute_rules_cache,
scan_categories_cache=scan_categories_cache,
scan_resource_groups_cache=scan_resource_groups_cache,
group_resources_cache=group_resources_cache,
)
# Update scan progress
@@ -933,6 +1017,38 @@ def perform_prowler_scan(
sentry_sdk.capture_exception(cat_exception)
logger.error(f"Error storing categories for scan {scan_id}: {cat_exception}")
try:
if scan_resource_groups_cache:
# Compute group-level resource counts (same value for all severity rows in a group)
group_resource_counts = {
grp: len(uids) for grp, uids in group_resources_cache.items()
}
resource_group_summaries = [
ScanGroupSummary(
tenant_id=tenant_id,
scan_id=scan_id,
resource_group=grp,
severity=severity,
total_findings=counts["total"],
failed_findings=counts["failed"],
new_failed_findings=counts["new_failed"],
resources_count=group_resource_counts.get(grp, 0),
)
for (
grp,
severity,
), counts in scan_resource_groups_cache.items()
]
with rls_transaction(tenant_id):
ScanGroupSummary.objects.bulk_create(
resource_group_summaries, batch_size=500, ignore_conflicts=True
)
except Exception as rg_exception:
sentry_sdk.capture_exception(rg_exception)
logger.error(
f"Error storing resource groups for scan {scan_id}: {rg_exception}"
)
serializer = ScanTaskSerializer(instance=scan_instance)
return serializer.data
+16
View File
@@ -14,6 +14,7 @@ from tasks.jobs.backfill import (
backfill_provider_compliance_scores,
backfill_resource_scan_summaries,
backfill_scan_category_summaries,
backfill_scan_resource_group_summaries,
)
from tasks.jobs.connection import (
check_integration_connection,
@@ -613,6 +614,21 @@ def backfill_scan_category_summaries_task(tenant_id: str, scan_id: str):
return backfill_scan_category_summaries(tenant_id=tenant_id, scan_id=scan_id)
@shared_task(name="backfill-scan-resource-group-summaries", queue="backfill")
@handle_provider_deletion
def backfill_scan_resource_group_summaries_task(tenant_id: str, scan_id: str):
"""
Backfill ScanGroupSummary for a completed scan.
Aggregates unique resource groups from findings and creates a summary row.
Args:
tenant_id (str): The tenant identifier.
scan_id (str): The scan identifier.
"""
return backfill_scan_resource_group_summaries(tenant_id=tenant_id, scan_id=scan_id)
@shared_task(name="backfill-provider-compliance-scores", queue="backfill")
def backfill_provider_compliance_scores_task(tenant_id: str):
"""
@@ -8,6 +8,7 @@ from tasks.jobs.backfill import (
backfill_provider_compliance_scores,
backfill_resource_scan_summaries,
backfill_scan_category_summaries,
backfill_scan_resource_group_summaries,
)
from api.models import (
@@ -16,6 +17,7 @@ from api.models import (
ResourceScanSummary,
Scan,
ScanCategorySummary,
ScanGroupSummary,
StateChoices,
)
from prowler.lib.check.models import Severity
@@ -265,6 +267,94 @@ class TestBackfillScanCategorySummaries:
assert summary.new_failed_findings == 1
@pytest.fixture(scope="function")
def findings_with_group_fixture(scans_fixture, resources_fixture):
scan = scans_fixture[0]
resource = resources_fixture[0]
finding = Finding.objects.create(
tenant_id=scan.tenant_id,
uid="finding_with_group",
scan=scan,
delta="new",
status=Status.FAIL,
status_extended="test status",
impact=Severity.high,
impact_extended="test impact",
severity=Severity.high,
raw_result={"status": Status.FAIL},
check_id="test_check",
check_metadata={"CheckId": "test_check"},
resource_groups="ai_ml",
first_seen_at="2024-01-02T00:00:00Z",
)
finding.add_resources([resource])
return finding
@pytest.fixture(scope="function")
def scan_resource_group_summary_fixture(scans_fixture):
scan = scans_fixture[0]
return ScanGroupSummary.objects.create(
tenant_id=scan.tenant_id,
scan=scan,
resource_group="existing-group",
severity=Severity.high,
total_findings=1,
failed_findings=0,
new_failed_findings=0,
resources_count=1,
)
@pytest.mark.django_db
class TestBackfillScanGroupSummaries:
def test_already_backfilled(self, scan_resource_group_summary_fixture):
tenant_id = scan_resource_group_summary_fixture.tenant_id
scan_id = scan_resource_group_summary_fixture.scan_id
result = backfill_scan_resource_group_summaries(str(tenant_id), str(scan_id))
assert result == {"status": "already backfilled"}
def test_not_completed_scan(self, get_not_completed_scans):
for scan in get_not_completed_scans:
result = backfill_scan_resource_group_summaries(
str(scan.tenant_id), str(scan.id)
)
assert result == {"status": "scan is not completed"}
def test_no_resource_groups_to_backfill(self, scans_fixture):
scan = scans_fixture[1] # Failed scan with no findings
result = backfill_scan_resource_group_summaries(
str(scan.tenant_id), str(scan.id)
)
assert result == {"status": "no resource groups to backfill"}
def test_successful_backfill(self, findings_with_group_fixture):
finding = findings_with_group_fixture
tenant_id = str(finding.tenant_id)
scan_id = str(finding.scan_id)
result = backfill_scan_resource_group_summaries(tenant_id, scan_id)
# 1 resource group × 1 severity = 1 row
assert result == {"status": "backfilled", "resource_groups_count": 1}
summaries = ScanGroupSummary.objects.filter(
tenant_id=tenant_id, scan_id=scan_id
)
assert summaries.count() == 1
summary = summaries.first()
assert summary.resource_group == "ai_ml"
assert summary.severity == Severity.high
assert summary.total_findings == 1
assert summary.failed_findings == 1
assert summary.new_failed_findings == 1
assert summary.resources_count == 1
@pytest.mark.django_db
class TestBackfillProviderComplianceScores:
def test_no_completed_scans(self, tenants_fixture):
+16
View File
@@ -1380,6 +1380,8 @@ class TestProcessFindingMicroBatch:
scan_resource_cache: set[tuple[str, str, str, str]] = set()
mute_rules_cache = {}
scan_categories_cache: dict[tuple[str, str], dict[str, int]] = {}
scan_resource_groups_cache: dict[tuple[str, str], dict[str, int]] = {}
group_resources_cache: dict[str, set] = {}
with (
patch("tasks.jobs.scan.rls_transaction", new=noop_rls_transaction),
@@ -1398,6 +1400,8 @@ class TestProcessFindingMicroBatch:
scan_resource_cache,
mute_rules_cache,
scan_categories_cache,
scan_resource_groups_cache,
group_resources_cache,
)
created_finding = Finding.objects.get(uid=finding.uid)
@@ -1491,6 +1495,8 @@ class TestProcessFindingMicroBatch:
scan_resource_cache: set[tuple[str, str, str, str]] = set()
mute_rules_cache = {finding.uid: "Muted via rule"}
scan_categories_cache: dict[tuple[str, str], dict[str, int]] = {}
scan_resource_groups_cache: dict[tuple[str, str], dict[str, int]] = {}
group_resources_cache: dict[str, set] = {}
with (
patch("tasks.jobs.scan.rls_transaction", new=noop_rls_transaction),
@@ -1509,6 +1515,8 @@ class TestProcessFindingMicroBatch:
scan_resource_cache,
mute_rules_cache,
scan_categories_cache,
scan_resource_groups_cache,
group_resources_cache,
)
existing_resource.refresh_from_db()
@@ -1617,6 +1625,8 @@ class TestProcessFindingMicroBatch:
scan_resource_cache: set[tuple[str, str, str, str]] = set()
mute_rules_cache = {}
scan_categories_cache: dict[tuple[str, str], dict[str, int]] = {}
scan_resource_groups_cache: dict[tuple[str, str], dict[str, int]] = {}
group_resources_cache: dict[str, set] = {}
with (
patch("tasks.jobs.scan.rls_transaction", new=noop_rls_transaction),
@@ -1636,6 +1646,8 @@ class TestProcessFindingMicroBatch:
scan_resource_cache,
mute_rules_cache,
scan_categories_cache,
scan_resource_groups_cache,
group_resources_cache,
)
# Verify the long UID finding was NOT created
@@ -1713,6 +1725,8 @@ class TestProcessFindingMicroBatch:
scan_resource_cache: set[tuple[str, str, str, str]] = set()
mute_rules_cache = {}
scan_categories_cache: dict[tuple[str, str], dict[str, int]] = {}
scan_resource_groups_cache: dict[tuple[str, str], dict[str, int]] = {}
group_resources_cache: dict[str, set] = {}
with (
patch("tasks.jobs.scan.rls_transaction", new=noop_rls_transaction),
@@ -1731,6 +1745,8 @@ class TestProcessFindingMicroBatch:
scan_resource_cache,
mute_rules_cache,
scan_categories_cache,
scan_resource_groups_cache,
group_resources_cache,
)
# finding1: PASS, severity=low, categories=["gen-ai", "security"]