From c20f07ced4a05bef7cf0ad9cf67c2b1ffc5299a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Fern=C3=A1ndez=20Poyatos?= Date: Mon, 12 May 2025 14:34:37 +0200 Subject: [PATCH] feat(findings): Improve performance on /findings/metadata, /overviews and filters (#7690) --- api/CHANGELOG.md | 8 + api/docker-entrypoint.sh | 2 +- api/pyproject.toml | 2 +- api/src/backend/api/db_utils.py | 71 ++++++ api/src/backend/api/filters.py | 25 +- .../api/management/commands/findings.py | 54 +++- .../0018_resource_scan_summaries.py | 81 ++++++ ...019_finding_denormalize_resource_fields.py | 42 ++++ ...ings_new_performance_indexes_partitions.py | 86 +++++++ ...findings_new_performance_indexes_parent.py | 37 +++ ...0022_scan_summaries_performance_indexes.py | 38 +++ .../0023_resources_lookup_optimization.py | 28 +++ api/src/backend/api/models.py | 120 +++++++-- api/src/backend/api/specs/v1.yaml | 4 +- api/src/backend/api/tests/test_views.py | 15 +- api/src/backend/api/utils.py | 35 ++- api/src/backend/api/v1/views.py | 238 +++++++++++------- api/src/backend/conftest.py | 9 + api/src/backend/tasks/jobs/backfill.py | 61 +++++ api/src/backend/tasks/jobs/scan.py | 150 ++++++----- api/src/backend/tasks/tasks.py | 13 + api/src/backend/tasks/tests/test_backfill.py | 79 ++++++ api/tests/performance/benchmark.py | 4 +- api/tests/performance/requirements.txt | 1 + api/tests/performance/scenarios/findings.py | 18 +- 25 files changed, 1031 insertions(+), 190 deletions(-) create mode 100644 api/src/backend/api/migrations/0018_resource_scan_summaries.py create mode 100644 api/src/backend/api/migrations/0019_finding_denormalize_resource_fields.py create mode 100644 api/src/backend/api/migrations/0020_findings_new_performance_indexes_partitions.py create mode 100644 api/src/backend/api/migrations/0021_findings_new_performance_indexes_parent.py create mode 100644 api/src/backend/api/migrations/0022_scan_summaries_performance_indexes.py create mode 100644 api/src/backend/api/migrations/0023_resources_lookup_optimization.py create mode 100644 api/src/backend/tasks/jobs/backfill.py create mode 100644 api/src/backend/tasks/tests/test_backfill.py diff --git a/api/CHANGELOG.md b/api/CHANGELOG.md index 2dbcd57ea6..f25354e058 100644 --- a/api/CHANGELOG.md +++ b/api/CHANGELOG.md @@ -2,6 +2,14 @@ All notable changes to the **Prowler API** are documented in this file. +## [v1.8.0] (Prowler UNRELEASED) + +### Added +- Added huge improvements to `/findings/metadata` and resource related filters for findings [(#7690)](https://github.com/prowler-cloud/prowler/pull/7690). +- Added improvements to `/overviews` endpoints [(#7690)](https://github.com/prowler-cloud/prowler/pull/7690). +- Added new queue to perform backfill background tasks [(#7690)](https://github.com/prowler-cloud/prowler/pull/7690). + +--- ## [v1.7.0] (Prowler v5.6.0) diff --git a/api/docker-entrypoint.sh b/api/docker-entrypoint.sh index 7d7711a321..0dff279d08 100755 --- a/api/docker-entrypoint.sh +++ b/api/docker-entrypoint.sh @@ -28,7 +28,7 @@ start_prod_server() { start_worker() { echo "Starting the worker..." - poetry run python -m celery -A config.celery worker -l "${DJANGO_LOGGING_LEVEL:-info}" -Q celery,scans,scan-reports,deletion -E --max-tasks-per-child 1 + poetry run python -m celery -A config.celery worker -l "${DJANGO_LOGGING_LEVEL:-info}" -Q celery,scans,scan-reports,deletion,backfill -E --max-tasks-per-child 1 } start_worker_beat() { diff --git a/api/pyproject.toml b/api/pyproject.toml index 6ad2ddc9df..ce79dcbc7a 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -35,7 +35,7 @@ name = "prowler-api" package-mode = false # Needed for the SDK compatibility requires-python = ">=3.11,<3.13" -version = "1.7.0" +version = "1.8.0" [project.scripts] celery = "src.backend.config.settings.celery" diff --git a/api/src/backend/api/db_utils.py b/api/src/backend/api/db_utils.py index 63a7769c25..ca98b6b592 100644 --- a/api/src/backend/api/db_utils.py +++ b/api/src/backend/api/db_utils.py @@ -227,6 +227,77 @@ def register_enum(apps, schema_editor, enum_class): # noqa: F841 register_adapter(enum_class, enum_adapter) +def create_index_on_partitions( + apps, # noqa: F841 + schema_editor, + parent_table: str, + index_name: str, + columns: str, + method: str = "BTREE", + where: str = "", +): + """ + Create an index on every existing partition of `parent_table`. + + Args: + parent_table: The name of the root table (e.g. "findings"). + index_name: A short name for the index (will be prefixed per-partition). + columns: The parenthesized column list, e.g. "tenant_id, scan_id, status". + method: The index method—BTREE, GIN, etc. Defaults to BTREE. + where: Optional WHERE clause (without the leading "WHERE"), e.g. "status = 'FAIL'". + """ + with connection.cursor() as cursor: + cursor.execute( + """ + SELECT inhrelid::regclass::text + FROM pg_inherits + WHERE inhparent = %s::regclass + """, + [parent_table], + ) + partitions = [row[0] for row in cursor.fetchall()] + + where_sql = f" WHERE {where}" if where else "" + for partition in partitions: + idx_name = f"{partition.replace('.', '_')}_{index_name}" + sql = ( + f"CREATE INDEX CONCURRENTLY IF NOT EXISTS {idx_name} " + f"ON {partition} USING {method} ({columns})" + f"{where_sql};" + ) + schema_editor.execute(sql) + + +def drop_index_on_partitions( + apps, # noqa: F841 + schema_editor, + parent_table: str, + index_name: str, +): + """ + Drop the per-partition indexes that were created by create_index_on_partitions. + + Args: + parent_table: The name of the root table (e.g. "findings"). + index_name: The same short name used when creating them. + """ + with connection.cursor() as cursor: + cursor.execute( + """ + SELECT inhrelid::regclass::text + FROM pg_inherits + WHERE inhparent = %s::regclass + """, + [parent_table], + ) + partitions = [row[0] for row in cursor.fetchall()] + + for partition in partitions: + idx_name = f"{partition.replace('.', '_')}_{index_name}" + sql = f"DROP INDEX CONCURRENTLY IF EXISTS {idx_name};" + schema_editor.execute(sql) + + # Postgres enum definition for member role diff --git a/api/src/backend/api/filters.py b/api/src/backend/api/filters.py index fc16cf394b..6d9512f2e2 100644 --- a/api/src/backend/api/filters.py +++ b/api/src/backend/api/filters.py @@ -293,16 +293,16 @@ class FindingFilter(FilterSet): resources = UUIDInFilter(field_name="resource__id", lookup_expr="in") - region = CharFilter(field_name="resources__region") - region__in = CharInFilter(field_name="resources__region", lookup_expr="in") + region = CharFilter(method="filter_resource_region") + region__in = CharInFilter(field_name="resource_regions", lookup_expr="overlap") region__icontains = CharFilter( - field_name="resources__region", lookup_expr="icontains" + field_name="resource_regions", lookup_expr="icontains" ) - service = CharFilter(field_name="resources__service") - service__in = CharInFilter(field_name="resources__service", lookup_expr="in") + service = CharFilter(method="filter_resource_service") + service__in = CharInFilter(field_name="resource_services", lookup_expr="overlap") service__icontains = CharFilter( - field_name="resources__service", lookup_expr="icontains" + field_name="resource_services", lookup_expr="icontains" ) resource_uid = CharFilter(field_name="resources__uid") @@ -317,8 +317,8 @@ class FindingFilter(FilterSet): field_name="resources__name", lookup_expr="icontains" ) - resource_type = CharFilter(field_name="resources__type") - resource_type__in = CharInFilter(field_name="resources__type", lookup_expr="in") + resource_type = CharFilter(method="filter_resource_type") + resource_type__in = CharInFilter(field_name="resource_types", lookup_expr="overlap") resource_type__icontains = CharFilter( field_name="resources__type", lookup_expr="icontains" ) @@ -385,6 +385,15 @@ class FindingFilter(FilterSet): }, } + def filter_resource_type(self, queryset, name, value): + return queryset.filter(resource_types__contains=[value]) + + def filter_resource_region(self, queryset, name, value): + return queryset.filter(resource_regions__contains=[value]) + + def filter_resource_service(self, queryset, name, value): + return queryset.filter(resource_services__contains=[value]) + def filter_queryset(self, queryset): if not (self.data.get("scan") or self.data.get("scan__in")) and not ( self.data.get("inserted_at") diff --git a/api/src/backend/api/management/commands/findings.py b/api/src/backend/api/management/commands/findings.py index f81f8012a3..e62f8f8e8e 100644 --- a/api/src/backend/api/management/commands/findings.py +++ b/api/src/backend/api/management/commands/findings.py @@ -12,6 +12,7 @@ from api.models import ( Provider, Resource, ResourceFindingMapping, + ResourceScanSummary, Scan, StatusChoices, ) @@ -133,6 +134,7 @@ class Command(BaseCommand): region=random.choice(possible_regions), service=random.choice(possible_services), type=random.choice(possible_types), + inserted_at="2024-10-01T00:00:00Z", ) ) @@ -181,6 +183,10 @@ class Command(BaseCommand): "servicename": assigned_resource.service, "resourcetype": assigned_resource.type, }, + resource_types=[assigned_resource.type], + resource_regions=[assigned_resource.region], + resource_services=[assigned_resource.service], + inserted_at="2024-10-01T00:00:00Z", ) ) @@ -197,12 +203,22 @@ class Command(BaseCommand): # Create ResourceFindingMapping mappings = [] - for index, f in enumerate(findings): + scan_resource_cache: set[tuple] = set() + for index, finding_instance in enumerate(findings): + resource_instance = resources[findings_resources_mapping[index]] mappings.append( ResourceFindingMapping( tenant_id=tenant_id, - resource=resources[findings_resources_mapping[index]], - finding=f, + resource=resource_instance, + finding=finding_instance, + ) + ) + scan_resource_cache.add( + ( + str(resource_instance.id), + resource_instance.service, + resource_instance.region, + resource_instance.type, ) ) @@ -220,6 +236,38 @@ class Command(BaseCommand): "Resource-finding mappings created successfully.\n\n" ) ) + + with rls_transaction(tenant_id): + scan.progress = 99 + scan.save() + + self.stdout.write(self.style.WARNING("Creating finding filter values...")) + resource_scan_summaries = [ + ResourceScanSummary( + tenant_id=tenant_id, + scan_id=str(scan.id), + resource_id=resource_id, + service=service, + region=region, + resource_type=resource_type, + ) + for resource_id, service, region, resource_type in scan_resource_cache + ] + num_batches = ceil(len(resource_scan_summaries) / batch_size) + with rls_transaction(tenant_id): + for i in tqdm( + range(0, len(resource_scan_summaries), batch_size), + total=num_batches, + ): + with rls_transaction(tenant_id): + ResourceScanSummary.objects.bulk_create( + resource_scan_summaries[i : i + batch_size], + ignore_conflicts=True, + ) + + self.stdout.write( + self.style.SUCCESS("Finding filter values created successfully.\n\n") + ) except Exception as e: self.stdout.write(self.style.ERROR(f"Failed to populate test data: {e}")) scan_state = "failed" diff --git a/api/src/backend/api/migrations/0018_resource_scan_summaries.py b/api/src/backend/api/migrations/0018_resource_scan_summaries.py new file mode 100644 index 0000000000..e9e2ffbe69 --- /dev/null +++ b/api/src/backend/api/migrations/0018_resource_scan_summaries.py @@ -0,0 +1,81 @@ +# Generated by Django 5.1.7 on 2025-05-05 10:01 + +import uuid + +import django.db.models.deletion +import uuid6 +from django.db import migrations, models + +import api.rls + + +class Migration(migrations.Migration): + dependencies = [ + ("api", "0017_m365_provider"), + ] + + operations = [ + migrations.CreateModel( + name="ResourceScanSummary", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("scan_id", models.UUIDField(db_index=True, default=uuid6.uuid7)), + ("resource_id", models.UUIDField(db_index=True, default=uuid.uuid4)), + ("service", models.CharField(max_length=100)), + ("region", models.CharField(max_length=100)), + ("resource_type", models.CharField(max_length=100)), + ( + "tenant", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="api.tenant" + ), + ), + ], + options={ + "db_table": "resource_scan_summaries", + "indexes": [ + models.Index( + fields=["tenant_id", "scan_id", "service"], + name="rss_tenant_scan_svc_idx", + ), + models.Index( + fields=["tenant_id", "scan_id", "region"], + name="rss_tenant_scan_reg_idx", + ), + models.Index( + fields=["tenant_id", "scan_id", "resource_type"], + name="rss_tenant_scan_type_idx", + ), + models.Index( + fields=["tenant_id", "scan_id", "region", "service"], + name="rss_tenant_scan_reg_svc_idx", + ), + models.Index( + fields=["tenant_id", "scan_id", "service", "resource_type"], + name="rss_tenant_scan_svc_type_idx", + ), + models.Index( + fields=["tenant_id", "scan_id", "region", "resource_type"], + name="rss_tenant_scan_reg_type_idx", + ), + ], + "unique_together": {("tenant_id", "scan_id", "resource_id")}, + }, + ), + migrations.AddConstraint( + model_name="resourcescansummary", + constraint=api.rls.RowLevelSecurityConstraint( + "tenant_id", + name="rls_on_resourcescansummary", + statements=["SELECT", "INSERT", "UPDATE", "DELETE"], + ), + ), + ] diff --git a/api/src/backend/api/migrations/0019_finding_denormalize_resource_fields.py b/api/src/backend/api/migrations/0019_finding_denormalize_resource_fields.py new file mode 100644 index 0000000000..837b741223 --- /dev/null +++ b/api/src/backend/api/migrations/0019_finding_denormalize_resource_fields.py @@ -0,0 +1,42 @@ +import django.contrib.postgres.fields +import django.contrib.postgres.indexes +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("api", "0018_resource_scan_summaries"), + ] + + operations = [ + migrations.AddField( + model_name="finding", + name="resource_regions", + field=django.contrib.postgres.fields.ArrayField( + base_field=models.CharField(max_length=100), + blank=True, + null=True, + size=None, + ), + ), + migrations.AddField( + model_name="finding", + name="resource_services", + field=django.contrib.postgres.fields.ArrayField( + base_field=models.CharField(max_length=100), + blank=True, + null=True, + size=None, + ), + ), + migrations.AddField( + model_name="finding", + name="resource_types", + field=django.contrib.postgres.fields.ArrayField( + base_field=models.CharField(max_length=100), + blank=True, + null=True, + size=None, + ), + ), + ] diff --git a/api/src/backend/api/migrations/0020_findings_new_performance_indexes_partitions.py b/api/src/backend/api/migrations/0020_findings_new_performance_indexes_partitions.py new file mode 100644 index 0000000000..eef7e10b99 --- /dev/null +++ b/api/src/backend/api/migrations/0020_findings_new_performance_indexes_partitions.py @@ -0,0 +1,86 @@ +from functools import partial + +from django.db import migrations + +from api.db_utils import create_index_on_partitions, drop_index_on_partitions + + +class Migration(migrations.Migration): + atomic = False + + dependencies = [ + ("api", "0019_finding_denormalize_resource_fields"), + ] + + operations = [ + migrations.RunPython( + partial( + create_index_on_partitions, + parent_table="findings", + index_name="gin_find_service_idx", + columns="resource_services", + method="GIN", + ), + reverse_code=partial( + drop_index_on_partitions, + parent_table="findings", + index_name="gin_find_service_idx", + ), + ), + migrations.RunPython( + partial( + create_index_on_partitions, + parent_table="findings", + index_name="gin_find_region_idx", + columns="resource_regions", + method="GIN", + ), + reverse_code=partial( + drop_index_on_partitions, + parent_table="findings", + index_name="gin_find_region_idx", + ), + ), + migrations.RunPython( + partial( + create_index_on_partitions, + parent_table="findings", + index_name="gin_find_rtype_idx", + columns="resource_types", + method="GIN", + ), + reverse_code=partial( + drop_index_on_partitions, + parent_table="findings", + index_name="gin_find_rtype_idx", + ), + ), + migrations.RunPython( + partial( + drop_index_on_partitions, + parent_table="findings", + index_name="findings_uid_idx", + ), + reverse_code=partial( + create_index_on_partitions, + parent_table="findings", + index_name="findings_uid_idx", + columns="uid", + method="BTREE", + ), + ), + migrations.RunPython( + partial( + drop_index_on_partitions, + parent_table="findings", + index_name="findings_filter_idx", + ), + reverse_code=partial( + create_index_on_partitions, + parent_table="findings", + index_name="findings_filter_idx", + columns="scan_id, impact, severity, status, check_id, delta", + method="BTREE", + ), + ), + ] diff --git a/api/src/backend/api/migrations/0021_findings_new_performance_indexes_parent.py b/api/src/backend/api/migrations/0021_findings_new_performance_indexes_parent.py new file mode 100644 index 0000000000..3165af9030 --- /dev/null +++ b/api/src/backend/api/migrations/0021_findings_new_performance_indexes_parent.py @@ -0,0 +1,37 @@ +import django.contrib.postgres.indexes +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("api", "0020_findings_new_performance_indexes_partitions"), + ] + + operations = [ + migrations.AddIndex( + model_name="finding", + index=django.contrib.postgres.indexes.GinIndex( + fields=["resource_services"], name="gin_find_service_idx" + ), + ), + migrations.AddIndex( + model_name="finding", + index=django.contrib.postgres.indexes.GinIndex( + fields=["resource_regions"], name="gin_find_region_idx" + ), + ), + migrations.AddIndex( + model_name="finding", + index=django.contrib.postgres.indexes.GinIndex( + fields=["resource_types"], name="gin_find_rtype_idx" + ), + ), + migrations.RemoveIndex( + model_name="finding", + name="findings_uid_idx", + ), + migrations.RemoveIndex( + model_name="finding", + name="findings_filter_idx", + ), + ] diff --git a/api/src/backend/api/migrations/0022_scan_summaries_performance_indexes.py b/api/src/backend/api/migrations/0022_scan_summaries_performance_indexes.py new file mode 100644 index 0000000000..d56f310b8f --- /dev/null +++ b/api/src/backend/api/migrations/0022_scan_summaries_performance_indexes.py @@ -0,0 +1,38 @@ +# Generated by Django 5.1.8 on 2025-05-12 10:04 + +from django.contrib.postgres.operations import AddIndexConcurrently +from django.db import migrations, models + + +class Migration(migrations.Migration): + atomic = False + + dependencies = [ + ("api", "0021_findings_new_performance_indexes_parent"), + ("django_celery_beat", "0019_alter_periodictasks_options"), + ] + + operations = [ + AddIndexConcurrently( + model_name="scan", + index=models.Index( + condition=models.Q(("state", "completed")), + fields=["tenant_id", "provider_id", "state", "-inserted_at"], + name="scans_prov_state_ins_desc_idx", + ), + ), + AddIndexConcurrently( + model_name="scansummary", + index=models.Index( + fields=["tenant_id", "scan_id", "service"], + name="ss_tenant_scan_service_idx", + ), + ), + AddIndexConcurrently( + model_name="scansummary", + index=models.Index( + fields=["tenant_id", "scan_id", "severity"], + name="ss_tenant_scan_severity_idx", + ), + ), + ] diff --git a/api/src/backend/api/migrations/0023_resources_lookup_optimization.py b/api/src/backend/api/migrations/0023_resources_lookup_optimization.py new file mode 100644 index 0000000000..9709f17eb4 --- /dev/null +++ b/api/src/backend/api/migrations/0023_resources_lookup_optimization.py @@ -0,0 +1,28 @@ +# Generated by Django 5.1.8 on 2025-05-12 10:18 + +from django.contrib.postgres.operations import AddIndexConcurrently +from django.db import migrations, models + + +class Migration(migrations.Migration): + atomic = False + + dependencies = [ + ("api", "0022_scan_summaries_performance_indexes"), + ] + + operations = [ + AddIndexConcurrently( + model_name="resource", + index=models.Index( + fields=["tenant_id", "id"], name="resources_tenant_id_idx" + ), + ), + AddIndexConcurrently( + model_name="resource", + index=models.Index( + fields=["tenant_id", "provider_id"], + name="resources_tenant_provider_idx", + ), + ), + ] diff --git a/api/src/backend/api/models.py b/api/src/backend/api/models.py index 01f88178f0..676b3b116c 100644 --- a/api/src/backend/api/models.py +++ b/api/src/backend/api/models.py @@ -5,6 +5,7 @@ from uuid import UUID, uuid4 from cryptography.fernet import Fernet from django.conf import settings from django.contrib.auth.models import AbstractBaseUser +from django.contrib.postgres.fields import ArrayField from django.contrib.postgres.indexes import GinIndex from django.contrib.postgres.search import SearchVector, SearchVectorField from django.core.validators import MinLengthValidator @@ -447,6 +448,11 @@ class Scan(RowLevelSecurityProtectedModel): fields=["tenant_id", "provider_id", "state", "inserted_at"], name="scans_prov_state_insert_idx", ), + models.Index( + fields=["tenant_id", "provider_id", "state", "-inserted_at"], + condition=Q(state=StateChoices.COMPLETED), + name="scans_prov_state_ins_desc_idx", + ), ] class JSONAPIMeta: @@ -573,6 +579,11 @@ class Resource(RowLevelSecurityProtectedModel): name="resource_tenant_metadata_idx", ), GinIndex(fields=["text_search"], name="gin_resources_search_idx"), + models.Index(fields=["tenant_id", "id"], name="resources_tenant_id_idx"), + models.Index( + fields=["tenant_id", "provider_id"], + name="resources_tenant_provider_idx", + ), ] constraints = [ @@ -673,6 +684,21 @@ class Finding(PostgresPartitionedModel, RowLevelSecurityProtectedModel): muted = models.BooleanField(default=False, null=False) compliance = models.JSONField(default=dict, null=True, blank=True) + # Denormalize resource data for performance + resource_regions = ArrayField( + models.CharField(max_length=100), blank=True, null=True + ) + resource_services = ArrayField( + models.CharField(max_length=100), + blank=True, + null=True, + ) + resource_types = ArrayField( + models.CharField(max_length=100), + blank=True, + null=True, + ) + # Relationships scan = models.ForeignKey(to=Scan, related_name="findings", on_delete=models.CASCADE) @@ -713,18 +739,6 @@ class Finding(PostgresPartitionedModel, RowLevelSecurityProtectedModel): ] indexes = [ - models.Index(fields=["uid"], name="findings_uid_idx"), - models.Index( - fields=[ - "scan_id", - "impact", - "severity", - "status", - "check_id", - "delta", - ], - name="findings_filter_idx", - ), models.Index(fields=["tenant_id", "id"], name="findings_tenant_and_id_idx"), GinIndex(fields=["text_search"], name="gin_findings_search_idx"), models.Index(fields=["tenant_id", "scan_id"], name="find_tenant_scan_idx"), @@ -736,19 +750,38 @@ class Finding(PostgresPartitionedModel, RowLevelSecurityProtectedModel): condition=Q(delta="new"), name="find_delta_new_idx", ), + GinIndex(fields=["resource_services"], name="gin_find_service_idx"), + GinIndex(fields=["resource_regions"], name="gin_find_region_idx"), + GinIndex(fields=["resource_types"], name="gin_find_rtype_idx"), ] class JSONAPIMeta: resource_name = "findings" def add_resources(self, resources: list[Resource] | None): - # Add new relationships with the tenant_id field + if not resources: + return + + self.resource_regions = self.resource_regions or [] + self.resource_services = self.resource_services or [] + self.resource_types = self.resource_types or [] + + # Deduplication + regions = set(self.resource_regions) + services = set(self.resource_services) + types = set(self.resource_types) + for resource in resources: ResourceFindingMapping.objects.update_or_create( resource=resource, finding=self, tenant_id=self.tenant_id ) + regions.add(resource.region) + services.add(resource.service) + types.add(resource.type) - # Save the instance + self.resource_regions = list(regions) + self.resource_services = list(services) + self.resource_types = list(types) self.save() @@ -1150,7 +1183,15 @@ class ScanSummary(RowLevelSecurityProtectedModel): models.Index( fields=["tenant_id", "scan_id"], name="scan_summaries_tenant_scan_idx", - ) + ), + models.Index( + fields=["tenant_id", "scan_id", "service"], + name="ss_tenant_scan_service_idx", + ), + models.Index( + fields=["tenant_id", "scan_id", "severity"], + name="ss_tenant_scan_severity_idx", + ), ] class JSONAPIMeta: @@ -1232,3 +1273,52 @@ class IntegrationProviderRelationship(RowLevelSecurityProtectedModel): statements=["SELECT", "INSERT", "UPDATE", "DELETE"], ), ] + + +class ResourceScanSummary(RowLevelSecurityProtectedModel): + scan_id = models.UUIDField(default=uuid7, db_index=True) + resource_id = models.UUIDField(default=uuid4, db_index=True) + service = models.CharField(max_length=100) + region = models.CharField(max_length=100) + resource_type = models.CharField(max_length=100) + + class Meta: + db_table = "resource_scan_summaries" + unique_together = (("tenant_id", "scan_id", "resource_id"),) + + indexes = [ + # Single-dimension lookups: + models.Index( + fields=["tenant_id", "scan_id", "service"], + name="rss_tenant_scan_svc_idx", + ), + models.Index( + fields=["tenant_id", "scan_id", "region"], + name="rss_tenant_scan_reg_idx", + ), + models.Index( + fields=["tenant_id", "scan_id", "resource_type"], + name="rss_tenant_scan_type_idx", + ), + # Two-dimension cross-filters: + models.Index( + fields=["tenant_id", "scan_id", "region", "service"], + name="rss_tenant_scan_reg_svc_idx", + ), + models.Index( + fields=["tenant_id", "scan_id", "service", "resource_type"], + name="rss_tenant_scan_svc_type_idx", + ), + models.Index( + fields=["tenant_id", "scan_id", "region", "resource_type"], + name="rss_tenant_scan_reg_type_idx", + ), + ] + + constraints = [ + RowLevelSecurityConstraint( + field="tenant_id", + name="rls_on_%(class)s", + statements=["SELECT", "INSERT", "UPDATE", "DELETE"], + ), + ] diff --git a/api/src/backend/api/specs/v1.yaml b/api/src/backend/api/specs/v1.yaml index 962ec773c2..534a232ca8 100644 --- a/api/src/backend/api/specs/v1.yaml +++ b/api/src/backend/api/specs/v1.yaml @@ -1,7 +1,7 @@ openapi: 3.0.3 info: title: Prowler API - version: 1.7.0 + version: 1.8.0 description: |- Prowler API specification. @@ -4505,7 +4505,7 @@ paths: description: '' /api/v1/scans/{id}/compliance/{name}: get: - operationId: scan_compliance_download + operationId: scans_compliance_retrieve description: Download a specific compliance report (e.g., 'cis_1.4_aws') as a CSV file. summary: Retrieve compliance report as CSV diff --git a/api/src/backend/api/tests/test_views.py b/api/src/backend/api/tests/test_views.py index 455f9f98af..91de5775c9 100644 --- a/api/src/backend/api/tests/test_views.py +++ b/api/src/backend/api/tests/test_views.py @@ -3096,7 +3096,9 @@ class TestFindingViewSet: ) assert response.status_code == status.HTTP_404_NOT_FOUND - def test_findings_metadata_retrieve(self, authenticated_client, findings_fixture): + def test_findings_metadata_retrieve( + self, authenticated_client, findings_fixture, backfill_scan_metadata_fixture + ): finding_1, *_ = findings_fixture response = authenticated_client.get( reverse("finding-metadata"), @@ -3119,14 +3121,14 @@ class TestFindingViewSet: ) # assert data["data"]["attributes"]["tags"] == expected_tags - def test_findings_metadata_severity_retrieve( - self, authenticated_client, findings_fixture + def test_findings_metadata_resource_filter_retrieve( + self, authenticated_client, findings_fixture, backfill_scan_metadata_fixture ): finding_1, *_ = findings_fixture response = authenticated_client.get( reverse("finding-metadata"), { - "filter[severity__in]": ["low", "medium"], + "filter[region]": "eu-west-1", "filter[inserted_at]": finding_1.inserted_at.strftime("%Y-%m-%d"), }, ) @@ -4814,9 +4816,8 @@ class TestOverviewViewSet: assert response.json()["data"][0]["attributes"]["findings"]["pass"] == 2 assert response.json()["data"][0]["attributes"]["findings"]["fail"] == 1 assert response.json()["data"][0]["attributes"]["findings"]["muted"] == 1 - assert response.json()["data"][0]["attributes"]["resources"]["total"] == len( - resources_fixture - ) + # Since we rely on completed scans, there are only 2 resources now + assert response.json()["data"][0]["attributes"]["resources"]["total"] == 2 def test_overview_services_list_no_required_filters( self, authenticated_client, scan_summaries_fixture diff --git a/api/src/backend/api/utils.py b/api/src/backend/api/utils.py index 63581c266e..ce3f4feb52 100644 --- a/api/src/backend/api/utils.py +++ b/api/src/backend/api/utils.py @@ -1,11 +1,14 @@ from datetime import datetime, timezone from allauth.socialaccount.providers.oauth2.client import OAuth2Client +from django.contrib.postgres.aggregates import ArrayAgg +from django.db.models import Subquery from rest_framework.exceptions import NotFound, ValidationError from api.db_router import MainRouter from api.exceptions import InvitationTokenExpiredException -from api.models import Invitation, Provider +from api.models import Invitation, Provider, Resource +from api.v1.serializers import FindingMetadataSerializer from prowler.providers.aws.aws_provider import AwsProvider from prowler.providers.azure.azure_provider import AzureProvider from prowler.providers.common.models import Connection @@ -205,3 +208,33 @@ def validate_invitation( ) return invitation + + +# ToRemove after removing the fallback mechanism in /findings/metadata +def get_findings_metadata_no_aggregations(tenant_id: str, filtered_queryset): + filtered_ids = filtered_queryset.order_by().values("id") + + relevant_resources = Resource.all_objects.filter( + tenant_id=tenant_id, findings__id__in=Subquery(filtered_ids) + ).only("service", "region", "type") + + aggregation = relevant_resources.aggregate( + services=ArrayAgg("service", flat=True), + regions=ArrayAgg("region", flat=True), + resource_types=ArrayAgg("type", flat=True), + ) + + services = sorted(set(aggregation["services"] or [])) + regions = sorted({region for region in aggregation["regions"] or [] if region}) + resource_types = sorted(set(aggregation["resource_types"] or [])) + + result = { + "services": services, + "regions": regions, + "resource_types": resource_types, + } + + serializer = FindingMetadataSerializer(data=result) + serializer.is_valid(raise_exception=True) + + return serializer.data diff --git a/api/src/backend/api/v1/views.py b/api/src/backend/api/v1/views.py index c0a2e824fe..19b21df92e 100644 --- a/api/src/backend/api/v1/views.py +++ b/api/src/backend/api/v1/views.py @@ -1,5 +1,6 @@ import glob import os +from datetime import datetime, timedelta, timezone import sentry_sdk from allauth.socialaccount.providers.github.views import GitHubOAuth2Adapter @@ -20,6 +21,7 @@ from django.db.models import Count, Exists, F, OuterRef, Prefetch, Q, Subquery, from django.db.models.functions import Coalesce from django.http import HttpResponse from django.urls import reverse +from django.utils.dateparse import parse_date from django.utils.decorators import method_decorator from django.views.decorators.cache import cache_control from django_celery_beat.models import PeriodicTask @@ -48,6 +50,7 @@ from rest_framework_simplejwt.exceptions import InvalidToken, TokenError from tasks.beat import schedule_provider_scan from tasks.jobs.export import get_s3_client from tasks.tasks import ( + backfill_scan_resource_summaries_task, check_provider_connection_task, delete_provider_task, delete_tenant_task, @@ -87,6 +90,7 @@ from api.models import ( ProviderSecret, Resource, ResourceFindingMapping, + ResourceScanSummary, Role, RoleProviderGroupRelationship, Scan, @@ -100,7 +104,12 @@ from api.models import ( from api.pagination import ComplianceOverviewPagination from api.rbac.permissions import Permissions, get_providers, get_role from api.rls import Tenant -from api.utils import CustomOAuth2Client, validate_invitation +from api.utils import ( + CustomOAuth2Client, + get_findings_metadata_no_aggregations, + validate_invitation, +) +from api.uuid_utils import datetime_to_uuid7, uuid7_start from api.v1.serializers import ( ComplianceOverviewFullSerializer, ComplianceOverviewMetadataSerializer, @@ -249,7 +258,7 @@ class SchemaView(SpectacularAPIView): def get(self, request, *args, **kwargs): spectacular_settings.TITLE = "Prowler API" - spectacular_settings.VERSION = "1.7.0" + spectacular_settings.VERSION = "1.8.0" spectacular_settings.DESCRIPTION = ( "Prowler API specification.\n\nThis file is auto-generated." ) @@ -1780,25 +1789,103 @@ class FindingViewSet(BaseRLSViewSet): @action(detail=False, methods=["get"], url_name="metadata") def metadata(self, request): - tenant_id = self.request.tenant_id - queryset = self.get_queryset() - filtered_queryset = self.filter_queryset(queryset) + # Force filter validation + filtered_queryset = self.filter_queryset(self.get_queryset()) - filtered_ids = filtered_queryset.order_by().values("id") + tenant_id = request.tenant_id + query_params = request.query_params - relevant_resources = Resource.all_objects.filter( - tenant_id=tenant_id, findings__id__in=Subquery(filtered_ids) - ).only("service", "region", "type") + queryset = ResourceScanSummary.objects.filter(tenant_id=tenant_id) + scan_based_filters = {} - aggregation = relevant_resources.aggregate( - services=ArrayAgg("service", flat=True), - regions=ArrayAgg("region", flat=True), - resource_types=ArrayAgg("type", flat=True), + if scans := query_params.get("filter[scan__in]") or query_params.get( + "filter[scan]" + ): + queryset = queryset.filter(scan_id__in=scans.split(",")) + scan_based_filters = {"id__in": scans.split(",")} + else: + exact = query_params.get("filter[inserted_at]") + gte = query_params.get("filter[inserted_at__gte]") + lte = query_params.get("filter[inserted_at__lte]") + + date_filters = {} + if exact: + date = parse_date(exact) + datetime_start = datetime.combine( + date, datetime.min.time(), tzinfo=timezone.utc + ) + datetime_end = datetime_start + timedelta(days=1) + date_filters["scan_id__gte"] = uuid7_start( + datetime_to_uuid7(datetime_start) + ) + date_filters["scan_id__lt"] = uuid7_start( + datetime_to_uuid7(datetime_end) + ) + else: + if gte: + date_start = parse_date(gte) + datetime_start = datetime.combine( + date_start, datetime.min.time(), tzinfo=timezone.utc + ) + date_filters["scan_id__gte"] = uuid7_start( + datetime_to_uuid7(datetime_start) + ) + if lte: + date_end = parse_date(lte) + datetime_end = datetime.combine( + date_end + timedelta(days=1), + datetime.min.time(), + tzinfo=timezone.utc, + ) + date_filters["scan_id__lt"] = uuid7_start( + datetime_to_uuid7(datetime_end) + ) + + if date_filters: + queryset = queryset.filter(**date_filters) + scan_based_filters = { + key.lstrip("scan_"): value for key, value in date_filters.items() + } + + # ToRemove: Temporary fallback mechanism + if not queryset.exists(): + scan_ids = Scan.objects.filter( + tenant_id=tenant_id, **scan_based_filters + ).values_list("id", flat=True) + for scan_id in scan_ids: + backfill_scan_resource_summaries_task.apply_async( + kwargs={"tenant_id": tenant_id, "scan_id": scan_id} + ) + return Response( + get_findings_metadata_no_aggregations(tenant_id, filtered_queryset) + ) + + if service_filter := query_params.get("filter[service]") or query_params.get( + "filter[service__in]" + ): + queryset = queryset.filter(service__in=service_filter.split(",")) + if region_filter := query_params.get("filter[region]") or query_params.get( + "filter[region__in]" + ): + queryset = queryset.filter(region__in=region_filter.split(",")) + if resource_type_filter := query_params.get( + "filter[resource_type]" + ) or query_params.get("filter[resource_type__in]"): + queryset = queryset.filter( + resource_type__in=resource_type_filter.split(",") + ) + + services = list( + queryset.values_list("service", flat=True).distinct().order_by("service") + ) + regions = list( + queryset.values_list("region", flat=True).distinct().order_by("region") + ) + resource_types = list( + queryset.values_list("resource_type", flat=True) + .distinct() + .order_by("resource_type") ) - - services = sorted(set(aggregation["services"] or [])) - regions = sorted({region for region in aggregation["regions"] or [] if region}) - resource_types = sorted(set(aggregation["resource_types"] or [])) result = { "services": services, @@ -1808,7 +1895,7 @@ class FindingViewSet(BaseRLSViewSet): serializer = self.get_serializer(data=result) serializer.is_valid(raise_exception=True) - return Response(serializer.data, status=status.HTTP_200_OK) + return Response(serializer.data) @extend_schema_view( @@ -2375,8 +2462,8 @@ class OverviewViewSet(BaseRLSViewSet): def _get_filtered_queryset(model): if role.unlimited_visibility: - return model.objects.filter(tenant_id=self.request.tenant_id) - return model.objects.filter( + return model.all_objects.filter(tenant_id=self.request.tenant_id) + return model.all_objects.filter( tenant_id=self.request.tenant_id, scan__provider__in=providers ) @@ -2420,51 +2507,38 @@ class OverviewViewSet(BaseRLSViewSet): tenant_id = self.request.tenant_id latest_scan_ids = ( - Scan.objects.filter( - tenant_id=tenant_id, - state=StateChoices.COMPLETED, - ) + Scan.all_objects.filter(tenant_id=tenant_id, state=StateChoices.COMPLETED) .order_by("provider_id", "-inserted_at") .distinct("provider_id") .values_list("id", flat=True) ) - findings_aggregated = ( - ScanSummary.objects.filter(tenant_id=tenant_id, scan_id__in=latest_scan_ids) - .values("scan__provider__provider") + resource_count_queryset = ( + Resource.all_objects.filter( + tenant_id=tenant_id, + provider_id=OuterRef("scan__provider_id"), + ) + .order_by() + .values("provider_id") + .annotate(cnt=Count("id")) + .values("cnt") + ) + + overview_queryset = ( + ScanSummary.all_objects.filter( + tenant_id=tenant_id, scan_id__in=latest_scan_ids + ) + .values(provider=F("scan__provider__provider")) .annotate( findings_passed=Coalesce(Sum("_pass"), 0), findings_failed=Coalesce(Sum("fail"), 0), findings_muted=Coalesce(Sum("muted"), 0), total_findings=Coalesce(Sum("total"), 0), + total_resources=Coalesce(Subquery(resource_count_queryset), 0), ) ) - resources_aggregated = ( - Resource.objects.filter(tenant_id=tenant_id) - .values("provider__provider") - .annotate(total_resources=Count("id")) - ) - resources_dict = { - row["provider__provider"]: row["total_resources"] - for row in resources_aggregated - } - - overview = [] - for row in findings_aggregated: - provider_type = row["scan__provider__provider"] - overview.append( - { - "provider": provider_type, - "total_resources": resources_dict.get(provider_type, 0), - "total_findings": row["total_findings"], - "findings_passed": row["findings_passed"], - "findings_failed": row["findings_failed"], - "findings_muted": row["findings_muted"], - } - ) - - serializer = OverviewProviderSerializer(overview, many=True) + serializer = OverviewProviderSerializer(overview_queryset, many=True) return Response(serializer.data, status=status.HTTP_200_OK) @action(detail=False, methods=["get"], url_name="findings") @@ -2473,22 +2547,16 @@ class OverviewViewSet(BaseRLSViewSet): queryset = self.get_queryset() filtered_queryset = self.filter_queryset(queryset) - latest_scan_subquery = ( - Scan.objects.filter( - tenant_id=tenant_id, - state=StateChoices.COMPLETED, - provider_id=OuterRef("scan__provider_id"), - ) - .order_by("-inserted_at") - .values("id")[:1] + latest_scan_ids = ( + Scan.all_objects.filter(tenant_id=tenant_id, state=StateChoices.COMPLETED) + .order_by("provider_id", "-inserted_at") + .distinct("provider_id") + .values_list("id", flat=True) ) - - annotated_queryset = filtered_queryset.annotate( - latest_scan_id=Subquery(latest_scan_subquery) + filtered_queryset = filtered_queryset.filter( + tenant_id=tenant_id, scan_id__in=latest_scan_ids ) - filtered_queryset = annotated_queryset.filter(scan_id=F("latest_scan_id")) - aggregated_totals = filtered_queryset.aggregate( _pass=Sum("_pass") or 0, fail=Sum("fail") or 0, @@ -2518,22 +2586,16 @@ class OverviewViewSet(BaseRLSViewSet): queryset = self.get_queryset() filtered_queryset = self.filter_queryset(queryset) - latest_scan_subquery = ( - Scan.objects.filter( - tenant_id=tenant_id, - state=StateChoices.COMPLETED, - provider_id=OuterRef("scan__provider_id"), - ) - .order_by("-inserted_at") - .values("id")[:1] + latest_scan_ids = ( + Scan.all_objects.filter(tenant_id=tenant_id, state=StateChoices.COMPLETED) + .order_by("provider_id", "-inserted_at") + .distinct("provider_id") + .values_list("id", flat=True) ) - - annotated_queryset = filtered_queryset.annotate( - latest_scan_id=Subquery(latest_scan_subquery) + filtered_queryset = filtered_queryset.filter( + tenant_id=tenant_id, scan_id__in=latest_scan_ids ) - filtered_queryset = annotated_queryset.filter(scan_id=F("latest_scan_id")) - severity_counts = ( filtered_queryset.values("severity") .annotate(count=Sum("total")) @@ -2554,22 +2616,16 @@ class OverviewViewSet(BaseRLSViewSet): queryset = self.get_queryset() filtered_queryset = self.filter_queryset(queryset) - latest_scan_subquery = ( - Scan.objects.filter( - tenant_id=tenant_id, - state=StateChoices.COMPLETED, - provider_id=OuterRef("scan__provider_id"), - ) - .order_by("-inserted_at") - .values("id")[:1] + latest_scan_ids = ( + Scan.all_objects.filter(tenant_id=tenant_id, state=StateChoices.COMPLETED) + .order_by("provider_id", "-inserted_at") + .distinct("provider_id") + .values_list("id", flat=True) ) - - annotated_queryset = filtered_queryset.annotate( - latest_scan_id=Subquery(latest_scan_subquery) + filtered_queryset = filtered_queryset.filter( + tenant_id=tenant_id, scan_id__in=latest_scan_ids ) - filtered_queryset = annotated_queryset.filter(scan_id=F("latest_scan_id")) - services_data = ( filtered_queryset.values("service") .annotate(_pass=Sum("_pass")) diff --git a/api/src/backend/conftest.py b/api/src/backend/conftest.py index acd33ea4d1..2fba4cebb7 100644 --- a/api/src/backend/conftest.py +++ b/api/src/backend/conftest.py @@ -10,6 +10,7 @@ from django.urls import reverse from django_celery_results.models import TaskResult from rest_framework import status from rest_framework.test import APIClient +from tasks.jobs.backfill import backfill_resource_scan_summaries from api.db_utils import rls_transaction from api.models import ( @@ -920,6 +921,14 @@ def integrations_fixture(providers_fixture): return integration1, integration2 +@pytest.fixture +def backfill_scan_metadata_fixture(scans_fixture, findings_fixture): + for scan_instance in scans_fixture: + tenant_id = scan_instance.tenant_id + scan_id = scan_instance.id + backfill_resource_scan_summaries(tenant_id=tenant_id, scan_id=scan_id) + + def get_authorization_header(access_token: str) -> dict: return {"Authorization": f"Bearer {access_token}"} diff --git a/api/src/backend/tasks/jobs/backfill.py b/api/src/backend/tasks/jobs/backfill.py new file mode 100644 index 0000000000..6c83e6a69b --- /dev/null +++ b/api/src/backend/tasks/jobs/backfill.py @@ -0,0 +1,61 @@ +from api.db_utils import rls_transaction +from api.models import ( + Resource, + ResourceFindingMapping, + ResourceScanSummary, + Scan, + StateChoices, +) + + +def backfill_resource_scan_summaries(tenant_id: str, scan_id: str): + with rls_transaction(tenant_id): + if ResourceScanSummary.objects.filter( + tenant_id=tenant_id, scan_id=scan_id + ).exists(): + return {"status": "already backfilled"} + + with rls_transaction(tenant_id): + if not Scan.objects.filter( + tenant_id=tenant_id, + id=scan_id, + state__in=(StateChoices.COMPLETED, StateChoices.FAILED), + ).exists(): + return {"status": "scan is not completed"} + + resource_ids_qs = ( + ResourceFindingMapping.objects.filter( + tenant_id=tenant_id, finding__scan_id=scan_id + ) + .values_list("resource_id", flat=True) + .distinct() + ) + + resource_ids = list(resource_ids_qs) + + if not resource_ids: + return {"status": "no resources to backfill"} + + resources_qs = Resource.objects.filter( + tenant_id=tenant_id, id__in=resource_ids + ).only("id", "service", "region", "type") + + summaries = [] + for resource in resources_qs.iterator(): + summaries.append( + ResourceScanSummary( + tenant_id=tenant_id, + scan_id=scan_id, + resource_id=str(resource.id), + service=resource.service, + region=resource.region, + resource_type=resource.type, + ) + ) + + for i in range(0, len(summaries), 500): + ResourceScanSummary.objects.bulk_create( + summaries[i : i + 500], ignore_conflicts=True + ) + + return {"status": "backfilled", "inserted": len(summaries)} diff --git a/api/src/backend/tasks/jobs/scan.py b/api/src/backend/tasks/jobs/scan.py index d143d5d2c6..81fbfbb0e9 100644 --- a/api/src/backend/tasks/jobs/scan.py +++ b/api/src/backend/tasks/jobs/scan.py @@ -19,6 +19,7 @@ from api.models import ( Finding, Provider, Resource, + ResourceScanSummary, ResourceTag, Scan, ScanSummary, @@ -121,6 +122,7 @@ def perform_prowler_scan( check_status_by_region = {} exception = None unique_resources = set() + scan_resource_cache: set[tuple[str, str, str, str]] = set() start_time = time.time() with rls_transaction(tenant_id): @@ -295,6 +297,16 @@ def perform_prowler_scan( continue region_dict[finding.check_id] = finding.status.value + # Update scan resource summaries + scan_resource_cache.add( + ( + str(resource_instance.id), + resource_instance.service, + resource_instance.region, + resource_instance.type, + ) + ) + # Update scan progress with rls_transaction(tenant_id): scan_instance.progress = progress @@ -314,66 +326,90 @@ def perform_prowler_scan( scan_instance.unique_resource_count = len(unique_resources) scan_instance.save() - if exception is None: - try: - regions = prowler_provider.get_regions() - except AttributeError: - regions = set() - - compliance_template = PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE[ - provider_instance.provider - ] - compliance_overview_by_region = { - region: deepcopy(compliance_template) for region in regions - } - - for region, check_status in check_status_by_region.items(): - compliance_data = compliance_overview_by_region.setdefault( - region, deepcopy(compliance_template) - ) - for check_name, status in check_status.items(): - generate_scan_compliance( - compliance_data, - provider_instance.provider, - check_name, - status, - ) - - # Prepare compliance overview objects - compliance_overview_objects = [] - for region, compliance_data in compliance_overview_by_region.items(): - for compliance_id, compliance in compliance_data.items(): - compliance_overview_objects.append( - ComplianceOverview( - tenant_id=tenant_id, - scan=scan_instance, - region=region, - compliance_id=compliance_id, - framework=compliance["framework"], - version=compliance["version"], - description=compliance["description"], - requirements=compliance["requirements"], - requirements_passed=compliance["requirements_status"]["passed"], - requirements_failed=compliance["requirements_status"]["failed"], - requirements_manual=compliance["requirements_status"]["manual"], - total_requirements=compliance["total_requirements"], - ) - ) - try: - with rls_transaction(tenant_id): - ComplianceOverview.objects.bulk_create( - compliance_overview_objects, batch_size=100 - ) - except Exception as overview_exception: - import sentry_sdk - - sentry_sdk.capture_exception(overview_exception) - logger.error( - f"Error storing compliance overview for scan {scan_id}: {overview_exception}" - ) if exception is not None: raise exception + try: + regions = prowler_provider.get_regions() + except AttributeError: + regions = set() + + compliance_template = PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE[ + provider_instance.provider + ] + compliance_overview_by_region = { + region: deepcopy(compliance_template) for region in regions + } + + for region, check_status in check_status_by_region.items(): + compliance_data = compliance_overview_by_region.setdefault( + region, deepcopy(compliance_template) + ) + for check_name, status in check_status.items(): + generate_scan_compliance( + compliance_data, + provider_instance.provider, + check_name, + status, + ) + + # Prepare compliance overview objects + compliance_overview_objects = [] + for region, compliance_data in compliance_overview_by_region.items(): + for compliance_id, compliance in compliance_data.items(): + compliance_overview_objects.append( + ComplianceOverview( + tenant_id=tenant_id, + scan=scan_instance, + region=region, + compliance_id=compliance_id, + framework=compliance["framework"], + version=compliance["version"], + description=compliance["description"], + requirements=compliance["requirements"], + requirements_passed=compliance["requirements_status"]["passed"], + requirements_failed=compliance["requirements_status"]["failed"], + requirements_manual=compliance["requirements_status"]["manual"], + total_requirements=compliance["total_requirements"], + ) + ) + try: + with rls_transaction(tenant_id): + ComplianceOverview.objects.bulk_create( + compliance_overview_objects, batch_size=500 + ) + except Exception as overview_exception: + import sentry_sdk + + sentry_sdk.capture_exception(overview_exception) + logger.error( + f"Error storing compliance overview for scan {scan_id}: {overview_exception}" + ) + + try: + resource_scan_summaries = [ + ResourceScanSummary( + tenant_id=tenant_id, + scan_id=scan_id, + resource_id=resource_id, + service=service, + region=region, + resource_type=resource_type, + ) + for resource_id, service, region, resource_type in scan_resource_cache + ] + with rls_transaction(tenant_id): + ResourceScanSummary.objects.bulk_create( + resource_scan_summaries, batch_size=500, ignore_conflicts=True + ) + except Exception as filter_exception: + import sentry_sdk + + sentry_sdk.capture_exception(filter_exception) + logger.error( + f"Error storing filter values for scan {scan_id}: {filter_exception}" + ) + serializer = ScanTaskSerializer(instance=scan_instance) return serializer.data diff --git a/api/src/backend/tasks/tasks.py b/api/src/backend/tasks/tasks.py index fa7012e7d8..090c0c33d3 100644 --- a/api/src/backend/tasks/tasks.py +++ b/api/src/backend/tasks/tasks.py @@ -7,6 +7,7 @@ from celery.utils.log import get_task_logger from config.celery import RLSTask from config.django.base import DJANGO_FINDINGS_BATCH_SIZE, DJANGO_TMP_OUTPUT_DIRECTORY from django_celery_beat.models import PeriodicTask +from tasks.jobs.backfill import backfill_resource_scan_summaries from tasks.jobs.connection import check_provider_connection from tasks.jobs.deletion import delete_provider, delete_tenant from tasks.jobs.export import ( @@ -358,3 +359,15 @@ def generate_outputs(scan_id: str, provider_id: str, tenant_id: str): Scan.all_objects.filter(id=scan_id).update(output_location=final_location) logger.info(f"Scan outputs at {final_location}") return {"upload": did_upload} + + +@shared_task(name="backfill-scan-resource-summaries", queue="backfill") +def backfill_scan_resource_summaries_task(tenant_id: str, scan_id: str): + """ + Tries to backfill the resource scan summaries table for a given scan. + + Args: + tenant_id (str): The tenant identifier. + scan_id (str): The scan identifier. + """ + return backfill_resource_scan_summaries(tenant_id=tenant_id, scan_id=scan_id) diff --git a/api/src/backend/tasks/tests/test_backfill.py b/api/src/backend/tasks/tests/test_backfill.py new file mode 100644 index 0000000000..b436f13151 --- /dev/null +++ b/api/src/backend/tasks/tests/test_backfill.py @@ -0,0 +1,79 @@ +from uuid import uuid4 + +import pytest +from tasks.jobs.backfill import backfill_resource_scan_summaries + +from api.models import ResourceScanSummary, Scan, StateChoices + + +@pytest.mark.django_db +class TestBackfillResourceScanSummaries: + @pytest.fixture(scope="function") + def resource_scan_summary_data(self, scans_fixture): + scan = scans_fixture[0] + return ResourceScanSummary.objects.create( + tenant_id=scan.tenant_id, + scan_id=scan.id, + resource_id=str(uuid4()), + service="aws", + region="us-east-1", + resource_type="instance", + ) + + @pytest.fixture(scope="function") + def get_not_completed_scans(self, providers_fixture): + provider_id = providers_fixture[0].id + tenant_id = providers_fixture[0].tenant_id + scan_1 = Scan.objects.create( + tenant_id=tenant_id, + trigger=Scan.TriggerChoices.MANUAL, + state=StateChoices.EXECUTING, + provider_id=provider_id, + ) + scan_2 = Scan.objects.create( + tenant_id=tenant_id, + trigger=Scan.TriggerChoices.MANUAL, + state=StateChoices.AVAILABLE, + provider_id=provider_id, + ) + return scan_1, scan_2 + + def test_already_backfilled(self, resource_scan_summary_data): + tenant_id = resource_scan_summary_data.tenant_id + scan_id = resource_scan_summary_data.scan_id + + result = backfill_resource_scan_summaries(tenant_id, scan_id) + + assert result == {"status": "already backfilled"} + + def test_not_completed_scan(self, get_not_completed_scans): + for scan_instance in get_not_completed_scans: + tenant_id = scan_instance.tenant_id + scan_id = scan_instance.id + result = backfill_resource_scan_summaries(tenant_id, scan_id) + + assert result == {"status": "scan is not completed"} + + def test_successful_backfill_inserts_one_summary( + self, resources_fixture, findings_fixture + ): + tenant_id = findings_fixture[0].tenant_id + scan_id = findings_fixture[0].scan_id + + # This scan affects the first two resources + resources = resources_fixture[:2] + + result = backfill_resource_scan_summaries(tenant_id, scan_id) + assert result == {"status": "backfilled", "inserted": len(resources)} + + # Verify correct values + summaries = ResourceScanSummary.objects.filter( + tenant_id=tenant_id, scan_id=scan_id + ) + assert summaries.count() == len(resources) + for resource in resources: + summary = summaries.get(resource_id=resource.id) + assert summary.resource_id == resource.id + assert summary.service == resource.service + assert summary.region == resource.region + assert summary.resource_type == resource.type diff --git a/api/tests/performance/benchmark.py b/api/tests/performance/benchmark.py index 1df7f81645..c527a3c729 100644 --- a/api/tests/performance/benchmark.py +++ b/api/tests/performance/benchmark.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 import argparse +import os import re import subprocess import sys @@ -122,8 +123,7 @@ def main() -> None: args = parser.parse_args() metrics_dir = Path(args.metrics_dir) - if not metrics_dir.is_dir(): - sys.exit(f"Metrics directory not found: {metrics_dir}") + os.makedirs(metrics_dir, exist_ok=True) metrics_data: dict[str, pd.DataFrame] = {} for csv_file in sorted(metrics_dir.glob("*.csv")): diff --git a/api/tests/performance/requirements.txt b/api/tests/performance/requirements.txt index 139690df29..3b713953c7 100644 --- a/api/tests/performance/requirements.txt +++ b/api/tests/performance/requirements.txt @@ -1,2 +1,3 @@ locust==2.34.1 matplotlib==3.10.1 +pandas==2.2.3 diff --git a/api/tests/performance/scenarios/findings.py b/api/tests/performance/scenarios/findings.py index aef484a7ea..acd32f2497 100644 --- a/api/tests/performance/scenarios/findings.py +++ b/api/tests/performance/scenarios/findings.py @@ -180,13 +180,27 @@ class APIUser(APIUserBase): ) endpoint = ( - f"/findings?filter[{filter_name}]={filter_value}" + f"/findings/metadata?filter[{filter_name}]={filter_value}" f"&filter[inserted_at]={TARGET_INSERTED_AT}" f"&{get_sort_value(FINDINGS_UI_SORT_VALUES)}" ) self.client.get(endpoint, headers=get_auth_headers(self.token), name=name) - @task + @task(3) + def findings_metadata_resource_filter_scan_large(self): + name = "/findings/metadata?filter[resource_filter]&filter[scan_id] - 500k" + filter_name, filter_value = get_next_resource_filter( + self.available_resource_filters + ) + + endpoint = ( + f"/findings/metadata?filter[{filter_name}]={filter_value}" + f"&filter[scan]={self.l_scan_id}" + f"&{get_sort_value(FINDINGS_UI_SORT_VALUES)}" + ) + self.client.get(endpoint, headers=get_auth_headers(self.token), name=name) + + @task(2) def findings_resource_filter_large_scan_include(self): name = "/findings?filter[resource_filter][scan]&include - 500k" filter_name, filter_value = get_next_resource_filter(