diff --git a/api/CHANGELOG.md b/api/CHANGELOG.md index b037366211..a2103d8a21 100644 --- a/api/CHANGELOG.md +++ b/api/CHANGELOG.md @@ -4,6 +4,14 @@ All notable changes to the **Prowler API** are documented in this file. ## [1.27.1] (Prowler v5.26.1) +### 🚀 Added + +- GIN index on `findings(categories, resource_services, resource_regions, resource_types)` to speed up `/api/v1/finding-groups` array filters [(#11001)](https://github.com/prowler-cloud/prowler/pull/11001) + +### 🔄 Changed + +- Remove orphaned `gin_resources_search_idx` declaration from `Resource.Meta.indexes` (DB index dropped in `0072_drop_unused_indexes`) [(#11001)](https://github.com/prowler-cloud/prowler/pull/11001) + ### 🐞 Fixed - `POST /api/v1/scans` was intermittently failing with `Scan matching query does not exist` in the `scan-perform` worker; the Celery task is now published via `transaction.on_commit` so the worker cannot read the Scan before the dispatch-wide transaction commits [(#11122)](https://github.com/prowler-cloud/prowler/pull/11122) @@ -15,6 +23,9 @@ All notable changes to the **Prowler API** are documented in this file. ### 🚀 Added - `scan-reset-ephemeral-resources` post-scan task zeroes `failed_findings_count` for resources missing from the latest full-scope scan, keeping ephemeral resources from polluting the Resources page sort [(#10929)](https://github.com/prowler-cloud/prowler/pull/10929) + +### 🔄 Changed + - ASD Essential Eight (AWS) compliance framework support [(#10982)](https://github.com/prowler-cloud/prowler/pull/10982) ### 🔐 Security diff --git a/api/src/backend/api/migrations/0091_findings_arrays_gin_index_partitions.py b/api/src/backend/api/migrations/0091_findings_arrays_gin_index_partitions.py new file mode 100644 index 0000000000..fc5716f828 --- /dev/null +++ b/api/src/backend/api/migrations/0091_findings_arrays_gin_index_partitions.py @@ -0,0 +1,31 @@ +from functools import partial + +from django.db import migrations + +from api.db_utils import create_index_on_partitions, drop_index_on_partitions + + +class Migration(migrations.Migration): + atomic = False + + dependencies = [ + ("api", "0090_attack_paths_cleanup_priority"), + ] + + operations = [ + migrations.RunPython( + partial( + create_index_on_partitions, + parent_table="findings", + index_name="gin_find_arrays_idx", + columns="categories, resource_services, resource_regions, resource_types", + method="GIN", + all_partitions=True, + ), + reverse_code=partial( + drop_index_on_partitions, + parent_table="findings", + index_name="gin_find_arrays_idx", + ), + ) + ] diff --git a/api/src/backend/api/migrations/0092_findings_arrays_gin_index_parent.py b/api/src/backend/api/migrations/0092_findings_arrays_gin_index_parent.py new file mode 100644 index 0000000000..b8baf448d6 --- /dev/null +++ b/api/src/backend/api/migrations/0092_findings_arrays_gin_index_parent.py @@ -0,0 +1,58 @@ +import django.contrib.postgres.indexes +from django.db import migrations + +INDEX_NAME = "gin_find_arrays_idx" +PARENT_TABLE = "findings" + + +def create_parent_and_attach(apps, schema_editor): + with schema_editor.connection.cursor() as cursor: + cursor.execute( + f"CREATE INDEX {INDEX_NAME} ON ONLY {PARENT_TABLE} " + f"USING gin (categories, resource_services, resource_regions, resource_types)" + ) + cursor.execute( + "SELECT inhrelid::regclass::text " + "FROM pg_inherits " + "WHERE inhparent = %s::regclass", + [PARENT_TABLE], + ) + for (partition,) in cursor.fetchall(): + child_idx = f"{partition.replace('.', '_')}_{INDEX_NAME}" + cursor.execute(f"ALTER INDEX {INDEX_NAME} ATTACH PARTITION {child_idx}") + + +def drop_parent_index(apps, schema_editor): + with schema_editor.connection.cursor() as cursor: + cursor.execute(f"DROP INDEX IF EXISTS {INDEX_NAME}") + + +class Migration(migrations.Migration): + dependencies = [ + ("api", "0091_findings_arrays_gin_index_partitions"), + ] + + operations = [ + migrations.SeparateDatabaseAndState( + state_operations=[ + migrations.AddIndex( + model_name="finding", + index=django.contrib.postgres.indexes.GinIndex( + fields=[ + "categories", + "resource_services", + "resource_regions", + "resource_types", + ], + name=INDEX_NAME, + ), + ), + ], + database_operations=[ + migrations.RunPython( + create_parent_and_attach, + reverse_code=drop_parent_index, + ), + ], + ), + ] diff --git a/api/src/backend/api/models.py b/api/src/backend/api/models.py index e7195cff5f..17d38e82f0 100644 --- a/api/src/backend/api/models.py +++ b/api/src/backend/api/models.py @@ -946,7 +946,6 @@ class Resource(RowLevelSecurityProtectedModel): OpClass(Upper("name"), name="gin_trgm_ops"), name="res_name_trgm_idx", ), - GinIndex(fields=["text_search"], name="gin_resources_search_idx"), models.Index(fields=["tenant_id", "id"], name="resources_tenant_id_idx"), models.Index( fields=["tenant_id", "provider_id"], @@ -1152,6 +1151,15 @@ class Finding(PostgresPartitionedModel, RowLevelSecurityProtectedModel): fields=["tenant_id", "scan_id", "check_id"], name="find_tenant_scan_check_idx", ), + GinIndex( + fields=[ + "categories", + "resource_services", + "resource_regions", + "resource_types", + ], + name="gin_find_arrays_idx", + ), ] class JSONAPIMeta: