feat(models): skip django check from partitioned indexes

api/CHANGELOG.md

@@ -9,6 +9,7 @@ All notable changes to the **Prowler API** are documented in this file.
 - Support GCP Service Account key [(#7824)](https://github.com/prowler-cloud/prowler/pull/7824)
 - `GET /compliance-overviews` endpoints to retrieve compliance metadata and specific requirements statuses [(#7877)](https://github.com/prowler-cloud/prowler/pull/7877)
 - Lighthouse configuration support [(#7848)](https://github.com/prowler-cloud/prowler/pull/7848)
+- Database concurrent index migration helpers [(#8045)](https://github.com/prowler-cloud/prowler/pull/8045)
 
 ### Changed
 - Reworked `GET /compliance-overviews` to return proper requirement metrics [(#7877)](https://github.com/prowler-cloud/prowler/pull/7877)

api/src/backend/api/db_utils.py

@@ -250,6 +250,7 @@ def register_enum(apps, schema_editor, enum_class):  # noqa: F841
         register_adapter(enum_class, enum_adapter)
 
 
+# DEPRECATED
 def _should_create_index_on_partition(
     partition_name: str, all_partitions: bool = False
 ) -> bool:
@@ -316,6 +317,7 @@ def _should_create_index_on_partition(
         return True
 
 
+# DEPRECATED
 def create_index_on_partitions(
     apps,  # noqa: F841
     schema_editor,
@@ -381,6 +383,7 @@ def create_index_on_partitions(
             schema_editor.execute(sql)
 
 
+# DEPRECATED
 def drop_index_on_partitions(
     apps,  # noqa: F841
     schema_editor,

api/src/backend/api/migrations/0028_findings_check_index_partitions.py

@@ -1,8 +1,6 @@
-from functools import partial
-
 from django.db import migrations
 
-from api.db_utils import create_index_on_partitions, drop_index_on_partitions
+from api.operations import CreatePartitionedIndex
 
 
 class Migration(migrations.Migration):
@@ -13,17 +11,12 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
-        migrations.RunPython(
-            partial(
-                create_index_on_partitions,
-                parent_table="findings",
-                index_name="find_tenant_scan_check_idx",
-                columns="tenant_id, scan_id, check_id",
-            ),
-            reverse_code=partial(
-                drop_index_on_partitions,
-                parent_table="findings",
-                index_name="find_tenant_scan_check_idx",
-            ),
+        CreatePartitionedIndex(
+            parent_table="findings",
+            index_name="find_tenant_scan_check_idx",
+            columns="tenant_id, scan_id, check_id",
+            method="BTREE",
+            all_partitions=False,
+            create_parent_index=True,
         )
     ]

api/src/backend/api/migrations/0029_findings_check_index_parent.py

@@ -1,4 +1,4 @@
-from django.db import migrations, models
+from django.db import migrations
 
 
 class Migration(migrations.Migration):
@@ -7,11 +7,6 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
-        migrations.AddIndex(
-            model_name="finding",
-            index=models.Index(
-                fields=["tenant_id", "scan_id", "check_id"],
-                name="find_tenant_scan_check_idx",
-            ),
-        ),
+        # No-op: Index managed manually via CratePartitionedIndex in the previous migrations
+        # Deprecated
     ]

api/src/backend/api/models.py

@@ -772,10 +772,11 @@ class Finding(PostgresPartitionedModel, RowLevelSecurityProtectedModel):
             GinIndex(fields=["resource_services"], name="gin_find_service_idx"),
             GinIndex(fields=["resource_regions"], name="gin_find_region_idx"),
             GinIndex(fields=["resource_types"], name="gin_find_rtype_idx"),
-            models.Index(
-                fields=["tenant_id", "scan_id", "check_id"],
-                name="find_tenant_scan_check_idx",
-            ),
+            # Indexes added through custom operation
+            # models.Index(
+            #     fields=["tenant_id", "scan_id", "check_id"],
+            #     name="find_tenant_scan_check_idx",
+            # ),
         ]
 
     class JSONAPIMeta:

api/src/backend/api/operations.py

@@ -0,0 +1,140 @@
+import re
+from datetime import datetime, timezone
+
+from django.db import connection
+from django.db.migrations.operations.base import Operation
+
+
+class CreatePartitionedIndex(Operation):
+    reversible = True
+
+    def __init__(
+        self,
+        parent_table: str,
+        index_name: str,
+        columns: str,
+        method: str = "BTREE",
+        where: str = "",
+        all_partitions: bool = False,
+        create_parent_index: bool = True,
+    ):
+        self.parent_table = parent_table
+        self.index_name = index_name
+        self.columns = columns
+        self.method = method
+        self.where = where
+        self.all_partitions = all_partitions
+        self.create_parent_index = create_parent_index
+
+    def state_forwards(self, app_label, state):  # noqa: F841
+        pass  # No state change
+
+    def database_forwards(self, app_label, schema_editor, from_state, to_state):  # noqa: F841
+        parent_index_name = f"{self.index_name}"
+        where_sql = f" WHERE {self.where}" if self.where else ""
+
+        with connection.cursor() as cursor:
+            cursor.execute(
+                """
+                SELECT inhrelid::regclass::text
+                FROM pg_inherits
+                WHERE inhparent = %s::regclass
+                """,
+                [self.parent_table],
+            )
+            partitions = [row[0] for row in cursor.fetchall()]
+
+        if self.create_parent_index:
+            sql = (
+                f"CREATE INDEX IF NOT EXISTS {parent_index_name} "
+                f"ON ONLY {self.parent_table} USING {self.method} ({self.columns})"
+                f"{where_sql};"
+            )
+            schema_editor.execute(sql)
+
+        for partition in partitions:
+            if self._should_create_index_on_partition(partition, self.all_partitions):
+                child_index_name = f"{partition.replace('.', '_')}_{self.index_name}"
+                create_sql = (
+                    f"CREATE INDEX CONCURRENTLY IF NOT EXISTS {child_index_name} "
+                    f"ON {partition} USING {self.method} ({self.columns})"
+                    f"{where_sql};"
+                )
+                schema_editor.execute(create_sql)
+
+                if self.create_parent_index:
+                    attach_sql = (
+                        f"ALTER INDEX {parent_index_name} "
+                        f"ATTACH PARTITION {child_index_name};"
+                    )
+                    try:
+                        schema_editor.execute(attach_sql)
+                    except Exception as e:
+                        print(
+                            f"Warning: Could not attach index {child_index_name}: {e}"
+                        )
+
+    def database_backwards(self, app_label, schema_editor, from_state, to_state):  # noqa: F841
+        if self.create_parent_index:
+            parent_index_name = self.index_name
+            drop_parent_sql = f"DROP INDEX IF EXISTS {parent_index_name};"
+            schema_editor.execute(drop_parent_sql)
+
+        with connection.cursor() as cursor:
+            cursor.execute(
+                """
+                SELECT inhrelid::regclass::text
+                FROM pg_inherits
+                WHERE inhparent = %s::regclass
+                """,
+                [self.parent_table],
+            )
+            partitions = [row[0] for row in cursor.fetchall()]
+
+        for partition in partitions:
+            idx_name = f"{partition.replace('.', '_')}_{self.index_name}"
+            drop_sql = f"DROP INDEX CONCURRENTLY IF EXISTS {idx_name};"
+            schema_editor.execute(drop_sql)
+
+    def describe(self):
+        return f"Create partitioned index {self.index_name} on {self.parent_table}"
+
+    def _should_create_index_on_partition(
+        self, partition_name: str, all_partitions: bool
+    ) -> bool:
+        if all_partitions:
+            return True
+
+        date_pattern = r"(\d{4})_([a-z]{3})$"
+        match = re.search(date_pattern, partition_name)
+        if not match:
+            return True
+
+        try:
+            year_str, month_abbr = match.groups()
+            year = int(year_str)
+            month_map = {
+                "jan": 1,
+                "feb": 2,
+                "mar": 3,
+                "apr": 4,
+                "may": 5,
+                "jun": 6,
+                "jul": 7,
+                "aug": 8,
+                "sep": 9,
+                "oct": 10,
+                "nov": 11,
+                "dec": 12,
+            }
+            month = month_map.get(month_abbr.lower())
+            if month is None:
+                return True
+
+            partition_date = datetime(year, month, 1, tzinfo=timezone.utc)
+            current_month_start = datetime.now(timezone.utc).replace(
+                day=1, hour=0, minute=0, second=0, microsecond=0
+            )
+            return partition_date >= current_month_start
+        except Exception:
+            return True

api/src/backend/api/rls.py

@@ -145,7 +145,7 @@ class BaseSecurityConstraint(models.BaseConstraint):
     """
 
     drop_sql_query = """
-        REVOKE ALL ON TABLE %(table_name) TO %(db_user)s;
+        REVOKE ALL ON TABLE %(table_name)s FROM %(db_user)s;
     """
 
     def __init__(self, name: str, statements: list | None = None) -> None: