feat(Findings): Partitioned database tables (#45)

* feat(Findings): initial findings model

* fix(Findings): add view, serializers, migration for enums

* fix(Findings): incomplete jsonb_to_tsvector wrapper

will not run as written

* fix(Findings): use Severity and Status enums from prowler SDK

* tests(Findings): add failing view tests

* fix(Finding): add resource relationship

not returning correct data from serializer, missing links

* fix(FindingSerializer): get Scan & Resource relationships to show up

* fix(FindingFilter): add more filter fields

* fix(FindingFilter): filter on provider id

* fix(FindingSerializer): return Resource in relationship

not ResourceFindingMapping

* fix(FindingModel): update migration

* fix(FindingFilter): full text search on findings

* fix(Resources): include Findings in ResourceSerializer

* fix(FindingFilter): expand text search columns

* fix(DbUtils): docstring, not comment

* fix(BaseViews): remove TODO

comment not applicable right now

* fix(Fixtures): add more findings to fixture file

and change on_delete behavior for resource_finding_mapping

* fix(Resources): rename index to match others

* fix(Findings): update Findigns RLS to allow for full CRUD

eventually we'll let users enter a manual finding
which implies INSERT, UPDATE, DELETE

* fix(Findings): use TextChoices directly for Status enum

* fix(FindingSerializer): build a set instead of a list

* consistency in fixtures

Co-authored-by: Víctor Fernández Poyatos <victor@prowler.com>

* fix(API): update v1 spec for findings

* feat(Findings): store findings in a partitioned table

* fix(Settings): change unit of partition max age

to match unit when creating

* docs(Partitions): document how to manage partitions via manage.py

* fix(Findings): add tag for spec/docs

* fix(Findings): clean up migrations

* fix(Findings): convert scan_id and inserted_at filters into finding.id filters

* fix(Findings): add default filter for today

and set default sort order

* fix(Findings): add default filter for today

and set default sort order

* fix(Findings): update fixtures so datetime matches id

* fix(Findings): partition the ResourceFindingMapping table to match Findings

* docs(Partitions): document postgres config values more

* docs(UUIDUtils): do not use raw query strigns (typo)

* docs(Partitions): change unit in comment description

* fix(Findings): change resource_name & tags to be Finding

* docs(Partitions): change unit in partitions settings docstring

* fix(Findings): remove conflicting logic & filters

* chore: apply suggested changes

* chore: optimize imports

---------

Co-authored-by: Víctor Fernández Poyatos <victor@prowler.com>

src/backend/api/models.py

@@ -7,13 +7,12 @@ from django.contrib.postgres.indexes import GinIndex
 from django.contrib.postgres.search import SearchVector, SearchVectorField
 from django.core.validators import MinLengthValidator
 from django.db import models
-
 from django.utils.translation import gettext_lazy as _
 from django_celery_results.models import TaskResult
-
-from uuid6 import uuid7
-
 from prowler.lib.outputs.finding import Severity
+from psqlextra.models import PostgresPartitionedModel
+from psqlextra.types import PostgresPartitioningMethod
+from uuid6 import uuid7
 
 from api.db_utils import (
     enum_to_choices,
@@ -26,14 +25,12 @@ from api.db_utils import (
     CustomUserManager,
 )
 from api.exceptions import ModelValidationError
-
 from api.rls import (
     RowLevelSecurityProtectedModel,
     RowLevelSecurityConstraint,
     BaseSecurityConstraint,
 )
 
-
 # Convert Prowler Severity enum to Django TextChoices
 SeverityChoices = enum_to_choices(Severity)
 
@@ -45,8 +42,8 @@ class StatusChoices(models.TextChoices):
     However it adds another state, MUTED, which is not in the CLI.
     """
 
-    PASS = "PASS", _("Pass")
     FAIL = "FAIL", _("Fail")
+    PASS = "PASS", _("Pass")
     MANUAL = "MANUAL", _("Manual")
     MUTED = "MUTED", _("Muted")
 
@@ -411,7 +408,19 @@ class ResourceTagMapping(RowLevelSecurityProtectedModel):
         ]
 
 
-class Finding(RowLevelSecurityProtectedModel):
+class Finding(PostgresPartitionedModel, RowLevelSecurityProtectedModel):
+    """
+    Defines the Finding model.
+
+    Findings uses a partitioned table to store findings. The partitions are created based on the UUIDv7 `id` field.
+
+    Note when creating migrations, you must use `python manage.py pgmakemigrations` to create the migrations.
+    """
+
+    class PartitioningMeta:
+        method = PostgresPartitioningMethod.RANGE
+        key = ["id"]
+
     class DeltaChoices(models.TextChoices):
         NEW = "new", _("New")
         CHANGED = "changed", _("Changed")
@@ -462,7 +471,7 @@ class Finding(RowLevelSecurityProtectedModel):
         editable=False,
     )
 
-    class Meta:
+    class Meta(RowLevelSecurityProtectedModel.Meta):
         db_table = "findings"
 
         constraints = [
@@ -471,6 +480,12 @@ class Finding(RowLevelSecurityProtectedModel):
                 name="rls_on_%(class)s",
                 statements=["SELECT", "UPDATE", "INSERT", "DELETE"],
             ),
+            RowLevelSecurityConstraint(
+                field="tenant_id",
+                name="rls_on_%(class)s_default",
+                partition_name="default",
+                statements=["SELECT", "UPDATE", "INSERT", "DELETE"],
+            ),
         ]
 
         indexes = [
@@ -499,13 +514,25 @@ class Finding(RowLevelSecurityProtectedModel):
         self.save()
 
 
-class ResourceFindingMapping(RowLevelSecurityProtectedModel):
+class ResourceFindingMapping(PostgresPartitionedModel, RowLevelSecurityProtectedModel):
+    """
+    Defines the ResourceFindingMapping model.
+
+    ResourceFindingMapping is used to map a Finding to a Resource.
+
+    It follows the same partitioning strategy as the Finding model.
+    """
+
     # NOTE that we don't really need a primary key here,
     #      but everything is easier with django if we do
     id = models.UUIDField(primary_key=True, default=uuid4, editable=False)
     resource = models.ForeignKey(Resource, on_delete=models.CASCADE)
     finding = models.ForeignKey(Finding, on_delete=models.CASCADE)
 
+    class PartitioningMeta:
+        method = PostgresPartitioningMethod.RANGE
+        key = ["finding_id"]
+
     class Meta(RowLevelSecurityProtectedModel.Meta):
         db_table = "resource_finding_mappings"