feat(Findings): Partitioned database tables (#45)

* feat(Findings): initial findings model

* fix(Findings): add view, serializers, migration for enums

* fix(Findings): incomplete jsonb_to_tsvector wrapper

will not run as written

* fix(Findings): use Severity and Status enums from prowler SDK

* tests(Findings): add failing view tests

* fix(Finding): add resource relationship

not returning correct data from serializer, missing links

* fix(FindingSerializer): get Scan & Resource relationships to show up

* fix(FindingFilter): add more filter fields

* fix(FindingFilter): filter on provider id

* fix(FindingSerializer): return Resource in relationship

not ResourceFindingMapping

* fix(FindingModel): update migration

* fix(FindingFilter): full text search on findings

* fix(Resources): include Findings in ResourceSerializer

* fix(FindingFilter): expand text search columns

* fix(DbUtils): docstring, not comment

* fix(BaseViews): remove TODO

comment not applicable right now

* fix(Fixtures): add more findings to fixture file

and change on_delete behavior for resource_finding_mapping

* fix(Resources): rename index to match others

* fix(Findings): update Findigns RLS to allow for full CRUD

eventually we'll let users enter a manual finding
which implies INSERT, UPDATE, DELETE

* fix(Findings): use TextChoices directly for Status enum

* fix(FindingSerializer): build a set instead of a list

* consistency in fixtures

Co-authored-by: Víctor Fernández Poyatos <victor@prowler.com>

* fix(API): update v1 spec for findings

* feat(Findings): store findings in a partitioned table

* fix(Settings): change unit of partition max age

to match unit when creating

* docs(Partitions): document how to manage partitions via manage.py

* fix(Findings): add tag for spec/docs

* fix(Findings): clean up migrations

* fix(Findings): convert scan_id and inserted_at filters into finding.id filters

* fix(Findings): add default filter for today

and set default sort order

* fix(Findings): add default filter for today

and set default sort order

* fix(Findings): update fixtures so datetime matches id

* fix(Findings): partition the ResourceFindingMapping table to match Findings

* docs(Partitions): document postgres config values more

* docs(UUIDUtils): do not use raw query strigns (typo)

* docs(Partitions): change unit in comment description

* fix(Findings): change resource_name & tags to be Finding

* docs(Partitions): change unit in partitions settings docstring

* fix(Findings): remove conflicting logic & filters

* chore: apply suggested changes

* chore: optimize imports

---------

Co-authored-by: Víctor Fernández Poyatos <victor@prowler.com>

src/backend/api/tests/test_uuid_utils.py

@@ -0,0 +1,128 @@
+from datetime import datetime, timezone
+from uuid import uuid4
+
+import pytest
+from dateutil.relativedelta import relativedelta
+from uuid6 import UUID
+
+from api.uuid_utils import (
+    datetime_to_uuid7,
+    datetime_from_uuid7,
+    uuid7_start,
+    uuid7_end,
+    uuid7_range,
+    parse_params_to_uuid7,
+)
+
+
+def test_parse_params_to_uuid7():
+    uuid = parse_params_to_uuid7("0191dff4-7a78-7031-8814-4fb51bf2bd5b")
+    assert isinstance(uuid, UUID)
+
+    uuids = parse_params_to_uuid7(
+        ["0191dff4-7a78-7031-8814-4fb51bf2bd5b", "01856aa0-c800-7b8f-bfa0-46a393f14d50"]
+    )
+    assert isinstance(uuids, list)
+    assert len(uuids) == 2
+
+    uuids = parse_params_to_uuid7(
+        "0191dff4-7a78-7031-8814-4fb51bf2bd5b,01856aa0-c800-7b8f-bfa0-46a393f14d50"
+    )
+    assert isinstance(uuids, list)
+    assert len(uuids) == 2
+
+
+@pytest.mark.parametrize(
+    "input_datetime",
+    [
+        datetime(2024, 9, 11, 7, 20, 27, tzinfo=timezone.utc),
+        datetime(2023, 1, 1, 0, 0, 0, tzinfo=timezone.utc),
+    ],
+)
+def test_timestamp_to_uuid7(input_datetime):
+    uuid7 = datetime_to_uuid7(input_datetime)
+    # assert instance is all we can do without reversing the math in the test
+    assert isinstance(uuid7, UUID)
+    assert uuid7.version == 7
+    assert uuid7.time == int(input_datetime.timestamp() * 1000)
+
+
+@pytest.mark.parametrize(
+    "uuid7, expected_datetime",
+    [
+        (
+            "0191dff4-7a78-7031-8814-4fb51bf2bd5b",
+            datetime(2024, 9, 11, 7, 20, 27, tzinfo=timezone.utc),
+        ),
+        (
+            "01856aa0-c800-7b8f-bfa0-46a393f14d50",
+            datetime(2023, 1, 1, 0, 0, 0, tzinfo=timezone.utc),
+        ),
+    ],
+)
+def test_datetime_from_uuid7(uuid7, expected_datetime):
+    extracted_datetime = datetime_from_uuid7(uuid7=UUID(uuid7))
+    assert extracted_datetime == expected_datetime
+
+
+def test_extract_timestamp_from_uuid4_invalid():
+    with pytest.raises(ValueError):
+        datetime_from_uuid7(uuid4())
+
+
+def test_uuid7_start():
+    dt = datetime.now(timezone.utc)
+    uuid = datetime_to_uuid7(dt)
+    dt = dt.replace(hour=0, minute=0, second=0, microsecond=0)
+    start = uuid7_start(uuid)
+    expected_ts = (
+        int(dt.replace(hour=0, minute=0, second=0, microsecond=0).timestamp()) * 1000
+    )
+    assert start.time == expected_ts
+
+
+@pytest.mark.parametrize(
+    "days_offset",
+    [0, 1, 10, 30, 60],
+)
+def test_uuid7_end(days_offset):
+    dt = datetime.now(timezone.utc)
+    uuid = datetime_to_uuid7(dt)
+
+    end = uuid7_end(uuid, days_offset)
+
+    dt = dt.replace(hour=0, minute=0, second=0, microsecond=0)
+    dt = dt + relativedelta(days=days_offset + 1, microseconds=-1)
+    expected_ts = int(dt.timestamp()) * 1000
+
+    assert end.time == expected_ts
+
+
+def test_uuid7_range():
+    uuids = [
+        datetime_to_uuid7(datetime.now(timezone.utc)),
+        datetime_to_uuid7(datetime.now(timezone.utc).replace(year=2023)),
+        datetime_to_uuid7(datetime.now(timezone.utc).replace(year=2024)),
+        datetime_to_uuid7(datetime.now(timezone.utc).replace(year=2025)),
+    ]
+    expected_start = (
+        int(
+            datetime.fromtimestamp(uuids[1].time / 1000, tz=timezone.utc)
+            .replace(hour=0, minute=0, second=0, microsecond=0)
+            .timestamp()
+        )
+        * 1000
+    )
+    expected_end = (
+        int(
+            datetime.fromtimestamp(uuids[-1].time / 1000, tz=timezone.utc)
+            .replace(hour=23, minute=59, second=59, microsecond=999999)
+            .timestamp()
+        )
+        * 1000
+    )
+
+    start, end = uuid7_range(uuids)
+
+    assert start.time == expected_start
+    assert end.time == expected_end

src/backend/api/tests/test_views.py

@@ -1390,7 +1390,10 @@ class TestFindingViewSet:
                 ("service", "ec2", 1),
                 ("service.in", "ec2,s3", 2),
                 ("service.icontains", "ec", 1),
+                ("inserted_at", "2024-01-01", 0),
+                ("inserted_at.date", "2024-01-01", 0),
                 ("inserted_at.gte", "2024-01-01", 2),
+                ("inserted_at.lte", "2024-12-31", 2),
                 ("updated_at.lte", "2024-01-01", 0),
                 ("resource_type.icontains", "prowler", 2),
                 # full text search on finding
@@ -1421,6 +1424,35 @@ class TestFindingViewSet:
         assert response.status_code == status.HTTP_200_OK
         assert len(response.json()["data"]) == expected_count
 
+    def test_finding_filter_by_scan_id(
+        self, authenticated_client, findings_fixture, tenant_header
+    ):
+        response = authenticated_client.get(
+            reverse("finding-list"),
+            {
+                "filter[scan]": findings_fixture[0].scan.id,
+            },
+            headers=tenant_header,
+        )
+        assert response.status_code == status.HTTP_200_OK
+        assert len(response.json()["data"]) == 2
+
+    def test_finding_filter_by_scan_id_in(
+        self, authenticated_client, findings_fixture, tenant_header
+    ):
+        response = authenticated_client.get(
+            reverse("finding-list"),
+            {
+                "filter[scan.in]": [
+                    findings_fixture[0].scan.id,
+                    findings_fixture[1].scan.id,
+                ]
+            },
+            headers=tenant_header,
+        )
+        assert response.status_code == status.HTTP_200_OK
+        assert len(response.json()["data"]) == 2
+
     def test_finding_filter_by_provider(
         self, authenticated_client, findings_fixture, tenant_header
     ):