From 4ca95b08e29dc87d620dff4dae5195a3406691ce Mon Sep 17 00:00:00 2001 From: Jon Young Date: Mon, 23 Sep 2024 05:39:03 -0400 Subject: [PATCH] feat(Findings): Partitioned database tables (#45) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(Findings): initial findings model * fix(Findings): add view, serializers, migration for enums * fix(Findings): incomplete jsonb_to_tsvector wrapper will not run as written * fix(Findings): use Severity and Status enums from prowler SDK * tests(Findings): add failing view tests * fix(Finding): add resource relationship not returning correct data from serializer, missing links * fix(FindingSerializer): get Scan & Resource relationships to show up * fix(FindingFilter): add more filter fields * fix(FindingFilter): filter on provider id * fix(FindingSerializer): return Resource in relationship not ResourceFindingMapping * fix(FindingModel): update migration * fix(FindingFilter): full text search on findings * fix(Resources): include Findings in ResourceSerializer * fix(FindingFilter): expand text search columns * fix(DbUtils): docstring, not comment * fix(BaseViews): remove TODO comment not applicable right now * fix(Fixtures): add more findings to fixture file and change on_delete behavior for resource_finding_mapping * fix(Resources): rename index to match others * fix(Findings): update Findigns RLS to allow for full CRUD eventually we'll let users enter a manual finding which implies INSERT, UPDATE, DELETE * fix(Findings): use TextChoices directly for Status enum * fix(FindingSerializer): build a set instead of a list * consistency in fixtures Co-authored-by: Víctor Fernández Poyatos * fix(API): update v1 spec for findings * feat(Findings): store findings in a partitioned table * fix(Settings): change unit of partition max age to match unit when creating * docs(Partitions): document how to manage partitions via manage.py * fix(Findings): add tag for spec/docs * fix(Findings): clean up migrations * fix(Findings): convert scan_id and inserted_at filters into finding.id filters * fix(Findings): add default filter for today and set default sort order * fix(Findings): add default filter for today and set default sort order * fix(Findings): update fixtures so datetime matches id * fix(Findings): partition the ResourceFindingMapping table to match Findings * docs(Partitions): document postgres config values more * docs(UUIDUtils): do not use raw query strigns (typo) * docs(Partitions): change unit in comment description * fix(Findings): change resource_name & tags to be Finding * docs(Partitions): change unit in partitions settings docstring * fix(Findings): remove conflicting logic & filters * chore: apply suggested changes * chore: optimize imports --------- Co-authored-by: Víctor Fernández Poyatos --- docs/partitions.md | 64 ++++++ poetry.lock | 23 ++- pyproject.toml | 1 + src/backend/api/db_utils.py | 15 +- src/backend/api/filters.py | 84 +++++++- .../{3_dev_scans.json => 4_dev_scans.json} | 0 ..._dev_findings.json => 5_dev_findings.json} | 32 +-- src/backend/api/migrations/0001_initial.py | 85 ++++++-- src/backend/api/models.py | 47 ++++- src/backend/api/partitions.py | 192 ++++++++++++++++++ src/backend/api/rls.py | 26 ++- src/backend/api/specs/v1.yaml | 26 +-- src/backend/api/tests/test_uuid_utils.py | 128 ++++++++++++ src/backend/api/tests/test_views.py | 32 +++ src/backend/api/uuid_utils.py | 96 +++++++++ src/backend/api/v1/serializers.py | 7 +- src/backend/api/v1/views.py | 18 +- src/backend/config/django/base.py | 2 + src/backend/config/django/devel.py | 4 +- src/backend/config/django/production.py | 2 +- src/backend/config/django/testing.py | 2 +- src/backend/config/settings/partitions.py | 16 ++ src/backend/conftest.py | 8 +- 23 files changed, 816 insertions(+), 94 deletions(-) create mode 100644 docs/partitions.md rename src/backend/api/fixtures/{3_dev_scans.json => 4_dev_scans.json} (100%) rename src/backend/api/fixtures/{4_dev_findings.json => 5_dev_findings.json} (92%) create mode 100644 src/backend/api/partitions.py create mode 100644 src/backend/api/tests/test_uuid_utils.py create mode 100644 src/backend/api/uuid_utils.py create mode 100644 src/backend/config/settings/partitions.py diff --git a/docs/partitions.md b/docs/partitions.md new file mode 100644 index 0000000000..31d3470469 --- /dev/null +++ b/docs/partitions.md @@ -0,0 +1,64 @@ +# Partitions + +## Overview + +Partitions are used to split the data in a table into smaller chunks, allowing for more efficient querying and storage. + +The Prowler API uses partitions to store findings. The partitions are created based on the UUIDv7 `id` field. + +You can use the Prowler API without ever creating additional partitions. This documentation is only relevant if you want to manage partitions to gain additional query performance. + +### Required Postgres Configuration + +There are 3 configuration options that need to be set in the `postgres.conf` file to get the most performance out of the partitioning: + +- `enable_partition_pruning = on` (default is on) +- `enable_partitionwise_join = on` (default is off) +- `enable_partitionwise_aggregate = on` (default is off) + +For more information on these options, see the [Postgres documentation](https://www.postgresql.org/docs/current/runtime-config-query.html). + +## Partitioning Strategy + +The partitioning strategy is defined in the `api.partitions` module. The strategy is responsible for creating and deleting partitions based on the provided configuration. + +## Managing Partitions + +The application will run without any extra work on your part. If you want to add or delete partitions, you can use the following commands: + +To manage the partitions, run `python manage.py pgpartition --using admin` + +This command will generate a list of partitions to create and delete based on the provided configuration. + +By default, the command will prompt you to accept the changes before applying them. + +```shell +Finding: + + 2024_oct_18 + name: 2024_oct_18 + from_values: 01929cec-8800-7988-a49d-54cbb64b2d3f + to_values: 0193376b-4c18-7ff7-99d5-95b8479ce39f + size_unit: days + size_value: 30 + + 2024_nov_17 + name: 2024_nov_17 + from_values: 0193376b-5000-7414-bbae-b1da382332f9 + to_values: 0193d1ea-1418-7b00-93fa-c4e06bf36bbe + size_unit: days + size_value: 30 + +0 partitions will be deleted +2 partitions will be created +``` + +If you choose to apply the partitions, tables will be generated with the following format: `___`. The date in the table name shows the first date of the partition. + +For more info on the partitioning manager, see https://github.com/SectorLabs/django-postgres-extra + +### Changing the Partitioning Parameters + +There are 3 environment variables that can be used to change the partitioning parameters: + +- `FINDINGS_TABLE_PARTITION_DAYS`: Set the days for each partition. Setting the partition days to 30 will create partitions with a size of 1 month. +- `FINDINGS_TABLE_PARTITION_COUNT`: Set the number of partitions to create +- `FINDINGS_TABLE_PARTITION_MAX_AGE_DAYS`: Set the number of days to keep partitions before deleting them. Setting this to `None` will keep partitions indefinitely. diff --git a/poetry.lock b/poetry.lock index 66aeeb9947..3cb0204841 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1406,6 +1406,27 @@ files = [ [package.dependencies] django = {version = ">=4.0,<6.0", markers = "python_version >= \"3.10\""} +[[package]] +name = "django-postgres-extra" +version = "2.0.9rc11" +description = "Bringing all of PostgreSQL's awesomeness to Django." +optional = false +python-versions = ">=3.6" +files = [ + {file = "django_postgres_extra-2.0.9rc11-py3-none-any.whl", hash = "sha256:23fb08261963bebf6560a2bb248b2c404f9c5f650734472ff074977952efbd56"}, + {file = "django_postgres_extra-2.0.9rc11.tar.gz", hash = "sha256:a7738125c84d133d1dbbdeab66dd14e897e01284b5e1e02e588c84891c8b8ded"}, +] + +[package.dependencies] +Django = ">=2.0,<6.0" +python-dateutil = ">=2.8.0,<=3.0.0" + +[package.extras] +analysis = ["autoflake (==1.4)", "autopep8 (==1.6.0)", "black (==22.3.0)", "django-stubs (==1.16.0)", "django-stubs (==1.9.0)", "docformatter (==1.4)", "flake8 (==4.0.1)", "isort (==5.10.0)", "mypy (==0.971)", "mypy (==1.2.0)", "types-dj-database-url (==1.3.0.0)", "types-psycopg2 (==2.9.21.9)", "types-python-dateutil (==2.8.19.12)", "typing-extensions (==4.1.0)", "typing-extensions (==4.5.0)"] +docs = ["Sphinx (==2.2.0)", "docutils (<0.18)", "sphinx-rtd-theme (==0.4.3)"] +publish = ["build (==0.7.0)", "twine (==3.7.1)"] +test = ["coveralls (==3.3.0)", "dj-database-url (==0.5.0)", "freezegun (==1.1.0)", "psycopg2 (>=2.8.4,<3.0.0)", "pytest (==6.2.5)", "pytest-benchmark (==3.4.1)", "pytest-cov (==3.0.0)", "pytest-django (==4.4.0)", "pytest-freezegun (==0.4.2)", "pytest-lazy-fixture (==0.6.3)", "snapshottest (==0.6.0)", "tox (==3.24.4)"] + [[package]] name = "djangorestframework" version = "3.15.2" @@ -4794,4 +4815,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = ">=3.11,<3.13" -content-hash = "bb69d8edb8b1e71769eb678b4104f5e7bd82ba548b94084e97a03c2e1215cda7" +content-hash = "b92ed16fc637dd09de05de5d76460cc2b47c6d93855b4fecbaaf64f3d9fcfdcf" diff --git a/pyproject.toml b/pyproject.toml index 9fcda05053..b7cdd3f4f6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,7 @@ django-cors-headers = "4.4.0" django-environ = "0.11.2" django-filter = "24.2" django-guid = "3.5.0" +django-postgres-extra = "^2.0.8" djangorestframework = "3.15.2" djangorestframework-jsonapi = "7.0.2" djangorestframework-simplejwt = "^5.3.1" diff --git a/src/backend/api/db_utils.py b/src/backend/api/db_utils.py index 92f6e8d551..935f2ffc65 100644 --- a/src/backend/api/db_utils.py +++ b/src/backend/api/db_utils.py @@ -3,9 +3,6 @@ from contextlib import contextmanager from django.conf import settings from django.contrib.auth.models import BaseUserManager from django.db import models -from django.contrib.postgres.search import SearchVector -from django.db.models import TextField - from psycopg2 import connect as psycopg2_connect from psycopg2.extensions import new_type, register_type, register_adapter, AsIs @@ -34,6 +31,7 @@ def psycopg_connection(database_alias: str): user=admin_db["USER"], password=admin_db["PASSWORD"], host=admin_db["HOST"], + port=admin_db["PORT"], ) yield psycopg2_connection finally: @@ -61,17 +59,6 @@ def enum_to_choices(enum_class): return [(item.value, item.name.replace("_", " ").title()) for item in enum_class] -# jsonb_to_tsvector - - -class JsonbToTsvector(SearchVector): - function = "jsonb_to_tsvector" - output_field = TextField() - - def __init__(self, expression): - super().__init__(expression) - - # Postgres Enums diff --git a/src/backend/api/filters.py b/src/backend/api/filters.py index 187b6968d1..ccc7e51726 100644 --- a/src/backend/api/filters.py +++ b/src/backend/api/filters.py @@ -1,5 +1,7 @@ +from datetime import date, datetime, timezone from uuid import UUID +from django.conf import settings from django.db.models import Q from django_filters.rest_framework import ( FilterSet, @@ -24,6 +26,13 @@ from api.models import ( StatusChoices, ) from api.rls import Tenant +from api.uuid_utils import ( + datetime_to_uuid7, + uuid7_start, + uuid7_end, + uuid7_range, + parse_params_to_uuid7, +) from api.v1.serializers import TaskBase @@ -307,7 +316,6 @@ class FindingFilter(ProviderRelationshipFilterSet): value = [value] return queryset.filter(scan__provider__id__in=value) - inserted_at = DateFilter(field_name="inserted_at", lookup_expr="date") updated_at = DateFilter(field_name="updated_at", lookup_expr="date") delta = CharFilter(method="filter_delta") @@ -341,6 +349,14 @@ class FindingFilter(ProviderRelationshipFilterSet): resource_type__in = CharFilter(method="filter_resource_type_in") resource_type__icontains = CharFilter(method="filter_resource_type_icontains") + scan = CharFilter(method="filter_scan_id") + scan__in = CharFilter(method="filter_scan_id_in") + + inserted_at = DateFilter(method="filter_inserted_at", lookup_expr="date") + inserted_at__date = DateFilter(method="filter_inserted_at", lookup_expr="date") + inserted_at__gte = DateFilter(method="filter_inserted_at_gte") + inserted_at__lte = DateFilter(method="filter_inserted_at_lte") + class Meta: model = Finding fields = { @@ -361,6 +377,23 @@ class FindingFilter(ProviderRelationshipFilterSet): self.provider_alias_icontains_lookup_field = "scan__provider__alias__icontains" super().__init__(*args, **kwargs) + data = self.data + if not data or ( + not data.get("scan") + and not data.get("scan__in") + and not data.get("inserted_at") + and not data.get("inserted_at.date") + and not data.get("inserted_at__gte") + and not data.get("inserted_at__lte") + ): + self.add_default_filter() + + def add_default_filter(self): + utc_now = datetime.now(timezone.utc) + start = uuid7_start(datetime_to_uuid7(utc_now)) + + self.queryset = self.queryset.filter(id__gte=start) + def filter_delta(self, queryset, name, value): return enum_filter( queryset, @@ -463,3 +496,52 @@ class FindingFilter(ProviderRelationshipFilterSet): def filter_resource_type_icontains(self, queryset, name, value): return queryset.filter(resources__type__icontains=value) + + # Convert filter values to UUIDv7 values for use with partitioning + + def filter_scan_id(self, queryset, name, value): + value = parse_params_to_uuid7(value) + + start = uuid7_start(value) + end = uuid7_end(value, settings.FINDINGS_TABLE_PARTITION_DAYS) + return queryset.filter(id__gte=start).filter(id__lt=end).filter(scan__id=value) + + def filter_scan_id_in(self, queryset, name, value): + value = parse_params_to_uuid7(value) + if isinstance(value, UUID): + value = [value] + + start, end = uuid7_range(value) + if start == end: + return queryset.filter(id__gte=start).filter(scan__id__in=value) + else: + return ( + queryset.filter(id__gte=start) + .filter(id__lt=end) + .filter(scan__id__in=value) + ) + + def filter_inserted_at(self, queryset, name, value): + value = self.maybe_date_to_datetime(value) + start = uuid7_start(datetime_to_uuid7(value)) + + return queryset.filter(id__gte=start).filter(inserted_at=value) + + def filter_inserted_at_gte(self, queryset, name, value): + value = self.maybe_date_to_datetime(value) + start = uuid7_start(datetime_to_uuid7(value)) + + return queryset.filter(id__gte=start).filter(inserted_at__gte=value) + + def filter_inserted_at_lte(self, queryset, name, value): + value = self.maybe_date_to_datetime(value) + end = uuid7_start(datetime_to_uuid7(value)) + + return queryset.filter(id__lte=end).filter(inserted_at__lte=value) + + @staticmethod + def maybe_date_to_datetime(value): + dt = value + if isinstance(value, date): + dt = datetime.combine(value, datetime.min.time(), tzinfo=timezone.utc) + return dt diff --git a/src/backend/api/fixtures/3_dev_scans.json b/src/backend/api/fixtures/4_dev_scans.json similarity index 100% rename from src/backend/api/fixtures/3_dev_scans.json rename to src/backend/api/fixtures/4_dev_scans.json diff --git a/src/backend/api/fixtures/4_dev_findings.json b/src/backend/api/fixtures/5_dev_findings.json similarity index 92% rename from src/backend/api/fixtures/4_dev_findings.json rename to src/backend/api/fixtures/5_dev_findings.json index b426552dec..f081de8f77 100644 --- a/src/backend/api/fixtures/4_dev_findings.json +++ b/src/backend/api/fixtures/5_dev_findings.json @@ -1,7 +1,7 @@ [ { "model": "api.finding", - "pk": "01920571-de39-7539-acb8-35f714afdcd9", + "pk": "01920bdb-faf8-7cb1-ba66-dee96f78f048", "fields": { "tenant": "12646005-9067-4d2a-a098-8bb378604362", "scan": "0191e280-9d2f-71c8-9b18-487a23ba185e", @@ -51,21 +51,21 @@ "impact": "critical", "severity": "critical" }, - "inserted_at": "2024-09-01T17:24:27.050Z", - "updated_at": "2024-09-01T17:24:27.050Z" + "inserted_at": "2024-09-19T19:56:59.590Z", + "updated_at": "2024-09-19T19:56:59.590Z" } }, { "model": "api.resourcefindingmapping", "fields": { - "finding": "01920571-de39-7539-acb8-35f714afdcd9", + "finding": "01920bdb-faf8-7cb1-ba66-dee96f78f048", "resource": "a3ba9470-a240-49a6-8196-9230a267a220", "tenant": "12646005-9067-4d2a-a098-8bb378604362" } }, { "model": "api.finding", - "pk": "01920571-3e43-7fae-9175-ee991af8a13c", + "pk": "01920bdb-faf8-7fbd-9aa4-ae144d4b4b91", "fields": { "tenant": "12646005-9067-4d2a-a098-8bb378604362", "scan": "01920573-aa9c-73c9-bcda-f2e35c9b19d2", @@ -113,21 +113,21 @@ "impact": "critical", "severity": "critical" }, - "inserted_at": "2024-09-01T17:24:27.050Z", - "updated_at": "2024-09-01T17:24:27.050Z" + "inserted_at": "2024-09-19T19:56:59.596Z", + "updated_at": "2024-09-19T19:56:59.596Z" } }, { "model": "api.resourcefindingmapping", "fields": { - "finding": "01920571-3e43-7fae-9175-ee991af8a13c", + "finding": "01920bdb-faf8-7fbd-9aa4-ae144d4b4b91", "resource": "a3ba9470-a240-49a6-8196-9230a267a220", "tenant": "12646005-9067-4d2a-a098-8bb378604362" } }, { "model": "api.finding", - "pk": "01920572-2def-77f4-ba03-d4c7ec46fe8d", + "pk": "01920bdf-8150-7630-a385-62a80cdc75b5", "fields": { "tenant": "12646005-9067-4d2a-a098-8bb378604362", "scan": "01920573-ea5b-77fd-a93f-1ed2ae12f728", @@ -177,21 +177,21 @@ "impact": "critical", "severity": "critical" }, - "inserted_at": "2024-09-01T17:24:27.050Z", - "updated_at": "2024-09-01T17:24:27.050Z" + "inserted_at": "2024-09-19T20:00:50.354Z", + "updated_at": "2024-09-19T20:00:50.354Z" } }, { "model": "api.resourcefindingmapping", "fields": { - "finding": "01920572-2def-77f4-ba03-d4c7ec46fe8d", + "finding": "01920bdf-8150-7630-a385-62a80cdc75b5", "resource": "a3ba9470-a240-49a6-8196-9230a267a220", "tenant": "12646005-9067-4d2a-a098-8bb378604362" } }, { "model": "api.finding", - "pk": "01920572-69bd-77c6-a038-fab6a5061477", + "pk": "01920be0-8320-75ce-a9a2-f927821769fa", "fields": { "tenant": "12646005-9067-4d2a-a098-8bb378604362", "scan": "01920573-ea5b-77fd-a93f-1ed2ae12f728", @@ -239,14 +239,14 @@ "impact": "critical", "severity": "critical" }, - "inserted_at": "2024-09-01T17:24:27.050Z", - "updated_at": "2024-09-01T17:24:27.050Z" + "inserted_at": "2024-09-19T20:01:56.306Z", + "updated_at": "2024-09-19T20:01:56.306Z" } }, { "model": "api.resourcefindingmapping", "fields": { - "finding": "01920572-69bd-77c6-a038-fab6a5061477", + "finding": "01920be0-8320-75ce-a9a2-f927821769fa", "resource": "a3ba9470-a240-49a6-8196-9230a267a220", "tenant": "12646005-9067-4d2a-a098-8bb378604362" } diff --git a/src/backend/api/migrations/0001_initial.py b/src/backend/api/migrations/0001_initial.py index 6116f761c5..b6e76c78d6 100644 --- a/src/backend/api/migrations/0001_initial.py +++ b/src/backend/api/migrations/0001_initial.py @@ -1,5 +1,4 @@ import uuid -from uuid6 import uuid7 from functools import partial import django.contrib.auth.models @@ -11,7 +10,16 @@ import django.db.models.deletion import django.utils.timezone from django.conf import settings from django.db import migrations, models - +from psqlextra.backend.migrations.operations.add_default_partition import ( + PostgresAddDefaultPartition, +) +from psqlextra.backend.migrations.operations.create_partitioned_model import ( + PostgresCreatePartitionedModel, +) +from psqlextra.manager.manager import PostgresManager +from psqlextra.models.partitioned import PostgresPartitionedModel +from psqlextra.types import PostgresPartitioningMethod +from uuid6 import uuid7 import api.rls from api.db_utils import ( @@ -702,7 +710,7 @@ class Migration(migrations.Migration): SeverityEnumMigration.create_enum_type, reverse_code=SeverityEnumMigration.drop_enum_type, ), - migrations.CreateModel( + PostgresCreatePartitionedModel( name="Finding", fields=[ ( @@ -782,20 +790,16 @@ class Migration(migrations.Migration): ], options={ "db_table": "findings", - "indexes": [ - models.Index( - fields=[ - "scan_id", - "impact", - "severity", - "status", - "check_id", - "delta", - ], - name="findings_filter_idx", - ), - ], + "base_manager_name": "objects", }, + partitioning_options={ + "method": PostgresPartitioningMethod["RANGE"], + "key": ["id"], + }, + bases=(PostgresPartitionedModel,), + managers=[ + ("objects", PostgresManager()), + ], ), migrations.RunSQL( sql=""" @@ -830,12 +834,24 @@ class Migration(migrations.Migration): ), ], ), + migrations.AddIndex( + model_name="finding", + index=models.Index( + fields=["scan_id", "impact", "severity", "status", "check_id", "delta"], + name="findings_filter_idx", + ), + ), migrations.AddIndex( model_name="finding", index=django.contrib.postgres.indexes.GinIndex( fields=["text_search"], name="gin_findings_search_idx" ), ), + PostgresAddDefaultPartition( + model_name="Finding", + name="default", + ), + # NOTE: the RLS policy needs to be explicitly set on the partitions migrations.AddConstraint( model_name="finding", constraint=api.rls.RowLevelSecurityConstraint( @@ -844,7 +860,16 @@ class Migration(migrations.Migration): statements=["SELECT", "INSERT", "UPDATE", "DELETE"], ), ), - migrations.CreateModel( + migrations.AddConstraint( + model_name="finding", + constraint=api.rls.RowLevelSecurityConstraint( + "tenant_id", + name="rls_on_finding_default", + partition_name="default", + statements=["SELECT", "INSERT", "UPDATE", "DELETE"], + ), + ), + PostgresCreatePartitionedModel( name="ResourceFindingMapping", fields=[ ( @@ -878,7 +903,16 @@ class Migration(migrations.Migration): options={ "db_table": "resource_finding_mappings", "abstract": False, + "base_manager_name": "objects", }, + partitioning_options={ + "method": PostgresPartitioningMethod["RANGE"], + "key": ["finding_id"], + }, + bases=(PostgresPartitionedModel,), + managers=[ + ("objects", PostgresManager()), + ], ), migrations.AddField( model_name="finding", @@ -905,4 +939,21 @@ class Migration(migrations.Migration): statements=["SELECT"], ), ), + PostgresAddDefaultPartition( + model_name="resourcefindingmapping", + name="default", + ), + migrations.AddConstraint( + model_name="resourcefindingmapping", + constraint=api.rls.RowLevelSecurityConstraint( + "tenant_id", + name="rls_on_resource_finding_mappings_default", + partition_name="default", + statements=["SELECT"], + ), + ), + migrations.AlterModelOptions( + name="finding", + options={}, + ), ] diff --git a/src/backend/api/models.py b/src/backend/api/models.py index a1d8170098..e4eb4f0dc3 100644 --- a/src/backend/api/models.py +++ b/src/backend/api/models.py @@ -7,13 +7,12 @@ from django.contrib.postgres.indexes import GinIndex from django.contrib.postgres.search import SearchVector, SearchVectorField from django.core.validators import MinLengthValidator from django.db import models - from django.utils.translation import gettext_lazy as _ from django_celery_results.models import TaskResult - -from uuid6 import uuid7 - from prowler.lib.outputs.finding import Severity +from psqlextra.models import PostgresPartitionedModel +from psqlextra.types import PostgresPartitioningMethod +from uuid6 import uuid7 from api.db_utils import ( enum_to_choices, @@ -26,14 +25,12 @@ from api.db_utils import ( CustomUserManager, ) from api.exceptions import ModelValidationError - from api.rls import ( RowLevelSecurityProtectedModel, RowLevelSecurityConstraint, BaseSecurityConstraint, ) - # Convert Prowler Severity enum to Django TextChoices SeverityChoices = enum_to_choices(Severity) @@ -45,8 +42,8 @@ class StatusChoices(models.TextChoices): However it adds another state, MUTED, which is not in the CLI. """ - PASS = "PASS", _("Pass") FAIL = "FAIL", _("Fail") + PASS = "PASS", _("Pass") MANUAL = "MANUAL", _("Manual") MUTED = "MUTED", _("Muted") @@ -411,7 +408,19 @@ class ResourceTagMapping(RowLevelSecurityProtectedModel): ] -class Finding(RowLevelSecurityProtectedModel): +class Finding(PostgresPartitionedModel, RowLevelSecurityProtectedModel): + """ + Defines the Finding model. + + Findings uses a partitioned table to store findings. The partitions are created based on the UUIDv7 `id` field. + + Note when creating migrations, you must use `python manage.py pgmakemigrations` to create the migrations. + """ + + class PartitioningMeta: + method = PostgresPartitioningMethod.RANGE + key = ["id"] + class DeltaChoices(models.TextChoices): NEW = "new", _("New") CHANGED = "changed", _("Changed") @@ -462,7 +471,7 @@ class Finding(RowLevelSecurityProtectedModel): editable=False, ) - class Meta: + class Meta(RowLevelSecurityProtectedModel.Meta): db_table = "findings" constraints = [ @@ -471,6 +480,12 @@ class Finding(RowLevelSecurityProtectedModel): name="rls_on_%(class)s", statements=["SELECT", "UPDATE", "INSERT", "DELETE"], ), + RowLevelSecurityConstraint( + field="tenant_id", + name="rls_on_%(class)s_default", + partition_name="default", + statements=["SELECT", "UPDATE", "INSERT", "DELETE"], + ), ] indexes = [ @@ -499,13 +514,25 @@ class Finding(RowLevelSecurityProtectedModel): self.save() -class ResourceFindingMapping(RowLevelSecurityProtectedModel): +class ResourceFindingMapping(PostgresPartitionedModel, RowLevelSecurityProtectedModel): + """ + Defines the ResourceFindingMapping model. + + ResourceFindingMapping is used to map a Finding to a Resource. + + It follows the same partitioning strategy as the Finding model. + """ + # NOTE that we don't really need a primary key here, # but everything is easier with django if we do id = models.UUIDField(primary_key=True, default=uuid4, editable=False) resource = models.ForeignKey(Resource, on_delete=models.CASCADE) finding = models.ForeignKey(Finding, on_delete=models.CASCADE) + class PartitioningMeta: + method = PostgresPartitioningMethod.RANGE + key = ["finding_id"] + class Meta(RowLevelSecurityProtectedModel.Meta): db_table = "resource_finding_mappings" diff --git a/src/backend/api/partitions.py b/src/backend/api/partitions.py new file mode 100644 index 0000000000..88d757c531 --- /dev/null +++ b/src/backend/api/partitions.py @@ -0,0 +1,192 @@ +from datetime import datetime, timezone +from typing import Generator, Optional + +from dateutil.relativedelta import relativedelta +from django.conf import settings +from psqlextra.partitioning import ( + PostgresPartitioningManager, + PostgresRangePartition, + PostgresRangePartitioningStrategy, + PostgresTimePartitionSize, + PostgresPartitioningError, +) +from psqlextra.partitioning.config import PostgresPartitioningConfig +from uuid6 import UUID + +from api.models import Finding, ResourceFindingMapping +from api.rls import RowLevelSecurityConstraint +from api.uuid_utils import datetime_to_uuid7 + + +class PostgresUUIDv7RangePartition(PostgresRangePartition): + def __init__( + self, + from_values: UUID, + to_values: UUID, + size: PostgresTimePartitionSize, + name_format: Optional[str] = None, + **kwargs, + ) -> None: + self.from_values = from_values + self.to_values = to_values + self.size = size + self.name_format = name_format + + self.rls_statements = None + if "rls_statements" in kwargs: + self.rls_statements = kwargs["rls_statements"] + + start_timestamp_ms = self.from_values.time + + self.start_datetime = datetime.fromtimestamp( + start_timestamp_ms / 1000, timezone.utc + ) + + def name(self) -> str: + if not self.name_format: + raise PostgresPartitioningError("Unknown size/unit") + + return self.start_datetime.strftime(self.name_format).lower() + + def deconstruct(self) -> dict: + return { + **super().deconstruct(), + "size_unit": self.size.unit.value, + "size_value": self.size.value, + } + + def create( + self, + model, + schema_editor, + comment, + ) -> None: + super().create(model, schema_editor, comment) + + # if this model has RLS statements, add them to the partition + if isinstance(self.rls_statements, list): + schema_editor.add_constraint( + model, + constraint=RowLevelSecurityConstraint( + "tenant_id", + name=f"rls_on_{self.name()}", + partition_name=self.name(), + statements=self.rls_statements, + ), + ) + + +class PostgresUUIDv7PartitioningStrategy(PostgresRangePartitioningStrategy): + def __init__( + self, + size: PostgresTimePartitionSize, + count: int, + start_date: datetime = None, + max_age: Optional[relativedelta] = None, + name_format: Optional[str] = None, + **kwargs, + ) -> None: + self.start_date = start_date.replace(hour=0, minute=0, second=0, microsecond=0) + self.size = size + self.count = count + self.max_age = max_age + self.name_format = name_format + + self.rls_statements = None + if "rls_statements" in kwargs: + self.rls_statements = kwargs["rls_statements"] + + def to_create(self) -> Generator[PostgresUUIDv7RangePartition, None, None]: + current_datetime = ( + self.start_date if self.start_date else self.get_start_datetime() + ) + + for _ in range(self.count): + end_datetime = ( + current_datetime + self.size.as_delta() - relativedelta(microseconds=1) + ) + start_uuid7 = datetime_to_uuid7(current_datetime) + end_uuid7 = datetime_to_uuid7(end_datetime) + + yield PostgresUUIDv7RangePartition( + from_values=start_uuid7, + to_values=end_uuid7, + size=self.size, + name_format=self.name_format, + rls_statements=self.rls_statements, + ) + + current_datetime += self.size.as_delta() + + def to_delete(self) -> Generator[PostgresUUIDv7RangePartition, None, None]: + if not self.max_age: + return + + current_datetime = self.get_start_datetime() - self.max_age + + while True: + end_datetime = current_datetime + self.size.as_delta() + start_uuid7 = datetime_to_uuid7(current_datetime) + end_uuid7 = datetime_to_uuid7(end_datetime) + + # dropping table will delete indexes and policies + yield PostgresUUIDv7RangePartition( + from_values=start_uuid7, + to_values=end_uuid7, + size=self.size, + name_format=self.name_format, + ) + + current_datetime -= self.size.as_delta() + + def get_start_datetime(self) -> datetime: + return datetime.now(timezone.utc).replace( + hour=0, minute=0, second=0, microsecond=0 + ) + + +def relative_days_or_none(value): + if value is None: + return None + return relativedelta(days=value) + + +# +# To manage the partitions, run `python manage.py pgpartition --using admin` +# +# For more info on the partitioning manager, see https://github.com/SectorLabs/django-postgres-extra +manager = PostgresPartitioningManager( + [ + PostgresPartitioningConfig( + model=Finding, + strategy=PostgresUUIDv7PartitioningStrategy( + start_date=datetime.now(timezone.utc), + size=PostgresTimePartitionSize( + days=settings.FINDINGS_TABLE_PARTITION_DAYS + ), + count=settings.FINDINGS_TABLE_PARTITION_COUNT, + max_age=relative_days_or_none( + settings.FINDINGS_TABLE_PARTITION_MAX_AGE_DAYS + ), + name_format="%Y_%b_%d", + rls_statements=["SELECT", "INSERT", "UPDATE", "DELETE"], + ), + ), + # ResourceFindingMapping should always follow the Finding partitioning + PostgresPartitioningConfig( + model=ResourceFindingMapping, + strategy=PostgresUUIDv7PartitioningStrategy( + start_date=datetime.now(timezone.utc), + size=PostgresTimePartitionSize( + days=settings.FINDINGS_TABLE_PARTITION_DAYS + ), + count=settings.FINDINGS_TABLE_PARTITION_COUNT, + max_age=relative_days_or_none( + settings.FINDINGS_TABLE_PARTITION_MAX_AGE_DAYS + ), + name_format="%Y_%b_%d", + rls_statements=["SELECT"], + ), + ), + ] +) diff --git a/src/backend/api/rls.py b/src/backend/api/rls.py index dd8f56353b..0f6e4a2400 100644 --- a/src/backend/api/rls.py +++ b/src/backend/api/rls.py @@ -26,7 +26,11 @@ class Tenant(models.Model): # TODO Add abstract class for non-RLS models class RowLevelSecurityConstraint(models.BaseConstraint): - """Model constraint to enforce row-level security on a tenant based model, in addition to the least privileges.""" + """ + Model constraint to enforce row-level security on a tenant based model, in addition to the least privileges. + + The constraint can be applied to a partitioned table by specifying the `partition_name` keyword argument. + """ rls_sql_query = """ ALTER TABLE %(table_name)s ENABLE ROW LEVEL SECURITY; @@ -60,10 +64,15 @@ class RowLevelSecurityConstraint(models.BaseConstraint): DROP POLICY IF EXISTS %(db_user)s_%(table_name)s_{statement} on %(table_name)s; """ - def __init__(self, field: str, name: str, statements: list | None = None) -> None: + def __init__( + self, field: str, name: str, statements: list | None = None, **kwargs + ) -> None: super().__init__(name=name) self.target_field: str = field self.statements = statements or ["SELECT"] + self.partition_name = None + if "partition_name" in kwargs: + self.partition_name = kwargs["partition_name"] def create_sql(self, model: Any, schema_editor: Any) -> Any: field_column = schema_editor.quote_name(self.target_field) @@ -81,12 +90,17 @@ class RowLevelSecurityConstraint(models.BaseConstraint): f"{self.rls_sql_query}" f"{policy_queries}" f"{grant_queries}" ) + table_name = model._meta.db_table + if self.partition_name: + table_name = f"{table_name}_{self.partition_name}" + return Statement( full_create_sql_query, - table_name=model._meta.db_table, + table_name=table_name, field_column=field_column, db_user=DB_USER, tenant_setting=POSTGRES_TENANT_VAR, + partition_name=self.partition_name, ) def remove_sql(self, model: Any, schema_editor: Any) -> Any: @@ -95,11 +109,15 @@ class RowLevelSecurityConstraint(models.BaseConstraint): f"{self.drop_sql_query}" f"{''.join([self.drop_policy_sql_query.format(statement) for statement in self.statements])}" ) + table_name = model._meta.db_table + if self.partition_name: + table_name = f"{table_name}_{self.partition_name}" return Statement( full_drop_sql_query, - table_name=Table(model._meta.db_table, schema_editor.quote_name), + table_name=Table(table_name, schema_editor.quote_name), field_column=field_column, db_user=DB_USER, + partition_name=self.partition_name, ) def __eq__(self, other: object) -> bool: diff --git a/src/backend/api/specs/v1.yaml b/src/backend/api/specs/v1.yaml index fdca8fbc84..846dd8da6a 100644 --- a/src/backend/api/specs/v1.yaml +++ b/src/backend/api/specs/v1.yaml @@ -11,7 +11,8 @@ paths: get: operationId: findings_list description: Retrieve a list of all findings with options for filtering by various - criteria. + criteria. If no `scan` or `inserted_at` filter is provided, a default filter + will be applied that will return all findings for the current day. summary: List all findings parameters: - in: query @@ -83,12 +84,12 @@ paths: name: filter[inserted_at__gte] schema: type: string - format: date-time + format: date - in: query name: filter[inserted_at__lte] schema: type: string - format: date-time + format: date - in: query name: filter[provider] schema: @@ -188,17 +189,10 @@ paths: name: filter[scan] schema: type: string - format: uuid - in: query name: filter[scan__in] schema: - type: array - items: - type: string - format: uuid - description: Multiple values may be separated by commas. - explode: false - style: form + type: string - name: filter[search] required: false in: query @@ -281,6 +275,8 @@ paths: items: type: string enum: + - id + - -id - status - -status - severity @@ -293,7 +289,7 @@ paths: - -updated_at explode: false tags: - - findings + - Findings security: - basicAuth: [] responses: @@ -351,7 +347,7 @@ paths: related resources should be returned. explode: false tags: - - findings + - Findings security: - basicAuth: [] responses: @@ -1826,14 +1822,14 @@ components: nullable: true status: enum: - - PASS - FAIL + - PASS - MANUAL - MUTED type: string description: |- - * `PASS` - Pass * `FAIL` - Fail + * `PASS` - Pass * `MANUAL` - Manual * `MUTED` - Muted status_extended: diff --git a/src/backend/api/tests/test_uuid_utils.py b/src/backend/api/tests/test_uuid_utils.py new file mode 100644 index 0000000000..a332241685 --- /dev/null +++ b/src/backend/api/tests/test_uuid_utils.py @@ -0,0 +1,128 @@ +from datetime import datetime, timezone +from uuid import uuid4 + +import pytest +from dateutil.relativedelta import relativedelta +from uuid6 import UUID + +from api.uuid_utils import ( + datetime_to_uuid7, + datetime_from_uuid7, + uuid7_start, + uuid7_end, + uuid7_range, + parse_params_to_uuid7, +) + + +def test_parse_params_to_uuid7(): + uuid = parse_params_to_uuid7("0191dff4-7a78-7031-8814-4fb51bf2bd5b") + assert isinstance(uuid, UUID) + + uuids = parse_params_to_uuid7( + ["0191dff4-7a78-7031-8814-4fb51bf2bd5b", "01856aa0-c800-7b8f-bfa0-46a393f14d50"] + ) + assert isinstance(uuids, list) + assert len(uuids) == 2 + + uuids = parse_params_to_uuid7( + "0191dff4-7a78-7031-8814-4fb51bf2bd5b,01856aa0-c800-7b8f-bfa0-46a393f14d50" + ) + assert isinstance(uuids, list) + assert len(uuids) == 2 + + +@pytest.mark.parametrize( + "input_datetime", + [ + datetime(2024, 9, 11, 7, 20, 27, tzinfo=timezone.utc), + datetime(2023, 1, 1, 0, 0, 0, tzinfo=timezone.utc), + ], +) +def test_timestamp_to_uuid7(input_datetime): + uuid7 = datetime_to_uuid7(input_datetime) + # assert instance is all we can do without reversing the math in the test + assert isinstance(uuid7, UUID) + assert uuid7.version == 7 + assert uuid7.time == int(input_datetime.timestamp() * 1000) + + +@pytest.mark.parametrize( + "uuid7, expected_datetime", + [ + ( + "0191dff4-7a78-7031-8814-4fb51bf2bd5b", + datetime(2024, 9, 11, 7, 20, 27, tzinfo=timezone.utc), + ), + ( + "01856aa0-c800-7b8f-bfa0-46a393f14d50", + datetime(2023, 1, 1, 0, 0, 0, tzinfo=timezone.utc), + ), + ], +) +def test_datetime_from_uuid7(uuid7, expected_datetime): + extracted_datetime = datetime_from_uuid7(uuid7=UUID(uuid7)) + assert extracted_datetime == expected_datetime + + +def test_extract_timestamp_from_uuid4_invalid(): + with pytest.raises(ValueError): + datetime_from_uuid7(uuid4()) + + +def test_uuid7_start(): + dt = datetime.now(timezone.utc) + uuid = datetime_to_uuid7(dt) + dt = dt.replace(hour=0, minute=0, second=0, microsecond=0) + start = uuid7_start(uuid) + expected_ts = ( + int(dt.replace(hour=0, minute=0, second=0, microsecond=0).timestamp()) * 1000 + ) + assert start.time == expected_ts + + +@pytest.mark.parametrize( + "days_offset", + [0, 1, 10, 30, 60], +) +def test_uuid7_end(days_offset): + dt = datetime.now(timezone.utc) + uuid = datetime_to_uuid7(dt) + + end = uuid7_end(uuid, days_offset) + + dt = dt.replace(hour=0, minute=0, second=0, microsecond=0) + dt = dt + relativedelta(days=days_offset + 1, microseconds=-1) + expected_ts = int(dt.timestamp()) * 1000 + + assert end.time == expected_ts + + +def test_uuid7_range(): + uuids = [ + datetime_to_uuid7(datetime.now(timezone.utc)), + datetime_to_uuid7(datetime.now(timezone.utc).replace(year=2023)), + datetime_to_uuid7(datetime.now(timezone.utc).replace(year=2024)), + datetime_to_uuid7(datetime.now(timezone.utc).replace(year=2025)), + ] + expected_start = ( + int( + datetime.fromtimestamp(uuids[1].time / 1000, tz=timezone.utc) + .replace(hour=0, minute=0, second=0, microsecond=0) + .timestamp() + ) + * 1000 + ) + expected_end = ( + int( + datetime.fromtimestamp(uuids[-1].time / 1000, tz=timezone.utc) + .replace(hour=23, minute=59, second=59, microsecond=999999) + .timestamp() + ) + * 1000 + ) + + start, end = uuid7_range(uuids) + + assert start.time == expected_start + assert end.time == expected_end diff --git a/src/backend/api/tests/test_views.py b/src/backend/api/tests/test_views.py index 7e20e7ac4d..08c5795ff5 100644 --- a/src/backend/api/tests/test_views.py +++ b/src/backend/api/tests/test_views.py @@ -1390,7 +1390,10 @@ class TestFindingViewSet: ("service", "ec2", 1), ("service.in", "ec2,s3", 2), ("service.icontains", "ec", 1), + ("inserted_at", "2024-01-01", 0), + ("inserted_at.date", "2024-01-01", 0), ("inserted_at.gte", "2024-01-01", 2), + ("inserted_at.lte", "2024-12-31", 2), ("updated_at.lte", "2024-01-01", 0), ("resource_type.icontains", "prowler", 2), # full text search on finding @@ -1421,6 +1424,35 @@ class TestFindingViewSet: assert response.status_code == status.HTTP_200_OK assert len(response.json()["data"]) == expected_count + def test_finding_filter_by_scan_id( + self, authenticated_client, findings_fixture, tenant_header + ): + response = authenticated_client.get( + reverse("finding-list"), + { + "filter[scan]": findings_fixture[0].scan.id, + }, + headers=tenant_header, + ) + assert response.status_code == status.HTTP_200_OK + assert len(response.json()["data"]) == 2 + + def test_finding_filter_by_scan_id_in( + self, authenticated_client, findings_fixture, tenant_header + ): + response = authenticated_client.get( + reverse("finding-list"), + { + "filter[scan.in]": [ + findings_fixture[0].scan.id, + findings_fixture[1].scan.id, + ] + }, + headers=tenant_header, + ) + assert response.status_code == status.HTTP_200_OK + assert len(response.json()["data"]) == 2 + def test_finding_filter_by_provider( self, authenticated_client, findings_fixture, tenant_header ): diff --git a/src/backend/api/uuid_utils.py b/src/backend/api/uuid_utils.py new file mode 100644 index 0000000000..aae7c21353 --- /dev/null +++ b/src/backend/api/uuid_utils.py @@ -0,0 +1,96 @@ +from datetime import datetime, timezone +from secrets import randbits + +from dateutil.relativedelta import relativedelta +from uuid6 import UUID + + +def parse_params_to_uuid7(value: str | list[str]) -> UUID | list[UUID]: + """ + Converts query parameters to UUIDv7 values. + """ + if isinstance(value, str): + if "," in value: + value = value.split(",") + else: + value = UUID(value) + + if isinstance(value, list) and isinstance(value[0], str): + value = [UUID(uuid) for uuid in value] + + return value + + +def datetime_to_uuid7(datetime: datetime) -> UUID: + """ + The body of this function is taken from the `uuid6` package. + https://github.com/oittaa/uuid6-python/blob/main/src/uuid6/__init__.py#L140-L147 + + That package only generates UUIDv7s using the current timestamp, but we want to + generate UUIDv7s using a timestamp from a different time. + + But there is an open issue for this: + https://github.com/oittaa/uuid6-python/issues/150 + """ + + timestamp_ms = int(datetime.timestamp()) * 1000 + uuid_int = (timestamp_ms & 0xFFFFFFFFFFFF) << 80 + uuid_int |= randbits(76) + + return UUID(int=uuid_int, version=7) + + +def datetime_from_uuid7(uuid7: UUID) -> datetime: + """ + For any given UUIDv7, extract the timestamp from the UUIDv7 and return it as a datetime. + + If any other UUID is provided, an exception will be raised. + """ + if not isinstance(uuid7, UUID): + raise ValueError("uuid7 must be a UUIDv7 object") + + timestamp_ms = uuid7.time + return datetime.fromtimestamp(timestamp_ms / 1000, tz=timezone.utc) + + +def uuid7_start(uuid: UUID) -> UUID: + """ + Returns a UUIDv7 that represents the start of the day for the given UUID. + """ + dt = datetime_from_uuid7(uuid).replace(hour=0, minute=0, second=0, microsecond=0) + + timestamp_ms = int(dt.timestamp()) * 1000 + uuid_int = (timestamp_ms & 0xFFFFFFFFFFFF) << 80 + + return UUID(int=uuid_int, version=7) + + +def uuid7_end(uuid: UUID, offset_days: int | None = 0) -> UUID: + """ + Returns a UUIDv7 that represents the end of the day for the given UUID. + + If the offset_days is provided, the end of the day will be offset by the specified number of days. + """ + dt = datetime_from_uuid7(uuid).replace(hour=0, minute=0, second=0, microsecond=0) + dt = dt + relativedelta(days=(offset_days + 1), microseconds=-1) + + timestamp_ms = int(dt.timestamp()) * 1000 + uuid_int = (timestamp_ms & 0xFFFFFFFFFFFF) << 80 + + return UUID(int=uuid_int, version=7) + + +def uuid7_range(uuids: list) -> list: + """ + For the given list of UUIDs, return a tuple of UUIDv7 values that represent the start and end of the range of days. + """ + if not uuids: + return [] + + if isinstance(uuids[0], str): + uuids = [UUID(uuid, version=7) for uuid in uuids] + + start = min(uuids) + end = max(uuids) + + return [uuid7_start(start), uuid7_end(end)] diff --git a/src/backend/api/v1/serializers.py b/src/backend/api/v1/serializers.py index 07bf0bb3ed..493f0092fc 100644 --- a/src/backend/api/v1/serializers.py +++ b/src/backend/api/v1/serializers.py @@ -3,9 +3,8 @@ import json from django.contrib.auth.password_validation import validate_password from drf_spectacular.utils import extend_schema_field from rest_framework_json_api import serializers -from rest_framework_json_api.serializers import ValidationError from rest_framework_json_api.relations import SerializerMethodResourceRelatedField - +from rest_framework_json_api.serializers import ValidationError from api.models import ( StateChoices, @@ -438,7 +437,7 @@ class ResourceSerializer(RLSSerializer): return fields -class FindingVisbilityEnumSerializerField(serializers.ChoiceField): +class FindingDeltaEnumSerializerField(serializers.ChoiceField): def __init__(self, **kwargs): kwargs["choices"] = Finding.DeltaChoices.choices super().__init__(**kwargs) @@ -498,7 +497,7 @@ class FindingSerializer(RLSSerializer): } class JSONAPIMeta: - resource_name = "findings" + resource_name = "Findings" def get_resources(self, obj): mappings = ResourceFindingMapping.objects.filter(finding=obj) diff --git a/src/backend/api/v1/views.py b/src/backend/api/v1/views.py index 55e9797d7e..78a639d758 100644 --- a/src/backend/api/v1/views.py +++ b/src/backend/api/v1/views.py @@ -15,7 +15,6 @@ from rest_framework.exceptions import MethodNotAllowed, NotFound from rest_framework.generics import get_object_or_404 from rest_framework_json_api.views import Response - from api.base_views import BaseRLSViewSet, BaseViewSet from api.filters import ( ProviderFilter, @@ -25,9 +24,9 @@ from api.filters import ( ResourceFilter, FindingFilter, ) - from api.models import User, Provider, Scan, Task, Resource, Finding from api.rls import Tenant +from api.uuid_utils import datetime_to_uuid7 from api.v1.serializers import ( UserSerializer, UserCreateSerializer, @@ -491,10 +490,12 @@ class ResourceViewSet(BaseRLSViewSet): @extend_schema_view( list=extend_schema( + tags=["Finding"], summary="List all findings", - description="Retrieve a list of all findings with options for filtering by various criteria.", + description="Retrieve a list of all findings with options for filtering by various criteria. If no `scan` or `inserted_at` filter is provided, a default filter will be applied that will return all findings for the current day.", ), retrieve=extend_schema( + tags=["Finding"], summary="Retrieve data from a specific finding", description="Fetch detailed information about a specific finding by its ID.", ), @@ -506,8 +507,12 @@ class FindingViewSet(BaseRLSViewSet): serializer_class = FindingSerializer http_method_names = ["get"] filterset_class = FindingFilter - ordering = ["inserted_at"] + # Default sort order will put failed findings first + # and then by severity + # and then by most recently inserted via the id field + ordering = ["status", "severity", "-id"] ordering_fields = [ + "id", "status", "severity", "check_id", @@ -515,6 +520,11 @@ class FindingViewSet(BaseRLSViewSet): "updated_at", ] + def inserted_at_to_uuidv7(self, inserted_at): + if inserted_at is None: + return None + return datetime_to_uuid7(inserted_at) + def get_queryset(self): # TODO: require scan_id filter, or if none provided, inject today diff --git a/src/backend/config/django/base.py b/src/backend/config/django/base.py index 1335019136..8d5bd912a7 100644 --- a/src/backend/config/django/base.py +++ b/src/backend/config/django/base.py @@ -3,6 +3,7 @@ from datetime import timedelta from config.custom_logging import LOGGING # noqa from config.env import BASE_DIR, env # noqa from config.settings.celery import * # noqa +from config.settings.partitions import * # noqa SECRET_KEY = env("SECRET_KEY", default="secret") DEBUG = env.bool("DJANGO_DEBUG", default=False) @@ -18,6 +19,7 @@ INSTALLED_APPS = [ "django.contrib.messages", "django.contrib.staticfiles", "django.contrib.postgres", + "psqlextra", "api", "rest_framework", "corsheaders", diff --git a/src/backend/config/django/devel.py b/src/backend/config/django/devel.py index 0424feb07a..9d135d25b5 100644 --- a/src/backend/config/django/devel.py +++ b/src/backend/config/django/devel.py @@ -8,7 +8,7 @@ ALLOWED_HOSTS = env.list("DJANGO_ALLOWED_HOSTS", default=["*"]) # Database DATABASES = { "prowler_user": { - "ENGINE": "django.db.backends.postgresql", + "ENGINE": "psqlextra.backend", "NAME": env("POSTGRES_DB", default="prowler_db"), "USER": env("POSTGRES_USER", default="prowler_user"), "PASSWORD": env("POSTGRES_PASSWORD", default="prowler"), @@ -16,7 +16,7 @@ DATABASES = { "PORT": env("POSTGRES_PORT", default="5432"), }, "admin": { - "ENGINE": "django.db.backends.postgresql", + "ENGINE": "psqlextra.backend", "NAME": env("POSTGRES_DB", default="prowler_db"), "USER": env("POSTGRES_ADMIN_USER", default="prowler"), "PASSWORD": env("POSTGRES_ADMIN_PASSWORD", default="S3cret"), diff --git a/src/backend/config/django/production.py b/src/backend/config/django/production.py index e80040a45a..99e9bc69bc 100644 --- a/src/backend/config/django/production.py +++ b/src/backend/config/django/production.py @@ -17,7 +17,7 @@ DATABASES = { "PORT": env("POSTGRES_PORT"), }, "admin": { - "ENGINE": "django.db.backends.postgresql", + "ENGINE": "psqlextra.backend", "NAME": env("POSTGRES_DB"), "USER": env("POSTGRES_ADMIN_USER"), "PASSWORD": env("POSTGRES_ADMIN_PASSWORD"), diff --git a/src/backend/config/django/testing.py b/src/backend/config/django/testing.py index ffa7a8a287..7846301f4e 100644 --- a/src/backend/config/django/testing.py +++ b/src/backend/config/django/testing.py @@ -8,7 +8,7 @@ ALLOWED_HOSTS = env.list("DJANGO_ALLOWED_HOSTS", default=["localhost", "127.0.0. DATABASES = { "default": { - "ENGINE": "django.db.backends.postgresql", + "ENGINE": "psqlextra.backend", "NAME": "prowler_db_test", "USER": env("POSTGRES_USER", default="prowler"), "PASSWORD": env("POSTGRES_PASSWORD", default="S3cret"), diff --git a/src/backend/config/settings/partitions.py b/src/backend/config/settings/partitions.py new file mode 100644 index 0000000000..7647e36528 --- /dev/null +++ b/src/backend/config/settings/partitions.py @@ -0,0 +1,16 @@ +from config.env import env + +# Partitioning +PSQLEXTRA_PARTITIONING_MANAGER = "api.partitions.manager" + +# Set the days for each partition. Setting the partition days to 30 will create partitions with a size of 1 month. +FINDINGS_TABLE_PARTITION_DAYS = env.int("FINDINGS_TABLE_PARTITION_DAYS", 30) + +# Set the number of partitions to create +FINDINGS_TABLE_PARTITION_COUNT = env.int("FINDINGS_TABLE_PARTITION_COUNT", 7) + +# Set the number of days to keep partitions before deleting them +# Setting this to None will keep partitions indefinitely +FINDINGS_TABLE_PARTITION_MAX_AGE_DAYS = env.int( + "FINDINGS_TABLE_PARTITION_MAX_AGE_DAYS", None +) diff --git a/src/backend/conftest.py b/src/backend/conftest.py index 5de04a4c54..02e679e925 100644 --- a/src/backend/conftest.py +++ b/src/backend/conftest.py @@ -295,13 +295,13 @@ def findings_fixture(scans_fixture, resources_fixture): delta="new", status=Status.FAIL, status_extended="Load Balancer exposed to internet", - impact=Severity.critical, + impact=Severity.medium, impact_extended="test impact extended two", - severity=Severity.critical, + severity=Severity.medium, raw_result={ "status": Status.FAIL, - "impact": Severity.critical, - "severity": Severity.critical, + "impact": Severity.medium, + "severity": Severity.medium, }, tags={"test": "test"}, check_id="test_check_id",