From 95dc87a91b90c3eb5b4a45e23e67b45992bc5665 Mon Sep 17 00:00:00 2001 From: Pepe Fagoaga Date: Fri, 8 Nov 2024 09:34:32 +0100 Subject: [PATCH] chore(partitions): add env to create partitions (#68) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(partitions): Use calendar months * fix: unit to get partition datetime * fix: imports * fix: format * chore: merge * fix(partitions): Only allow month as unit * fix(uuid7_end): default months to 1 * test: fix test_uuid7_end * test: reset expected dt to start of month * fix: tests uuid utils * docs: we only allow months --------- Co-authored-by: Víctor Fernández Poyatos --- docs/partitions.md | 33 ++++++++++++----------- src/backend/api/filters.py | 3 +-- src/backend/api/partitions.py | 27 +++++++++++++------ src/backend/api/tests/test_uuid_utils.py | 14 +++++----- src/backend/api/uuid_utils.py | 17 ++++++------ src/backend/config/settings/partitions.py | 10 +++---- 6 files changed, 58 insertions(+), 46 deletions(-) diff --git a/docs/partitions.md b/docs/partitions.md index 31d3470469..3423488bb1 100644 --- a/docs/partitions.md +++ b/docs/partitions.md @@ -34,31 +34,32 @@ By default, the command will prompt you to accept the changes before applying th ```shell Finding: - + 2024_oct_18 - name: 2024_oct_18 - from_values: 01929cec-8800-7988-a49d-54cbb64b2d3f - to_values: 0193376b-4c18-7ff7-99d5-95b8479ce39f - size_unit: days - size_value: 30 - + 2024_nov_17 - name: 2024_nov_17 - from_values: 0193376b-5000-7414-bbae-b1da382332f9 - to_values: 0193d1ea-1418-7b00-93fa-c4e06bf36bbe - size_unit: days - size_value: 30 + + 2024_nov + name: 2024_nov + from_values: 0192e505-9000-72c8-a47c-cce719d8fb93 + to_values: 01937f84-5418-7eb8-b2a6-e3be749e839d + size_unit: months + size_value: 1 + + 2024_dec + name: 2024_dec + from_values: 01937f84-5800-7b55-879c-9cdb46f023f6 + to_values: 01941f29-7818-7f9f-b4be-20b05bb2f574 + size_unit: months + size_value: 1 0 partitions will be deleted 2 partitions will be created ``` -If you choose to apply the partitions, tables will be generated with the following format: `___`. The date in the table name shows the first date of the partition. +If you choose to apply the partitions, tables will be generated with the following format: `__`. For more info on the partitioning manager, see https://github.com/SectorLabs/django-postgres-extra ### Changing the Partitioning Parameters -There are 3 environment variables that can be used to change the partitioning parameters: +There are 4 environment variables that can be used to change the partitioning parameters: -- `FINDINGS_TABLE_PARTITION_DAYS`: Set the days for each partition. Setting the partition days to 30 will create partitions with a size of 1 month. +- `DJANGO_MANAGE_DB_PARTITIONS`: Allow Django to manage database partitons. By default is set to `False`. +- `FINDINGS_TABLE_PARTITION_MONTHS`: Set the months for each partition. Setting the partition monts to 1 will create partitions with a size of 1 natural month. - `FINDINGS_TABLE_PARTITION_COUNT`: Set the number of partitions to create -- `FINDINGS_TABLE_PARTITION_MAX_AGE_DAYS`: Set the number of days to keep partitions before deleting them. Setting this to `None` will keep partitions indefinitely. +- `FINDINGS_TABLE_PARTITION_MAX_AGE_MONTHS`: Set the number of months to keep partitions before deleting them. Setting this to `None` will keep partitions indefinitely. diff --git a/src/backend/api/filters.py b/src/backend/api/filters.py index 176469011e..94d9f0ee3d 100644 --- a/src/backend/api/filters.py +++ b/src/backend/api/filters.py @@ -321,12 +321,11 @@ class FindingFilter(FilterSet): } # Convert filter values to UUIDv7 values for use with partitioning - def filter_scan_id(self, queryset, name, value): try: value_uuid = transform_into_uuid7(value) start = uuid7_start(value_uuid) - end = uuid7_end(value_uuid, settings.FINDINGS_TABLE_PARTITION_DAYS) + end = uuid7_end(value_uuid, settings.FINDINGS_TABLE_PARTITION_MONTHS) except ValidationError as validation_error: detail = str(validation_error.detail[0]) raise ValidationError( diff --git a/src/backend/api/partitions.py b/src/backend/api/partitions.py index 88d757c531..92390ffdec 100644 --- a/src/backend/api/partitions.py +++ b/src/backend/api/partitions.py @@ -86,7 +86,9 @@ class PostgresUUIDv7PartitioningStrategy(PostgresRangePartitioningStrategy): name_format: Optional[str] = None, **kwargs, ) -> None: - self.start_date = start_date.replace(hour=0, minute=0, second=0, microsecond=0) + self.start_date = start_date.replace( + day=1, hour=0, minute=0, second=0, microsecond=0 + ) self.size = size self.count = count self.max_age = max_age @@ -140,8 +142,17 @@ class PostgresUUIDv7PartitioningStrategy(PostgresRangePartitioningStrategy): current_datetime -= self.size.as_delta() def get_start_datetime(self) -> datetime: + """ + Gets the start of the current month in UTC timezone. + + This function returns a `datetime` object set to the first day of the current + month, at midnight (00:00:00), in UTC. + + Returns: + datetime: A `datetime` object representing the start of the current month in UTC. + """ return datetime.now(timezone.utc).replace( - hour=0, minute=0, second=0, microsecond=0 + day=1, hour=0, minute=0, second=0, microsecond=0 ) @@ -162,13 +173,13 @@ manager = PostgresPartitioningManager( strategy=PostgresUUIDv7PartitioningStrategy( start_date=datetime.now(timezone.utc), size=PostgresTimePartitionSize( - days=settings.FINDINGS_TABLE_PARTITION_DAYS + months=settings.FINDINGS_TABLE_PARTITION_MONTHS ), count=settings.FINDINGS_TABLE_PARTITION_COUNT, max_age=relative_days_or_none( - settings.FINDINGS_TABLE_PARTITION_MAX_AGE_DAYS + settings.FINDINGS_TABLE_PARTITION_MAX_AGE_MONTHS ), - name_format="%Y_%b_%d", + name_format="%Y_%b", rls_statements=["SELECT", "INSERT", "UPDATE", "DELETE"], ), ), @@ -178,13 +189,13 @@ manager = PostgresPartitioningManager( strategy=PostgresUUIDv7PartitioningStrategy( start_date=datetime.now(timezone.utc), size=PostgresTimePartitionSize( - days=settings.FINDINGS_TABLE_PARTITION_DAYS + months=settings.FINDINGS_TABLE_PARTITION_MONTHS ), count=settings.FINDINGS_TABLE_PARTITION_COUNT, max_age=relative_days_or_none( - settings.FINDINGS_TABLE_PARTITION_MAX_AGE_DAYS + settings.FINDINGS_TABLE_PARTITION_MAX_AGE_MONTHS ), - name_format="%Y_%b_%d", + name_format="%Y_%b", rls_statements=["SELECT"], ), ), diff --git a/src/backend/api/tests/test_uuid_utils.py b/src/backend/api/tests/test_uuid_utils.py index d7e78f4e47..e202d087f3 100644 --- a/src/backend/api/tests/test_uuid_utils.py +++ b/src/backend/api/tests/test_uuid_utils.py @@ -74,13 +74,13 @@ def test_uuid7_start(): assert start_uuid.version == 7 -@pytest.mark.parametrize("days_offset", [0, 1, 10, 30, 60]) -def test_uuid7_end(days_offset): +@pytest.mark.parametrize("months_offset", [0, 1, 10, 30, 60]) +def test_uuid7_end(months_offset): dt = datetime.now(timezone.utc) uuid = datetime_to_uuid7(dt) - end_uuid = uuid7_end(uuid, days_offset) - expected_dt = dt.replace(hour=0, minute=0, second=0, microsecond=0) - expected_dt += relativedelta(days=(days_offset + 1), microseconds=-1) + end_uuid = uuid7_end(uuid, months_offset) + expected_dt = dt.replace(day=1, hour=0, minute=0, second=0, microsecond=0) + expected_dt += relativedelta(months=months_offset, microseconds=-1) expected_timestamp_ms = int(expected_dt.timestamp() * 1000) & 0xFFFFFFFFFFFF assert end_uuid.time == expected_timestamp_ms assert end_uuid.version == 7 @@ -103,8 +103,8 @@ def test_uuid7_range(): # Expected end of range end_dt = datetime_from_uuid7(max(uuid_list, key=lambda u: u.time)) - end_dt = end_dt.replace(hour=0, minute=0, second=0, microsecond=0) - end_dt += relativedelta(days=1, microseconds=-1) + end_dt = end_dt.replace(day=1, hour=0, minute=0, second=0, microsecond=0) + end_dt += relativedelta(months=1, microseconds=-1) expected_end_timestamp_ms = int(end_dt.timestamp() * 1000) & 0xFFFFFFFFFFFF assert start_uuid.time == expected_start_timestamp_ms diff --git a/src/backend/api/uuid_utils.py b/src/backend/api/uuid_utils.py index 0d63db57da..8b5dcf7874 100644 --- a/src/backend/api/uuid_utils.py +++ b/src/backend/api/uuid_utils.py @@ -100,22 +100,23 @@ def uuid7_start(uuid_obj: UUID) -> UUID: return datetime_to_uuid7(start_of_day) -def uuid7_end(uuid_obj: UUID, offset_days: int = 0) -> UUID: +def uuid7_end(uuid_obj: UUID, offset_months: int = 1) -> UUID: """ - Returns a UUIDv7 that represents the end of the day for the given UUID. + Returns a UUIDv7 that represents the end of the month for the given UUID. Args: uuid_obj: A UUIDv7 object. - offset_days: Number of days to offset from the given UUID's date. Defaults to 0. + offset_days: Number of months to offset from the given UUID's date. Defaults to 1 to handle if + partitions are not being used, if so the value will be the one set at FINDINGS_TABLE_PARTITION_MONTHS. Returns: - A UUIDv7 object representing the end of the day for the given UUID's date plus offset_days. + A UUIDv7 object representing the end of the month for the given UUID's date plus offset_months. """ - end_of_day = datetime_from_uuid7(uuid_obj).replace( - hour=0, minute=0, second=0, microsecond=0 + end_of_month = datetime_from_uuid7(uuid_obj).replace( + day=1, hour=0, minute=0, second=0, microsecond=0 ) - end_of_day += relativedelta(days=(offset_days + 1), microseconds=-1) - return datetime_to_uuid7(end_of_day) + end_of_month += relativedelta(months=offset_months, microseconds=-1) + return datetime_to_uuid7(end_of_month) def uuid7_range(uuid_list: list[UUID]) -> list[UUID]: diff --git a/src/backend/config/settings/partitions.py b/src/backend/config/settings/partitions.py index 7647e36528..caceee70c3 100644 --- a/src/backend/config/settings/partitions.py +++ b/src/backend/config/settings/partitions.py @@ -3,14 +3,14 @@ from config.env import env # Partitioning PSQLEXTRA_PARTITIONING_MANAGER = "api.partitions.manager" -# Set the days for each partition. Setting the partition days to 30 will create partitions with a size of 1 month. -FINDINGS_TABLE_PARTITION_DAYS = env.int("FINDINGS_TABLE_PARTITION_DAYS", 30) +# Set the months for each partition. Setting the partition months to 1 will create partitions with a size of 1 natural month. +FINDINGS_TABLE_PARTITION_MONTHS = env.int("FINDINGS_TABLE_PARTITION_MONTHS", 1) # Set the number of partitions to create FINDINGS_TABLE_PARTITION_COUNT = env.int("FINDINGS_TABLE_PARTITION_COUNT", 7) -# Set the number of days to keep partitions before deleting them +# Set the number of months to keep partitions before deleting them # Setting this to None will keep partitions indefinitely -FINDINGS_TABLE_PARTITION_MAX_AGE_DAYS = env.int( - "FINDINGS_TABLE_PARTITION_MAX_AGE_DAYS", None +FINDINGS_TABLE_PARTITION_MAX_AGE_MONTHS = env.int( + "FINDINGS_TABLE_PARTITION_MAX_AGE_MONTHS", None )