ben.carrasco/prowler
1
0
Fork 0
mirror of https://github.com/prowler-cloud/prowler.git synced 2026-07-23 12:31:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore(partitions): add env to create partitions (#68)

Browse Source 
* fix(partitions): Use calendar months

* fix: unit to get partition datetime

* fix: imports

* fix: format

* chore: merge

* fix(partitions): Only allow month as unit

* fix(uuid7_end): default months to 1

* test: fix test_uuid7_end

* test: reset expected dt to start of month

* fix: tests uuid utils

* docs: we only allow months

---------

Co-authored-by: Víctor Fernández Poyatos <victor@prowler.com>
This commit is contained in:
Pepe Fagoaga
2024-11-08 09:34:32 +01:00
committed by GitHub
parent d8d831c2a0
commit 95dc87a91b
6 changed files with 58 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/partitions.md
+17 -16
View File
@@ -34,31 +34,32 @@ By default, the command will prompt you to accept the changes before applying th
```shell
Finding:
+ 2024_oct_18
name: 2024_oct_18
from_values: 01929cec-8800-7988-a49d-54cbb64b2d3f
to_values: 0193376b-4c18-7ff7-99d5-95b8479ce39f
size_unit: days
size_value: 30
+ 2024_nov_17
name: 2024_nov_17
from_values: 0193376b-5000-7414-bbae-b1da382332f9
to_values: 0193d1ea-1418-7b00-93fa-c4e06bf36bbe
size_unit: days
size_value: 30
+ 2024_nov
name: 2024_nov
from_values: 0192e505-9000-72c8-a47c-cce719d8fb93
to_values: 01937f84-5418-7eb8-b2a6-e3be749e839d
size_unit: months
size_value: 1
+ 2024_dec
name: 2024_dec
from_values: 01937f84-5800-7b55-879c-9cdb46f023f6
to_values: 01941f29-7818-7f9f-b4be-20b05bb2f574
size_unit: months
size_value: 1
0 partitions will be deleted
2 partitions will be created
```
If you choose to apply the partitions, tables will be generated with the following format: `<table_name>_<year>_<month>_<day>`. The date in the table name shows the first date of the partition.
If you choose to apply the partitions, tables will be generated with the following format: `<table_name>_<year>_<month>`.
For more info on the partitioning manager, see https://github.com/SectorLabs/django-postgres-extra
### Changing the Partitioning Parameters
There are 3 environment variables that can be used to change the partitioning parameters:
There are 4 environment variables that can be used to change the partitioning parameters:
- `FINDINGS_TABLE_PARTITION_DAYS`: Set the days for each partition. Setting the partition days to 30 will create partitions with a size of 1 month.
- `DJANGO_MANAGE_DB_PARTITIONS`: Allow Django to manage database partitons. By default is set to `False`.
- `FINDINGS_TABLE_PARTITION_MONTHS`: Set the months for each partition. Setting the partition monts to 1 will create partitions with a size of 1 natural month.
- `FINDINGS_TABLE_PARTITION_COUNT`: Set the number of partitions to create
- `FINDINGS_TABLE_PARTITION_MAX_AGE_DAYS`: Set the number of days to keep partitions before deleting them. Setting this to `None` will keep partitions indefinitely.
- `FINDINGS_TABLE_PARTITION_MAX_AGE_MONTHS`: Set the number of months to keep partitions before deleting them. Setting this to `None` will keep partitions indefinitely.
src/backend/api/filters.py
+1 -2
View File
@@ -321,12 +321,11 @@ class FindingFilter(FilterSet):
}
# Convert filter values to UUIDv7 values for use with partitioning
def filter_scan_id(self, queryset, name, value):
try:
value_uuid = transform_into_uuid7(value)
start = uuid7_start(value_uuid)
end = uuid7_end(value_uuid, settings.FINDINGS_TABLE_PARTITION_DAYS)
end = uuid7_end(value_uuid, settings.FINDINGS_TABLE_PARTITION_MONTHS)
except ValidationError as validation_error:
detail = str(validation_error.detail[0])
raise ValidationError(
src/backend/api/partitions.py
+19 -8
View File
@@ -86,7 +86,9 @@ class PostgresUUIDv7PartitioningStrategy(PostgresRangePartitioningStrategy):
name_format: Optional[str] = None,
**kwargs,
) -> None:
self.start_date = start_date.replace(hour=0, minute=0, second=0, microsecond=0)
self.start_date = start_date.replace(
day=1, hour=0, minute=0, second=0, microsecond=0
)
self.size = size
self.count = count
self.max_age = max_age
@@ -140,8 +142,17 @@ class PostgresUUIDv7PartitioningStrategy(PostgresRangePartitioningStrategy):
current_datetime -= self.size.as_delta()
def get_start_datetime(self) -> datetime:
"""
Gets the start of the current month in UTC timezone.
This function returns a `datetime` object set to the first day of the current
month, at midnight (00:00:00), in UTC.
Returns:
datetime: A `datetime` object representing the start of the current month in UTC.
"""
return datetime.now(timezone.utc).replace(
hour=0, minute=0, second=0, microsecond=0
day=1, hour=0, minute=0, second=0, microsecond=0
)
@@ -162,13 +173,13 @@ manager = PostgresPartitioningManager(
strategy=PostgresUUIDv7PartitioningStrategy(
start_date=datetime.now(timezone.utc),
size=PostgresTimePartitionSize(
days=settings.FINDINGS_TABLE_PARTITION_DAYS
months=settings.FINDINGS_TABLE_PARTITION_MONTHS
),
count=settings.FINDINGS_TABLE_PARTITION_COUNT,
max_age=relative_days_or_none(
settings.FINDINGS_TABLE_PARTITION_MAX_AGE_DAYS
settings.FINDINGS_TABLE_PARTITION_MAX_AGE_MONTHS
),
name_format="%Y_%b_%d",
name_format="%Y_%b",
rls_statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
),
),
@@ -178,13 +189,13 @@ manager = PostgresPartitioningManager(
strategy=PostgresUUIDv7PartitioningStrategy(
start_date=datetime.now(timezone.utc),
size=PostgresTimePartitionSize(
days=settings.FINDINGS_TABLE_PARTITION_DAYS
months=settings.FINDINGS_TABLE_PARTITION_MONTHS
),
count=settings.FINDINGS_TABLE_PARTITION_COUNT,
max_age=relative_days_or_none(
settings.FINDINGS_TABLE_PARTITION_MAX_AGE_DAYS
settings.FINDINGS_TABLE_PARTITION_MAX_AGE_MONTHS
),
name_format="%Y_%b_%d",
name_format="%Y_%b",
rls_statements=["SELECT"],
),
),
src/backend/api/tests/test_uuid_utils.py
+7 -7
View File
@@ -74,13 +74,13 @@ def test_uuid7_start():
assert start_uuid.version == 7
@pytest.mark.parametrize("days_offset", [0, 1, 10, 30, 60])
def test_uuid7_end(days_offset):
@pytest.mark.parametrize("months_offset", [0, 1, 10, 30, 60])
def test_uuid7_end(months_offset):
dt = datetime.now(timezone.utc)
uuid = datetime_to_uuid7(dt)
end_uuid = uuid7_end(uuid, days_offset)
expected_dt = dt.replace(hour=0, minute=0, second=0, microsecond=0)
expected_dt += relativedelta(days=(days_offset + 1), microseconds=-1)
end_uuid = uuid7_end(uuid, months_offset)
expected_dt = dt.replace(day=1, hour=0, minute=0, second=0, microsecond=0)
expected_dt += relativedelta(months=months_offset, microseconds=-1)
expected_timestamp_ms = int(expected_dt.timestamp() * 1000) & 0xFFFFFFFFFFFF
assert end_uuid.time == expected_timestamp_ms
assert end_uuid.version == 7
@@ -103,8 +103,8 @@ def test_uuid7_range():
# Expected end of range
end_dt = datetime_from_uuid7(max(uuid_list, key=lambda u: u.time))
end_dt = end_dt.replace(hour=0, minute=0, second=0, microsecond=0)
end_dt += relativedelta(days=1, microseconds=-1)
end_dt = end_dt.replace(day=1, hour=0, minute=0, second=0, microsecond=0)
end_dt += relativedelta(months=1, microseconds=-1)
expected_end_timestamp_ms = int(end_dt.timestamp() * 1000) & 0xFFFFFFFFFFFF
assert start_uuid.time == expected_start_timestamp_ms
src/backend/api/uuid_utils.py
+9 -8
View File
@@ -100,22 +100,23 @@ def uuid7_start(uuid_obj: UUID) -> UUID:
return datetime_to_uuid7(start_of_day)
def uuid7_end(uuid_obj: UUID, offset_days: int = 0) -> UUID:
def uuid7_end(uuid_obj: UUID, offset_months: int = 1) -> UUID:
"""
Returns a UUIDv7 that represents the end of the day for the given UUID.
Returns a UUIDv7 that represents the end of the month for the given UUID.
Args:
uuid_obj: A UUIDv7 object.
offset_days: Number of days to offset from the given UUID's date. Defaults to 0.
offset_days: Number of months to offset from the given UUID's date. Defaults to 1 to handle if
partitions are not being used, if so the value will be the one set at FINDINGS_TABLE_PARTITION_MONTHS.
Returns:
A UUIDv7 object representing the end of the day for the given UUID's date plus offset_days.
A UUIDv7 object representing the end of the month for the given UUID's date plus offset_months.
"""
end_of_day = datetime_from_uuid7(uuid_obj).replace(
hour=0, minute=0, second=0, microsecond=0
end_of_month = datetime_from_uuid7(uuid_obj).replace(
day=1, hour=0, minute=0, second=0, microsecond=0
)
end_of_day += relativedelta(days=(offset_days + 1), microseconds=-1)
return datetime_to_uuid7(end_of_day)
end_of_month += relativedelta(months=offset_months, microseconds=-1)
return datetime_to_uuid7(end_of_month)
def uuid7_range(uuid_list: list[UUID]) -> list[UUID]:
src/backend/config/settings/partitions.py
+5 -5
View File
@@ -3,14 +3,14 @@ from config.env import env
# Partitioning
PSQLEXTRA_PARTITIONING_MANAGER = "api.partitions.manager"
# Set the days for each partition. Setting the partition days to 30 will create partitions with a size of 1 month.
FINDINGS_TABLE_PARTITION_DAYS = env.int("FINDINGS_TABLE_PARTITION_DAYS", 30)
# Set the months for each partition. Setting the partition months to 1 will create partitions with a size of 1 natural month.
FINDINGS_TABLE_PARTITION_MONTHS = env.int("FINDINGS_TABLE_PARTITION_MONTHS", 1)
# Set the number of partitions to create
FINDINGS_TABLE_PARTITION_COUNT = env.int("FINDINGS_TABLE_PARTITION_COUNT", 7)
# Set the number of days to keep partitions before deleting them
# Set the number of months to keep partitions before deleting them
# Setting this to None will keep partitions indefinitely
FINDINGS_TABLE_PARTITION_MAX_AGE_DAYS = env.int(
"FINDINGS_TABLE_PARTITION_MAX_AGE_DAYS", None
FINDINGS_TABLE_PARTITION_MAX_AGE_MONTHS = env.int(
"FINDINGS_TABLE_PARTITION_MAX_AGE_MONTHS", None
)