chore(Filters): PRWLR-4948 Refactor filter module to reduce complexity and improve docs (#50)

* chore(Django-filter): PRWLR-4948 bump django-filter version to 24.3

* fix(Middleware): PRWLR-4948 fix logging middleware when auth is not present in request

* fix(Task): PRWLR-4948 fix serializer state values

* chore(Filters): PRWLR-4948 use ChoiceFilter instead of implementing method for basic filters

* chore(Filters): PRWLR-4948 refactor complex filters to reduce complexity
This commit is contained in:
Víctor Fernández Poyatos authored and GitHub committed 2024-10-04 14:13:04 +02:00
1 parent 6bd8a17a5f
commit d98b716dfc
5 files changed
+116 -300

No files matched your search

+109 -293
View File
@@ -4,16 +4,23 @@ from uuid import UUID
from django.conf import settings
from django.db.models import Q
from django_filters.rest_framework import (
BaseInFilter,
FilterSet,
BooleanFilter,
CharFilter,
UUIDFilter,
DateFilter,
ChoiceFilter,
)
from rest_framework_json_api.django_filters.backends import DjangoFilterBackend
from rest_framework_json_api.serializers import ValidationError
from api.db_utils import ProviderEnumField
from api.db_utils import (
ProviderEnumField,
FindingDeltaEnumField,
StatusEnumField,
SeverityEnumField,
)
from api.models import (
Membership,
Provider,
@@ -37,55 +44,6 @@ from api.uuid_utils import (
from api.v1.serializers import TaskBase
def enum_filter(queryset, value, enum_choices, lookup_field: str):
"""
Filter a queryset based on a provided value, using a specified lookup field
and validating against a given enumeration.
This function filters a given queryset by checking if the provided `value`
matches a valid choice in the specified `enum_choices` (a Django `TextChoices` enum).
If the `value` is valid, the queryset is filtered using the specified `lookup_field`.
Otherwise, a `ValidationError` is raised.
Args:
queryset (QuerySet): The Django queryset to be filtered.
value (str): The value to filter the queryset by, which must be a valid
option in the specified `enum_choices`.
enum_choices: The enumeration class that defines the valid
choices for the `value`.
lookup_field (str): The field or lookup path within the model used for
filtering the queryset.
Returns:
QuerySet: A filtered queryset based on the provided `value` and `lookup_field`.
Raises:
ValidationError: If the provided `value` is not a valid choice in
the specified `enum_choices`.
"""
if "__in" in lookup_field:
values = value
if isinstance(values, str):
values = value.split(",")
values = [v for v in values if v in enum_choices]
return queryset.filter(**{lookup_field: values})
if value not in enum_choices:
raise ValidationError(
f"Invalid provider value: '{value}'. Valid values are: "
f"{', '.join(enum_choices)}"
)
return queryset.filter(**{lookup_field: value})
def extract_lookup_expr(name):
"""Extract the lookup expression from the filter name."""
parts = name.split("__")
return parts[-1] if len(parts) > 1 else "exact"
class CustomDjangoFilterBackend(DjangoFilterBackend):
def to_html(self, _request, _queryset, _view):
"""Override this method to use the Browsable API in dev environments.
@@ -95,6 +53,18 @@ class CustomDjangoFilterBackend(DjangoFilterBackend):
return None
class UUIDInFilter(BaseInFilter, UUIDFilter):
pass
class CharInFilter(BaseInFilter, CharFilter):
pass
class ChoiceInFilter(BaseInFilter, ChoiceFilter):
pass
class TenantFilter(FilterSet):
inserted_at = DateFilter(field_name="inserted_at", lookup_expr="date")
updated_at = DateFilter(field_name="updated_at", lookup_expr="date")
@@ -110,15 +80,7 @@ class TenantFilter(FilterSet):
class MembershipFilter(FilterSet):
date_joined = DateFilter(field_name="date_joined", lookup_expr="date")
role = CharFilter(method="filter_role")
def filter_role(self, queryset, name, value):
return enum_filter(
queryset,
value,
enum_choices=Membership.RoleChoices,
lookup_field="role",
)
role = ChoiceFilter(choices=Membership.RoleChoices.choices)
class Meta:
model = Membership
@@ -133,24 +95,7 @@ class ProviderFilter(FilterSet):
inserted_at = DateFilter(field_name="inserted_at", lookup_expr="date")
updated_at = DateFilter(field_name="updated_at", lookup_expr="date")
connected = BooleanFilter()
provider = CharFilter(method="filter_provider_type")
provider__in = CharFilter(method="filter_provider_type_in")
def filter_provider_type(self, queryset, name, value):
return enum_filter(
queryset,
value,
enum_choices=Provider.ProviderChoices,
lookup_field="provider",
)
def filter_provider_type_in(self, queryset, name, value):
return enum_filter(
queryset,
value,
enum_choices=Provider.ProviderChoices,
lookup_field="provider__in",
)
provider = ChoiceFilter(choices=Provider.ProviderChoices.choices)
class Meta:
model = Provider
@@ -170,81 +115,29 @@ class ProviderFilter(FilterSet):
class ProviderRelationshipFilterSet(FilterSet):
provider_type = CharFilter(method="filter_provider_type")
provider_type__in = CharFilter(method="filter_provider_type_in")
provider_uid = CharFilter(method="filter_provider_uid")
provider_uid__in = CharFilter(method="filter_provider_uid_in")
provider_uid__icontains = CharFilter(method="filter_provider_uid_icontains")
provider_alias = CharFilter(method="filter_provider_alias")
provider_alias__in = CharFilter(method="filter_provider_alias_in")
provider_alias__icontains = CharFilter(method="filter_provider_alias_icontains")
# this can be overridden in subclasses
provider_type_lookup_field = "provider__provider__exact"
provider_type_in_lookup_field = "provider__provider__in"
provider_uid_lookup_field = "provider__uid"
provider_uid_in_lookup_field = "provider__uid__in"
provider_uid_icontains_lookup_field = "provider__uid__icontains"
provider_alias_lookup_field = "provider__alias"
provider_alias_in_lookup_field = "provider__alias__in"
provider_alias_icontains_lookup_field = "provider__alias__icontains"
def filter_provider_type(self, queryset, name, value):
return enum_filter(
queryset,
value,
enum_choices=Provider.ProviderChoices,
lookup_field=self.provider_type_lookup_field,
)
def filter_provider_type_in(self, queryset, name, value):
if not isinstance(value, list):
value = value.split(",")
return enum_filter(
queryset,
value,
enum_choices=Provider.ProviderChoices,
lookup_field=self.provider_type_in_lookup_field,
)
def filter_provider_uid(self, queryset, name, value):
return queryset.filter(**{self.provider_uid_lookup_field: value})
def filter_provider_uid_in(self, queryset, name, value):
if isinstance(value, str):
value = value.split(",")
return queryset.filter(**{self.provider_uid_in_lookup_field: value})
def filter_provider_uid_icontains(self, queryset, name, value):
return queryset.filter(**{self.provider_uid_icontains_lookup_field: value})
def filter_provider_alias(self, queryset, name, value):
return queryset.filter(**{self.provider_alias_lookup_field: value})
def filter_provider_alias_in(self, queryset, name, value):
if isinstance(value, str):
value = value.split(",")
return queryset.filter(**{self.provider_alias_in_lookup_field: value})
def filter_provider_alias_icontains(self, queryset, name, value):
return queryset.filter(**{self.provider_alias_icontains_lookup_field: value})
provider_type = ChoiceFilter(
choices=Provider.ProviderChoices.choices, field_name="provider__provider"
)
provider_type__in = ChoiceInFilter(
choices=Provider.ProviderChoices.choices, field_name="provider__provider"
)
provider_uid = CharFilter(field_name="provider__uid", lookup_expr="exact")
provider_uid__in = CharInFilter(field_name="provider__uid", lookup_expr="in")
provider_uid__icontains = CharFilter(
field_name="provider__uid", lookup_expr="icontains"
)
provider_alias = CharFilter(field_name="provider__alias", lookup_expr="exact")
provider_alias__in = CharInFilter(field_name="provider__alias", lookup_expr="in")
provider_alias__icontains = CharFilter(
field_name="provider__alias", lookup_expr="icontains"
)
class ScanFilter(ProviderRelationshipFilterSet):
inserted_at = DateFilter(field_name="inserted_at", lookup_expr="date")
completed_at = DateFilter(field_name="completed_at", lookup_expr="date")
started_at = DateFilter(field_name="started_at", lookup_expr="date")
trigger = CharFilter(method="filter_trigger")
def filter_trigger(self, queryset, name, value):
if value not in Scan.TriggerChoices:
raise ValidationError(
f"Invalid scan trigger value: '{value}'. Valid values are: "
f"{', '.join(Scan.TriggerChoices)}"
)
return queryset.filter(trigger=value)
trigger = ChoiceFilter(choices=Scan.TriggerChoices.choices)
class Meta:
model = Scan
@@ -261,8 +154,9 @@ class TaskFilter(FilterSet):
name__icontains = CharFilter(
field_name="task_runner_task__task_name", lookup_expr="icontains"
)
state = CharFilter(method="filter_state", lookup_expr="exact")
state = ChoiceFilter(
choices=StateChoices.choices, method="filter_state", lookup_expr="exact"
)
task_state_inverse_mapping_values = {
v: k for k, v in TaskBase.state_mapping.items()
}
@@ -321,55 +215,73 @@ class ResourceFilter(ProviderRelationshipFilterSet):
return queryset.filter(Q(tags__value=value) | Q(tags__value__icontains=value))
def filter_tag(self, queryset, name, value):
# we won't know what the user wants to filter on just based on the value
# We won't know what the user wants to filter on just based on the value,
# and we don't want to build special filtering logic for every possible
# provider tag spec, so we'll just do a full text search
return queryset.filter(tags__text_search=value)
class FindingFilter(ProviderRelationshipFilterSet):
class FindingFilter(FilterSet):
# We filter providers from the scan in findings
provider = UUIDFilter(field_name="scan__provider__id", lookup_expr="exact")
provider__in = UUIDFilter(method="filter_provider_id_in")
def filter_provider_id_in(self, queryset, name, value):
if isinstance(value, str):
value = value.split(",")
if isinstance(value, UUID):
value = [value]
return queryset.filter(scan__provider__id__in=value)
provider__in = UUIDInFilter(field_name="scan__provider__id", lookup_expr="in")
provider_type = ChoiceFilter(
choices=Provider.ProviderChoices.choices, field_name="scan__provider__provider"
)
provider_type__in = ChoiceInFilter(
choices=Provider.ProviderChoices.choices, field_name="scan__provider__provider"
)
provider_uid = CharFilter(field_name="scan__provider__uid", lookup_expr="exact")
provider_uid__in = CharInFilter(field_name="scan__provider__uid", lookup_expr="in")
provider_uid__icontains = CharFilter(
field_name="scan__provider__uid", lookup_expr="icontains"
)
provider_alias = CharFilter(field_name="scan__provider__alias", lookup_expr="exact")
provider_alias__in = CharInFilter(
field_name="scan__provider__alias", lookup_expr="in"
)
provider_alias__icontains = CharFilter(
field_name="scan__provider__alias", lookup_expr="icontains"
)
updated_at = DateFilter(field_name="updated_at", lookup_expr="date")
delta = CharFilter(method="filter_delta")
delta__in = CharFilter(method="filter_delta_in")
status = CharFilter(method="filter_status")
status__in = CharFilter(method="filter_status_in")
severity = CharFilter(method="filter_severity")
severity__in = CharFilter(method="filter_severity_in")
impact = CharFilter(method="filter_severity")
impact__in = CharFilter(method="filter_severity_in")
delta = ChoiceFilter(choices=Finding.DeltaChoices.choices)
status = ChoiceFilter(choices=StatusChoices.choices)
severity = ChoiceFilter(choices=SeverityChoices)
impact = ChoiceFilter(choices=SeverityChoices)
resources = UUIDFilter(field_name="resource__id", lookup_expr="in")
resources = UUIDInFilter(field_name="resource__id", lookup_expr="in")
region = CharFilter(method="filter_region")
region__in = CharFilter(method="filter_region_in")
region__icontains = CharFilter(method="filter_region_icontains")
region = CharFilter(field_name="resources__region")
region__in = CharInFilter(field_name="resources__region", lookup_expr="in")
region__icontains = CharFilter(
field_name="resources__region", lookup_expr="icontains"
)
service = CharFilter(method="filter_service")
service__in = CharFilter(method="filter_service_in")
service__icontains = CharFilter(method="filter_service_icontains")
service = CharFilter(field_name="resources__service")
service__in = CharInFilter(field_name="resources__service", lookup_expr="in")
service__icontains = CharFilter(
field_name="resources__service", lookup_expr="icontains"
)
resource_uid = CharFilter(method="filter_resource_uid")
resource_uid__in = CharFilter(method="filter_resource_uid_in")
resource_uid__icontains = CharFilter(method="filter_resource_uid_icontains")
resource_uid = CharFilter(field_name="resources__uid")
resource_uid__in = CharInFilter(field_name="resources__uid", lookup_expr="in")
resource_uid__icontains = CharFilter(
field_name="resources__uid", lookup_expr="icontains"
)
resource_name = CharFilter(method="filter_resource_name")
resource_name__in = CharFilter(method="filter_resource_name_in")
resource_name__icontains = CharFilter(method="filter_resource_name_icontains")
resource_name = CharFilter(field_name="resources__name")
resource_name__in = CharInFilter(field_name="resources__name", lookup_expr="in")
resource_name__icontains = CharFilter(
field_name="resources__name", lookup_expr="icontains"
)
resource_type = CharFilter(method="filter_resource_type")
resource_type__in = CharFilter(method="filter_resource_type_in")
resource_type__icontains = CharFilter(method="filter_resource_type_icontains")
resource_type = CharFilter(field_name="resources__type")
resource_type__in = CharInFilter(field_name="resources__type", lookup_expr="in")
resource_type__icontains = CharFilter(
field_name="resources__type", lookup_expr="icontains"
)
scan = CharFilter(method="filter_scan_id")
scan__in = CharFilter(method="filter_scan_id_in")
@@ -383,20 +295,27 @@ class FindingFilter(ProviderRelationshipFilterSet):
model = Finding
fields = {
"scan": ["exact", "in"],
"delta": ["exact", "in"],
"status": ["exact", "in"],
"severity": ["exact", "in"],
"impact": ["exact", "in"],
"check_id": ["exact", "in", "icontains"],
"inserted_at": ["date", "gte", "lte"],
"updated_at": ["gte", "lte"],
}
filter_overrides = {
FindingDeltaEnumField: {
"filter_class": CharFilter,
},
StatusEnumField: {
"filter_class": CharFilter,
},
SeverityEnumField: {
"filter_class": CharFilter,
},
}
def __init__(self, *args, **kwargs):
self.provider_type_lookup_field = "scan__provider__provider__exact"
self.provider_type_in_lookup_field = "scan__provider__provider__in"
self.provider_uid_lookup_field = "scan__provider__uid"
self.provider_uid_in_lookup_field = "scan__provider__uid__in"
self.provider_uid_icontains_lookup_field = "scan__provider__uid__icontains"
self.provider_alias_lookup_field = "scan__provider__alias"
self.provider_alias_in_lookup_field = "scan__provider__alias__in"
self.provider_alias_icontains_lookup_field = "scan__provider__alias__icontains"
super().__init__(*args, **kwargs)
data = self.data
@@ -416,109 +335,6 @@ class FindingFilter(ProviderRelationshipFilterSet):
self.queryset = self.queryset.filter(id__gte=start)
def filter_delta(self, queryset, name, value):
return enum_filter(
queryset,
value,
enum_choices=Finding.DeltaChoices,
lookup_field="delta",
)
def filter_delta_in(self, queryset, name, value):
return enum_filter(
queryset,
value,
enum_choices=Finding.DeltaChoices,
lookup_field="delta__in",
)
def filter_severity(self, queryset, name, value):
return enum_filter(
queryset,
value,
enum_choices=SeverityChoices,
lookup_field="severity",
)
def filter_severity_in(self, queryset, name, value):
return enum_filter(
queryset,
value,
enum_choices=SeverityChoices,
lookup_field="severity__in",
)
def filter_status(self, queryset, name, value):
return enum_filter(
queryset,
value,
enum_choices=StatusChoices,
lookup_field="status",
)
def filter_status_in(self, queryset, name, value):
return enum_filter(
queryset,
value,
enum_choices=StatusChoices,
lookup_field="status__in",
)
def filter_region(self, queryset, name, value):
return queryset.filter(resources__region=value)
def filter_region_in(self, queryset, name, value):
if isinstance(value, str):
value = value.split(",")
return queryset.filter(resources__region__in=value)
def filter_region_icontains(self, queryset, name, value):
return queryset.filter(resources__region__icontains=value)
def filter_service(self, queryset, name, value):
return queryset.filter(resources__service=value)
def filter_service_in(self, queryset, name, value):
if isinstance(value, str):
value = value.split(",")
return queryset.filter(resources__service__in=value)
def filter_service_icontains(self, queryset, name, value):
return queryset.filter(resources__service__icontains=value)
def filter_resource_uid(self, queryset, name, value):
return queryset.filter(resources__uid=value)
def filter_resource_uid_in(self, queryset, name, value):
if isinstance(value, str):
value = value.split(",")
return queryset.filter(resources__uid__in=value)
def filter_resource_uid_icontains(self, queryset, name, value):
return queryset.filter(resources__uid__icontains=value)
def filter_resource_name(self, queryset, name, value):
return queryset.filter(resources__name=value)
def filter_resource_name_in(self, queryset, name, value):
if isinstance(value, str):
value = value.split(",")
return queryset.filter(resources__name__in=value)
def filter_resource_name_icontains(self, queryset, name, value):
return queryset.filter(resources__name__icontains=value)
def filter_resource_type(self, queryset, name, value):
return queryset.filter(resources__type=value)
def filter_resource_type_in(self, queryset, name, value):
if isinstance(value, str):
value = value.split(",")
return queryset.filter(resources__type__in=value)
def filter_resource_type_icontains(self, queryset, name, value):
return queryset.filter(resources__type__icontains=value)
# Convert filter values to UUIDv7 values for use with partitioning
def filter_scan_id(self, queryset, name, value):
+1 -1
View File
@@ -5,7 +5,7 @@ from config.custom_logging import BackendLogger
def extract_auth_info(request) -> dict:
if not hasattr(request, "auth") or request.auth is not None:
if getattr(request, "auth", None) is not None:
tenant_id = request.auth.get("tenant_id", "N/A")
user_id = request.auth.get("user_id", "N/A")
else:
+1 -1
View File
@@ -217,7 +217,7 @@ class UserUpdateSerializer(BaseWriteSerializer):
# Tasks
class TaskBase(serializers.ModelSerializer):
state_mapping = {
"PENDING": StateChoices.SCHEDULED,
"PENDING": StateChoices.AVAILABLE,
"STARTED": StateChoices.EXECUTING,
"PROGRESS": StateChoices.EXECUTING,
"SUCCESS": StateChoices.COMPLETED,