Merge branch 'master' into PRWLR-5956-Export-Artifacts-only

api/src/backend/api/filters.py

@@ -1,4 +1,4 @@
-from datetime import date, datetime, timezone
+from datetime import date, datetime, timedelta, timezone
 
 from django.conf import settings
 from django.db.models import Q
@@ -346,8 +346,14 @@ class FindingFilter(FilterSet):
 
     inserted_at = DateFilter(method="filter_inserted_at", lookup_expr="date")
     inserted_at__date = DateFilter(method="filter_inserted_at", lookup_expr="date")
-    inserted_at__gte = DateFilter(method="filter_inserted_at_gte")
-    inserted_at__lte = DateFilter(method="filter_inserted_at_lte")
+    inserted_at__gte = DateFilter(
+        method="filter_inserted_at_gte",
+        help_text=f"Maximum date range is {settings.FINDINGS_MAX_DAYS_IN_RANGE} days.",
+    )
+    inserted_at__lte = DateFilter(
+        method="filter_inserted_at_lte",
+        help_text=f"Maximum date range is {settings.FINDINGS_MAX_DAYS_IN_RANGE} days.",
+    )
 
     class Meta:
         model = Finding
@@ -375,6 +381,52 @@ class FindingFilter(FilterSet):
             },
         }
 
+    def filter_queryset(self, queryset):
+        if not (self.data.get("scan") or self.data.get("scan__in")) and not (
+            self.data.get("inserted_at")
+            or self.data.get("inserted_at__date")
+            or self.data.get("inserted_at__gte")
+            or self.data.get("inserted_at__lte")
+        ):
+            raise ValidationError(
+                [
+                    {
+                        "detail": "At least one date filter is required: filter[inserted_at], filter[inserted_at.gte], "
+                        "or filter[inserted_at.lte].",
+                        "status": 400,
+                        "source": {"pointer": "/data/attributes/inserted_at"},
+                        "code": "required",
+                    }
+                ]
+            )
+
+        gte_date = (
+            datetime.strptime(self.data.get("inserted_at__gte"), "%Y-%m-%d").date()
+            if self.data.get("inserted_at__gte")
+            else datetime.now(timezone.utc).date()
+        )
+        lte_date = (
+            datetime.strptime(self.data.get("inserted_at__lte"), "%Y-%m-%d").date()
+            if self.data.get("inserted_at__lte")
+            else datetime.now(timezone.utc).date()
+        )
+
+        if abs(lte_date - gte_date) > timedelta(
+            days=settings.FINDINGS_MAX_DAYS_IN_RANGE
+        ):
+            raise ValidationError(
+                [
+                    {
+                        "detail": f"The date range cannot exceed {settings.FINDINGS_MAX_DAYS_IN_RANGE} days.",
+                        "status": 400,
+                        "source": {"pointer": "/data/attributes/inserted_at"},
+                        "code": "invalid",
+                    }
+                ]
+            )
+
+        return super().filter_queryset(queryset)
+
     #  Convert filter values to UUIDv7 values for use with partitioning
     def filter_scan_id(self, queryset, name, value):
         try:

api/src/backend/api/migrations/0009_increase_provider_uid_maximum_length.py

@@ -0,0 +1,22 @@
+# Generated by Django 5.1.5 on 2025-02-07 09:42
+
+import django.core.validators
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("api", "0008_daily_scheduled_tasks_update"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="provider",
+            name="uid",
+            field=models.CharField(
+                max_length=250,
+                validators=[django.core.validators.MinLengthValidator(3)],
+                verbose_name="Unique identifier for the provider, set by the provider",
+            ),
+        ),
+    ]

api/src/backend/api/models.py

@@ -227,13 +227,13 @@ class Provider(RowLevelSecurityProtectedModel):
     @staticmethod
     def validate_kubernetes_uid(value):
         if not re.match(
-            r"(^[a-z0-9]([-a-z0-9]{1,61}[a-z0-9])?$)|(^arn:aws(-cn|-us-gov|-iso|-iso-b)?:[a-zA-Z0-9\-]+:([a-z]{2}-[a-z]+-\d{1})?:(\d{12})?:[a-zA-Z0-9\-_\/:\.\*]+(:\d+)?$)",
+            r"^[a-z0-9][A-Za-z0-9_.:\/-]{1,250}$",
             value,
         ):
             raise ModelValidationError(
                 detail="The value must either be a valid Kubernetes UID (up to 63 characters, "
                 "starting and ending with a lowercase letter or number, containing only "
-                "lowercase alphanumeric characters and hyphens) or a valid EKS ARN.",
+                "lowercase alphanumeric characters and hyphens) or a valid AWS EKS Cluster ARN, GCP GKE Context Name or Azure AKS Cluster Name.",
                 code="kubernetes-uid",
                 pointer="/data/attributes/uid",
             )
@@ -247,7 +247,7 @@ class Provider(RowLevelSecurityProtectedModel):
     )
     uid = models.CharField(
         "Unique identifier for the provider, set by the provider",
-        max_length=63,
+        max_length=250,
         blank=False,
         validators=[MinLengthValidator(3)],
     )

api/src/backend/api/specs/v1.yaml

@@ -346,6 +346,9 @@ paths:
         schema:
           type: string
           format: date
+        description: At least one of the variations of the `filter[inserted_at]` filter
+          must be provided.
+        required: true
       - in: query
         name: filter[inserted_at__date]
         schema:
@@ -356,11 +359,13 @@ paths:
         schema:
           type: string
           format: date
+        description: Maximum date range is 7 days.
       - in: query
         name: filter[inserted_at__lte]
         schema:
           type: string
           format: date
+        description: Maximum date range is 7 days.
       - in: query
         name: filter[provider]
         schema:
@@ -861,11 +866,13 @@ paths:
         schema:
           type: string
           format: date
+        description: Maximum date range is 7 days.
       - in: query
         name: filter[inserted_at__lte]
         schema:
           type: string
           format: date
+        description: Maximum date range is 7 days.
       - in: query
         name: filter[provider]
         schema:
@@ -1275,6 +1282,9 @@ paths:
         schema:
           type: string
           format: date
+        description: At least one of the variations of the `filter[inserted_at]` filter
+          must be provided.
+        required: true
       - in: query
         name: filter[inserted_at__date]
         schema:
@@ -1285,11 +1295,13 @@ paths:
         schema:
           type: string
           format: date
+        description: Maximum date range is 7 days.
       - in: query
         name: filter[inserted_at__lte]
         schema:
           type: string
           format: date
+        description: Maximum date range is 7 days.
       - in: query
         name: filter[provider]
         schema:
@@ -7665,7 +7677,7 @@ components:
             uid:
               type: string
               title: Unique identifier for the provider, set by the provider
-              maxLength: 63
+              maxLength: 250
               minLength: 3
             alias:
               type: string
@@ -7777,7 +7789,7 @@ components:
             uid:
               type: string
               title: Unique identifier for the provider, set by the provider
-              maxLength: 63
+              maxLength: 250
               minLength: 3
           required:
           - uid
@@ -7821,7 +7833,7 @@ components:
                   type: string
                   minLength: 3
                   title: Unique identifier for the provider, set by the provider
-                  maxLength: 63
+                  maxLength: 250
               required:
               - uid
       required:

api/src/backend/api/tests/test_views.py

@@ -5,6 +5,7 @@ from unittest.mock import ANY, Mock, patch
 import jwt
 import pytest
 from conftest import API_JSON_CONTENT_TYPE, TEST_PASSWORD, TEST_USER
+from django.conf import settings
 from django.urls import reverse
 from rest_framework import status
 
@@ -27,6 +28,12 @@ from api.rls import Tenant
 TODAY = str(datetime.today().date())
 
 
+def today_after_n_days(n_days: int) -> str:
+    return datetime.strftime(
+        datetime.today().date() + timedelta(days=n_days), "%Y-%m-%d"
+    )
+
+
 @pytest.mark.django_db
 class TestUserViewSet:
     def test_users_list(self, authenticated_client, create_test_user):
@@ -878,6 +885,16 @@ class TestProviderViewSet:
                     "uid": "kubernetes-test-123456789",
                     "alias": "test",
                 },
+                {
+                    "provider": "kubernetes",
+                    "uid": "arn:aws:eks:us-east-1:111122223333:cluster/test-cluster-long-name-123456789",
+                    "alias": "EKS",
+                },
+                {
+                    "provider": "kubernetes",
+                    "uid": "gke_aaaa-dev_europe-test1_dev-aaaa-test-cluster-long-name-123456789",
+                    "alias": "GKE",
+                },
                 {
                     "provider": "azure",
                     "uid": "8851db6b-42e5-4533-aa9e-30a32d67e875",
@@ -2379,12 +2396,33 @@ class TestResourceViewSet:
 @pytest.mark.django_db
 class TestFindingViewSet:
     def test_findings_list_none(self, authenticated_client):
-        response = authenticated_client.get(reverse("finding-list"))
+        response = authenticated_client.get(
+            reverse("finding-list"), {"filter[inserted_at]": TODAY}
+        )
         assert response.status_code == status.HTTP_200_OK
         assert len(response.json()["data"]) == 0
 
-    def test_findings_list(self, authenticated_client, findings_fixture):
+    def test_findings_list_no_date_filter(self, authenticated_client):
         response = authenticated_client.get(reverse("finding-list"))
+        assert response.status_code == status.HTTP_400_BAD_REQUEST
+        assert response.json()["errors"][0]["code"] == "required"
+
+    def test_findings_date_range_too_large(self, authenticated_client):
+        response = authenticated_client.get(
+            reverse("finding-list"),
+            {
+                "filter[inserted_at.lte]": today_after_n_days(
+                    -(settings.FINDINGS_MAX_DAYS_IN_RANGE + 1)
+                ),
+            },
+        )
+        assert response.status_code == status.HTTP_400_BAD_REQUEST
+        assert response.json()["errors"][0]["code"] == "invalid"
+
+    def test_findings_list(self, authenticated_client, findings_fixture):
+        response = authenticated_client.get(
+            reverse("finding-list"), {"filter[inserted_at]": TODAY}
+        )
         assert response.status_code == status.HTTP_200_OK
         assert len(response.json()["data"]) == len(findings_fixture)
         assert (
@@ -2404,7 +2442,8 @@ class TestFindingViewSet:
         self, include_values, expected_resources, authenticated_client, findings_fixture
     ):
         response = authenticated_client.get(
-            reverse("finding-list"), {"include": include_values}
+            reverse("finding-list"),
+            {"include": include_values, "filter[inserted_at]": TODAY},
         )
         assert response.status_code == status.HTTP_200_OK
         assert len(response.json()["data"]) == len(findings_fixture)
@@ -2439,13 +2478,13 @@ class TestFindingViewSet:
                 ("service.icontains", "ec", 1),
                 ("inserted_at", "2024-01-01", 0),
                 ("inserted_at.date", "2024-01-01", 0),
-                ("inserted_at.gte", "2024-01-01", 2),
+                ("inserted_at.gte", today_after_n_days(-1), 2),
                 (
                     "inserted_at.lte",
-                    "2028-12-31",
+                    today_after_n_days(1),
                     2,
-                ),  # TODO: To avoid having to modify this value and to ensure that the tests always work, we should set the time before the fixtures are inserted
-                ("updated_at.lte", "2024-01-01", 0),
+                ),
+                ("updated_at.lte", today_after_n_days(-1), 0),
                 ("resource_type.icontains", "prowler", 2),
                 # full text search on finding
                 ("search", "dev-qa", 1),
@@ -2475,9 +2514,13 @@ class TestFindingViewSet:
         filter_value,
         expected_count,
     ):
+        filters = {f"filter[{filter_name}]": filter_value}
+        if "inserted_at" not in filter_name:
+            filters["filter[inserted_at]"] = TODAY
+
         response = authenticated_client.get(
             reverse("finding-list"),
-            {f"filter[{filter_name}]": filter_value},
+            filters,
         )
 
         assert response.status_code == status.HTTP_200_OK
@@ -2486,9 +2529,7 @@ class TestFindingViewSet:
     def test_finding_filter_by_scan_id(self, authenticated_client, findings_fixture):
         response = authenticated_client.get(
             reverse("finding-list"),
-            {
-                "filter[scan]": findings_fixture[0].scan.id,
-            },
+            {"filter[scan]": findings_fixture[0].scan.id},
         )
         assert response.status_code == status.HTTP_200_OK
         assert len(response.json()["data"]) == 2
@@ -2511,6 +2552,7 @@ class TestFindingViewSet:
             reverse("finding-list"),
             {
                 "filter[provider]": findings_fixture[0].scan.provider.id,
+                "filter[inserted_at]": TODAY,
             },
         )
         assert response.status_code == status.HTTP_200_OK
@@ -2525,7 +2567,8 @@ class TestFindingViewSet:
                 "filter[provider.in]": [
                     findings_fixture[0].scan.provider.id,
                     findings_fixture[1].scan.provider.id,
-                ]
+                ],
+                "filter[inserted_at]": TODAY,
             },
         )
         assert response.status_code == status.HTTP_200_OK
@@ -2559,13 +2602,13 @@ class TestFindingViewSet:
     )
     def test_findings_sort(self, authenticated_client, sort_field):
         response = authenticated_client.get(
-            reverse("finding-list"), {"sort": sort_field}
+            reverse("finding-list"), {"sort": sort_field, "filter[inserted_at]": TODAY}
         )
         assert response.status_code == status.HTTP_200_OK
 
     def test_findings_sort_invalid(self, authenticated_client):
         response = authenticated_client.get(
-            reverse("finding-list"), {"sort": "invalid"}
+            reverse("finding-list"), {"sort": "invalid", "filter[inserted_at]": TODAY}
         )
         assert response.status_code == status.HTTP_400_BAD_REQUEST
         assert response.json()["errors"][0]["code"] == "invalid"
@@ -2633,7 +2676,7 @@ class TestFindingViewSet:
             reverse("finding-metadata"),
             {
                 "filter[severity__in]": ["low", "medium"],
-                "filter[inserted_at]": finding_1.updated_at.strftime("%Y-%m-%d"),
+                "filter[inserted_at]": finding_1.inserted_at.strftime("%Y-%m-%d"),
             },
         )
         data = response.json()

api/src/backend/api/v1/views.py

@@ -1352,6 +1352,14 @@ class ResourceViewSet(BaseRLSViewSet):
         tags=["Finding"],
         summary="List all findings",
         description="Retrieve a list of all findings with options for filtering by various criteria.",
+        parameters=[
+            OpenApiParameter(
+                name="filter[inserted_at]",
+                description="At least one of the variations of the `filter[inserted_at]` filter must be provided.",
+                required=True,
+                type=OpenApiTypes.DATE,
+            )
+        ],
     ),
     retrieve=extend_schema(
         tags=["Finding"],
@@ -1369,6 +1377,14 @@ class ResourceViewSet(BaseRLSViewSet):
         tags=["Finding"],
         summary="Retrieve metadata values from findings",
         description="Fetch unique metadata values from a set of findings. This is useful for dynamic filtering.",
+        parameters=[
+            OpenApiParameter(
+                name="filter[inserted_at]",
+                description="At least one of the variations of the `filter[inserted_at]` filter must be provided.",
+                required=True,
+                type=OpenApiTypes.DATE,
+            )
+        ],
         filters=True,
     ),
 )
@@ -1445,6 +1461,12 @@ class FindingViewSet(BaseRLSViewSet):
 
         return queryset
 
+    def filter_queryset(self, queryset):
+        # Do not apply filters when retrieving specific finding
+        if self.action == "retrieve":
+            return queryset
+        return super().filter_queryset(queryset)
+
     def inserted_at_to_uuidv7(self, inserted_at):
         if inserted_at is None:
             return None
@@ -1477,50 +1499,26 @@ class FindingViewSet(BaseRLSViewSet):
         queryset = self.get_queryset()
         filtered_queryset = self.filter_queryset(queryset)
 
-        relevant_resources = Resource.objects.filter(
-            tenant_id=tenant_id, findings__in=filtered_queryset
-        ).distinct()
+        filtered_ids = filtered_queryset.order_by().values("id")
 
-        services = (
-            relevant_resources.values_list("service", flat=True)
-            .distinct()
-            .order_by("service")
+        relevant_resources = Resource.all_objects.filter(
+            tenant_id=tenant_id, findings__id__in=Subquery(filtered_ids)
+        ).only("service", "region", "type")
+
+        aggregation = relevant_resources.aggregate(
+            services=ArrayAgg("service", flat=True),
+            regions=ArrayAgg("region", flat=True),
+            resource_types=ArrayAgg("type", flat=True),
         )
 
-        regions = (
-            relevant_resources.exclude(region="")
-            .values_list("region", flat=True)
-            .distinct()
-            .order_by("region")
-        )
-
-        resource_types = (
-            relevant_resources.values_list("type", flat=True)
-            .distinct()
-            .order_by("type")
-        )
-
-        # Temporarily disabled until we implement tag filtering in the UI
-        # tag_data = (
-        #     relevant_resources
-        #     .filter(tags__key__isnull=False, tags__value__isnull=False)
-        #     .exclude(tags__key="")
-        #     .exclude(tags__value="")
-        #     .values("tags__key", "tags__value")
-        #     .distinct()
-        #     .order_by("tags__key", "tags__value")
-        # )
-        #
-        # tags_dict = {}
-        # for row in tag_data:
-        #     k, v = row["tags__key"], row["tags__value"]
-        #     tags_dict.setdefault(k, []).append(v)
+        services = sorted(set(aggregation["services"] or []))
+        regions = sorted({region for region in aggregation["regions"] or [] if region})
+        resource_types = sorted(set(aggregation["resource_types"] or []))
 
         result = {
-            "services": list(services),
-            "regions": list(regions),
-            "resource_types": list(resource_types),
-            # "tags": tags_dict
+            "services": services,
+            "regions": regions,
+            "resource_types": resource_types,
         }
 
         serializer = self.get_serializer(data=result)

api/src/backend/config/django/base.py

@@ -207,3 +207,4 @@ CACHE_STALE_WHILE_REVALIDATE = env.int("DJANGO_STALE_WHILE_REVALIDATE", 60)
 
 
 TESTING = False
+FINDINGS_MAX_DAYS_IN_RANGE = env.int("DJANGO_FINDINGS_MAX_DAYS_IN_RANGE", 7)