diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index e9a83ff79f..9a417cc5d0 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -17,6 +17,11 @@ Please include a summary of the change and which issue is fixed. List any depend - [ ] Review if backport is needed. - [ ] Review if is needed to change the [Readme.md](https://github.com/prowler-cloud/prowler/blob/master/README.md) +#### API +- [ ] Verify if API specs need to be regenerated. +- [ ] Check if version updates are required (e.g., specs, Poetry, etc.). +- [ ] Ensure new entries are added to [CHANGELOG.md](https://github.com/prowler-cloud/prowler/blob/master/api/CHANGELOG.md), if applicable. + ### License By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. diff --git a/.github/workflows/find-secrets.yml b/.github/workflows/find-secrets.yml index 0eb7bbd6fe..3f2c135ddf 100644 --- a/.github/workflows/find-secrets.yml +++ b/.github/workflows/find-secrets.yml @@ -11,7 +11,7 @@ jobs: with: fetch-depth: 0 - name: TruffleHog OSS - uses: trufflesecurity/trufflehog@v3.88.4 + uses: trufflesecurity/trufflehog@v3.88.5 with: path: ./ base: ${{ github.event.repository.default_branch }} diff --git a/api/CHANGELOG.md b/api/CHANGELOG.md new file mode 100644 index 0000000000..0a3acb7fd4 --- /dev/null +++ b/api/CHANGELOG.md @@ -0,0 +1,20 @@ +# Prowler API Changelog + +All notable changes to the **Prowler API** are documented in this file. + +--- + +## [Unreleased] + + +--- + +## [v1.4.0] (Prowler v5.3.0) - 2025-02-10 + +### Changed +- Daily scheduled scan instances are now created beforehand with `SCHEDULED` state [(#6700)](https://github.com/prowler-cloud/prowler/pull/6700). +- Findings endpoints now require at least one date filter [(#6800)](https://github.com/prowler-cloud/prowler/pull/6800). +- Findings metadata endpoint received a performance improvement [(#6863)](https://github.com/prowler-cloud/prowler/pull/6863). +- Increase the allowed length of the provider UID for Kubernetes providers [(#6869)](https://github.com/prowler-cloud/prowler/pull/6869). + +--- diff --git a/api/src/backend/api/filters.py b/api/src/backend/api/filters.py index 0ace11fe9c..ed4ef1c152 100644 --- a/api/src/backend/api/filters.py +++ b/api/src/backend/api/filters.py @@ -1,4 +1,4 @@ -from datetime import date, datetime, timezone +from datetime import date, datetime, timedelta, timezone from django.conf import settings from django.db.models import Q @@ -346,8 +346,14 @@ class FindingFilter(FilterSet): inserted_at = DateFilter(method="filter_inserted_at", lookup_expr="date") inserted_at__date = DateFilter(method="filter_inserted_at", lookup_expr="date") - inserted_at__gte = DateFilter(method="filter_inserted_at_gte") - inserted_at__lte = DateFilter(method="filter_inserted_at_lte") + inserted_at__gte = DateFilter( + method="filter_inserted_at_gte", + help_text=f"Maximum date range is {settings.FINDINGS_MAX_DAYS_IN_RANGE} days.", + ) + inserted_at__lte = DateFilter( + method="filter_inserted_at_lte", + help_text=f"Maximum date range is {settings.FINDINGS_MAX_DAYS_IN_RANGE} days.", + ) class Meta: model = Finding @@ -375,6 +381,52 @@ class FindingFilter(FilterSet): }, } + def filter_queryset(self, queryset): + if not (self.data.get("scan") or self.data.get("scan__in")) and not ( + self.data.get("inserted_at") + or self.data.get("inserted_at__date") + or self.data.get("inserted_at__gte") + or self.data.get("inserted_at__lte") + ): + raise ValidationError( + [ + { + "detail": "At least one date filter is required: filter[inserted_at], filter[inserted_at.gte], " + "or filter[inserted_at.lte].", + "status": 400, + "source": {"pointer": "/data/attributes/inserted_at"}, + "code": "required", + } + ] + ) + + gte_date = ( + datetime.strptime(self.data.get("inserted_at__gte"), "%Y-%m-%d").date() + if self.data.get("inserted_at__gte") + else datetime.now(timezone.utc).date() + ) + lte_date = ( + datetime.strptime(self.data.get("inserted_at__lte"), "%Y-%m-%d").date() + if self.data.get("inserted_at__lte") + else datetime.now(timezone.utc).date() + ) + + if abs(lte_date - gte_date) > timedelta( + days=settings.FINDINGS_MAX_DAYS_IN_RANGE + ): + raise ValidationError( + [ + { + "detail": f"The date range cannot exceed {settings.FINDINGS_MAX_DAYS_IN_RANGE} days.", + "status": 400, + "source": {"pointer": "/data/attributes/inserted_at"}, + "code": "invalid", + } + ] + ) + + return super().filter_queryset(queryset) + # Convert filter values to UUIDv7 values for use with partitioning def filter_scan_id(self, queryset, name, value): try: diff --git a/api/src/backend/api/migrations/0009_increase_provider_uid_maximum_length.py b/api/src/backend/api/migrations/0009_increase_provider_uid_maximum_length.py new file mode 100644 index 0000000000..f595710626 --- /dev/null +++ b/api/src/backend/api/migrations/0009_increase_provider_uid_maximum_length.py @@ -0,0 +1,22 @@ +# Generated by Django 5.1.5 on 2025-02-07 09:42 + +import django.core.validators +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("api", "0008_daily_scheduled_tasks_update"), + ] + + operations = [ + migrations.AlterField( + model_name="provider", + name="uid", + field=models.CharField( + max_length=250, + validators=[django.core.validators.MinLengthValidator(3)], + verbose_name="Unique identifier for the provider, set by the provider", + ), + ), + ] diff --git a/api/src/backend/api/models.py b/api/src/backend/api/models.py index 45d2f66bcf..6424d691dd 100644 --- a/api/src/backend/api/models.py +++ b/api/src/backend/api/models.py @@ -227,13 +227,13 @@ class Provider(RowLevelSecurityProtectedModel): @staticmethod def validate_kubernetes_uid(value): if not re.match( - r"(^[a-z0-9]([-a-z0-9]{1,61}[a-z0-9])?$)|(^arn:aws(-cn|-us-gov|-iso|-iso-b)?:[a-zA-Z0-9\-]+:([a-z]{2}-[a-z]+-\d{1})?:(\d{12})?:[a-zA-Z0-9\-_\/:\.\*]+(:\d+)?$)", + r"^[a-z0-9][A-Za-z0-9_.:\/-]{1,250}$", value, ): raise ModelValidationError( detail="The value must either be a valid Kubernetes UID (up to 63 characters, " "starting and ending with a lowercase letter or number, containing only " - "lowercase alphanumeric characters and hyphens) or a valid EKS ARN.", + "lowercase alphanumeric characters and hyphens) or a valid AWS EKS Cluster ARN, GCP GKE Context Name or Azure AKS Cluster Name.", code="kubernetes-uid", pointer="/data/attributes/uid", ) @@ -247,7 +247,7 @@ class Provider(RowLevelSecurityProtectedModel): ) uid = models.CharField( "Unique identifier for the provider, set by the provider", - max_length=63, + max_length=250, blank=False, validators=[MinLengthValidator(3)], ) diff --git a/api/src/backend/api/specs/v1.yaml b/api/src/backend/api/specs/v1.yaml index ab25ef0e27..ca5f1a5b07 100644 --- a/api/src/backend/api/specs/v1.yaml +++ b/api/src/backend/api/specs/v1.yaml @@ -346,6 +346,9 @@ paths: schema: type: string format: date + description: At least one of the variations of the `filter[inserted_at]` filter + must be provided. + required: true - in: query name: filter[inserted_at__date] schema: @@ -356,11 +359,13 @@ paths: schema: type: string format: date + description: Maximum date range is 7 days. - in: query name: filter[inserted_at__lte] schema: type: string format: date + description: Maximum date range is 7 days. - in: query name: filter[provider] schema: @@ -861,11 +866,13 @@ paths: schema: type: string format: date + description: Maximum date range is 7 days. - in: query name: filter[inserted_at__lte] schema: type: string format: date + description: Maximum date range is 7 days. - in: query name: filter[provider] schema: @@ -1275,6 +1282,9 @@ paths: schema: type: string format: date + description: At least one of the variations of the `filter[inserted_at]` filter + must be provided. + required: true - in: query name: filter[inserted_at__date] schema: @@ -1285,11 +1295,13 @@ paths: schema: type: string format: date + description: Maximum date range is 7 days. - in: query name: filter[inserted_at__lte] schema: type: string format: date + description: Maximum date range is 7 days. - in: query name: filter[provider] schema: @@ -7665,7 +7677,7 @@ components: uid: type: string title: Unique identifier for the provider, set by the provider - maxLength: 63 + maxLength: 250 minLength: 3 alias: type: string @@ -7777,7 +7789,7 @@ components: uid: type: string title: Unique identifier for the provider, set by the provider - maxLength: 63 + maxLength: 250 minLength: 3 required: - uid @@ -7821,7 +7833,7 @@ components: type: string minLength: 3 title: Unique identifier for the provider, set by the provider - maxLength: 63 + maxLength: 250 required: - uid required: diff --git a/api/src/backend/api/tests/test_views.py b/api/src/backend/api/tests/test_views.py index 0173797062..73f2bab027 100644 --- a/api/src/backend/api/tests/test_views.py +++ b/api/src/backend/api/tests/test_views.py @@ -5,6 +5,7 @@ from unittest.mock import ANY, Mock, patch import jwt import pytest from conftest import API_JSON_CONTENT_TYPE, TEST_PASSWORD, TEST_USER +from django.conf import settings from django.urls import reverse from rest_framework import status @@ -27,6 +28,12 @@ from api.rls import Tenant TODAY = str(datetime.today().date()) +def today_after_n_days(n_days: int) -> str: + return datetime.strftime( + datetime.today().date() + timedelta(days=n_days), "%Y-%m-%d" + ) + + @pytest.mark.django_db class TestUserViewSet: def test_users_list(self, authenticated_client, create_test_user): @@ -878,6 +885,16 @@ class TestProviderViewSet: "uid": "kubernetes-test-123456789", "alias": "test", }, + { + "provider": "kubernetes", + "uid": "arn:aws:eks:us-east-1:111122223333:cluster/test-cluster-long-name-123456789", + "alias": "EKS", + }, + { + "provider": "kubernetes", + "uid": "gke_aaaa-dev_europe-test1_dev-aaaa-test-cluster-long-name-123456789", + "alias": "GKE", + }, { "provider": "azure", "uid": "8851db6b-42e5-4533-aa9e-30a32d67e875", @@ -2379,12 +2396,33 @@ class TestResourceViewSet: @pytest.mark.django_db class TestFindingViewSet: def test_findings_list_none(self, authenticated_client): - response = authenticated_client.get(reverse("finding-list")) + response = authenticated_client.get( + reverse("finding-list"), {"filter[inserted_at]": TODAY} + ) assert response.status_code == status.HTTP_200_OK assert len(response.json()["data"]) == 0 - def test_findings_list(self, authenticated_client, findings_fixture): + def test_findings_list_no_date_filter(self, authenticated_client): response = authenticated_client.get(reverse("finding-list")) + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert response.json()["errors"][0]["code"] == "required" + + def test_findings_date_range_too_large(self, authenticated_client): + response = authenticated_client.get( + reverse("finding-list"), + { + "filter[inserted_at.lte]": today_after_n_days( + -(settings.FINDINGS_MAX_DAYS_IN_RANGE + 1) + ), + }, + ) + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert response.json()["errors"][0]["code"] == "invalid" + + def test_findings_list(self, authenticated_client, findings_fixture): + response = authenticated_client.get( + reverse("finding-list"), {"filter[inserted_at]": TODAY} + ) assert response.status_code == status.HTTP_200_OK assert len(response.json()["data"]) == len(findings_fixture) assert ( @@ -2404,7 +2442,8 @@ class TestFindingViewSet: self, include_values, expected_resources, authenticated_client, findings_fixture ): response = authenticated_client.get( - reverse("finding-list"), {"include": include_values} + reverse("finding-list"), + {"include": include_values, "filter[inserted_at]": TODAY}, ) assert response.status_code == status.HTTP_200_OK assert len(response.json()["data"]) == len(findings_fixture) @@ -2439,13 +2478,13 @@ class TestFindingViewSet: ("service.icontains", "ec", 1), ("inserted_at", "2024-01-01", 0), ("inserted_at.date", "2024-01-01", 0), - ("inserted_at.gte", "2024-01-01", 2), + ("inserted_at.gte", today_after_n_days(-1), 2), ( "inserted_at.lte", - "2028-12-31", + today_after_n_days(1), 2, - ), # TODO: To avoid having to modify this value and to ensure that the tests always work, we should set the time before the fixtures are inserted - ("updated_at.lte", "2024-01-01", 0), + ), + ("updated_at.lte", today_after_n_days(-1), 0), ("resource_type.icontains", "prowler", 2), # full text search on finding ("search", "dev-qa", 1), @@ -2475,9 +2514,13 @@ class TestFindingViewSet: filter_value, expected_count, ): + filters = {f"filter[{filter_name}]": filter_value} + if "inserted_at" not in filter_name: + filters["filter[inserted_at]"] = TODAY + response = authenticated_client.get( reverse("finding-list"), - {f"filter[{filter_name}]": filter_value}, + filters, ) assert response.status_code == status.HTTP_200_OK @@ -2486,9 +2529,7 @@ class TestFindingViewSet: def test_finding_filter_by_scan_id(self, authenticated_client, findings_fixture): response = authenticated_client.get( reverse("finding-list"), - { - "filter[scan]": findings_fixture[0].scan.id, - }, + {"filter[scan]": findings_fixture[0].scan.id}, ) assert response.status_code == status.HTTP_200_OK assert len(response.json()["data"]) == 2 @@ -2511,6 +2552,7 @@ class TestFindingViewSet: reverse("finding-list"), { "filter[provider]": findings_fixture[0].scan.provider.id, + "filter[inserted_at]": TODAY, }, ) assert response.status_code == status.HTTP_200_OK @@ -2525,7 +2567,8 @@ class TestFindingViewSet: "filter[provider.in]": [ findings_fixture[0].scan.provider.id, findings_fixture[1].scan.provider.id, - ] + ], + "filter[inserted_at]": TODAY, }, ) assert response.status_code == status.HTTP_200_OK @@ -2559,13 +2602,13 @@ class TestFindingViewSet: ) def test_findings_sort(self, authenticated_client, sort_field): response = authenticated_client.get( - reverse("finding-list"), {"sort": sort_field} + reverse("finding-list"), {"sort": sort_field, "filter[inserted_at]": TODAY} ) assert response.status_code == status.HTTP_200_OK def test_findings_sort_invalid(self, authenticated_client): response = authenticated_client.get( - reverse("finding-list"), {"sort": "invalid"} + reverse("finding-list"), {"sort": "invalid", "filter[inserted_at]": TODAY} ) assert response.status_code == status.HTTP_400_BAD_REQUEST assert response.json()["errors"][0]["code"] == "invalid" @@ -2633,7 +2676,7 @@ class TestFindingViewSet: reverse("finding-metadata"), { "filter[severity__in]": ["low", "medium"], - "filter[inserted_at]": finding_1.updated_at.strftime("%Y-%m-%d"), + "filter[inserted_at]": finding_1.inserted_at.strftime("%Y-%m-%d"), }, ) data = response.json() diff --git a/api/src/backend/api/v1/views.py b/api/src/backend/api/v1/views.py index 4b125eaca7..4a097a03ec 100644 --- a/api/src/backend/api/v1/views.py +++ b/api/src/backend/api/v1/views.py @@ -1352,6 +1352,14 @@ class ResourceViewSet(BaseRLSViewSet): tags=["Finding"], summary="List all findings", description="Retrieve a list of all findings with options for filtering by various criteria.", + parameters=[ + OpenApiParameter( + name="filter[inserted_at]", + description="At least one of the variations of the `filter[inserted_at]` filter must be provided.", + required=True, + type=OpenApiTypes.DATE, + ) + ], ), retrieve=extend_schema( tags=["Finding"], @@ -1369,6 +1377,14 @@ class ResourceViewSet(BaseRLSViewSet): tags=["Finding"], summary="Retrieve metadata values from findings", description="Fetch unique metadata values from a set of findings. This is useful for dynamic filtering.", + parameters=[ + OpenApiParameter( + name="filter[inserted_at]", + description="At least one of the variations of the `filter[inserted_at]` filter must be provided.", + required=True, + type=OpenApiTypes.DATE, + ) + ], filters=True, ), ) @@ -1445,6 +1461,12 @@ class FindingViewSet(BaseRLSViewSet): return queryset + def filter_queryset(self, queryset): + # Do not apply filters when retrieving specific finding + if self.action == "retrieve": + return queryset + return super().filter_queryset(queryset) + def inserted_at_to_uuidv7(self, inserted_at): if inserted_at is None: return None @@ -1477,50 +1499,26 @@ class FindingViewSet(BaseRLSViewSet): queryset = self.get_queryset() filtered_queryset = self.filter_queryset(queryset) - relevant_resources = Resource.objects.filter( - tenant_id=tenant_id, findings__in=filtered_queryset - ).distinct() + filtered_ids = filtered_queryset.order_by().values("id") - services = ( - relevant_resources.values_list("service", flat=True) - .distinct() - .order_by("service") + relevant_resources = Resource.all_objects.filter( + tenant_id=tenant_id, findings__id__in=Subquery(filtered_ids) + ).only("service", "region", "type") + + aggregation = relevant_resources.aggregate( + services=ArrayAgg("service", flat=True), + regions=ArrayAgg("region", flat=True), + resource_types=ArrayAgg("type", flat=True), ) - regions = ( - relevant_resources.exclude(region="") - .values_list("region", flat=True) - .distinct() - .order_by("region") - ) - - resource_types = ( - relevant_resources.values_list("type", flat=True) - .distinct() - .order_by("type") - ) - - # Temporarily disabled until we implement tag filtering in the UI - # tag_data = ( - # relevant_resources - # .filter(tags__key__isnull=False, tags__value__isnull=False) - # .exclude(tags__key="") - # .exclude(tags__value="") - # .values("tags__key", "tags__value") - # .distinct() - # .order_by("tags__key", "tags__value") - # ) - # - # tags_dict = {} - # for row in tag_data: - # k, v = row["tags__key"], row["tags__value"] - # tags_dict.setdefault(k, []).append(v) + services = sorted(set(aggregation["services"] or [])) + regions = sorted({region for region in aggregation["regions"] or [] if region}) + resource_types = sorted(set(aggregation["resource_types"] or [])) result = { - "services": list(services), - "regions": list(regions), - "resource_types": list(resource_types), - # "tags": tags_dict + "services": services, + "regions": regions, + "resource_types": resource_types, } serializer = self.get_serializer(data=result) diff --git a/api/src/backend/config/django/base.py b/api/src/backend/config/django/base.py index f78cc94ab8..6494bd0ff3 100644 --- a/api/src/backend/config/django/base.py +++ b/api/src/backend/config/django/base.py @@ -207,3 +207,4 @@ CACHE_STALE_WHILE_REVALIDATE = env.int("DJANGO_STALE_WHILE_REVALIDATE", 60) TESTING = False +FINDINGS_MAX_DAYS_IN_RANGE = env.int("DJANGO_FINDINGS_MAX_DAYS_IN_RANGE", 7) diff --git a/dashboard/pages/compliance.py b/dashboard/pages/compliance.py index ce47ac1e8c..a1198d777d 100644 --- a/dashboard/pages/compliance.py +++ b/dashboard/pages/compliance.py @@ -532,8 +532,8 @@ def get_bar_graph(df, column_name): # Cut the text if it is too long for i in range(len(colums)): - if len(colums[i]) > 15: - colums[i] = colums[i][:15] + "..." + if len(colums[i]) > 43: + colums[i] = colums[i][:43] + "..." fig = px.bar( df, diff --git a/dashboard/pages/overview.py b/dashboard/pages/overview.py index 24ffae5de6..afa7ecba04 100644 --- a/dashboard/pages/overview.py +++ b/dashboard/pages/overview.py @@ -1720,7 +1720,7 @@ def generate_table(data, index, color_mapping_severity, color_mapping_status): [ html.P( html.Strong( - "Recomendation: ", + "Recommendation: ", style={ "margin-right": "5px" }, @@ -1744,7 +1744,7 @@ def generate_table(data, index, color_mapping_severity, color_mapping_status): [ html.P( html.Strong( - "RecomendationUrl: ", + "RecommendationUrl: ", style={ "margin-right": "5px" }, diff --git a/docs/index.md b/docs/index.md index f43bfb38ea..648564bc18 100644 --- a/docs/index.md +++ b/docs/index.md @@ -377,6 +377,19 @@ Go to [http://localhost:3000](http://localhost:3000) after installing the app (s Sign Up Button Sign Up +???+ note "User creation and default tenant behavior" + + When creating a new user, the behavior depends on whether an invitation is provided: + + - **Without an invitation**: + + - A new tenant is automatically created. + - The new user is assigned to this tenant. + - A set of **RBAC admin permissions** is generated and assigned to the user for the newly created tenant. + - **With an invitation**: The user is added to the specified tenant with the permissions defined in the invitation. + + This mechanism ensures that the first user in a newly created tenant has administrative permissions within that tenant. + #### **Log In** Log in with your email and password to start using the Prowler App. @@ -535,6 +548,7 @@ prowler kubernetes --kubeconfig-file path For in-cluster execution, you can use the supplied yaml to run Prowler as a job within a new Prowler namespace: ```console +kubectl apply -f kubernetes/prowler-sa.yaml kubectl apply -f kubernetes/job.yaml kubectl apply -f kubernetes/prowler-role.yaml kubectl apply -f kubernetes/prowler-rolebinding.yaml diff --git a/docs/tutorials/kubernetes/in-cluster.md b/docs/tutorials/kubernetes/in-cluster.md index cf401f1ff6..ae733db84a 100644 --- a/docs/tutorials/kubernetes/in-cluster.md +++ b/docs/tutorials/kubernetes/in-cluster.md @@ -9,6 +9,7 @@ For in-cluster execution, you can use the supplied yaml files inside `/kubernete They can be used to run Prowler as a job within a new Prowler namespace: ```console +kubectl apply -f kubernetes/prowler-sa.yaml kubectl apply -f kubernetes/job.yaml kubectl apply -f kubernetes/prowler-role.yaml kubectl apply -f kubernetes/prowler-rolebinding.yaml diff --git a/docs/tutorials/prowler-app.md b/docs/tutorials/prowler-app.md index 3184d9e958..d4d3d2f890 100644 --- a/docs/tutorials/prowler-app.md +++ b/docs/tutorials/prowler-app.md @@ -99,6 +99,32 @@ By default, the `kubeconfig` file is located at `~/.kube/config`. Kubernetes Credentials +???+ note + If you are adding an **Amazon EKS** cluster, follow these additional steps to ensure proper authentication: + + 1. Apply the necessary Kubernetes resources to your EKS cluster (you can find the files in the [`kubernetes` directory of the Prowler repository](https://github.com/prowler-cloud/prowler/tree/master/kubernetes)): + ```console + kubectl apply -f kubernetes/prowler-sa.yaml + kubectl apply -f kubernetes/prowler-role.yaml + kubectl apply -f kubernetes/prowler-rolebinding.yaml + ``` + + 2. Generate a long-lived token for authentication: + ```console + kubectl create token prowler-sa -n prowler-ns --duration=0 + ``` + - **Security Note:** The `--duration=0` option generates a non-expiring token, which may pose a security risk if not managed properly. Users should decide on an appropriate expiration time based on their security policies. If a limited-time token is preferred, set `--duration=