ben.carrasco/prowler
1
0
Fork 0
mirror of https://github.com/prowler-cloud/prowler.git synced 2026-03-22 03:08:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(services): Add GET /overviews/services to API (#6029)

Browse Source
This commit is contained in:
Víctor Fernández Poyatos
2024-12-17 08:47:44 +01:00
committed by GitHub
parent 3ed0b8a464
commit 0ec3ed8be7
6 changed files with 423 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
api/src/backend/api/filters.py
View File

@@ -22,22 +22,22 @@ from api.db_utils import (
StatusEnumField,
)
from api.models import (
ComplianceOverview,
Finding,
Invitation,
Membership,
PermissionChoices,
Provider,
ProviderGroup,
ProviderSecret,
Resource,
ResourceTag,
Role,
Scan,
ScanSummary,
SeverityChoices,
StateChoices,
StatusChoices,
ProviderSecret,
Invitation,
Role,
ComplianceOverview,
Task,
User,
)
@@ -543,3 +543,25 @@ class ScanSummaryFilter(FilterSet):
"inserted_at": ["date", "gte", "lte"],
"region": ["exact", "icontains", "in"],
}
class ServiceOverviewFilter(ScanSummaryFilter):
muted_findings = None
def is_valid(self):
# Check if at least one of the inserted_at filters is present
inserted_at_filters = [
self.data.get("inserted_at"),
self.data.get("inserted_at__gte"),
self.data.get("inserted_at__lte"),
]
if not any(inserted_at_filters):
raise ValidationError(
{
"inserted_at": [
"At least one of filter[inserted_at], filter[inserted_at__gte], or "
"filter[inserted_at__lte] is required."
]
}
)
return super().is_valid()

181
api/src/backend/api/specs/v1.yaml
View File

@@ -1551,6 +1551,143 @@ paths:
schema:
$ref: '#/components/schemas/OverviewProviderResponse'
description: ''
/api/v1/overviews/services:
get:
operationId: overviews_services_retrieve
description: Retrieve an aggregated summary of findings grouped by service.
The response includes the total count of findings for each service, as long
as there are at least one finding for that service. At least one of the `inserted_at`
filters must be provided.
summary: Get findings data by service
parameters:
- in: query
name: fields[services-overview]
schema:
type: array
items:
type: string
enum:
- id
- total
- fail
- muted
- pass
description: endpoint return only specific fields in the response on a per-type
basis by including a fields[TYPE] query parameter.
explode: false
- in: query
name: filter[inserted_at]
schema:
type: string
format: date
- in: query
name: filter[inserted_at__date]
schema:
type: string
format: date
- in: query
name: filter[inserted_at__gte]
schema:
type: string
format: date-time
- in: query
name: filter[inserted_at__lte]
schema:
type: string
format: date-time
- in: query
name: filter[provider_id]
schema:
type: string
format: uuid
- in: query
name: filter[provider_type]
schema:
type: string
enum:
- aws
- azure
- gcp
- kubernetes
description: |-
* `aws` - AWS
* `azure` - Azure
* `gcp` - GCP
* `kubernetes` - Kubernetes
- in: query
name: filter[provider_type__in]
schema:
type: array
items:
type: string
enum:
- aws
- azure
- gcp
- kubernetes
description: |-
Multiple values may be separated by commas.
* `aws` - AWS
* `azure` - Azure
* `gcp` - GCP
* `kubernetes` - Kubernetes
explode: false
style: form
- in: query
name: filter[region]
schema:
type: string
- in: query
name: filter[region__icontains]
schema:
type: string
- in: query
name: filter[region__in]
schema:
type: array
items:
type: string
description: Multiple values may be separated by commas.
explode: false
style: form
- name: filter[search]
required: false
in: query
description: A search term.
schema:
type: string
- name: sort
required: false
in: query
description: '[list of fields to sort by](https://jsonapi.org/format/#fetching-sorting)'
schema:
type: array
items:
type: string
enum:
- id
- -id
- total
- -total
- fail
- -fail
- muted
- -muted
- pass
- -pass
explode: false
tags:
- Overview
security:
- jwtAuth: []
responses:
'200':
content:
application/vnd.api+json:
schema:
$ref: '#/components/schemas/OverviewServiceResponse'
description: ''
/api/v1/provider-groups:
get:
operationId: provider_groups_list
@@ -5996,6 +6133,50 @@ components:
type: string
enum:
- providers-overview
OverviewService:
type: object
required:
- type
- id
additionalProperties: false
properties:
type:
allOf:
- $ref: '#/components/schemas/OverviewServiceTypeEnum'
description: The [type](https://jsonapi.org/format/#document-resource-object-identification)
member is used to describe resource objects that share common attributes
and relationships.
id: {}
attributes:
type: object
properties:
id:
type: string
total:
type: integer
fail:
type: integer
muted:
type: integer
pass:
type: integer
required:
- id
- total
- fail
- muted
- pass
OverviewServiceResponse:
type: object
properties:
data:
$ref: '#/components/schemas/OverviewService'
required:
- data
OverviewServiceTypeEnum:
type: string
enum:
- services-overview
OverviewSeverity:
type: object
required:

38
api/src/backend/api/tests/test_views.py
View File

@@ -9,18 +9,18 @@ from django.urls import reverse
from rest_framework import status
from api.models import (
Invitation,
Membership,
Provider,
ProviderGroup,
ProviderGroupMembership,
ProviderSecret,
Role,
RoleProviderGroupRelationship,
Invitation,
UserRoleRelationship,
ProviderSecret,
Scan,
StateChoices,
User,
UserRoleRelationship,
)
from api.rls import Tenant
@@ -3909,7 +3909,37 @@ class TestOverviewViewSet:
resources_fixture
)
# TODO Add more tests for the rest of overviews
def test_overview_services_list_no_required_filters(
self, authenticated_client, scan_summaries_fixture
):
response = authenticated_client.get(reverse("overview-services"))
assert response.status_code == status.HTTP_400_BAD_REQUEST
def test_overview_services_list(self, authenticated_client, scan_summaries_fixture):
response = authenticated_client.get(
reverse("overview-services"), {"filter[inserted_at]": TODAY}
)
assert response.status_code == status.HTTP_200_OK
# Only two different services
assert len(response.json()["data"]) == 2
# Fixed data from the fixture, TODO improve this at some point with something more dynamic
service1_data = response.json()["data"][0]
service2_data = response.json()["data"][1]
assert service1_data["id"] == "service1"
assert service2_data["id"] == "service2"
# TODO fix numbers when muted_findings filter is fixed
assert service1_data["attributes"]["total"] == 3
assert service2_data["attributes"]["total"] == 1
assert service1_data["attributes"]["pass"] == 1
assert service2_data["attributes"]["pass"] == 1
assert service1_data["attributes"]["fail"] == 1
assert service2_data["attributes"]["fail"] == 0
assert service1_data["attributes"]["muted"] == 1
assert service2_data["attributes"]["muted"] == 0
@pytest.mark.django_db

30
api/src/backend/api/v1/serializers.py
View File

@@ -14,24 +14,24 @@ from rest_framework_simplejwt.serializers import TokenObtainPairSerializer
from rest_framework_simplejwt.tokens import RefreshToken
from api.models import (
ComplianceOverview,
Finding,
Invitation,
InvitationRoleRelationship,
Membership,
Provider,
ProviderGroup,
ProviderGroupMembership,
ProviderSecret,
Resource,
ResourceTag,
Finding,
ProviderSecret,
Invitation,
InvitationRoleRelationship,
Role,
RoleProviderGroupRelationship,
UserRoleRelationship,
ComplianceOverview,
Scan,
StateChoices,
Task,
User,
UserRoleRelationship,
)
from api.rls import Tenant
@@ -1655,6 +1655,24 @@ class OverviewSeveritySerializer(serializers.Serializer):
return {"version": "v1"}
class OverviewServiceSerializer(serializers.Serializer):
id = serializers.CharField(source="service")
total = serializers.IntegerField()
_pass = serializers.IntegerField()
fail = serializers.IntegerField()
muted = serializers.IntegerField()
class JSONAPIMeta:
resource_name = "services-overview"
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.fields["pass"] = self.fields.pop("_pass")
def get_root_meta(self, _resource, _many):
return {"version": "v1"}
# Schedules

101
api/src/backend/api/v1/views.py
View File

@@ -8,7 +8,6 @@ from django.urls import reverse
from django.utils.decorators import method_decorator
from django.views.decorators.cache import cache_control
from drf_spectacular.settings import spectacular_settings
from drf_spectacular_jsonapi.schemas.openapi import JsonApiAutoSchema
from drf_spectacular.utils import (
OpenApiParameter,
OpenApiResponse,
@@ -17,6 +16,7 @@ from drf_spectacular.utils import (
extend_schema_view,
)
from drf_spectacular.views import SpectacularAPIView
from drf_spectacular_jsonapi.schemas.openapi import JsonApiAutoSchema
from rest_framework import permissions, status
from rest_framework.decorators import action
from rest_framework.exceptions import (
@@ -26,10 +26,9 @@ from rest_framework.exceptions import (
ValidationError,
)
from rest_framework.generics import GenericAPIView, get_object_or_404
from rest_framework.permissions import SAFE_METHODS
from rest_framework_json_api.views import RelationshipView, Response
from rest_framework_simplejwt.exceptions import InvalidToken, TokenError
from rest_framework.permissions import SAFE_METHODS
from tasks.beat import schedule_provider_scan
from tasks.tasks import (
check_provider_connection_task,
@@ -50,17 +49,15 @@ from api.filters import (
ProviderGroupFilter,
ProviderSecretFilter,
ResourceFilter,
RoleFilter,
ScanFilter,
ScanSummaryFilter,
ServiceOverviewFilter,
TaskFilter,
TenantFilter,
UserFilter,
RoleFilter,
)
from api.models import (
StatusChoices,
User,
UserRoleRelationship,
ComplianceOverview,
Finding,
Invitation,
@@ -69,14 +66,17 @@ from api.models import (
ProviderGroup,
ProviderGroupMembership,
ProviderSecret,
Resource,
Role,
RoleProviderGroupRelationship,
Resource,
Scan,
ScanSummary,
SeverityChoices,
StateChoices,
StatusChoices,
Task,
User,
UserRoleRelationship,
)
from api.pagination import ComplianceOverviewPagination
from api.rbac.permissions import HasPermissions, Permissions
@@ -84,12 +84,6 @@ from api.rls import Tenant
from api.utils import validate_invitation
from api.uuid_utils import datetime_to_uuid7
from api.v1.serializers import (
TokenSerializer,
TokenRefreshSerializer,
UserSerializer,
UserCreateSerializer,
UserUpdateSerializer,
UserRoleRelationshipSerializer,
ComplianceOverviewFullSerializer,
ComplianceOverviewSerializer,
FindingDynamicFilterSerializer,
@@ -101,30 +95,36 @@ from api.v1.serializers import (
MembershipSerializer,
OverviewFindingSerializer,
OverviewProviderSerializer,
OverviewServiceSerializer,
OverviewSeveritySerializer,
ProviderCreateSerializer,
ProviderGroupMembershipSerializer,
ProviderGroupSerializer,
ProviderGroupUpdateSerializer,
RoleProviderGroupRelationshipSerializer,
ProviderSerializer,
ProviderUpdateSerializer,
TenantSerializer,
TaskSerializer,
ScanSerializer,
ScanCreateSerializer,
ScanUpdateSerializer,
ResourceSerializer,
ProviderSecretCreateSerializer,
ProviderSecretSerializer,
ProviderSecretUpdateSerializer,
ProviderSecretCreateSerializer,
RoleSerializer,
ProviderSerializer,
ProviderUpdateSerializer,
ResourceSerializer,
RoleCreateSerializer,
RoleProviderGroupRelationshipSerializer,
RoleSerializer,
RoleUpdateSerializer,
ScanCreateSerializer,
ScanSerializer,
ScanUpdateSerializer,
ScheduleDailyCreateSerializer,
TaskSerializer,
TenantSerializer,
TokenRefreshSerializer,
TokenSerializer,
UserCreateSerializer,
UserRoleRelationshipSerializer,
UserSerializer,
UserUpdateSerializer,
)
CACHE_DECORATOR = cache_control(
max_age=django_settings.CACHE_MAX_AGE,
stale_while_revalidate=django_settings.CACHE_STALE_WHILE_REVALIDATE,
@@ -191,7 +191,7 @@ class SchemaView(SpectacularAPIView):
def get(self, request, *args, **kwargs):
spectacular_settings.TITLE = "Prowler API"
spectacular_settings.VERSION = "1.0.1"
spectacular_settings.VERSION = "1.1.0"
spectacular_settings.DESCRIPTION = (
"Prowler API specification.\n\nThis file is auto-generated."
)
@@ -1888,6 +1888,15 @@ class ComplianceOverviewViewSet(BaseRLSViewSet):
),
filters=True,
),
services=extend_schema(
summary="Get findings data by service",
description=(
"Retrieve an aggregated summary of findings grouped by service. The response includes the total count "
"of findings for each service, as long as there are at least one finding for that service. At least "
"one of the `inserted_at` filters must be provided."
),
filters=True,
),
)
@method_decorator(CACHE_DECORATOR, name="list")
class OverviewViewSet(BaseRLSViewSet):
@@ -1902,6 +1911,8 @@ class OverviewViewSet(BaseRLSViewSet):
return ScanSummary.objects.all()
elif self.action == "findings_severity":
return ScanSummary.objects.all()
elif self.action == "services":
return ScanSummary.objects.all()
else:
return super().get_queryset()
@@ -1912,6 +1923,8 @@ class OverviewViewSet(BaseRLSViewSet):
return OverviewFindingSerializer
elif self.action == "findings_severity":
return OverviewSeveritySerializer
elif self.action == "services":
return OverviewServiceSerializer
return super().get_serializer_class()
def get_filterset_class(self):
@@ -1919,6 +1932,8 @@ class OverviewViewSet(BaseRLSViewSet):
return None
elif self.action in ["findings", "findings_severity"]:
return ScanSummaryFilter
elif self.action == "services":
return ServiceOverviewFilter
return None
@extend_schema(exclude=True)
@@ -2064,6 +2079,38 @@ class OverviewViewSet(BaseRLSViewSet):
serializer = OverviewSeveritySerializer(severity_data)
return Response(serializer.data, status=status.HTTP_200_OK)
@action(detail=False, methods=["get"], url_name="services")
def services(self, request):
queryset = self.get_queryset()
filtered_queryset = self.filter_queryset(queryset)
latest_scan_subquery = (
Scan.objects.filter(
state=StateChoices.COMPLETED, provider_id=OuterRef("scan__provider_id")
)
.order_by("-id")
.values("id")[:1]
)
annotated_queryset = filtered_queryset.annotate(
latest_scan_id=Subquery(latest_scan_subquery)
)
filtered_queryset = annotated_queryset.filter(scan_id=F("latest_scan_id"))
services_data = (
filtered_queryset.values("service")
.annotate(_pass=Sum("_pass"))
.annotate(fail=Sum("fail"))
.annotate(muted=Sum("muted"))
.annotate(total=Sum("total"))
.order_by("service")
)
serializer = OverviewServiceSerializer(services_data, many=True)
return Response(serializer.data, status=status.HTTP_200_OK)
@extend_schema(tags=["Schedule"])
@extend_schema_view(

86
api/src/backend/conftest.py
View File

@@ -24,6 +24,7 @@ from api.models import (
ResourceTag,
Role,
Scan,
ScanSummary,
StateChoices,
Task,
User,
@@ -762,6 +763,85 @@ def get_api_tokens(
)
@pytest.fixture
def scan_summaries_fixture(tenants_fixture, providers_fixture):
tenant = tenants_fixture[0]
provider = providers_fixture[0]
scan = Scan.objects.create(
name="overview scan",
provider=provider,
trigger=Scan.TriggerChoices.MANUAL,
state=StateChoices.COMPLETED,
tenant=tenant,
)
ScanSummary.objects.create(
tenant=tenant,
check_id="check1",
service="service1",
severity="high",
region="region1",
_pass=1,
fail=0,
muted=0,
total=1,
new=1,
changed=0,
unchanged=0,
fail_new=0,
fail_changed=0,
pass_new=1,
pass_changed=0,
muted_new=0,
muted_changed=0,
scan=scan,
)
ScanSummary.objects.create(
tenant=tenant,
check_id="check1",
service="service1",
severity="high",
region="region2",
_pass=0,
fail=1,
muted=1,
total=2,
new=2,
changed=0,
unchanged=0,
fail_new=1,
fail_changed=0,
pass_new=0,
pass_changed=0,
muted_new=1,
muted_changed=0,
scan=scan,
)
ScanSummary.objects.create(
tenant=tenant,
check_id="check2",
service="service2",
severity="critical",
region="region1",
_pass=1,
fail=0,
muted=0,
total=1,
new=1,
changed=0,
unchanged=0,
fail_new=0,
fail_changed=0,
pass_new=1,
pass_changed=0,
muted_new=0,
muted_changed=0,
scan=scan,
)
def get_authorization_header(access_token: str) -> dict:
return {"Authorization": f"Bearer {access_token}"}
@@ -772,10 +852,12 @@ def pytest_collection_modifyitems(items):
def pytest_configure(config):
# Apply the mock before the test session starts. This is necessary to avoid admin error when running the 0004_rbac_missing_admin_roles migration
# Apply the mock before the test session starts. This is necessary to avoid admin error when running the
# 0004_rbac_missing_admin_roles migration
patch("api.db_router.MainRouter.admin_db", new="default").start()
def pytest_unconfigure(config):
# Stop all patches after the test session ends. This is necessary to avoid admin error when running the 0004_rbac_missing_admin_roles migration
# Stop all patches after the test session ends. This is necessary to avoid admin error when running the
# 0004_rbac_missing_admin_roles migration
patch.stopall()