feat(Findings): add the /findings endpoint (#38)

* feat(Findings): initial findings model

* fix(Findings): add view, serializers, migration for enums

* fix(Findings): incomplete jsonb_to_tsvector wrapper

will not run as written

* fix(Findings): use Severity and Status enums from prowler SDK

* tests(Findings): add failing view tests

* fix(Finding): add resource relationship

not returning correct data from serializer, missing links

* fix(FindingSerializer): get Scan & Resource relationships to show up

* fix(FindingFilter): add more filter fields

* fix(FindingFilter): filter on provider id

* fix(FindingSerializer): return Resource in relationship

not ResourceFindingMapping

* fix(FindingModel): update migration

* fix(FindingFilter): full text search on findings

* fix(Resources): include Findings in ResourceSerializer

* fix(FindingFilter): expand text search columns

* fix(DbUtils): docstring, not comment

* fix(BaseViews): remove TODO

comment not applicable right now

* fix(Fixtures): add more findings to fixture file

and change on_delete behavior for resource_finding_mapping

* fix(Resources): rename index to match others

* fix(Findings): update Findigns RLS to allow for full CRUD

eventually we'll let users enter a manual finding
which implies INSERT, UPDATE, DELETE

* fix(Findings): use TextChoices directly for Status enum

* fix(FindingSerializer): build a set instead of a list

* consistency in fixtures

Co-authored-by: Víctor Fernández Poyatos <victor@prowler.com>

* fix(API): update v1 spec for findings

---------

Co-authored-by: Víctor Fernández Poyatos <victor@prowler.com>
This commit is contained in:
Jon Young
2024-09-18 13:28:03 -04:00
committed by GitHub
parent 9ffde34198
commit 489ac20141
13 changed files with 2071 additions and 61 deletions
+59
View File
@@ -3,6 +3,9 @@ from contextlib import contextmanager
from django.conf import settings
from django.contrib.auth.models import BaseUserManager
from django.db import models
from django.contrib.postgres.search import SearchVector
from django.db.models import TextField
from psycopg2 import connect as psycopg2_connect
from psycopg2.extensions import new_type, register_type, register_adapter, AsIs
@@ -49,6 +52,26 @@ class CustomUserManager(BaseUserManager):
return user
def enum_to_choices(enum_class):
"""
This function converts a Python Enum to a list of tuples, where the first element is the value and the second element is the name.
It's for use with Django's `choices` attribute, which expects a list of tuples.
"""
return [(item.value, item.name.replace("_", " ").title()) for item in enum_class]
# jsonb_to_tsvector
class JsonbToTsvector(SearchVector):
function = "jsonb_to_tsvector"
output_field = TextField()
def __init__(self, expression):
super().__init__(expression)
# Postgres Enums
@@ -158,3 +181,39 @@ class StateEnum(EnumType):
class StateEnumField(PostgresEnumField):
def __init__(self, *args, **kwargs):
super().__init__("state", *args, **kwargs)
# Postgres enum definition for Finding.Delta
class FindingDeltaEnum(EnumType):
enum_type_name = "finding_delta"
class FindingDeltaEnumField(PostgresEnumField):
def __init__(self, *args, **kwargs):
super().__init__("finding_delta", *args, **kwargs)
# Postgres enum definition for Severity
class SeverityEnum(EnumType):
enum_type_name = "severity"
class SeverityEnumField(PostgresEnumField):
def __init__(self, *args, **kwargs):
super().__init__("severity", *args, **kwargs)
# Postgres enum definition for Status
class StatusEnum(EnumType):
enum_type_name = "status"
class StatusEnumField(PostgresEnumField):
def __init__(self, *args, **kwargs):
super().__init__("status", *args, **kwargs)
+201 -9
View File
@@ -1,15 +1,28 @@
from uuid import UUID
from django.db.models import Q
from django_filters.rest_framework import (
FilterSet,
BooleanFilter,
CharFilter,
UUIDFilter,
DateFilter,
)
from rest_framework_json_api.django_filters.backends import DjangoFilterBackend
from rest_framework_json_api.serializers import ValidationError
from api.db_utils import ProviderEnumField
from api.models import Provider, Resource, ResourceTag, Scan, Task, StateChoices
from api.models import (
Provider,
Resource,
ResourceTag,
Scan,
Task,
StateChoices,
Finding,
SeverityChoices,
StatusChoices,
)
from api.rls import Tenant
from api.v1.serializers import TaskBase
@@ -135,12 +148,22 @@ class ProviderRelationshipFilterSet(FilterSet):
provider_alias__in = CharFilter(method="filter_provider_alias_in")
provider_alias__icontains = CharFilter(method="filter_provider_alias_icontains")
# this can be overridden in subclasses
provider_type_lookup_field = "provider__provider__exact"
provider_type_in_lookup_field = "provider__provider__in"
provider_uid_lookup_field = "provider__uid"
provider_uid_in_lookup_field = "provider__uid__in"
provider_uid_icontains_lookup_field = "provider__uid__icontains"
provider_alias_lookup_field = "provider__alias"
provider_alias_in_lookup_field = "provider__alias__in"
provider_alias_icontains_lookup_field = "provider__alias__icontains"
def filter_provider_type(self, queryset, name, value):
return enum_filter(
queryset,
value,
enum_choices=Provider.ProviderChoices,
lookup_field="provider__provider__exact",
lookup_field=self.provider_type_lookup_field,
)
def filter_provider_type_in(self, queryset, name, value):
@@ -151,30 +174,30 @@ class ProviderRelationshipFilterSet(FilterSet):
queryset,
value,
enum_choices=Provider.ProviderChoices,
lookup_field="provider__provider__in",
lookup_field=self.provider_type_in_lookup_field,
)
def filter_provider_uid(self, queryset, name, value):
return queryset.filter(provider__uid=value)
return queryset.filter(**{self.provider_uid_lookup_field: value})
def filter_provider_uid_in(self, queryset, name, value):
if isinstance(value, str):
value = value.split(",")
return queryset.filter(provider__uid__in=value)
return queryset.filter(**{self.provider_uid_in_lookup_field: value})
def filter_provider_uid_icontains(self, queryset, name, value):
return queryset.filter(provider__uid__icontains=value)
return queryset.filter(**{self.provider_uid_icontains_lookup_field: value})
def filter_provider_alias(self, queryset, name, value):
return queryset.filter(provider__alias=value)
return queryset.filter(**{self.provider_alias_lookup_field: value})
def filter_provider_alias_in(self, queryset, name, value):
if isinstance(value, str):
value = value.split(",")
return queryset.filter(provider__alias__in=value)
return queryset.filter(**{self.provider_alias_in_lookup_field: value})
def filter_provider_alias_icontains(self, queryset, name, value):
return queryset.filter(provider__alias__icontains=value)
return queryset.filter(**{self.provider_alias_icontains_lookup_field: value})
class ScanFilter(ProviderRelationshipFilterSet):
@@ -271,3 +294,172 @@ class ResourceFilter(ProviderRelationshipFilterSet):
# and we don't want to build special filtering logic for every possible
# provider tag spec, so we'll just do a full text search
return queryset.filter(tags__text_search=value)
class FindingFilter(ProviderRelationshipFilterSet):
provider = UUIDFilter(field_name="scan__provider__id", lookup_expr="exact")
provider__in = UUIDFilter(method="filter_provider_id_in")
def filter_provider_id_in(self, queryset, name, value):
if isinstance(value, str):
value = value.split(",")
if isinstance(value, UUID):
value = [value]
return queryset.filter(scan__provider__id__in=value)
inserted_at = DateFilter(field_name="inserted_at", lookup_expr="date")
updated_at = DateFilter(field_name="updated_at", lookup_expr="date")
delta = CharFilter(method="filter_delta")
delta__in = CharFilter(method="filter_delta_in")
status = CharFilter(method="filter_status")
status__in = CharFilter(method="filter_status_in")
severity = CharFilter(method="filter_severity")
severity__in = CharFilter(method="filter_severity_in")
impact = CharFilter(method="filter_severity")
impact__in = CharFilter(method="filter_severity_in")
resources = UUIDFilter(field_name="resource__id", lookup_expr="in")
region = CharFilter(method="filter_region")
region__in = CharFilter(method="filter_region_in")
region__icontains = CharFilter(method="filter_region_icontains")
service = CharFilter(method="filter_service")
service__in = CharFilter(method="filter_service_in")
service__icontains = CharFilter(method="filter_service_icontains")
resource_uid = CharFilter(method="filter_resource_uid")
resource_uid__in = CharFilter(method="filter_resource_uid_in")
resource_uid__icontains = CharFilter(method="filter_resource_uid_icontains")
resource_name = CharFilter(method="filter_resource_name")
resource_name__in = CharFilter(method="filter_resource_name_in")
resource_name__icontains = CharFilter(method="filter_resource_name_icontains")
resource_type = CharFilter(method="filter_resource_type")
resource_type__in = CharFilter(method="filter_resource_type_in")
resource_type__icontains = CharFilter(method="filter_resource_type_icontains")
class Meta:
model = Finding
fields = {
"scan": ["exact", "in"],
"check_id": ["exact", "in", "icontains"],
"inserted_at": ["date", "gte", "lte"],
"updated_at": ["gte", "lte"],
}
def __init__(self, *args, **kwargs):
self.provider_type_lookup_field = "scan__provider__provider__exact"
self.provider_type_in_lookup_field = "scan__provider__provider__in"
self.provider_uid_lookup_field = "scan__provider__uid"
self.provider_uid_in_lookup_field = "scan__provider__uid__in"
self.provider_uid_icontains_lookup_field = "scan__provider__uid__icontains"
self.provider_alias_lookup_field = "scan__provider__alias"
self.provider_alias_in_lookup_field = "scan__provider__alias__in"
self.provider_alias_icontains_lookup_field = "scan__provider__alias__icontains"
super().__init__(*args, **kwargs)
def filter_delta(self, queryset, name, value):
return enum_filter(
queryset,
value,
enum_choices=Finding.DeltaChoices,
lookup_field="delta",
)
def filter_delta_in(self, queryset, name, value):
return enum_filter(
queryset,
value,
enum_choices=Finding.DeltaChoices,
lookup_field="delta__in",
)
def filter_severity(self, queryset, name, value):
return enum_filter(
queryset,
value,
enum_choices=SeverityChoices,
lookup_field="severity",
)
def filter_severity_in(self, queryset, name, value):
return enum_filter(
queryset,
value,
enum_choices=SeverityChoices,
lookup_field="severity__in",
)
def filter_status(self, queryset, name, value):
return enum_filter(
queryset,
value,
enum_choices=StatusChoices,
lookup_field="status",
)
def filter_status_in(self, queryset, name, value):
return enum_filter(
queryset,
value,
enum_choices=StatusChoices,
lookup_field="status__in",
)
def filter_region(self, queryset, name, value):
return queryset.filter(resources__region=value)
def filter_region_in(self, queryset, name, value):
if isinstance(value, str):
value = value.split(",")
return queryset.filter(resources__region__in=value)
def filter_region_icontains(self, queryset, name, value):
return queryset.filter(resources__region__icontains=value)
def filter_service(self, queryset, name, value):
return queryset.filter(resources__service=value)
def filter_service_in(self, queryset, name, value):
if isinstance(value, str):
value = value.split(",")
return queryset.filter(resources__service__in=value)
def filter_service_icontains(self, queryset, name, value):
return queryset.filter(resources__service__icontains=value)
def filter_resource_uid(self, queryset, name, value):
return queryset.filter(resources__uid=value)
def filter_resource_uid_in(self, queryset, name, value):
if isinstance(value, str):
value = value.split(",")
return queryset.filter(resources__uid__in=value)
def filter_resource_uid_icontains(self, queryset, name, value):
return queryset.filter(resources__uid__icontains=value)
def filter_resource_name(self, queryset, name, value):
return queryset.filter(resources__name=value)
def filter_resource_name_in(self, queryset, name, value):
if isinstance(value, str):
value = value.split(",")
return queryset.filter(resources__name__in=value)
def filter_resource_name_icontains(self, queryset, name, value):
return queryset.filter(resources__name__icontains=value)
def filter_resource_type(self, queryset, name, value):
return queryset.filter(resources__type=value)
def filter_resource_type_in(self, queryset, name, value):
if isinstance(value, str):
value = value.split(",")
return queryset.filter(resources__type__in=value)
def filter_resource_type_icontains(self, queryset, name, value):
return queryset.filter(resources__type__icontains=value)
@@ -7,6 +7,8 @@
"provider": "37b065f8-26b0-4218-a665-0b23d07b27d9",
"uid": "unique-1",
"name": "testing 1",
"region": "eu-west-1",
"service": "ec2",
"inserted_at": "2024-08-01T17:20:27.050Z",
"updated_at": "2024-08-01T17:20:27.050Z"
}
@@ -19,6 +21,8 @@
"provider": "37b065f8-26b0-4218-a665-0b23d07b27d9",
"uid": "unique-2",
"name": "testing 2",
"region": "eu-west-1",
"service": "ec2",
"inserted_at": "2024-08-01T17:20:27.050Z",
"updated_at": "2024-08-01T17:20:27.050Z"
}
+68
View File
@@ -0,0 +1,68 @@
[
{
"model": "api.scan",
"pk": "0191e280-9d2f-71c8-9b18-487a23ba185e",
"fields": {
"tenant": "12646005-9067-4d2a-a098-8bb378604362",
"provider": "37b065f8-26b0-4218-a665-0b23d07b27d9",
"trigger": "manual",
"name": "test scan 1",
"state": "completed",
"unique_resource_count": 1,
"duration": 10,
"scanner_args": {
"key1": "value1",
"key2": {
"key21": "value21"
}
},
"scheduled_at": "2024-09-01T17:20:27.050Z",
"inserted_at": "2024-09-01T17:24:27.050Z",
"updated_at": "2024-09-01T17:24:27.050Z"
}
},
{
"model": "api.scan",
"pk": "01920573-aa9c-73c9-bcda-f2e35c9b19d2",
"fields": {
"tenant": "12646005-9067-4d2a-a098-8bb378604362",
"provider": "b85601a8-4b45-4194-8135-03fb980ef428",
"trigger": "manual",
"name": "test aws scan 2",
"state": "completed",
"unique_resource_count": 1,
"duration": 10,
"scanner_args": {
"key1": "value1",
"key2": {
"key21": "value21"
}
},
"scheduled_at": "2024-09-02T17:20:27.050Z",
"inserted_at": "2024-09-02T17:24:27.050Z",
"updated_at": "2024-09-02T17:24:27.050Z"
}
},
{
"model": "api.scan",
"pk": "01920573-ea5b-77fd-a93f-1ed2ae12f728",
"fields": {
"tenant": "12646005-9067-4d2a-a098-8bb378604362",
"provider": "b85601a8-4b45-4194-8135-03fb980ef428",
"trigger": "manual",
"name": "test aws scan",
"state": "completed",
"unique_resource_count": 1,
"duration": 10,
"scanner_args": {
"key1": "value1",
"key2": {
"key21": "value21"
}
},
"scheduled_at": "2024-09-02T19:20:27.050Z",
"inserted_at": "2024-09-02T19:24:27.050Z",
"updated_at": "2024-09-02T19:24:27.050Z"
}
}
]
@@ -0,0 +1,254 @@
[
{
"model": "api.finding",
"pk": "01920571-de39-7539-acb8-35f714afdcd9",
"fields": {
"tenant": "12646005-9067-4d2a-a098-8bb378604362",
"scan": "0191e280-9d2f-71c8-9b18-487a23ba185e",
"status": "FAIL",
"status_extended": "test status extended",
"impact": "low",
"impact_extended": "test impact extended",
"severity": "low",
"check_id": "accessanalyzer_enabled",
"check_metadata": {
"check_id": "accessanalyzer_enabled",
"metadata": {
"Categories": [],
"CheckID": "accessanalyzer_enabled",
"CheckTitle": "Check if IAM Access Analyzer is enabled",
"CheckType": [
"IAM"
],
"DependsOn": [],
"Description": "Check if IAM Access Analyzer is enabled",
"Notes": "",
"Provider": "aws",
"RelatedTo": [],
"RelatedUrl": "https://docs.aws.amazon.com/IAM/latest/UserGuide/what-is-access-analyzer.html",
"Remediation": {
"Code": {
"CLI": "aws accessanalyzer create-analyzer --analyzer-name <NAME> --type <ACCOUNT|ORGANIZATION>",
"NativeIaC": "",
"Other": "",
"Terraform": ""
},
"Recommendation": {
"Text": "Enable IAM Access Analyzer for all accounts, create analyzer and take action over it is recommendations (IAM Access Analyzer is available at no additional cost).",
"Url": "https://docs.aws.amazon.com/IAM/latest/UserGuide/what-is-access-analyzer.html"
}
},
"ResourceIdTemplate": "arn:partition:access-analyzer:region:account-id:analyzer/resource-id",
"ResourceType": "Other",
"Risk": "AWS IAM Access Analyzer helps you identify the resources in your organization and accounts, such as Amazon S3 buckets or IAM roles, that are shared with an external entity. This lets you identify unintended access to your resources and data, which is a security risk. IAM Access Analyzer uses a form of mathematical analysis called automated reasoning, which applies logic and mathematical inference to determine all possible access paths allowed by a resource policy.",
"ServiceName": "accessanalyzer",
"Severity": "low",
"SubServiceName": ""
}
},
"raw_result": {
"status": "FAIL",
"impact": "critical",
"severity": "critical"
},
"inserted_at": "2024-09-01T17:24:27.050Z",
"updated_at": "2024-09-01T17:24:27.050Z"
}
},
{
"model": "api.resourcefindingmapping",
"fields": {
"finding": "01920571-de39-7539-acb8-35f714afdcd9",
"resource": "a3ba9470-a240-49a6-8196-9230a267a220",
"tenant": "12646005-9067-4d2a-a098-8bb378604362"
}
},
{
"model": "api.finding",
"pk": "01920571-3e43-7fae-9175-ee991af8a13c",
"fields": {
"tenant": "12646005-9067-4d2a-a098-8bb378604362",
"scan": "01920573-aa9c-73c9-bcda-f2e35c9b19d2",
"status": "PASS",
"status_extended": "test status extended",
"impact": "medium",
"impact_extended": "test impact extended",
"severity": "medium",
"check_id": "workspaces_vpc_2private_1public_subnets_nat",
"check_metadata": {
"check_id": "workspaces_vpc_2private_1public_subnets_nat",
"metadata": {
"Categories": [],
"CheckID": "workspaces_vpc_2private_1public_subnets_nat",
"CheckTitle": "Ensure that the Workspaces VPC are deployed following the best practices using 1 public subnet and 2 private subnets with a NAT Gateway attached",
"CheckType": [],
"DependsOn": [],
"Description": "Ensure that the Workspaces VPC are deployed following the best practices using 1 public subnet and 2 private subnets with a NAT Gateway attached",
"Notes": "",
"Provider": "aws",
"RelatedTo": [],
"RelatedUrl": "https://docs.aws.amazon.com/workspaces/latest/adminguide/amazon-workspaces-vpc.html",
"Remediation": {
"Code": {
"CLI": "",
"NativeIaC": "",
"Other": "",
"Terraform": ""
},
"Recommendation": {
"Text": "Follow the documentation and deploy Workspaces VPC using 1 public subnet and 2 private subnets with a NAT Gateway attached",
"Url": "https://docs.aws.amazon.com/workspaces/latest/adminguide/amazon-workspaces-vpc.html"
}
},
"ResourceIdTemplate": "arn:aws:workspaces:region:account-id:workspace",
"ResourceType": "AwsWorkspaces",
"Risk": "Proper network segmentation is a key security best practice. Workspaces VPC should be deployed using 1 public subnet and 2 private subnets with a NAT Gateway attached",
"ServiceName": "workspaces",
"Severity": "medium",
"SubServiceName": ""
}
},
"raw_result": {
"status": "FAIL",
"impact": "critical",
"severity": "critical"
},
"inserted_at": "2024-09-01T17:24:27.050Z",
"updated_at": "2024-09-01T17:24:27.050Z"
}
},
{
"model": "api.resourcefindingmapping",
"fields": {
"finding": "01920571-3e43-7fae-9175-ee991af8a13c",
"resource": "a3ba9470-a240-49a6-8196-9230a267a220",
"tenant": "12646005-9067-4d2a-a098-8bb378604362"
}
},
{
"model": "api.finding",
"pk": "01920572-2def-77f4-ba03-d4c7ec46fe8d",
"fields": {
"tenant": "12646005-9067-4d2a-a098-8bb378604362",
"scan": "01920573-ea5b-77fd-a93f-1ed2ae12f728",
"status": "FAIL",
"status_extended": "test status extended",
"impact": "low",
"impact_extended": "test impact extended",
"severity": "low",
"check_id": "accessanalyzer_enabled",
"check_metadata": {
"check_id": "accessanalyzer_enabled",
"metadata": {
"Categories": [],
"CheckID": "accessanalyzer_enabled",
"CheckTitle": "Check if IAM Access Analyzer is enabled",
"CheckType": [
"IAM"
],
"DependsOn": [],
"Description": "Check if IAM Access Analyzer is enabled",
"Notes": "",
"Provider": "aws",
"RelatedTo": [],
"RelatedUrl": "https://docs.aws.amazon.com/IAM/latest/UserGuide/what-is-access-analyzer.html",
"Remediation": {
"Code": {
"CLI": "aws accessanalyzer create-analyzer --analyzer-name <NAME> --type <ACCOUNT|ORGANIZATION>",
"NativeIaC": "",
"Other": "",
"Terraform": ""
},
"Recommendation": {
"Text": "Enable IAM Access Analyzer for all accounts, create analyzer and take action over it is recommendations (IAM Access Analyzer is available at no additional cost).",
"Url": "https://docs.aws.amazon.com/IAM/latest/UserGuide/what-is-access-analyzer.html"
}
},
"ResourceIdTemplate": "arn:partition:access-analyzer:region:account-id:analyzer/resource-id",
"ResourceType": "Other",
"Risk": "AWS IAM Access Analyzer helps you identify the resources in your organization and accounts, such as Amazon S3 buckets or IAM roles, that are shared with an external entity. This lets you identify unintended access to your resources and data, which is a security risk. IAM Access Analyzer uses a form of mathematical analysis called automated reasoning, which applies logic and mathematical inference to determine all possible access paths allowed by a resource policy.",
"ServiceName": "accessanalyzer",
"Severity": "low",
"SubServiceName": ""
}
},
"raw_result": {
"status": "FAIL",
"impact": "critical",
"severity": "critical"
},
"inserted_at": "2024-09-01T17:24:27.050Z",
"updated_at": "2024-09-01T17:24:27.050Z"
}
},
{
"model": "api.resourcefindingmapping",
"fields": {
"finding": "01920572-2def-77f4-ba03-d4c7ec46fe8d",
"resource": "a3ba9470-a240-49a6-8196-9230a267a220",
"tenant": "12646005-9067-4d2a-a098-8bb378604362"
}
},
{
"model": "api.finding",
"pk": "01920572-69bd-77c6-a038-fab6a5061477",
"fields": {
"tenant": "12646005-9067-4d2a-a098-8bb378604362",
"scan": "01920573-ea5b-77fd-a93f-1ed2ae12f728",
"status": "PASS",
"status_extended": "test status extended",
"impact": "medium",
"impact_extended": "test impact extended",
"severity": "medium",
"check_id": "workspaces_vpc_2private_1public_subnets_nat",
"check_metadata": {
"check_id": "workspaces_vpc_2private_1public_subnets_nat",
"metadata": {
"Categories": [],
"CheckID": "workspaces_vpc_2private_1public_subnets_nat",
"CheckTitle": "Ensure that the Workspaces VPC are deployed following the best practices using 1 public subnet and 2 private subnets with a NAT Gateway attached",
"CheckType": [],
"DependsOn": [],
"Description": "Ensure that the Workspaces VPC are deployed following the best practices using 1 public subnet and 2 private subnets with a NAT Gateway attached",
"Notes": "",
"Provider": "aws",
"RelatedTo": [],
"RelatedUrl": "https://docs.aws.amazon.com/workspaces/latest/adminguide/amazon-workspaces-vpc.html",
"Remediation": {
"Code": {
"CLI": "",
"NativeIaC": "",
"Other": "",
"Terraform": ""
},
"Recommendation": {
"Text": "Follow the documentation and deploy Workspaces VPC using 1 public subnet and 2 private subnets with a NAT Gateway attached",
"Url": "https://docs.aws.amazon.com/workspaces/latest/adminguide/amazon-workspaces-vpc.html"
}
},
"ResourceIdTemplate": "arn:aws:workspaces:region:account-id:workspace",
"ResourceType": "AwsWorkspaces",
"Risk": "Proper network segmentation is a key security best practice. Workspaces VPC should be deployed using 1 public subnet and 2 private subnets with a NAT Gateway attached",
"ServiceName": "workspaces",
"Severity": "medium",
"SubServiceName": ""
}
},
"raw_result": {
"status": "FAIL",
"impact": "critical",
"severity": "critical"
},
"inserted_at": "2024-09-01T17:24:27.050Z",
"updated_at": "2024-09-01T17:24:27.050Z"
}
},
{
"model": "api.resourcefindingmapping",
"fields": {
"finding": "01920572-69bd-77c6-a038-fab6a5061477",
"resource": "a3ba9470-a240-49a6-8196-9230a267a220",
"tenant": "12646005-9067-4d2a-a098-8bb378604362"
}
}
]
+251 -10
View File
@@ -1,4 +1,5 @@
import uuid
from uuid6 import uuid7
from functools import partial
import django.contrib.auth.models
@@ -10,7 +11,7 @@ import django.db.models.deletion
import django.utils.timezone
from django.conf import settings
from django.db import migrations, models
from uuid6 import uuid7
import api.rls
from api.db_utils import (
@@ -27,7 +28,14 @@ from api.db_utils import (
TASK_RUNNER_DB_TABLE,
POSTGRES_TENANT_VAR,
)
from api.models import Provider, Scan, StateChoices
from api.models import (
Provider,
Scan,
StateChoices,
Finding,
StatusChoices,
SeverityChoices,
)
DB_NAME = settings.DATABASES["default"]["NAME"]
@@ -47,6 +55,23 @@ StateEnumMigration = PostgresEnumMigration(
enum_values=tuple(state[0] for state in StateChoices.choices),
)
FindingDeltaEnumMigration = PostgresEnumMigration(
enum_name="finding_delta",
enum_values=tuple(
finding_delta[0] for finding_delta in Finding.DeltaChoices.choices
),
)
StatusEnumMigration = PostgresEnumMigration(
enum_name="status",
enum_values=tuple(status[0] for status in StatusChoices.choices),
)
SeverityEnumMigration = PostgresEnumMigration(
enum_name="severity",
enum_values=tuple(severity[0] for severity in SeverityChoices),
)
class Migration(migrations.Migration):
initial = True
@@ -506,6 +531,13 @@ class Migration(migrations.Migration):
"abstract": False,
},
),
migrations.AddIndex(
model_name="resource",
index=models.Index(
fields=["uid", "region", "service", "name"],
name="resource_uid_reg_serv_name_idx",
),
),
migrations.CreateModel(
name="ResourceTag",
fields=[
@@ -631,7 +663,7 @@ class Migration(migrations.Migration):
model_name="resourcetagmapping",
constraint=models.UniqueConstraint(
fields=("tenant_id", "resource_id", "tag_id"),
name="unique_resource_tag_mappings_by_tenant_resource_tag",
name="unique_resource_tag_mappings_by_tenant",
),
),
migrations.AddConstraint(
@@ -642,13 +674,6 @@ class Migration(migrations.Migration):
statements=["SELECT"],
),
),
migrations.AddIndex(
model_name="resource",
index=models.Index(
fields=["uid", "region", "service", "name"],
name="idx_resource_uid_reg_serv_name",
),
),
migrations.AddConstraint(
model_name="resource",
constraint=models.UniqueConstraint(
@@ -664,4 +689,220 @@ class Migration(migrations.Migration):
statements=["SELECT"],
),
),
# Create and register ScanTypeEnum type
migrations.RunPython(
FindingDeltaEnumMigration.create_enum_type,
reverse_code=FindingDeltaEnumMigration.drop_enum_type,
),
migrations.RunPython(
StatusEnumMigration.create_enum_type,
reverse_code=StatusEnumMigration.drop_enum_type,
),
migrations.RunPython(
SeverityEnumMigration.create_enum_type,
reverse_code=SeverityEnumMigration.drop_enum_type,
),
migrations.CreateModel(
name="Finding",
fields=[
(
"id",
models.UUIDField(
default=uuid7,
editable=False,
primary_key=True,
serialize=False,
),
),
("inserted_at", models.DateTimeField(auto_now_add=True)),
("updated_at", models.DateTimeField(auto_now=True)),
(
"delta",
api.db_utils.FindingDeltaEnumField(
choices=[("new", "New"), ("changed", "Changed")],
blank=True,
null=True,
),
),
(
"status",
api.db_utils.StatusEnumField(
choices=[
("PASS", "Pass"),
("FAIL", "Fail"),
("MANUAL", "Manual"),
("MUTED", "Muted"),
]
),
),
("status_extended", models.TextField(blank=True, null=True)),
(
"severity",
api.db_utils.SeverityEnumField(
choices=[
("critical", "Critical"),
("high", "High"),
("medium", "Medium"),
("low", "Low"),
("informational", "Informational"),
]
),
),
(
"impact",
api.db_utils.SeverityEnumField(
choices=[
("critical", "Critical"),
("high", "High"),
("medium", "Medium"),
("low", "Low"),
("informational", "Informational"),
]
),
),
("impact_extended", models.TextField(blank=True, null=True)),
("raw_result", models.JSONField(default=dict)),
("check_id", models.CharField(max_length=100, null=False)),
("check_metadata", models.JSONField(default=dict, null=False)),
("tags", models.JSONField(default=dict, blank=True, null=True)),
(
"scan",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="findings",
to="api.scan",
),
),
(
"tenant",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE, to="api.tenant"
),
),
],
options={
"db_table": "findings",
"indexes": [
models.Index(
fields=[
"scan_id",
"impact",
"severity",
"status",
"check_id",
"delta",
],
name="findings_filter_idx",
),
],
},
),
migrations.RunSQL(
sql="""
ALTER TABLE findings
ADD COLUMN text_search tsvector
GENERATED ALWAYS AS (
setweight(to_tsvector('english', coalesce(impact_extended, '')), 'A') ||
setweight(to_tsvector('english', coalesce(status_extended, '')), 'B') ||
setweight(jsonb_to_tsvector('simple', check_metadata, '["string", "numeric"]'), 'D') ||
setweight(jsonb_to_tsvector('simple', tags, '["string", "numeric"]'), 'D')
) STORED;
""",
reverse_sql="""
ALTER TABLE findings
DROP COLUMN text_search;
""",
state_operations=[
migrations.AddField(
model_name="finding",
name="text_search",
field=models.GeneratedField(
db_persist=True,
expression=django.contrib.postgres.search.SearchVector(
"impact_extended",
"status_extended",
config="simple",
weight="A",
),
null=True,
output_field=django.contrib.postgres.search.SearchVectorField(),
),
),
],
),
migrations.AddIndex(
model_name="finding",
index=django.contrib.postgres.indexes.GinIndex(
fields=["text_search"], name="gin_findings_search_idx"
),
),
migrations.AddConstraint(
model_name="finding",
constraint=api.rls.RowLevelSecurityConstraint(
"tenant_id",
name="rls_on_finding",
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
),
),
migrations.CreateModel(
name="ResourceFindingMapping",
fields=[
(
"id",
models.UUIDField(
default=uuid.uuid4,
editable=False,
primary_key=True,
serialize=False,
),
),
(
"finding",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE, to="api.finding"
),
),
(
"resource",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE, to="api.resource"
),
),
(
"tenant",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE, to="api.tenant"
),
),
],
options={
"db_table": "resource_finding_mappings",
"abstract": False,
},
),
migrations.AddField(
model_name="finding",
name="resources",
field=models.ManyToManyField(
related_name="findings",
through="api.ResourceFindingMapping",
to="api.resource",
verbose_name="Resources associated with the finding",
),
),
migrations.AddConstraint(
model_name="resourcefindingmapping",
constraint=models.UniqueConstraint(
fields=("tenant_id", "resource_id", "finding_id"),
name="unique_resource_finding_mappings_by_tenant",
),
),
migrations.AddConstraint(
model_name="resourcefindingmapping",
constraint=api.rls.RowLevelSecurityConstraint(
"tenant_id",
name="rls_on_resourcefindingmapping",
statements=["SELECT"],
),
),
]
+145 -2
View File
@@ -7,17 +7,26 @@ from django.contrib.postgres.indexes import GinIndex
from django.contrib.postgres.search import SearchVector, SearchVectorField
from django.core.validators import MinLengthValidator
from django.db import models
from django.utils.translation import gettext_lazy as _
from django_celery_results.models import TaskResult
from uuid6 import uuid7
from prowler.lib.outputs.finding import Severity
from api.db_utils import (
enum_to_choices,
ProviderEnumField,
StateEnumField,
ScanTriggerEnumField,
FindingDeltaEnumField,
SeverityEnumField,
StatusEnumField,
CustomUserManager,
)
from api.exceptions import ModelValidationError
from api.rls import (
RowLevelSecurityProtectedModel,
RowLevelSecurityConstraint,
@@ -25,6 +34,23 @@ from api.rls import (
)
# Convert Prowler Severity enum to Django TextChoices
SeverityChoices = enum_to_choices(Severity)
class StatusChoices(models.TextChoices):
"""
This list is based on the finding status in the Prowler CLI.
However it adds another state, MUTED, which is not in the CLI.
"""
PASS = "PASS", _("Pass")
FAIL = "FAIL", _("Fail")
MANUAL = "MANUAL", _("Manual")
MUTED = "MUTED", _("Muted")
class StateChoices(models.TextChoices):
AVAILABLE = "available", _("Available")
SCHEDULED = "scheduled", _("Scheduled")
@@ -338,7 +364,7 @@ class Resource(RowLevelSecurityProtectedModel):
indexes = [
models.Index(
fields=["uid", "region", "service", "name"],
name="idx_resource_uid_reg_serv_name",
name="resource_uid_reg_serv_name_idx",
),
GinIndex(fields=["text_search"], name="gin_resources_search_idx"),
]
@@ -375,7 +401,124 @@ class ResourceTagMapping(RowLevelSecurityProtectedModel):
constraints = [
models.UniqueConstraint(
fields=("tenant_id", "resource_id", "tag_id"),
name="unique_resource_tag_mappings_by_tenant_resource_tag",
name="unique_resource_tag_mappings_by_tenant",
),
RowLevelSecurityConstraint(
field="tenant_id",
name="rls_on_%(class)s",
statements=["SELECT"],
),
]
class Finding(RowLevelSecurityProtectedModel):
class DeltaChoices(models.TextChoices):
NEW = "new", _("New")
CHANGED = "changed", _("Changed")
id = models.UUIDField(primary_key=True, default=uuid7, editable=False)
inserted_at = models.DateTimeField(auto_now_add=True, editable=False)
updated_at = models.DateTimeField(auto_now=True, editable=False)
delta = FindingDeltaEnumField(
choices=DeltaChoices.choices,
blank=True,
null=True,
)
status = StatusEnumField(choices=StatusChoices)
status_extended = models.TextField(blank=True, null=True)
severity = SeverityEnumField(choices=SeverityChoices)
impact = SeverityEnumField(choices=SeverityChoices)
impact_extended = models.TextField(blank=True, null=True)
raw_result = models.JSONField(default=dict)
tags = models.JSONField(default=dict, null=True, blank=True)
check_id = models.CharField(max_length=100, blank=False, null=False)
check_metadata = models.JSONField(default=dict, null=False)
# Relationships
scan = models.ForeignKey(to=Scan, related_name="findings", on_delete=models.CASCADE)
# many-to-many Resources. Relationship is defined on Resource
resources = models.ManyToManyField(
Resource,
verbose_name="Resources associated with the finding",
through="ResourceFindingMapping",
related_name="findings",
)
# TODO: Add resource search
text_search = models.GeneratedField(
expression=SearchVector(
"impact_extended", "status_extended", weight="A", config="simple"
),
output_field=SearchVectorField(),
db_persist=True,
null=True,
editable=False,
)
class Meta:
db_table = "findings"
constraints = [
RowLevelSecurityConstraint(
field="tenant_id",
name="rls_on_%(class)s",
statements=["SELECT", "UPDATE", "INSERT", "DELETE"],
),
]
indexes = [
models.Index(
fields=[
"scan_id",
"impact",
"severity",
"status",
"check_id",
"delta",
],
name="findings_filter_idx",
),
GinIndex(fields=["text_search"], name="gin_findings_search_idx"),
]
def add_resources(self, resources: list[Resource] | None):
# Add new relationships with the tenant_id field
for resource in resources:
ResourceFindingMapping.objects.update_or_create(
resource=resource, finding=self, tenant_id=self.tenant_id
)
# Save the instance
self.save()
class ResourceFindingMapping(RowLevelSecurityProtectedModel):
# NOTE that we don't really need a primary key here,
# but everything is easier with django if we do
id = models.UUIDField(primary_key=True, default=uuid4, editable=False)
resource = models.ForeignKey(Resource, on_delete=models.CASCADE)
finding = models.ForeignKey(Finding, on_delete=models.CASCADE)
class Meta(RowLevelSecurityProtectedModel.Meta):
db_table = "resource_finding_mappings"
# django will automatically create indexes for:
# - resource_id
# - finding_id
# - tenant_id
# - id
constraints = [
models.UniqueConstraint(
fields=("tenant_id", "resource_id", "finding_id"),
name="unique_resource_finding_mappings_by_tenant",
),
RowLevelSecurityConstraint(
field="tenant_id",
+673 -35
View File
@@ -7,6 +7,360 @@ info:
This file is auto-generated.
paths:
/api/v1/findings:
get:
operationId: findings_list
description: Retrieve a list of all findings with options for filtering by various
criteria.
summary: List all findings
parameters:
- in: query
name: fields[findings]
schema:
type: array
items:
type: string
enum:
- delta
- status
- status_extended
- severity
- check_id
- check_metadata
- raw_result
- inserted_at
- updated_at
- url
- scan
- resources
description: endpoint return only specific fields in the response on a per-type
basis by including a fields[TYPE] query parameter.
explode: false
- in: query
name: filter[check_id]
schema:
type: string
- in: query
name: filter[check_id__icontains]
schema:
type: string
- in: query
name: filter[check_id__in]
schema:
type: array
items:
type: string
description: Multiple values may be separated by commas.
explode: false
style: form
- in: query
name: filter[delta]
schema:
type: string
- in: query
name: filter[delta__in]
schema:
type: string
- in: query
name: filter[impact]
schema:
type: string
- in: query
name: filter[impact__in]
schema:
type: string
- in: query
name: filter[inserted_at]
schema:
type: string
format: date
- in: query
name: filter[inserted_at__date]
schema:
type: string
format: date
- in: query
name: filter[inserted_at__gte]
schema:
type: string
format: date-time
- in: query
name: filter[inserted_at__lte]
schema:
type: string
format: date-time
- in: query
name: filter[provider]
schema:
type: string
format: uuid
- in: query
name: filter[provider__in]
schema:
type: string
format: uuid
- in: query
name: filter[provider_alias]
schema:
type: string
- in: query
name: filter[provider_alias__icontains]
schema:
type: string
- in: query
name: filter[provider_alias__in]
schema:
type: string
- in: query
name: filter[provider_type]
schema:
type: string
- in: query
name: filter[provider_type__in]
schema:
type: string
- in: query
name: filter[provider_uid]
schema:
type: string
- in: query
name: filter[provider_uid__icontains]
schema:
type: string
- in: query
name: filter[provider_uid__in]
schema:
type: string
- in: query
name: filter[region]
schema:
type: string
- in: query
name: filter[region__icontains]
schema:
type: string
- in: query
name: filter[region__in]
schema:
type: string
- in: query
name: filter[resource_name]
schema:
type: string
- in: query
name: filter[resource_name__icontains]
schema:
type: string
- in: query
name: filter[resource_name__in]
schema:
type: string
- in: query
name: filter[resource_type]
schema:
type: string
- in: query
name: filter[resource_type__icontains]
schema:
type: string
- in: query
name: filter[resource_type__in]
schema:
type: string
- in: query
name: filter[resource_uid]
schema:
type: string
- in: query
name: filter[resource_uid__icontains]
schema:
type: string
- in: query
name: filter[resource_uid__in]
schema:
type: string
- in: query
name: filter[resources]
schema:
type: string
format: uuid
- in: query
name: filter[scan]
schema:
type: string
format: uuid
- in: query
name: filter[scan__in]
schema:
type: array
items:
type: string
format: uuid
description: Multiple values may be separated by commas.
explode: false
style: form
- name: filter[search]
required: false
in: query
description: A search term.
schema:
type: string
- in: query
name: filter[service]
schema:
type: string
- in: query
name: filter[service__icontains]
schema:
type: string
- in: query
name: filter[service__in]
schema:
type: string
- in: query
name: filter[severity]
schema:
type: string
- in: query
name: filter[severity__in]
schema:
type: string
- in: query
name: filter[status]
schema:
type: string
- in: query
name: filter[status__in]
schema:
type: string
- in: query
name: filter[updated_at]
schema:
type: string
format: date
- in: query
name: filter[updated_at__gte]
schema:
type: string
format: date-time
- in: query
name: filter[updated_at__lte]
schema:
type: string
format: date-time
- in: query
name: include
schema:
type: array
items:
type: string
enum:
- scan
- resources
description: include query parameter to allow the client to customize which
related resources should be returned.
explode: false
- name: page[number]
required: false
in: query
description: A page number within the paginated result set.
schema:
type: integer
- name: page[size]
required: false
in: query
description: Number of results to return per page.
schema:
type: integer
- name: sort
required: false
in: query
description: '[list of fields to sort by](https://jsonapi.org/format/#fetching-sorting)'
schema:
type: array
items:
type: string
enum:
- status
- -status
- severity
- -severity
- check_id
- -check_id
- inserted_at
- -inserted_at
- updated_at
- -updated_at
explode: false
tags:
- findings
security:
- basicAuth: []
responses:
'200':
content:
application/vnd.api+json:
schema:
$ref: '#/components/schemas/PaginatedFindingList'
description: ''
/api/v1/findings/{id}:
get:
operationId: findings_retrieve
description: Fetch detailed information about a specific finding by its ID.
summary: Retrieve data from a specific finding
parameters:
- in: query
name: fields[findings]
schema:
type: array
items:
type: string
enum:
- delta
- status
- status_extended
- severity
- check_id
- check_metadata
- raw_result
- inserted_at
- updated_at
- url
- scan
- resources
description: endpoint return only specific fields in the response on a per-type
basis by including a fields[TYPE] query parameter.
explode: false
- in: path
name: id
schema:
type: string
format: uuid
description: A UUID string identifying this finding.
required: true
- in: query
name: include
schema:
type: array
items:
type: string
enum:
- scan
- resources
description: include query parameter to allow the client to customize which
related resources should be returned.
explode: false
tags:
- findings
security:
- basicAuth: []
responses:
'200':
content:
application/vnd.api+json:
schema:
$ref: '#/components/schemas/FindingResponse'
description: ''
/api/v1/providers:
get:
operationId: providers_list
@@ -24,7 +378,7 @@ paths:
- inserted_at
- updated_at
- provider
- provider_id
- uid
- alias
- connection
- scanner_args
@@ -40,10 +394,34 @@ paths:
name: filter[alias__icontains]
schema:
type: string
- in: query
name: filter[alias__in]
schema:
type: array
items:
type: string
description: Multiple values may be separated by commas.
explode: false
style: form
- in: query
name: filter[connected]
schema:
type: boolean
- in: query
name: filter[id]
schema:
type: string
format: uuid
- in: query
name: filter[id__in]
schema:
type: array
items:
type: string
format: uuid
description: Multiple values may be separated by commas.
explode: false
style: form
- in: query
name: filter[inserted_at]
schema:
@@ -64,11 +442,7 @@ paths:
schema:
type: string
- in: query
name: filter[provider_id]
schema:
type: string
- in: query
name: filter[provider_id__icontains]
name: filter[provider__in]
schema:
type: string
- name: filter[search]
@@ -77,6 +451,23 @@ paths:
description: A search term.
schema:
type: string
- in: query
name: filter[uid]
schema:
type: string
- in: query
name: filter[uid__icontains]
schema:
type: string
- in: query
name: filter[uid__in]
schema:
type: array
items:
type: string
description: Multiple values may be separated by commas.
explode: false
style: form
- in: query
name: filter[updated_at]
schema:
@@ -115,8 +506,8 @@ paths:
enum:
- provider
- -provider
- provider_id
- -provider_id
- uid
- -uid
- alias
- -alias
- connected
@@ -181,7 +572,7 @@ paths:
- inserted_at
- updated_at
- provider
- provider_id
- uid
- alias
- connection
- scanner_args
@@ -315,6 +706,7 @@ paths:
- tags
- provider
- url
- findings
- type
description: endpoint return only specific fields in the response on a per-type
basis by including a fields[TYPE] query parameter.
@@ -344,15 +736,11 @@ paths:
type: string
- in: query
name: filter[provider]
schema:
type: string
- in: query
name: filter[provider_id]
schema:
type: string
format: uuid
- in: query
name: filter[provider_id__in]
name: filter[provider__in]
schema:
type: array
items:
@@ -361,6 +749,38 @@ paths:
description: Multiple values may be separated by commas.
explode: false
style: form
- in: query
name: filter[provider_alias]
schema:
type: string
- in: query
name: filter[provider_alias__icontains]
schema:
type: string
- in: query
name: filter[provider_alias__in]
schema:
type: string
- in: query
name: filter[provider_type]
schema:
type: string
- in: query
name: filter[provider_type__in]
schema:
type: string
- in: query
name: filter[provider_uid]
schema:
type: string
- in: query
name: filter[provider_uid__icontains]
schema:
type: string
- in: query
name: filter[provider_uid__in]
schema:
type: string
- in: query
name: filter[region]
schema:
@@ -457,6 +877,17 @@ paths:
schema:
type: string
format: date-time
- in: query
name: include
schema:
type: array
items:
type: string
enum:
- findings
description: include query parameter to allow the client to customize which
related resources should be returned.
explode: false
- name: page[number]
required: false
in: query
@@ -478,8 +909,8 @@ paths:
items:
type: string
enum:
- provider_id
- -provider_id
- provider_uid
- -provider_uid
- uid
- -uid
- name
@@ -530,6 +961,7 @@ paths:
- tags
- provider
- url
- findings
- type
description: endpoint return only specific fields in the response on a per-type
basis by including a fields[TYPE] query parameter.
@@ -541,6 +973,17 @@ paths:
format: uuid
description: A UUID string identifying this resource.
required: true
- in: query
name: include
schema:
type: array
items:
type: string
enum:
- findings
description: include query parameter to allow the client to customize which
related resources should be returned.
explode: false
tags:
- Resource
security:
@@ -601,15 +1044,11 @@ paths:
type: string
- in: query
name: filter[provider]
schema:
type: string
- in: query
name: filter[provider_id]
schema:
type: string
format: uuid
- in: query
name: filter[provider_id__in]
name: filter[provider__in]
schema:
type: array
items:
@@ -618,6 +1057,38 @@ paths:
description: Multiple values may be separated by commas.
explode: false
style: form
- in: query
name: filter[provider_alias]
schema:
type: string
- in: query
name: filter[provider_alias__icontains]
schema:
type: string
- in: query
name: filter[provider_alias__in]
schema:
type: string
- in: query
name: filter[provider_type]
schema:
type: string
- in: query
name: filter[provider_type__in]
schema:
type: string
- in: query
name: filter[provider_uid]
schema:
type: string
- in: query
name: filter[provider_uid__icontains]
schema:
type: string
- in: query
name: filter[provider_uid__in]
schema:
type: string
- name: filter[search]
required: false
in: query
@@ -664,8 +1135,6 @@ paths:
items:
type: string
enum:
- provider_id
- -provider_id
- name
- -name
- trigger
@@ -1279,7 +1748,6 @@ paths:
type: string
enum:
- username
- password
- email
- date_joined
description: endpoint return only specific fields in the response on a per-type
@@ -1327,6 +1795,154 @@ components:
readOnly: true
required:
- id
Finding:
type: object
required:
- type
- id
additionalProperties: false
properties:
type:
allOf:
- $ref: '#/components/schemas/FindingTypeEnum'
description: The [type](https://jsonapi.org/format/#document-resource-object-identification)
member is used to describe resource objects that share common attributes
and relationships.
id:
type: string
format: uuid
attributes:
type: object
properties:
delta:
enum:
- new
- changed
- null
type: string
description: |-
* `new` - New
* `changed` - Changed
nullable: true
status:
enum:
- PASS
- FAIL
- MANUAL
- MUTED
type: string
description: |-
* `PASS` - Pass
* `FAIL` - Fail
* `MANUAL` - Manual
* `MUTED` - Muted
status_extended:
type: string
nullable: true
severity:
enum:
- critical
- high
- medium
- low
- informational
type: string
description: |-
* `critical` - Critical
* `high` - High
* `medium` - Medium
* `low` - Low
* `informational` - Informational
check_id:
type: string
maxLength: 100
check_metadata: {}
raw_result: {}
inserted_at:
type: string
format: date-time
readOnly: true
updated_at:
type: string
format: date-time
readOnly: true
required:
- status
- severity
- check_id
relationships:
type: object
properties:
scan:
type: object
properties:
data:
type: object
properties:
id:
type: string
format: uuid
type:
type: string
enum:
- Scan
title: Resource Type Name
description: The [type](https://jsonapi.org/format/#document-resource-object-identification)
member is used to describe resource objects that share common
attributes and relationships.
required:
- id
- type
required:
- data
description: The identifier of the related object.
title: Resource Identifier
resources:
type: object
properties:
data:
type: object
properties:
id:
type: string
type:
type: string
enum:
- Resource
title: Resource Type Name
description: The [type](https://jsonapi.org/format/#document-resource-object-identification)
member is used to describe resource objects that share common
attributes and relationships.
required:
- id
- type
required:
- data
description: The identifier of the related object.
title: Resource Identifier
readOnly: true
required:
- scan
FindingResponse:
type: object
properties:
data:
$ref: '#/components/schemas/Finding'
required:
- data
FindingTypeEnum:
type: string
enum:
- findings
PaginatedFindingList:
type: object
properties:
data:
type: array
items:
$ref: '#/components/schemas/Finding'
required:
- data
PaginatedProviderList:
type: object
properties:
@@ -1545,8 +2161,9 @@ components:
* `azure` - Azure
* `gcp` - GCP
* `kubernetes` - Kubernetes
provider_id:
uid:
type: string
title: Unique identifier for the provider, set by the provider
maxLength: 63
minLength: 3
alias:
@@ -1566,7 +2183,7 @@ components:
scanner_args: {}
required:
- provider
- provider_id
- uid
ProviderCreate:
type: object
required:
@@ -1599,13 +2216,14 @@ components:
* `azure` - Azure
* `gcp` - GCP
* `kubernetes` - Kubernetes
provider_id:
uid:
type: string
title: Unique identifier for the provider, set by the provider
maxLength: 63
minLength: 3
scanner_args: {}
required:
- provider_id
- uid
ProviderCreateRequest:
type: object
properties:
@@ -1642,13 +2260,14 @@ components:
* `azure` - Azure
* `gcp` - GCP
* `kubernetes` - Kubernetes
provider_id:
uid:
type: string
minLength: 3
title: Unique identifier for the provider, set by the provider
maxLength: 63
scanner_args: {}
required:
- provider_id
- uid
required:
- data
ProviderCreateResponse:
@@ -1746,6 +2365,30 @@ components:
- data
description: The identifier of the related object.
title: Resource Identifier
findings:
type: object
properties:
data:
type: object
properties:
id:
type: string
type:
type: string
enum:
- Finding
title: Resource Type Name
description: The [type](https://jsonapi.org/format/#document-resource-object-identification)
member is used to describe resource objects that share common
attributes and relationships.
required:
- id
- type
required:
- data
description: The identifier of the related object.
title: Resource Identifier
readOnly: true
required:
- provider
ResourceResponse:
@@ -2161,10 +2804,6 @@ components:
type: string
pattern: ^[\w.@+-]+$
maxLength: 150
password:
type: string
writeOnly: true
maxLength: 128
email:
type: string
format: email
@@ -2175,7 +2814,6 @@ components:
readOnly: true
required:
- username
- password
- email
UserCreate:
type: object
+180
View File
@@ -1347,3 +1347,183 @@ class TestResourceViewSet:
headers=tenant_header,
)
assert response.status_code == status.HTTP_404_NOT_FOUND
@pytest.mark.django_db
class TestFindingViewSet:
def test_findings_list_none(self, authenticated_client, tenant_header):
response = authenticated_client.get(
reverse("finding-list"), headers=tenant_header
)
assert response.status_code == status.HTTP_200_OK
assert len(response.json()["data"]) == 0
def test_findings_list(self, authenticated_client, findings_fixture, tenant_header):
response = authenticated_client.get(
reverse("finding-list"), headers=tenant_header
)
assert response.status_code == status.HTTP_200_OK
assert len(response.json()["data"]) == len(findings_fixture)
assert (
response.json()["data"][0]["attributes"]["status"]
== findings_fixture[0].status
)
@pytest.mark.parametrize(
"filter_name, filter_value, expected_count",
(
[
("delta", "new", 1),
("provider_type", "aws", 2),
("provider_uid", "123456789012", 2),
(
"resource_uid",
"arn:aws:ec2:us-east-1:123456789012:instance/i-1234567890abcdef0",
1,
),
("resource_uid.icontains", "i-1234567890abcdef", 2),
("resource_name", "My Instance 2", 1),
("resource_name.icontains", "ce 2", 1),
("region", "eu-west-1", 1),
("region.in", "eu-west-1,eu-west-2", 1),
("region.icontains", "east", 1),
("service", "ec2", 1),
("service.in", "ec2,s3", 2),
("service.icontains", "ec", 1),
("inserted_at.gte", "2024-01-01", 2),
("updated_at.lte", "2024-01-01", 0),
("resource_type.icontains", "prowler", 2),
# full text search on finding
("search", "dev-qa", 1),
("search", "orange juice", 1),
# full text search on resource
("search", "ec2", 2),
# full text search on finding tags
("search", "value2", 2),
]
),
)
def test_finding_filters(
self,
authenticated_client,
findings_fixture,
tenant_header,
filter_name,
filter_value,
expected_count,
):
response = authenticated_client.get(
reverse("finding-list"),
{f"filter[{filter_name}]": filter_value},
headers=tenant_header,
)
assert response.status_code == status.HTTP_200_OK
assert len(response.json()["data"]) == expected_count
def test_finding_filter_by_provider(
self, authenticated_client, findings_fixture, tenant_header
):
response = authenticated_client.get(
reverse("finding-list"),
{
"filter[provider]": findings_fixture[0].scan.provider.id,
},
headers=tenant_header,
)
assert response.status_code == status.HTTP_200_OK
assert len(response.json()["data"]) == 2
def test_finding_filter_by_provider_id_in(
self, authenticated_client, findings_fixture, tenant_header
):
response = authenticated_client.get(
reverse("finding-list"),
{
"filter[provider.in]": [
findings_fixture[0].scan.provider.id,
findings_fixture[1].scan.provider.id,
]
},
headers=tenant_header,
)
assert response.status_code == status.HTTP_200_OK
assert len(response.json()["data"]) == 2
@pytest.mark.parametrize(
"filter_name",
(
[
"finding", # Invalid filter name
"invalid",
]
),
)
def test_findings_filters_invalid(
self, authenticated_client, tenant_header, filter_name
):
response = authenticated_client.get(
reverse("finding-list"),
{f"filter[{filter_name}]": "whatever"},
headers=tenant_header,
)
assert response.status_code == status.HTTP_400_BAD_REQUEST
@pytest.mark.parametrize(
"sort_field",
[
"status",
"severity",
"check_id",
"inserted_at",
"updated_at",
],
)
def test_findings_sort(self, authenticated_client, tenant_header, sort_field):
response = authenticated_client.get(
reverse("finding-list"), {"sort": sort_field}, headers=tenant_header
)
assert response.status_code == status.HTTP_200_OK
def test_findings_sort_invalid(self, authenticated_client, tenant_header):
response = authenticated_client.get(
reverse("finding-list"), {"sort": "invalid"}, headers=tenant_header
)
assert response.status_code == status.HTTP_400_BAD_REQUEST
assert response.json()["errors"][0]["code"] == "invalid"
assert response.json()["errors"][0]["source"]["pointer"] == "/data"
assert (
response.json()["errors"][0]["detail"] == "invalid sort parameter: invalid"
)
def test_findings_retrieve(
self, authenticated_client, findings_fixture, tenant_header
):
finding_1, *_ = findings_fixture
response = authenticated_client.get(
reverse("finding-detail", kwargs={"pk": finding_1.id}),
headers=tenant_header,
)
assert response.status_code == status.HTTP_200_OK
assert response.json()["data"]["attributes"]["status"] == finding_1.status
assert (
response.json()["data"]["attributes"]["status_extended"]
== finding_1.status_extended
)
assert response.json()["data"]["attributes"]["severity"] == finding_1.severity
assert response.json()["data"]["attributes"]["check_id"] == finding_1.check_id
assert response.json()["data"]["relationships"]["scan"]["data"]["id"] == str(
finding_1.scan.id
)
assert response.json()["data"]["relationships"]["resources"]["data"][0][
"id"
] == str(finding_1.resources.first().id)
def test_findings_invalid_retrieve(self, authenticated_client, tenant_header):
response = authenticated_client.get(
reverse("finding-detail", kwargs={"pk": "random_id"}),
headers=tenant_header,
)
assert response.status_code == status.HTTP_404_NOT_FOUND
+95 -1
View File
@@ -4,8 +4,22 @@ from django.contrib.auth.password_validation import validate_password
from drf_spectacular.utils import extend_schema_field
from rest_framework_json_api import serializers
from rest_framework_json_api.serializers import ValidationError
from rest_framework_json_api.relations import SerializerMethodResourceRelatedField
from api.models import StateChoices, User, Provider, Scan, Task, Resource, ResourceTag
from api.models import (
StateChoices,
User,
Provider,
Scan,
Task,
Resource,
ResourceTag,
SeverityChoices,
StatusChoices,
Finding,
ResourceFindingMapping,
)
from api.rls import Tenant
from api.utils import merge_dicts
@@ -372,6 +386,10 @@ class ResourceSerializer(RLSSerializer):
tags = serializers.SerializerMethodField()
type_ = serializers.CharField(read_only=True)
findings = SerializerMethodResourceRelatedField(
method_name="get_findings", many=True, read_only=True
)
class Meta:
model = Resource
fields = [
@@ -386,6 +404,7 @@ class ResourceSerializer(RLSSerializer):
"tags",
"provider",
"url",
"findings",
]
extra_kwargs = {
"id": {"read_only": True},
@@ -393,6 +412,14 @@ class ResourceSerializer(RLSSerializer):
"updated_at": {"read_only": True},
}
included_serializers = {
"findings": "api.v1.serializers.FindingSerializer",
}
def get_findings(self, obj):
mappings = ResourceFindingMapping.objects.filter(resource=obj)
return Finding.objects.filter(id__in=[m.finding_id for m in mappings])
@extend_schema_field(
{
"type": "object",
@@ -409,3 +436,70 @@ class ResourceSerializer(RLSSerializer):
type_ = fields.pop("type_")
fields["type"] = type_
return fields
class FindingVisbilityEnumSerializerField(serializers.ChoiceField):
def __init__(self, **kwargs):
kwargs["choices"] = Finding.DeltaChoices.choices
super().__init__(**kwargs)
class SeverityEnumSerializerField(serializers.ChoiceField):
def __init__(self, **kwargs):
kwargs["choices"] = SeverityChoices.choices
super().__init__(**kwargs)
class StatusEnumSerializerField(serializers.ChoiceField):
def __init__(self, **kwargs):
kwargs["choices"] = StatusChoices.choices
super().__init__(**kwargs)
class ResourceFindingMappingSerializer(serializers.ModelSerializer):
resource = ResourceSerializer(read_only=True)
class Meta:
model = ResourceFindingMapping
fields = ["resource"]
class FindingSerializer(RLSSerializer):
"""
Serializer for the Finding model.
"""
resources = SerializerMethodResourceRelatedField(
method_name="get_resources", many=True, read_only=True
)
class Meta:
model = Finding
fields = [
"id",
"delta",
"status",
"status_extended",
"severity",
"check_id",
"check_metadata",
"raw_result",
"inserted_at",
"updated_at",
"url",
# Relationships
"scan",
"resources",
]
included_serializers = {
"scan": "api.v1.serializers.ScanSerializer",
"resources": "api.v1.serializers.ResourceSerializer",
}
class JSONAPIMeta:
resource_name = "findings"
def get_resources(self, obj):
mappings = ResourceFindingMapping.objects.filter(finding=obj)
return Resource.objects.filter(id__in={m.resource_id for m in mappings})
+3
View File
@@ -10,6 +10,7 @@ from api.v1.views import (
ScanViewSet,
TaskViewSet,
ResourceViewSet,
FindingViewSet,
)
router = routers.DefaultRouter(trailing_slash=False)
@@ -20,6 +21,8 @@ router.register(r"providers", ProviderViewSet, basename="provider")
router.register(r"scans", ScanViewSet, basename="scan")
router.register(r"tasks", TaskViewSet, basename="task")
router.register(r"resources", ResourceViewSet, basename="resource")
router.register(r"findings", FindingViewSet, basename="finding")
urlpatterns = [
path("", include(router.urls)),
+67 -1
View File
@@ -15,6 +15,7 @@ from rest_framework.exceptions import MethodNotAllowed, NotFound
from rest_framework.generics import get_object_or_404
from rest_framework_json_api.views import Response
from api.base_views import BaseRLSViewSet, BaseViewSet
from api.filters import (
ProviderFilter,
@@ -22,8 +23,10 @@ from api.filters import (
ScanFilter,
TaskFilter,
ResourceFilter,
FindingFilter,
)
from api.models import User, Provider, Scan, Task, Resource
from api.models import User, Provider, Scan, Task, Resource, Finding
from api.rls import Tenant
from api.v1.serializers import (
UserSerializer,
@@ -39,6 +42,7 @@ from api.v1.serializers import (
ScanCreateSerializer,
ScanUpdateSerializer,
ResourceSerializer,
FindingSerializer,
)
from tasks.tasks import check_provider_connection_task, delete_provider_task
@@ -473,11 +477,73 @@ class ResourceViewSet(BaseRLSViewSet):
| Q(name=search_value)
| Q(region=search_value)
| Q(service=search_value)
| Q(type=search_value)
| Q(text_search=search_query)
| Q(uid__contains=search_value)
| Q(name__contains=search_value)
| Q(region__contains=search_value)
| Q(service__contains=search_value)
| Q(type__contains=search_value)
).distinct()
return queryset
@extend_schema_view(
list=extend_schema(
summary="List all findings",
description="Retrieve a list of all findings with options for filtering by various criteria.",
),
retrieve=extend_schema(
summary="Retrieve data from a specific finding",
description="Fetch detailed information about a specific finding by its ID.",
),
)
@method_decorator(CACHE_DECORATOR, name="list")
@method_decorator(CACHE_DECORATOR, name="retrieve")
class FindingViewSet(BaseRLSViewSet):
queryset = Finding.objects.all()
serializer_class = FindingSerializer
http_method_names = ["get"]
filterset_class = FindingFilter
ordering = ["inserted_at"]
ordering_fields = [
"status",
"severity",
"check_id",
"inserted_at",
"updated_at",
]
def get_queryset(self):
# TODO: require scan_id filter, or if none provided, inject today
queryset = Finding.objects.all()
search_value = self.request.query_params.get("filter[search]", None)
if search_value:
# Django's ORM will build a LEFT JOIN and OUTER JOIN on any "through" tables, resulting in duplicates
# The duplicates then require a `distinct` query
search_query = SearchQuery(
search_value, config="simple", search_type="plain"
)
queryset = queryset.filter(
Q(impact_extended__contains=search_value)
| Q(status_extended__contains=search_value)
| Q(check_id=search_value)
| Q(check_id__icontains=search_value)
| Q(text_search=search_query)
| Q(resources__uid=search_value)
| Q(resources__name=search_value)
| Q(resources__region=search_value)
| Q(resources__service=search_value)
| Q(resources__type=search_value)
| Q(resources__uid__contains=search_value)
| Q(resources__name__contains=search_value)
| Q(resources__region__contains=search_value)
| Q(resources__service__contains=search_value)
| Q(resources__tags__text_search=search_query)
| Q(resources__text_search=search_query)
).distinct()
return queryset
+71 -3
View File
@@ -8,7 +8,19 @@ from rest_framework.test import APIClient
from rest_framework import status
from api.models import User, Provider, Resource, ResourceTag, Scan, StateChoices, Task
from prowler.lib.outputs.finding import Severity, Status
from api.models import (
User,
Provider,
Finding,
Resource,
ResourceTag,
Scan,
StateChoices,
Task,
)
from api.rls import Tenant
API_JSON_CONTENT_TYPE = "application/vnd.api+json"
@@ -221,7 +233,7 @@ def resources_fixture(providers_fixture):
uid="arn:aws:ec2:us-east-1:123456789012:instance/i-1234567890abcdef1",
name="My Instance 2",
region="eu-west-1",
service="ec2",
service="s3",
type="prowler-test",
)
resource2.upsert_or_delete_tags(tags)
@@ -232,7 +244,7 @@ def resources_fixture(providers_fixture):
uid="arn:aws:ec2:us-east-1:123456789012:bucket/i-1234567890abcdef2",
name="My Bucket 3",
region="us-east-1",
service="s3",
service="ec2",
type="test",
)
@@ -248,6 +260,62 @@ def resources_fixture(providers_fixture):
return resource1, resource2, resource3
@pytest.fixture
def findings_fixture(scans_fixture, resources_fixture):
scan, *_ = scans_fixture
resource1, resource2, *_ = resources_fixture
finding1 = Finding.objects.create(
tenant_id=scan.tenant_id,
scan=scan,
delta=None,
status=Status.FAIL,
status_extended="test status extended ",
impact=Severity.critical,
impact_extended="test impact extended one",
severity=Severity.critical,
raw_result={
"status": Status.FAIL,
"impact": Severity.critical,
"severity": Severity.critical,
},
tags={"test": "dev-qa"},
check_id="test_check_id",
check_metadata={
"CheckId": "test_check_id",
"Description": "test description apple sauce",
},
)
finding1.add_resources([resource1])
finding2 = Finding.objects.create(
tenant_id=scan.tenant_id,
scan=scan,
delta="new",
status=Status.FAIL,
status_extended="Load Balancer exposed to internet",
impact=Severity.critical,
impact_extended="test impact extended two",
severity=Severity.critical,
raw_result={
"status": Status.FAIL,
"impact": Severity.critical,
"severity": Severity.critical,
},
tags={"test": "test"},
check_id="test_check_id",
check_metadata={
"CheckId": "test_check_id",
"Description": "test description orange juice",
},
)
finding2.add_resources([resource2])
return finding1, finding2
@pytest.fixture
def tenant_header(tenants_fixture):
return {"X-Tenant-ID": str(tenants_fixture[0].id)}