mirror of
https://github.com/prowler-cloud/prowler.git
synced 2026-07-24 04:51:51 +00:00
feat(overviews): Add API overview endpoints for findings and severity (#5910)
This commit is contained in:
Víctor Fernández Poyatos
committed by
GitHub
GitHub
parent
f576b24fc8
commit
b69a0d5137
@@ -1,4 +1,4 @@
|
||||
name: "API - Pull Request"
|
||||
name: "API - Pull Request"
|
||||
|
||||
on:
|
||||
push:
|
||||
@@ -148,7 +148,7 @@ jobs:
|
||||
working-directory: ./api
|
||||
if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true'
|
||||
run: |
|
||||
poetry run pytest -n auto --cov=./src/backend --cov-report=xml src/backend
|
||||
poetry run pytest --cov=./src/backend --cov-report=xml src/backend
|
||||
- name: Upload coverage reports to Codecov
|
||||
if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true'
|
||||
uses: codecov/codecov-action@v5
|
||||
|
||||
@@ -12,6 +12,7 @@ build/
|
||||
/dist/
|
||||
*.egg-info/
|
||||
*/__pycache__/*.pyc
|
||||
.idea/
|
||||
|
||||
# Session
|
||||
Session.vim
|
||||
|
||||
@@ -97,12 +97,13 @@ repos:
|
||||
- id: safety
|
||||
name: safety
|
||||
description: "Safety is a tool that checks your installed dependencies for known security vulnerabilities"
|
||||
entry: bash -c 'safety check --ignore 70612'
|
||||
entry: bash -c 'safety check --ignore 70612,66963'
|
||||
language: system
|
||||
|
||||
- id: vulture
|
||||
name: vulture
|
||||
description: "Vulture finds unused code in Python programs."
|
||||
entry: bash -c 'vulture --exclude "contrib" --min-confidence 100 .'
|
||||
exclude: 'api/src/backend/'
|
||||
language: system
|
||||
files: '.*\.py'
|
||||
|
||||
Generated
+40
-23
@@ -439,13 +439,13 @@ azure-core = ">=1.26.2,<2.0.0"
|
||||
|
||||
[[package]]
|
||||
name = "azure-mgmt-cosmosdb"
|
||||
version = "9.6.0"
|
||||
version = "9.7.0"
|
||||
description = "Microsoft Azure Cosmos DB Management Client Library for Python"
|
||||
optional = false
|
||||
python-versions = ">=3.8"
|
||||
files = [
|
||||
{file = "azure_mgmt_cosmosdb-9.6.0-py3-none-any.whl", hash = "sha256:02b4108867de58e0b89a206ee7b7588b439e1f6fef2377ce1979b803a0d02d5a"},
|
||||
{file = "azure_mgmt_cosmosdb-9.6.0.tar.gz", hash = "sha256:667c7d8a8f542b0e7972e63274af536ad985187e24a6cc2e3c8eef35560881fc"},
|
||||
{file = "azure_mgmt_cosmosdb-9.7.0-py3-none-any.whl", hash = "sha256:be735a554d16995c8cefe413e62119985f8fabae1cb45a6f6ad2c3958bed14da"},
|
||||
{file = "azure_mgmt_cosmosdb-9.7.0.tar.gz", hash = "sha256:b5072d319f11953d8f12e22459aded1912d5f27e442e1d8b49596a85005410a1"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
@@ -537,6 +537,22 @@ azure-mgmt-core = ">=1.3.2"
|
||||
isodate = ">=0.6.1"
|
||||
typing-extensions = ">=4.6.0"
|
||||
|
||||
[[package]]
|
||||
name = "azure-mgmt-search"
|
||||
version = "9.1.0"
|
||||
description = "Microsoft Azure Search Management Client Library for Python"
|
||||
optional = false
|
||||
python-versions = ">=3.7"
|
||||
files = [
|
||||
{file = "azure-mgmt-search-9.1.0.tar.gz", hash = "sha256:53bc6eeadb0974d21f120bb21bb5e6827df6d650e17347460fd83e2d68883599"},
|
||||
{file = "azure_mgmt_search-9.1.0-py3-none-any.whl", hash = "sha256:488ff81477e980e2b7abf0b857387c74ebbad419e6f6126044e3e6fad2da72b6"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
azure-common = ">=1.1,<2.0"
|
||||
azure-mgmt-core = ">=1.3.2,<2.0.0"
|
||||
isodate = ">=0.6.1,<1.0.0"
|
||||
|
||||
[[package]]
|
||||
name = "azure-mgmt-security"
|
||||
version = "7.0.0"
|
||||
@@ -686,17 +702,17 @@ files = [
|
||||
|
||||
[[package]]
|
||||
name = "boto3"
|
||||
version = "1.35.60"
|
||||
version = "1.35.66"
|
||||
description = "The AWS SDK for Python"
|
||||
optional = false
|
||||
python-versions = ">=3.8"
|
||||
files = [
|
||||
{file = "boto3-1.35.60-py3-none-any.whl", hash = "sha256:a34d28de1a1f6ca6ec3edd05c26db16e422293d8f9dcd94f308059a434596753"},
|
||||
{file = "boto3-1.35.60.tar.gz", hash = "sha256:e573504c67c3e438fd4b0222119ed1a73b644c78eb3b6dee0b36a6c70ecf7677"},
|
||||
{file = "boto3-1.35.66-py3-none-any.whl", hash = "sha256:09a610f8cf4d3c22d4ca69c1f89079e3a1c82805ce94fa0eb4ecdd4d2ba6c4bc"},
|
||||
{file = "boto3-1.35.66.tar.gz", hash = "sha256:c392b9168b65e9c23483eaccb5b68d1f960232d7f967a1e00a045ba065ce050d"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
botocore = ">=1.35.60,<1.36.0"
|
||||
botocore = ">=1.35.66,<1.36.0"
|
||||
jmespath = ">=0.7.1,<2.0.0"
|
||||
s3transfer = ">=0.10.0,<0.11.0"
|
||||
|
||||
@@ -705,13 +721,13 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"]
|
||||
|
||||
[[package]]
|
||||
name = "botocore"
|
||||
version = "1.35.60"
|
||||
version = "1.35.69"
|
||||
description = "Low-level, data-driven core of boto 3."
|
||||
optional = false
|
||||
python-versions = ">=3.8"
|
||||
files = [
|
||||
{file = "botocore-1.35.60-py3-none-any.whl", hash = "sha256:ddccfc39a0a55ac0321191a36d29c2ea9be2c96ceefb3928dd3c91c79c494d50"},
|
||||
{file = "botocore-1.35.60.tar.gz", hash = "sha256:378f53037d817bed2c04a006b7319745e664030182211429c924647273b29bc9"},
|
||||
{file = "botocore-1.35.69-py3-none-any.whl", hash = "sha256:cad8d9305f873404eee4b197d84e60a40975d43cbe1ab63abe893420ddfe6e3c"},
|
||||
{file = "botocore-1.35.69.tar.gz", hash = "sha256:f9f23dd76fb247d9b0e8d411d2995e6f847fc451c026f1e58e300f815b0b36eb"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
@@ -1891,13 +1907,13 @@ grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"]
|
||||
|
||||
[[package]]
|
||||
name = "google-api-python-client"
|
||||
version = "2.153.0"
|
||||
version = "2.154.0"
|
||||
description = "Google API Client Library for Python"
|
||||
optional = false
|
||||
python-versions = ">=3.7"
|
||||
files = [
|
||||
{file = "google_api_python_client-2.153.0-py2.py3-none-any.whl", hash = "sha256:6ff13bbfa92a57972e33ec3808e18309e5981b8ca1300e5da23bf2b4d6947384"},
|
||||
{file = "google_api_python_client-2.153.0.tar.gz", hash = "sha256:35cce8647f9c163fc04fb4d811fc91aae51954a2bdd74918decbe0e65d791dd2"},
|
||||
{file = "google_api_python_client-2.154.0-py2.py3-none-any.whl", hash = "sha256:a521bbbb2ec0ba9d6f307cdd64ed6e21eeac372d1bd7493a4ab5022941f784ad"},
|
||||
{file = "google_api_python_client-2.154.0.tar.gz", hash = "sha256:1b420062e03bfcaa1c79e2e00a612d29a6a934151ceb3d272fe150a656dc8f17"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
@@ -3275,7 +3291,7 @@ files = [
|
||||
|
||||
[[package]]
|
||||
name = "prowler"
|
||||
version = "4.6.0"
|
||||
version = "5.0.0"
|
||||
description = "Prowler is an Open Source security tool to perform AWS, GCP and Azure security best practices assessments, audits, incident response, continuous monitoring, hardening and forensics readiness. It contains hundreds of controls covering CIS, NIST 800, NIST CSF, CISA, RBI, FedRAMP, PCI-DSS, GDPR, HIPAA, FFIEC, SOC2, GXP, AWS Well-Architected Framework Security Pillar, AWS Foundational Technical Review (FTR), ENS (Spanish National Security Scheme) and your custom security frameworks."
|
||||
optional = false
|
||||
python-versions = ">=3.9,<3.13"
|
||||
@@ -3292,26 +3308,27 @@ azure-mgmt-authorization = "4.0.0"
|
||||
azure-mgmt-compute = "33.0.0"
|
||||
azure-mgmt-containerregistry = "10.3.0"
|
||||
azure-mgmt-containerservice = "33.0.0"
|
||||
azure-mgmt-cosmosdb = "9.6.0"
|
||||
azure-mgmt-cosmosdb = "9.7.0"
|
||||
azure-mgmt-keyvault = "10.3.1"
|
||||
azure-mgmt-monitor = "6.0.2"
|
||||
azure-mgmt-network = "28.0.0"
|
||||
azure-mgmt-rdbms = "10.1.0"
|
||||
azure-mgmt-resource = "23.2.0"
|
||||
azure-mgmt-search = "9.1.0"
|
||||
azure-mgmt-security = "7.0.0"
|
||||
azure-mgmt-sql = "3.0.1"
|
||||
azure-mgmt-storage = "21.2.1"
|
||||
azure-mgmt-subscription = "3.1.1"
|
||||
azure-mgmt-web = "7.3.1"
|
||||
azure-storage-blob = "12.24.0"
|
||||
boto3 = "1.35.60"
|
||||
botocore = "1.35.60"
|
||||
boto3 = "1.35.66"
|
||||
botocore = "1.35.69"
|
||||
colorama = "0.4.6"
|
||||
cryptography = "43.0.1"
|
||||
dash = "2.18.2"
|
||||
dash-bootstrap-components = "1.6.0"
|
||||
detect-secrets = "1.5.0"
|
||||
google-api-python-client = "2.153.0"
|
||||
google-api-python-client = "2.154.0"
|
||||
google-auth-httplib2 = ">=0.1,<0.3"
|
||||
jsonschema = "4.23.0"
|
||||
kubernetes = "31.0.0"
|
||||
@@ -3325,7 +3342,7 @@ python-dateutil = "^2.9.0.post0"
|
||||
pytz = "2024.2"
|
||||
schema = "0.7.7"
|
||||
shodan = "1.31.0"
|
||||
slack-sdk = "3.33.3"
|
||||
slack-sdk = "3.33.4"
|
||||
tabulate = "0.9.0"
|
||||
tzlocal = "5.2"
|
||||
|
||||
@@ -3333,7 +3350,7 @@ tzlocal = "5.2"
|
||||
type = "git"
|
||||
url = "https://github.com/prowler-cloud/prowler.git"
|
||||
reference = "master"
|
||||
resolved_reference = "8be83fc632445cd25eeb90ed20257716b673cead"
|
||||
resolved_reference = "9c383baff309d868b37934b694df9aacba397fad"
|
||||
|
||||
[[package]]
|
||||
name = "psutil"
|
||||
@@ -4458,13 +4475,13 @@ files = [
|
||||
|
||||
[[package]]
|
||||
name = "slack-sdk"
|
||||
version = "3.33.3"
|
||||
version = "3.33.4"
|
||||
description = "The Slack API Platform SDK for Python"
|
||||
optional = false
|
||||
python-versions = ">=3.6"
|
||||
files = [
|
||||
{file = "slack_sdk-3.33.3-py2.py3-none-any.whl", hash = "sha256:0515fb93cd03b18de61f876a8304c4c3cef4dd3c2a3bad62d7394d2eb5a3c8e6"},
|
||||
{file = "slack_sdk-3.33.3.tar.gz", hash = "sha256:4cc44c9ffe4bb28a01fbe3264c2f466c783b893a4eca62026ab845ec7c176ff1"},
|
||||
{file = "slack_sdk-3.33.4-py2.py3-none-any.whl", hash = "sha256:9f30cb3c9c07b441c49d53fc27f9f1837ad1592a7e9d4ca431f53cdad8826cc6"},
|
||||
{file = "slack_sdk-3.33.4.tar.gz", hash = "sha256:5e109847f6b6a22d227609226ba4ed936109dc00675bddeb7e0bee502d3ee7e0"},
|
||||
]
|
||||
|
||||
[package.extras]
|
||||
|
||||
@@ -4,47 +4,48 @@ from django.conf import settings
|
||||
from django.db.models import Q
|
||||
from django_filters.rest_framework import (
|
||||
BaseInFilter,
|
||||
FilterSet,
|
||||
BooleanFilter,
|
||||
CharFilter,
|
||||
UUIDFilter,
|
||||
DateFilter,
|
||||
ChoiceFilter,
|
||||
DateFilter,
|
||||
FilterSet,
|
||||
UUIDFilter,
|
||||
)
|
||||
from rest_framework_json_api.django_filters.backends import DjangoFilterBackend
|
||||
from rest_framework_json_api.serializers import ValidationError
|
||||
|
||||
from api.db_utils import (
|
||||
ProviderEnumField,
|
||||
FindingDeltaEnumField,
|
||||
StatusEnumField,
|
||||
SeverityEnumField,
|
||||
InvitationStateEnumField,
|
||||
ProviderEnumField,
|
||||
SeverityEnumField,
|
||||
StatusEnumField,
|
||||
)
|
||||
from api.models import (
|
||||
User,
|
||||
ComplianceOverview,
|
||||
Finding,
|
||||
Invitation,
|
||||
Membership,
|
||||
Provider,
|
||||
ProviderGroup,
|
||||
ProviderSecret,
|
||||
Resource,
|
||||
ResourceTag,
|
||||
Scan,
|
||||
Task,
|
||||
StateChoices,
|
||||
Finding,
|
||||
ScanSummary,
|
||||
SeverityChoices,
|
||||
StateChoices,
|
||||
StatusChoices,
|
||||
ProviderSecret,
|
||||
Invitation,
|
||||
ComplianceOverview,
|
||||
Task,
|
||||
User,
|
||||
)
|
||||
from api.rls import Tenant
|
||||
from api.uuid_utils import (
|
||||
datetime_to_uuid7,
|
||||
uuid7_start,
|
||||
transform_into_uuid7,
|
||||
uuid7_end,
|
||||
uuid7_range,
|
||||
transform_into_uuid7,
|
||||
uuid7_start,
|
||||
)
|
||||
from api.v1.serializers import TaskBase
|
||||
|
||||
@@ -57,6 +58,13 @@ class CustomDjangoFilterBackend(DjangoFilterBackend):
|
||||
"""
|
||||
return None
|
||||
|
||||
def get_filterset_class(self, view, queryset=None):
|
||||
# Check if the view has 'get_filterset_class' method
|
||||
if hasattr(view, "get_filterset_class"):
|
||||
return view.get_filterset_class()
|
||||
# Fallback to the default implementation
|
||||
return super().get_filterset_class(view, queryset)
|
||||
|
||||
|
||||
class UUIDInFilter(BaseInFilter, UUIDFilter):
|
||||
pass
|
||||
@@ -482,3 +490,28 @@ class ComplianceOverviewFilter(FilterSet):
|
||||
"version": ["exact", "icontains"],
|
||||
"region": ["exact", "icontains", "in"],
|
||||
}
|
||||
|
||||
|
||||
class ScanSummaryFilter(FilterSet):
|
||||
inserted_at = DateFilter(field_name="inserted_at", lookup_expr="date")
|
||||
provider_id = UUIDFilter(field_name="scan__provider__id", lookup_expr="exact")
|
||||
provider_type = ChoiceFilter(
|
||||
field_name="scan__provider__provider", choices=Provider.ProviderChoices.choices
|
||||
)
|
||||
provider_type__in = ChoiceInFilter(
|
||||
field_name="scan__provider__provider", choices=Provider.ProviderChoices.choices
|
||||
)
|
||||
region = CharFilter(field_name="region")
|
||||
muted_findings = BooleanFilter(method="filter_muted_findings")
|
||||
|
||||
def filter_muted_findings(self, queryset, name, value):
|
||||
if not value:
|
||||
return queryset.exclude(muted__gt=0)
|
||||
return queryset
|
||||
|
||||
class Meta:
|
||||
model = ScanSummary
|
||||
fields = {
|
||||
"inserted_at": ["date", "gte", "lte"],
|
||||
"region": ["exact", "icontains", "in"],
|
||||
}
|
||||
|
||||
@@ -22,36 +22,36 @@ from uuid6 import uuid7
|
||||
|
||||
import api.rls
|
||||
from api.db_utils import (
|
||||
PostgresEnumMigration,
|
||||
MemberRoleEnumField,
|
||||
DB_PROWLER_PASSWORD,
|
||||
DB_PROWLER_USER,
|
||||
POSTGRES_TENANT_VAR,
|
||||
POSTGRES_USER_VAR,
|
||||
TASK_RUNNER_DB_TABLE,
|
||||
InvitationStateEnum,
|
||||
InvitationStateEnumField,
|
||||
MemberRoleEnum,
|
||||
MemberRoleEnumField,
|
||||
PostgresEnumMigration,
|
||||
ProviderEnum,
|
||||
ProviderEnumField,
|
||||
ProviderSecretTypeEnum,
|
||||
ProviderSecretTypeEnumField,
|
||||
ScanTriggerEnum,
|
||||
StateEnumField,
|
||||
StateEnum,
|
||||
ScanTriggerEnumField,
|
||||
InvitationStateEnum,
|
||||
InvitationStateEnumField,
|
||||
StateEnum,
|
||||
StateEnumField,
|
||||
register_enum,
|
||||
DB_PROWLER_USER,
|
||||
DB_PROWLER_PASSWORD,
|
||||
TASK_RUNNER_DB_TABLE,
|
||||
POSTGRES_TENANT_VAR,
|
||||
POSTGRES_USER_VAR,
|
||||
)
|
||||
from api.models import (
|
||||
Provider,
|
||||
Scan,
|
||||
StateChoices,
|
||||
Finding,
|
||||
StatusChoices,
|
||||
SeverityChoices,
|
||||
Membership,
|
||||
ProviderSecret,
|
||||
Invitation,
|
||||
Membership,
|
||||
Provider,
|
||||
ProviderSecret,
|
||||
Scan,
|
||||
SeverityChoices,
|
||||
StateChoices,
|
||||
StatusChoices,
|
||||
)
|
||||
|
||||
DB_NAME = settings.DATABASES["default"]["NAME"]
|
||||
@@ -289,7 +289,8 @@ class Migration(migrations.Migration):
|
||||
),
|
||||
),
|
||||
# Enable tenants RLS based on memberships
|
||||
migrations.RunSQL(f"""
|
||||
migrations.RunSQL(
|
||||
f"""
|
||||
ALTER TABLE tenants ENABLE ROW LEVEL SECURITY;
|
||||
|
||||
-- Policy for SELECT
|
||||
@@ -364,7 +365,8 @@ class Migration(migrations.Migration):
|
||||
FOR INSERT
|
||||
TO {DB_PROWLER_USER}
|
||||
WITH CHECK (true);
|
||||
"""),
|
||||
"""
|
||||
),
|
||||
# Create and register ProviderEnum type
|
||||
migrations.RunPython(
|
||||
ProviderEnumMigration.create_enum_type,
|
||||
@@ -1482,4 +1484,81 @@ class Migration(migrations.Migration):
|
||||
name="comp_ov_cp_id_req_fail_idx",
|
||||
),
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name="ScanSummary",
|
||||
fields=[
|
||||
(
|
||||
"id",
|
||||
models.UUIDField(
|
||||
default=uuid.uuid4,
|
||||
editable=False,
|
||||
primary_key=True,
|
||||
serialize=False,
|
||||
),
|
||||
),
|
||||
("inserted_at", models.DateTimeField(auto_now_add=True)),
|
||||
("check_id", models.CharField(max_length=100)),
|
||||
("service", models.TextField()),
|
||||
(
|
||||
"severity",
|
||||
api.db_utils.SeverityEnumField(
|
||||
choices=[
|
||||
("critical", "Critical"),
|
||||
("high", "High"),
|
||||
("medium", "Medium"),
|
||||
("low", "Low"),
|
||||
("informational", "Informational"),
|
||||
]
|
||||
),
|
||||
),
|
||||
("region", models.TextField()),
|
||||
("_pass", models.IntegerField(db_column="pass", default=0)),
|
||||
("fail", models.IntegerField(default=0)),
|
||||
("muted", models.IntegerField(default=0)),
|
||||
("total", models.IntegerField(default=0)),
|
||||
("new", models.IntegerField(default=0)),
|
||||
("changed", models.IntegerField(default=0)),
|
||||
("unchanged", models.IntegerField(default=0)),
|
||||
("fail_new", models.IntegerField(default=0)),
|
||||
("fail_changed", models.IntegerField(default=0)),
|
||||
("pass_new", models.IntegerField(default=0)),
|
||||
("pass_changed", models.IntegerField(default=0)),
|
||||
("muted_new", models.IntegerField(default=0)),
|
||||
("muted_changed", models.IntegerField(default=0)),
|
||||
(
|
||||
"scan",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
related_name="aggregations",
|
||||
related_query_name="aggregation",
|
||||
to="api.scan",
|
||||
),
|
||||
),
|
||||
(
|
||||
"tenant",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE, to="api.tenant"
|
||||
),
|
||||
),
|
||||
],
|
||||
options={
|
||||
"db_table": "scan_summaries",
|
||||
"abstract": False,
|
||||
},
|
||||
),
|
||||
migrations.AddConstraint(
|
||||
model_name="scansummary",
|
||||
constraint=api.rls.RowLevelSecurityConstraint(
|
||||
"tenant_id",
|
||||
name="rls_on_scansummary",
|
||||
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
|
||||
),
|
||||
),
|
||||
migrations.AddConstraint(
|
||||
model_name="scansummary",
|
||||
constraint=models.UniqueConstraint(
|
||||
fields=("tenant", "scan", "check_id", "service", "severity", "region"),
|
||||
name="unique_scan_summary",
|
||||
),
|
||||
),
|
||||
]
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import json
|
||||
import re
|
||||
from uuid import uuid4, UUID
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from cryptography.fernet import Fernet
|
||||
from django.conf import settings
|
||||
@@ -11,35 +11,33 @@ from django.core.validators import MinLengthValidator
|
||||
from django.db import models
|
||||
from django.utils.translation import gettext_lazy as _
|
||||
from django_celery_results.models import TaskResult
|
||||
from prowler.lib.check.models import Severity
|
||||
from psqlextra.models import PostgresPartitionedModel
|
||||
from psqlextra.types import PostgresPartitioningMethod
|
||||
from uuid6 import uuid7
|
||||
|
||||
from api.db_utils import (
|
||||
MemberRoleEnumField,
|
||||
enum_to_choices,
|
||||
ProviderEnumField,
|
||||
StateEnumField,
|
||||
ScanTriggerEnumField,
|
||||
FindingDeltaEnumField,
|
||||
SeverityEnumField,
|
||||
StatusEnumField,
|
||||
CustomUserManager,
|
||||
ProviderSecretTypeEnumField,
|
||||
FindingDeltaEnumField,
|
||||
InvitationStateEnumField,
|
||||
one_week_from_now,
|
||||
MemberRoleEnumField,
|
||||
ProviderEnumField,
|
||||
ProviderSecretTypeEnumField,
|
||||
ScanTriggerEnumField,
|
||||
SeverityEnumField,
|
||||
StateEnumField,
|
||||
StatusEnumField,
|
||||
enum_to_choices,
|
||||
generate_random_token,
|
||||
one_week_from_now,
|
||||
)
|
||||
from api.exceptions import ModelValidationError
|
||||
from api.rls import (
|
||||
RowLevelSecurityProtectedModel,
|
||||
)
|
||||
from api.rls import (
|
||||
Tenant,
|
||||
RowLevelSecurityConstraint,
|
||||
BaseSecurityConstraint,
|
||||
RowLevelSecurityConstraint,
|
||||
RowLevelSecurityProtectedModel,
|
||||
Tenant,
|
||||
)
|
||||
from prowler.lib.check.models import Severity
|
||||
|
||||
fernet = Fernet(settings.SECRETS_ENCRYPTION_KEY.encode())
|
||||
|
||||
@@ -856,3 +854,51 @@ class ComplianceOverview(RowLevelSecurityProtectedModel):
|
||||
|
||||
class JSONAPIMeta:
|
||||
resource_name = "compliance-overviews"
|
||||
|
||||
|
||||
class ScanSummary(RowLevelSecurityProtectedModel):
|
||||
id = models.UUIDField(primary_key=True, default=uuid4, editable=False)
|
||||
inserted_at = models.DateTimeField(auto_now_add=True, editable=False)
|
||||
check_id = models.CharField(max_length=100, blank=False, null=False)
|
||||
service = models.TextField(blank=False)
|
||||
severity = SeverityEnumField(choices=SeverityChoices)
|
||||
region = models.TextField(blank=False)
|
||||
_pass = models.IntegerField(db_column="pass", default=0)
|
||||
fail = models.IntegerField(default=0)
|
||||
muted = models.IntegerField(default=0)
|
||||
total = models.IntegerField(default=0)
|
||||
new = models.IntegerField(default=0)
|
||||
changed = models.IntegerField(default=0)
|
||||
unchanged = models.IntegerField(default=0)
|
||||
|
||||
fail_new = models.IntegerField(default=0)
|
||||
fail_changed = models.IntegerField(default=0)
|
||||
pass_new = models.IntegerField(default=0)
|
||||
pass_changed = models.IntegerField(default=0)
|
||||
muted_new = models.IntegerField(default=0)
|
||||
muted_changed = models.IntegerField(default=0)
|
||||
|
||||
scan = models.ForeignKey(
|
||||
Scan,
|
||||
on_delete=models.CASCADE,
|
||||
related_name="aggregations",
|
||||
related_query_name="aggregation",
|
||||
)
|
||||
|
||||
class Meta(RowLevelSecurityProtectedModel.Meta):
|
||||
db_table = "scan_summaries"
|
||||
|
||||
constraints = [
|
||||
models.UniqueConstraint(
|
||||
fields=("tenant", "scan", "check_id", "service", "severity", "region"),
|
||||
name="unique_scan_summary",
|
||||
),
|
||||
RowLevelSecurityConstraint(
|
||||
field="tenant_id",
|
||||
name="rls_on_%(class)s",
|
||||
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
|
||||
),
|
||||
]
|
||||
|
||||
class JSONAPIMeta:
|
||||
resource_name = "scan-summaries"
|
||||
|
||||
@@ -780,16 +780,332 @@ paths:
|
||||
schema:
|
||||
$ref: '#/components/schemas/OpenApiResponseResponse'
|
||||
description: ''
|
||||
/api/v1/overviews/findings:
|
||||
get:
|
||||
operationId: overviews_findings_retrieve
|
||||
description: Fetch aggregated findings data across all providers, grouped by
|
||||
various metrics such as passed, failed, muted, and total findings. This endpoint
|
||||
calculates summary statistics based on the latest scans for each provider
|
||||
and applies any provided filters, such as region, provider type, and scan
|
||||
date.
|
||||
summary: Get aggregated findings data
|
||||
parameters:
|
||||
- in: query
|
||||
name: fields[findings-overview]
|
||||
schema:
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
enum:
|
||||
- id
|
||||
- new
|
||||
- changed
|
||||
- unchanged
|
||||
- fail_new
|
||||
- fail_changed
|
||||
- pass_new
|
||||
- pass_changed
|
||||
- muted_new
|
||||
- muted_changed
|
||||
- total
|
||||
- fail
|
||||
- muted
|
||||
- pass
|
||||
description: endpoint return only specific fields in the response on a per-type
|
||||
basis by including a fields[TYPE] query parameter.
|
||||
explode: false
|
||||
- in: query
|
||||
name: filter[inserted_at]
|
||||
schema:
|
||||
type: string
|
||||
format: date
|
||||
- in: query
|
||||
name: filter[inserted_at__date]
|
||||
schema:
|
||||
type: string
|
||||
format: date
|
||||
- in: query
|
||||
name: filter[inserted_at__gte]
|
||||
schema:
|
||||
type: string
|
||||
format: date-time
|
||||
- in: query
|
||||
name: filter[inserted_at__lte]
|
||||
schema:
|
||||
type: string
|
||||
format: date-time
|
||||
- in: query
|
||||
name: filter[muted_findings]
|
||||
schema:
|
||||
type: boolean
|
||||
- in: query
|
||||
name: filter[provider_id]
|
||||
schema:
|
||||
type: string
|
||||
format: uuid
|
||||
- in: query
|
||||
name: filter[provider_type]
|
||||
schema:
|
||||
type: string
|
||||
enum:
|
||||
- aws
|
||||
- azure
|
||||
- gcp
|
||||
- kubernetes
|
||||
description: |-
|
||||
* `aws` - AWS
|
||||
* `azure` - Azure
|
||||
* `gcp` - GCP
|
||||
* `kubernetes` - Kubernetes
|
||||
- in: query
|
||||
name: filter[provider_type__in]
|
||||
schema:
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
enum:
|
||||
- aws
|
||||
- azure
|
||||
- gcp
|
||||
- kubernetes
|
||||
description: |-
|
||||
Multiple values may be separated by commas.
|
||||
|
||||
* `aws` - AWS
|
||||
* `azure` - Azure
|
||||
* `gcp` - GCP
|
||||
* `kubernetes` - Kubernetes
|
||||
explode: false
|
||||
style: form
|
||||
- in: query
|
||||
name: filter[region]
|
||||
schema:
|
||||
type: string
|
||||
- in: query
|
||||
name: filter[region__icontains]
|
||||
schema:
|
||||
type: string
|
||||
- in: query
|
||||
name: filter[region__in]
|
||||
schema:
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
description: Multiple values may be separated by commas.
|
||||
explode: false
|
||||
style: form
|
||||
- name: filter[search]
|
||||
required: false
|
||||
in: query
|
||||
description: A search term.
|
||||
schema:
|
||||
type: string
|
||||
- name: sort
|
||||
required: false
|
||||
in: query
|
||||
description: '[list of fields to sort by](https://jsonapi.org/format/#fetching-sorting)'
|
||||
schema:
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
enum:
|
||||
- id
|
||||
- -id
|
||||
- new
|
||||
- -new
|
||||
- changed
|
||||
- -changed
|
||||
- unchanged
|
||||
- -unchanged
|
||||
- fail_new
|
||||
- -fail_new
|
||||
- fail_changed
|
||||
- -fail_changed
|
||||
- pass_new
|
||||
- -pass_new
|
||||
- pass_changed
|
||||
- -pass_changed
|
||||
- muted_new
|
||||
- -muted_new
|
||||
- muted_changed
|
||||
- -muted_changed
|
||||
- total
|
||||
- -total
|
||||
- fail
|
||||
- -fail
|
||||
- muted
|
||||
- -muted
|
||||
- pass
|
||||
- -pass
|
||||
explode: false
|
||||
tags:
|
||||
- Overview
|
||||
security:
|
||||
- jwtAuth: []
|
||||
responses:
|
||||
'200':
|
||||
content:
|
||||
application/vnd.api+json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/OverviewFindingResponse'
|
||||
description: ''
|
||||
/api/v1/overviews/findings_severity:
|
||||
get:
|
||||
operationId: overviews_findings_severity_retrieve
|
||||
description: Retrieve an aggregated summary of findings grouped by severity
|
||||
levels, such as low, medium, high, and critical. The response includes the
|
||||
total count of findings for each severity, considering only the latest scans
|
||||
for each provider. Additional filters can be applied to narrow down results
|
||||
by region, provider type, or other attributes.
|
||||
summary: Get findings data by severity
|
||||
parameters:
|
||||
- in: query
|
||||
name: fields[findings-severity-overview]
|
||||
schema:
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
enum:
|
||||
- id
|
||||
- critical
|
||||
- high
|
||||
- medium
|
||||
- low
|
||||
- informational
|
||||
description: endpoint return only specific fields in the response on a per-type
|
||||
basis by including a fields[TYPE] query parameter.
|
||||
explode: false
|
||||
- in: query
|
||||
name: filter[inserted_at]
|
||||
schema:
|
||||
type: string
|
||||
format: date
|
||||
- in: query
|
||||
name: filter[inserted_at__date]
|
||||
schema:
|
||||
type: string
|
||||
format: date
|
||||
- in: query
|
||||
name: filter[inserted_at__gte]
|
||||
schema:
|
||||
type: string
|
||||
format: date-time
|
||||
- in: query
|
||||
name: filter[inserted_at__lte]
|
||||
schema:
|
||||
type: string
|
||||
format: date-time
|
||||
- in: query
|
||||
name: filter[muted_findings]
|
||||
schema:
|
||||
type: boolean
|
||||
- in: query
|
||||
name: filter[provider_id]
|
||||
schema:
|
||||
type: string
|
||||
format: uuid
|
||||
- in: query
|
||||
name: filter[provider_type]
|
||||
schema:
|
||||
type: string
|
||||
enum:
|
||||
- aws
|
||||
- azure
|
||||
- gcp
|
||||
- kubernetes
|
||||
description: |-
|
||||
* `aws` - AWS
|
||||
* `azure` - Azure
|
||||
* `gcp` - GCP
|
||||
* `kubernetes` - Kubernetes
|
||||
- in: query
|
||||
name: filter[provider_type__in]
|
||||
schema:
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
enum:
|
||||
- aws
|
||||
- azure
|
||||
- gcp
|
||||
- kubernetes
|
||||
description: |-
|
||||
Multiple values may be separated by commas.
|
||||
|
||||
* `aws` - AWS
|
||||
* `azure` - Azure
|
||||
* `gcp` - GCP
|
||||
* `kubernetes` - Kubernetes
|
||||
explode: false
|
||||
style: form
|
||||
- in: query
|
||||
name: filter[region]
|
||||
schema:
|
||||
type: string
|
||||
- in: query
|
||||
name: filter[region__icontains]
|
||||
schema:
|
||||
type: string
|
||||
- in: query
|
||||
name: filter[region__in]
|
||||
schema:
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
description: Multiple values may be separated by commas.
|
||||
explode: false
|
||||
style: form
|
||||
- name: filter[search]
|
||||
required: false
|
||||
in: query
|
||||
description: A search term.
|
||||
schema:
|
||||
type: string
|
||||
- name: sort
|
||||
required: false
|
||||
in: query
|
||||
description: '[list of fields to sort by](https://jsonapi.org/format/#fetching-sorting)'
|
||||
schema:
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
enum:
|
||||
- id
|
||||
- -id
|
||||
- critical
|
||||
- -critical
|
||||
- high
|
||||
- -high
|
||||
- medium
|
||||
- -medium
|
||||
- low
|
||||
- -low
|
||||
- informational
|
||||
- -informational
|
||||
explode: false
|
||||
tags:
|
||||
- Overview
|
||||
security:
|
||||
- jwtAuth: []
|
||||
responses:
|
||||
'200':
|
||||
content:
|
||||
application/vnd.api+json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/OverviewSeverityResponse'
|
||||
description: ''
|
||||
/api/v1/overviews/providers:
|
||||
get:
|
||||
operationId: overviews_providers_retrieve
|
||||
description: Fetch aggregated summaries of the latest findings and resources
|
||||
for each provider. This includes counts of passed, failed, and manual findings,
|
||||
as well as the total number of resources managed by each provider.
|
||||
summary: List aggregated overview data for providers
|
||||
description: Retrieve an aggregated overview of findings and resources grouped
|
||||
by providers. The response includes the count of passed, failed, and manual
|
||||
findings, along with the total number of resources managed by each provider.
|
||||
Only the latest findings for each provider are considered in the aggregation
|
||||
to ensure accurate and up-to-date insights.
|
||||
summary: Get aggregated provider data
|
||||
parameters:
|
||||
- in: query
|
||||
name: fields[provider-overviews]
|
||||
name: fields[providers-overview]
|
||||
schema:
|
||||
type: array
|
||||
items:
|
||||
@@ -4400,6 +4716,77 @@ components:
|
||||
$ref: '#/components/schemas/Membership'
|
||||
required:
|
||||
- data
|
||||
OverviewFinding:
|
||||
type: object
|
||||
required:
|
||||
- type
|
||||
- id
|
||||
additionalProperties: false
|
||||
properties:
|
||||
type:
|
||||
allOf:
|
||||
- $ref: '#/components/schemas/OverviewFindingTypeEnum'
|
||||
description: The [type](https://jsonapi.org/format/#document-resource-object-identification)
|
||||
member is used to describe resource objects that share common attributes
|
||||
and relationships.
|
||||
id: {}
|
||||
attributes:
|
||||
type: object
|
||||
properties:
|
||||
id:
|
||||
type: string
|
||||
default: n/a
|
||||
new:
|
||||
type: integer
|
||||
changed:
|
||||
type: integer
|
||||
unchanged:
|
||||
type: integer
|
||||
fail_new:
|
||||
type: integer
|
||||
fail_changed:
|
||||
type: integer
|
||||
pass_new:
|
||||
type: integer
|
||||
pass_changed:
|
||||
type: integer
|
||||
muted_new:
|
||||
type: integer
|
||||
muted_changed:
|
||||
type: integer
|
||||
total:
|
||||
type: integer
|
||||
fail:
|
||||
type: integer
|
||||
muted:
|
||||
type: integer
|
||||
pass:
|
||||
type: integer
|
||||
required:
|
||||
- new
|
||||
- changed
|
||||
- unchanged
|
||||
- fail_new
|
||||
- fail_changed
|
||||
- pass_new
|
||||
- pass_changed
|
||||
- muted_new
|
||||
- muted_changed
|
||||
- total
|
||||
- fail
|
||||
- muted
|
||||
- pass
|
||||
OverviewFindingResponse:
|
||||
type: object
|
||||
properties:
|
||||
data:
|
||||
$ref: '#/components/schemas/OverviewFinding'
|
||||
required:
|
||||
- data
|
||||
OverviewFindingTypeEnum:
|
||||
type: string
|
||||
enum:
|
||||
- findings-overview
|
||||
OverviewProvider:
|
||||
type: object
|
||||
required:
|
||||
@@ -4449,7 +4836,54 @@ components:
|
||||
OverviewProviderTypeEnum:
|
||||
type: string
|
||||
enum:
|
||||
- provider-overviews
|
||||
- providers-overview
|
||||
OverviewSeverity:
|
||||
type: object
|
||||
required:
|
||||
- type
|
||||
- id
|
||||
additionalProperties: false
|
||||
properties:
|
||||
type:
|
||||
allOf:
|
||||
- $ref: '#/components/schemas/OverviewSeverityTypeEnum'
|
||||
description: The [type](https://jsonapi.org/format/#document-resource-object-identification)
|
||||
member is used to describe resource objects that share common attributes
|
||||
and relationships.
|
||||
id: {}
|
||||
attributes:
|
||||
type: object
|
||||
properties:
|
||||
id:
|
||||
type: string
|
||||
default: n/a
|
||||
critical:
|
||||
type: integer
|
||||
high:
|
||||
type: integer
|
||||
medium:
|
||||
type: integer
|
||||
low:
|
||||
type: integer
|
||||
informational:
|
||||
type: integer
|
||||
required:
|
||||
- critical
|
||||
- high
|
||||
- medium
|
||||
- low
|
||||
- informational
|
||||
OverviewSeverityResponse:
|
||||
type: object
|
||||
properties:
|
||||
data:
|
||||
$ref: '#/components/schemas/OverviewSeverity'
|
||||
required:
|
||||
- data
|
||||
OverviewSeverityTypeEnum:
|
||||
type: string
|
||||
enum:
|
||||
- findings-severity-overview
|
||||
PaginatedComplianceOverviewList:
|
||||
type: object
|
||||
required:
|
||||
|
||||
@@ -1,29 +1,24 @@
|
||||
import json
|
||||
from datetime import datetime
|
||||
from datetime import timezone, timedelta
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from unittest.mock import ANY, Mock, patch
|
||||
|
||||
import jwt
|
||||
import pytest
|
||||
from conftest import API_JSON_CONTENT_TYPE, TEST_PASSWORD, TEST_USER
|
||||
from django.urls import reverse
|
||||
from rest_framework import status
|
||||
|
||||
from api.models import (
|
||||
User,
|
||||
Invitation,
|
||||
Membership,
|
||||
Provider,
|
||||
ProviderGroup,
|
||||
ProviderGroupMembership,
|
||||
Scan,
|
||||
ProviderSecret,
|
||||
Invitation,
|
||||
Scan,
|
||||
User,
|
||||
)
|
||||
from api.rls import Tenant
|
||||
from conftest import (
|
||||
API_JSON_CONTENT_TYPE,
|
||||
TEST_PASSWORD,
|
||||
TEST_USER,
|
||||
)
|
||||
|
||||
TODAY = str(datetime.today().date())
|
||||
|
||||
@@ -1794,7 +1789,7 @@ class TestScanViewSet:
|
||||
],
|
||||
)
|
||||
@patch("api.v1.views.Task.objects.get")
|
||||
@patch("api.v1.views.perform_scan_task.delay")
|
||||
@patch("api.v1.views.perform_scan_task.apply_async")
|
||||
def test_scans_create_valid(
|
||||
self,
|
||||
mock_perform_scan_task,
|
||||
@@ -3009,9 +3004,9 @@ class TestInvitationViewSet:
|
||||
response = authenticated_client.get(
|
||||
reverse("invitation-list"),
|
||||
{
|
||||
f"filter[{filter_name}]": filter_value
|
||||
if filter_name != "inviter"
|
||||
else str(user.id)
|
||||
f"filter[{filter_name}]": (
|
||||
filter_value if filter_name != "inviter" else str(user.id)
|
||||
)
|
||||
},
|
||||
)
|
||||
|
||||
@@ -3262,3 +3257,5 @@ class TestOverviewViewSet:
|
||||
assert response.json()["data"][0]["attributes"]["resources"]["total"] == len(
|
||||
resources_fixture
|
||||
)
|
||||
|
||||
# TODO Add more tests for the rest of overviews
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import json
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from django.conf import settings
|
||||
from django.contrib.auth import authenticate
|
||||
@@ -14,24 +14,23 @@ from rest_framework_simplejwt.serializers import TokenObtainPairSerializer
|
||||
from rest_framework_simplejwt.tokens import RefreshToken
|
||||
|
||||
from api.models import (
|
||||
StateChoices,
|
||||
User,
|
||||
ComplianceOverview,
|
||||
Finding,
|
||||
Invitation,
|
||||
Membership,
|
||||
Provider,
|
||||
ProviderGroup,
|
||||
ProviderGroupMembership,
|
||||
Scan,
|
||||
Task,
|
||||
ProviderSecret,
|
||||
Resource,
|
||||
ResourceTag,
|
||||
Finding,
|
||||
ProviderSecret,
|
||||
Invitation,
|
||||
ComplianceOverview,
|
||||
Scan,
|
||||
StateChoices,
|
||||
Task,
|
||||
User,
|
||||
)
|
||||
from api.rls import Tenant
|
||||
|
||||
|
||||
# Tokens
|
||||
|
||||
|
||||
@@ -1234,7 +1233,7 @@ class OverviewProviderSerializer(serializers.Serializer):
|
||||
resources = serializers.SerializerMethodField(read_only=True)
|
||||
|
||||
class JSONAPIMeta:
|
||||
resource_name = "provider-overviews"
|
||||
resource_name = "providers-overview"
|
||||
|
||||
def get_root_meta(self, _resource, _many):
|
||||
return {"version": "v1"}
|
||||
@@ -1270,3 +1269,45 @@ class OverviewProviderSerializer(serializers.Serializer):
|
||||
return {
|
||||
"total": obj["total_resources"],
|
||||
}
|
||||
|
||||
|
||||
class OverviewFindingSerializer(serializers.Serializer):
|
||||
id = serializers.CharField(default="n/a")
|
||||
new = serializers.IntegerField()
|
||||
changed = serializers.IntegerField()
|
||||
unchanged = serializers.IntegerField()
|
||||
fail_new = serializers.IntegerField()
|
||||
fail_changed = serializers.IntegerField()
|
||||
pass_new = serializers.IntegerField()
|
||||
pass_changed = serializers.IntegerField()
|
||||
muted_new = serializers.IntegerField()
|
||||
muted_changed = serializers.IntegerField()
|
||||
total = serializers.IntegerField()
|
||||
_pass = serializers.IntegerField()
|
||||
fail = serializers.IntegerField()
|
||||
muted = serializers.IntegerField()
|
||||
|
||||
class JSONAPIMeta:
|
||||
resource_name = "findings-overview"
|
||||
|
||||
def get_root_meta(self, _resource, _many):
|
||||
return {"version": "v1"}
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.fields["pass"] = self.fields.pop("_pass")
|
||||
|
||||
|
||||
class OverviewSeveritySerializer(serializers.Serializer):
|
||||
id = serializers.CharField(default="n/a")
|
||||
critical = serializers.IntegerField()
|
||||
high = serializers.IntegerField()
|
||||
medium = serializers.IntegerField()
|
||||
low = serializers.IntegerField()
|
||||
informational = serializers.IntegerField()
|
||||
|
||||
class JSONAPIMeta:
|
||||
resource_name = "findings-severity-overview"
|
||||
|
||||
def get_root_meta(self, _resource, _many):
|
||||
return {"version": "v1"}
|
||||
|
||||
+190
-67
@@ -2,20 +2,20 @@ from celery.result import AsyncResult
|
||||
from django.conf import settings as django_settings
|
||||
from django.contrib.postgres.search import SearchQuery
|
||||
from django.db import transaction
|
||||
from django.db.models import Prefetch, Subquery, OuterRef, Count, Q, F
|
||||
from django.db.models import Count, F, OuterRef, Prefetch, Q, Subquery, Sum
|
||||
from django.urls import reverse
|
||||
from django.utils.decorators import method_decorator
|
||||
from django.views.decorators.cache import cache_control
|
||||
from drf_spectacular.settings import spectacular_settings
|
||||
from drf_spectacular.utils import (
|
||||
extend_schema,
|
||||
extend_schema_view,
|
||||
OpenApiParameter,
|
||||
OpenApiResponse,
|
||||
OpenApiTypes,
|
||||
extend_schema,
|
||||
extend_schema_view,
|
||||
)
|
||||
from drf_spectacular.views import SpectacularAPIView
|
||||
from rest_framework import status, permissions
|
||||
from rest_framework import permissions, status
|
||||
from rest_framework.decorators import action
|
||||
from rest_framework.exceptions import (
|
||||
MethodNotAllowed,
|
||||
@@ -23,82 +23,87 @@ from rest_framework.exceptions import (
|
||||
PermissionDenied,
|
||||
ValidationError,
|
||||
)
|
||||
from rest_framework.generics import get_object_or_404, GenericAPIView
|
||||
from rest_framework.generics import GenericAPIView, get_object_or_404
|
||||
from rest_framework_json_api.views import Response
|
||||
from rest_framework_simplejwt.exceptions import InvalidToken
|
||||
from rest_framework_simplejwt.exceptions import TokenError
|
||||
from rest_framework_simplejwt.exceptions import InvalidToken, TokenError
|
||||
from tasks.beat import schedule_provider_scan
|
||||
from tasks.tasks import (
|
||||
check_provider_connection_task,
|
||||
delete_provider_task,
|
||||
perform_scan_summary_task,
|
||||
perform_scan_task,
|
||||
)
|
||||
|
||||
from api.base_views import BaseTenantViewset, BaseRLSViewSet, BaseUserViewset
|
||||
from api.base_views import BaseRLSViewSet, BaseTenantViewset, BaseUserViewset
|
||||
from api.db_router import MainRouter
|
||||
from api.filters import (
|
||||
ComplianceOverviewFilter,
|
||||
FindingFilter,
|
||||
InvitationFilter,
|
||||
MembershipFilter,
|
||||
ProviderFilter,
|
||||
ProviderGroupFilter,
|
||||
TenantFilter,
|
||||
MembershipFilter,
|
||||
ScanFilter,
|
||||
TaskFilter,
|
||||
ResourceFilter,
|
||||
FindingFilter,
|
||||
ProviderSecretFilter,
|
||||
InvitationFilter,
|
||||
ResourceFilter,
|
||||
ScanFilter,
|
||||
ScanSummaryFilter,
|
||||
TaskFilter,
|
||||
TenantFilter,
|
||||
UserFilter,
|
||||
ComplianceOverviewFilter,
|
||||
)
|
||||
from api.models import (
|
||||
StatusChoices,
|
||||
User,
|
||||
ComplianceOverview,
|
||||
Finding,
|
||||
Invitation,
|
||||
Membership,
|
||||
Provider,
|
||||
ProviderGroup,
|
||||
ProviderGroupMembership,
|
||||
Scan,
|
||||
Task,
|
||||
Resource,
|
||||
Finding,
|
||||
ProviderSecret,
|
||||
Invitation,
|
||||
ComplianceOverview,
|
||||
Resource,
|
||||
Scan,
|
||||
ScanSummary,
|
||||
SeverityChoices,
|
||||
StatusChoices,
|
||||
Task,
|
||||
User,
|
||||
)
|
||||
from api.pagination import ComplianceOverviewPagination
|
||||
from api.rls import Tenant
|
||||
from api.utils import validate_invitation
|
||||
from api.uuid_utils import datetime_to_uuid7
|
||||
from api.v1.serializers import (
|
||||
TokenSerializer,
|
||||
TokenRefreshSerializer,
|
||||
UserSerializer,
|
||||
UserCreateSerializer,
|
||||
UserUpdateSerializer,
|
||||
ComplianceOverviewFullSerializer,
|
||||
ComplianceOverviewSerializer,
|
||||
FindingSerializer,
|
||||
InvitationAcceptSerializer,
|
||||
InvitationCreateSerializer,
|
||||
InvitationSerializer,
|
||||
InvitationUpdateSerializer,
|
||||
MembershipSerializer,
|
||||
OverviewFindingSerializer,
|
||||
OverviewProviderSerializer,
|
||||
OverviewSeveritySerializer,
|
||||
ProviderCreateSerializer,
|
||||
ProviderGroupMembershipUpdateSerializer,
|
||||
ProviderGroupSerializer,
|
||||
ProviderGroupUpdateSerializer,
|
||||
ProviderGroupMembershipUpdateSerializer,
|
||||
ProviderSerializer,
|
||||
ProviderCreateSerializer,
|
||||
ProviderUpdateSerializer,
|
||||
TenantSerializer,
|
||||
TaskSerializer,
|
||||
ScanSerializer,
|
||||
ScanCreateSerializer,
|
||||
ScanUpdateSerializer,
|
||||
ResourceSerializer,
|
||||
FindingSerializer,
|
||||
ProviderSecretCreateSerializer,
|
||||
ProviderSecretSerializer,
|
||||
ProviderSecretUpdateSerializer,
|
||||
ProviderSecretCreateSerializer,
|
||||
InvitationSerializer,
|
||||
InvitationCreateSerializer,
|
||||
InvitationUpdateSerializer,
|
||||
InvitationAcceptSerializer,
|
||||
ComplianceOverviewSerializer,
|
||||
ComplianceOverviewFullSerializer,
|
||||
OverviewProviderSerializer,
|
||||
)
|
||||
from tasks.beat import schedule_provider_scan
|
||||
from tasks.tasks import (
|
||||
check_provider_connection_task,
|
||||
delete_provider_task,
|
||||
perform_scan_task,
|
||||
ProviderSerializer,
|
||||
ProviderUpdateSerializer,
|
||||
ResourceSerializer,
|
||||
ScanCreateSerializer,
|
||||
ScanSerializer,
|
||||
ScanUpdateSerializer,
|
||||
TaskSerializer,
|
||||
TenantSerializer,
|
||||
TokenRefreshSerializer,
|
||||
TokenSerializer,
|
||||
UserCreateSerializer,
|
||||
UserSerializer,
|
||||
UserUpdateSerializer,
|
||||
)
|
||||
|
||||
CACHE_DECORATOR = cache_control(
|
||||
@@ -803,12 +808,18 @@ class ScanViewSet(BaseRLSViewSet):
|
||||
with transaction.atomic():
|
||||
scan = input_serializer.save()
|
||||
with transaction.atomic():
|
||||
task = perform_scan_task.delay(
|
||||
tenant_id=request.tenant_id,
|
||||
scan_id=str(scan.id),
|
||||
provider_id=str(scan.provider_id),
|
||||
# Disabled for now
|
||||
# checks_to_execute=scan.scanner_args.get("checks_to_execute"),
|
||||
task = perform_scan_task.apply_async(
|
||||
kwargs={
|
||||
"tenant_id": request.tenant_id,
|
||||
"scan_id": str(scan.id),
|
||||
"provider_id": str(scan.provider_id),
|
||||
# Disabled for now
|
||||
# checks_to_execute=scan.scanner_args.get("checks_to_execute"),
|
||||
},
|
||||
link=perform_scan_summary_task.si(
|
||||
tenant_id=request.tenant_id,
|
||||
scan_id=str(scan.id),
|
||||
),
|
||||
)
|
||||
|
||||
scan.task_id = task.id
|
||||
@@ -1295,26 +1306,67 @@ class ComplianceOverviewViewSet(BaseRLSViewSet):
|
||||
@extend_schema(tags=["Overview"])
|
||||
@extend_schema_view(
|
||||
providers=extend_schema(
|
||||
summary="List aggregated overview data for providers",
|
||||
description="Fetch aggregated summaries of the latest findings and resources for each provider. "
|
||||
"This includes counts of passed, failed, and manual findings, as well as the total number "
|
||||
"of resources managed by each provider.",
|
||||
summary="Get aggregated provider data",
|
||||
description=(
|
||||
"Retrieve an aggregated overview of findings and resources grouped by providers. "
|
||||
"The response includes the count of passed, failed, and manual findings, along with "
|
||||
"the total number of resources managed by each provider. Only the latest findings for "
|
||||
"each provider are considered in the aggregation to ensure accurate and up-to-date insights."
|
||||
),
|
||||
),
|
||||
findings=extend_schema(
|
||||
summary="Get aggregated findings data",
|
||||
description=(
|
||||
"Fetch aggregated findings data across all providers, grouped by various metrics such as "
|
||||
"passed, failed, muted, and total findings. This endpoint calculates summary statistics "
|
||||
"based on the latest scans for each provider and applies any provided filters, such as "
|
||||
"region, provider type, and scan date."
|
||||
),
|
||||
filters=True,
|
||||
),
|
||||
findings_severity=extend_schema(
|
||||
summary="Get findings data by severity",
|
||||
description=(
|
||||
"Retrieve an aggregated summary of findings grouped by severity levels, such as low, medium, "
|
||||
"high, and critical. The response includes the total count of findings for each severity, "
|
||||
"considering only the latest scans for each provider. Additional filters can be applied to "
|
||||
"narrow down results by region, provider type, or other attributes."
|
||||
),
|
||||
filters=True,
|
||||
),
|
||||
)
|
||||
@method_decorator(CACHE_DECORATOR, name="list")
|
||||
class OverviewViewSet(BaseRLSViewSet):
|
||||
queryset = ComplianceOverview.objects.all()
|
||||
http_method_names = ["get"]
|
||||
ordering = ["compliance_id"]
|
||||
ordering = ["-id"]
|
||||
|
||||
def get_queryset(self):
|
||||
return Finding.objects.all()
|
||||
if self.action == "providers":
|
||||
return Finding.objects.all()
|
||||
elif self.action == "findings":
|
||||
return ScanSummary.objects.all()
|
||||
elif self.action == "findings_severity":
|
||||
return ScanSummary.objects.all()
|
||||
else:
|
||||
return super().get_queryset()
|
||||
|
||||
def get_serializer_class(self):
|
||||
if self.action == "providers":
|
||||
return OverviewProviderSerializer
|
||||
elif self.action == "findings":
|
||||
return OverviewFindingSerializer
|
||||
elif self.action == "findings_severity":
|
||||
return OverviewSeveritySerializer
|
||||
return super().get_serializer_class()
|
||||
|
||||
def get_filterset_class(self):
|
||||
if self.action == "providers":
|
||||
return None
|
||||
elif self.action in ["findings", "findings_severity"]:
|
||||
return ScanSummaryFilter
|
||||
return None
|
||||
|
||||
@extend_schema(exclude=True)
|
||||
def list(self, request, *args, **kwargs):
|
||||
raise MethodNotAllowed(method="GET")
|
||||
@@ -1366,7 +1418,7 @@ class OverviewViewSet(BaseRLSViewSet):
|
||||
for res in resources_aggregated
|
||||
if res["provider__provider"] == provider
|
||||
),
|
||||
0, # Default to 0 if no resources are found
|
||||
0,
|
||||
)
|
||||
overview.append(
|
||||
{
|
||||
@@ -1382,3 +1434,74 @@ class OverviewViewSet(BaseRLSViewSet):
|
||||
serializer = OverviewProviderSerializer(overview, many=True)
|
||||
|
||||
return Response(serializer.data, status=status.HTTP_200_OK)
|
||||
|
||||
@action(detail=False, methods=["get"], url_name="findings")
|
||||
def findings(self, request):
|
||||
queryset = self.get_queryset()
|
||||
filtered_queryset = self.filter_queryset(queryset)
|
||||
|
||||
latest_scan_subquery = (
|
||||
Scan.objects.filter(provider_id=OuterRef("scan__provider_id"))
|
||||
.order_by("-id")
|
||||
.values("id")[:1]
|
||||
)
|
||||
|
||||
annotated_queryset = filtered_queryset.annotate(
|
||||
latest_scan_id=Subquery(latest_scan_subquery)
|
||||
)
|
||||
|
||||
filtered_queryset = annotated_queryset.filter(scan_id=F("latest_scan_id"))
|
||||
|
||||
aggregated_totals = filtered_queryset.aggregate(
|
||||
_pass=Sum("_pass") or 0,
|
||||
fail=Sum("fail") or 0,
|
||||
muted=Sum("muted") or 0,
|
||||
total=Sum("total") or 0,
|
||||
new=Sum("new") or 0,
|
||||
changed=Sum("changed") or 0,
|
||||
unchanged=Sum("unchanged") or 0,
|
||||
fail_new=Sum("fail_new") or 0,
|
||||
fail_changed=Sum("fail_changed") or 0,
|
||||
pass_new=Sum("pass_new") or 0,
|
||||
pass_changed=Sum("pass_changed") or 0,
|
||||
muted_new=Sum("muted_new") or 0,
|
||||
muted_changed=Sum("muted_changed") or 0,
|
||||
)
|
||||
|
||||
for key in aggregated_totals:
|
||||
if aggregated_totals[key] is None:
|
||||
aggregated_totals[key] = 0
|
||||
|
||||
serializer = self.get_serializer(aggregated_totals)
|
||||
return Response(serializer.data, status=status.HTTP_200_OK)
|
||||
|
||||
@action(detail=False, methods=["get"], url_name="findings_severity")
|
||||
def findings_severity(self, request):
|
||||
queryset = self.get_queryset()
|
||||
filtered_queryset = self.filter_queryset(queryset)
|
||||
|
||||
latest_scan_subquery = (
|
||||
Scan.objects.filter(provider_id=OuterRef("scan__provider_id"))
|
||||
.order_by("-id")
|
||||
.values("id")[:1]
|
||||
)
|
||||
|
||||
annotated_queryset = filtered_queryset.annotate(
|
||||
latest_scan_id=Subquery(latest_scan_subquery)
|
||||
)
|
||||
|
||||
filtered_queryset = annotated_queryset.filter(scan_id=F("latest_scan_id"))
|
||||
|
||||
severity_counts = (
|
||||
filtered_queryset.values("severity")
|
||||
.annotate(count=Sum("total"))
|
||||
.order_by("severity")
|
||||
)
|
||||
|
||||
severity_data = {sev[0]: 0 for sev in SeverityChoices}
|
||||
|
||||
for item in severity_counts:
|
||||
severity_data[item["severity"]] = item["count"]
|
||||
|
||||
serializer = OverviewSeveritySerializer(severity_data)
|
||||
return Response(serializer.data, status=status.HTTP_200_OK)
|
||||
|
||||
@@ -3,8 +3,7 @@ from copy import deepcopy
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from celery.utils.log import get_task_logger
|
||||
from prowler.lib.outputs.finding import Finding as ProwlerFinding
|
||||
from prowler.lib.scan.scan import Scan as ProwlerScan
|
||||
from django.db.models import Case, Count, IntegerField, Sum, When
|
||||
|
||||
from api.compliance import (
|
||||
PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE,
|
||||
@@ -12,17 +11,20 @@ from api.compliance import (
|
||||
)
|
||||
from api.db_utils import tenant_transaction
|
||||
from api.models import (
|
||||
Provider,
|
||||
Scan,
|
||||
ComplianceOverview,
|
||||
Finding,
|
||||
Provider,
|
||||
Resource,
|
||||
ResourceTag,
|
||||
StatusChoices as FindingStatus,
|
||||
Scan,
|
||||
ScanSummary,
|
||||
StateChoices,
|
||||
ComplianceOverview,
|
||||
)
|
||||
from api.models import StatusChoices as FindingStatus
|
||||
from api.utils import initialize_prowler_provider
|
||||
from api.v1.serializers import ScanTaskSerializer
|
||||
from prowler.lib.outputs.finding import Finding as ProwlerFinding
|
||||
from prowler.lib.scan.scan import Scan as ProwlerScan
|
||||
|
||||
logger = get_task_logger(__name__)
|
||||
|
||||
@@ -268,7 +270,7 @@ def perform_prowler_scan(
|
||||
scan_instance.unique_resource_count = len(unique_resources)
|
||||
scan_instance.save()
|
||||
|
||||
if generate_compliance:
|
||||
if exception is None and generate_compliance:
|
||||
try:
|
||||
regions = prowler_provider.get_regions()
|
||||
except AttributeError:
|
||||
@@ -321,3 +323,155 @@ def perform_prowler_scan(
|
||||
|
||||
serializer = ScanTaskSerializer(instance=scan_instance)
|
||||
return serializer.data
|
||||
|
||||
|
||||
def aggregate_findings(tenant_id: str, scan_id: str):
|
||||
"""
|
||||
Aggregates findings for a given scan and stores the results in the ScanSummary table.
|
||||
|
||||
This function retrieves all findings associated with a given `scan_id` and calculates various
|
||||
metrics such as counts of failed, passed, and muted findings, as well as their deltas (new,
|
||||
changed, unchanged). The results are grouped by `check_id`, `service`, `severity`, and `region`.
|
||||
These aggregated metrics are then stored in the `ScanSummary` table.
|
||||
|
||||
Args:
|
||||
tenant_id (str): The ID of the tenant to which the scan belongs.
|
||||
scan_id (str): The ID of the scan for which findings need to be aggregated.
|
||||
|
||||
Aggregated Metrics:
|
||||
- fail: Total number of failed findings.
|
||||
- _pass: Total number of passed findings.
|
||||
- muted: Total number of muted findings.
|
||||
- total: Total number of findings.
|
||||
- new: Total number of new findings.
|
||||
- changed: Total number of changed findings.
|
||||
- unchanged: Total number of unchanged findings.
|
||||
- fail_new: Failed findings with a delta of 'new'.
|
||||
- fail_changed: Failed findings with a delta of 'changed'.
|
||||
- pass_new: Passed findings with a delta of 'new'.
|
||||
- pass_changed: Passed findings with a delta of 'changed'.
|
||||
- muted_new: Muted findings with a delta of 'new'.
|
||||
- muted_changed: Muted findings with a delta of 'changed'.
|
||||
"""
|
||||
with tenant_transaction(tenant_id):
|
||||
findings = Finding.objects.filter(scan_id=scan_id)
|
||||
|
||||
aggregation = findings.values(
|
||||
"check_id",
|
||||
"resources__service",
|
||||
"severity",
|
||||
"resources__region",
|
||||
).annotate(
|
||||
fail=Sum(
|
||||
Case(
|
||||
When(status="FAIL", then=1),
|
||||
default=0,
|
||||
output_field=IntegerField(),
|
||||
)
|
||||
),
|
||||
_pass=Sum(
|
||||
Case(
|
||||
When(status="PASS", then=1),
|
||||
default=0,
|
||||
output_field=IntegerField(),
|
||||
)
|
||||
),
|
||||
muted=Sum(
|
||||
Case(
|
||||
When(status="MUTED", then=1),
|
||||
default=0,
|
||||
output_field=IntegerField(),
|
||||
)
|
||||
),
|
||||
total=Count("id"),
|
||||
new=Sum(
|
||||
Case(
|
||||
When(delta="new", then=1),
|
||||
default=0,
|
||||
output_field=IntegerField(),
|
||||
)
|
||||
),
|
||||
changed=Sum(
|
||||
Case(
|
||||
When(delta="changed", then=1),
|
||||
default=0,
|
||||
output_field=IntegerField(),
|
||||
)
|
||||
),
|
||||
unchanged=Sum(
|
||||
Case(
|
||||
When(delta__isnull=True, then=1),
|
||||
default=0,
|
||||
output_field=IntegerField(),
|
||||
)
|
||||
),
|
||||
fail_new=Sum(
|
||||
Case(
|
||||
When(delta="new", status="FAIL", then=1),
|
||||
default=0,
|
||||
output_field=IntegerField(),
|
||||
)
|
||||
),
|
||||
fail_changed=Sum(
|
||||
Case(
|
||||
When(delta="changed", status="FAIL", then=1),
|
||||
default=0,
|
||||
output_field=IntegerField(),
|
||||
)
|
||||
),
|
||||
pass_new=Sum(
|
||||
Case(
|
||||
When(delta="new", status="PASS", then=1),
|
||||
default=0,
|
||||
output_field=IntegerField(),
|
||||
)
|
||||
),
|
||||
pass_changed=Sum(
|
||||
Case(
|
||||
When(delta="changed", status="PASS", then=1),
|
||||
default=0,
|
||||
output_field=IntegerField(),
|
||||
)
|
||||
),
|
||||
muted_new=Sum(
|
||||
Case(
|
||||
When(delta="new", status="MUTED", then=1),
|
||||
default=0,
|
||||
output_field=IntegerField(),
|
||||
)
|
||||
),
|
||||
muted_changed=Sum(
|
||||
Case(
|
||||
When(delta="changed", status="MUTED", then=1),
|
||||
default=0,
|
||||
output_field=IntegerField(),
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
with tenant_transaction(tenant_id):
|
||||
scan_aggregations = {
|
||||
ScanSummary(
|
||||
tenant_id=tenant_id,
|
||||
scan_id=scan_id,
|
||||
check_id=agg["check_id"],
|
||||
service=agg["resources__service"],
|
||||
severity=agg["severity"],
|
||||
region=agg["resources__region"],
|
||||
fail=agg["fail"],
|
||||
_pass=agg["_pass"],
|
||||
muted=agg["muted"],
|
||||
total=agg["total"],
|
||||
new=agg["new"],
|
||||
changed=agg["changed"],
|
||||
unchanged=agg["unchanged"],
|
||||
fail_new=agg["fail_new"],
|
||||
fail_changed=agg["fail_changed"],
|
||||
pass_new=agg["pass_new"],
|
||||
pass_changed=agg["pass_changed"],
|
||||
muted_new=agg["muted_new"],
|
||||
muted_changed=agg["muted_changed"],
|
||||
)
|
||||
for agg in aggregation
|
||||
}
|
||||
ScanSummary.objects.bulk_create(scan_aggregations, batch_size=3000)
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
from celery import shared_task
|
||||
from config.celery import RLSTask
|
||||
from tasks.jobs.connection import check_provider_connection
|
||||
from tasks.jobs.deletion import delete_instance
|
||||
from tasks.jobs.scan import aggregate_findings, perform_prowler_scan
|
||||
|
||||
from api.db_utils import tenant_transaction
|
||||
from api.decorators import set_tenant
|
||||
from api.models import Provider, Scan
|
||||
from config.celery import RLSTask
|
||||
from tasks.jobs.connection import check_provider_connection
|
||||
from tasks.jobs.deletion import delete_instance
|
||||
from tasks.jobs.scan import perform_prowler_scan
|
||||
|
||||
|
||||
@shared_task(base=RLSTask, name="provider-connection-check")
|
||||
@@ -110,3 +110,8 @@ def perform_scheduled_scan_task(self, tenant_id: str, provider_id: str):
|
||||
scan_id=str(scan_instance.id),
|
||||
provider_id=provider_id,
|
||||
)
|
||||
|
||||
|
||||
@shared_task(name="scan-summary")
|
||||
def perform_scan_summary_task(tenant_id: str, scan_id: str):
|
||||
return aggregate_findings(tenant_id=tenant_id, scan_id=scan_id)
|
||||
|
||||
@@ -1,19 +1,19 @@
|
||||
from unittest.mock import patch, MagicMock
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from api.models import (
|
||||
StateChoices,
|
||||
Severity,
|
||||
Finding,
|
||||
Resource,
|
||||
StatusChoices,
|
||||
Provider,
|
||||
)
|
||||
from tasks.jobs.scan import (
|
||||
perform_prowler_scan,
|
||||
_create_finding_delta,
|
||||
_store_resources,
|
||||
perform_prowler_scan,
|
||||
)
|
||||
|
||||
from api.models import (
|
||||
Finding,
|
||||
Provider,
|
||||
Resource,
|
||||
Severity,
|
||||
StateChoices,
|
||||
StatusChoices,
|
||||
)
|
||||
|
||||
|
||||
@@ -358,3 +358,6 @@ class TestPerformScan:
|
||||
|
||||
assert resource == resource_instance
|
||||
assert resource_uid_tuple == (resource_instance.uid, resource_instance.region)
|
||||
|
||||
|
||||
# TODO Add tests for aggregations
|
||||
|
||||
Reference in New Issue
Block a user