ben.carrasco/prowler
1
0
Fork 0
mirror of https://github.com/prowler-cloud/prowler.git synced 2025-12-19 05:17:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: add missing SDK fields to API findings and resources (#7318)

Browse Source
This commit is contained in:
Adrián Jesús Peña Rodríguez
2025-04-04 14:57:49 +02:00
committed by GitHub
parent b1569ac2f3
commit 6dbf2ac606
10 changed files with 189 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
api/CHANGELOG.md
View File

@@ -12,6 +12,7 @@ All notable changes to the **Prowler API** are documented in this file.
- HTTP Security Headers [(#7289)](https://github.com/prowler-cloud/prowler/pull/7289).
- New endpoint to get the compliance overviews metadata [(#7333)](https://github.com/prowler-cloud/prowler/pull/7333).
- Support for muted findings [(#7378)](https://github.com/prowler-cloud/prowler/pull/7378).
- Added missing fields to API findings and resources [(#7318)](https://github.com/prowler-cloud/prowler/pull/7318).
---

32
api/src/backend/api/migrations/0016_finding_compliance_resource_details_and_more.py Normal file
View File

@@ -0,0 +1,32 @@
# Generated by Django 5.1.5 on 2025-03-31 10:46
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("api", "0015_finding_muted"),
]
operations = [
migrations.AddField(
model_name="finding",
name="compliance",
field=models.JSONField(blank=True, default=dict, null=True),
),
migrations.AddField(
model_name="resource",
name="details",
field=models.TextField(blank=True, null=True),
),
migrations.AddField(
model_name="resource",
name="metadata",
field=models.TextField(blank=True, null=True),
),
migrations.AddField(
model_name="resource",
name="partition",
field=models.TextField(blank=True, null=True),
),
]

6
api/src/backend/api/models.py
View File

@@ -518,6 +518,11 @@ class Resource(RowLevelSecurityProtectedModel):
editable=False,
)
metadata = models.TextField(blank=True, null=True)
details = models.TextField(blank=True, null=True)
partition = models.TextField(blank=True, null=True)
# Relationships
tags = models.ManyToManyField(
ResourceTag,
verbose_name="Tags associated with the resource, by provider",
@@ -656,6 +661,7 @@ class Finding(PostgresPartitionedModel, RowLevelSecurityProtectedModel):
check_id = models.CharField(max_length=100, blank=False, null=False)
check_metadata = models.JSONField(default=dict, null=False)
muted = models.BooleanField(default=False, null=False)
compliance = models.JSONField(default=dict, null=True, blank=True)
# Relationships
scan = models.ForeignKey(to=Scan, related_name="findings", on_delete=models.CASCADE)

14
api/src/backend/tasks/jobs/scan.py
View File

@@ -1,3 +1,4 @@
import json
import time
from copy import deepcopy
from datetime import datetime, timezone
@@ -6,6 +7,7 @@ from celery.utils.log import get_task_logger
from config.settings.celery import CELERY_DEADLOCK_ATTEMPTS
from django.db import IntegrityError, OperationalError
from django.db.models import Case, Count, IntegerField, Sum, When
from tasks.utils import CustomEncoder
from api.compliance import (
PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE,
@@ -191,6 +193,17 @@ def perform_prowler_scan(
if resource_instance.type != finding.resource_type:
resource_instance.type = finding.resource_type
updated_fields.append("type")
if resource_instance.metadata != finding.resource_metadata:
resource_instance.metadata = json.dumps(
finding.resource_metadata, cls=CustomEncoder
)
updated_fields.append("metadata")
if resource_instance.details != finding.resource_details:
resource_instance.details = finding.resource_details
updated_fields.append("details")
if resource_instance.partition != finding.partition:
resource_instance.partition = finding.partition
updated_fields.append("partition")
if updated_fields:
with rls_transaction(tenant_id):
resource_instance.save(update_fields=updated_fields)
@@ -268,6 +281,7 @@ def perform_prowler_scan(
scan=scan_instance,
first_seen_at=last_first_seen_at,
muted=finding.muted,
compliance=finding.compliance,
)
finding_instance.add_resources([resource_instance])

5
api/src/backend/tasks/tasks.py
View File

@@ -246,6 +246,11 @@ def generate_outputs(scan_id: str, provider_id: str, tenant_id: str):
scan_id (str): The scan identifier.
provider_id (str): The provider_id id to be used in generating outputs.
"""
# Check if the scan has findings
if not ScanSummary.objects.filter(scan_id=scan_id).exists():
logger.info(f"No findings found for scan {scan_id}")
return {"upload": False}
# Initialize the prowler provider
prowler_provider = initialize_prowler_provider(Provider.objects.get(id=provider_id))

14
api/src/backend/tasks/tests/test_scan.py
View File

@@ -1,3 +1,4 @@
import json
import uuid
from unittest.mock import MagicMock, patch
@@ -7,6 +8,7 @@ from tasks.jobs.scan import (
_store_resources,
perform_prowler_scan,
)
from tasks.utils import CustomEncoder
from api.models import (
Finding,
@@ -109,6 +111,11 @@ class TestPerformScan:
finding.resource_tags = {"tag1": "value1", "tag2": "value2"}
finding.muted = False
finding.raw = {}
finding.resource_metadata = {"test": "metadata"}
finding.resource_details = {"details": "test"}
finding.partition = "partition"
finding.muted = True
finding.compliance = {"compliance1": "PASS"}
# Mock the ProwlerScan instance
mock_prowler_scan_instance = MagicMock()
@@ -146,6 +153,8 @@ class TestPerformScan:
assert scan_finding.severity == finding.severity
assert scan_finding.check_id == finding.check_id
assert scan_finding.raw_result == finding.raw
assert scan_finding.muted
assert scan_finding.compliance == finding.compliance
assert scan_resource.tenant == tenant
assert scan_resource.uid == finding.resource_uid
@@ -153,6 +162,11 @@ class TestPerformScan:
assert scan_resource.service == finding.service_name
assert scan_resource.type == finding.resource_type
assert scan_resource.name == finding.resource_name
assert scan_resource.metadata == json.dumps(
finding.resource_metadata, cls=CustomEncoder
)
assert scan_resource.details == f"{finding.resource_details}"
assert scan_resource.partition == finding.partition
# Assert that the resource tags have been created and associated
tags = scan_resource.tags.all()

23
api/src/backend/tasks/utils.py
View File

@@ -1,9 +1,32 @@
import json
from datetime import datetime, timedelta, timezone
from enum import Enum
from django_celery_beat.models import PeriodicTask
from django_celery_results.models import TaskResult
class CustomEncoder(json.JSONEncoder):
def default(self, o):
# Enum serialization
if isinstance(o, Enum):
return o.value
# Datetime and timedelta serialization
if isinstance(o, datetime):
return o.isoformat(timespec="seconds")
if isinstance(o, timedelta):
return o.total_seconds()
# Custom object serialization
try:
return super().default(o)
except TypeError:
try:
return o.__dict__
except AttributeError:
return str(o)
def get_next_execution_datetime(task_id: int, provider_id: str) -> datetime:
task_instance = TaskResult.objects.get(task_id=task_id)
try:

2
examples/output/example_output_azure.ocsf.json
View File

@@ -377,7 +377,7 @@
"product_uid": "prowler",
"title": "Ensure Image Vulnerability Scanning using Azure Defender image scanning or a third party provider",
"types": [],
"uid": "prowler-azure-defender_container_images_scan_enabled-<subscription_uid>-global-Dender plan for Containers"
"uid": "prowler-azure-defender_container_images_scan_enabled-<subscription_uid>-global-Defender plan for Containers"
},
"resources": [
{

25
prowler/lib/outputs/finding.py
View File

@@ -1,3 +1,4 @@
import json
from datetime import datetime
from types import SimpleNamespace
from typing import Optional, Union
@@ -94,7 +95,6 @@ class Finding(BaseModel):
Returns:
dict: A dictionary containing the metadata with keys converted to lowercase.
"""
return dict_to_lowercase(self.metadata.dict())
@classmethod
@@ -120,13 +120,16 @@ class Finding(BaseModel):
output_data = {}
output_data.update(common_finding_data)
bulk_checks_metadata = {}
if hasattr(output_options, "bulk_checks_metadata"):
bulk_checks_metadata = output_options.bulk_checks_metadata
try:
output_data["compliance"] = check_output.compliance
except AttributeError:
bulk_checks_metadata = {}
if hasattr(output_options, "bulk_checks_metadata"):
bulk_checks_metadata = output_options.bulk_checks_metadata
output_data["compliance"] = get_check_compliance(
check_output, provider.type, bulk_checks_metadata
)
output_data["compliance"] = get_check_compliance(
check_output, provider.type, bulk_checks_metadata
)
try:
output_data["provider"] = provider.type
output_data["resource_metadata"] = check_output.resource
@@ -305,14 +308,11 @@ class Finding(BaseModel):
Finding: A new Finding instance populated with data from the provided model.
"""
# Missing Finding's API values
finding.muted = False
finding.resource_details = ""
resource = finding.resources.first()
finding.resource_arn = resource.uid
finding.resource_name = resource.name
# TODO: Change this when the API has all the values
finding.resource = {}
finding.resource = json.loads(resource.metadata)
finding.resource_details = resource.details
finding.resource_id = resource.name if provider.type == "aws" else resource.uid
@@ -367,6 +367,7 @@ class Finding(BaseModel):
finding.resource_tags = unroll_tags(
[{"key": tag.key, "value": tag.value} for tag in resource.tags.all()]
)
return cls.generate_output(provider, finding, SimpleNamespace())
def _transform_findings_stats(scan_summaries: list[dict]) -> dict:

108
tests/lib/outputs/finding_test.py
View File

@@ -53,7 +53,12 @@ def mock_check_metadata(provider):
def mock_get_check_compliance(*_):
return {"mock_compliance_key": "mock_compliance_value"}
return {
"CIS-2.0": ["1.12"],
"CIS-3.0": ["1.12"],
"ENS-RD2022": ["op.acc.2.gcp.rbak.1"],
"MITRE-ATTACK": ["T1098"],
}
class DummyTag:
@@ -71,12 +76,25 @@ class DummyTags:
class DummyResource:
def __init__(self, uid, name, resource_arn, region, tags):
def __init__(
self,
uid,
name,
resource_arn,
region,
tags,
details=None,
metadata=None,
partition=None,
):
self.uid = uid
self.name = name
self.resource_arn = resource_arn
self.region = region
self.tags = DummyTags(tags)
self.details = details or ""
self.metadata = metadata or "{}"
self.partition = partition
def __iter__(self):
yield "uid", self.uid
@@ -112,14 +130,8 @@ class DummyAPIFinding:
Attributes will be added dynamically.
"""
pass
class TestFinding:
@patch(
"prowler.lib.outputs.finding.get_check_compliance",
new=mock_get_check_compliance,
)
def test_generate_output_aws(self):
# Mock provider
provider = MagicMock()
@@ -145,7 +157,13 @@ class TestFinding:
check_output.status_extended = "mock_status_extended"
check_output.muted = False
check_output.check_metadata = mock_check_metadata(provider="aws")
check_output.resource = {}
check_output.resource = {"metadata": "mock_metadata"}
check_output.compliance = {
"CIS-2.0": ["1.12"],
"CIS-3.0": ["1.12"],
"ENS-RD2022": ["op.acc.2.gcp.rbak.1"],
"MITRE-ATTACK": ["T1098"],
}
# Mock output options
output_options = MagicMock()
@@ -160,9 +178,14 @@ class TestFinding:
assert finding_output.resource_name == "test_resource_id"
assert finding_output.resource_uid == "test_resource_arn"
assert finding_output.resource_details == "test_resource_details"
assert finding_output.resource_metadata == {"metadata": "mock_metadata"}
assert finding_output.partition == "aws"
assert finding_output.region == "us-west-1"
assert finding_output.compliance == {
"mock_compliance_key": "mock_compliance_value"
"CIS-2.0": ["1.12"],
"CIS-3.0": ["1.12"],
"ENS-RD2022": ["op.acc.2.gcp.rbak.1"],
"MITRE-ATTACK": ["T1098"],
}
assert finding_output.status == Status.PASS
assert finding_output.status_extended == "mock_status_extended"
@@ -211,10 +234,6 @@ class TestFinding:
assert finding_output.service_name == "mock_service_name"
assert finding_output.raw == {}
@patch(
"prowler.lib.outputs.finding.get_check_compliance",
new=mock_get_check_compliance,
)
def test_generate_output_azure(self):
# Mock provider
provider = MagicMock()
@@ -243,6 +262,12 @@ class TestFinding:
check_output.muted = False
check_output.check_metadata = mock_check_metadata(provider="azure")
check_output.resource = {}
check_output.compliance = {
"CIS-2.0": ["1.12"],
"CIS-3.0": ["1.12"],
"ENS-RD2022": ["op.acc.2.gcp.rbak.1"],
"MITRE-ATTACK": ["T1098"],
}
# Mock output options
output_options = MagicMock()
@@ -262,7 +287,10 @@ class TestFinding:
assert finding_output.resource_uid == "test_resource_id"
assert finding_output.region == "us-west-1"
assert finding_output.compliance == {
"mock_compliance_key": "mock_compliance_value"
"CIS-2.0": ["1.12"],
"CIS-3.0": ["1.12"],
"ENS-RD2022": ["op.acc.2.gcp.rbak.1"],
"MITRE-ATTACK": ["T1098"],
}
assert finding_output.status == Status.PASS
assert finding_output.status_extended == "mock_status_extended"
@@ -298,10 +326,6 @@ class TestFinding:
assert finding_output.metadata.Notes == "mock_notes"
assert finding_output.metadata.Compliance == []
@patch(
"prowler.lib.outputs.finding.get_check_compliance",
new=mock_get_check_compliance,
)
def test_generate_output_gcp(self):
# Mock provider
provider = MagicMock()
@@ -332,6 +356,12 @@ class TestFinding:
check_output.muted = False
check_output.check_metadata = mock_check_metadata(provider="gcp")
check_output.resource = {}
check_output.compliance = {
"CIS-2.0": ["1.12"],
"CIS-3.0": ["1.12"],
"ENS-RD2022": ["op.acc.2.gcp.rbak.1"],
"MITRE-ATTACK": ["T1098"],
}
# Mock output options
output_options = MagicMock()
@@ -347,7 +377,10 @@ class TestFinding:
assert finding_output.resource_uid == "test_resource_id"
assert finding_output.region == "us-west-1"
assert finding_output.compliance == {
"mock_compliance_key": "mock_compliance_value"
"CIS-2.0": ["1.12"],
"CIS-3.0": ["1.12"],
"ENS-RD2022": ["op.acc.2.gcp.rbak.1"],
"MITRE-ATTACK": ["T1098"],
}
assert finding_output.status == Status.PASS
assert finding_output.status_extended == "mock_status_extended"
@@ -388,10 +421,6 @@ class TestFinding:
assert finding_output.metadata.Notes == "mock_notes"
assert finding_output.metadata.Compliance == []
@patch(
"prowler.lib.outputs.finding.get_check_compliance",
new=mock_get_check_compliance,
)
def test_generate_output_kubernetes(self):
# Mock provider
provider = MagicMock()
@@ -411,6 +440,12 @@ class TestFinding:
check_output.check_metadata = mock_check_metadata(provider="kubernetes")
check_output.timestamp = datetime.now()
check_output.resource = {}
check_output.compliance = {
"CIS-2.0": ["1.12"],
"CIS-3.0": ["1.12"],
"ENS-RD2022": ["op.acc.2.gcp.rbak.1"],
"MITRE-ATTACK": ["T1098"],
}
# Mock Output Options
output_options = MagicMock()
@@ -427,7 +462,10 @@ class TestFinding:
assert finding_output.region == "namespace: test_namespace"
assert finding_output.account_name == "context: In-Cluster"
assert finding_output.compliance == {
"mock_compliance_key": "mock_compliance_value"
"CIS-2.0": ["1.12"],
"CIS-3.0": ["1.12"],
"ENS-RD2022": ["op.acc.2.gcp.rbak.1"],
"MITRE-ATTACK": ["T1098"],
}
assert finding_output.status == Status.PASS
assert finding_output.status_extended == "mock_status_extended"
@@ -586,6 +624,7 @@ class TestFinding:
dummy_finding.status_extended = "extended"
dummy_finding.check_metadata = check_metadata
dummy_finding.resources = resources
dummy_finding.muted = True
# Call the transform_api_finding classmethod
finding_obj = Finding.transform_api_finding(dummy_finding, provider)
@@ -627,7 +666,10 @@ class TestFinding:
assert finding_obj.resource_tags == {"env": "prod"}
assert finding_obj.region == "us-east-1"
assert finding_obj.compliance == {
"mock_compliance_key": "mock_compliance_value"
"CIS-2.0": ["1.12"],
"CIS-3.0": ["1.12"],
"ENS-RD2022": ["op.acc.2.gcp.rbak.1"],
"MITRE-ATTACK": ["T1098"],
}
@patch(
@@ -709,6 +751,7 @@ class TestFinding:
)
api_finding.resources = DummyResources(api_resource)
api_finding.subscription = "default"
api_finding.muted = False
finding_obj = Finding.transform_api_finding(api_finding, provider)
assert finding_obj.account_organization_uid == "test-ing-432a-a828-d9c965196f87"
@@ -718,7 +761,10 @@ class TestFinding:
assert finding_obj.region == api_resource.region
assert finding_obj.resource_tags == {}
assert finding_obj.compliance == {
"mock_compliance_key": "mock_compliance_value"
"CIS-2.0": ["1.12"],
"CIS-3.0": ["1.12"],
"ENS-RD2022": ["op.acc.2.gcp.rbak.1"],
"MITRE-ATTACK": ["T1098"],
}
assert finding_obj.status == Status("FAIL")
@@ -807,6 +853,7 @@ class TestFinding:
dummy_finding.check_metadata = check_metadata
dummy_finding.raw_result = {}
dummy_finding.project_id = "project1"
dummy_finding.muted = True
resource = DummyResource(
uid="gcp-resource-uid",
@@ -831,7 +878,10 @@ class TestFinding:
== dummy_project.organization.display_name
)
assert finding_obj.compliance == {
"mock_compliance_key": "mock_compliance_value"
"CIS-2.0": ["1.12"],
"CIS-3.0": ["1.12"],
"ENS-RD2022": ["op.acc.2.gcp.rbak.1"],
"MITRE-ATTACK": ["T1098"],
}
assert finding_obj.status == Status("PASS")
assert finding_obj.status_extended == "GCP check extended"
@@ -893,6 +943,7 @@ class TestFinding:
)
resource.region = "namespace: default"
api_finding.resources = DummyResources(resource)
api_finding.muted = True
finding_obj = Finding.transform_api_finding(api_finding, provider)
assert finding_obj.auth_method == "in-cluster"
assert finding_obj.resource_name == "k8s-resource-name"
@@ -958,6 +1009,7 @@ class TestFinding:
tags=[],
)
dummy_finding.resources = DummyResources(resource)
dummy_finding.muted = True
finding_obj = Finding.transform_api_finding(dummy_finding, provider)
assert finding_obj.auth_method == "ms_identity_type: ms_identity_id"
assert finding_obj.account_uid == "ms-tenant-id"