From a790a5060eee680e8b0e8670ecd9cc71a84fad33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Jes=C3=BAs=20Pe=C3=B1a=20Rodr=C3=ADguez?= Date: Tue, 4 Feb 2025 14:44:24 +0100 Subject: [PATCH] feat(export): add api export system --- .env | 17 ++ api/src/backend/api/decorators.py | 41 ++-- .../api/migrations/0009_scan_output.py | 22 +++ api/src/backend/api/models.py | 2 + api/src/backend/api/rls.py | 4 +- api/src/backend/api/tests/test_utils.py | 7 +- api/src/backend/api/tests/test_views.py | 42 ++-- api/src/backend/api/v1/serializers.py | 8 + api/src/backend/api/v1/views.py | 91 ++++++++- api/src/backend/tasks/jobs/export.py | 138 ++++++++++++++ api/src/backend/tasks/tasks.py | 148 ++++++++++++++- docker-compose-dev.yml | 3 + prowler/__main__.py | 23 --- .../outputs/compliance/compliance_output.py | 3 +- prowler/lib/outputs/csv/csv.py | 3 +- prowler/lib/outputs/finding.py | 179 +++++++++++++++++- prowler/lib/outputs/ocsf/ocsf.py | 7 +- prowler/lib/outputs/output.py | 10 +- 18 files changed, 657 insertions(+), 91 deletions(-) create mode 100644 api/src/backend/api/migrations/0009_scan_output.py create mode 100644 api/src/backend/tasks/jobs/export.py diff --git a/.env b/.env index b0f3dfd7f1..f667ad6183 100644 --- a/.env +++ b/.env @@ -30,6 +30,23 @@ VALKEY_HOST=valkey VALKEY_PORT=6379 VALKEY_DB=0 +# API scan settings +# The AWS access key to be used when uploading scan artifacts to an S3 bucket +# If left empty, default AWS credentials resolution behavior will be used +DJANGO_ARTIFACTS_AWS_ACCESS_KEY_ID="" + +# The AWS secret key to be used when uploading scan artifacts to an S3 bucket +DJANGO_ARTIFACTS_AWS_SECRET_ACCESS_KEY="" + +# An optional AWS session token +DJANGO_ARTIFACTS_AWS_SESSION_TOKEN="" + +# The AWS region where your S3 bucket is located (e.g., "us-east-1") +DJANGO_ARTIFACTS_AWS_DEFAULT_REGION="" + +# The name of the S3 bucket where scan artifacts should be stored +DJANGO_ARTIFACTS_AWS_S3_OUTPUT_BUCKET="" + # Django settings DJANGO_ALLOWED_HOSTS=localhost,127.0.0.1,prowler-api DJANGO_BIND_ADDRESS=0.0.0.0 diff --git a/api/src/backend/api/decorators.py b/api/src/backend/api/decorators.py index a98b38b5eb..b74d12df5d 100644 --- a/api/src/backend/api/decorators.py +++ b/api/src/backend/api/decorators.py @@ -7,7 +7,7 @@ from rest_framework_json_api.serializers import ValidationError from api.db_utils import POSTGRES_TENANT_VAR, SET_CONFIG_QUERY -def set_tenant(func): +def set_tenant(func=None, *, keep_tenant=False): """ Decorator to set the tenant context for a Celery task based on the provided tenant_id. @@ -40,20 +40,29 @@ def set_tenant(func): # The tenant context will be set before the task logic executes. """ - @wraps(func) - @transaction.atomic - def wrapper(*args, **kwargs): - try: - tenant_id = kwargs.pop("tenant_id") - except KeyError: - raise KeyError("This task requires the tenant_id") - try: - uuid.UUID(tenant_id) - except ValueError: - raise ValidationError("Tenant ID must be a valid UUID") - with connection.cursor() as cursor: - cursor.execute(SET_CONFIG_QUERY, [POSTGRES_TENANT_VAR, tenant_id]) + def decorator(func): + @wraps(func) + @transaction.atomic + def wrapper(*args, **kwargs): + try: + if not keep_tenant: + tenant_id = kwargs.pop("tenant_id") + else: + tenant_id = kwargs["tenant_id"] + except KeyError: + raise KeyError("This task requires the tenant_id") + try: + uuid.UUID(tenant_id) + except ValueError: + raise ValidationError("Tenant ID must be a valid UUID") + with connection.cursor() as cursor: + cursor.execute(SET_CONFIG_QUERY, [POSTGRES_TENANT_VAR, tenant_id]) - return func(*args, **kwargs) + return func(*args, **kwargs) - return wrapper + return wrapper + + if func is None: + return decorator + else: + return decorator(func) diff --git a/api/src/backend/api/migrations/0009_scan_output.py b/api/src/backend/api/migrations/0009_scan_output.py new file mode 100644 index 0000000000..ea0510b458 --- /dev/null +++ b/api/src/backend/api/migrations/0009_scan_output.py @@ -0,0 +1,22 @@ +# Generated by Django 5.1.5 on 2025-02-07 10:59 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("api", "0008_daily_scheduled_tasks_update"), + ] + + operations = [ + migrations.AddField( + model_name="scan", + name="output_path", + field=models.CharField(blank=True, max_length=200, null=True), + ), + migrations.AddField( + model_name="scan", + name="upload_to_s3", + field=models.BooleanField(blank=True, null=True), + ), + ] diff --git a/api/src/backend/api/models.py b/api/src/backend/api/models.py index ef982be8c7..45d2f66bcf 100644 --- a/api/src/backend/api/models.py +++ b/api/src/backend/api/models.py @@ -414,6 +414,8 @@ class Scan(RowLevelSecurityProtectedModel): scheduler_task = models.ForeignKey( PeriodicTask, on_delete=models.CASCADE, null=True, blank=True ) + output_path = models.CharField(blank=True, null=True, max_length=200) + upload_to_s3 = models.BooleanField(blank=True, null=True) # TODO: mutelist foreign key class Meta(RowLevelSecurityProtectedModel.Meta): diff --git a/api/src/backend/api/rls.py b/api/src/backend/api/rls.py index c7b15a82dd..f9a044e4c5 100644 --- a/api/src/backend/api/rls.py +++ b/api/src/backend/api/rls.py @@ -88,9 +88,7 @@ class RowLevelSecurityConstraint(models.BaseConstraint): f"{grant_queries}{self.grant_sql_query.format(statement=statement)}" ) - full_create_sql_query = ( - f"{self.rls_sql_query}" f"{policy_queries}" f"{grant_queries}" - ) + full_create_sql_query = f"{self.rls_sql_query}{policy_queries}{grant_queries}" table_name = model._meta.db_table if self.partition_name: diff --git a/api/src/backend/api/tests/test_utils.py b/api/src/backend/api/tests/test_utils.py index fcde1a9999..bfeec54852 100644 --- a/api/src/backend/api/tests/test_utils.py +++ b/api/src/backend/api/tests/test_utils.py @@ -274,9 +274,10 @@ class TestValidateInvitation: expired_time = datetime.now(timezone.utc) - timedelta(days=1) invitation.expires_at = expired_time - with patch("api.utils.Invitation.objects.using") as mock_using, patch( - "api.utils.datetime" - ) as mock_datetime: + with ( + patch("api.utils.Invitation.objects.using") as mock_using, + patch("api.utils.datetime") as mock_datetime, + ): mock_db = mock_using.return_value mock_db.get.return_value = invitation mock_datetime.now.return_value = datetime.now(timezone.utc) diff --git a/api/src/backend/api/tests/test_views.py b/api/src/backend/api/tests/test_views.py index 0173797062..c3c654b20d 100644 --- a/api/src/backend/api/tests/test_views.py +++ b/api/src/backend/api/tests/test_views.py @@ -847,9 +847,9 @@ class TestProviderViewSet: included_data = response.json()["included"] for expected_type in expected_resources: - assert any( - d.get("type") == expected_type for d in included_data - ), f"Expected type '{expected_type}' not found in included data" + assert any(d.get("type") == expected_type for d in included_data), ( + f"Expected type '{expected_type}' not found in included data" + ) def test_providers_retrieve(self, authenticated_client, providers_fixture): provider1, *_ = providers_fixture @@ -2226,9 +2226,9 @@ class TestResourceViewSet: included_data = response.json()["included"] for expected_type in expected_resources: - assert any( - d.get("type") == expected_type for d in included_data - ), f"Expected type '{expected_type}' not found in included data" + assert any(d.get("type") == expected_type for d in included_data), ( + f"Expected type '{expected_type}' not found in included data" + ) @pytest.mark.parametrize( "filter_name, filter_value, expected_count", @@ -2412,9 +2412,9 @@ class TestFindingViewSet: included_data = response.json()["included"] for expected_type in expected_resources: - assert any( - d.get("type") == expected_type for d in included_data - ), f"Expected type '{expected_type}' not found in included data" + assert any(d.get("type") == expected_type for d in included_data), ( + f"Expected type '{expected_type}' not found in included data" + ) @pytest.mark.parametrize( "filter_name, filter_value, expected_count", @@ -2692,9 +2692,9 @@ class TestJWTFields: reverse("token-obtain"), data, format="json" ) - assert ( - response.status_code == status.HTTP_200_OK - ), f"Unexpected status code: {response.status_code}" + assert response.status_code == status.HTTP_200_OK, ( + f"Unexpected status code: {response.status_code}" + ) access_token = response.data["attributes"]["access"] payload = jwt.decode(access_token, options={"verify_signature": False}) @@ -2708,23 +2708,23 @@ class TestJWTFields: # Verify expected fields for field in expected_fields: assert field in payload, f"The field '{field}' is not in the JWT" - assert ( - payload[field] == expected_fields[field] - ), f"The value of '{field}' does not match" + assert payload[field] == expected_fields[field], ( + f"The value of '{field}' does not match" + ) # Verify time fields are integers for time_field in ["exp", "iat", "nbf"]: assert time_field in payload, f"The field '{time_field}' is not in the JWT" - assert isinstance( - payload[time_field], int - ), f"The field '{time_field}' is not an integer" + assert isinstance(payload[time_field], int), ( + f"The field '{time_field}' is not an integer" + ) # Verify identification fields are non-empty strings for id_field in ["jti", "sub", "tenant_id"]: assert id_field in payload, f"The field '{id_field}' is not in the JWT" - assert ( - isinstance(payload[id_field], str) and payload[id_field] - ), f"The field '{id_field}' is not a valid string" + assert isinstance(payload[id_field], str) and payload[id_field], ( + f"The field '{id_field}' is not a valid string" + ) @pytest.mark.django_db diff --git a/api/src/backend/api/v1/serializers.py b/api/src/backend/api/v1/serializers.py index 4f17d2d61c..8ff4568691 100644 --- a/api/src/backend/api/v1/serializers.py +++ b/api/src/backend/api/v1/serializers.py @@ -819,6 +819,14 @@ class ScanTaskSerializer(RLSSerializer): ] +class ScanReportSerializer(RLSSerializer): + class Meta: + model = Scan + fields = [ + "id", + ] + + class ResourceTagSerializer(RLSSerializer): """ Serializer for the ResourceTag model diff --git a/api/src/backend/api/v1/views.py b/api/src/backend/api/v1/views.py index b11626c829..4b125eaca7 100644 --- a/api/src/backend/api/v1/views.py +++ b/api/src/backend/api/v1/views.py @@ -1,9 +1,16 @@ +import glob +import os + +import boto3 +from botocore.exceptions import ClientError, NoCredentialsError, ParamValidationError from celery.result import AsyncResult +from config.env import env from django.conf import settings as django_settings from django.contrib.postgres.aggregates import ArrayAgg from django.contrib.postgres.search import SearchQuery from django.db import transaction from django.db.models import Count, F, OuterRef, Prefetch, Q, Subquery, Sum +from django.http import HttpResponse from django.db.models.functions import Coalesce from django.urls import reverse from django.utils.decorators import method_decorator @@ -35,7 +42,6 @@ from tasks.tasks import ( check_provider_connection_task, delete_provider_task, delete_tenant_task, - perform_scan_summary_task, perform_scan_task, ) @@ -114,6 +120,7 @@ from api.v1.serializers import ( RoleSerializer, RoleUpdateSerializer, ScanCreateSerializer, + ScanReportSerializer, ScanSerializer, ScanUpdateSerializer, ScheduleDailyCreateSerializer, @@ -1073,6 +1080,8 @@ class ScanViewSet(BaseRLSViewSet): return ScanCreateSerializer elif self.action == "partial_update": return ScanUpdateSerializer + elif self.action == "report": + return ScanReportSerializer return super().get_serializer_class() def partial_update(self, request, *args, **kwargs): @@ -1090,6 +1099,82 @@ class ScanViewSet(BaseRLSViewSet): ) return Response(data=read_serializer.data, status=status.HTTP_200_OK) + @extend_schema( + tags=["Scan"], + summary="Download ZIP report", + description="Returns a ZIP file containing the requested report", + request=ScanReportSerializer, + responses={ + 200: OpenApiResponse(description="Report obtained successfully"), + 423: OpenApiResponse( + description="There is a problem with the AWS credentials" + ), + }, + ) + @action(detail=True, methods=["get"], url_name="report") + def report(self, request, pk=None): + scan_instance = Scan.objects.get(pk=pk) + output_path = scan_instance.output_path + if 1 == 2 and scan_instance.upload_to_s3: + s3_client = None + try: + s3_client = boto3.client("s3") + s3_client.list_buckets() + except (ClientError, NoCredentialsError, ParamValidationError): + try: + s3_client = boto3.client( + "s3", + aws_access_key_id=env.str("DJANGO_ARTIFACTS_AWS_ACCESS_KEY_ID"), + aws_secret_access_key=env.str( + "DJANGO_ARTIFACTS_AWS_SECRET_ACCESS_KEY" + ), + aws_session_token=env.str("DJANGO_ARTIFACTS_AWS_SESSION_TOKEN"), + region_name=env.str("DJANGO_ARTIFACTS_AWS_DEFAULT_REGION"), + ) + s3_client.list_buckets() + except (ClientError, NoCredentialsError, ParamValidationError): + return Response( + {"detail": "There is a problem with the AWS credentials."}, + status=status.HTTP_423_LOCKED, + ) + + bucket_name = env.str("DJANGO_ARTIFACTS_AWS_S3_OUTPUT_BUCKET") + + try: + key = output_path[len(f"s3://{bucket_name}/") :] + s3_object = s3_client.get_object(Bucket=bucket_name, Key=key) + file_content = s3_object["Body"].read() + filename = os.path.basename(output_path.split("/")[-1]) + except ClientError: + return Response( + {"detail": "Error accessing cloud storage"}, + status=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) + + else: + zip_files = glob.glob(output_path) + if not zip_files: + return Response( + {"detail": "No local files found"}, status=status.HTTP_404_NOT_FOUND + ) + + try: + file_path = zip_files[0] + with open(file_path, "rb") as f: + file_content = f.read() + filename = os.path.basename(file_path) + except IOError: + return Response( + {"detail": "Error reading local file"}, + status=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) + + response = HttpResponse( + file_content, content_type="application/x-zip-compressed" + ) + response["Content-Disposition"] = f'attachment; filename="{filename}"' + return response + def create(self, request, *args, **kwargs): input_serializer = self.get_serializer(data=request.data) input_serializer.is_valid(raise_exception=True) @@ -1104,10 +1189,6 @@ class ScanViewSet(BaseRLSViewSet): # Disabled for now # checks_to_execute=scan.scanner_args.get("checks_to_execute"), }, - link=perform_scan_summary_task.si( - tenant_id=self.request.tenant_id, - scan_id=str(scan.id), - ), ) scan.task_id = task.id diff --git a/api/src/backend/tasks/jobs/export.py b/api/src/backend/tasks/jobs/export.py new file mode 100644 index 0000000000..c344976a9f --- /dev/null +++ b/api/src/backend/tasks/jobs/export.py @@ -0,0 +1,138 @@ +import os +import zipfile + +import boto3 +from botocore.exceptions import ClientError +from celery.utils.log import get_task_logger +from config.env import env +from prowler.config.config import ( + csv_file_suffix, + html_file_suffix, + json_ocsf_file_suffix, + output_file_timestamp, +) +from prowler.lib.outputs.csv.csv import CSV +from prowler.lib.outputs.html.html import HTML +from prowler.lib.outputs.ocsf.ocsf import OCSF + +logger = get_task_logger(__name__) + +tmp_output_directory = "/tmp/prowler_api_output" + +# Predefined mapping for output formats and their configurations +OUTPUT_FORMATS_MAPPING = { + "csv": { + "class": CSV, + "suffix": csv_file_suffix, + "kwargs": {}, + }, + "json-ocsf": {"class": OCSF, "suffix": json_ocsf_file_suffix, "kwargs": {}}, + "html": {"class": HTML, "suffix": html_file_suffix, "kwargs": {"stats": {}}}, +} + +# Mapping provider types to their identity components for output paths +PROVIDER_IDENTITY_MAP = { + "aws": lambda p: p.identity.account, + "azure": lambda p: p.identity.tenant_domain, + "gcp": lambda p: p.identity.profile, + "kubernetes": lambda p: p.identity.context.replace(":", "_").replace("/", "_"), +} + + +def _compress_output_files(output_directory: str) -> str: + """ + Compress output files from all configured output formats into a ZIP archive. + Args: + output_directory (str): The directory where the output files are located. + The function looks up all known suffixes in OUTPUT_FORMATS_MAPPING + and compresses those files into a single ZIP. + Returns: + str: The full path to the newly created ZIP archive. + """ + zip_path = f"{output_directory}.zip" + + with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zipf: + for suffix in [config["suffix"] for config in OUTPUT_FORMATS_MAPPING.values()]: + zipf.write( + f"{output_directory}{suffix}", + f"artifacts/{output_directory.split('/')[-1]}{suffix}", + ) + + return zip_path + + +def _upload_to_s3(tenant_id: str, zip_path: str, scan_id: str) -> str: + """ + Upload the specified ZIP file to an S3 bucket. + If the S3 bucket environment variables are not configured, + the function returns None without performing an upload. + Args: + tenant_id (str): The tenant identifier, used as part of the S3 key prefix. + zip_path (str): The local file system path to the ZIP file to be uploaded. + scan_id (str): The scan identifier, used as part of the S3 key prefix. + Returns: + str: The S3 URI of the uploaded file (e.g., "s3:///") if successful. + None: If the required environment variables for the S3 bucket are not set. + Raises: + botocore.exceptions.ClientError: If the upload attempt to S3 fails for any reason. + """ + if not env.str("DJANGO_ARTIFACTS_AWS_S3_OUTPUT_BUCKET", ""): + return + + if env.str("DJANGO_ARTIFACTS_AWS_ACCESS_KEY_ID", ""): + s3 = boto3.client( + "s3", + aws_access_key_id=env.str("DJANGO_ARTIFACTS_AWS_ACCESS_KEY_ID"), + aws_secret_access_key=env.str("DJANGO_ARTIFACTS_AWS_SECRET_ACCESS_KEY"), + aws_session_token=env.str("DJANGO_ARTIFACTS_AWS_SESSION_TOKEN"), + region_name=env.str("DJANGO_ARTIFACTS_AWS_DEFAULT_REGION"), + ) + else: + s3 = boto3.client("s3") + + s3_key = f"{tenant_id}/{scan_id}/{os.path.basename(zip_path)}" + try: + s3.upload_file( + Filename=zip_path, + Bucket=env.str("DJANGO_ARTIFACTS_AWS_S3_OUTPUT_BUCKET"), + Key=s3_key, + ) + return f"s3://{env.str('DJANGO_ARTIFACTS_AWS_S3_OUTPUT_BUCKET')}/{s3_key}" + except ClientError as e: + logger.error(f"S3 upload failed: {str(e)}") + raise e + + +def _generate_output_directory( + prowler_provider: object, tenant_id: str, scan_id: str +) -> str: + """ + Generate a file system path for the output directory of a prowler scan. + + This function constructs the output directory path by combining a base + temporary output directory, the tenant ID, the scan ID, and details about + the prowler provider along with a timestamp. The resulting path is used to + store the output files of a prowler scan. + + Note: + This function depends on two external variables: + - `tmp_output_directory`: The base directory where temporary outputs are stored. + - `output_file_timestamp`: A timestamp (as a string) used to uniquely identify the output. + + Args: + prowler_provider (object): An identifier or descriptor for the prowler provider. + Typically, this is a string indicating the provider (e.g., "aws"). + tenant_id (str): The unique identifier for the tenant. + scan_id (str): The unique identifier for the scan. + + Returns: + str: The constructed file system path for the prowler scan output directory. + + Example: + >>> _generate_output_directory("aws", "tenant-1234", "scan-5678") + '/tmp/tenant-1234/scan-5678/prowler-output-aws-2023-02-15T12:34:56' + """ + return ( + f"{tmp_output_directory}/{tenant_id}/{scan_id}/prowler-output-" + f"{prowler_provider}-{output_file_timestamp}" + ) diff --git a/api/src/backend/tasks/tasks.py b/api/src/backend/tasks/tasks.py index 792900c4e7..f4f876b1d3 100644 --- a/api/src/backend/tasks/tasks.py +++ b/api/src/backend/tasks/tasks.py @@ -1,14 +1,24 @@ -from celery import shared_task +import os + +from celery import chain, shared_task from config.celery import RLSTask +from api.utils import initialize_prowler_provider from django_celery_beat.models import PeriodicTask from tasks.jobs.connection import check_provider_connection from tasks.jobs.deletion import delete_provider, delete_tenant +from tasks.jobs.export import ( + OUTPUT_FORMATS_MAPPING, + _compress_output_files, + _generate_output_directory, + _upload_to_s3, +) from tasks.jobs.scan import aggregate_findings, perform_prowler_scan from tasks.utils import get_next_execution_datetime from api.db_utils import rls_transaction from api.decorators import set_tenant -from api.models import Scan, StateChoices +from api.models import Finding, Provider, Scan, ScanSummary, StateChoices +from prowler.lib.outputs.finding import Finding as FindingOutput @shared_task(base=RLSTask, name="provider-connection-check") @@ -68,13 +78,20 @@ def perform_scan_task( Returns: dict: The result of the scan execution, typically including the status and results of the performed checks. """ - return perform_prowler_scan( + result = perform_prowler_scan( tenant_id=tenant_id, scan_id=scan_id, provider_id=provider_id, checks_to_execute=checks_to_execute, ) + chain( + perform_scan_summary_task.s(tenant_id, scan_id), + generate_outputs.si(scan_id, provider_id, tenant_id=tenant_id), + ).apply_async() + + return result + @shared_task(base=RLSTask, bind=True, name="scan-perform-scheduled", queue="scans") def perform_scheduled_scan_task(self, tenant_id: str, provider_id: str): @@ -135,12 +152,11 @@ def perform_scheduled_scan_task(self, tenant_id: str, provider_id: str): scheduler_task_id=periodic_task_instance.id, ) - perform_scan_summary_task.apply_async( - kwargs={ - "tenant_id": tenant_id, - "scan_id": str(scan_instance.id), - } - ) + chain( + perform_scan_summary_task.s(tenant_id, scan_instance.id), + generate_outputs.si(str(scan_instance.id), provider_id, tenant_id=tenant_id), + ).apply_async() + return result @@ -152,3 +168,117 @@ def perform_scan_summary_task(tenant_id: str, scan_id: str): @shared_task(name="tenant-deletion") def delete_tenant_task(tenant_id: str): return delete_tenant(pk=tenant_id) + + +def batched(iterable, batch_size): + """ + Yield successive batches from an iterable. + + Args: + iterable: An iterable source of items. + batch_size (int): The number of items per batch. + + Yields: + tuple: A pair (batch, is_last_batch) where: + - batch (list): A list of items (with length equal to batch_size, + except possibly for the last batch). + - is_last_batch (bool): True if this is the final batch, False otherwise. + """ + batch = [] + for item in iterable: + batch.append(item) + if len(batch) == batch_size: + yield batch, False + batch = [] + + yield batch, True + + +@shared_task(base=RLSTask, name="scan-output", queue="scans") +@set_tenant(keep_tenant=True) +def generate_outputs(scan_id: str, provider_id: str, tenant_id: str): + """ + Process findings in batches and generate output files in multiple formats. + + This function retrieves findings associated with a scan, processes them + in batches of 50, and writes each batch to the corresponding output files. + It reuses output writer instances across batches, updates them with each + batch of transformed findings, and uses a flag to indicate when the final + batch is being processed. Finally, the output files are compressed and + uploaded to S3. + + Args: + tenant_id (str): The tenant identifier. + scan_id (str): The scan identifier. + provider_id (str): The provider_id id to be used in generating outputs. + """ + # Generate and ensure the output directory exists + output_directory = _generate_output_directory(provider_id, tenant_id, scan_id) + os.makedirs("/".join(output_directory.split("/")[:-1]), exist_ok=True) + + # Define auxiliary variables + output_writers = {} + scan_summary = FindingOutput._transform_findings_stats( + ScanSummary.objects.filter(scan_id=scan_id) + ) + + # Retrieve findings queryset + findings_qs = Finding.objects.filter(scan_id=scan_id) + + # Process findings in batches + for batch, is_last_batch in batched(findings_qs.iterator(), 50): + finding_outputs = [ + FindingOutput.transform_api_finding(finding) for finding in batch + ] + + # Generate output files + for mode, config in OUTPUT_FORMATS_MAPPING.items(): + kwargs = dict(config.get("kwargs", {})) + if mode == "html": + kwargs["provider"] = initialize_prowler_provider( + Provider.objects.get(id=provider_id) + ) + kwargs["stats"] = scan_summary + + writer_class = config["class"] + if writer_class in output_writers: + # Reuse the existing writer and update with the new batch + writer = output_writers[writer_class] + writer.findings = finding_outputs + writer.close_file = is_last_batch + else: + # Create a new writer for this format + writer = writer_class( + findings=finding_outputs, + file_path=output_directory, + file_extension=config["suffix"], + ) + writer.close_file = is_last_batch + output_writers[writer_class] = writer + + # Write the current batch using the writer + writer.batch_write_data_to_file(**kwargs) + + # Compress output files + output_directory = _compress_output_files(output_directory) + + # Save to configured storage + uploaded = _upload_to_s3(tenant_id, output_directory, scan_id) + + if uploaded: + output_directory = uploaded + uploaded = True + else: + uploaded = False + + # Update the scan instance with the output path and upload status + Scan.objects.filter(id=scan_id).update( + output_path=output_directory, upload_to_s3=uploaded + ) + + return { + "output_path": output_directory, + "upload_to_s3": uploaded, + "scan_id": scan_id, + "provider_id": provider_id, + } diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index c8ecea96d5..a85cec27c0 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -16,6 +16,7 @@ services: volumes: - "./api/src/backend:/home/prowler/backend" - "./api/pyproject.toml:/home/prowler/pyproject.toml" + - "/tmp/prowler_api_output:/tmp/prowler_api_output" depends_on: postgres: condition: service_healthy @@ -85,6 +86,8 @@ services: env_file: - path: .env required: false + volumes: + - "/tmp/prowler_api_output:/tmp/prowler_api_output" depends_on: valkey: condition: service_healthy diff --git a/prowler/__main__.py b/prowler/__main__.py index c01f2a0a56..f3643deb8a 100644 --- a/prowler/__main__.py +++ b/prowler/__main__.py @@ -351,7 +351,6 @@ def prowler(): if mode == "csv": csv_output = CSV( findings=finding_outputs, - create_file_descriptor=True, file_path=f"{filename}{csv_file_suffix}", ) generated_outputs["regular"].append(csv_output) @@ -361,7 +360,6 @@ def prowler(): if mode == "json-asff": asff_output = ASFF( findings=finding_outputs, - create_file_descriptor=True, file_path=f"{filename}{json_asff_file_suffix}", ) generated_outputs["regular"].append(asff_output) @@ -371,7 +369,6 @@ def prowler(): if mode == "json-ocsf": json_output = OCSF( findings=finding_outputs, - create_file_descriptor=True, file_path=f"{filename}{json_ocsf_file_suffix}", ) generated_outputs["regular"].append(json_output) @@ -379,7 +376,6 @@ def prowler(): if mode == "html": html_output = HTML( findings=finding_outputs, - create_file_descriptor=True, file_path=f"{filename}{html_file_suffix}", ) generated_outputs["regular"].append(html_output) @@ -402,7 +398,6 @@ def prowler(): cis = AWSCIS( findings=finding_outputs, compliance=bulk_compliance_frameworks[compliance_name], - create_file_descriptor=True, file_path=filename, ) generated_outputs["compliance"].append(cis) @@ -416,7 +411,6 @@ def prowler(): mitre_attack = AWSMitreAttack( findings=finding_outputs, compliance=bulk_compliance_frameworks[compliance_name], - create_file_descriptor=True, file_path=filename, ) generated_outputs["compliance"].append(mitre_attack) @@ -430,7 +424,6 @@ def prowler(): ens = AWSENS( findings=finding_outputs, compliance=bulk_compliance_frameworks[compliance_name], - create_file_descriptor=True, file_path=filename, ) generated_outputs["compliance"].append(ens) @@ -444,7 +437,6 @@ def prowler(): aws_well_architected = AWSWellArchitected( findings=finding_outputs, compliance=bulk_compliance_frameworks[compliance_name], - create_file_descriptor=True, file_path=filename, ) generated_outputs["compliance"].append(aws_well_architected) @@ -458,7 +450,6 @@ def prowler(): iso27001 = AWSISO27001( findings=finding_outputs, compliance=bulk_compliance_frameworks[compliance_name], - create_file_descriptor=True, file_path=filename, ) generated_outputs["compliance"].append(iso27001) @@ -472,7 +463,6 @@ def prowler(): kisa_ismsp = AWSKISAISMSP( findings=finding_outputs, compliance=bulk_compliance_frameworks[compliance_name], - create_file_descriptor=True, file_path=filename, ) generated_outputs["compliance"].append(kisa_ismsp) @@ -485,7 +475,6 @@ def prowler(): generic_compliance = GenericCompliance( findings=finding_outputs, compliance=bulk_compliance_frameworks[compliance_name], - create_file_descriptor=True, file_path=filename, ) generated_outputs["compliance"].append(generic_compliance) @@ -502,7 +491,6 @@ def prowler(): cis = AzureCIS( findings=finding_outputs, compliance=bulk_compliance_frameworks[compliance_name], - create_file_descriptor=True, file_path=filename, ) generated_outputs["compliance"].append(cis) @@ -516,7 +504,6 @@ def prowler(): mitre_attack = AzureMitreAttack( findings=finding_outputs, compliance=bulk_compliance_frameworks[compliance_name], - create_file_descriptor=True, file_path=filename, ) generated_outputs["compliance"].append(mitre_attack) @@ -530,7 +517,6 @@ def prowler(): ens = AzureENS( findings=finding_outputs, compliance=bulk_compliance_frameworks[compliance_name], - create_file_descriptor=True, file_path=filename, ) generated_outputs["compliance"].append(ens) @@ -543,7 +529,6 @@ def prowler(): generic_compliance = GenericCompliance( findings=finding_outputs, compliance=bulk_compliance_frameworks[compliance_name], - create_file_descriptor=True, file_path=filename, ) generated_outputs["compliance"].append(generic_compliance) @@ -560,7 +545,6 @@ def prowler(): cis = GCPCIS( findings=finding_outputs, compliance=bulk_compliance_frameworks[compliance_name], - create_file_descriptor=True, file_path=filename, ) generated_outputs["compliance"].append(cis) @@ -574,7 +558,6 @@ def prowler(): mitre_attack = GCPMitreAttack( findings=finding_outputs, compliance=bulk_compliance_frameworks[compliance_name], - create_file_descriptor=True, file_path=filename, ) generated_outputs["compliance"].append(mitre_attack) @@ -588,7 +571,6 @@ def prowler(): ens = GCPENS( findings=finding_outputs, compliance=bulk_compliance_frameworks[compliance_name], - create_file_descriptor=True, file_path=filename, ) generated_outputs["compliance"].append(ens) @@ -601,7 +583,6 @@ def prowler(): generic_compliance = GenericCompliance( findings=finding_outputs, compliance=bulk_compliance_frameworks[compliance_name], - create_file_descriptor=True, file_path=filename, ) generated_outputs["compliance"].append(generic_compliance) @@ -618,7 +599,6 @@ def prowler(): cis = KubernetesCIS( findings=finding_outputs, compliance=bulk_compliance_frameworks[compliance_name], - create_file_descriptor=True, file_path=filename, ) generated_outputs["compliance"].append(cis) @@ -631,7 +611,6 @@ def prowler(): generic_compliance = GenericCompliance( findings=finding_outputs, compliance=bulk_compliance_frameworks[compliance_name], - create_file_descriptor=True, file_path=filename, ) generated_outputs["compliance"].append(generic_compliance) @@ -648,7 +627,6 @@ def prowler(): cis = Microsoft365CIS( findings=finding_outputs, compliance=bulk_compliance_frameworks[compliance_name], - create_file_descriptor=True, file_path=filename, ) generated_outputs["compliance"].append(cis) @@ -661,7 +639,6 @@ def prowler(): generic_compliance = GenericCompliance( findings=finding_outputs, compliance=bulk_compliance_frameworks[compliance_name], - create_file_descriptor=True, file_path=filename, ) generated_outputs["compliance"].append(generic_compliance) diff --git a/prowler/lib/outputs/compliance/compliance_output.py b/prowler/lib/outputs/compliance/compliance_output.py index 5b6cda9574..09fb746f33 100644 --- a/prowler/lib/outputs/compliance/compliance_output.py +++ b/prowler/lib/outputs/compliance/compliance_output.py @@ -29,7 +29,6 @@ class ComplianceOutput(Output): self, findings: List[Finding], compliance: Compliance, - create_file_descriptor: bool = False, file_path: str = None, file_extension: str = "", ) -> None: @@ -48,7 +47,7 @@ class ComplianceOutput(Output): else compliance.Framework ) self.transform(findings, compliance, compliance_name) - if create_file_descriptor: + if not self._file_descriptor and file_path: self.create_file_descriptor(file_path) def batch_write_data_to_file(self) -> None: diff --git a/prowler/lib/outputs/csv/csv.py b/prowler/lib/outputs/csv/csv.py index 9fb37cd151..37b573cb85 100644 --- a/prowler/lib/outputs/csv/csv.py +++ b/prowler/lib/outputs/csv/csv.py @@ -98,7 +98,8 @@ class CSV(Output): fieldnames=self._data[0].keys(), delimiter=";", ) - csv_writer.writeheader() + if self._file_descriptor.tell() == 0: + csv_writer.writeheader() for finding in self._data: csv_writer.writerow(finding) self._file_descriptor.close() diff --git a/prowler/lib/outputs/finding.py b/prowler/lib/outputs/finding.py index 62c5cfe47d..ca1c7fdea4 100644 --- a/prowler/lib/outputs/finding.py +++ b/prowler/lib/outputs/finding.py @@ -4,10 +4,11 @@ from typing import Optional, Union from pydantic import BaseModel, Field, ValidationError from prowler.config.config import prowler_version -from prowler.lib.check.models import Check_Report, CheckMetadata +from prowler.lib.check.models import Check_Report, CheckMetadata, Remediation, Code, Recommendation from prowler.lib.logger import logger from prowler.lib.outputs.common import Status, fill_common_finding_data from prowler.lib.outputs.compliance.compliance import get_check_compliance +from prowler.lib.outputs.utils import unroll_tags from prowler.lib.utils.utils import dict_to_lowercase, get_nested_attribute from prowler.providers.common.provider import Provider @@ -267,3 +268,179 @@ class Finding(BaseModel): f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" ) raise error + + @classmethod + def transform_api_finding( + cls, finding + ) -> "Finding": + """ + Transform a FindingModel instance into an API-friendly Finding object. + + This class method extracts data from a FindingModel instance and maps its + properties to a new Finding object. The transformation populates various + fields including authentication details, timestamp, account information, + check metadata (such as provider, check ID, title, type, service, severity, + and remediation details), as well as resource-specific data. The resulting + Finding object is structured for use in API responses or further processing. + + Args: + finding (API Finding): An API Finding instance containing data from the database. + + Returns: + Finding: A new Finding instance populated with data from the provided model. + """ + output_data = {} + output_data["auth_method"] = "" # Pepe + output_data["timestamp"] = finding.inserted_at + output_data["account_uid"] = finding.scan.provider.uid + output_data["account_name"] = "" + output_data["metadata"] = CheckMetadata( + Provider=finding.check_metadata["provider"], + CheckID=finding.check_metadata["checkid"], + CheckTitle=finding.check_metadata["checktitle"], + CheckType=finding.check_metadata["checktype"], + ServiceName=finding.check_metadata["servicename"], + SubServiceName=finding.check_metadata["subservicename"], + Severity=finding.check_metadata["severity"], + ResourceType=finding.check_metadata["resourcetype"], + Description=finding.check_metadata["description"], + Risk=finding.check_metadata["risk"], + RelatedUrl=finding.check_metadata["relatedurl"], + Remediation=Remediation( + Recommendation=Recommendation( + Text=finding.check_metadata["remediation"]["recommendation"]["text"], + Url=finding.check_metadata["remediation"]["recommendation"]["url"], + ), + Code=Code( + NativeIaC=finding.check_metadata["remediation"]["code"]["nativeiac"], + Terraform=finding.check_metadata["remediation"]["code"]["terraform"], + CLI=finding.check_metadata["remediation"]["code"]["cli"], + Other=finding.check_metadata["remediation"]["code"]["other"], + ), + ), + ResourceIdTemplate=finding.check_metadata["resourceidtemplate"], + Categories=finding.check_metadata["categories"], + DependsOn=finding.check_metadata["dependson"], + RelatedTo=finding.check_metadata["relatedto"], + Notes=finding.check_metadata["notes"], + ) + output_data["uid"] = finding.uid + output_data["status"] = Status(finding.status) + output_data["status_extended"] = finding.status_extended + output_data["resource_uid"] = finding.resources.first().uid + output_data["resource_name"] = finding.resources.first().name + output_data["resource_details"] = "" + resource_tags = finding.resources.first().tags.all() + output_data["resource_tags"] = unroll_tags([{"key": tag.key, "value": tag.value} for tag in resource_tags]) + output_data["region"] = finding.resources.first().region + output_data["compliance"] = {} + + return cls(**output_data) + + def _transform_findings_stats( + scan_summaries: list[dict] + ) -> dict: + """ + Aggregate and transform scan summary data into findings statistics. + + This function processes a list of scan summary objects and calculates overall + metrics such as the total number of passed and failed findings (including muted counts), + as well as a breakdown of results by severity (critical, high, medium, and low). + It also retrieves the unique resource count from the associated scan information. + The final output is a dictionary of aggregated statistics intended for reporting or + further analysis. + + Args: + scan_summaries (list[dict]): A list of scan summary objects. Each object is expected + to have attributes including: + - _pass: Number of passed findings. + - fail: Number of failed findings. + - total: Total number of findings. + - muted: Number indicating if the finding is muted. + - severity: A string representing the severity level. + Additionally, the first scan summary should have an associated + `scan` attribute with a `unique_resource_count`. + + Returns: + dict: A dictionary containing aggregated findings statistics: + - total_pass: Total number of passed findings. + - total_muted_pass: Total number of muted passed findings. + - total_fail: Total number of failed findings. + - total_muted_fail: Total number of muted failed findings. + - resources_count: The unique resource count extracted from the scan. + - findings_count: Total number of findings. + - total_critical_severity_fail: Failed findings with critical severity. + - total_critical_severity_pass: Passed findings with critical severity. + - total_high_severity_fail: Failed findings with high severity. + - total_high_severity_pass: Passed findings with high severity. + - total_medium_severity_fail: Failed findings with medium severity. + - total_medium_severity_pass: Passed findings with medium severity. + - total_low_severity_fail: Failed findings with low severity. + - total_low_severity_pass: Passed findings with low severity. + - all_fails_are_muted: A boolean indicating whether all failing findings are muted. + """ + # Initialize overall counters + total_pass = 0 + total_fail = 0 + muted_pass = 0 + muted_fail = 0 + findings_count = 0 + resources_count = scan_summaries[0].scan.unique_resource_count + + # Initialize severity breakdown counters + critical_severity_pass = 0 + critical_severity_fail = 0 + high_severity_pass = 0 + high_severity_fail = 0 + medium_severity_pass = 0 + medium_severity_fail = 0 + low_severity_pass = 0 + low_severity_fail = 0 + + # Loop over each row from the database + for row in scan_summaries: + # Accumulate overall totals + total_pass += row._pass + total_fail += row.fail + findings_count += row.total + + if row.muted > 0: + if row._pass > 0: + muted_pass += row._pass + if row.fail > 0: + muted_fail += row.fail + + sev = row.severity.lower() + if sev == "critical": + critical_severity_pass += row._pass + critical_severity_fail += row.fail + elif sev == "high": + high_severity_pass += row._pass + high_severity_fail += row.fail + elif sev == "medium": + medium_severity_pass += row._pass + medium_severity_fail += row.fail + elif sev == "low": + low_severity_pass += row._pass + low_severity_fail += row.fail + + all_fails_are_muted = (total_fail > 0) and (total_fail == muted_fail) + + stats = { + "total_pass": total_pass, + "total_muted_pass": muted_pass, + "total_fail": total_fail, + "total_muted_fail": muted_fail, + "resources_count": resources_count, + "findings_count": findings_count, + "total_critical_severity_fail": critical_severity_fail, + "total_critical_severity_pass": critical_severity_pass, + "total_high_severity_fail": high_severity_fail, + "total_high_severity_pass": high_severity_pass, + "total_medium_severity_fail": medium_severity_fail, + "total_medium_severity_pass": medium_severity_pass, + "total_low_severity_fail": low_severity_fail, + "total_low_severity_pass": low_severity_pass, + "all_fails_are_muted": all_fails_are_muted, + } + return stats diff --git a/prowler/lib/outputs/ocsf/ocsf.py b/prowler/lib/outputs/ocsf/ocsf.py index 00d85f5273..2ddba0ddd8 100644 --- a/prowler/lib/outputs/ocsf/ocsf.py +++ b/prowler/lib/outputs/ocsf/ocsf.py @@ -200,7 +200,8 @@ class OCSF(Output): and not self._file_descriptor.closed and self._data ): - self._file_descriptor.write("[") + if self._file_descriptor.tell() == 0: + self._file_descriptor.write("[") for finding in self._data: try: self._file_descriptor.write( @@ -211,14 +212,14 @@ class OCSF(Output): logger.error( f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" ) - if self._file_descriptor.tell() > 0: + if self.close_file: if self._file_descriptor.tell() != 1: self._file_descriptor.seek( self._file_descriptor.tell() - 1, os.SEEK_SET ) self._file_descriptor.truncate() self._file_descriptor.write("]") - self._file_descriptor.close() + self._file_descriptor.close() except Exception as error: logger.error( f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" diff --git a/prowler/lib/outputs/output.py b/prowler/lib/outputs/output.py index 290eeabfc9..da05640f3a 100644 --- a/prowler/lib/outputs/output.py +++ b/prowler/lib/outputs/output.py @@ -23,7 +23,6 @@ class Output(ABC): file_descriptor: Property to access the file descriptor. transform: Abstract method to transform findings into a specific format. batch_write_data_to_file: Abstract method to write data to a file in batches. - create_file_descriptor: Method to create a file descriptor for writing data to a file. """ _data: list @@ -33,21 +32,24 @@ class Output(ABC): def __init__( self, findings: List[Finding], - create_file_descriptor: bool = False, file_path: str = None, file_extension: str = "", ) -> None: self._data = [] + self.close_file = False + self.file_path = file_path + self.file_descriptor = None if not file_extension and file_path: self._file_extension = "".join(Path(file_path).suffixes) if file_extension: self._file_extension = file_extension + self.file_path = f"{file_path}{self.file_extension}" if findings: self.transform(findings) - if create_file_descriptor and file_path: - self.create_file_descriptor(file_path) + if not self.file_descriptor and file_path: + self.create_file_descriptor(self.file_path) @property def data(self):