From 019dec744be62cc1fe879f22c8cd11d39e94a4e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Jes=C3=BAs=20Pe=C3=B1a=20Rodr=C3=ADguez?= Date: Thu, 24 Apr 2025 13:23:52 +0200 Subject: [PATCH] feat: add compliance to API report files --- api/src/backend/tasks/jobs/export.py | 113 +++++++++++++++++- api/src/backend/tasks/tasks.py | 33 +++++ .../outputs/compliance/compliance_output.py | 15 ++- 3 files changed, 153 insertions(+), 8 deletions(-) diff --git a/api/src/backend/tasks/jobs/export.py b/api/src/backend/tasks/jobs/export.py index 11c9e5c4cf..c7d6b7fff9 100644 --- a/api/src/backend/tasks/jobs/export.py +++ b/api/src/backend/tasks/jobs/export.py @@ -13,6 +13,30 @@ from prowler.config.config import ( json_ocsf_file_suffix, output_file_timestamp, ) +from prowler.lib.outputs.compliance.aws_well_architected.aws_well_architected import ( + AWSWellArchitected, +) +from prowler.lib.outputs.compliance.cis.cis_aws import AWSCIS +from prowler.lib.outputs.compliance.cis.cis_azure import AzureCIS +from prowler.lib.outputs.compliance.cis.cis_gcp import GCPCIS +from prowler.lib.outputs.compliance.cis.cis_kubernetes import KubernetesCIS +from prowler.lib.outputs.compliance.cis.cis_m365 import M365CIS +from prowler.lib.outputs.compliance.ens.ens_aws import AWSENS +from prowler.lib.outputs.compliance.ens.ens_azure import AzureENS +from prowler.lib.outputs.compliance.ens.ens_gcp import GCPENS +from prowler.lib.outputs.compliance.iso27001.iso27001_aws import AWSISO27001 +from prowler.lib.outputs.compliance.iso27001.iso27001_azure import AzureISO27001 +from prowler.lib.outputs.compliance.iso27001.iso27001_gcp import GCPISO27001 +from prowler.lib.outputs.compliance.iso27001.iso27001_kubernetes import ( + KubernetesISO27001, +) +from prowler.lib.outputs.compliance.iso27001.iso27001_nhn import NHNISO27001 +from prowler.lib.outputs.compliance.kisa_ismsp.kisa_ismsp_aws import AWSKISAISMSP +from prowler.lib.outputs.compliance.mitre_attack.mitre_attack_aws import AWSMitreAttack +from prowler.lib.outputs.compliance.mitre_attack.mitre_attack_azure import ( + AzureMitreAttack, +) +from prowler.lib.outputs.compliance.mitre_attack.mitre_attack_gcp import GCPMitreAttack from prowler.lib.outputs.csv.csv import CSV from prowler.lib.outputs.html.html import HTML from prowler.lib.outputs.ocsf.ocsf import OCSF @@ -20,6 +44,43 @@ from prowler.lib.outputs.ocsf.ocsf import OCSF logger = get_task_logger(__name__) +COMPLIANCE_CLASS_MAP = { + "aws": [ + (lambda name: name.startswith("cis_"), AWSCIS), + (lambda name: name == "mitre_attack_aws", AWSMitreAttack), + (lambda name: name.startswith("ens_"), AWSENS), + ( + lambda name: name.startswith("aws_well_architected_framework"), + AWSWellArchitected, + ), + (lambda name: name.startswith("iso27001_"), AWSISO27001), + (lambda name: name.startswith("kisa"), AWSKISAISMSP), + ], + "azure": [ + (lambda name: name.startswith("cis_"), AzureCIS), + (lambda name: name == "mitre_attack_azure", AzureMitreAttack), + (lambda name: name.startswith("ens_"), AzureENS), + (lambda name: name.startswith("iso27001_"), AzureISO27001), + ], + "gcp": [ + (lambda name: name.startswith("cis_"), GCPCIS), + (lambda name: name == "mitre_attack_gcp", GCPMitreAttack), + (lambda name: name.startswith("ens_"), GCPENS), + (lambda name: name.startswith("iso27001_"), GCPISO27001), + ], + "kubernetes": [ + (lambda name: name.startswith("cis_"), KubernetesCIS), + (lambda name: name.startswith("iso27001_"), KubernetesISO27001), + ], + "m365": [ + (lambda name: name.startswith("cis_"), M365CIS), + ], + "nhn": [ + (lambda name: name.startswith("iso27001_"), NHNISO27001), + ], +} + + # Predefined mapping for output formats and their configurations OUTPUT_FORMATS_MAPPING = { "csv": { @@ -43,13 +104,17 @@ def _compress_output_files(output_directory: str) -> str: str: The full path to the newly created ZIP archive. """ zip_path = f"{output_directory}.zip" + parent_dir = os.path.dirname(output_directory) + zip_path_abs = os.path.abspath(zip_path) with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zipf: - for suffix in [config["suffix"] for config in OUTPUT_FORMATS_MAPPING.values()]: - zipf.write( - f"{output_directory}{suffix}", - f"output/{output_directory.split('/')[-1]}{suffix}", - ) + for foldername, _, filenames in os.walk(parent_dir): + for filename in filenames: + file_path = os.path.join(foldername, filename) + if os.path.abspath(file_path) == zip_path_abs: + continue + arcname = os.path.relpath(file_path, start=parent_dir) + zipf.write(file_path, arcname) return zip_path @@ -154,3 +219,41 @@ def _generate_output_directory( os.makedirs("/".join(path.split("/")[:-1]), exist_ok=True) return path + + +def _generate_compliance_output_directory( + output_directory, prowler_provider: object, tenant_id: str, scan_id: str +) -> str: + """ + Generate a file system path for the compliance output directory of a prowler scan. + + This function constructs the compliance-specific output directory path by combining + a base temporary output directory, the tenant ID, the scan ID, and details about + the prowler provider along with a timestamp. The resulting path is used to store + compliance-related output files generated during a prowler scan. + + Note: + This function depends on one external variable: + - `output_file_timestamp`: A timestamp (as a string) used to uniquely identify the output. + + Args: + output_directory (str): The base output directory. + prowler_provider (object): An identifier or descriptor for the prowler provider. + Typically, this is a string indicating the provider (e.g., "aws"). + tenant_id (str): The unique identifier for the tenant. + scan_id (str): The unique identifier for the scan. + + Returns: + str: The constructed file system path for the prowler compliance output directory. + + Example: + >>> _generate_compliance_output_directory("/tmp", "aws", "tenant-1234", "scan-5678") + '/tmp/tenant-1234/scan-5678/compliance/prowler-output-aws-2023-02-15T12:34:56' + """ + path = ( + f"{output_directory}/{tenant_id}/{scan_id}/compliance/prowler-output-" + f"{prowler_provider}-{output_file_timestamp}" + ) + os.makedirs("/".join(path.split("/")[:-1]), exist_ok=True) + + return path diff --git a/api/src/backend/tasks/tasks.py b/api/src/backend/tasks/tasks.py index 5af7895394..096ef66ade 100644 --- a/api/src/backend/tasks/tasks.py +++ b/api/src/backend/tasks/tasks.py @@ -10,8 +10,10 @@ from django_celery_beat.models import PeriodicTask from tasks.jobs.connection import check_provider_connection from tasks.jobs.deletion import delete_provider, delete_tenant from tasks.jobs.export import ( + COMPLIANCE_CLASS_MAP, OUTPUT_FORMATS_MAPPING, _compress_output_files, + _generate_compliance_output_directory, _generate_output_directory, _upload_to_s3, ) @@ -23,6 +25,9 @@ from api.decorators import set_tenant from api.models import Finding, Provider, Scan, ScanSummary, StateChoices from api.utils import initialize_prowler_provider from api.v1.serializers import ScanTaskSerializer +from prowler.config.config import get_available_compliance_frameworks +from prowler.lib.check.compliance_models import Compliance +from prowler.lib.outputs.compliance.generic.generic import GenericCompliance from prowler.lib.outputs.finding import Finding as FindingOutput logger = get_task_logger(__name__) @@ -256,12 +261,22 @@ def generate_outputs(scan_id: str, provider_id: str, tenant_id: str): # Get the provider UID provider_uid = Provider.objects.get(id=provider_id).uid + provider_type = Provider.objects.get(id=provider_id).provider + + # Get the compliance frameworks for the provider + bulk_compliance_frameworks = Compliance.get_bulk(provider_type) + available_compliance_frameworks = get_available_compliance_frameworks(provider_type) # Generate and ensure the output directory exists output_directory = _generate_output_directory( DJANGO_TMP_OUTPUT_DIRECTORY, provider_uid, tenant_id, scan_id ) + # Generate compliance and ensure the output directory exists + compliance_output_directory = _generate_compliance_output_directory( + DJANGO_TMP_OUTPUT_DIRECTORY, provider_uid, tenant_id, scan_id + ) + # Define auxiliary variables output_writers = {} scan_summary = FindingOutput._transform_findings_stats( @@ -308,6 +323,24 @@ def generate_outputs(scan_id: str, provider_id: str, tenant_id: str): # TODO: Refactor the output classes to avoid this manual reset writer._data = [] + for compliance_name in available_compliance_frameworks: + filename = f"{compliance_output_directory}_{compliance_name}.csv" + compliance = bulk_compliance_frameworks[compliance_name] + writer_class = GenericCompliance # Default + + for condition, cls in COMPLIANCE_CLASS_MAP.get(provider_type, []): + if condition(compliance_name): + writer_class = cls + break + + writer = writer_class( + findings=finding_outputs, + compliance=compliance, + file_path=filename, + ) + writer.batch_write_data_to_file() + writer.close_file = is_last_batch + # Compress output files output_directory = _compress_output_files(output_directory) diff --git a/prowler/lib/outputs/compliance/compliance_output.py b/prowler/lib/outputs/compliance/compliance_output.py index bb0e49d452..d3b855521e 100644 --- a/prowler/lib/outputs/compliance/compliance_output.py +++ b/prowler/lib/outputs/compliance/compliance_output.py @@ -31,14 +31,21 @@ class ComplianceOutput(Output): compliance: Compliance, file_path: str = None, file_extension: str = "", + from_cli: bool = True, ) -> None: + # TODO: This class needs to be refactored to use the Output class init, methods and properties self._data = [] + self.close_file = False + self.file_path = file_path self.file_descriptor = None + # This parameter is to avoid refactoring more code, the CLI does not write in batches, the API does + self._from_cli = from_cli if not file_extension and file_path: self._file_extension = "".join(Path(file_path).suffixes) if file_extension: self._file_extension = file_extension + self.file_path = f"{file_path}{self.file_extension}" if findings: # Get the compliance name of the model @@ -49,7 +56,7 @@ class ComplianceOutput(Output): ) self.transform(findings, compliance, compliance_name) if not self._file_descriptor and file_path: - self.create_file_descriptor(file_path) + self.create_file_descriptor(self.file_path) def batch_write_data_to_file(self) -> None: """ @@ -69,12 +76,14 @@ class ComplianceOutput(Output): fieldnames=[field.upper() for field in self._data[0].dict().keys()], delimiter=";", ) - csv_writer.writeheader() + if self._file_descriptor.tell() == 0: + csv_writer.writeheader() for finding in self._data: csv_writer.writerow( {k.upper(): v for k, v in finding.dict().items()} ) - self._file_descriptor.close() + if self.close_file or self._from_cli: + self._file_descriptor.close() except Exception as error: logger.error( f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"