feat: add compliance to API report files

api/src/backend/tasks/jobs/export.py

@@ -13,6 +13,30 @@ from prowler.config.config import (
     json_ocsf_file_suffix,
     output_file_timestamp,
 )
+from prowler.lib.outputs.compliance.aws_well_architected.aws_well_architected import (
+    AWSWellArchitected,
+)
+from prowler.lib.outputs.compliance.cis.cis_aws import AWSCIS
+from prowler.lib.outputs.compliance.cis.cis_azure import AzureCIS
+from prowler.lib.outputs.compliance.cis.cis_gcp import GCPCIS
+from prowler.lib.outputs.compliance.cis.cis_kubernetes import KubernetesCIS
+from prowler.lib.outputs.compliance.cis.cis_m365 import M365CIS
+from prowler.lib.outputs.compliance.ens.ens_aws import AWSENS
+from prowler.lib.outputs.compliance.ens.ens_azure import AzureENS
+from prowler.lib.outputs.compliance.ens.ens_gcp import GCPENS
+from prowler.lib.outputs.compliance.iso27001.iso27001_aws import AWSISO27001
+from prowler.lib.outputs.compliance.iso27001.iso27001_azure import AzureISO27001
+from prowler.lib.outputs.compliance.iso27001.iso27001_gcp import GCPISO27001
+from prowler.lib.outputs.compliance.iso27001.iso27001_kubernetes import (
+    KubernetesISO27001,
+)
+from prowler.lib.outputs.compliance.iso27001.iso27001_nhn import NHNISO27001
+from prowler.lib.outputs.compliance.kisa_ismsp.kisa_ismsp_aws import AWSKISAISMSP
+from prowler.lib.outputs.compliance.mitre_attack.mitre_attack_aws import AWSMitreAttack
+from prowler.lib.outputs.compliance.mitre_attack.mitre_attack_azure import (
+    AzureMitreAttack,
+)
+from prowler.lib.outputs.compliance.mitre_attack.mitre_attack_gcp import GCPMitreAttack
 from prowler.lib.outputs.csv.csv import CSV
 from prowler.lib.outputs.html.html import HTML
 from prowler.lib.outputs.ocsf.ocsf import OCSF
@@ -20,6 +44,43 @@ from prowler.lib.outputs.ocsf.ocsf import OCSF
 logger = get_task_logger(__name__)
 
 
+COMPLIANCE_CLASS_MAP = {
+    "aws": [
+        (lambda name: name.startswith("cis_"), AWSCIS),
+        (lambda name: name == "mitre_attack_aws", AWSMitreAttack),
+        (lambda name: name.startswith("ens_"), AWSENS),
+        (
+            lambda name: name.startswith("aws_well_architected_framework"),
+            AWSWellArchitected,
+        ),
+        (lambda name: name.startswith("iso27001_"), AWSISO27001),
+        (lambda name: name.startswith("kisa"), AWSKISAISMSP),
+    ],
+    "azure": [
+        (lambda name: name.startswith("cis_"), AzureCIS),
+        (lambda name: name == "mitre_attack_azure", AzureMitreAttack),
+        (lambda name: name.startswith("ens_"), AzureENS),
+        (lambda name: name.startswith("iso27001_"), AzureISO27001),
+    ],
+    "gcp": [
+        (lambda name: name.startswith("cis_"), GCPCIS),
+        (lambda name: name == "mitre_attack_gcp", GCPMitreAttack),
+        (lambda name: name.startswith("ens_"), GCPENS),
+        (lambda name: name.startswith("iso27001_"), GCPISO27001),
+    ],
+    "kubernetes": [
+        (lambda name: name.startswith("cis_"), KubernetesCIS),
+        (lambda name: name.startswith("iso27001_"), KubernetesISO27001),
+    ],
+    "m365": [
+        (lambda name: name.startswith("cis_"), M365CIS),
+    ],
+    "nhn": [
+        (lambda name: name.startswith("iso27001_"), NHNISO27001),
+    ],
+}
+
+
 # Predefined mapping for output formats and their configurations
 OUTPUT_FORMATS_MAPPING = {
     "csv": {
@@ -43,13 +104,17 @@ def _compress_output_files(output_directory: str) -> str:
         str: The full path to the newly created ZIP archive.
     """
     zip_path = f"{output_directory}.zip"
+    parent_dir = os.path.dirname(output_directory)
+    zip_path_abs = os.path.abspath(zip_path)
 
     with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zipf:
-        for suffix in [config["suffix"] for config in OUTPUT_FORMATS_MAPPING.values()]:
-            zipf.write(
-                f"{output_directory}{suffix}",
-                f"output/{output_directory.split('/')[-1]}{suffix}",
-            )
+        for foldername, _, filenames in os.walk(parent_dir):
+            for filename in filenames:
+                file_path = os.path.join(foldername, filename)
+                if os.path.abspath(file_path) == zip_path_abs:
+                    continue
+                arcname = os.path.relpath(file_path, start=parent_dir)
+                zipf.write(file_path, arcname)
 
     return zip_path
 
@@ -154,3 +219,41 @@ def _generate_output_directory(
     os.makedirs("/".join(path.split("/")[:-1]), exist_ok=True)
 
     return path
+
+
+def _generate_compliance_output_directory(
+    output_directory, prowler_provider: object, tenant_id: str, scan_id: str
+) -> str:
+    """
+    Generate a file system path for the compliance output directory of a prowler scan.
+
+    This function constructs the compliance-specific output directory path by combining
+    a base temporary output directory, the tenant ID, the scan ID, and details about
+    the prowler provider along with a timestamp. The resulting path is used to store
+    compliance-related output files generated during a prowler scan.
+
+    Note:
+        This function depends on one external variable:
+          - `output_file_timestamp`: A timestamp (as a string) used to uniquely identify the output.
+
+    Args:
+        output_directory (str): The base output directory.
+        prowler_provider (object): An identifier or descriptor for the prowler provider.
+                                   Typically, this is a string indicating the provider (e.g., "aws").
+        tenant_id (str): The unique identifier for the tenant.
+        scan_id (str): The unique identifier for the scan.
+
+    Returns:
+        str: The constructed file system path for the prowler compliance output directory.
+
+    Example:
+        >>> _generate_compliance_output_directory("/tmp", "aws", "tenant-1234", "scan-5678")
+        '/tmp/tenant-1234/scan-5678/compliance/prowler-output-aws-2023-02-15T12:34:56'
+    """
+    path = (
+        f"{output_directory}/{tenant_id}/{scan_id}/compliance/prowler-output-"
+        f"{prowler_provider}-{output_file_timestamp}"
+    )
+    os.makedirs("/".join(path.split("/")[:-1]), exist_ok=True)
+
+    return path

api/src/backend/tasks/tasks.py

@@ -10,8 +10,10 @@ from django_celery_beat.models import PeriodicTask
 from tasks.jobs.connection import check_provider_connection
 from tasks.jobs.deletion import delete_provider, delete_tenant
 from tasks.jobs.export import (
+    COMPLIANCE_CLASS_MAP,
     OUTPUT_FORMATS_MAPPING,
     _compress_output_files,
+    _generate_compliance_output_directory,
     _generate_output_directory,
     _upload_to_s3,
 )
@@ -23,6 +25,9 @@ from api.decorators import set_tenant
 from api.models import Finding, Provider, Scan, ScanSummary, StateChoices
 from api.utils import initialize_prowler_provider
 from api.v1.serializers import ScanTaskSerializer
+from prowler.config.config import get_available_compliance_frameworks
+from prowler.lib.check.compliance_models import Compliance
+from prowler.lib.outputs.compliance.generic.generic import GenericCompliance
 from prowler.lib.outputs.finding import Finding as FindingOutput
 
 logger = get_task_logger(__name__)
@@ -256,12 +261,22 @@ def generate_outputs(scan_id: str, provider_id: str, tenant_id: str):
 
     # Get the provider UID
     provider_uid = Provider.objects.get(id=provider_id).uid
+    provider_type = Provider.objects.get(id=provider_id).provider
+
+    # Get the compliance frameworks for the provider
+    bulk_compliance_frameworks = Compliance.get_bulk(provider_type)
+    available_compliance_frameworks = get_available_compliance_frameworks(provider_type)
 
     # Generate and ensure the output directory exists
     output_directory = _generate_output_directory(
         DJANGO_TMP_OUTPUT_DIRECTORY, provider_uid, tenant_id, scan_id
     )
 
+    # Generate compliance and ensure the output directory exists
+    compliance_output_directory = _generate_compliance_output_directory(
+        DJANGO_TMP_OUTPUT_DIRECTORY, provider_uid, tenant_id, scan_id
+    )
+
     # Define auxiliary variables
     output_writers = {}
     scan_summary = FindingOutput._transform_findings_stats(
@@ -308,6 +323,24 @@ def generate_outputs(scan_id: str, provider_id: str, tenant_id: str):
             # TODO: Refactor the output classes to avoid this manual reset
             writer._data = []
 
+        for compliance_name in available_compliance_frameworks:
+            filename = f"{compliance_output_directory}_{compliance_name}.csv"
+            compliance = bulk_compliance_frameworks[compliance_name]
+            writer_class = GenericCompliance  # Default
+
+            for condition, cls in COMPLIANCE_CLASS_MAP.get(provider_type, []):
+                if condition(compliance_name):
+                    writer_class = cls
+                    break
+
+            writer = writer_class(
+                findings=finding_outputs,
+                compliance=compliance,
+                file_path=filename,
+            )
+            writer.batch_write_data_to_file()
+            writer.close_file = is_last_batch
+
     # Compress output files
     output_directory = _compress_output_files(output_directory)
 

prowler/lib/outputs/compliance/compliance_output.py

@@ -31,14 +31,21 @@ class ComplianceOutput(Output):
         compliance: Compliance,
         file_path: str = None,
         file_extension: str = "",
+        from_cli: bool = True,
     ) -> None:
+        # TODO: This class needs to be refactored to use the Output class init, methods and properties
         self._data = []
+        self.close_file = False
+        self.file_path = file_path
         self.file_descriptor = None
+        # This parameter is to avoid refactoring more code, the CLI does not write in batches, the API does
+        self._from_cli = from_cli
 
         if not file_extension and file_path:
             self._file_extension = "".join(Path(file_path).suffixes)
         if file_extension:
             self._file_extension = file_extension
+            self.file_path = f"{file_path}{self.file_extension}"
 
         if findings:
             # Get the compliance name of the model
@@ -49,7 +56,7 @@ class ComplianceOutput(Output):
             )
             self.transform(findings, compliance, compliance_name)
             if not self._file_descriptor and file_path:
-                self.create_file_descriptor(file_path)
+                self.create_file_descriptor(self.file_path)
 
     def batch_write_data_to_file(self) -> None:
         """
@@ -69,12 +76,14 @@ class ComplianceOutput(Output):
                     fieldnames=[field.upper() for field in self._data[0].dict().keys()],
                     delimiter=";",
                 )
-                csv_writer.writeheader()
+                if self._file_descriptor.tell() == 0:
+                    csv_writer.writeheader()
                 for finding in self._data:
                     csv_writer.writerow(
                         {k.upper(): v for k, v in finding.dict().items()}
                     )
-                self._file_descriptor.close()
+                if self.close_file or self._from_cli:
+                    self._file_descriptor.close()
         except Exception as error:
             logger.error(
                 f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"