fix(outputs): parse resource metadata once per batch (#13149)

api/changelog.d/report-metadata-memory.fixed.md

@@ -0,0 +1 @@
+Report generation shares the parsed resource metadata between the findings of a batch and releases each batch before building the next one

api/src/backend/tasks/tasks.py

@@ -897,8 +897,12 @@ def generate_outputs_task(scan_id: str, provider_id: str, tenant_id: str):
     )
     with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
         for batch, is_last in batched(qs, DJANGO_FINDINGS_BATCH_SIZE):
+            metadata_cache = {}
             fos = [
-                FindingOutput.transform_api_finding(f, prowler_provider) for f in batch
+                FindingOutput.transform_api_finding(
+                    f, prowler_provider, metadata_cache=metadata_cache
+                )
+                for f in batch
             ]
 
             # Outputs
@@ -979,6 +983,8 @@ def generate_outputs_task(scan_id: str, provider_id: str, tenant_id: str):
                 writer.batch_write_data_to_file()
                 writer._data.clear()
 
+            del fos, metadata_cache
+
     compressed = _compress_output_files(out_dir)
 
     upload_uri = _upload_to_s3(

api/src/backend/tasks/tests/test_tasks.py

@@ -742,7 +742,7 @@ class TestGenerateOutputs:
             patch("tasks.tasks.FindingOutput._transform_findings_stats"),
             patch(
                 "tasks.tasks.FindingOutput.transform_api_finding",
-                side_effect=lambda f, _prov: f,
+                side_effect=lambda f, _prov, **_kw: f,
             ),
             patch("tasks.tasks._compress_output_files", return_value="outdir.zip"),
             patch("tasks.tasks._upload_to_s3", return_value="s3://bucket/outdir.zip"),