Merge branch 'master' of github.com:prowler-cloud/prowler into showdown-demo

api/CHANGELOG.md

@@ -19,8 +19,12 @@ All notable changes to the **Prowler API** are documented in this file.
 
 ## [v1.8.3] (Prowler v5.7.3)
 
+### Added
+- Database backend to handle already closed connections [(#7935)](https://github.com/prowler-cloud/prowler/pull/7935).
+
 ### Fixed
 - Fixed transaction persistence with RLS operations [(#7916)](https://github.com/prowler-cloud/prowler/pull/7916).
+- Reverted the change `get_with_retry` to use the original `get` method for retrieving tasks [(#7932)](https://github.com/prowler-cloud/prowler/pull/7932).
 
 ---
 

api/src/backend/api/compliance.py

@@ -190,6 +190,8 @@ def generate_compliance_overview_template(prowler_compliance: dict):
                 total_checks = len(requirement.Checks)
                 checks_dict = {check: None for check in requirement.Checks}
 
+                req_status_val = "MANUAL" if total_checks == 0 else "PASS"
+
                 # Build requirement dictionary
                 requirement_dict = {
                     "name": requirement.Name or requirement.Id,
@@ -204,20 +206,18 @@ def generate_compliance_overview_template(prowler_compliance: dict):
                         "manual": 0,
                         "total": total_checks,
                     },
-                    "status": "PASS",
+                    "status": req_status_val,
                 }
 
-                # Update requirements status
-                if total_checks == 0:
+                # Update requirements status counts for the framework
+                if req_status_val == "MANUAL":
                     requirements_status["manual"] += 1
+                elif req_status_val == "PASS":
+                    requirements_status["passed"] += 1
 
                 # Add requirement to compliance requirements
                 compliance_requirements[requirement.Id] = requirement_dict
 
-            # Calculate pending requirements
-            pending_requirements = total_requirements - requirements_status["manual"]
-            requirements_status["passed"] = pending_requirements
-
             # Build compliance dictionary
             compliance_dict = {
                 "framework": compliance_data.Framework,

api/src/backend/api/models.py

@@ -1,11 +1,9 @@
 import json
 import logging
 import re
-import time
 from uuid import UUID, uuid4
 
 from config.custom_logging import BackendLogger
-from config.env import env
 from cryptography.fernet import Fernet, InvalidToken
 from django.conf import settings
 from django.contrib.auth.models import AbstractBaseUser
@@ -358,42 +356,6 @@ class ProviderGroupMembership(RowLevelSecurityProtectedModel):
         resource_name = "provider_groups-provider"
 
 
-class TaskManager(models.Manager):
-    def get_with_retry(
-        self,
-        id: str,
-        max_retries: int = None,
-        delay_seconds: float = None,
-    ):
-        """
-        Retry fetching a Task by ID in case it hasn't been created yet.
-
-        Args:
-            id (str): The Celery task ID (expected to match Task model PK).
-            max_retries (int, optional): Number of retry attempts. Defaults to env TASK_RETRY_ATTEMPTS or 5.
-            delay_seconds (float, optional): Delay between retries in seconds. Defaults to env TASK_RETRY_DELAY_SECONDS or 0.1.
-
-        Returns:
-            Task: The retrieved Task instance.
-
-        Raises:
-            Task.DoesNotExist: If the task is not found after all retries.
-        """
-        max_retries = max_retries or env.int("TASK_RETRY_ATTEMPTS", default=5)
-        delay_seconds = delay_seconds or env.float(
-            "TASK_RETRY_DELAY_SECONDS", default=0.1
-        )
-
-        for _attempt in range(max_retries):
-            try:
-                return self.get(id=id)
-            except self.model.DoesNotExist:
-                time.sleep(delay_seconds)
-        raise self.model.DoesNotExist(
-            f"Task with ID {id} not found after {max_retries} retries."
-        )
-
-
 class Task(RowLevelSecurityProtectedModel):
     id = models.UUIDField(primary_key=True, default=uuid4, editable=False)
     inserted_at = models.DateTimeField(auto_now_add=True, editable=False)
@@ -406,8 +368,6 @@ class Task(RowLevelSecurityProtectedModel):
         blank=True,
     )
 
-    objects = TaskManager()
-
     class Meta(RowLevelSecurityProtectedModel.Meta):
         db_table = "tasks"
 

api/src/backend/api/tests/test_compliance.py

@@ -1,12 +1,12 @@
-from unittest.mock import patch, MagicMock
+from unittest.mock import MagicMock, patch
 
 from api.compliance import (
+    generate_compliance_overview_template,
+    generate_scan_compliance,
     get_prowler_provider_checks,
     get_prowler_provider_compliance,
-    load_prowler_compliance,
     load_prowler_checks,
-    generate_scan_compliance,
-    generate_compliance_overview_template,
+    load_prowler_compliance,
 )
 from api.models import Provider
 
@@ -69,7 +69,7 @@ class TestCompliance:
 
         load_prowler_compliance()
 
-        from api.compliance import PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE, PROWLER_CHECKS
+        from api.compliance import PROWLER_CHECKS, PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE
 
         assert PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE == {
             "template_key": "template_value"
@@ -268,7 +268,7 @@ class TestCompliance:
                                 "manual": 0,
                                 "total": 0,
                             },
-                            "status": "PASS",
+                            "status": "MANUAL",
                         },
                     },
                     "requirements_status": {

api/src/backend/api/tests/test_models.py

@@ -1,9 +1,6 @@
-import uuid
-from unittest import mock
-
 import pytest
 
-from api.models import Resource, ResourceTag, Task
+from api.models import Resource, ResourceTag
 
 
 @pytest.mark.django_db
@@ -123,35 +120,3 @@ class TestResourceModel:
 #             compliance={},
 #         )
 #         assert Finding.objects.filter(uid=long_uid).exists()
-
-
-@pytest.mark.django_db
-class TestTaskManager:
-    def test_get_with_retry_success(self):
-        task_id = uuid.uuid4()
-        call_counter = {"count": 0}
-
-        def side_effect(*args, **kwargs):
-            if call_counter["count"] < 2:
-                call_counter["count"] += 1
-                raise Task.DoesNotExist()
-            return Task(id=task_id)
-
-        with mock.patch.object(Task.objects, "get", side_effect=side_effect):
-            task = Task.objects.get_with_retry(
-                task_id, max_retries=5, delay_seconds=0.01
-            )
-
-        assert task.id == task_id
-        assert call_counter["count"] == 2
-
-    def test_get_with_retry_fail(self):
-        non_existent_id = uuid.uuid4()
-
-        with mock.patch.object(Task.objects, "get", side_effect=Task.DoesNotExist):
-            with pytest.raises(Task.DoesNotExist) as excinfo:
-                Task.objects.get_with_retry(
-                    non_existent_id, max_retries=3, delay_seconds=0.01
-                )
-
-        assert str(non_existent_id) in str(excinfo.value)

api/src/backend/api/tests/test_views.py

@@ -4837,6 +4837,24 @@ class TestComplianceOverviewViewSet:
             assert "description" in attributes
             assert "status" in attributes
 
+    def test_compliance_overview_requirements_manual(
+        self, authenticated_client, compliance_requirements_overviews_fixture
+    ):
+        scan_id = str(compliance_requirements_overviews_fixture[0].scan.id)
+        # Compliance with a manual requirement
+        compliance_id = "aws_account_security_onboarding_aws"
+
+        response = authenticated_client.get(
+            reverse("complianceoverview-requirements"),
+            {
+                "filter[scan_id]": scan_id,
+                "filter[compliance_id]": compliance_id,
+            },
+        )
+        assert response.status_code == status.HTTP_200_OK
+        data = response.json()["data"]
+        assert data[-1]["attributes"]["status"] == "MANUAL"
+
     def test_compliance_overview_requirements_missing_scan_id(
         self, authenticated_client
     ):

api/src/backend/api/v1/views.py

@@ -1097,7 +1097,7 @@ class ProviderViewSet(BaseRLSViewSet):
             task = check_provider_connection_task.delay(
                 provider_id=pk, tenant_id=self.request.tenant_id
             )
-        prowler_task = Task.objects.get_with_retry(id=task.id)
+        prowler_task = Task.objects.get(id=task.id)
         serializer = TaskSerializer(prowler_task)
         return Response(
             data=serializer.data,
@@ -1120,7 +1120,7 @@ class ProviderViewSet(BaseRLSViewSet):
             task = delete_provider_task.delay(
                 provider_id=pk, tenant_id=self.request.tenant_id
             )
-        prowler_task = Task.objects.get_with_retry(id=task.id)
+        prowler_task = Task.objects.get(id=task.id)
         serializer = TaskSerializer(prowler_task)
         return Response(
             data=serializer.data,
@@ -1500,7 +1500,7 @@ class ScanViewSet(BaseRLSViewSet):
                 },
             )
 
-        prowler_task = Task.objects.get_with_retry(id=task.id)
+        prowler_task = Task.objects.get(id=task.id)
         scan.task_id = task.id
         scan.save(update_fields=["task_id"])
 
@@ -2738,7 +2738,10 @@ class ComplianceOverviewViewSet(BaseRLSViewSet, TaskManagementMixin):
                 "requirement_id", "framework", "version", "description"
             )
             .distinct()
-            .annotate(total_instances=Count("id"))
+            .annotate(
+                total_instances=Count("id"),
+                manual_count=Count("id", filter=Q(requirement_status="MANUAL")),
+            )
         )
 
         passed_instances = (
@@ -2756,8 +2759,13 @@ class ComplianceOverviewViewSet(BaseRLSViewSet, TaskManagementMixin):
             requirement_id = requirement["requirement_id"]
             total_instances = requirement["total_instances"]
             passed_count = passed_counts.get(requirement_id, 0)
-
-            requirement_status = "PASS" if passed_count == total_instances else "FAIL"
+            is_manual = requirement["manual_count"] == total_instances
+            if is_manual:
+                requirement_status = "MANUAL"
+            elif passed_count == total_instances:
+                requirement_status = "PASS"
+            else:
+                requirement_status = "FAIL"
 
             requirements_summary.append(
                 {
@@ -3128,7 +3136,7 @@ class ScheduleViewSet(BaseRLSViewSet):
         with transaction.atomic():
             task = schedule_provider_scan(provider_instance)
 
-        prowler_task = Task.objects.get_with_retry(id=task.id)
+        prowler_task = Task.objects.get(id=task.id)
         self.response_serializer_class = TaskSerializer
         output_serializer = self.get_serializer(prowler_task)
 

api/src/backend/config/django/base.py

@@ -128,6 +128,7 @@ DJANGO_GUID = {
 }
 
 DATABASE_ROUTERS = ["api.db_router.MainRouter"]
+POSTGRES_EXTRA_DB_BACKEND_BASE = "database_backend"
 
 
 # Password validation

api/src/backend/conftest.py

@@ -847,8 +847,23 @@ def compliance_requirements_overviews_fixture(scans_fixture, tenants_fixture):
         total_checks=2,
     )
 
-    # Create a different compliance framework for testing
     requirement_overview5 = ComplianceRequirementOverview.objects.create(
+        tenant=tenant,
+        scan=scan1,
+        compliance_id="aws_account_security_onboarding_aws",
+        framework="AWS-Account-Security-Onboarding",
+        version="1.0",
+        description="Description for AWS Account Security Onboarding (MANUAL)",
+        region="eu-west-2",
+        requirement_id="requirement3",
+        requirement_status=StatusChoices.MANUAL,
+        passed_checks=0,
+        failed_checks=0,
+        total_checks=0,
+    )
+
+    # Create a different compliance framework for testing
+    requirement_overview6 = ComplianceRequirementOverview.objects.create(
         tenant=tenant,
         scan=scan1,
         compliance_id="cis_1.4_aws",
@@ -869,6 +884,7 @@ def compliance_requirements_overviews_fixture(scans_fixture, tenants_fixture):
         requirement_overview3,
         requirement_overview4,
         requirement_overview5,
+        requirement_overview6,
     )
 
 

api/src/backend/database_backend/base.py

@@ -0,0 +1,15 @@
+import django.db
+from django.db.backends.postgresql.base import (
+    DatabaseWrapper as BuiltinPostgresDatabaseWrapper,
+)
+from psycopg2 import InterfaceError
+
+
+class DatabaseWrapper(BuiltinPostgresDatabaseWrapper):
+    def create_cursor(self, name=None):
+        try:
+            return super().create_cursor(name=name)
+        except InterfaceError:
+            django.db.close_old_connections()
+            django.db.connection.connect()
+            return super().create_cursor(name=name)