refactor(api): lazy load providers and compliance (#9857)

api/src/backend/api/apps.py

@@ -30,7 +30,6 @@ class ApiConfig(AppConfig):
     def ready(self):
         from api import schema_extensions  # noqa: F401
         from api import signals  # noqa: F401
-        from api.compliance import load_prowler_compliance
 
         # Generate required cryptographic keys if not present, but only if:
         #   `"manage.py" not in sys.argv[0]`: If an external server (e.g., Gunicorn) is running the app
@@ -44,8 +43,6 @@ class ApiConfig(AppConfig):
         # Neo4j driver is initialized lazily on first use (see api.attack_paths.database)
         # This avoids connection attempts during regular scans that don't need graph database
 
-        load_prowler_compliance()
-
     def _ensure_crypto_keys(self):
         """
         Orchestrator method that ensures all required cryptographic keys are present.

api/src/backend/api/compliance.py

@@ -1,15 +1,99 @@
-from types import MappingProxyType
+from collections.abc import Iterable, Mapping
 
 from api.models import Provider
 from prowler.config.config import get_available_compliance_frameworks
 from prowler.lib.check.compliance_models import Compliance
 from prowler.lib.check.models import CheckMetadata
 
-PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE = {}
-PROWLER_CHECKS = {}
 AVAILABLE_COMPLIANCE_FRAMEWORKS = {}
 
 
+class LazyComplianceTemplate(Mapping):
+    """Lazy-load compliance templates per provider on first access."""
+
+    def __init__(self, provider_types: Iterable[str] | None = None) -> None:
+        if provider_types is None:
+            provider_types = Provider.ProviderChoices.values
+        self._provider_types = tuple(provider_types)
+        self._provider_types_set = set(self._provider_types)
+        self._cache: dict[str, dict] = {}
+
+    def _load_provider(self, provider_type: str) -> dict:
+        if provider_type not in self._provider_types_set:
+            raise KeyError(provider_type)
+        cached = self._cache.get(provider_type)
+        if cached is not None:
+            return cached
+        _ensure_provider_loaded(provider_type)
+        return self._cache[provider_type]
+
+    def __getitem__(self, key: str) -> dict:
+        return self._load_provider(key)
+
+    def __iter__(self):
+        return iter(self._provider_types)
+
+    def __len__(self) -> int:
+        return len(self._provider_types)
+
+    def __contains__(self, key: object) -> bool:
+        return key in self._provider_types_set
+
+    def get(self, key: str, default=None):
+        if key not in self._provider_types_set:
+            return default
+        return self._load_provider(key)
+
+    def __repr__(self) -> str:  # pragma: no cover - debugging helper
+        loaded = ", ".join(sorted(self._cache))
+        return f"{self.__class__.__name__}(loaded=[{loaded}])"
+
+
+class LazyChecksMapping(Mapping):
+    """Lazy-load checks mapping per provider on first access."""
+
+    def __init__(self, provider_types: Iterable[str] | None = None) -> None:
+        if provider_types is None:
+            provider_types = Provider.ProviderChoices.values
+        self._provider_types = tuple(provider_types)
+        self._provider_types_set = set(self._provider_types)
+        self._cache: dict[str, dict] = {}
+
+    def _load_provider(self, provider_type: str) -> dict:
+        if provider_type not in self._provider_types_set:
+            raise KeyError(provider_type)
+        cached = self._cache.get(provider_type)
+        if cached is not None:
+            return cached
+        _ensure_provider_loaded(provider_type)
+        return self._cache[provider_type]
+
+    def __getitem__(self, key: str) -> dict:
+        return self._load_provider(key)
+
+    def __iter__(self):
+        return iter(self._provider_types)
+
+    def __len__(self) -> int:
+        return len(self._provider_types)
+
+    def __contains__(self, key: object) -> bool:
+        return key in self._provider_types_set
+
+    def get(self, key: str, default=None):
+        if key not in self._provider_types_set:
+            return default
+        return self._load_provider(key)
+
+    def __repr__(self) -> str:  # pragma: no cover - debugging helper
+        loaded = ", ".join(sorted(self._cache))
+        return f"{self.__class__.__name__}(loaded=[{loaded}])"
+
+
+PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE = LazyComplianceTemplate()
+PROWLER_CHECKS = LazyChecksMapping()
+
+
 def get_compliance_frameworks(provider_type: Provider.ProviderChoices) -> list[str]:
     """
     Retrieve and cache the list of available compliance frameworks for a specific cloud provider.
@@ -70,28 +154,35 @@ def get_prowler_provider_compliance(provider_type: Provider.ProviderChoices) ->
     return Compliance.get_bulk(provider_type)
 
 
-def load_prowler_compliance():
-    """
-    Load and initialize the Prowler compliance data and checks for all provider types.
-
-    This function retrieves compliance data for all supported provider types,
-    generates a compliance overview template, and populates the global variables
-    `PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE` and `PROWLER_CHECKS` with read-only mappings
-    of the compliance templates and checks, respectively.
-    """
-    global PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE
-    global PROWLER_CHECKS
-
-    prowler_compliance = {
-        provider_type: get_prowler_provider_compliance(provider_type)
-        for provider_type in Provider.ProviderChoices.values
-    }
-    template = generate_compliance_overview_template(prowler_compliance)
-    PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE = MappingProxyType(template)
-    PROWLER_CHECKS = MappingProxyType(load_prowler_checks(prowler_compliance))
+def _load_provider_assets(provider_type: Provider.ProviderChoices) -> tuple[dict, dict]:
+    prowler_compliance = {provider_type: get_prowler_provider_compliance(provider_type)}
+    template = generate_compliance_overview_template(
+        prowler_compliance, provider_types=[provider_type]
+    )
+    checks = load_prowler_checks(prowler_compliance, provider_types=[provider_type])
+    return template.get(provider_type, {}), checks.get(provider_type, {})
 
 
-def load_prowler_checks(prowler_compliance):
+def _ensure_provider_loaded(provider_type: Provider.ProviderChoices) -> None:
+    if (
+        provider_type in PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE._cache
+        and provider_type in PROWLER_CHECKS._cache
+    ):
+        return
+    template_cached = PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE._cache.get(provider_type)
+    checks_cached = PROWLER_CHECKS._cache.get(provider_type)
+    if template_cached is not None and checks_cached is not None:
+        return
+    template, checks = _load_provider_assets(provider_type)
+    if template_cached is None:
+        PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE._cache[provider_type] = template
+    if checks_cached is None:
+        PROWLER_CHECKS._cache[provider_type] = checks
+
+
+def load_prowler_checks(
+    prowler_compliance, provider_types: Iterable[str] | None = None
+):
     """
     Generate a mapping of checks to the compliance frameworks that include them.
 
@@ -100,21 +191,25 @@ def load_prowler_checks(prowler_compliance):
     of compliance names that include that check.
 
     Args:
-        prowler_compliance (dict): The compliance data for all provider types,
+        prowler_compliance (dict): The compliance data for provider types,
             as returned by `get_prowler_provider_compliance`.
+        provider_types (Iterable[str] | None): Optional subset of provider types to
+            process. Defaults to all providers.
 
     Returns:
         dict: A nested dictionary where the first-level keys are provider types,
             and the values are dictionaries mapping check IDs to sets of compliance names.
     """
     checks = {}
-    for provider_type in Provider.ProviderChoices.values:
+    if provider_types is None:
+        provider_types = Provider.ProviderChoices.values
+    for provider_type in provider_types:
         checks[provider_type] = {
             check_id: set() for check_id in get_prowler_provider_checks(provider_type)
         }
-        for compliance_name, compliance_data in prowler_compliance[
-            provider_type
-        ].items():
+        for compliance_name, compliance_data in prowler_compliance.get(
+            provider_type, {}
+        ).items():
             for requirement in compliance_data.Requirements:
                 for check in requirement.Checks:
                     try:
@@ -163,7 +258,9 @@ def generate_scan_compliance(
                     ] += 1
 
 
-def generate_compliance_overview_template(prowler_compliance: dict):
+def generate_compliance_overview_template(
+    prowler_compliance: dict, provider_types: Iterable[str] | None = None
+):
     """
     Generate a compliance overview template for all provider types.
 
@@ -173,17 +270,21 @@ def generate_compliance_overview_template(prowler_compliance: dict):
     counts for requirements status.
 
     Args:
-        prowler_compliance (dict): The compliance data for all provider types,
+        prowler_compliance (dict): The compliance data for provider types,
             as returned by `get_prowler_provider_compliance`.
+        provider_types (Iterable[str] | None): Optional subset of provider types to
+            process. Defaults to all providers.
 
     Returns:
         dict: A nested dictionary representing the compliance overview template,
             structured by provider type and compliance framework.
     """
     template = {}
-    for provider_type in Provider.ProviderChoices.values:
+    if provider_types is None:
+        provider_types = Provider.ProviderChoices.values
+    for provider_type in provider_types:
         provider_compliance = template.setdefault(provider_type, {})
-        compliance_data_dict = prowler_compliance[provider_type]
+        compliance_data_dict = prowler_compliance.get(provider_type, {})
 
         for compliance_name, compliance_data in compliance_data_dict.items():
             compliance_requirements = {}

api/src/backend/api/tests/test_compliance.py

@@ -6,7 +6,6 @@ from api.compliance import (
     get_prowler_provider_checks,
     get_prowler_provider_compliance,
     load_prowler_checks,
-    load_prowler_compliance,
 )
 from api.models import Provider
 
@@ -35,55 +34,6 @@ class TestCompliance:
         assert compliance_data == mock_compliance.get_bulk.return_value
         mock_compliance.get_bulk.assert_called_once_with(provider_type)
 
-    @patch("api.models.Provider.ProviderChoices")
-    @patch("api.compliance.get_prowler_provider_compliance")
-    @patch("api.compliance.generate_compliance_overview_template")
-    @patch("api.compliance.load_prowler_checks")
-    def test_load_prowler_compliance(
-        self,
-        mock_load_prowler_checks,
-        mock_generate_compliance_overview_template,
-        mock_get_prowler_provider_compliance,
-        mock_provider_choices,
-    ):
-        mock_provider_choices.values = ["aws", "azure"]
-
-        compliance_data_aws = {"compliance_aws": MagicMock()}
-        compliance_data_azure = {"compliance_azure": MagicMock()}
-
-        compliance_data_dict = {
-            "aws": compliance_data_aws,
-            "azure": compliance_data_azure,
-        }
-
-        def mock_get_compliance(provider_type):
-            return compliance_data_dict[provider_type]
-
-        mock_get_prowler_provider_compliance.side_effect = mock_get_compliance
-
-        mock_generate_compliance_overview_template.return_value = {
-            "template_key": "template_value"
-        }
-
-        mock_load_prowler_checks.return_value = {"checks_key": "checks_value"}
-
-        load_prowler_compliance()
-
-        from api.compliance import PROWLER_CHECKS, PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE
-
-        assert PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE == {
-            "template_key": "template_value"
-        }
-        assert PROWLER_CHECKS == {"checks_key": "checks_value"}
-
-        expected_prowler_compliance = compliance_data_dict
-        mock_get_prowler_provider_compliance.assert_any_call("aws")
-        mock_get_prowler_provider_compliance.assert_any_call("azure")
-        mock_generate_compliance_overview_template.assert_called_once_with(
-            expected_prowler_compliance
-        )
-        mock_load_prowler_checks.assert_called_once_with(expected_prowler_compliance)
-
     @patch("api.compliance.get_prowler_provider_checks")
     @patch("api.models.Provider.ProviderChoices")
     def test_load_prowler_checks(

api/src/backend/api/utils.py

@@ -1,4 +1,7 @@
+from __future__ import annotations
+
 from datetime import datetime, timezone
+from typing import TYPE_CHECKING
 
 from allauth.socialaccount.providers.oauth2.client import OAuth2Client
 from django.contrib.postgres.aggregates import ArrayAgg
@@ -11,19 +14,25 @@ from api.exceptions import InvitationTokenExpiredException
 from api.models import Integration, Invitation, Processor, Provider, Resource
 from api.v1.serializers import FindingMetadataSerializer
 from prowler.lib.outputs.jira.jira import Jira, JiraBasicAuthError
-from prowler.providers.alibabacloud.alibabacloud_provider import AlibabacloudProvider
-from prowler.providers.aws.aws_provider import AwsProvider
 from prowler.providers.aws.lib.s3.s3 import S3
 from prowler.providers.aws.lib.security_hub.security_hub import SecurityHub
-from prowler.providers.azure.azure_provider import AzureProvider
 from prowler.providers.common.models import Connection
-from prowler.providers.gcp.gcp_provider import GcpProvider
-from prowler.providers.github.github_provider import GithubProvider
-from prowler.providers.iac.iac_provider import IacProvider
-from prowler.providers.kubernetes.kubernetes_provider import KubernetesProvider
-from prowler.providers.m365.m365_provider import M365Provider
-from prowler.providers.mongodbatlas.mongodbatlas_provider import MongodbatlasProvider
-from prowler.providers.oraclecloud.oraclecloud_provider import OraclecloudProvider
+
+if TYPE_CHECKING:
+    from prowler.providers.alibabacloud.alibabacloud_provider import (
+        AlibabacloudProvider,
+    )
+    from prowler.providers.aws.aws_provider import AwsProvider
+    from prowler.providers.azure.azure_provider import AzureProvider
+    from prowler.providers.gcp.gcp_provider import GcpProvider
+    from prowler.providers.github.github_provider import GithubProvider
+    from prowler.providers.iac.iac_provider import IacProvider
+    from prowler.providers.kubernetes.kubernetes_provider import KubernetesProvider
+    from prowler.providers.m365.m365_provider import M365Provider
+    from prowler.providers.mongodbatlas.mongodbatlas_provider import (
+        MongodbatlasProvider,
+    )
+    from prowler.providers.oraclecloud.oraclecloud_provider import OraclecloudProvider
 
 
 class CustomOAuth2Client(OAuth2Client):
@@ -89,24 +98,52 @@ def return_prowler_provider(
     """
     match provider.provider:
         case Provider.ProviderChoices.AWS.value:
+            from prowler.providers.aws.aws_provider import AwsProvider
+
             prowler_provider = AwsProvider
         case Provider.ProviderChoices.GCP.value:
+            from prowler.providers.gcp.gcp_provider import GcpProvider
+
             prowler_provider = GcpProvider
         case Provider.ProviderChoices.AZURE.value:
+            from prowler.providers.azure.azure_provider import AzureProvider
+
             prowler_provider = AzureProvider
         case Provider.ProviderChoices.KUBERNETES.value:
+            from prowler.providers.kubernetes.kubernetes_provider import (
+                KubernetesProvider,
+            )
+
             prowler_provider = KubernetesProvider
         case Provider.ProviderChoices.M365.value:
+            from prowler.providers.m365.m365_provider import M365Provider
+
             prowler_provider = M365Provider
         case Provider.ProviderChoices.GITHUB.value:
+            from prowler.providers.github.github_provider import GithubProvider
+
             prowler_provider = GithubProvider
         case Provider.ProviderChoices.MONGODBATLAS.value:
+            from prowler.providers.mongodbatlas.mongodbatlas_provider import (
+                MongodbatlasProvider,
+            )
+
             prowler_provider = MongodbatlasProvider
         case Provider.ProviderChoices.IAC.value:
+            from prowler.providers.iac.iac_provider import IacProvider
+
             prowler_provider = IacProvider
         case Provider.ProviderChoices.ORACLECLOUD.value:
+            from prowler.providers.oraclecloud.oraclecloud_provider import (
+                OraclecloudProvider,
+            )
+
             prowler_provider = OraclecloudProvider
         case Provider.ProviderChoices.ALIBABACLOUD.value:
+            from prowler.providers.alibabacloud.alibabacloud_provider import (
+                AlibabacloudProvider,
+            )
+
             prowler_provider = AlibabacloudProvider
         case _:
             raise ValueError(f"Provider type {provider.provider} not supported")

api/src/backend/api/v1/views.py

@@ -3,7 +3,6 @@ import glob
 import json
 import logging
 import os
-
 from collections import defaultdict
 from copy import deepcopy
 from datetime import datetime, timedelta, timezone
@@ -11,7 +10,6 @@ from decimal import ROUND_HALF_UP, Decimal, InvalidOperation
 from urllib.parse import urljoin
 
 import sentry_sdk
-
 from allauth.socialaccount.models import SocialAccount, SocialApp
 from allauth.socialaccount.providers.github.views import GitHubOAuth2Adapter
 from allauth.socialaccount.providers.google.views import GoogleOAuth2Adapter
@@ -75,12 +73,26 @@ from rest_framework.generics import GenericAPIView, get_object_or_404
 from rest_framework.permissions import SAFE_METHODS
 from rest_framework_json_api.views import RelationshipView, Response
 from rest_framework_simplejwt.exceptions import InvalidToken, TokenError
-
-from api.attack_paths import (
-    get_queries_for_provider,
-    get_query_by_id,
-    views_helpers as attack_paths_views_helpers,
+from tasks.beat import schedule_provider_scan
+from tasks.jobs.attack_paths import db_utils as attack_paths_db_utils
+from tasks.jobs.export import get_s3_client
+from tasks.tasks import (
+    backfill_compliance_summaries_task,
+    backfill_scan_resource_summaries_task,
+    check_integration_connection_task,
+    check_lighthouse_connection_task,
+    check_lighthouse_provider_connection_task,
+    check_provider_connection_task,
+    delete_provider_task,
+    delete_tenant_task,
+    jira_integration_task,
+    mute_historical_findings_task,
+    perform_scan_task,
+    refresh_lighthouse_provider_models_task,
 )
+
+from api.attack_paths import get_queries_for_provider, get_query_by_id
+from api.attack_paths import views_helpers as attack_paths_views_helpers
 from api.base_views import BaseRLSViewSet, BaseTenantViewset, BaseUserViewset
 from api.compliance import (
     PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE,
@@ -90,6 +102,7 @@ from api.db_router import MainRouter
 from api.db_utils import rls_transaction
 from api.exceptions import TaskFailedException
 from api.filters import (
+    AttackPathsScanFilter,
     AttackSurfaceOverviewFilter,
     CategoryOverviewFilter,
     ComplianceOverviewFilter,
@@ -102,7 +115,6 @@ from api.filters import (
     InvitationFilter,
     LatestFindingFilter,
     LatestResourceFilter,
-    AttackPathsScanFilter,
     LighthouseProviderConfigFilter,
     LighthouseProviderModelsFilter,
     MembershipFilter,
@@ -124,6 +136,7 @@ from api.filters import (
     UserFilter,
 )
 from api.models import (
+    AttackPathsScan,
     AttackSurfaceOverview,
     ComplianceOverviewSummary,
     ComplianceRequirementOverview,
@@ -131,7 +144,6 @@ from api.models import (
     Finding,
     Integration,
     Invitation,
-    AttackPathsScan,
     LighthouseConfiguration,
     LighthouseProviderConfiguration,
     LighthouseProviderModels,
@@ -177,9 +189,9 @@ from api.utils import (
 from api.uuid_utils import datetime_to_uuid7, uuid7_start
 from api.v1.mixins import DisablePaginationMixin, PaginateByPkMixin, TaskManagementMixin
 from api.v1.serializers import (
+    AttackPathsQueryResultSerializer,
     AttackPathsQueryRunRequestSerializer,
     AttackPathsQuerySerializer,
-    AttackPathsQueryResultSerializer,
     AttackPathsScanSerializer,
     AttackSurfaceOverviewSerializer,
     CategoryOverviewSerializer,
@@ -263,23 +275,6 @@ from api.v1.serializers import (
     UserSerializer,
     UserUpdateSerializer,
 )
-from tasks.beat import schedule_provider_scan
-from tasks.jobs.attack_paths import db_utils as attack_paths_db_utils
-from tasks.jobs.export import get_s3_client
-from tasks.tasks import (
-    backfill_compliance_summaries_task,
-    backfill_scan_resource_summaries_task,
-    check_integration_connection_task,
-    check_lighthouse_connection_task,
-    check_lighthouse_provider_connection_task,
-    check_provider_connection_task,
-    delete_provider_task,
-    delete_tenant_task,
-    jira_integration_task,
-    mute_historical_findings_task,
-    perform_scan_task,
-    refresh_lighthouse_provider_models_task,
-)
 
 logger = logging.getLogger(BackendLogger.API)
 
@@ -4199,7 +4194,7 @@ class ComplianceOverviewViewSet(BaseRLSViewSet, TaskManagementMixin):
         # If we couldn't determine from database, try each provider type
         if not provider_type:
             for pt in Provider.ProviderChoices.values:
-                if compliance_id in PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE.get(pt, {}):
+                if compliance_id in get_compliance_frameworks(pt):
                     provider_type = pt
                     break
 

api/src/backend/tasks/tests/test_integrations.py

@@ -417,9 +417,8 @@ class TestProwlerIntegrationConnectionTest:
             raise_on_exception=False,
         )
 
-    @patch("api.utils.AwsProvider")
     @patch("api.utils.S3")
-    def test_s3_integration_connection_failure(self, mock_s3_class, mock_aws_provider):
+    def test_s3_integration_connection_failure(self, mock_s3_class):
         """Test S3 integration connection failure."""
         integration = MagicMock()
         integration.integration_type = Integration.IntegrationChoices.AMAZON_S3
@@ -429,9 +428,6 @@ class TestProwlerIntegrationConnectionTest:
         }
         integration.configuration = {"bucket_name": "test-bucket"}
 
-        mock_session = MagicMock()
-        mock_aws_provider.return_value.session.current_session = mock_session
-
         mock_connection = Connection(
             is_connected=False, error=Exception("Bucket not found")
         )