fix(alibabacloud): retry STS connection failures (#12353)

prowler/changelog.d/alibabacloud-sts-connection-retries.fixed.md

@@ -0,0 +1 @@
+Alibaba Cloud STS credential validation retries transient connection failures and reports exhausted attempts as connection errors instead of invalid credentials

prowler/providers/alibabacloud/alibabacloud_provider.py

@@ -1,11 +1,17 @@
 import os
 import pathlib
+import socket
 
 from alibabacloud_credentials.client import Client as CredClient
 from alibabacloud_credentials.models import Config as CredConfig
 from alibabacloud_sts20150401.client import Client as StsClient
 from alibabacloud_tea_openapi import models as open_api_models
+from alibabacloud_tea_openapi.exceptions import ClientException
 from colorama import Fore, Style
+from darabonba.exceptions import RetryError
+from darabonba.policy.retry import RetryCondition, RetryOptions
+from requests.exceptions import ConnectionError as RequestsConnectionError
+from requests.exceptions import Timeout as RequestsTimeout
 
 from prowler.config.config import (
     default_config_file_path,
@@ -17,9 +23,12 @@ from prowler.lib.utils.utils import print_boxes
 from prowler.providers.alibabacloud.config import (
     ALIBABACLOUD_DEFAULT_REGION,
     ALIBABACLOUD_REGIONS,
+    ALIBABACLOUD_STS_MAX_ATTEMPTS,
+    ALIBABACLOUD_STS_RETRY_DELAY_MS,
     ROLE_SESSION_NAME,
 )
 from prowler.providers.alibabacloud.exceptions.exceptions import (
+    AlibabaCloudConnectionError,
     AlibabaCloudInvalidCredentialsError,
     AlibabaCloudNoCredentialsError,
     AlibabaCloudSetUpSessionError,
@@ -34,6 +43,61 @@ from prowler.providers.common.models import Audit_Metadata, Connection
 from prowler.providers.common.provider import Provider
 
 
+def _exception_chain(error: Exception):
+    """Yield structured exceptions wrapped by SDK and Python exception chains."""
+    pending = [error]
+    seen = set()
+
+    while pending:
+        current = pending.pop()
+        if id(current) in seen:
+            continue
+        seen.add(id(current))
+        yield current
+
+        for attribute in ("inner_exception", "__cause__", "__context__"):
+            nested = getattr(current, attribute, None)
+            if isinstance(nested, BaseException):
+                pending.append(nested)
+        pending.extend(arg for arg in current.args if isinstance(arg, BaseException))
+
+
+def _is_connection_error(error: Exception) -> bool:
+    """Return whether an SDK exception chain contains a transport failure."""
+    connection_errors = (
+        ConnectionError,
+        TimeoutError,
+        socket.gaierror,
+        RetryError,
+        RequestsConnectionError,
+        RequestsTimeout,
+    )
+    return any(
+        isinstance(exception, connection_errors)
+        for exception in _exception_chain(error)
+    )
+
+
+def _is_authentication_error(error: Exception) -> bool:
+    """Return whether an SDK exception chain contains an authentication failure."""
+    authentication_code_prefixes = (
+        "InvalidAccessKeyId",
+        "InvalidSecurityToken",
+        "MissingSecurityToken",
+        "SecurityTokenExpired",
+        "SignatureDoesNotMatch",
+    )
+    for exception in _exception_chain(error):
+        if not isinstance(exception, ClientException):
+            continue
+        code = exception.code or ""
+        if exception.status_code == 401 or code == "InvalidCredentials":
+            return True
+        if code.startswith(authentication_code_prefixes):
+            return True
+    return False
+
+
 class AlibabacloudProvider(Provider):
     """
     AlibabacloudProvider class is the main class for the Alibaba Cloud provider.
@@ -435,6 +499,7 @@ class AlibabacloudProvider(Provider):
             AlibabaCloudCallerIdentity: An object containing the caller identity information.
 
         Raises:
+            AlibabaCloudConnectionError: If STS cannot be reached after retries.
             AlibabaCloudInvalidCredentialsError: If credentials are invalid.
         """
         try:
@@ -445,6 +510,18 @@ class AlibabacloudProvider(Provider):
             sts_config = open_api_models.Config(
                 access_key_id=cred.access_key_id,
                 access_key_secret=cred.access_key_secret,
+                retry_options=RetryOptions(
+                    retryCondition=[
+                        RetryCondition(
+                            maxAttempts=ALIBABACLOUD_STS_MAX_ATTEMPTS,
+                            exception=["RetryError"],
+                            backoff={
+                                "policy": "Fixed",
+                                "period": ALIBABACLOUD_STS_RETRY_DELAY_MS,
+                            },
+                        )
+                    ]
+                ),
             )
             if cred.security_token:
                 sts_config.security_token = cred.security_token
@@ -477,10 +554,17 @@ class AlibabacloudProvider(Provider):
 
         except Exception as sts_error:
             logger.error(f"Could not get caller identity from STS: {sts_error}. ")
-            raise AlibabaCloudInvalidCredentialsError(
-                file=pathlib.Path(__file__).name,
-                original_exception=sts_error,
-            )
+            if _is_authentication_error(sts_error):
+                raise AlibabaCloudInvalidCredentialsError(
+                    file=pathlib.Path(__file__).name,
+                    original_exception=sts_error,
+                ) from sts_error
+            if _is_connection_error(sts_error):
+                raise AlibabaCloudConnectionError(
+                    file=pathlib.Path(__file__).name,
+                    original_exception=sts_error,
+                ) from sts_error
+            raise
 
     @staticmethod
     def get_profile_region() -> str:
@@ -742,6 +826,7 @@ class AlibabacloudProvider(Provider):
 
         Raises:
             AlibabaCloudSetUpSessionError: If there is an error setting up the session.
+            AlibabaCloudConnectionError: If STS cannot be reached after retries.
             AlibabaCloudInvalidCredentialsError: If there is an authentication error.
             Exception: If there is an unexpected error.
 
@@ -809,6 +894,14 @@ class AlibabacloudProvider(Provider):
                 raise auth_error
             return Connection(error=auth_error)
 
+        except AlibabaCloudConnectionError as connection_error:
+            logger.error(
+                f"{connection_error.__class__.__name__}[{connection_error.__traceback__.tb_lineno}]: {connection_error}"
+            )
+            if raise_on_exception:
+                raise connection_error
+            return Connection(error=connection_error)
+
         except Exception as error:
             logger.critical(
                 f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"

prowler/providers/alibabacloud/config.py

@@ -6,6 +6,10 @@ ROLE_SESSION_NAME = "ProwlerAssessmentSession"
 # Alibaba Cloud SDK Configuration
 ALIBABACLOUD_SDK_READ_TIMEOUT = 60  # seconds
 ALIBABACLOUD_SDK_CONNECT_TIMEOUT = 10  # seconds
+ALIBABACLOUD_STS_MAX_ATTEMPTS = 3
+# Avoid immediate retry bursts while bounding added retry delay to two seconds:
+# three total attempts introduce at most two fixed one-second waits.
+ALIBABACLOUD_STS_RETRY_DELAY_MS = 1000
 
 # Alibaba Cloud Regions - Only publicly accessible regions
 # Note: Some regions may require special approval or are not globally available

prowler/providers/alibabacloud/exceptions/exceptions.py

@@ -38,6 +38,10 @@ class AlibabaCloudBaseException(ProwlerException):
             "message": "Alibaba Cloud HTTP/API error",
             "remediation": "Check the Alibaba Cloud API request and response, and ensure the service is accessible.",
         },
+        (10008, "AlibabaCloudConnectionError"): {
+            "message": "Could not connect to Alibaba Cloud",
+            "remediation": "Check network connectivity and ensure the Alibaba Cloud service endpoint is accessible.",
+        },
     }
 
     def __init__(self, code, file=None, original_exception=None, message=None):
@@ -114,3 +118,12 @@ class AlibabaCloudHTTPError(AlibabaCloudBaseException):
         super().__init__(
             10007, file=file, original_exception=original_exception, message=message
         )
+
+
+class AlibabaCloudConnectionError(AlibabaCloudBaseException):
+    """Raised when Alibaba Cloud cannot be reached after retry attempts."""
+
+    def __init__(self, file=None, original_exception=None, message=None):
+        super().__init__(
+            10008, file=file, original_exception=original_exception, message=message
+        )

tests/providers/alibabacloud/alibabacloud_provider_test.py

@@ -1,10 +1,16 @@
 import os
-from unittest.mock import MagicMock, patch
+from unittest.mock import MagicMock, call, patch
 
 import pytest
+from alibabacloud_tea_openapi.exceptions import ClientException
+from darabonba.core import DaraCore
+from darabonba.exceptions import RetryError, UnretryableException
+from Tea.exceptions import UnretryableException as TeaUnretryableException
+from Tea.response import TeaResponse
 
 from prowler.providers.alibabacloud.alibabacloud_provider import AlibabacloudProvider
 from prowler.providers.alibabacloud.exceptions.exceptions import (
+    AlibabaCloudConnectionError,
     AlibabaCloudInvalidCredentialsError,
     AlibabaCloudSetUpSessionError,
 )
@@ -12,9 +18,243 @@ from prowler.providers.alibabacloud.models import AlibabaCloudCallerIdentity
 from prowler.providers.common.models import Connection
 
 
+def test_validate_credentials_non_authentication_api_error_is_not_invalid_credentials():
+    """Test non-authentication STS API errors are not relabeled as credentials."""
+    session = MagicMock()
+    session.get_credentials.return_value = MagicMock(
+        access_key_id="LTAI1234567890",
+        access_key_secret="test-secret-key",
+        security_token=None,
+    )
+    api_error = ClientException(
+        code="Forbidden",
+        message="The caller is not authorized",
+        status_code=403,
+    )
+    wrapped_api_error = TeaUnretryableException(request=None, ex=api_error)
+
+    with (
+        patch.object(DaraCore, "do_action", side_effect=wrapped_api_error),
+        patch.object(DaraCore, "sleep") as sleep,
+    ):
+        with pytest.raises(UnretryableException) as exception:
+            AlibabacloudProvider.validate_credentials(session)
+
+    assert not isinstance(exception.value, AlibabaCloudInvalidCredentialsError)
+    assert exception.value.inner_exception is wrapped_api_error
+    assert exception.value.inner_exception.inner_exception is api_error
+    sleep.assert_not_called()
+
+
+def test_validate_credentials_retries_transport_failure_then_succeeds():
+    """Test STS caller identity retries a transient transport failure."""
+    session = MagicMock()
+    session.get_credentials.return_value = MagicMock(
+        access_key_id="LTAI1234567890",
+        access_key_secret="test-secret-key",
+        security_token=None,
+    )
+    response = TeaResponse()
+    response.status_code = 200
+    response.headers = {"content-type": "application/json"}
+    response.body = (
+        b'{"AccountId":"1234567890","PrincipalId":"123456",'
+        b'"Arn":"acs:ram::1234567890:user/test-user",'
+        b'"IdentityType":"RamUser"}'
+    )
+
+    with (
+        patch.object(
+            DaraCore,
+            "do_action",
+            side_effect=[RetryError("connection reset"), response],
+        ) as do_action,
+        patch.object(DaraCore, "sleep") as sleep,
+    ):
+        caller_identity = AlibabacloudProvider.validate_credentials(session)
+
+    assert caller_identity.account_id == "1234567890"
+    assert do_action.call_count == 2
+    sleep.assert_called_once_with(1000)
+
+
+def test_validate_credentials_connection_failure_is_not_invalid_credentials():
+    """Test exhausted STS transport retries raise a connection-specific error."""
+    session = MagicMock()
+    session.get_credentials.return_value = MagicMock(
+        access_key_id="LTAI1234567890",
+        access_key_secret="test-secret-key",
+        security_token=None,
+    )
+    retry_errors = []
+    for _ in range(3):
+        connection_reset = ConnectionResetError(104, "Connection reset by peer")
+        retry_error = RetryError(str(connection_reset))
+        retry_error.__cause__ = connection_reset
+        retry_errors.append(retry_error)
+
+    with (
+        patch.object(DaraCore, "do_action", side_effect=retry_errors) as do_action,
+        patch.object(DaraCore, "sleep") as sleep,
+    ):
+        with pytest.raises(AlibabaCloudConnectionError) as exception:
+            AlibabacloudProvider.validate_credentials(session)
+
+    assert not isinstance(exception.value, AlibabaCloudInvalidCredentialsError)
+    assert do_action.call_count == 3
+    assert sleep.call_args_list == [call(1000), call(1000)]
+    assert isinstance(exception.value.original_exception, UnretryableException)
+    assert exception.value.original_exception.inner_exception is retry_errors[-1]
+    assert exception.value.__cause__ is exception.value.original_exception
+
+
+def test_validate_credentials_genuine_invalid_credentials():
+    """Test an explicit STS authentication failure remains a credentials error."""
+    session = MagicMock()
+    session.get_credentials.return_value = MagicMock(
+        access_key_id="LTAI-invalid",
+        access_key_secret="invalid-secret",
+        security_token=None,
+    )
+    authentication_error = ClientException(
+        code="InvalidAccessKeyId.NotFound",
+        message="The AccessKey ID does not exist",
+        status_code=400,
+    )
+    wrapped_authentication_error = TeaUnretryableException(
+        request=None, ex=authentication_error
+    )
+
+    with (
+        patch.object(DaraCore, "do_action", side_effect=wrapped_authentication_error),
+        patch.object(DaraCore, "sleep") as sleep,
+    ):
+        with pytest.raises(AlibabaCloudInvalidCredentialsError) as exception:
+            AlibabacloudProvider.validate_credentials(session)
+
+    assert isinstance(exception.value.original_exception, UnretryableException)
+    assert (
+        exception.value.original_exception.inner_exception
+        is wrapped_authentication_error
+    )
+    assert (
+        exception.value.original_exception.inner_exception.inner_exception
+        is authentication_error
+    )
+    assert exception.value.__cause__ is exception.value.original_exception
+    sleep.assert_not_called()
+
+
+def test_validate_credentials_authentication_error_wins_over_transport_error():
+    """Test a definitive nested authentication error takes precedence over transport."""
+    session = MagicMock()
+    session.get_credentials.return_value = MagicMock(
+        access_key_id="LTAI-invalid",
+        access_key_secret="invalid-secret",
+        security_token=None,
+    )
+    authentication_error = ClientException(
+        code="InvalidAccessKeyId.NotFound",
+        message="The AccessKey ID does not exist",
+        status_code=400,
+    )
+    retry_errors = []
+    for _ in range(3):
+        retry_error = RetryError("connection reset")
+        retry_error.__cause__ = authentication_error
+        retry_errors.append(retry_error)
+
+    with (
+        patch.object(DaraCore, "do_action", side_effect=retry_errors),
+        patch.object(DaraCore, "sleep") as sleep,
+    ):
+        with pytest.raises(AlibabaCloudInvalidCredentialsError) as exception:
+            AlibabacloudProvider.validate_credentials(session)
+
+    assert isinstance(exception.value.original_exception, UnretryableException)
+    assert exception.value.original_exception.inner_exception is retry_errors[-1]
+    assert exception.value.__cause__ is exception.value.original_exception
+    assert sleep.call_args_list == [call(1000), call(1000)]
+
+
 class TestAlibabacloudProviderTestConnection:
     """Tests for the AlibabacloudProvider.test_connection method."""
 
+    def test_test_connection_connection_error_no_raise(self):
+        """Test connection failures are returned with their dedicated type."""
+        mock_session = MagicMock()
+        connection_error = AlibabaCloudConnectionError(
+            file="test_file",
+            original_exception=RetryError("connection reset"),
+        )
+
+        with (
+            patch.object(
+                AlibabacloudProvider,
+                "setup_session",
+                return_value=mock_session,
+            ),
+            patch.object(
+                AlibabacloudProvider,
+                "validate_credentials",
+                side_effect=connection_error,
+            ),
+            patch(
+                "prowler.providers.alibabacloud.alibabacloud_provider.logger.error"
+            ) as logger_error,
+            patch(
+                "prowler.providers.alibabacloud.alibabacloud_provider.logger.critical"
+            ) as logger_critical,
+        ):
+            result = AlibabacloudProvider.test_connection(
+                access_key_id="LTAI1234567890",
+                access_key_secret="test-secret-key",
+                raise_on_exception=False,
+            )
+
+        assert result.is_connected is False
+        assert result.error is connection_error
+        assert result.error.code == 10008
+        logger_error.assert_called_once()
+        logger_critical.assert_not_called()
+
+    def test_test_connection_connection_error_raises(self):
+        """Test connection failures retain raise-on-exception behavior."""
+        mock_session = MagicMock()
+        connection_error = AlibabaCloudConnectionError(
+            file="test_file",
+            original_exception=RetryError("connection reset"),
+        )
+
+        with (
+            patch.object(
+                AlibabacloudProvider,
+                "setup_session",
+                return_value=mock_session,
+            ),
+            patch.object(
+                AlibabacloudProvider,
+                "validate_credentials",
+                side_effect=connection_error,
+            ),
+            patch(
+                "prowler.providers.alibabacloud.alibabacloud_provider.logger.error"
+            ) as logger_error,
+            patch(
+                "prowler.providers.alibabacloud.alibabacloud_provider.logger.critical"
+            ) as logger_critical,
+        ):
+            with pytest.raises(AlibabaCloudConnectionError) as exception:
+                AlibabacloudProvider.test_connection(
+                    access_key_id="LTAI1234567890",
+                    access_key_secret="test-secret-key",
+                    raise_on_exception=True,
+                )
+
+        assert exception.value is connection_error
+        logger_error.assert_called_once()
+        logger_critical.assert_not_called()
+
     def test_test_connection_with_static_credentials_success(self):
         """Test successful connection with static access key credentials."""
         mock_session = MagicMock()

tests/providers/alibabacloud/conftest.py

@@ -5,6 +5,7 @@ Mocks Alibaba Cloud SDK modules to avoid import issues when the real
 dependencies are not installed in the test environment.
 """
 
+import importlib
 import sys
 from unittest.mock import MagicMock
 
@@ -38,4 +39,7 @@ MOCKED_MODULES = [
 ]
 
 for module_name in MOCKED_MODULES:
-    sys.modules.setdefault(module_name, MagicMock())
+    try:
+        importlib.import_module(module_name)
+    except ModuleNotFoundError:
+        sys.modules.setdefault(module_name, MagicMock())