From 865eebe7fbd0de32d91856b897588e0c74439ff4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pedro=20Mart=C3=ADn?= Date: Thu, 10 Sep 2026 07:21:27 +0100 Subject: [PATCH] fix(aws): configurable boto3 timeouts, 10s connect default (#12774) --- .../basic-usage/prowler-cli.mdx | 14 +++ .../providers/aws/boto3-configuration.mdx | 29 +++++- ...s-boto3-connect-timeout-default.changed.md | 1 + .../changelog.d/aws-boto3-timeouts.added.md | 1 + .../aws-retries-max-attempts-zero.fixed.md | 1 + prowler/providers/aws/aws_provider.py | 39 +++++--- prowler/providers/aws/config.py | 27 +++++- .../providers/aws/exceptions/exceptions.py | 13 +++ .../providers/aws/lib/arguments/arguments.py | 23 ++++- .../aws/lib/session/aws_set_up_session.py | 8 +- prowler/providers/common/provider.py | 2 + tests/lib/cli/parser_test.py | 29 ++++++ tests/providers/aws/aws_provider_test.py | 88 +++++++++++++++++++ 13 files changed, 258 insertions(+), 17 deletions(-) create mode 100644 prowler/changelog.d/aws-boto3-connect-timeout-default.changed.md create mode 100644 prowler/changelog.d/aws-boto3-timeouts.added.md create mode 100644 prowler/changelog.d/aws-retries-max-attempts-zero.fixed.md diff --git a/docs/getting-started/basic-usage/prowler-cli.mdx b/docs/getting-started/basic-usage/prowler-cli.mdx index ebf3c6515e..d64bef0b77 100644 --- a/docs/getting-started/basic-usage/prowler-cli.mdx +++ b/docs/getting-started/basic-usage/prowler-cli.mdx @@ -2,6 +2,8 @@ title: 'Basic Usage' --- +import { VersionBadge } from "/snippets/version-badge.mdx" + ## Running Prowler Running Prowler requires specifying the provider (e.g. `aws`, `gcp`, `azure`, `kubernetes`, `m365`, `github`, `iac` or `mongodbatlas`): @@ -91,6 +93,18 @@ By default, `prowler` will scan all AWS regions. See more details about AWS Authentication in the [Authentication Section](/user-guide/providers/aws/authentication) section. +- **AWS Retrier and Timeout Configuration** + + + + Tune the Boto3 standard retrier and the endpoint timeouts when AWS throttles the scan or when some endpoints are unreachable from the network Prowler runs in: + + ```console + prowler aws --aws-retries-max-attempts 5 --aws-connect-timeout 5 --aws-read-timeout 30 + ``` + + See the [Boto3 configuration](/user-guide/providers/aws/boto3-configuration) page for defaults and environment variables. + ## Azure Azure requires specifying the auth method: diff --git a/docs/user-guide/providers/aws/boto3-configuration.mdx b/docs/user-guide/providers/aws/boto3-configuration.mdx index 83d01d5e80..d4e348b2b7 100644 --- a/docs/user-guide/providers/aws/boto3-configuration.mdx +++ b/docs/user-guide/providers/aws/boto3-configuration.mdx @@ -1,14 +1,39 @@ --- -title: "Boto3 Retrier Configuration in Prowler" +title: "Boto3 Retrier and Timeout Configuration in Prowler" --- +import { VersionBadge } from "/snippets/version-badge.mdx" + Prowler's AWS Provider leverages Boto3's [Standard](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/retries.html) retry mode to automatically retry client calls to AWS services when encountering errors or exceptions. +## Timeout Configuration + + + +Every AWS API call is bounded by two timeouts: + +- Connect timeout: seconds to wait to establish a connection (TCP, proxy tunnel and TLS handshake) to the AWS endpoint. Prowler's default is 10 seconds, configurable via `--aws-connect-timeout 5`. +- Read timeout: seconds to wait for a response once connected. Prowler's default is 60 seconds, configurable via `--aws-read-timeout 30`. + +Both timeouts can also be set through environment variables, which is the way to tune them in Prowler Cloud and other deployments without a CLI: + +```console +export PROWLER_AWS_BOTO3_CONNECT_TIMEOUT=5 +export PROWLER_AWS_BOTO3_READ_TIMEOUT=30 +``` + +CLI flags take precedence over the environment variables. Prowler sets both timeouts explicitly, so `AWS_DEFAULTS_MODE` and a `connect_timeout` in `~/.aws/config` are ignored; use the flag or the environment variable instead. + + +Boto3 defaults both timeouts to 60 seconds. In networks with restricted egress (for example VPC endpoints for a subset of services, GovCloud or private deployments), every AWS service without a reachable endpoint used to cost up to 4 attempts × 60 seconds (the first call plus the 3 retries) for each region. Prowler lowers the connect timeout to 10 seconds so unreachable endpoints fail fast; lower it further together with `--aws-retries-max-attempts 0`, which disables retries and leaves a single attempt per call, if a scan still spends most of its time waiting on unreachable services. + + + ## Retry Behavior Overview Boto3's Standard retry mode includes the following mechanisms: -- Maximum Retry Attempts: Default value set to 3, configurable via the `--aws-retries-max-attempts 5` argument. +- Maximum Retry Attempts: Default value set to 3, configurable via the `--aws-retries-max-attempts 5` argument. `0` disables retries. - Expanded Error Handling: Retries occur for a comprehensive set of errors. diff --git a/prowler/changelog.d/aws-boto3-connect-timeout-default.changed.md b/prowler/changelog.d/aws-boto3-connect-timeout-default.changed.md new file mode 100644 index 0000000000..a204443b2a --- /dev/null +++ b/prowler/changelog.d/aws-boto3-connect-timeout-default.changed.md @@ -0,0 +1 @@ +AWS provider default Boto3 connect timeout lowered from 60 to 10 seconds, so scans in restricted-egress networks (VPC endpoints for a subset of services, GovCloud, private deployments) no longer spend 4 minutes per region on every service whose endpoint is unreachable diff --git a/prowler/changelog.d/aws-boto3-timeouts.added.md b/prowler/changelog.d/aws-boto3-timeouts.added.md new file mode 100644 index 0000000000..df5d6e4f2b --- /dev/null +++ b/prowler/changelog.d/aws-boto3-timeouts.added.md @@ -0,0 +1 @@ +`--aws-connect-timeout` and `--aws-read-timeout` CLI flags, plus `PROWLER_AWS_BOTO3_CONNECT_TIMEOUT` and `PROWLER_AWS_BOTO3_READ_TIMEOUT` environment variables, to bound how long each AWS API call waits for an endpoint diff --git a/prowler/changelog.d/aws-retries-max-attempts-zero.fixed.md b/prowler/changelog.d/aws-retries-max-attempts-zero.fixed.md new file mode 100644 index 0000000000..8eb2ad9e07 --- /dev/null +++ b/prowler/changelog.d/aws-retries-max-attempts-zero.fixed.md @@ -0,0 +1 @@ +`--aws-retries-max-attempts 0` now disables Boto3 retries instead of being silently ignored in favour of the default of 3 diff --git a/prowler/providers/aws/aws_provider.py b/prowler/providers/aws/aws_provider.py index d0ca3b98ee..07e3ecab89 100644 --- a/prowler/providers/aws/aws_provider.py +++ b/prowler/providers/aws/aws_provider.py @@ -126,6 +126,8 @@ class AwsProvider(Provider): aws_access_key_id: str = None, aws_secret_access_key: str = None, aws_session_token: Optional[str] = None, + connect_timeout: Optional[int] = None, + read_timeout: Optional[int] = None, ): """ Initializes the AWS provider. @@ -155,6 +157,8 @@ class AwsProvider(Provider): - aws_access_key_id: The AWS access key ID. - aws_secret_access_key: The AWS secret access key. - aws_session_token: The AWS session token, optional. + - connect_timeout: Seconds to wait to establish a connection to an AWS endpoint. + - read_timeout: Seconds to wait for a response from an AWS endpoint. Raises: - ArgumentTypeError: If the input MFA ARN is invalid. @@ -229,7 +233,9 @@ class AwsProvider(Provider): # TODO: Use AwsSetUpSession ????? # Configure the initial AWS Session using the local credentials: profile or environment variables - session_config = self.set_session_config(retries_max_attempts) + session_config = self.set_session_config( + retries_max_attempts, connect_timeout, read_timeout + ) aws_session = self.setup_session( mfa=mfa, profile=profile, @@ -1165,26 +1171,35 @@ class AwsProvider(Provider): return AWSMFAInfo(arn=mfa_ARN, totp=mfa_TOTP) @staticmethod - def set_session_config(retries_max_attempts: int) -> Config: + def set_session_config( + retries_max_attempts: int, + connect_timeout: Optional[int] = None, + read_timeout: Optional[int] = None, + ) -> Config: """ - set_session_config returns a botocore Config object with the Prowler user agent and the default retrier configuration if nothing is passed as argument + set_session_config returns a botocore Config object with the Prowler user agent and the default retrier and timeout configuration if nothing is passed as argument Args: - retries_max_attempts: The maximum number of retries for the standard retrier config + - connect_timeout: Seconds to wait to establish a connection to an AWS endpoint + - read_timeout: Seconds to wait for a response from an AWS endpoint Returns: - Config: The botocore Config object """ default_session_config = get_default_session_config() - if retries_max_attempts: - default_session_config = default_session_config.merge( - Config( - retries={ - "max_attempts": retries_max_attempts, - "mode": "standard", - }, - ) - ) + overrides = {} + if retries_max_attempts is not None: + overrides["retries"] = { + "max_attempts": retries_max_attempts, + "mode": "standard", + } + if connect_timeout: + overrides["connect_timeout"] = connect_timeout + if read_timeout: + overrides["read_timeout"] = read_timeout + if overrides: + default_session_config = default_session_config.merge(Config(**overrides)) return default_session_config diff --git a/prowler/providers/aws/config.py b/prowler/providers/aws/config.py index ea55d1a314..ed2ca503d0 100644 --- a/prowler/providers/aws/config.py +++ b/prowler/providers/aws/config.py @@ -2,14 +2,39 @@ import os from botocore.config import Config +from prowler.providers.aws.exceptions.exceptions import AWSInvalidBoto3TimeoutError + AWS_STS_GLOBAL_ENDPOINT_REGION = "us-east-1" AWS_REGION_US_EAST_1 = "us-east-1" BOTO3_USER_AGENT_EXTRA = os.getenv("PROWLER_AWS_BOTO3_USER_AGENT_EXTRA", "APN_1826889") +BOTO3_RETRIES_MAX_ATTEMPTS = 3 +# botocore defaults both to 60s +BOTO3_CONNECT_TIMEOUT = 10 +BOTO3_READ_TIMEOUT = 60 ROLE_SESSION_NAME = "ProwlerAssessmentSession" +def get_boto3_timeout_from_env(name: str, default: int) -> int: + """Positive integer seconds read from the environment, or default when unset.""" + raw = os.getenv(name, "").strip() + if not raw: + return default + if not raw.isdecimal() or int(raw) == 0: + raise AWSInvalidBoto3TimeoutError( + file=os.path.basename(__file__), + message=f"{name} must be a positive integer number of seconds, got {raw!r}", + ) + return int(raw) + + def get_default_session_config() -> Config: return Config( user_agent_extra=BOTO3_USER_AGENT_EXTRA, - retries={"max_attempts": 3, "mode": "standard"}, + retries={"max_attempts": BOTO3_RETRIES_MAX_ATTEMPTS, "mode": "standard"}, + connect_timeout=get_boto3_timeout_from_env( + "PROWLER_AWS_BOTO3_CONNECT_TIMEOUT", BOTO3_CONNECT_TIMEOUT + ), + read_timeout=get_boto3_timeout_from_env( + "PROWLER_AWS_BOTO3_READ_TIMEOUT", BOTO3_READ_TIMEOUT + ), ) diff --git a/prowler/providers/aws/exceptions/exceptions.py b/prowler/providers/aws/exceptions/exceptions.py index 4ea3d5e177..089e0d99c7 100644 --- a/prowler/providers/aws/exceptions/exceptions.py +++ b/prowler/providers/aws/exceptions/exceptions.py @@ -78,6 +78,10 @@ class AWSBaseException(ProwlerException): "message": "The provided AWS partition is invalid", "remediation": "Check the provided AWS partition and ensure it is valid.", }, + (1918, "AWSInvalidBoto3TimeoutError"): { + "message": "The Boto3 timeout configured through the environment is invalid", + "remediation": "Set PROWLER_AWS_BOTO3_CONNECT_TIMEOUT and PROWLER_AWS_BOTO3_READ_TIMEOUT to a positive integer number of seconds.", + }, } def __init__(self, code, file=None, original_exception=None, message=None): @@ -231,3 +235,12 @@ class AWSInvalidPartitionError(AWSBaseException): super().__init__( 1917, file=file, original_exception=original_exception, message=message ) + + +class AWSInvalidBoto3TimeoutError(AWSBaseException): + """Boto3 timeout configured through the environment is not a positive integer.""" + + def __init__(self, file=None, original_exception=None, message=None): + super().__init__( + 1918, file=file, original_exception=original_exception, message=message + ) diff --git a/prowler/providers/aws/lib/arguments/arguments.py b/prowler/providers/aws/lib/arguments/arguments.py index 2d1632422b..84f3b4adfa 100644 --- a/prowler/providers/aws/lib/arguments/arguments.py +++ b/prowler/providers/aws/lib/arguments/arguments.py @@ -156,7 +156,21 @@ def init_parser(self): nargs="?", default=None, type=int, - help="Set the maximum attemps for the Boto3 standard retrier config (Default: 3)", + help="Set the maximum retries for the Boto3 standard retrier config, 0 disables retries (Default: 3)", + ) + boto3_config_subparser.add_argument( + "--aws-connect-timeout", + nargs="?", + default=None, + type=validate_timeout, + help="Seconds to wait to establish a connection (TCP, proxy tunnel and TLS) to an AWS endpoint before retrying (Default: 10)", + ) + boto3_config_subparser.add_argument( + "--aws-read-timeout", + nargs="?", + default=None, + type=validate_timeout, + help="Seconds to wait for a response from an AWS endpoint before retrying (Default: 60)", ) # Scan Unused Services @@ -190,6 +204,13 @@ def validate_session_duration(session_duration: int) -> int: return duration +def validate_timeout(value: str) -> int: + """validate_timeout validates that the input is a whole number of seconds greater than zero""" + if not value.isdecimal() or int(value) == 0: + raise ArgumentTypeError(f"{value} is not a positive integer") + return int(value) + + def validate_role_session_name(session_name) -> str: """ Validates that the role session name is valid. diff --git a/prowler/providers/aws/lib/session/aws_set_up_session.py b/prowler/providers/aws/lib/session/aws_set_up_session.py index 3189400040..8f0b4130ca 100644 --- a/prowler/providers/aws/lib/session/aws_set_up_session.py +++ b/prowler/providers/aws/lib/session/aws_set_up_session.py @@ -42,6 +42,8 @@ class AwsSetUpSession: aws_session_token: Optional[str] = None, retries_max_attempts: int = 3, regions: set = set(), + connect_timeout: Optional[int] = None, + read_timeout: Optional[int] = None, ) -> None: """ The constructor for the AwsSetUpSession class. @@ -58,6 +60,8 @@ class AwsSetUpSession: - aws_session_token: The AWS session token, optional. - retries_max_attempts: The maximum number of retries for the AWS client. - regions: A set of regions to audit. + - connect_timeout: Seconds to wait to establish a connection to an AWS endpoint. + - read_timeout: Seconds to wait for a response from an AWS endpoint. Returns: @@ -73,7 +77,9 @@ class AwsSetUpSession: aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, ) - session_config = AwsProvider.set_session_config(retries_max_attempts) + session_config = AwsProvider.set_session_config( + retries_max_attempts, connect_timeout, read_timeout + ) aws_session = AwsProvider.setup_session( mfa=mfa, profile=profile, diff --git a/prowler/providers/common/provider.py b/prowler/providers/common/provider.py index 2e81bad121..7e23b8de7f 100644 --- a/prowler/providers/common/provider.py +++ b/prowler/providers/common/provider.py @@ -382,6 +382,8 @@ class Provider(ABC): ) provider_class( retries_max_attempts=arguments.aws_retries_max_attempts, + connect_timeout=arguments.aws_connect_timeout, + read_timeout=arguments.aws_read_timeout, role_arn=arguments.role, session_duration=arguments.session_duration, external_id=arguments.external_id, diff --git a/tests/lib/cli/parser_test.py b/tests/lib/cli/parser_test.py index da16f437b5..a811186b3b 100644 --- a/tests/lib/cli/parser_test.py +++ b/tests/lib/cli/parser_test.py @@ -1152,6 +1152,35 @@ class Test_Parser: parsed = self.parser.parse(command) assert parsed.aws_retries_max_attempts == int(max_retries) + def test_aws_parser_retries_max_attempts_zero(self): + command = [prowler_command, "--aws-retries-max-attempts", "0"] + parsed = self.parser.parse(command) + assert parsed.aws_retries_max_attempts == 0 + + def test_aws_parser_timeouts_default_to_none(self): + parsed = self.parser.parse([prowler_command]) + assert parsed.aws_connect_timeout is None + assert parsed.aws_read_timeout is None + + @pytest.mark.parametrize( + "argument, attribute", + [ + ("--aws-connect-timeout", "aws_connect_timeout"), + ("--aws-read-timeout", "aws_read_timeout"), + ], + ) + def test_aws_parser_timeouts(self, argument, attribute): + timeout = "5" + command = [prowler_command, argument, timeout] + parsed = self.parser.parse(command) + assert getattr(parsed, attribute) == int(timeout) + + @pytest.mark.parametrize("value", ["0", "-1", "abc"]) + def test_aws_parser_connect_timeout_rejects_non_positive(self, value): + command = [prowler_command, "--aws-connect-timeout", value] + with pytest.raises(SystemExit): + self.parser.parse(command) + def test_aws_parser_scan_unused_services(self): argument = "--scan-unused-services" command = [prowler_command, argument] diff --git a/tests/providers/aws/aws_provider_test.py b/tests/providers/aws/aws_provider_test.py index 7ce61b6862..cd8bb4f049 100644 --- a/tests/providers/aws/aws_provider_test.py +++ b/tests/providers/aws/aws_provider_test.py @@ -24,19 +24,24 @@ from prowler.providers.aws.aws_provider import ( ) from prowler.providers.aws.config import ( AWS_STS_GLOBAL_ENDPOINT_REGION, + BOTO3_CONNECT_TIMEOUT, + BOTO3_READ_TIMEOUT, BOTO3_USER_AGENT_EXTRA, ROLE_SESSION_NAME, + get_boto3_timeout_from_env, get_default_session_config, ) from prowler.providers.aws.exceptions.exceptions import ( AWSArgumentTypeValidationError, AWSIAMRoleARNInvalidResourceTypeError, + AWSInvalidBoto3TimeoutError, AWSInvalidPartitionError, AWSInvalidProviderIdError, AWSNoCredentialsError, ) from prowler.providers.aws.lib.arn.models import ARN from prowler.providers.aws.lib.mutelist.mutelist import AWSMutelist +from prowler.providers.aws.lib.session.aws_set_up_session import AwsSetUpSession from prowler.providers.aws.models import ( AWSAssumeRoleInfo, AWSCallerIdentity, @@ -2735,6 +2740,8 @@ aws: assert session_config.user_agent_extra == BOTO3_USER_AGENT_EXTRA assert session_config.retries == {"max_attempts": 3, "mode": "standard"} + assert session_config.connect_timeout == BOTO3_CONNECT_TIMEOUT + assert session_config.read_timeout == BOTO3_READ_TIMEOUT @mock_aws def test_set_session_config_10_max_attempts(self): @@ -2743,12 +2750,93 @@ aws: assert session_config.user_agent_extra == BOTO3_USER_AGENT_EXTRA assert session_config.retries == {"max_attempts": 10, "mode": "standard"} + assert session_config.connect_timeout == BOTO3_CONNECT_TIMEOUT + assert session_config.read_timeout == BOTO3_READ_TIMEOUT + + def test_set_session_config_0_max_attempts_disables_retries(self): + session_config = AwsProvider.set_session_config(0) + + assert session_config.retries == {"max_attempts": 0, "mode": "standard"} + + @mock_aws + def test_aws_provider_0_max_attempts_reaches_clients(self): + aws_provider = AwsProvider(retries_max_attempts=0) + client = aws_provider.session.current_session.client( + "ec2", region_name=AWS_REGION_US_EAST_1 + ) + + # botocore rewrites max_attempts into total_max_attempts (retries + 1) + assert client.meta.config.retries["total_max_attempts"] == 1 + + def test_set_session_config_timeouts(self): + session_config = AwsProvider.set_session_config( + None, connect_timeout=2, read_timeout=15 + ) + + assert session_config.retries == {"max_attempts": 3, "mode": "standard"} + assert session_config.connect_timeout == 2 + assert session_config.read_timeout == 15 + + @mock_aws + def test_aws_provider_timeouts_reach_session_config(self): + aws_provider = AwsProvider(connect_timeout=2, read_timeout=15) + + assert aws_provider.session.session_config.connect_timeout == 2 + assert aws_provider.session.session_config.read_timeout == 15 + + @mock_aws + def test_aws_set_up_session_forwards_timeouts(self): + aws_session = AwsSetUpSession( + aws_access_key_id="testing", + aws_secret_access_key="testing", + connect_timeout=2, + read_timeout=15, + ) + + assert aws_session._session.session_config.connect_timeout == 2 + assert aws_session._session.session_config.read_timeout == 15 def test_get_default_session_config(self): config = get_default_session_config() assert config.user_agent_extra == BOTO3_USER_AGENT_EXTRA assert config.retries == {"max_attempts": 3, "mode": "standard"} + assert config.connect_timeout == BOTO3_CONNECT_TIMEOUT + assert config.read_timeout == BOTO3_READ_TIMEOUT + + def test_get_default_session_config_timeouts_from_env(self): + with mock.patch.dict( + os.environ, + { + "PROWLER_AWS_BOTO3_CONNECT_TIMEOUT": "3", + "PROWLER_AWS_BOTO3_READ_TIMEOUT": "20", + }, + ): + config = get_default_session_config() + + assert config.connect_timeout == 3 + assert config.read_timeout == 20 + + def test_set_session_config_argument_overrides_env_timeouts(self): + with mock.patch.dict(os.environ, {"PROWLER_AWS_BOTO3_CONNECT_TIMEOUT": "3"}): + config = AwsProvider.set_session_config(None, connect_timeout=7) + + assert config.connect_timeout == 7 + + @pytest.mark.parametrize("raw", ["0", "-5", "ten", "1.5"]) + def test_get_boto3_timeout_from_env_rejects_non_positive_integers(self, raw): + with mock.patch.dict(os.environ, {"PROWLER_AWS_BOTO3_CONNECT_TIMEOUT": raw}): + with raises( + AWSInvalidBoto3TimeoutError, match="PROWLER_AWS_BOTO3_CONNECT_TIMEOUT" + ): + get_boto3_timeout_from_env("PROWLER_AWS_BOTO3_CONNECT_TIMEOUT", 10) + + def test_get_boto3_timeout_from_env_blank_falls_back_to_default(self): + with mock.patch.dict(os.environ, {"PROWLER_AWS_BOTO3_CONNECT_TIMEOUT": " "}): + assert ( + get_boto3_timeout_from_env("PROWLER_AWS_BOTO3_CONNECT_TIMEOUT", 10) + == 10 + ) @mock_aws @patch(