Skip to content

Make EcsCredentialProvider retry behavior configurable #3279

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
kieranbrown wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Make EcsCredentialProvider retry behavior configurable #3279

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 42 additions & 1 deletion src/Credentials/EcsCredentialProvider.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,11 +41,22 @@ class EcsCredentialProvider
/** @var int */
private $attempts;

/** @var string[] */
private $retryableExceptions;

@stobrien89 stobrien89 May 5, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This should have a default value of [ConnectException::class] and any exceptions passed in retryable_exceptions should be added to it. Open to alternatives if your use case requires the current pattern


/** @var int[] */
private $retryableErrorCodes;

/**
* The constructor accepts following options:
* - timeout: (optional) Connection timeout, in seconds, default 1.0
* - retries: Optional number of retries to be attempted, default 3.
* - client: An EcsClient to make request from
* - retryable_exceptions: Optional array of additional exception class
* names that should be retried. Connection errors are always retried,
* regardless of this option.
* - retryable_error_codes: Optional array of HTTP status codes that
* should be retried. Defaults to an empty array.
*
* @param array $config Configuration options
*/
Expand All @@ -58,6 +69,8 @@ public function __construct(array $config = [])
: ((int) getenv(self::ENV_RETRIES) ?: self::DEFAULT_ENV_RETRIES);

$this->client = $config['client'] ?? \Aws\default_http_handler();
$this->retryableExceptions = $config['retryable_exceptions'] ?? [];
$this->retryableErrorCodes = $config['retryable_error_codes'] ?? [];
}

/**
Expand Down Expand Up @@ -106,7 +119,8 @@ public function __invoke()
})->otherwise(function ($reason) {
$connectionError = is_array($reason) && !empty($reason['connection_error']);
$exception = is_array($reason) ? ($reason['exception'] ?? null) : $reason;
$isRetryable = $connectionError || ($exception instanceof \Throwable && HttpHandlerError::isConnectionError($exception));
$isRetryable = $connectionError
|| ($exception instanceof \Throwable && $this->isRetryable($exception));

if ($isRetryable && ($this->attempts < $this->retries)) {
sleep((int)pow(1.2, $this->attempts));
Expand Down Expand Up @@ -221,6 +235,33 @@ private function getEcsUri()
return self::SERVER_URI . $credsUri;
}

/**
* Determines whether a failed request should be retried. Connection
* errors are always retried; the configured retryable_exceptions and
* retryable_error_codes are checked in addition to them.
*/
private function isRetryable(\Throwable $exception): bool
{
if (HttpHandlerError::isConnectionError($exception)) {
return true;
}

foreach ($this->retryableExceptions as $exceptionClass) {
if ($exception instanceof $exceptionClass) {
return true;
}
}

$response = HttpHandlerError::getResponse($exception);

return $response !== null
&& in_array(
$response->getStatusCode(),
$this->retryableErrorCodes,
true
);
}

private function decodeResult($response)
{
$result = json_decode($response, true);
Expand Down
183 changes: 183 additions & 0 deletions tests/Credentials/EcsCredentialProviderTest.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -462,6 +462,15 @@ public static function successDataProvider(): array
]);
$rejectionRawConnectException = Promise\Create::rejectionFor($connectException);

$tooManyRequestsException = self::createRequestException(
'429 Too Many Requests',
new Psr7\Request('GET', '/latest'),
new Psr7\Response(429)
);
$rejectionTooManyRequests = Promise\Create::rejectionFor([
'exception' => $tooManyRequestsException,
]);

$promiseCreds = Promise\Create::promiseFor(
new Response(200, [], Psr7\Utils::streamFor(
json_encode(call_user_func_array(
Expand Down Expand Up @@ -524,6 +533,135 @@ public static function successDataProvider(): array
];
}

public function testRetriesOptedInErrorCode()
{
$expiry = time() + 1000;
$creds = ['foo_key', 'baz_secret', 'qux_token', "@{$expiry}"];

$rejectionTooManyRequests = Promise\Create::rejectionFor([
'exception' => self::createRequestException(
'429 Too Many Requests',
new Psr7\Request('GET', '/latest'),
new Psr7\Response(429)
),
]);
$promiseCreds = Promise\Create::promiseFor(
new Response(200, [], Psr7\Utils::streamFor(
json_encode(call_user_func_array(
[self::class, 'getCredentialArray'],
$creds
)))
)
);

$provider = new EcsCredentialProvider([
'client' => $this->getTestClient([
$rejectionTooManyRequests,
$promiseCreds,
], $creds),
'retries' => 2,
'retryable_error_codes' => [429],
]);

$credentials = $provider()->wait();
$this->assertSame('foo_key', $credentials->getAccessKeyId());
$this->assertSame('baz_secret', $credentials->getSecretKey());
}

public function testDoesNotRetry429ByDefault()
{
$rejectionTooManyRequests = Promise\Create::rejectionFor([
'exception' => self::createRequestException(
'429 Too Many Requests',
new Psr7\Request('GET', '/latest'),
new Psr7\Response(429)
),
]);

$provider = new EcsCredentialProvider([
'client' => $this->getTestClient([
$rejectionTooManyRequests,
]),
'retries' => 3,
]);

try {
$provider()->wait();
$this->fail('Provider should have thrown an exception.');
} catch (CredentialsException $e) {
$this->assertStringContainsString(
'attempt 0/3',
$e->getMessage()
);
$this->assertStringContainsString('429 Too Many Requests', $e->getMessage());
}

$this->assertSame(0, $provider->getAttempts());
}

public function testRetriesOptedInExceptionClass()
{
$expiry = time() + 1000;
$creds = ['foo_key', 'baz_secret', 'qux_token', "@{$expiry}"];

$rejectionRequest = Promise\Create::rejectionFor([
'exception' => new \DomainException('Boom'),
]);
$promiseCreds = Promise\Create::promiseFor(
new Response(200, [], Psr7\Utils::streamFor(
json_encode(call_user_func_array(
[self::class, 'getCredentialArray'],
$creds
)))
)
);

$provider = new EcsCredentialProvider([
'client' => $this->getTestClient([
$rejectionRequest,
$promiseCreds,
], $creds),
'retries' => 2,
'retryable_exceptions' => [\DomainException::class],
]);

$credentials = $provider()->wait();
$this->assertSame('foo_key', $credentials->getAccessKeyId());
}

public function testCustomRetryableExceptionsAreAddedToDefaults()
{
$expiry = time() + 1000;
$creds = ['foo_key', 'baz_secret', 'qux_token', "@{$expiry}"];

$rejectionConnection = Promise\Create::rejectionFor([
'exception' => new ConnectException(
'cURL error 28: Connection timed out after 1000 milliseconds',
new Psr7\Request('GET', '/latest')
),
]);
$promiseCreds = Promise\Create::promiseFor(
new Response(200, [], Psr7\Utils::streamFor(
json_encode(call_user_func_array(
[self::class, 'getCredentialArray'],
$creds
)))
)
);

$provider = new EcsCredentialProvider([
'client' => $this->getTestClient([
$rejectionConnection,
$promiseCreds,
], $creds),
'retries' => 2,
'retryable_exceptions' => [\DomainException::class],
]);

$credentials = $provider()->wait();
$this->assertSame('foo_key', $credentials->getAccessKeyId());
}

/**
* @param $client
* @param \Exception $expected
Expand Down Expand Up @@ -566,6 +704,13 @@ public static function failureDataProvider(): array
'connection_error' => true,
'exception' => new \Exception('cURL error 28: Connection timed out after 1000 milliseconds'),
]);
$rejectionTooManyRequests = Promise\Create::rejectionFor([
'exception' => self::createRequestException(
'429 Too Many Requests',
$getRequest,
new Psr7\Response(429)
)
]);

return [
'Non-retryable error' => [
Expand All @@ -585,9 +730,47 @@ public static function failureDataProvider(): array
'Error retrieving credentials from container metadata after attempt 1/1 (cURL error 28: Connection timed out after 1000 milliseconds)'
)
],
'Non-retryable HTTP 429 by default' => [
[
$rejectionTooManyRequests,
],
new CredentialsException(
'Error retrieving credentials from container metadata after attempt 0/1 (429 Too Many Requests)'
)
],
];
}

public function testOptedInHTTP429RetryExhaustsAttempts()
{
$rejectionTooManyRequests = Promise\Create::rejectionFor([
'exception' => self::createRequestException(
'429 Too Many Requests',
new Psr7\Request('GET', '/latest'),
new Psr7\Response(429)
),
]);

$provider = new EcsCredentialProvider([
'client' => $this->getTestClient([
$rejectionTooManyRequests,
$rejectionTooManyRequests,
]),
'retries' => 1,
'retryable_error_codes' => [429],
]);

try {
$provider()->wait();
$this->fail('Provider should have thrown an exception.');
} catch (CredentialsException $e) {
$this->assertSame(
'Error retrieving credentials from container metadata after attempt 1/1 (429 Too Many Requests)',
$e->getMessage()
);
}
}

public function testReadsRetriesFromEnvironment()
{
putenv('AWS_METADATA_SERVICE_NUM_ATTEMPTS=1');
Expand Down