Add credential_cache setting to keep role credentials out of the file cache

.changes/next-release/feature-credentials-75017.json

@@ -0,0 +1,5 @@
+{
+  "type": "feature",
+  "category": "credentials",
+  "description": "Add the ``credential_cache`` setting for assume role and assume role with web identity profiles. Set ``credential_cache = memory`` to keep a profile's temporary credentials out of ``~/.aws/cli/cache``."
+}

awscli/botocore/credentials.py

@@ -1027,6 +1027,56 @@ def _assume_role_kwargs(self):
         return assume_role_kwargs
 
 
+_CREDENTIAL_CACHE_FILE = 'file'
+_CREDENTIAL_CACHE_MEMORY = 'memory'
+_CREDENTIAL_CACHE_VALUES = (_CREDENTIAL_CACHE_FILE, _CREDENTIAL_CACHE_MEMORY)
+# Used by ``credential_cache = file`` where no cache was injected. It's the
+# directory the AWS CLI's injected role cache uses.
+_DEFAULT_CREDENTIAL_CACHE_DIR = os.path.join('~', '.aws', 'cli', 'cache')
+
+
+def _get_credential_cache_config(profile_name, profile):
+    """Read and validate a role profile's ``credential_cache`` setting.
+
+    The setting controls whether a role profile's temporary credentials
+    are cached on disk or only in memory. Returns ``file``, ``memory`` or
+    ``None`` when unset.
+    """
+    credential_cache = profile.get('credential_cache')
+    if (
+        credential_cache is not None
+        and credential_cache not in _CREDENTIAL_CACHE_VALUES
+    ):
+        valid = ', '.join(_CREDENTIAL_CACHE_VALUES)
+        raise InvalidConfigError(
+            error_msg=(
+                f'The credential_cache "{credential_cache}" in profile '
+                f'"{profile_name}" is not valid. Valid values are: {valid}.'
+            )
+        )
+    return credential_cache
+
+
+def _select_credential_cache(credential_cache, default_cache, memory_cache):
+    """Pick the cache a role hop uses, from its profile's setting.
+
+    ``memory`` uses ``memory_cache`` (a ``dict`` owned by the provider), so
+    nothing reaches the file cache. A profile that doesn't set the key, or
+    sets ``file``, uses the provider's injected cache (``default_cache``),
+    which is the behavior before this setting existed. Where nothing was
+    injected (``default_cache`` is the plain ``dict`` default, as on a web
+    identity source hop), an explicit ``file`` still caches on disk, in a
+    plain ``JSONFileCache`` in ``~/.aws/cli/cache``.
+    """
+    if credential_cache == _CREDENTIAL_CACHE_MEMORY:
+        return memory_cache
+    if credential_cache == _CREDENTIAL_CACHE_FILE and (
+        default_cache is None or isinstance(default_cache, dict)
+    ):
+        return JSONFileCache(os.path.expanduser(_DEFAULT_CREDENTIAL_CACHE_DIR))
+    return default_cache
+
+
 class CredentialProvider:
     # A short name to identify the provider within botocore.
     METHOD = None
@@ -1622,6 +1672,8 @@ def __init__(
         self._profile_provider_builder = profile_provider_builder
         self._visited_profiles = [self._profile_name]
         self._feature_ids = set()
+        # Used by profiles that set ``credential_cache = memory``.
+        self._memory_cache = {}
 
     def load(self):
         self._loaded_config = self._load_config()
@@ -1670,7 +1722,11 @@ def _load_creds_via_assume_role(self, profile_name):
             role_arn=role_config['role_arn'],
             extra_args=extra_args,
             mfa_prompter=self._prompter,
-            cache=self.cache,
+            cache=_select_credential_cache(
+                role_config['credential_cache'],
+                default_cache=self.cache,
+                memory_cache=self._memory_cache,
+            ),
         )
         fetcher.feature_ids = self._feature_ids.copy()
         refresher = fetcher.fetch_credentials
@@ -1700,6 +1756,7 @@ def _get_role_config(self, profile_name):
         external_id = profile.get('external_id')
         role_session_name = profile.get('role_session_name')
         duration_seconds = profile.get('duration_seconds')
+        credential_cache = _get_credential_cache_config(profile_name, profile)
 
         role_config = {
             'role_arn': role_arn,
@@ -1708,6 +1765,7 @@ def _get_role_config(self, profile_name):
             'role_session_name': role_session_name,
             'source_profile': source_profile,
             'credential_source': credential_source,
+            'credential_cache': credential_cache,
         }
 
         if duration_seconds is not None:
@@ -1911,6 +1969,8 @@ def __init__(
             token_loader_cls = FileWebIdentityTokenLoader
         self._token_loader_cls = token_loader_cls
         self._feature_ids = set()
+        # Used by profiles that set ``credential_cache = memory``.
+        self._memory_cache = {}
 
     def load(self):
         return self._assume_role_with_web_identity()
@@ -1943,6 +2003,20 @@ def _get_config(self, key):
 
         return None
 
+    def _get_cache(self):
+        # credential_cache is a key of the profile in use, even when the
+        # token file or role ARN comes from the environment. A profile
+        # without it (or no profile) keeps the injected cache.
+        credential_cache = _get_credential_cache_config(
+            self._profile_name,
+            {'credential_cache': self._get_profile_config('credential_cache')},
+        )
+        return _select_credential_cache(
+            credential_cache,
+            default_cache=self.cache,
+            memory_cache=self._memory_cache,
+        )
+
     def _assume_role_with_web_identity(self):
         token_path = self._get_config('web_identity_token_file')
         if not token_path:
@@ -1969,7 +2043,7 @@ def _assume_role_with_web_identity(self):
             web_identity_token_loader=token_loader,
             role_arn=role_arn,
             extra_args=extra_args,
-            cache=self.cache,
+            cache=self._get_cache(),
         )
         fetcher.feature_ids = self._feature_ids.copy()
 

awscli/topics/config-vars.rst

@@ -283,13 +283,48 @@ in the AWS CLI config file:
   The value can range from 900 seconds (15 minutes) up to  the  maximum 
   session  duration setting  for  the role.  This is an optional parameter
   and by default, the value is set to 3600 seconds.
+* ``credential_cache`` - Where the AWS CLI caches the temporary credentials
+  from the ``AssumeRole`` call for this profile. Valid values are ``file``
+  (the default) and ``memory``. With ``file``, the credentials are stored on
+  disk, in ``~/.aws/cli/cache``, and reused by later commands until they
+  expire. With ``memory``, they are kept only for the duration of a single
+  AWS CLI command and are never written to disk. This is an optional
+  parameter.
+
+Switching a profile to ``credential_cache = memory`` does not delete the
+credentials it has already cached. To remove them, delete the files in
+``~/.aws/cli/cache``; the AWS CLI recreates the ones it needs.
+
+Only the AWS CLI reads ``credential_cache``. The AWS SDKs and other tools
+that read the shared config file ignore it, so a tool that keeps its own file
+cache may still write credentials to disk for a profile set to ``memory``.
+
+In a chain of roles, where a ``source_profile`` is itself a profile that
+assumes a role, each profile's ``credential_cache`` applies only to that
+profile's ``AssumeRole`` call.
+
+When the ``source_profile`` is an assume role with web identity profile that
+doesn't set ``credential_cache``, its credentials are cached only in memory,
+as in earlier versions of the AWS CLI. To cache that hop on disk in
+``~/.aws/cli/cache``, set ``credential_cache = file`` in that profile.
+
+.. warning::
+
+    With ``credential_cache = memory``, every AWS CLI command makes its own
+    ``AssumeRole`` call for each role in the chain. These calls count against
+    the AWS STS request quota for the account, which is shared with every
+    other workload in the account, so running many commands in parallel can
+    lead to throttling. If the profile sets ``mfa_serial``, you are prompted
+    for an MFA code on every command, and commands that cannot prompt (for
+    example, those run without a terminal) fail.
 
 If you do not have MFA authentication required, then you only need to specify a
 ``role_arn`` and either a ``source_profile`` or a ``credential_source``.
 
 When you specify a profile that has IAM role configuration, the AWS CLI
 will make an ``AssumeRole`` call to retrieve temporary credentials.  These
-credentials are then stored (in ``~/.aws/cli/cache``).  Subsequent AWS CLI
+credentials are then stored (in ``~/.aws/cli/cache``, unless the profile sets
+``credential_cache = memory``).  Subsequent AWS CLI
 commands will use the cached temporary credentials until they expire, in which
 case the AWS CLI will automatically refresh credentials.
 
@@ -349,6 +384,14 @@ assume role with web identity profile in the shared config:
   ``AssumeRoleWithWebIdentity`` operation.  This is an optional parameter. If
   you do not provide this value, a session name will be automatically
   generated.
+* ``credential_cache`` - Where the AWS CLI caches the temporary credentials
+  from the ``AssumeRoleWithWebIdentity`` call: ``file`` (the default) or
+  ``memory``. See ``credential_cache`` under `Using AWS IAM Roles`_, including
+  the warning about AWS STS quotas.
+
+Only the AWS CLI reads ``credential_cache``. The AWS SDKs and other tools
+that read the shared config file ignore it, so a tool that keeps its own file
+cache may still write credentials to disk for a profile set to ``memory``.
 
 Below is an example configuration for the minimal amount of configuration
 needed to configure an assume role with web identity profile::
@@ -373,7 +416,10 @@ This provider can also be configured via the environment:
 
     These environment variables currently only apply to the assume role with
     web identity provider and do not apply to the general assume role provider
-    configuration.
+    configuration. When the provider is configured through these environment
+    variables, the ``credential_cache`` setting of the profile in use still
+    applies. If the profile does not set it, credentials are cached in
+    ``~/.aws/cli/cache``.
 
 
 Sourcing Credentials From External Processes