From 0bc6b41f2edfdd283b081897690e037d2f129104 Mon Sep 17 00:00:00 2001 From: tdviet Date: Wed, 24 May 2023 21:20:29 +0200 Subject: [PATCH 1/5] Introducing locker as temporary secret storage --- fedcloudclient/decorators.py | 76 ++++++++++++++++++ fedcloudclient/secret.py | 150 ++++++++++++++++++++++++++++++----- 2 files changed, 205 insertions(+), 21 deletions(-) diff --git a/fedcloudclient/decorators.py b/fedcloudclient/decorators.py index 54416f8..bfb7d94 100644 --- a/fedcloudclient/decorators.py +++ b/fedcloudclient/decorators.py @@ -34,6 +34,14 @@ metavar="site-name", ) +# Output format for secret module +secret_output_params = click.option( + "--output-format", + "-f", + required=False, + help="Output format", + type=click.Choice(["text", "YAML", "JSON"], case_sensitive=False), +) def all_site_params(func): """ @@ -332,3 +340,71 @@ def wrapper(*args, **kwargs): return func(*args, **kwargs) return wrapper + + +def secret_token_params(func): + """ + Decorator for secret token. + If locker token is not defined, get access token from oidc-* parameters + and replace them in the wrapper function + """ + + @optgroup.group("Token options", help="Choose one of options for providing token") + @optgroup.option( + "--locker-token", + help="Locker token", + envvar="FEDCLOUD_LOCKER_TOKEN", + metavar="locker_token", + ) + @optgroup.option( + "--oidc-agent-account", + help="Account name in oidc-agent", + envvar="OIDC_AGENT_ACCOUNT", + metavar="account", + ) + @optgroup.option( + "--oidc-access-token", + help="OIDC access token", + envvar="OIDC_ACCESS_TOKEN", + metavar="token", + ) + @optgroup.option( + "--mytoken", + help="Mytoken string", + envvar="FEDCLOUD_MYTOKEN", + metavar="mytoken", + ) + @optgroup.option( + "--mytoken-server", + help="Mytoken sever", + envvar="FEDCLOUD_MYTOKEN_SERVER", + default=DEFAULT_MYTOKEN_SERVER, + show_default=True, + metavar="mytoken-server", + ) + @wraps(func) + def wrapper(*args, **kwargs): + from fedcloudclient.checkin import get_access_token + + # If locker token is given, ignore OIDC token options + locker_token = kwargs.pop("locker_token") + if locker_token: + kwargs.pop("oidc_access_token") + kwargs.pop("oidc_agent_account") + kwargs.pop("mytoken") + kwargs.pop("mytoken_server") + kwargs["access_token"] = None + kwargs["locker_token"] = locker_token + return func(*args, **kwargs) + + access_token = get_access_token( + kwargs.pop("oidc_access_token"), + kwargs.pop("oidc_agent_account"), + kwargs.pop("mytoken"), + kwargs.pop("mytoken_server"), + ) + kwargs["access_token"] = access_token + kwargs["locker_token"] = None + return func(*args, **kwargs) + + return wrapper diff --git a/fedcloudclient/secret.py b/fedcloudclient/secret.py index 6ff61ea..04c2b08 100644 --- a/fedcloudclient/secret.py +++ b/fedcloudclient/secret.py @@ -8,6 +8,7 @@ import click import hvac +import requests import yaml from cryptography.fernet import Fernet, InvalidToken @@ -18,12 +19,13 @@ from yaml import YAMLError from fedcloudclient.checkin import get_checkin_id -from fedcloudclient.decorators import oidc_params +from fedcloudclient.decorators import oidc_params, secret_output_params, secret_token_params VAULT_ADDR = "https://vault.services.fedcloud.eu:8200" VAULT_ROLE = "demo" -VAULT_MOUNT_POINT = "/secrets" +VAULT_MOUNT_POINT = "/secrets/" VAULT_SALT = "fedcloud_salt" +VAULT_LOCKER_MOUNT_POINT = "/v1/cubbyhole/" def secret_client(access_token, command, path, data): @@ -64,6 +66,41 @@ def secret_client(access_token, command, path, data): ) +def locker_client(locker_token, command, path, data): + """ + Client function for accessing secrets + :param path: path to secret + :param command: the command to perform + :param data: input data + :param locker_token: locker token + :return: Output data from the service + """ + + try: + headers = {"X-Vault-Token": locker_token} + url = VAULT_ADDR + VAULT_LOCKER_MOUNT_POINT + path + if command == "list_secrets": + response = requests.get(url, headers=headers, params={"list": "true"}) + elif command == "read_secret": + response = requests.get(url, headers=headers) + elif command == "delete_secret": + response = requests.delete(url, headers=headers) + elif command == "put": + response = requests.post(url, headers=headers, data=data) + else: + raise SystemExit(f"Invalid command {command}") + response.raise_for_status() + if command in ["list_secrets", "read_secret"]: + response_json = response.json() + return dict(response_json) + else: + return None + except requests.exceptions.HTTPError as exception: + raise SystemExit( + f"Error: Error when accessing secrets on server. Server response: {type(exception).__name__}: {exception}" + ) + + def read_data_from_file(input_format, input_file): """ Read data from file. Format may be text, yaml, json or auto-detect according to file extension @@ -243,14 +280,8 @@ def secret(): @secret.command() -@oidc_params -@click.option( - "--output-format", - "-f", - required=False, - help="Output format", - type=click.Choice(["text", "YAML", "JSON"], case_sensitive=False), -) +@secret_token_params +@secret_output_params @click.argument("short_path", metavar="[secret path]") @click.argument("key", metavar="[key]", required=False) @click.option( @@ -276,6 +307,7 @@ def secret(): ) def get( access_token, + locker_token, short_path, key, output_format, @@ -286,8 +318,10 @@ def get( """ Get the secret object in the path. If a key is given, print only the value of the key """ - - response = secret_client(access_token, "read_secret", short_path, None) + if locker_token: + response = locker_client(locker_token, "read_secret", short_path, None) + else: + response = secret_client(access_token, "read_secret", short_path, None) if decrypt_key: decrypt_data(decrypt_key, response["data"]) if not key: @@ -300,40 +334,42 @@ def get( @secret.command("list") -@oidc_params +@secret_token_params @click.argument("short_path", metavar="[secret path]", required=False, default="") def list_( access_token, + locker_token, short_path, ): """ List secret objects in the path """ - - response = secret_client(access_token, "list_secrets", short_path, None) + if locker_token: + response = locker_client(locker_token, "list_secrets", short_path, None) + else: + response = secret_client(access_token, "list_secrets", short_path, None) print("\n".join(map(str, response["data"]["keys"]))) @secret.command() -@oidc_params +@secret_token_params @click.argument("short_path", metavar="[secret path]") @click.argument("secrets", nargs=-1, metavar="[key=value...]") @click.option( "--encrypt-key", "-e", metavar="[key]", - required=False, help="Encryption key or passphrase", ) @click.option( "--binary-file", "-b", - required=False, is_flag=True, help="True for reading secrets from binary files", ) def put( access_token, + locker_token, short_path, secrets, encrypt_key, @@ -346,18 +382,90 @@ def put( secret_dict = secret_params_to_dict(secrets, binary_file) if encrypt_key: encrypt_data(encrypt_key, secret_dict) - secret_client(access_token, "put", short_path, secret_dict) + if locker_token: + locker_client(locker_token, "put", short_path, secret_dict) + else: + secret_client(access_token, "put", short_path, secret_dict) + @secret.command() -@oidc_params +@secret_token_params @click.argument("short_path", metavar="[secret path]") def delete( access_token, + locker_token, short_path, ): """ Delete the secret object in the path """ + if locker_token: + locker_client(locker_token, "delete_secret", short_path, None) + else: + secret_client(access_token, "delete_secret", short_path, None) - secret_client(access_token, "delete_secret", short_path, None) +@secret.group() +def locker(): + """ + Commands for creating and accessing locker objects + """ + + +@locker.command() +@oidc_params +@secret_output_params +@click.option("--ttl", default="24h", help="Time-to-live for the new locker") +@click.option("--num_uses", default=0, help="Max number of uses") +@click.option("--token-only", is_flag=True, help="True for print token only") +def create(access_token, ttl, num_uses, output_format, token_only): + """ + Create a locker and return the locker token + """ + try: + client = hvac.Client(url=VAULT_ADDR) + client.auth.jwt.jwt_login(role=VAULT_ROLE, jwt=access_token) + client.auth.token.renew_self(increment=ttl) + locker_token = client.auth.token.create(policies=["default"], ttl=ttl, num_uses=num_uses) + if token_only: + print(locker_token["auth"]["client_token"]) + else: + print_secrets(None, output_format, locker_token["auth"]) + except VaultError as e: + raise SystemExit( + f"Error: Error when accessing secrets on server. Server response: {type(e).__name__}: {e}" + ) + +@locker.command() +@secret_output_params +@click.argument("locker_token", metavar="[locker_token]", envvar="FEDCLOUD_LOCKER_TOKEN") +def check(locker_token, output_format): + """ + Check status of locker token + """ + + try: + client = hvac.Client(url=VAULT_ADDR) + client.token = locker_token + locker_info = client.auth.token.lookup_self() + print_secrets(None, output_format, locker_info["data"]) + except VaultError as e: + raise SystemExit( + f"Error: Error when accessing secrets on server. Server response: {type(e).__name__}: {e}" + ) + + +@locker.command() +@click.argument("locker_token", metavar="[locker_token]", envvar="FEDCLOUD_LOCKER_TOKEN") +def delete(locker_token): + """ + Delete the locker token + """ + try: + client = hvac.Client(url=VAULT_ADDR) + client.token = locker_token + client.auth.token.revoke_self() + except VaultError as e: + raise SystemExit( + f"Error: Error when accessing secrets on server. Server response: {type(e).__name__}: {e}" + ) \ No newline at end of file From a90818fb7c98b4cde7aae22dc6c8073be2d4223a Mon Sep 17 00:00:00 2001 From: tdviet Date: Wed, 24 May 2023 21:41:36 +0200 Subject: [PATCH 2/5] reformat --- fedcloudclient/decorators.py | 1 + fedcloudclient/secret.py | 28 ++++++++++++++++++++-------- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/fedcloudclient/decorators.py b/fedcloudclient/decorators.py index bfb7d94..832464c 100644 --- a/fedcloudclient/decorators.py +++ b/fedcloudclient/decorators.py @@ -43,6 +43,7 @@ type=click.Choice(["text", "YAML", "JSON"], case_sensitive=False), ) + def all_site_params(func): """ Decorator for all-sites options diff --git a/fedcloudclient/secret.py b/fedcloudclient/secret.py index 04c2b08..664e140 100644 --- a/fedcloudclient/secret.py +++ b/fedcloudclient/secret.py @@ -10,7 +10,6 @@ import hvac import requests import yaml - from cryptography.fernet import Fernet, InvalidToken from cryptography.hazmat.primitives import hashes from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC @@ -19,7 +18,11 @@ from yaml import YAMLError from fedcloudclient.checkin import get_checkin_id -from fedcloudclient.decorators import oidc_params, secret_output_params, secret_token_params +from fedcloudclient.decorators import ( + oidc_params, + secret_output_params, + secret_token_params, +) VAULT_ADDR = "https://vault.services.fedcloud.eu:8200" VAULT_ROLE = "demo" @@ -388,7 +391,6 @@ def put( secret_client(access_token, "put", short_path, secret_dict) - @secret.command() @secret_token_params @click.argument("short_path", metavar="[secret path]") @@ -405,6 +407,7 @@ def delete( else: secret_client(access_token, "delete_secret", short_path, None) + @secret.group() def locker(): """ @@ -426,7 +429,11 @@ def create(access_token, ttl, num_uses, output_format, token_only): client = hvac.Client(url=VAULT_ADDR) client.auth.jwt.jwt_login(role=VAULT_ROLE, jwt=access_token) client.auth.token.renew_self(increment=ttl) - locker_token = client.auth.token.create(policies=["default"], ttl=ttl, num_uses=num_uses) + locker_token = client.auth.token.create( + policies=["default"], + ttl=ttl, + num_uses=num_uses + ) if token_only: print(locker_token["auth"]["client_token"]) else: @@ -436,9 +443,14 @@ def create(access_token, ttl, num_uses, output_format, token_only): f"Error: Error when accessing secrets on server. Server response: {type(e).__name__}: {e}" ) + @locker.command() @secret_output_params -@click.argument("locker_token", metavar="[locker_token]", envvar="FEDCLOUD_LOCKER_TOKEN") +@click.argument( + "locker_token", + metavar="[locker_token]", + envvar="FEDCLOUD_LOCKER_TOKEN" +) def check(locker_token, output_format): """ Check status of locker token @@ -457,9 +469,9 @@ def check(locker_token, output_format): @locker.command() @click.argument("locker_token", metavar="[locker_token]", envvar="FEDCLOUD_LOCKER_TOKEN") -def delete(locker_token): +def revoke(locker_token): """ - Delete the locker token + Revoke the locker token and delete all data in the locker """ try: client = hvac.Client(url=VAULT_ADDR) @@ -468,4 +480,4 @@ def delete(locker_token): except VaultError as e: raise SystemExit( f"Error: Error when accessing secrets on server. Server response: {type(e).__name__}: {e}" - ) \ No newline at end of file + ) From e0598c7c87b49030d5358b6559f7dd57bafaa961 Mon Sep 17 00:00:00 2001 From: tdviet Date: Wed, 24 May 2023 21:49:37 +0200 Subject: [PATCH 3/5] reformat --- fedcloudclient/secret.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/fedcloudclient/secret.py b/fedcloudclient/secret.py index 664e140..4877af4 100644 --- a/fedcloudclient/secret.py +++ b/fedcloudclient/secret.py @@ -430,9 +430,7 @@ def create(access_token, ttl, num_uses, output_format, token_only): client.auth.jwt.jwt_login(role=VAULT_ROLE, jwt=access_token) client.auth.token.renew_self(increment=ttl) locker_token = client.auth.token.create( - policies=["default"], - ttl=ttl, - num_uses=num_uses + policies=["default"], ttl=ttl, num_uses=num_uses ) if token_only: print(locker_token["auth"]["client_token"]) @@ -447,9 +445,7 @@ def create(access_token, ttl, num_uses, output_format, token_only): @locker.command() @secret_output_params @click.argument( - "locker_token", - metavar="[locker_token]", - envvar="FEDCLOUD_LOCKER_TOKEN" + "locker_token", metavar="[locker_token]", envvar="FEDCLOUD_LOCKER_TOKEN" ) def check(locker_token, output_format): """ @@ -468,7 +464,9 @@ def check(locker_token, output_format): @locker.command() -@click.argument("locker_token", metavar="[locker_token]", envvar="FEDCLOUD_LOCKER_TOKEN") +@click.argument( + "locker_token", metavar="[locker_token]", envvar="FEDCLOUD_LOCKER_TOKEN" +) def revoke(locker_token): """ Revoke the locker token and delete all data in the locker From dc5a0804c1393038e28c97d7dcdefa59c2c585db Mon Sep 17 00:00:00 2001 From: tdviet Date: Thu, 25 May 2023 11:37:43 +0200 Subject: [PATCH 4/5] Fix typo --- fedcloudclient/secret.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fedcloudclient/secret.py b/fedcloudclient/secret.py index 4877af4..0ab25bc 100644 --- a/fedcloudclient/secret.py +++ b/fedcloudclient/secret.py @@ -419,7 +419,7 @@ def locker(): @oidc_params @secret_output_params @click.option("--ttl", default="24h", help="Time-to-live for the new locker") -@click.option("--num_uses", default=0, help="Max number of uses") +@click.option("--num-uses", default=0, help="Max number of uses") @click.option("--token-only", is_flag=True, help="True for print token only") def create(access_token, ttl, num_uses, output_format, token_only): """ From abc3294bd3f2e3eb23f18a2a2e221e111cea80e4 Mon Sep 17 00:00:00 2001 From: tdviet Date: Sun, 28 May 2023 10:37:13 +0200 Subject: [PATCH 5/5] - Change option to print locker token - Set default number uses of locker to 10 --- fedcloudclient/secret.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/fedcloudclient/secret.py b/fedcloudclient/secret.py index 0ab25bc..4fbe375 100644 --- a/fedcloudclient/secret.py +++ b/fedcloudclient/secret.py @@ -419,9 +419,9 @@ def locker(): @oidc_params @secret_output_params @click.option("--ttl", default="24h", help="Time-to-live for the new locker") -@click.option("--num-uses", default=0, help="Max number of uses") -@click.option("--token-only", is_flag=True, help="True for print token only") -def create(access_token, ttl, num_uses, output_format, token_only): +@click.option("--num-uses", default=10, help="Max number of uses") +@click.option("--verbose", is_flag=True, help="Print token details") +def create(access_token, ttl, num_uses, output_format, verbose): """ Create a locker and return the locker token """ @@ -430,9 +430,9 @@ def create(access_token, ttl, num_uses, output_format, token_only): client.auth.jwt.jwt_login(role=VAULT_ROLE, jwt=access_token) client.auth.token.renew_self(increment=ttl) locker_token = client.auth.token.create( - policies=["default"], ttl=ttl, num_uses=num_uses + policies=["default"], ttl=ttl, num_uses=num_uses, renewable=False ) - if token_only: + if not verbose: print(locker_token["auth"]["client_token"]) else: print_secrets(None, output_format, locker_token["auth"])