From 3f02d0998343aed70f7d6f8523aa8937919d7bd4 Mon Sep 17 00:00:00 2001 From: Srinivas Gorur-Shandilya Date: Wed, 20 Nov 2024 20:08:51 -0500 Subject: [PATCH 1/8] feat: started working on platform-api-sdk --- Makefile | 2 +- pyproject.toml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 98fc987..1d6e99c 100644 --- a/Makefile +++ b/Makefile @@ -38,7 +38,7 @@ install: @python3 -m venv venv @source $(CURDIR)/venv/bin/activate && \ pip install --upgrade pip && \ - pip install -e .[lint,test,dev,docs,plots] && \ + pip install --no-cache-dir -e .[lint,test,dev,docs,plots] && \ deactivate @-mkdir -p ~/.deeporigin @test -f ~/.deeporigin/deeporigin || ln -s $(CURDIR)/venv/bin/deeporigin ~/.deeporigin/deeporigin diff --git a/pyproject.toml b/pyproject.toml index 40b772e..b5f6498 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,7 @@ dependencies = [ "pyjwt", "cryptography", "python-box", + "do_sdk_platform @ file:///Users/srinivas/code/generated-sdks/deeporigin-platform-sdk" ] dynamic = ["version"] From a01d7384b8441f42d538bc4467e39a2d2061874d Mon Sep 17 00:00:00 2001 From: Srinivas Gorur-Shandilya Date: Thu, 21 Nov 2024 10:46:47 -0500 Subject: [PATCH 2/8] feat: autogenerated platform SDK --- src/platform/__init__.py | 0 src/platform/users.py | 36 ++++++++++++++ src/platform/utils.py | 102 +++++++++++++++++++++++++++++++++++++++ src/utils/core.py | 2 +- 4 files changed, 139 insertions(+), 1 deletion(-) create mode 100644 src/platform/__init__.py create mode 100644 src/platform/users.py create mode 100644 src/platform/utils.py diff --git a/src/platform/__init__.py b/src/platform/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/platform/users.py b/src/platform/users.py new file mode 100644 index 0000000..f9457d6 --- /dev/null +++ b/src/platform/users.py @@ -0,0 +1,36 @@ +"""bridge between autogenerated code and high level API""" + +import sys + +from deeporigin.platform.utils import ( + _create_function, + _get_api_client, + _get_client_methods, +) + +API_NAME = "UsersApi" + + +this_module_name = sys.modules[__name__] + + +methods = _get_client_methods(_get_api_client(api_name=API_NAME)) + + +for method in methods: + # clean up the name so that it's more readable + sanitized_method_name = method.split("controller")[0].rstrip("_") + + # add this function as an attribute to this module + # so that we can call it + setattr( + this_module_name, + sanitized_method_name, + _create_function( + method_path=method, + api_name=API_NAME, + ), + ) + + +__all__ = list(methods) diff --git a/src/platform/utils.py b/src/platform/utils.py new file mode 100644 index 0000000..533f8ea --- /dev/null +++ b/src/platform/utils.py @@ -0,0 +1,102 @@ +import inspect +from urllib.parse import urljoin + +import do_sdk_platform +from beartype import beartype +from box import Box +from deeporigin.auth import get_tokens +from deeporigin.config import get_value +from deeporigin.utils.core import PersistentDict, _get_method, _redact_responses + + +@beartype +def _get_api_client(*, api_name: str, configure: bool = True): + """return a configured client for the API we want to access + + Args: + api_name (str): name of the API + + Returns: + configured client + """ + + if configure: + configuration = do_sdk_platform.configuration.Configuration( + host=urljoin(get_value()["api_endpoint"], "/api"), + access_token=get_tokens()["access"], + ) + + client = do_sdk_platform.ApiClient(configuration=configuration) + else: + client = do_sdk_platform.ApiClient() + + api_class = getattr(do_sdk_platform, api_name) + client = api_class(api_client=client) + return client + + +@beartype +def _get_client_methods(client) -> set: + """utility function to get methods from the client that return raw responses from the server""" + methods = set( + [ + attr + for attr in dir(client) + if callable(getattr(client, attr)) + and not attr.startswith("_") + and "without_preload_content" in attr + ] + ) + + return methods + + +def _create_function(*, method_path: str, api_name: str): + """utility function the dynamically creates functions + that wrap low-level functions in the DeepOrigin data API""" + + # we're constructing a client solely for the purposes + # of inspecting its methods and extracting + # function signatures. So we don't need any + # authentication + + client = _get_api_client( + configure=False, + api_name=api_name, + ) + + method = _get_method(client, method_path) + + signature = inspect.signature(method) + + def dynamic_function( + *, + client=None, + **kwargs, + ): + if client is None: + client = _get_api_client(api_name=api_name) + method = _get_method(client, method_path) + + # call the low level API method + response = method(**kwargs) + + if not isinstance(response, dict): + response = response.json() + + if "data" in response.keys(): + response = response["data"] + if isinstance(response, list): + response = [Box(item) for item in response] + else: + response = Box(response) + else: + response = Box(response) + + return response + + # attach the signature of the underlying method to the + # function so that IDEs can display it properly + dynamic_function.__signature__ = signature + + return dynamic_function diff --git a/src/utils/core.py b/src/utils/core.py index ca4ff40..da640c8 100644 --- a/src/utils/core.py +++ b/src/utils/core.py @@ -129,7 +129,7 @@ def clear(self): self._save(self._data) -def _get_method(obj, method_path): +def _get_method(obj, method_path: str): # Split the method path into components methods = method_path.split(".") From e54b87b6fa1f7f1f8bcb131aebf53d779a1ecc8e Mon Sep 17 00:00:00 2001 From: Srinivas Gorur-Shandilya Date: Thu, 21 Nov 2024 11:01:51 -0500 Subject: [PATCH 3/8] fix: install platform-api-sdk from pypi --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index b5f6498..c30e683 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,7 @@ dependencies = [ "pyjwt", "cryptography", "python-box", - "do_sdk_platform @ file:///Users/srinivas/code/generated-sdks/deeporigin-platform-sdk" + "do-sdk-platform", ] dynamic = ["version"] From 9066a5fe554e8cfec1cd79b22d36c663703fcddc Mon Sep 17 00:00:00 2001 From: Srinivas Gorur-Shandilya Date: Thu, 21 Nov 2024 11:09:16 -0500 Subject: [PATCH 4/8] feat: added wrappers for more APIs --- pyproject.toml | 2 +- src/platform/organizations.py | 36 +++++++++++++++++++++++++++++++++++ src/platform/tools.py | 36 +++++++++++++++++++++++++++++++++++ src/platform/users.py | 2 +- src/platform/volumes.py | 36 +++++++++++++++++++++++++++++++++++ 5 files changed, 110 insertions(+), 2 deletions(-) create mode 100644 src/platform/organizations.py create mode 100644 src/platform/tools.py create mode 100644 src/platform/volumes.py diff --git a/pyproject.toml b/pyproject.toml index c30e683..56e54b9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,7 @@ dependencies = [ "pyjwt", "cryptography", "python-box", - "do-sdk-platform", + "do-sdk-platform==1.0.0", ] dynamic = ["version"] diff --git a/src/platform/organizations.py b/src/platform/organizations.py new file mode 100644 index 0000000..0e259c7 --- /dev/null +++ b/src/platform/organizations.py @@ -0,0 +1,36 @@ +"""bridge module to interact with the platform organizations api""" + +import sys + +from deeporigin.platform.utils import ( + _create_function, + _get_api_client, + _get_client_methods, +) + +API_NAME = "OrganizationsApi" + + +this_module_name = sys.modules[__name__] + + +methods = _get_client_methods(_get_api_client(api_name=API_NAME)) + + +for method in methods: + # clean up the name so that it's more readable + sanitized_method_name = method.split("controller")[0].rstrip("_") + + # add this function as an attribute to this module + # so that we can call it + setattr( + this_module_name, + sanitized_method_name, + _create_function( + method_path=method, + api_name=API_NAME, + ), + ) + + +__all__ = list(methods) diff --git a/src/platform/tools.py b/src/platform/tools.py new file mode 100644 index 0000000..de46aa4 --- /dev/null +++ b/src/platform/tools.py @@ -0,0 +1,36 @@ +"""bridge module to interact with the platform tools api""" + +import sys + +from deeporigin.platform.utils import ( + _create_function, + _get_api_client, + _get_client_methods, +) + +API_NAME = "ToolsApi" + + +this_module_name = sys.modules[__name__] + + +methods = _get_client_methods(_get_api_client(api_name=API_NAME)) + + +for method in methods: + # clean up the name so that it's more readable + sanitized_method_name = method.split("controller")[0].rstrip("_") + + # add this function as an attribute to this module + # so that we can call it + setattr( + this_module_name, + sanitized_method_name, + _create_function( + method_path=method, + api_name=API_NAME, + ), + ) + + +__all__ = list(methods) diff --git a/src/platform/users.py b/src/platform/users.py index f9457d6..8733bdf 100644 --- a/src/platform/users.py +++ b/src/platform/users.py @@ -1,4 +1,4 @@ -"""bridge between autogenerated code and high level API""" +"""bridge module to interact with the platform users api""" import sys diff --git a/src/platform/volumes.py b/src/platform/volumes.py new file mode 100644 index 0000000..2fe18af --- /dev/null +++ b/src/platform/volumes.py @@ -0,0 +1,36 @@ +"""bridge module to interact with the platform tools api""" + +import sys + +from deeporigin.platform.utils import ( + _create_function, + _get_api_client, + _get_client_methods, +) + +API_NAME = "VolumesApi" + + +this_module_name = sys.modules[__name__] + + +methods = _get_client_methods(_get_api_client(api_name=API_NAME)) + + +for method in methods: + # clean up the name so that it's more readable + sanitized_method_name = method.split("controller")[0].rstrip("_") + + # add this function as an attribute to this module + # so that we can call it + setattr( + this_module_name, + sanitized_method_name, + _create_function( + method_path=method, + api_name=API_NAME, + ), + ) + + +__all__ = list(methods) From f40d372f3db57111fb8cebf7b4dd4a55e7c845f5 Mon Sep 17 00:00:00 2001 From: Srinivas Gorur-Shandilya Date: Thu, 21 Nov 2024 11:23:57 -0500 Subject: [PATCH 5/8] feat: added workstations API --- src/platform/utils.py | 6 +++++- src/platform/workstations.py | 36 ++++++++++++++++++++++++++++++++++++ tests/utils.py | 2 ++ 3 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 src/platform/workstations.py diff --git a/src/platform/utils.py b/src/platform/utils.py index 533f8ea..4ad3875 100644 --- a/src/platform/utils.py +++ b/src/platform/utils.py @@ -1,3 +1,5 @@ +"""module to automatically wrap methods in autogenerated low-level code and re-expose them as high-level functions""" + import inspect from urllib.parse import urljoin @@ -6,7 +8,7 @@ from box import Box from deeporigin.auth import get_tokens from deeporigin.config import get_value -from deeporigin.utils.core import PersistentDict, _get_method, _redact_responses +from deeporigin.utils.core import _get_method @beartype @@ -74,6 +76,8 @@ def dynamic_function( client=None, **kwargs, ): + """dynamic function that wraps low-level functions in the DeepOrigin platform API""" + if client is None: client = _get_api_client(api_name=api_name) method = _get_method(client, method_path) diff --git a/src/platform/workstations.py b/src/platform/workstations.py new file mode 100644 index 0000000..60bef6b --- /dev/null +++ b/src/platform/workstations.py @@ -0,0 +1,36 @@ +"""bridge module to interact with the platform tools api""" + +import sys + +from deeporigin.platform.utils import ( + _create_function, + _get_api_client, + _get_client_methods, +) + +API_NAME = "ComputebenchesApi" + + +this_module_name = sys.modules[__name__] + + +methods = _get_client_methods(_get_api_client(api_name=API_NAME)) + + +for method in methods: + # clean up the name so that it's more readable + sanitized_method_name = method.split("controller")[0].rstrip("_") + + # add this function as an attribute to this module + # so that we can call it + setattr( + this_module_name, + sanitized_method_name, + _create_function( + method_path=method, + api_name=API_NAME, + ), + ) + + +__all__ = list(methods) diff --git a/tests/utils.py b/tests/utils.py index ca46e7d..17d22d1 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,3 +1,5 @@ +"""helper module to set up tests""" + import io from contextlib import redirect_stderr, redirect_stdout From 67178150d2f2ab8ac5a805efca9683afe7562cfe Mon Sep 17 00:00:00 2001 From: Srinivas Gorur-Shandilya Date: Thu, 21 Nov 2024 13:57:39 -0500 Subject: [PATCH 6/8] fix: refactored and eliminated a lot of code --- src/data_hub/_api.py | 4 +++- src/platform/organizations.py | 31 +++++-------------------------- src/platform/tools.py | 31 +++++-------------------------- src/platform/users.py | 31 +++++-------------------------- src/platform/utils.py | 28 ++++++++++++++++++++++++++++ src/platform/volumes.py | 31 +++++-------------------------- src/platform/workstations.py | 31 +++++-------------------------- 7 files changed, 56 insertions(+), 131 deletions(-) diff --git a/src/data_hub/_api.py b/src/data_hub/_api.py index 3545e61..b34cdf1 100644 --- a/src/data_hub/_api.py +++ b/src/data_hub/_api.py @@ -9,6 +9,7 @@ import sys from pathlib import Path +from beartype import beartype from box import Box from deeporigin import auth from deeporigin.exceptions import DeepOriginException @@ -47,7 +48,8 @@ } -def _get_client_methods(): +@beartype +def _get_client_methods() -> set: # the only reason we're creating this client is to # extract methods from it. So no need to # authenticate diff --git a/src/platform/organizations.py b/src/platform/organizations.py index 0e259c7..b3ab1d5 100644 --- a/src/platform/organizations.py +++ b/src/platform/organizations.py @@ -2,35 +2,14 @@ import sys -from deeporigin.platform.utils import ( - _create_function, - _get_api_client, - _get_client_methods, -) +from deeporigin.platform.utils import add_functions_to_module API_NAME = "OrganizationsApi" -this_module_name = sys.modules[__name__] - - -methods = _get_client_methods(_get_api_client(api_name=API_NAME)) - - -for method in methods: - # clean up the name so that it's more readable - sanitized_method_name = method.split("controller")[0].rstrip("_") - - # add this function as an attribute to this module - # so that we can call it - setattr( - this_module_name, - sanitized_method_name, - _create_function( - method_path=method, - api_name=API_NAME, - ), - ) - +methods = add_functions_to_module( + module=sys.modules[__name__], + api_name=API_NAME, +) __all__ = list(methods) diff --git a/src/platform/tools.py b/src/platform/tools.py index de46aa4..835e6bd 100644 --- a/src/platform/tools.py +++ b/src/platform/tools.py @@ -2,35 +2,14 @@ import sys -from deeporigin.platform.utils import ( - _create_function, - _get_api_client, - _get_client_methods, -) +from deeporigin.platform.utils import add_functions_to_module API_NAME = "ToolsApi" -this_module_name = sys.modules[__name__] - - -methods = _get_client_methods(_get_api_client(api_name=API_NAME)) - - -for method in methods: - # clean up the name so that it's more readable - sanitized_method_name = method.split("controller")[0].rstrip("_") - - # add this function as an attribute to this module - # so that we can call it - setattr( - this_module_name, - sanitized_method_name, - _create_function( - method_path=method, - api_name=API_NAME, - ), - ) - +methods = add_functions_to_module( + module=sys.modules[__name__], + api_name=API_NAME, +) __all__ = list(methods) diff --git a/src/platform/users.py b/src/platform/users.py index 8733bdf..6bbc5bb 100644 --- a/src/platform/users.py +++ b/src/platform/users.py @@ -2,35 +2,14 @@ import sys -from deeporigin.platform.utils import ( - _create_function, - _get_api_client, - _get_client_methods, -) +from deeporigin.platform.utils import add_functions_to_module API_NAME = "UsersApi" -this_module_name = sys.modules[__name__] - - -methods = _get_client_methods(_get_api_client(api_name=API_NAME)) - - -for method in methods: - # clean up the name so that it's more readable - sanitized_method_name = method.split("controller")[0].rstrip("_") - - # add this function as an attribute to this module - # so that we can call it - setattr( - this_module_name, - sanitized_method_name, - _create_function( - method_path=method, - api_name=API_NAME, - ), - ) - +methods = add_functions_to_module( + module=sys.modules[__name__], + api_name=API_NAME, +) __all__ = list(methods) diff --git a/src/platform/utils.py b/src/platform/utils.py index 4ad3875..2ca6545 100644 --- a/src/platform/utils.py +++ b/src/platform/utils.py @@ -11,6 +11,34 @@ from deeporigin.utils.core import _get_method +def add_functions_to_module( + module: str, + api_name: str, +) -> set: + methods = _get_client_methods( + _get_api_client( + api_name=api_name, + ) + ) + + for method in methods: + # clean up the name so that it's more readable + sanitized_method_name = method.split("controller")[0].rstrip("_") + + # add this function as an attribute to this module + # so that we can call it + setattr( + module, + sanitized_method_name, + _create_function( + method_path=method, + api_name=api_name, + ), + ) + + return methods + + @beartype def _get_api_client(*, api_name: str, configure: bool = True): """return a configured client for the API we want to access diff --git a/src/platform/volumes.py b/src/platform/volumes.py index 2fe18af..b246f13 100644 --- a/src/platform/volumes.py +++ b/src/platform/volumes.py @@ -2,35 +2,14 @@ import sys -from deeporigin.platform.utils import ( - _create_function, - _get_api_client, - _get_client_methods, -) +from deeporigin.platform.utils import add_functions_to_module API_NAME = "VolumesApi" -this_module_name = sys.modules[__name__] - - -methods = _get_client_methods(_get_api_client(api_name=API_NAME)) - - -for method in methods: - # clean up the name so that it's more readable - sanitized_method_name = method.split("controller")[0].rstrip("_") - - # add this function as an attribute to this module - # so that we can call it - setattr( - this_module_name, - sanitized_method_name, - _create_function( - method_path=method, - api_name=API_NAME, - ), - ) - +methods = add_functions_to_module( + module=sys.modules[__name__], + api_name=API_NAME, +) __all__ = list(methods) diff --git a/src/platform/workstations.py b/src/platform/workstations.py index 60bef6b..249bd85 100644 --- a/src/platform/workstations.py +++ b/src/platform/workstations.py @@ -2,35 +2,14 @@ import sys -from deeporigin.platform.utils import ( - _create_function, - _get_api_client, - _get_client_methods, -) +from deeporigin.platform.utils import add_functions_to_module API_NAME = "ComputebenchesApi" -this_module_name = sys.modules[__name__] - - -methods = _get_client_methods(_get_api_client(api_name=API_NAME)) - - -for method in methods: - # clean up the name so that it's more readable - sanitized_method_name = method.split("controller")[0].rstrip("_") - - # add this function as an attribute to this module - # so that we can call it - setattr( - this_module_name, - sanitized_method_name, - _create_function( - method_path=method, - api_name=API_NAME, - ), - ) - +methods = add_functions_to_module( + module=sys.modules[__name__], + api_name=API_NAME, +) __all__ = list(methods) From 5213dbd0c461c8f3d952fa7e1ca5e3e14f92c4f7 Mon Sep 17 00:00:00 2001 From: Srinivas Gorur-Shandilya Date: Thu, 21 Nov 2024 14:00:02 -0500 Subject: [PATCH 7/8] fix: removed another loc --- src/platform/organizations.py | 5 +---- src/platform/tools.py | 5 +---- src/platform/users.py | 5 +---- src/platform/volumes.py | 5 +---- src/platform/workstations.py | 5 +---- 5 files changed, 5 insertions(+), 20 deletions(-) diff --git a/src/platform/organizations.py b/src/platform/organizations.py index b3ab1d5..ba0ad2d 100644 --- a/src/platform/organizations.py +++ b/src/platform/organizations.py @@ -4,12 +4,9 @@ from deeporigin.platform.utils import add_functions_to_module -API_NAME = "OrganizationsApi" - - methods = add_functions_to_module( module=sys.modules[__name__], - api_name=API_NAME, + api_name="OrganizationsApi", ) __all__ = list(methods) diff --git a/src/platform/tools.py b/src/platform/tools.py index 835e6bd..d0508e0 100644 --- a/src/platform/tools.py +++ b/src/platform/tools.py @@ -4,12 +4,9 @@ from deeporigin.platform.utils import add_functions_to_module -API_NAME = "ToolsApi" - - methods = add_functions_to_module( module=sys.modules[__name__], - api_name=API_NAME, + api_name="ToolsApi", ) __all__ = list(methods) diff --git a/src/platform/users.py b/src/platform/users.py index 6bbc5bb..9459693 100644 --- a/src/platform/users.py +++ b/src/platform/users.py @@ -4,12 +4,9 @@ from deeporigin.platform.utils import add_functions_to_module -API_NAME = "UsersApi" - - methods = add_functions_to_module( module=sys.modules[__name__], - api_name=API_NAME, + api_name="UsersApi", ) __all__ = list(methods) diff --git a/src/platform/volumes.py b/src/platform/volumes.py index b246f13..a4a0bba 100644 --- a/src/platform/volumes.py +++ b/src/platform/volumes.py @@ -4,12 +4,9 @@ from deeporigin.platform.utils import add_functions_to_module -API_NAME = "VolumesApi" - - methods = add_functions_to_module( module=sys.modules[__name__], - api_name=API_NAME, + api_name="VolumesApi", ) __all__ = list(methods) diff --git a/src/platform/workstations.py b/src/platform/workstations.py index 249bd85..b87de61 100644 --- a/src/platform/workstations.py +++ b/src/platform/workstations.py @@ -4,12 +4,9 @@ from deeporigin.platform.utils import add_functions_to_module -API_NAME = "ComputebenchesApi" - - methods = add_functions_to_module( module=sys.modules[__name__], - api_name=API_NAME, + api_name="ComputebenchesApi", ) __all__ = list(methods) From 917356e02dcc2cd54b30925d058330045c8602d4 Mon Sep 17 00:00:00 2001 From: Srinivas Gorur-Shandilya Date: Thu, 21 Nov 2024 14:02:26 -0500 Subject: [PATCH 8/8] fix: added docstring --- src/platform/utils.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/platform/utils.py b/src/platform/utils.py index 2ca6545..243598f 100644 --- a/src/platform/utils.py +++ b/src/platform/utils.py @@ -11,10 +11,22 @@ from deeporigin.utils.core import _get_method +@beartype def add_functions_to_module( module: str, api_name: str, ) -> set: + """utility function to dynamically add functions to a module + + This function works by calling setattr on the module. + + Args: + module (str): name of the module + api_name (str): name of the API + + Returns: + set of methods that were added + """ methods = _get_client_methods( _get_api_client( api_name=api_name,