diff --git a/README.md b/README.md index 1792301..2cb03fc 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ APRICOT has been constructed using the following components: - [**Jupyter**](https://jupyter.org/), an open-source web application that allows you to create and share documents that contain live code, equations, visualizations and narrative text. - [**IM**](https://www.grycap.upv.es/im/index.php), an open-source virtual infrastructure provisioning tool for multi-Clouds. -### Infrastructure management +## Infrastructure management To manage and use previous deployed infrastructures within Jupyter notebook environment, a set of Ipython magic functions have been implemented. These functions are listed below: @@ -58,7 +58,7 @@ To manage and use previous deployed infrastructures within Jupyter notebook envi Like any Jupyter magics, these must be lodaded at the notebook using _%reload_ext apricot_magic_ or configure jupyter to load these magics in all notebooks. -### Docker +## Docker A Dockerfile has been provided to construct a docker image with Jupyter and APRICOT configured. Use @@ -70,7 +70,7 @@ to build the image. Then, use to create and execute a container. The container will start automatically a Jupyter server with APRICOT preconfigured. Then, use the url provided by Jupyter to access to the server. -### Development install +## Development install Note: You will need NodeJS to build the extension package. @@ -106,7 +106,7 @@ By default, the `jlpm build` command generates the source maps for this extensio jupyter lab build --minimize=False ``` -### Development uninstall +## Development uninstall ```bash pip uninstall apricot @@ -116,6 +116,6 @@ In development mode, you will also need to remove the symlink created by `jupyte command. To find its location, you can run `jupyter labextension list` to figure out where the `labextensions` folder is located. Then you can remove the symlink named `apricot` within that folder. -### Packaging the extension +## Packaging the extension See [RELEASE](RELEASE.md) diff --git a/src/apricot-tutorial.ipynb b/apricot-tutorial.ipynb similarity index 94% rename from src/apricot-tutorial.ipynb rename to apricot-tutorial.ipynb index 74c802d..8633f4f 100644 --- a/src/apricot-tutorial.ipynb +++ b/apricot-tutorial.ipynb @@ -54,11 +54,11 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 11, "metadata": {}, "outputs": [], "source": [ - "infrastructure_id = \"aa624830-4b20-11ef-a01f-1abcb0121196\"\n", + "infrastructure_id = \"dc678ce2-7eff-11ef-9af1-968d5abd162c\"\n", "vm_id = \"0\"" ] }, @@ -156,7 +156,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Congratulations! You finished the tutorial and now you are free to keep exploring and exploiting the APRICOT extension!" + "Congratulations! You finished the tutorial and now you are free to keep exploring and exploiting the APRICOT extension!\n", + "\n", + "All the magics implemented are available in README.md." ] } ], diff --git a/resources/deployed-template.yaml b/resources/deployed-template.yaml index fb6ad20..e69de29 100644 --- a/resources/deployed-template.yaml +++ b/resources/deployed-template.yaml @@ -1,217 +0,0 @@ -tosca_definitions_version: tosca_simple_yaml_1_0 -imports: - - grycap_custom_types: https://raw.githubusercontent.com/grycap/tosca/main/custom_types.yaml -description: > - Deploy a compute node getting the IP and SSH credentials to access via ssh - with an extra HD disk. -metadata: - template_name: VM - template_version: 1.1.0 - template_author: Miguel Caballer - creation_date: 2020-09-08T00:00:00.000Z - display_name: Deploy a VM - icon: images/vm-icon-disk.png - tag: VM - order: 1 - tabs: - VM Data: - - num_cpus - - mem_size - - disk_size - - num_instances - - storage_size - - mount_path - - ports - GPU Data: .*gpu.* - childs: - - docker.yaml - - galaxy.yaml - - users.yml - - ssh_keys.yml - - proxy_host.yaml - - minio_compose.yaml - - nvidia.yml - - ansible_tasks.yml - - ssh_oidc.yaml - - noderedvm.yaml - - cernvmfs.yaml - - image-service.yaml - - ai4eoscvm.yaml - - mlflowvm.yaml - - wget.yml - - sgde.yaml - - dydns_egi_update_vm.yml - - flowfuse.yaml - infra_name: jupyter_c9a1f2bba6932c40e79b0e7498319e1ccb714786741c49522630b4fb52026f72 -topology_template: - inputs: - num_cpus: - type: integer - description: Number of virtual cpus for the VM - default: 1 - constraints: - - valid_values: - - 1 - - 2 - - 4 - - 8 - - 16 - - 32 - - 64 - mem_size: - type: scalar-unit.size - description: Amount of memory for the VM - default: 2 GB - constraints: - - valid_values: - - 2 GB - - 4 GB - - 8 GB - - 16 GB - - 32 GB - - 64 GB - - 128 GB - - 256 GB - - 512 GB - disk_size: - type: scalar-unit.size - description: Size of the root disk of the WNs - default: 20 GB - constraints: - - valid_values: - - 20 GB - - 50 GB - - 100 GB - - 200 GB - num_instances: - type: integer - description: Number of VMs to be spawned - default: 1 - storage_size: - type: scalar-unit.size - description: Size of the extra HD added to the instance (Set 0 if disk is not needed) - default: 0 GB - constraints: - - valid_values: - - 0 GB - - 10 GB - - 20 GB - - 50 GB - - 100 GB - - 200 GB - - 500 GB - - 1 TB - - 2 TB - - 10 TB - - 20 TB - - 40 TB - - 100 TB - mount_path: - type: string - description: Path to mount the extra disk - default: /mnt/disk - num_gpus: - type: integer - description: Number of GPUs to assing to this VM - default: 1 - constraints: - - valid_values: - - 0 - - 1 - - 2 - - 3 - - 4 - gpu_vendor: - type: string - description: GPU Vendor - default: - constraints: - - valid_values: - - - - NVIDIA - - AMD - gpu_model: - type: string - description: GPU Model - default: - ports: - type: map - entry_schema: - type: PortSpec - description: | - List of ports to be Opened in the Cloud site (eg. 22,80,443,2000:2100). - You can also include the remote CIDR (eg. 8.8.0.0/24). - default: - ssh_port: - source: 22 - protocol: tcp - image: - type: string - description: Image to be used for the VM - default: ost://horsemen.i3m.upv.es/79095fa5-7baf-493c-9514-e24d52dd1527 - node_templates: - simple_node: - type: tosca.nodes.indigo.Compute - capabilities: - endpoint: - properties: - network_name: PUBLIC - ports: - get_input: ports - scalable: - properties: - count: - get_input: num_instances - host: - properties: - disk_size: - get_input: disk_size - num_cpus: - get_input: num_cpus - mem_size: - get_input: mem_size - num_gpus: - get_input: num_gpus - gpu_vendor: - get_input: gpu_vendor - gpu_model: - get_input: gpu_model - os: - properties: - image: - get_input: image - interfaces: - Standard: - configure: - implementation: >- - https://raw.githubusercontent.com/grycap/tosca/main/artifacts/dummy.yml - requirements: - - local_storage: - node: my_block_storage - capability: tosca.capabilities.Attachment - relationship: - type: tosca.relationships.AttachesTo - properties: - location: - get_input: mount_path - device: hdb - my_block_storage: - type: tosca.nodes.BlockStorage - properties: - size: - get_input: storage_size - outputs: - node_ip: - value: - get_attribute: - - simple_node - - public_address - - 0 - node_creds: - value: - get_attribute: - - simple_node - - endpoint - - credential - - 0 - diff --git a/src/deploymentMenu.ts b/src/deploymentMenu.ts index 0c942aa..d54190b 100644 --- a/src/deploymentMenu.ts +++ b/src/deploymentMenu.ts @@ -211,8 +211,6 @@ async function createImagesDropdown( return; } - console.log('Images:', output); - // Check if the output contains "error" in the message if (output.toLowerCase().includes('error')) { alert(output); @@ -226,7 +224,6 @@ async function createImagesDropdown( } const jsonOutput = output.substring(jsonStartIndex).trim(); - console.log('JSON output:', jsonOutput); try { const images: { uri: string; name: string }[] = JSON.parse(jsonOutput); @@ -444,7 +441,7 @@ async function selectImage(obj: IDeployInfo): Promise { mkdir -p $PWD/templates &> /dev/null # Create pipes mkfifo $PWD/${pipeAuth} - `; + `; // Command to create the IM-cli credentials let authContent = `id = im; type = InfrastructureManager; username = ${obj.IMuser}; password = ${obj.IMpass};\n`; @@ -479,9 +476,9 @@ async function selectImage(obj: IDeployInfo): Promise { const getEGIToken = async () => { const code = `%%bash - TOKEN=$(cat /var/run/secrets/egi.eu/access_token) - echo $TOKEN - `; + TOKEN=$(cat /var/run/secrets/egi.eu/access_token) + echo $TOKEN + `; const kernelManager = new KernelManager(); const kernel = await kernelManager.startNew(); const future = kernel.requestExecute({ code }); @@ -514,7 +511,7 @@ async function deployIMCommand( mkfifo $PWD/${pipeAuth} # Save mergedTemplate as a YAML file echo '${mergedTemplate}' > ${deployedTemplatePath} - `; + `; // Command to create the IM-cli credentials let authContent = `id = im; type = InfrastructureManager; username = ${obj.IMuser}; password = ${obj.IMpass};\n`; @@ -543,22 +540,21 @@ async function deployIMCommand( fi `; - console.log('cmd', cmd); + console.log('TOSCA recipe deployed:', cmd); return cmd; } async function saveToInfrastructureList(obj: IInfrastructureData) { const infrastructuresListPath = await getInfrastructuresListPath(); // Construct the bash command - const cmd = ` - %%bash - PWD=$(pwd) - existingJson=$(cat ${infrastructuresListPath}) - newJson=$(echo "$existingJson" | jq -c '.infrastructures += [${JSON.stringify(obj)}]') - echo "$newJson" > ${infrastructuresListPath} - `; - - console.log('Bash command:', cmd); + const cmd = `%%bash + PWD=$(pwd) + existingJson=$(cat ${infrastructuresListPath}) + newJson=$(echo "$existingJson" | jq -c '.infrastructures += [${JSON.stringify(obj)}]') + echo "$newJson" > ${infrastructuresListPath} + `; + + console.log('Credentials saved to infrastructuresList.json:', cmd); return cmd; } diff --git a/src/listDeployments.ts b/src/listDeployments.ts index 36eb256..8157ea0 100644 --- a/src/listDeployments.ts +++ b/src/listDeployments.ts @@ -66,8 +66,9 @@ async function populateTable(table: HTMLTableElement): Promise { try { // Read the contents of infrastructuresList.json - const cmdReadJson = `%%bash cat "${infrastructuresListPath}" -`; + const cmdReadJson = `%%bash + cat "${infrastructuresListPath}" + `; const futureReadJson = kernel.requestExecute({ code: cmdReadJson }); futureReadJson.onIOPub = msg => { @@ -228,25 +229,25 @@ async function infrastructureState( } const cmd = `%%bash - PWD=$(pwd) - # Remove pipes if they exist - rm -f $PWD/${pipeAuth} &> /dev/null - # Create pipes - mkfifo $PWD/${pipeAuth} - # Command to create the infrastructure manager client credentials - echo -e "${authContent}" > $PWD/${pipeAuth} & - - stateOut=$(python3 ${imClientPath} getstate ${infrastructureID} -r https://im.egi.eu/im -a $PWD/${pipeAuth}) - # Remove pipe - rm -f $PWD/${pipeAuth} &> /dev/null - # Print state output on stderr or stdout - if [ $? -ne 0 ]; then - >&2 echo -e $stateOut - exit 1 - else - echo -e $stateOut - fi - `; + PWD=$(pwd) + # Remove pipes if they exist + rm -f $PWD/${pipeAuth} &> /dev/null + # Create pipes + mkfifo $PWD/${pipeAuth} + # Command to create the infrastructure manager client credentials + echo -e "${authContent}" > $PWD/${pipeAuth} & + + stateOut=$(python3 ${imClientPath} getstate ${infrastructureID} -r https://im.egi.eu/im -a $PWD/${pipeAuth}) + # Remove pipe + rm -f $PWD/${pipeAuth} &> /dev/null + # Print state output on stderr or stdout + if [ $? -ne 0 ]; then + >&2 echo -e $stateOut + exit 1 + else + echo -e $stateOut + fi + `; console.log('cmdState', cmd); return cmd; @@ -294,25 +295,25 @@ async function infrastructureIP( } const cmd = `%%bash - PWD=$(pwd) - # Remove pipes if they exist - rm -f $PWD/${pipeAuth} &> /dev/null - # Create pipes - mkfifo $PWD/${pipeAuth} - # Command to create the infrastructure manager client credentials - echo -e "${authContent}" > $PWD/${pipeAuth} & - - stateOut=$(python3 ${imClientPath} getvminfo ${infrastructureID} 0 net_interface.1.ip -r https://im.egi.eu/im -a $PWD/${pipeAuth}) - # Remove pipe - rm -f $PWD/${pipeAuth} &> /dev/null - # Print state output on stderr or stdout - if [ $? -ne 0 ]; then - >&2 echo -e $stateOut - exit 1 - else - echo -e $stateOut - fi - `; + PWD=$(pwd) + # Remove pipes if they exist + rm -f $PWD/${pipeAuth} &> /dev/null + # Create pipes + mkfifo $PWD/${pipeAuth} + # Command to create the infrastructure manager client credentials + echo -e "${authContent}" > $PWD/${pipeAuth} & + + stateOut=$(python3 ${imClientPath} getvminfo ${infrastructureID} 0 net_interface.1.ip -r https://im.egi.eu/im -a $PWD/${pipeAuth}) + # Remove pipe + rm -f $PWD/${pipeAuth} &> /dev/null + # Print state output on stderr or stdout + if [ $? -ne 0 ]; then + >&2 echo -e $stateOut + exit 1 + else + echo -e $stateOut + fi + `; console.log('cmdState', cmd); return cmd; diff --git a/src/utils.ts b/src/utils.ts index 749a3ca..573c78a 100755 --- a/src/utils.ts +++ b/src/utils.ts @@ -33,7 +33,10 @@ export async function executeKernelCommand( export async function getIMClientPath(): Promise { return new Promise((resolve, reject) => { - const cmdIMClientPath = '%%bash\n' + 'which im_client.py'; + const cmdIMClientPath = ` + %%bash + which im_client.py + `; executeKernelCommand(cmdIMClientPath, output => { if (output.trim()) { @@ -93,7 +96,8 @@ export async function getInfrastructuresListPath(): Promise { export async function getDeployableTemplatesPath(): Promise { return new Promise((resolve, reject) => { - const cmdTemplatesPath = `%%bash\n + const cmdTemplatesPath = ` + %%bash realpath --relative-to="$(pwd)" resources/deployable_templates `; @@ -110,44 +114,3 @@ export async function getDeployableTemplatesPath(): Promise { }).catch(reject); }); } - -// // Function to batch run commands in parallel -// export async function getPathsInParallel(): Promise<{ -// imClientPath: string; -// infrastructuresListPath: string; -// deployedTemplatePath: string; -// templatesPath: string; -// }> { -// try { -// const [imClientPath, infrastructuresListPath, deployedTemplatePath, templatesPath] = await Promise.all([ -// getIMClientPath(), -// getInfrastructuresListPath(), -// getDeployedTemplatePath(), -// getTemplatesPath(), -// ]); - -// return { -// imClientPath, -// infrastructuresListPath, -// deployedTemplatePath, -// templatesPath, -// }; -// } catch (error) { -// console.error('Error getting paths in parallel:', error); -// throw error; -// } -// } - -// Function to get the infrastructures list path -// export async function getCurrentWorkingDirectory(): Promise { -// return new Promise((resolve, reject) => { -// const cmdCurrentDir = ` -// %%bash -// pwd -// `; - -// executeKernelCommand(cmdCurrentDir, (output) => { -// resolve(output.trim()); -// }).catch(reject); -// }); -// }