From 1e81c50a48dfb522892914744bff85f5c1372e7b Mon Sep 17 00:00:00 2001 From: Mohammed Ahmed Date: Mon, 30 Oct 2023 13:17:51 +0530 Subject: [PATCH] feat: Enabling disconnected cluster installation (#205) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Part of #143 This PR sets the core requirements for disconnected cluster installation. This includes required variables, roles for mirroring (legacy for the platform and `oc-mirror` for everything else), applying `oc-mirror` manifests post cluster up, and corresponding changes to the rest of the playbook and docs. ```bash # update the vars files ❯ ansible-playbook playbooks/0_setup.yaml ❯ ansible-playbook playbooks/3_setup_kvm_host.yaml # If using in in-place registry (for testing) set this up on the plar here and bind it to the interface where the cluster will be setup. Make sure the lpar can resolve to this and then the cluster forwarder IP is set to the gateway IP of the interface Eg 192.168.122.1 and then patch ins reg and pull secret values accordingly. A sample script for this is provided below ❯ ansible-playbook playbooks/4_create_bastion.yaml ❯ ansible-playbook playbooks/disconnected_mirror_artifacts.yaml ❯ ansible-playbook playbooks/5_setup_bastion.yaml ❯ ansible-playbook playbooks/6_create_nodes.yaml ❯ ansible-playbook playbooks/7_ocp_verification.yaml ❯ ansible-playbook playbooks/disconnected_apply_operator_manifests.yaml ``` Sample script for in-place registry on the LPAR itself for NAT-based cluster setup. To be run only after the libvirt interface is created/available on lpar: https://github.com/mohammedzee1000/IBM-Ansible-OpenShift-Provisioning-TestNotes/blob/main/scripts/testing_utils/create_reg.sh --------- Signed-off-by: Mohammed Zeeshan Ahmed Signed-off-by: Mohammed Zeeshan Ahmed Co-authored-by: Mohammed Zeeshan Ahmed Co-authored-by: Jacob Emery --- .gitignore | 3 +- docs/prerequisites.md | 2 +- docs/run-the-playbooks-for-disconnected.md | 80 +++++++++++ docs/run-the-playbooks.md | 2 +- docs/set-variables-group-vars.md | 31 ++++- .../default/group_vars/all.yaml.template | 91 ++++++++++--- mkdocs.yaml | 5 +- playbooks/0_setup.yaml | 7 + ...disconnected_apply_operator_manifests.yaml | 17 +++ playbooks/disconnected_mirror_artifacts.yaml | 14 ++ playbooks/pre-existing_site.yaml | 4 + playbooks/reinstall_cluster.yaml | 7 + playbooks/site.yaml | 6 +- .../tasks/main.yaml | 17 +++ roles/disconnected_check_vars/tasks/main.yaml | 19 +++ .../tasks/main.yaml | 124 ++++++++++++++++++ .../templates/ca.crt.j2 | 1 + .../templates/imageset.yaml.j2 | 11 ++ .../templates/mirror-secret.json.j2 | 1 + roles/get_ocp/tasks/main.yaml | 10 +- .../get_ocp/templates/install-config.yaml.j2 | 14 +- roles/set_inventory/templates/hosts.j2 | 5 + 22 files changed, 444 insertions(+), 27 deletions(-) create mode 100644 docs/run-the-playbooks-for-disconnected.md create mode 100644 playbooks/disconnected_apply_operator_manifests.yaml create mode 100644 playbooks/disconnected_mirror_artifacts.yaml create mode 100644 roles/disconnected_apply_operator_manifests_to_cluster/tasks/main.yaml create mode 100644 roles/disconnected_check_vars/tasks/main.yaml create mode 100644 roles/disconnected_mirror_images/tasks/main.yaml create mode 100644 roles/disconnected_mirror_images/templates/ca.crt.j2 create mode 100644 roles/disconnected_mirror_images/templates/imageset.yaml.j2 create mode 100644 roles/disconnected_mirror_images/templates/mirror-secret.json.j2 diff --git a/.gitignore b/.gitignore index f05903c7..a33ceacd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .DS_Store .iso .vscode -site \ No newline at end of file +site +.oc-mirror-results diff --git a/docs/prerequisites.md b/docs/prerequisites.md index 0e1c529b..5485bda3 100644 --- a/docs/prerequisites.md +++ b/docs/prerequisites.md @@ -41,7 +41,7 @@ ``` sudo mkdir /home//ocp-config ``` - or HTTP: + * or HTTP: ``` sudo mkdir /var/www/html/ocp-config ``` diff --git a/docs/run-the-playbooks-for-disconnected.md b/docs/run-the-playbooks-for-disconnected.md new file mode 100644 index 00000000..e7fc0e02 --- /dev/null +++ b/docs/run-the-playbooks-for-disconnected.md @@ -0,0 +1,80 @@ +# Run the Playbooks +## Overview +For installing disconnected clusters, you will mostly be following rhe same process as a standard connected cluster. + +The main additional steps we would be doing is mirroring the OCP images to another registry which is accessible to +the cluster and post the cluster coming up, we will be applying operator hub manifests such as image content source +policy and catalog source, generated by `oc-mirror`, to the cluster. + +Disconnected playbook are mentioned below. Please refer the **4 Run the Playbooks** documentation for details of rest of the playbooks: + +* disconnected_mirror_artifacts.yaml ([code](https://github.com/IBM/Ansible-OpenShift-Provisioning/blob/main/playbooks/disconnected_mirror_artifacts.yaml)) - Run before **6_create_nodes.yaml** +* disconnected_apply_operator_manifests.yaml ([code](https://github.com/IBM/Ansible-OpenShift-Provisioning/blob/main/playbooks/disconnected_apply_operator_manifests.yaml)) - Run after **7_ocp_verification.yaml** +## Pre-requisites +* A running registry where the OCP and operator hub images will be mirrored. If the CA of this registry is not automatically trusted, then keep the + CA cert content handy to update in inventory file. The CA cert is the file with which, do dont need to skip tls to access the registry. +* Make sure you have required pull secrets handy. You will need 2 pull secrets, one to apply on the cluster and another which will be used for + mirroring. The mirroring pull secret MUST have push access to the mirror registry as well as must give you access to Red Hat registries. + A good way to create this would be take the Red Hat pull secret from **Get Info page** and do a podman login with creds having write access. + + ``` + cp -avrf /path/to/redhat-pull-secrets.json ./mirror-secret.json + podman login -u admin -p admin --tls-verify=false --authfile=./mirror-secret.json + cat ./mirror-secret.json | jq -r tostring + + ``` + +* A mirror host. This can be any host that can access the internet (mainly the registry being mirrored from) as well as the registry being mirrored to. + This registries being mirrored from would typically be the Red Hat registries (registry.redhat.io, quay.io etc) +* The file server, configured mentioned below. +* Appropriately updated variables in your `all.yaml`. Refer the variables documentation. +### File Server +This configuration will take place on the file server mentioned under **File Server** section in overall pre-requisites documentaion. The additional +configurations are mentioned over here. + +* Make sure to have a directory housing the clients + + * For FTP: + + ``` + sudo mkdir /home//clients + ``` + + * or HTTP: + + ``` + sudo mkdir /var/www/html/clients + ``` + + Make sure this directory contains a pre-downloaded `oc-mirror` binary in `tar.gz` format. Currently the supported binary is available for `x86_64` on Red Hat Customer portal openshift [downloads](https://console.redhat.com/openshift/downloads) page. It can also be found on mirror.openshift.com from `4.14` onwards for other architectures. +### NOTE +* At this stage, only oc-mirror binary is fetched from File Server, so it is expected that the lpar for disconnected cluster can at least reach `mirror.openshift.com` to download the + other artifacts for cluster installation. +* The platorm related image content source policy will be baked into the install config as part of **5 Setup Bastion Playbook**. +* Right now on legacy platform mirroring is supported in this playook during the creation of the cluster. +* Manifests generated by `oc-mirror` will be applied to the cluster cluster up. So if you add platform details in image set, it + will be applied on cluster only after the cluster is up. + +## Disconnected Mirror Artifacts Playbook +### Overview +Mirror the ocp platform and other necessary images to the mirror registry. Please run this playbook before you run **6 Create Nodes Playbook** and after +**0 Setup Playbook**. +### Outcomes +* Download `oc` and `oc-mirror` to the mirror host. +* Template the mirror pull secret to the mirror host. +* Add the ca cert to the mirror host anchors if ca is not trusted. +* Mirror the platform images using `oc adm release mirror` if legacy mirroring is enabled. +* Template the image set to mirror host and then mirror it using `oc-mirror` plogin. +* Copy the results on the `oc-mirror` to ansible controller to apply to cluster in future steps. +### Notes +* Currently, platform can **only** be mirrored the legacy way. While the image set can contain platform mirroring configs, it will **not** be applied to cluster during creation. +* This playbook can be run at any stage after the **0 Setup** playbook. Make sure to run this before the cluster starts pulling at the images from the registry + which typically happens where the **Create Nodes Playbook** is run. + +# Disconnected apply oc mirror manifests to cluster Playbook +### Overview +Post cluster creation, `oc-mirror` manifests are applied to the cluster. Please run this playbook after **7 OCP Verification Playbook**. +### Outcomes +* Copy the `oc-mirror` results manifests to the bastion. +* Apply the copied manifests to the cluster. +* Disable default content sources. diff --git a/docs/run-the-playbooks.md b/docs/run-the-playbooks.md index 59bdbad6..a6138bcf 100644 --- a/docs/run-the-playbooks.md +++ b/docs/run-the-playbooks.md @@ -94,7 +94,7 @@ Configuration of the bastion to host essential infrastructure services for the c * CoreOS roofts is pulled to the bastion if not already there. * OCP client and installer are pulled down if not there already. * oc, kubectl and openshift-install binaries are installed. -* OCP install-config is templated and backed up. +* OCP install-config is templated and backed up. In disconnected mode, if platform is mirrored (currently only legacy), image content source policy and additionalTrustBundle is also patched. * Manfifests are created. * OCP install directory found at /root/ocpinst/ is created and populated with necessary files. * Ignition files for the bootstrap, control, and compute nodes are transferred to HTTP-accessible directory for booting nodes. diff --git a/docs/set-variables-group-vars.md b/docs/set-variables-group-vars.md index eeda9cb0..202653a8 100644 --- a/docs/set-variables-group-vars.md +++ b/docs/set-variables-group-vars.md @@ -242,7 +242,36 @@ **hypershift.agents_parms.vcpus** | vCPUs for agents | 4 **hypershift.agents_parms.nameserver** | Nameserver to be used for agents | 192.168.10.1 -## 17 - (Optional) Create compute node in a day-2 operation +## 17 - (Optional) Disconnected cluster setup +**Variable Name** | **Description** | **Example** +:--- | :--- | :--- +**disconnected.enabled** | True or False, to enable disconnected mode | False +**disconnected.registry.url** | String containing url of disconnected registry with or without port and without protocol | registry.tt.testing:5000 +**disconnected.registry.pull_secret** | String containing pull secret of the disconnected registry to be applied on the *cluster*.
Make sure to enclose pull_secret in 'single quotes' and it has appropriate pull access. | '{"auths":{"registry.tt.
.testing:5000":{"auth":"b3Blb
...
4yQQ==","email":"test.
user@example.com"}}}' +**disconnected.registry.mirror_pull_ecret** | String containing pull secret to use for mirroring. Contains Red Hat secret and registry pull
secret. Make sure to enclose pull_secret in 'single quotes' and must be able to push to mirror
registry. | '{"auths":{"cloud.openshift
.com":{"auth":"b3Blb
...
4yQQ==","email":"redhat.
user@gmail.com", "registry.tt..testing:5000":
...
user@example.com"}}}' +**disconnected.registry.ca_trusted** | True or False to indicate that mirror registry CA is implicitly trusted or needs to be made
trusted on mirror host and cluster. | False +**disconnected.registry.ca_cert** | Multiline string containing the mirror registry CA bundle | -----BEGIN CERTIFICATE-----
MIIDqDCCApCgAwIBAgIULL+d1HTYsiP+8jeWnqBis3N4BskwDQYJKoZIhvcNAQEF
...
-----END CERTIFICATE----- +**disconnected.mirroring.host.name** | String containing the hostname of the host, which will be used for mirroring | mirror-host-1 +**disconnected.mirroring.host.ip** | String containing ip of the host, which will be used for mirroring | 192.168.10.99 +**disconnected.mirroring.host.user** | String containing the username of the host, which will be used for mirroring | mirroruser +**disconnected.mirroring.host.pass** | String containing the password of the host, which will be used for mirroring | mirrorpassword +**disconnected.mirroring.file_server.clients_dir** | Directory path relative to the HTTP/FTP accessible directory on **env.file_server**
where client binary tarballs are kept | clients +**disconnected.mirroring.file_server.oc_mirror_tgz** | Name of oc-mirror tarball on **env.file_server** in **disconnected.mirroring.file_server.clients_dir** | oc-mirror.tar.gz +**disconnected.mirroring.legacy.platform** | True or False if the platform should be mirrored using `oc adm release mirror`. | True +**disconnected.mirroring.legacy.ocp_quay_release_image_tag** | The tag of the release image *quay.io/openshift-release-dev/ocp-release* to mirror and use | 4.13.1-s390x +**disconnected.mirroring.legacy.ocp_org** | The org part of the repo on the mirror registry where the release image will be pushed | ocp4 +**disconnected.mirroring.legacy.ocp_repo** | The repo part of the repo on the mirror registry where the release image will be pushed | openshift4 +**disconnected.mirroring.legacy.ocp_tag** | The tag part of the repo on the mirror registry where the release image will be pushed.
Full image would be as below.:

disconnected.registry.url/disconnected.mirroring.legacy.ocp_org/disconnected...ocp_repo
:disconnected..ocp_tag | v4.13.1 +**disconnected.mirroring.oc_mirror.oc_mirror_args.continue_on_error** | True or False to give `--continue-on-error` flag to `oc-mirror` | False +**disconnected.mirroring.oc_mirror.oc_mirror_args.source_skip_tls** | True or False to give `--source-skip-tls` flag to `oc-mirror` | False +**disconnected.mirroring.oc_mirror.image_set** | YAML fields containing a standard `oc-mirror` [image set](https://docs.openshift.com/container-platform/latest/installing/disconnected_install/installing-mirroring-disconnected.html#oc-mirror-creating-image-set-config_installing-mirroring-disconnected) with some minor changes to schema.
Differences are documented as needed. Used to generate final image set. | see template +**disconnected.mirroring.oc_mirror.image_set.storageConfig.registry.enabled** | True or False to use registry storage backend for pushing mirrored content directly to the registry.
Currently only this backend is supported.| True +**disconnected.mirroring.oc_mirror.image_set.storageConfig.registry.imageURL.org** | The org part of registry imageURL from standard image set. | mirror +**disconnected.mirroring.oc_mirror.image_set.storageConfig.registry.imageURL.repo** | The repo part of registry imageURL from standard image set.
Final imageURL will be as below:

disconnected.registry.url/disconnected.mirroring.oc_mirror.image_set.storageConfig
.registry.imageURL.org/disconnected...imageURL.repo | oc-mirror-metadata +**disconnected.mirroring.oc_mirror.image_set.storageConfig.registry.skipTLS** | True of False same purpose served as in standard image set i.e. skip the tls for the registry
during mirroring.| false +**disconnected.mirrroing.oc_mirror.image_set.mirror** | YAML containing a list of what needs to be mirrored. See the oc mirror image set documentation.
*WARNING*: Platform mirroring in this way is not supported. Use legacy way of platform mirroring | see oc-mirror [image set](https://docs.openshift.com/container-platform/latest/installing/disconnected_install/installing-mirroring-disconnected.html#oc-mirror-creating-image-set-config_installing-mirroring-disconnected) documentation + +## 18 - (Optional) Create compute node in a day-2 operation **Variable Name** | **Description** | **Example** :--- | :--- | :--- diff --git a/inventories/default/group_vars/all.yaml.template b/inventories/default/group_vars/all.yaml.template index a669dd59..87a3ed38 100644 --- a/inventories/default/group_vars/all.yaml.template +++ b/inventories/default/group_vars/all.yaml.template @@ -246,19 +246,19 @@ hypershift: create_bastion: true networking_device: enc1100 - gateway: + gateway: bastion_parms: - interface: - hostname: - base_domain: - os_variant: + interface: + hostname: + base_domain: + os_variant: nameserver: - gateway: - subnet_mask: + gateway: + subnet_mask: - mgmt_cluster_nameserver: - oc_url: + mgmt_cluster_nameserver: + oc_url: #Hosted Control Plane Parameters @@ -275,7 +275,7 @@ hypershift: # MultiClusterEngine Parameters mce: - version: + version: instance_name: engine delete: false @@ -294,25 +294,82 @@ hypershift: static_ip_parms: static_ip: true ip: # Required only if static_ip is true - #- - #- + #- + #- interface: eth0 - agents_count: + agents_count: # If you want to use specific mac addresses, provide them here - agent_mac_addr: - #- + agent_mac_addr: + #- disk_size: 100G ram: 16384 vcpus: 4 nameserver: -# Section 17 - (Optional) Create additional compute node in a day-2 operation +# Section 17 - (Optional) Setup disconnected clusters +# Warning: currently, the oc-mirror plugin is officially downloadable to amd64 only. +disconnected: + enabled: False + registry: + url: 'registry url' + pull_secret: '' # this is similar to env.redhat.pull_secret but it will only contain secrets to be applied to the cluster in disconnected mode. + mirror_pull_secret: '' # this should contain a pull secret that contains the combination of env.redhat.pull_secret and pull secret with push access to mirror registry for mirroring + ca_trusted: False + ca_cert: | + -----BEGIN CERTIFICATE----- + if ca_trusted is False, then this ca will be added to mirror host anchors as well as to the install config of cluster + -----END CERTIFICATE----- + mirroring: + host: # this is the host that can access the internet as well as the registry + name: hosname + ip: x.x.x.x + user: mirroruser # with become access + pass: mirrorpassword + file_server: # in disconnected mode, the client binaries and rhcos will be put on env.file_server and then downloaded to the final destination from there. For now, its only oc-mirror. Rest of artifacts will be downloaded from urls + clients_dir: 'clients' + oc_mirror_tgz: 'oc-mirror.tar.gz' # name of oc-mirror plugin binary in clients_dir. should be a tar.gz file. You must place this in your ftp server after downloading it yourself from https://console.redhat.com/openshift/downloads for amd64 (or building it yourself if your mirror host is s390x) + client_download: # this will download oc binary to the mirror host for use on the mirror host for mirroring + ocp_download_url: "https://mirror.openshift.com/pub/openshift-v4/multi/clients/ocp/4.13.1/amd64/" + ocp_client_tgz: 'openshift-client-linux.tar.gz' # name of the oc binary. Should be a tar.gz file + legacy: + platform: True # if true then platform is mirrored in the old way and install config will be patched with the imagecontentsourcepolicy + ocp_quay_release_image_tag: '4.13.1-s390x' + ocp_org: 'ocp4' + ocp_repo: 'openshift4' + ocp_tag: 'v4.13.1' # platform images will be pushed to {tegistry_url}/{ocp_org}/{ocp_repo}:{ocp_tag} + oc_mirror: + oc_mirror_args: + continue_on_error: False + source_skip_tls: False + image_set: # this field is a standard image set from oc-mirror documentation. The only exception is the storageConfig which is altered to allow substitution of disconnected.registry.url + storageConfig: + registry: + enabled: True # use registry storage backend. Currently only method supproted + imageURL: # the final value will be {imageURL: disconnected.registry.url/org/repo} + org: mirror + repo: oc-mirror-metadata + skipTLS: false # standard field form oc-mirror schema + mirror: # this field is also atandard from the oc-mirror schema. It will be substituted as is into the final image set # WARNING: platform mirroring through oc-mirror is currently not supported, use legacy + operators: + - catalog: registry.redhat.io/redhat/redhat-operator-index:v4.13 + full: false + packages: + - name: serverless-operator + channels: + - name: stable + #minVersion: '2.4.1-0' + #maxVersion: '2.4.1-0' + additionalImages: + - name: registry.redhat.io/ubi8/ubi:latest + helm: {} + +# Section 18 - (Optional) Create additional compute node in a day-2 operation day2_compute_node: vm_name: vm_hostname: vm_ip: vm_ipv6: - vm_interface: + vm_interface: hostname: host_arch: diff --git a/mkdocs.yaml b/mkdocs.yaml index 91cb7892..c7dbe84f 100644 --- a/mkdocs.yaml +++ b/mkdocs.yaml @@ -3,7 +3,7 @@ repo_url: https://github.com/IBM/Ansible-OpenShift-Provisioning site_url: https://ibm.github.io/Ansible-OpenShift-Provisioning/ edit_uri: edit/main/docs -nav: +nav: - Home: 'index.md' - Read Me: - Before You Begin: 'before-you-begin.md' @@ -13,6 +13,7 @@ nav: - 2 Set Variables (group_vars): 'set-variables-group-vars.md' - 3 Set Variables (host_vars): 'set-variables-host-vars.md' - 4 Run the Playbooks: 'run-the-playbooks.md' + - Run the Playbooks (Disconnected): 'run-the-playbooks-for-disconnected.md' - Run the Playbooks (HyperShift): 'run-the-playbooks-for-hypershift.md' - Misc: - Troubleshooting: 'troubleshooting.md' @@ -37,4 +38,4 @@ markdown_extensions: - toc: permalink: "#" - attr_list - + diff --git a/playbooks/0_setup.yaml b/playbooks/0_setup.yaml index ea718146..cce1b782 100644 --- a/playbooks/0_setup.yaml +++ b/playbooks/0_setup.yaml @@ -8,6 +8,13 @@ roles: - set_inventory + pre_tasks: + - name: Check disconnected variables if disconnected cluster will be installed. + tags: disconnected + ansible.builtin.include_role: + name: disconnected_check_vars + when: disconnected.enabled + post_tasks: - name: Ensure Ansible Galaxy collections have been installed. tags: galaxy diff --git a/playbooks/disconnected_apply_operator_manifests.yaml b/playbooks/disconnected_apply_operator_manifests.yaml new file mode 100644 index 00000000..23657338 --- /dev/null +++ b/playbooks/disconnected_apply_operator_manifests.yaml @@ -0,0 +1,17 @@ +--- +- name: Disconnected apply operator manifests + hosts: bastion + become: true + environment: + KUBECONFIG: "{{ '/home/' if (env.bastion.access.user != 'root') else '/'}}{{ env.bastion.access.user }}/.kube/config" + gather_facts: true + vars_files: + - "{{ inventory_dir }}/group_vars/all.yaml" + tasks: + - name: apply operator manifests when cluster is disconnected + ansible.builtin.include_role: + name: '{{ item }}' + loop: + - disconnected_check_vars + - disconnected_apply_operator_manifests_to_cluster + when: disconnected.enabled diff --git a/playbooks/disconnected_mirror_artifacts.yaml b/playbooks/disconnected_mirror_artifacts.yaml new file mode 100644 index 00000000..a270e311 --- /dev/null +++ b/playbooks/disconnected_mirror_artifacts.yaml @@ -0,0 +1,14 @@ +--- +- name: Mirror ocp artifacts + hosts: mirrorhost + gather_facts: false + vars_files: + - "{{ inventory_dir }}/group_vars/all.yaml" + tasks: + - name: mirror ocp artifacts when disconnected install + ansible.builtin.include_role: + name: '{{ item }}' + loop: + - disconnected_check_vars + - disconnected_mirror_images + when: disconnected.enabled diff --git a/playbooks/pre-existing_site.yaml b/playbooks/pre-existing_site.yaml index 6456527c..5347411b 100644 --- a/playbooks/pre-existing_site.yaml +++ b/playbooks/pre-existing_site.yaml @@ -3,6 +3,10 @@ - import_playbook: 0_setup.yaml - import_playbook: 4_create_bastion.yaml +- import_playbook: disconnected_mirror_artifacts.yaml + when: disconnected.enabled - import_playbook: 5_setup_bastion.yaml - import_playbook: 6_create_nodes.yaml - import_playbook: 7_ocp_verification.yaml +- import_playbook: disconnected_apply_operator_manifests.yaml + when: disconnected.enabled diff --git a/playbooks/reinstall_cluster.yaml b/playbooks/reinstall_cluster.yaml index 9bdc19fb..6bdb8572 100644 --- a/playbooks/reinstall_cluster.yaml +++ b/playbooks/reinstall_cluster.yaml @@ -1,6 +1,8 @@ # Use this if you want to re-install nodes with a new OCP version --- +- import_playbook: 0_setup.yaml + - name: Re-Install cluster - Copy SSH key from localhost to access bastion hosts: localhost tags: ssh, ssh_copy_id, section_1 @@ -73,6 +75,9 @@ name: ssh_copy_id when: env.z.lpar3.hostname is defined +- import_playbook: disconnected_mirror_artifacts.yaml + when: disconnected.enabled + - name: Re-Install cluster - Update ignitions and other install files hosts: bastion become: true @@ -84,3 +89,5 @@ - import_playbook: 6_create_nodes.yaml - import_playbook: 7_ocp_verification.yaml +- import_playbook: disconnected_apply_operator_manifests.yaml + when: disconnected.enabled diff --git a/playbooks/site.yaml b/playbooks/site.yaml index cda7fd23..177f3ae8 100644 --- a/playbooks/site.yaml +++ b/playbooks/site.yaml @@ -6,6 +6,10 @@ - import_playbook: 2_create_kvm_host.yaml - import_playbook: 3_setup_kvm_host.yaml - import_playbook: 4_create_bastion.yaml +- import_playbook: disconnected_mirror_artifacts.yaml + when: disconnected.enabled - import_playbook: 5_setup_bastion.yaml - import_playbook: 6_create_nodes.yaml -- import_playbook: 7_ocp_verification.yaml \ No newline at end of file +- import_playbook: 7_ocp_verification.yaml +- import_playbook: disconnected_apply_operator_manifests.yaml + when: disconnected.enabled diff --git a/roles/disconnected_apply_operator_manifests_to_cluster/tasks/main.yaml b/roles/disconnected_apply_operator_manifests_to_cluster/tasks/main.yaml new file mode 100644 index 00000000..7d7c86e9 --- /dev/null +++ b/roles/disconnected_apply_operator_manifests_to_cluster/tasks/main.yaml @@ -0,0 +1,17 @@ +--- +- name: disconnected apply operator manifests + tags: disconnected_apply_operator_manifests + block: + - name: sync the image set results to install directory + tags: disconnected_apply_operator_manifests + become: true + ansible.posix.synchronize: + src: "../.oc-mirror-results" + dest: /root/ocpinst + - name: apply the manifests on the cluster + tags: disconnected_apply_operator_manifests + ansible.builtin.command: oc apply -f /root/ocpinst/.oc-mirror-results + - name: disable default content sources + tags: disconnected_apply_operator_manifests + ansible.builtin.command: "oc patch OperatorHub cluster --type json -p '[{\"op\": \"add\", \"path\": \"/spec/disableAllDefaultSources\", \"value\": true}]'" + when: disconnected.enabled diff --git a/roles/disconnected_check_vars/tasks/main.yaml b/roles/disconnected_check_vars/tasks/main.yaml new file mode 100644 index 00000000..fdb75c9f --- /dev/null +++ b/roles/disconnected_check_vars/tasks/main.yaml @@ -0,0 +1,19 @@ +--- + +- name: 'Check mandatory variables are defined for disconnected' + block: + - name: "check mandatory variables for disconnected are set" + assert: + that: + - disconnected.registry.url is defined and disconnected.registry.url != None + - disconnected.registry.pull_secret is defined and disconnected.registry.pull_secret != None + - disconnected.registry.mirror_pull_secret is defined and disconnected.registry.mirror_pull_secret != None + - disconnected.mirroring.host.name is defined and disconnected.mirroring.host.name != None + - disconnected.mirroring.host.ip is defined and disconnected.mirroring.host.ip != None + - disconnected.mirroring.host.user is defined and disconnected.mirroring.host.user != None + - disconnected.mirroring.host.pass is defined and disconnected.mirroring.host.pass != None + - disconnected.mirroring.file_server.clients_dir is defined and disconnected.mirroring.file_server.clients_dir != None + - disconnected.mirroring.file_server.oc_mirror_tgz is defined and disconnected.mirroring.file_server.oc_mirror_tgz != None + - disconnected.mirroring.client_download.ocp_download_url is defined and disconnected.mirroring.client_download.ocp_download_url != None + - disconnected.mirroring.client_download.ocp_client_tgz is defined and disconnected.mirroring.client_download.ocp_client_tgz != None + - disconnected.mirroring.oc_mirror.image_set is defined and disconnected.mirroring.oc_mirror.image_set != None diff --git a/roles/disconnected_mirror_images/tasks/main.yaml b/roles/disconnected_mirror_images/tasks/main.yaml new file mode 100644 index 00000000..c2b9ca7f --- /dev/null +++ b/roles/disconnected_mirror_images/tasks/main.yaml @@ -0,0 +1,124 @@ +--- +- name: mirror artifacts in disconnected mode + tags: mirror_artifacts + block: + - name: Get user home directory + tags: mirror_artifacts + shell: > + getent passwd {{ ansible_user }} | awk -F: '{ print $6 }' + changed_when: false + register: user_home + - name: Delete OCP download directory for idempotency. + tags: mirror_artifacts + become: true + file: + path: "{{ user_home.stdout }}/ocpinst" + state: absent + - name: Create OCP download directory + tags: mirror_artifacts + file: + path: "{{ user_home.stdout }}/ocpinst" + state: directory + - name: Unzip OCP client and oc-mirror + tags: mirror_artifacts + ansible.builtin.unarchive: + src: "{{ item }}" + dest: "{{ user_home.stdout }}/ocpinst/" + remote_src: yes + loop: + - "{{ disconnected.mirroring.client_download.ocp_download_url }}{{ disconnected.mirroring.client_download.ocp_client_tgz }}" + - "{{ env.file_server.protocol }}://{{ env.file_server.user + ':' + env.file_server.pass + '@' if env.file_server.protocol == 'ftp' else '' }}{{ env.file_server.ip }}/{{ disconnected.mirroring.file_server.clients_dir }}/{{ disconnected.mirroring.file_server.oc_mirror_tgz }}" + - name: Copy kubectl, oc, and oc-mirror binaries to /usr/local/sbin + tags: mirror_artifacts + become: true + ansible.builtin.copy: + src: "{{ user_home.stdout }}/ocpinst/{{ item }}" + dest: /usr/sbin/{{ item }} + owner: root + group: root + mode: "755" + remote_src: yes + loop: + - kubectl + - oc + - oc-mirror + - name: Check if directory {{ user_home.stdout }}/.docker exists + tags: mirror_artifacts + ansible.builtin.stat: + path: "{{ user_home.stdout }}/.docker" + register: home_docker + - name: Create directory {{ user_home.stdout }}/.docker + tags: mirror_artifacts + file: + path: "{{ user_home.stdout }}/.docker" + state: directory + when: not home_docker.stat.exists + - name: create pull secret file for mirroring + tags: mirror_artifacts + ansible.builtin.template: + src: mirror-secret.json.j2 + dest: "{{ user_home.stdout }}/.docker/config.json" + backup: yes + force: yes + - name: create ca cert file for adding to ca trust when ca is not trusted and updating ca trust + tags: mirror_artifacts + become: true + block: + - name: create ca cert file when ca is untrusted + ansible.builtin.template: + src: ca.crt.j2 + dest: /etc/pki/ca-trust/source/anchors/registry.crt + force: yes + - name: update ca trust with the cert file + ansible.builtin.shell: | + set -o pipefail + update-ca-trust + when: not disconnected.registry.ca_trusted + - name: mirror the platform the legacy way if legacy mode is enabled + tags: mirror_artifacts + block: + - name: run the oc adm release mirror command + ansible.builtin.shell: | + set -o pipefail + oc adm -a {{ user_home.stdout }}/.docker/config.json release mirror \ + --from=quay.io/openshift-release-dev/ocp-release:{{ disconnected.mirroring.legacy.ocp_quay_release_image_tag }} \ + --to={{ disconnected.registry.url }}/{{ disconnected.mirroring.legacy.ocp_org }}/{{ disconnected.mirroring.legacy.ocp_repo }} \ + --to-release-image={{ disconnected.registry.url }}/{{ disconnected.mirroring.legacy.ocp_org }}/{{ disconnected.mirroring.legacy.ocp_repo }}:{{ disconnected.mirroring.legacy.ocp_tag }} + register: cmd_oc_legacy_mirroring + - name: print the output of the mirroring + ansible.builtin.debug: + var: cmd_oc_legacy_mirroring.stdout_lines + when: disconnected.mirroring.legacy.platform + - name: prepare imageset for oc-mirror mirroring + tags: mirror_artifacts + ansible.builtin.template: + src: imageset.yaml.j2 + dest: "{{ user_home.stdout }}/ocpinst/imageset.yaml" + - name: mirror the imageset to registry + tags: mirror_artifacts + block: + - name: mirror the imageset to registry + ansible.builtin.shell: | + set -o pipefail + oc mirror --config {{ user_home.stdout }}/ocpinst/imageset.yaml docker://{{ disconnected.registry.url}} --ignore-history{{ ' --continue-on-error' if disconnected.mirroring.oc_mirror.oc_mirror_args.continue_on_error == True }} \ + {{ ' --source-skip-tls' if disconnected.mirroring.oc_mirror.oc_mirror_args.source_skip_tls == True }} + # ignore-history set by default for idempotency + register: cmd_oc_mirror + args: + chdir: "{{ user_home.stdout }}/ocpinst" + - name: print output of imageset mirroring + ansible.builtin.debug: + var: cmd_oc_mirror + - name: Get the results directory from oc mirror operation + tags: mirror_artifacts + ansible.builtin.shell: | + set -o pipefail + ls {{ user_home.stdout }}/ocpinst/oc-mirror-workspace/ | grep results + register: oc_mirror_results_dir_name + - name: Fetch the files that need to be applied on the cluster to the controller + tags: mirror_artifacts + ansible.posix.synchronize: + mode: pull + src: "{{ user_home.stdout }}/ocpinst/oc-mirror-workspace/{{ oc_mirror_results_dir_name.stdout }}/*" + dest: "../.oc-mirror-results" + when: disconnected.enabled diff --git a/roles/disconnected_mirror_images/templates/ca.crt.j2 b/roles/disconnected_mirror_images/templates/ca.crt.j2 new file mode 100644 index 00000000..7101b74f --- /dev/null +++ b/roles/disconnected_mirror_images/templates/ca.crt.j2 @@ -0,0 +1 @@ +{{ disconnected.registry.ca_cert }} diff --git a/roles/disconnected_mirror_images/templates/imageset.yaml.j2 b/roles/disconnected_mirror_images/templates/imageset.yaml.j2 new file mode 100644 index 00000000..3ebfb32f --- /dev/null +++ b/roles/disconnected_mirror_images/templates/imageset.yaml.j2 @@ -0,0 +1,11 @@ +kind: ImageSetConfiguration +apiVersion: mirror.openshift.io/v1alpha2 +archiveSize: 4 +storageConfig: +{% if ( disconnected.enabled ) and ( disconnected.mirroring.oc_mirror.image_set.storageConfig.registry.enabled ) %} + registry: + imageURL: {{ disconnected.registry.url | string + '/' + disconnected.mirroring.oc_mirror.image_set.storageConfig.registry.imageURL.org | string + '/' + disconnected.mirroring.oc_mirror.image_set.storageConfig.registry.imageURL.repo | string }} + skipTLS: {{ disconnected.mirroring.oc_mirror.image_set.storageConfig.registry.skipTLS }} +{% endif %} +mirror: +{{ disconnected.mirroring.oc_mirror.image_set.mirror | to_nice_yaml | indent(2, true) }} diff --git a/roles/disconnected_mirror_images/templates/mirror-secret.json.j2 b/roles/disconnected_mirror_images/templates/mirror-secret.json.j2 new file mode 100644 index 00000000..88354b06 --- /dev/null +++ b/roles/disconnected_mirror_images/templates/mirror-secret.json.j2 @@ -0,0 +1 @@ +{{ disconnected.registry.mirror_pull_secret }} diff --git a/roles/get_ocp/tasks/main.yaml b/roles/get_ocp/tasks/main.yaml index 3c8f2ad4..2b42c278 100644 --- a/roles/get_ocp/tasks/main.yaml +++ b/roles/get_ocp/tasks/main.yaml @@ -88,7 +88,10 @@ - name: Create manifests tags: get_ocp - command: /root/ocpinst/openshift-install create manifests --dir=/root/ocpinst/ + ansible.builtin.shell: | + set -o pipefail + {{ 'export OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE=quay.io/openshift-release-dev/ocp-release:' + disconnected.mirroring.legacy.ocp_quay_release_image_tag if disconnected.enabled and disconnected.mirroring.legacy.platform }} + /root/ocpinst/openshift-install create manifests --dir=/root/ocpinst/ become: true - name: Set masters schedulable parameter to false @@ -122,7 +125,10 @@ - name: Create ignition files tags: get_ocp become: true - command: /root/ocpinst/openshift-install create ignition-configs --dir=/root/ocpinst/ + ansible.builtin.shell: | + set -o pipefail + {{ 'export OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE=quay.io/openshift-release-dev/ocp-release:' + disconnected.mirroring.legacy.ocp_quay_release_image_tag if disconnected.enabled and disconnected.mirroring.legacy.platform }} + /root/ocpinst/openshift-install create ignition-configs --dir=/root/ocpinst/ - name: Set ownership to root and permissions of ignitions and related files. tags: get_ocp diff --git a/roles/get_ocp/templates/install-config.yaml.j2 b/roles/get_ocp/templates/install-config.yaml.j2 index f36de9c6..b897922f 100644 --- a/roles/get_ocp/templates/install-config.yaml.j2 +++ b/roles/get_ocp/templates/install-config.yaml.j2 @@ -39,4 +39,16 @@ networking: platform: none: {} fips: {{ env.install_config.fips }} -pullSecret: '{{ env.redhat.pull_secret }}' +pullSecret: '{{ env.redhat.pull_secret if not disconnected.enabled else disconnected.registry.pull_secret }}' +{% if disconnected.enabled and disconnected.mirroring.legacy.platform %} +{{ 'imageContentSources: ' }} +{{ '- mirrors:'}} +{{ ' - ' + disconnected.registry.url + '/' + disconnected.mirroring.legacy.ocp_org + '/' + disconnected.mirroring.legacy.ocp_repo }} +{{ ' source: quay.io/openshift-release-dev/ocp-release' }} +{{ '- mirrors:'}} +{{ ' - ' + disconnected.registry.url + '/' + disconnected.mirroring.legacy.ocp_org + '/' + disconnected.mirroring.legacy.ocp_repo }} +{{ ' source: quay.io/openshift-release-dev/ocp-v4.0-art-dev' }} +{% endif %} +{% if disconnected.enabled and not disconnected.registry.ca_trusted %} +{{ 'additionalTrustBundle: |' }}{% for line in disconnected.registry.ca_cert.split('\n') %}{{ '\n ' + line }}{% endfor %} +{% endif %} diff --git a/roles/set_inventory/templates/hosts.j2 b/roles/set_inventory/templates/hosts.j2 index 7c4a1f9a..08784805 100644 --- a/roles/set_inventory/templates/hosts.j2 +++ b/roles/set_inventory/templates/hosts.j2 @@ -16,3 +16,8 @@ {{ '[jumphost]' }} {{ env.jumphost.name | string + ' ansible_host=' + env.jumphost.ip | string + ' ansible_user=' + env.jumphost.user | string + ' ansible_become_password=' + env.jumphost.pass | string }} {% endif -%} + +{% if ( disconnected.enabled ) %} +{{ '[mirrorhost]' }} +{{ disconnected.mirroring.host.name | string + ' ansible_host=' + disconnected.mirroring.host.ip | string + ' ansible_user=' + disconnected.mirroring.host.user | string + ' ansible_become_password=' + disconnected.mirroring.host.pass | string }} +{% endif -%}