diff --git a/migration_scripts/README.md b/migration_scripts/README.md new file mode 100644 index 000000000..35d7ecb7a --- /dev/null +++ b/migration_scripts/README.md @@ -0,0 +1,145 @@ +- [Prerequisites](#prerequisites) +- [Migration Scripts](#migration-scripts) +- [Backup Content and Config](#backup-content-and-config) +- [Create Mappings](#create-mappings) +- [Move files](#move-files) +- [Create redirects](#create-redirects) +- [Sidebar Unification](#sidebar-unification) +- [Check That All Pages Are Linked to in the Sidebar](#check-that-all-pages-are-linked-to-in-the-sidebar) +- [Position Files and Folders - Ready For Testing](#position-files-and-folders---ready-for-testing) +- [Testing Changes Locally](#testing-changes-locally) +- [Push Changes](#push-changes) +- [Clean Up](#clean-up) + +# Prerequisites + +```bash +pip3 install openpyxl +``` + +# Migration Scripts + +Fork the developer repository, clone your fork, create a new branch, set the remote: + +```bash +cd ~ +git clone git@github.com:yourusername/developer.git +cd developer +git checkout -b my_new_branch +git remote add upstream https://github.com/fermyon/developer +``` + +Change into the migration_scripts directory and then run the following scripts: + +```bash +cd migration_scripts +``` + +# Backup Content and Config + +Run the following shell script to make it easy to reset (iterate while making decisions about the migration) + +```bash +./backup_content.sh +``` + +# Create Mappings + +Run the create_url_vs_markdown_file_vs_toc_label_mapping.py script. It will generate a spreadsheet called mapping_information.xlsx which will contain all of the current url paths from the existing spin v2 sidebar template and the cloud sidebar template: + +```bash +python3 create_url_vs_markdown_file_vs_toc_label_mapping.py +``` + +Open the mapping_information.xlsx spreadsheet and fill in the: +- unified_url_path (the new url where the resource will be found online i.e. `/spin/v2/quickstart` vs `/latest/quickstart` etc.) +- unified_file_path (the new file path where the markdown file will live i.e. `/spin/v2/quickstart` vs `/latest/quickstart` etc.) +- unified_toc_label (the label that will appear in the ToC for that given markdown file) + +> **Please note** - the new labels have already been decided. See the `sidebar_structure.py` file for label strings to use in the spreadsheet (you are essentially mapping ToC labels to existing markdown files recorded in the spreadsheet). **A label that you add to the spreadsheet's `unified_toc_label` has to exactly match one of the text entries in the `sidebar_structure.py` file. Change one or the other until you have a match to ensure that the generation of the `unified_sidebar.hbs` will succeed.** + +Save the Excel spreadsheet file back to the same location before proceeding with any of the following scripts. + +# Move files + +Run the following scripts to move files to their new location on disk: + +```bash +python3 move_markdown_files_to_unified_location.py +``` + +> If you are not satisfied with the outcome of this script, please run `./reset_content.sh` and try again. + + +Check that no files are left over in either `/spin/v2` or `/cloud` directories (except for cloud changelog which should stay where it is). + +# Create redirects + +Run the following scripts to generate the redirect code for the spin.toml file: + +```bash +python3 create_redirect_syntax_for_manifest.py +``` + +The script above will generate a new `updated_spin.toml` file. + +> If you are not satisfied with the outcome of this script, please re-run it. The `updated_spin.toml` file will be overwritten automatically. + +Go ahead and copy the `spin.toml` file from the migration_scripts directory, over to the application's root: + +```bash +mv updated_spin.toml ../spin.toml +``` + +# Sidebar Unification + +The following script reads the mappings from above and generates a new sidebar `.hbs` file (that will be the replacement sidebar for the original cloud and spin sidebars): + +```bash +python3 create_unified_sidebar.py +``` + +> If you are not satisfied with the outcome of this script, just re-run it. It will override the `latest_sidebar.hbs` file automatically. + +# Check That All Pages Are Linked to in the Sidebar + +Run the following script to make sure that all pages on the website are listed in the sidebar, where a user can click and open the page: + +```bash +python3 check_that_all_pages_are_linked_to_in_sidebar.py +``` + +The above script will list all pages with either a cross ❌ or a check mark ✔ next to the file's name (we want all ✔ before we proceed). + +# Position Files and Folders - Ready For Testing + +```bash +# Put the freshly generated sidebar file into place +mv latest_sidebar.hbs ../templates/latest_sidebar.hbs +``` + +# Testing Changes Locally + +Perform the following to test changes locally: + +```bash +cd ../../developer +npm ci +cd spin-up-hub +npm ci +cd ../../developer +spin build +spin up -e PREVIEW_MODE=1 +``` + +# Push Changes + +```bash +git add . +git commit -S --signoff -m "Adding new unification structure" +git push origin unification +``` + +# Clean Up + +Please run the `./clean_up_after_migration.sh` once the changes are approved and merged. \ No newline at end of file diff --git a/migration_scripts/backup_content.sh b/migration_scripts/backup_content.sh new file mode 100755 index 000000000..78cfe9906 --- /dev/null +++ b/migration_scripts/backup_content.sh @@ -0,0 +1,3 @@ +#!/bin/bash +cp -rp ../content ./content_backup + diff --git a/migration_scripts/check_that_all_pages_are_linked_to_in_sidebar.py b/migration_scripts/check_that_all_pages_are_linked_to_in_sidebar.py new file mode 100644 index 000000000..db113c55d --- /dev/null +++ b/migration_scripts/check_that_all_pages_are_linked_to_in_sidebar.py @@ -0,0 +1,36 @@ +import os +from pathlib import Path + +def construct_paths(base_dir, *sub_dirs): + return [os.path.join(base_dir, *sub_dir) for sub_dir in sub_dirs] + +base_directory = os.path.dirname(__file__) + +markdown_file_subdirectories = [ + ["..", "content", "spin", "v2"], + ["..", "content", "cloud"] +] + +handlebars_template_subdirectories = [ + ["latest_sidebar.hbs"] +] + +directories = construct_paths(base_directory, *markdown_file_subdirectories) +try: + menu = construct_paths(base_directory, *handlebars_template_subdirectories)[0] + if not os.path.exists(menu): + raise FileNotFoundError(f"The file {menu} does not exist. \nPlease run the create_unified_sidebar.py script and try again.") + with open(menu, 'r') as file: + content = file.read() + for directory in directories: + print(f"Checking {directory}...") + for filename in os.listdir(directory): + f = os.path.join(directory, filename) + if os.path.isfile(f): + docFileName = Path(f).stem + if docFileName in content: + print(docFileName + "\U00002714") + else: + print(docFileName + "\U0000274C") +except FileNotFoundError as e: + print(e) diff --git a/migration_scripts/clean_up_after_migration.sh b/migration_scripts/clean_up_after_migration.sh new file mode 100755 index 000000000..09d46bca4 --- /dev/null +++ b/migration_scripts/clean_up_after_migration.sh @@ -0,0 +1,8 @@ +#!/bin/bash +rm -rf ./content_backup +rm -rf ./templates_backup +rm -rf ./spin_toml_backup.toml +rm -rf mapping_information.xlsx +rm -rf latest_sidebar.hbs + + diff --git a/migration_scripts/create_redirect_syntax_for_manifest.py b/migration_scripts/create_redirect_syntax_for_manifest.py new file mode 100644 index 000000000..9fb91a48d --- /dev/null +++ b/migration_scripts/create_redirect_syntax_for_manifest.py @@ -0,0 +1,58 @@ +import re +from urllib.parse import urljoin +import requests +import openpyxl + +def read_excel_to_dict(file_path): + wb = openpyxl.load_workbook(file_path) + ws = wb.active + data = [] + + # Skip the header row + for row in ws.iter_rows(min_row=2, values_only=True): + original_url_path, unified_url_path, original_file_path, unified_file_path, original_toc_label, unified_toc_label = row + data.append({ + "original_url_path": original_url_path, + "unified_url_path": unified_url_path, + "original_file_path": original_file_path, + "unified_file_path": unified_file_path, + "original_toc_label": original_toc_label, + "unified_toc_label": unified_toc_label + }) + return data + +file_path = "mapping_information.xlsx" +mapping_information = read_excel_to_dict(file_path) + +# Step 1: Download the raw version of the spin.toml file +url = "https://raw.githubusercontent.com/fermyon/developer/main/spin.toml" +response = requests.get(url) +spin_toml_content = response.text + +last_redirect_index = spin_toml_content.rfind("[component.redirect") + +new_config_blocks = "" +for entry in mapping_information: + new_config_block = f""" +# Redirect {entry['original_url_path']} to {entry['unified_url_path']} +[component.redirect-{entry['unified_url_path'].strip('/').replace('/', '-')}] +source = "modules/redirect.wasm" +environment = {{ DESTINATION = "urljoin('https://developer.fermyon.com', {entry['unified_url_path']})" }} + +[[trigger.http]] +id = "trigger-{entry['unified_url_path'].strip('/').replace('/', '-')}" +component = "redirect-{entry['unified_url_path'].strip('/').replace('/', '-')}" +route = "{entry['original_url_path']}" +""" + new_config_blocks += new_config_block + +updated_spin_toml_content = ( + spin_toml_content[:last_redirect_index] + + spin_toml_content[last_redirect_index:] + + new_config_blocks +) + +with open("updated_spin.toml", "w") as file: + file.write(updated_spin_toml_content) + +print("Updated spin.toml file has been written as 'updated_spin.toml'.") \ No newline at end of file diff --git a/migration_scripts/create_unified_sidebar.py b/migration_scripts/create_unified_sidebar.py new file mode 100644 index 000000000..9d5c1cda2 --- /dev/null +++ b/migration_scripts/create_unified_sidebar.py @@ -0,0 +1,94 @@ +import re +import requests +import openpyxl +from sidebar_structure import sidebar_structure + +def read_excel_to_dict(file_path): + wb = openpyxl.load_workbook(file_path) + ws = wb.active + data = [] + + # Skip the header row + for row in ws.iter_rows(min_row=2, values_only=True): + original_url_path, unified_url_path, original_file_path, unified_file_path, original_toc_label, unified_toc_label = row + data.append({ + "original_url_path": original_url_path, + "unified_url_path": unified_url_path, + "original_file_path": original_file_path, + "unified_file_path": unified_file_path, + "original_toc_label": original_toc_label, + "unified_toc_label": unified_toc_label + }) + return data + +file_path = "mapping_information.xlsx" +mapping_information = read_excel_to_dict(file_path) + +# Function to generate Handlebars template using actual elements and attributes from the original templates +def generate_handlebars_template(structure, indent=0, idx=0): + handlebars = "" + for key, value in structure.items(): + current_id = f"rd{idx}" + if isinstance(value, dict): + handlebars += ' ' * indent + f'
\n' + elif isinstance(value, list): + if len(value) > 0 and isinstance(value[0], dict): + handlebars += ' ' * indent + f' \n' + else: + handlebars += ' ' * indent + f' \n' + else: + handlebars += ' ' * indent + f' \n' + idx += 1 + return handlebars + +# Generating the combined sidebar Handlebars template +handlebars_template = f""" +