diff --git a/migration_scripts/README.md b/migration_scripts/README.md new file mode 100644 index 000000000..35d7ecb7a --- /dev/null +++ b/migration_scripts/README.md @@ -0,0 +1,145 @@ +- [Prerequisites](#prerequisites) +- [Migration Scripts](#migration-scripts) +- [Backup Content and Config](#backup-content-and-config) +- [Create Mappings](#create-mappings) +- [Move files](#move-files) +- [Create redirects](#create-redirects) +- [Sidebar Unification](#sidebar-unification) +- [Check That All Pages Are Linked to in the Sidebar](#check-that-all-pages-are-linked-to-in-the-sidebar) +- [Position Files and Folders - Ready For Testing](#position-files-and-folders---ready-for-testing) +- [Testing Changes Locally](#testing-changes-locally) +- [Push Changes](#push-changes) +- [Clean Up](#clean-up) + +# Prerequisites + +```bash +pip3 install openpyxl +``` + +# Migration Scripts + +Fork the developer repository, clone your fork, create a new branch, set the remote: + +```bash +cd ~ +git clone git@github.com:yourusername/developer.git +cd developer +git checkout -b my_new_branch +git remote add upstream https://github.com/fermyon/developer +``` + +Change into the migration_scripts directory and then run the following scripts: + +```bash +cd migration_scripts +``` + +# Backup Content and Config + +Run the following shell script to make it easy to reset (iterate while making decisions about the migration) + +```bash +./backup_content.sh +``` + +# Create Mappings + +Run the create_url_vs_markdown_file_vs_toc_label_mapping.py script. It will generate a spreadsheet called mapping_information.xlsx which will contain all of the current url paths from the existing spin v2 sidebar template and the cloud sidebar template: + +```bash +python3 create_url_vs_markdown_file_vs_toc_label_mapping.py +``` + +Open the mapping_information.xlsx spreadsheet and fill in the: +- unified_url_path (the new url where the resource will be found online i.e. `/spin/v2/quickstart` vs `/latest/quickstart` etc.) +- unified_file_path (the new file path where the markdown file will live i.e. `/spin/v2/quickstart` vs `/latest/quickstart` etc.) +- unified_toc_label (the label that will appear in the ToC for that given markdown file) + +> **Please note** - the new labels have already been decided. See the `sidebar_structure.py` file for label strings to use in the spreadsheet (you are essentially mapping ToC labels to existing markdown files recorded in the spreadsheet). **A label that you add to the spreadsheet's `unified_toc_label` has to exactly match one of the text entries in the `sidebar_structure.py` file. Change one or the other until you have a match to ensure that the generation of the `unified_sidebar.hbs` will succeed.** + +Save the Excel spreadsheet file back to the same location before proceeding with any of the following scripts. + +# Move files + +Run the following scripts to move files to their new location on disk: + +```bash +python3 move_markdown_files_to_unified_location.py +``` + +> If you are not satisfied with the outcome of this script, please run `./reset_content.sh` and try again. + + +Check that no files are left over in either `/spin/v2` or `/cloud` directories (except for cloud changelog which should stay where it is). + +# Create redirects + +Run the following scripts to generate the redirect code for the spin.toml file: + +```bash +python3 create_redirect_syntax_for_manifest.py +``` + +The script above will generate a new `updated_spin.toml` file. + +> If you are not satisfied with the outcome of this script, please re-run it. The `updated_spin.toml` file will be overwritten automatically. + +Go ahead and copy the `spin.toml` file from the migration_scripts directory, over to the application's root: + +```bash +mv updated_spin.toml ../spin.toml +``` + +# Sidebar Unification + +The following script reads the mappings from above and generates a new sidebar `.hbs` file (that will be the replacement sidebar for the original cloud and spin sidebars): + +```bash +python3 create_unified_sidebar.py +``` + +> If you are not satisfied with the outcome of this script, just re-run it. It will override the `latest_sidebar.hbs` file automatically. + +# Check That All Pages Are Linked to in the Sidebar + +Run the following script to make sure that all pages on the website are listed in the sidebar, where a user can click and open the page: + +```bash +python3 check_that_all_pages_are_linked_to_in_sidebar.py +``` + +The above script will list all pages with either a cross ❌ or a check mark ✔ next to the file's name (we want all ✔ before we proceed). + +# Position Files and Folders - Ready For Testing + +```bash +# Put the freshly generated sidebar file into place +mv latest_sidebar.hbs ../templates/latest_sidebar.hbs +``` + +# Testing Changes Locally + +Perform the following to test changes locally: + +```bash +cd ../../developer +npm ci +cd spin-up-hub +npm ci +cd ../../developer +spin build +spin up -e PREVIEW_MODE=1 +``` + +# Push Changes + +```bash +git add . +git commit -S --signoff -m "Adding new unification structure" +git push origin unification +``` + +# Clean Up + +Please run the `./clean_up_after_migration.sh` once the changes are approved and merged. \ No newline at end of file diff --git a/migration_scripts/backup_content.sh b/migration_scripts/backup_content.sh new file mode 100755 index 000000000..78cfe9906 --- /dev/null +++ b/migration_scripts/backup_content.sh @@ -0,0 +1,3 @@ +#!/bin/bash +cp -rp ../content ./content_backup + diff --git a/migration_scripts/check_that_all_pages_are_linked_to_in_sidebar.py b/migration_scripts/check_that_all_pages_are_linked_to_in_sidebar.py new file mode 100644 index 000000000..db113c55d --- /dev/null +++ b/migration_scripts/check_that_all_pages_are_linked_to_in_sidebar.py @@ -0,0 +1,36 @@ +import os +from pathlib import Path + +def construct_paths(base_dir, *sub_dirs): + return [os.path.join(base_dir, *sub_dir) for sub_dir in sub_dirs] + +base_directory = os.path.dirname(__file__) + +markdown_file_subdirectories = [ + ["..", "content", "spin", "v2"], + ["..", "content", "cloud"] +] + +handlebars_template_subdirectories = [ + ["latest_sidebar.hbs"] +] + +directories = construct_paths(base_directory, *markdown_file_subdirectories) +try: + menu = construct_paths(base_directory, *handlebars_template_subdirectories)[0] + if not os.path.exists(menu): + raise FileNotFoundError(f"The file {menu} does not exist. \nPlease run the create_unified_sidebar.py script and try again.") + with open(menu, 'r') as file: + content = file.read() + for directory in directories: + print(f"Checking {directory}...") + for filename in os.listdir(directory): + f = os.path.join(directory, filename) + if os.path.isfile(f): + docFileName = Path(f).stem + if docFileName in content: + print(docFileName + "\U00002714") + else: + print(docFileName + "\U0000274C") +except FileNotFoundError as e: + print(e) diff --git a/migration_scripts/clean_up_after_migration.sh b/migration_scripts/clean_up_after_migration.sh new file mode 100755 index 000000000..09d46bca4 --- /dev/null +++ b/migration_scripts/clean_up_after_migration.sh @@ -0,0 +1,8 @@ +#!/bin/bash +rm -rf ./content_backup +rm -rf ./templates_backup +rm -rf ./spin_toml_backup.toml +rm -rf mapping_information.xlsx +rm -rf latest_sidebar.hbs + + diff --git a/migration_scripts/create_redirect_syntax_for_manifest.py b/migration_scripts/create_redirect_syntax_for_manifest.py new file mode 100644 index 000000000..9fb91a48d --- /dev/null +++ b/migration_scripts/create_redirect_syntax_for_manifest.py @@ -0,0 +1,58 @@ +import re +from urllib.parse import urljoin +import requests +import openpyxl + +def read_excel_to_dict(file_path): + wb = openpyxl.load_workbook(file_path) + ws = wb.active + data = [] + + # Skip the header row + for row in ws.iter_rows(min_row=2, values_only=True): + original_url_path, unified_url_path, original_file_path, unified_file_path, original_toc_label, unified_toc_label = row + data.append({ + "original_url_path": original_url_path, + "unified_url_path": unified_url_path, + "original_file_path": original_file_path, + "unified_file_path": unified_file_path, + "original_toc_label": original_toc_label, + "unified_toc_label": unified_toc_label + }) + return data + +file_path = "mapping_information.xlsx" +mapping_information = read_excel_to_dict(file_path) + +# Step 1: Download the raw version of the spin.toml file +url = "https://raw.githubusercontent.com/fermyon/developer/main/spin.toml" +response = requests.get(url) +spin_toml_content = response.text + +last_redirect_index = spin_toml_content.rfind("[component.redirect") + +new_config_blocks = "" +for entry in mapping_information: + new_config_block = f""" +# Redirect {entry['original_url_path']} to {entry['unified_url_path']} +[component.redirect-{entry['unified_url_path'].strip('/').replace('/', '-')}] +source = "modules/redirect.wasm" +environment = {{ DESTINATION = "urljoin('https://developer.fermyon.com', {entry['unified_url_path']})" }} + +[[trigger.http]] +id = "trigger-{entry['unified_url_path'].strip('/').replace('/', '-')}" +component = "redirect-{entry['unified_url_path'].strip('/').replace('/', '-')}" +route = "{entry['original_url_path']}" +""" + new_config_blocks += new_config_block + +updated_spin_toml_content = ( + spin_toml_content[:last_redirect_index] + + spin_toml_content[last_redirect_index:] + + new_config_blocks +) + +with open("updated_spin.toml", "w") as file: + file.write(updated_spin_toml_content) + +print("Updated spin.toml file has been written as 'updated_spin.toml'.") \ No newline at end of file diff --git a/migration_scripts/create_unified_sidebar.py b/migration_scripts/create_unified_sidebar.py new file mode 100644 index 000000000..9d5c1cda2 --- /dev/null +++ b/migration_scripts/create_unified_sidebar.py @@ -0,0 +1,94 @@ +import re +import requests +import openpyxl +from sidebar_structure import sidebar_structure + +def read_excel_to_dict(file_path): + wb = openpyxl.load_workbook(file_path) + ws = wb.active + data = [] + + # Skip the header row + for row in ws.iter_rows(min_row=2, values_only=True): + original_url_path, unified_url_path, original_file_path, unified_file_path, original_toc_label, unified_toc_label = row + data.append({ + "original_url_path": original_url_path, + "unified_url_path": unified_url_path, + "original_file_path": original_file_path, + "unified_file_path": unified_file_path, + "original_toc_label": original_toc_label, + "unified_toc_label": unified_toc_label + }) + return data + +file_path = "mapping_information.xlsx" +mapping_information = read_excel_to_dict(file_path) + +# Function to generate Handlebars template using actual elements and attributes from the original templates +def generate_handlebars_template(structure, indent=0, idx=0): + handlebars = "" + for key, value in structure.items(): + current_id = f"rd{idx}" + if isinstance(value, dict): + handlebars += ' ' * indent + f'
\n' + handlebars += ' ' * indent + f' \n' + handlebars += ' ' * indent + f' \n' + handlebars += ' ' * indent + f' \n' + handlebars += ' ' * indent + f'
\n' + elif isinstance(value, list): + if len(value) > 0 and isinstance(value[0], dict): + handlebars += ' ' * indent + f'
\n' + handlebars += ' ' * indent + f' \n' + handlebars += ' ' * indent + f' \n' + handlebars += ' ' * indent + f' \n' + handlebars += ' ' * indent + f'
\n' + else: + handlebars += ' ' * indent + f'
\n' + handlebars += ' ' * indent + f' \n' + handlebars += ' ' * indent + f' \n' + handlebars += ' ' * indent + f' \n' + handlebars += ' ' * indent + f'
\n' + else: + handlebars += ' ' * indent + f'
\n' + handlebars += ' ' * indent + f' \n' + handlebars += ' ' * indent + f' \n' + handlebars += ' ' * indent + f' \n' + handlebars += ' ' * indent + f'
\n' + idx += 1 + return handlebars + +# Generating the combined sidebar Handlebars template +handlebars_template = f""" +
+ {generate_handlebars_template(sidebar_structure)} +
+""" + +# Save the Handlebars template to a file +with open("latest_sidebar.hbs", "w") as file: + file.write(handlebars_template) + +print("Combined sidebar Handlebars template generated successfully.") \ No newline at end of file diff --git a/migration_scripts/create_url_vs_markdown_file_vs_toc_label_mapping.py b/migration_scripts/create_url_vs_markdown_file_vs_toc_label_mapping.py new file mode 100644 index 000000000..e5c508fd0 --- /dev/null +++ b/migration_scripts/create_url_vs_markdown_file_vs_toc_label_mapping.py @@ -0,0 +1,61 @@ +import re +import openpyxl +from openpyxl import Workbook +import requests + +# Remove formatting from the Handlebars template +def remove_formatting(text): + text = re.sub(r'\s+', ' ', text) + text = re.sub(r'\t+', '', text) + text = re.sub(r'\n+', '', text) + return text + +# Function to fetch the content of a file from a GitHub repository +def fetch_file_from_github(repo, filepath): + url = f"https://raw.githubusercontent.com/{repo}/main/{filepath}" + response = requests.get(url) + if response.status_code == 200: + return response.text + else: + raise Exception(f"Failed to fetch {filepath} from {repo}") + +repo = "fermyon/developer" +original_spin_sidebar_v2 = fetch_file_from_github(repo, "templates/spin_sidebar_v2.hbs") +spin_sidebar_v2 = remove_formatting(original_spin_sidebar_v2) +original_cloud_sidebar = fetch_file_from_github(repo, "templates/cloud_sidebar.hbs") +cloud_sidebar = remove_formatting(original_cloud_sidebar) + +#print(f"Cleaned Spin Sidebar Content\n{spin_sidebar_v2}") +#print(f"Cleaned Cloud Sidebar Content\n{cloud_sidebar}") + +# Parsing the original Handlebars templates to extract links +def extract_links(handlebars_template): + pattern = r'"{{site\.info\.base_url}}(/(?:spin/v2|cloud|wasm-languages|static/image)/[\.a-zA-Z0-9/-]*)">([^<]+)' + matches = re.findall(pattern, handlebars_template) + print(f"Found {matches} links in the sidebar.") + return [match for match in matches] + +spin_links = extract_links(spin_sidebar_v2) +print(f"Extracted {len(spin_links)} links from the Spin sidebar.") +cloud_links = extract_links(cloud_sidebar) +print(f"Extracted {len(cloud_links)} links from the Cloud sidebar.") + +# Combine the links from both templates +all_links = spin_links + cloud_links + +# Create a new Excel workbook and select the active worksheet +wb = Workbook() +ws = wb.active +ws.title = "Links" + +# Write the headers +ws.append(["original_url_path", "unified_url_path", "original_file_path", "unified_file_path", "original_toc_label", "unified_toc_label"]) + +# Write the links to column A and leave columns B and C blank +for link in all_links: + ws.append([link[0], "", "", link[0], link[1], ""]) + +# Save the workbook to a file +wb.save("mapping_information.xlsx") + +print("Links have been written to extracted_links.xlsx") \ No newline at end of file diff --git a/migration_scripts/latest_sidebar.hbs b/migration_scripts/latest_sidebar.hbs new file mode 100644 index 000000000..3ed9aaed2 --- /dev/null +++ b/migration_scripts/latest_sidebar.hbs @@ -0,0 +1,224 @@ + +
+
+ + + +
+
+ + + +
+
+ + + +
+
+ + + +
+
+ + + +
+
+ + + +
+
+ + + +
+ +
diff --git a/migration_scripts/mapping_information.xlsx b/migration_scripts/mapping_information.xlsx new file mode 100644 index 000000000..892e4baca Binary files /dev/null and b/migration_scripts/mapping_information.xlsx differ diff --git a/migration_scripts/move_markdown_files_to_unified_location.py b/migration_scripts/move_markdown_files_to_unified_location.py new file mode 100644 index 000000000..a0a61ca93 --- /dev/null +++ b/migration_scripts/move_markdown_files_to_unified_location.py @@ -0,0 +1,72 @@ +import re +import os +import shutil +import requests +import openpyxl +from openpyxl import Workbook +from pathlib import Path +from urllib.parse import urljoin + +def construct_paths(base_dir, *sub_dirs): + return [os.path.join(base_dir, *sub_dir) for sub_dir in sub_dirs] + +base_directory = os.path.dirname(__file__) + +markdown_file_subdirectories = [ + ["..", "content"] +] + +def read_excel_to_dict(file_path): + wb = openpyxl.load_workbook(file_path) + ws = wb.active + data = [] + + # Skip the header row + for row in ws.iter_rows(min_row=2, values_only=True): + original_url_path, unified_url_path, original_file_path, unified_file_path, original_toc_label, unified_toc_label = row + data.append({ + "original_url_path": original_url_path, + "unified_url_path": unified_url_path, + "original_file_path": original_file_path, + "unified_file_path": unified_file_path, + "original_toc_label": original_toc_label, + "unified_toc_label": unified_toc_label + }) + return data + +def update_url_in_markdown(file_path, old_url_path, new_url_path): + # Read the file content + with open(file_path, 'r', encoding='utf-8') as file: + content = file.read() + + # Define the old URL pattern and the new URL string + old_url_pattern = f'https://github.com/fermyon/developer/blob/main/content/{old_url_path}.md' + new_url_string = f'https://github.com/fermyon/developer/blob/main/content/{new_url_path}.md' + + # Update the URL in the front matter + updated_content = content.replace(old_url_pattern, new_url_string) + + # Write the updated content back to the file + with open(file_path, 'w', encoding='utf-8') as file: + file.write(updated_content) + + print(f"Updated URL in: {file_path}") + + +file_path = "mapping_information.xlsx" +mapping_information = read_excel_to_dict(file_path) + +directory_prefix = construct_paths(base_directory, *markdown_file_subdirectories)[0] + +for entry in mapping_information: + original_file_path = os.path.join(directory_prefix, entry['original_file_path']) + unified_file_path = os.path.join(directory_prefix, entry['unified_file_path']) + # Update the URL in the markdown file + try: + update_url_in_markdown(original_file_path, entry['original_file_path'], entry['unified_file_path']) + shutil.move(original_file_path, unified_file_path) + print(f"Moved: {original_file_path} to {unified_file_path}") + except FileNotFoundError: + print(f"File not found: {original_file_path}") + except Exception as e: + print(f"Error updating {original_file_path} and {unified_file_path}: {e}") \ No newline at end of file diff --git a/migration_scripts/reset_content.sh b/migration_scripts/reset_content.sh new file mode 100755 index 000000000..fe8898c11 --- /dev/null +++ b/migration_scripts/reset_content.sh @@ -0,0 +1,4 @@ +#!/bin/bash +rm -rf ../content +cp -rp ./content_backup ../content + diff --git a/migration_scripts/sidebar_structure.py b/migration_scripts/sidebar_structure.py new file mode 100644 index 000000000..f74c4d733 --- /dev/null +++ b/migration_scripts/sidebar_structure.py @@ -0,0 +1,128 @@ +sidebar_structure = { + "Quick Start - Create Your First App With Spin": [], + "Wasm Language Guides": [ + "Language Support Overview", + "Rust", + "Go", + "Javascript", + "Python", + "Other Languages" + ], + "Creating Apps With Spin": { + "Get Started": [ + "Introduction", + "Install", + "Upgrade", + "Resources (Discord link etc.)" + ], + "Concepts": [ + "Learn About Spin App Fundamentals - Trigger, Components, and Application Manifest", + "Learn About Spin Internal Data Layout", + "Learn About Spin App Swappable Resources - Runtime configuration" + ], + "How To": { + "Applications": [ + "Writing Applications", + "Structuring Applications", + "Compiling Applications", + "Running Applications", + "Publishing and Distribution", + "Observing Applications", + "Troubleshooting Application Development" + ], + "Triggers": [ + "Triggers", + "The HTTP Trigger", + "The Redis Trigger", + "Building Custom Triggers - Extending and Embedding Spin" + ], + "API Guides": { + "API Support Overview": [], + "Storage": [ + "Key Value Store", + "SQLite Storage", + "Redis Storage", + "Relational Database Storage" + ], + "Outbound connectivity": [ + "Making HTTP Requests", + "MQTT Messaging" + ], + "Serverless AI": [], + "Application Variables": [] + }, + "Tools": { + "Templates": [ + "Creating Spin Templates", + "Managing Templates" + ], + "Plugins": [ + "Creating Spin Plugins", + "Managing Plugins" + ], + "Triggers": [] + } + }, + "Tutorials": [ + "Build Your First Serverless AI App", + "Sentiment Analysis With Serverless AI", + "Building a URL Shortener With Spin", + "Spin Apps in Registries", + "Spin Key-Value Store" + ] + }, + "Deploying your Spin App": { + "Fermyon Cloud": [ + "Quickstart", + "Introduction", + { + "Applications": [ + "Develop a Spin Application", + "Deploy an Application", + "Upgrade an Application", + "Delete an Application" + ], + "Concepts": [ + "The Fermyon Cloud", + "Deployment Concepts", + "Custom Domain", + "Linking Apps To Resources Using Labels" + ], + "Tutorials": [ + "Using VS Code Extension", + "Cloud Key-Value Store", + "Persisting Data With SQLite Database", + "Persisting Data With Redis", + "Persisting Data With PostgreSQL", + "Deploying Spin Apps Using GitHub Action", + "Apply Custom Fermyon Subdomain", + "Apply Custom Domain", + "Configuring App Variables and Secrets" + ], + "Support": [ + "The User Settings Screen", + "Quotas, Limitations, and Technical FAQ", + "Pricing and Billing", + "How to Get Help?" + ] + } + ], + "Fermyon Platform For Kubernetes": [ + "Kubernetes", + "Spin on Kubernetes", + "Spin in Pods (Legacy)" + ] + }, + "References": [ + "CLI Reference", + "Application Manifest (Version 1) Reference", + "Application Manifest Reference" + ], + "Contributing": [ + "Contributing to Docs", + "Contributing to Spin", + "Spin Improvement Proposals", + "Contributing to SpinKube" + ], + "Resources": [] +}