From 9d5d5d7a966688d29f8b5ca6cb382d872becf73d Mon Sep 17 00:00:00 2001
From: Amisha Singla <amishas157@gmail.com>
Date: Wed, 30 Oct 2024 16:07:50 -0500
Subject: [PATCH] Add workflow to calculate source data schema diff

---
 .../update_source_data_schema_changelog.yml   | 43 ++++++++++++
 changelog/source_data.md                      |  0
 schemas/account_signers_schema.json           |  5 ++
 scripts/get_source_data_schema_changelog.py   | 67 +++++++++++++++++++
 4 files changed, 115 insertions(+)
 create mode 100644 .github/workflows/update_source_data_schema_changelog.yml
 create mode 100644 changelog/source_data.md
 create mode 100644 scripts/get_source_data_schema_changelog.py

diff --git a/.github/workflows/update_source_data_schema_changelog.yml b/.github/workflows/update_source_data_schema_changelog.yml
new file mode 100644
index 00000000..83cae905
--- /dev/null
+++ b/.github/workflows/update_source_data_schema_changelog.yml
@@ -0,0 +1,43 @@
+name: Update changelog for Source Data
+
+on:
+  push:
+    branches:
+      - patch/update-source-data-schema-changelog
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref_protected == 'true' && github.sha || github.ref }}-{{ github.event_name }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v2
+
+      - name: Run Bash Script
+        run: |
+          cd $GITHUB_WORKSPACE
+          cp -r schemas/ new_schemas/
+          git checkout main
+          cp -r schemas/ old_schemas
+          output=$(python3 scripts/get_source_data_schema_changelog.py)
+          echo "$output" > changelog/source_data.md
+
+      - name: Commit changes
+        id: commit_changes
+        run: |
+          git add changelog/source_data.md
+          if git commit -m "Update changelog for Source data"; then
+            echo "Changes committed."
+            echo "changes_committed=true" >> $GITHUB_OUTPUT
+          else
+            echo "No changes to commit."
+            echo "changes_committed=false" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Push branch
+        if: steps.commit_changes.outputs.changes_committed == 'true'
+        run: |
+          git push
diff --git a/changelog/source_data.md b/changelog/source_data.md
new file mode 100644
index 00000000..e69de29b
diff --git a/schemas/account_signers_schema.json b/schemas/account_signers_schema.json
index fd92d8a6..63f7796b 100644
--- a/schemas/account_signers_schema.json
+++ b/schemas/account_signers_schema.json
@@ -58,5 +58,10 @@
     "mode": "NULLABLE",
     "name": "ledger_sequence",
     "type": "INTEGER"
+  },
+  {
+    "mode": "NULLABLE",
+    "name": "ledger_sequences",
+    "type": "INTEGER"
   }
 ]
diff --git a/scripts/get_source_data_schema_changelog.py b/scripts/get_source_data_schema_changelog.py
new file mode 100644
index 00000000..df0d2092
--- /dev/null
+++ b/scripts/get_source_data_schema_changelog.py
@@ -0,0 +1,67 @@
+import os
+import json
+
+# Directories containing old and new schemas
+old_schemas_dir = "original_schemas"
+new_schemas_dir = "new_schemas"
+
+# Check if the directories exist
+if not os.path.exists(old_schemas_dir):
+    print(f"Directory {old_schemas_dir} does not exist.")
+    exit(1)
+
+if not os.path.exists(new_schemas_dir):
+    print(f"Directory {new_schemas_dir} does not exist.")
+    exit(1)
+
+def read_json_file(filepath):
+    with open(filepath, 'r') as f:
+        return json.load(f)
+
+new_schemas = [file.replace("_schema.json", "") for file in os.listdir(new_schemas_dir)]
+old_schemas = [file.replace("_schema.json", "") for file in os.listdir(old_schemas_dir)]
+
+tables_added = [model for model in new_schemas if model not in old_schemas]
+tables_removed = [model for model in old_schemas if model not in new_schemas]
+common_tables = [model for model in new_schemas if model in old_schemas]
+
+if tables_added:
+    print("")
+    print("## Tables Added:")
+    print([table for table in tables_added])
+
+if tables_removed:
+    print("")
+    print("## Tables Removed:")
+    print([table for table in tables_removed])
+
+for schema in common_tables:
+    old_file_path = os.path.join(old_schemas_dir, schema + '_schema.json')
+    new_file_path = os.path.join(new_schemas_dir, schema + '_schema.json')
+
+    new_data = read_json_file(new_file_path)
+    old_data = read_json_file(old_file_path)
+
+    old_dict = {item['name']: item for item in old_data}
+    new_dict = {item['name']: item for item in new_data}
+
+    added = [new_dict[name]['name'] for name in new_dict if name not in old_dict]
+    deleted = [old_dict[name]['name'] for name in old_dict if name not in new_dict]
+    type_changed = [
+        (new_dict[name]['name'], new_dict[name]['type'], old_dict[name]['type']) for name in new_dict 
+        if name in old_dict and new_dict[name]['type'] != old_dict[name]['type']
+    ]
+
+    if added or deleted or type_changed:
+        print("")
+        print(f"## {schema}")
+
+    if added:
+        print(f'**Added columns:** {[field for field in added]}')
+
+    if deleted:
+        print(f'**Deleted columns:** {[field for field in deleted]}')
+
+    if type_changed:
+        for (field, new_type, old_type) in type_changed:
+            print(f'Type changed for column {field} from {old_type} to {new_type}')