From 8f9f5b2b455228ae7b3a935d81909f593d403d75 Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Wed, 30 Oct 2024 16:29:47 -0500 Subject: [PATCH] Check all edge cases of changelog --- .../update_source_data_schema_changelog.yml | 5 +- changelog/source_data.md | 1 + schemas/dimAccounts_schema.json | 12 -- schemas/dimOffers_schema.json | 5 + schemas/history_assets_schema.json | 2 +- schemas/history_trades_schema.json | 5 - schemas/new_table_schema.json | 118 ++++++++++++++++++ schemas/offers_schema.json | 2 +- schemas/trust_lines_schema.json | 5 - 9 files changed, 130 insertions(+), 25 deletions(-) delete mode 100644 schemas/dimAccounts_schema.json create mode 100644 schemas/new_table_schema.json diff --git a/.github/workflows/update_source_data_schema_changelog.yml b/.github/workflows/update_source_data_schema_changelog.yml index eb6269d3..0564c789 100644 --- a/.github/workflows/update_source_data_schema_changelog.yml +++ b/.github/workflows/update_source_data_schema_changelog.yml @@ -29,7 +29,10 @@ jobs: git clone --branch master https://github.com/stellar/stellar-etl-airflow.git repo_master_copy cp -r repo_master_copy/schemas/ original_schemas/ output=$(python3 scripts/get_source_data_schema_changelog.py) - echo "$output" > changelog/source_data.md + if [ -n "$output" ]; then + existing_changelog=$( changelog/source_data.md + echo "$existing_changelog" >> changelog/source_data.md - name: Commit changes id: commit_changes diff --git a/changelog/source_data.md b/changelog/source_data.md index e69de29b..d7b218ef 100644 --- a/changelog/source_data.md +++ b/changelog/source_data.md @@ -0,0 +1 @@ +## Junk data to check if append works diff --git a/schemas/dimAccounts_schema.json b/schemas/dimAccounts_schema.json deleted file mode 100644 index 67c848ff..00000000 --- a/schemas/dimAccounts_schema.json +++ /dev/null @@ -1,12 +0,0 @@ -[ - { - "mode": "NULLABLE", - "name": "address", - "type": "STRING" - }, - { - "mode": "NULLABLE", - "name": "account_id", - "type": "FLOAT" - } -] diff --git a/schemas/dimOffers_schema.json b/schemas/dimOffers_schema.json index 63b4e757..eb6b4f23 100644 --- a/schemas/dimOffers_schema.json +++ b/schemas/dimOffers_schema.json @@ -38,5 +38,10 @@ "mode": "NULLABLE", "name": "horizon_offer_id", "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "new_id", + "type": "INTEGER" } ] diff --git a/schemas/history_assets_schema.json b/schemas/history_assets_schema.json index c6576a96..a92dc485 100644 --- a/schemas/history_assets_schema.json +++ b/schemas/history_assets_schema.json @@ -7,7 +7,7 @@ { "mode": "NULLABLE", "name": "asset_code", - "type": "STRING" + "type": "INTEGER" }, { "mode": "NULLABLE", diff --git a/schemas/history_trades_schema.json b/schemas/history_trades_schema.json index 494d1893..e61d456a 100644 --- a/schemas/history_trades_schema.json +++ b/schemas/history_trades_schema.json @@ -29,11 +29,6 @@ "name": "selling_asset_issuer", "type": "STRING" }, - { - "mode": "NULLABLE", - "name": "selling_asset_type", - "type": "STRING" - }, { "mode": "NULLABLE", "name": "selling_asset_id", diff --git a/schemas/new_table_schema.json b/schemas/new_table_schema.json new file mode 100644 index 00000000..a0f621bf --- /dev/null +++ b/schemas/new_table_schema.json @@ -0,0 +1,118 @@ +[ + { + "mode": "NULLABLE", + "name": "liquidity_pool_id", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "type", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "fee", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "trustline_count", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "pool_share_count", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "asset_a_type", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "asset_a_code", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "asset_a_issuer", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "asset_a_id", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "asset_a_amount", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "asset_b_type", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "asset_b_code", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "asset_b_issuer", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "asset_b_id", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "asset_b_amount", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "last_modified_ledger", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "ledger_entry_change", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "deleted", + "type": "BOOLEAN" + }, + { + "mode": "NULLABLE", + "name": "batch_id", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "batch_run_date", + "type": "DATETIME" + }, + { + "mode": "NULLABLE", + "name": "batch_insert_ts", + "type": "TIMESTAMP" + }, + { + "mode": "NULLABLE", + "name": "closed_at", + "type": "TIMESTAMP" + }, + { + "mode": "NULLABLE", + "name": "ledger_sequence", + "type": "INTEGER" + } + ] + \ No newline at end of file diff --git a/schemas/offers_schema.json b/schemas/offers_schema.json index e5104ca0..0a8a11e4 100644 --- a/schemas/offers_schema.json +++ b/schemas/offers_schema.json @@ -47,7 +47,7 @@ { "mode": "NULLABLE", "name": "buying_asset_id", - "type": "INTEGER" + "type": "STRING" }, { "mode": "NULLABLE", diff --git a/schemas/trust_lines_schema.json b/schemas/trust_lines_schema.json index 1502d701..c6370df8 100644 --- a/schemas/trust_lines_schema.json +++ b/schemas/trust_lines_schema.json @@ -9,11 +9,6 @@ "name": "account_id", "type": "STRING" }, - { - "mode": "NULLABLE", - "name": "asset_type", - "type": "STRING" - }, { "mode": "NULLABLE", "name": "asset_issuer",