ci: Set motherduck db

nf-core · Jan 8, 2025 · 9cd6de6 · 9cd6de6
1 parent f20d559
commit 9cd6de6
Show file tree

Hide file tree

Showing 3 changed files with 12 additions and 5 deletions.
diff --git a/.github/workflows/run_github.yml b/.github/workflows/run_github.yml
@@ -14,9 +14,8 @@ env:
   # DLT specific environment variables
   # DLT__SECRETS__SOURCES__GITHUB__TOKEN: ${{ secrets.GITHUB_TOKEN }}
   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  MOTHERDUCK_DATABASE: ${{ secrets.MOTHERDUCK_DATABASE }}
   motherduck_token: ${{ secrets.MOTHERDUCK_TOKEN }}
-
-  # Optional: Add any other environment variables needed
   PYTHONUNBUFFERED: "1"
 
 jobs:

diff --git a/.github/workflows/run_slack_workflow.yml b/.github/workflows/run_slack_workflow.yml
@@ -13,8 +13,8 @@ name: Run Slack Pipeline
 env:
   # DLT specific environment variables
   SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+  MOTHERDUCK_DATABASE: ${{ secrets.MOTHERDUCK_DATABASE }}
   motherduck_token: ${{ secrets.MOTHERDUCK_TOKEN }}
-  # Remove the DLT__ prefixed version since we're using direct env vars
   PYTHONUNBUFFERED: "1"
 
 jobs:

diff --git a/.github/workflows/run_twitter.yml b/.github/workflows/run_twitter.yml
@@ -13,9 +13,8 @@ name: Run Twitter Pipeline
 env:
   # DLT specific environment variables
   TWITTER_BEARER_TOKEN: ${{ secrets.TWITTER_BEARER_TOKEN }}
+  MOTHERDUCK_DATABASE: ${{ secrets.MOTHERDUCK_DATABASE }}
   motherduck_token: ${{ secrets.MOTHERDUCK_TOKEN }}
-
-  # Optional: Add any other environment variables needed
   PYTHONUNBUFFERED: "1"
 
 jobs:
@@ -51,5 +50,14 @@ jobs:
             pipeline/pyproject.toml
             pipeline/uv.lock
 
+      - name: Setup DLT credentials
+        run: |
+          mkdir -p pipeline/.dlt
+          cat << EOF > pipeline/.dlt/secrets.toml
+          [destination.motherduck.credentials]
+          database = "nf_core_dlt"
+          password = "${{ secrets.MOTHERDUCK_TOKEN }}"
+          EOF
+          
       - name: Run Twitter pipeline
         run: uv run python twitter_pipeline.py