diff --git a/.github/workflows/run_github.yml b/.github/workflows/run_github.yml index 95e40ce..b6ffcd6 100644 --- a/.github/workflows/run_github.yml +++ b/.github/workflows/run_github.yml @@ -14,9 +14,8 @@ env: # DLT specific environment variables # DLT__SECRETS__SOURCES__GITHUB__TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + MOTHERDUCK_DATABASE: ${{ secrets.MOTHERDUCK_DATABASE }} motherduck_token: ${{ secrets.MOTHERDUCK_TOKEN }} - - # Optional: Add any other environment variables needed PYTHONUNBUFFERED: "1" jobs: diff --git a/.github/workflows/run_slack_workflow.yml b/.github/workflows/run_slack_workflow.yml index 1e98a8e..b00ef9a 100644 --- a/.github/workflows/run_slack_workflow.yml +++ b/.github/workflows/run_slack_workflow.yml @@ -13,8 +13,8 @@ name: Run Slack Pipeline env: # DLT specific environment variables SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} + MOTHERDUCK_DATABASE: ${{ secrets.MOTHERDUCK_DATABASE }} motherduck_token: ${{ secrets.MOTHERDUCK_TOKEN }} - # Remove the DLT__ prefixed version since we're using direct env vars PYTHONUNBUFFERED: "1" jobs: diff --git a/.github/workflows/run_twitter.yml b/.github/workflows/run_twitter.yml index e1f8da4..d0e9655 100644 --- a/.github/workflows/run_twitter.yml +++ b/.github/workflows/run_twitter.yml @@ -13,9 +13,8 @@ name: Run Twitter Pipeline env: # DLT specific environment variables TWITTER_BEARER_TOKEN: ${{ secrets.TWITTER_BEARER_TOKEN }} + MOTHERDUCK_DATABASE: ${{ secrets.MOTHERDUCK_DATABASE }} motherduck_token: ${{ secrets.MOTHERDUCK_TOKEN }} - - # Optional: Add any other environment variables needed PYTHONUNBUFFERED: "1" jobs: @@ -51,5 +50,14 @@ jobs: pipeline/pyproject.toml pipeline/uv.lock + - name: Setup DLT credentials + run: | + mkdir -p pipeline/.dlt + cat << EOF > pipeline/.dlt/secrets.toml + [destination.motherduck.credentials] + database = "nf_core_dlt" + password = "${{ secrets.MOTHERDUCK_TOKEN }}" + EOF + - name: Run Twitter pipeline run: uv run python twitter_pipeline.py