From 601294bc5694909b48d218d5f18259234a8eef34 Mon Sep 17 00:00:00 2001 From: fujistone Date: Mon, 9 Dec 2024 16:43:58 +0100 Subject: [PATCH 1/3] feat(arcueil) : adding arcueil dbt project --- .github/workflows/arcueil_models_run.yml | 46 +++++++++++++++++++ .../arcueil_models_run_preproduction.yml | 46 +++++++++++++++++++ projects/arcueil/.gitignore | 4 ++ projects/arcueil/README.md | 15 ++++++ projects/arcueil/analyses/.gitkeep | 0 projects/arcueil/config.yml | 3 ++ projects/arcueil/dbt_project.yml | 36 +++++++++++++++ projects/arcueil/macros/.gitkeep | 0 projects/arcueil/packages.yml | 4 ++ projects/arcueil/profiles.yml | 31 +++++++++++++ projects/arcueil/requirements.txt | 2 + projects/arcueil/seeds/.gitkeep | 0 projects/arcueil/snapshots/.gitkeep | 0 projects/arcueil/tests/.gitkeep | 0 14 files changed, 187 insertions(+) create mode 100644 .github/workflows/arcueil_models_run.yml create mode 100644 .github/workflows/arcueil_models_run_preproduction.yml create mode 100644 projects/arcueil/.gitignore create mode 100644 projects/arcueil/README.md create mode 100644 projects/arcueil/analyses/.gitkeep create mode 100644 projects/arcueil/config.yml create mode 100644 projects/arcueil/dbt_project.yml create mode 100644 projects/arcueil/macros/.gitkeep create mode 100644 projects/arcueil/packages.yml create mode 100644 projects/arcueil/profiles.yml create mode 100644 projects/arcueil/requirements.txt create mode 100644 projects/arcueil/seeds/.gitkeep create mode 100644 projects/arcueil/snapshots/.gitkeep create mode 100644 projects/arcueil/tests/.gitkeep diff --git a/.github/workflows/arcueil_models_run.yml b/.github/workflows/arcueil_models_run.yml new file mode 100644 index 00000000..2a0f928c --- /dev/null +++ b/.github/workflows/arcueil_models_run.yml @@ -0,0 +1,46 @@ +name: arcueil_models_run + +on: + workflow_dispatch: + +env: + DBNAME: ${{ secrets.ARCUEIL_DBNAME_PROD }} + HOST: ${{ secrets.HOST }} + PASSWORD: ${{ secrets.PASSWORD }} + PORT: ${{ secrets.PORT }} + TARGET_SCHEMA: ${{ secrets.SCHEMA_PROD }} + USER: ${{ secrets.USER }} + PARTICIPATION_HOST_NAME: ${{ secrets.ARCUEIL_PARTICIPATION_HOST_NAME }} + WORKING_DIRECTORY: ./projects/arcueil + + +jobs: + arcueil_models_run: + name: arcueil_models_run + runs-on: ubuntu-latest + defaults: + run: + working-directory: ${{ env.WORKING_DIRECTORY }} + + steps: + - name: Check out + uses: actions/checkout@master + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.12.x" + cache: 'pip' + + - name: Install dependencies + run: | + pip install -r requirements.txt + dbt deps --target prod + + - name: Run dbt models + run: dbt run --target prod + + - name: Test dbt models + run: dbt test --target prod + + diff --git a/.github/workflows/arcueil_models_run_preproduction.yml b/.github/workflows/arcueil_models_run_preproduction.yml new file mode 100644 index 00000000..caa2d4b1 --- /dev/null +++ b/.github/workflows/arcueil_models_run_preproduction.yml @@ -0,0 +1,46 @@ +name: arcueil_models_run_preproduction + +on: + workflow_dispatch: + +env: + DBNAME: ${{ secrets.ARCUEIL_DBNAME_PROD }} + HOST_PREPROD: ${{ secrets.HOST_PREPROD }} + PASSWORD_PREPROD: ${{ secrets.PASSWORD_PREPROD }} + PORT_PREPROD: ${{ secrets.PORT_PREPROD }} + TARGET_SCHEMA: ${{ secrets.SCHEMA_PROD }} + USER_PREPROD: ${{ secrets.USER_PREPROD }} + PARTICIPATION_HOST_NAME: ${{ secrets.ARCUEIL_PARTICIPATION_HOST_NAME }} + WORKING_DIRECTORY: ./projects/arcueil + + +jobs: + arcueil_models_run: + name: arcueil_models_run_preproduction + runs-on: ubuntu-latest + defaults: + run: + working-directory: ${{ env.WORKING_DIRECTORY }} + + steps: + - name: Check out + uses: actions/checkout@master + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.12.x" + cache: 'pip' + + - name: Install dependencies + run: | + pip install -r requirements.txt + dbt deps --target preprod + + - name: Run dbt models + run: dbt run --target preprod + + - name: Test dbt models + run: dbt test --target preprod + + diff --git a/projects/arcueil/.gitignore b/projects/arcueil/.gitignore new file mode 100644 index 00000000..49f147cb --- /dev/null +++ b/projects/arcueil/.gitignore @@ -0,0 +1,4 @@ + +target/ +dbt_packages/ +logs/ diff --git a/projects/arcueil/README.md b/projects/arcueil/README.md new file mode 100644 index 00000000..7874ac84 --- /dev/null +++ b/projects/arcueil/README.md @@ -0,0 +1,15 @@ +Welcome to your new dbt project! + +### Using the starter project + +Try running the following commands: +- dbt run +- dbt test + + +### Resources: +- Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction) +- Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers +- Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support +- Find [dbt events](https://events.getdbt.com) near you +- Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices diff --git a/projects/arcueil/analyses/.gitkeep b/projects/arcueil/analyses/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/projects/arcueil/config.yml b/projects/arcueil/config.yml new file mode 100644 index 00000000..ff3d0f0d --- /dev/null +++ b/projects/arcueil/config.yml @@ -0,0 +1,3 @@ +db_name_dev: ARCUEIL_DBNAME_DEV +db_name_prod: ARCUEIL_DBNAME_PROD +participation_host_name: ARCUEIL_PARTICIPATION_HOST_NAME \ No newline at end of file diff --git a/projects/arcueil/dbt_project.yml b/projects/arcueil/dbt_project.yml new file mode 100644 index 00000000..97b48dae --- /dev/null +++ b/projects/arcueil/dbt_project.yml @@ -0,0 +1,36 @@ + +# Name your project! Project names should contain only lowercase characters +# and underscores. A good package name should reflect your organization's +# name or the intended use of these models +name: 'arcueil' +version: '1.0.0' + +# This setting configures which "profile" dbt uses for this project. +profile: 'arcueil' + +# These configurations specify where dbt should look for different types of files. +# The `model-paths` config, for example, states that models in this project can be +# found in the "models/" directory. You probably won't need to change these! +model-paths: ["models"] +analysis-paths: ["analyses"] +test-paths: ["tests"] +seed-paths: ["seeds"] +macro-paths: ["macros"] +snapshot-paths: ["snapshots"] + +clean-targets: # directories to be removed by `dbt clean` + - "target" + - "dbt_packages" + + +# Configuring models +# Full documentation: https://docs.getdbt.com/docs/configuring-models + +# In this example config, we tell dbt to build all models in the example/ +# directory as views. These settings can be overridden in the individual model +# files using the `{{ config(...) }}` macro. +models: + arcueil: + # Config indicated by + and applies to all files under models/example/ + example: + +materialized: view diff --git a/projects/arcueil/macros/.gitkeep b/projects/arcueil/macros/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/projects/arcueil/packages.yml b/projects/arcueil/packages.yml new file mode 100644 index 00000000..f46b0908 --- /dev/null +++ b/projects/arcueil/packages.yml @@ -0,0 +1,4 @@ +packages: + - local: ../demo + - package: dbt-labs/dbt_utils + version: 1.2.0 \ No newline at end of file diff --git a/projects/arcueil/profiles.yml b/projects/arcueil/profiles.yml new file mode 100644 index 00000000..c3ea8905 --- /dev/null +++ b/projects/arcueil/profiles.yml @@ -0,0 +1,31 @@ +arcueil: + target: preprod + outputs: + dev: + dbname: "{{ env_var('DBNAME') }}" + host: "{{ env_var('HOST') }}" + pass: "{{ env_var('PASSWORD') }}" + port: "{{ env_var('PORT') | as_number }}" + schema: "{{ env_var('TARGET_SCHEMA') }}" + threads: 2 + type: postgres + user: "{{ env_var('USER') }}" + preprod: + dbname: "{{ env_var('DBNAME') }}" + host: "{{ env_var('HOST_PREPROD') }}" + pass: "{{ env_var('PASSWORD_PREPROD') }}" + port: "{{ env_var('PORT_PREPROD') | as_number }}" + schema: "{{ env_var('TARGET_SCHEMA') }}" + threads: 2 + type: postgres + user: "{{ env_var('USER_PREPROD') }}" + prod: + dbname: "{{ env_var('DBNAME') }}" + host: "{{ env_var('HOST') }}" + pass: "{{ env_var('PASSWORD') }}" + port: "{{ env_var('PORT') | as_number }}" + schema: "{{ env_var('TARGET_SCHEMA') }}" + threads: 2 + type: postgres + user: "{{ env_var('USER') }}" + diff --git a/projects/arcueil/requirements.txt b/projects/arcueil/requirements.txt new file mode 100644 index 00000000..576228e4 --- /dev/null +++ b/projects/arcueil/requirements.txt @@ -0,0 +1,2 @@ +dbt-postgres==1.8.2 +dbt-core==1.8.2 \ No newline at end of file diff --git a/projects/arcueil/seeds/.gitkeep b/projects/arcueil/seeds/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/projects/arcueil/snapshots/.gitkeep b/projects/arcueil/snapshots/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/projects/arcueil/tests/.gitkeep b/projects/arcueil/tests/.gitkeep new file mode 100644 index 00000000..e69de29b From 80dc7796601c1c9c944ae31765dd692adda0216a Mon Sep 17 00:00:00 2001 From: fujistone Date: Mon, 9 Dec 2024 17:15:40 +0100 Subject: [PATCH 2/3] correcting errors after rebase --- ...cueil_models_run.yml => arcueil_models_run_production.yml} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename .github/workflows/{arcueil_models_run.yml => arcueil_models_run_production.yml} (92%) diff --git a/.github/workflows/arcueil_models_run.yml b/.github/workflows/arcueil_models_run_production.yml similarity index 92% rename from .github/workflows/arcueil_models_run.yml rename to .github/workflows/arcueil_models_run_production.yml index 2a0f928c..17ad9727 100644 --- a/.github/workflows/arcueil_models_run.yml +++ b/.github/workflows/arcueil_models_run_production.yml @@ -1,4 +1,4 @@ -name: arcueil_models_run +name: arcueil_models_run_production on: workflow_dispatch: @@ -16,7 +16,7 @@ env: jobs: arcueil_models_run: - name: arcueil_models_run + name: arcueil_models_run_production runs-on: ubuntu-latest defaults: run: From 2393317c7262550df162032e6e65e302b5854782 Mon Sep 17 00:00:00 2001 From: Fujistone <73913825+Fujistone@users.noreply.github.com> Date: Thu, 26 Dec 2024 15:00:49 +0100 Subject: [PATCH 3/3] Changing materialization type --- projects/arcueil/dbt_project.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/projects/arcueil/dbt_project.yml b/projects/arcueil/dbt_project.yml index 97b48dae..494df3d3 100644 --- a/projects/arcueil/dbt_project.yml +++ b/projects/arcueil/dbt_project.yml @@ -31,6 +31,4 @@ clean-targets: # directories to be removed by `dbt clean` # files using the `{{ config(...) }}` macro. models: arcueil: - # Config indicated by + and applies to all files under models/example/ - example: - +materialized: view + +materialized: table