From d37ca73232eeca83d123d5bd3873958c2ffdac20 Mon Sep 17 00:00:00 2001 From: TJ Murphy Date: Wed, 18 Dec 2024 17:20:50 -0800 Subject: [PATCH] [CHORE] adjust warehouses to reduce cost (#181) * adjust warehouses --------- Co-authored-by: TJ Murphy <1796+teej@users.noreply.github.com> --- examples/dbt-labs-how-we-configure-snowflake.yml | 9 ++++++--- .../dicom-image-classification-to-detect-pneumonia.yml | 10 ++++++---- ...flake-tutorials-create-your-first-iceberg-table.yml | 3 ++- tests/fixtures/json/warehouse.json | 2 +- tests/fixtures/sql/warehouse.sql | 1 + tests/integration/test_lifecycle.py | 2 +- tests/integration/test_resources.py | 1 + tools/test_account_configs/base.yml | 6 ++++++ 8 files changed, 24 insertions(+), 10 deletions(-) diff --git a/examples/dbt-labs-how-we-configure-snowflake.yml b/examples/dbt-labs-how-we-configure-snowflake.yml index 978db844..fd4ec9ca 100644 --- a/examples/dbt-labs-how-we-configure-snowflake.yml +++ b/examples/dbt-labs-how-we-configure-snowflake.yml @@ -25,18 +25,21 @@ databases: warehouses: - name: loading warehouse_size: XSMALL - auto_suspend: 60 + auto_suspend: 30 auto_resume: true + initially_suspended: true comment: "Tools like Fivetran and Stitch will use this warehouse to perform their regular loads of new data. We separate this workload from the other workloads because, at scale, loading can put significant strain on your warehouse and we don't want to cause slowness for your BI users." - name: transforming warehouse_size: XSMALL - auto_suspend: 60 + auto_suspend: 30 auto_resume: true + initially_suspended: true comment: "This is the warehouse that dbt will use to perform all data transformations. It will only be in use (and charging you credits) when regular jobs are being run." - name: reporting warehouse_size: XSMALL - auto_suspend: 60 + auto_suspend: 30 auto_resume: true + initially_suspended: true comment: "Mode and any other BI tools will connect to this warehouse to run analytical queries and report the results to end users. This warehouse will be spun up only when a user is actively running a query against it." grants: diff --git a/examples/dicom-image-classification-to-detect-pneumonia.yml b/examples/dicom-image-classification-to-detect-pneumonia.yml index 1579ab28..25b7cdf0 100644 --- a/examples/dicom-image-classification-to-detect-pneumonia.yml +++ b/examples/dicom-image-classification-to-detect-pneumonia.yml @@ -22,14 +22,16 @@ schemas: warehouses: - name: snowopt_wh - warehouse_size: LARGE - auto_suspend: 120 + warehouse_size: XSMALL + auto_suspend: 30 auto_resume: true + initially_suspended: true comment: "Warehouse optimized for model training and heavy computations." - name: standard_wh - warehouse_size: MEDIUM - auto_suspend: 120 + warehouse_size: XSMALL + auto_suspend: 30 auto_resume: true + initially_suspended: true comment: "General purpose warehouse for regular workloads." stages: diff --git a/examples/snowflake-tutorials-create-your-first-iceberg-table.yml b/examples/snowflake-tutorials-create-your-first-iceberg-table.yml index 4af5ef08..54bd871f 100644 --- a/examples/snowflake-tutorials-create-your-first-iceberg-table.yml +++ b/examples/snowflake-tutorials-create-your-first-iceberg-table.yml @@ -30,8 +30,9 @@ warehouses: - name: iceberg_tutorial_wh owner: iceberg_tutorial_role warehouse_size: XSMALL - auto_suspend: 60 + auto_suspend: 30 auto_resume: true + initially_suspended: true grants: - GRANT CREATE DATABASE ON ACCOUNT TO ROLE iceberg_tutorial_role diff --git a/tests/fixtures/json/warehouse.json b/tests/fixtures/json/warehouse.json index ec91936e..e88e6090 100644 --- a/tests/fixtures/json/warehouse.json +++ b/tests/fixtures/json/warehouse.json @@ -6,7 +6,7 @@ "max_cluster_count": 1, "min_cluster_count": 1, "scaling_policy": "STANDARD", - "auto_suspend": 60, + "auto_suspend": 30, "auto_resume": false, "initially_suspended": true, "resource_monitor": null, diff --git a/tests/fixtures/sql/warehouse.sql b/tests/fixtures/sql/warehouse.sql index c787a9cb..f72a6bad 100644 --- a/tests/fixtures/sql/warehouse.sql +++ b/tests/fixtures/sql/warehouse.sql @@ -18,6 +18,7 @@ CREATE WAREHOUSE lowercase_wh warehouse_size = x6large warehouse_type = snowpark-optimized scaling_policy = economy +initially_suspended = true ; diff --git a/tests/integration/test_lifecycle.py b/tests/integration/test_lifecycle.py index 32d23ef8..f7865656 100644 --- a/tests/integration/test_lifecycle.py +++ b/tests/integration/test_lifecycle.py @@ -144,7 +144,7 @@ def test_task_lifecycle(cursor, suffix, marked_for_cleanup): owner=TEST_ROLE, comment="This is a test task modified", allow_overlapping_execution=False, - user_task_managed_initial_warehouse_size="LARGE", + user_task_managed_initial_warehouse_size="XSMALL", user_task_timeout_ms=2000, suspend_task_after_num_failures=2, config='{"output_dir": "/temp/test_directory/", "learning_rate": 0.2}', diff --git a/tests/integration/test_resources.py b/tests/integration/test_resources.py index 6a0f3e8f..ed964610 100644 --- a/tests/integration/test_resources.py +++ b/tests/integration/test_resources.py @@ -120,6 +120,7 @@ def test_fetch_warehouse_snowpark_optimized(cursor, suffix, marked_for_cleanup): name=f"TEST_FETCH_WAREHOUSE_SNOWPARK_OPTIMIZED_{suffix}", warehouse_type="SNOWPARK-OPTIMIZED", warehouse_size="MEDIUM", + initially_suspended=True, ) cursor.execute(warehouse.create_sql()) diff --git a/tools/test_account_configs/base.yml b/tools/test_account_configs/base.yml index f834d707..e71b9ce6 100644 --- a/tools/test_account_configs/base.yml +++ b/tools/test_account_configs/base.yml @@ -293,8 +293,14 @@ views: warehouses: - name: static_warehouse + warehouse_size: XSMALL + auto_suspend: 30 + auto_resume: true + initially_suspended: true - name: CI warehouse_size: XSMALL + auto_suspend: 30 + initially_suspended: true