Add Azure Lake Loader as a destination (closes #78)

terraform/azure/pipeline/README.md

@@ -19,6 +19,8 @@ No providers.
 | <a name="module_eh_namespace"></a> [eh\_namespace](#module\_eh\_namespace) | snowplow-devops/event-hub-namespace/azurerm | 0.1.1 |
 | <a name="module_enrich_eh"></a> [enrich\_eh](#module\_enrich\_eh) | snowplow-devops/enrich-event-hub-vmss/azurerm | 0.1.2 |
 | <a name="module_enriched_eh_topic"></a> [enriched\_eh\_topic](#module\_enriched\_eh\_topic) | snowplow-devops/event-hub/azurerm | 0.1.1 |
+| <a name="module_lake_loader"></a> [lake\_loader](#module\_lake\_loader) | snowplow-devops/lake-loader-vmss/azurerm | 0.1.1 |
+| <a name="module_lake_storage_container"></a> [lake\_storage\_container](#module\_lake\_storage\_container) | snowplow-devops/storage-container/azurerm | 0.1.1 |
 | <a name="module_raw_eh_topic"></a> [raw\_eh\_topic](#module\_raw\_eh\_topic) | snowplow-devops/event-hub/azurerm | 0.1.1 |
 | <a name="module_sf_loader"></a> [sf\_loader](#module\_sf\_loader) | snowplow-devops/snowflake-loader-vmss/azurerm | 0.1.1 |
 | <a name="module_sf_message_queue_eh_topic"></a> [sf\_message\_queue\_eh\_topic](#module\_sf\_message\_queue\_eh\_topic) | snowplow-devops/event-hub/azurerm | 0.1.1 |
@@ -43,6 +45,7 @@ No resources.
 | <a name="input_storage_account_name"></a> [storage\_account\_name](#input\_storage\_account\_name) | The name of the Storage Account the data will be loaded into | `string` | n/a | yes |
 | <a name="input_subnet_id_lb"></a> [subnet\_id\_lb](#input\_subnet\_id\_lb) | The ID of the subnet to deploy the load balancer into (e.g. collector-agw1) | `string` | n/a | yes |
 | <a name="input_subnet_id_servers"></a> [subnet\_id\_servers](#input\_subnet\_id\_servers) | The ID of the subnet to deploy the servers into (e.g. pipeline1) | `string` | n/a | yes |
+| <a name="input_lake_enabled"></a> [lake\_enabled](#input\_lake\_enabled) | Whether to load all data into a Storage Container to build a data-lake based on Delta format | `bool` | `false` | no |
 | <a name="input_snowflake_account"></a> [snowflake\_account](#input\_snowflake\_account) | Snowflake account to use | `string` | `""` | no |
 | <a name="input_snowflake_database"></a> [snowflake\_database](#input\_snowflake\_database) | Snowflake database name | `string` | `""` | no |
 | <a name="input_snowflake_enabled"></a> [snowflake\_enabled](#input\_snowflake\_enabled) | Whether to enable loading into a Snowflake Database | `bool` | `false` | no |

terraform/azure/pipeline/target_lake_loader.tf

@@ -0,0 +1,42 @@
+module "lake_storage_container" {
+  source  = "snowplow-devops/storage-container/azurerm"
+  version = "0.1.1"
+
+  count = var.lake_enabled ? 1 : 0
+
+  name                 = "lake-container"
+  storage_account_name = local.storage_account_name
+}
+
+module "lake_loader" {
+  source  = "snowplow-devops/lake-loader-vmss/azurerm"
+  version = "0.1.1"
+
+  count = var.lake_enabled ? 1 : 0
+
+  name                = "${var.prefix}-lake-loader"
+  resource_group_name = var.resource_group_name
+  subnet_id           = var.subnet_id_servers
+
+  enriched_topic_name              = module.enriched_eh_topic.name
+  enriched_topic_connection_string = module.enriched_eh_topic.read_only_primary_connection_string
+  bad_topic_name                   = module.bad_1_eh_topic.name
+  bad_topic_connection_string      = module.bad_1_eh_topic.read_write_primary_connection_string
+  eh_namespace_name                = module.eh_namespace.name
+  eh_namespace_broker              = module.eh_namespace.broker
+
+  storage_account_name   = local.storage_account_name
+  storage_container_name = module.lake_storage_container[0].name
+
+  ssh_public_key   = var.ssh_public_key
+  ssh_ip_allowlist = var.ssh_ip_allowlist
+
+  telemetry_enabled = var.telemetry_enabled
+  user_provided_id  = var.user_provided_id
+
+  custom_iglu_resolvers = local.custom_iglu_resolvers
+
+  tags = var.tags
+
+  depends_on = [module.lake_storage_container]
+}

terraform/azure/pipeline/terraform.tfvars

@@ -51,6 +51,11 @@ snowflake_warehouse       = "<WAREHOUSE>"
 # This controls how often data will be loading into Snowflake
 snowflake_transformer_window_period_min = 1
 
+# --- Target: Lake
+# Follow the guide to get input values for the loader:
+# https://docs.snowplow.io/docs/getting-started-on-snowplow-open-source/quick-start/
+lake_enabled = false
+
 # --- ADVANCED CONFIGURATION ZONE --- #
 
 # Telemetry principles: https://docs.snowplowanalytics.com/docs/open-source-quick-start/what-is-the-quick-start-for-open-source/telemetry-principles/

terraform/azure/pipeline/variables.tf

@@ -96,7 +96,6 @@ variable "snowflake_enabled" {
   type        = bool
 }
 
-
 variable "snowflake_account" {
   description = "Snowflake account to use"
   type        = string
@@ -145,3 +144,11 @@ variable "snowflake_transformer_window_period_min" {
   type        = number
   default     = 5
 }
+
+# --- Target: Lake
+
+variable "lake_enabled" {
+  description = "Whether to load all data into a Storage Container to build a data-lake based on Delta format"
+  default     = false
+  type        = bool
+}