diff --git a/README.md b/README.md index b84c9ee..5b3c06f 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ No requirements. | Name | Version | |------|---------| +| [compute](#provider\_compute) | n/a | | [google](#provider\_google) | n/a | | [google-beta](#provider\_google-beta) | n/a | | [random](#provider\_random) | n/a | @@ -14,9 +15,8 @@ No requirements. | Name | Source | Version | |------|--------|---------| -| [cloud\_router](#module\_cloud\_router) | terraform-google-modules/cloud-router/google | ~> 6.0 | -| [dataproc-firewall-rules](#module\_dataproc-firewall-rules) | terraform-google-modules/network/google//modules/firewall-rules | 6.0.1 | | [kms\_crypto\_key-iam-bindings](#module\_kms\_crypto\_key-iam-bindings) | terraform-google-modules/iam/google//modules/kms_crypto_keys_iam | n/a | +| [network](#module\_network) | ./modules/network | n/a | ## Resources @@ -26,12 +26,7 @@ No requirements. | [google-beta_google_storage_bucket.tenant_input_bucket](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_storage_bucket) | resource | | [google-beta_google_storage_bucket.tenant_output_bucket](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_storage_bucket) | resource | | [google_bigquery_dataset.tenant_dataset](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/bigquery_dataset) | resource | -| [google_compute_address.cloud_nat_static_ip_address](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_address) | resource | -| [google_compute_firewall.allow_idapi_egress](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_firewall) | resource | -| [google_compute_firewall.allow_metastore_egress](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_firewall) | resource | | [google_compute_network.vpc_network](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_network) | resource | -| [google_compute_subnetwork.dataproc_subnet](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_subnetwork) | resource | -| [google_compute_subnetwork_iam_member.vpc_subnetwork_user](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_subnetwork_iam_member) | resource | | [google_kms_crypto_key.tenant_crypto_key](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/kms_crypto_key) | resource | | [google_kms_key_ring.kms](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/kms_key_ring) | resource | | [google_project_iam_member.allow_bq_connector_push_down](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_iam_member) | resource | @@ -47,6 +42,7 @@ No requirements. | [google_storage_bucket_iam_policy.tenant_input_bucket](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_iam_policy) | resource | | [google_storage_bucket_iam_policy.tenant_output_bucket](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_iam_policy) | resource | | [random_id.generator](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | +| [compute_compute.vpc_network](https://registry.terraform.io/providers/hashicorp/compute/latest/docs/data-sources/compute) | data source | | [google_iam_policy.tenant_build_bucket](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/iam_policy) | data source | | [google_iam_policy.tenant_input_bucket](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/iam_policy) | data source | | [google_iam_policy.tenant_output_bucket](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/iam_policy) | data source | @@ -65,7 +61,6 @@ No requirements. | [data\_retention\_period\_days](#input\_data\_retention\_period\_days) | The number of days this customers data will be stored before its automatically deleted | `number` | `0` | no | | [data\_viewers](#input\_data\_viewers) | The users, groups & service accounts that should have read only access to this customers data | 
object({
    service_accounts = list(string)
    groups           = list(string)
    users            = list(string)
  })
| n/a | yes | | [dataproc\_subnet\_ip4\_cidr](#input\_dataproc\_subnet\_ip4\_cidr) | Subnet used for Dataproc clusters | `string` | n/a | yes | -| [enable\_dataproc\_network](#input\_enable\_dataproc\_network) | Configure network bits for Dataproc - VPC, firewall rules etc | `bool` | `true` | no | | [enable\_kms](#input\_enable\_kms) | Configure KMS to encrypt build, input and output buckets | `bool` | `true` | no | | [environment](#input\_environment) | The environment this infrastructure is supported (eg.: dev, staging or prod) | `string` | n/a | yes | | [gcp\_region](#input\_gcp\_region) | The GCP region to be used | `string` | n/a | yes | @@ -81,6 +76,7 @@ No requirements. | [storage\_location](#input\_storage\_location) | The storage location for BigQuery and GCS. | `string` | n/a | yes | | [tenant\_orchestration\_sa](#input\_tenant\_orchestration\_sa) | Tenant Orchestration ServiceAccount for remote execution | `string` | n/a | yes | | [tenant\_service\_account\_name](#input\_tenant\_service\_account\_name) | Service Account name | `string` | `""` | no | +| [vpc\_network\_name](#input\_vpc\_network\_name) | The network to connect the data-plane to, if not specified, module will provision a dedicated one | `bool` | `null` | no | ## Outputs @@ -91,3 +87,4 @@ No requirements. | [output\_bucket\_name](#output\_output\_bucket\_name) | The name of the GCS bucket that will be used to store the output files | | [tenant\_bigquery\_dataset\_name](#output\_tenant\_bigquery\_dataset\_name) | The name of the BigQuery dataset that will be used to store the tenant data | | [tenant\_data\_access\_svc\_account](#output\_tenant\_data\_access\_svc\_account) | The service account object that will be used to access the tenant data | +| [vpc\_network\_name](#output\_vpc\_network\_name) | The name of the VPC network | diff --git a/main.tf b/main.tf index 4c50c18..85ccb10 100644 --- a/main.tf +++ b/main.tf @@ -1,20 +1,52 @@ -data "google_project" "data_plane_project" { - project_id = var.data_plane_project +resource "google_compute_network" "vpc_network" { + count = (var.vpc_network_name == null || var.vpc_network_name == "") ? 1 : 0 + project = var.project_id + name = lower("${var.installation_name}-${var.region}-vpc") + auto_create_subnetworks = false + routing_mode = "GLOBAL" + description = "The shared network for the identity graph" } -data "google_storage_project_service_account" "data_plane_gcs_account" { - project = var.data_plane_project +data "google_compute_network" "vpc_network" { + project = var.project_id + name = (var.vpc_network_name == null || var.vpc_network_name == "") ? google_compute_network.vpc_network[0].name : var.vpc_network_name } -resource "google_project_service" "enable_api" { - for_each = toset([ - "accesscontextmanager.googleapis.com", - "cloudkms.googleapis.com", - "dataproc.googleapis.com", - "pubsub.googleapis.com" - ]) - project = var.data_plane_project - service = each.value +module "root" { + for_each = var.data_planes + source = "./modules/root" - disable_on_destroy = false + environment = var.environment + data_plane_project = var.project_id + gcp_region = var.region + installation_name = var.installation_name + storage_location = each.value.storage_region + key_management_location = var.key_management_location + dataproc_subnet_ip4_cidr = var.subnet_ip4_cidr + + organisation_id = each.value.organisation_id + name = each.value.tenant_name + country_code = each.value.country_code + data_viewers = each.value.data_viewers + data_editors = each.value.data_editors + + tenant_orchestration_sa = each.value.tenant_orchestration_sa + data_retention_period_days = each.value.data_retention_period_days + key_rotation_period_days = each.value.key_rotation_period_days + + metastore_cidr_ip_address = var.metastore_cidr_ip_address + idapi_cidr_ip_addresses = var.idapi_cidr_ip_addresses +} + +module "network" { + depends_on = [ module.root ] + source = "./modules/network" + installation_name = var.installation_name + project_id = var.project_id + region = var.region + network_name = data.google_compute_network.vpc_network.name + subnet_ip4_cidr = var.subnet_ip4_cidr + subnet_users = [] + idapi_cidr_ip_addresses = var.idapi_cidr_ip_addresses + metastore_cidr_ip_address = var.metastore_cidr_ip_address } diff --git a/firewalls.tf b/modules/network/firewall.tf similarity index 77% rename from firewalls.tf rename to modules/network/firewall.tf index 7408852..083b544 100644 --- a/firewalls.tf +++ b/modules/network/firewall.tf @@ -1,8 +1,7 @@ resource "google_compute_firewall" "allow_metastore_egress" { - count = var.enable_dataproc_network ? 1 : 0 - project = google_compute_network.vpc_network[0].project + project = var.project_id name = "allow-${var.installation_name}-metastore-egress" - network = google_compute_network.vpc_network[0].name + network = var.network_name direction = "EGRESS" priority = "1000" description = "Allow EGRESS to Identity Engine Metastore CloudSQL instance" @@ -14,17 +13,14 @@ resource "google_compute_firewall" "allow_metastore_egress" { "3306" ] } - - destination_ranges = [ - var.metastore_cidr_ip_address - ] + destination_ranges = [var.metastore_cidr_ip_address] } + resource "google_compute_firewall" "allow_idapi_egress" { - count = var.enable_dataproc_network ? 1 : 0 - project = google_compute_network.vpc_network[0].project + project = var.project_id name = "allow-${var.installation_name}-idapi-egress" - network = google_compute_network.vpc_network[0].name + network = var.network_name direction = "EGRESS" priority = "1000" description = "Allow EGRESS to LiveRamp ID-API instance" @@ -36,16 +32,15 @@ resource "google_compute_firewall" "allow_idapi_egress" { "443" ] } - destination_ranges = var.idapi_cidr_ip_addresses } + module "dataproc-firewall-rules" { - count = var.enable_dataproc_network ? 1 : 0 source = "terraform-google-modules/network/google//modules/firewall-rules" version = "6.0.1" - project_id = var.data_plane_project - network_name = google_compute_network.vpc_network[0].name + project_id = var.project_id + network_name = var.network_name rules = [ { @@ -53,7 +48,7 @@ module "dataproc-firewall-rules" { description = "Allow Dataproc clusters to communicate over private IP to google APIs and other nodes" direction = "INGRESS" priority = 1000 - ranges = [var.dataproc_subnet_ip4_cidr] + ranges = [var.subnet_ip4_cidr] source_tags = null source_service_accounts = null target_tags = null @@ -80,7 +75,7 @@ module "dataproc-firewall-rules" { description = "Allow Dataproc clusters to communicate over private IP to google APIs and other nodes" direction = "EGRESS" priority = 1000 - ranges = [var.dataproc_subnet_ip4_cidr] + ranges = [var.subnet_ip4_cidr] source_tags = null source_service_accounts = null target_tags = null diff --git a/modules/network/iam.tf b/modules/network/iam.tf new file mode 100644 index 0000000..72b1121 --- /dev/null +++ b/modules/network/iam.tf @@ -0,0 +1,7 @@ +resource "google_compute_subnetwork_iam_member" "subnet_user" { + for_each = toset(var.subnet_users) + project = var.project_id + subnetwork = google_compute_subnetwork.dataproc_subnet.name + role = "roles/compute.networkUser" + member = "serviceAccount:${each.value}" +} diff --git a/cloud_router.tf b/modules/network/router.tf similarity index 63% rename from cloud_router.tf rename to modules/network/router.tf index 2ad605e..74040e6 100644 --- a/cloud_router.tf +++ b/modules/network/router.tf @@ -1,21 +1,20 @@ resource "google_compute_address" "cloud_nat_static_ip_address" { - project = var.data_plane_project - region = var.gcp_region - count = var.enable_dataproc_network ? 2 : 0 - name = lower("${var.installation_name}-${var.gcp_region}-nat-ip-${count.index}") + count = 2 + project = var.project_id + region = var.region + name = lower("${var.installation_name}-${var.region}-nat-ip-${count.index}") } module "cloud_router" { - count = var.enable_dataproc_network ? 1 : 0 source = "terraform-google-modules/cloud-router/google" version = "~> 6.0" - name = lower("${var.installation_name}-${var.gcp_region}-router") - project = var.data_plane_project - network = google_compute_network.vpc_network[0].name - region = var.gcp_region + name = lower("${var.installation_name}-${var.region}-router") + project = var.project_id + network = var.network_name + region = var.region nats = [{ - name = lower("${var.installation_name}-${var.gcp_region}-nat") + name = lower("${var.installation_name}-${var.region}-nat") nat_ip_allocate_option = "MANUAL_ONLY" source_subnetwork_ip_ranges_to_nat = "ALL_SUBNETWORKS_ALL_IP_RANGES" nat_ips = google_compute_address.cloud_nat_static_ip_address.*.self_link diff --git a/modules/network/subnetwork.tf b/modules/network/subnetwork.tf new file mode 100644 index 0000000..9743103 --- /dev/null +++ b/modules/network/subnetwork.tf @@ -0,0 +1,8 @@ +resource "google_compute_subnetwork" "dataproc_subnet" { + project = var.project_id + region = var.region + name = lower("${var.installation_name}-${var.region}-dataproc") + ip_cidr_range = var.subnet_ip4_cidr + network = var.network_name + private_ip_google_access = true +} \ No newline at end of file diff --git a/modules/network/variables.tf b/modules/network/variables.tf new file mode 100644 index 0000000..d801cbf --- /dev/null +++ b/modules/network/variables.tf @@ -0,0 +1,32 @@ +variable "installation_name" { + type = string +} + +variable "project_id" { + type = string +} + +variable "region" { + type = string +} + +variable "network_name" { + type = string +} + +variable "subnet_ip4_cidr" { + type = string +} + +variable "subnet_users" { + type = list(string) +} + +variable "metastore_cidr_ip_address" { + type = string +} + +variable "idapi_cidr_ip_addresses" { + type = list(string) + default = [] +} diff --git a/bigquery.tf b/modules/root/bigquery.tf similarity index 100% rename from bigquery.tf rename to modules/root/bigquery.tf diff --git a/buckets.tf b/modules/root/buckets.tf similarity index 100% rename from buckets.tf rename to modules/root/buckets.tf diff --git a/crypto-key.tf b/modules/root/crypto-key.tf similarity index 100% rename from crypto-key.tf rename to modules/root/crypto-key.tf diff --git a/iam.tf b/modules/root/iam.tf similarity index 82% rename from iam.tf rename to modules/root/iam.tf index 3014c9e..b88a1bc 100644 --- a/iam.tf +++ b/modules/root/iam.tf @@ -16,14 +16,6 @@ resource "google_project_iam_member" "dataproc_editor" { member = "serviceAccount:${google_service_account.tenant_data_access.email}" } -resource "google_compute_subnetwork_iam_member" "vpc_subnetwork_user" { - count = var.enable_dataproc_network ? 1 : 0 - project = var.data_plane_project - subnetwork = google_compute_subnetwork.dataproc_subnet[0].name - role = "roles/compute.networkUser" - member = "serviceAccount:${google_service_account.tenant_data_access.email}" -} - resource "google_project_iam_member" "bigquery_job_creator" { project = var.data_plane_project role = "roles/bigquery.jobUser" diff --git a/locals.tf b/modules/root/locals.tf similarity index 100% rename from locals.tf rename to modules/root/locals.tf diff --git a/modules/root/main.tf b/modules/root/main.tf new file mode 100644 index 0000000..4c50c18 --- /dev/null +++ b/modules/root/main.tf @@ -0,0 +1,20 @@ +data "google_project" "data_plane_project" { + project_id = var.data_plane_project +} + +data "google_storage_project_service_account" "data_plane_gcs_account" { + project = var.data_plane_project +} + +resource "google_project_service" "enable_api" { + for_each = toset([ + "accesscontextmanager.googleapis.com", + "cloudkms.googleapis.com", + "dataproc.googleapis.com", + "pubsub.googleapis.com" + ]) + project = var.data_plane_project + service = each.value + + disable_on_destroy = false +} diff --git a/outputs.tf b/modules/root/outputs.tf similarity index 100% rename from outputs.tf rename to modules/root/outputs.tf diff --git a/service-accounts.tf b/modules/root/service-accounts.tf similarity index 100% rename from service-accounts.tf rename to modules/root/service-accounts.tf diff --git a/variables.tf b/modules/root/variables.tf similarity index 95% rename from variables.tf rename to modules/root/variables.tf index 3e57b44..c688d55 100644 --- a/variables.tf +++ b/modules/root/variables.tf @@ -94,12 +94,6 @@ variable "idapi_cidr_ip_addresses" { description = "Portrait Engine ID-API instance CIDR IP addresses" } -variable "enable_dataproc_network" { - type = bool - description = "Configure network bits for Dataproc - VPC, firewall rules etc" - default = true -} - variable "enable_kms" { type = bool description = "Configure KMS to encrypt build, input and output buckets" diff --git a/network.tf b/network.tf deleted file mode 100644 index 67d29d4..0000000 --- a/network.tf +++ /dev/null @@ -1,18 +0,0 @@ -resource "google_compute_network" "vpc_network" { - count = var.enable_dataproc_network ? 1 : 0 - project = var.data_plane_project - name = lower("${var.installation_name}-${var.country_code}-vpc") - auto_create_subnetworks = false - routing_mode = "GLOBAL" - description = "The shared network for the identity graph" -} - -resource "google_compute_subnetwork" "dataproc_subnet" { - count = var.enable_dataproc_network ? 1 : 0 - project = var.data_plane_project - region = var.gcp_region - name = lower("${var.installation_name}-${var.country_code}-dataproc") - ip_cidr_range = var.dataproc_subnet_ip4_cidr - network = google_compute_network.vpc_network[0].id - private_ip_google_access = true -} diff --git a/output.tf b/output.tf new file mode 100644 index 0000000..64d8783 --- /dev/null +++ b/output.tf @@ -0,0 +1,4 @@ +output "vpc_network_name" { + value = data.google_compute_network.vpc_network.name + description = "The name of the VPC network" +} diff --git a/variable.tf b/variable.tf new file mode 100644 index 0000000..7295a91 --- /dev/null +++ b/variable.tf @@ -0,0 +1,64 @@ +variable "installation_name" { + type = string +} + +variable "project_id" { + type = string +} + +variable "region" { + type = string +} + +variable "environment" { + type = string + description = "The environment this infrastructure is supported (dev, staging or prod)" +} + +variable "vpc_network_name" { + type = string + description = "The network to connect the data-plane to, if not specified, module will provision a dedicated one" + default = null +} + +variable "subnet_ip4_cidr" { + type = string +} + +variable "key_management_location" { + type = string +} + +variable "metastore_cidr_ip_address" { + type = string + description = "Identity Engine Metastore CloudSQL instance CIDR IP address" +} + +variable "idapi_cidr_ip_addresses" { + type = list(string) + default = [] + description = "Identity Engine ID-API instance CIDR IP addresses" +} + +variable "data_planes" { + type = map(object({ + tenant_name = string + organisation_id = string + country_code = string + storage_region = optional(string) + data_editors = object({ + service_accounts = list(string) + groups = list(string) + users = list(string) + }) + data_viewers = object({ + service_accounts = list(string) + groups = list(string) + users = list(string) + }) + tenant_orchestration_sa = string + data_retention_period_days = number + key_rotation_period_days = number + })) + description = "A map of portrait engine data-planes with their customer specific configuration. Set storage_region to enable regional storage" +}