Allow for specyfing service account to use with `terraform-google-modules/log-export/google`

[piotrekkr]

TL;DR

I cannot reuse modules terraform-google-modules/log-export/google with for_each because it generate circular reference errors with submodules on terraform plan. If I could specify existing service account as writer identity in module I could overcome this.

Terraform Resources

No response

Detailed design

This fails

locals {
  projects = ["project1", "project2"]
  api_key = "api_key"
}

module "gcp_log_export_qa" {
  for_each = local.projects
  source                 = "terraform-google-modules/log-export/google"
  destination_uri        = module.datadog_log_pub_sub_exporter_qa[each.value].destination_uri
  filter                 = "resource.type=\"cloud_run_revision\" OR resource.type=\"http_load_balancer\""
  log_sink_name          = "datadog-log-sink"
  parent_resource_id     = each.value
  parent_resource_type   = "project"
  unique_writer_identity = true
}

module "datadog_log_pub_sub_exporter_qa" {
  for_each = local.projects
  source                   = "terraform-google-modules/log-export/google//modules/pubsub"
  project_id               = each.value
  topic_name               = "datadog-log-exporter"
  log_sink_writer_identity = module.gcp_log_export_qa[each.value].writer_identity
  create_subscriber        = false
  create_push_subscriber   = true
  push_endpoint            = "https://gcp-intake.logs.datadoghq.eu/v1/input/${local.api_key}/"
}

with error:

╷
│ Error: Cycle: module.datadog.module.datadog_log_pub_sub_exporter_qa.google_pubsub_topic_iam_member.pubsub_sink_member, module.datadog.module.gcp_log_export_qa.local.log_sink_resource_id (expand), module.datadog.module.gcp_log_export_qa.output.log_sink_resource_id (expand), module.datadog.module.gcp_log_export_qa.local.log_sink_resource_name (expand), module.datadog.module.gcp_log_export_qa.output.log_sink_resource_name (expand), module.datadog.module.gcp_log_export_qa.google_logging_project_sink.sink, module.datadog.module.gcp_log_export_qa.google_logging_organization_sink.sink, module.datadog.module.gcp_log_export_qa.google_logging_billing_account_sink.sink, module.datadog.module.gcp_log_export_qa.local.log_sink_parent_id (expand), module.datadog.module.gcp_log_export_qa.output.parent_resource_id (expand), module.datadog.module.datadog_log_pub_sub_exporter_qa.var.log_sink_writer_identity (expand), module.datadog.module.datadog_log_pub_sub_exporter_qa (close), module.datadog.module.gcp_log_export_qa.var.destination_uri (expand), module.datadog.module.gcp_log_export_qa.google_logging_folder_sink.sink, module.datadog.module.gcp_log_export_qa.local.log_sink_writer_identity (expand), module.datadog.module.gcp_log_export_qa.output.writer_identity (expand), module.datadog.module.gcp_log_export_qa (close)

To break this circular ref we could just create service account and specify it like this:

module "gcp_log_export_qa" {
  for_each = local.projects
  source                 = "terraform-google-modules/log-export/google"
  destination_uri        = module.datadog_log_pub_sub_exporter_qa[each.value].destination_uri
  filter                 = "resource.type=\"cloud_run_revision\" OR resource.type=\"http_load_balancer\""
  log_sink_name          = "datadog-log-sink"
  parent_resource_id     = each.value
  parent_resource_type   = "project"
  // instead of specifying SA (uniq or not) we use existing SA
  writer_identity        = google_service_account.log_exporter.email 
}

module "datadog_log_pub_sub_exporter_qa" {
  for_each = local.projects
  source                   = "terraform-google-modules/log-export/google//modules/pubsub"
  project_id               = each.value
  topic_name               = "datadog-log-exporter"
  log_sink_writer_identity = google_service_account.log_exporter.email // no ref to module
  create_subscriber        = false
  create_push_subscriber   = true
  push_endpoint            = "https://gcp-intake.logs.datadoghq.eu/v1/input/${local.api_key}/"
}

### Additional information

_No response_

[piotrekkr]

Okay just realized that this is not possible since google_logging_project_sink does not allow for this...

[piotrekkr]

For anyone who may have same problem. To overcome this you will just need to create your own module with something like this:

#-----------#
# Log sinks #
#-----------#
resource "google_logging_project_sink" "sink" {
  name                   = var.log_sink_name
  project                = var.project_id
  filter                 = var.filter
  destination            = "pubsub.googleapis.com/projects/${google_pubsub_topic.topic.project}/topics/${google_pubsub_topic.topic.name}"
  unique_writer_identity = var.unique_writer_identity
}

#----------------#
# API activation #
#----------------#
resource "google_project_service" "enable_destination_api" {
  project            = var.project_id
  service            = "pubsub.googleapis.com"
  disable_on_destroy = false
}

#--------------#
# Pubsub topic #
#--------------#
resource "google_pubsub_topic" "topic" {
  project = google_project_service.enable_destination_api.project
  name    = var.topic_name
}

#--------------------------#
# Pubsub push subscription #
#--------------------------#

resource "google_pubsub_subscription" "pubsub_push_subscription" {
  name    = "${google_pubsub_topic.topic.name}-push-subscription"
  project = google_pubsub_topic.topic.project
  topic   = google_pubsub_topic.topic.name

  push_config {
    push_endpoint = "https://gcp-intake.logs.datadoghq.eu/v1/input/${var.api_key}/"
  }
}

#--------------------------------#
# Service account IAM membership #
#--------------------------------#
resource "google_pubsub_topic_iam_member" "pubsub_sink_member" {
  project = google_pubsub_topic.topic.project
  topic   = google_pubsub_topic.topic.name
  role    = "roles/pubsub.publisher"
  member  = google_logging_project_sink.sink.writer_identity
}

It works fine with for_each like:

module "datadog_log_export" {
  for_each      = { for k, v in var.log_export_projects : v => v }
  source        = "./log-exporter"
  api_key       = var.log_export_api_key
  filter        = "resource.type=\"cloud_run_revision\" OR resource.type=\"http_load_balancer\""
  topic_name    = "datadog-log-exporter"
  project_id    = each.value
  log_sink_name = "datadog-log-sink"
}