feat!: Create project destination and remove log bucket destination for the centralized logging (#1148)

Author: daniel-cit

1-org/envs/shared/README.md

@@ -39,8 +39,8 @@
 | domains\_to\_allow | The list of domains to allow users from in IAM. |
 | interconnect\_project\_id | The Dedicated Interconnect project ID |
 | interconnect\_project\_number | The Dedicated Interconnect project number |
-| logs\_export\_logbucket\_linked\_dataset\_name | The resource name of the Log Bucket linked BigQuery dataset created for Log Analytics. See https://cloud.google.com/logging/docs/log-analytics . |
-| logs\_export\_logbucket\_name | The log bucket for destination of log exports. See https://cloud.google.com/logging/docs/routing/overview#buckets . |
+| logs\_export\_project\_linked\_dataset\_name | The resource name of the Log Bucket linked BigQuery dataset for the project destination. |
+| logs\_export\_project\_logbucket\_name | The resource name for the Log Bucket created for the project destination. |
 | logs\_export\_pubsub\_topic | The Pub/Sub topic for destination of log exports |
 | logs\_export\_storage\_bucket\_name | The storage bucket for destination of log exports |
 | network\_folder\_name | The network folder name. |

1-org/envs/shared/log_sinks.tf

@@ -72,16 +72,16 @@ module "logs_export" {
   }
 
   /******************************************
-    Send logs to Logbucket
+    Send logs to Logging project
   *****************************************/
-  logbucket_options = {
-    logging_sink_name          = "sk-c-logging-logbkt"
+  project_options = {
+    logging_sink_name          = "sk-c-logging-prj"
     logging_sink_filter        = local.logs_filter
-    name                       = "logbkt-org-logs-${random_string.suffix.result}"
+    log_bucket_id              = "AggregatedLogs"
+    log_bucket_description     = "Project destination log bucket for aggregated logs"
     location                   = local.default_region
-    enable_analytics           = true
-    linked_dataset_id          = "ds_c_logbkt_analytics"
-    linked_dataset_description = "BigQuery Dataset for Logbucket analytics"
+    linked_dataset_id          = "ds_c_prj_aggregated_logs_analytics"
+    linked_dataset_description = "Project destination BigQuery Dataset for Logbucket analytics"
   }
 }
 

1-org/envs/shared/outputs.tf

@@ -114,21 +114,21 @@ output "logs_export_storage_bucket_name" {
   description = "The storage bucket for destination of log exports"
 }
 
-output "logs_export_logbucket_name" {
-  value       = module.logs_export.logbucket_destination_name
-  description = "The log bucket for destination of log exports. See https://cloud.google.com/logging/docs/routing/overview#buckets ."
+output "logs_export_project_logbucket_name" {
+  description = "The resource name for the Log Bucket created for the project destination."
+  value       = module.logs_export.project_logbucket_name
+}
+
+output "logs_export_project_linked_dataset_name" {
+  description = "The resource name of the Log Bucket linked BigQuery dataset for the project destination."
+  value       = module.logs_export.project_linked_dataset_name
 }
 
 output "billing_sink_names" {
   value       = module.logs_export.billing_sink_names
   description = "The name of the sinks under billing account level."
 }
 
-output "logs_export_logbucket_linked_dataset_name" {
-  value       = module.logs_export.logbucket_linked_dataset_name
-  description = "The resource name of the Log Bucket linked BigQuery dataset created for Log Analytics. See https://cloud.google.com/logging/docs/log-analytics ."
-}
-
 output "tags" {
   value       = local.tags_output
   description = "Tag Values to be applied on next steps."

1-org/modules/centralized-logging/README.md

@@ -61,9 +61,9 @@ module "logging_logbucket" {
 |------|-------------|------|---------|:--------:|
 | billing\_account | Billing Account ID used in case sinks are under billing account level. Format 000000-000000-000000. | `string` | `null` | no |
 | enable\_billing\_account\_sink | If true, a log router sink will be created for the billing account. The billing\_account variable cannot be null. | `bool` | `false` | no |
-| logbucket\_options | Destination LogBucket options:<br>- name: The name of the log bucket to be created and used for log entries matching the filter.<br>- logging\_sink\_name: The name of the log sink to be created.<br>- logging\_sink\_filter: The filter to apply when exporting logs. Only log entries that match the filter are exported. Default is "" which exports all logs.<br>- location: The location of the log bucket. Default: global.<br>- enable\_analytics: Whether or not Log Analytics is enabled. A Log bucket with Log Analytics enabled can be queried in the Log Analytics page using SQL queries. Cannot be disabled once enabled.<br>- linked\_dataset\_id: The ID of the linked BigQuery dataset. A valid link dataset ID must only have alphanumeric characters and underscores within it and have up to 100 characters.<br>- linked\_dataset\_description: A use-friendly description of the linked BigQuery dataset. The maximum length of the description is 8000 characters.<br>- retention\_days: The number of days data should be retained for the log bucket. Default 30. | <pre>object({<br>    name                       = optional(string, null)<br>    logging_sink_name          = optional(string, null)<br>    logging_sink_filter        = optional(string, "")<br>    location                   = optional(string, "global")<br>    enable_analytics           = optional(bool, true)<br>    linked_dataset_id          = optional(string, null)<br>    linked_dataset_description = optional(string, null)<br>    retention_days             = optional(number, 30)<br>  })</pre> | `null` | no |
 | logging\_destination\_project\_id | The ID of the project that will have the resources where the logs will be created. | `string` | n/a | yes |
 | logging\_project\_key | (Optional) The key of logging destination project if it is inside resources map. It is mandatory when resource\_type = project and logging\_target\_type = logbucket. | `string` | `""` | no |
+| project\_options | Destination Project options:<br>- logging\_sink\_name: The name of the log sink to be created.<br>- logging\_sink\_filter: The filter to apply when exporting logs. Only log entries that match the filter are exported. Default is "" which exports all logs.<br>- log\_bucket\_id: Id of the log bucket create to store the logs exported to the project.<br>- log\_bucket\_description: Description of the log bucket create to store the logs exported to the project.<br>- location: The location of the log bucket. Default: global.<br>- enable\_analytics: Whether or not Log Analytics is enabled in the \_Default log bucket. A Log bucket with Log Analytics enabled can be queried in the Log Analytics page using SQL queries. Cannot be disabled once enabled.<br>- retention\_days: The number of days data should be retained for the \_Default log bucket. Default 30.<br>- linked\_dataset\_id: The ID of the linked BigQuery dataset for the \_Default log bucket. A valid link dataset ID must only have alphanumeric characters and underscores within it and have up to 100 characters.<br>- linked\_dataset\_description: A use-friendly description of the linked BigQuery dataset for the \_Default log bucket. The maximum length of the description is 8000 characters. | <pre>object({<br>    logging_sink_name          = optional(string, null)<br>    logging_sink_filter        = optional(string, "")<br>    log_bucket_id              = optional(string, null)<br>    log_bucket_description     = optional(string, null)<br>    location                   = optional(string, "global")<br>    enable_analytics           = optional(bool, true)<br>    retention_days             = optional(number, 30)<br>    linked_dataset_id          = optional(string, null)<br>    linked_dataset_description = optional(string, null)<br>  })</pre> | `null` | no |
 | pubsub\_options | Destination Pubsub options:<br>- topic\_name: The name of the pubsub topic to be created and used for log entries matching the filter.<br>- logging\_sink\_name: The name of the log sink to be created.<br>- logging\_sink\_filter: The filter to apply when exporting logs. Only log entries that match the filter are exported. Default is "" which exports all logs.<br>- create\_subscriber: Whether to create a subscription to the topic that was created and used for log entries matching the filter. If 'true', a pull subscription is created along with a service account that is granted roles/pubsub.subscriber and roles/pubsub.viewer to the topic. | <pre>object({<br>    topic_name          = optional(string, null)<br>    logging_sink_name   = optional(string, null)<br>    logging_sink_filter = optional(string, "")<br>    create_subscriber   = optional(bool, true)<br>  })</pre> | `null` | no |
 | resource\_type | Resource type of the resource that will export logs to destination. Must be: project, organization, or folder. | `string` | n/a | yes |
 | resources | Export logs from the specified resources. | `map(string)` | n/a | yes |
@@ -74,8 +74,8 @@ module "logging_logbucket" {
 | Name | Description |
 |------|-------------|
 | billing\_sink\_names | Map of log sink names with billing suffix |
-| logbucket\_destination\_name | The resource name for the destination Log Bucket. |
-| logbucket\_linked\_dataset\_name | The resource name of the Log Bucket linked BigQuery dataset. |
+| project\_linked\_dataset\_name | The resource name of the Log Bucket linked BigQuery dataset for the project destination. |
+| project\_logbucket\_name | The resource name for the Log Bucket created for the project destination. |
 | pubsub\_destination\_name | The resource name for the destination Pub/Sub. |
 | storage\_destination\_name | The resource name for the destination Storage. |
 

1-org/modules/centralized-logging/main.tf

@@ -15,9 +15,8 @@
  */
 
 locals {
-  value_first_resource  = values(var.resources)[0]
-  logbucket_sink_member = { for k, v in var.resources : k => v if k != var.logging_project_key }
-  include_children      = (var.resource_type == "organization" || var.resource_type == "folder")
+  value_first_resource = values(var.resources)[0]
+  include_children     = (var.resource_type == "organization" || var.resource_type == "folder")
 
   # Create an intermediate list with all resources X all destinations
   exports_list = flatten([
@@ -38,40 +37,41 @@ locals {
   log_exports = {
     for v in local.exports_list : "${v.res}_${v.type}" => v
   }
+
   destinations_options = {
     pub = var.pubsub_options
     sto = var.storage_options
-    lbk = var.logbucket_options
+    prj = var.project_options
   }
 
   logging_sink_name_map = {
     pub = try("sk-to-tp-logs-${var.logging_destination_project_id}", "sk-to-tp-logs")
     sto = try("sk-to-bkt-logs-${var.logging_destination_project_id}", "sk-to-bkt-logs")
-    lbk = try("sk-to-logbkt-logs-${var.logging_destination_project_id}", "sk-to-logbkt-logs")
+    prj = try("sk-to-prj-logs-${var.logging_destination_project_id}", "sk-to-prj-logs")
   }
 
   logging_tgt_name = {
     pub = "${local.logging_tgt_prefix.pub}${random_string.suffix.result}"
     sto = "${local.logging_tgt_prefix.sto}${random_string.suffix.result}"
-    lbk = "${local.logging_tgt_prefix.lbk}${random_string.suffix.result}"
+    prj = ""
   }
 
   destination_uri_map = {
     pub = try(module.destination_pubsub[0].destination_uri, "")
     sto = try(module.destination_storage[0].destination_uri, "")
-    lbk = try(module.destination_logbucket[0].destination_uri, "")
+    prj = try(module.destination_project[0].destination_uri, "")
   }
 
   destination_resource_name = merge(
     var.pubsub_options != null ? { pub = module.destination_pubsub[0].resource_name } : {},
     var.storage_options != null ? { sto = module.destination_storage[0].resource_name } : {},
-    var.logbucket_options != null ? { lbk = module.destination_logbucket[0].resource_name } : {}
+    var.project_options != null ? { prj = module.destination_project[0].project } : {}
   )
 
   logging_tgt_prefix = {
     pub = "tp-logs-"
     sto = try("bkt-logs-${var.logging_destination_project_id}-", "bkt-logs-")
-    lbk = "logbkt-logs-"
+    prj = ""
   }
 }
 
@@ -83,7 +83,7 @@ resource "random_string" "suffix" {
 
 module "log_export" {
   source  = "terraform-google-modules/log-export/google"
-  version = "~> 7.4"
+  version = "~> 7.8"
 
   for_each = local.log_exports
 
@@ -96,7 +96,6 @@ module "log_export" {
   include_children       = local.include_children
 }
 
-
 module "log_export_billing" {
   source  = "terraform-google-modules/log-export/google"
   version = "~> 7.4"
@@ -118,52 +117,115 @@ resource "time_sleep" "wait_sa_iam_membership" {
   ]
 }
 
-#-------------------------#
-# Send logs to Log Bucket #
-#-------------------------#
-module "destination_logbucket" {
-  source  = "terraform-google-modules/log-export/google//modules/logbucket"
-  version = "~> 7.7"
+#--------------------------#
+# Send logs to Log project #
+#--------------------------#
 
-  count = var.logbucket_options != null ? 1 : 0
+module "destination_project" {
+  source  = "terraform-google-modules/log-export/google//modules/project"
+  version = "~> 7.8"
+  count   = var.project_options != null ? 1 : 0
 
-  project_id                    = var.logging_destination_project_id
-  name                          = coalesce(var.logbucket_options.name, local.logging_tgt_name.lbk)
-  log_sink_writer_identity      = module.log_export["${local.value_first_resource}_lbk"].writer_identity
-  location                      = var.logbucket_options.location
-  enable_analytics              = var.logbucket_options.enable_analytics
-  linked_dataset_id             = var.logbucket_options.linked_dataset_id
-  linked_dataset_description    = var.logbucket_options.linked_dataset_description
-  retention_days                = var.logbucket_options.retention_days
-  grant_write_permission_on_bkt = false
+  project_id               = var.logging_destination_project_id
+  log_sink_writer_identity = module.log_export["${local.value_first_resource}_prj"].writer_identity
 }
 
-#-------------------------------------------#
-# Log Bucket Service account IAM membership #
-#-------------------------------------------#
-resource "google_project_iam_member" "logbucket_sink_member" {
-  for_each = var.logbucket_options != null ? local.logbucket_sink_member : {}
+#---------------------------------------------#
+# Log Projects Service account IAM membership #
+#---------------------------------------------#
+
+resource "google_project_iam_member" "project_sink_member" {
+  for_each = var.project_options != null ? var.resources : {}
 
   project = var.logging_destination_project_id
-  role    = "roles/logging.bucketWriter"
+  role    = "roles/logging.logWriter"
 
   # Set permission only on sinks for this destination using
   # module.log_export key "<resource>_<dest>"
-  member = module.log_export["${each.value}_lbk"].writer_identity
+  member = module.log_export["${each.value}_prj"].writer_identity
+}
+
+#----------------------------------------------#
+# Send logs to Log project - Internal Log sink #
+#----------------------------------------------#
+
+module "internal_project_log_export" {
+  source  = "terraform-google-modules/log-export/google"
+  version = "~> 7.8"
+  count   = var.project_options != null ? 1 : 0
+
+  destination_uri        = "logging.googleapis.com/projects/${var.logging_destination_project_id}/locations/${var.project_options.location}/buckets/${coalesce(var.project_options.log_bucket_id, "AggregatedLogs")}"
+  filter                 = var.project_options.logging_sink_filter
+  log_sink_name          = "${coalesce(var.project_options.logging_sink_name, local.logging_sink_name_map["prj"])}-la"
+  parent_resource_id     = var.logging_destination_project_id
+  parent_resource_type   = "project"
+  unique_writer_identity = true
+}
+
+module "destination_aggregated_logs" {
+  source  = "terraform-google-modules/log-export/google//modules/logbucket"
+  version = "~> 7.8"
+  count   = var.project_options != null ? 1 : 0
+
+  project_id                    = var.logging_destination_project_id
+  name                          = coalesce(var.project_options.log_bucket_id, "AggregatedLogs")
+  log_sink_writer_identity      = module.internal_project_log_export[0].writer_identity
+  location                      = var.project_options.location
+  enable_analytics              = var.project_options.enable_analytics
+  linked_dataset_id             = var.project_options.linked_dataset_id
+  linked_dataset_description    = var.project_options.linked_dataset_description
+  retention_days                = var.project_options.retention_days
+  grant_write_permission_on_bkt = false
+}
+
+#-------------------------------------------------#
+# Send logs to Log project - update _Default sink #
+#-------------------------------------------------#
+
+data "google_client_config" "default" {
 }
 
-#------------------------------------------------------------------#
-# Log Bucket Service account IAM membership for log_export_billing #
-#------------------------------------------------------------------#
-resource "google_project_iam_member" "logbucket_sink_member_billing" {
-  count = var.enable_billing_account_sink == true && var.logbucket_options != null ? 1 : 0
+resource "terracurl_request" "exclude_external_logs" {
+  count = var.project_options != null ? 1 : 0
+
+  name           = "exclude_external_logs"
+  url            = "https://logging.googleapis.com/v2/projects/${var.logging_destination_project_id}/sinks/_Default?updateMask=exclusions"
+  method         = "PUT"
+  response_codes = [200]
+  headers = {
+    Authorization = "Bearer ${data.google_client_config.default.access_token}"
+    Content-Type  = "application/json",
+  }
+  request_body = <<EOF
+{
+  "exclusions": [
+    {
+      "name": "exclude_external_logs",
+      "filter": "-logName : \"/${var.logging_destination_project_id}/\""
+    }
+  ],
+}
+EOF
+
+  lifecycle {
+    ignore_changes = [
+      headers,
+    ]
+  }
+}
+
+#---------------------------------------------------------------#
+# Project Service account IAM membership for log_export_billing #
+#---------------------------------------------------------------#
+resource "google_project_iam_member" "project_sink_member_billing" {
+  count = var.enable_billing_account_sink == true && var.project_options != null ? 1 : 0
 
   project = var.logging_destination_project_id
-  role    = "roles/logging.bucketWriter"
+  role    = "roles/logging.logWriter"
 
   # Set permission only on sinks for this destination using
   # module.log_export_billing key "<resource>_<dest>"
-  member = module.log_export_billing["lbk"].writer_identity
+  member = module.log_export_billing["prj"].writer_identity
 
 
   depends_on = [
@@ -176,7 +238,7 @@ resource "google_project_iam_member" "logbucket_sink_member_billing" {
 #----------------------#
 module "destination_storage" {
   source  = "terraform-google-modules/log-export/google//modules/storage"
-  version = "~> 7.4"
+  version = "~> 7.8"
 
   count = var.storage_options != null ? 1 : 0
 
@@ -217,7 +279,7 @@ resource "google_storage_bucket_iam_member" "storage_sink_member_billing" {
 
 
   depends_on = [
-    google_project_iam_member.logbucket_sink_member_billing
+    google_project_iam_member.project_sink_member_billing
   ]
 }
 
@@ -227,7 +289,7 @@ resource "google_storage_bucket_iam_member" "storage_sink_member_billing" {
 #----------------------#
 module "destination_pubsub" {
   source  = "terraform-google-modules/log-export/google//modules/pubsub"
-  version = "~> 7.4"
+  version = "~> 7.8"
 
   count = var.pubsub_options != null ? 1 : 0