Kubernetes cluster と Grafana Cloud を連携してみる（前編） 〜Getting Started Guide の先へ〜

# Source: k8s-monitoring/templates/alloy-config.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: grafana-k8s-monitoring-alloy-logs
  namespace: alloy
data:
  config.alloy: |
    // Feature: Pod Logs
    declare "pod_logs" {
      argument "logs_destinations" {
        comment = "Must be a list of log destinations where collected logs should be forwarded to"
      }
    
      discovery.relabel "filtered_pods" {
        targets = discovery.kubernetes.pods.targets
        rule {
          source_labels = ["__meta_kubernetes_namespace"]
          action = "replace"
          target_label = "namespace"
        }
        rule {
          source_labels = ["__meta_kubernetes_pod_name"]
          action = "replace"
          target_label = "pod"
        }
        rule {
          source_labels = ["__meta_kubernetes_pod_container_name"]
          action = "replace"
          target_label = "container"
        }
        rule {
          source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_container_name"]
          separator = "/"
          action = "replace"
          replacement = "$1"
          target_label = "job"
        }
    
        // set the container runtime as a label
        rule {
          action = "replace"
          source_labels = ["__meta_kubernetes_pod_container_id"]
          regex = "^(\\S+):\\/\\/.+$"
          replacement = "$1"
          target_label = "tmp_container_runtime"
        }
    
        // make all labels on the pod available to the pipeline as labels,
        // they are omitted before write to loki via stage.label_keep unless explicitly set
        rule {
          action = "labelmap"
          regex = "__meta_kubernetes_pod_label_(.+)"
        }
    
        // make all annotations on the pod available to the pipeline as labels,
        // they are omitted before write to loki via stage.label_keep unless explicitly set
        rule {
          action = "labelmap"
          regex = "__meta_kubernetes_pod_annotation_(.+)"
        }
    
        // explicitly set service_name. if not set, loki will automatically try to populate a default.
        // see https://grafana.com/docs/loki/latest/get-started/labels/#default-labels-for-all-users
        //
        // choose the first value found from the following ordered list:
        // - pod.annotation[resource.opentelemetry.io/service.name]
        // - pod.label[app.kubernetes.io/name]
        // - k8s.pod.name
        // - k8s.container.name
        rule {
          action = "replace"
          source_labels = [
            "__meta_kubernetes_pod_annotation_resource_opentelemetry_io_service_name",
            "__meta_kubernetes_pod_label_app_kubernetes_io_name",
            "__meta_kubernetes_pod_container_name",
          ]
          separator = ";"
          regex = "^(?:;*)?([^;]+).*$"
          replacement = "$1"
          target_label = "service_name"
        }
    
        // explicitly set service_namespace.
        //
        // choose the first value found from the following ordered list:
        // - pod.annotation[resource.opentelemetry.io/service.namespace]
        // - pod.namespace
        rule {
          action = "replace"
          source_labels = [
            "__meta_kubernetes_pod_annotation_resource_opentelemetry_io_service_namespace",
            "namespace",
          ]
          separator = ";"
          regex = "^(?:;*)?([^;]+).*$"
          replacement = "$1"
          target_label = "service_namespace"
        }
    
        // explicitly set service_instance_id.
        //
        // choose the first value found from the following ordered list:
        // - pod.annotation[resource.opentelemetry.io/service.instance.id]
        // - concat([k8s.namespace.name, k8s.pod.name, k8s.container.name], '.')
        rule {
          source_labels = ["__meta_kubernetes_pod_annotation_resource_opentelemetry_io_service_instance_id"]
          target_label = "service_instance_id"
        }
        rule {
          source_labels = ["service_instance_id", "namespace", "pod", "container"]
          separator = "."
          regex = "^\\.([^.]+\\.[^.]+\\.[^.]+)$"
          target_label = "service_instance_id"
        }
    
        // set resource attributes
        rule {
          action = "labelmap"
          regex = "__meta_kubernetes_pod_annotation_resource_opentelemetry_io_(.+)"
        }
        rule {
          source_labels = ["__meta_kubernetes_pod_annotation_k8s_grafana_com_logs_job"]
          regex = "(.+)"
          target_label = "job"
        }
        rule {
          source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_name"]
          regex = "(.+)"
          target_label = "app_kubernetes_io_name"
        }
      }
    
      discovery.kubernetes "pods" {
        role = "pod"
        selectors {
          role = "pod"
          field = "spec.nodeName=" + sys.env("HOSTNAME")
        }
      }
    
      discovery.relabel "filtered_pods_with_paths" {
        targets = discovery.relabel.filtered_pods.output
    
        rule {
          source_labels = ["__meta_kubernetes_pod_uid", "__meta_kubernetes_pod_container_name"]
          separator = "/"
          action = "replace"
          replacement = "/var/log/pods/*$1/*.log"
          target_label = "__path__"
        }
      }
    
      local.file_match "pod_logs" {
        path_targets = discovery.relabel.filtered_pods_with_paths.output
      }
    
      loki.source.file "pod_logs" {
        targets    = local.file_match.pod_logs.targets
        forward_to = [loki.process.pod_logs.receiver]
      }
    
      loki.process "pod_logs" {
        stage.match {
          selector = "{tmp_container_runtime=~\"containerd|cri-o\"}"
          // the cri processing stage extracts the following k/v pairs: log, stream, time, flags
          stage.cri {}
    
          // Set the extract flags and stream values as labels
          stage.labels {
            values = {
              flags  = "",
              stream  = "",
            }
          }
        }
    
        stage.match {
          selector = "{tmp_container_runtime=\"docker\"}"
          // the docker processing stage extracts the following k/v pairs: log, stream, time
          stage.docker {}
    
          // Set the extract stream value as a label
          stage.labels {
            values = {
              stream  = "",
            }
          }
        }
    
        // Drop the filename label, since it's not really useful in the context of Kubernetes, where we already have cluster,
        // namespace, pod, and container labels. Drop any structured metadata. Also drop the temporary
        // container runtime label as it is no longer needed.
        stage.label_drop {
          values = [
            "filename",
            "tmp_container_runtime",
          ]
        }
        stage.structured_metadata {
          values = {
            "k8s_pod_name" = "k8s_pod_name",
            "pod" = "pod",
            "service_instance_id" = "service_instance_id",
          }
        }
    
        // Only keep the labels that are defined in the `keepLabels` list.
        stage.label_keep {
          values = ["__tenant_id__","app_kubernetes_io_name","container","instance","job","level","namespace","service_name","service_namespace","deployment_environment","deployment_environment_name","k8s_namespace_name","k8s_deployment_name","k8s_statefulset_name","k8s_daemonset_name","k8s_cronjob_name","k8s_job_name","k8s_node_name"]
        }
    
        forward_to = argument.logs_destinations.value
      }
    }
    pod_logs "feature" {
      logs_destinations = [
        loki.write.grafana_cloud_logs.receiver,
      ]
    }
    
    
    
    
    // Destination: grafana-cloud-logs (loki)
    otelcol.exporter.loki "grafana_cloud_logs" {
      forward_to = [loki.write.grafana_cloud_logs.receiver]
    }
    
    loki.write "grafana_cloud_logs" {
      endpoint {
        url = "https://logs-prod-030.grafana.net/loki/api/v1/push"
        basic_auth {
          username = convert.nonsensitive(remote.kubernetes.secret.grafana_cloud_logs.data["username"])
          password = remote.kubernetes.secret.grafana_cloud_logs.data["password"]
        }
        tls_config {
          insecure_skip_verify = false
        }
        min_backoff_period = "500ms"
        max_backoff_period = "5m"
        max_backoff_retries = "10"
      }
      external_labels = {
        "cluster" = "my-cluster",
        "k8s_cluster_name" = "my-cluster",
      }
    }
    
    remote.kubernetes.secret "grafana_cloud_logs" {
      name      = "grafana-cloud-logs-grafana-k8s-monitoring"
      namespace = "alloy"
    }