Grafana と Loki でアプリケーションエラーを意味のある単位で監視する

otelcol.processor.transform "severity" {
  log_statements {
    context = "log"
    statements = [
      // Normalize custom slog texts such as ERROR+4 to the OTLP FATAL range.
      `set(severity_text, "FATAL") where severity_number >= 21 and severity_number <= 24`,
      // Loki's OTLP attribute mapping indexes attributes, not LogRecord fields.
      `set(attributes["severity_text"], severity_text)`,
    ]
  }
  output {
    logs = [otelcol.processor.batch.logs.input]
  }
}

sum by (deployment_environment_name, error_group, exception_type, severity) (
  count_over_time(
    {service_name="otel-log-demo", severity_text=~"ERROR|FATAL|WARN"}
    # Exclude errors handled by dedicated metric- or log-based rules from this general rule.
    | alert_general_rule_exclude!="true"
    # errors.Formattable's first two lines and Wraps messages can contain variable error text,
    # such as IP addresses and timestamps.
    # Remove them only from the fingerprint, preserving frame order and line numbers.
    | label_format error_group=`{{ if and .exception_type .exception_stacktrace }}{{ regexReplaceAll "(?m)^Wraps: [^\\r\\n]*(\\r?\\n|$)" (regexReplaceAll "^[^\\r\\n]*\\r?\\n[^\\r\\n]*(\\r?\\n|$)" .exception_stacktrace "") "" }}{{ end }}`
    # Without a usable typed stack, fingerprint the body without hex values or digits.
    | label_format error_group=`{{ if .error_group }}{{ .error_group }}{{ else }}{{ regexReplaceAll "[0-9]+" (regexReplaceAll "0[xX][0-9a-fA-F]+" (__line__) "") "" }}{{ end }}`
    # Use the normalized severity in both alert identity and notification grouping.
    | label_format severity=`{{ .severity_text }}`
    [10m]
  )
)

{service_name="otel-log-demo", severity_text=~"ERROR|FATAL|WARN"}

# Exclude errors handled by dedicated metric- or log-based rules from this general rule.
| alert_general_rule_exclude!="true"

# errors.Formattable's first two lines and Wraps messages can contain variable error text,
# such as IP addresses and timestamps.
# Remove them only from the fingerprint, preserving frame order and line numbers.
| label_format error_group=`{{ if and .exception_type .exception_stacktrace }}{{ regexReplaceAll "(?m)^Wraps: [^\\r\\n]*(\\r?\\n|$)" (regexReplaceAll "^[^\\r\\n]*\\r?\\n[^\\r\\n]*(\\r?\\n|$)" .exception_stacktrace "") "" }}{{ end }}`

# Use the normalized severity in both alert identity and notification grouping.
| label_format severity=`{{ .severity_text }}`

topk by (deployment_environment_name, error_group, exception_type, severity) (
  1,
  max by (deployment_environment_name, error_group, exception_type, severity, error_message, exception_stacktrace, runbook_url, error_time_from, error_time_to) (
    max_over_time(
      {service_name="otel-log-demo", severity_text=~"ERROR|FATAL|WARN"}
      # Exclude errors handled by dedicated metric- or log-based rules from this general rule.
      | alert_general_rule_exclude!="true"
      # errors.Formattable's first two lines and Wraps messages can contain variable error text,
      # such as IP addresses and timestamps.
      # Remove them only from the fingerprint, preserving frame order and line numbers.
      | label_format error_group=`{{ if and .exception_type .exception_stacktrace }}{{ regexReplaceAll "(?m)^Wraps: [^\\r\\n]*(\\r?\\n|$)" (regexReplaceAll "^[^\\r\\n]*\\r?\\n[^\\r\\n]*(\\r?\\n|$)" .exception_stacktrace "") "" }}{{ end }}`
      # Without a usable typed stack, fingerprint the body without hex values or digits.
      | label_format error_group=`{{ if .error_group }}{{ .error_group }}{{ else }}{{ regexReplaceAll "[0-9]+" (regexReplaceAll "0[xX][0-9a-fA-F]+" (__line__) "") "" }}{{ end }}`
      # Use the normalized severity in both alert identity and notification grouping.
      | label_format severity=`{{ .severity_text }}`
      # Include the message for notification display, but not in the labels that identify an alert.
      | label_format error_message=`{{ __line__ }}{{ if and .exception_message (not (contains .exception_message (__line__))) }}: {{ .exception_message }}{{ end }}`
      # Save the Explore time range and expose the timestamp as a numeric value for latest-log selection.
      | label_format error_time_from=`{{ unixEpochMillis ((__timestamp__).Add -10000000000) }}`
      | label_format error_time_to=`{{ unixEpochMillis (__timestamp__) }}`
      | label_format error_epoch=`{{ unixEpochMillis (__timestamp__) }}`
      | unwrap error_epoch
      [10m]
    )
  )
)

max_over_time(
  -- snip --
  | unwrap error_epoch
  [10m]
)

{error_group="database", error_message="database request 100 timed out", ...} 1000
{error_group="database", error_message="database request 101 timed out", ...} 5000
{error_group="payment",  error_message="payment request 200 failed", ...}    3000

label_replace(vector(1), "severity", "ERROR", "", ".*")
or
label_replace(vector(0), "severity", "FATAL", "", ".*")
or
label_replace(vector(10), "severity", "WARN", "", ".*")

{{ with $labels.exception_type }}
  {{ . }}
{{ end }}
{{ with $values.latest_error }}
{{ with .Labels.error_message }}
  {{ if $labels.exception_type }}: {{ end }}{{ . }}{{ end }}
{{ end }}

{{ $runbook := "" -}}
{{ with $values.latest_error -}}
  {{ with .Labels.runbook_url -}}
    {{ $runbook = . -}}
  {{ end -}}
{{ end -}}
{{ if not $runbook -}}
  {{ if eq $labels.exception_type "main.databaseError" -}}
    {{ $runbook = "https://example.com/runbooks/database" -}}
  {{ end -}}
{{ end -}}
{{ $runbook -}}

{{ with $values.latest_error }}{{ with .Labels.exception_stacktrace }}{{ . }}{{ end }}{{ end }}

{{ printf "{service_name=%q, deployment_environment_name=%q}" $labels.service_name $labels.deployment_environment_name }}

{{ with $values.latest_error }}{{ .Labels.error_time_from }}{{ end }}

{{ with $values.latest_error }}{{ .Labels.error_time_to }}{{ end }}

{{ .CommonLabels.service_name }}: {{ if .Alerts.Firing }}{{ (index .Alerts.Firing 0).Annotations.summary }}
{{ else if .Alerts.Resolved }}{{ (index .Alerts.Resolved 0).Annotations.summary }}
{{ else }}No alerts in notification
{{ end }}

{{ range .Alerts -}}
{{ .Annotations.description }}
  {{ $hasExplore := and .Annotations.explore_query .Annotations.explore_from .Annotations.explore_to -}}
  {{ $hasSilence := and .SilenceURL (le (len .SilenceURL) 5000) -}}
  {{ if or .Annotations.runbook_url .GeneratorURL .DashboardURL $hasExplore $hasSilence }}
*Actions*

    {{ with .Annotations.runbook_url -}}
- <{{ . }}|Go to runbook>
    {{ end -}}
    {{ with .GeneratorURL -}}
- <{{ . }}|Go to alert rule>
    {{ end -}}
    {{ with .DashboardURL -}}
- <{{ . }}|Go to dashboard>
    {{ end -}}
    {{ if $hasExplore -}}
      {{/* Use data.ToJSON instead of printf "%q", which can emit Go-only escapes such as \x01 that are invalid in JSON. */ -}}
      {{ $panes := printf "{\"logs\":{\"datasource\":\"loki\",\"queries\":[{\"datasource\":{\"type\":\"loki\",\"uid\":\"loki\"},\"editorMode\":\"code\",\"expr\":%s,\"queryType\":\"range\",\"refId\":\"logs\"}],\"range\":{\"from\":\"%s\",\"to\":\"%s\"}}}" (data.ToJSON .Annotations.explore_query) .Annotations.explore_from .Annotations.explore_to -}}
- <http://localhost:3000/explore?schemaVersion=1&panes={{ urlquery $panes }}|Explore logs>
    {{ end -}}
    {{ if $hasSilence -}}
- <{{ .SilenceURL }}|Create silence>
    {{ end -}}
  {{ end -}}
  {{ with .Annotations.stacktrace }}
*Stack trace*

```
{{ printf "%.3000s" . }}
```
  {{ end }}
  {{ $firstLabel := true -}}
  {{ range $key, $value := .Labels -}}
    {{ if and (ne $key "error_group") (ne $key "service_name") -}}
      {{ if $firstLabel -}}
*Other labels:*

        {{ $firstLabel = false -}}
      {{ end -}}
- {{ $key }}: {{ $value }}
    {{ end -}}
  {{ end }}
  {{ $firstAnnotation := true -}}
  {{ range $key, $value := .Annotations -}}
    {{ if and (ne $key "summary") (ne $key "description") (ne $key "runbook_url") (ne $key "stacktrace") (ne $key "explore_query") (ne $key "explore_from") (ne $key "explore_to") -}}
      {{ if $firstAnnotation -}}
*Other annotations:*

        {{ $firstAnnotation = false -}}
      {{ end -}}
- {{ $key }}: {{ $value }}
    {{ end -}}
  {{ end }}
{{ end -}}

  {{ with .Annotations.stacktrace }}
*Stack trace*

```
{{ printf "%.3000s" . }}
```
  {{ end }}