From 1fcd0ff68b6748cf5d4e50d12a9d7959f181baed Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Tue, 11 Aug 2026 01:33:23 +0900 Subject: [PATCH 1/2] =?UTF-8?q?fix:=20kube-vip=20=EB=A1=9C=EA=B7=B8=20?= =?UTF-8?q?=EC=88=98=EC=A7=91=20=EA=B2=BD=EB=A1=9C=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../grafana-dashboard-monitoring-stack.yaml | 18 +++++++++++++++++- kubernetes/monitoring/logs/README.md | 3 ++- kubernetes/monitoring/logs/alloy-values.yaml | 4 ++-- 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/kubernetes/monitoring/grafana-dashboard-monitoring-stack.yaml b/kubernetes/monitoring/grafana-dashboard-monitoring-stack.yaml index b8f47ea4..a24d8f59 100644 --- a/kubernetes/monitoring/grafana-dashboard-monitoring-stack.yaml +++ b/kubernetes/monitoring/grafana-dashboard-monitoring-stack.yaml @@ -10,7 +10,7 @@ data: "uid": "codeplace-monitoring-stack", "title": "CodePlace Monitoring Stack", "schemaVersion": 39, - "version": 2, + "version": 3, "refresh": "30s", "tags": ["codeplace", "monitoring", "scrape-contract"], "links": [ @@ -571,6 +571,22 @@ data: "legendFormat": "{{namespace}} {{job}}" } ] + }, + { + "type": "logs", + "title": "Recent kube-vip Logs", + "description": "Cluster control-plane kube-vip logs for VIP ownership, leader election, and ARP troubleshooting.", + "gridPos": {"x": 0, "y": 49, "w": 24, "h": 8}, + "datasource": {"type": "loki", "uid": "Loki"}, + "options": { + "showLabels": false, + "showTime": true, + "wrapLogMessage": true, + "dedupStrategy": "none" + }, + "targets": [ + {"refId": "A", "expr": "{namespace=\"kube-system\", app_kubernetes_io_name=\"kube-vip-ds\"}"} + ] } ] } diff --git a/kubernetes/monitoring/logs/README.md b/kubernetes/monitoring/logs/README.md index 3292490e..384c2594 100644 --- a/kubernetes/monitoring/logs/README.md +++ b/kubernetes/monitoring/logs/README.md @@ -4,7 +4,7 @@ This directory contains Helm values for the Kubernetes logs stack. - Loki: SingleBinary mode, filesystem storage, Longhorn PVC. - Loki gateway: 2 replicas for the stateless write/query entrypoint. -- Alloy: DaemonSet log collector for every pod in `code-place-dev`, `code-place-prod`, and `monitoring`, plus Traefik pods in `kube-system`. It mounts host `/var/log` so `/var/log/pods` is readable from the Alloy container. It tolerates the standard `control-plane` and `master` NoSchedule taints so tainted control-plane nodes are still covered. +- Alloy: DaemonSet log collector for every pod in `code-place-dev`, `code-place-prod`, and `monitoring`, plus Traefik and kube-vip pods in `kube-system`. It mounts host `/var/log` so `/var/log/pods` is readable from the Alloy container. It tolerates the standard `control-plane` and `master` NoSchedule taints so tainted control-plane nodes are still covered. - Kubernetes Event Exporter: Warning events are written to stdout as JSON and collected by Alloy from the `monitoring` namespace. - Retention: `code-place-dev` 3 days, `code-place-prod` 7 days. - Monitoring: Loki and Alloy ServiceMonitors are enabled for kube-prometheus-stack. @@ -79,6 +79,7 @@ Grafana should show a `Loki` datasource. Useful Explore queries: {namespace="code-place-prod", app="backend"} | json {namespace="code-place-dev", container=~"backend|celery-worker|judge-server"} {namespace="kube-system", app_kubernetes_io_name="traefik"} | json | DownstreamStatus >= 400 +{namespace="kube-system", app_kubernetes_io_name="kube-vip-ds"} {namespace="monitoring", app_kubernetes_io_name="kubernetes-event-exporter"} ``` diff --git a/kubernetes/monitoring/logs/alloy-values.yaml b/kubernetes/monitoring/logs/alloy-values.yaml index 8903943a..0e9ecffa 100644 --- a/kubernetes/monitoring/logs/alloy-values.yaml +++ b/kubernetes/monitoring/logs/alloy-values.yaml @@ -1,5 +1,5 @@ # Helm values for grafana/alloy. -# Alloy runs as a DaemonSet and collects CodePlace, monitoring, and Traefik logs. +# Alloy runs as a DaemonSet and collects CodePlace, monitoring, Traefik, and kube-vip logs. controller: type: daemonset @@ -37,7 +37,7 @@ alloy: rule { source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_label_app", "__meta_kubernetes_pod_label_app_kubernetes_io_name"] separator = "/" - regex = "(code-place-dev|code-place-prod|monitoring)/.*|kube-system/.*/traefik" + regex = "(code-place-dev|code-place-prod|monitoring)/.*|kube-system/.*/(traefik|kube-vip-ds)" action = "keep" } From 952fe277e1234d06632c69a9028bf9913f458f16 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Tue, 11 Aug 2026 01:47:38 +0900 Subject: [PATCH 2/2] =?UTF-8?q?fix:=20Kubernetes=20=EC=9D=B4=EB=B2=A4?= =?UTF-8?q?=ED=8A=B8=20=EB=A9=94=ED=83=80=EB=8D=B0=EC=9D=B4=ED=84=B0=20?= =?UTF-8?q?=EC=A1=B0=ED=9A=8C=20=EC=98=A4=EB=A5=98=20=EC=A0=9C=EA=B1=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kubernetes/monitoring/kubernetes-event-exporter.yaml | 1 + kubernetes/monitoring/logs/README.md | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/kubernetes/monitoring/kubernetes-event-exporter.yaml b/kubernetes/monitoring/kubernetes-event-exporter.yaml index 17addff2..49b63f71 100644 --- a/kubernetes/monitoring/kubernetes-event-exporter.yaml +++ b/kubernetes/monitoring/kubernetes-event-exporter.yaml @@ -44,6 +44,7 @@ data: config.yaml: | logLevel: warn logFormat: json + omitLookup: true maxEventAgeSeconds: 60 metricsNamePrefix: event_exporter_ route: diff --git a/kubernetes/monitoring/logs/README.md b/kubernetes/monitoring/logs/README.md index 384c2594..c5dca2d9 100644 --- a/kubernetes/monitoring/logs/README.md +++ b/kubernetes/monitoring/logs/README.md @@ -47,6 +47,10 @@ helm upgrade --install alloy grafana/alloy \ kubectl apply -k kubernetes/monitoring +# Event exporter reads its ConfigMap only at startup. +kubectl -n monitoring rollout restart deployment/kubernetes-event-exporter +kubectl -n monitoring rollout status deployment/kubernetes-event-exporter --timeout=3m + # One-time migration after the replacement Traefik PodMonitor is present. kubectl -n monitoring get podmonitor traefik kubectl -n monitoring delete servicemonitor traefik --ignore-not-found