From 1739ddb5ba6d28af388e7085c1673e6582a254d6 Mon Sep 17 00:00:00 2001
From: r0zehnal0vak <k.rozehnalka@seznam.cz>
Date: Mon, 30 Sep 2024 14:49:38 +0200
Subject: [PATCH] update monitoring
---
.../roles/monitoring/files/values.yaml | 1 +
ansible/docs/metrics.md | 50 +++++++++++++++++++
2 files changed, 51 insertions(+)
create mode 100644 ansible/docs/metrics.md
diff --git a/ansible/01-playbook/roles/monitoring/files/values.yaml b/ansible/01-playbook/roles/monitoring/files/values.yaml
index b5d8ebc..17b3be0 100644
--- a/ansible/01-playbook/roles/monitoring/files/values.yaml
+++ b/ansible/01-playbook/roles/monitoring/files/values.yaml
@@ -35,6 +35,7 @@ grafana:
token_url: 'https://login.e-infra.cz/oidc/token'
api_url: 'https://login.e-infra.cz/oidc/userinfo'
use_pkce: true
+ # role_attribute_path: to_string('Viewer') # assign role (Admin, Editor, Viewer) to all logged in users
prometheus:
prometheusSpec:
diff --git a/ansible/docs/metrics.md b/ansible/docs/metrics.md
new file mode 100644
index 0000000..3bce909
--- /dev/null
+++ b/ansible/docs/metrics.md
@@ -0,0 +1,50 @@
+### Labeling
+For additional metrics labeling, add label to service monitor in order to progapate labels to Prometheus. Check [this](https://grafana.com/blog/2022/03/21/how-relabeling-in-prometheus-works/) for more information.
+```
+# Examle of adding label to etcd service monitor:
+
+kubeEtcd:
+ serviceMonitor:
+ enabled: true
+ port: http-metrics
+ scheme: http
+ relabelings:
+ - targetLabel: env
+ replacement: prod
+```
+Query it with [PromQL](https://prometheus.io/docs/prometheus/latest/querying/basics/) using <code>up{env='prod'}</code>. To add labels to other service monitor follow the [default values](https://github.com/prometheus-community/helm-charts/blob/main/charts/kube-prometheus-stack/values.yaml).
+### Filtering
+When connected to the master node via SSH, you can curl the available metrics using <code>curl `http://target_ip:port/metrics`</code>.
+```
+# Example of keeping only specific metrics of the etcd target, the rest is dropped:
+
+kubeEtcd:
+ serviceMonitor:
+ enabled: true
+ metricRelabelings:
+ - action: keep
+ regex: 'etcd_network_(client_grpc_received_bytes_total|client_grpc_sent_bytes_total)|etcd_server_has_leader'
+ sourceLabels: [__name__]
+
+# Example of dropping specific metrics of the kubeApiServer target and keeping the rest
+
+kubeApiServer:
+ serviceMonitor:
+ metricRelabelings:
+ # Drop excessively noisy apiserver buckets.
+ - action: drop
+ regex: apiserver_request_duration_seconds_bucket;(0.15|0.2|0.3|0.35|0.4|0.45|0.6|0.7|0.8|0.9|1.25|1.5|1.75|2|3|3.5|4|4.5|6|7|8|9|15|25|40|50)
+ sourceLabels:
+ - __name__
+ - le
+
+# Keep metrics only with custom label env=prod:
+ - action: keep
+ regex: '.*' # match all metrics
+ sourceLabels: [env] # Filters based on the env label
+ separator: ; # Use this if you want to keep the default labels too
+ replacement: prod
+```
+Source labels:
+- <code>\_\_name__</code>: Use this when you want to filter based on the metric name.
+- <code>le</code>: Use this when working with metrics that have defined buckets, such as request durations or sizes.
\ No newline at end of file
--
GitLab