Merge branch 'update-monitoring' into 'master'

update monitoring

See merge request !21

ansible/01-playbook/roles/monitoring/files/values.yaml

@@ -35,6 +35,7 @@ grafana:
         token_url: 'https://login.e-infra.cz/oidc/token'
         api_url: 'https://login.e-infra.cz/oidc/userinfo'
         use_pkce: true
+        # role_attribute_path: to_string('Viewer') # assign role (Admin, Editor, Viewer) to all logged in users 
 
 prometheus:
   prometheusSpec:
@@ -94,7 +95,7 @@ alertmanager:
       routes:
       - receiver: 'null'
         matchers:
-          - alertname =~ "(Watchdog|etcdHighCommitDurations|etcdHighFsyncDurations|PrometheusDuplicateTimestamps|CPUThrottlingHigh)" # alerts to be ignored
+          - alertname =~ "(Watchdog|InfoInhibitor|etcdHighCommitDurations|etcdHighFsyncDurations|CPUThrottlingHigh)" # alerts to be ignored
       - receiver: 'slack-notifications'
         continue: true
       - receiver: 'outlook-notifications'
@@ -138,4 +139,12 @@ kubeEtcd:
   #   scheme: http
   #   relabelings:
   #     - targetLabel: env
-  #       replacement: prod
+  #       replacement: prod
\ No newline at end of file
+
+kube-state-metrics:
+  prometheus:
+    monitor:
+      metricRelabelings:
+        - action: drop
+          regex: "kube_endpoint_ports;etcd-metrics" # fix PrometheusDuplicateTimestamps alert
+          sourceLabels: [__name__, endpoint]
\ No newline at end of file

ansible/docs/metrics.md

+### Labeling
+For additional metrics labeling, add label to service monitor in order to progapate labels to Prometheus. Check [this](https://grafana.com/blog/2022/03/21/how-relabeling-in-prometheus-works/) for more information.
+```
+# Examle of adding label to etcd service monitor:
+
+kubeEtcd:
+  serviceMonitor:
+    enabled: true
+    port: http-metrics
+    scheme: http
+    relabelings:
+      - targetLabel: env
+        replacement: prod
+```
+Query it with [PromQL](https://prometheus.io/docs/prometheus/latest/querying/basics/) using <code>up{env='prod'}</code>. To add labels to other service monitor follow the [default values](https://github.com/prometheus-community/helm-charts/blob/main/charts/kube-prometheus-stack/values.yaml).
+### Filtering
+When connected to the master node via SSH, you can curl the available metrics using <code>curl `http://target_ip:port/metrics`</code>.
+```
+# Example of keeping only specific metrics of the etcd target, the rest is dropped:
+
+kubeEtcd:
+  serviceMonitor:
+    enabled: true
+    metricRelabelings: 
+      - action: keep
+        regex: 'etcd_network_(client_grpc_received_bytes_total|client_grpc_sent_bytes_total)|etcd_server_has_leader'
+        sourceLabels: [__name__]
+
+# Example of dropping specific metrics of the kubeApiServer target and keeping the rest
+
+kubeApiServer:
+  serviceMonitor:
+    metricRelabelings: 
+      # Drop excessively noisy apiserver buckets.
+      - action: drop
+        regex: apiserver_request_duration_seconds_bucket;(0.15|0.2|0.3|0.35|0.4|0.45|0.6|0.7|0.8|0.9|1.25|1.5|1.75|2|3|3.5|4|4.5|6|7|8|9|15|25|40|50)
+        sourceLabels: 
+          - __name__
+          - le
+
+# Keep metrics only with custom label env=prod:
+      - action: keep
+        regex: '.*' # match all metrics
+        sourceLabels: [env] # Filters based on the env label
+        separator: ; # Use this if you want to keep the default labels too
+        replacement: prod
+```
+Source labels:
+- <code>\_\_name__</code>: Use this when you want to filter based on the metric name.
+- <code>le</code>: Use this when working with metrics that have defined buckets, such as request durations or sizes.
\ No newline at end of file