Merge branch 'loki' into 'master'

Loki

See merge request !25

ansible/01-playbook/play.yml

@@ -10,6 +10,7 @@
   roles:
     - modify
     - storageClass
-    - nginx_ingress
     - cert-manager
-    - monitoring
+    - monitoring
\ No newline at end of file
+    - loki
+    - nginx_ingress
\ No newline at end of file

ansible/01-playbook/roles/loki/files/configmap.yaml

+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: loki-dashboard
+  namespace: monitoring
+  labels:
+    grafana_dashboard: "1"
+data:
+  loki_dashboard.json: |
+    {
+      "annotations": {
+        "list": [
+          {
+            "builtIn": 1,
+            "datasource": {
+              "type": "datasource",
+              "uid": "grafana"
+            },
+            "enable": true,
+            "hide": true,
+            "iconColor": "rgba(0, 211, 255, 1)",
+            "name": "Annotations & Alerts",
+            "target": {
+              "limit": 100,
+              "matchAny": false,
+              "tags": [],
+              "type": "dashboard"
+            },
+            "type": "dashboard"
+          }
+        ]
+      },
+      "description": "Logs collected from Kubernetes, stored in Loki",
+      "editable": true,
+      "fiscalYearStartMonth": 0,
+      "gnetId": 15141,
+      "graphTooltip": 0,
+      "id": 33,
+      "links": [],
+      "panels": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "P8E80F9AEF21F6940"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "bars",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              }
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 4,
+            "w": 24,
+            "x": 0,
+            "y": 0
+          },
+          "id": 4,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": false
+            },
+            "tooltip": {
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "loki",
+                "uid": "P8E80F9AEF21F6940"
+              },
+              "expr": "sum(count_over_time({namespace=~\"$namespace\", stream=~\"$stream\", container =~\"$container\"} |= \"$query\" [$__interval]))",
+              "instant": false,
+              "legendFormat": "Log count",
+              "range": true,
+              "refId": "A"
+            }
+          ],
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "P8E80F9AEF21F6940"
+          },
+          "description": "Logs from services running in Kubernetes",
+          "gridPos": {
+            "h": 25,
+            "w": 24,
+            "x": 0,
+            "y": 4
+          },
+          "id": 2,
+          "options": {
+            "dedupStrategy": "none",
+            "enableLogDetails": true,
+            "prettifyLogMessage": false,
+            "showCommonLabels": false,
+            "showLabels": false,
+            "showTime": false,
+            "sortOrder": "Descending",
+            "wrapLogMessage": false
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "loki",
+                "uid": "P8E80F9AEF21F6940"
+              },
+              "expr": "{namespace=~\"$namespace\", stream=~\"$stream\", container =~\"$container\"} |= \"$query\"",
+              "refId": "A"
+            }
+          ],
+          "type": "logs"
+        }
+      ],
+      "refresh": false,
+      "schemaVersion": 39,
+      "tags": ["loki", "logging"],
+      "templating": {
+        "list": [
+          {
+            "current": {
+              "selected": false,
+              "text": "",
+              "value": ""
+            },
+            "description": "String to search for",
+            "hide": 0,
+            "label": "Search Query",
+            "name": "query",
+            "options": [
+              {
+                "selected": true,
+                "text": "",
+                "value": ""
+              }
+            ],
+            "query": "",
+            "skipUrlSync": false,
+            "type": "textbox"
+          },
+          {
+            "allValue": ".+",
+            "current": {
+              "selected": false,
+              "text": "All",
+              "value": "$__all"
+            },
+            "datasource": {
+              "type": "loki",
+              "uid": "P8E80F9AEF21F6940"
+            },
+            "definition": "label_values(namespace)",
+            "hide": 0,
+            "includeAll": true,
+            "multi": true,
+            "name": "namespace",
+            "options": [],
+            "query": "label_values(namespace)",
+            "refresh": 1,
+            "regex": "",
+            "skipUrlSync": false,
+            "sort": 0,
+            "type": "query"
+          },
+          {
+            "allValue": ".+",
+            "current": {
+              "selected": false,
+              "text": "All",
+              "value": "$__all"
+            },
+            "datasource": {
+              "type": "loki",
+              "uid": "P8E80F9AEF21F6940"
+            },
+            "definition": "label_values(stream)",
+            "hide": 0,
+            "includeAll": true,
+            "multi": true,
+            "name": "stream",
+            "options": [],
+            "query": "label_values(stream)",
+            "refresh": 1,
+            "regex": "",
+            "skipUrlSync": false,
+            "sort": 0,
+            "type": "query"
+          },
+          {
+            "allValue": ".+",
+            "current": {
+              "selected": false,
+              "text": "All",
+              "value": "$__all"
+            },
+            "datasource": {
+              "type": "loki",
+              "uid": "P8E80F9AEF21F6940"
+            },
+            "definition": "label_values(container)",
+            "hide": 0,
+            "includeAll": true,
+            "multi": true,
+            "name": "container",
+            "options": [],
+            "query": "label_values(container)",
+            "refresh": 1,
+            "regex": "",
+            "skipUrlSync": false,
+            "sort": 0,
+            "type": "query"
+          }
+        ]
+      },
+      "time": {
+        "from": "now-6h",
+        "to": "now"
+      },
+      "timepicker": {},
+      "timezone": "",
+      "title": "Loki Kubernetes Logs",
+      "tags": ["loki", "logging"],
+      "uid": "o6-BGgnnk",
+      "version": 1,
+      "weekStart": ""
+    }
\ No newline at end of file

ansible/01-playbook/roles/loki/files/servicemonitor.yaml

+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: loki
+  namespace: monitoring
+  labels:
+    release: prometheus-community
+spec:
+  selector:
+    matchLabels:
+      app: loki
+      release: loki-stack
+  endpoints:
+    - port: http-metrics
+      path: /metrics
+      interval: 30s
\ No newline at end of file

ansible/01-playbook/roles/loki/files/values.yaml

+test_pod:
+  enabled: false
+
+loki:
+  enabled: true
+  isDefault: false
+  auth_enabled: false
+  url: http://{{ .Release.Name }}:3100
+  readinessProbe:
+    httpGet:
+      path: /ready
+      port: http-metrics
+    initialDelaySeconds: 90
+    periodSeconds: 10
+  livenessProbe:
+    httpGet:
+      path: /ready
+      port: http-metrics
+    initialDelaySeconds: 90
+    periodSeconds: 10
+  config:
+    compactor:
+      shared_store: aws
+      working_directory: /data/tsdb-index
+    schema_config:
+      configs:
+        - from: "2024-12-03"
+          store: tsdb
+          object_store: s3
+          schema: v11
+          index:
+            prefix: loki_ops_index_
+            period: 24h
+    storage_config:
+      tsdb_shipper:
+        active_index_directory: /data/tsdb-index
+        cache_location: /data/tsdb-cache
+      aws:
+        bucketnames: bucket_name
+        endpoint: https://object-store.cloud.muni.cz # custom_endpoint
+        access_key_id: access_key
+        secret_access_key: secret_access_key
+        s3forcepathstyle: true
+
+promtail:
+  enabled: true
+  config:
+    logLevel: info
+    serverPort: 3101
+    clients:
+      - url: http://{{ .Release.Name }}:3100/loki/api/v1/push
+
+fluent-bit:
+  enabled: false
+
+grafana:
+  enabled: false
+
+prometheus:
+  enabled: false
\ No newline at end of file

ansible/01-playbook/roles/loki/tasks/main.yaml

+- name: Add grafana helm repository
+  kubernetes.core.helm_repository:
+    name: grafana
+    repo_url: https://grafana.github.io/helm-charts
+  when: install_loki_task | bool
+  tags:
+  - loki
+
+- name: Deploy loki chart
+  kubernetes.core.helm:
+    name: loki-stack 
+    chart_ref: grafana/loki-stack
+    release_namespace: monitoring
+    create_namespace: true
+    kubeconfig: "{{ playbook_dir }}/../artifacts/admin.conf"
+    chart_version: "2.10.2" 
+    update_repo_cache: True
+    values_files:
+    - "{{ role_path }}/files/values.yaml"
+  when: install_loki_task | bool
+  tags:
+    - loki
+
+- name: Apply the service monitor manifest
+  kubernetes.core.k8s:
+    kubeconfig: "{{ playbook_dir }}/../artifacts/admin.conf"
+    state: present
+    src: "{{ role_path }}/files/servicemonitor.yaml"
+  when:  install_loki_task | bool
+  tags:
+    - loki
+
+- name: Apply the configmap manifest
+  kubernetes.core.k8s:
+    kubeconfig: "{{ playbook_dir }}/../artifacts/admin.conf"
+    state: present
+    src: "{{ role_path }}/files/configmap.yaml"
+  when:  install_loki_task | bool
+  tags:
+    - loki
\ No newline at end of file

ansible/01-playbook/roles/monitoring/files/values.yaml

@@ -36,6 +36,18 @@ grafana:
         api_url: 'https://login.e-infra.cz/oidc/userinfo'
         use_pkce: true
         # role_attribute_path: to_string('Viewer') # assign role (Admin, Editor, Viewer) to all logged in users 
+  sidecar:
+    dashboards:
+      enabled: true
+      label: grafana_dashboard
+      labelValue: "1"
+      searchNamespace: ALL
+  additionalDataSources:
+    - name: Loki
+      type: loki
+      access: proxy
+      url: http://loki-stack.monitoring.svc.cluster.local:3100
+      isDefault: false
 
 prometheus:
   prometheusSpec:
@@ -56,8 +68,19 @@ prometheus:
       limits:
         memory: '2Gi'
         cpu: '1000m'
+    additionalServiceMonitors:
+      - name: loki
+        namespace: monitoring
+        selector:
+          matchLabels:
+            app: loki
+        endpoints:
+          - port: http-metrics
+            path: /metrics
+            interval: 30s
 
 alertmanager:
+  enabled: true
   alertmanagerSpec:
     replicas: 3
   config:
@@ -104,7 +127,7 @@ alertmanager:
     - name: 'null'
     - name: 'slack-notifications'
       slack_configs:
-      - api_url: '<api-url>' # how to get webhook https://docs.kubermatic.com/kubermatic/v2.25/tutorials-howtos/monitoring-logging-alerting/user-cluster/setting-up-alertmanager-with-slack-notifications/#setting-up-slack-incoming-webhooks
+      - api_url: 'https://hooks.slack.com/services/webhook' # how to get webhook https://docs.kubermatic.com/kubermatic/v2.25/tutorials-howtos/monitoring-logging-alerting/user-cluster/setting-up-alertmanager-with-slack-notifications/#setting-up-slack-incoming-webhooks
         channel: '#alertmanager-testing'
         send_resolved: true
         text: '{{ .CommonAnnotations.description }} Runbook: {{ .CommonAnnotations.runbook_url }}'

ansible/docs/loki.md

+# Loki
+
+To deploy Loki-stack, set <code>install_loki_task: true</code>.
+
+## Storage
+
+In OpenStack UI, navigate to **Object Storage > Containers** and create a new container.
+
+Run the following commands to create EC2 credentials for accessing the S3 storage:
+```
+CREDENTIALS=$(openstack ec2 credentials create -f shell)
+access=$(echo "$CREDENTIALS" | grep -o 'access="[^"]*"' | cut -d'"' -f2)
+secret=$(echo "$CREDENTIALS" | grep -o 'secret="[^"]*"' | cut -d'"' -f2)
+echo "EC2 credentials created."
+```
+Retrieve the Access and Secret Keys:
+```
+echo "Access Key: $access"
+echo "Secret Key: $secret"
+```
+Add the credentials to the <code>loki/values.yaml<code>.
+
+Validate s3 storage:    
+```
+aws configure # Configure with generated access and secret keys
+aws s3 ls s3://loki-logs/ --endpoint-url=https://object-store.cloud.muni.cz
+aws s3 ls s3://loki-logs/index --endpoint-url=https://object-store.cloud.muni.cz --recursive
+aws s3 ls s3://loki-logs/fake --endpoint-url=https://object-store.cloud.muni.cz --recursive
+```
\ No newline at end of file

ansible/group_vars/all/all.yml

@@ -26,6 +26,7 @@ external_cloud_provider: openstack
 
 install_nginx_task: false # tag: ingress
 install_monitoring_task: false # tag: monitoring
+install_loki_task: false # tag: loki
 install_cert_manager_task: false # tag: certmanager
 kubeconfig_localhost: true
 install_storageclass: true # tag: storageclass
\ No newline at end of file