PR from branch users/artem-trof/ydbops-6676-e2e-tools

Утилита для запуска workload и ее развертывания для ydb-oss кластера (на baremetal и в k8s)

5 files changed, 204 insertions, 95 deletions

diff --git a/ydb/deploy/helm/ydb-prometheus/README.md b/ydb/deploy/helm/ydb-prometheus/README.md
index 25eccdf80f..a84d5fa544 100644
--- a/ydb/deploy/helm/ydb-prometheus/README.md
+++ b/ydb/deploy/helm/ydb-prometheus/README.md
@@ -32,18 +32,48 @@ kube-prometheus-stack:
 
 ydb:
   clusters:
-  - cluster: <cluster-name>
+  - name: <cluster-name>
     type: external
-    ports:
-      static: <static nodes port>
-      dynamic:
-      - <dynamic nodes ports, one per database (tenant)>
-    hosts:
-    - <ydb host>
+    storage:
+      name: <prometheus job name, will be formatted as ydb/<name>/counter/<counter name>>
+      hosts:
+      - <ydb storage hosts>
+      port:
+        number: <ydb storage monitoring port number>
+    databases:
+    - name: <prometheus job name, will be formatted as ydb/<name>/counter/<counter name>>
+      hosts:
+      - <ydb dynamic hosts>
+      port:
+        number: <ydb dynamic monitoring port number>
+
 ```
 
 4. Install chart with `helm`
 
 ## Monitoring Internal YDB Cluster (deployed with ydb-operator)
 
-Work in progress
+1. Set following in values.yaml to cluster monitor:
+
+```yaml
+
+ydb:
+  clusters:
+  - name: <cluster-name>
+    type: internal
+    storage:
+      name: <prometheus job name, will be formatted as serviceMonitor/<release namespace>/<release-name>/<counter number>
+      namespace: <ydb cluster namespace>
+      selector: <kubernetes label selector for ydb storage pods>
+      port:
+        name: <ydb storage service (status) port number>
+    databases:
+    - name: <prometheus job name, will be formatted as serviceMonitor/<release namespace>/<release-name>/<counter number>
+      namespace: <ydb cluster namespace>
+      selector: <kubernetes label selector for ydb database pods>
+      port:
+        name: <ydb database service (status) port number>
+
+```
+
+2. Install chart with `helm`
diff --git a/ydb/deploy/helm/ydb-prometheus/templates/_helpers.tpl b/ydb/deploy/helm/ydb-prometheus/templates/_helpers.tpl
index e2b4a0424c..da7a4d3f42 100644
--- a/ydb/deploy/helm/ydb-prometheus/templates/_helpers.tpl
+++ b/ydb/deploy/helm/ydb-prometheus/templates/_helpers.tpl
@@ -48,72 +48,89 @@ app.kubernetes.io/managed-by: {{ .Release.Service }}
 {{ $context.Values.grafanaDashboards.markerLabel.key }}: {{ $context.Values.grafanaDashboards.markerLabel.value | quote }}
 {{- end -}}
 
-{{- define "ydb-prometheus.externalCluster.config" -}}
+{{- define "ydb-prometheus.externalCluster.jobs" }}
 {{- $context := . -}}
-{{- $counters := .Values.ydb.counters -}}
-{{- $clusters := .Values.ydb.clusters -}}
-{{- $result := list -}}
-{{- range $cluster := $clusters }}
-  {{- if eq $cluster.type "external" }}
-
-    {{- range $counter := $counters }}
-      {{- $name := "staticNode" }}
-      {{- $port := $cluster.ports.static }}
-      {{- $type := default "all" $counter.type }}
-      {{- if or (eq $type "static") (eq $type "all") }}
-        {{- $config := include "ydb-prometheus.externalCluster.targetCounter" (tuple $cluster $counter $name $port) | fromYaml }}
-        {{- $result = append $result $config }}
-      {{- end }}
+{{- $spec := index $context 0 -}}
+{{- $counters := index $context 1 -}}
+{{- $specType := index $context 2 -}}
+  {{- range $counter := $counters }}
+    {{- $type := include "ydb-prometheus.counter.type" $counter }}
+    {{- if or (eq $type "all") (eq $type $specType) }}
+- job_name: {{ include "ydb-prometheus.counter.jobName" (tuple $spec.name $counter) }}
+  metrics_path: {{ include "ydb-prometheus.counter.metricsPath" $counter | quote }}
+  relabel_configs:
+  - source_labels:
+    - __address__
+    target_label: instance
+    regex: '([^:]+)(:[0-9]+)?'
+    replacement: '${1}'
+  metric_relabel_configs:
+  - source_labels:
+    - __name__
+    target_label: __name__
+    regex: (.*)
+    replacement: {{ $counter.counter }}_$1
+  static_configs:
+  - targets:
+    {{- range $host := $spec.hosts }}
+      {{- printf "- %s:%d" $host ($spec.port.number | int) | nindent 4 }}
     {{- end }}
+    labels:
+      counter: {{ $counter.counter | quote }}
+      container: ydb-dynamic
+      {{- end }}
+  {{- end }}
+{{- end }}
 
-    {{- range $port := $cluster.ports.dynamic }}
-      {{- range $counter := $counters }}
-        {{- $name := printf "dynamicNode/%d" ($port | int) }}
-        {{- $type := default "all" $counter.type }}
-        {{- if or (eq $type "dynamic") (eq $type "all") }}
-          {{- $config := include "ydb-prometheus.externalCluster.targetCounter" (tuple $cluster $counter $name $port) | fromYaml }}
-          {{- $result = append $result $config }}
-        {{- end }}
+{{- define "ydb-prometheus.internalCluster.serviceMonitor" -}}
+{{- $context := . -}}
+{{- $spec := index $context 0 -}}
+{{- $counters := index $context 1 -}}
+{{- $specType := index $context 2 -}}
+spec:
+  endpoints:
+    {{- range $counter := $counters }}
+      {{- $type := include "ydb-prometheus.counter.type" $counter }}
+      {{- if or (eq $type "all") (eq $type $specType) }}
+  - path: {{ include "ydb-prometheus.counter.metricsPath" $counter }}
+    port: {{ $spec.port.name }}
+    metricRelabelings:
+    - sourceLabels:
+      - __name__
+      targetLabel: __name__
+      regex: (.*)
+      replacement: {{ $counter.counter }}_$1
+    relabelings:
+    - sourceLabels:
+      - __meta_kubernetes_namespace
+      targetLabel: job
+      regex: (.*)
+      replacement: {{ include "ydb-prometheus.counter.jobName" (tuple (printf "$1/%s" $spec.name) $counter) }}
       {{- end }}
     {{- end }}
+  namespaceSelector:
+    matchNames:
+    - {{ $spec.namespace }}
+  selector:
+    matchLabels: {{ $spec.selector | toYaml | nindent 6 }}
+{{- end -}}
 
-{{- end }}
-{{- end }}
+{{- define "ydb-prometheus.counter.jobName" -}}
+{{- $context := . -}}
+{{- $name := index $context 0 -}}
+{{- $counter := index $context 1 -}}
+{{- $name := printf "ydb/%s/counter/%s" $name $counter.counter | quote -}}
+{{- $name -}}
+{{- end -}}
 
-{{- $result | toYaml }}
+{{- define "ydb-prometheus.counter.type" -}}
+{{- $counter := . -}}
+{{- $type := default "all" $counter.type -}}
+{{- $type -}}
 {{- end -}}
 
-{{- define "ydb-prometheus.externalCluster.targetCounter" }}
-{{- $context := . }}
-{{- $cluster := index $context 0 }}
-{{- $counter := index $context 1 }}
-{{- $name := index $context 2 }}
-{{- $port := index $context 3 }}
-{{- $metricsPath := (printf "/counters/counters=%s/prometheus" $counter.counter) }}
-{{- if $counter.metricsPath }}
-{{- $metricsPath = $counter.metricsPath }}
-{{- end }}
-job_name: {{ printf "ydb/%s/%s/counter/%s" $cluster.cluster $name $counter.counter | quote }}
-metrics_path: {{ $metricsPath | quote }}
-relabel_configs:
-- source_labels:
-  - __address__
-  target_label: instance
-  regex: '([^:]+)(:[0-9]+)?'
-  replacement: '${1}'
-metric_relabel_configs:
-- source_labels:
-  - __name__
-  target_label: __name__
-  regex: (.*)
-  replacement: {{ $counter.counter }}_$1
-static_configs:
-- targets:
-  {{- range $host := $cluster.hosts }}
-    {{- printf "- %s:%d" $host ($port | int) | nindent 4 }}
-  {{- end }}
-  labels:
-    project: {{ $cluster.cluster }}
-    counter: {{ $counter.counter | quote }}
-    container: ydb-dynamic
-{{- end }}
\ No newline at end of file
+{{- define "ydb-prometheus.counter.metricsPath" -}}
+{{- $counter := . -}}
+{{- $metricsPath := default (printf "/counters/counters=%s/prometheus" $counter.counter) $counter.metricsPath -}}
+{{- $metricsPath -}}
+{{- end -}}
\ No newline at end of file
diff --git a/ydb/deploy/helm/ydb-prometheus/templates/secret-prometheus-static-targets.yaml b/ydb/deploy/helm/ydb-prometheus/templates/secret-prometheus-static-targets.yaml
index 9f1081d18c..e83d00219f 100644
--- a/ydb/deploy/helm/ydb-prometheus/templates/secret-prometheus-static-targets.yaml
+++ b/ydb/deploy/helm/ydb-prometheus/templates/secret-prometheus-static-targets.yaml
@@ -1,10 +1,20 @@
-{{- if .Values.prometheusConfig.enabled -}}
 {{- $namespace := .Release.Namespace -}}
-{{- $name := .Values.prometheusConfig.additionalScrapeConfigs.name -}}
-  {{- if not $name -}}
-  {{- $name = printf "%s-additional-scrape-configs" (include "ydb-prometheus.fullname" $) -}}
-  {{- end -}}
-  {{- $data := include "ydb-prometheus.externalCluster.config" $ -}}
+{{- if .Values.prometheusConfig.enabled -}}
+  {{- $name := default (printf "%s-additional-scrape-configs" (include "ydb-prometheus.fullname" .)) .Values.prometheusConfig.additionalScrapeConfigs.name -}}
+  {{- $data := list -}}
+  {{- $counters := .Values.ydb.counters -}}
+
+  {{- range $cluster := .Values.ydb.clusters }}
+    {{- if eq $cluster.type "external" }}
+      {{- $jobs := include "ydb-prometheus.externalCluster.jobs" (tuple $cluster.storage $counters "static") | fromYamlArray -}}
+      {{- $data = concat $data $jobs -}}
+      {{- range $database := $cluster.databases -}}
+        {{- $jobs := include "ydb-prometheus.externalCluster.jobs" (tuple $database $counters "dynamic") | fromYamlArray -}}
+        {{- $data = concat $data $jobs -}}
+      {{- end -}}
+    {{- end }}
+  {{- end }}
+
   {{- if $data -}}
 apiVersion: v1
 kind: Secret
@@ -14,6 +24,6 @@ metadata:
   labels:
 {{- include "ydb-prometheus.labels" $ | nindent 4 }}
 data:
-  {{ .Values.prometheusConfig.additionalScrapeConfigs.key }}: {{ $data | b64enc | quote }}
+  {{ .Values.prometheusConfig.additionalScrapeConfigs.key }}: {{ $data | toYaml | b64enc | quote }}
   {{- end -}}
 {{- end -}}
\ No newline at end of file
diff --git a/ydb/deploy/helm/ydb-prometheus/templates/service-monitor-cluster.yaml b/ydb/deploy/helm/ydb-prometheus/templates/service-monitor-cluster.yaml
index dc805a4865..76526f068e 100644
--- a/ydb/deploy/helm/ydb-prometheus/templates/service-monitor-cluster.yaml
+++ b/ydb/deploy/helm/ydb-prometheus/templates/service-monitor-cluster.yaml
@@ -1,2 +1,26 @@
+{{- $namespace := .Release.Namespace -}}
 {{- if .Values.prometheusConfig.enabled -}}
+{{- $counters := .Values.ydb.counters -}}
+  {{- range $cluster := .Values.ydb.clusters -}}
+    {{- if eq $cluster.type "internal" -}}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ printf "%s-%s" (include "ydb-prometheus.fullname" $) $cluster.storage.name }}
+  namespace: {{ $namespace }}
+  labels: {{- include "ydb-prometheus.labels" $ | nindent 4 }}
+{{ include "ydb-prometheus.internalCluster.serviceMonitor" (tuple $cluster.storage $counters "static") }}
+---
+      {{- range $database := $cluster.databases -}}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ printf "%s-%s" (include "ydb-prometheus.fullname" $) $database.name }}
+  namespace: {{ $namespace }}
+  labels: {{- include "ydb-prometheus.labels" $ | nindent 4 }}
+{{ include "ydb-prometheus.internalCluster.serviceMonitor" (tuple $database $counters "dynamic") }}
+---
+      {{- end -}}
+    {{- end -}}
+  {{- end -}}
 {{- end -}}
\ No newline at end of file
diff --git a/ydb/deploy/helm/ydb-prometheus/values.yaml b/ydb/deploy/helm/ydb-prometheus/values.yaml
index f7ee1d284a..2ce913ce4d 100644
--- a/ydb/deploy/helm/ydb-prometheus/values.yaml
+++ b/ydb/deploy/helm/ydb-prometheus/values.yaml
@@ -55,43 +55,71 @@ ydb:
     # If unspecified, will be used /counters/counters=%s/prometheus
     # metricsPath: <url to prometheus metrics>
     #
+    - counter: ydb
+      metricsPath: /counters/counters=ydb/name_label=name/prometheus
+    # - counter: ydb_serverless
     - counter: auth
-    - counter: compile
+    - counter: coordinator
+    - counter: dsproxy_queue
     - counter: dsproxy
     - counter: dsproxy_percentile
-    - counter: dsproxy_queue
-    - counter: dsproxynode
     - counter: grpc
-    - counter: interconnect
-    - counter: kqp
-    - counter: netclassifier
     - counter: pdisks
       type: static
-    - counter: pqproxy
+    - counter: kqp
+      type: dynamic
+    - counter: processing
     - counter: proxy
-    - counter: quoter_service
+    - counter: followers
+    - counter: storage_pool_stat
+    # - counter: streaming
     - counter: tablets
+    - counter: utils
+    # - counter: yql
+    - counter: dsproxynode
+    - counter: interconnect
     - counter: vdisks
       type: static
-    - counter: utils
-    - counter: ydb
-      metricsPath: /counters/counters=ydb/name_label=name/prometheus
 
   # Cluster which will be monitored
   clusters: []
   # Cluster name
-  # cluster: <cluster name>
+  # name: <cluster name>
   #
   # Cluster type
   # For external clusters, additional scrape config secret will be generated
   # For internal clusters, service monitor with appropriate configuration will be generated
   # type: <external|internal>
   #
-  # External cluster targets, used only if type == external
-  # external:
-  # Monitoring ports for static and dynamic nodes
-  #   ports:
-  #     static: <static (storage) node port>
-  #     dynamic: <dynamic (tenants) node ports>
-  # Monitoring hosts
-  #   hosts: <cluster targets>
+  # Storage nodes monitoring configuration
+  # storage:
+  #   name: <name of prometheus job/target>
+  #
+  #   Internal cluster specific spec
+  #   namespace: <namespace of ydb storage pods>
+  #   selector: <label selector for ydb storage pods>
+  #   port:
+  #     name: <service port name for ydb storage pods>
+  #
+  #   External cluster specific spec
+  #   hosts:
+  #   - <ydb storage host>
+  #   port:
+  #     number: <ydb storage monitoring (status) port number>
+  #
+  # Dynamic nodes monitoring configuration
+  # Each item represents ydb database
+  # databases:
+  # - name: <name of prometheus job/target>
+  #
+  #   Internal cluster specific spec
+  #   namespace: <namespace of ydb database pods>
+  #   selector: <label selector for ydb database pods>
+  #   port:
+  #     name: <service port name for ydb database pods>
+  #
+  #   External cluster specific spec
+  #   hosts:
+  #   - <ydb database host>
+  #   port:
+  #     number: <ydb database monitoring (status) port number>