diff options
author | artem-trof <artem-trof@yandex-team.com> | 2023-02-17 14:23:27 +0300 |
---|---|---|
committer | artem-trof <artem-trof@yandex-team.com> | 2023-02-17 14:23:27 +0300 |
commit | 49a708acc9002a668fa866efccb3d752cd60ccbd (patch) | |
tree | 8c18b5f99bd015ad3dcb38a614d585ea8a7ff005 | |
parent | ed241323ffc1d09cf1999a40708af44b1f48326b (diff) | |
download | ydb-49a708acc9002a668fa866efccb3d752cd60ccbd.tar.gz |
PR from branch users/artem-trof/ydbops-6676-e2e-tools
Утилита для запуска workload и ее развертывания для ydb-oss кластера (на baremetal и в k8s)
5 files changed, 204 insertions, 95 deletions
diff --git a/ydb/deploy/helm/ydb-prometheus/README.md b/ydb/deploy/helm/ydb-prometheus/README.md index 25eccdf80f..a84d5fa544 100644 --- a/ydb/deploy/helm/ydb-prometheus/README.md +++ b/ydb/deploy/helm/ydb-prometheus/README.md @@ -32,18 +32,48 @@ kube-prometheus-stack: ydb: clusters: - - cluster: <cluster-name> + - name: <cluster-name> type: external - ports: - static: <static nodes port> - dynamic: - - <dynamic nodes ports, one per database (tenant)> - hosts: - - <ydb host> + storage: + name: <prometheus job name, will be formatted as ydb/<name>/counter/<counter name>> + hosts: + - <ydb storage hosts> + port: + number: <ydb storage monitoring port number> + databases: + - name: <prometheus job name, will be formatted as ydb/<name>/counter/<counter name>> + hosts: + - <ydb dynamic hosts> + port: + number: <ydb dynamic monitoring port number> + ``` 4. Install chart with `helm` ## Monitoring Internal YDB Cluster (deployed with ydb-operator) -Work in progress +1. Set following in values.yaml to cluster monitor: + +```yaml + +ydb: + clusters: + - name: <cluster-name> + type: internal + storage: + name: <prometheus job name, will be formatted as serviceMonitor/<release namespace>/<release-name>/<counter number> + namespace: <ydb cluster namespace> + selector: <kubernetes label selector for ydb storage pods> + port: + name: <ydb storage service (status) port number> + databases: + - name: <prometheus job name, will be formatted as serviceMonitor/<release namespace>/<release-name>/<counter number> + namespace: <ydb cluster namespace> + selector: <kubernetes label selector for ydb database pods> + port: + name: <ydb database service (status) port number> + +``` + +2. Install chart with `helm` diff --git a/ydb/deploy/helm/ydb-prometheus/templates/_helpers.tpl b/ydb/deploy/helm/ydb-prometheus/templates/_helpers.tpl index e2b4a0424c..da7a4d3f42 100644 --- a/ydb/deploy/helm/ydb-prometheus/templates/_helpers.tpl +++ b/ydb/deploy/helm/ydb-prometheus/templates/_helpers.tpl @@ -48,72 +48,89 @@ app.kubernetes.io/managed-by: {{ .Release.Service }} {{ $context.Values.grafanaDashboards.markerLabel.key }}: {{ $context.Values.grafanaDashboards.markerLabel.value | quote }} {{- end -}} -{{- define "ydb-prometheus.externalCluster.config" -}} +{{- define "ydb-prometheus.externalCluster.jobs" }} {{- $context := . -}} -{{- $counters := .Values.ydb.counters -}} -{{- $clusters := .Values.ydb.clusters -}} -{{- $result := list -}} -{{- range $cluster := $clusters }} - {{- if eq $cluster.type "external" }} - - {{- range $counter := $counters }} - {{- $name := "staticNode" }} - {{- $port := $cluster.ports.static }} - {{- $type := default "all" $counter.type }} - {{- if or (eq $type "static") (eq $type "all") }} - {{- $config := include "ydb-prometheus.externalCluster.targetCounter" (tuple $cluster $counter $name $port) | fromYaml }} - {{- $result = append $result $config }} - {{- end }} +{{- $spec := index $context 0 -}} +{{- $counters := index $context 1 -}} +{{- $specType := index $context 2 -}} + {{- range $counter := $counters }} + {{- $type := include "ydb-prometheus.counter.type" $counter }} + {{- if or (eq $type "all") (eq $type $specType) }} +- job_name: {{ include "ydb-prometheus.counter.jobName" (tuple $spec.name $counter) }} + metrics_path: {{ include "ydb-prometheus.counter.metricsPath" $counter | quote }} + relabel_configs: + - source_labels: + - __address__ + target_label: instance + regex: '([^:]+)(:[0-9]+)?' + replacement: '${1}' + metric_relabel_configs: + - source_labels: + - __name__ + target_label: __name__ + regex: (.*) + replacement: {{ $counter.counter }}_$1 + static_configs: + - targets: + {{- range $host := $spec.hosts }} + {{- printf "- %s:%d" $host ($spec.port.number | int) | nindent 4 }} {{- end }} + labels: + counter: {{ $counter.counter | quote }} + container: ydb-dynamic + {{- end }} + {{- end }} +{{- end }} - {{- range $port := $cluster.ports.dynamic }} - {{- range $counter := $counters }} - {{- $name := printf "dynamicNode/%d" ($port | int) }} - {{- $type := default "all" $counter.type }} - {{- if or (eq $type "dynamic") (eq $type "all") }} - {{- $config := include "ydb-prometheus.externalCluster.targetCounter" (tuple $cluster $counter $name $port) | fromYaml }} - {{- $result = append $result $config }} - {{- end }} +{{- define "ydb-prometheus.internalCluster.serviceMonitor" -}} +{{- $context := . -}} +{{- $spec := index $context 0 -}} +{{- $counters := index $context 1 -}} +{{- $specType := index $context 2 -}} +spec: + endpoints: + {{- range $counter := $counters }} + {{- $type := include "ydb-prometheus.counter.type" $counter }} + {{- if or (eq $type "all") (eq $type $specType) }} + - path: {{ include "ydb-prometheus.counter.metricsPath" $counter }} + port: {{ $spec.port.name }} + metricRelabelings: + - sourceLabels: + - __name__ + targetLabel: __name__ + regex: (.*) + replacement: {{ $counter.counter }}_$1 + relabelings: + - sourceLabels: + - __meta_kubernetes_namespace + targetLabel: job + regex: (.*) + replacement: {{ include "ydb-prometheus.counter.jobName" (tuple (printf "$1/%s" $spec.name) $counter) }} {{- end }} {{- end }} + namespaceSelector: + matchNames: + - {{ $spec.namespace }} + selector: + matchLabels: {{ $spec.selector | toYaml | nindent 6 }} +{{- end -}} -{{- end }} -{{- end }} +{{- define "ydb-prometheus.counter.jobName" -}} +{{- $context := . -}} +{{- $name := index $context 0 -}} +{{- $counter := index $context 1 -}} +{{- $name := printf "ydb/%s/counter/%s" $name $counter.counter | quote -}} +{{- $name -}} +{{- end -}} -{{- $result | toYaml }} +{{- define "ydb-prometheus.counter.type" -}} +{{- $counter := . -}} +{{- $type := default "all" $counter.type -}} +{{- $type -}} {{- end -}} -{{- define "ydb-prometheus.externalCluster.targetCounter" }} -{{- $context := . }} -{{- $cluster := index $context 0 }} -{{- $counter := index $context 1 }} -{{- $name := index $context 2 }} -{{- $port := index $context 3 }} -{{- $metricsPath := (printf "/counters/counters=%s/prometheus" $counter.counter) }} -{{- if $counter.metricsPath }} -{{- $metricsPath = $counter.metricsPath }} -{{- end }} -job_name: {{ printf "ydb/%s/%s/counter/%s" $cluster.cluster $name $counter.counter | quote }} -metrics_path: {{ $metricsPath | quote }} -relabel_configs: -- source_labels: - - __address__ - target_label: instance - regex: '([^:]+)(:[0-9]+)?' - replacement: '${1}' -metric_relabel_configs: -- source_labels: - - __name__ - target_label: __name__ - regex: (.*) - replacement: {{ $counter.counter }}_$1 -static_configs: -- targets: - {{- range $host := $cluster.hosts }} - {{- printf "- %s:%d" $host ($port | int) | nindent 4 }} - {{- end }} - labels: - project: {{ $cluster.cluster }} - counter: {{ $counter.counter | quote }} - container: ydb-dynamic -{{- end }}
\ No newline at end of file +{{- define "ydb-prometheus.counter.metricsPath" -}} +{{- $counter := . -}} +{{- $metricsPath := default (printf "/counters/counters=%s/prometheus" $counter.counter) $counter.metricsPath -}} +{{- $metricsPath -}} +{{- end -}}
\ No newline at end of file diff --git a/ydb/deploy/helm/ydb-prometheus/templates/secret-prometheus-static-targets.yaml b/ydb/deploy/helm/ydb-prometheus/templates/secret-prometheus-static-targets.yaml index 9f1081d18c..e83d00219f 100644 --- a/ydb/deploy/helm/ydb-prometheus/templates/secret-prometheus-static-targets.yaml +++ b/ydb/deploy/helm/ydb-prometheus/templates/secret-prometheus-static-targets.yaml @@ -1,10 +1,20 @@ -{{- if .Values.prometheusConfig.enabled -}} {{- $namespace := .Release.Namespace -}} -{{- $name := .Values.prometheusConfig.additionalScrapeConfigs.name -}} - {{- if not $name -}} - {{- $name = printf "%s-additional-scrape-configs" (include "ydb-prometheus.fullname" $) -}} - {{- end -}} - {{- $data := include "ydb-prometheus.externalCluster.config" $ -}} +{{- if .Values.prometheusConfig.enabled -}} + {{- $name := default (printf "%s-additional-scrape-configs" (include "ydb-prometheus.fullname" .)) .Values.prometheusConfig.additionalScrapeConfigs.name -}} + {{- $data := list -}} + {{- $counters := .Values.ydb.counters -}} + + {{- range $cluster := .Values.ydb.clusters }} + {{- if eq $cluster.type "external" }} + {{- $jobs := include "ydb-prometheus.externalCluster.jobs" (tuple $cluster.storage $counters "static") | fromYamlArray -}} + {{- $data = concat $data $jobs -}} + {{- range $database := $cluster.databases -}} + {{- $jobs := include "ydb-prometheus.externalCluster.jobs" (tuple $database $counters "dynamic") | fromYamlArray -}} + {{- $data = concat $data $jobs -}} + {{- end -}} + {{- end }} + {{- end }} + {{- if $data -}} apiVersion: v1 kind: Secret @@ -14,6 +24,6 @@ metadata: labels: {{- include "ydb-prometheus.labels" $ | nindent 4 }} data: - {{ .Values.prometheusConfig.additionalScrapeConfigs.key }}: {{ $data | b64enc | quote }} + {{ .Values.prometheusConfig.additionalScrapeConfigs.key }}: {{ $data | toYaml | b64enc | quote }} {{- end -}} {{- end -}}
\ No newline at end of file diff --git a/ydb/deploy/helm/ydb-prometheus/templates/service-monitor-cluster.yaml b/ydb/deploy/helm/ydb-prometheus/templates/service-monitor-cluster.yaml index dc805a4865..76526f068e 100644 --- a/ydb/deploy/helm/ydb-prometheus/templates/service-monitor-cluster.yaml +++ b/ydb/deploy/helm/ydb-prometheus/templates/service-monitor-cluster.yaml @@ -1,2 +1,26 @@ +{{- $namespace := .Release.Namespace -}} {{- if .Values.prometheusConfig.enabled -}} +{{- $counters := .Values.ydb.counters -}} + {{- range $cluster := .Values.ydb.clusters -}} + {{- if eq $cluster.type "internal" -}} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ printf "%s-%s" (include "ydb-prometheus.fullname" $) $cluster.storage.name }} + namespace: {{ $namespace }} + labels: {{- include "ydb-prometheus.labels" $ | nindent 4 }} +{{ include "ydb-prometheus.internalCluster.serviceMonitor" (tuple $cluster.storage $counters "static") }} +--- + {{- range $database := $cluster.databases -}} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ printf "%s-%s" (include "ydb-prometheus.fullname" $) $database.name }} + namespace: {{ $namespace }} + labels: {{- include "ydb-prometheus.labels" $ | nindent 4 }} +{{ include "ydb-prometheus.internalCluster.serviceMonitor" (tuple $database $counters "dynamic") }} +--- + {{- end -}} + {{- end -}} + {{- end -}} {{- end -}}
\ No newline at end of file diff --git a/ydb/deploy/helm/ydb-prometheus/values.yaml b/ydb/deploy/helm/ydb-prometheus/values.yaml index f7ee1d284a..2ce913ce4d 100644 --- a/ydb/deploy/helm/ydb-prometheus/values.yaml +++ b/ydb/deploy/helm/ydb-prometheus/values.yaml @@ -55,43 +55,71 @@ ydb: # If unspecified, will be used /counters/counters=%s/prometheus # metricsPath: <url to prometheus metrics> # + - counter: ydb + metricsPath: /counters/counters=ydb/name_label=name/prometheus + # - counter: ydb_serverless - counter: auth - - counter: compile + - counter: coordinator + - counter: dsproxy_queue - counter: dsproxy - counter: dsproxy_percentile - - counter: dsproxy_queue - - counter: dsproxynode - counter: grpc - - counter: interconnect - - counter: kqp - - counter: netclassifier - counter: pdisks type: static - - counter: pqproxy + - counter: kqp + type: dynamic + - counter: processing - counter: proxy - - counter: quoter_service + - counter: followers + - counter: storage_pool_stat + # - counter: streaming - counter: tablets + - counter: utils + # - counter: yql + - counter: dsproxynode + - counter: interconnect - counter: vdisks type: static - - counter: utils - - counter: ydb - metricsPath: /counters/counters=ydb/name_label=name/prometheus # Cluster which will be monitored clusters: [] # Cluster name - # cluster: <cluster name> + # name: <cluster name> # # Cluster type # For external clusters, additional scrape config secret will be generated # For internal clusters, service monitor with appropriate configuration will be generated # type: <external|internal> # - # External cluster targets, used only if type == external - # external: - # Monitoring ports for static and dynamic nodes - # ports: - # static: <static (storage) node port> - # dynamic: <dynamic (tenants) node ports> - # Monitoring hosts - # hosts: <cluster targets> + # Storage nodes monitoring configuration + # storage: + # name: <name of prometheus job/target> + # + # Internal cluster specific spec + # namespace: <namespace of ydb storage pods> + # selector: <label selector for ydb storage pods> + # port: + # name: <service port name for ydb storage pods> + # + # External cluster specific spec + # hosts: + # - <ydb storage host> + # port: + # number: <ydb storage monitoring (status) port number> + # + # Dynamic nodes monitoring configuration + # Each item represents ydb database + # databases: + # - name: <name of prometheus job/target> + # + # Internal cluster specific spec + # namespace: <namespace of ydb database pods> + # selector: <label selector for ydb database pods> + # port: + # name: <service port name for ydb database pods> + # + # External cluster specific spec + # hosts: + # - <ydb database host> + # port: + # number: <ydb database monitoring (status) port number> |