I have used the prometheus deployment from the link https://raw.githubusercontent.com/istio/istio/release-1.16/samples/addons/prometheus.yaml.
We want prometheus to monitor a single namespace which shall help us use prometheus and grafana in single namespace as we do not get to have users separation based on namespaces in a single deployment for the complete cluster. I did follow the suggestion in this Stack Overflow question but no complete configmap YAML was provided there.
However with a basic idea I have updated the configmap as below:
# Please edit the object below. Lines beginning with a '#' will be ignored,
# and an empty file will abort the edit. If an error occurs while saving this file will be
# reopened with the relevant failures.
#
apiVersion: v1
data:
alerting_rules.yml: |
{}
alerts: |
{}
prometheus.yml: |
global:
evaluation_interval: 1m
scrape_interval: 15s
scrape_timeout: 10s
rule_files:
- /etc/config/recording_rules.yml
- /etc/config/alerting_rules.yml
- /etc/config/rules
- /etc/config/alerts
scrape_configs:
- job_name: prometheus
static_configs:
- targets:
- localhost:9090
- bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
job_name: kubernetes-apiservers
kubernetes_sd_configs:
- role: endpoints
relabel_configs:
- action: keep
regex: default;kubernetes;https
source_labels:
- __meta_kubernetes_namespace
- __meta_kubernetes_service_name
- __meta_kubernetes_endpoint_port_name
scheme: https
tls_config:
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
insecure_skip_verify: true
- bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
job_name: kubernetes-nodes
kubernetes_sd_configs:
- role: node
relabel_configs:
- action: labelmap
regex: __meta_kubernetes_node_label_(.+)
- replacement: kubernetes.default.svc:443
target_label: __address__
- regex: (.+)
replacement: /api/v1/nodes/$1/proxy/metrics
source_labels:
- __meta_kubernetes_node_name
target_label: __metrics_path__
scheme: https
tls_config:
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
insecure_skip_verify: true
- bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
job_name: kubernetes-nodes-cadvisor
kubernetes_sd_configs:
- role: node
relabel_configs:
- action: labelmap
regex: __meta_kubernetes_node_label_(.+)
- replacement: kubernetes.default.svc:443
target_label: __address__
- regex: (.+)
replacement: /api/v1/nodes/$1/proxy/metrics/cadvisor
source_labels:
- __meta_kubernetes_node_name
target_label: __metrics_path__
scheme: https
tls_config:
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
insecure_skip_verify: true
- honor_labels: true
job_name: kubernetes-service-endpoints
kubernetes_sd_configs:
- role: endpoints
relabel_configs:
- action: keep
regex: true
source_labels:
- __meta_kubernetes_service_annotation_prometheus_io_scrape
- action: drop
regex: true
source_labels:
- __meta_kubernetes_service_annotation_prometheus_io_scrape_slow
- action: replace
regex: (https?)
source_labels:
- __meta_kubernetes_service_annotation_prometheus_io_scheme
target_label: __scheme__
- action: replace
regex: (.+)
source_labels:
- __meta_kubernetes_service_annotation_prometheus_io_path
target_label: __metrics_path__
- action: replace
regex: ([^:]+)(?::\d+)?;(\d+)
replacement: $1:$2
source_labels:
- __address__
- __meta_kubernetes_service_annotation_prometheus_io_port
target_label: __address__
- action: labelmap
regex: __meta_kubernetes_service_annotation_prometheus_io_param_(.+)
replacement: __param_$1
- action: labelmap
regex: __meta_kubernetes_service_label_(.+)
- action: replace
source_labels:
- __meta_kubernetes_namespace
target_label: namespace
- action: replace
source_labels:
- __meta_kubernetes_service_name
target_label: service
- action: replace
source_labels:
- __meta_kubernetes_pod_node_name
target_label: node
- honor_labels: true
job_name: kubernetes-service-endpoints-slow
kubernetes_sd_configs:
- role: endpoints
relabel_configs:
- action: keep
regex: true
source_labels:
- __meta_kubernetes_service_annotation_prometheus_io_scrape_slow
- action: replace
regex: (https?)
source_labels:
- __meta_kubernetes_service_annotation_prometheus_io_scheme
target_label: __scheme__
- action: replace
regex: (.+)
source_labels:
- __meta_kubernetes_service_annotation_prometheus_io_path
target_label: __metrics_path__
- action: replace
regex: ([^:]+)(?::\d+)?;(\d+)
replacement: $1:$2
source_labels:
- __address__
- __meta_kubernetes_service_annotation_prometheus_io_port
target_label: __address__
- action: labelmap
regex: __meta_kubernetes_service_annotation_prometheus_io_param_(.+)
replacement: __param_$1
- action: labelmap
regex: __meta_kubernetes_service_label_(.+)
- action: replace
source_labels:
- __meta_kubernetes_namespace
target_label: namespace
- action: replace
source_labels:
- __meta_kubernetes_service_name
target_label: service
- action: replace
source_labels:
- __meta_kubernetes_pod_node_name
target_label: node
scrape_interval: 5m
scrape_timeout: 30s
- honor_labels: true
job_name: prometheus-pushgateway
kubernetes_sd_configs:
- role: service
relabel_configs:
- action: keep
regex: pushgateway
source_labels:
- __meta_kubernetes_service_annotation_prometheus_io_probe
- honor_labels: true
job_name: kubernetes-services
kubernetes_sd_configs:
- role: service
metrics_path: /probe
params:
module:
- http_2xx
relabel_configs:
- action: keep
regex: true
source_labels:
- __meta_kubernetes_service_annotation_prometheus_io_probe
- source_labels:
- __address__
target_label: __param_target
- replacement: blackbox
target_label: __address__
- source_labels:
- __param_target
target_label: instance
- action: labelmap
regex: __meta_kubernetes_service_label_(.+)
- source_labels:
- __meta_kubernetes_namespace
target_label: namespace
- source_labels:
- __meta_kubernetes_service_name
target_label: service
- honor_labels: true
job_name: kubernetes-pods
kubernetes_sd_configs:
- role: pod
namespaces:
names:
- canary
relabel_configs:
- action: keep
regex: true
source_labels:
- __meta_kubernetes_pod_annotation_prometheus_io_scrape
- action: drop
regex: true
source_labels:
- __meta_kubernetes_pod_annotation_prometheus_io_scrape_slow
- action: replace
regex: (https?)
source_labels:
- __meta_kubernetes_pod_annotation_prometheus_io_scheme
target_label: __scheme__
- action: replace
regex: (.+)
source_labels:
- __meta_kubernetes_pod_annotation_prometheus_io_path
target_label: __metrics_path__
- action: replace
regex: ([^:]+)(?::\d+)?;(\d+)
replacement: $1:$2
source_labels:
- __address__
- __meta_kubernetes_pod_annotation_prometheus_io_port
target_label: __address__
- action: labelmap
regex: __meta_kubernetes_pod_annotation_prometheus_io_param_(.+)
replacement: __param_$1
- action: labelmap
regex: __meta_kubernetes_pod_label_(.+)
- action: replace
source_labels:
- __meta_kubernetes_namespace
target_label: namespace
- action: replace
source_labels:
- __meta_kubernetes_pod_name
target_label: pod
- action: drop
regex: Pending|Succeeded|Failed|Completed
source_labels:
- __meta_kubernetes_pod_phase
- job_name: "kubernetes-cadvisor"
scheme: https
metrics_path: /metrics/cadvisor
tls_config:
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
# disable certificate verification by uncommenting the line below.
#
# insecure_skip_verify: true
authorization:
credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token
kubernetes_sd_configs:
- role: node
relabel_configs:
- action: labelmap
regex: __meta_kubernetes_node_label_(.+)
metric_relabel_configs:
- action: keep
source_labels: [namespace]
regex: canary
- honor_labels: true
job_name: kubernetes-pods-slow
kubernetes_sd_configs:
- role: pod
namespaces:
names:
- canary
relabel_configs:
- action: keep
regex: true
source_labels:
- __meta_kubernetes_pod_annotation_prometheus_io_scrape_slow
- action: replace
regex: (https?)
source_labels:
- __meta_kubernetes_pod_annotation_prometheus_io_scheme
target_label: __scheme__
- action: replace
regex: (.+)
source_labels:
- __meta_kubernetes_pod_annotation_prometheus_io_path
target_label: __metrics_path__
- action: replace
regex: ([^:]+)(?::\d+)?;(\d+)
replacement: $1:$2
source_labels:
- __address__
- __meta_kubernetes_pod_annotation_prometheus_io_port
target_label: __address__
- action: labelmap
regex: __meta_kubernetes_pod_annotation_prometheus_io_param_(.+)
replacement: __param_$1
- action: labelmap
regex: __meta_kubernetes_pod_label_(.+)
- action: replace
source_labels:
- __meta_kubernetes_namespace
target_label: namespace
- action: replace
source_labels:
- __meta_kubernetes_pod_name
target_label: pod
- action: drop
regex: Pending|Succeeded|Failed|Completed
source_labels:
- __meta_kubernetes_pod_phase
scrape_interval: 5m
scrape_timeout: 30s
recording_rules.yml: |
{}
rules: |
{}
kind: ConfigMap
metadata:
annotations:
kubectl.kubernetes.io/last-applied-configuration: |
{"apiVersion":"v1","data":{"alerting_rules.yml":"{}\n","alerts":"{}\n","prometheus.yml":"global:\n evaluation_interval: 1m\n scrape_interval: 15s\n scrape_timeout: 10s\nrule_files:\n- /etc/config/recording_rules.yml\n- /etc/config/alerting_rules.yml\n- /etc/config/rules\n- /etc/config/alerts\nscrape_configs:\n- job_name: prometheus\n static_configs:\n - targets:\n - localhost:9090\n- bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token\n job_name: kubernetes-apiservers\n kubernetes_sd_configs:\n - role: endpoints\n relabel_configs:\n - action: keep\n regex: default;kubernetes;https\n source_labels:\n - __meta_kubernetes_namespace\n - __meta_kubernetes_service_name\n - __meta_kubernetes_endpoint_port_name\n scheme: https\n tls_config:\n ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt\n insecure_skip_verify: true\n- bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token\n job_name: kubernetes-nodes\n kubernetes_sd_configs:\n - role: node\n relabel_configs:\n - action: labelmap\n regex: __meta_kubernetes_node_label_(.+)\n - replacement: kubernetes.default.svc:443\n target_label: __address__\n - regex: (.+)\n replacement: /api/v1/nodes/$1/proxy/metrics\n source_labels:\n - __meta_kubernetes_node_name\n target_label: __metrics_path__\n scheme: https\n tls_config:\n ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt\n insecure_skip_verify: true\n- bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token\n job_name: kubernetes-nodes-cadvisor\n kubernetes_sd_configs:\n - role: node\n relabel_configs:\n - action: labelmap\n regex: __meta_kubernetes_node_label_(.+)\n - replacement: kubernetes.default.svc:443\n target_label: __address__\n - regex: (.+)\n replacement: /api/v1/nodes/$1/proxy/metrics/cadvisor\n source_labels:\n - __meta_kubernetes_node_name\n target_label: __metrics_path__\n scheme: https\n tls_config:\n ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt\n insecure_skip_verify: true\n- honor_labels: true\n job_name: kubernetes-service-endpoints\n kubernetes_sd_configs:\n - role: endpoints\n relabel_configs:\n - action: keep\n regex: true\n source_labels:\n - __meta_kubernetes_service_annotation_prometheus_io_scrape\n - action: drop\n regex: true\n source_labels:\n - __meta_kubernetes_service_annotation_prometheus_io_scrape_slow\n - action: replace\n regex: (https?)\n source_labels:\n - __meta_kubernetes_service_annotation_prometheus_io_scheme\n target_label: __scheme__\n - action: replace\n regex: (.+)\n source_labels:\n - __meta_kubernetes_service_annotation_prometheus_io_path\n target_label: __metrics_path__\n - action: replace\n regex: ([^:]+)(?::\\d+)?;(\\d+)\n replacement: $1:$2\n source_labels:\n - __address__\n - __meta_kubernetes_service_annotation_prometheus_io_port\n target_label: __address__\n - action: labelmap\n regex: __meta_kubernetes_service_annotation_prometheus_io_param_(.+)\n replacement: __param_$1\n - action: labelmap\n regex: __meta_kubernetes_service_label_(.+)\n - action: replace\n source_labels:\n - __meta_kubernetes_namespace\n target_label: namespace\n - action: replace\n source_labels:\n - __meta_kubernetes_service_name\n target_label: service\n - action: replace\n source_labels:\n - __meta_kubernetes_pod_node_name\n target_label: node\n- honor_labels: true\n job_name: kubernetes-service-endpoints-slow\n kubernetes_sd_configs:\n - role: endpoints\n relabel_configs:\n - action: keep\n regex: true\n source_labels:\n - __meta_kubernetes_service_annotation_prometheus_io_scrape_slow\n - action: replace\n regex: (https?)\n source_labels:\n - __meta_kubernetes_service_annotation_prometheus_io_scheme\n target_label: __scheme__\n - action: replace\n regex: (.+)\n source_labels:\n - __meta_kubernetes_service_annotation_prometheus_io_path\n target_label: __metrics_path__\n - action: replace\n regex: ([^:]+)(?::\\d+)?;(\\d+)\n replacement: $1:$2\n source_labels:\n - __address__\n - __meta_kubernetes_service_annotation_prometheus_io_port\n target_label: __address__\n - action: labelmap\n regex: __meta_kubernetes_service_annotation_prometheus_io_param_(.+)\n replacement: __param_$1\n - action: labelmap\n regex: __meta_kubernetes_service_label_(.+)\n - action: replace\n source_labels:\n - __meta_kubernetes_namespace\n target_label: namespace\n - action: replace\n source_labels:\n - __meta_kubernetes_service_name\n target_label: service\n - action: replace\n source_labels:\n - __meta_kubernetes_pod_node_name\n target_label: node\n scrape_interval: 5m\n scrape_timeout: 30s\n- honor_labels: true\n job_name: prometheus-pushgateway\n kubernetes_sd_configs:\n - role: service\n relabel_configs:\n - action: keep\n regex: pushgateway\n source_labels:\n - __meta_kubernetes_service_annotation_prometheus_io_probe\n- honor_labels: true\n job_name: kubernetes-services\n kubernetes_sd_configs:\n - role: service\n metrics_path: /probe\n params:\n module:\n - http_2xx\n relabel_configs:\n - action: keep\n regex: true\n source_labels:\n - __meta_kubernetes_service_annotation_prometheus_io_probe\n - source_labels:\n - __address__\n target_label: __param_target\n - replacement: blackbox\n target_label: __address__\n - source_labels:\n - __param_target\n target_label: instance\n - action: labelmap\n regex: __meta_kubernetes_service_label_(.+)\n - source_labels:\n - __meta_kubernetes_namespace\n target_label: namespace\n - source_labels:\n - __meta_kubernetes_service_name\n target_label: service\n- honor_labels: true\n job_name: kubernetes-pods\n kubernetes_sd_configs:\n - role: pod\n relabel_configs:\n - action: keep\n regex: true\n source_labels:\n - __meta_kubernetes_pod_annotation_prometheus_io_scrape\n - action: drop\n regex: true\n source_labels:\n - __meta_kubernetes_pod_annotation_prometheus_io_scrape_slow\n - action: replace\n regex: (https?)\n source_labels:\n - __meta_kubernetes_pod_annotation_prometheus_io_scheme\n target_label: __scheme__\n - action: replace\n regex: (.+)\n source_labels:\n - __meta_kubernetes_pod_annotation_prometheus_io_path\n target_label: __metrics_path__\n - action: replace\n regex: ([^:]+)(?::\\d+)?;(\\d+)\n replacement: $1:$2\n source_labels:\n - __address__\n - __meta_kubernetes_pod_annotation_prometheus_io_port\n target_label: __address__\n - action: labelmap\n regex: __meta_kubernetes_pod_annotation_prometheus_io_param_(.+)\n replacement: __param_$1\n - action: labelmap\n regex: __meta_kubernetes_pod_label_(.+)\n - action: replace\n source_labels:\n - __meta_kubernetes_namespace\n target_label: namespace\n - action: replace\n source_labels:\n - __meta_kubernetes_pod_name\n target_label: pod\n - action: drop\n regex: Pending|Succeeded|Failed|Completed\n source_labels:\n - __meta_kubernetes_pod_phase\n- honor_labels: true\n job_name: kubernetes-pods-slow\n kubernetes_sd_configs:\n - role: pod\n relabel_configs:\n - action: keep\n regex: true\n source_labels:\n - __meta_kubernetes_pod_annotation_prometheus_io_scrape_slow\n - action: replace\n regex: (https?)\n source_labels:\n - __meta_kubernetes_pod_annotation_prometheus_io_scheme\n target_label: __scheme__\n - action: replace\n regex: (.+)\n source_labels:\n - __meta_kubernetes_pod_annotation_prometheus_io_path\n target_label: __metrics_path__\n - action: replace\n regex: ([^:]+)(?::\\d+)?;(\\d+)\n replacement: $1:$2\n source_labels:\n - __address__\n - __meta_kubernetes_pod_annotation_prometheus_io_port\n target_label: __address__\n - action: labelmap\n regex: __meta_kubernetes_pod_annotation_prometheus_io_param_(.+)\n replacement: __param_$1\n - action: labelmap\n regex: __meta_kubernetes_pod_label_(.+)\n - action: replace\n source_labels:\n - __meta_kubernetes_namespace\n target_label: namespace\n - action: replace\n source_labels:\n - __meta_kubernetes_pod_name\n target_label: pod\n - action: drop\n regex: Pending|Succeeded|Failed|Completed\n source_labels:\n - __meta_kubernetes_pod_phase\n scrape_interval: 5m\n scrape_timeout: 30s\n","recording_rules.yml":"{}\n","rules":"{}\n"},"kind":"ConfigMap","metadata":{"annotations":{},"labels":{"app":"prometheus","chart":"prometheus-15.9.0","component":"server","heritage":"Helm","release":"prometheus"},"name":"prometheus","namespace":"istio-system"}}
creationTimestamp: "2023-02-01T04:57:40Z"
labels:
app: prometheus
chart: prometheus-15.9.0
component: server
heritage: Helm
release: prometheus
name: prometheus
namespace: istio-system
resourceVersion: "10173050"
uid: 2df90740-9fed-4944-8d1a-89f0e7295996
The pod is in running state but on grafana I am able to see all namespaces.
To observe just the metric of a single namespace.
If you want monitoring only the current namespace where Prometheus pod runs, then put the following namespaces config option inside every kubernetes_sd_config:
kubernetes_sd_configs:
- ... your config here
namespaces:
own_namespace: true
For example:
scrape_configs:
- job_name: kubernetes-service-endpoints
kubernetes_sd_configs:
- role: endpoints
namespaces:
own_namespace: true
If you need monitoring only the specified namespace (for example, foobar), then use the following config:
kubernetes_sd_configs:
- ... your config here
namespaces:
names:
- foobar
See kubernetes_sd_configs docs for more details.
Related
I am new to promtail and struggling to understand location from where it picks logs from each pod. I installed loki stack with promtail below are my default scrape_config
scrape_configs:
# See also https://github.com/grafana/loki/blob/master/production/ksonnet/promtail/scrape_config.libsonnet for reference
- job_name: kubernetes-pods
pipeline_stages:
- cri: {}
kubernetes_sd_configs:
- role: pod
relabel_configs:
- source_labels:
- __meta_kubernetes_pod_controller_name
regex: ([0-9a-z-.]+?)(-[0-9a-f]{8,10})?
action: replace
target_label: __tmp_controller_name
- source_labels:
- __meta_kubernetes_pod_label_app_kubernetes_io_name
- __meta_kubernetes_pod_label_app
- __tmp_controller_name
- __meta_kubernetes_pod_name
regex: ^;*([^;]+)(;.*)?$
action: replace
target_label: app
- source_labels:
- __meta_kubernetes_pod_label_app_kubernetes_io_instance
- __meta_kubernetes_pod_label_release
regex: ^;*([^;]+)(;.*)?$
action: replace
target_label: instance
- source_labels:
- __meta_kubernetes_pod_label_app_kubernetes_io_component
- __meta_kubernetes_pod_label_component
regex: ^;*([^;]+)(;.*)?$
action: replace
target_label: component
- action: replace
source_labels:
- __meta_kubernetes_pod_node_name
target_label: node_name
- action: replace
source_labels:
- __meta_kubernetes_namespace
target_label: namespace
- action: replace
replacement: $1
separator: /
source_labels:
- namespace
- app
target_label: job
- action: replace
source_labels:
- __meta_kubernetes_pod_name
target_label: pod
- action: replace
source_labels:
- __meta_kubernetes_pod_container_name
target_label: container
- action: replace
replacement: /var/log/pods/*$1/*.log
separator: /
source_labels:
- __meta_kubernetes_pod_uid
- __meta_kubernetes_pod_container_name
target_label: __path__
- action: replace
regex: true/(.*)
replacement: /var/log/pods/*$1/*.log
separator: /
source_labels:
- __meta_kubernetes_pod_annotationpresent_kubernetes_io_config_hash
- __meta_kubernetes_pod_annotation_kubernetes_io_config_hash
- __meta_kubernetes_pod_container_name
target_label: __path__
1- I want loki/promtail to pick some other log files but I am not sure on which location should I put these logs so that it is picked up by loki/promtails. I tried following paths but it did work
/app/logs
/var/log/containers
/var/log/pods
/var/log
/app
Look forward to replies. Thanks in advance
I am using Kubernetes service discovery in Prometheus but for one of the services I need to pass an authentication block with username and password like:
basic_auth:
username: prometheus
password: prom123456
I understand that the authentication is per job so how can I combine both the discovery and the authentication for only one service of the discovery?
- job_name: 'kubernetes-service-endpoints'
kubernetes_sd_configs:
- role: endpoints
relabel_configs:
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
action: keep
regex: true
- source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
action: replace
regex: ([^:]+)(?::\d+)?;(\d+)
replacement: $1:$2
target_label: __address__
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
action: replace
target_label: __metrics_path__
regex: (.+)
- source_labels: [__meta_kubernetes_namespace]
action: replace
target_label: namespace
regex: (.+)
- regex: __meta_kubernetes_service_label_(.+)
action: labelmap
- regex: 'app_kubernetes_io_(.+)'
action: labeldrop
- regex: 'helm_sh_(.+)'
action: labeldrop
I have the below prometheus configuration to fetch the kubernetes pods metrics but however , in prometheus i could see only 2 metrics related to pods which Kube_pod_CPU, Kube_pod_memory but not any other metrics related to pods. Any help would be appreciated if there is any configuration issue.
- job_name: 'kubernetes-pods'
scrape_interval: 15s
scrape_timeout: 10s
metrics_path: /metrics
scheme: http
kubernetes_sd_configs:
- role: pod
namespaces:
names: ["namespace-name"]
relabel_configs:
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
action: keep
separator: ":"
replacement: $1
regex: true
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
action: replace
separator: ":"
replacement: $1
target_label: __metrics_path__
regex: (.+)
- source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
action: replace
separator: ":"
/prometheus-kubernetes.yml
regex: (.+):(?:\d+);(\d+)
#regex: ([^:]+)(?::\d+)?;(\d+)
replacement: ${1}:${2}
target_label: __address__
- source_labels: [__meta_kubernetes_pod_container_name]
action: keep
regex: prometheus.*
- action: labelmap
separator: ":"
replacement: $1
regex: __meta_kubernetes_pod_label_(.+)
- source_labels: [__meta_kubernetes_namespace]
separator: ":"
replacement: $1
action: replace
target_label: kubernetes_namespace
- source_labels: [__meta_kubernetes_pod_name]
separator: ":"
replacement: $1
action: replace
target_label: kubernetes_pod_name
Thanks
> You have to add this three annotation to your deployment:
prometheus.io/scrape: 'true'
prometheus.io/path: 'metrics'
prometheus.io/port: '80'
How it will work?
Look at the kubernetes-pods job of config-map.yaml you are using to configure prometheus,
- job_name: 'kubernetes-pods'
kubernetes_sd_configs:
- role: pod
relabel_configs:
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
action: keep
regex: true
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
action: replace
target_label: __metrics_path__
regex: (.+)
- source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
action: replace
regex: ([^:]+)(?::\d+)?;(\d+)
replacement: $1:$2
target_label: __address__
- action: labelmap
regex: __meta_kubernetes_pod_label_(.+)
- source_labels: [__meta_kubernetes_namespace]
action: replace
target_label: kubernetes_namespace
- source_labels: [__meta_kubernetes_pod_name]
action: replace
target_label: kubernetes_pod_name
I'm starting playing with custom exporters(Using kubernetes,grafana and prometheus) and I have a problem. I managed to expose my metrics correctly but every time I kill the pod that is sending them, the vars change and grafana plots a different colour(like a new info).
Is there any way to only keep app as var, I think that the problem are the vars that change(pod name and ip)?
MyMetric{app="prometheus-export-mymetric",instance="172.26.32.69:3000",job="kubernetes-pods",kubernetes_namespace="default",kubernetes_pod_name="prometheus-export-mymetric-66694564b8-r4pqc",pod_template_hash="66694564b8"}
Thanks in advance.
instead of kubernetes_pod_name you shall use pod labels that stay the same after redeploying.
in prometheus config we are using something like this:
- job_name: kubernetes-pods
honor_timestamps: true
scrape_interval: 15s
scrape_timeout: 10s
metrics_path: /metrics
scheme: http
kubernetes_sd_configs:
- role: pod
relabel_configs:
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
separator: ;
regex: "true"
replacement: $1
action: keep
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
separator: ;
regex: (.+)
target_label: __metrics_path__
replacement: $1
action: replace
- source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
separator: ;
regex: ([^:]+)(?::\d+)?;(\d+)
target_label: __address__
replacement: $1:$2
action: replace
- separator: ;
regex: __meta_kubernetes_pod_label_(.+)
replacement: $1
action: labelmap
- source_labels: [__meta_kubernetes_namespace]
separator: ;
regex: (.*)
target_label: kubernetes_namespace
replacement: $1
action: replace
- source_labels: [__meta_kubernetes_pod_name]
separator: ;
regex: (.*)
target_label: kubernetes_pod_name
replacement: $1
action: replace
I have deployed prometheus 2.0 on my multi node kubernetes cluster made from kubeadm. While accessing the prometheus dashboard i am not able to view pods and service job even after configuring it in prometheus configuration yaml file.
prometheus target are as follow https://i.stack.imgur.com/jiQPG.png .
Does this problem has anything to do with the prometheus version. I think i am going wrong with the syntax part of the configuration.
global:
scrape_interval: 5s
scrape_timeout: 5s
evaluation_interval: 5s
scrape_configs:
- job_name: kubernetes-apiservers
scrape_interval: 5s
scrape_timeout: 5s
metrics_path: /metrics
scheme: https
kubernetes_sd_configs:
- api_server: null
role: endpoints
namespaces:
names: []
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
tls_config:
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
insecure_skip_verify: false
relabel_configs:
- source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
separator: ;
regex: default;kubernetes;https
replacement: $1
action: keep
- job_name: kubernetes-nodes
scrape_interval: 5s
scrape_timeout: 5s
metrics_path: /metrics
scheme: https
kubernetes_sd_configs:
- api_server: null
role: node
namespaces:
names: []
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
tls_config:
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
insecure_skip_verify: false
relabel_configs:
- separator: ;
regex: __meta_kubernetes_node_label_(.+)
replacement: $1
action: labelmap
- separator: ;
regex: (.*)
target_label: __address__
replacement: kubernetes.default.svc:443
action: replace
- source_labels: [__meta_kubernetes_node_name]
separator: ;
regex: (.+)
target_label: __metrics_path__
replacement: /api/v1/nodes/${1}/proxy/metrics
action: replace
- job_name: kubernetes-pods
scrape_interval: 5s
scrape_timeout: 5s
metrics_path: /metrics
scheme: https
kubernetes_sd_configs:
- api_server: null
role: pod
namespaces:
names: []
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
tls_config:
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
insecure_skip_verify: false
relabel_configs:
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
separator: ;
regex: "true"
replacement: $1
action: keep
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
separator: ;
regex: (.+)
target_label: __metrics_path__
replacement: $1
action: replace
- source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
separator: ;
regex: ([^:]+)(?::\d+)?;(\d+)
target_label: __address__
replacement: $1:$2
action: replace
- separator: ;
regex: __meta_kubernetes_pod_label_(.+)
replacement: $1
action: labelmap
- source_labels: [__meta_kubernetes_namespace]
separator: ;
regex: (.*)
target_label: kubernetes_namespace
replacement: $1
action: replace
- source_labels: [__meta_kubernetes_pod_name]
separator: ;
regex: (.*)
target_label: kubernetes_pod_name
replacement: $1
action: replace
- job_name: kubernetes-cadvisor
scrape_interval: 5s
scrape_timeout: 5s
metrics_path: /metrics
scheme: https
kubernetes_sd_configs:
- api_server: null
role: node
namespaces:
names: []
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
tls_config:
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
insecure_skip_verify: false
relabel_configs:
- separator: ;
regex: __meta_kubernetes_node_label_(.+)
replacement: $1
action: labelmap
- separator: ;
regex: (.*)
target_label: __address__
replacement: kubernetes.default.svc:443
action: replace
- source_labels: [__meta_kubernetes_node_name]
separator: ;
regex: (.+)
target_label: __metrics_path__
replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
action: replace
- job_name: kubernetes-service-endpoints
scrape_interval: 5s
scrape_timeout: 5s
metrics_path: /metrics
scheme: https
kubernetes_sd_configs:
- api_server: null
role: endpoints
namespaces:
names: []
relabel_configs:
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
separator: ;
regex: "true"
replacement: $1
action: keep
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
separator: ;
regex: (https?)
target_label: __scheme__
replacement: $1
action: replace
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
separator: ;
regex: (.+)
target_label: __metrics_path__
replacement: $1
action: replace
- source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
separator: ;
regex: ([^:]+)(?::\d+)?;(\d+)
target_label: __address__
replacement: $1:$2
action: replace
- separator: ;
regex: __meta_kubernetes_service_label_(.+)
replacement: $1
action: labelmap
- source_labels: [__meta_kubernetes_namespace]
separator: ;
regex: (.*)
target_label: kubernetes_namespace
replacement: $1
action: replace
- source_labels: [__meta_kubernetes_service_name]
separator: ;
regex: (.*)
target_label: kubernetes_name
replacement: $1
action: replace
Thanks
I assume you are doing the query you posted in the comments:
container_memory_usage_bytes{job="kubernetes-pods"}
This doesn't work, because you are filtering by the job name kubernetes-pods, but container_memory_usage_bytes comes from cAdvisor. So according to your config the job is named kubernetes-cadvisor.
Therefore this should work:
container_memory_usage_bytes{job="kubernetes-cadvisor"}
Since the series name is rather unique, you can just omit the job name:
container_memory_usage_bytes