prometheus

Query Prometheus via curl Ad-hoc PromQL query from CLI curl -sG 'http://prometheus:9090/api/v1/query' --data-urlencode 'query=up' | jq . #prometheus#monitoring Instant query via promtool Run an instant PromQL query using promtool promtool query instant http://prometheus:9090 'up{job="kubelet"}' #prometheus#monitoring#debug Range query via curl Query Prometheus for a time range via HTTP API curl -sG 'http://prometheus:9090/api/v1/query_range' --data-urlencode 'query=rate(http_requests_total[5m])' --data-urlencode 'start=2024-01-01T00:00:00Z' --data-urlencode 'end=2024-01-01T01:00:00Z' --data-urlencode 'step=60s' | jq . #prometheus#monitoring Check alert rules Validate Prometheus alert/recording rule files for syntax promtool check rules rules.yml #prometheus#monitoring Silence active alert Create a silence to mute alert notifications during maintenance amtool silence add --alertmanager.url=http://alertmanager:9093 alertname=<name> --duration=2h --comment='Maintenance' #alertmanager#monitoring List silences Show all active silences in Alertmanager amtool silence query --alertmanager.url=http://alertmanager:9093 #alertmanager#monitoring Expire silence Immediately end an active silence amtool silence expire <id> --alertmanager.url=http://alertmanager:9093 #alertmanager#monitoring Check alertmanager config Validate Alertmanager configuration file amtool check-config alertmanager.yml #alertmanager#monitoring List active alerts Show all firing alerts in Alertmanager amtool alert query --alertmanager.url=http://alertmanager:9093 #alertmanager#monitoring Reload Prometheus config Hot-reload Prometheus configuration without restart curl -X POST http://prometheus:9090/-/reload #prometheus#monitoring Query instant vector Run an instant PromQL query via HTTP API curl -sG 'http://prometheus:9090/api/v1/query' --data-urlencode 'query=up' | jq '.data.result' #prometheus#monitoring Range query for graph data Fetch time-series data over a range for a rate metric curl -sG 'http://prometheus:9090/api/v1/query_range' --data-urlencode 'query=rate(http_requests_total[5m])' --data-urlencode 'start=1h' --data-urlencode 'end=now' --data-urlencode 'step=60' #prometheus#monitoring Top N series by value PromQL: top 10 pods by CPU usage rate topk(10, sum by (pod) (rate(container_cpu_usage_seconds_total[5m]))) #prometheus#monitoring Disk usage alert rule PromQL expression: fire when root disk > 85% used 100 - (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"} * 100) > 85 #prometheus#monitoring Reload Prometheus config Hot reload Prometheus config and rules without restart curl -X POST http://prometheus:9090/-/reload #prometheus List active alerts Show only firing alerts with name and severity curl -s http://prometheus:9090/api/v1/alerts | jq '.data.alerts[] | select(.state=="firing") | {name: .labels.alertname, severity: .labels.severity}' #prometheus#monitoring List all metric names Count total metric names and preview first 20 curl -s http://prometheus:9090/api/v1/label/__name__/values | jq '.data | length, .[0:20]' #prometheus#debug Series cardinality per job PromQL: count active time series grouped by job label count by (job) ({__name__=~".+"}) #prometheus#monitoring Silence an alert Create an Alertmanager silence for 2 hours amtool silence add alertname=HighMemory --duration=2h --comment='Investigating' #prometheus#monitoring Check TSDB status View TSDB head stats: chunks and active series curl -s http://prometheus:9090/api/v1/status/tsdb | jq '{head_chunks: .data.headStats.numChunks, series: .data.headStats.numSeries}' #prometheus#monitoring 95th percentile latency from histogram p95 latency per job; note: always aggregate _bucket by le before histogram_quantile histogram_quantile(0.95, sum by(le,job)(rate(http_request_duration_seconds_bucket[5m]))) #prometheus#performance#observability#monitoring Predict disk full window alert when linear trend predicts disk full within 4h based on last 6h predict_linear(node_filesystem_avail_bytes{mountpoint="/"}[6h], 4*3600) < 0 #prometheus#disk#monitoring#troubleshooting Detect dead target with absent fires when no series match; catches a job that vanished from discovery entirely absent(up{job="node"} == 1) #prometheus#monitoring#troubleshooting#observability Rewrite label with label_replace extract/derive a new label via regex capture; useful for joins and tidy dashboards label_replace(node_uname_info, "host", "$1", "nodename", "(.+)") #prometheus#observability#monitoring Max of per-second rate over window subquery: peak 5m rate sampled each minute over the last hour; spot bursts max_over_time(rate(node_network_receive_bytes_total[5m])[1h:1m]) #prometheus#performance#network#observability Aggregate keeping all labels but instance without() keeps every label except instance; cleaner than listing many by() labels sum without(instance) (rate(http_requests_total{code=~"5.."}[5m])) #prometheus#observability#monitoring Recording rule for expensive ratio precompute SLO error ratio to keep dashboards and alerts fast groups: - name: slo rules: - record: job:request_errors:ratio5m expr: sum by(job)(rate(http_requests_total{code=~"5.."}[5m])) / sum by(job)(rate(http_requests_total[5m])) #prometheus#gitops#observability#performance Validate rules and config with promtool lint before reload; run in CI to catch broken expr/yaml before deploy promtool check rules rules.yml && promtool check config prometheus.yml #prometheus#ci#gitops#troubleshooting Instant query from CLI run PromQL without a browser; great for scripted top-N investigations promtool query instant http://localhost:9090 'topk(5, sum by(pod)(rate(container_cpu_usage_seconds_total[5m])))' #prometheus#debug#troubleshooting#performance Find cardinality offenders in TSDB lists highest-cardinality metrics and label pairs; first stop for memory blowups promtool tsdb analyze /prometheus --limit 20 #prometheus#performance#troubleshooting#observability