# Two histograms with 4 buckets each (x_sum and x_count not included, # only buckets). Lowest bucket for one histogram < 0, for the other > # 0. They have the same name, just separated by label. Not useful in # practice, but can happen (if clients change bucketing), and the # server has to cope with it. # Test histogram. load_with_nhcb 5m testhistogram_bucket{le="0.1", start="positive"} 0+5x10 testhistogram_bucket{le=".2", start="positive"} 0+7x10 testhistogram_bucket{le="1e0", start="positive"} 0+11x10 testhistogram_bucket{le="+Inf", start="positive"} 0+12x10 testhistogram_bucket{le="-.2", start="negative"} 0+1x10 testhistogram_bucket{le="-0.1", start="negative"} 0+2x10 testhistogram_bucket{le="0.3", start="negative"} 0+2x10 testhistogram_bucket{le="+Inf", start="negative"} 0+3x10 # Another test histogram, where q(1/6), q(1/2), and q(5/6) are each in # the middle of a bucket and should therefore be 1, 3, and 5, # respectively. load_with_nhcb 5m testhistogram2_bucket{le="0"} 0+0x10 testhistogram2_bucket{le="2"} 0+1x10 testhistogram2_bucket{le="4"} 0+2x10 testhistogram2_bucket{le="6"} 0+3x10 testhistogram2_bucket{le="+Inf"} 0+3x10 # Another test histogram, this time without any observations in the +Inf bucket. # This enables a meaningful calculation of standard deviation and variance. load_with_nhcb 5m testhistogram3_bucket{le="0", start="positive"} 0+0x10 testhistogram3_bucket{le="0.1", start="positive"} 0+5x10 testhistogram3_bucket{le=".2", start="positive"} 0+7x10 testhistogram3_bucket{le="1e0", start="positive"} 0+11x10 testhistogram3_bucket{le="+Inf", start="positive"} 0+11x10 testhistogram3_sum{start="positive"} 0+33x10 testhistogram3_count{start="positive"} 0+11x10 testhistogram3_bucket{le="-.25", start="negative"} 0+0x10 testhistogram3_bucket{le="-.2", start="negative"} 0+1x10 testhistogram3_bucket{le="-0.1", start="negative"} 0+2x10 testhistogram3_bucket{le="0.3", start="negative"} 0+2x10 testhistogram3_bucket{le="+Inf", start="negative"} 0+2x10 testhistogram3_sum{start="negative"} 0+8x10 testhistogram3_count{start="negative"} 0+2x10 # Now a more realistic histogram per job and instance to test aggregation. load_with_nhcb 5m request_duration_seconds_bucket{job="job1", instance="ins1", le="0.1"} 0+1x10 request_duration_seconds_bucket{job="job1", instance="ins1", le="0.2"} 0+3x10 request_duration_seconds_bucket{job="job1", instance="ins1", le="+Inf"} 0+4x10 request_duration_seconds_bucket{job="job1", instance="ins2", le="0.1"} 0+2x10 request_duration_seconds_bucket{job="job1", instance="ins2", le="0.2"} 0+5x10 request_duration_seconds_bucket{job="job1", instance="ins2", le="+Inf"} 0+6x10 request_duration_seconds_bucket{job="job2", instance="ins1", le="0.1"} 0+3x10 request_duration_seconds_bucket{job="job2", instance="ins1", le="0.2"} 0+4x10 request_duration_seconds_bucket{job="job2", instance="ins1", le="+Inf"} 0+6x10 request_duration_seconds_bucket{job="job2", instance="ins2", le="0.1"} 0+4x10 request_duration_seconds_bucket{job="job2", instance="ins2", le="0.2"} 0+7x10 request_duration_seconds_bucket{job="job2", instance="ins2", le="+Inf"} 0+9x10 # Different le representations in one histogram. load_with_nhcb 5m mixed_bucket{job="job1", instance="ins1", le="0.1"} 0+1x10 mixed_bucket{job="job1", instance="ins1", le="0.2"} 0+1x10 mixed_bucket{job="job1", instance="ins1", le="2e-1"} 0+1x10 mixed_bucket{job="job1", instance="ins1", le="2.0e-1"} 0+1x10 mixed_bucket{job="job1", instance="ins1", le="+Inf"} 0+4x10 mixed_bucket{job="job1", instance="ins2", le="+inf"} 0+0x10 mixed_bucket{job="job1", instance="ins2", le="+Inf"} 0+0x10 # Test histogram_count. eval instant at 50m histogram_count(testhistogram3) {start="positive"} 110 {start="negative"} 20 # Classic way of accessing the count still works. eval instant at 50m testhistogram3_count testhistogram3_count{start="positive"} 110 testhistogram3_count{start="negative"} 20 # Test histogram_sum. eval instant at 50m histogram_sum(testhistogram3) {start="positive"} 330 {start="negative"} 80 # Classic way of accessing the sum still works. eval instant at 50m testhistogram3_sum testhistogram3_sum{start="positive"} 330 testhistogram3_sum{start="negative"} 80 # Test histogram_avg. This has no classic equivalent. eval instant at 50m histogram_avg(testhistogram3) {start="positive"} 3 {start="negative"} 4 # Test histogram_stddev. This has no classic equivalent. eval instant at 50m histogram_stddev(testhistogram3) {start="positive"} 2.8189265757336734 {start="negative"} 4.182715937754936 # Test histogram_stdvar. This has no classic equivalent. eval instant at 50m histogram_stdvar(testhistogram3) {start="positive"} 7.946347039377573 {start="negative"} 17.495112615949154 # Test histogram_fraction. eval instant at 50m histogram_fraction(0, 0.2, testhistogram3) {start="positive"} 0.6363636363636364 {start="negative"} 0 eval instant at 50m histogram_fraction(0, 0.2, rate(testhistogram3[10m])) {start="positive"} 0.6363636363636364 {start="negative"} 0 # In the classic histogram, we can access the corresponding bucket (if # it exists) and divide by the count to get the same result. eval instant at 50m testhistogram3_bucket{le=".2"} / ignoring(le) testhistogram3_count {start="positive"} 0.6363636363636364 eval instant at 50m rate(testhistogram3_bucket{le=".2"}[10m]) / ignoring(le) rate(testhistogram3_count[10m]) {start="positive"} 0.6363636363636364 # Test histogram_quantile, native and classic. eval instant at 50m histogram_quantile(0, testhistogram3) {start="positive"} 0 {start="negative"} -0.25 eval instant at 50m histogram_quantile(0, testhistogram3_bucket) {start="positive"} 0 {start="negative"} -0.25 eval instant at 50m histogram_quantile(0.25, testhistogram3) {start="positive"} 0.055 {start="negative"} -0.225 eval instant at 50m histogram_quantile(0.25, testhistogram3_bucket) {start="positive"} 0.055 {start="negative"} -0.225 eval instant at 50m histogram_quantile(0.5, testhistogram3) {start="positive"} 0.125 {start="negative"} -0.2 eval instant at 50m histogram_quantile(0.5, testhistogram3_bucket) {start="positive"} 0.125 {start="negative"} -0.2 eval instant at 50m histogram_quantile(0.75, testhistogram3) {start="positive"} 0.45 {start="negative"} -0.15 eval instant at 50m histogram_quantile(0.75, testhistogram3_bucket) {start="positive"} 0.45 {start="negative"} -0.15 eval instant at 50m histogram_quantile(1, testhistogram3) {start="positive"} 1 {start="negative"} -0.1 eval instant at 50m histogram_quantile(1, testhistogram3_bucket) {start="positive"} 1 {start="negative"} -0.1 # Quantile too low. eval_warn instant at 50m histogram_quantile(-0.1, testhistogram) {start="positive"} -Inf {start="negative"} -Inf eval_warn instant at 50m histogram_quantile(-0.1, testhistogram_bucket) {start="positive"} -Inf {start="negative"} -Inf # Quantile too high. eval_warn instant at 50m histogram_quantile(1.01, testhistogram) {start="positive"} +Inf {start="negative"} +Inf eval_warn instant at 50m histogram_quantile(1.01, testhistogram_bucket) {start="positive"} +Inf {start="negative"} +Inf # Quantile invalid. eval_warn instant at 50m histogram_quantile(NaN, testhistogram) {start="positive"} NaN {start="negative"} NaN eval_warn instant at 50m histogram_quantile(NaN, testhistogram_bucket) {start="positive"} NaN {start="negative"} NaN # Quantile value in lowest bucket. eval instant at 50m histogram_quantile(0, testhistogram) {start="positive"} 0 {start="negative"} -0.2 eval instant at 50m histogram_quantile(0, testhistogram_bucket) {start="positive"} 0 {start="negative"} -0.2 # Quantile value in highest bucket. eval instant at 50m histogram_quantile(1, testhistogram) {start="positive"} 1 {start="negative"} 0.3 eval instant at 50m histogram_quantile(1, testhistogram_bucket) {start="positive"} 1 {start="negative"} 0.3 # Finally some useful quantiles. eval instant at 50m histogram_quantile(0.2, testhistogram) {start="positive"} 0.048 {start="negative"} -0.2 eval instant at 50m histogram_quantile(0.2, testhistogram_bucket) {start="positive"} 0.048 {start="negative"} -0.2 eval instant at 50m histogram_quantile(0.5, testhistogram) {start="positive"} 0.15 {start="negative"} -0.15 eval instant at 50m histogram_quantile(0.5, testhistogram_bucket) {start="positive"} 0.15 {start="negative"} -0.15 eval instant at 50m histogram_quantile(0.8, testhistogram) {start="positive"} 0.72 {start="negative"} 0.3 eval instant at 50m histogram_quantile(0.8, testhistogram_bucket) {start="positive"} 0.72 {start="negative"} 0.3 # More realistic with rates. eval instant at 50m histogram_quantile(0.2, rate(testhistogram[10m])) {start="positive"} 0.048 {start="negative"} -0.2 eval instant at 50m histogram_quantile(0.2, rate(testhistogram_bucket[10m])) {start="positive"} 0.048 {start="negative"} -0.2 eval instant at 50m histogram_quantile(0.5, rate(testhistogram[10m])) {start="positive"} 0.15 {start="negative"} -0.15 eval instant at 50m histogram_quantile(0.5, rate(testhistogram_bucket[10m])) {start="positive"} 0.15 {start="negative"} -0.15 eval instant at 50m histogram_quantile(0.8, rate(testhistogram[10m])) {start="positive"} 0.72 {start="negative"} 0.3 eval instant at 50m histogram_quantile(0.8, rate(testhistogram_bucket[10m])) {start="positive"} 0.72 {start="negative"} 0.3 # Want results exactly in the middle of the bucket. eval instant at 7m histogram_quantile(1./6., testhistogram2) {} 1 eval instant at 7m histogram_quantile(1./6., testhistogram2_bucket) {} 1 eval instant at 7m histogram_quantile(0.5, testhistogram2) {} 3 eval instant at 7m histogram_quantile(0.5, testhistogram2_bucket) {} 3 eval instant at 7m histogram_quantile(5./6., testhistogram2) {} 5 eval instant at 7m histogram_quantile(5./6., testhistogram2_bucket) {} 5 eval instant at 47m histogram_quantile(1./6., rate(testhistogram2[15m])) {} 1 eval instant at 47m histogram_quantile(1./6., rate(testhistogram2_bucket[15m])) {} 1 eval instant at 47m histogram_quantile(0.5, rate(testhistogram2[15m])) {} 3 eval instant at 47m histogram_quantile(0.5, rate(testhistogram2_bucket[15m])) {} 3 eval instant at 47m histogram_quantile(5./6., rate(testhistogram2[15m])) {} 5 eval instant at 47m histogram_quantile(5./6., rate(testhistogram2_bucket[15m])) {} 5 # Aggregated histogram: Everything in one. Note how native histograms # don't require aggregation by le. eval instant at 50m histogram_quantile(0.3, sum(rate(request_duration_seconds[10m]))) {} 0.075 eval instant at 50m histogram_quantile(0.3, sum(rate(request_duration_seconds_bucket[10m])) by (le)) {} 0.075 eval instant at 50m histogram_quantile(0.5, sum(rate(request_duration_seconds[10m]))) {} 0.1277777777777778 eval instant at 50m histogram_quantile(0.5, sum(rate(request_duration_seconds_bucket[10m])) by (le)) {} 0.1277777777777778 # Aggregated histogram: Everything in one. Now with avg, which does not change anything. eval instant at 50m histogram_quantile(0.3, avg(rate(request_duration_seconds[10m]))) {} 0.075 eval instant at 50m histogram_quantile(0.3, avg(rate(request_duration_seconds_bucket[10m])) by (le)) {} 0.075 eval instant at 50m histogram_quantile(0.5, avg(rate(request_duration_seconds[10m]))) {} 0.12777777777777778 eval instant at 50m histogram_quantile(0.5, avg(rate(request_duration_seconds_bucket[10m])) by (le)) {} 0.12777777777777778 # Aggregated histogram: By instance. eval instant at 50m histogram_quantile(0.3, sum(rate(request_duration_seconds[10m])) by (instance)) {instance="ins1"} 0.075 {instance="ins2"} 0.075 eval instant at 50m histogram_quantile(0.3, sum(rate(request_duration_seconds_bucket[10m])) by (le, instance)) {instance="ins1"} 0.075 {instance="ins2"} 0.075 eval instant at 50m histogram_quantile(0.5, sum(rate(request_duration_seconds[10m])) by (instance)) {instance="ins1"} 0.1333333333 {instance="ins2"} 0.125 eval instant at 50m histogram_quantile(0.5, sum(rate(request_duration_seconds_bucket[10m])) by (le, instance)) {instance="ins1"} 0.1333333333 {instance="ins2"} 0.125 # Aggregated histogram: By job. eval instant at 50m histogram_quantile(0.3, sum(rate(request_duration_seconds[10m])) by (job)) {job="job1"} 0.1 {job="job2"} 0.0642857142857143 eval instant at 50m histogram_quantile(0.3, sum(rate(request_duration_seconds_bucket[10m])) by (le, job)) {job="job1"} 0.1 {job="job2"} 0.0642857142857143 eval instant at 50m histogram_quantile(0.5, sum(rate(request_duration_seconds[10m])) by (job)) {job="job1"} 0.14 {job="job2"} 0.1125 eval instant at 50m histogram_quantile(0.5, sum(rate(request_duration_seconds_bucket[10m])) by (le, job)) {job="job1"} 0.14 {job="job2"} 0.1125 # Aggregated histogram: By job and instance. eval instant at 50m histogram_quantile(0.3, sum(rate(request_duration_seconds[10m])) by (job, instance)) {instance="ins1", job="job1"} 0.11 {instance="ins2", job="job1"} 0.09 {instance="ins1", job="job2"} 0.06 {instance="ins2", job="job2"} 0.0675 eval instant at 50m histogram_quantile(0.3, sum(rate(request_duration_seconds_bucket[10m])) by (le, job, instance)) {instance="ins1", job="job1"} 0.11 {instance="ins2", job="job1"} 0.09 {instance="ins1", job="job2"} 0.06 {instance="ins2", job="job2"} 0.0675 eval instant at 50m histogram_quantile(0.5, sum(rate(request_duration_seconds[10m])) by (job, instance)) {instance="ins1", job="job1"} 0.15 {instance="ins2", job="job1"} 0.1333333333333333 {instance="ins1", job="job2"} 0.1 {instance="ins2", job="job2"} 0.1166666666666667 eval instant at 50m histogram_quantile(0.5, sum(rate(request_duration_seconds_bucket[10m])) by (le, job, instance)) {instance="ins1", job="job1"} 0.15 {instance="ins2", job="job1"} 0.1333333333333333 {instance="ins1", job="job2"} 0.1 {instance="ins2", job="job2"} 0.1166666666666667 # The unaggregated histogram for comparison. Same result as the previous one. eval instant at 50m histogram_quantile(0.3, rate(request_duration_seconds[10m])) {instance="ins1", job="job1"} 0.11 {instance="ins2", job="job1"} 0.09 {instance="ins1", job="job2"} 0.06 {instance="ins2", job="job2"} 0.0675 eval instant at 50m histogram_quantile(0.3, rate(request_duration_seconds_bucket[10m])) {instance="ins1", job="job1"} 0.11 {instance="ins2", job="job1"} 0.09 {instance="ins1", job="job2"} 0.06 {instance="ins2", job="job2"} 0.0675 eval instant at 50m histogram_quantile(0.5, rate(request_duration_seconds[10m])) {instance="ins1", job="job1"} 0.15 {instance="ins2", job="job1"} 0.13333333333333333 {instance="ins1", job="job2"} 0.1 {instance="ins2", job="job2"} 0.11666666666666667 eval instant at 50m histogram_quantile(0.5, rate(request_duration_seconds_bucket[10m])) {instance="ins1", job="job1"} 0.15 {instance="ins2", job="job1"} 0.13333333333333333 {instance="ins1", job="job2"} 0.1 {instance="ins2", job="job2"} 0.11666666666666667 # All NHCBs summed into one. eval instant at 50m sum(request_duration_seconds) {} {{schema:-53 count:250 custom_values:[0.1 0.2] buckets:[100 90 60]}} eval instant at 50m sum(request_duration_seconds{job="job1",instance="ins1"} + ignoring(job,instance) request_duration_seconds{job="job1",instance="ins2"} + ignoring(job,instance) request_duration_seconds{job="job2",instance="ins1"} + ignoring(job,instance) request_duration_seconds{job="job2",instance="ins2"}) {} {{schema:-53 count:250 custom_values:[0.1 0.2] buckets:[100 90 60]}} eval instant at 50m avg(request_duration_seconds) {} {{schema:-53 count:62.5 custom_values:[0.1 0.2] buckets:[25 22.5 15]}} # To verify the result above, calculate from classic histogram as well. eval instant at 50m avg (request_duration_seconds_bucket{le="0.1"}) {} 25 eval instant at 50m avg (request_duration_seconds_bucket{le="0.2"}) - avg (request_duration_seconds_bucket{le="0.1"}) {} 22.5 eval instant at 50m avg (request_duration_seconds_bucket{le="+Inf"}) - avg (request_duration_seconds_bucket{le="0.2"}) {} 15 eval instant at 50m count(request_duration_seconds) {} 4 # A histogram with nonmonotonic bucket counts. This may happen when recording # rule evaluation or federation races scrape ingestion, causing some buckets # counts to be derived from fewer samples. load 5m nonmonotonic_bucket{le="0.1"} 0+2x10 nonmonotonic_bucket{le="1"} 0+1x10 nonmonotonic_bucket{le="10"} 0+5x10 nonmonotonic_bucket{le="100"} 0+4x10 nonmonotonic_bucket{le="1000"} 0+9x10 nonmonotonic_bucket{le="+Inf"} 0+8x10 # Nonmonotonic buckets, triggering an info annotation. eval_info instant at 50m histogram_quantile(0.01, nonmonotonic_bucket) {} 0.0045 eval_info instant at 50m histogram_quantile(0.5, nonmonotonic_bucket) {} 8.5 eval_info instant at 50m histogram_quantile(0.99, nonmonotonic_bucket) {} 979.75 # Buckets with different representations of the same upper bound. eval instant at 50m histogram_quantile(0.5, rate(mixed_bucket[10m])) {instance="ins1", job="job1"} 0.15 {instance="ins2", job="job1"} NaN eval instant at 50m histogram_quantile(0.5, rate(mixed[10m])) {instance="ins1", job="job1"} 0.2 {instance="ins2", job="job1"} NaN eval instant at 50m histogram_quantile(0.75, rate(mixed_bucket[10m])) {instance="ins1", job="job1"} 0.2 {instance="ins2", job="job1"} NaN eval instant at 50m histogram_quantile(1, rate(mixed_bucket[10m])) {instance="ins1", job="job1"} 0.2 {instance="ins2", job="job1"} NaN load_with_nhcb 5m empty_bucket{le="0.1", job="job1", instance="ins1"} 0x10 empty_bucket{le="0.2", job="job1", instance="ins1"} 0x10 empty_bucket{le="+Inf", job="job1", instance="ins1"} 0x10 eval instant at 50m histogram_quantile(0.2, rate(empty_bucket[10m])) {instance="ins1", job="job1"} NaN # Load a duplicate histogram with a different name to test failure scenario on multiple histograms with the same label set. # https://github.com/prometheus/prometheus/issues/9910 load_with_nhcb 5m request_duration_seconds2_bucket{job="job1", instance="ins1", le="0.1"} 0+1x10 request_duration_seconds2_bucket{job="job1", instance="ins1", le="0.2"} 0+3x10 request_duration_seconds2_bucket{job="job1", instance="ins1", le="+Inf"} 0+4x10 eval_fail instant at 50m histogram_quantile(0.99, {__name__=~"request_duration_seconds\\d*_bucket"}) eval_fail instant at 50m histogram_quantile(0.99, {__name__=~"request_duration_seconds\\d*"}) # Histogram with constant buckets. load_with_nhcb 1m const_histogram_bucket{le="0.0"} 1 1 1 1 1 const_histogram_bucket{le="1.0"} 1 1 1 1 1 const_histogram_bucket{le="2.0"} 1 1 1 1 1 const_histogram_bucket{le="+Inf"} 1 1 1 1 1 # There is no change to the bucket count over time, thus rate is 0 in each bucket. eval instant at 5m rate(const_histogram_bucket[5m]) {le="0.0"} 0 {le="1.0"} 0 {le="2.0"} 0 {le="+Inf"} 0 # Native histograms do not represent empty buckets, so here the zeros are implicit. eval instant at 5m rate(const_histogram[5m]) {} {{schema:-53 sum:0 count:0 custom_values:[0.0 1.0 2.0]}} # Zero buckets mean no observations, so there is no value that observations fall below, # which means that any quantile is a NaN. eval instant at 5m histogram_quantile(1.0, sum by (le) (rate(const_histogram_bucket[5m]))) {} NaN eval instant at 5m histogram_quantile(1.0, sum(rate(const_histogram[5m]))) {} NaN load_with_nhcb 1m histogram_over_time_bucket{le="0"} 0 1 3 9 histogram_over_time_bucket{le="1"} 2 3 3 9 histogram_over_time_bucket{le="2"} 3 8 5 10 histogram_over_time_bucket{le="4"} 3 10 6 18 # Test custom buckets with sum_over_time, avg_over_time. eval instant at 3m sum_over_time(histogram_over_time[4m:1m]) {} {{schema:-53 count:37 custom_values:[0 1 2 4] buckets:[13 4 9 11]}} eval instant at 3m avg_over_time(histogram_over_time[4m:1m]) {} {{schema:-53 count:9.25 custom_values:[0 1 2 4] buckets:[3.25 1 2.25 2.75]}} # Test custom buckets with counter reset load_with_nhcb 5m histogram_with_reset_bucket{le="1"} 1 3 9 histogram_with_reset_bucket{le="2"} 3 3 9 histogram_with_reset_bucket{le="4"} 8 5 12 histogram_with_reset_bucket{le="8"} 10 6 18 histogram_with_reset_sum{} 36 16 61 eval instant at 10m increase(histogram_with_reset[15m]) {} {{schema:-53 count:27 sum:91.5 custom_values:[1 2 4 8] counter_reset_hint:gauge buckets:[13.5 0 4.5 9]}} eval instant at 10m resets(histogram_with_reset[15m]) {} 1 eval instant at 10m histogram_count(increase(histogram_with_reset[15m])) {} 27 eval instant at 10m histogram_sum(increase(histogram_with_reset[15m])) {} 91.5