diff --git a/.circleci/config.yml b/.circleci/config.yml index d6fd08755..30564395a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,7 +14,7 @@ executors: - image: quay.io/prometheus/golang-builder:1.18-base golang_oldest: docker: - - image: quay.io/prometheus/golang-builder:1.17-base + - image: quay.io/prometheus/golang-builder:1.18-base jobs: test_go: diff --git a/go.mod b/go.mod index 98f9cb855..61f8afc7f 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/prometheus/prometheus -go 1.17 +go 1.18 require ( github.com/Azure/azure-sdk-for-go v65.0.0+incompatible @@ -65,7 +65,7 @@ require ( golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 golang.org/x/sys v0.0.0-20220731174439-a90be440212d golang.org/x/time v0.0.0-20220609170525-579cf78fd858 - golang.org/x/tools v0.1.12 + golang.org/x/tools v0.1.13-0.20220908144252-ce397412b6a4 google.golang.org/api v0.91.0 google.golang.org/genproto v0.0.0-20220802133213-ce4fa296bf78 google.golang.org/grpc v1.48.0 @@ -162,6 +162,7 @@ require ( github.com/opencontainers/image-spec v1.0.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect + github.com/rogpeppe/go-internal v1.8.1 // indirect github.com/sirupsen/logrus v1.8.1 // indirect github.com/spf13/pflag v1.0.5 // indirect go.mongodb.org/mongo-driver v1.10.0 // indirect diff --git a/go.sum b/go.sum index 3ae093c2b..3331adf27 100644 --- a/go.sum +++ b/go.sum @@ -26,7 +26,6 @@ cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A= -cloud.google.com/go v0.102.0 h1:DAq3r8y4mDgyB/ZPJ9v/5VJNqjgJAxTn6ZYLlUywOu8= cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+nc= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= @@ -133,24 +132,19 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= -github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4= github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/logex v1.2.0/go.mod h1:9+9sk7u7pGNWYMkh0hdiL++6OeibzJccyQU4p4MedaY= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/readline v1.5.0/go.mod h1:x22KAscuvRqlLoK9CsoYsmxoXZMMFVyOl86cAH8qUic= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/chzyer/test v0.0.0-20210722231415-061457976a23/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= @@ -186,14 +180,12 @@ github.com/dgryski/go-sip13 v0.0.0-20200911182023-62edffca9245/go.mod h1:vAd38F8 github.com/digitalocean/godo v1.82.0 h1:lqAit46H1CqJGjh7LDbsamng/UMBME5rvmfH3Vb5Yy8= github.com/digitalocean/godo v1.82.0/go.mod h1:BPCqvwbjbGqxuUnIKB4EvS/AX7IDnNmt5fwvIkWo+ew= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= -github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v20.10.17+incompatible h1:JYCuMrWaVNophQTOrMMoSwudOVEfcegoZZrleKc1xwE= github.com/docker/docker v20.10.17+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= -github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= @@ -535,7 +527,6 @@ github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmK github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20220319035150-800ac71e25c2/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= @@ -586,7 +577,6 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -637,7 +627,6 @@ github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/go-testing-interface v1.14.1 h1:jrgshOhYAUVNMAJiKbEu7EqAwgJJ2JqpQmpLJOu07cU= -github.com/mitchellh/go-testing-interface v1.14.1/go.mod h1:gfgS7OtZj6MA4U1UrDRp04twqAjfvlZyCfX3sDjEym8= github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= @@ -657,7 +646,6 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= @@ -723,6 +711,7 @@ github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -734,7 +723,6 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= -github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= github.com/prometheus/alertmanager v0.24.0 h1:HBWR3lk4uy3ys+naDZthDdV7yEsxpaNeZuUS+hJgrOw= github.com/prometheus/alertmanager v0.24.0/go.mod h1:r6fy/D7FRuZh5YbnX6J3MBY0eI4Pb5yPYS7/bPSXXqI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -785,8 +773,8 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= -github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.8.1 h1:geMPLpDpQOgVyCg5z5GoRwLHepNdb71NXb67XFkP+Eg= +github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= github.com/rs/cors v1.8.2/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= @@ -841,7 +829,6 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.5/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= @@ -866,7 +853,6 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.mongodb.org/mongo-driver v1.7.3/go.mod h1:NqaYOwnXWr5Pm7AOpO5QFxKJ503nbMse/R79oO62zWg= @@ -886,7 +872,6 @@ go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.34.0 h1:9NkMW03wwEzPtP/KciZ4Ozu/Uz5ZA7kfqXJIObnrjGU= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.34.0/go.mod h1:548ZsYzmT4PL4zWKRd8q/N4z0Wxzn/ZxUE+lkEpwWQA= -go.opentelemetry.io/otel v1.8.0/go.mod h1:2pkj+iMj0o03Y+cW6/m8Y4WkRdYN3AvCXCnzRMp9yvM= go.opentelemetry.io/otel v1.9.0 h1:8WZNQFIB2a71LnANS9JeyidJKKGOOremcUtb/OtHISw= go.opentelemetry.io/otel v1.9.0/go.mod h1:np4EoPGzoPs3O67xUVNoPPcmSvsfOxNlNA4F4AC+0Eo= go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.9.0 h1:ggqApEjDKczicksfvZUCxuvoyDmR6Sbm56LwiK8DVR0= @@ -901,7 +886,6 @@ go.opentelemetry.io/otel/metric v0.31.0 h1:6SiklT+gfWAwWUR0meEMxQBtihpiEs4c+vL9s go.opentelemetry.io/otel/metric v0.31.0/go.mod h1:ohmwj9KTSIeBnDBm/ZwH2PSZxZzoOaG2xZeekTRzL5A= go.opentelemetry.io/otel/sdk v1.9.0 h1:LNXp1vrr83fNXTHgU8eO89mhzxb/bbWAsHG6fNf3qWo= go.opentelemetry.io/otel/sdk v1.9.0/go.mod h1:AEZc8nt5bd2F7BC24J5R0mrjYnpEgYHyTcM/vrSple4= -go.opentelemetry.io/otel/trace v1.8.0/go.mod h1:0Bt3PXY8w+3pheS3hQUt+wow8b1ojPaTBoTCh2zIFI4= go.opentelemetry.io/otel/trace v1.9.0 h1:oZaCNJUjWcg60VXWee8lJKlqhPbXAPB51URuR47pQYc= go.opentelemetry.io/otel/trace v1.9.0/go.mod h1:2737Q0MuG8q1uILYm2YYVkAyLtOofiTNGg6VODnOiPo= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= @@ -1042,7 +1026,6 @@ golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220802222814-0bcc04d9c69b h1:3ogNYyK4oIQdIKzTu68hQrr4iuVxF3AxKl9Aj/eDrw0= golang.org/x/net v0.0.0-20220802222814-0bcc04d9c69b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1149,7 +1132,6 @@ golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1170,7 +1152,6 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1178,8 +1159,6 @@ golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220731174439-a90be440212d h1:Sv5ogFZatcgIMMtBSTTAgMYsicp25MXBubjXNDKwm80= golang.org/x/sys v0.0.0-20220731174439-a90be440212d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -1270,8 +1249,8 @@ golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= -golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.1.13-0.20220908144252-ce397412b6a4 h1:glzimF7qHZuKVEiMbE7UqBu44MyTjt5u6j3Jz+rfMRM= +golang.org/x/tools v0.1.13-0.20220908144252-ce397412b6a4/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1409,7 +1388,6 @@ google.golang.org/genproto v0.0.0-20220523171625-347a074981d8/go.mod h1:RAyBrSAP google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= google.golang.org/genproto v0.0.0-20220624142145-8cd45d7dbd1f/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= -google.golang.org/genproto v0.0.0-20220728213248-dd149ef739b9/go.mod h1:iHe1svFLAZg9VWz891+QbRMwUv9O/1Ww+/mngYeThbc= google.golang.org/genproto v0.0.0-20220802133213-ce4fa296bf78 h1:QntLWYqZeuBtJkth3m/6DLznnI0AHJr+AgJXvVh/izw= google.golang.org/genproto v0.0.0-20220802133213-ce4fa296bf78/go.mod h1:iHe1svFLAZg9VWz891+QbRMwUv9O/1Ww+/mngYeThbc= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= diff --git a/model/histogram/compact.go b/model/histogram/compact.go new file mode 100644 index 000000000..96a377e08 --- /dev/null +++ b/model/histogram/compact.go @@ -0,0 +1,223 @@ +// Copyright 2022 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package histogram + +// compactBuckets is a generic function used by both Histogram.Compact and +// FloatHistogram.Compact. Set deltaBuckets to true if the provided buckets are +// deltas. Set it to false if the buckets contain absolute counts. +func compactBuckets[Bucket float64 | int64](buckets []Bucket, spans []Span, maxEmptyBuckets int, deltaBuckets bool) ([]Bucket, []Span) { + // Fast path: If there are no empty buckets AND no offset in any span is + // <= maxEmptyBuckets AND no span has length 0, there is nothing to do and we can return + // immediately. We check that first because it's cheap and presumably + // common. + nothingToDo := true + var currentBucketAbsolute Bucket + for _, bucket := range buckets { + if deltaBuckets { + currentBucketAbsolute += bucket + } else { + currentBucketAbsolute = bucket + } + if currentBucketAbsolute == 0 { + nothingToDo = false + break + } + } + if nothingToDo { + for _, span := range spans { + if int(span.Offset) <= maxEmptyBuckets || span.Length == 0 { + nothingToDo = false + break + } + } + if nothingToDo { + return buckets, spans + } + } + + var iBucket, iSpan int + var posInSpan uint32 + currentBucketAbsolute = 0 + + // Helper function. + emptyBucketsHere := func() int { + i := 0 + abs := currentBucketAbsolute + for uint32(i)+posInSpan < spans[iSpan].Length && abs == 0 { + i++ + if i+iBucket >= len(buckets) { + break + } + abs = buckets[i+iBucket] + } + return i + } + + // Merge spans with zero-offset to avoid special cases later. + if len(spans) > 1 { + for i, span := range spans[1:] { + if span.Offset == 0 { + spans[iSpan].Length += span.Length + continue + } + iSpan++ + if i+1 != iSpan { + spans[iSpan] = span + } + } + spans = spans[:iSpan+1] + iSpan = 0 + } + + // Merge spans with zero-length to avoid special cases later. + for i, span := range spans { + if span.Length == 0 { + if i+1 < len(spans) { + spans[i+1].Offset += span.Offset + } + continue + } + if i != iSpan { + spans[iSpan] = span + } + iSpan++ + } + spans = spans[:iSpan] + iSpan = 0 + + // Cut out empty buckets from start and end of spans, no matter + // what. Also cut out empty buckets from the middle of a span but only + // if there are more than maxEmptyBuckets consecutive empty buckets. + for iBucket < len(buckets) { + if deltaBuckets { + currentBucketAbsolute += buckets[iBucket] + } else { + currentBucketAbsolute = buckets[iBucket] + } + if nEmpty := emptyBucketsHere(); nEmpty > 0 { + if posInSpan > 0 && + nEmpty < int(spans[iSpan].Length-posInSpan) && + nEmpty <= maxEmptyBuckets { + // The empty buckets are in the middle of a + // span, and there are few enough to not bother. + // Just fast-forward. + iBucket += nEmpty + if deltaBuckets { + currentBucketAbsolute = 0 + } + posInSpan += uint32(nEmpty) + continue + } + // In all other cases, we cut out the empty buckets. + if deltaBuckets && iBucket+nEmpty < len(buckets) { + currentBucketAbsolute = -buckets[iBucket] + buckets[iBucket+nEmpty] += buckets[iBucket] + } + buckets = append(buckets[:iBucket], buckets[iBucket+nEmpty:]...) + if posInSpan == 0 { + // Start of span. + if nEmpty == int(spans[iSpan].Length) { + // The whole span is empty. + offset := spans[iSpan].Offset + spans = append(spans[:iSpan], spans[iSpan+1:]...) + if len(spans) > iSpan { + spans[iSpan].Offset += offset + int32(nEmpty) + } + continue + } + spans[iSpan].Length -= uint32(nEmpty) + spans[iSpan].Offset += int32(nEmpty) + continue + } + // It's in the middle or in the end of the span. + // Split the current span. + newSpan := Span{ + Offset: int32(nEmpty), + Length: spans[iSpan].Length - posInSpan - uint32(nEmpty), + } + spans[iSpan].Length = posInSpan + // In any case, we have to split to the next span. + iSpan++ + posInSpan = 0 + if newSpan.Length == 0 { + // The span is empty, so we were already at the end of a span. + // We don't have to insert the new span, just adjust the next + // span's offset, if there is one. + if iSpan < len(spans) { + spans[iSpan].Offset += int32(nEmpty) + } + continue + } + // Insert the new span. + spans = append(spans, Span{}) + if iSpan+1 < len(spans) { + copy(spans[iSpan+1:], spans[iSpan:]) + } + spans[iSpan] = newSpan + continue + } + iBucket++ + posInSpan++ + if posInSpan >= spans[iSpan].Length { + posInSpan = 0 + iSpan++ + } + } + if maxEmptyBuckets == 0 || len(buckets) == 0 { + return buckets, spans + } + + // Finally, check if any offsets between spans are small enough to merge + // the spans. + iBucket = int(spans[0].Length) + if deltaBuckets { + currentBucketAbsolute = 0 + for _, bucket := range buckets[:iBucket] { + currentBucketAbsolute += bucket + } + } + iSpan = 1 + for iSpan < len(spans) { + if int(spans[iSpan].Offset) > maxEmptyBuckets { + l := int(spans[iSpan].Length) + if deltaBuckets { + for _, bucket := range buckets[iBucket : iBucket+l] { + currentBucketAbsolute += bucket + } + } + iBucket += l + iSpan++ + continue + } + // Merge span with previous one and insert empty buckets. + offset := int(spans[iSpan].Offset) + spans[iSpan-1].Length += uint32(offset) + spans[iSpan].Length + spans = append(spans[:iSpan], spans[iSpan+1:]...) + newBuckets := make([]Bucket, len(buckets)+offset) + copy(newBuckets, buckets[:iBucket]) + copy(newBuckets[iBucket+offset:], buckets[iBucket:]) + if deltaBuckets { + newBuckets[iBucket] = -currentBucketAbsolute + newBuckets[iBucket+offset] += currentBucketAbsolute + } + iBucket += offset + buckets = newBuckets + currentBucketAbsolute = buckets[iBucket] + // Note that with many merges, it would be more efficient to + // first record all the chunks of empty buckets to insert and + // then do it in one go through all the buckets. + } + + return buckets, spans +} diff --git a/model/histogram/float_histogram.go b/model/histogram/float_histogram.go index af1bc0cdf..a0aef7f38 100644 --- a/model/histogram/float_histogram.go +++ b/model/histogram/float_histogram.go @@ -348,164 +348,34 @@ func addBucket( // maxEmptyBuckets. (The actual implementation might do something more efficient // but with the same result.) The compaction happens "in place" in the // receiving histogram, but a pointer to it is returned for convenience. +// +// The ideal value for maxEmptyBuckets depends on circumstances. The motivation +// to set maxEmptyBuckets > 0 is the assumption that is is less overhead to +// represent very few empty buckets explicitly within one span than cutting the +// one span into two to treat the empty buckets as a gap between the two spans, +// both in terms of storage requirement as well as in terms of encoding and +// decoding effort. However, the tradeoffs are subtle. For one, they are +// different in the exposition format vs. in a TSDB chunk vs. for the in-memory +// representation as Go types. In the TSDB, as an additional aspects, the span +// layout is only stored once per chunk, while many histograms with that same +// chunk layout are then only stored with their buckets (so that even a single +// empty bucket will be stored many times). +// +// For the Go types, an additional Span takes 8 bytes. Similarly, an additional +// bucket takes 8 bytes. Therefore, with a single separating empty bucket, both +// options have the same storage requirement, but the single-span solution is +// easier to iterate through. Still, the safest bet is to use maxEmptyBuckets==0 +// and only use a larger number if you know what you are doing. func (h *FloatHistogram) Compact(maxEmptyBuckets int) *FloatHistogram { h.PositiveBuckets, h.PositiveSpans = compactBuckets( - h.PositiveBuckets, h.PositiveSpans, maxEmptyBuckets, + h.PositiveBuckets, h.PositiveSpans, maxEmptyBuckets, false, ) h.NegativeBuckets, h.NegativeSpans = compactBuckets( - h.NegativeBuckets, h.NegativeSpans, maxEmptyBuckets, + h.NegativeBuckets, h.NegativeSpans, maxEmptyBuckets, false, ) return h } -func compactBuckets(buckets []float64, spans []Span, maxEmptyBuckets int) ([]float64, []Span) { - if len(buckets) == 0 { - return buckets, spans - } - - var iBucket, iSpan int - var posInSpan uint32 - - // Helper function. - emptyBucketsHere := func() int { - i := 0 - for i+iBucket < len(buckets) && - uint32(i)+posInSpan < spans[iSpan].Length && - buckets[i+iBucket] == 0 { - i++ - } - return i - } - - // Merge spans with zero-offset to avoid special cases later. - if len(spans) > 1 { - for i, span := range spans[1:] { - if span.Offset == 0 { - spans[iSpan].Length += span.Length - continue - } - iSpan++ - if i+1 != iSpan { - spans[iSpan] = span - } - } - spans = spans[:iSpan+1] - iSpan = 0 - } - - // Merge spans with zero-length to avoid special cases later. - for i, span := range spans { - if span.Length == 0 { - if i+1 < len(spans) { - spans[i+1].Offset += span.Offset - } - continue - } - if i != iSpan { - spans[iSpan] = span - } - iSpan++ - } - spans = spans[:iSpan] - iSpan = 0 - - // Cut out empty buckets from start and end of spans, no matter - // what. Also cut out empty buckets from the middle of a span but only - // if there are more than maxEmptyBuckets consecutive empty buckets. - for iBucket < len(buckets) { - if nEmpty := emptyBucketsHere(); nEmpty > 0 { - if posInSpan > 0 && - nEmpty < int(spans[iSpan].Length-posInSpan) && - nEmpty <= maxEmptyBuckets { - // The empty buckets are in the middle of a - // span, and there are few enough to not bother. - // Just fast-forward. - iBucket += nEmpty - posInSpan += uint32(nEmpty) - continue - } - // In all other cases, we cut out the empty buckets. - buckets = append(buckets[:iBucket], buckets[iBucket+nEmpty:]...) - if posInSpan == 0 { - // Start of span. - if nEmpty == int(spans[iSpan].Length) { - // The whole span is empty. - offset := spans[iSpan].Offset - spans = append(spans[:iSpan], spans[iSpan+1:]...) - if len(spans) > iSpan { - spans[iSpan].Offset += offset + int32(nEmpty) - } - continue - } - spans[iSpan].Length -= uint32(nEmpty) - spans[iSpan].Offset += int32(nEmpty) - continue - } - // It's in the middle or in the end of the span. - // Split the current span. - newSpan := Span{ - Offset: int32(nEmpty), - Length: spans[iSpan].Length - posInSpan - uint32(nEmpty), - } - spans[iSpan].Length = posInSpan - // In any case, we have to split to the next span. - iSpan++ - posInSpan = 0 - if newSpan.Length == 0 { - // The span is empty, so we were already at the end of a span. - // We don't have to insert the new span, just adjust the next - // span's offset, if there is one. - if iSpan < len(spans) { - spans[iSpan].Offset += int32(nEmpty) - } - continue - } - // Insert the new span. - spans = append(spans, Span{}) - if iSpan+1 < len(spans) { - copy(spans[iSpan+1:], spans[iSpan:]) - } - spans[iSpan] = newSpan - continue - } - iBucket++ - posInSpan++ - if posInSpan >= spans[iSpan].Length { - posInSpan = 0 - iSpan++ - } - } - if maxEmptyBuckets == 0 || len(buckets) == 0 { - return buckets, spans - } - - // Finally, check if any offsets between spans are small enough to merge - // the spans. - iBucket = int(spans[0].Length) - iSpan = 1 - for iSpan < len(spans) { - if int(spans[iSpan].Offset) > maxEmptyBuckets { - iBucket += int(spans[iSpan].Length) - iSpan++ - continue - } - // Merge span with previous one and insert empty buckets. - offset := int(spans[iSpan].Offset) - spans[iSpan-1].Length += uint32(offset) + spans[iSpan].Length - spans = append(spans[:iSpan], spans[iSpan+1:]...) - newBuckets := make([]float64, len(buckets)+offset) - copy(newBuckets, buckets[:iBucket]) - copy(newBuckets[iBucket+offset:], buckets[iBucket:]) - iBucket += offset - buckets = newBuckets - // Note that with many merges, it would be more efficient to - // first record all the chunks of empty buckets to insert and - // then do it in one go through all the buckets. - } - - return buckets, spans -} - // DetectReset returns true if the receiving histogram is missing any buckets // that have a non-zero population in the provided previous histogram. It also // returns true if any count (in any bucket, in the zero count, or in the count @@ -515,21 +385,21 @@ func compactBuckets(buckets []float64, spans []Span, maxEmptyBuckets int) ([]flo // Special behavior in case the Schema or the ZeroThreshold are not the same in // both histograms: // -// * A decrease of the ZeroThreshold or an increase of the Schema (i.e. an -// increase of resolution) can only happen together with a reset. Thus, the -// method returns true in either case. +// - A decrease of the ZeroThreshold or an increase of the Schema (i.e. an +// increase of resolution) can only happen together with a reset. Thus, the +// method returns true in either case. // -// * Upon an increase of the ZeroThreshold, the buckets in the previous -// histogram that fall within the new ZeroThreshold are added to the ZeroCount -// of the previous histogram (without mutating the provided previous -// histogram). The scenario that a populated bucket of the previous histogram -// is partially within, partially outside of the new ZeroThreshold, can only -// happen together with a counter reset and therefore shortcuts to returning -// true. +// - Upon an increase of the ZeroThreshold, the buckets in the previous +// histogram that fall within the new ZeroThreshold are added to the ZeroCount +// of the previous histogram (without mutating the provided previous +// histogram). The scenario that a populated bucket of the previous histogram +// is partially within, partially outside of the new ZeroThreshold, can only +// happen together with a counter reset and therefore shortcuts to returning +// true. // -// * Upon a decrease of the Schema, the buckets of the previous histogram are -// merged so that they match the new, lower-resolution schema (again without -// mutating the provided previous histogram). +// - Upon a decrease of the Schema, the buckets of the previous histogram are +// merged so that they match the new, lower-resolution schema (again without +// mutating the provided previous histogram). // // Note that this kind of reset detection is quite expensive. Ideally, resets // are detected at ingest time and stored in the TSDB, so that the reset @@ -764,7 +634,7 @@ func (h *FloatHistogram) trimBucketsInZeroBucket() { // We are abusing Compact to trim the buckets set to zero // above. Premature compacting could cause additional cost, but this // code path is probably rarely used anyway. - h.Compact(3) + h.Compact(0) } // reconcileZeroBuckets finds a zero bucket large enough to include the zero diff --git a/model/histogram/float_histogram_test.go b/model/histogram/float_histogram_test.go index 2b49b5b71..bdf9839bc 100644 --- a/model/histogram/float_histogram_test.go +++ b/model/histogram/float_histogram_test.go @@ -938,7 +938,7 @@ func TestFloatHistogramCompact(t *testing.T) { for _, c := range cases { t.Run(c.name, func(t *testing.T) { require.Equal(t, c.expected, c.in.Compact(c.maxEmptyBuckets)) - // Has it also happened in-place? + // Compact has happened in-place, too. require.Equal(t, c.expected, c.in) }) } @@ -1244,8 +1244,8 @@ func TestFloatHistogramAdd(t *testing.T) { Sum: 3.579, PositiveSpans: []Span{{1, 5}}, PositiveBuckets: []float64{2, 6, 10, 9, 5}, - NegativeSpans: []Span{{3, 7}}, - NegativeBuckets: []float64{3, 2, 1, 0, 4, 9, 6}, + NegativeSpans: []Span{{3, 3}, {1, 3}}, + NegativeBuckets: []float64{3, 2, 1, 4, 9, 6}, }, }, { @@ -1277,8 +1277,8 @@ func TestFloatHistogramAdd(t *testing.T) { Sum: 3.579, PositiveSpans: []Span{{-1, 1}, {1, 5}}, PositiveBuckets: []float64{0, 2, 6, 10, 9, 5}, - NegativeSpans: []Span{{3, 7}}, - NegativeBuckets: []float64{3, 2, 1, 0, 4, 9, 6}, + NegativeSpans: []Span{{3, 3}, {1, 3}}, + NegativeBuckets: []float64{3, 2, 1, 4, 9, 6}, }, }, { @@ -1310,8 +1310,8 @@ func TestFloatHistogramAdd(t *testing.T) { Sum: 3.579, PositiveSpans: []Span{{1, 5}}, PositiveBuckets: []float64{2, 6, 10, 9, 5}, - NegativeSpans: []Span{{3, 7}}, - NegativeBuckets: []float64{3, 2, 1, 0, 4, 9, 6}, + NegativeSpans: []Span{{3, 3}, {1, 3}}, + NegativeBuckets: []float64{3, 2, 1, 4, 9, 6}, }, }, { @@ -1345,8 +1345,8 @@ func TestFloatHistogramAdd(t *testing.T) { Sum: 3.579, PositiveSpans: []Span{{-1, 7}}, PositiveBuckets: []float64{6, 4, 2, 6, 10, 9, 5}, - NegativeSpans: []Span{{3, 7}}, - NegativeBuckets: []float64{3, 2, 1, 0, 4, 9, 6}, + NegativeSpans: []Span{{3, 3}, {1, 3}}, + NegativeBuckets: []float64{3, 2, 1, 4, 9, 6}, }, }, { diff --git a/model/histogram/histogram.go b/model/histogram/histogram.go index 9eaf6e28e..b70381498 100644 --- a/model/histogram/histogram.go +++ b/model/histogram/histogram.go @@ -27,11 +27,11 @@ import ( // An example for schema 0 (by which each bucket is twice as wide as the // previous bucket): // -// Bucket boundaries → [-2,-1) [-1,-0.5) [-0.5,-0.25) ... [-0.001,0.001] ... (0.25,0.5] (0.5,1] (1,2] .... -// ↑ ↑ ↑ ↑ ↑ ↑ ↑ -// Zero bucket (width e.g. 0.001) → | | | ZB | | | -// Positive bucket indices → | | | ... -1 0 1 2 3 -// Negative bucket indices → 3 2 1 0 -1 ... +// Bucket boundaries → [-2,-1) [-1,-0.5) [-0.5,-0.25) ... [-0.001,0.001] ... (0.25,0.5] (0.5,1] (1,2] .... +// ↑ ↑ ↑ ↑ ↑ ↑ ↑ +// Zero bucket (width e.g. 0.001) → | | | ZB | | | +// Positive bucket indices → | | | ... -1 0 1 2 3 +// Negative bucket indices → 3 2 1 0 -1 ... // // Which bucket indices are actually used is determined by the spans. type Histogram struct { @@ -262,6 +262,18 @@ func bucketsMatch(b1, b2 []int64) bool { return true } +// Compact works like FloatHistogram.Compact. See there for detailed +// explanations. +func (h *Histogram) Compact(maxEmptyBuckets int) *Histogram { + h.PositiveBuckets, h.PositiveSpans = compactBuckets( + h.PositiveBuckets, h.PositiveSpans, maxEmptyBuckets, true, + ) + h.NegativeBuckets, h.NegativeSpans = compactBuckets( + h.NegativeBuckets, h.NegativeSpans, maxEmptyBuckets, true, + ) + return h +} + // ToFloat returns a FloatHistogram representation of the Histogram. It is a // deep copy (e.g. spans are not shared). func (h *Histogram) ToFloat() *FloatHistogram { diff --git a/model/histogram/histogram_test.go b/model/histogram/histogram_test.go index 3b136edce..6b502e426 100644 --- a/model/histogram/histogram_test.go +++ b/model/histogram/histogram_test.go @@ -524,3 +524,259 @@ func TestHistogramMatches(t *testing.T) { h2.NegativeBuckets = append(h2.NegativeBuckets, 1) require.False(t, h1.Equals(h2)) } + +func TestHistogramCompact(t *testing.T) { + cases := []struct { + name string + in *Histogram + maxEmptyBuckets int + expected *Histogram + }{ + { + "empty histogram", + &Histogram{}, + 0, + &Histogram{}, + }, + { + "nothing should happen", + &Histogram{ + PositiveSpans: []Span{{-2, 1}, {2, 3}}, + PositiveBuckets: []int64{1, 3, -3, 42}, + NegativeSpans: []Span{{3, 2}, {3, 2}}, + NegativeBuckets: []int64{5, 3, 1.234e5, 1000}, + }, + 0, + &Histogram{ + PositiveSpans: []Span{{-2, 1}, {2, 3}}, + PositiveBuckets: []int64{1, 3, -3, 42}, + NegativeSpans: []Span{{3, 2}, {3, 2}}, + NegativeBuckets: []int64{5, 3, 1.234e5, 1000}, + }, + }, + { + "eliminate zero offsets", + &Histogram{ + PositiveSpans: []Span{{-2, 1}, {0, 3}, {0, 1}}, + PositiveBuckets: []int64{1, 3, -3, 42, 3}, + NegativeSpans: []Span{{0, 2}, {0, 2}, {2, 1}, {0, 1}}, + NegativeBuckets: []int64{5, 3, 1.234e5, 1000, 3, 4}, + }, + 0, + &Histogram{ + PositiveSpans: []Span{{-2, 5}}, + PositiveBuckets: []int64{1, 3, -3, 42, 3}, + NegativeSpans: []Span{{0, 4}, {2, 2}}, + NegativeBuckets: []int64{5, 3, 1.234e5, 1000, 3, 4}, + }, + }, + { + "eliminate zero length", + &Histogram{ + PositiveSpans: []Span{{-2, 2}, {2, 0}, {3, 3}}, + PositiveBuckets: []int64{1, 3, -3, 42, 3}, + NegativeSpans: []Span{{0, 2}, {0, 0}, {2, 0}, {1, 4}}, + NegativeBuckets: []int64{5, 3, 1.234e5, 1000, 3, 4}, + }, + 0, + &Histogram{ + PositiveSpans: []Span{{-2, 2}, {5, 3}}, + PositiveBuckets: []int64{1, 3, -3, 42, 3}, + NegativeSpans: []Span{{0, 2}, {3, 4}}, + NegativeBuckets: []int64{5, 3, 1.234e5, 1000, 3, 4}, + }, + }, + { + "eliminate multiple zero length spans", + &Histogram{ + PositiveSpans: []Span{{-2, 2}, {2, 0}, {2, 0}, {2, 0}, {3, 3}}, + PositiveBuckets: []int64{1, 3, -3, 42, 3}, + }, + 0, + &Histogram{ + PositiveSpans: []Span{{-2, 2}, {9, 3}}, + PositiveBuckets: []int64{1, 3, -3, 42, 3}, + }, + }, + { + "cut empty buckets at start or end", + &Histogram{ + PositiveSpans: []Span{{-4, 4}, {5, 3}}, + PositiveBuckets: []int64{0, 0, 1, 3, -3, 42, 3}, + NegativeSpans: []Span{{0, 2}, {3, 5}}, + NegativeBuckets: []int64{5, 3, -4, -2, 3, 4, -9}, + }, + 0, + &Histogram{ + PositiveSpans: []Span{{-2, 2}, {5, 3}}, + PositiveBuckets: []int64{1, 3, -3, 42, 3}, + NegativeSpans: []Span{{0, 2}, {3, 4}}, + NegativeBuckets: []int64{5, 3, -4, -2, 3, 4}, + }, + }, + { + "cut empty buckets at start and end", + &Histogram{ + PositiveSpans: []Span{{-4, 4}, {5, 6}}, + PositiveBuckets: []int64{0, 0, 1, 3, -3, 42, 3, -46, 0, 0}, + NegativeSpans: []Span{{-2, 4}, {3, 5}}, + NegativeBuckets: []int64{0, 0, 5, 3, -4, -2, 3, 4, -9}, + }, + 0, + &Histogram{ + PositiveSpans: []Span{{-2, 2}, {5, 3}}, + PositiveBuckets: []int64{1, 3, -3, 42, 3}, + NegativeSpans: []Span{{0, 2}, {3, 4}}, + NegativeBuckets: []int64{5, 3, -4, -2, 3, 4}, + }, + }, + { + "cut empty buckets at start or end of spans, even in the middle", + &Histogram{ + PositiveSpans: []Span{{-4, 6}, {3, 6}}, + PositiveBuckets: []int64{0, 0, 1, 3, -4, 0, 1, 42, 3, -46, 0, 0}, + NegativeSpans: []Span{{0, 2}, {2, 6}}, + NegativeBuckets: []int64{5, 3, -8, 4, -2, 3, 4, -9}, + }, + 0, + &Histogram{ + PositiveSpans: []Span{{-2, 2}, {5, 3}}, + PositiveBuckets: []int64{1, 3, -3, 42, 3}, + NegativeSpans: []Span{{0, 2}, {3, 4}}, + NegativeBuckets: []int64{5, 3, -4, -2, 3, 4}, + }, + }, + { + "cut empty buckets at start or end but merge spans due to maxEmptyBuckets", + &Histogram{ + PositiveSpans: []Span{{-4, 4}, {5, 3}}, + PositiveBuckets: []int64{0, 0, 1, 3, -3, 42, 3}, + NegativeSpans: []Span{{0, 2}, {3, 5}}, + NegativeBuckets: []int64{5, 3, -4, -2, 3, 4, -9}, + }, + 10, + &Histogram{ + PositiveSpans: []Span{{-2, 10}}, + PositiveBuckets: []int64{1, 3, -4, 0, 0, 0, 0, 1, 42, 3}, + NegativeSpans: []Span{{0, 9}}, + NegativeBuckets: []int64{5, 3, -8, 0, 0, 4, -2, 3, 4}, + }, + }, + { + "cut empty buckets from the middle of a span", + &Histogram{ + PositiveSpans: []Span{{-4, 6}, {3, 3}}, + PositiveBuckets: []int64{0, 0, 1, -1, 0, 3, -2, 42, 3}, + NegativeSpans: []Span{{0, 2}, {3, 5}}, + NegativeBuckets: []int64{5, 3, -4, -2, -2, 3, 4}, + }, + 0, + &Histogram{ + PositiveSpans: []Span{{-2, 1}, {2, 1}, {3, 3}}, + PositiveBuckets: []int64{1, 2, -2, 42, 3}, + NegativeSpans: []Span{{0, 2}, {3, 2}, {1, 2}}, + NegativeBuckets: []int64{5, 3, -4, -2, 1, 4}, + }, + }, + { + "cut out a span containing only empty buckets", + &Histogram{ + PositiveSpans: []Span{{-4, 3}, {2, 2}, {3, 4}}, + PositiveBuckets: []int64{0, 0, 1, -1, 0, 3, -2, 42, 3}, + }, + 0, + &Histogram{ + PositiveSpans: []Span{{-2, 1}, {7, 4}}, + PositiveBuckets: []int64{1, 2, -2, 42, 3}, + }, + }, + { + "cut empty buckets from the middle of a span, avoiding some due to maxEmptyBuckets", + &Histogram{ + PositiveSpans: []Span{{-4, 6}, {3, 3}}, + PositiveBuckets: []int64{0, 0, 1, -1, 0, 3, -2, 42, 3}, + NegativeSpans: []Span{{0, 2}, {3, 5}}, + NegativeBuckets: []int64{5, 3, -4, -2, -2, 3, 4}, + }, + 1, + &Histogram{ + PositiveSpans: []Span{{-2, 1}, {2, 1}, {3, 3}}, + PositiveBuckets: []int64{1, 2, -2, 42, 3}, + NegativeSpans: []Span{{0, 2}, {3, 5}}, + NegativeBuckets: []int64{5, 3, -4, -2, -2, 3, 4}, + }, + }, + { + "avoiding all cutting of empty buckets from the middle of a chunk due to maxEmptyBuckets", + &Histogram{ + PositiveSpans: []Span{{-4, 6}, {3, 3}}, + PositiveBuckets: []int64{0, 0, 1, -1, 0, 3, -2, 42, 3}, + NegativeSpans: []Span{{0, 2}, {3, 5}}, + NegativeBuckets: []int64{5, 3, -4, -2, -2, 3, 4}, + }, + 2, + &Histogram{ + PositiveSpans: []Span{{-2, 4}, {3, 3}}, + PositiveBuckets: []int64{1, -1, 0, 3, -2, 42, 3}, + NegativeSpans: []Span{{0, 2}, {3, 5}}, + NegativeBuckets: []int64{5, 3, -4, -2, -2, 3, 4}, + }, + }, + { + "everything merged into one span due to maxEmptyBuckets", + &Histogram{ + PositiveSpans: []Span{{-4, 6}, {3, 3}}, + PositiveBuckets: []int64{0, 0, 1, -1, 0, 3, -2, 42, 3}, + NegativeSpans: []Span{{0, 2}, {3, 5}}, + NegativeBuckets: []int64{5, 3, -4, -2, -2, 3, 4}, + }, + 3, + &Histogram{ + PositiveSpans: []Span{{-2, 10}}, + PositiveBuckets: []int64{1, -1, 0, 3, -3, 0, 0, 1, 42, 3}, + NegativeSpans: []Span{{0, 10}}, + NegativeBuckets: []int64{5, 3, -8, 0, 0, 4, -2, -2, 3, 4}, + }, + }, + { + "only empty buckets and maxEmptyBuckets greater zero", + &Histogram{ + PositiveSpans: []Span{{-4, 6}, {3, 3}}, + PositiveBuckets: []int64{0, 0, 0, 0, 0, 0, 0, 0, 0}, + NegativeSpans: []Span{{0, 7}}, + NegativeBuckets: []int64{0, 0, 0, 0, 0, 0, 0}, + }, + 3, + &Histogram{ + PositiveSpans: []Span{}, + PositiveBuckets: []int64{}, + NegativeSpans: []Span{}, + NegativeBuckets: []int64{}, + }, + }, + { + "multiple spans of only empty buckets", + &Histogram{ + PositiveSpans: []Span{{-10, 2}, {2, 1}, {3, 3}}, + PositiveBuckets: []int64{0, 0, 0, 0, 2, 3}, + NegativeSpans: []Span{{-10, 2}, {2, 1}, {3, 3}}, + NegativeBuckets: []int64{2, 3, -5, 0, 0, 0}, + }, + 0, + &Histogram{ + PositiveSpans: []Span{{-1, 2}}, + PositiveBuckets: []int64{2, 3}, + NegativeSpans: []Span{{-10, 2}}, + NegativeBuckets: []int64{2, 3}, + }, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + require.Equal(t, c.expected, c.in.Compact(c.maxEmptyBuckets)) + // Compact has happened in-place, too. + require.Equal(t, c.expected, c.in) + }) + } +} diff --git a/model/textparse/protobufparse.go b/model/textparse/protobufparse.go index ffabadddc..a9c940879 100644 --- a/model/textparse/protobufparse.go +++ b/model/textparse/protobufparse.go @@ -132,9 +132,16 @@ func (p *ProtobufParser) Series() ([]byte, *int64, float64) { return p.metricBytes.Bytes(), nil, v } -// Histogram returns the bytes of a series with a native histogram as a -// value, the timestamp if set, and the native histogram in the current -// sample. +// Histogram returns the bytes of a series with a native histogram as a value, +// the timestamp if set, and the native histogram in the current sample. +// +// The Compact method is called before returning the Histogram (or FloatHistogram). +// +// If the SampleCountFloat or the ZeroCountFloat in the proto message is > 0, +// the histogram is parsed and returned as a FloatHistogram and nil is returned +// as the (integer) Histogram return value. Otherwise, it is parsed and returned +// as an (integer) Histogram and nil is returned as the FloatHistogram return +// value. func (p *ProtobufParser) Histogram() ([]byte, *int64, *histogram.Histogram, *histogram.FloatHistogram) { var ( m = p.mf.GetMetric()[p.metricPos] @@ -162,6 +169,7 @@ func (p *ProtobufParser) Histogram() ([]byte, *int64, *histogram.Histogram, *his fh.NegativeSpans[i].Offset = span.GetOffset() fh.NegativeSpans[i].Length = span.GetLength() } + fh.Compact(0) if ts != 0 { return p.metricBytes.Bytes(), &ts, nil, &fh } @@ -190,7 +198,7 @@ func (p *ProtobufParser) Histogram() ([]byte, *int64, *histogram.Histogram, *his sh.NegativeSpans[i].Offset = span.GetOffset() sh.NegativeSpans[i].Length = span.GetLength() } - + sh.Compact(0) if ts != 0 { return p.metricBytes.Bytes(), &ts, &sh, nil } diff --git a/promql/engine_test.go b/promql/engine_test.go index f7f9ad69c..4d084348a 100644 --- a/promql/engine_test.go +++ b/promql/engine_test.go @@ -3159,8 +3159,8 @@ func TestSparseHistogramRate(t *testing.T) { ZeroCount: 1. / 15., Count: 4. / 15., Sum: 1.226666666666667, - PositiveSpans: []histogram.Span{{Offset: 0, Length: 5}}, - PositiveBuckets: []float64{1. / 15., 1. / 15., 0, 1. / 15., 1. / 15.}, + PositiveSpans: []histogram.Span{{Offset: 0, Length: 2}, {Offset: 1, Length: 2}}, + PositiveBuckets: []float64{1. / 15., 1. / 15., 1. / 15., 1. / 15.}, } require.Equal(t, expectedHistogram, actualHistogram) } diff --git a/promql/functions.go b/promql/functions.go index c8bd5aaa2..dd148458d 100644 --- a/promql/functions.go +++ b/promql/functions.go @@ -31,18 +31,23 @@ import ( // FunctionCall is the type of a PromQL function implementation // -// vals is a list of the evaluated arguments for the function call. -// For range vectors it will be a Matrix with one series, instant vectors a -// Vector, scalars a Vector with one series whose value is the scalar -// value,and nil for strings. +// vals is a list of the evaluated arguments for the function call. For range +// vectors it will be a Matrix with one series, instant vectors a Vector, +// scalars a Vector with one series whose value is the scalar value,and nil for +// strings. +// // args are the original arguments to the function, where you can access -// matrixSelectors, vectorSelectors, and StringLiterals. -// enh.Out is a pre-allocated empty vector that you may use to accumulate -// output before returning it. The vectors in vals should not be returned.a -// Range vector functions need only return a vector with the right value, -// the metric and timestamp are not needed. +// matrixSelectors, vectorSelectors, and StringLiterals. +// +// enh.Out is a pre-allocated empty vector that you may use to accumulate output +// before returning it. The vectors in vals should not be returned. +// +// Range vector functions need only return a vector with the right value, the +// metric and timestamp are not needed. +// // Instant vector functions need only return a vector with the right values and -// metrics, the timestamp are not needed. +// metrics, the timestamp are not needed. +// // Scalar results should be returned as the value of a sample in a Vector. type FunctionCall func(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector @@ -159,9 +164,7 @@ func extrapolatedRate(vals []parser.Value, args parser.Expressions, enh *EvalNod // histogramRate is a helper function for extrapolatedRate. It requires // points[0] to be a histogram. It returns nil if any other Point in points is -// not a histogram. Currently, it also returns nil on mixed schemas or zero -// thresholds in the histograms, because it cannot handle those schema changes -// yet. +// not a histogram. func histogramRate(points []Point, isCounter bool) *histogram.FloatHistogram { prev := points[0].H // We already know that this is a histogram. last := points[len(points)-1].H @@ -204,7 +207,7 @@ func histogramRate(points []Point, isCounter bool) *histogram.FloatHistogram { prev = curr } } - return h.Compact(3) + return h.Compact(0) } // === delta(Matrix parser.ValueTypeMatrix) Vector ===