From fe7e91e2ebd8d1af63d0d144eb759b77f4c6afe2 Mon Sep 17 00:00:00 2001 From: Fabian Reinartz Date: Mon, 15 Feb 2016 15:22:57 +0100 Subject: [PATCH] Make scraping offset consistent. To evenly distribute scraping load we currently rely on random jittering. This commit hashes over the target's identity and calculates a consistent offset. This also ensures that scrape intervals are constantly spaced between config/target changes. --- retrieval/target.go | 31 ++++++++++-- retrieval/target_test.go | 46 ++++++++++++++++++ web/ui/bindata.go | 102 +++++++++++++++++++-------------------- 3 files changed, 123 insertions(+), 56 deletions(-) diff --git a/retrieval/target.go b/retrieval/target.go index 7c470e0bc..a706c9cf6 100644 --- a/retrieval/target.go +++ b/retrieval/target.go @@ -18,7 +18,6 @@ import ( "fmt" "io" "io/ioutil" - "math/rand" "net/http" "net/url" "strings" @@ -267,6 +266,30 @@ func (t *Target) String() string { return t.host() } +// fingerprint returns an identifying hash for the target. +func (t *Target) fingerprint() model.Fingerprint { + t.RLock() + defer t.RUnlock() + + return t.labels.Fingerprint() +} + +// offset returns the time until the next scrape cycle for the target. +func (t *Target) offset(interval time.Duration) time.Duration { + now := time.Now().UnixNano() + + var ( + base = now % int64(interval) + offset = uint64(t.fingerprint()) % uint64(interval) + next = base + int64(offset) + ) + + if next > int64(interval) { + next -= int64(interval) + } + return time.Duration(next) +} + func (t *Target) client() (*http.Client, error) { t.RLock() defer t.RUnlock() @@ -366,14 +389,12 @@ func (t *Target) RunScraper(sampleAppender storage.SampleAppender) { log.Debugf("Starting scraper for target %v...", t) - jitterTimer := time.NewTimer(time.Duration(float64(lastScrapeInterval) * rand.Float64())) select { - case <-jitterTimer.C: + case <-time.After(t.offset(lastScrapeInterval)): + // Continue after scraping offset. case <-t.scraperStopping: - jitterTimer.Stop() return } - jitterTimer.Stop() ticker := time.NewTicker(lastScrapeInterval) defer ticker.Stop() diff --git a/retrieval/target_test.go b/retrieval/target_test.go index 3a0ee8857..4cef0d4ef 100644 --- a/retrieval/target_test.go +++ b/retrieval/target_test.go @@ -45,6 +45,52 @@ func TestTargetLabels(t *testing.T) { } } +func TestTargetOffset(t *testing.T) { + interval := 10 * time.Second + + offsets := make([]time.Duration, 10000) + + // Calculate offsets for 10000 different targets. + for i := range offsets { + target := newTestTarget("example.com:80", 0, model.LabelSet{ + "label": model.LabelValue(fmt.Sprintf("%d", i)), + }) + offsets[i] = target.offset(interval) + } + + // Put the offsets into buckets and validate that they are all + // within bounds. + bucketSize := 1 * time.Second + buckets := make([]int, interval/bucketSize) + + for _, offset := range offsets { + if offset < 0 || offset >= interval { + t.Fatalf("Offset %v out of bounds", offset) + } + + bucket := offset / bucketSize + buckets[bucket]++ + } + + t.Log(buckets) + + // Calculate whether the the number of targets per bucket + // does not differ more than a given tolerance. + avg := len(offsets) / len(buckets) + tolerance := 0.15 + + for _, bucket := range buckets { + diff := bucket - avg + if diff < 0 { + diff = -diff + } + + if float64(diff)/float64(avg) > tolerance { + t.Fatalf("Bucket out of tolerance bounds") + } + } +} + func TestOverwriteLabels(t *testing.T) { type test struct { metric string diff --git a/web/ui/bindata.go b/web/ui/bindata.go index 6db3b1018..43c15e3b7 100644 --- a/web/ui/bindata.go +++ b/web/ui/bindata.go @@ -791,38 +791,38 @@ func AssetNames() []string { // _bindata is a table, holding each asset generator, mapped to its name. var _bindata = map[string]func() (*asset, error){ - "web/ui/templates/_base.html": webUiTemplates_baseHtml, - "web/ui/templates/alerts.html": webUiTemplatesAlertsHtml, - "web/ui/templates/graph.html": webUiTemplatesGraphHtml, - "web/ui/templates/status.html": webUiTemplatesStatusHtml, - "web/ui/static/css/alerts.css": webUiStaticCssAlertsCss, - "web/ui/static/css/graph.css": webUiStaticCssGraphCss, - "web/ui/static/css/prom_console.css": webUiStaticCssProm_consoleCss, - "web/ui/static/css/prometheus.css": webUiStaticCssPrometheusCss, - "web/ui/static/img/ajax-loader.gif": webUiStaticImgAjaxLoaderGif, - "web/ui/static/js/alerts.js": webUiStaticJsAlertsJs, - "web/ui/static/js/graph.js": webUiStaticJsGraphJs, - "web/ui/static/js/graph_template.handlebar": webUiStaticJsGraph_templateHandlebar, - "web/ui/static/js/prom_console.js": webUiStaticJsProm_consoleJs, - "web/ui/static/vendor/bootstrap-3.3.1/css/bootstrap-theme.min.css": webUiStaticVendorBootstrap331CssBootstrapThemeMinCss, - "web/ui/static/vendor/bootstrap-3.3.1/css/bootstrap.min.css": webUiStaticVendorBootstrap331CssBootstrapMinCss, - "web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.eot": webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularEot, - "web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.svg": webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularSvg, - "web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.ttf": webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularTtf, - "web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.woff": webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularWoff, - "web/ui/static/vendor/bootstrap-3.3.1/js/bootstrap.min.js": webUiStaticVendorBootstrap331JsBootstrapMinJs, - "web/ui/static/vendor/bootstrap-3.3.1/js/npm.js": webUiStaticVendorBootstrap331JsNpmJs, - "web/ui/static/vendor/bootstrap-datetimepicker/bootstrap-datetimepicker.js": webUiStaticVendorBootstrapDatetimepickerBootstrapDatetimepickerJs, + "web/ui/templates/_base.html": webUiTemplates_baseHtml, + "web/ui/templates/alerts.html": webUiTemplatesAlertsHtml, + "web/ui/templates/graph.html": webUiTemplatesGraphHtml, + "web/ui/templates/status.html": webUiTemplatesStatusHtml, + "web/ui/static/css/alerts.css": webUiStaticCssAlertsCss, + "web/ui/static/css/graph.css": webUiStaticCssGraphCss, + "web/ui/static/css/prom_console.css": webUiStaticCssProm_consoleCss, + "web/ui/static/css/prometheus.css": webUiStaticCssPrometheusCss, + "web/ui/static/img/ajax-loader.gif": webUiStaticImgAjaxLoaderGif, + "web/ui/static/js/alerts.js": webUiStaticJsAlertsJs, + "web/ui/static/js/graph.js": webUiStaticJsGraphJs, + "web/ui/static/js/graph_template.handlebar": webUiStaticJsGraph_templateHandlebar, + "web/ui/static/js/prom_console.js": webUiStaticJsProm_consoleJs, + "web/ui/static/vendor/bootstrap-3.3.1/css/bootstrap-theme.min.css": webUiStaticVendorBootstrap331CssBootstrapThemeMinCss, + "web/ui/static/vendor/bootstrap-3.3.1/css/bootstrap.min.css": webUiStaticVendorBootstrap331CssBootstrapMinCss, + "web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.eot": webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularEot, + "web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.svg": webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularSvg, + "web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.ttf": webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularTtf, + "web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.woff": webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularWoff, + "web/ui/static/vendor/bootstrap-3.3.1/js/bootstrap.min.js": webUiStaticVendorBootstrap331JsBootstrapMinJs, + "web/ui/static/vendor/bootstrap-3.3.1/js/npm.js": webUiStaticVendorBootstrap331JsNpmJs, + "web/ui/static/vendor/bootstrap-datetimepicker/bootstrap-datetimepicker.js": webUiStaticVendorBootstrapDatetimepickerBootstrapDatetimepickerJs, "web/ui/static/vendor/bootstrap-datetimepicker/bootstrap-datetimepicker.min.css": webUiStaticVendorBootstrapDatetimepickerBootstrapDatetimepickerMinCss, - "web/ui/static/vendor/bootstrap3-typeahead/bootstrap3-typeahead.min.js": webUiStaticVendorBootstrap3TypeaheadBootstrap3TypeaheadMinJs, - "web/ui/static/vendor/js/handlebars.js": webUiStaticVendorJsHandlebarsJs, - "web/ui/static/vendor/js/jquery.hotkeys.js": webUiStaticVendorJsJqueryHotkeysJs, - "web/ui/static/vendor/js/jquery.min.js": webUiStaticVendorJsJqueryMinJs, - "web/ui/static/vendor/js/jquery.selection.js": webUiStaticVendorJsJquerySelectionJs, - "web/ui/static/vendor/rickshaw/rickshaw.min.css": webUiStaticVendorRickshawRickshawMinCss, - "web/ui/static/vendor/rickshaw/rickshaw.min.js": webUiStaticVendorRickshawRickshawMinJs, - "web/ui/static/vendor/rickshaw/vendor/d3.layout.min.js": webUiStaticVendorRickshawVendorD3LayoutMinJs, - "web/ui/static/vendor/rickshaw/vendor/d3.v3.js": webUiStaticVendorRickshawVendorD3V3Js, + "web/ui/static/vendor/bootstrap3-typeahead/bootstrap3-typeahead.min.js": webUiStaticVendorBootstrap3TypeaheadBootstrap3TypeaheadMinJs, + "web/ui/static/vendor/js/handlebars.js": webUiStaticVendorJsHandlebarsJs, + "web/ui/static/vendor/js/jquery.hotkeys.js": webUiStaticVendorJsJqueryHotkeysJs, + "web/ui/static/vendor/js/jquery.min.js": webUiStaticVendorJsJqueryMinJs, + "web/ui/static/vendor/js/jquery.selection.js": webUiStaticVendorJsJquerySelectionJs, + "web/ui/static/vendor/rickshaw/rickshaw.min.css": webUiStaticVendorRickshawRickshawMinCss, + "web/ui/static/vendor/rickshaw/rickshaw.min.js": webUiStaticVendorRickshawRickshawMinJs, + "web/ui/static/vendor/rickshaw/vendor/d3.layout.min.js": webUiStaticVendorRickshawVendorD3LayoutMinJs, + "web/ui/static/vendor/rickshaw/vendor/d3.v3.js": webUiStaticVendorRickshawVendorD3V3Js, } // AssetDir returns the file names below a certain @@ -864,69 +864,70 @@ type bintree struct { Func func() (*asset, error) Children map[string]*bintree } + var _bintree = &bintree{nil, map[string]*bintree{ "web": &bintree{nil, map[string]*bintree{ "ui": &bintree{nil, map[string]*bintree{ "static": &bintree{nil, map[string]*bintree{ "css": &bintree{nil, map[string]*bintree{ - "alerts.css": &bintree{webUiStaticCssAlertsCss, map[string]*bintree{}}, - "graph.css": &bintree{webUiStaticCssGraphCss, map[string]*bintree{}}, + "alerts.css": &bintree{webUiStaticCssAlertsCss, map[string]*bintree{}}, + "graph.css": &bintree{webUiStaticCssGraphCss, map[string]*bintree{}}, "prom_console.css": &bintree{webUiStaticCssProm_consoleCss, map[string]*bintree{}}, - "prometheus.css": &bintree{webUiStaticCssPrometheusCss, map[string]*bintree{}}, + "prometheus.css": &bintree{webUiStaticCssPrometheusCss, map[string]*bintree{}}, }}, "img": &bintree{nil, map[string]*bintree{ "ajax-loader.gif": &bintree{webUiStaticImgAjaxLoaderGif, map[string]*bintree{}}, }}, "js": &bintree{nil, map[string]*bintree{ - "alerts.js": &bintree{webUiStaticJsAlertsJs, map[string]*bintree{}}, - "graph.js": &bintree{webUiStaticJsGraphJs, map[string]*bintree{}}, + "alerts.js": &bintree{webUiStaticJsAlertsJs, map[string]*bintree{}}, + "graph.js": &bintree{webUiStaticJsGraphJs, map[string]*bintree{}}, "graph_template.handlebar": &bintree{webUiStaticJsGraph_templateHandlebar, map[string]*bintree{}}, - "prom_console.js": &bintree{webUiStaticJsProm_consoleJs, map[string]*bintree{}}, + "prom_console.js": &bintree{webUiStaticJsProm_consoleJs, map[string]*bintree{}}, }}, "vendor": &bintree{nil, map[string]*bintree{ "bootstrap-3.3.1": &bintree{nil, map[string]*bintree{ "css": &bintree{nil, map[string]*bintree{ "bootstrap-theme.min.css": &bintree{webUiStaticVendorBootstrap331CssBootstrapThemeMinCss, map[string]*bintree{}}, - "bootstrap.min.css": &bintree{webUiStaticVendorBootstrap331CssBootstrapMinCss, map[string]*bintree{}}, + "bootstrap.min.css": &bintree{webUiStaticVendorBootstrap331CssBootstrapMinCss, map[string]*bintree{}}, }}, "fonts": &bintree{nil, map[string]*bintree{ - "glyphicons-halflings-regular.eot": &bintree{webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularEot, map[string]*bintree{}}, - "glyphicons-halflings-regular.svg": &bintree{webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularSvg, map[string]*bintree{}}, - "glyphicons-halflings-regular.ttf": &bintree{webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularTtf, map[string]*bintree{}}, + "glyphicons-halflings-regular.eot": &bintree{webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularEot, map[string]*bintree{}}, + "glyphicons-halflings-regular.svg": &bintree{webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularSvg, map[string]*bintree{}}, + "glyphicons-halflings-regular.ttf": &bintree{webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularTtf, map[string]*bintree{}}, "glyphicons-halflings-regular.woff": &bintree{webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularWoff, map[string]*bintree{}}, }}, "js": &bintree{nil, map[string]*bintree{ "bootstrap.min.js": &bintree{webUiStaticVendorBootstrap331JsBootstrapMinJs, map[string]*bintree{}}, - "npm.js": &bintree{webUiStaticVendorBootstrap331JsNpmJs, map[string]*bintree{}}, + "npm.js": &bintree{webUiStaticVendorBootstrap331JsNpmJs, map[string]*bintree{}}, }}, }}, "bootstrap-datetimepicker": &bintree{nil, map[string]*bintree{ - "bootstrap-datetimepicker.js": &bintree{webUiStaticVendorBootstrapDatetimepickerBootstrapDatetimepickerJs, map[string]*bintree{}}, + "bootstrap-datetimepicker.js": &bintree{webUiStaticVendorBootstrapDatetimepickerBootstrapDatetimepickerJs, map[string]*bintree{}}, "bootstrap-datetimepicker.min.css": &bintree{webUiStaticVendorBootstrapDatetimepickerBootstrapDatetimepickerMinCss, map[string]*bintree{}}, }}, "bootstrap3-typeahead": &bintree{nil, map[string]*bintree{ "bootstrap3-typeahead.min.js": &bintree{webUiStaticVendorBootstrap3TypeaheadBootstrap3TypeaheadMinJs, map[string]*bintree{}}, }}, "js": &bintree{nil, map[string]*bintree{ - "handlebars.js": &bintree{webUiStaticVendorJsHandlebarsJs, map[string]*bintree{}}, - "jquery.hotkeys.js": &bintree{webUiStaticVendorJsJqueryHotkeysJs, map[string]*bintree{}}, - "jquery.min.js": &bintree{webUiStaticVendorJsJqueryMinJs, map[string]*bintree{}}, + "handlebars.js": &bintree{webUiStaticVendorJsHandlebarsJs, map[string]*bintree{}}, + "jquery.hotkeys.js": &bintree{webUiStaticVendorJsJqueryHotkeysJs, map[string]*bintree{}}, + "jquery.min.js": &bintree{webUiStaticVendorJsJqueryMinJs, map[string]*bintree{}}, "jquery.selection.js": &bintree{webUiStaticVendorJsJquerySelectionJs, map[string]*bintree{}}, }}, "rickshaw": &bintree{nil, map[string]*bintree{ "rickshaw.min.css": &bintree{webUiStaticVendorRickshawRickshawMinCss, map[string]*bintree{}}, - "rickshaw.min.js": &bintree{webUiStaticVendorRickshawRickshawMinJs, map[string]*bintree{}}, + "rickshaw.min.js": &bintree{webUiStaticVendorRickshawRickshawMinJs, map[string]*bintree{}}, "vendor": &bintree{nil, map[string]*bintree{ "d3.layout.min.js": &bintree{webUiStaticVendorRickshawVendorD3LayoutMinJs, map[string]*bintree{}}, - "d3.v3.js": &bintree{webUiStaticVendorRickshawVendorD3V3Js, map[string]*bintree{}}, + "d3.v3.js": &bintree{webUiStaticVendorRickshawVendorD3V3Js, map[string]*bintree{}}, }}, }}, }}, }}, "templates": &bintree{nil, map[string]*bintree{ - "_base.html": &bintree{webUiTemplates_baseHtml, map[string]*bintree{}}, + "_base.html": &bintree{webUiTemplates_baseHtml, map[string]*bintree{}}, "alerts.html": &bintree{webUiTemplatesAlertsHtml, map[string]*bintree{}}, - "graph.html": &bintree{webUiTemplatesGraphHtml, map[string]*bintree{}}, + "graph.html": &bintree{webUiTemplatesGraphHtml, map[string]*bintree{}}, "status.html": &bintree{webUiTemplatesStatusHtml, map[string]*bintree{}}, }}, }}, @@ -979,4 +980,3 @@ func _filePath(dir, name string) string { cannonicalName := strings.Replace(name, "\\", "/", -1) return filepath.Join(append([]string{dir}, strings.Split(cannonicalName, "/")...)...) } -