From fb69a38e671db03edff09e4d90c0f228251d67bb Mon Sep 17 00:00:00 2001 From: Ben Kochie Date: Mon, 11 Nov 2024 17:29:33 +0100 Subject: [PATCH] Enable auto-gomemlimit by default (#15373) * Enable auto-gomemlimit by default Enable the `auto-gomemlimit` feature flag by default. * Add command line flag `--no-auto-gomemlimit` to disable. --------- Signed-off-by: SuperQ --- cmd/prometheus/main.go | 16 ++++++++-------- docs/command-line/prometheus.md | 3 ++- docs/feature_flags.md | 8 -------- 3 files changed, 10 insertions(+), 17 deletions(-) diff --git a/cmd/prometheus/main.go b/cmd/prometheus/main.go index 8fb6d4d38..8b8f42d18 100644 --- a/cmd/prometheus/main.go +++ b/cmd/prometheus/main.go @@ -196,13 +196,14 @@ type flagConfig struct { enableAutoReload bool autoReloadInterval model.Duration - featureList []string - memlimitRatio float64 + memlimitEnable bool + memlimitRatio float64 + + featureList []string // These options are extracted from featureList // for ease of use. enablePerStepStats bool enableAutoGOMAXPROCS bool - enableAutoGOMEMLIMIT bool enableConcurrentRuleEval bool prometheusURL string @@ -243,9 +244,6 @@ func (c *flagConfig) setFeatureListOptions(logger *slog.Logger) error { c.autoReloadInterval, _ = model.ParseDuration("1s") } logger.Info("Enabled automatic configuration file reloading. Checking for configuration changes every", "interval", c.autoReloadInterval) - case "auto-gomemlimit": - c.enableAutoGOMEMLIMIT = true - logger.Info("Automatically set GOMEMLIMIT to match Linux container or system memory limit") case "concurrent-rule-eval": c.enableConcurrentRuleEval = true logger.Info("Experimental concurrent rule evaluation enabled.") @@ -332,6 +330,8 @@ func main() { a.Flag("web.listen-address", "Address to listen on for UI, API, and telemetry. Can be repeated."). Default("0.0.0.0:9090").StringsVar(&cfg.web.ListenAddresses) + a.Flag("auto-gomemlimit", "Automatically set GOMEMLIMIT to match Linux container or system memory limit"). + Default("true").BoolVar(&cfg.memlimitEnable) a.Flag("auto-gomemlimit.ratio", "The ratio of reserved GOMEMLIMIT memory to the detected maximum container or system memory"). Default("0.9").FloatVar(&cfg.memlimitRatio) @@ -515,7 +515,7 @@ func main() { a.Flag("scrape.discovery-reload-interval", "Interval used by scrape manager to throttle target groups updates."). Hidden().Default("5s").SetValue(&cfg.scrape.DiscoveryReloadInterval) - a.Flag("enable-feature", "Comma separated feature names to enable. Valid options: auto-gomemlimit, exemplar-storage, expand-external-labels, memory-snapshot-on-shutdown, promql-per-step-stats, promql-experimental-functions, extra-scrape-metrics, auto-gomaxprocs, native-histograms, created-timestamp-zero-ingestion, concurrent-rule-eval, delayed-compaction, old-ui. See https://prometheus.io/docs/prometheus/latest/feature_flags/ for more details."). + a.Flag("enable-feature", "Comma separated feature names to enable. Valid options: exemplar-storage, expand-external-labels, memory-snapshot-on-shutdown, promql-per-step-stats, promql-experimental-functions, extra-scrape-metrics, auto-gomaxprocs, native-histograms, created-timestamp-zero-ingestion, concurrent-rule-eval, delayed-compaction, old-ui. See https://prometheus.io/docs/prometheus/latest/feature_flags/ for more details."). Default("").StringsVar(&cfg.featureList) a.Flag("agent", "Run Prometheus in 'Agent mode'.").BoolVar(&agentMode) @@ -775,7 +775,7 @@ func main() { } } - if cfg.enableAutoGOMEMLIMIT { + if cfg.memlimitEnable { if _, err := memlimit.SetGoMemLimitWithOpts( memlimit.WithRatio(cfg.memlimitRatio), memlimit.WithProvider( diff --git a/docs/command-line/prometheus.md b/docs/command-line/prometheus.md index a179a2f9f..9f5400c41 100644 --- a/docs/command-line/prometheus.md +++ b/docs/command-line/prometheus.md @@ -17,6 +17,7 @@ The Prometheus monitoring server | --config.file | Prometheus configuration file path. | `prometheus.yml` | | --config.auto-reload-interval | Specifies the interval for checking and automatically reloading the Prometheus configuration file upon detecting changes. | `30s` | | --web.listen-address ... | Address to listen on for UI, API, and telemetry. Can be repeated. | `0.0.0.0:9090` | +| --auto-gomemlimit | Automatically set GOMEMLIMIT to match Linux container or system memory limit | `true` | | --auto-gomemlimit.ratio | The ratio of reserved GOMEMLIMIT memory to the detected maximum container or system memory | `0.9` | | --web.config.file | [EXPERIMENTAL] Path to configuration file that can enable TLS or authentication. | | | --web.read-timeout | Maximum duration before timing out read of the request, and closing idle connections. | `5m` | @@ -58,7 +59,7 @@ The Prometheus monitoring server | --query.timeout | Maximum time a query may take before being aborted. Use with server mode only. | `2m` | | --query.max-concurrency | Maximum number of queries executed concurrently. Use with server mode only. | `20` | | --query.max-samples | Maximum number of samples a single query can load into memory. Note that queries will fail if they try to load more samples than this into memory, so this also limits the number of samples a query can return. Use with server mode only. | `50000000` | -| --enable-feature ... | Comma separated feature names to enable. Valid options: auto-gomemlimit, exemplar-storage, expand-external-labels, memory-snapshot-on-shutdown, promql-per-step-stats, promql-experimental-functions, extra-scrape-metrics, auto-gomaxprocs, native-histograms, created-timestamp-zero-ingestion, concurrent-rule-eval, delayed-compaction, old-ui. See https://prometheus.io/docs/prometheus/latest/feature_flags/ for more details. | | +| --enable-feature ... | Comma separated feature names to enable. Valid options: exemplar-storage, expand-external-labels, memory-snapshot-on-shutdown, promql-per-step-stats, promql-experimental-functions, extra-scrape-metrics, auto-gomaxprocs, native-histograms, created-timestamp-zero-ingestion, concurrent-rule-eval, delayed-compaction, old-ui. See https://prometheus.io/docs/prometheus/latest/feature_flags/ for more details. | | | --agent | Run Prometheus in 'Agent mode'. | | | --log.level | Only log messages with the given severity or above. One of: [debug, info, warn, error] | `info` | | --log.format | Output format of log messages. One of: [logfmt, json] | `logfmt` | diff --git a/docs/feature_flags.md b/docs/feature_flags.md index 4be11ed47..0541961f2 100644 --- a/docs/feature_flags.md +++ b/docs/feature_flags.md @@ -53,14 +53,6 @@ computed at all. When enabled, GOMAXPROCS variable is automatically set to match Linux container CPU quota. -## Auto GOMEMLIMIT - -`--enable-feature=auto-gomemlimit` - -When enabled, the GOMEMLIMIT variable is automatically set to match the Linux container memory limit. If there is no container limit, or the process is running outside of containers, the system memory total is used. - -There is also an additional tuning flag, `--auto-gomemlimit.ratio`, which allows controlling how much of the memory is used for Prometheus. The remainder is reserved for memory outside the process. For example, kernel page cache. Page cache is important for Prometheus TSDB query performance. The default is `0.9`, which means 90% of the memory limit will be used for Prometheus. - ## Native Histograms `--enable-feature=native-histograms`