diff --git a/docs/querying/api.md b/docs/querying/api.md index 76e9e231f..fe8b723a6 100644 --- a/docs/querying/api.md +++ b/docs/querying/api.md @@ -797,6 +797,67 @@ $ curl http://localhost:9090/api/v1/status/buildinfo **NOTE**: The exact returned build properties may change without notice between Prometheus versions. +### TSDB Stats + +The following endpoint returns various cardinality statistics about the Prometheus TSDB: + +``` +GET /api/v1/status/tsdb +``` +- **seriesCountByMetricName:** This will provide a list of metrics names and their series count. +- **labelValueCountByLabelName:** This will provide a list of the label names and their value count. +- **memoryInBytesByLabelName** This will provide a list of the label names and memory used in bytes. Memory usage is calculated by adding the length of all values for a given label name. +- **seriesCountByLabelPair** This will provide a list of label value pairs and their series count. + +```json +$ curl http://localhost:9090/api/v1/status/tsdb +{ + "status": "success", + "data": { + "seriesCountByMetricName": [ + { + "name": "net_conntrack_dialer_conn_failed_total", + "value": 20 + }, + { + "name": "prometheus_http_request_duration_seconds_bucket", + "value": 20 + } + ], + "labelValueCountByLabelName": [ + { + "name": "__name__", + "value": 211 + }, + { + "name": "event", + "value": 3 + } + ], + "memoryInBytesByLabelName": [ + { + "name": "__name__", + "value": 8266 + }, + { + "name": "instance", + "value": 28 + } + ], + "seriesCountByLabelValuePair": [ + { + "name": "job=prometheus", + "value": 425 + }, + { + "name": "instance=localhost:9090", + "value": 425 + } + ] + } +} +``` + *New in v2.14* ## TSDB Admin APIs diff --git a/web/api/v1/api.go b/web/api/v1/api.go index 0f33d33b6..735dbc526 100644 --- a/web/api/v1/api.go +++ b/web/api/v1/api.go @@ -36,6 +36,8 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" "github.com/prometheus/common/route" + "github.com/prometheus/prometheus/tsdb" + "github.com/prometheus/prometheus/tsdb/index" tsdbLabels "github.com/prometheus/prometheus/tsdb/labels" "github.com/prometheus/prometheus/config" @@ -158,6 +160,7 @@ type TSDBAdmin interface { Delete(mint, maxt int64, ms ...tsdbLabels.Matcher) error Dir() string Snapshot(dir string, withHead bool) error + Head() *tsdb.Head } // API can register a set of endpoints in a router and handle @@ -278,6 +281,7 @@ func (api *API) Register(r *route.Router) { r.Get("/status/runtimeinfo", wrap(api.serveRuntimeInfo)) r.Get("/status/buildinfo", wrap(api.serveBuildInfo)) r.Get("/status/flags", wrap(api.serveFlags)) + r.Get("/status/tsdb", wrap(api.serveTSDBStatus)) r.Post("/read", api.ready(http.HandlerFunc(api.remoteRead))) r.Get("/alerts", wrap(api.alerts)) @@ -915,6 +919,45 @@ func (api *API) serveFlags(r *http.Request) apiFuncResult { return apiFuncResult{api.flagsMap, nil, nil, nil} } +// stat holds the information about individual cardinality. +type stat struct { + Name string `json:"name"` + Value uint64 `json:"value"` +} + +// tsdbStatus has information of cardinality statistics from postings. +type tsdbStatus struct { + SeriesCountByMetricName []stat `json:"seriesCountByMetricName"` + LabelValueCountByLabelName []stat `json:"labelValueCountByLabelName"` + MemoryInBytesByLabelName []stat `json:"memoryInBytesByLabelName"` + SeriesCountByLabelValuePair []stat `json:"seriesCountByLabelValuePair"` +} + +func (api *API) serveTSDBStatus(r *http.Request) apiFuncResult { + db := api.db() + if db == nil { + return apiFuncResult{nil, &apiError{errorUnavailable, errors.New("TSDB not ready")}, nil, nil} + } + convert := func(stats []index.Stat) []stat { + result := make([]stat, 0, len(stats)) + for _, item := range stats { + item := stat{Name: item.Name, Value: item.Count} + result = append(result, item) + } + return result + } + + posting := db.Head().PostingsCardinalityStats(model.MetricNameLabel) + response := tsdbStatus{ + SeriesCountByMetricName: convert(posting.CardinalityMetricsStats), + LabelValueCountByLabelName: convert(posting.CardinalityLabelStats), + MemoryInBytesByLabelName: convert(posting.LabelValueStats), + SeriesCountByLabelValuePair: convert(posting.LabelValuePairsStats), + } + + return apiFuncResult{response, nil, nil, nil} +} + func (api *API) remoteRead(w http.ResponseWriter, r *http.Request) { ctx := r.Context() if err := api.remoteReadGate.Start(ctx); err != nil { diff --git a/web/api/v1/api_test.go b/web/api/v1/api_test.go index 78160e388..63ab63e87 100644 --- a/web/api/v1/api_test.go +++ b/web/api/v1/api_test.go @@ -39,6 +39,7 @@ import ( "github.com/prometheus/common/model" "github.com/prometheus/common/promlog" "github.com/prometheus/common/route" + "github.com/prometheus/prometheus/tsdb" tsdbLabels "github.com/prometheus/prometheus/tsdb/labels" "github.com/prometheus/prometheus/config" @@ -1332,6 +1333,10 @@ func (f *fakeDB) Dir() string { return dir } func (f *fakeDB) Snapshot(dir string, withHead bool) error { return f.err } +func (f *fakeDB) Head() *tsdb.Head { + h, _ := tsdb.NewHead(nil, nil, nil, 1000) + return h +} func TestAdminEndpoints(t *testing.T) { tsdb, tsdbWithError := &fakeDB{}, &fakeDB{err: errors.New("some error")} @@ -1842,6 +1847,47 @@ func TestRespond(t *testing.T) { } } +func TestTSDBStatus(t *testing.T) { + tsdb := &fakeDB{} + tsdbStatusAPI := func(api *API) apiFunc { return api.serveTSDBStatus } + + for i, tc := range []struct { + db *fakeDB + endpoint func(api *API) apiFunc + method string + values url.Values + + errType errorType + }{ + // Tests for the TSDB Status endpoint. + { + db: tsdb, + endpoint: tsdbStatusAPI, + + errType: errorNone, + }, + } { + tc := tc + t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { + api := &API{ + db: func() TSDBAdmin { + if tc.db != nil { + return tc.db + } + return nil + }, + } + endpoint := tc.endpoint(api) + req, err := http.NewRequest(tc.method, fmt.Sprintf("?%s", tc.values.Encode()), nil) + if err != nil { + t.Fatalf("Error when creating test request: %s", err) + } + res := endpoint(req) + assertAPIError(t, res.err, tc.errType) + }) + } +} + // This is a global to avoid the benchmark being optimized away. var testResponseWriter = httptest.ResponseRecorder{} diff --git a/web/ui/react-app/src/App.test.tsx b/web/ui/react-app/src/App.test.tsx index a1f99a6f5..813dbb0b6 100755 --- a/web/ui/react-app/src/App.test.tsx +++ b/web/ui/react-app/src/App.test.tsx @@ -4,7 +4,7 @@ import App from './App'; import Navigation from './Navbar'; import { Container } from 'reactstrap'; import { Router } from '@reach/router'; -import { Alerts, Config, Flags, Rules, Services, Status, Targets, PanelList } from './pages'; +import { Alerts, Config, Flags, Rules, Services, Status, Targets, TSDBStatus, PanelList } from './pages'; describe('App', () => { const app = shallow(); @@ -13,7 +13,7 @@ describe('App', () => { expect(app.find(Navigation)).toHaveLength(1); }); it('routes', () => { - [Alerts, Config, Flags, Rules, Services, Status, Targets, PanelList].forEach(component => { + [Alerts, Config, Flags, Rules, Services, Status, Targets, TSDBStatus, PanelList].forEach(component => { const c = app.find(component); expect(c).toHaveLength(1); expect(c.prop('pathPrefix')).toBe('/path/prefix'); diff --git a/web/ui/react-app/src/App.tsx b/web/ui/react-app/src/App.tsx index f4f2ada1c..2666adef9 100755 --- a/web/ui/react-app/src/App.tsx +++ b/web/ui/react-app/src/App.tsx @@ -4,7 +4,7 @@ import { Container } from 'reactstrap'; import './App.css'; import { Router, Redirect } from '@reach/router'; -import { Alerts, Config, Flags, Rules, Services, Status, Targets, PanelList } from './pages'; +import { Alerts, Config, Flags, Rules, Services, Status, Targets, TSDBStatus, PanelList } from './pages'; import PathPrefixProps from './PathPrefixProps'; const App: FC = ({ pathPrefix }) => { @@ -22,6 +22,7 @@ const App: FC = ({ pathPrefix }) => { + diff --git a/web/ui/react-app/src/Navbar.tsx b/web/ui/react-app/src/Navbar.tsx index ed945dd0d..606a806c6 100644 --- a/web/ui/react-app/src/Navbar.tsx +++ b/web/ui/react-app/src/Navbar.tsx @@ -43,6 +43,9 @@ const Navigation: FC = ({ pathPrefix }) => { Runtime & Build Information + + TSDB Status + Command-Line Flags diff --git a/web/ui/react-app/src/pages/TSDBStatus.test.tsx b/web/ui/react-app/src/pages/TSDBStatus.test.tsx new file mode 100644 index 000000000..cefdcd375 --- /dev/null +++ b/web/ui/react-app/src/pages/TSDBStatus.test.tsx @@ -0,0 +1,125 @@ +import * as React from 'react'; +import { mount, shallow, ReactWrapper } from 'enzyme'; +import { act } from 'react-dom/test-utils'; +import { Alert, Table, Badge } from 'reactstrap'; +import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; +import { faSpinner } from '@fortawesome/free-solid-svg-icons'; +import { TSDBStatus } from '.'; +import { TSDBMap, Stats } from './TSDBStatus'; + +const fakeTSDBStatusResponse: { + status: string; + data: TSDBMap; +} = { + status: 'success', + data: { + labelValueCountByLabelName: [ + { + name: '__name__', + value: 5, + }, + ], + seriesCountByMetricName: [ + { + name: 'scrape_duration_seconds', + value: 1, + }, + { + name: 'scrape_samples_scraped', + value: 1, + }, + ], + memoryInBytesByLabelName: [ + { + name: '__name__', + value: 103, + }, + ], + seriesCountByLabelValuePair: [ + { + name: 'instance=localhost:9100', + value: 5, + }, + ], + }, +}; + +describe('TSDB Stats', () => { + beforeEach(() => { + fetch.resetMocks(); + }); + + it('before data is returned', () => { + const tsdbStatus = shallow(); + const icon = tsdbStatus.find(FontAwesomeIcon); + expect(icon.prop('icon')).toEqual(faSpinner); + expect(icon.prop('spin')).toBeTruthy(); + }); + + describe('when an error is returned', () => { + it('displays an alert', async () => { + const mock = fetch.mockReject(new Error('error loading tsdb status')); + + let page: ReactWrapper; + await act(async () => { + page = mount(); + }); + page.update(); + + expect(mock).toHaveBeenCalledWith('/path/prefix/api/v1/status/tsdb', undefined); + const alert = page.find(Alert); + expect(alert.prop('color')).toBe('danger'); + expect(alert.text()).toContain('error loading tsdb status'); + }); + }); + + describe('Table Data Validation', () => { + it('Table Test', async () => { + const tables = [ + { + data: fakeTSDBStatusResponse.data.labelValueCountByLabelName, + table_index: 0, + }, + { + data: fakeTSDBStatusResponse.data.seriesCountByMetricName, + table_index: 1, + }, + { + data: fakeTSDBStatusResponse.data.memoryInBytesByLabelName, + table_index: 2, + }, + { + data: fakeTSDBStatusResponse.data.seriesCountByLabelValuePair, + table_index: 3, + }, + ]; + + const mock = fetch.mockResponse(JSON.stringify(fakeTSDBStatusResponse)); + let page: ReactWrapper; + await act(async () => { + page = mount(); + }); + page.update(); + + expect(mock).toHaveBeenCalledWith('/path/prefix/api/v1/status/tsdb', undefined); + + for (let i = 0; i < tables.length; i++) { + let data = tables[i].data; + let table = page + .find(Table) + .at(tables[i].table_index) + .find('tbody'); + let rows = table.find('tr'); + for (let i = 0; i < data.length; i++) { + const firstRowColumns = rows + .at(i) + .find('td') + .map(column => column.text()); + expect(rows.length).toBe(data.length); + expect(firstRowColumns[0]).toBe(data[i].name); + expect(firstRowColumns[1]).toBe(data[i].value.toString()); + } + } + }); + }); +}); diff --git a/web/ui/react-app/src/pages/TSDBStatus.tsx b/web/ui/react-app/src/pages/TSDBStatus.tsx new file mode 100644 index 000000000..826350793 --- /dev/null +++ b/web/ui/react-app/src/pages/TSDBStatus.tsx @@ -0,0 +1,87 @@ +import React, { FC, Fragment } from 'react'; +import { RouteComponentProps } from '@reach/router'; +import { Alert, Table } from 'reactstrap'; +import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; +import { faSpinner } from '@fortawesome/free-solid-svg-icons'; +import { useFetch } from '../utils/useFetch'; +import PathPrefixProps from '../PathPrefixProps'; + +export interface Stats { + name: string; + value: number; +} + +export interface TSDBMap { + seriesCountByMetricName: Array; + labelValueCountByLabelName: Array; + memoryInBytesByLabelName: Array; + seriesCountByLabelValuePair: Array; +} + +const paddingStyle = { + padding: '10px', +}; + +function createTable(title: string, unit: string, stats: Array) { + return ( +
+

{title}

+ + + + + + + + + {stats.map((element: Stats, i: number) => { + return ( + + + + + + + ); + })} + +
Name{unit}
{element.name}{element.value}
+
+ ); +} + +const TSDBStatus: FC = ({ pathPrefix }) => { + const { response, error } = useFetch(`${pathPrefix}/api/v1/status/tsdb`); + const headStats = () => { + const stats: TSDBMap = response && response.data; + if (error) { + return ( + + Error: Error fetching TSDB Status: {error.message} + + ); + } else if (stats) { + return ( +
+
+

Head Cardinality Stats

+
+ {createTable('Top 10 label names with value count', 'Count', stats.labelValueCountByLabelName)} + {createTable('Top 10 series count by metric names', 'Count', stats.seriesCountByMetricName)} + {createTable('Top 10 label names with high memory usage', 'Bytes', stats.memoryInBytesByLabelName)} + {createTable('Top 10 series count by label value pairs', 'Count', stats.seriesCountByLabelValuePair)} +
+ ); + } + return ; + }; + + return ( +
+

TSDB Status

+ {headStats()} +
+ ); +}; + +export default TSDBStatus; diff --git a/web/ui/react-app/src/pages/index.ts b/web/ui/react-app/src/pages/index.ts index b9dd5e9cc..dba245f10 100644 --- a/web/ui/react-app/src/pages/index.ts +++ b/web/ui/react-app/src/pages/index.ts @@ -6,5 +6,6 @@ import Services from './Services'; import Status from './Status'; import Targets from './targets/Targets'; import PanelList from './PanelList'; +import TSDBStatus from './TSDBStatus'; -export { Alerts, Config, Flags, Rules, Services, Status, Targets, PanelList }; +export { Alerts, Config, Flags, Rules, Services, Status, Targets, TSDBStatus, PanelList };