github
/
consul
mirror of https://github.com/hashicorp/consul
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Merge pull request #11720 from hashicorp/bbolt

pull/10778/head
Freddy 3 years ago committed by GitHub
parent
047aa2ffb0 09fd8f727e
commit
e246defb6c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 62 additions and 4 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 11
      .changelog/11720.txt
  2. 1
      agent/agent.go
  3. 4
      agent/config/builder.go
  4. 4
      agent/config/config.go
  5. 2
      agent/config/runtime.go
  6. 1
      agent/config/runtime_test.go
  7. 3
      agent/config/testdata/TestRuntimeConfig_Sanitize.golden
  8. 3
      agent/config/testdata/full-config.hcl
  9. 3
      agent/config/testdata/full-config.json
  10. 6
      agent/consul/config.go
  11. 13
      agent/consul/server.go
  12. 4
      go.mod
  13. 11
      go.sum

11
.changelog/11720.txt
Unescape View File

@ -0,0 +1,11 @@
```release-note:improvement
raft: Use bbolt instead of the legacy boltdb implementation
```
```release-note:improvement
raft: Emit boltdb related performance metrics
```
```release-note:improvement
raft: Added a configuration to disable boltdb freelist syncing
```

1
agent/agent.go
Unescape View File

@ -1263,6 +1263,7 @@ func newConsulConfig(runtimeCfg *config.RuntimeConfig, logger hclog.Logger) (*co
}
cfg.ConfigEntryBootstrap = runtimeCfg.ConfigEntryBootstrap
cfg.RaftBoltDBConfig = runtimeCfg.RaftBoltDBConfig
// Duplicate our own serf config once to make sure that the duplication
// function does not drift.

4
agent/config/builder.go
Unescape View File

@ -1094,6 +1094,10 @@ func (b *builder) build() (rt RuntimeConfig, err error) {
rt.UseStreamingBackend = boolValWithDefault(c.UseStreamingBackend, true)
if c.RaftBoltDBConfig != nil {
rt.RaftBoltDBConfig = *c.RaftBoltDBConfig
}
if rt.Cache.EntryFetchMaxBurst <= 0 {
return RuntimeConfig{}, fmt.Errorf("cache.entry_fetch_max_burst must be strictly positive, was: %v", rt.Cache.EntryFetchMaxBurst)
}

4
agent/config/config.go
Unescape View File

@ -4,6 +4,8 @@ import (
"encoding/json"
"fmt"
"github.com/hashicorp/consul/agent/consul"
"github.com/hashicorp/hcl"
"github.com/mitchellh/mapstructure"
@ -256,6 +258,8 @@ type Config struct {
RPC RPC `mapstructure:"rpc"`
RaftBoltDBConfig *consul.RaftBoltDBConfig `mapstructure:"raft_boltdb"`
// UseStreamingBackend instead of blocking queries for service health and
// any other endpoints which support streaming.
UseStreamingBackend *bool `mapstructure:"use_streaming_backend"`

2
agent/config/runtime.go
Unescape View File

@ -943,6 +943,8 @@ type RuntimeConfig struct {
// hcl: raft_trailing_logs = int
RaftTrailingLogs int
RaftBoltDBConfig consul.RaftBoltDBConfig
// ReconnectTimeoutLAN specifies the amount of time to wait to reconnect with
// another agent before deciding it's permanently gone. This can be used to
// control the time it takes to reap failed nodes from the cluster.

1
agent/config/runtime_test.go
Unescape View File

@ -6015,6 +6015,7 @@ func TestLoad_FullConfig(t *testing.T) {
"args": []interface{}{"dltjDJ2a", "flEa7C2d"},
},
},
RaftBoltDBConfig: consul.RaftBoltDBConfig{NoFreelistSync: true},
}
entFullRuntimeConfig(expected)

3
agent/config/testdata/TestRuntimeConfig_Sanitize.golden vendored
Unescape View File

@ -252,6 +252,9 @@
"RPCMaxConnsPerClient": 0,
"RPCProtocol": 0,
"RPCRateLimit": 0,
"RaftBoltDBConfig": {
"NoFreelistSync": false
},
"RaftProtocol": 3,
"RaftSnapshotInterval": "0s",
"RaftSnapshotThreshold": 0,

3
agent/config/testdata/full-config.hcl vendored
Unescape View File

@ -328,6 +328,9 @@ raft_protocol = 3
raft_snapshot_threshold = 16384
raft_snapshot_interval = "30s"
raft_trailing_logs = 83749
raft_boltdb {
NoFreelistSync = true
}
read_replica = true
reconnect_timeout = "23739s"
reconnect_timeout_wan = "26694s"

3
agent/config/testdata/full-config.json vendored
Unescape View File

@ -326,6 +326,9 @@
"raft_snapshot_threshold": 16384,
"raft_snapshot_interval": "30s",
"raft_trailing_logs": 83749,
"raft_boltdb": {
"NoFreelistSync": true
},
"read_replica": true,
"reconnect_timeout": "23739s",
"reconnect_timeout_wan": "26694s",

6
agent/consul/config.go
Unescape View File

@ -391,6 +391,8 @@ type Config struct {
RPCConfig RPCConfig
RaftBoltDBConfig RaftBoltDBConfig
// Embedded Consul Enterprise specific configuration
*EnterpriseConfig
}
@ -603,3 +605,7 @@ type ReloadableConfig struct {
RaftSnapshotInterval time.Duration
RaftTrailingLogs int
}
type RaftBoltDBConfig struct {
NoFreelistSync bool
}

13
agent/consul/server.go
Unescape View File

@ -18,6 +18,7 @@ import (
"time"
"github.com/hashicorp/go-version"
"go.etcd.io/bbolt"
"github.com/armon/go-metrics"
connlimit "github.com/hashicorp/go-connlimit"
@ -25,7 +26,7 @@ import (
"github.com/hashicorp/go-memdb"
"github.com/hashicorp/raft"
autopilot "github.com/hashicorp/raft-autopilot"
raftboltdb "github.com/hashicorp/raft-boltdb"
raftboltdb "github.com/hashicorp/raft-boltdb/v2"
"github.com/hashicorp/serf/serf"
"golang.org/x/time/rate"
"google.golang.org/grpc"
@ -729,13 +730,21 @@ func (s *Server) setupRaft() error {
}
// Create the backend raft store for logs and stable storage.
store, err := raftboltdb.NewBoltStore(filepath.Join(path, "raft.db"))
store, err := raftboltdb.New(raftboltdb.Options{
BoltOptions: &bbolt.Options{
NoFreelistSync: s.config.RaftBoltDBConfig.NoFreelistSync,
},
Path: filepath.Join(path, "raft.db"),
})
if err != nil {
return err
}
s.raftStore = store
stable = store
// start publishing boltdb metrics
go store.RunMetrics(&lib.StopChannelContext{StopCh: s.shutdownCh}, 0)
// Wrap the store in a LogCache to improve performance.
cacheStore, err := raft.NewLogCache(raftLogCacheSize, store)
if err != nil {

4
go.mod
Unescape View File

@ -53,7 +53,8 @@ require (
github.com/hashicorp/net-rpc-msgpackrpc v0.0.0-20151116020338-a14192a58a69
github.com/hashicorp/raft v1.3.2
github.com/hashicorp/raft-autopilot v0.1.5
github.com/hashicorp/raft-boltdb v0.0.0-20171010151810-6e5ba93211ea
github.com/hashicorp/raft-boltdb v0.0.0-20211202195631-7d34b9fb3f42 // indirect
github.com/hashicorp/raft-boltdb/v2 v2.2.0
github.com/hashicorp/serf v0.9.6
github.com/hashicorp/vault/api v1.0.5-0.20200717191844-f687267c8086
github.com/hashicorp/vault/sdk v0.1.14-0.20200519221838-e0cfd64bc267
@ -79,6 +80,7 @@ require (
github.com/ryanuber/columnize v2.1.0+incompatible
github.com/shirou/gopsutil/v3 v3.21.10
github.com/stretchr/testify v1.7.0
go.etcd.io/bbolt v1.3.5
go.opencensus.io v0.22.0 // indirect
go.uber.org/goleak v1.1.10
golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a

11
go.sum
Unescape View File

@ -59,6 +59,7 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg=
github.com/armon/go-metrics v0.3.0/go.mod h1:zXjbSimjXTd7vOpY8B0/2LpvNvDoXBuplAD+gJD3GYs=
github.com/armon/go-metrics v0.3.8/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc=
github.com/armon/go-metrics v0.3.10 h1:FR+drcQStOe+32sYyJYyZ7FIdgoGGBnwLl+flodp8Uo=
github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc=
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
@ -284,14 +285,19 @@ github.com/hashicorp/memberlist v0.3.0 h1:8+567mCcFDnS5ADl7lrpxPMWiFCElyUEeW0gtj
github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE=
github.com/hashicorp/net-rpc-msgpackrpc v0.0.0-20151116020338-a14192a58a69 h1:lc3c72qGlIMDqQpQH82Y4vaglRMMFdJbziYWriR4UcE=
github.com/hashicorp/net-rpc-msgpackrpc v0.0.0-20151116020338-a14192a58a69/go.mod h1:/z+jUGRBlwVpUZfjute9jWaF6/HuhjuFQuL1YXzVD1Q=
github.com/hashicorp/raft v1.1.0/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7HVbyzqM=
github.com/hashicorp/raft v1.1.1/go.mod h1:vPAJM8Asw6u8LxC3eJCUZmRP/E4QmUGE1R7g7k8sG/8=
github.com/hashicorp/raft v1.2.0/go.mod h1:vPAJM8Asw6u8LxC3eJCUZmRP/E4QmUGE1R7g7k8sG/8=
github.com/hashicorp/raft v1.3.2 h1:j2tqHqFnDdWCepLxzuo3b6WzS2krIweBrvEoqBbWMTo=
github.com/hashicorp/raft v1.3.2/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7HVbyzqM=
github.com/hashicorp/raft-autopilot v0.1.5 h1:onEfMH5uHVdXQqtas36zXUHEZxLdsJVu/nXHLcLdL1I=
github.com/hashicorp/raft-autopilot v0.1.5/go.mod h1:Af4jZBwaNOI+tXfIqIdbcAnh/UyyqIMj/pOISIfhArw=
github.com/hashicorp/raft-boltdb v0.0.0-20171010151810-6e5ba93211ea h1:xykPFhrBAS2J0VBzVa5e80b5ZtYuNQtgXjN40qBZlD4=
github.com/hashicorp/raft-boltdb v0.0.0-20171010151810-6e5ba93211ea/go.mod h1:pNv7Wc3ycL6F5oOWn+tPGo2gWD4a5X+yp/ntwdKLjRk=
github.com/hashicorp/raft-boltdb v0.0.0-20210409134258-03c10cc3d4ea/go.mod h1:qRd6nFJYYS6Iqnc/8HcUmko2/2Gw8qTFEmxDLii6W5I=
github.com/hashicorp/raft-boltdb v0.0.0-20211202195631-7d34b9fb3f42 h1:Ye8SofeDHJzu9xvvaMmpMkqHELWW7rTcXwdUR0CWW48=
github.com/hashicorp/raft-boltdb v0.0.0-20211202195631-7d34b9fb3f42/go.mod h1:wcXL8otVu5cpJVLjcmq7pmfdRCdaP+xnvu7WQcKJAhs=
github.com/hashicorp/raft-boltdb/v2 v2.2.0 h1:/CVN9LSAcH50L3yp2TsPFIpeyHn1m3VF6kiutlDE3Nw=
github.com/hashicorp/raft-boltdb/v2 v2.2.0/go.mod h1:SgPUD5TP20z/bswEr210SnkUFvQP/YjKV95aaiTbeMQ=
github.com/hashicorp/serf v0.9.6 h1:uuEX1kLR6aoda1TBttmJQKDLZE1Ob7KN0NPdE7EtCDc=
github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4=
github.com/hashicorp/vault/api v1.0.5-0.20200717191844-f687267c8086 h1:OKsyxKi2sNmqm1Gv93adf2AID2FOBFdCbbZn9fGtIdg=
@ -520,6 +526,8 @@ github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0=
go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
go.opencensus.io v0.22.0 h1:C9hSCOW830chIVkdja34wa6Ky+IzWllkUinR+BtRZd4=
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
@ -620,6 +628,7 @@ golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

Write Preview
Loading…
Cancel
Save