github
/
consul
mirror of https://github.com/hashicorp/consul
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Add `testing/deployer` (neé `consul-topology`) [NET-4610] (#17823) 

Co-authored-by: R.B. Boyer <4903+rboyer@users.noreply.github.com>
Co-authored-by: R.B. Boyer <rb@hashicorp.com>
Co-authored-by: Freddy <freddygv@users.noreply.github.com>
pull/18169/head
Nick Irvine 1 year ago committed by GitHub
parent
07fce869af
commit
62005369b5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
60 changed files with 7818 additions and 0 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 1
      .github/workflows/reusable-lint.yml
  2. 4
      testing/deployer/.gitignore
  3. 179
      testing/deployer/README.md
  4. 9
      testing/deployer/TODO.md
  5. 44
      testing/deployer/go.mod
  6. 241
      testing/deployer/go.sum
  7. 332
      testing/deployer/sprawl/acl.go
  8. 160
      testing/deployer/sprawl/acl_rules.go
  9. 520
      testing/deployer/sprawl/boot.go
  10. 425
      testing/deployer/sprawl/catalog.go
  11. 58
      testing/deployer/sprawl/configentries.go
  12. 98
      testing/deployer/sprawl/consul.go
  13. 8
      testing/deployer/sprawl/debug.go
  14. 170
      testing/deployer/sprawl/details.go
  15. 174
      testing/deployer/sprawl/ent.go
  16. 11
      testing/deployer/sprawl/helpers.go
  17. 83
      testing/deployer/sprawl/internal/build/docker.go
  18. 120
      testing/deployer/sprawl/internal/runner/exec.go
  19. 70
      testing/deployer/sprawl/internal/secrets/store.go
  20. 215
      testing/deployer/sprawl/internal/tfgen/agent.go
  21. 45
      testing/deployer/sprawl/internal/tfgen/digest.go
  22. 180
      testing/deployer/sprawl/internal/tfgen/dns.go
  23. 39
      testing/deployer/sprawl/internal/tfgen/docker.go
  24. 15
      testing/deployer/sprawl/internal/tfgen/docker_test.go
  25. 475
      testing/deployer/sprawl/internal/tfgen/gen.go
  26. 70
      testing/deployer/sprawl/internal/tfgen/io.go
  27. 249
      testing/deployer/sprawl/internal/tfgen/nodes.go
  28. 16
      testing/deployer/sprawl/internal/tfgen/prelude.go
  29. 87
      testing/deployer/sprawl/internal/tfgen/proxy.go
  30. 95
      testing/deployer/sprawl/internal/tfgen/res.go
  31. 29
      testing/deployer/sprawl/internal/tfgen/templates/container-app-dataplane.tf.tmpl
  32. 31
      testing/deployer/sprawl/internal/tfgen/templates/container-app-sidecar.tf.tmpl
  33. 25
      testing/deployer/sprawl/internal/tfgen/templates/container-app.tf.tmpl
  34. 40
      testing/deployer/sprawl/internal/tfgen/templates/container-consul.tf.tmpl
  35. 28
      testing/deployer/sprawl/internal/tfgen/templates/container-coredns.tf.tmpl
  36. 25
      testing/deployer/sprawl/internal/tfgen/templates/container-mgw.tf.tmpl
  37. 38
      testing/deployer/sprawl/internal/tfgen/templates/container-pause.tf.tmpl
  38. 33
      testing/deployer/sprawl/internal/tfgen/templates/container-proxy.tf.tmpl
  39. 15
      testing/deployer/sprawl/internal/tfgen/tfgen.go
  40. 165
      testing/deployer/sprawl/peering.go
  41. 464
      testing/deployer/sprawl/sprawl.go
  42. 202
      testing/deployer/sprawl/sprawltest/sprawltest.go
  43. 180
      testing/deployer/sprawl/sprawltest/test_test.go
  44. 114
      testing/deployer/sprawl/tls.go
  45. 671
      testing/deployer/topology/compile.go
  46. 3
      testing/deployer/topology/default_cdp.go
  47. 4
      testing/deployer/topology/default_consul.go
  48. 3
      testing/deployer/topology/default_envoy.go
  49. 142
      testing/deployer/topology/ids.go
  50. 123
      testing/deployer/topology/images.go
  51. 98
      testing/deployer/topology/images_test.go
  52. 787
      testing/deployer/topology/topology.go
  53. 17
      testing/deployer/topology/util.go
  54. 11
      testing/deployer/topology/util_test.go
  55. 63
      testing/deployer/util/consul.go
  56. 57
      testing/deployer/util/files.go
  57. 21
      testing/deployer/util/internal/ipamutils/doc.go
  58. 117
      testing/deployer/util/internal/ipamutils/utils.go
  59. 102
      testing/deployer/util/internal/ipamutils/utils_test.go
  60. 17
      testing/deployer/util/net.go

1
.github/workflows/reusable-lint.yml
Unescape View File

@ -34,6 +34,7 @@ jobs:
- "envoyextensions"
- "troubleshoot"
- "test/integration/consul-container"
- "testing/deployer"
fail-fast: true
name: lint ${{ matrix.directory }}
steps:

4
testing/deployer/.gitignore vendored
Unescape View File

@ -0,0 +1,4 @@
/terraform
/workdir
/sample-cli
workdir

179
testing/deployer/README.md
Unescape View File

@ -0,0 +1,179 @@
[![GoDoc](https://pkg.go.dev/badge/github.com/hashicorp/consul/testing/deployer)](https://pkg.go.dev/github.com/hashicorp/consul/testing/deployer)
## Summary
This is a Go library used to launch one or more Consul clusters that can be
peered using the cluster peering feature. Under the covers `terraform` is used
in conjunction with the
[`kreuzwerker/docker`](https://registry.terraform.io/providers/kreuzwerker/docker/latest)
provider to manage a fleet of local docker containers and networks.
### Configuration
The complete topology of Consul clusters is defined using a topology.Config
which allows you to define a set of networks and reference those networks when
assigning nodes and services to clusters. Both Consul clients and
`consul-dataplane` instances are supported.
Here is an example configuration with two peered clusters:
```
cfg := &topology.Config{
Networks: []*topology.Network{
{Name: "dc1"},
{Name: "dc2"},
{Name: "wan", Type: "wan"},
},
Clusters: []*topology.Cluster{
{
Name: "dc1",
Nodes: []*topology.Node{
{
Kind: topology.NodeKindServer,
Name: "dc1-server1",
Addresses: []*topology.Address{
{Network: "dc1"},
{Network: "wan"},
},
},
{
Kind: topology.NodeKindClient,
Name: "dc1-client1",
Services: []*topology.Service{
{
ID: topology.ServiceID{Name: "mesh-gateway"},
Port: 8443,
EnvoyAdminPort: 19000,
IsMeshGateway: true,
},
},
},
{
Kind: topology.NodeKindClient,
Name: "dc1-client2",
Services: []*topology.Service{
{
ID: topology.ServiceID{Name: "ping"},
Image: "rboyer/pingpong:latest",
Port: 8080,
EnvoyAdminPort: 19000,
Command: []string{
"-bind", "0.0.0.0:8080",
"-dial", "127.0.0.1:9090",
"-pong-chaos",
"-dialfreq", "250ms",
"-name", "ping",
},
Upstreams: []*topology.Upstream{{
ID: topology.ServiceID{Name: "pong"},
LocalPort: 9090,
Peer: "peer-dc2-default",
}},
},
},
},
},
InitialConfigEntries: []api.ConfigEntry{
&api.ExportedServicesConfigEntry{
Name: "default",
Services: []api.ExportedService{{
Name: "ping",
Consumers: []api.ServiceConsumer{{
Peer: "peer-dc2-default",
}},
}},
},
},
},
{
Name: "dc2",
Nodes: []*topology.Node{
{
Kind: topology.NodeKindServer,
Name: "dc2-server1",
Addresses: []*topology.Address{
{Network: "dc2"},
{Network: "wan"},
},
},
{
Kind: topology.NodeKindClient,
Name: "dc2-client1",
Services: []*topology.Service{
{
ID: topology.ServiceID{Name: "mesh-gateway"},
Port: 8443,
EnvoyAdminPort: 19000,
IsMeshGateway: true,
},
},
},
{
Kind: topology.NodeKindDataplane,
Name: "dc2-client2",
Services: []*topology.Service{
{
ID: topology.ServiceID{Name: "pong"},
Image: "rboyer/pingpong:latest",
Port: 8080,
EnvoyAdminPort: 19000,
Command: []string{
"-bind", "0.0.0.0:8080",
"-dial", "127.0.0.1:9090",
"-pong-chaos",
"-dialfreq", "250ms",
"-name", "pong",
},
Upstreams: []*topology.Upstream{{
ID: topology.ServiceID{Name: "ping"},
LocalPort: 9090,
Peer: "peer-dc1-default",
}},
},
},
},
},
InitialConfigEntries: []api.ConfigEntry{
&api.ExportedServicesConfigEntry{
Name: "default",
Services: []api.ExportedService{{
Name: "ping",
Consumers: []api.ServiceConsumer{{
Peer: "peer-dc2-default",
}},
}},
},
},
},
},
Peerings: []*topology.Peering{{
Dialing: topology.PeerCluster{
Name: "dc1",
},
Accepting: topology.PeerCluster{
Name: "dc2",
},
}},
}
```
Once you have a topology configuration, you simply call the appropriate
`Launch` function to validate and boot the cluster.
You may also modify your original configuration (in some allowed ways) and call
`Relaunch` on an existing topology which will differentially adjust the running
infrastructure. This can be useful to do things like upgrade instances in place
or subly reconfigure them.
### For Testing
It is meant to be consumed primarily by unit tests desiring a complex
reasonably realistic Consul setup. For that use case use the `sprawl/sprawltest` wrapper:
```
func TestSomething(t *testing.T) {
cfg := &topology.Config{...}
sp := sprawltest.Launch(t, cfg)
// do stuff with 'sp'
}
```

9
testing/deployer/TODO.md
Unescape View File

@ -0,0 +1,9 @@
Missing things that should probably be added;
- consul-dataplane support for running mesh gateways
- consul-dataplane health check updates (automatic; manual)
- ServerExternalAddresses in a peering; possibly rig up a DNS name for this.
- after creating a token, verify it exists on all servers before proceding (rather than sleep looping on not-founds)
- investigate strange gRPC bug that is currently papered over
- allow services to override their mesh gateway modes
- remove some of the debug prints of various things

44
testing/deployer/go.mod
Unescape View File

@ -0,0 +1,44 @@
module github.com/hashicorp/consul/testing/deployer
go 1.20
require (
github.com/google/go-cmp v0.5.9
github.com/hashicorp/consul/api v1.20.0
github.com/hashicorp/consul/sdk v0.13.1
github.com/hashicorp/go-cleanhttp v0.5.2
github.com/hashicorp/go-hclog v1.5.0
github.com/hashicorp/go-multierror v1.1.1
github.com/hashicorp/hcl/v2 v2.16.2
github.com/mitchellh/copystructure v1.2.0
github.com/rboyer/safeio v0.2.2
github.com/stretchr/testify v1.8.2
golang.org/x/crypto v0.7.0
)
require (
github.com/agext/levenshtein v1.2.1 // indirect
github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect
github.com/armon/go-metrics v0.3.10 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/fatih/color v1.13.0 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
github.com/hashicorp/go-rootcerts v1.0.2 // indirect
github.com/hashicorp/go-uuid v1.0.3 // indirect
github.com/hashicorp/go-version v1.2.1 // indirect
github.com/hashicorp/golang-lru v0.5.4 // indirect
github.com/hashicorp/serf v0.10.1 // indirect
github.com/mattn/go-colorable v0.1.12 // indirect
github.com/mattn/go-isatty v0.0.14 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/mitchellh/go-wordwrap v1.0.0 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/mitchellh/reflectwalk v1.0.2 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/zclconf/go-cty v1.12.1 // indirect
golang.org/x/sys v0.6.0 // indirect
golang.org/x/text v0.8.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

241
testing/deployer/go.sum
Unescape View File

@ -0,0 +1,241 @@
github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
github.com/agext/levenshtein v1.2.1 h1:QmvMAjj2aEICytGiWzmxoE0x2KZvE0fvmqMOfy2tjT8=
github.com/agext/levenshtein v1.2.1/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/apparentlymart/go-textseg/v13 v13.0.0 h1:Y+KvPE1NYz0xl601PVImeQfFyEy6iT90AvPUL1NNfNw=
github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo=
github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
github.com/armon/go-metrics v0.3.10 h1:FR+drcQStOe+32sYyJYyZ7FIdgoGGBnwLl+flodp8Uo=
github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc=
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag=
github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=
github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/go-test/deep v1.0.3 h1:ZrJSEWsXzPOxaZnFteGEfooLba+ju3FYIbOrS+rQd68=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCydnIczKteheJEzHRToSGK3Bnlw=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/hashicorp/consul/api v1.20.0 h1:9IHTjNVSZ7MIwjlW3N3a7iGiykCMDpxZu8jsxFJh0yc=
github.com/hashicorp/consul/api v1.20.0/go.mod h1:nR64eD44KQ59Of/ECwt2vUmIK2DKsDzAwTmwmLl8Wpo=
github.com/hashicorp/consul/sdk v0.13.1 h1:EygWVWWMczTzXGpO93awkHFzfUka6hLYJ0qhETd+6lY=
github.com/hashicorp/consul/sdk v0.13.1/go.mod h1:SW/mM4LbKfqmMvcFu8v+eiQQ7oitXEFeiBe9StxERb0=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
github.com/hashicorp/go-hclog v1.5.0 h1:bI2ocEMgcVlz55Oj1xZNBsVi900c7II+fWDyV9o+13c=
github.com/hashicorp/go-hclog v1.5.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc=
github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
github.com/hashicorp/go-msgpack v0.5.3 h1:zKjpN5BK/P5lMYrLmBHdBULWbJ0XpYR+7NGzqkZzoD4=
github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA=
github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs=
github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc=
github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8=
github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
github.com/hashicorp/go-sockaddr v1.0.2 h1:ztczhD1jLxIRjVejw8gFomI1BQZOe2WoVOu0SyteCQc=
github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8=
github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
github.com/hashicorp/go-version v1.2.1 h1:zEfKbn2+PDgroKdiOzqiE8rsmLqU2uwi5PB5pBJ3TkI=
github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
github.com/hashicorp/hcl/v2 v2.16.2 h1:mpkHZh/Tv+xet3sy3F9Ld4FyI2tUpWe9x3XtPx9f1a0=
github.com/hashicorp/hcl/v2 v2.16.2/go.mod h1:JRmR89jycNkrrqnMmvPDMd56n1rQJ2Q6KocSLCMCXng=
github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc=
github.com/hashicorp/memberlist v0.5.0 h1:EtYPN8DpAURiapus508I4n9CzHs2W+8NZGbmmR/prTM=
github.com/hashicorp/memberlist v0.5.0/go.mod h1:yvyXLpo0QaGE59Y7hDTsTzDD25JYBZ4mHgHUZ8lrOI0=
github.com/hashicorp/serf v0.10.1 h1:Z1H2J60yRKvfDYAOZLd2MU0ND4AH/WDz7xYHDWQsIPY=
github.com/hashicorp/serf v0.10.1/go.mod h1:yL2t6BqATOLGc5HF7qbFkTfXoPIY0WZdWHfEvMqbG+4=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4=
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40=
github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso=
github.com/miekg/dns v1.1.41 h1:WMszZWJG0XmzbK9FEmzH2TVcqYzFesusSIB41b8KHxY=
github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI=
github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI=
github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw=
github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s=
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-wordwrap v1.0.0 h1:6GlHJ/LTGMrIJbwgdqdl2eEH8o+Exx/0m8ir9Gns0u4=
github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo=
github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ=
github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY=
github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
github.com/rboyer/safeio v0.2.2 h1:XhtqyUTRleMYGyBt3ni4j2BtEh669U2ry2INnnd+B4k=
github.com/rboyer/safeio v0.2.2/go.mod h1:pSnr2LFXyn/c/fotxotyOdYy7pP/XSh6MpBmzXPjiNc=
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
github.com/zclconf/go-cty v1.12.1 h1:PcupnljUm9EIvbgSHQnHhUr3fO6oFmkOrvs2BAFNXXY=
github.com/zclconf/go-cty v1.12.1/go.mod h1:s9IfD1LK5ccNMSWCVFCE2rJfHiZgi7JijgeWIMfhLvA=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY=
golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A=
golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8=
golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68=
golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

332
testing/deployer/sprawl/acl.go
Unescape View File

@ -0,0 +1,332 @@
package sprawl
import (
"fmt"
"strings"
"time"
"github.com/hashicorp/consul/api"
"github.com/hashicorp/consul/testing/deployer/sprawl/internal/secrets"
"github.com/hashicorp/consul/testing/deployer/topology"
)
// TODO: fix this by checking that a token/policy works on ALL servers before
// returning from create.
func isACLNotFound(err error) bool {
if err == nil {
return false
}
return strings.Contains(err.Error(), `ACL not found`)
}
func (s *Sprawl) bootstrapACLs(cluster string) error {
var (
client = s.clients[cluster]
logger = s.logger.With("cluster", cluster)
mgmtToken = s.secrets.ReadGeneric(cluster, secrets.BootstrapToken)
)
ac := client.ACL()
if mgmtToken != "" {
NOT_BOOTED:
ready, err := s.isACLBootstrapped(cluster, client)
if err != nil {
return fmt.Errorf("error checking if the acl system is bootstrapped: %w", err)
} else if !ready {
logger.Warn("ACL system is not ready yet")
time.Sleep(250 * time.Millisecond)
goto NOT_BOOTED
}
TRYAGAIN:
// check to see if it works
_, _, err = ac.TokenReadSelf(&api.QueryOptions{Token: mgmtToken})
if err != nil {
if isACLNotBootstrapped(err) {
logger.Warn("system is rebooting", "error", err)
time.Sleep(250 * time.Millisecond)
goto TRYAGAIN
}
return fmt.Errorf("management token no longer works: %w", err)
}
logger.Info("current management token", "token", mgmtToken)
return nil
}
TRYAGAIN2:
logger.Info("bootstrapping ACLs")
tok, _, err := ac.Bootstrap()
if err != nil {
if isACLNotBootstrapped(err) {
logger.Warn("system is rebooting", "error", err)
time.Sleep(250 * time.Millisecond)
goto TRYAGAIN2
}
return err
}
mgmtToken = tok.SecretID
s.secrets.SaveGeneric(cluster, secrets.BootstrapToken, mgmtToken)
logger.Info("current management token", "token", mgmtToken)
return nil
}
func isACLNotBootstrapped(err error) bool {
switch {
case strings.Contains(err.Error(), "ACL system must be bootstrapped before making any requests that require authorization"):
return true
case strings.Contains(err.Error(), "The ACL system is currently in legacy mode"):
return true
}
return false
}
func (s *Sprawl) isACLBootstrapped(cluster string, client *api.Client) (bool, error) {
policy, _, err := client.ACL().PolicyReadByName("global-management", &api.QueryOptions{
Token: s.secrets.ReadGeneric(cluster, secrets.BootstrapToken),
})
if err != nil {
if strings.Contains(err.Error(), "Unexpected response code: 403 (ACL not found)") {
return false, nil
} else if isACLNotBootstrapped(err) {
return false, nil
}
return false, err
}
return policy != nil, nil
}
func (s *Sprawl) createAnonymousToken(cluster *topology.Cluster) error {
var (
client = s.clients[cluster.Name]
logger = s.logger.With("cluster", cluster.Name)
)
if err := s.createAnonymousPolicy(cluster); err != nil {
return err
}
token, err := CreateOrUpdateToken(client, anonymousToken())
if err != nil {
return err
}
logger.Info("created anonymous token",
"token", token.SecretID,
)
return nil
}
func (s *Sprawl) createAnonymousPolicy(cluster *topology.Cluster) error {
var (
client = s.clients[cluster.Name]
logger = s.logger.With("cluster", cluster.Name)
)
op, err := CreateOrUpdatePolicy(client, anonymousPolicy(cluster.Enterprise))
if err != nil {
return err
}
logger.Info("created anonymous policy",
"policy-name", op.Name,
"policy-id", op.ID,
)
return nil
}
func (s *Sprawl) createAgentTokens(cluster *topology.Cluster) error {
var (
client = s.clients[cluster.Name]
logger = s.logger.With("cluster", cluster.Name)
)
for _, node := range cluster.Nodes {
// NOTE: always create tokens even for disabled nodes.
if !node.IsAgent() {
continue
}
if tok := s.secrets.ReadAgentToken(cluster.Name, node.ID()); tok == "" {
token, err := CreateOrUpdateToken(client, tokenForNode(node, cluster.Enterprise))
if err != nil {
return err
}
logger.Info("created agent token",
"node", node.ID(),
"token", token.SecretID,
)
s.secrets.SaveAgentToken(cluster.Name, node.ID(), token.SecretID)
}
}
return nil
}
// Create a policy to allow super permissive catalog reads across namespace
// boundaries.
func (s *Sprawl) createCrossNamespaceCatalogReadPolicies(cluster *topology.Cluster, partition string) error {
if !cluster.Enterprise {
return nil
}
var (
client = s.clients[cluster.Name]
logger = s.logger.With("cluster", cluster.Name)
)
op, err := CreateOrUpdatePolicy(client, policyForCrossNamespaceRead(partition))
if err != nil {
return err
}
logger.Info("created cross-ns-catalog-read policy",
"policy-name", op.Name,
"policy-id", op.ID,
"partition", partition,
)
return nil
}
func (s *Sprawl) createAllServiceTokens() error {
for _, cluster := range s.topology.Clusters {
if err := s.createServiceTokens(cluster); err != nil {
return fmt.Errorf("createServiceTokens[%s]: %w", cluster.Name, err)
}
}
return nil
}
func (s *Sprawl) createServiceTokens(cluster *topology.Cluster) error {
var (
client = s.clients[cluster.Name]
logger = s.logger.With("cluster", cluster.Name)
)
sids := make(map[topology.ServiceID]struct{})
for _, node := range cluster.Nodes {
if !node.RunsWorkloads() || len(node.Services) == 0 || node.Disabled {
continue
}
for _, svc := range node.Services {
sid := svc.ID
if _, done := sids[sid]; done {
continue
}
var overridePolicy *api.ACLPolicy
if svc.IsMeshGateway {
var err error
overridePolicy, err = CreateOrUpdatePolicy(client, policyForMeshGateway(svc, cluster.Enterprise))
if err != nil {
return fmt.Errorf("could not create policy: %w", err)
}
}
token, err := CreateOrUpdateToken(client, tokenForService(svc, overridePolicy, cluster.Enterprise))
if err != nil {
return fmt.Errorf("could not create token: %w", err)
}
logger.Info("created service token",
"service", svc.ID.Name,
"namespace", svc.ID.Namespace,
"partition", svc.ID.Partition,
"token", token.SecretID,
)
s.secrets.SaveServiceToken(cluster.Name, sid, token.SecretID)
sids[sid] = struct{}{}
}
}
return nil
}
func CreateOrUpdateToken(client *api.Client, t *api.ACLToken) (*api.ACLToken, error) {
ac := client.ACL()
currentToken, err := getTokenByDescription(client, t.Description, &api.QueryOptions{
Partition: t.Partition,
Namespace: t.Namespace,
})
if err != nil {
return nil, err
} else if currentToken != nil {
t.AccessorID = currentToken.AccessorID
t.SecretID = currentToken.SecretID
}
if t.AccessorID != "" {
t, _, err = ac.TokenUpdate(t, nil)
} else {
t, _, err = ac.TokenCreate(t, nil)
}
if err != nil {
return nil, err
}
return t, nil
}
func getTokenByDescription(client *api.Client, description string, opts *api.QueryOptions) (*api.ACLToken, error) {
ac := client.ACL()
tokens, _, err := ac.TokenList(opts)
if err != nil {
return nil, err
}
for _, tokenEntry := range tokens {
if tokenEntry.Description == description {
token, _, err := ac.TokenRead(tokenEntry.AccessorID, opts)
if err != nil {
return nil, err
}
return token, nil
}
}
return nil, nil
}
func CreateOrUpdatePolicy(client *api.Client, p *api.ACLPolicy) (*api.ACLPolicy, error) {
ac := client.ACL()
currentPolicy, _, err := ac.PolicyReadByName(p.Name, &api.QueryOptions{
Partition: p.Partition,
Namespace: p.Namespace,
})
// There is a quirk about Consul 1.14.x, where: if reading a policy yields
// an empty result, we return "ACL not found". It's safe to ignore this here,
// because if the Client's ACL token truly doesn't exist, then the create fails below.
if err != nil && !strings.Contains(err.Error(), "ACL not found") {
return nil, err
} else if currentPolicy != nil {
p.ID = currentPolicy.ID
}
if p.ID != "" {
p, _, err = ac.PolicyUpdate(p, nil)
} else {
p, _, err = ac.PolicyCreate(p, nil)
}
if err != nil {
return nil, err
}
return p, nil
}

160
testing/deployer/sprawl/acl_rules.go
Unescape View File

@ -0,0 +1,160 @@
package sprawl
import (
"fmt"
"github.com/hashicorp/consul/api"
"github.com/hashicorp/consul/testing/deployer/topology"
)
func policyForCrossNamespaceRead(partition string) *api.ACLPolicy {
return &api.ACLPolicy{
Name: "cross-ns-catalog-read",
Description: "cross-ns-catalog-read",
Partition: partition,
Rules: fmt.Sprintf(`
partition %[1]q {
namespace_prefix "" {
node_prefix "" { policy = "read" }
service_prefix "" { policy = "read" }
}
}
`, partition),
}
}
const anonymousTokenAccessorID = "00000000-0000-0000-0000-000000000002"
func anonymousToken() *api.ACLToken {
return &api.ACLToken{
AccessorID: anonymousTokenAccessorID,
// SecretID: "anonymous",
Description: "anonymous",
Local: false,
Policies: []*api.ACLTokenPolicyLink{
{
Name: "anonymous",
},
},
}
}
func anonymousPolicy(enterprise bool) *api.ACLPolicy {
p := &api.ACLPolicy{
Name: "anonymous",
Description: "anonymous",
}
if enterprise {
p.Rules = `
partition_prefix "" {
namespace_prefix "" {
node_prefix "" { policy = "read" }
service_prefix "" { policy = "read" }
}
}
`
} else {
p.Rules = `
node_prefix "" { policy = "read" }
service_prefix "" { policy = "read" }
`
}
return p
}
func tokenForNode(node *topology.Node, enterprise bool) *api.ACLToken {
nid := node.ID()
tokenName := "agent--" + nid.ACLString()
token := &api.ACLToken{
Description: tokenName,
Local: false,
NodeIdentities: []*api.ACLNodeIdentity{{
NodeName: node.PodName(),
Datacenter: node.Datacenter,
}},
}
if enterprise {
token.Partition = node.Partition
token.Namespace = "default"
}
return token
}
func tokenForService(svc *topology.Service, overridePolicy *api.ACLPolicy, enterprise bool) *api.ACLToken {
token := &api.ACLToken{
Description: "service--" + svc.ID.ACLString(),
Local: false,
}
if overridePolicy != nil {
token.Policies = []*api.ACLTokenPolicyLink{{ID: overridePolicy.ID}}
} else {
token.ServiceIdentities = []*api.ACLServiceIdentity{{
ServiceName: svc.ID.Name,
}}
}
if enterprise {
token.Namespace = svc.ID.Namespace
token.Partition = svc.ID.Partition
}
return token
}
func policyForMeshGateway(svc *topology.Service, enterprise bool) *api.ACLPolicy {
policyName := "mesh-gateway--" + svc.ID.ACLString()
policy := &api.ACLPolicy{
Name: policyName,
Description: policyName,
}
if enterprise {
policy.Partition = svc.ID.Partition
policy.Namespace = "default"
}
if enterprise {
policy.Rules = `
namespace_prefix "" {
service "mesh-gateway" {
policy = "write"
}
service_prefix "" {
policy = "read"
}
node_prefix "" {
policy = "read"
}
}
agent_prefix "" {
policy = "read"
}
# for peering
mesh = "write"
peering = "read"
`
} else {
policy.Rules = `
service "mesh-gateway" {
policy = "write"
}
service_prefix "" {
policy = "read"
}
node_prefix "" {
policy = "read"
}
agent_prefix "" {
policy = "read"
}
# for peering
mesh = "write"
peering = "read"
`
}
return policy
}

520
testing/deployer/sprawl/boot.go
Unescape View File

@ -0,0 +1,520 @@
package sprawl
import (
"context"
"crypto/rand"
"encoding/base64"
"errors"
"fmt"
"strings"
"time"
"github.com/hashicorp/consul/api"
"github.com/hashicorp/go-multierror"
"github.com/hashicorp/consul/testing/deployer/sprawl/internal/build"
"github.com/hashicorp/consul/testing/deployer/sprawl/internal/secrets"
"github.com/hashicorp/consul/testing/deployer/sprawl/internal/tfgen"
"github.com/hashicorp/consul/testing/deployer/topology"
"github.com/hashicorp/consul/testing/deployer/util"
)
const (
sharedBootstrapToken = "root"
// sharedBootstrapToken = "ec59aa56-1996-4ff1-911a-f5d782552a13"
sharedAgentRecoveryToken = "22082b05-05c9-4a0a-b3da-b9685ac1d688"
)
func (s *Sprawl) launch() error {
return s.launchType(true)
}
func (s *Sprawl) relaunch() error {
return s.launchType(false)
}
func (s *Sprawl) launchType(firstTime bool) (launchErr error) {
if err := build.DockerImages(s.logger, s.runner, s.topology); err != nil {
return fmt.Errorf("build.DockerImages: %w", err)
}
if firstTime {
// Initialize secrets the easy way for now (same in all clusters).
gossipKey, err := newGossipKey()
if err != nil {
return fmt.Errorf("newGossipKey: %w", err)
}
for _, cluster := range s.topology.Clusters {
s.secrets.SaveGeneric(cluster.Name, secrets.BootstrapToken, sharedBootstrapToken)
s.secrets.SaveGeneric(cluster.Name, secrets.AgentRecovery, sharedAgentRecoveryToken)
s.secrets.SaveGeneric(cluster.Name, secrets.GossipKey, gossipKey)
// Give servers a copy of the bootstrap token for use as their agent tokens
// to avoid complicating the chicken/egg situation for startup.
for _, node := range cluster.Nodes {
if node.IsServer() { // include disabled
s.secrets.SaveAgentToken(cluster.Name, node.ID(), sharedBootstrapToken)
}
}
}
}
var cleanupFuncs []func()
defer func() {
for i := len(cleanupFuncs) - 1; i >= 0; i-- {
cleanupFuncs[i]()
}
}()
if firstTime {
var err error
s.generator, err = tfgen.NewGenerator(
s.logger.Named("tfgen"),
s.runner,
s.topology,
&s.secrets,
s.workdir,
s.license,
)
if err != nil {
return err
}
} else {
s.generator.SetTopology(s.topology)
}
cleanupFuncs = append(cleanupFuncs, func() {
// Log the error before the cleanup so you don't have to wait to see
// the cause.
if launchErr != nil {
s.logger.Error("fatal error during launch", "error", launchErr)
}
_ = s.generator.DestroyAllQuietly()
})
if firstTime {
// The networking phase is special. We have to pick a random subnet and
// hope. Once we have this established once it is immutable for future
// runs.
if err := s.initNetworkingAndVolumes(); err != nil {
return fmt.Errorf("initNetworkingAndVolumes: %w", err)
}
}
if err := s.assignIPAddresses(); err != nil {
return fmt.Errorf("assignIPAddresses: %w", err)
}
// The previous terraform run should have made the special volume for us.
if err := s.initTLS(context.TODO()); err != nil {
return fmt.Errorf("initTLS: %w", err)
}
if firstTime {
if err := s.createFirstTime(); err != nil {
return err
}
s.generator.MarkLaunched()
} else {
if err := s.updateExisting(); err != nil {
return err
}
}
if err := s.waitForPeeringEstablishment(); err != nil {
return fmt.Errorf("waitForPeeringEstablishment: %w", err)
}
cleanupFuncs = nil // reset
return nil
}
func (s *Sprawl) Stop() error {
var merr error
if s.generator != nil {
if err := s.generator.DestroyAllQuietly(); err != nil {
merr = multierror.Append(merr, err)
}
}
return merr
}
const dockerOutOfNetworksErrorMessage = `Unable to create network: Error response from daemon: Pool overlaps with other one on this address space`
var ErrDockerNetworkCollision = errors.New("could not create one or more docker networks for use due to subnet collision")
func (s *Sprawl) initNetworkingAndVolumes() error {
var lastErr error
for attempts := 0; attempts < 5; attempts++ {
err := s.generator.Generate(tfgen.StepNetworks)
if err != nil && strings.Contains(err.Error(), dockerOutOfNetworksErrorMessage) {
lastErr = ErrDockerNetworkCollision
s.logger.Warn(ErrDockerNetworkCollision.Error()+"; retrying", "attempt", attempts+1)
time.Sleep(1 * time.Second)
continue
} else if err != nil {
return fmt.Errorf("generator[networks]: %w", err)
}
return nil
}
return lastErr
}
func (s *Sprawl) assignIPAddresses() error {
// assign ips now that we have network ips known to us
for _, net := range s.topology.Networks {
if len(net.IPPool) == 0 {
return fmt.Errorf("network %q does not have any ip assignments", net.Name)
}
}
for _, cluster := range s.topology.Clusters {
for _, node := range cluster.Nodes {
for _, addr := range node.Addresses {
net, ok := s.topology.Networks[addr.Network]
if !ok {
return fmt.Errorf("unknown network %q", addr.Network)
}
addr.IPAddress = net.IPByIndex(node.Index)
}
}
}
return nil
}
func (s *Sprawl) initConsulServers() error {
if err := s.generator.Generate(tfgen.StepServers); err != nil {
return fmt.Errorf("generator[servers]: %w", err)
}
// s.logger.Info("ALL", "t", jd(s.topology)) // TODO
// Create token-less api clients first.
for _, cluster := range s.topology.Clusters {
node := cluster.FirstServer()
var err error
s.clients[cluster.Name], err = util.ProxyAPIClient(
node.LocalProxyPort(),
node.LocalAddress(),
8500,
"", /*no token yet*/
)
if err != nil {
return fmt.Errorf("error creating initial bootstrap client for cluster=%s: %w", cluster.Name, err)
}
}
if err := s.rejoinAllConsulServers(); err != nil {
return err
}
for _, cluster := range s.topology.Clusters {
err := s.bootstrapACLs(cluster.Name)
if err != nil {
return fmt.Errorf("bootstrap[%s]: %w", cluster.Name, err)
}
mgmtToken := s.secrets.ReadGeneric(cluster.Name, secrets.BootstrapToken)
// Reconfigure the clients to use a management token.
node := cluster.FirstServer()
s.clients[cluster.Name], err = util.ProxyAPIClient(
node.LocalProxyPort(),
node.LocalAddress(),
8500,
mgmtToken,
)
if err != nil {
return fmt.Errorf("error creating final client for cluster=%s: %v", cluster.Name, err)
}
// For some reason the grpc resolver stuff for partitions takes some
// time to get ready.
s.waitForLocalWrites(cluster, mgmtToken)
// Create tenancies so that the ACL tokens and clients have somewhere to go.
if cluster.Enterprise {
if err := s.initTenancies(cluster); err != nil {
return fmt.Errorf("initTenancies[%s]: %w", cluster.Name, err)
}
}
if err := s.populateInitialConfigEntries(cluster); err != nil {
return fmt.Errorf("populateInitialConfigEntries[%s]: %w", cluster.Name, err)
}
if err := s.createAnonymousToken(cluster); err != nil {
return fmt.Errorf("createAnonymousToken[%s]: %w", cluster.Name, err)
}
// Create tokens for all of the agents to use for anti-entropy.
//
// NOTE: this will cause the servers to roll to pick up the change to
// the acl{tokens{agent=XXX}}} section.
if err := s.createAgentTokens(cluster); err != nil {
return fmt.Errorf("createAgentTokens[%s]: %w", cluster.Name, err)
}
}
return nil
}
func (s *Sprawl) createFirstTime() error {
if err := s.initConsulServers(); err != nil {
return fmt.Errorf("initConsulServers: %w", err)
}
if err := s.generator.Generate(tfgen.StepAgents); err != nil {
return fmt.Errorf("generator[agents]: %w", err)
}
for _, cluster := range s.topology.Clusters {
if err := s.waitForClientAntiEntropyOnce(cluster); err != nil {
return fmt.Errorf("waitForClientAntiEntropyOnce[%s]: %w", cluster.Name, err)
}
}
// Ideally we start services WITH a token initially, so we pre-create them
// before running terraform for them.
if err := s.createAllServiceTokens(); err != nil {
return fmt.Errorf("createAllServiceTokens: %w", err)
}
if err := s.registerAllServicesForDataplaneInstances(); err != nil {
return fmt.Errorf("registerAllServicesForDataplaneInstances: %w", err)
}
// We can do this ahead, because we've incrementally run terraform as
// we went.
if err := s.registerAllServicesToAgents(); err != nil {
return fmt.Errorf("registerAllServicesToAgents: %w", err)
}
// NOTE: start services WITH token initially
if err := s.generator.Generate(tfgen.StepServices); err != nil {
return fmt.Errorf("generator[services]: %w", err)
}
if err := s.initPeerings(); err != nil {
return fmt.Errorf("initPeerings: %w", err)
}
return nil
}
func (s *Sprawl) updateExisting() error {
if err := s.preRegenTasks(); err != nil {
return fmt.Errorf("preRegenTasks: %w", err)
}
// We save all of the terraform to the end. Some of the containers will
// be a little broken until we can do stuff like register services to
// new agents, which we cannot do until they come up.
if err := s.generator.Generate(tfgen.StepRelaunch); err != nil {
return fmt.Errorf("generator[relaunch]: %w", err)
}
if err := s.postRegenTasks(); err != nil {
return fmt.Errorf("postRegenTasks: %w", err)
}
// TODO: enforce that peering relationships cannot change
// TODO: include a fixup version of new peerings?
return nil
}
func (s *Sprawl) preRegenTasks() error {
for _, cluster := range s.topology.Clusters {
// Create tenancies so that the ACL tokens and clients have somewhere to go.
if cluster.Enterprise {
if err := s.initTenancies(cluster); err != nil {
return fmt.Errorf("initTenancies[%s]: %w", cluster.Name, err)
}
}
if err := s.populateInitialConfigEntries(cluster); err != nil {
return fmt.Errorf("populateInitialConfigEntries[%s]: %w", cluster.Name, err)
}
// Create tokens for all of the agents to use for anti-entropy.
if err := s.createAgentTokens(cluster); err != nil {
return fmt.Errorf("createAgentTokens[%s]: %w", cluster.Name, err)
}
}
// Ideally we start services WITH a token initially, so we pre-create them
// before running terraform for them.
if err := s.createAllServiceTokens(); err != nil {
return fmt.Errorf("createAllServiceTokens: %w", err)
}
if err := s.registerAllServicesForDataplaneInstances(); err != nil {
return fmt.Errorf("registerAllServicesForDataplaneInstances: %w", err)
}
return nil
}
func (s *Sprawl) postRegenTasks() error {
if err := s.rejoinAllConsulServers(); err != nil {
return err
}
for _, cluster := range s.topology.Clusters {
var err error
mgmtToken := s.secrets.ReadGeneric(cluster.Name, secrets.BootstrapToken)
// Reconfigure the clients to use a management token.
node := cluster.FirstServer()
s.clients[cluster.Name], err = util.ProxyAPIClient(
node.LocalProxyPort(),
node.LocalAddress(),
8500,
mgmtToken,
)
if err != nil {
return fmt.Errorf("error creating final client for cluster=%s: %v", cluster.Name, err)
}
s.waitForLeader(cluster)
// For some reason the grpc resolver stuff for partitions takes some
// time to get ready.
s.waitForLocalWrites(cluster, mgmtToken)
}
for _, cluster := range s.topology.Clusters {
if err := s.waitForClientAntiEntropyOnce(cluster); err != nil {
return fmt.Errorf("waitForClientAntiEntropyOnce[%s]: %w", cluster.Name, err)
}
}
if err := s.registerAllServicesToAgents(); err != nil {
return fmt.Errorf("registerAllServicesToAgents: %w", err)
}
return nil
}
func (s *Sprawl) waitForLocalWrites(cluster *topology.Cluster, token string) {
var (
client = s.clients[cluster.Name]
logger = s.logger.With("cluster", cluster.Name)
)
tryKV := func() error {
_, err := client.KV().Put(&api.KVPair{
Key: "local-test",
Value: []byte("payload-for-local-test-in-" + cluster.Name),
}, nil)
return err
}
tryAP := func() error {
if !cluster.Enterprise {
return nil
}
_, _, err := client.Partitions().Create(context.Background(), &api.Partition{
Name: "placeholder",
}, &api.WriteOptions{Token: token})
return err
}
start := time.Now()
for attempts := 0; ; attempts++ {
if err := tryKV(); err != nil {
logger.Warn("local kv write failed; something is not ready yet", "error", err)
time.Sleep(500 * time.Millisecond)
continue
} else {
dur := time.Since(start)
logger.Info("local kv write success", "elapsed", dur, "retries", attempts)
}
break
}
if cluster.Enterprise {
start = time.Now()
for attempts := 0; ; attempts++ {
if err := tryAP(); err != nil {
logger.Warn("local partition write failed; something is not ready yet", "error", err)
time.Sleep(500 * time.Millisecond)
continue
} else {
dur := time.Since(start)
logger.Info("local partition write success", "elapsed", dur, "retries", attempts)
}
break
}
}
}
func (s *Sprawl) waitForClientAntiEntropyOnce(cluster *topology.Cluster) error {
var (
client = s.clients[cluster.Name]
logger = s.logger.With("cluster", cluster.Name)
)
var (
queryOptionList = cluster.PartitionQueryOptionsList()
start = time.Now()
cc = client.Catalog()
)
for {
// Enumerate all of the nodes that are currently in the catalog. This
// will overmatch including things like fake nodes for agentless but
// that's ok.
current := make(map[topology.NodeID]*api.Node)
for _, queryOpts := range queryOptionList {
nodes, _, err := cc.Nodes(queryOpts)
if err != nil {
return err
}
for _, node := range nodes {
nid := topology.NewNodeID(node.Node, node.Partition)
current[nid] = node
}
}
// See if we have them all.
var stragglers []topology.NodeID
for _, node := range cluster.Nodes {
if !node.IsAgent() || node.Disabled {
continue
}
nid := node.CatalogID()
got, ok := current[nid]
if ok && len(got.TaggedAddresses) > 0 {
// this is a field that is not updated just due to serf reconcile
continue
}
stragglers = append(stragglers, nid)
}
if len(stragglers) == 0 {
dur := time.Since(start)
logger.Info("all nodes have posted node updates, so first anti-entropy has happened", "elapsed", dur)
return nil
}
logger.Info("not all client nodes have posted node updates yet", "nodes", stragglers)
time.Sleep(1 * time.Second)
}
}
func newGossipKey() (string, error) {
key := make([]byte, 16)
n, err := rand.Reader.Read(key)
if err != nil {
return "", fmt.Errorf("Error reading random data: %s", err)
}
if n != 16 {
return "", fmt.Errorf("Couldn't read enough entropy. Generate more entropy!")
}
return base64.StdEncoding.EncodeToString(key), nil
}

425
testing/deployer/sprawl/catalog.go
Unescape View File

@ -0,0 +1,425 @@
package sprawl
import (
"fmt"
"net/http"
"time"
"github.com/hashicorp/consul/api"
"github.com/hashicorp/consul/testing/deployer/topology"
"github.com/hashicorp/consul/testing/deployer/util"
)
func (s *Sprawl) registerAllServicesToAgents() error {
for _, cluster := range s.topology.Clusters {
if err := s.registerServicesToAgents(cluster); err != nil {
return fmt.Errorf("registerServicesToAgents[%s]: %w", cluster.Name, err)
}
}
return nil
}
func (s *Sprawl) registerAllServicesForDataplaneInstances() error {
for _, cluster := range s.topology.Clusters {
if err := s.registerServicesForDataplaneInstances(cluster); err != nil {
return fmt.Errorf("registerServicesForDataplaneInstances[%s]: %w", cluster.Name, err)
}
}
return nil
}
func (s *Sprawl) registerServicesToAgents(cluster *topology.Cluster) error {
for _, node := range cluster.Nodes {
if !node.RunsWorkloads() || len(node.Services) == 0 || node.Disabled {
continue
}
if !node.IsAgent() {
continue
}
agentClient, err := util.ProxyAPIClient(
node.LocalProxyPort(),
node.LocalAddress(),
8500,
"", /*token will be in request*/
)
if err != nil {
return err
}
for _, svc := range node.Services {
if err := s.registerAgentService(agentClient, cluster, node, svc); err != nil {
return err
}
}
}
return nil
}
func (s *Sprawl) registerAgentService(
agentClient *api.Client,
cluster *topology.Cluster,
node *topology.Node,
svc *topology.Service,
) error {
if !node.IsAgent() {
panic("called wrong method type")
}
if svc.IsMeshGateway {
return nil // handled at startup time for agent-full, but won't be for agent-less
}
var (
logger = s.logger.With("cluster", cluster.Name)
)
reg := &api.AgentServiceRegistration{
ID: svc.ID.Name,
Name: svc.ID.Name,
Port: svc.Port,
Meta: svc.Meta,
}
if cluster.Enterprise {
reg.Namespace = svc.ID.Namespace
reg.Partition = svc.ID.Partition
}
if !svc.DisableServiceMesh {
var upstreams []api.Upstream
for _, u := range svc.Upstreams {
uAPI := api.Upstream{
DestinationPeer: u.Peer,
DestinationName: u.ID.Name,
LocalBindAddress: u.LocalAddress,
LocalBindPort: u.LocalPort,
// Config map[string]interface{} `json:",omitempty" bexpr:"-"`
// MeshGateway MeshGatewayConfig `json:",omitempty"`
}
if cluster.Enterprise {
uAPI.DestinationNamespace = u.ID.Namespace
if u.Peer == "" {
uAPI.DestinationPartition = u.ID.Partition
}
}
upstreams = append(upstreams, uAPI)
}
reg.Connect = &api.AgentServiceConnect{
SidecarService: &api.AgentServiceRegistration{
Proxy: &api.AgentServiceConnectProxyConfig{
Upstreams: upstreams,
},
},
}
}
switch {
case svc.CheckTCP != "":
chk := &api.AgentServiceCheck{
Name: "up",
TCP: svc.CheckTCP,
Interval: "5s",
Timeout: "1s",
}
reg.Checks = append(reg.Checks, chk)
case svc.CheckHTTP != "":
chk := &api.AgentServiceCheck{
Name: "up",
HTTP: svc.CheckHTTP,
Method: "GET",
Interval: "5s",
Timeout: "1s",
}
reg.Checks = append(reg.Checks, chk)
}
// Switch token for every request.
hdr := make(http.Header)
hdr.Set("X-Consul-Token", s.secrets.ReadServiceToken(cluster.Name, svc.ID))
agentClient.SetHeaders(hdr)
RETRY:
if err := agentClient.Agent().ServiceRegister(reg); err != nil {
if isACLNotFound(err) {
time.Sleep(50 * time.Millisecond)
goto RETRY
}
return fmt.Errorf("failed to register service %q to node %q: %w", svc.ID, node.ID(), err)
}
logger.Info("registered service to client agent",
"service", svc.ID.Name,
"node", node.Name,
"namespace", svc.ID.Namespace,
"partition", svc.ID.Partition,
)
return nil
}
func (s *Sprawl) registerServicesForDataplaneInstances(cluster *topology.Cluster) error {
for _, node := range cluster.Nodes {
if !node.RunsWorkloads() || len(node.Services) == 0 || node.Disabled {
continue
}
if !node.IsDataplane() {
continue
}
if err := s.registerCatalogNode(cluster, node); err != nil {
return fmt.Errorf("error registering virtual node: %w", err)
}
for _, svc := range node.Services {
if err := s.registerCatalogService(cluster, node, svc); err != nil {
return fmt.Errorf("error registering service: %w", err)
}
if !svc.DisableServiceMesh {
if err := s.registerCatalogSidecarService(cluster, node, svc); err != nil {
return fmt.Errorf("error registering sidecar service: %w", err)
}
}
}
}
return nil
}
func (s *Sprawl) registerCatalogNode(
cluster *topology.Cluster,
node *topology.Node,
) error {
if !node.IsDataplane() {
panic("called wrong method type")
}
var (
client = s.clients[cluster.Name]
logger = s.logger.With("cluster", cluster.Name)
)
reg := &api.CatalogRegistration{
Node: node.PodName(),
Address: node.LocalAddress(),
NodeMeta: map[string]string{
"dataplane-faux": "1",
},
}
if cluster.Enterprise {
reg.Partition = node.Partition
}
// register synthetic node
RETRY:
if _, err := client.Catalog().Register(reg, nil); err != nil {
if isACLNotFound(err) {
time.Sleep(50 * time.Millisecond)
goto RETRY
}
return fmt.Errorf("error registering virtual node %s: %w", node.ID(), err)
}
logger.Info("virtual node created",
"node", node.ID(),
)
return nil
}
func (s *Sprawl) registerCatalogService(
cluster *topology.Cluster,
node *topology.Node,
svc *topology.Service,
) error {
if !node.IsDataplane() {
panic("called wrong method type")
}
var (
client = s.clients[cluster.Name]
logger = s.logger.With("cluster", cluster.Name)
)
reg := serviceToCatalogRegistration(cluster, node, svc)
RETRY:
if _, err := client.Catalog().Register(reg, nil); err != nil {
if isACLNotFound(err) {
time.Sleep(50 * time.Millisecond)
goto RETRY
}
return fmt.Errorf("error registering service %s to node %s: %w", svc.ID, node.ID(), err)
}
logger.Info("dataplane service created",
"service", svc.ID,
"node", node.ID(),
)
return nil
}
func (s *Sprawl) registerCatalogSidecarService(
cluster *topology.Cluster,
node *topology.Node,
svc *topology.Service,
) error {
if !node.IsDataplane() {
panic("called wrong method type")
}
if svc.DisableServiceMesh {
panic("not valid")
}
var (
client = s.clients[cluster.Name]
logger = s.logger.With("cluster", cluster.Name)
)
pid, reg := serviceToSidecarCatalogRegistration(cluster, node, svc)
RETRY:
if _, err := client.Catalog().Register(reg, nil); err != nil {
if isACLNotFound(err) {
time.Sleep(50 * time.Millisecond)
goto RETRY
}
return fmt.Errorf("error registering service %s to node %s: %w", svc.ID, node.ID(), err)
}
logger.Info("dataplane sidecar service created",
"service", pid,
"node", node.ID(),
)
return nil
}
func serviceToCatalogRegistration(
cluster *topology.Cluster,
node *topology.Node,
svc *topology.Service,
) *api.CatalogRegistration {
reg := &api.CatalogRegistration{
Node: node.PodName(),
SkipNodeUpdate: true,
Service: &api.AgentService{
Kind: api.ServiceKindTypical,
ID: svc.ID.Name,
Service: svc.ID.Name,
Meta: svc.Meta,
Port: svc.Port,
Address: node.LocalAddress(),
},
}
if node.HasPublicAddress() {
reg.TaggedAddresses = map[string]string{
"lan": node.LocalAddress(),
"lan_ipv4": node.LocalAddress(),
"wan": node.PublicAddress(),
"wan_ipv4": node.PublicAddress(),
}
}
if cluster.Enterprise {
reg.Partition = svc.ID.Partition
reg.Service.Namespace = svc.ID.Namespace
reg.Service.Partition = svc.ID.Partition
}
if svc.HasCheck() {
chk := &api.HealthCheck{
Name: "external sync",
// Type: "external-sync",
Status: "passing", // TODO
ServiceID: svc.ID.Name,
ServiceName: svc.ID.Name,
Output: "",
}
if cluster.Enterprise {
chk.Namespace = svc.ID.Namespace
chk.Partition = svc.ID.Partition
}
switch {
case svc.CheckTCP != "":
chk.Definition.TCP = svc.CheckTCP
case svc.CheckHTTP != "":
chk.Definition.HTTP = svc.CheckHTTP
chk.Definition.Method = "GET"
}
reg.Checks = append(reg.Checks, chk)
}
return reg
}
func serviceToSidecarCatalogRegistration(
cluster *topology.Cluster,
node *topology.Node,
svc *topology.Service,
) (topology.ServiceID, *api.CatalogRegistration) {
pid := svc.ID
pid.Name += "-sidecar-proxy"
reg := &api.CatalogRegistration{
Node: node.PodName(),
SkipNodeUpdate: true,
Service: &api.AgentService{
Kind: api.ServiceKindConnectProxy,
ID: pid.Name,
Service: pid.Name,
Meta: svc.Meta,
Port: svc.EnvoyPublicListenerPort,
Address: node.LocalAddress(),
Proxy: &api.AgentServiceConnectProxyConfig{
DestinationServiceName: svc.ID.Name,
DestinationServiceID: svc.ID.Name,
LocalServicePort: svc.Port,
},
},
Checks: []*api.HealthCheck{{
Name: "external sync",
// Type: "external-sync",
Status: "passing", // TODO
ServiceID: pid.Name,
ServiceName: pid.Name,
Definition: api.HealthCheckDefinition{
TCP: fmt.Sprintf("%s:%d", node.LocalAddress(), svc.EnvoyPublicListenerPort),
},
Output: "",
}},
}
if node.HasPublicAddress() {
reg.TaggedAddresses = map[string]string{
"lan": node.LocalAddress(),
"lan_ipv4": node.LocalAddress(),
"wan": node.PublicAddress(),
"wan_ipv4": node.PublicAddress(),
}
}
if cluster.Enterprise {
reg.Partition = pid.Partition
reg.Service.Namespace = pid.Namespace
reg.Service.Partition = pid.Partition
reg.Checks[0].Namespace = pid.Namespace
reg.Checks[0].Partition = pid.Partition
}
for _, u := range svc.Upstreams {
pu := api.Upstream{
DestinationName: u.ID.Name,
DestinationPeer: u.Peer,
LocalBindAddress: u.LocalAddress,
LocalBindPort: u.LocalPort,
}
if cluster.Enterprise {
pu.DestinationNamespace = u.ID.Namespace
if u.Peer == "" {
pu.DestinationPartition = u.ID.Partition
}
}
reg.Service.Proxy.Upstreams = append(reg.Service.Proxy.Upstreams, pu)
}
return pid, reg
}

58
testing/deployer/sprawl/configentries.go
Unescape View File

@ -0,0 +1,58 @@
package sprawl
import (
"fmt"
"strings"
"time"
"github.com/hashicorp/consul/api"
"github.com/hashicorp/consul/testing/deployer/topology"
)
func (s *Sprawl) populateInitialConfigEntries(cluster *topology.Cluster) error {
if len(cluster.InitialConfigEntries) == 0 {
return nil
}
var (
client = s.clients[cluster.Name]
logger = s.logger.With("cluster", cluster.Name)
)
for _, ce := range cluster.InitialConfigEntries {
_, _, err := client.ConfigEntries().Set(ce, nil)
if err != nil {
if ce.GetKind() == api.ServiceIntentions && strings.Contains(err.Error(), intentionsMigrationError) {
logger.Warn("known error writing initial config entry; trying again",
"kind", ce.GetKind(),
"name", ce.GetName(),
"namespace", ce.GetNamespace(),
"partition", ce.GetPartition(),
"error", err,
)
time.Sleep(500 * time.Millisecond)
continue
}
return fmt.Errorf(
"error persisting config entry kind=%q name=%q namespace=%q partition=%q: %w",
ce.GetKind(),
ce.GetName(),
ce.GetNamespace(),
ce.GetPartition(),
err,
)
}
logger.Info("wrote initial config entry",
"kind", ce.GetKind(),
"name", ce.GetName(),
"namespace", ce.GetNamespace(),
"partition", ce.GetPartition(),
)
}
return nil
}
const intentionsMigrationError = `Intentions are read only while being upgraded to config entries`

98
testing/deployer/sprawl/consul.go
Unescape View File

@ -0,0 +1,98 @@
package sprawl
import (
"errors"
"fmt"
"time"
"github.com/hashicorp/consul/api"
"github.com/hashicorp/consul/testing/deployer/sprawl/internal/secrets"
"github.com/hashicorp/consul/testing/deployer/topology"
"github.com/hashicorp/consul/testing/deployer/util"
)
func getLeader(client *api.Client) (string, error) {
leaderAdd, err := client.Status().Leader()
if err != nil {
return "", fmt.Errorf("could not query leader: %w", err)
}
if leaderAdd == "" {
return "", errors.New("no leader available")
}
return leaderAdd, nil
}
func (s *Sprawl) waitForLeader(cluster *topology.Cluster) {
var (
client = s.clients[cluster.Name]
logger = s.logger.With("cluster", cluster.Name)
)
for {
leader, err := client.Status().Leader()
if leader != "" && err == nil {
logger.Info("cluster has leader", "leader_addr", leader)
return
}
logger.Info("cluster has no leader yet", "error", err)
time.Sleep(500 * time.Millisecond)
}
}
func (s *Sprawl) rejoinAllConsulServers() error {
// Join the servers together.
for _, cluster := range s.topology.Clusters {
if err := s.rejoinServers(cluster); err != nil {
return fmt.Errorf("rejoinServers[%s]: %w", cluster.Name, err)
}
s.waitForLeader(cluster)
}
return nil
}
func (s *Sprawl) rejoinServers(cluster *topology.Cluster) error {
var (
// client = s.clients[cluster.Name]
logger = s.logger.With("cluster", cluster.Name)
)
servers := cluster.ServerNodes()
recoveryToken := s.secrets.ReadGeneric(cluster.Name, secrets.AgentRecovery)
node0, rest := servers[0], servers[1:]
client, err := util.ProxyNotPooledAPIClient(
node0.LocalProxyPort(),
node0.LocalAddress(),
8500,
recoveryToken,
)
if err != nil {
return fmt.Errorf("could not get client for %q: %w", node0.ID(), err)
}
logger.Info("joining servers together",
"from", node0.ID(),
"rest", nodeSliceToNodeIDSlice(rest),
)
for _, node := range rest {
for {
err = client.Agent().Join(node.LocalAddress(), false)
if err == nil {
break
}
logger.Warn("could not join", "from", node0.ID(), "to", node.ID(), "error", err)
time.Sleep(500 * time.Millisecond)
}
}
return nil
}
func nodeSliceToNodeIDSlice(nodes []*topology.Node) []topology.NodeID {
var out []topology.NodeID
for _, node := range nodes {
out = append(out, node.ID())
}
return out
}

8
testing/deployer/sprawl/debug.go
Unescape View File

@ -0,0 +1,8 @@
package sprawl
import "encoding/json"
func jd(v any) string {
b, _ := json.MarshalIndent(v, "", " ")
return string(b)
}

170
testing/deployer/sprawl/details.go
Unescape View File

@ -0,0 +1,170 @@
package sprawl
import (
"bytes"
"fmt"
"sort"
"strconv"
"strings"
"text/tabwriter"
)
// PrintDetails will dump relevant addressing and naming data to the logger for
// human interaction purposes.
func (s *Sprawl) PrintDetails() error {
det := logDetails{
TopologyID: s.topology.ID,
}
for _, cluster := range s.topology.Clusters {
client := s.clients[cluster.Name]
cfg, err := client.Operator().RaftGetConfiguration(nil)
if err != nil {
return fmt.Errorf("could not get raft config for cluster %q: %w", cluster.Name, err)
}
var leaderNode string
for _, svr := range cfg.Servers {
if svr.Leader {
leaderNode = strings.TrimSuffix(svr.Node, "-pod")
}
}
cd := clusterDetails{
Name: cluster.Name,
Leader: leaderNode,
}
for _, node := range cluster.Nodes {
if node.Disabled {
continue
}
var addrs []string
for _, addr := range node.Addresses {
addrs = append(addrs, addr.Network+"="+addr.IPAddress)
}
sort.Strings(addrs)
if node.IsServer() {
cd.Apps = append(cd.Apps, appDetail{
Type: "server",
Container: node.DockerName(),
Addresses: addrs,
ExposedPort: node.ExposedPort(8500),
})
}
for _, svc := range node.Services {
if svc.IsMeshGateway {
cd.Apps = append(cd.Apps, appDetail{
Type: "mesh-gateway",
Container: node.DockerName(),
ExposedPort: node.ExposedPort(svc.Port),
ExposedEnvoyAdminPort: node.ExposedPort(svc.EnvoyAdminPort),
Addresses: addrs,
Service: svc.ID.String(),
})
} else {
cd.Apps = append(cd.Apps, appDetail{
Type: "app",
Container: node.DockerName(),
ExposedPort: node.ExposedPort(svc.Port),
ExposedEnvoyAdminPort: node.ExposedPort(svc.EnvoyAdminPort),
Addresses: addrs,
Service: svc.ID.String(),
})
}
}
}
det.Clusters = append(det.Clusters, cd)
}
var buf bytes.Buffer
w := tabwriter.NewWriter(&buf, 0, 0, 3, ' ', tabwriter.Debug)
score := map[string]int{
"server": 0,
"mesh-gateway": 1,
"app": 2,
}
for _, cluster := range det.Clusters {
fmt.Fprintf(w, "CLUSTER\tTYPE\tCONTAINER\tNAME\tADDRS\tPORTS\t\n")
sort.Slice(cluster.Apps, func(i, j int) bool {
a := cluster.Apps[i]
b := cluster.Apps[j]
asc := score[a.Type]
bsc := score[b.Type]
if asc < bsc {
return true
} else if asc > bsc {
return false
}
if a.Container < b.Container {
return true
} else if a.Container > b.Container {
return false
}
if a.Service < b.Service {
return true
} else if a.Service > b.Service {
return false
}
return a.ExposedPort < b.ExposedPort
})
for _, d := range cluster.Apps {
if d.Type == "server" && d.Container == cluster.Leader {
d.Type = "leader"
}
portStr := "app=" + strconv.Itoa(d.ExposedPort)
if d.ExposedEnvoyAdminPort > 0 {
portStr += " envoy=" + strconv.Itoa(d.ExposedEnvoyAdminPort)
}
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t\n",
cluster.Name,
d.Type,
d.Container,
d.Service,
strings.Join(d.Addresses, ", "),
portStr,
)
}
fmt.Fprintf(w, "\t\t\t\t\t\n")
}
w.Flush()
s.logger.Info("CURRENT SPRAWL DETAILS", "details", buf.String())
return nil
}
type logDetails struct {
TopologyID string
Clusters []clusterDetails
}
type clusterDetails struct {
Name string
Leader string
Apps []appDetail
}
type appDetail struct {
Type string // server|mesh-gateway|app
Container string
Addresses []string
ExposedPort int `json:",omitempty"`
ExposedEnvoyAdminPort int `json:",omitempty"`
// just services
Service string `json:",omitempty"`
}

174
testing/deployer/sprawl/ent.go
Unescape View File

@ -0,0 +1,174 @@
package sprawl
import (
"bytes"
"context"
"fmt"
"os"
"strings"
"time"
"github.com/hashicorp/consul/api"
"github.com/hashicorp/consul/testing/deployer/topology"
)
func (s *Sprawl) ensureLicense() error {
if s.license != "" {
return nil
}
v, err := readLicense()
if err != nil {
return err
}
s.license = v
return nil
}
func readLicense() (string, error) {
if license := os.Getenv("CONSUL_LICENSE"); license != "" {
return license, nil
}
licensePath := os.Getenv("CONSUL_LICENSE_PATH")
if licensePath == "" {
return "", nil
}
licenseBytes, err := os.ReadFile(licensePath)
if err != nil {
return "", err
}
return strings.TrimSpace(string(licenseBytes)), nil
}
func (s *Sprawl) initTenancies(cluster *topology.Cluster) error {
var (
client = s.clients[cluster.Name]
logger = s.logger.With("cluster", cluster.Name)
)
// TODO: change this to UPSERT
var (
partClient = client.Partitions()
nsClient = client.Namespaces()
partitionNameList []string
)
for _, ap := range cluster.Partitions {
if ap.Name != "default" {
old, _, err := partClient.Read(context.Background(), ap.Name, nil)
if err != nil {
return fmt.Errorf("error reading partition %q: %w", ap.Name, err)
}
if old == nil {
obj := &api.Partition{
Name: ap.Name,
}
_, _, err := partClient.Create(context.Background(), obj, nil)
if err != nil {
return fmt.Errorf("error creating partition %q: %w", ap.Name, err)
}
logger.Info("created partition", "partition", ap.Name)
}
partitionNameList = append(partitionNameList, ap.Name)
}
if err := s.createCrossNamespaceCatalogReadPolicies(cluster, ap.Name); err != nil {
return fmt.Errorf("createCrossNamespaceCatalogReadPolicies[%s]: %w", ap.Name, err)
}
for _, ns := range ap.Namespaces {
old, _, err := nsClient.Read(ns, &api.QueryOptions{Partition: ap.Name})
if err != nil {
return err
}
if old == nil {
obj := &api.Namespace{
Partition: ap.Name,
Name: ns,
ACLs: &api.NamespaceACLConfig{
PolicyDefaults: []api.ACLLink{
{Name: "cross-ns-catalog-read"},
},
},
}
if ns == "default" {
_, _, err := nsClient.Update(obj, nil)
if err != nil {
return err
}
logger.Info("updated namespace", "namespace", ns, "partition", ap.Name)
} else {
_, _, err := nsClient.Create(obj, nil)
if err != nil {
return err
}
logger.Info("created namespace", "namespace", ns, "partition", ap.Name)
}
}
}
}
if err := s.waitUntilPartitionedSerfIsReady(context.TODO(), cluster, partitionNameList); err != nil {
return fmt.Errorf("waitUntilPartitionedSerfIsReady: %w", err)
}
return nil
}
func (s *Sprawl) waitUntilPartitionedSerfIsReady(ctx context.Context, cluster *topology.Cluster, partitions []string) error {
var (
logger = s.logger.With("cluster", cluster.Name)
)
readyLogs := make(map[string]string)
for _, partition := range partitions {
readyLogs[partition] = `agent.server: Added serf partition to gossip network: partition=` + partition
}
start := time.Now()
logger.Info("waiting for partitioned serf to be ready on all servers", "partitions", partitions)
for _, node := range cluster.Nodes {
if !node.IsServer() || node.Disabled {
continue
}
var buf bytes.Buffer
for {
buf.Reset()
err := s.runner.DockerExec(ctx, []string{
"logs", node.DockerName(),
}, &buf, nil)
if err != nil {
return fmt.Errorf("could not fetch docker logs from node %q: %w", node.ID(), err)
}
var (
body = buf.String()
found []string
)
for partition, readyLog := range readyLogs {
if strings.Contains(body, readyLog) {
found = append(found, partition)
}
}
if len(found) == len(readyLogs) {
break
}
}
time.Sleep(500 * time.Millisecond)
}
logger.Info("partitioned serf is ready on all servers", "partitions", partitions, "elapsed", time.Since(start))
return nil
}

11
testing/deployer/sprawl/helpers.go
Unescape View File

@ -0,0 +1,11 @@
package sprawl
// Deprecated: remove
func TruncateSquidError(err error) error {
return err
}
// Deprecated: remove
func IsSquid503(err error) bool {
return false
}

83
testing/deployer/sprawl/internal/build/docker.go
Unescape View File

@ -0,0 +1,83 @@
package build
import (
"context"
"strings"
"github.com/hashicorp/go-hclog"
"github.com/hashicorp/consul/testing/deployer/sprawl/internal/runner"
"github.com/hashicorp/consul/testing/deployer/topology"
)
const dockerfileEnvoy = `
ARG CONSUL_IMAGE
ARG ENVOY_IMAGE
FROM ${CONSUL_IMAGE}
FROM ${ENVOY_IMAGE}
COPY --from=0 /bin/consul /bin/consul
`
// FROM hashicorp/consul-dataplane:latest
// COPY --from=busybox:uclibc /bin/sh /bin/sh
const dockerfileDataplane = `
ARG DATAPLANE_IMAGE
FROM busybox:latest
FROM ${DATAPLANE_IMAGE}
COPY --from=0 /bin/busybox /bin/busybox
USER 0:0
RUN ["busybox", "--install", "/bin", "-s"]
USER 100:0
ENTRYPOINT []
`
func DockerImages(
logger hclog.Logger,
run *runner.Runner,
t *topology.Topology,
) error {
logw := logger.Named("docker").StandardWriter(&hclog.StandardLoggerOptions{ForceLevel: hclog.Info})
built := make(map[string]struct{})
for _, c := range t.Clusters {
for _, n := range c.Nodes {
joint := n.Images.EnvoyConsulImage()
if _, ok := built[joint]; joint != "" && !ok {
logger.Info("building image", "image", joint)
err := run.DockerExec(context.TODO(), []string{
"build",
"--build-arg",
"CONSUL_IMAGE=" + n.Images.Consul,
"--build-arg",
"ENVOY_IMAGE=" + n.Images.Envoy,
"-t", joint,
"-",
}, logw, strings.NewReader(dockerfileEnvoy))
if err != nil {
return err
}
built[joint] = struct{}{}
}
cdp := n.Images.LocalDataplaneImage()
if _, ok := built[cdp]; cdp != "" && !ok {
logger.Info("building image", "image", cdp)
err := run.DockerExec(context.TODO(), []string{
"build",
"--build-arg",
"DATAPLANE_IMAGE=" + n.Images.Dataplane,
"-t", cdp,
"-",
}, logw, strings.NewReader(dockerfileDataplane))
if err != nil {
return err
}
built[cdp] = struct{}{}
}
}
}
return nil
}

120
testing/deployer/sprawl/internal/runner/exec.go
Unescape View File

@ -0,0 +1,120 @@
package runner
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"os"
"os/exec"
"github.com/hashicorp/go-hclog"
)
type Runner struct {
logger hclog.Logger
tfBin string
dockerBin string
}
func Load(logger hclog.Logger) (*Runner, error) {
r := &Runner{
logger: logger,
}
type item struct {
name string
dest *string
warn string // optional
}
lookup := []item{
{"docker", &r.dockerBin, ""},
{"terraform", &r.tfBin, ""},
}
var (
bins []string
err error
)
for _, i := range lookup {
*i.dest, err = exec.LookPath(i.name)
if err != nil {
if errors.Is(err, exec.ErrNotFound) {
if i.warn != "" {
return nil, fmt.Errorf("Could not find %q on path (%s): %w", i.name, i.warn, err)
} else {
return nil, fmt.Errorf("Could not find %q on path: %w", i.name, err)
}
}
return nil, fmt.Errorf("Unexpected failure looking for %q on path: %w", i.name, err)
}
bins = append(bins, *i.dest)
}
r.logger.Trace("using binaries", "paths", bins)
return r, nil
}
func (r *Runner) DockerExec(ctx context.Context, args []string, stdout io.Writer, stdin io.Reader) error {
return cmdExec(ctx, "docker", r.dockerBin, args, stdout, nil, stdin, "")
}
func (r *Runner) DockerExecWithStderr(ctx context.Context, args []string, stdout, stderr io.Writer, stdin io.Reader) error {
return cmdExec(ctx, "docker", r.dockerBin, args, stdout, stderr, stdin, "")
}
func (r *Runner) TerraformExec(ctx context.Context, args []string, stdout io.Writer, workdir string) error {
return cmdExec(ctx, "terraform", r.tfBin, args, stdout, nil, nil, workdir)
}
func cmdExec(ctx context.Context, name, binary string, args []string, stdout, stderr io.Writer, stdin io.Reader, dir string) error {
if binary == "" {
panic("binary named " + name + " was not detected")
}
var errWriter bytes.Buffer
if stdout == nil {
stdout = os.Stdout // TODO: wrap logs
}
cmd := exec.CommandContext(ctx, binary, args...)
if dir != "" {
cmd.Dir = dir
}
cmd.Stdout = stdout
cmd.Stderr = &errWriter
if stderr != nil {
cmd.Stderr = io.MultiWriter(stderr, cmd.Stderr)
}
cmd.Stdin = stdin
if err := cmd.Run(); err != nil {
return &ExecError{
BinaryName: name,
Err: err,
ErrorOutput: errWriter.String(),
}
}
return nil
}
type ExecError struct {
BinaryName string
ErrorOutput string
Err error
}
func (e *ExecError) Unwrap() error {
return e.Err
}
func (e *ExecError) Error() string {
return fmt.Sprintf(
"could not invoke %q: %v : %s",
e.BinaryName,
e.Err,
e.ErrorOutput,
)
}

70
testing/deployer/sprawl/internal/secrets/store.go
Unescape View File

@ -0,0 +1,70 @@
package secrets
import (
"net/url"
"strings"
"github.com/hashicorp/consul/testing/deployer/topology"
)
type Store struct {
m map[string]string
}
const (
GossipKey = "gossip"
BootstrapToken = "bootstrap-token"
AgentRecovery = "agent-recovery"
)
func (s *Store) SaveGeneric(cluster, name, value string) {
s.save(encode(cluster, "generic", name), value)
}
func (s *Store) ReadGeneric(cluster, name string) string {
return s.read(encode(cluster, "generic", name))
}
func (s *Store) SaveAgentToken(cluster string, nid topology.NodeID, value string) {
s.save(encode(cluster, "agent", nid.String()), value)
}
func (s *Store) ReadAgentToken(cluster string, nid topology.NodeID) string {
return s.read(encode(cluster, "agent", nid.String()))
}
func (s *Store) SaveServiceToken(cluster string, sid topology.ServiceID, value string) {
s.save(encode(cluster, "service", sid.String()), value)
}
func (s *Store) ReadServiceToken(cluster string, sid topology.ServiceID) string {
return s.read(encode(cluster, "service", sid.String()))
}
func (s *Store) save(key, value string) {
if s.m == nil {
s.m = make(map[string]string)
}
s.m[key] = value
}
func (s *Store) read(key string) string {
if s.m == nil {
return ""
}
v, ok := s.m[key]
if !ok {
return ""
}
return v
}
func encode(parts ...string) string {
var out []string
for _, p := range parts {
out = append(out, url.QueryEscape(p))
}
return strings.Join(out, "/")
}

215
testing/deployer/sprawl/internal/tfgen/agent.go
Unescape View File

@ -0,0 +1,215 @@
package tfgen
import (
"fmt"
"strings"
"github.com/hashicorp/hcl/v2/hclwrite"
"github.com/hashicorp/consul/testing/deployer/sprawl/internal/secrets"
"github.com/hashicorp/consul/testing/deployer/topology"
)
func (g *Generator) generateAgentHCL(node *topology.Node) (string, error) {
if !node.IsAgent() {
return "", fmt.Errorf("not an agent")
}
cluster, ok := g.topology.Clusters[node.Cluster]
if !ok {
return "", fmt.Errorf("no such cluster: %s", node.Cluster)
}
var b HCLBuilder
b.add("server", node.IsServer())
b.add("bind_addr", "0.0.0.0")
b.add("client_addr", "0.0.0.0")
b.add("advertise_addr", `{{ GetInterfaceIP "eth0" }}`)
b.add("datacenter", node.Datacenter)
b.add("disable_update_check", true)
b.add("log_level", "trace")
b.add("enable_debug", true)
b.add("use_streaming_backend", true)
// speed up leaves
b.addBlock("performance", func() {
b.add("leave_drain_time", "50ms")
})
b.add("primary_datacenter", node.Datacenter)
// Using retry_join here is bad because changing server membership will
// destroy and recreate all of the servers
// if !node.IsServer() {
b.addSlice("retry_join", []string{"server." + node.Cluster + "-consulcluster.lan"})
b.add("retry_interval", "1s")
// }
if node.IsServer() {
b.addBlock("peering", func() {
b.add("enabled", true)
})
}
b.addBlock("ui_config", func() {
b.add("enabled", true)
})
b.addBlock("telemetry", func() {
b.add("disable_hostname", true)
b.add("prometheus_retention_time", "168h")
})
b.add("encrypt", g.sec.ReadGeneric(node.Cluster, secrets.GossipKey))
{
var (
root = "/consul/config/certs"
caFile = root + "/consul-agent-ca.pem"
certFile = root + "/" + node.TLSCertPrefix + ".pem"
certKey = root + "/" + node.TLSCertPrefix + "-key.pem"
)
b.addBlock("tls", func() {
b.addBlock("internal_rpc", func() {
b.add("ca_file", caFile)
b.add("cert_file", certFile)
b.add("key_file", certKey)
b.add("verify_incoming", true)
b.add("verify_server_hostname", true)
b.add("verify_outgoing", true)
})
// if cfg.EncryptionTLSAPI {
// b.addBlock("https", func() {
// b.add("ca_file", caFile)
// b.add("cert_file", certFile)
// b.add("key_file", certKey)
// // b.add("verify_incoming", true)
// })
// }
if node.IsServer() {
b.addBlock("grpc", func() {
b.add("ca_file", caFile)
b.add("cert_file", certFile)
b.add("key_file", certKey)
// b.add("verify_incoming", true)
})
}
})
}
b.addBlock("ports", func() {
if node.IsServer() {
b.add("grpc_tls", 8503)
b.add("grpc", -1)
} else {
b.add("grpc", 8502)
b.add("grpc_tls", -1)
}
b.add("http", 8500)
b.add("dns", 8600)
})
b.addSlice("recursors", []string{"8.8.8.8"})
b.addBlock("acl", func() {
b.add("enabled", true)
b.add("default_policy", "deny")
b.add("down_policy", "extend-cache")
b.add("enable_token_persistence", true)
b.addBlock("tokens", func() {
if node.IsServer() {
b.add("initial_management", g.sec.ReadGeneric(node.Cluster, secrets.BootstrapToken))
}
b.add("agent_recovery", g.sec.ReadGeneric(node.Cluster, secrets.AgentRecovery))
b.add("agent", g.sec.ReadAgentToken(node.Cluster, node.ID()))
})
})
if node.IsServer() {
b.add("bootstrap_expect", len(cluster.ServerNodes()))
// b.add("translate_wan_addrs", true)
b.addBlock("rpc", func() {
b.add("enable_streaming", true)
})
if node.HasPublicAddress() {
b.add("advertise_addr_wan", `{{ GetInterfaceIP "eth1" }}`) // note: can't use 'node.PublicAddress()' b/c we don't know that yet
}
// Exercise config entry bootstrap
// b.addBlock("config_entries", func() {
// b.addBlock("bootstrap", func() {
// b.add("kind", "service-defaults")
// b.add("name", "placeholder")
// b.add("protocol", "grpc")
// })
// b.addBlock("bootstrap", func() {
// b.add("kind", "service-intentions")
// b.add("name", "placeholder")
// b.addBlock("sources", func() {
// b.add("name", "placeholder-client")
// b.add("action", "allow")
// })
// })
// })
b.addBlock("connect", func() {
b.add("enabled", true)
})
} else {
if cluster.Enterprise {
b.add("partition", node.Partition)
}
}
return b.String(), nil
}
type HCLBuilder struct {
parts []string
}
func (b *HCLBuilder) format(s string, a ...any) {
if len(a) == 0 {
b.parts = append(b.parts, s)
} else {
b.parts = append(b.parts, fmt.Sprintf(s, a...))
}
}
func (b *HCLBuilder) add(k string, v any) {
switch x := v.(type) {
case string:
if x != "" {
b.format("%s = %q", k, x)
}
case int:
b.format("%s = %d", k, x)
case bool:
b.format("%s = %v", k, x)
default:
panic(fmt.Sprintf("unexpected type %T", v))
}
}
func (b *HCLBuilder) addBlock(block string, fn func()) {
b.format(block + "{")
fn()
b.format("}")
}
func (b *HCLBuilder) addSlice(name string, vals []string) {
b.format(name + " = [")
for _, v := range vals {
b.format("%q,", v)
}
b.format("]")
}
func (b *HCLBuilder) String() string {
joined := strings.Join(b.parts, "\n")
// Ensure it looks tidy
return string(hclwrite.Format([]byte(joined)))
}

45
testing/deployer/sprawl/internal/tfgen/digest.go
Unescape View File

@ -0,0 +1,45 @@
package tfgen
import (
"fmt"
)
// digestOutputs takes the data extracted from terraform output variables and
// updates various fields on the topology.Topology with that data.
func (g *Generator) digestOutputs(out *Outputs) error {
for clusterName, nodeMap := range out.Nodes {
cluster, ok := g.topology.Clusters[clusterName]
if !ok {
return fmt.Errorf("found output cluster that does not exist: %s", clusterName)
}
for nid, nodeOut := range nodeMap {
node := cluster.NodeByID(nid)
if node == nil {
return fmt.Errorf("found output node that does not exist in cluster %q: %s", nid, clusterName)
}
if node.DigestExposedPorts(nodeOut.Ports) {
g.logger.Info("discovered exposed port mappings",
"cluster", clusterName,
"node", nid.String(),
"ports", nodeOut.Ports,
)
}
}
}
for netName, proxyPort := range out.ProxyPorts {
changed, err := g.topology.DigestExposedProxyPort(netName, proxyPort)
if err != nil {
return err
}
if changed {
g.logger.Info("discovered exposed forward proxy port",
"network", netName,
"port", proxyPort,
)
}
}
return nil
}

180
testing/deployer/sprawl/internal/tfgen/dns.go
Unescape View File

@ -0,0 +1,180 @@
package tfgen
import (
"bytes"
"fmt"
"os"
"path/filepath"
"strings"
"text/template"
"github.com/hashicorp/consul/testing/deployer/topology"
"github.com/hashicorp/consul/testing/deployer/util"
)
func (g *Generator) getCoreDNSContainer(
net *topology.Network,
ipAddress string,
hashes []string,
) Resource {
var env []string
for i, hv := range hashes {
env = append(env, fmt.Sprintf("HASH_FILE_%d_VALUE=%s", i, hv))
}
coredns := struct {
Name string
DockerNetworkName string
IPAddress string
HashValues string
Env []string
}{
Name: net.Name,
DockerNetworkName: net.DockerName,
IPAddress: ipAddress,
Env: env,
}
return Eval(tfCorednsT, &coredns)
}
func (g *Generator) writeCoreDNSFiles(net *topology.Network, dnsIPAddress string) (bool, []string, error) {
if net.IsPublic() {
return false, nil, fmt.Errorf("coredns only runs on local networks")
}
rootdir := filepath.Join(g.workdir, "terraform", "coredns-config-"+net.Name)
if err := os.MkdirAll(rootdir, 0755); err != nil {
return false, nil, err
}
for _, cluster := range g.topology.Clusters {
if cluster.NetworkName != net.Name {
continue
}
var addrs []string
for _, node := range cluster.SortedNodes() {
if node.Kind != topology.NodeKindServer || node.Disabled {
continue
}
addr := node.AddressByNetwork(net.Name)
if addr.IPAddress != "" {
addrs = append(addrs, addr.IPAddress)
}
}
var (
clusterDNSName = cluster.Name + "-consulcluster.lan"
)
corefilePath := filepath.Join(rootdir, "Corefile")
zonefilePath := filepath.Join(rootdir, "servers")
_, err := UpdateFileIfDifferent(
g.logger,
generateCoreDNSConfigFile(
clusterDNSName,
addrs,
),
corefilePath,
0644,
)
if err != nil {
return false, nil, fmt.Errorf("error writing %q: %w", corefilePath, err)
}
corefileHash, err := util.HashFile(corefilePath)
if err != nil {
return false, nil, fmt.Errorf("error hashing %q: %w", corefilePath, err)
}
_, err = UpdateFileIfDifferent(
g.logger,
generateCoreDNSZoneFile(
dnsIPAddress,
clusterDNSName,
addrs,
),
zonefilePath,
0644,
)
if err != nil {
return false, nil, fmt.Errorf("error writing %q: %w", zonefilePath, err)
}
zonefileHash, err := util.HashFile(zonefilePath)
if err != nil {
return false, nil, fmt.Errorf("error hashing %q: %w", zonefilePath, err)
}
return true, []string{corefileHash, zonefileHash}, nil
}
return false, nil, nil
}
func generateCoreDNSConfigFile(
clusterDNSName string,
addrs []string,
) []byte {
serverPart := ""
if len(addrs) > 0 {
var servers []string
for _, addr := range addrs {
servers = append(servers, addr+":8600")
}
serverPart = fmt.Sprintf(`
consul:53 {
forward . %s
log
errors
whoami
}
`, strings.Join(servers, " "))
}
return []byte(fmt.Sprintf(`
%[1]s:53 {
file /config/servers %[1]s
log
errors
whoami
}
%[2]s
.:53 {
forward . 8.8.8.8:53
log
errors
whoami
}
`, clusterDNSName, serverPart))
}
func generateCoreDNSZoneFile(
dnsIPAddress string,
clusterDNSName string,
addrs []string,
) []byte {
var buf bytes.Buffer
buf.WriteString(fmt.Sprintf(`
$TTL 60
$ORIGIN %[1]s.
@ IN SOA ns.%[1]s. webmaster.%[1]s. (
2017042745 ; serial
7200 ; refresh (2 hours)
3600 ; retry (1 hour)
1209600 ; expire (2 weeks)
3600 ; minimum (1 hour)
)
@ IN NS ns.%[1]s. ; Name server
ns IN A %[2]s ; self
`, clusterDNSName, dnsIPAddress))
for _, addr := range addrs {
buf.WriteString(fmt.Sprintf(`
server IN A %s ; Consul server
`, addr))
}
return buf.Bytes()
}
var tfCorednsT = template.Must(template.ParseFS(content, "templates/container-coredns.tf.tmpl"))

39
testing/deployer/sprawl/internal/tfgen/docker.go
Unescape View File

@ -0,0 +1,39 @@
package tfgen
import (
"fmt"
"regexp"
)
var invalidResourceName = regexp.MustCompile(`[^a-z0-9-]+`)
func DockerImageResourceName(image string) string {
return invalidResourceName.ReplaceAllLiteralString(image, "-")
}
func DockerNetwork(name, subnet string) Resource {
return Text(fmt.Sprintf(`
resource "docker_network" %[1]q {
name = %[1]q
attachable = true
ipam_config {
subnet = %[2]q
}
}
`, name, subnet))
}
func DockerVolume(name string) Resource {
return Text(fmt.Sprintf(`
resource "docker_volume" %[1]q {
name = %[1]q
}`, name))
}
func DockerImage(name, image string) Resource {
return Text(fmt.Sprintf(`
resource "docker_image" %[1]q {
name = %[2]q
keep_locally = true
}`, name, image))
}

15
testing/deployer/sprawl/internal/tfgen/docker_test.go
Unescape View File

@ -0,0 +1,15 @@
package tfgen
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestDockerImageResourceName(t *testing.T) {
fn := DockerImageResourceName
assert.Equal(t, "", fn(""))
assert.Equal(t, "abcdefghijklmnopqrstuvwxyz0123456789-", fn("abcdefghijklmnopqrstuvwxyz0123456789-"))
assert.Equal(t, "hashicorp-consul-1-15-0", fn("hashicorp/consul:1.15.0"))
}

475
testing/deployer/sprawl/internal/tfgen/gen.go
Unescape View File

@ -0,0 +1,475 @@
package tfgen
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"os"
"path/filepath"
"strconv"
"strings"
"github.com/hashicorp/go-hclog"
"github.com/hashicorp/consul/testing/deployer/sprawl/internal/runner"
"github.com/hashicorp/consul/testing/deployer/sprawl/internal/secrets"
"github.com/hashicorp/consul/testing/deployer/topology"
"github.com/hashicorp/consul/testing/deployer/util"
)
type Generator struct {
logger hclog.Logger
runner *runner.Runner
topology *topology.Topology
sec *secrets.Store
workdir string
license string
tfLogger io.Writer
// set during network phase
remainingSubnets map[string]struct{}
launched bool
}
func NewGenerator(
logger hclog.Logger,
runner *runner.Runner,
topo *topology.Topology,
sec *secrets.Store,
workdir string,
license string,
) (*Generator, error) {
if logger == nil {
panic("logger is required")
}
if runner == nil {
panic("runner is required")
}
if topo == nil {
panic("topology is required")
}
if sec == nil {
panic("secrets store is required")
}
if workdir == "" {
panic("workdir is required")
}
g := &Generator{
logger: logger,
runner: runner,
sec: sec,
workdir: workdir,
license: license,
tfLogger: logger.Named("terraform").StandardWriter(&hclog.StandardLoggerOptions{
ForceLevel: hclog.Info,
}),
}
g.SetTopology(topo)
_ = g.terraformDestroy(context.Background(), true) // cleanup prior run
return g, nil
}
func (g *Generator) MarkLaunched() {
g.launched = true
}
func (g *Generator) SetTopology(topo *topology.Topology) {
if topo == nil {
panic("topology is required")
}
g.topology = topo
}
type Step int
const (
StepAll Step = 0
StepNetworks Step = 1
StepServers Step = 2
StepAgents Step = 3
StepServices Step = 4
// StepPeering Step = XXX5
StepRelaunch Step = 5
)
func (s Step) String() string {
switch s {
case StepAll:
return "all"
case StepNetworks:
return "networks"
case StepServers:
return "servers"
case StepAgents:
return "agents"
case StepServices:
return "services"
case StepRelaunch:
return "relaunch"
// case StepPeering:
// return "peering"
default:
return "UNKNOWN--" + strconv.Itoa(int(s))
}
}
func (s Step) StartServers() bool { return s >= StepServers }
func (s Step) StartAgents() bool { return s >= StepAgents }
func (s Step) StartServices() bool { return s >= StepServices }
// func (s Step) InitiatePeering() bool { return s >= StepPeering }
func (g *Generator) Regenerate() error {
return g.Generate(StepRelaunch)
}
func (g *Generator) Generate(step Step) error {
if g.launched && step != StepRelaunch {
return fmt.Errorf("cannot use step %q after successful launch; see Regenerate()", step)
}
g.logger.Info("generating and creating resources", "step", step.String())
var (
networks []Resource
volumes []Resource
images []Resource
containers []Resource
imageNames = make(map[string]string)
)
addVolume := func(name string) {
volumes = append(volumes, DockerVolume(name))
}
addImage := func(name, image string) {
if image == "" {
return
}
if _, ok := imageNames[image]; ok {
return
}
if name == "" {
name = DockerImageResourceName(image)
}
imageNames[image] = name
g.logger.Info("registering image", "resource", name, "image", image)
images = append(images, DockerImage(name, image))
}
if g.remainingSubnets == nil {
g.remainingSubnets = util.GetPossibleDockerNetworkSubnets()
}
if len(g.remainingSubnets) == 0 {
return fmt.Errorf("exhausted all docker networks")
}
addImage("nginx", "nginx:latest")
addImage("coredns", "coredns/coredns:latest")
for _, net := range g.topology.SortedNetworks() {
if net.Subnet == "" {
// Because this harness runs on a linux or macos host, we can't
// directly invoke the moby libnetwork calls to check for free
// subnets as it would have to cross into the docker desktop vm on
// mac.
//
// Instead rely on map iteration order being random to avoid
// collisions, but detect the terraform failure and retry until
// success.
var ipnet string
for ipnet = range g.remainingSubnets {
}
if ipnet == "" {
return fmt.Errorf("could not get a free docker network")
}
delete(g.remainingSubnets, ipnet)
if _, err := net.SetSubnet(ipnet); err != nil {
return fmt.Errorf("assigned subnet is invalid %q: %w", ipnet, err)
}
}
networks = append(networks, DockerNetwork(net.DockerName, net.Subnet))
var (
// We always ask for a /24, so just blindly pick x.x.x.252 as our
// proxy address. There's an offset of 2 in the list of available
// addresses here because we removed x.x.x.0 and x.x.x.1 from the
// pool.
proxyIPAddress = net.IPByIndex(250)
// Grab x.x.x.253 for the dns server
dnsIPAddress = net.IPByIndex(251)
)
{
// wrote, hashes, err := g.write
}
{ // nginx forward proxy
_, hash, err := g.writeNginxConfig(net)
if err != nil {
return fmt.Errorf("writeNginxConfig[%s]: %w", net.Name, err)
}
containers = append(containers, g.getForwardProxyContainer(net, proxyIPAddress, hash))
}
net.ProxyAddress = proxyIPAddress
net.DNSAddress = ""
if net.IsLocal() {
wrote, hashes, err := g.writeCoreDNSFiles(net, dnsIPAddress)
if err != nil {
return fmt.Errorf("writeCoreDNSFiles[%s]: %w", net.Name, err)
}
if wrote {
net.DNSAddress = dnsIPAddress
containers = append(containers, g.getCoreDNSContainer(net, dnsIPAddress, hashes))
}
}
}
for _, c := range g.topology.SortedClusters() {
if c.TLSVolumeName == "" {
c.TLSVolumeName = c.Name + "-tls-material-" + g.topology.ID
}
addVolume(c.TLSVolumeName)
}
addImage("pause", "registry.k8s.io/pause:3.3")
if step.StartServers() {
for _, c := range g.topology.SortedClusters() {
for _, node := range c.SortedNodes() {
if node.Disabled {
continue
}
addImage("", node.Images.Consul)
addImage("", node.Images.EnvoyConsulImage())
addImage("", node.Images.LocalDataplaneImage())
if node.IsAgent() {
addVolume(node.DockerName())
}
for _, svc := range node.Services {
addImage("", svc.Image)
}
myContainers, err := g.generateNodeContainers(step, c, node)
if err != nil {
return err
}
containers = append(containers, myContainers...)
}
}
}
tfpath := func(p string) string {
return filepath.Join(g.workdir, "terraform", p)
}
if _, err := WriteHCLResourceFile(g.logger, []Resource{Text(terraformPrelude)}, tfpath("init.tf"), 0644); err != nil {
return err
}
if netResult, err := WriteHCLResourceFile(g.logger, networks, tfpath("networks.tf"), 0644); err != nil {
return err
} else if netResult == UpdateResultModified {
if step != StepNetworks {
return fmt.Errorf("cannot change networking details after they are established")
}
}
if _, err := WriteHCLResourceFile(g.logger, volumes, tfpath("volumes.tf"), 0644); err != nil {
return err
}
if _, err := WriteHCLResourceFile(g.logger, images, tfpath("images.tf"), 0644); err != nil {
return err
}
if _, err := WriteHCLResourceFile(g.logger, containers, tfpath("containers.tf"), 0644); err != nil {
return err
}
if err := g.terraformApply(context.TODO()); err != nil {
return err
}
out, err := g.terraformOutputs(context.TODO())
if err != nil {
return err
}
return g.digestOutputs(out)
}
func (g *Generator) DestroyAll() error {
return g.terraformDestroy(context.TODO(), false)
}
func (g *Generator) DestroyAllQuietly() error {
return g.terraformDestroy(context.TODO(), true)
}
func (g *Generator) terraformApply(ctx context.Context) error {
tfdir := filepath.Join(g.workdir, "terraform")
if _, err := os.Stat(filepath.Join(tfdir, ".terraform")); err != nil {
if !os.IsNotExist(err) {
return err
}
// On the fly init
g.logger.Info("Running 'terraform init'...")
if err := g.runner.TerraformExec(ctx, []string{"init", "-input=false"}, g.tfLogger, tfdir); err != nil {
return err
}
}
g.logger.Info("Running 'terraform apply'...")
return g.runner.TerraformExec(ctx, []string{"apply", "-input=false", "-auto-approve"}, g.tfLogger, tfdir)
}
func (g *Generator) terraformDestroy(ctx context.Context, quiet bool) error {
g.logger.Info("Running 'terraform destroy'...")
var out io.Writer
if quiet {
out = io.Discard
} else {
out = g.tfLogger
}
tfdir := filepath.Join(g.workdir, "terraform")
return g.runner.TerraformExec(ctx, []string{
"destroy", "-input=false", "-auto-approve", "-refresh=false",
}, out, tfdir)
}
func (g *Generator) terraformOutputs(ctx context.Context) (*Outputs, error) {
tfdir := filepath.Join(g.workdir, "terraform")
var buf bytes.Buffer
err := g.runner.TerraformExec(ctx, []string{
"output", "-json",
}, &buf, tfdir)
if err != nil {
return nil, err
}
type outputVar struct {
// may be map[string]any
Value any `json:"value"`
}
raw := make(map[string]*outputVar)
dec := json.NewDecoder(&buf)
if err := dec.Decode(&raw); err != nil {
return nil, err
}
out := &Outputs{}
for key, rv := range raw {
switch {
case strings.HasPrefix(key, "ports_"):
cluster, nid, ok := extractNodeOutputKey("ports_", key)
if !ok {
return nil, fmt.Errorf("unexpected output var: %s", key)
}
ports := make(map[int]int)
for k, v := range rv.Value.(map[string]any) {
ki, err := strconv.Atoi(k)
if err != nil {
return nil, fmt.Errorf("unexpected port value %q: %w", k, err)
}
ports[ki] = int(v.(float64))
}
out.SetNodePorts(cluster, nid, ports)
case strings.HasPrefix(key, "forwardproxyport_"):
netname := strings.TrimPrefix(key, "forwardproxyport_")
found := rv.Value.(map[string]any)
if len(found) != 1 {
return nil, fmt.Errorf("found unexpected ports: %v", found)
}
got, ok := found[strconv.Itoa(proxyInternalPort)]
if !ok {
return nil, fmt.Errorf("found unexpected ports: %v", found)
}
out.SetProxyPort(netname, int(got.(float64)))
}
}
return out, nil
}
func extractNodeOutputKey(prefix, key string) (string, topology.NodeID, bool) {
clusterNode := strings.TrimPrefix(key, prefix)
cluster, nodeid, ok := strings.Cut(clusterNode, "_")
if !ok {
return "", topology.NodeID{}, false
}
partition, node, ok := strings.Cut(nodeid, "_")
if !ok {
return "", topology.NodeID{}, false
}
nid := topology.NewNodeID(node, partition)
return cluster, nid, true
}
type Outputs struct {
ProxyPorts map[string]int // net -> exposed port
Nodes map[string]map[topology.NodeID]*NodeOutput // clusterID -> node -> stuff
}
func (o *Outputs) SetNodePorts(cluster string, nid topology.NodeID, ports map[int]int) {
nodeOut := o.getNode(cluster, nid)
nodeOut.Ports = ports
}
func (o *Outputs) SetProxyPort(net string, port int) {
if o.ProxyPorts == nil {
o.ProxyPorts = make(map[string]int)
}
o.ProxyPorts[net] = port
}
func (o *Outputs) getNode(cluster string, nid topology.NodeID) *NodeOutput {
if o.Nodes == nil {
o.Nodes = make(map[string]map[topology.NodeID]*NodeOutput)
}
cnodes, ok := o.Nodes[cluster]
if !ok {
cnodes = make(map[topology.NodeID]*NodeOutput)
o.Nodes[cluster] = cnodes
}
nodeOut, ok := cnodes[nid]
if !ok {
nodeOut = &NodeOutput{}
cnodes[nid] = nodeOut
}
return nodeOut
}
type NodeOutput struct {
Ports map[int]int `json:",omitempty"`
}

70
testing/deployer/sprawl/internal/tfgen/io.go
Unescape View File

@ -0,0 +1,70 @@
package tfgen
import (
"bytes"
"os"
"strings"
"github.com/hashicorp/go-hclog"
"github.com/hashicorp/hcl/v2/hclwrite"
"github.com/rboyer/safeio"
)
func WriteHCLResourceFile(
logger hclog.Logger,
res []Resource,
path string,
perm os.FileMode,
) (UpdateResult, error) {
var text []string
for _, r := range res {
val, err := r.Render()
if err != nil {
return UpdateResultNone, err
}
text = append(text, strings.TrimSpace(val))
}
body := strings.Join(text, "\n\n")
// Ensure it looks tidy
out := hclwrite.Format(bytes.TrimSpace([]byte(body)))
return UpdateFileIfDifferent(logger, out, path, perm)
}
type UpdateResult int
const (
UpdateResultNone UpdateResult = iota
UpdateResultCreated
UpdateResultModified
)
func UpdateFileIfDifferent(
logger hclog.Logger,
body []byte,
path string,
perm os.FileMode,
) (UpdateResult, error) {
prev, err := os.ReadFile(path)
result := UpdateResultNone
if err != nil {
if !os.IsNotExist(err) {
return result, err
}
logger.Info("writing new file", "path", path)
result = UpdateResultCreated
} else {
// loaded
if bytes.Equal(body, prev) {
return result, nil
}
logger.Info("file has changed", "path", path)
result = UpdateResultModified
}
_, err = safeio.WriteToFile(bytes.NewReader(body), path, perm)
return result, err
}

249
testing/deployer/sprawl/internal/tfgen/nodes.go
Unescape View File

@ -0,0 +1,249 @@
package tfgen
import (
"fmt"
"sort"
"strconv"
"text/template"
"github.com/hashicorp/consul/testing/deployer/topology"
)
type terraformPod struct {
PodName string
Node *topology.Node
Ports []int
Labels map[string]string
TLSVolumeName string
DNSAddress string
DockerNetworkName string
}
type terraformConsulAgent struct {
terraformPod
ImageResource string
HCL string
EnterpriseLicense string
Env []string
}
type terraformMeshGatewayService struct {
terraformPod
EnvoyImageResource string
Service *topology.Service
Command []string
}
type terraformService struct {
terraformPod
AppImageResource string
EnvoyImageResource string // agentful
DataplaneImageResource string // agentless
Service *topology.Service
Env []string
Command []string
EnvoyCommand []string // agentful
}
func (g *Generator) generateNodeContainers(
step Step,
cluster *topology.Cluster,
node *topology.Node,
) ([]Resource, error) {
if node.Disabled {
return nil, fmt.Errorf("cannot generate containers for a disabled node")
}
pod := terraformPod{
PodName: node.PodName(),
Node: node,
Labels: map[string]string{
"consulcluster-topology-id": g.topology.ID,
"consulcluster-cluster-name": node.Cluster,
},
TLSVolumeName: cluster.TLSVolumeName,
DNSAddress: "8.8.8.8",
}
cluster, ok := g.topology.Clusters[node.Cluster]
if !ok {
return nil, fmt.Errorf("no such cluster: %s", node.Cluster)
}
net, ok := g.topology.Networks[cluster.NetworkName]
if !ok {
return nil, fmt.Errorf("no local network: %s", cluster.NetworkName)
}
if net.DNSAddress != "" {
pod.DNSAddress = net.DNSAddress
}
pod.DockerNetworkName = net.DockerName
var (
containers []Resource
)
if node.IsAgent() {
agentHCL, err := g.generateAgentHCL(node)
if err != nil {
return nil, err
}
agent := terraformConsulAgent{
terraformPod: pod,
ImageResource: DockerImageResourceName(node.Images.Consul),
HCL: agentHCL,
EnterpriseLicense: g.license,
Env: node.AgentEnv,
}
switch {
case node.IsServer() && step.StartServers(),
!node.IsServer() && step.StartAgents():
containers = append(containers, Eval(tfConsulT, &agent))
}
}
for _, svc := range node.SortedServices() {
if svc.IsMeshGateway {
if node.Kind == topology.NodeKindDataplane {
panic("NOT READY YET")
}
gw := terraformMeshGatewayService{
terraformPod: pod,
EnvoyImageResource: DockerImageResourceName(node.Images.EnvoyConsulImage()),
Service: svc,
Command: []string{
"consul", "connect", "envoy",
"-register",
"-mesh-gateway",
},
}
if token := g.sec.ReadServiceToken(node.Cluster, svc.ID); token != "" {
gw.Command = append(gw.Command, "-token", token)
}
if cluster.Enterprise {
gw.Command = append(gw.Command,
"-partition",
svc.ID.Partition,
)
}
gw.Command = append(gw.Command,
"-address",
`{{ GetInterfaceIP \"eth0\" }}:`+strconv.Itoa(svc.Port),
"-wan-address",
`{{ GetInterfaceIP \"eth1\" }}:`+strconv.Itoa(svc.Port),
)
gw.Command = append(gw.Command,
"-grpc-addr", "http://127.0.0.1:8502",
"-admin-bind",
// for demo purposes
"0.0.0.0:"+strconv.Itoa(svc.EnvoyAdminPort),
"--",
"-l",
"trace",
)
if step.StartServices() {
containers = append(containers, Eval(tfMeshGatewayT, &gw))
}
} else {
tfsvc := terraformService{
terraformPod: pod,
AppImageResource: DockerImageResourceName(svc.Image),
Service: svc,
Command: svc.Command,
}
tfsvc.Env = append(tfsvc.Env, svc.Env...)
if step.StartServices() {
containers = append(containers, Eval(tfAppT, &tfsvc))
}
setenv := func(k, v string) {
tfsvc.Env = append(tfsvc.Env, k+"="+v)
}
if !svc.DisableServiceMesh {
if node.IsDataplane() {
tfsvc.DataplaneImageResource = DockerImageResourceName(node.Images.LocalDataplaneImage())
tfsvc.EnvoyImageResource = ""
tfsvc.EnvoyCommand = nil
// --- REQUIRED ---
setenv("DP_CONSUL_ADDRESSES", "server."+node.Cluster+"-consulcluster.lan")
setenv("DP_SERVICE_NODE_NAME", node.PodName())
setenv("DP_PROXY_SERVICE_ID", svc.ID.Name+"-sidecar-proxy")
} else {
tfsvc.DataplaneImageResource = ""
tfsvc.EnvoyImageResource = DockerImageResourceName(node.Images.EnvoyConsulImage())
tfsvc.EnvoyCommand = []string{
"consul", "connect", "envoy",
"-sidecar-for", svc.ID.Name,
}
}
if cluster.Enterprise {
if node.IsDataplane() {
setenv("DP_SERVICE_NAMESPACE", svc.ID.Namespace)
setenv("DP_SERVICE_PARTITION", svc.ID.Partition)
} else {
tfsvc.EnvoyCommand = append(tfsvc.EnvoyCommand,
"-partition",
svc.ID.Partition,
"-namespace",
svc.ID.Namespace,
)
}
}
if token := g.sec.ReadServiceToken(node.Cluster, svc.ID); token != "" {
if node.IsDataplane() {
setenv("DP_CREDENTIAL_TYPE", "static")
setenv("DP_CREDENTIAL_STATIC_TOKEN", token)
} else {
tfsvc.EnvoyCommand = append(tfsvc.EnvoyCommand, "-token", token)
}
}
if node.IsDataplane() {
setenv("DP_ENVOY_ADMIN_BIND_ADDRESS", "0.0.0.0") // for demo purposes
setenv("DP_ENVOY_ADMIN_BIND_PORT", "19000")
setenv("DP_LOG_LEVEL", "trace")
setenv("DP_CA_CERTS", "/consul/config/certs/consul-agent-ca.pem")
setenv("DP_CONSUL_GRPC_PORT", "8503")
setenv("DP_TLS_SERVER_NAME", "server."+node.Datacenter+".consul")
} else {
tfsvc.EnvoyCommand = append(tfsvc.EnvoyCommand,
"-grpc-addr", "http://127.0.0.1:8502",
"-admin-bind",
// for demo purposes
"0.0.0.0:"+strconv.Itoa(svc.EnvoyAdminPort),
"--",
"-l",
"trace",
)
}
if step.StartServices() {
sort.Strings(tfsvc.Env)
if node.IsDataplane() {
containers = append(containers, Eval(tfAppDataplaneT, &tfsvc))
} else {
containers = append(containers, Eval(tfAppSidecarT, &tfsvc))
}
}
}
}
}
// Wait until the very end to render the pod so we know all of the ports.
pod.Ports = node.SortedPorts()
// pod placeholder container
containers = append(containers, Eval(tfPauseT, &pod))
return containers, nil
}
var tfPauseT = template.Must(template.ParseFS(content, "templates/container-pause.tf.tmpl"))
var tfConsulT = template.Must(template.ParseFS(content, "templates/container-consul.tf.tmpl"))
var tfMeshGatewayT = template.Must(template.ParseFS(content, "templates/container-mgw.tf.tmpl"))
var tfAppT = template.Must(template.ParseFS(content, "templates/container-app.tf.tmpl"))
var tfAppSidecarT = template.Must(template.ParseFS(content, "templates/container-app-sidecar.tf.tmpl"))
var tfAppDataplaneT = template.Must(template.ParseFS(content, "templates/container-app-dataplane.tf.tmpl"))

16
testing/deployer/sprawl/internal/tfgen/prelude.go
Unescape View File

@ -0,0 +1,16 @@
package tfgen
const terraformPrelude = `provider "docker" {
host = "unix:///var/run/docker.sock"
}
terraform {
required_providers {
docker = {
source = "kreuzwerker/docker"
version = "~> 2.0"
}
}
required_version = ">= 0.13"
}
`

87
testing/deployer/sprawl/internal/tfgen/proxy.go
Unescape View File

@ -0,0 +1,87 @@
package tfgen
import (
"fmt"
"os"
"path/filepath"
"text/template"
"github.com/hashicorp/consul/testing/deployer/topology"
"github.com/hashicorp/consul/testing/deployer/util"
)
const proxyInternalPort = 80
func (g *Generator) writeNginxConfig(net *topology.Network) (bool, string, error) {
rootdir := filepath.Join(g.workdir, "terraform", "nginx-config-"+net.Name)
if err := os.MkdirAll(rootdir, 0755); err != nil {
return false, "", err
}
configFile := filepath.Join(rootdir, "nginx.conf")
body := fmt.Sprintf(`
server {
listen %d;
location / {
resolver 8.8.8.8;
##############
# Relevant config knobs are here: https://nginx.org/en/docs/http/ngx_http_proxy_module.html
##############
proxy_pass http://$http_host$uri$is_args$args;
proxy_cache off;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
proxy_connect_timeout 5s;
proxy_read_timeout 5s;
proxy_send_timeout 5s;
proxy_request_buffering off;
proxy_buffering off;
}
}
`, proxyInternalPort)
_, err := UpdateFileIfDifferent(
g.logger,
[]byte(body),
configFile,
0644,
)
if err != nil {
return false, "", fmt.Errorf("error writing %q: %w", configFile, err)
}
hash, err := util.HashFile(configFile)
if err != nil {
return false, "", fmt.Errorf("error hashing %q: %w", configFile, err)
}
return true, hash, err
}
func (g *Generator) getForwardProxyContainer(
net *topology.Network,
ipAddress string,
hash string,
) Resource {
env := []string{"HASH_FILE_VALUE=" + hash}
proxy := struct {
Name string
DockerNetworkName string
InternalPort int
IPAddress string
Env []string
}{
Name: net.Name,
DockerNetworkName: net.DockerName,
InternalPort: proxyInternalPort,
IPAddress: ipAddress,
Env: env,
}
return Eval(tfForwardProxyT, &proxy)
}
var tfForwardProxyT = template.Must(template.ParseFS(content, "templates/container-proxy.tf.tmpl"))

95
testing/deployer/sprawl/internal/tfgen/res.go
Unescape View File

@ -0,0 +1,95 @@
package tfgen
import (
"bytes"
"text/template"
"github.com/hashicorp/go-hclog"
"github.com/hashicorp/hcl/v2/hclwrite"
)
type FileResource struct {
name string
res Resource
}
func (r *FileResource) Name() string { return r.name }
func (r *FileResource) Commit(logger hclog.Logger) error {
val, err := r.res.Render()
if err != nil {
return err
}
_, err = UpdateFileIfDifferent(logger, []byte(val), r.name, 0644)
return err
}
func File(name string, res Resource) *FileResource {
return &FileResource{name: name, res: res}
}
func Text(s string) Resource {
return &textResource{text: s}
}
func Embed(name string) Resource {
return &embedResource{name: name}
}
func Eval(t *template.Template, data any) Resource {
return &evalResource{template: t, data: data, hcl: false}
}
func HCL(t *template.Template, data any) Resource {
return &evalResource{template: t, data: data, hcl: true}
}
type Resource interface {
Render() (string, error)
}
type embedResource struct {
name string
}
func (r *embedResource) Render() (string, error) {
val, err := content.ReadFile(r.name)
if err != nil {
return "", err
}
return string(val), nil
}
type textResource struct {
text string
}
func (r *textResource) Render() (string, error) {
return r.text, nil
}
type evalResource struct {
template *template.Template
data any
hcl bool
}
func (r *evalResource) Render() (string, error) {
out, err := StringTemplate(r.template, r.data)
if err != nil {
return "", err
}
if r.hcl {
return string(hclwrite.Format([]byte(out))), nil
}
return out, nil
}
func StringTemplate(t *template.Template, data any) (string, error) {
var res bytes.Buffer
if err := t.Execute(&res, data); err != nil {
return "", err
}
return res.String(), nil
}

29
testing/deployer/sprawl/internal/tfgen/templates/container-app-dataplane.tf.tmpl
Unescape View File

@ -0,0 +1,29 @@
resource "docker_container" "{{.Node.DockerName}}-{{.Service.ID.TFString}}-sidecar" {
name = "{{.Node.DockerName}}-{{.Service.ID.TFString}}-sidecar"
network_mode = "container:${docker_container.{{.PodName}}.id}"
image = docker_image.{{.DataplaneImageResource}}.latest
restart = "on-failure"
{{- range $k, $v := .Labels }}
labels {
label = "{{ $k }}"
value = "{{ $v }}"
}
{{- end }}
volumes {
volume_name = "{{.TLSVolumeName}}"
container_path = "/consul/config/certs"
read_only = true
}
env = [
{{- range .Env }}
"{{.}}",
{{- end}}
]
command = [
"/usr/local/bin/consul-dataplane",
]
}

31
testing/deployer/sprawl/internal/tfgen/templates/container-app-sidecar.tf.tmpl
Unescape View File

@ -0,0 +1,31 @@
resource "docker_container" "{{.Node.DockerName}}-{{.Service.ID.TFString}}-sidecar" {
name = "{{.Node.DockerName}}-{{.Service.ID.TFString}}-sidecar"
network_mode = "container:${docker_container.{{.PodName}}.id}"
image = docker_image.{{.EnvoyImageResource}}.latest
restart = "on-failure"
{{- range $k, $v := .Labels }}
labels {
label = "{{ $k }}"
value = "{{ $v }}"
}
{{- end }}
volumes {
volume_name = "{{.TLSVolumeName}}"
container_path = "/consul/config/certs"
read_only = true
}
env = [
{{- range .Env }}
"{{.}}",
{{- end}}
]
command = [
{{- range .EnvoyCommand }}
"{{.}}",
{{- end }}
]
}

25
testing/deployer/sprawl/internal/tfgen/templates/container-app.tf.tmpl
Unescape View File

@ -0,0 +1,25 @@
resource "docker_container" "{{.Node.DockerName}}-{{.Service.ID.TFString}}" {
name = "{{.Node.DockerName}}-{{.Service.ID.TFString}}"
network_mode = "container:${docker_container.{{.PodName}}.id}"
image = docker_image.{{.AppImageResource}}.latest
restart = "on-failure"
{{- range $k, $v := .Labels }}
labels {
label = "{{ $k }}"
value = "{{ $v }}"
}
{{- end }}
env = [
{{- range .Env }}
"{{.}}",
{{- end}}
]
command = [
{{- range .Command }}
"{{.}}",
{{- end }}
]
}

40
testing/deployer/sprawl/internal/tfgen/templates/container-consul.tf.tmpl
Unescape View File

@ -0,0 +1,40 @@
resource "docker_container" "{{.Node.DockerName}}" {
name = "{{.Node.DockerName}}"
network_mode = "container:${docker_container.{{.PodName}}.id}"
image = docker_image.{{.ImageResource}}.latest
restart = "always"
env = [
"CONSUL_UID=0",
"CONSUL_GID=0",
"CONSUL_LICENSE={{.EnterpriseLicense}}",
{{- range .Env }}
"{{.}}",
{{- end}}
]
{{- range $k, $v := .Labels }}
labels {
label = "{{ $k }}"
value = "{{ $v }}"
}
{{- end }}
command = [
"agent",
"-hcl",
<<-EOT
{{ .HCL }}
EOT
]
volumes {
volume_name = "{{.Node.DockerName}}"
container_path = "/consul/data"
}
volumes {
volume_name = "{{.TLSVolumeName}}"
container_path = "/consul/config/certs"
}
}

28
testing/deployer/sprawl/internal/tfgen/templates/container-coredns.tf.tmpl
Unescape View File

@ -0,0 +1,28 @@
resource "docker_container" "{{.DockerNetworkName}}-coredns" {
name = "{{.DockerNetworkName}}-coredns"
image = docker_image.coredns.latest
restart = "always"
dns = ["8.8.8.8"]
networks_advanced {
name = docker_network.{{.DockerNetworkName}}.name
ipv4_address = "{{.IPAddress}}"
}
env = [
{{- range .Env }}
"{{.}}",
{{- end}}
]
volumes {
host_path = abspath("coredns-config-{{.Name}}")
container_path = "/config"
read_only = true
}
command = [
"-conf",
"/config/Corefile",
]
}

25
testing/deployer/sprawl/internal/tfgen/templates/container-mgw.tf.tmpl
Unescape View File

@ -0,0 +1,25 @@
resource "docker_container" "{{.Node.DockerName}}-{{.Service.ID.TFString}}" {
name = "{{.Node.DockerName}}-{{.Service.ID.TFString}}"
network_mode = "container:${docker_container.{{.PodName}}.id}"
image = docker_image.{{.EnvoyImageResource}}.latest
restart = "on-failure"
{{- range $k, $v := .Labels }}
labels {
label = "{{ $k }}"
value = "{{ $v }}"
}
{{- end }}
volumes {
volume_name = "{{.TLSVolumeName}}"
container_path = "/consul/config/certs"
read_only = true
}
command = [
{{- range .Command }}
"{{.}}",
{{- end }}
]
}

38
testing/deployer/sprawl/internal/tfgen/templates/container-pause.tf.tmpl
Unescape View File

@ -0,0 +1,38 @@
resource "docker_container" "{{.PodName}}" {
name = "{{.PodName}}"
image = docker_image.pause.latest
hostname = "{{.PodName}}"
restart = "always"
dns = ["{{.DNSAddress}}"]
{{- range $k, $v := .Labels }}
labels {
label = "{{ $k }}"
value = "{{ $v }}"
}
{{- end }}
depends_on = [
docker_container.{{.DockerNetworkName}}-coredns,
docker_container.{{.DockerNetworkName}}-forwardproxy,
]
{{- range .Ports }}
ports {
internal = {{.}}
}
{{- end }}
{{- range .Node.Addresses }}
networks_advanced {
name = docker_network.{{.DockerNetworkName}}.name
ipv4_address = "{{.IPAddress}}"
}
{{- end }}
}
output "ports_{{.Node.Cluster}}_{{.Node.Partition}}_{{.Node.Name}}" {
value = {
for port in docker_container.{{.PodName}}.ports : port.internal => port.external
}
}

33
testing/deployer/sprawl/internal/tfgen/templates/container-proxy.tf.tmpl
Unescape View File

@ -0,0 +1,33 @@
resource "docker_container" "{{.DockerNetworkName}}-forwardproxy" {
name = "{{.DockerNetworkName}}-forwardproxy"
image = docker_image.nginx.latest
restart = "always"
dns = ["8.8.8.8"]
ports {
internal = {{.InternalPort}}
}
networks_advanced {
name = docker_network.{{.DockerNetworkName}}.name
ipv4_address = "{{.IPAddress}}"
}
env = [
{{- range .Env }}
"{{.}}",
{{- end}}
]
volumes {
host_path = abspath("nginx-config-{{.Name}}/nginx.conf")
container_path = "/etc/nginx/conf.d/default.conf"
read_only = true
}
}
output "forwardproxyport_{{.Name}}" {
value = {
for port in docker_container.{{.DockerNetworkName}}-forwardproxy.ports : port.internal => port.external
}
}

15
testing/deployer/sprawl/internal/tfgen/tfgen.go
Unescape View File

@ -0,0 +1,15 @@
package tfgen
import (
"embed"
)
//go:embed templates/container-app-dataplane.tf.tmpl
//go:embed templates/container-app-sidecar.tf.tmpl
//go:embed templates/container-app.tf.tmpl
//go:embed templates/container-consul.tf.tmpl
//go:embed templates/container-mgw.tf.tmpl
//go:embed templates/container-pause.tf.tmpl
//go:embed templates/container-proxy.tf.tmpl
//go:embed templates/container-coredns.tf.tmpl
var content embed.FS

165
testing/deployer/sprawl/peering.go
Unescape View File

@ -0,0 +1,165 @@
package sprawl
import (
"context"
"fmt"
"strings"
"time"
"github.com/hashicorp/consul/api"
"github.com/hashicorp/go-hclog"
"github.com/hashicorp/consul/testing/deployer/topology"
)
// TODO: this is definitely a grpc resolver/balancer issue to look into
const grpcWeirdError = `transport: Error while dialing failed to find Consul server for global address`
func isWeirdGRPCError(err error) bool {
if err == nil {
return false
}
return strings.Contains(err.Error(), grpcWeirdError)
}
func (s *Sprawl) initPeerings() error {
// TODO: wait until services are healthy? wait until mesh gateways work?
// if err := s.generator.Generate(tfgen.StepPeering); err != nil {
// return fmt.Errorf("generator[peering]: %w", err)
// }
var (
logger = s.logger.Named("peering")
_ = logger
)
for _, peering := range s.topology.Peerings {
dialingCluster, ok := s.topology.Clusters[peering.Dialing.Name]
if !ok {
return fmt.Errorf("peering references dialing cluster that does not exist: %s", peering.String())
}
acceptingCluster, ok := s.topology.Clusters[peering.Accepting.Name]
if !ok {
return fmt.Errorf("peering references accepting cluster that does not exist: %s", peering.String())
}
var (
dialingClient = s.clients[dialingCluster.Name]
acceptingClient = s.clients[acceptingCluster.Name]
)
// TODO: allow for use of ServerExternalAddresses
req1 := api.PeeringGenerateTokenRequest{
PeerName: peering.Accepting.PeerName,
}
if acceptingCluster.Enterprise {
req1.Partition = peering.Accepting.Partition
}
GENTOKEN:
resp, _, err := acceptingClient.Peerings().GenerateToken(context.Background(), req1, nil)
if err != nil {
if isWeirdGRPCError(err) {
time.Sleep(50 * time.Millisecond)
goto GENTOKEN
}
return fmt.Errorf("error generating peering token for %q: %w", peering.String(), err)
}
peeringToken := resp.PeeringToken
logger.Info("generated peering token", "peering", peering.String())
req2 := api.PeeringEstablishRequest{
PeerName: peering.Dialing.PeerName,
PeeringToken: peeringToken,
}
if dialingCluster.Enterprise {
req2.Partition = peering.Dialing.Partition
}
logger.Info("establishing peering with token", "peering", peering.String())
ESTABLISH:
_, _, err = dialingClient.Peerings().Establish(context.Background(), req2, nil)
if err != nil {
if isWeirdGRPCError(err) {
time.Sleep(50 * time.Millisecond)
goto ESTABLISH
}
return fmt.Errorf("error establishing peering with token for %q: %w", peering.String(), err)
}
logger.Info("peering established", "peering", peering.String())
}
return nil
}
func (s *Sprawl) waitForPeeringEstablishment() error {
var (
logger = s.logger.Named("peering")
)
for _, peering := range s.topology.Peerings {
dialingCluster, ok := s.topology.Clusters[peering.Dialing.Name]
if !ok {
return fmt.Errorf("peering references dialing cluster that does not exist: %s", peering.String())
}
acceptingCluster, ok := s.topology.Clusters[peering.Accepting.Name]
if !ok {
return fmt.Errorf("peering references accepting cluster that does not exist: %s", peering.String())
}
var (
dialingClient = s.clients[dialingCluster.Name]
acceptingClient = s.clients[acceptingCluster.Name]
dialingLogger = logger.With(
"cluster", dialingCluster.Name,
"peering", peering.String(),
)
acceptingLogger = logger.With(
"cluster", acceptingCluster.Name,
"peering", peering.String(),
)
)
s.checkPeeringDirection(dialingLogger, dialingClient, peering.Dialing, dialingCluster.Enterprise)
s.checkPeeringDirection(acceptingLogger, acceptingClient, peering.Accepting, acceptingCluster.Enterprise)
}
return nil
}
func (s *Sprawl) checkPeeringDirection(logger hclog.Logger, client *api.Client, pc topology.PeerCluster, enterprise bool) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
for {
opts := &api.QueryOptions{}
if enterprise {
opts.Partition = pc.Partition
}
res, _, err := client.Peerings().Read(ctx, pc.PeerName, opts)
if isWeirdGRPCError(err) {
time.Sleep(50 * time.Millisecond)
continue
}
if err != nil {
logger.Info("error looking up peering", "error", err)
time.Sleep(100 * time.Millisecond)
continue
}
if res == nil {
logger.Info("peering not found")
time.Sleep(100 * time.Millisecond)
continue
}
if res.State == api.PeeringStateActive {
logger.Info("peering is active")
return
}
logger.Info("peering not active yet", "state", res.State)
time.Sleep(500 * time.Millisecond)
}
}

464
testing/deployer/sprawl/sprawl.go
Unescape View File

@ -0,0 +1,464 @@
package sprawl
import (
"bufio"
"bytes"
"context"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"strings"
"time"
"github.com/hashicorp/consul/api"
"github.com/hashicorp/go-hclog"
"github.com/hashicorp/go-multierror"
"github.com/mitchellh/copystructure"
"github.com/hashicorp/consul/testing/deployer/sprawl/internal/runner"
"github.com/hashicorp/consul/testing/deployer/sprawl/internal/secrets"
"github.com/hashicorp/consul/testing/deployer/sprawl/internal/tfgen"
"github.com/hashicorp/consul/testing/deployer/topology"
"github.com/hashicorp/consul/testing/deployer/util"
)
// TODO: manage workdir externally without chdir
// Sprawl is the definition of a complete running Consul deployment topology.
type Sprawl struct {
logger hclog.Logger
runner *runner.Runner
license string
secrets secrets.Store
workdir string
// set during Run
config *topology.Config
topology *topology.Topology
generator *tfgen.Generator
clients map[string]*api.Client // one per cluster
}
// Topology allows access to the topology that defines the resources. Do not
// write to any of these fields.
func (s *Sprawl) Topology() *topology.Topology {
return s.topology
}
func (s *Sprawl) Config() *topology.Config {
c2, err := copyConfig(s.config)
if err != nil {
panic(err)
}
return c2
}
func (s *Sprawl) HTTPClientForCluster(clusterName string) (*http.Client, error) {
cluster, ok := s.topology.Clusters[clusterName]
if !ok {
return nil, fmt.Errorf("no such cluster: %s", clusterName)
}
// grab the local network for the cluster
network, ok := s.topology.Networks[cluster.NetworkName]
if !ok {
return nil, fmt.Errorf("no such network: %s", cluster.NetworkName)
}
transport, err := util.ProxyHTTPTransport(network.ProxyPort)
if err != nil {
return nil, err
}
return &http.Client{Transport: transport}, nil
}
// APIClientForNode gets a pooled api.Client connected to the agent running on
// the provided node.
//
// Passing an empty token will assume the bootstrap token. If you want to
// actually use the anonymous token say "-".
func (s *Sprawl) APIClientForNode(clusterName string, nid topology.NodeID, token string) (*api.Client, error) {
cluster, ok := s.topology.Clusters[clusterName]
if !ok {
return nil, fmt.Errorf("no such cluster: %s", clusterName)
}
nid.Normalize()
node := cluster.NodeByID(nid)
if !node.IsAgent() {
return nil, fmt.Errorf("node is not an agent")
}
switch token {
case "":
token = s.secrets.ReadGeneric(clusterName, secrets.BootstrapToken)
case "-":
token = ""
}
return util.ProxyAPIClient(
node.LocalProxyPort(),
node.LocalAddress(),
8500,
token,
)
}
func copyConfig(cfg *topology.Config) (*topology.Config, error) {
dup, err := copystructure.Copy(cfg)
if err != nil {
return nil, err
}
return dup.(*topology.Config), nil
}
// Launch will create the topology defined by the provided configuration and
// bring up all of the relevant clusters. Once created the Stop method must be
// called to destroy everything.
func Launch(
logger hclog.Logger,
workdir string,
cfg *topology.Config,
) (*Sprawl, error) {
if logger == nil {
panic("logger is required")
}
if workdir == "" {
panic("workdir is required")
}
if err := os.MkdirAll(filepath.Join(workdir, "terraform"), 0755); err != nil {
return nil, err
}
runner, err := runner.Load(logger)
if err != nil {
return nil, err
}
// Copy this to avoid leakage.
cfg, err = copyConfig(cfg)
if err != nil {
return nil, err
}
s := &Sprawl{
logger: logger,
runner: runner,
workdir: workdir,
clients: make(map[string]*api.Client),
}
if err := s.ensureLicense(); err != nil {
return nil, err
}
// Copy this AGAIN, BEFORE compiling so we capture the original definition, without denorms.
s.config, err = copyConfig(cfg)
if err != nil {
return nil, err
}
s.topology, err = topology.Compile(logger.Named("compile"), cfg)
if err != nil {
return nil, fmt.Errorf("topology.Compile: %w", err)
}
s.logger.Info("compiled topology", "ct", jd(s.topology)) // TODO
start := time.Now()
if err := s.launch(); err != nil {
return nil, err
}
s.logger.Info("topology is ready for use", "elapsed", time.Since(start))
if err := s.PrintDetails(); err != nil {
return nil, fmt.Errorf("error gathering diagnostic details: %w", err)
}
return s, nil
}
func (s *Sprawl) Relaunch(
cfg *topology.Config,
) error {
// Copy this BEFORE compiling so we capture the original definition, without denorms.
var err error
s.config, err = copyConfig(cfg)
if err != nil {
return err
}
newTopology, err := topology.Recompile(s.logger.Named("recompile"), cfg, s.topology)
if err != nil {
return fmt.Errorf("topology.Compile: %w", err)
}
s.topology = newTopology
s.logger.Info("compiled replacement topology", "ct", jd(s.topology)) // TODO
start := time.Now()
if err := s.relaunch(); err != nil {
return err
}
s.logger.Info("topology is ready for use", "elapsed", time.Since(start))
if err := s.PrintDetails(); err != nil {
return fmt.Errorf("error gathering diagnostic details: %w", err)
}
return nil
}
// Leader returns the cluster leader agent, or an error if no leader is
// available.
func (s *Sprawl) Leader(clusterName string) (*topology.Node, error) {
cluster, ok := s.topology.Clusters[clusterName]
if !ok {
return nil, fmt.Errorf("no such cluster: %s", clusterName)
}
var (
client = s.clients[cluster.Name]
// logger = s.logger.With("cluster", cluster.Name)
)
leaderAddr, err := getLeader(client)
if err != nil {
return nil, err
}
for _, node := range cluster.Nodes {
if !node.IsServer() || node.Disabled {
continue
}
if strings.HasPrefix(leaderAddr, node.LocalAddress()+":") {
return node, nil
}
}
return nil, fmt.Errorf("leader not found")
}
// Followers returns the cluster following servers.
func (s *Sprawl) Followers(clusterName string) ([]*topology.Node, error) {
cluster, ok := s.topology.Clusters[clusterName]
if !ok {
return nil, fmt.Errorf("no such cluster: %s", clusterName)
}
leaderNode, err := s.Leader(clusterName)
if err != nil {
return nil, fmt.Errorf("could not determine leader: %w", err)
}
var followers []*topology.Node
for _, node := range cluster.Nodes {
if !node.IsServer() || node.Disabled {
continue
}
if node.ID() != leaderNode.ID() {
followers = append(followers, node)
}
}
return followers, nil
}
func (s *Sprawl) DisabledServers(clusterName string) ([]*topology.Node, error) {
cluster, ok := s.topology.Clusters[clusterName]
if !ok {
return nil, fmt.Errorf("no such cluster: %s", clusterName)
}
var servers []*topology.Node
for _, node := range cluster.Nodes {
if !node.IsServer() || !node.Disabled {
continue
}
servers = append(servers, node)
}
return servers, nil
}
func (s *Sprawl) StopContainer(ctx context.Context, containerName string) error {
return s.runner.DockerExec(ctx, []string{"stop", containerName}, nil, nil)
}
func (s *Sprawl) SnapshotEnvoy(ctx context.Context) error {
snapDir := filepath.Join(s.workdir, "envoy-snapshots")
if err := os.MkdirAll(snapDir, 0755); err != nil {
return fmt.Errorf("could not create envoy snapshot output dir %s: %w", snapDir, err)
}
targets := map[string]string{
"config_dump.json": "config_dump",
"clusters.json": "clusters?format=json",
"stats.txt": "stats",
"stats_prometheus.txt": "stats/prometheus",
}
var merr error
for _, c := range s.topology.Clusters {
client, err := s.HTTPClientForCluster(c.Name)
if err != nil {
return fmt.Errorf("could not get http client for cluster %q: %w", c.Name, err)
}
for _, n := range c.Nodes {
if n.Disabled {
continue
}
for _, s := range n.Services {
if s.Disabled || s.EnvoyAdminPort <= 0 {
continue
}
prefix := fmt.Sprintf("http://%s:%d", n.LocalAddress(), s.EnvoyAdminPort)
for fn, target := range targets {
u := prefix + "/" + target
body, err := scrapeURL(client, u)
if err != nil {
merr = multierror.Append(merr, fmt.Errorf("could not scrape %q for %s on %s: %w",
target, s.ID.String(), n.ID().String(), err,
))
continue
}
outFn := filepath.Join(snapDir, n.DockerName()+"--"+s.ID.TFString()+"."+fn)
if err := os.WriteFile(outFn+".tmp", body, 0644); err != nil {
merr = multierror.Append(merr, fmt.Errorf("could not write output %q for %s on %s: %w",
target, s.ID.String(), n.ID().String(), err,
))
continue
}
if err := os.Rename(outFn+".tmp", outFn); err != nil {
merr = multierror.Append(merr, fmt.Errorf("could not write output %q for %s on %s: %w",
target, s.ID.String(), n.ID().String(), err,
))
continue
}
}
}
}
}
return merr
}
func scrapeURL(client *http.Client, url string) ([]byte, error) {
res, err := client.Get(url)
if err != nil {
return nil, err
}
defer res.Body.Close()
body, err := io.ReadAll(res.Body)
if err != nil {
return nil, err
}
return body, nil
}
func (s *Sprawl) CaptureLogs(ctx context.Context) error {
logDir := filepath.Join(s.workdir, "logs")
if err := os.MkdirAll(logDir, 0755); err != nil {
return fmt.Errorf("could not create log output dir %s: %w", logDir, err)
}
containers, err := s.listContainers(ctx)
if err != nil {
return err
}
s.logger.Info("Capturing logs")
var merr error
for _, container := range containers {
if err := s.dumpContainerLogs(ctx, container, logDir); err != nil {
merr = multierror.Append(merr, fmt.Errorf("could not dump logs for container %s: %w", container, err))
}
}
return merr
}
// Dump known containers out of terraform state file.
func (s *Sprawl) listContainers(ctx context.Context) ([]string, error) {
tfdir := filepath.Join(s.workdir, "terraform")
var buf bytes.Buffer
if err := s.runner.TerraformExec(ctx, []string{"state", "list"}, &buf, tfdir); err != nil {
return nil, fmt.Errorf("error listing containers in terraform state file: %w", err)
}
var (
scan = bufio.NewScanner(&buf)
containers []string
)
for scan.Scan() {
line := strings.TrimSpace(scan.Text())
name := strings.TrimPrefix(line, "docker_container.")
if name != line {
containers = append(containers, name)
continue
}
}
if err := scan.Err(); err != nil {
return nil, err
}
return containers, nil
}
func (s *Sprawl) dumpContainerLogs(ctx context.Context, containerName, outputRoot string) error {
path := filepath.Join(outputRoot, containerName+".log")
f, err := os.Create(path + ".tmp")
if err != nil {
return err
}
keep := false
defer func() {
_ = f.Close()
if !keep {
_ = os.Remove(path + ".tmp")
_ = os.Remove(path)
}
}()
err = s.runner.DockerExecWithStderr(
ctx,
[]string{"logs", containerName},
f,
f,
nil,
)
if err != nil {
return err
}
if err := f.Close(); err != nil {
return err
}
if err := os.Rename(path+".tmp", path); err != nil {
return err
}
keep = true
return nil
}

202
testing/deployer/sprawl/sprawltest/sprawltest.go
Unescape View File

@ -0,0 +1,202 @@
package sprawltest
import (
"context"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"sync"
"testing"
"github.com/hashicorp/consul/sdk/testutil"
"github.com/hashicorp/go-hclog"
"github.com/hashicorp/go-multierror"
"github.com/stretchr/testify/require"
"github.com/hashicorp/consul/testing/deployer/sprawl"
"github.com/hashicorp/consul/testing/deployer/sprawl/internal/runner"
"github.com/hashicorp/consul/testing/deployer/topology"
)
// TODO(rb): move comments to doc.go
var (
// set SPRAWL_WORKDIR_ROOT in the environment to have the test output
// coalesced in here. By default it uses a directory called "workdir" in
// each package.
workdirRoot string
// set SPRAWL_KEEP_WORKDIR=1 in the environment to keep the workdir output
// intact. Files are all destroyed by default.
keepWorkdirOnFail bool
// set SPRAWL_KEEP_RUNNING=1 in the environment to keep the workdir output
// intact and also refrain from tearing anything down. Things are all
// destroyed by default.
//
// SPRAWL_KEEP_RUNNING=1 implies SPRAWL_KEEP_WORKDIR=1
keepRunningOnFail bool
// set SPRAWL_SKIP_OLD_CLEANUP to prevent the library from tearing down and
// removing anything found in the working directory at init time. The
// default behavior is to do this.
skipOldCleanup bool
)
var cleanupPriorRunOnce sync.Once
func init() {
if root := os.Getenv("SPRAWL_WORKDIR_ROOT"); root != "" {
fmt.Fprintf(os.Stdout, "INFO: sprawltest: SPRAWL_WORKDIR_ROOT set; using %q as output root\n", root)
workdirRoot = root
} else {
workdirRoot = "workdir"
}
if os.Getenv("SPRAWL_KEEP_WORKDIR") == "1" {
keepWorkdirOnFail = true
fmt.Fprintf(os.Stdout, "INFO: sprawltest: SPRAWL_KEEP_WORKDIR set; not destroying workdir on failure\n")
}
if os.Getenv("SPRAWL_KEEP_RUNNING") == "1" {
keepRunningOnFail = true
keepWorkdirOnFail = true
fmt.Fprintf(os.Stdout, "INFO: sprawltest: SPRAWL_KEEP_RUNNING set; not tearing down resources on failure\n")
}
if os.Getenv("SPRAWL_SKIP_OLD_CLEANUP") == "1" {
skipOldCleanup = true
fmt.Fprintf(os.Stdout, "INFO: sprawltest: SPRAWL_SKIP_OLD_CLEANUP set; not cleaning up anything found in %q\n", workdirRoot)
}
if !skipOldCleanup {
cleanupPriorRunOnce.Do(func() {
fmt.Fprintf(os.Stdout, "INFO: sprawltest: triggering cleanup of any prior test runs\n")
CleanupWorkingDirectories()
})
}
}
// Launch will create the topology defined by the provided configuration and
// bring up all of the relevant clusters.
//
// - Logs will be routed to (*testing.T).Logf.
//
// - By default everything will be stopped and removed via
// (*testing.T).Cleanup. For failed tests, this can be skipped by setting the
// environment variable SKIP_TEARDOWN=1.
func Launch(t *testing.T, cfg *topology.Config) *sprawl.Sprawl {
SkipIfTerraformNotPresent(t)
sp, err := sprawl.Launch(
testutil.Logger(t),
initWorkingDirectory(t),
cfg,
)
require.NoError(t, err)
stopOnCleanup(t, sp)
return sp
}
func initWorkingDirectory(t *testing.T) string {
// TODO(rb): figure out how to get the calling package which we can put in
// the middle here, which is likely 2 call frames away so maybe
// runtime.Callers can help
scratchDir := filepath.Join(workdirRoot, t.Name())
_ = os.RemoveAll(scratchDir) // cleanup prior runs
if err := os.MkdirAll(scratchDir, 0755); err != nil {
t.Fatalf("error: %v", err)
}
t.Cleanup(func() {
if t.Failed() && keepWorkdirOnFail {
t.Logf("test failed; leaving sprawl terraform definitions in: %s", scratchDir)
} else {
_ = os.RemoveAll(scratchDir)
}
})
return scratchDir
}
func stopOnCleanup(t *testing.T, sp *sprawl.Sprawl) {
t.Cleanup(func() {
if t.Failed() && keepWorkdirOnFail {
// It's only worth it to capture the logs if we aren't going to
// immediately discard them.
if err := sp.CaptureLogs(context.Background()); err != nil {
t.Logf("log capture encountered failures: %v", err)
}
if err := sp.SnapshotEnvoy(context.Background()); err != nil {
t.Logf("envoy snapshot capture encountered failures: %v", err)
}
}
if t.Failed() && keepRunningOnFail {
t.Log("test failed; leaving sprawl running")
} else {
//nolint:errcheck
sp.Stop()
}
})
}
// CleanupWorkingDirectories is meant to run in an init() once at the start of
// any tests.
func CleanupWorkingDirectories() {
fi, err := os.ReadDir(workdirRoot)
if os.IsNotExist(err) {
return
} else if err != nil {
fmt.Fprintf(os.Stderr, "WARN: sprawltest: unable to scan 'workdir' for prior runs to cleanup\n")
return
} else if len(fi) == 0 {
fmt.Fprintf(os.Stdout, "INFO: sprawltest: no prior tests to clean up\n")
return
}
r, err := runner.Load(hclog.NewNullLogger())
if err != nil {
fmt.Fprintf(os.Stderr, "WARN: sprawltest: unable to look for 'terraform' and 'docker' binaries\n")
return
}
ctx := context.Background()
for _, d := range fi {
if !d.IsDir() {
continue
}
path := filepath.Join(workdirRoot, d.Name(), "terraform")
fmt.Fprintf(os.Stdout, "INFO: sprawltest: cleaning up failed prior run in: %s\n", path)
err := r.TerraformExec(ctx, []string{
"init", "-input=false",
}, io.Discard, path)
err2 := r.TerraformExec(ctx, []string{
"destroy", "-input=false", "-auto-approve", "-refresh=false",
}, io.Discard, path)
if err2 != nil {
err = multierror.Append(err, err2)
}
if err != nil {
fmt.Fprintf(os.Stderr, "WARN: sprawltest: could not clean up failed prior run in: %s: %v\n", path, err)
} else {
_ = os.RemoveAll(path)
}
}
}
func SkipIfTerraformNotPresent(t *testing.T) {
const terraformBinaryName = "terraform"
path, err := exec.LookPath(terraformBinaryName)
if err != nil || path == "" {
t.Skipf("%q not found on $PATH - download and install to run this test", terraformBinaryName)
}
}

180
testing/deployer/sprawl/sprawltest/test_test.go
Unescape View File

@ -0,0 +1,180 @@
package sprawltest_test
import (
"strconv"
"testing"
"github.com/hashicorp/consul/api"
"github.com/stretchr/testify/require"
"github.com/hashicorp/consul/testing/deployer/sprawl/sprawltest"
"github.com/hashicorp/consul/testing/deployer/topology"
)
func TestSprawl(t *testing.T) {
serversDC1 := newTopologyServerSet("dc1-server", 3, []string{"dc1", "wan"}, nil)
serversDC2 := newTopologyServerSet("dc2-server", 3, []string{"dc2", "wan"}, nil)
cfg := &topology.Config{
Networks: []*topology.Network{
{Name: "dc1"},
{Name: "dc2"},
{Name: "wan", Type: "wan"},
},
Clusters: []*topology.Cluster{
{
Name: "dc1",
Nodes: topology.MergeSlices(serversDC1, []*topology.Node{
{
Kind: topology.NodeKindClient,
Name: "dc1-client1",
Services: []*topology.Service{
{
ID: topology.ServiceID{Name: "mesh-gateway"},
Port: 8443,
EnvoyAdminPort: 19000,
IsMeshGateway: true,
},
},
},
{
Kind: topology.NodeKindClient,
Name: "dc1-client2",
Services: []*topology.Service{
{
ID: topology.ServiceID{Name: "ping"},
Image: "rboyer/pingpong:latest",
Port: 8080,
EnvoyAdminPort: 19000,
Command: []string{
"-bind", "0.0.0.0:8080",
"-dial", "127.0.0.1:9090",
"-pong-chaos",
"-dialfreq", "250ms",
"-name", "ping",
},
Upstreams: []*topology.Upstream{{
ID: topology.ServiceID{Name: "pong"},
LocalPort: 9090,
Peer: "peer-dc2-default",
}},
},
},
},
}),
InitialConfigEntries: []api.ConfigEntry{
&api.ExportedServicesConfigEntry{
Name: "default",
Services: []api.ExportedService{{
Name: "ping",
Consumers: []api.ServiceConsumer{{
Peer: "peer-dc2-default",
}},
}},
},
},
},
{
Name: "dc2",
Nodes: topology.MergeSlices(serversDC2, []*topology.Node{
{
Kind: topology.NodeKindClient,
Name: "dc2-client1",
Services: []*topology.Service{
{
ID: topology.ServiceID{Name: "mesh-gateway"},
Port: 8443,
EnvoyAdminPort: 19000,
IsMeshGateway: true,
},
},
},
{
Kind: topology.NodeKindDataplane,
Name: "dc2-client2",
Services: []*topology.Service{
{
ID: topology.ServiceID{Name: "pong"},
Image: "rboyer/pingpong:latest",
Port: 8080,
EnvoyAdminPort: 19000,
Command: []string{
"-bind", "0.0.0.0:8080",
"-dial", "127.0.0.1:9090",
"-pong-chaos",
"-dialfreq", "250ms",
"-name", "pong",
},
Upstreams: []*topology.Upstream{{
ID: topology.ServiceID{Name: "ping"},
LocalPort: 9090,
Peer: "peer-dc1-default",
}},
},
},
},
}),
InitialConfigEntries: []api.ConfigEntry{
&api.ExportedServicesConfigEntry{
Name: "default",
Services: []api.ExportedService{{
Name: "ping",
Consumers: []api.ServiceConsumer{{
Peer: "peer-dc2-default",
}},
}},
},
},
},
},
Peerings: []*topology.Peering{{
Dialing: topology.PeerCluster{
Name: "dc1",
},
Accepting: topology.PeerCluster{
Name: "dc2",
},
}},
}
sp := sprawltest.Launch(t, cfg)
for _, cluster := range sp.Topology().Clusters {
leader, err := sp.Leader(cluster.Name)
require.NoError(t, err)
t.Logf("%s: leader = %s", cluster.Name, leader.ID())
followers, err := sp.Followers(cluster.Name)
require.NoError(t, err)
for _, f := range followers {
t.Logf("%s: follower = %s", cluster.Name, f.ID())
}
}
}
func newTopologyServerSet(
namePrefix string,
num int,
networks []string,
mutateFn func(i int, node *topology.Node),
) []*topology.Node {
var out []*topology.Node
for i := 1; i <= num; i++ {
name := namePrefix + strconv.Itoa(i)
node := &topology.Node{
Kind: topology.NodeKindServer,
Name: name,
}
for _, net := range networks {
node.Addresses = append(node.Addresses, &topology.Address{Network: net})
}
if mutateFn != nil {
mutateFn(i, node)
}
out = append(out, node)
}
return out
}

114
testing/deployer/sprawl/tls.go
Unescape View File

@ -0,0 +1,114 @@
package sprawl
import (
"bytes"
"context"
"fmt"
"io"
"github.com/hashicorp/consul/testing/deployer/topology"
)
const (
consulUID = "100"
consulGID = "1000"
consulUserArg = consulUID + ":" + consulGID
)
func tlsPrefixFromNode(node *topology.Node) string {
switch node.Kind {
case topology.NodeKindServer:
return node.Partition + "." + node.Name + ".server"
case topology.NodeKindClient:
return node.Partition + "." + node.Name + ".client"
default:
return ""
}
}
func tlsCertCreateCommand(node *topology.Node) string {
if node.IsServer() {
return fmt.Sprintf(`consul tls cert create -server -dc=%s -node=%s`, node.Datacenter, node.PodName())
} else {
return fmt.Sprintf(`consul tls cert create -client -dc=%s`, node.Datacenter)
}
}
func (s *Sprawl) initTLS(ctx context.Context) error {
for _, cluster := range s.topology.Clusters {
var buf bytes.Buffer
// Create the CA if not already done, and proceed to do all of the
// consul CLI calls inside of a throwaway temp directory.
buf.WriteString(`
if [[ ! -f consul-agent-ca-key.pem || ! -f consul-agent-ca.pem ]]; then
consul tls ca create
fi
rm -rf tmp
mkdir -p tmp
cp -a consul-agent-ca-key.pem consul-agent-ca.pem tmp
cd tmp
`)
for _, node := range cluster.Nodes {
if !node.IsAgent() || node.Disabled {
continue
}
node.TLSCertPrefix = tlsPrefixFromNode(node)
if node.TLSCertPrefix == "" {
continue
}
expectPrefix := cluster.Datacenter + "-" + string(node.Kind) + "-consul-0"
// Conditionally generate these in isolation and rename them to
// not rely upon the numerical indexing.
buf.WriteString(fmt.Sprintf(`
if [[ ! -f %[1]s || ! -f %[2]s ]]; then
rm -f %[3]s %[4]s
%[5]s
mv -f %[3]s %[1]s
mv -f %[4]s %[2]s
fi
`,
"../"+node.TLSCertPrefix+"-key.pem", "../"+node.TLSCertPrefix+".pem",
expectPrefix+"-key.pem", expectPrefix+".pem",
tlsCertCreateCommand(node),
))
}
err := s.runner.DockerExec(ctx, []string{
"run",
"--rm",
"-i",
"--net=none",
"-v", cluster.TLSVolumeName + ":/data",
"busybox:latest",
"sh", "-c",
// Need this so the permissions stick; docker seems to treat unused volumes differently.
`touch /data/VOLUME_PLACEHOLDER && chown -R ` + consulUserArg + ` /data`,
}, io.Discard, nil)
if err != nil {
return fmt.Errorf("could not initialize docker volume for cert data %q: %w", cluster.TLSVolumeName, err)
}
err = s.runner.DockerExec(ctx, []string{"run",
"--rm",
"-i",
"--net=none",
"-u", consulUserArg,
"-v", cluster.TLSVolumeName + ":/data",
"-w", "/data",
"--entrypoint", "",
cluster.Images.Consul,
"/bin/sh", "-ec", buf.String(),
}, io.Discard, nil)
if err != nil {
return fmt.Errorf("could not create all necessary TLS certificates in docker volume: %v", err)
}
}
return nil
}

671
testing/deployer/topology/compile.go
Unescape View File

@ -0,0 +1,671 @@
package topology
import (
crand "crypto/rand"
"encoding/hex"
"errors"
"fmt"
"reflect"
"regexp"
"sort"
"github.com/google/go-cmp/cmp"
"github.com/hashicorp/go-hclog"
)
const DockerPrefix = "consulcluster"
func Compile(logger hclog.Logger, raw *Config) (*Topology, error) {
return compile(logger, raw, nil)
}
func Recompile(logger hclog.Logger, raw *Config, prev *Topology) (*Topology, error) {
if prev == nil {
return nil, errors.New("missing previous topology")
}
return compile(logger, raw, prev)
}
func compile(logger hclog.Logger, raw *Config, prev *Topology) (*Topology, error) {
var id string
if prev == nil {
var err error
id, err = newTopologyID()
if err != nil {
return nil, err
}
} else {
id = prev.ID
}
images := DefaultImages().OverrideWith(raw.Images)
if images.Consul != "" {
return nil, fmt.Errorf("topology.images.consul cannot be set at this level")
}
if len(raw.Networks) == 0 {
return nil, fmt.Errorf("topology.networks is empty")
}
networks := make(map[string]*Network)
for _, net := range raw.Networks {
if net.DockerName != "" {
return nil, fmt.Errorf("network %q should not specify DockerName", net.Name)
}
if !IsValidLabel(net.Name) {
return nil, fmt.Errorf("network name is not valid: %s", net.Name)
}
if _, exists := networks[net.Name]; exists {
return nil, fmt.Errorf("cannot have two networks with the same name %q", net.Name)
}
switch net.Type {
case "":
net.Type = "lan"
case "wan", "lan":
default:
return nil, fmt.Errorf("network %q has unknown type %q", net.Name, net.Type)
}
networks[net.Name] = net
net.DockerName = DockerPrefix + "-" + net.Name + "-" + id
}
if len(raw.Clusters) == 0 {
return nil, fmt.Errorf("topology.clusters is empty")
}
var (
clusters = make(map[string]*Cluster)
nextIndex int // use a global index so any shared networks work properly with assignments
)
foundPeerNames := make(map[string]map[string]struct{})
for _, c := range raw.Clusters {
if c.Name == "" {
return nil, fmt.Errorf("cluster has no name")
}
foundPeerNames[c.Name] = make(map[string]struct{})
if !IsValidLabel(c.Name) {
return nil, fmt.Errorf("cluster name is not valid: %s", c.Name)
}
if _, exists := clusters[c.Name]; exists {
return nil, fmt.Errorf("cannot have two clusters with the same name %q; use unique names and override the Datacenter field if that's what you want", c.Name)
}
if c.Datacenter == "" {
c.Datacenter = c.Name
} else {
if !IsValidLabel(c.Datacenter) {
return nil, fmt.Errorf("datacenter name is not valid: %s", c.Datacenter)
}
}
clusters[c.Name] = c
if c.NetworkName == "" {
c.NetworkName = c.Name
}
c.Images = images.OverrideWith(c.Images).ChooseConsul(c.Enterprise)
if _, ok := networks[c.NetworkName]; !ok {
return nil, fmt.Errorf("cluster %q uses network name %q that does not exist", c.Name, c.NetworkName)
}
if len(c.Nodes) == 0 {
return nil, fmt.Errorf("cluster %q has no nodes", c.Name)
}
if c.TLSVolumeName != "" {
return nil, fmt.Errorf("user cannot specify the TLSVolumeName field")
}
tenancies := make(map[string]map[string]struct{})
addTenancy := func(partition, namespace string) {
partition = PartitionOrDefault(partition)
namespace = NamespaceOrDefault(namespace)
m, ok := tenancies[partition]
if !ok {
m = make(map[string]struct{})
tenancies[partition] = m
}
m[namespace] = struct{}{}
}
for _, ap := range c.Partitions {
addTenancy(ap.Name, "default")
for _, ns := range ap.Namespaces {
addTenancy(ap.Name, ns)
}
}
for _, ce := range c.InitialConfigEntries {
addTenancy(ce.GetPartition(), ce.GetNamespace())
}
seenNodes := make(map[NodeID]struct{})
for _, n := range c.Nodes {
if n.Name == "" {
return nil, fmt.Errorf("cluster %q node has no name", c.Name)
}
if !IsValidLabel(n.Name) {
return nil, fmt.Errorf("node name is not valid: %s", n.Name)
}
switch n.Kind {
case NodeKindServer, NodeKindClient, NodeKindDataplane:
default:
return nil, fmt.Errorf("cluster %q node %q has invalid kind: %s", c.Name, n.Name, n.Kind)
}
n.Partition = PartitionOrDefault(n.Partition)
if !IsValidLabel(n.Partition) {
return nil, fmt.Errorf("node partition is not valid: %s", n.Partition)
}
addTenancy(n.Partition, "default")
if _, exists := seenNodes[n.ID()]; exists {
return nil, fmt.Errorf("cannot have two nodes in the same cluster %q with the same name %q", c.Name, n.ID())
}
seenNodes[n.ID()] = struct{}{}
if len(n.usedPorts) != 0 {
return nil, fmt.Errorf("user cannot specify the usedPorts field")
}
n.usedPorts = make(map[int]int)
exposePort := func(v int) bool {
if _, ok := n.usedPorts[v]; ok {
return false
}
n.usedPorts[v] = 0
return true
}
if n.IsAgent() {
// TODO: the ux here is awful; we should be able to examine the topology to guess properly
exposePort(8500)
if n.IsServer() {
exposePort(8503)
} else {
exposePort(8502)
}
}
if n.Index != 0 {
return nil, fmt.Errorf("user cannot specify the node index")
}
n.Index = nextIndex
nextIndex++
n.Images = c.Images.OverrideWith(n.Images).ChooseNode(n.Kind)
n.Cluster = c.Name
n.Datacenter = c.Datacenter
n.dockerName = DockerPrefix + "-" + n.Name + "-" + id
if len(n.Addresses) == 0 {
n.Addresses = append(n.Addresses, &Address{Network: c.NetworkName})
}
var (
numPublic int
numLocal int
)
for _, addr := range n.Addresses {
if addr.Network == "" {
return nil, fmt.Errorf("cluster %q node %q has invalid address", c.Name, n.Name)
}
if addr.Type != "" {
return nil, fmt.Errorf("user cannot specify the address type directly")
}
net, ok := networks[addr.Network]
if !ok {
return nil, fmt.Errorf("cluster %q node %q uses network name %q that does not exist", c.Name, n.Name, addr.Network)
}
if net.IsPublic() {
numPublic++
} else if net.IsLocal() {
numLocal++
}
addr.Type = net.Type
addr.DockerNetworkName = net.DockerName
}
if numLocal == 0 {
return nil, fmt.Errorf("cluster %q node %q has no local addresses", c.Name, n.Name)
}
if numPublic > 1 {
return nil, fmt.Errorf("cluster %q node %q has more than one public address", c.Name, n.Name)
}
seenServices := make(map[ServiceID]struct{})
for _, svc := range n.Services {
if n.IsAgent() {
// Default to that of the enclosing node.
svc.ID.Partition = n.Partition
}
svc.ID.Normalize()
// Denormalize
svc.Node = n
if !IsValidLabel(svc.ID.Partition) {
return nil, fmt.Errorf("service partition is not valid: %s", svc.ID.Partition)
}
if !IsValidLabel(svc.ID.Namespace) {
return nil, fmt.Errorf("service namespace is not valid: %s", svc.ID.Namespace)
}
if !IsValidLabel(svc.ID.Name) {
return nil, fmt.Errorf("service name is not valid: %s", svc.ID.Name)
}
addTenancy(svc.ID.Partition, svc.ID.Namespace)
if _, exists := seenServices[svc.ID]; exists {
return nil, fmt.Errorf("cannot have two services on the same node %q in the same cluster %q with the same name %q", n.ID(), c.Name, svc.ID)
}
seenServices[svc.ID] = struct{}{}
if !svc.DisableServiceMesh && n.IsDataplane() {
if svc.EnvoyPublicListenerPort <= 0 {
if _, ok := n.usedPorts[20000]; !ok {
// For convenience the FIRST service on a node can get 20000 for free.
svc.EnvoyPublicListenerPort = 20000
} else {
return nil, fmt.Errorf("envoy public listener port is required")
}
}
}
// add all of the service ports
for _, port := range svc.ports() {
if ok := exposePort(port); !ok {
return nil, fmt.Errorf("port used more than once on cluster %q node %q: %d", c.Name, n.ID(), port)
}
}
// TODO(rb): re-expose?
// switch svc.Protocol {
// case "":
// svc.Protocol = "tcp"
// fallthrough
// case "tcp":
// if svc.CheckHTTP != "" {
// return nil, fmt.Errorf("cannot set CheckHTTP for tcp service")
// }
// case "http":
// if svc.CheckTCP != "" {
// return nil, fmt.Errorf("cannot set CheckTCP for tcp service")
// }
// default:
// return nil, fmt.Errorf("service has invalid protocol: %s", svc.Protocol)
// }
for _, u := range svc.Upstreams {
// Default to that of the enclosing service.
if u.Peer == "" {
if u.ID.Partition == "" {
u.ID.Partition = svc.ID.Partition
}
if u.ID.Namespace == "" {
u.ID.Namespace = svc.ID.Namespace
}
} else {
if u.ID.Partition != "" {
u.ID.Partition = "" // irrelevant here; we'll set it to the value of the OTHER side for plumbing purposes in tests
}
u.ID.Namespace = NamespaceOrDefault(u.ID.Namespace)
foundPeerNames[c.Name][u.Peer] = struct{}{}
}
if u.ID.Name == "" {
return nil, fmt.Errorf("upstream service name is required")
}
addTenancy(u.ID.Partition, u.ID.Namespace)
}
if err := svc.Validate(); err != nil {
return nil, fmt.Errorf("cluster %q node %q service %q is not valid: %w", c.Name, n.Name, svc.ID.String(), err)
}
}
}
// Explode this into the explicit list based on stray references made.
c.Partitions = nil
for ap, nsMap := range tenancies {
p := &Partition{
Name: ap,
}
for ns := range nsMap {
p.Namespaces = append(p.Namespaces, ns)
}
sort.Strings(p.Namespaces)
c.Partitions = append(c.Partitions, p)
}
sort.Slice(c.Partitions, func(i, j int) bool {
return c.Partitions[i].Name < c.Partitions[j].Name
})
if !c.Enterprise {
expect := []*Partition{{Name: "default", Namespaces: []string{"default"}}}
if !reflect.DeepEqual(c.Partitions, expect) {
return nil, fmt.Errorf("cluster %q references non-default partitions or namespaces but is OSS", c.Name)
}
}
}
clusteredPeerings := make(map[string]map[string]*PeerCluster) // local-cluster -> local-peer -> info
addPeerMapEntry := func(pc PeerCluster) {
pm, ok := clusteredPeerings[pc.Name]
if !ok {
pm = make(map[string]*PeerCluster)
clusteredPeerings[pc.Name] = pm
}
pm[pc.PeerName] = &pc
}
for _, p := range raw.Peerings {
dialingCluster, ok := clusters[p.Dialing.Name]
if !ok {
return nil, fmt.Errorf("peering references a dialing cluster that does not exist: %s", p.Dialing.Name)
}
acceptingCluster, ok := clusters[p.Accepting.Name]
if !ok {
return nil, fmt.Errorf("peering references an accepting cluster that does not exist: %s", p.Accepting.Name)
}
if p.Dialing.Name == p.Accepting.Name {
return nil, fmt.Errorf("self peerings are not allowed: %s", p.Dialing.Name)
}
p.Dialing.Partition = PartitionOrDefault(p.Dialing.Partition)
p.Accepting.Partition = PartitionOrDefault(p.Accepting.Partition)
if dialingCluster.Enterprise {
if !dialingCluster.hasPartition(p.Dialing.Partition) {
return nil, fmt.Errorf("dialing side of peering cannot reference a partition that does not exist: %s", p.Dialing.Partition)
}
} else {
if p.Dialing.Partition != "default" {
return nil, fmt.Errorf("dialing side of peering cannot reference a partition when OSS")
}
}
if acceptingCluster.Enterprise {
if !acceptingCluster.hasPartition(p.Accepting.Partition) {
return nil, fmt.Errorf("accepting side of peering cannot reference a partition that does not exist: %s", p.Accepting.Partition)
}
} else {
if p.Accepting.Partition != "default" {
return nil, fmt.Errorf("accepting side of peering cannot reference a partition when OSS")
}
}
if p.Dialing.PeerName == "" {
p.Dialing.PeerName = "peer-" + p.Accepting.Name + "-" + p.Accepting.Partition
}
if p.Accepting.PeerName == "" {
p.Accepting.PeerName = "peer-" + p.Dialing.Name + "-" + p.Dialing.Partition
}
{ // Ensure the link fields do not have recursive links.
p.Dialing.Link = nil
p.Accepting.Link = nil
// Copy the un-linked data before setting the link
pa := p.Accepting
pd := p.Dialing
p.Accepting.Link = &pd
p.Dialing.Link = &pa
}
addPeerMapEntry(p.Accepting)
addPeerMapEntry(p.Dialing)
delete(foundPeerNames[p.Accepting.Name], p.Accepting.PeerName)
delete(foundPeerNames[p.Dialing.Name], p.Dialing.PeerName)
}
for cluster, peers := range foundPeerNames {
if len(peers) > 0 {
var pretty []string
for name := range peers {
pretty = append(pretty, name)
}
sort.Strings(pretty)
return nil, fmt.Errorf("cluster[%s] found topology references to peerings that do not exist: %v", cluster, pretty)
}
}
// after we decoded the peering stuff, we can fill in some computed data in the upstreams
for _, c := range clusters {
c.Peerings = clusteredPeerings[c.Name]
for _, n := range c.Nodes {
for _, svc := range n.Services {
for _, u := range svc.Upstreams {
if u.Peer == "" {
u.Cluster = c.Name
u.Peering = nil
continue
}
remotePeer, ok := c.Peerings[u.Peer]
if !ok {
return nil, fmt.Errorf("not possible")
}
u.Cluster = remotePeer.Link.Name
u.Peering = remotePeer.Link
// this helps in generating fortio assertions; otherwise field is ignored
u.ID.Partition = remotePeer.Link.Partition
}
}
}
}
t := &Topology{
ID: id,
Networks: networks,
Clusters: clusters,
Images: images,
Peerings: raw.Peerings,
}
if prev != nil {
// networks cannot change
if !sameKeys(prev.Networks, t.Networks) {
return nil, fmt.Errorf("cannot create or destroy networks")
}
for _, newNetwork := range t.Networks {
oldNetwork := prev.Networks[newNetwork.Name]
// Carryover
newNetwork.inheritFromExisting(oldNetwork)
if err := isSame(oldNetwork, newNetwork); err != nil {
return nil, fmt.Errorf("networks cannot change: %w", err)
}
}
// cannot add or remove an entire cluster
if !sameKeys(prev.Clusters, t.Clusters) {
return nil, fmt.Errorf("cannot create or destroy clusters")
}
for _, newCluster := range t.Clusters {
oldCluster := prev.Clusters[newCluster.Name]
// Carryover
newCluster.inheritFromExisting(oldCluster)
if newCluster.Name != oldCluster.Name ||
newCluster.NetworkName != oldCluster.NetworkName ||
newCluster.Datacenter != oldCluster.Datacenter ||
newCluster.Enterprise != oldCluster.Enterprise {
return nil, fmt.Errorf("cannot edit some cluster fields for %q", newCluster.Name)
}
// WARN on presence of some things.
if len(newCluster.InitialConfigEntries) > 0 {
logger.Warn("initial config entries were provided, but are skipped on recompile")
}
// Check NODES
if err := inheritAndValidateNodes(oldCluster.Nodes, newCluster.Nodes); err != nil {
return nil, fmt.Errorf("some immutable aspects of nodes were changed in cluster %q: %w", newCluster.Name, err)
}
}
}
return t, nil
}
const permutedWarning = "use the disabled node kind if you want to ignore a node"
func inheritAndValidateNodes(
prev, curr []*Node,
) error {
nodeMap := mapifyNodes(curr)
for prevIdx, node := range prev {
currNode, ok := nodeMap[node.ID()]
if !ok {
return fmt.Errorf("node %q has vanished; "+permutedWarning, node.ID())
}
// Ensure it hasn't been permuted.
if currNode.Pos != prevIdx {
return fmt.Errorf(
"node %q has been shuffled %d -> %d; "+permutedWarning,
node.ID(),
prevIdx,
currNode.Pos,
)
}
if currNode.Node.Kind != node.Kind ||
currNode.Node.Partition != node.Partition ||
currNode.Node.Name != node.Name ||
currNode.Node.Index != node.Index ||
len(currNode.Node.Addresses) != len(node.Addresses) ||
!sameKeys(currNode.Node.usedPorts, node.usedPorts) {
return fmt.Errorf("cannot edit some node fields for %q", node.ID())
}
currNode.Node.inheritFromExisting(node)
for i := 0; i < len(currNode.Node.Addresses); i++ {
prevAddr := node.Addresses[i]
currAddr := currNode.Node.Addresses[i]
if prevAddr.Network != currAddr.Network {
return fmt.Errorf("addresses were shuffled for node %q", node.ID())
}
if prevAddr.Type != currAddr.Type {
return fmt.Errorf("cannot edit some address fields for %q", node.ID())
}
currAddr.inheritFromExisting(prevAddr)
}
svcMap := mapifyServices(currNode.Node.Services)
for _, svc := range node.Services {
currSvc, ok := svcMap[svc.ID]
if !ok {
continue // service has vanished, this is ok
}
// don't care about index permutation
if currSvc.ID != svc.ID ||
currSvc.Port != svc.Port ||
currSvc.EnvoyAdminPort != svc.EnvoyAdminPort ||
currSvc.EnvoyPublicListenerPort != svc.EnvoyPublicListenerPort ||
isSame(currSvc.Command, svc.Command) != nil ||
isSame(currSvc.Env, svc.Env) != nil {
return fmt.Errorf("cannot edit some address fields for %q", svc.ID)
}
currSvc.inheritFromExisting(svc)
}
}
return nil
}
func newTopologyID() (string, error) {
const n = 16
id := make([]byte, n)
if _, err := crand.Read(id[:]); err != nil {
return "", err
}
return hex.EncodeToString(id)[:n], nil
}
// matches valid DNS labels according to RFC 1123, should be at most 63
// characters according to the RFC
var validLabel = regexp.MustCompile(`^[a-zA-Z0-9]([a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])?$`)
// IsValidLabel returns true if the string given is a valid DNS label (RFC 1123).
// Note: the only difference between RFC 1035 and RFC 1123 labels is that in
// RFC 1123 labels can begin with a number.
func IsValidLabel(name string) bool {
return validLabel.MatchString(name)
}
// ValidateLabel is similar to IsValidLabel except it returns an error
// instead of false when name is not a valid DNS label. The error will contain
// reference to what constitutes a valid DNS label.
func ValidateLabel(name string) error {
if !IsValidLabel(name) {
return errors.New("a valid DNS label must consist of lower case alphanumeric characters or '-', and must start and end with an alphanumeric character")
}
return nil
}
func isSame(x, y any) error {
diff := cmp.Diff(x, y)
if diff != "" {
return fmt.Errorf("values are not equal\n--- expected\n+++ actual\n%v", diff)
}
return nil
}
func sameKeys[K comparable, V any](x, y map[K]V) bool {
if len(x) != len(y) {
return false
}
for kx := range x {
if _, ok := y[kx]; !ok {
return false
}
}
return true
}
func mapifyNodes(nodes []*Node) map[NodeID]nodeWithPosition {
m := make(map[NodeID]nodeWithPosition)
for i, node := range nodes {
m[node.ID()] = nodeWithPosition{
Pos: i,
Node: node,
}
}
return m
}
type nodeWithPosition struct {
Pos int
Node *Node
}
func mapifyServices(services []*Service) map[ServiceID]*Service {
m := make(map[ServiceID]*Service)
for _, svc := range services {
m[svc.ID] = svc
}
return m
}

3
testing/deployer/topology/default_cdp.go
Unescape View File

@ -0,0 +1,3 @@
package topology
const DefaultDataplaneImage = "hashicorp/consul-dataplane:1.1.0"

4
testing/deployer/topology/default_consul.go
Unescape View File

@ -0,0 +1,4 @@
package topology
const DefaultConsulImage = "hashicorp/consul:1.15.2"
const DefaultConsulEnterpriseImage = "hashicorp/consul-enterprise:1.15.2-ent"

3
testing/deployer/topology/default_envoy.go
Unescape View File

@ -0,0 +1,3 @@
package topology
const DefaultEnvoyImage = "envoyproxy/envoy:v1.25.1"

142
testing/deployer/topology/ids.go
Unescape View File

@ -0,0 +1,142 @@
package topology
import (
"fmt"
"github.com/hashicorp/consul/api"
)
type NodeServiceID struct {
Node string
Service string `json:",omitempty"`
Namespace string `json:",omitempty"`
Partition string `json:",omitempty"`
}
func NewNodeServiceID(node, service, namespace, partition string) NodeServiceID {
id := NodeServiceID{
Node: node,
Service: service,
Namespace: namespace,
Partition: partition,
}
id.Normalize()
return id
}
func (id NodeServiceID) NodeID() NodeID {
return NewNodeID(id.Node, id.Partition)
}
func (id NodeServiceID) ServiceID() ServiceID {
return NewServiceID(id.Service, id.Namespace, id.Partition)
}
func (id *NodeServiceID) Normalize() {
id.Namespace = NamespaceOrDefault(id.Namespace)
id.Partition = PartitionOrDefault(id.Partition)
}
func (id NodeServiceID) String() string {
return fmt.Sprintf("%s/%s/%s/%s", id.Partition, id.Node, id.Namespace, id.Service)
}
type NodeID struct {
Name string `json:",omitempty"`
Partition string `json:",omitempty"`
}
func NewNodeID(name, partition string) NodeID {
id := NodeID{
Name: name,
Partition: partition,
}
id.Normalize()
return id
}
func (id *NodeID) Normalize() {
id.Partition = PartitionOrDefault(id.Partition)
}
func (id NodeID) String() string {
return fmt.Sprintf("%s/%s", id.Partition, id.Name)
}
func (id NodeID) ACLString() string {
return fmt.Sprintf("%s--%s", id.Partition, id.Name)
}
func (id NodeID) TFString() string {
return id.ACLString()
}
type ServiceID struct {
Name string `json:",omitempty"`
Namespace string `json:",omitempty"`
Partition string `json:",omitempty"`
}
func NewServiceID(name, namespace, partition string) ServiceID {
id := ServiceID{
Name: name,
Namespace: namespace,
Partition: partition,
}
id.Normalize()
return id
}
func (id ServiceID) Less(other ServiceID) bool {
if id.Partition != other.Partition {
return id.Partition < other.Partition
}
if id.Namespace != other.Namespace {
return id.Namespace < other.Namespace
}
return id.Name < other.Name
}
func (id *ServiceID) Normalize() {
id.Namespace = NamespaceOrDefault(id.Namespace)
id.Partition = PartitionOrDefault(id.Partition)
}
func (id ServiceID) String() string {
return fmt.Sprintf("%s/%s/%s", id.Partition, id.Namespace, id.Name)
}
func (id ServiceID) ACLString() string {
return fmt.Sprintf("%s--%s--%s", id.Partition, id.Namespace, id.Name)
}
func (id ServiceID) TFString() string {
return id.ACLString()
}
func PartitionOrDefault(name string) string {
if name == "" {
return "default"
}
return name
}
func NamespaceOrDefault(name string) string {
if name == "" {
return "default"
}
return name
}
func DefaultToEmpty(name string) string {
if name == "default" {
return ""
}
return name
}
// PartitionQueryOptions returns an *api.QueryOptions with the given partition
// field set only if the partition is non-default. This helps when writing
// tests for joint use in OSS and ENT.
func PartitionQueryOptions(partition string) *api.QueryOptions {
return &api.QueryOptions{
Partition: DefaultToEmpty(partition),
}
}

123
testing/deployer/topology/images.go
Unescape View File

@ -0,0 +1,123 @@
package topology
import (
"strings"
)
type Images struct {
Consul string `json:",omitempty"`
ConsulOSS string `json:",omitempty"`
ConsulEnterprise string `json:",omitempty"`
Envoy string
Dataplane string
}
func (i Images) LocalDataplaneImage() string {
if i.Dataplane == "" {
return ""
}
img, tag, ok := strings.Cut(i.Dataplane, ":")
if !ok {
tag = "latest"
}
repo, name, ok := strings.Cut(img, "/")
if ok {
name = repo + "-" + name
}
// ex: local/hashicorp-consul-dataplane:1.1.0
return "local/" + name + ":" + tag
}
func (i Images) EnvoyConsulImage() string {
if i.Consul == "" || i.Envoy == "" {
return ""
}
img1, tag1, ok1 := strings.Cut(i.Consul, ":")
img2, tag2, ok2 := strings.Cut(i.Envoy, ":")
if !ok1 {
tag1 = "latest"
}
if !ok2 {
tag2 = "latest"
}
repo1, name1, ok1 := strings.Cut(img1, "/")
repo2, name2, ok2 := strings.Cut(img2, "/")
if ok1 {
name1 = repo1 + "-" + name1
} else {
name1 = repo1
}
if ok2 {
name2 = repo2 + "-" + name2
} else {
name2 = repo2
}
// ex: local/hashicorp-consul-and-envoyproxy-envoy:1.15.0-with-v1.26.2
return "local/" + name1 + "-and-" + name2 + ":" + tag1 + "-with-" + tag2
}
func (i Images) ChooseNode(kind NodeKind) Images {
switch kind {
case NodeKindServer:
i.Envoy = ""
i.Dataplane = ""
case NodeKindClient:
i.Dataplane = ""
case NodeKindDataplane:
i.Envoy = ""
default:
// do nothing
}
return i
}
func (i Images) ChooseConsul(enterprise bool) Images {
if enterprise {
i.Consul = i.ConsulEnterprise
} else {
i.Consul = i.ConsulOSS
}
i.ConsulEnterprise = ""
i.ConsulOSS = ""
return i
}
func (i Images) OverrideWith(i2 Images) Images {
if i2.Consul != "" {
i.Consul = i2.Consul
}
if i2.ConsulOSS != "" {
i.ConsulOSS = i2.ConsulOSS
}
if i2.ConsulEnterprise != "" {
i.ConsulEnterprise = i2.ConsulEnterprise
}
if i2.Envoy != "" {
i.Envoy = i2.Envoy
}
if i2.Dataplane != "" {
i.Dataplane = i2.Dataplane
}
return i
}
// DefaultImages controls which specific docker images are used as default
// values for topology components that do not specify values.
//
// These can be bulk-updated using the make target 'make update-defaults'
func DefaultImages() Images {
return Images{
Consul: "",
ConsulOSS: DefaultConsulImage,
ConsulEnterprise: DefaultConsulEnterpriseImage,
Envoy: DefaultEnvoyImage,
Dataplane: DefaultDataplaneImage,
}
}

98
testing/deployer/topology/images_test.go
Unescape View File

@ -0,0 +1,98 @@
package topology
import (
"strconv"
"testing"
"github.com/stretchr/testify/require"
)
func TestImages_EnvoyConsulImage(t *testing.T) {
type testcase struct {
consul, envoy string
expect string
}
run := func(t *testing.T, tc testcase) {
i := Images{Consul: tc.consul, Envoy: tc.envoy}
j := i.EnvoyConsulImage()
require.Equal(t, tc.expect, j)
}
cases := []testcase{
{
consul: "",
envoy: "",
expect: "",
},
{
consul: "consul",
envoy: "",
expect: "",
},
{
consul: "",
envoy: "envoy",
expect: "",
},
{
consul: "consul",
envoy: "envoy",
expect: "local/consul-and-envoy:latest-with-latest",
},
// repos
{
consul: "hashicorp/consul",
envoy: "envoy",
expect: "local/hashicorp-consul-and-envoy:latest-with-latest",
},
{
consul: "consul",
envoy: "envoyproxy/envoy",
expect: "local/consul-and-envoyproxy-envoy:latest-with-latest",
},
{
consul: "hashicorp/consul",
envoy: "envoyproxy/envoy",
expect: "local/hashicorp-consul-and-envoyproxy-envoy:latest-with-latest",
},
// tags
{
consul: "consul:1.15.0",
envoy: "envoy",
expect: "local/consul-and-envoy:1.15.0-with-latest",
},
{
consul: "consul",
envoy: "envoy:v1.26.1",
expect: "local/consul-and-envoy:latest-with-v1.26.1",
},
{
consul: "consul:1.15.0",
envoy: "envoy:v1.26.1",
expect: "local/consul-and-envoy:1.15.0-with-v1.26.1",
},
// repos+tags
{
consul: "hashicorp/consul:1.15.0",
envoy: "envoy:v1.26.1",
expect: "local/hashicorp-consul-and-envoy:1.15.0-with-v1.26.1",
},
{
consul: "consul:1.15.0",
envoy: "envoyproxy/envoy:v1.26.1",
expect: "local/consul-and-envoyproxy-envoy:1.15.0-with-v1.26.1",
},
{
consul: "hashicorp/consul:1.15.0",
envoy: "envoyproxy/envoy:v1.26.1",
expect: "local/hashicorp-consul-and-envoyproxy-envoy:1.15.0-with-v1.26.1",
},
}
for i, tc := range cases {
t.Run(strconv.Itoa(i), func(t *testing.T) {
run(t, tc)
})
}
}

787
testing/deployer/topology/topology.go
Unescape View File

@ -0,0 +1,787 @@
package topology
import (
"errors"
"fmt"
"net"
"net/netip"
"reflect"
"sort"
"github.com/hashicorp/consul/api"
)
type Topology struct {
ID string
// Images controls which specific docker images are used when running this
// node. Non-empty fields here override non-empty fields inherited from the
// general default values from DefaultImages().
Images Images
// Networks is the list of networks to create for this set of clusters.
Networks map[string]*Network
// Clusters defines the list of Consul clusters that should be created, and
// their associated workloads.
Clusters map[string]*Cluster
// Peerings defines the list of pairwise peerings that should be established
// between clusters.
Peerings []*Peering `json:",omitempty"`
}
func (t *Topology) DigestExposedProxyPort(netName string, proxyPort int) (bool, error) {
net, ok := t.Networks[netName]
if !ok {
return false, fmt.Errorf("found output network that does not exist: %s", netName)
}
if net.ProxyPort == proxyPort {
return false, nil
}
net.ProxyPort = proxyPort
// Denormalize for UX.
for _, cluster := range t.Clusters {
for _, node := range cluster.Nodes {
for _, addr := range node.Addresses {
if addr.Network == netName {
addr.ProxyPort = proxyPort
}
}
}
}
return true, nil
}
func (t *Topology) SortedNetworks() []*Network {
var out []*Network
for _, n := range t.Networks {
out = append(out, n)
}
sort.Slice(out, func(i, j int) bool {
return out[i].Name < out[j].Name
})
return out
}
func (t *Topology) SortedClusters() []*Cluster {
var out []*Cluster
for _, c := range t.Clusters {
out = append(out, c)
}
sort.Slice(out, func(i, j int) bool {
return out[i].Name < out[j].Name
})
return out
}
type Config struct {
// Images controls which specific docker images are used when running this
// node. Non-empty fields here override non-empty fields inherited from the
// general default values from DefaultImages().
Images Images
// Networks is the list of networks to create for this set of clusters.
Networks []*Network
// Clusters defines the list of Consul clusters that should be created, and
// their associated workloads.
Clusters []*Cluster
// Peerings defines the list of pairwise peerings that should be established
// between clusters.
Peerings []*Peering
}
func (c *Config) Cluster(name string) *Cluster {
for _, cluster := range c.Clusters {
if cluster.Name == name {
return cluster
}
}
return nil
}
type Network struct {
Type string // lan/wan ; empty means lan
Name string // logical name
// computed at topology compile
DockerName string
// generated during network-and-tls
Subnet string
IPPool []string `json:"-"`
// generated during network-and-tls
ProxyAddress string `json:",omitempty"`
DNSAddress string `json:",omitempty"`
// filled in from terraform outputs after network-and-tls
ProxyPort int `json:",omitempty"`
}
func (n *Network) IsLocal() bool {
return n.Type == "" || n.Type == "lan"
}
func (n *Network) IsPublic() bool {
return n.Type == "wan"
}
func (n *Network) inheritFromExisting(existing *Network) {
n.Subnet = existing.Subnet
n.IPPool = existing.IPPool
n.ProxyAddress = existing.ProxyAddress
n.DNSAddress = existing.DNSAddress
n.ProxyPort = existing.ProxyPort
}
func (n *Network) IPByIndex(index int) string {
if index >= len(n.IPPool) {
panic(fmt.Sprintf(
"not enough ips on this network to assign index %d: %d",
len(n.IPPool), index,
))
}
return n.IPPool[index]
}
func (n *Network) SetSubnet(subnet string) (bool, error) {
if n.Subnet == subnet {
return false, nil
}
p, err := netip.ParsePrefix(subnet)
if err != nil {
return false, err
}
if !p.IsValid() {
return false, errors.New("not valid")
}
p = p.Masked()
var ipPool []string
addr := p.Addr()
for {
if !p.Contains(addr) {
break
}
ipPool = append(ipPool, addr.String())
addr = addr.Next()
}
ipPool = ipPool[2:] // skip the x.x.x.{0,1}
n.Subnet = subnet
n.IPPool = ipPool
return true, nil
}
// Cluster represents a single standalone install of Consul. This is the unit
// of what is peered when using cluster peering. Older consul installs would
// call this a datacenter.
type Cluster struct {
Name string
NetworkName string // empty assumes same as Name
// Images controls which specific docker images are used when running this
// cluster. Non-empty fields here override non-empty fields inherited from
// the enclosing Topology.
Images Images
// Enterprise marks this cluster as desiring to run Consul Enterprise
// components.
Enterprise bool `json:",omitempty"`
// Nodes is the definition of the nodes (agent-less and agent-ful).
Nodes []*Node
// Partitions is a list of tenancy configurations that should be created
// after the servers come up but before the clients and the rest of the
// topology starts.
//
// Enterprise Only.
Partitions []*Partition `json:",omitempty"`
// Datacenter defaults to "Name" if left unspecified. It lets you possibly
// create multiple peer clusters with identical datacenter names.
Datacenter string
// InitialConfigEntries is a convenience function to have some config
// entries created after the servers start up but before the rest of the
// topology comes up.
InitialConfigEntries []api.ConfigEntry `json:",omitempty"`
// TLSVolumeName is the docker volume name containing the various certs
// generated by 'consul tls cert create'
//
// This is generated during the networking phase and is not user specified.
TLSVolumeName string `json:",omitempty"`
// Peerings is a map of peering names to information about that peering in this cluster
//
// Denormalized during compile.
Peerings map[string]*PeerCluster `json:",omitempty"`
}
func (c *Cluster) inheritFromExisting(existing *Cluster) {
c.TLSVolumeName = existing.TLSVolumeName
}
type Partition struct {
Name string
Namespaces []string
}
func (c *Cluster) hasPartition(p string) bool {
for _, partition := range c.Partitions {
if partition.Name == p {
return true
}
}
return false
}
func (c *Cluster) PartitionQueryOptionsList() []*api.QueryOptions {
if !c.Enterprise {
return []*api.QueryOptions{{}}
}
var out []*api.QueryOptions
for _, p := range c.Partitions {
out = append(out, &api.QueryOptions{Partition: p.Name})
}
return out
}
func (c *Cluster) ServerNodes() []*Node {
var out []*Node
for _, node := range c.SortedNodes() {
if node.Kind != NodeKindServer || node.Disabled {
continue
}
out = append(out, node)
}
return out
}
func (c *Cluster) ServerByAddr(addr string) *Node {
expect, _, err := net.SplitHostPort(addr)
if err != nil {
return nil
}
for _, node := range c.Nodes {
if node.Kind != NodeKindServer || node.Disabled {
continue
}
if node.LocalAddress() == expect {
return node
}
}
return nil
}
func (c *Cluster) FirstServer() *Node {
for _, node := range c.Nodes {
if node.IsServer() && !node.Disabled && node.ExposedPort(8500) > 0 {
return node
}
}
return nil
}
func (c *Cluster) FirstClient() *Node {
for _, node := range c.Nodes {
if node.Kind != NodeKindClient || node.Disabled {
continue
}
return node
}
return nil
}
func (c *Cluster) ActiveNodes() []*Node {
var out []*Node
for _, node := range c.Nodes {
if !node.Disabled {
out = append(out, node)
}
}
return out
}
func (c *Cluster) SortedNodes() []*Node {
var out []*Node
out = append(out, c.Nodes...)
kindOrder := map[NodeKind]int{
NodeKindServer: 1,
NodeKindClient: 2,
NodeKindDataplane: 2,
}
sort.Slice(out, func(i, j int) bool {
ni, nj := out[i], out[j]
// servers before clients/dataplanes
ki, kj := kindOrder[ni.Kind], kindOrder[nj.Kind]
if ki < kj {
return true
} else if ki > kj {
return false
}
// lex sort by partition
if ni.Partition < nj.Partition {
return true
} else if ni.Partition > nj.Partition {
return false
}
// lex sort by name
return ni.Name < nj.Name
})
return out
}
func (c *Cluster) FindService(id NodeServiceID) *Service {
id.Normalize()
nid := id.NodeID()
sid := id.ServiceID()
return c.ServiceByID(nid, sid)
}
func (c *Cluster) ServiceByID(nid NodeID, sid ServiceID) *Service {
return c.NodeByID(nid).ServiceByID(sid)
}
func (c *Cluster) ServicesByID(sid ServiceID) []*Service {
sid.Normalize()
var out []*Service
for _, n := range c.Nodes {
for _, svc := range n.Services {
if svc.ID == sid {
out = append(out, svc)
}
}
}
return out
}
func (c *Cluster) NodeByID(nid NodeID) *Node {
nid.Normalize()
for _, n := range c.Nodes {
if n.ID() == nid {
return n
}
}
panic("node not found: " + nid.String())
}
type Address struct {
Network string
// denormalized at topology compile
Type string
// denormalized at topology compile
DockerNetworkName string
// generated after network-and-tls
IPAddress string
// denormalized from terraform outputs stored in the Network
ProxyPort int `json:",omitempty"`
}
func (a *Address) inheritFromExisting(existing *Address) {
a.IPAddress = existing.IPAddress
a.ProxyPort = existing.ProxyPort
}
func (a Address) IsLocal() bool {
return a.Type == "" || a.Type == "lan"
}
func (a Address) IsPublic() bool {
return a.Type == "wan"
}
type NodeKind string
const (
NodeKindUnknown NodeKind = ""
NodeKindServer NodeKind = "server"
NodeKindClient NodeKind = "client"
NodeKindDataplane NodeKind = "dataplane"
)
// TODO: rename pod
type Node struct {
Kind NodeKind
Partition string // will be not empty
Name string // logical name
// Images controls which specific docker images are used when running this
// node. Non-empty fields here override non-empty fields inherited from
// the enclosing Cluster.
Images Images
// AgentEnv contains optional environment variables to attach to Consul agents.
AgentEnv []string
Disabled bool `json:",omitempty"`
Addresses []*Address
Services []*Service
// denormalized at topology compile
Cluster string
Datacenter string
// computed at topology compile
Index int
// generated during network-and-tls
TLSCertPrefix string `json:",omitempty"`
// dockerName is computed at topology compile
dockerName string
// usedPorts has keys that are computed at topology compile (internal
// ports) and values initialized to zero until terraform creates the pods
// and extracts the exposed port values from output variables.
usedPorts map[int]int // keys are from compile / values are from terraform output vars
}
func (n *Node) DockerName() string {
return n.dockerName
}
func (n *Node) ExposedPort(internalPort int) int {
return n.usedPorts[internalPort]
}
func (n *Node) SortedPorts() []int {
var out []int
for internalPort := range n.usedPorts {
out = append(out, internalPort)
}
sort.Ints(out)
return out
}
func (n *Node) inheritFromExisting(existing *Node) {
n.TLSCertPrefix = existing.TLSCertPrefix
merged := existing.usedPorts
for k, vNew := range n.usedPorts {
if _, present := merged[k]; !present {
merged[k] = vNew
}
}
n.usedPorts = merged
}
func (n *Node) String() string {
return n.ID().String()
}
func (n *Node) ID() NodeID {
return NewNodeID(n.Name, n.Partition)
}
func (n *Node) CatalogID() NodeID {
return NewNodeID(n.PodName(), n.Partition)
}
func (n *Node) PodName() string {
return n.dockerName + "-pod"
}
func (n *Node) AddressByNetwork(name string) *Address {
for _, a := range n.Addresses {
if a.Network == name {
return a
}
}
return nil
}
func (n *Node) LocalAddress() string {
for _, a := range n.Addresses {
if a.IsLocal() {
if a.IPAddress == "" {
panic("node has no assigned local address")
}
return a.IPAddress
}
}
panic("node has no local network")
}
func (n *Node) HasPublicAddress() bool {
for _, a := range n.Addresses {
if a.IsPublic() {
return true
}
}
return false
}
func (n *Node) LocalProxyPort() int {
for _, a := range n.Addresses {
if a.IsLocal() {
if a.ProxyPort > 0 {
return a.ProxyPort
}
panic("node has no assigned local address")
}
}
panic("node has no local network")
}
func (n *Node) PublicAddress() string {
for _, a := range n.Addresses {
if a.IsPublic() {
if a.IPAddress == "" {
panic("node has no assigned public address")
}
return a.IPAddress
}
}
panic("node has no public network")
}
func (n *Node) PublicProxyPort() int {
for _, a := range n.Addresses {
if a.IsPublic() {
if a.ProxyPort > 0 {
return a.ProxyPort
}
panic("node has no assigned public address")
}
}
panic("node has no public network")
}
func (n *Node) IsServer() bool {
return n.Kind == NodeKindServer
}
func (n *Node) IsAgent() bool {
return n.Kind == NodeKindServer || n.Kind == NodeKindClient
}
func (n *Node) RunsWorkloads() bool {
return n.IsAgent() || n.IsDataplane()
}
func (n *Node) IsDataplane() bool {
return n.Kind == NodeKindDataplane
}
func (n *Node) SortedServices() []*Service {
var out []*Service
out = append(out, n.Services...)
sort.Slice(out, func(i, j int) bool {
mi := out[i].IsMeshGateway
mj := out[j].IsMeshGateway
if mi && !mi {
return false
} else if !mi && mj {
return true
}
return out[i].ID.Less(out[j].ID)
})
return out
}
// DigestExposedPorts returns true if it was changed.
func (n *Node) DigestExposedPorts(ports map[int]int) bool {
if reflect.DeepEqual(n.usedPorts, ports) {
return false
}
for internalPort := range n.usedPorts {
if v, ok := ports[internalPort]; ok {
n.usedPorts[internalPort] = v
} else {
panic(fmt.Sprintf(
"cluster %q node %q port %d not found in exposed list",
n.Cluster,
n.ID(),
internalPort,
))
}
}
for _, svc := range n.Services {
svc.DigestExposedPorts(ports)
}
return true
}
func (n *Node) ServiceByID(sid ServiceID) *Service {
sid.Normalize()
for _, svc := range n.Services {
if svc.ID == sid {
return svc
}
}
panic("service not found: " + sid.String())
}
type ServiceAndNode struct {
Service *Service
Node *Node
}
type Service struct {
ID ServiceID
Image string
Port int
ExposedPort int `json:",omitempty"`
Disabled bool `json:",omitempty"` // TODO
// TODO: expose extra port here?
Meta map[string]string `json:",omitempty"`
// TODO(rb): re-expose this perhaps? Protocol string `json:",omitempty"` // tcp|http (empty == tcp)
CheckHTTP string `json:",omitempty"` // url; will do a GET
CheckTCP string `json:",omitempty"` // addr; will do a socket open/close
EnvoyAdminPort int
ExposedEnvoyAdminPort int `json:",omitempty"`
EnvoyPublicListenerPort int `json:",omitempty"` // agentless
Command []string `json:",omitempty"` // optional
Env []string `json:",omitempty"` // optional
DisableServiceMesh bool `json:",omitempty"`
IsMeshGateway bool `json:",omitempty"`
Upstreams []*Upstream
// denormalized at topology compile
Node *Node `json:"-"`
}
func (s *Service) inheritFromExisting(existing *Service) {
s.ExposedPort = existing.ExposedPort
s.ExposedEnvoyAdminPort = existing.ExposedEnvoyAdminPort
}
func (s *Service) ports() []int {
var out []int
if s.Port > 0 {
out = append(out, s.Port)
}
if s.EnvoyAdminPort > 0 {
out = append(out, s.EnvoyAdminPort)
}
if s.EnvoyPublicListenerPort > 0 {
out = append(out, s.EnvoyPublicListenerPort)
}
for _, u := range s.Upstreams {
if u.LocalPort > 0 {
out = append(out, u.LocalPort)
}
}
return out
}
func (s *Service) HasCheck() bool {
return s.CheckTCP != "" || s.CheckHTTP != ""
}
func (s *Service) DigestExposedPorts(ports map[int]int) {
s.ExposedPort = ports[s.Port]
if s.EnvoyAdminPort > 0 {
s.ExposedEnvoyAdminPort = ports[s.EnvoyAdminPort]
} else {
s.ExposedEnvoyAdminPort = 0
}
}
func (s *Service) Validate() error {
if s.ID.Name == "" {
return fmt.Errorf("service name is required")
}
if s.Image == "" && !s.IsMeshGateway {
return fmt.Errorf("service image is required")
}
if s.Port <= 0 {
return fmt.Errorf("service has invalid port")
}
if s.DisableServiceMesh && s.IsMeshGateway {
return fmt.Errorf("cannot disable service mesh and still run a mesh gateway")
}
if s.DisableServiceMesh && len(s.Upstreams) > 0 {
return fmt.Errorf("cannot disable service mesh and configure upstreams")
}
if s.DisableServiceMesh {
if s.EnvoyAdminPort != 0 {
return fmt.Errorf("cannot use envoy admin port without a service mesh")
}
} else {
if s.EnvoyAdminPort <= 0 {
return fmt.Errorf("envoy admin port is required")
}
}
for _, u := range s.Upstreams {
if u.ID.Name == "" {
return fmt.Errorf("upstream service name is required")
}
if u.LocalPort <= 0 {
return fmt.Errorf("upstream local port is required")
}
if u.LocalAddress != "" {
ip := net.ParseIP(u.LocalAddress)
if ip == nil {
return fmt.Errorf("upstream local address is invalid: %s", u.LocalAddress)
}
}
}
return nil
}
type Upstream struct {
ID ServiceID
LocalAddress string `json:",omitempty"` // defaults to 127.0.0.1
LocalPort int
Peer string `json:",omitempty"`
// TODO: what about mesh gateway mode overrides?
// computed at topology compile
Cluster string `json:",omitempty"`
Peering *PeerCluster `json:",omitempty"` // this will have Link!=nil
}
type Peering struct {
Dialing PeerCluster
Accepting PeerCluster
}
type PeerCluster struct {
Name string
Partition string
PeerName string // name to call it on this side; defaults if not specified
// computed at topology compile (pointer so it can be empty in json)
Link *PeerCluster `json:",omitempty"`
}
func (c PeerCluster) String() string {
return c.Name + ":" + c.Partition
}
func (p *Peering) String() string {
return "(" + p.Dialing.String() + ")->(" + p.Accepting.String() + ")"
}

17
testing/deployer/topology/util.go
Unescape View File

@ -0,0 +1,17 @@
package topology
func MergeSlices[V any](x, y []V) []V {
switch {
case len(x) == 0 && len(y) == 0:
return nil
case len(x) == 0:
return y
case len(y) == 0:
return x
}
out := make([]V, 0, len(x)+len(y))
out = append(out, x...)
out = append(out, y...)
return out
}

11
testing/deployer/topology/util_test.go
Unescape View File

@ -0,0 +1,11 @@
package topology
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestMergeSlices(t *testing.T) {
require.Nil(t, MergeSlices[int](nil, nil))
}

63
testing/deployer/util/consul.go
Unescape View File

@ -0,0 +1,63 @@
package util
import (
"fmt"
"net/http"
"net/url"
"strconv"
"github.com/hashicorp/consul/api"
"github.com/hashicorp/go-cleanhttp"
)
func ProxyNotPooledAPIClient(proxyPort int, containerIP string, containerPort int, token string) (*api.Client, error) {
return proxyAPIClient(cleanhttp.DefaultTransport(), proxyPort, containerIP, containerPort, token)
}
func ProxyAPIClient(proxyPort int, containerIP string, containerPort int, token string) (*api.Client, error) {
return proxyAPIClient(cleanhttp.DefaultPooledTransport(), proxyPort, containerIP, containerPort, token)
}
func proxyAPIClient(baseTransport *http.Transport, proxyPort int, containerIP string, containerPort int, token string) (*api.Client, error) {
if proxyPort <= 0 {
return nil, fmt.Errorf("cannot use an http proxy on port %d", proxyPort)
}
if containerIP == "" {
return nil, fmt.Errorf("container IP is required")
}
if containerPort <= 0 {
return nil, fmt.Errorf("cannot dial api client on port %d", containerPort)
}
proxyURL, err := url.Parse("http://127.0.0.1:" + strconv.Itoa(proxyPort))
if err != nil {
return nil, err
}
cfg := api.DefaultConfig()
cfg.Transport = baseTransport
cfg.Transport.Proxy = http.ProxyURL(proxyURL)
cfg.Address = fmt.Sprintf("http://%s:%d", containerIP, containerPort)
cfg.Token = token
return api.NewClient(cfg)
}
func ProxyNotPooledHTTPTransport(proxyPort int) (*http.Transport, error) {
return proxyHTTPTransport(cleanhttp.DefaultTransport(), proxyPort)
}
func ProxyHTTPTransport(proxyPort int) (*http.Transport, error) {
return proxyHTTPTransport(cleanhttp.DefaultPooledTransport(), proxyPort)
}
func proxyHTTPTransport(baseTransport *http.Transport, proxyPort int) (*http.Transport, error) {
if proxyPort <= 0 {
return nil, fmt.Errorf("cannot use an http proxy on port %d", proxyPort)
}
proxyURL, err := url.Parse("http://127.0.0.1:" + strconv.Itoa(proxyPort))
if err != nil {
return nil, err
}
baseTransport.Proxy = http.ProxyURL(proxyURL)
return baseTransport, nil
}

57
testing/deployer/util/files.go
Unescape View File

@ -0,0 +1,57 @@
package util
import (
"fmt"
"io"
"os"
"path/filepath"
"golang.org/x/crypto/blake2b"
)
func FilesExist(parent string, paths ...string) (bool, error) {
for _, p := range paths {
ok, err := FileExists(filepath.Join(parent, p))
if err != nil {
return false, err
}
if !ok {
return false, nil
}
}
return true, nil
}
func FileExists(path string) (bool, error) {
_, err := os.Stat(path)
if os.IsNotExist(err) {
return false, nil
} else if err != nil {
return false, err
} else {
return true, nil
}
}
func HashFile(path string) (string, error) {
hash, err := blake2b.New256(nil)
if err != nil {
return "", err
}
if err := AddFileToHash(path, hash); err != nil {
return "", err
}
return fmt.Sprintf("%x", hash.Sum(nil)), nil
}
func AddFileToHash(path string, w io.Writer) error {
f, err := os.Open(path)
if err != nil {
return err
}
defer f.Close()
_, err = io.Copy(w, f)
return err
}

21
testing/deployer/util/internal/ipamutils/doc.go
Unescape View File

@ -0,0 +1,21 @@
// Copyright 2015 Docker Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Originally from:
// https://github.com/moby/moby/blob/7489b51f610104ab5acc43f4e77142927e7b522e/libnetwork/ipamutils
//
// The only changes were to remove dead code from the package that we did not
// need, and to edit the tests to use github.com/stretchr/testify to avoid an
// extra dependency.
package ipamutils

117
testing/deployer/util/internal/ipamutils/utils.go
Unescape View File

@ -0,0 +1,117 @@
// Package ipamutils provides utility functions for ipam management
package ipamutils
import (
"fmt"
"net"
"sync"
)
var (
// predefinedLocalScopeDefaultNetworks contains a list of 31 IPv4 private networks with host size 16 and 12
// (172.17-31.x.x/16, 192.168.x.x/20) which do not overlap with the networks in `PredefinedGlobalScopeDefaultNetworks`
predefinedLocalScopeDefaultNetworks []*net.IPNet
// predefinedGlobalScopeDefaultNetworks contains a list of 64K IPv4 private networks with host size 8
// (10.x.x.x/24) which do not overlap with the networks in `PredefinedLocalScopeDefaultNetworks`
predefinedGlobalScopeDefaultNetworks []*net.IPNet
mutex sync.Mutex
localScopeDefaultNetworks = []*NetworkToSplit{{"172.17.0.0/16", 16}, {"172.18.0.0/16", 16}, {"172.19.0.0/16", 16},
{"172.20.0.0/14", 16}, {"172.24.0.0/14", 16}, {"172.28.0.0/14", 16},
{"192.168.0.0/16", 20}}
globalScopeDefaultNetworks = []*NetworkToSplit{{"10.0.0.0/8", 24}}
)
// NetworkToSplit represent a network that has to be split in chunks with mask length Size.
// Each subnet in the set is derived from the Base pool. Base is to be passed
// in CIDR format.
// Example: a Base "10.10.0.0/16 with Size 24 will define the set of 256
// 10.10.[0-255].0/24 address pools
type NetworkToSplit struct {
Base string `json:"base"`
Size int `json:"size"`
}
func init() {
var err error
if predefinedGlobalScopeDefaultNetworks, err = SplitNetworks(globalScopeDefaultNetworks); err != nil {
panic("failed to initialize the global scope default address pool: " + err.Error())
}
if predefinedLocalScopeDefaultNetworks, err = SplitNetworks(localScopeDefaultNetworks); err != nil {
panic("failed to initialize the local scope default address pool: " + err.Error())
}
}
// ConfigGlobalScopeDefaultNetworks configures global default pool.
// Ideally this will be called from SwarmKit as part of swarm init
func ConfigGlobalScopeDefaultNetworks(defaultAddressPool []*NetworkToSplit) error {
if defaultAddressPool == nil {
return nil
}
mutex.Lock()
defer mutex.Unlock()
defaultNetworks, err := SplitNetworks(defaultAddressPool)
if err != nil {
return err
}
predefinedGlobalScopeDefaultNetworks = defaultNetworks
return nil
}
// GetGlobalScopeDefaultNetworks returns a copy of the global-sopce network list.
func GetGlobalScopeDefaultNetworks() []*net.IPNet {
mutex.Lock()
defer mutex.Unlock()
return append([]*net.IPNet(nil), predefinedGlobalScopeDefaultNetworks...)
}
// GetLocalScopeDefaultNetworks returns a copy of the default local-scope network list.
func GetLocalScopeDefaultNetworks() []*net.IPNet {
return append([]*net.IPNet(nil), predefinedLocalScopeDefaultNetworks...)
}
// SplitNetworks takes a slice of networks, split them accordingly and returns them
func SplitNetworks(list []*NetworkToSplit) ([]*net.IPNet, error) {
localPools := make([]*net.IPNet, 0, len(list))
for _, p := range list {
_, b, err := net.ParseCIDR(p.Base)
if err != nil {
return nil, fmt.Errorf("invalid base pool %q: %v", p.Base, err)
}
ones, _ := b.Mask.Size()
if p.Size <= 0 || p.Size < ones {
return nil, fmt.Errorf("invalid pools size: %d", p.Size)
}
localPools = append(localPools, splitNetwork(p.Size, b)...)
}
return localPools, nil
}
func splitNetwork(size int, base *net.IPNet) []*net.IPNet {
one, bits := base.Mask.Size()
mask := net.CIDRMask(size, bits)
n := 1 << uint(size-one)
s := uint(bits - size)
list := make([]*net.IPNet, 0, n)
for i := 0; i < n; i++ {
ip := copyIP(base.IP)
addIntToIP(ip, uint(i<<s))
list = append(list, &net.IPNet{IP: ip, Mask: mask})
}
return list
}
func copyIP(from net.IP) net.IP {
ip := make([]byte, len(from))
copy(ip, from)
return ip
}
func addIntToIP(array net.IP, ordinal uint) {
for i := len(array) - 1; i >= 0; i-- {
array[i] |= (byte)(ordinal & 0xff)
ordinal >>= 8
}
}

102
testing/deployer/util/internal/ipamutils/utils_test.go
Unescape View File

@ -0,0 +1,102 @@
package ipamutils
import (
"net"
"testing"
"github.com/stretchr/testify/assert"
)
func initBroadPredefinedNetworks() []*net.IPNet {
pl := make([]*net.IPNet, 0, 31)
mask := []byte{255, 255, 0, 0}
for i := 17; i < 32; i++ {
pl = append(pl, &net.IPNet{IP: []byte{172, byte(i), 0, 0}, Mask: mask})
}
mask20 := []byte{255, 255, 240, 0}
for i := 0; i < 16; i++ {
pl = append(pl, &net.IPNet{IP: []byte{192, 168, byte(i << 4), 0}, Mask: mask20})
}
return pl
}
func initGranularPredefinedNetworks() []*net.IPNet {
pl := make([]*net.IPNet, 0, 256*256)
mask := []byte{255, 255, 255, 0}
for i := 0; i < 256; i++ {
for j := 0; j < 256; j++ {
pl = append(pl, &net.IPNet{IP: []byte{10, byte(i), byte(j), 0}, Mask: mask})
}
}
return pl
}
func initGlobalScopeNetworks() []*net.IPNet {
pl := make([]*net.IPNet, 0, 256*256)
mask := []byte{255, 255, 255, 0}
for i := 0; i < 256; i++ {
for j := 0; j < 256; j++ {
pl = append(pl, &net.IPNet{IP: []byte{30, byte(i), byte(j), 0}, Mask: mask})
}
}
return pl
}
func TestDefaultNetwork(t *testing.T) {
for _, nw := range GetGlobalScopeDefaultNetworks() {
if ones, bits := nw.Mask.Size(); bits != 32 || ones != 24 {
t.Fatalf("Unexpected size for network in granular list: %v", nw)
}
}
for _, nw := range GetLocalScopeDefaultNetworks() {
if ones, bits := nw.Mask.Size(); bits != 32 || (ones != 20 && ones != 16) {
t.Fatalf("Unexpected size for network in broad list: %v", nw)
}
}
originalBroadNets := initBroadPredefinedNetworks()
m := make(map[string]bool)
for _, v := range originalBroadNets {
m[v.String()] = true
}
for _, nw := range GetLocalScopeDefaultNetworks() {
_, ok := m[nw.String()]
assert.True(t, ok)
delete(m, nw.String())
}
assert.Empty(t, m)
originalGranularNets := initGranularPredefinedNetworks()
m = make(map[string]bool)
for _, v := range originalGranularNets {
m[v.String()] = true
}
for _, nw := range GetGlobalScopeDefaultNetworks() {
_, ok := m[nw.String()]
assert.True(t, ok)
delete(m, nw.String())
}
assert.Empty(t, m)
}
func TestConfigGlobalScopeDefaultNetworks(t *testing.T) {
err := ConfigGlobalScopeDefaultNetworks([]*NetworkToSplit{{"30.0.0.0/8", 24}})
assert.NoError(t, err)
originalGlobalScopeNetworks := initGlobalScopeNetworks()
m := make(map[string]bool)
for _, v := range originalGlobalScopeNetworks {
m[v.String()] = true
}
for _, nw := range GetGlobalScopeDefaultNetworks() {
_, ok := m[nw.String()]
assert.True(t, ok)
delete(m, nw.String())
}
assert.Empty(t, m)
}

17
testing/deployer/util/net.go
Unescape View File

@ -0,0 +1,17 @@
package util
import (
"github.com/hashicorp/consul/testing/deployer/util/internal/ipamutils"
)
// GetPossibleDockerNetworkSubnets returns a copy of the global-scope network list.
func GetPossibleDockerNetworkSubnets() map[string]struct{} {
list := ipamutils.GetGlobalScopeDefaultNetworks()
out := make(map[string]struct{})
for _, ipnet := range list {
subnet := ipnet.String()
out[subnet] = struct{}{}
}
return out
}
Write Preview
Loading…
Cancel
Save