github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Switch e2e_node to etcd3

pull/58/head
Jordan Liggitt 2018-10-03 01:39:57 -04:00
parent 85ae79500f
commit ad46728158
8 changed files with 42 additions and 261 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
staging/src/k8s.io/kube-aggregator/Godeps/Godeps.json generated
View File

@ -38,10 +38,6 @@
"ImportPath": "github.com/coreos/etcd/auth/authpb",
"Rev": "420a452267a7ce45b3fcbed04d54030d69964fc1"
},
{
"ImportPath": "github.com/coreos/etcd/client",
"Rev": "420a452267a7ce45b3fcbed04d54030d69964fc1"
},
{
"ImportPath": "github.com/coreos/etcd/clientv3",
"Rev": "420a452267a7ce45b3fcbed04d54030d69964fc1"
@ -58,14 +54,6 @@
"ImportPath": "github.com/coreos/etcd/mvcc/mvccpb",
"Rev": "420a452267a7ce45b3fcbed04d54030d69964fc1"
},
{
"ImportPath": "github.com/coreos/etcd/pkg/pathutil",
"Rev": "420a452267a7ce45b3fcbed04d54030d69964fc1"
},
{
"ImportPath": "github.com/coreos/etcd/pkg/srv",
"Rev": "420a452267a7ce45b3fcbed04d54030d69964fc1"
},
{
"ImportPath": "github.com/coreos/etcd/pkg/tlsutil",
"Rev": "420a452267a7ce45b3fcbed04d54030d69964fc1"
@ -78,14 +66,6 @@
"ImportPath": "github.com/coreos/etcd/pkg/types",
"Rev": "420a452267a7ce45b3fcbed04d54030d69964fc1"
},
{
"ImportPath": "github.com/coreos/etcd/version",
"Rev": "420a452267a7ce45b3fcbed04d54030d69964fc1"
},
{
"ImportPath": "github.com/coreos/go-semver/semver",
"Rev": "568e959cd89871e61434c1143528d9162da89ef2"
},
{
"ImportPath": "github.com/coreos/go-systemd/daemon",
"Rev": "48702e0da86bd25e76cfef347e2adeb434a0d0a6"
@ -322,10 +302,6 @@
"ImportPath": "github.com/stretchr/testify/assert",
"Rev": "c679ae2cc0cb27ec3293fea7e254e47386f05d69"
},
{
"ImportPath": "github.com/ugorji/go/codec",
"Rev": "ded73eae5db7e7a0ef6f55aace87a2873c5d2b74"
},
{
"ImportPath": "golang.org/x/crypto/ssh/terminal",
"Rev": "de0752318171da717af4ce24d0a2e8626afaeb11"
@ -1162,10 +1138,6 @@
"ImportPath": "k8s.io/apiserver/pkg/storage/etcd/metrics",
"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
},
{
"ImportPath": "k8s.io/apiserver/pkg/storage/etcd/util",
"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
},
{
"ImportPath": "k8s.io/apiserver/pkg/storage/etcd3",
"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"

28
staging/src/k8s.io/sample-apiserver/Godeps/Godeps.json generated
View File

@ -38,10 +38,6 @@
"ImportPath": "github.com/coreos/etcd/auth/authpb",
"Rev": "420a452267a7ce45b3fcbed04d54030d69964fc1"
},
{
"ImportPath": "github.com/coreos/etcd/client",
"Rev": "420a452267a7ce45b3fcbed04d54030d69964fc1"
},
{
"ImportPath": "github.com/coreos/etcd/clientv3",
"Rev": "420a452267a7ce45b3fcbed04d54030d69964fc1"
@ -58,14 +54,6 @@
"ImportPath": "github.com/coreos/etcd/mvcc/mvccpb",
"Rev": "420a452267a7ce45b3fcbed04d54030d69964fc1"
},
{
"ImportPath": "github.com/coreos/etcd/pkg/pathutil",
"Rev": "420a452267a7ce45b3fcbed04d54030d69964fc1"
},
{
"ImportPath": "github.com/coreos/etcd/pkg/srv",
"Rev": "420a452267a7ce45b3fcbed04d54030d69964fc1"
},
{
"ImportPath": "github.com/coreos/etcd/pkg/tlsutil",
"Rev": "420a452267a7ce45b3fcbed04d54030d69964fc1"
@ -78,14 +66,6 @@
"ImportPath": "github.com/coreos/etcd/pkg/types",
"Rev": "420a452267a7ce45b3fcbed04d54030d69964fc1"
},
{
"ImportPath": "github.com/coreos/etcd/version",
"Rev": "420a452267a7ce45b3fcbed04d54030d69964fc1"
},
{
"ImportPath": "github.com/coreos/go-semver/semver",
"Rev": "568e959cd89871e61434c1143528d9162da89ef2"
},
{
"ImportPath": "github.com/coreos/go-systemd/daemon",
"Rev": "48702e0da86bd25e76cfef347e2adeb434a0d0a6"
@ -302,10 +282,6 @@
"ImportPath": "github.com/spf13/pflag",
"Rev": "583c0c0531f06d5278b7d917446061adc344b5cd"
},
{
"ImportPath": "github.com/ugorji/go/codec",
"Rev": "ded73eae5db7e7a0ef6f55aace87a2873c5d2b74"
},
{
"ImportPath": "golang.org/x/crypto/ssh/terminal",
"Rev": "de0752318171da717af4ce24d0a2e8626afaeb11"
@ -1126,10 +1102,6 @@
"ImportPath": "k8s.io/apiserver/pkg/storage/etcd/metrics",
"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
},
{
"ImportPath": "k8s.io/apiserver/pkg/storage/etcd/util",
"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
},
{
"ImportPath": "k8s.io/apiserver/pkg/storage/etcd3",
"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"

2
test/e2e_node/e2e_node_suite_test.go
View File

@ -86,7 +86,7 @@ const rootfs = "/rootfs"
func TestE2eNode(t *testing.T) {
if *runServicesMode {
// If run-services-mode is specified, only run services in current process.
services.RunE2EServices()
services.RunE2EServices(t)
return
}
if *runKubeletMode {

7
test/e2e_node/services/BUILD
View File

@ -9,7 +9,6 @@ go_library(
name = "go_default_library",
srcs = [
"apiserver.go",
"etcd.go",
"internal_services.go",
"kubelet.go",
"logs.go",
@ -29,6 +28,8 @@ go_library(
"//pkg/kubelet/kubeletconfig/util/codec:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/storage/etcd/testing:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/storage/storagebackend:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/util/flag:go_default_library",
"//staging/src/k8s.io/client-go/dynamic:go_default_library",
@ -39,10 +40,6 @@ go_library(
"//test/e2e/framework:go_default_library",
"//test/e2e_node/builder:go_default_library",
"//test/e2e_node/remote:go_default_library",
"//vendor/github.com/coreos/etcd/etcdserver:go_default_library",
"//vendor/github.com/coreos/etcd/etcdserver/api/v2http:go_default_library",
"//vendor/github.com/coreos/etcd/pkg/transport:go_default_library",
"//vendor/github.com/coreos/etcd/pkg/types:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/github.com/kardianos/osext:go_default_library",
"//vendor/github.com/spf13/pflag:go_default_library",

27
test/e2e_node/services/apiserver.go
View File

@ -20,6 +20,7 @@ import (
"fmt"
"net"
"k8s.io/apiserver/pkg/storage/storagebackend"
apiserver "k8s.io/kubernetes/cmd/kube-apiserver/app"
"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
)
@ -31,21 +32,23 @@ const (
)
// APIServer is a server which manages apiserver.
type APIServer struct{}
type APIServer struct {
storageConfig storagebackend.Config
stopCh chan struct{}
}
// NewAPIServer creates an apiserver.
func NewAPIServer() *APIServer {
return &APIServer{}
func NewAPIServer(storageConfig storagebackend.Config) *APIServer {
return &APIServer{
storageConfig: storageConfig,
stopCh: make(chan struct{}),
}
}
// Start starts the apiserver, returns when apiserver is ready.
func (a *APIServer) Start() error {
o := options.NewServerRunOptions()
o.Etcd.StorageConfig.ServerList = []string{getEtcdClientURL()}
// TODO: Current setup of etcd in e2e-node tests doesn't support etcd v3
// protocol. We should migrate it to use the same infrastructure as all
// other tests (pkg/storage/etcd/testing).
o.Etcd.StorageConfig.Type = "etcd2"
o.Etcd.StorageConfig = a.storageConfig
_, ipnet, err := net.ParseCIDR(clusterIPRange)
if err != nil {
return err
@ -56,14 +59,12 @@ func (a *APIServer) Start() error {
errCh := make(chan error)
go func() {
defer close(errCh)
stopCh := make(chan struct{})
defer close(stopCh)
completedOptions, err := apiserver.Complete(o)
if err != nil {
errCh <- fmt.Errorf("set apiserver default options error: %v", err)
return
}
err = apiserver.Run(completedOptions, stopCh)
err = apiserver.Run(completedOptions, a.stopCh)
if err != nil {
errCh <- fmt.Errorf("run apiserver error: %v", err)
return
@ -80,6 +81,10 @@ func (a *APIServer) Start() error {
// Stop stops the apiserver. Currently, there is no way to stop the apiserver.
// The function is here only for completion.
func (a *APIServer) Stop() error {
if a.stopCh != nil {
close(a.stopCh)
a.stopCh = nil
}
return nil
}

161
test/e2e_node/services/etcd.go
View File

@ -1,161 +0,0 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package services
import (
"crypto/tls"
"net"
"net/http"
"net/url"
"time"
"github.com/coreos/etcd/etcdserver"
"github.com/coreos/etcd/etcdserver/api/v2http"
"github.com/coreos/etcd/pkg/transport"
"github.com/coreos/etcd/pkg/types"
"github.com/golang/glog"
)
// TODO: These tests should not be leveraging v2http
// TODO(random-liu): Add service interface to manage services with the same behaviour.
// All following configurations are got from etcd source code.
// TODO(random-liu): Use embed.NewConfig after etcd3 is supported.
const (
etcdName = "etcd"
clientURLStr = "http://localhost:4001" // clientURL has listener created and handles etcd API traffic
peerURLStr = "http://localhost:7001" // peerURL does't have listener created, it is used to pass Etcd validation
snapCount = etcdserver.DefaultSnapCount
maxSnapFiles = 5
maxWALFiles = 5
tickMs = 100
electionTicks = 10
etcdHealthCheckURL = clientURLStr + "/v2/keys/" // Trailing slash is required,
)
// EtcdServer is a server which manages etcd.
type EtcdServer struct {
*etcdserver.EtcdServer
config *etcdserver.ServerConfig
clientListen net.Listener
}
// NewEtcd creates a new default etcd server using 'dataDir' for persistence.
func NewEtcd(dataDir string) *EtcdServer {
clientURLs, err := types.NewURLs([]string{clientURLStr})
if err != nil {
glog.Fatalf("Failed to parse client url %q: %v", clientURLStr, err)
}
peerURLs, err := types.NewURLs([]string{peerURLStr})
if err != nil {
glog.Fatalf("Failed to parse peer url %q: %v", peerURLStr, err)
}
config := &etcdserver.ServerConfig{
Name: etcdName,
ClientURLs: clientURLs,
PeerURLs: peerURLs,
DataDir: dataDir,
InitialPeerURLsMap: map[string]types.URLs{etcdName: peerURLs},
NewCluster: true,
SnapCount: snapCount,
MaxSnapFiles: maxSnapFiles,
MaxWALFiles: maxWALFiles,
TickMs: tickMs,
ElectionTicks: electionTicks,
AuthToken: "simple",
}
return &EtcdServer{
config: config,
}
}
// Start starts the etcd server and listening for client connections
func (e *EtcdServer) Start() error {
var err error
e.EtcdServer, err = etcdserver.NewServer(e.config)
if err != nil {
return err
}
// create client listener, there should be only one url
e.clientListen, err = createListener(e.config.ClientURLs[0])
if err != nil {
return err
}
// start etcd
e.EtcdServer.Start()
// setup client listener
ch := v2http.NewClientHandler(e.EtcdServer, e.config.ReqTimeout())
errCh := make(chan error)
go func(l net.Listener) {
defer close(errCh)
srv := &http.Server{
Handler: ch,
ReadTimeout: 5 * time.Minute,
}
// Serve always returns a non-nil error.
errCh <- srv.Serve(l)
}(e.clientListen)
err = readinessCheck("etcd", []string{etcdHealthCheckURL}, errCh)
if err != nil {
return err
}
return nil
}
// Stop closes all connections and stops the Etcd server
func (e *EtcdServer) Stop() error {
if e.EtcdServer != nil {
e.EtcdServer.Stop()
}
if e.clientListen != nil {
err := e.clientListen.Close()
if err != nil {
return err
}
}
return nil
}
// Name returns the server's unique name
func (e *EtcdServer) Name() string {
return etcdName
}
func createListener(url url.URL) (net.Listener, error) {
l, err := net.Listen("tcp", url.Host)
if err != nil {
return nil, err
}
l, err = transport.NewKeepAliveListener(l, url.Scheme, &tls.Config{})
if err != nil {
return nil, err
}
return l, nil
}
func getEtcdClientURL() string {
return clientURLStr
}
func getEtcdHealthCheckURL() string {
return etcdHealthCheckURL
}

45
test/e2e_node/services/internal_services.go
View File

@ -17,9 +17,11 @@ limitations under the License.
package services
import (
"io/ioutil"
"os"
"testing"
etcdtesting "k8s.io/apiserver/pkg/storage/etcd/testing"
"k8s.io/apiserver/pkg/storage/storagebackend"
"k8s.io/kubernetes/test/e2e/framework"
"github.com/golang/glog"
@ -29,7 +31,8 @@ import (
type e2eServices struct {
rmDirs []string
// statically linked e2e services
etcdServer *EtcdServer
etcdServer *etcdtesting.EtcdTestServer
etcdStorage *storagebackend.Config
apiServer *APIServer
nsController *NamespaceController
}
@ -40,9 +43,9 @@ func newE2EServices() *e2eServices {
// run starts all e2e services and wait for the termination signal. Once receives the
// termination signal, it will stop the e2e services gracefully.
func (es *e2eServices) run() error {
defer es.stop()
if err := es.start(); err != nil {
func (es *e2eServices) run(t *testing.T) error {
defer es.stop(t)
if err := es.start(t); err != nil {
return err
}
// Wait until receiving a termination signal.
@ -51,13 +54,13 @@ func (es *e2eServices) run() error {
}
// start starts the tests embedded services or returns an error.
func (es *e2eServices) start() error {
func (es *e2eServices) start(t *testing.T) error {
glog.Info("Starting e2e services...")
err := es.startEtcd()
err := es.startEtcd(t)
if err != nil {
return err
}
err = es.startApiServer()
err = es.startApiServer(es.etcdStorage)
if err != nil {
return err
}
@ -70,7 +73,7 @@ func (es *e2eServices) start() error {
}
// stop stops the embedded e2e services.
func (es *e2eServices) stop() {
func (es *e2eServices) stop(t *testing.T) {
glog.Info("Stopping e2e services...")
// TODO(random-liu): Use a loop to stop all services after introducing
// service interface.
@ -90,9 +93,7 @@ func (es *e2eServices) stop() {
glog.Info("Stopping etcd")
if es.etcdServer != nil {
if err := es.etcdServer.Stop(); err != nil {
glog.Errorf("Failed to stop %q: %v", es.etcdServer.Name(), err)
}
es.etcdServer.Terminate(t)
}
for _, d := range es.rmDirs {
@ -107,23 +108,18 @@ func (es *e2eServices) stop() {
}
// startEtcd starts the embedded etcd instance or returns an error.
func (es *e2eServices) startEtcd() error {
func (es *e2eServices) startEtcd(t *testing.T) error {
glog.Info("Starting etcd")
// Create data directory in current working space.
dataDir, err := ioutil.TempDir(".", "etcd")
if err != nil {
return err
}
// Mark the dataDir as directories to remove.
es.rmDirs = append(es.rmDirs, dataDir)
es.etcdServer = NewEtcd(dataDir)
return es.etcdServer.Start()
server, etcdStorage := etcdtesting.NewUnsecuredEtcd3TestClientServer(t)
es.etcdServer = server
es.etcdStorage = etcdStorage
return nil
}
// startApiServer starts the embedded API server or returns an error.
func (es *e2eServices) startApiServer() error {
func (es *e2eServices) startApiServer(etcdStorage *storagebackend.Config) error {
glog.Info("Starting API server")
es.apiServer = NewAPIServer()
es.apiServer = NewAPIServer(*etcdStorage)
return es.apiServer.Start()
}
@ -137,7 +133,6 @@ func (es *e2eServices) startNamespaceController() error {
// getServicesHealthCheckURLs returns the health check urls for the internal services.
func getServicesHealthCheckURLs() []string {
return []string{
getEtcdHealthCheckURL(),
getAPIServerHealthCheckURL(),
}
}

5
test/e2e_node/services/services.go
View File

@ -22,6 +22,7 @@ import (
"os"
"os/exec"
"path"
"testing"
"github.com/golang/glog"
"github.com/kardianos/osext"
@ -105,12 +106,12 @@ func (e *E2EServices) Stop() {
// RunE2EServices actually start the e2e services. This function is used to
// start e2e services in current process. This is only used in run-services-mode.
func RunE2EServices() {
func RunE2EServices(t *testing.T) {
// Populate global DefaultFeatureGate with value from TestContext.FeatureGates.
// This way, statically-linked components see the same feature gate config as the test context.
utilfeature.DefaultFeatureGate.SetFromMap(framework.TestContext.FeatureGates)
e := newE2EServices()
if err := e.run(); err != nil {
if err := e.run(t); err != nil {
glog.Fatalf("Failed to run e2e services: %v", err)
}
}