github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

pass listener to genericapiserver

pull/6/head
hzxuzhonghu 2017-11-16 13:32:12 +08:00
parent 2d64ce5e8e
commit 6ba30f678c
17 changed files with 142 additions and 122 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cmd/kube-apiserver/app/server.go
View File

@ -355,6 +355,7 @@ func BuildGenericConfig(s *options.ServerRunOptions, proxyTransport *http.Transp
if err := s.GenericServerRunOptions.ApplyTo(genericConfig); err != nil {
return nil, nil, nil, nil, nil, err
}
insecureServingOptions, err := s.InsecureServing.ApplyTo(genericConfig)
if err != nil {
return nil, nil, nil, nil, nil, err

65
cmd/kube-apiserver/app/testing/testserver.go
View File

@ -74,12 +74,16 @@ func StartTestServer(t *testing.T) (result *restclient.Config, tearDownForCaller
tmpDir, err = ioutil.TempDir("", "kubernetes-kube-apiserver")
if err != nil {
return nil, nil, fmt.Errorf("Failed to create temp dir: %v", err)
return nil, nil, fmt.Errorf("failed to create temp dir: %v", err)
}
s := options.NewServerRunOptions()
s.InsecureServing.BindPort = 0
s.SecureServing.BindPort = freePort()
s.SecureServing.Listener, s.SecureServing.BindPort, err = createListenerOnFreePort()
if err != nil {
return nil, nil, fmt.Errorf("failed to create listener: %v", err)
}
s.SecureServing.ServerCert.CertDirectory = tmpDir
s.ServiceClusterIPRange.IP = net.IPv4(10, 0, 0, 0)
s.ServiceClusterIPRange.Mask = net.CIDRMask(16, 32)
@ -88,31 +92,23 @@ func StartTestServer(t *testing.T) (result *restclient.Config, tearDownForCaller
s.Admission.PluginNames = strings.Split("Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds", ",")
s.APIEnablement.RuntimeConfig.Set("api/all=true")
t.Logf("Starting kube-apiserver...")
runErrCh := make(chan error, 1)
t.Logf("Starting kube-apiserver on port %d...", s.SecureServing.BindPort)
server, err := app.CreateServerChain(s, stopCh)
if err != nil {
return nil, nil, fmt.Errorf("Failed to create server chain: %v", err)
return nil, nil, fmt.Errorf("failed to create server chain: %v", err)
}
go func(stopCh <-chan struct{}) {
if err := server.PrepareRun().Run(stopCh); err != nil {
t.Logf("kube-apiserver exited uncleanly: %v", err)
runErrCh <- err
t.Errorf("kube-apiserver failed run: %v", err)
}
}(stopCh)
t.Logf("Waiting for /healthz to be ok...")
client, err := kubernetes.NewForConfig(server.LoopbackClientConfig)
if err != nil {
return nil, nil, fmt.Errorf("Failed to create a client: %v", err)
return nil, nil, fmt.Errorf("failed to create a client: %v", err)
}
err = wait.Poll(100*time.Millisecond, 30*time.Second, func() (bool, error) {
select {
case err := <-runErrCh:
return false, err
default:
}
result := client.CoreV1().RESTClient().Get().AbsPath("/healthz").Do()
status := 0
result.StatusCode(&status)
@ -122,7 +118,7 @@ func StartTestServer(t *testing.T) (result *restclient.Config, tearDownForCaller
return false, nil
})
if err != nil {
return nil, nil, fmt.Errorf("Failed to wait for /healthz to return ok: %v", err)
return nil, nil, fmt.Errorf("failed to wait for /healthz to return ok: %v", err)
}
// from here the caller must call tearDown
@ -132,40 +128,27 @@ func StartTestServer(t *testing.T) (result *restclient.Config, tearDownForCaller
// StartTestServerOrDie calls StartTestServer with up to 5 retries on bind error and dies with
// t.Fatal if it does not succeed.
func StartTestServerOrDie(t *testing.T) (*restclient.Config, TearDownFunc) {
// retry test because the bind might fail due to a race with another process
// binding to the port. We cannot listen to :0 (then the kernel would give us
// a port which is free for sure), so we need this workaround.
var err error
for retry := 0; retry < 5 && !t.Failed(); retry++ {
var config *restclient.Config
var td TearDownFunc
config, td, err = StartTestServer(t)
if err == nil {
return config, td
}
if err != nil && !strings.Contains(err.Error(), "bind") {
break
}
t.Logf("Bind error, retrying...")
config, td, err := StartTestServer(t)
if err == nil {
return config, td
}
t.Fatalf("Failed to launch server: %v", err)
return nil, nil
}
func freePort() int {
addr, err := net.ResolveTCPAddr("tcp", "localhost:0")
func createListenerOnFreePort() (net.Listener, int, error) {
ln, err := net.Listen("tcp", ":0")
if err != nil {
panic(err)
return nil, 0, err
}
l, err := net.ListenTCP("tcp", addr)
if err != nil {
panic(err)
// get port
tcpAddr, ok := ln.Addr().(*net.TCPAddr)
if !ok {
ln.Close()
return nil, 0, fmt.Errorf("invalid listen address: %q", ln.Addr().String())
}
defer l.Close()
return l.Addr().(*net.TCPAddr).Port
return ln, tcpAddr.Port, nil
}

1
pkg/kubeapiserver/server/BUILD
View File

@ -17,6 +17,7 @@ go_library(
"//vendor/k8s.io/apiserver/pkg/features:go_default_library",
"//vendor/k8s.io/apiserver/pkg/server:go_default_library",
"//vendor/k8s.io/apiserver/pkg/server/filters:go_default_library",
"//vendor/k8s.io/apiserver/pkg/server/options:go_default_library",
"//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
"//vendor/k8s.io/client-go/rest:go_default_library",
],

8
pkg/kubeapiserver/server/insecure_handler.go
View File

@ -29,6 +29,7 @@ import (
"k8s.io/apiserver/pkg/features"
"k8s.io/apiserver/pkg/server"
genericfilters "k8s.io/apiserver/pkg/server/filters"
"k8s.io/apiserver/pkg/server/options"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/client-go/rest"
)
@ -118,8 +119,11 @@ func serveInsecurely(insecureServingInfo *InsecureServingInfo, insecureHandler h
MaxHeaderBytes: 1 << 20,
}
glog.Infof("Serving insecurely on %s", insecureServingInfo.BindAddress)
var err error
_, err = server.RunServer(insecureServer, insecureServingInfo.BindNetwork, shutDownTimeout, stopCh)
ln, _, err := options.CreateListener(insecureServingInfo.BindNetwork, insecureServingInfo.BindAddress)
if err != nil {
return err
}
err = server.RunServer(insecureServer, ln, shutDownTimeout, stopCh)
return err
}

4
staging/src/k8s.io/apiextensions-apiserver/Godeps/Godeps.json generated
View File

@ -294,10 +294,6 @@
"ImportPath": "github.com/peterbourgon/diskv",
"Rev": "5f041e8faa004a95c88a202771f4cc3e991971e6"
},
{
"ImportPath": "github.com/pkg/errors",
"Rev": "a22138067af1c4942683050411a841ade67fe1eb"
},
{
"ImportPath": "github.com/pmezard/go-difflib/difflib",
"Rev": "d8ed2627bdf02c080bf22230dbb337003b7aba2d"

4
staging/src/k8s.io/apiserver/Godeps/Godeps.json generated
View File

@ -518,10 +518,6 @@
"ImportPath": "github.com/peterbourgon/diskv",
"Rev": "5f041e8faa004a95c88a202771f4cc3e991971e6"
},
{
"ImportPath": "github.com/pkg/errors",
"Rev": "a22138067af1c4942683050411a841ade67fe1eb"
},
{
"ImportPath": "github.com/pmezard/go-difflib/difflib",
"Rev": "d8ed2627bdf02c080bf22230dbb337003b7aba2d"

1
staging/src/k8s.io/apiserver/pkg/server/BUILD
View File

@ -71,7 +71,6 @@ go_library(
"//vendor/github.com/go-openapi/spec:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/github.com/pborman/uuid:go_default_library",
"//vendor/github.com/pkg/errors:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apimachinery:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library",

7
staging/src/k8s.io/apiserver/pkg/server/config.go
View File

@ -203,11 +203,8 @@ type RecommendedConfig struct {
}
type SecureServingInfo struct {
// BindAddress is the ip:port to serve on
BindAddress string
// BindNetwork is the type of network to bind to - defaults to "tcp", accepts "tcp",
// "tcp4", and "tcp6".
BindNetwork string
// Listener is the secure server network listener.
Listener net.Listener
// Cert is the main server cert which is used if SNI does not match. Cert must be non-nil and is
// allowed to be in SNICerts.

4
staging/src/k8s.io/apiserver/pkg/server/config_selfclient.go
View File

@ -32,7 +32,7 @@ func (s *SecureServingInfo) NewLoopbackClientConfig(token string, loopbackCert [
return nil, nil
}
host, port, err := LoopbackHostPort(s.BindAddress)
host, port, err := LoopbackHostPort(s.Listener.Addr().String())
if err != nil {
return nil, err
}
@ -64,7 +64,7 @@ func LoopbackHostPort(bindAddress string) (string, string, error) {
}
// Value is expected to be an IP or DNS name, not "0.0.0.0".
if host == "0.0.0.0" {
if host == "0.0.0.0" || host == "::" {
host = "localhost"
// Get ip of local interface, but fall back to "localhost".
// Note that "localhost" is resolved with the external nameserver first with Go's stdlib.

22
staging/src/k8s.io/apiserver/pkg/server/config_selfclient_test.go
View File

@ -43,4 +43,26 @@ func TestLoopbackHostPort(t *testing.T) {
if port != "443" {
t.Fatalf("expected 443 as port, got %q", port)
}
host, port, err = LoopbackHostPort("[ff06:0:0:0:0:0:0:c3]:443")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if host != "ff06:0:0:0:0:0:0:c3" {
t.Fatalf("expected ff06:0:0:0:0:0:0:c3 as host, got %q", host)
}
if port != "443" {
t.Fatalf("expected 443 as port, got %q", port)
}
host, port, err = LoopbackHostPort("[::]:443")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if ip := net.ParseIP(host); ip == nil || !ip.IsLoopback() {
t.Fatalf("expected host to be loopback, got %q", host)
}
if port != "443" {
t.Fatalf("expected 443 as port, got %q", port)
}
}

8
staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go
View File

@ -100,9 +100,6 @@ type GenericAPIServer struct {
SecureServingInfo *SecureServingInfo
// numerical ports, set after listening
effectiveSecurePort int
// ExternalAddress is the address (hostname or IP and port) that should be used in
// external (public internet) URLs for this GenericAPIServer.
ExternalAddress string
@ -338,11 +335,6 @@ func (s preparedGenericAPIServer) NonBlockingRun(stopCh <-chan struct{}) error {
return nil
}
// EffectiveSecurePort returns the secure port we bound to.
func (s *GenericAPIServer) EffectiveSecurePort() int {
return s.effectiveSecurePort
}
// installAPIResources is a private method for installing the REST storage backing each api groupversionresource
func (s *GenericAPIServer) installAPIResources(apiPrefix string, apiGroupInfo *APIGroupInfo) error {
for _, groupVersion := range apiGroupInfo.GroupMeta.GroupVersions {

10
staging/src/k8s.io/apiserver/pkg/server/genericapiserver_test.go
View File

@ -550,7 +550,15 @@ func TestGracefulShutdown(t *testing.T) {
Handler: s.Handler,
}
stopCh := make(chan struct{})
serverPort, err := RunServer(insecureServer, "tcp", 10*time.Second, stopCh)
ln, err := net.Listen("tcp", insecureServer.Addr)
if err != nil {
t.Errorf("failed to listen on %v: %v", insecureServer.Addr, err)
}
// get port
serverPort := ln.Addr().(*net.TCPAddr).Port
err = RunServer(insecureServer, ln, 10*time.Second, stopCh)
if err != nil {
t.Errorf("RunServer err: %v", err)
}

51
staging/src/k8s.io/apiserver/pkg/server/options/serving.go
View File

@ -38,6 +38,14 @@ import (
type SecureServingOptions struct {
BindAddress net.IP
BindPort int
// BindNetwork is the type of network to bind to - defaults to "tcp", accepts "tcp",
// "tcp4", and "tcp6".
BindNetwork string
// Listener is the secure server network listener.
// either Listener or BindAddress/BindPort/BindNetwork is set,
// if Listener is set, use it and omit BindAddress/BindPort/BindNetwork.
Listener net.Listener
// ServerCert is the TLS cert info for serving secure traffic
ServerCert GeneratableKeyCert
@ -147,14 +155,25 @@ func (s *SecureServingOptions) ApplyTo(c *server.Config) error {
if s == nil {
return nil
}
if s.BindPort <= 0 {
return nil
}
if s.Listener == nil {
var err error
addr := net.JoinHostPort(s.BindAddress.String(), strconv.Itoa(s.BindPort))
s.Listener, s.BindPort, err = CreateListener(s.BindNetwork, addr)
if err != nil {
return fmt.Errorf("failed to create listener: %v", err)
}
}
if err := s.applyServingInfoTo(c); err != nil {
return err
}
c.SecureServingInfo.Listener = s.Listener
// create self-signed cert+key with the fake server.LoopbackClientServerNameOverride and
// let the server return it when the loopback client connects.
certPem, keyPem, err := certutil.GenerateSelfSignedCertKey(server.LoopbackClientServerNameOverride, nil, nil)
@ -184,16 +203,9 @@ func (s *SecureServingOptions) ApplyTo(c *server.Config) error {
}
func (s *SecureServingOptions) applyServingInfoTo(c *server.Config) error {
if s.BindPort <= 0 {
return nil
}
secureServingInfo := &server.SecureServingInfo{
BindAddress: net.JoinHostPort(s.BindAddress.String(), strconv.Itoa(s.BindPort)),
}
secureServingInfo := &server.SecureServingInfo{}
serverCertFile, serverKeyFile := s.ServerCert.CertKey.CertFile, s.ServerCert.CertKey.KeyFile
// load main cert
if len(serverCertFile) != 0 || len(serverKeyFile) != 0 {
tlsCert, err := tls.LoadX509KeyPair(serverCertFile, serverKeyFile)
@ -250,7 +262,7 @@ func (s *SecureServingOptions) MaybeDefaultWithSelfSignedCerts(publicAddress str
return nil
}
keyCert := &s.ServerCert.CertKey
if s.BindPort == 0 || len(keyCert.CertFile) != 0 || len(keyCert.KeyFile) != 0 {
if len(keyCert.CertFile) != 0 || len(keyCert.KeyFile) != 0 {
return nil
}
@ -286,3 +298,22 @@ func (s *SecureServingOptions) MaybeDefaultWithSelfSignedCerts(publicAddress str
return nil
}
func CreateListener(network, addr string) (net.Listener, int, error) {
if len(network) == 0 {
network = "tcp"
}
ln, err := net.Listen(network, addr)
if err != nil {
return nil, 0, fmt.Errorf("failed to listen on %v: %v", addr, err)
}
// get port
tcpAddr, ok := ln.Addr().(*net.TCPAddr)
if !ok {
ln.Close()
return nil, 0, fmt.Errorf("invalid listen address: %q", ln.Addr().String())
}
return ln, tcpAddr.Port, nil
}

18
staging/src/k8s.io/apiserver/pkg/server/options/serving_test.go
View File

@ -47,6 +47,7 @@ import (
utilflag "k8s.io/apiserver/pkg/util/flag"
"k8s.io/client-go/discovery"
restclient "k8s.io/client-go/rest"
"strconv"
)
func setUp(t *testing.T) Config {
@ -481,6 +482,15 @@ NextTest:
},
SNICertKeys: namedCertKeys,
}
// use a random free port
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Errorf("failed to listen on 127.0.0.1:0")
}
secureOptions.Listener = ln
// get port
secureOptions.BindPort = ln.Addr().(*net.TCPAddr).Port
config.LoopbackClientConfig = &restclient.Config{}
if err := secureOptions.ApplyTo(&config); err != nil {
t.Errorf("%q - failed applying the SecureServingOptions: %v", title, err)
@ -493,9 +503,6 @@ NextTest:
return
}
// patch in a 0-port to enable auto port allocation
s.SecureServingInfo.BindAddress = "127.0.0.1:0"
// add poststart hook to know when the server is up.
startedCh := make(chan struct{})
s.AddPostStartHook("test-notifier", func(context PostStartHookContext) error {
@ -517,9 +524,8 @@ NextTest:
<-startedCh
effectiveSecurePort := fmt.Sprintf("%d", preparedServer.EffectiveSecurePort())
// try to dial
addr := fmt.Sprintf("localhost:%s", effectiveSecurePort)
addr := fmt.Sprintf("localhost:%d", secureOptions.BindPort)
t.Logf("Dialing %s as %q", addr, test.ServerName)
conn, err := tls.Dial("tcp", addr, &tls.Config{
RootCAs: roots,
@ -547,7 +553,7 @@ NextTest:
if len(test.LoopbackClientBindAddressOverride) != 0 {
host = test.LoopbackClientBindAddressOverride
}
s.LoopbackClientConfig.Host = net.JoinHostPort(host, effectiveSecurePort)
s.LoopbackClientConfig.Host = net.JoinHostPort(host, strconv.Itoa(secureOptions.BindPort))
if test.ExpectLoopbackClientError {
if err == nil {
t.Errorf("%q - expected error creating loopback client config", title)

52
staging/src/k8s.io/apiserver/pkg/server/serve.go
View File

@ -26,11 +26,10 @@ import (
"strings"
"time"
"github.com/golang/glog"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/validation"
"github.com/golang/glog"
"github.com/pkg/errors"
)
const (
@ -41,8 +40,12 @@ const (
// be loaded or the initial listen call fails. The actual server loop (stoppable by closing
// stopCh) runs in a go routine, i.e. serveSecurely does not block.
func (s *GenericAPIServer) serveSecurely(stopCh <-chan struct{}) error {
if s.SecureServingInfo.Listener == nil {
return fmt.Errorf("listener must not be nil")
}
secureServer := &http.Server{
Addr: s.SecureServingInfo.BindAddress,
Addr: s.SecureServingInfo.Listener.Addr().String(),
Handler: s.Handler,
MaxHeaderBytes: 1 << 20,
TLSConfig: &tls.Config{
@ -83,33 +86,22 @@ func (s *GenericAPIServer) serveSecurely(stopCh <-chan struct{}) error {
secureServer.TLSConfig.ClientCAs = s.SecureServingInfo.ClientCA
}
glog.Infof("Serving securely on %s", s.SecureServingInfo.BindAddress)
var err error
s.effectiveSecurePort, err = RunServer(secureServer, s.SecureServingInfo.BindNetwork, s.ShutdownTimeout, stopCh)
glog.Infof("Serving securely on %s", secureServer.Addr)
err := RunServer(secureServer, s.SecureServingInfo.Listener, s.ShutdownTimeout, stopCh)
return err
}
// RunServer listens on the given port, then spawns a go-routine continuously serving
// until the stopCh is closed. The port is returned. This function does not block.
func RunServer(server *http.Server, network string, shutDownTimeout time.Duration, stopCh <-chan struct{}) (int, error) {
if len(server.Addr) == 0 {
return 0, errors.New("address cannot be empty")
}
if len(network) == 0 {
network = "tcp"
}
ln, err := net.Listen(network, server.Addr)
if err != nil {
return 0, fmt.Errorf("failed to listen on %v: %v", server.Addr, err)
}
// get port
tcpAddr, ok := ln.Addr().(*net.TCPAddr)
if !ok {
ln.Close()
return 0, fmt.Errorf("invalid listen address: %q", ln.Addr().String())
// RunServer listens on the given port if listener is not given,
// then spawns a go-routine continuously serving
// until the stopCh is closed. This function does not block.
func RunServer(
server *http.Server,
ln net.Listener,
shutDownTimeout time.Duration,
stopCh <-chan struct{},
) error {
if ln == nil {
return fmt.Errorf("listener must not be nil")
}
// Shutdown server gracefully.
@ -131,7 +123,7 @@ func RunServer(server *http.Server, network string, shutDownTimeout time.Duratio
err := server.Serve(listener)
msg := fmt.Sprintf("Stopped listening on %s", tcpAddr.String())
msg := fmt.Sprintf("Stopped listening on %s", ln.Addr().String())
select {
case <-stopCh:
glog.Info(msg)
@ -140,7 +132,7 @@ func RunServer(server *http.Server, network string, shutDownTimeout time.Duratio
}
}()
return tcpAddr.Port, nil
return nil
}
type NamedTLSCert struct {

4
staging/src/k8s.io/kube-aggregator/Godeps/Godeps.json generated
View File

@ -274,10 +274,6 @@
"ImportPath": "github.com/peterbourgon/diskv",
"Rev": "5f041e8faa004a95c88a202771f4cc3e991971e6"
},
{
"ImportPath": "github.com/pkg/errors",
"Rev": "a22138067af1c4942683050411a841ade67fe1eb"
},
{
"ImportPath": "github.com/pmezard/go-difflib/difflib",
"Rev": "d8ed2627bdf02c080bf22230dbb337003b7aba2d"

4
staging/src/k8s.io/sample-apiserver/Godeps/Godeps.json generated
View File

@ -266,10 +266,6 @@
"ImportPath": "github.com/peterbourgon/diskv",
"Rev": "5f041e8faa004a95c88a202771f4cc3e991971e6"
},
{
"ImportPath": "github.com/pkg/errors",
"Rev": "a22138067af1c4942683050411a841ade67fe1eb"
},
{
"ImportPath": "github.com/prometheus/client_golang/prometheus",
"Rev": "e7e903064f5e9eb5da98208bae10b475d4db0f8c"