From 7267299c3c4517ebcd0d0440c9491618e6e5bda9 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Tue, 6 Dec 2016 12:22:49 +0100 Subject: [PATCH] genericapiserver: move MasterCount and service options into master --- cmd/kube-apiserver/app/options/options.go | 49 ++++++++++++++++--- .../kube-apiserver/app/options/validation.go | 13 +++-- cmd/kube-apiserver/app/server.go | 14 ++++-- examples/apiserver/apiserver.go | 6 +-- hack/update-federation-openapi-spec.sh | 3 +- pkg/genericapiserver/config.go | 3 -- .../options/server_run_options.go | 33 ------------- pkg/master/master.go | 3 +- test/e2e_node/services/apiserver.go | 2 +- test/integration/federation/server_test.go | 3 -- 10 files changed, 63 insertions(+), 66 deletions(-) rename pkg/genericapiserver/validation/universal_validation.go => cmd/kube-apiserver/app/options/validation.go (84%) diff --git a/cmd/kube-apiserver/app/options/options.go b/cmd/kube-apiserver/app/options/options.go index 8a412bb130..cbfcd221c6 100644 --- a/cmd/kube-apiserver/app/options/options.go +++ b/cmd/kube-apiserver/app/options/options.go @@ -18,6 +18,7 @@ limitations under the License. package options import ( + "net" "time" "k8s.io/kubernetes/pkg/api" @@ -25,10 +26,14 @@ import ( genericoptions "k8s.io/kubernetes/pkg/genericapiserver/options" kubeletclient "k8s.io/kubernetes/pkg/kubelet/client" "k8s.io/kubernetes/pkg/master/ports" + utilnet "k8s.io/kubernetes/pkg/util/net" "github.com/spf13/pflag" ) +// DefaultServiceNodePortRange is the default port range for NodePort services. +var DefaultServiceNodePortRange = utilnet.PortRange{Base: 30000, Size: 2768} + // ServerRunOptions runs a kubernetes api server. type ServerRunOptions struct { GenericServerRunOptions *genericoptions.ServerRunOptions @@ -38,12 +43,16 @@ type ServerRunOptions struct { Authentication *genericoptions.BuiltInAuthenticationOptions Authorization *genericoptions.BuiltInAuthorizationOptions - AllowPrivileged bool - EventTTL time.Duration - KubeletConfig kubeletclient.KubeletClientConfig - MaxConnectionBytesPerSec int64 - SSHKeyfile string - SSHUser string + AllowPrivileged bool + EventTTL time.Duration + KubeletConfig kubeletclient.KubeletClientConfig + KubernetesServiceNodePort int + MasterCount int + MaxConnectionBytesPerSec int64 + ServiceClusterIPRange net.IPNet // TODO: make this a list + ServiceNodePortRange utilnet.PortRange + SSHKeyfile string + SSHUser string } // NewServerRunOptions creates a new ServerRunOptions object with default parameters @@ -56,7 +65,8 @@ func NewServerRunOptions() *ServerRunOptions { Authentication: genericoptions.NewBuiltInAuthenticationOptions().WithAll(), Authorization: genericoptions.NewBuiltInAuthorizationOptions(), - EventTTL: 1 * time.Hour, + EventTTL: 1 * time.Hour, + MasterCount: 1, KubeletConfig: kubeletclient.KubeletClientConfig{ Port: ports.KubeletPort, PreferredAddressTypes: []string{ @@ -68,6 +78,7 @@ func NewServerRunOptions() *ServerRunOptions { EnableHttps: true, HTTPTimeout: time.Duration(5) * time.Second, }, + ServiceNodePortRange: DefaultServiceNodePortRange, } return &s } @@ -104,6 +115,30 @@ func (s *ServerRunOptions) AddFlags(fs *pflag.FlagSet) { "If non-zero, throttle each user connection to this number of bytes/sec. "+ "Currently only applies to long-running requests.") + fs.IntVar(&s.MasterCount, "apiserver-count", s.MasterCount, + "The number of apiservers running in the cluster.") + + // See #14282 for details on how to test/try this option out. + // TODO: remove this comment once this option is tested in CI. + fs.IntVar(&s.KubernetesServiceNodePort, "kubernetes-service-node-port", s.KubernetesServiceNodePort, ""+ + "If non-zero, the Kubernetes master service (which apiserver creates/maintains) will be "+ + "of type NodePort, using this as the value of the port. If zero, the Kubernetes master "+ + "service will be of type ClusterIP.") + + fs.IPNetVar(&s.ServiceClusterIPRange, "service-cluster-ip-range", s.ServiceClusterIPRange, ""+ + "A CIDR notation IP range from which to assign service cluster IPs. This must not "+ + "overlap with any IP ranges assigned to nodes for pods.") + + fs.IPNetVar(&s.ServiceClusterIPRange, "portal-net", s.ServiceClusterIPRange, + "DEPRECATED: see --service-cluster-ip-range instead.") + fs.MarkDeprecated("portal-net", "see --service-cluster-ip-range instead") + + fs.Var(&s.ServiceNodePortRange, "service-node-port-range", ""+ + "A port range to reserve for services with NodePort visibility. "+ + "Example: '30000-32767'. Inclusive at both ends of the range.") + fs.Var(&s.ServiceNodePortRange, "service-node-ports", "DEPRECATED: see --service-node-port-range instead") + fs.MarkDeprecated("service-node-ports", "see --service-node-port-range instead") + // Kubelet related flags: fs.BoolVar(&s.KubeletConfig.EnableHttps, "kubelet-https", s.KubeletConfig.EnableHttps, "Use https for kubelet connections.") diff --git a/pkg/genericapiserver/validation/universal_validation.go b/cmd/kube-apiserver/app/options/validation.go similarity index 84% rename from pkg/genericapiserver/validation/universal_validation.go rename to cmd/kube-apiserver/app/options/validation.go index 72e683acfc..de273a1f44 100644 --- a/pkg/genericapiserver/validation/universal_validation.go +++ b/cmd/kube-apiserver/app/options/validation.go @@ -14,18 +14,16 @@ See the License for the specific language governing permissions and limitations under the License. */ -package validation +package options import ( "fmt" - "github.com/golang/glog" - "k8s.io/kubernetes/pkg/genericapiserver/options" utilerrors "k8s.io/kubernetes/pkg/util/errors" ) // TODO: Longer term we should read this from some config store, rather than a flag. -func verifyClusterIPFlags(options *options.ServerRunOptions) []error { +func verifyClusterIPFlags(options *ServerRunOptions) []error { errors := []error{} if options.ServiceClusterIPRange.IP == nil { errors = append(errors, fmt.Errorf("No --service-cluster-ip-range specified")) @@ -37,7 +35,7 @@ func verifyClusterIPFlags(options *options.ServerRunOptions) []error { return errors } -func verifyServiceNodePort(options *options.ServerRunOptions) []error { +func verifyServiceNodePort(options *ServerRunOptions) []error { errors := []error{} if options.KubernetesServiceNodePort < 0 || options.KubernetesServiceNodePort > 65535 { errors = append(errors, fmt.Errorf("--kubernetes-service-node-port %v must be between 0 and 65535, inclusive. If 0, the Kubernetes master service will be of type ClusterIP.", options.KubernetesServiceNodePort)) @@ -49,7 +47,7 @@ func verifyServiceNodePort(options *options.ServerRunOptions) []error { return errors } -func ValidateRunOptions(options *options.ServerRunOptions) { +func ValidateRunOptions(options *ServerRunOptions) error { errors := []error{} if errs := verifyClusterIPFlags(options); len(errs) > 0 { errors = append(errors, errs...) @@ -58,6 +56,7 @@ func ValidateRunOptions(options *options.ServerRunOptions) { errors = append(errors, errs...) } if err := utilerrors.NewAggregate(errors); err != nil { - glog.Fatalf("Validate server run options failed: %v", err) + return fmt.Errorf("validate server run options failed: %v", err) } + return nil } diff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go index 7e22f3dfc0..8c9d7b1c57 100644 --- a/cmd/kube-apiserver/app/server.go +++ b/cmd/kube-apiserver/app/server.go @@ -88,7 +88,7 @@ func Run(s *options.ServerRunOptions) error { return err } - serviceIPRange, apiServerServiceIP, err := master.DefaultServiceIPRange(s.GenericServerRunOptions.ServiceClusterIPRange) + serviceIPRange, apiServerServiceIP, err := master.DefaultServiceIPRange(s.ServiceClusterIPRange) if err != nil { return fmt.Errorf("error determining service IP ranges: %v", err) } @@ -97,8 +97,14 @@ func Run(s *options.ServerRunOptions) error { return fmt.Errorf("error creating self-signed certificates: %v", err) } + // TODO(sttts): change signature of DefaultAndValidateRunOptions to aggregate errors genericapiserver.DefaultAndValidateRunOptions(s.GenericServerRunOptions) + // TODO(sttts): move all defaulting and validation above into cmd/kube-apiserver/app/options.DefaultAndValidateRunOptions() + if err != options.ValidateRunOptions(s) { + return err + } + genericConfig := genericapiserver.NewConfig(). // create the new config ApplyOptions(s.GenericServerRunOptions). // apply the options selected ApplyInsecureServingOptions(s.InsecureServing) @@ -313,10 +319,10 @@ func Run(s *options.ServerRunOptions) error { APIServerServiceIP: apiServerServiceIP, APIServerServicePort: 443, - ServiceNodePortRange: s.GenericServerRunOptions.ServiceNodePortRange, - KubernetesServiceNodePort: s.GenericServerRunOptions.KubernetesServiceNodePort, + ServiceNodePortRange: s.ServiceNodePortRange, + KubernetesServiceNodePort: s.KubernetesServiceNodePort, - MasterCount: s.GenericServerRunOptions.MasterCount, + MasterCount: s.MasterCount, } if s.GenericServerRunOptions.EnableWatchCache { diff --git a/examples/apiserver/apiserver.go b/examples/apiserver/apiserver.go index e5f95b5867..873a149998 100644 --- a/examples/apiserver/apiserver.go +++ b/examples/apiserver/apiserver.go @@ -80,12 +80,10 @@ func NewServerRunOptions() *ServerRunOptions { } func (serverOptions *ServerRunOptions) Run(stopCh <-chan struct{}) error { - // Set ServiceClusterIPRange - _, serviceClusterIPRange, _ := net.ParseCIDR("10.0.0.0/24") - serverOptions.GenericServerRunOptions.ServiceClusterIPRange = *serviceClusterIPRange serverOptions.Etcd.StorageConfig.ServerList = []string{"http://127.0.0.1:2379"} - genericvalidation.ValidateRunOptions(serverOptions.GenericServerRunOptions) + // TODO(sttts): unify signature of DefaultAndValidateRunOptions with the others + genericapiserver.DefaultAndValidateRunOptions(serverOptions.GenericServerRunOptions) if errs := serverOptions.Etcd.Validate(); len(errs) > 0 { return utilerrors.NewAggregate(errs) } diff --git a/hack/update-federation-openapi-spec.sh b/hack/update-federation-openapi-spec.sh index 2faf357197..68a4a534a1 100755 --- a/hack/update-federation-openapi-spec.sh +++ b/hack/update-federation-openapi-spec.sh @@ -61,8 +61,7 @@ kube::log::status "Starting federation-apiserver" --etcd-servers="http://${ETCD_HOST}:${ETCD_PORT}" \ --advertise-address="10.10.10.10" \ --cert-dir="${TMP_DIR}/certs" \ - --token-auth-file=$TMP_DIR/tokenauth.csv \ - --service-cluster-ip-range="10.0.0.0/24" >/tmp/openapi-federation-api-server.log 2>&1 & + --token-auth-file=$TMP_DIR/tokenauth.csv >/tmp/openapi-federation-api-server.log 2>&1 & APISERVER_PID=$! kube::util::wait_for_url "${API_HOST}:${API_PORT}/" "apiserver: " diff --git a/pkg/genericapiserver/config.go b/pkg/genericapiserver/config.go index f0c2c6b631..bf8333773e 100644 --- a/pkg/genericapiserver/config.go +++ b/pkg/genericapiserver/config.go @@ -59,7 +59,6 @@ import ( openapicommon "k8s.io/kubernetes/pkg/genericapiserver/openapi/common" "k8s.io/kubernetes/pkg/genericapiserver/options" "k8s.io/kubernetes/pkg/genericapiserver/routes" - genericvalidation "k8s.io/kubernetes/pkg/genericapiserver/validation" "k8s.io/kubernetes/pkg/healthz" "k8s.io/kubernetes/pkg/runtime" certutil "k8s.io/kubernetes/pkg/util/cert" @@ -631,8 +630,6 @@ func (s *GenericAPIServer) installAPI(c *Config) { } func DefaultAndValidateRunOptions(options *options.ServerRunOptions) { - genericvalidation.ValidateRunOptions(options) - glog.Infof("Will report %v as public IP address.", options.AdvertiseAddress) // Set default value for ExternalAddress if not specified. diff --git a/pkg/genericapiserver/options/server_run_options.go b/pkg/genericapiserver/options/server_run_options.go index 042013317a..64308cc053 100644 --- a/pkg/genericapiserver/options/server_run_options.go +++ b/pkg/genericapiserver/options/server_run_options.go @@ -26,13 +26,10 @@ import ( "k8s.io/kubernetes/pkg/apimachinery/registered" "k8s.io/kubernetes/pkg/runtime/schema" "k8s.io/kubernetes/pkg/util/config" - utilnet "k8s.io/kubernetes/pkg/util/net" "github.com/spf13/pflag" ) -var DefaultServiceNodePortRange = utilnet.PortRange{Base: 30000, Size: 2768} - // ServerRunOptions contains the options while running a generic api server. type ServerRunOptions struct { AdmissionControl string @@ -54,14 +51,10 @@ type ServerRunOptions struct { EnableSwaggerUI bool EnableWatchCache bool ExternalHost string - KubernetesServiceNodePort int - MasterCount int MaxRequestsInFlight int MaxMutatingRequestsInFlight int MinRequestTimeout int RuntimeConfig config.ConfigurationMap - ServiceClusterIPRange net.IPNet // TODO: make this a list - ServiceNodePortRange utilnet.PortRange StorageVersions string // The default values for StorageVersions. StorageVersions overrides // these; you can change this if you want to change the defaults (e.g., @@ -81,12 +74,10 @@ func NewServerRunOptions() *ServerRunOptions { EnableProfiling: true, EnableContentionProfiling: false, EnableWatchCache: true, - MasterCount: 1, MaxRequestsInFlight: 400, MaxMutatingRequestsInFlight: 200, MinRequestTimeout: 1800, RuntimeConfig: make(config.ConfigurationMap), - ServiceNodePortRange: DefaultServiceNodePortRange, StorageVersions: registered.AllPreferredGroupVersions(), } } @@ -227,22 +218,12 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) { fs.StringVar(&s.ExternalHost, "external-hostname", s.ExternalHost, "The hostname to use when generating externalized URLs for this master (e.g. Swagger API Docs).") - // See #14282 for details on how to test/try this option out. - // TODO: remove this comment once this option is tested in CI. - fs.IntVar(&s.KubernetesServiceNodePort, "kubernetes-service-node-port", s.KubernetesServiceNodePort, ""+ - "If non-zero, the Kubernetes master service (which apiserver creates/maintains) will be "+ - "of type NodePort, using this as the value of the port. If zero, the Kubernetes master "+ - "service will be of type ClusterIP.") - // TODO: remove post-1.6 fs.String("long-running-request-regexp", "", ""+ "A regular expression matching long running requests which should "+ "be excluded from maximum inflight request handling.") fs.MarkDeprecated("long-running-request-regexp", "regular expression matching of long-running requests is no longer supported") - fs.IntVar(&s.MasterCount, "apiserver-count", s.MasterCount, - "The number of apiservers running in the cluster.") - deprecatedMasterServiceNamespace := api.NamespaceDefault fs.StringVar(&deprecatedMasterServiceNamespace, "master-service-namespace", deprecatedMasterServiceNamespace, ""+ "DEPRECATED: the namespace from which the kubernetes master services should be injected into pods.") @@ -267,20 +248,6 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) { "apis// can be used to turn on/off specific resources. api/all and "+ "api/legacy are special keys to control all and legacy api versions respectively.") - fs.IPNetVar(&s.ServiceClusterIPRange, "service-cluster-ip-range", s.ServiceClusterIPRange, ""+ - "A CIDR notation IP range from which to assign service cluster IPs. This must not "+ - "overlap with any IP ranges assigned to nodes for pods.") - - fs.IPNetVar(&s.ServiceClusterIPRange, "portal-net", s.ServiceClusterIPRange, - "DEPRECATED: see --service-cluster-ip-range instead.") - fs.MarkDeprecated("portal-net", "see --service-cluster-ip-range instead") - - fs.Var(&s.ServiceNodePortRange, "service-node-port-range", ""+ - "A port range to reserve for services with NodePort visibility. "+ - "Example: '30000-32767'. Inclusive at both ends of the range.") - fs.Var(&s.ServiceNodePortRange, "service-node-ports", "DEPRECATED: see --service-node-port-range instead") - fs.MarkDeprecated("service-node-ports", "see --service-node-port-range instead") - deprecatedStorageVersion := "" fs.StringVar(&deprecatedStorageVersion, "storage-version", deprecatedStorageVersion, "DEPRECATED: the version to store the legacy v1 resources with. Defaults to server preferred.") diff --git a/pkg/master/master.go b/pkg/master/master.go index 90dd647abb..ddcbef1a07 100644 --- a/pkg/master/master.go +++ b/pkg/master/master.go @@ -24,6 +24,7 @@ import ( "strconv" "time" + "k8s.io/kubernetes/cmd/kube-apiserver/app/options" "k8s.io/kubernetes/pkg/api" apiv1 "k8s.io/kubernetes/pkg/api/v1" appsapi "k8s.io/kubernetes/pkg/apis/apps/v1beta1" @@ -39,7 +40,6 @@ import ( corev1client "k8s.io/kubernetes/pkg/client/clientset_generated/clientset/typed/core/v1" coreclient "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/core/internalversion" "k8s.io/kubernetes/pkg/genericapiserver" - "k8s.io/kubernetes/pkg/genericapiserver/options" "k8s.io/kubernetes/pkg/healthz" kubeletclient "k8s.io/kubernetes/pkg/kubelet/client" "k8s.io/kubernetes/pkg/master/thirdparty" @@ -116,7 +116,6 @@ type Config struct { // Port names should align with ports defined in ExtraServicePorts ExtraEndpointPorts []api.EndpointPort // If non-zero, the "kubernetes" services uses this port as NodePort. - // TODO(sttts): move into master KubernetesServiceNodePort int // Number of masters running; all masters must be started with the diff --git a/test/e2e_node/services/apiserver.go b/test/e2e_node/services/apiserver.go index 7e7eeee498..02a0f8acf7 100644 --- a/test/e2e_node/services/apiserver.go +++ b/test/e2e_node/services/apiserver.go @@ -46,7 +46,7 @@ func (a *APIServer) Start() error { if err != nil { return err } - config.GenericServerRunOptions.ServiceClusterIPRange = *ipnet + config.ServiceClusterIPRange = *ipnet config.AllowPrivileged = true errCh := make(chan error) go func() { diff --git a/test/integration/federation/server_test.go b/test/integration/federation/server_test.go index 5110aa0c94..1c494a6092 100644 --- a/test/integration/federation/server_test.go +++ b/test/integration/federation/server_test.go @@ -20,7 +20,6 @@ import ( "encoding/json" "fmt" "io/ioutil" - "net" "net/http" "testing" "time" @@ -48,8 +47,6 @@ func TestRun(t *testing.T) { s := options.NewServerRunOptions() s.SecureServing.ServingOptions.BindPort = securePort s.InsecureServing.BindPort = insecurePort - _, ipNet, _ := net.ParseCIDR("10.10.10.0/24") - s.GenericServerRunOptions.ServiceClusterIPRange = *ipNet s.Etcd.StorageConfig.ServerList = []string{"http://localhost:2379"} go func() { if err := app.Run(s); err != nil {