From 4ea363d98e9cf1fb7c83da3fe6d7db95a904cae3 Mon Sep 17 00:00:00 2001 From: Alin-Gheorghe Balutoiu Date: Wed, 21 Feb 2018 16:48:38 +0200 Subject: [PATCH] Add support for binaries to run as Windows services This patch adds support for kubernetes to integrate with Windows SCM. As a first step both `kubelet` and `kube-proxy` can be registered as a service. To create the service: PS > sc.exe create binPath= " --service " CMD > sc create binPath= " --service " Please note that if the arguments contain spaces, it must be escaped. Example: PS > sc.exe create kubelet binPath= "C:\kubelet.exe --service --hostname-override 'minion' " CMD > sc create kubelet binPath= "C:\kubelet.exe --service --hostname-override 'minion' " Example to start the service: PS > Start-Service kubelet; Start-Service kube-proxy CMD > net start kubelet && net start kube-proxy Example to stop the service: PS > Stop-Service kubelet (-Force); Stop-Service kube-proxy (-Force) CMD > net stop kubelet && net stop kube-proxy Example to query the service: PS > Get-Service kubelet; Get-Service kube-proxy; CMD > sc.exe queryex kubelet && sc qc kubelet && sc.exe queryex kube-proxy && sc.exe qc kube-proxy Signed-off-by: Alin Balutoiu Signed-off-by: Alin Gabriel Serdean Co-authored-by: Alin Gabriel Serdean --- cmd/kube-proxy/app/BUILD | 12 +++ cmd/kube-proxy/app/init_others.go | 30 +++++++ cmd/kube-proxy/app/init_windows.go | 40 ++++++++++ cmd/kube-proxy/app/server.go | 8 ++ cmd/kubelet/app/BUILD | 14 ++++ cmd/kubelet/app/init_others.go | 23 ++++++ cmd/kubelet/app/init_windows.go | 34 ++++++++ cmd/kubelet/app/options/BUILD | 11 +++ cmd/kubelet/app/options/options.go | 5 ++ cmd/kubelet/app/options/osflags_others.go | 26 +++++++ cmd/kubelet/app/options/osflags_windows.go | 27 +++++++ cmd/kubelet/app/server.go | 3 + pkg/BUILD | 1 + pkg/windows/service/BUILD | 38 +++++++++ pkg/windows/service/service.go | 91 ++++++++++++++++++++++ 15 files changed, 363 insertions(+) create mode 100644 cmd/kube-proxy/app/init_others.go create mode 100644 cmd/kube-proxy/app/init_windows.go create mode 100644 cmd/kubelet/app/init_others.go create mode 100644 cmd/kubelet/app/init_windows.go create mode 100644 cmd/kubelet/app/options/osflags_others.go create mode 100644 cmd/kubelet/app/options/osflags_windows.go create mode 100644 pkg/windows/service/BUILD create mode 100644 pkg/windows/service/service.go diff --git a/cmd/kube-proxy/app/BUILD b/cmd/kube-proxy/app/BUILD index 7082c8f51b..3ae5ab8f04 100644 --- a/cmd/kube-proxy/app/BUILD +++ b/cmd/kube-proxy/app/BUILD @@ -13,36 +13,47 @@ go_library( "server.go", ] + select({ "@io_bazel_rules_go//go/platform:android": [ + "init_others.go", "server_others.go", ], "@io_bazel_rules_go//go/platform:darwin": [ + "init_others.go", "server_others.go", ], "@io_bazel_rules_go//go/platform:dragonfly": [ + "init_others.go", "server_others.go", ], "@io_bazel_rules_go//go/platform:freebsd": [ + "init_others.go", "server_others.go", ], "@io_bazel_rules_go//go/platform:linux": [ + "init_others.go", "server_others.go", ], "@io_bazel_rules_go//go/platform:nacl": [ + "init_others.go", "server_others.go", ], "@io_bazel_rules_go//go/platform:netbsd": [ + "init_others.go", "server_others.go", ], "@io_bazel_rules_go//go/platform:openbsd": [ + "init_others.go", "server_others.go", ], "@io_bazel_rules_go//go/platform:plan9": [ + "init_others.go", "server_others.go", ], "@io_bazel_rules_go//go/platform:solaris": [ + "init_others.go", "server_others.go", ], "@io_bazel_rules_go//go/platform:windows": [ + "init_windows.go", "server_windows.go", ], "//conditions:default": [], @@ -177,6 +188,7 @@ go_library( "//pkg/proxy/winkernel:go_default_library", "//pkg/proxy/winuserspace:go_default_library", "//pkg/util/netsh:go_default_library", + "//pkg/windows/service:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library", ], diff --git a/cmd/kube-proxy/app/init_others.go b/cmd/kube-proxy/app/init_others.go new file mode 100644 index 0000000000..6c0e6b8c7d --- /dev/null +++ b/cmd/kube-proxy/app/init_others.go @@ -0,0 +1,30 @@ +// +build !windows + +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "github.com/spf13/pflag" +) + +func initForOS(service bool) error { + return nil +} + +func (o *Options) addOSFlags(fs *pflag.FlagSet) { +} diff --git a/cmd/kube-proxy/app/init_windows.go b/cmd/kube-proxy/app/init_windows.go new file mode 100644 index 0000000000..85cafcda53 --- /dev/null +++ b/cmd/kube-proxy/app/init_windows.go @@ -0,0 +1,40 @@ +// +build windows + +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "k8s.io/kubernetes/pkg/windows/service" + + "github.com/spf13/pflag" +) + +const ( + serviceName = "kube-proxy" +) + +func initForOS(windowsService bool) error { + if windowsService { + return service.InitService(serviceName) + } + return nil +} + +func (o *Options) addOSFlags(fs *pflag.FlagSet) { + fs.BoolVar(&o.WindowsService, "windows-service", o.WindowsService, "Enable Windows Service Control Manager API integration") +} diff --git a/cmd/kube-proxy/app/server.go b/cmd/kube-proxy/app/server.go index ff30980da2..7b42e66168 100644 --- a/cmd/kube-proxy/app/server.go +++ b/cmd/kube-proxy/app/server.go @@ -100,6 +100,9 @@ type Options struct { CleanupAndExit bool // CleanupIPVS, when true, makes the proxy server clean up ipvs rules before running. CleanupIPVS bool + // WindowsService should be set to true if kube-proxy is running as a service on Windows. + // Its corresponding flag only gets registered in Windows builds + WindowsService bool // config is the proxy server's configuration object. config *kubeproxyconfig.KubeProxyConfiguration @@ -119,6 +122,7 @@ type Options struct { // AddFlags adds flags to fs and binds them to options. func (o *Options) AddFlags(fs *pflag.FlagSet) { + o.addOSFlags(fs) fs.StringVar(&o.ConfigFile, "config", o.ConfigFile, "The path to the configuration file.") fs.StringVar(&o.WriteConfigTo, "write-config-to", o.WriteConfigTo, "If set, write the default configuration values to this file and exit.") fs.BoolVar(&o.CleanupAndExit, "cleanup-iptables", o.CleanupAndExit, "If true cleanup iptables and ipvs rules and exit.") @@ -344,6 +348,10 @@ with the apiserver API to configure the proxy.`, verflag.PrintAndExitIfRequested() utilflag.PrintFlags(cmd.Flags()) + if err := initForOS(opts.WindowsService); err != nil { + glog.Fatalf("failed OS init: %v", err) + } + cmdutil.CheckErr(opts.Complete()) cmdutil.CheckErr(opts.Validate(args)) cmdutil.CheckErr(opts.Run()) diff --git a/cmd/kubelet/app/BUILD b/cmd/kubelet/app/BUILD index 968b7a4512..cbfb90ff13 100644 --- a/cmd/kubelet/app/BUILD +++ b/cmd/kubelet/app/BUILD @@ -20,36 +20,47 @@ go_library( "server.go", ] + select({ "@io_bazel_rules_go//go/platform:android": [ + "init_others.go", "server_unsupported.go", ], "@io_bazel_rules_go//go/platform:darwin": [ + "init_others.go", "server_unsupported.go", ], "@io_bazel_rules_go//go/platform:dragonfly": [ + "init_others.go", "server_unsupported.go", ], "@io_bazel_rules_go//go/platform:freebsd": [ + "init_others.go", "server_unsupported.go", ], "@io_bazel_rules_go//go/platform:linux": [ + "init_others.go", "server_linux.go", ], "@io_bazel_rules_go//go/platform:nacl": [ + "init_others.go", "server_unsupported.go", ], "@io_bazel_rules_go//go/platform:netbsd": [ + "init_others.go", "server_unsupported.go", ], "@io_bazel_rules_go//go/platform:openbsd": [ + "init_others.go", "server_unsupported.go", ], "@io_bazel_rules_go//go/platform:plan9": [ + "init_others.go", "server_unsupported.go", ], "@io_bazel_rules_go//go/platform:solaris": [ + "init_others.go", "server_unsupported.go", ], "@io_bazel_rules_go//go/platform:windows": [ + "init_windows.go", "server_unsupported.go", ], "//conditions:default": [], @@ -162,6 +173,9 @@ go_library( "@io_bazel_rules_go//go/platform:linux": [ "//vendor/golang.org/x/exp/inotify:go_default_library", ], + "@io_bazel_rules_go//go/platform:windows": [ + "//pkg/windows/service:go_default_library", + ], "//conditions:default": [], }), ) diff --git a/cmd/kubelet/app/init_others.go b/cmd/kubelet/app/init_others.go new file mode 100644 index 0000000000..4b19ac91b2 --- /dev/null +++ b/cmd/kubelet/app/init_others.go @@ -0,0 +1,23 @@ +// +build !windows + +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +func initForOS(service bool) error { + return nil +} diff --git a/cmd/kubelet/app/init_windows.go b/cmd/kubelet/app/init_windows.go new file mode 100644 index 0000000000..a2da4cf049 --- /dev/null +++ b/cmd/kubelet/app/init_windows.go @@ -0,0 +1,34 @@ +// +build windows + +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "k8s.io/kubernetes/pkg/windows/service" +) + +const ( + serviceName = "kubelet" +) + +func initForOS(windowsService bool) error { + if windowsService { + return service.InitService(serviceName) + } + return nil +} diff --git a/cmd/kubelet/app/options/BUILD b/cmd/kubelet/app/options/BUILD index 0194dc7894..db6e6a9a86 100644 --- a/cmd/kubelet/app/options/BUILD +++ b/cmd/kubelet/app/options/BUILD @@ -15,36 +15,47 @@ go_library( ] + select({ "@io_bazel_rules_go//go/platform:android": [ "globalflags_other.go", + "osflags_others.go", ], "@io_bazel_rules_go//go/platform:darwin": [ "globalflags_other.go", + "osflags_others.go", ], "@io_bazel_rules_go//go/platform:dragonfly": [ "globalflags_other.go", + "osflags_others.go", ], "@io_bazel_rules_go//go/platform:freebsd": [ "globalflags_other.go", + "osflags_others.go", ], "@io_bazel_rules_go//go/platform:linux": [ "globalflags_linux.go", + "osflags_others.go", ], "@io_bazel_rules_go//go/platform:nacl": [ "globalflags_other.go", + "osflags_others.go", ], "@io_bazel_rules_go//go/platform:netbsd": [ "globalflags_other.go", + "osflags_others.go", ], "@io_bazel_rules_go//go/platform:openbsd": [ "globalflags_other.go", + "osflags_others.go", ], "@io_bazel_rules_go//go/platform:plan9": [ "globalflags_other.go", + "osflags_others.go", ], "@io_bazel_rules_go//go/platform:solaris": [ "globalflags_other.go", + "osflags_others.go", ], "@io_bazel_rules_go//go/platform:windows": [ "globalflags_other.go", + "osflags_windows.go", ], "//conditions:default": [], }), diff --git a/cmd/kubelet/app/options/options.go b/cmd/kubelet/app/options/options.go index c881423451..d359ac0705 100644 --- a/cmd/kubelet/app/options/options.go +++ b/cmd/kubelet/app/options/options.go @@ -125,6 +125,10 @@ type KubeletFlags struct { // cAdvisorPort is the port of the localhost cAdvisor endpoint (set to 0 to disable) CAdvisorPort int32 + // WindowsService should be set to true if kubelet is running as a service on Windows. + // Its corresponding flag only gets registered in Windows builds. + WindowsService bool + // EXPERIMENTAL FLAGS // Whitelist of unsafe sysctls or sysctl patterns (ending in *). // +optional @@ -329,6 +333,7 @@ func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) { // AddFlags adds flags for a specific KubeletFlags to the specified FlagSet func (f *KubeletFlags) AddFlags(fs *pflag.FlagSet) { f.ContainerRuntimeOptions.AddFlags(fs) + f.addOSFlags(fs) fs.StringVar(&f.KubeletConfigFile, "config", f.KubeletConfigFile, "The Kubelet will load its initial configuration from this file. The path may be absolute or relative; relative paths start at the Kubelet's current working directory. Omit this flag to use the built-in default configuration values. Command-line flags override configuration from this file.") fs.StringVar(&f.KubeConfig, "kubeconfig", f.KubeConfig, "Path to a kubeconfig file, specifying how to connect to the API server. Providing --kubeconfig enables API server mode, omitting --kubeconfig enables standalone mode.") diff --git a/cmd/kubelet/app/options/osflags_others.go b/cmd/kubelet/app/options/osflags_others.go new file mode 100644 index 0000000000..ab4c0ac1c6 --- /dev/null +++ b/cmd/kubelet/app/options/osflags_others.go @@ -0,0 +1,26 @@ +// +build !windows + +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +import ( + "github.com/spf13/pflag" +) + +func (f *KubeletFlags) addOSFlags(fs *pflag.FlagSet) { +} diff --git a/cmd/kubelet/app/options/osflags_windows.go b/cmd/kubelet/app/options/osflags_windows.go new file mode 100644 index 0000000000..8923f5d6f1 --- /dev/null +++ b/cmd/kubelet/app/options/osflags_windows.go @@ -0,0 +1,27 @@ +// +build windows + +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +import ( + "github.com/spf13/pflag" +) + +func (f *KubeletFlags) addOSFlags(fs *pflag.FlagSet) { + fs.BoolVar(&f.WindowsService, "windows-service", f.WindowsService, "Enable Windows Service Control Manager API integration") +} diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index b355640d6e..327d372f15 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -373,6 +373,9 @@ func UnsecuredDependencies(s *options.KubeletServer) (*kubelet.Dependencies, err func Run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies) error { // To help debugging, immediately log version glog.Infof("Version: %+v", version.Get()) + if err := initForOS(s.KubeletFlags.WindowsService); err != nil { + return fmt.Errorf("failed OS init: %v", err) + } if err := run(s, kubeDeps); err != nil { return fmt.Errorf("failed to run Kubelet: %v", err) } diff --git a/pkg/BUILD b/pkg/BUILD index 9b5868573b..037dbecae9 100644 --- a/pkg/BUILD +++ b/pkg/BUILD @@ -102,6 +102,7 @@ filegroup( "//pkg/version:all-srcs", "//pkg/volume:all-srcs", "//pkg/watch/json:all-srcs", + "//pkg/windows/service:all-srcs", ], tags = ["automanaged"], ) diff --git a/pkg/windows/service/BUILD b/pkg/windows/service/BUILD new file mode 100644 index 0000000000..ea07fc9e4d --- /dev/null +++ b/pkg/windows/service/BUILD @@ -0,0 +1,38 @@ +package(default_visibility = ["//visibility:public"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_library", +) + +go_library( + name = "go_default_library", + srcs = select({ + "@io_bazel_rules_go//go/platform:windows": [ + "service.go", + ], + "//conditions:default": [], + }), + importpath = "k8s.io/kubernetes/pkg/windows/service", + deps = select({ + "@io_bazel_rules_go//go/platform:windows": [ + "//vendor/github.com/golang/glog:go_default_library", + "//vendor/golang.org/x/sys/windows:go_default_library", + "//vendor/golang.org/x/sys/windows/svc:go_default_library", + ], + "//conditions:default": [], + }), +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], +) diff --git a/pkg/windows/service/service.go b/pkg/windows/service/service.go new file mode 100644 index 0000000000..acc48246f1 --- /dev/null +++ b/pkg/windows/service/service.go @@ -0,0 +1,91 @@ +// +build windows + +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "os" + + "github.com/golang/glog" + + "golang.org/x/sys/windows" + "golang.org/x/sys/windows/svc" +) + +var ( + service *handler +) + +type handler struct { + tosvc chan bool + fromsvc chan error +} + +// InitService is the entry point for running the daemon as a Windows +// service. It returns an indication of whether it is running as a service; +// and an error. +func InitService(serviceName string) error { + h := &handler{ + tosvc: make(chan bool), + fromsvc: make(chan error), + } + + service = h + var err error + go func() { + err = svc.Run(serviceName, h) + h.fromsvc <- err + }() + + // Wait for the first signal from the service handler. + err = <-h.fromsvc + if err != nil { + return err + } + glog.Infof("Running %s as a Windows service!", serviceName) + return nil +} + +func (h *handler) Execute(_ []string, r <-chan svc.ChangeRequest, s chan<- svc.Status) (bool, uint32) { + s <- svc.Status{State: svc.StartPending, Accepts: 0} + // Unblock initService() + h.fromsvc <- nil + + s <- svc.Status{State: svc.Running, Accepts: svc.AcceptStop | svc.AcceptShutdown | svc.Accepted(windows.SERVICE_ACCEPT_PARAMCHANGE)} + glog.Infof("Service running") +Loop: + for { + select { + case <-h.tosvc: + break Loop + case c := <-r: + switch c.Cmd { + case svc.Cmd(windows.SERVICE_CONTROL_PARAMCHANGE): + s <- c.CurrentStatus + case svc.Interrogate: + s <- c.CurrentStatus + case svc.Stop, svc.Shutdown: + s <- svc.Status{State: svc.Stopped} + // TODO: Stop the kubelet gracefully instead of killing the process + os.Exit(0) + } + } + } + + return false, 0 +}