2019-01-01 08:23:01 +00:00
|
|
|
package server
|
|
|
|
|
|
|
|
import (
|
2020-12-22 22:17:00 +00:00
|
|
|
"context"
|
2019-05-29 18:53:51 +00:00
|
|
|
"crypto"
|
2019-04-19 18:20:34 +00:00
|
|
|
"crypto/x509"
|
2021-04-21 22:56:20 +00:00
|
|
|
"fmt"
|
2019-04-19 18:20:34 +00:00
|
|
|
"net"
|
2019-01-01 08:23:01 +00:00
|
|
|
"net/http"
|
2020-12-22 22:17:00 +00:00
|
|
|
"os"
|
2023-07-07 16:14:49 +00:00
|
|
|
"path"
|
2019-03-18 18:50:20 +00:00
|
|
|
"path/filepath"
|
2019-01-26 04:56:12 +00:00
|
|
|
"strconv"
|
2019-04-23 17:53:06 +00:00
|
|
|
"strings"
|
2020-12-22 22:17:00 +00:00
|
|
|
"sync"
|
|
|
|
"time"
|
2019-01-01 08:23:01 +00:00
|
|
|
|
|
|
|
"github.com/gorilla/mux"
|
2022-03-02 23:47:27 +00:00
|
|
|
"github.com/k3s-io/k3s/pkg/bootstrap"
|
|
|
|
"github.com/k3s-io/k3s/pkg/cli/cmds"
|
|
|
|
"github.com/k3s-io/k3s/pkg/daemons/config"
|
2022-04-27 09:09:58 +00:00
|
|
|
"github.com/k3s-io/k3s/pkg/generated/clientset/versioned/scheme"
|
2022-03-02 23:47:27 +00:00
|
|
|
"github.com/k3s-io/k3s/pkg/nodepassword"
|
2022-03-29 18:36:48 +00:00
|
|
|
"github.com/k3s-io/k3s/pkg/util"
|
2022-03-02 23:47:27 +00:00
|
|
|
"github.com/k3s-io/k3s/pkg/version"
|
2020-12-22 22:17:00 +00:00
|
|
|
"github.com/pkg/errors"
|
2019-05-09 22:05:51 +00:00
|
|
|
certutil "github.com/rancher/dynamiclistener/cert"
|
2021-07-03 08:37:19 +00:00
|
|
|
coreclient "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
|
2019-04-19 18:20:34 +00:00
|
|
|
"github.com/sirupsen/logrus"
|
2023-05-31 18:54:03 +00:00
|
|
|
corev1 "k8s.io/api/core/v1"
|
2022-04-27 09:09:58 +00:00
|
|
|
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
2022-03-29 18:36:48 +00:00
|
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
2022-04-27 09:09:58 +00:00
|
|
|
"k8s.io/apimachinery/pkg/runtime/schema"
|
2023-05-31 18:54:03 +00:00
|
|
|
"k8s.io/apimachinery/pkg/types"
|
2019-01-09 16:54:15 +00:00
|
|
|
"k8s.io/apimachinery/pkg/util/json"
|
2023-05-31 18:54:03 +00:00
|
|
|
"k8s.io/apimachinery/pkg/util/wait"
|
2022-04-04 21:53:12 +00:00
|
|
|
"k8s.io/apiserver/pkg/authentication/user"
|
2022-04-27 09:09:58 +00:00
|
|
|
"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
|
2022-12-24 01:58:55 +00:00
|
|
|
"k8s.io/apiserver/pkg/endpoints/request"
|
2022-12-07 01:44:53 +00:00
|
|
|
bootstrapapi "k8s.io/cluster-bootstrap/token/api"
|
2022-12-24 01:58:55 +00:00
|
|
|
"k8s.io/kubernetes/pkg/auth/nodeidentifier"
|
2019-01-01 08:23:01 +00:00
|
|
|
)
|
|
|
|
|
2019-01-26 04:56:12 +00:00
|
|
|
const (
|
2019-10-27 05:53:25 +00:00
|
|
|
staticURL = "/static/"
|
2019-01-26 04:56:12 +00:00
|
|
|
)
|
|
|
|
|
2022-12-24 01:58:55 +00:00
|
|
|
var (
|
|
|
|
identifier = nodeidentifier.NewDefaultNodeIdentifier()
|
|
|
|
)
|
|
|
|
|
2021-07-28 20:56:59 +00:00
|
|
|
func router(ctx context.Context, config *Config, cfg *cmds.Server) http.Handler {
|
2020-12-22 22:17:00 +00:00
|
|
|
serverConfig := &config.ControlConfig
|
|
|
|
nodeAuth := passwordBootstrap(ctx, config)
|
|
|
|
|
2020-05-05 22:09:04 +00:00
|
|
|
prefix := "/v1-" + version.Program
|
2022-04-04 21:54:50 +00:00
|
|
|
authed := mux.NewRouter().SkipClean(true)
|
2022-12-07 01:44:53 +00:00
|
|
|
authed.Use(authMiddleware(serverConfig, version.Program+":agent", user.NodesGroup, bootstrapapi.BootstrapDefaultGroup))
|
2020-12-22 22:17:00 +00:00
|
|
|
authed.Path(prefix + "/serving-kubelet.crt").Handler(servingKubeletCert(serverConfig, serverConfig.Runtime.ServingKubeletKey, nodeAuth))
|
|
|
|
authed.Path(prefix + "/client-kubelet.crt").Handler(clientKubeletCert(serverConfig, serverConfig.Runtime.ClientKubeletKey, nodeAuth))
|
2020-05-05 22:09:04 +00:00
|
|
|
authed.Path(prefix + "/client-kube-proxy.crt").Handler(fileHandler(serverConfig.Runtime.ClientKubeProxyCert, serverConfig.Runtime.ClientKubeProxyKey))
|
|
|
|
authed.Path(prefix + "/client-" + version.Program + "-controller.crt").Handler(fileHandler(serverConfig.Runtime.ClientK3sControllerCert, serverConfig.Runtime.ClientK3sControllerKey))
|
|
|
|
authed.Path(prefix + "/client-ca.crt").Handler(fileHandler(serverConfig.Runtime.ClientCA))
|
|
|
|
authed.Path(prefix + "/server-ca.crt").Handler(fileHandler(serverConfig.Runtime.ServerCA))
|
2022-03-29 18:36:48 +00:00
|
|
|
authed.Path(prefix + "/apiservers").Handler(apiserversHandler(serverConfig))
|
2021-07-28 20:56:59 +00:00
|
|
|
authed.Path(prefix + "/config").Handler(configHandler(serverConfig, cfg))
|
2021-07-27 21:56:05 +00:00
|
|
|
authed.Path(prefix + "/readyz").Handler(readyzHandler(serverConfig))
|
2019-01-01 08:23:01 +00:00
|
|
|
|
2022-02-16 22:19:58 +00:00
|
|
|
if cfg.DisableAPIServer {
|
|
|
|
authed.NotFoundHandler = apiserverDisabled()
|
|
|
|
} else {
|
|
|
|
authed.NotFoundHandler = apiserver(serverConfig.Runtime)
|
|
|
|
}
|
|
|
|
|
2022-04-04 21:54:50 +00:00
|
|
|
nodeAuthed := mux.NewRouter().SkipClean(true)
|
|
|
|
nodeAuthed.NotFoundHandler = authed
|
2022-04-04 21:53:12 +00:00
|
|
|
nodeAuthed.Use(authMiddleware(serverConfig, user.NodesGroup))
|
2020-12-10 18:26:16 +00:00
|
|
|
nodeAuthed.Path(prefix + "/connect").Handler(serverConfig.Runtime.Tunnel)
|
2019-10-27 05:53:25 +00:00
|
|
|
|
2022-04-04 21:54:50 +00:00
|
|
|
serverAuthed := mux.NewRouter().SkipClean(true)
|
2019-10-27 05:53:25 +00:00
|
|
|
serverAuthed.NotFoundHandler = nodeAuthed
|
2022-04-04 21:54:50 +00:00
|
|
|
serverAuthed.Use(authMiddleware(serverConfig, version.Program+":server"))
|
2022-01-06 18:38:22 +00:00
|
|
|
serverAuthed.Path(prefix + "/encrypt/status").Handler(encryptionStatusHandler(serverConfig))
|
|
|
|
serverAuthed.Path(prefix + "/encrypt/config").Handler(encryptionConfigHandler(ctx, serverConfig))
|
2022-12-10 00:20:51 +00:00
|
|
|
serverAuthed.Path(prefix + "/cert/cacerts").Handler(caCertReplaceHandler(serverConfig))
|
2019-10-31 02:05:40 +00:00
|
|
|
serverAuthed.Path("/db/info").Handler(nodeAuthed)
|
2022-04-15 00:30:04 +00:00
|
|
|
serverAuthed.Path(prefix + "/server-bootstrap").Handler(bootstrapHandler(serverConfig.Runtime))
|
2019-10-27 05:53:25 +00:00
|
|
|
|
2022-04-04 21:54:50 +00:00
|
|
|
systemAuthed := mux.NewRouter().SkipClean(true)
|
|
|
|
systemAuthed.NotFoundHandler = serverAuthed
|
|
|
|
systemAuthed.MethodNotAllowedHandler = serverAuthed
|
|
|
|
systemAuthed.Use(authMiddleware(serverConfig, user.SystemPrivilegedGroup))
|
|
|
|
systemAuthed.Methods(http.MethodConnect).Handler(serverConfig.Runtime.Tunnel)
|
|
|
|
|
2019-03-18 18:50:20 +00:00
|
|
|
staticDir := filepath.Join(serverConfig.DataDir, "static")
|
2022-04-04 21:54:50 +00:00
|
|
|
router := mux.NewRouter().SkipClean(true)
|
|
|
|
router.NotFoundHandler = systemAuthed
|
2019-03-18 18:50:20 +00:00
|
|
|
router.PathPrefix(staticURL).Handler(serveStatic(staticURL, staticDir))
|
2020-12-10 18:26:16 +00:00
|
|
|
router.Path("/cacerts").Handler(cacerts(serverConfig.Runtime.ServerCA))
|
2019-01-31 22:42:27 +00:00
|
|
|
router.Path("/ping").Handler(ping())
|
2019-01-01 08:23:01 +00:00
|
|
|
|
2020-12-10 18:26:16 +00:00
|
|
|
return router
|
2019-01-01 08:23:01 +00:00
|
|
|
}
|
|
|
|
|
2021-10-12 06:13:10 +00:00
|
|
|
func apiserver(runtime *config.ControlRuntime) http.Handler {
|
|
|
|
return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
|
|
|
|
if runtime != nil && runtime.APIServer != nil {
|
|
|
|
runtime.APIServer.ServeHTTP(resp, req)
|
|
|
|
} else {
|
2022-04-27 09:09:58 +00:00
|
|
|
responsewriters.ErrorNegotiated(
|
|
|
|
apierrors.NewServiceUnavailable("apiserver not ready"),
|
|
|
|
scheme.Codecs.WithoutConversion(), schema.GroupVersion{}, resp, req,
|
|
|
|
)
|
2021-10-12 06:13:10 +00:00
|
|
|
}
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
2022-02-16 22:19:58 +00:00
|
|
|
func apiserverDisabled() http.Handler {
|
|
|
|
return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
|
2022-04-27 09:09:58 +00:00
|
|
|
responsewriters.ErrorNegotiated(
|
|
|
|
apierrors.NewServiceUnavailable("apiserver disabled"),
|
|
|
|
scheme.Codecs.WithoutConversion(), schema.GroupVersion{}, resp, req,
|
|
|
|
)
|
2022-04-15 00:30:04 +00:00
|
|
|
})
|
|
|
|
}
|
|
|
|
|
|
|
|
func bootstrapHandler(runtime *config.ControlRuntime) http.Handler {
|
|
|
|
if runtime.HTTPBootstrap {
|
|
|
|
return bootstrap.Handler(&runtime.ControlRuntimeBootstrap)
|
|
|
|
}
|
|
|
|
return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
|
|
|
|
logrus.Warnf("Received HTTP bootstrap request from %s, but embedded etcd is not enabled.", req.RemoteAddr)
|
2022-04-27 09:09:58 +00:00
|
|
|
responsewriters.ErrorNegotiated(
|
|
|
|
apierrors.NewBadRequest("etcd disabled"),
|
|
|
|
scheme.Codecs.WithoutConversion(), schema.GroupVersion{}, resp, req,
|
|
|
|
)
|
2022-02-16 22:19:58 +00:00
|
|
|
})
|
|
|
|
}
|
|
|
|
|
2020-12-10 18:26:16 +00:00
|
|
|
func cacerts(serverCA string) http.Handler {
|
|
|
|
var ca []byte
|
2019-01-01 08:23:01 +00:00
|
|
|
return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
|
2020-12-10 18:26:16 +00:00
|
|
|
if ca == nil {
|
|
|
|
var err error
|
2022-10-08 00:36:57 +00:00
|
|
|
ca, err = os.ReadFile(serverCA)
|
2020-12-10 18:26:16 +00:00
|
|
|
if err != nil {
|
2023-05-31 21:01:21 +00:00
|
|
|
sendError(err, resp, req)
|
2020-12-10 18:26:16 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
}
|
2019-01-01 08:23:01 +00:00
|
|
|
resp.Header().Set("content-type", "text/plain")
|
2019-10-27 05:53:25 +00:00
|
|
|
resp.Write(ca)
|
2019-01-01 08:23:01 +00:00
|
|
|
})
|
|
|
|
}
|
|
|
|
|
2022-12-24 01:58:55 +00:00
|
|
|
func getNodeInfo(req *http.Request) (*nodeInfo, error) {
|
|
|
|
user, ok := request.UserFrom(req.Context())
|
|
|
|
if !ok {
|
|
|
|
return nil, errors.New("auth user not set")
|
|
|
|
}
|
|
|
|
|
2020-05-05 22:09:04 +00:00
|
|
|
nodeName := req.Header.Get(version.Program + "-Node-Name")
|
|
|
|
if nodeName == "" {
|
2022-12-24 01:58:55 +00:00
|
|
|
return nil, errors.New("node name not set")
|
2019-05-29 18:53:51 +00:00
|
|
|
}
|
|
|
|
|
2020-05-05 22:09:04 +00:00
|
|
|
nodePassword := req.Header.Get(version.Program + "-Node-Password")
|
|
|
|
if nodePassword == "" {
|
2022-12-24 01:58:55 +00:00
|
|
|
return nil, errors.New("node password not set")
|
2019-05-29 18:53:51 +00:00
|
|
|
}
|
|
|
|
|
2022-12-24 01:58:55 +00:00
|
|
|
return &nodeInfo{
|
|
|
|
Name: strings.ToLower(nodeName),
|
|
|
|
Password: nodePassword,
|
|
|
|
User: user,
|
|
|
|
}, nil
|
2019-05-29 18:53:51 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func getCACertAndKeys(caCertFile, caKeyFile, signingKeyFile string) ([]*x509.Certificate, crypto.Signer, crypto.Signer, error) {
|
2022-10-08 00:36:57 +00:00
|
|
|
keyBytes, err := os.ReadFile(signingKeyFile)
|
2019-05-29 18:53:51 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, nil, nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
key, err := certutil.ParsePrivateKeyPEM(keyBytes)
|
|
|
|
if err != nil {
|
|
|
|
return nil, nil, nil, err
|
|
|
|
}
|
|
|
|
|
2022-10-08 00:36:57 +00:00
|
|
|
caKeyBytes, err := os.ReadFile(caKeyFile)
|
2019-05-29 18:53:51 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, nil, nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
caKey, err := certutil.ParsePrivateKeyPEM(caKeyBytes)
|
|
|
|
if err != nil {
|
|
|
|
return nil, nil, nil, err
|
|
|
|
}
|
|
|
|
|
2022-10-08 00:36:57 +00:00
|
|
|
caBytes, err := os.ReadFile(caCertFile)
|
2019-05-29 18:53:51 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, nil, nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
caCert, err := certutil.ParseCertsPEM(caBytes)
|
|
|
|
if err != nil {
|
|
|
|
return nil, nil, nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
return caCert, caKey.(crypto.Signer), key.(crypto.Signer), nil
|
|
|
|
}
|
|
|
|
|
2020-12-22 22:17:00 +00:00
|
|
|
func servingKubeletCert(server *config.Control, keyFile string, auth nodePassBootstrapper) http.Handler {
|
2019-01-01 08:23:01 +00:00
|
|
|
return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
|
|
|
|
if req.TLS == nil {
|
|
|
|
resp.WriteHeader(http.StatusNotFound)
|
|
|
|
return
|
|
|
|
}
|
2019-04-19 18:20:34 +00:00
|
|
|
|
2020-12-23 23:55:47 +00:00
|
|
|
nodeName, errCode, err := auth(req)
|
|
|
|
if err != nil {
|
2023-05-31 21:01:21 +00:00
|
|
|
sendError(err, resp, req, errCode)
|
2019-04-23 17:53:06 +00:00
|
|
|
return
|
2019-04-19 18:20:34 +00:00
|
|
|
}
|
|
|
|
|
2023-03-07 23:05:52 +00:00
|
|
|
caCerts, caKey, key, err := getCACertAndKeys(server.Runtime.ServerCA, server.Runtime.ServerCAKey, server.Runtime.ServingKubeletKey)
|
2019-04-19 18:20:34 +00:00
|
|
|
if err != nil {
|
2023-05-31 21:01:21 +00:00
|
|
|
sendError(err, resp, req)
|
2019-04-19 18:20:34 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2020-09-03 02:15:09 +00:00
|
|
|
ips := []net.IP{net.ParseIP("127.0.0.1")}
|
2021-04-21 22:56:20 +00:00
|
|
|
|
2020-09-04 01:48:30 +00:00
|
|
|
if nodeIP := req.Header.Get(version.Program + "-Node-IP"); nodeIP != "" {
|
2021-04-21 22:56:20 +00:00
|
|
|
for _, v := range strings.Split(nodeIP, ",") {
|
|
|
|
ip := net.ParseIP(v)
|
|
|
|
if ip == nil {
|
2023-05-31 21:01:21 +00:00
|
|
|
sendError(fmt.Errorf("invalid node IP address %s", ip), resp, req)
|
2021-04-21 22:56:20 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
ips = append(ips, ip)
|
|
|
|
}
|
2020-09-03 02:15:09 +00:00
|
|
|
}
|
|
|
|
|
2019-05-29 18:53:51 +00:00
|
|
|
cert, err := certutil.NewSignedCert(certutil.Config{
|
|
|
|
CommonName: nodeName,
|
|
|
|
Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
|
|
|
|
AltNames: certutil.AltNames{
|
|
|
|
DNSNames: []string{nodeName, "localhost"},
|
2020-09-03 02:15:09 +00:00
|
|
|
IPs: ips,
|
2019-05-29 18:53:51 +00:00
|
|
|
},
|
2023-03-07 23:05:52 +00:00
|
|
|
}, key, caCerts[0], caKey)
|
2019-04-19 18:20:34 +00:00
|
|
|
if err != nil {
|
2023-05-31 21:01:21 +00:00
|
|
|
sendError(err, resp, req)
|
2019-04-19 18:20:34 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2022-10-08 00:36:57 +00:00
|
|
|
keyBytes, err := os.ReadFile(keyFile)
|
2019-11-15 23:48:03 +00:00
|
|
|
if err != nil {
|
|
|
|
http.Error(resp, err.Error(), http.StatusInternalServerError)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2023-03-07 23:05:52 +00:00
|
|
|
resp.Write(util.EncodeCertsPEM(cert, caCerts))
|
2019-11-15 23:48:03 +00:00
|
|
|
resp.Write(keyBytes)
|
2019-05-29 18:53:51 +00:00
|
|
|
})
|
|
|
|
}
|
2019-04-19 18:20:34 +00:00
|
|
|
|
2020-12-22 22:17:00 +00:00
|
|
|
func clientKubeletCert(server *config.Control, keyFile string, auth nodePassBootstrapper) http.Handler {
|
2019-05-29 18:53:51 +00:00
|
|
|
return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
|
|
|
|
if req.TLS == nil {
|
|
|
|
resp.WriteHeader(http.StatusNotFound)
|
2019-04-19 18:20:34 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2020-12-23 23:55:47 +00:00
|
|
|
nodeName, errCode, err := auth(req)
|
|
|
|
if err != nil {
|
2023-05-31 21:01:21 +00:00
|
|
|
sendError(err, resp, req, errCode)
|
2019-04-19 18:20:34 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2023-03-07 23:05:52 +00:00
|
|
|
caCerts, caKey, key, err := getCACertAndKeys(server.Runtime.ClientCA, server.Runtime.ClientCAKey, server.Runtime.ClientKubeletKey)
|
2019-04-19 18:20:34 +00:00
|
|
|
if err != nil {
|
2023-05-31 21:01:21 +00:00
|
|
|
sendError(err, resp, req)
|
2019-04-19 18:20:34 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2019-05-29 18:53:51 +00:00
|
|
|
cert, err := certutil.NewSignedCert(certutil.Config{
|
|
|
|
CommonName: "system:node:" + nodeName,
|
2022-04-04 21:53:12 +00:00
|
|
|
Organization: []string{user.NodesGroup},
|
2019-05-29 18:53:51 +00:00
|
|
|
Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
|
2023-03-07 23:05:52 +00:00
|
|
|
}, key, caCerts[0], caKey)
|
2019-04-19 18:20:34 +00:00
|
|
|
if err != nil {
|
2023-05-31 21:01:21 +00:00
|
|
|
sendError(err, resp, req)
|
2019-04-19 18:20:34 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2022-10-08 00:36:57 +00:00
|
|
|
keyBytes, err := os.ReadFile(keyFile)
|
2019-11-15 23:48:03 +00:00
|
|
|
if err != nil {
|
|
|
|
http.Error(resp, err.Error(), http.StatusInternalServerError)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2023-03-07 23:05:52 +00:00
|
|
|
resp.Write(util.EncodeCertsPEM(cert, caCerts))
|
2019-11-15 23:48:03 +00:00
|
|
|
resp.Write(keyBytes)
|
2019-01-01 08:23:01 +00:00
|
|
|
})
|
|
|
|
}
|
|
|
|
|
2019-11-15 23:48:03 +00:00
|
|
|
func fileHandler(fileName ...string) http.Handler {
|
2019-01-01 08:23:01 +00:00
|
|
|
return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
|
|
|
|
if req.TLS == nil {
|
|
|
|
resp.WriteHeader(http.StatusNotFound)
|
|
|
|
return
|
|
|
|
}
|
2019-11-15 23:48:03 +00:00
|
|
|
resp.Header().Set("Content-Type", "text/plain")
|
|
|
|
|
|
|
|
if len(fileName) == 1 {
|
|
|
|
http.ServeFile(resp, req, fileName[0])
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
for _, f := range fileName {
|
2022-10-08 00:36:57 +00:00
|
|
|
bytes, err := os.ReadFile(f)
|
2019-11-15 23:48:03 +00:00
|
|
|
if err != nil {
|
|
|
|
logrus.Errorf("Failed to read %s: %v", f, err)
|
|
|
|
resp.WriteHeader(http.StatusInternalServerError)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
resp.Write(bytes)
|
|
|
|
}
|
2019-01-01 08:23:01 +00:00
|
|
|
})
|
|
|
|
}
|
|
|
|
|
2022-03-29 18:36:48 +00:00
|
|
|
func apiserversHandler(server *config.Control) http.Handler {
|
|
|
|
var endpointsClient coreclient.EndpointsClient
|
|
|
|
return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
|
|
|
|
var endpoints []string
|
|
|
|
if endpointsClient == nil {
|
|
|
|
if server.Runtime.Core != nil {
|
|
|
|
endpointsClient = server.Runtime.Core.Core().V1().Endpoints()
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if endpointsClient != nil {
|
|
|
|
if endpoint, _ := endpointsClient.Get("default", "kubernetes", metav1.GetOptions{}); endpoint != nil {
|
|
|
|
endpoints = util.GetAddresses(endpoint)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
resp.Header().Set("content-type", "application/json")
|
|
|
|
if err := json.NewEncoder(resp).Encode(endpoints); err != nil {
|
|
|
|
logrus.Errorf("Failed to encode apiserver endpoints: %v", err)
|
|
|
|
resp.WriteHeader(http.StatusInternalServerError)
|
|
|
|
}
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
2021-07-28 20:56:59 +00:00
|
|
|
func configHandler(server *config.Control, cfg *cmds.Server) http.Handler {
|
2019-01-01 08:23:01 +00:00
|
|
|
return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
|
|
|
|
if req.TLS == nil {
|
|
|
|
resp.WriteHeader(http.StatusNotFound)
|
|
|
|
return
|
|
|
|
}
|
2021-07-28 20:56:59 +00:00
|
|
|
// Startup hooks may read and modify cmds.Server in a goroutine, but as these are copied into
|
|
|
|
// config.Control before the startup hooks are called, any modifications need to be sync'd back
|
|
|
|
// into the struct before it is sent to agents.
|
|
|
|
// At this time we don't sync all the fields, just those known to be touched by startup hooks.
|
|
|
|
server.DisableKubeProxy = cfg.DisableKubeProxy
|
2019-01-01 08:23:01 +00:00
|
|
|
resp.Header().Set("content-type", "application/json")
|
2021-10-12 06:13:10 +00:00
|
|
|
if err := json.NewEncoder(resp).Encode(server); err != nil {
|
|
|
|
logrus.Errorf("Failed to encode agent config: %v", err)
|
|
|
|
resp.WriteHeader(http.StatusInternalServerError)
|
|
|
|
}
|
2019-01-01 08:23:01 +00:00
|
|
|
})
|
|
|
|
}
|
2019-01-26 04:56:12 +00:00
|
|
|
|
2021-07-27 21:56:05 +00:00
|
|
|
func readyzHandler(server *config.Control) http.Handler {
|
|
|
|
return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
|
|
|
|
code := http.StatusOK
|
|
|
|
data := []byte("ok")
|
|
|
|
if server.Runtime.Core == nil {
|
|
|
|
code = http.StatusInternalServerError
|
|
|
|
data = []byte("runtime core not ready")
|
|
|
|
}
|
|
|
|
resp.WriteHeader(code)
|
|
|
|
resp.Header().Set("Content-Type", "text/plain")
|
|
|
|
resp.Header().Set("Content-length", strconv.Itoa(len(data)))
|
|
|
|
resp.Write(data)
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
2019-01-31 22:42:27 +00:00
|
|
|
func ping() http.Handler {
|
|
|
|
return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
|
|
|
|
data := []byte("pong")
|
|
|
|
resp.Header().Set("Content-Type", "text/plain")
|
|
|
|
resp.Header().Set("Content-Length", strconv.Itoa(len(data)))
|
|
|
|
resp.Write(data)
|
|
|
|
})
|
|
|
|
}
|
2019-03-18 18:50:20 +00:00
|
|
|
|
|
|
|
func serveStatic(urlPrefix, staticDir string) http.Handler {
|
|
|
|
return http.StripPrefix(urlPrefix, http.FileServer(http.Dir(staticDir)))
|
|
|
|
}
|
2019-04-19 18:20:34 +00:00
|
|
|
|
2023-05-31 21:01:21 +00:00
|
|
|
func sendError(err error, resp http.ResponseWriter, req *http.Request, status ...int) {
|
2020-12-23 23:55:47 +00:00
|
|
|
var code int
|
|
|
|
if len(status) == 1 {
|
2019-04-23 17:53:06 +00:00
|
|
|
code = status[0]
|
|
|
|
}
|
2020-12-23 23:55:47 +00:00
|
|
|
if code == 0 || code == http.StatusOK {
|
|
|
|
code = http.StatusInternalServerError
|
2020-12-22 22:17:00 +00:00
|
|
|
}
|
2023-05-31 21:01:21 +00:00
|
|
|
logrus.Errorf("Sending HTTP %d response to %s: %v", code, req.RemoteAddr, err)
|
|
|
|
responsewriters.ErrorNegotiated(
|
|
|
|
apierrors.NewGenericServerResponse(code, req.Method, schema.GroupResource{}, req.URL.Path, err.Error(), 0, true),
|
|
|
|
scheme.Codecs.WithoutConversion(), schema.GroupVersion{}, resp, req,
|
|
|
|
)
|
2019-04-19 18:20:34 +00:00
|
|
|
}
|
2020-12-22 22:17:00 +00:00
|
|
|
|
2020-12-23 23:55:47 +00:00
|
|
|
// nodePassBootstrapper returns a node name, or http error code and error
|
2020-12-22 22:17:00 +00:00
|
|
|
type nodePassBootstrapper func(req *http.Request) (string, int, error)
|
|
|
|
|
2022-12-24 01:58:55 +00:00
|
|
|
// nodeInfo contains information on the requesting node, derived from auth creds
|
|
|
|
// and request headers.
|
|
|
|
type nodeInfo struct {
|
|
|
|
Name string
|
|
|
|
Password string
|
|
|
|
User user.Info
|
|
|
|
}
|
|
|
|
|
2020-12-22 22:17:00 +00:00
|
|
|
func passwordBootstrap(ctx context.Context, config *Config) nodePassBootstrapper {
|
|
|
|
runtime := config.ControlConfig.Runtime
|
2022-03-18 09:07:19 +00:00
|
|
|
deferredNodes := map[string]bool{}
|
2020-12-22 22:17:00 +00:00
|
|
|
var secretClient coreclient.SecretClient
|
2022-12-24 01:58:55 +00:00
|
|
|
var nodeClient coreclient.NodeClient
|
2022-03-18 09:07:19 +00:00
|
|
|
var mu sync.Mutex
|
2020-12-22 22:17:00 +00:00
|
|
|
|
|
|
|
return nodePassBootstrapper(func(req *http.Request) (string, int, error) {
|
2022-12-24 01:58:55 +00:00
|
|
|
node, err := getNodeInfo(req)
|
2020-12-22 22:17:00 +00:00
|
|
|
if err != nil {
|
|
|
|
return "", http.StatusBadRequest, err
|
|
|
|
}
|
|
|
|
|
2022-12-24 01:58:55 +00:00
|
|
|
nodeName, isNodeAuth := identifier.NodeIdentity(node.User)
|
|
|
|
if isNodeAuth && nodeName != node.Name {
|
|
|
|
return "", http.StatusBadRequest, errors.New("header node name does not match auth node name")
|
|
|
|
}
|
|
|
|
|
|
|
|
if secretClient == nil || nodeClient == nil {
|
2020-12-22 22:17:00 +00:00
|
|
|
if runtime.Core != nil {
|
|
|
|
// initialize the client if we can
|
|
|
|
secretClient = runtime.Core.Core().V1().Secret()
|
2022-12-24 01:58:55 +00:00
|
|
|
nodeClient = runtime.Core.Core().V1().Node()
|
|
|
|
} else if node.Name == os.Getenv("NODE_NAME") {
|
|
|
|
// If we're verifying our own password, verify it locally and ensure a secret later.
|
|
|
|
return verifyLocalPassword(ctx, config, &mu, deferredNodes, node)
|
|
|
|
} else if config.ControlConfig.DisableAPIServer && !isNodeAuth {
|
|
|
|
// If we're running on an etcd-only node, and the request didn't use Node Identity auth,
|
|
|
|
// defer node password verification until an apiserver joins the cluster.
|
|
|
|
return verifyRemotePassword(ctx, config, &mu, deferredNodes, node)
|
2020-12-22 22:17:00 +00:00
|
|
|
} else {
|
2022-12-24 01:58:55 +00:00
|
|
|
// Otherwise, reject the request until the core is ready.
|
2020-12-22 22:17:00 +00:00
|
|
|
return "", http.StatusServiceUnavailable, errors.New("runtime core not ready")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-05-31 18:54:03 +00:00
|
|
|
// verify that the node exists, if using Node Identity auth
|
2022-12-24 01:58:55 +00:00
|
|
|
if err := verifyNode(ctx, nodeClient, node); err != nil {
|
|
|
|
return "", http.StatusUnauthorized, err
|
|
|
|
}
|
|
|
|
|
2023-05-31 18:54:03 +00:00
|
|
|
// verify that the node password secret matches, or create it if it does not
|
2022-12-24 01:58:55 +00:00
|
|
|
if err := nodepassword.Ensure(secretClient, node.Name, node.Password); err != nil {
|
2023-05-31 18:54:03 +00:00
|
|
|
// if the verification failed, reject the request
|
|
|
|
if errors.Is(err, nodepassword.ErrVerifyFailed) {
|
|
|
|
return "", http.StatusForbidden, err
|
|
|
|
}
|
|
|
|
// If verification failed due to an error creating the node password secret, allow
|
|
|
|
// the request, but retry verification until the outage is resolved. This behavior
|
|
|
|
// allows nodes to join the cluster during outages caused by validating webhooks
|
|
|
|
// blocking secret creation - if the outage requires new nodes to join in order to
|
|
|
|
// run the webhook pods, we must fail open here to resolve the outage.
|
|
|
|
return verifyRemotePassword(ctx, config, &mu, deferredNodes, node)
|
2020-12-22 22:17:00 +00:00
|
|
|
}
|
|
|
|
|
2022-12-24 01:58:55 +00:00
|
|
|
return node.Name, http.StatusOK, nil
|
2020-12-22 22:17:00 +00:00
|
|
|
})
|
|
|
|
}
|
|
|
|
|
2022-12-24 01:58:55 +00:00
|
|
|
func verifyLocalPassword(ctx context.Context, config *Config, mu *sync.Mutex, deferredNodes map[string]bool, node *nodeInfo) (string, int, error) {
|
2020-12-22 22:17:00 +00:00
|
|
|
// use same password file location that the agent creates
|
|
|
|
nodePasswordRoot := "/"
|
2022-09-29 20:16:33 +00:00
|
|
|
if config.ControlConfig.Rootless {
|
2023-07-07 16:14:49 +00:00
|
|
|
nodePasswordRoot = filepath.Join(path.Dir(config.ControlConfig.DataDir), "agent")
|
2020-12-22 22:17:00 +00:00
|
|
|
}
|
|
|
|
nodeConfigPath := filepath.Join(nodePasswordRoot, "etc", "rancher", "node")
|
|
|
|
nodePasswordFile := filepath.Join(nodeConfigPath, "password")
|
|
|
|
|
2022-10-08 00:36:57 +00:00
|
|
|
passBytes, err := os.ReadFile(nodePasswordFile)
|
2020-12-22 22:17:00 +00:00
|
|
|
if err != nil {
|
2020-12-23 22:05:55 +00:00
|
|
|
return "", http.StatusInternalServerError, errors.Wrap(err, "unable to read node password file")
|
2020-12-22 22:17:00 +00:00
|
|
|
}
|
|
|
|
|
2023-05-08 21:29:03 +00:00
|
|
|
passHash, err := nodepassword.Hasher.CreateHash(strings.TrimSpace(string(passBytes)))
|
|
|
|
if err != nil {
|
|
|
|
return "", http.StatusInternalServerError, errors.Wrap(err, "unable to hash node password file")
|
|
|
|
}
|
|
|
|
|
|
|
|
if err := nodepassword.Hasher.VerifyHash(passHash, node.Password); err != nil {
|
2023-05-31 18:54:03 +00:00
|
|
|
return "", http.StatusForbidden, errors.Wrap(err, "unable to verify local node password")
|
2020-12-22 22:17:00 +00:00
|
|
|
}
|
|
|
|
|
2022-03-18 09:07:19 +00:00
|
|
|
mu.Lock()
|
|
|
|
defer mu.Unlock()
|
|
|
|
|
2022-12-24 01:58:55 +00:00
|
|
|
if _, ok := deferredNodes[node.Name]; !ok {
|
|
|
|
deferredNodes[node.Name] = true
|
|
|
|
go ensureSecret(ctx, config, node)
|
2023-05-31 18:54:03 +00:00
|
|
|
logrus.Infof("Password verified locally for node %s", node.Name)
|
2020-12-22 22:17:00 +00:00
|
|
|
}
|
|
|
|
|
2022-12-24 01:58:55 +00:00
|
|
|
return node.Name, http.StatusOK, nil
|
2022-03-18 09:07:19 +00:00
|
|
|
}
|
2020-12-22 22:17:00 +00:00
|
|
|
|
2022-12-24 01:58:55 +00:00
|
|
|
func verifyRemotePassword(ctx context.Context, config *Config, mu *sync.Mutex, deferredNodes map[string]bool, node *nodeInfo) (string, int, error) {
|
2022-03-18 09:07:19 +00:00
|
|
|
mu.Lock()
|
|
|
|
defer mu.Unlock()
|
|
|
|
|
2022-12-24 01:58:55 +00:00
|
|
|
if _, ok := deferredNodes[node.Name]; !ok {
|
|
|
|
deferredNodes[node.Name] = true
|
|
|
|
go ensureSecret(ctx, config, node)
|
2023-05-31 18:54:03 +00:00
|
|
|
logrus.Infof("Password verification deferred for node %s", node.Name)
|
2022-03-18 09:07:19 +00:00
|
|
|
}
|
2020-12-22 22:17:00 +00:00
|
|
|
|
2022-12-24 01:58:55 +00:00
|
|
|
return node.Name, http.StatusOK, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func verifyNode(ctx context.Context, nodeClient coreclient.NodeClient, node *nodeInfo) error {
|
|
|
|
if nodeName, isNodeAuth := identifier.NodeIdentity(node.User); isNodeAuth {
|
|
|
|
if _, err := nodeClient.Get(nodeName, metav1.GetOptions{}); err != nil {
|
|
|
|
return errors.Wrap(err, "unable to verify node identity")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return nil
|
2020-12-22 22:17:00 +00:00
|
|
|
}
|
2022-03-18 09:07:19 +00:00
|
|
|
|
2022-12-24 01:58:55 +00:00
|
|
|
func ensureSecret(ctx context.Context, config *Config, node *nodeInfo) {
|
2022-03-18 09:07:19 +00:00
|
|
|
runtime := config.ControlConfig.Runtime
|
2023-05-31 18:54:03 +00:00
|
|
|
wait.PollImmediateUntilWithContext(ctx, time.Second*5, func(ctx context.Context) (bool, error) {
|
|
|
|
if runtime.Core != nil {
|
|
|
|
secretClient := runtime.Core.Core().V1().Secret()
|
|
|
|
// This is consistent with events attached to the node generated by the kubelet
|
|
|
|
// https://github.com/kubernetes/kubernetes/blob/612130dd2f4188db839ea5c2dea07a96b0ad8d1c/pkg/kubelet/kubelet.go#L479-L485
|
|
|
|
nodeRef := &corev1.ObjectReference{
|
|
|
|
Kind: "Node",
|
|
|
|
Name: node.Name,
|
|
|
|
UID: types.UID(node.Name),
|
|
|
|
Namespace: "",
|
|
|
|
}
|
|
|
|
if err := nodepassword.Ensure(secretClient, node.Name, node.Password); err != nil {
|
|
|
|
runtime.Event.Eventf(nodeRef, corev1.EventTypeWarning, "NodePasswordValidationFailed", "Deferred node password secret validation failed: %v", err)
|
|
|
|
// Return true to stop polling if the password verification failed; only retry on secret creation errors.
|
|
|
|
return errors.Is(err, nodepassword.ErrVerifyFailed), nil
|
2022-03-18 09:07:19 +00:00
|
|
|
}
|
2023-05-31 18:54:03 +00:00
|
|
|
runtime.Event.Event(nodeRef, corev1.EventTypeNormal, "NodePasswordValidationComplete", "Deferred node password secret validation complete")
|
|
|
|
return true, nil
|
2022-03-18 09:07:19 +00:00
|
|
|
}
|
2023-05-31 18:54:03 +00:00
|
|
|
return false, nil
|
|
|
|
})
|
2022-03-18 09:07:19 +00:00
|
|
|
}
|