Merge pull request #35805 from dgoodwin/token-mgmt

Automatic merge from submit-queue

Implement kubeadm bootstrap token management

Creates bootstrap tokens as secrets per the specification in #30707 

_WARNING_: These are not currently hooked up to the discovery service or the token it creates.

Still TODO:
- [x] delete tokens
- [x] merge with #35144 and adopt it's testing approach
- [x] determine if we want wholesale json output & templating like kubectl (we do not have an API object with the data we want here) may require a bit of plumbing.
- [x] allow specifying a token duration on the CLI
- [x] allow configuring the default token duration
- [x] hook up the initial token created during init

Sample output:

```
(root@centos1 ~) $ kubeadm token create
Running pre-flight checks
<cmd/token> Token secret created: f6dc69.c43e491752c4a0fd
(root@centos1 ~) $ kubeadm token create
Running pre-flight checks
<cmd/token> Token secret created: 8fad2f.e7b78c8a5f7c7b9a
(root@centos1 ~) $ kubeadm token list  
Running pre-flight checks
ID        TOKEN                     EXPIRATION
44d805    44d805.a4e78b6cf6435e33   23h
4f65bb    4f65bb.d006a3c7a0e428c9   23h
6a086e    6a086e.2ff99f0823236b5b   23h
8fad2f    8fad2f.e7b78c8a5f7c7b9a   23h
f6dc69    f6dc69.c43e491752c4a0fd   23h
f81653    f81653.9ab82a2926c7e985   23h
```
pull/6/head
Kubernetes Submit Queue 2016-12-20 14:44:40 -08:00 committed by GitHub
commit 52df372f9b
13 changed files with 306 additions and 51 deletions

View File

@ -30,7 +30,10 @@ go_library(
"//cmd/kubeadm/app/preflight:go_default_library",
"//cmd/kubeadm/app/util:go_default_library",
"//pkg/api:go_default_library",
"//pkg/api/v1:go_default_library",
"//pkg/client/unversioned/clientcmd/api:go_default_library",
"//pkg/fields:go_default_library",
"//pkg/kubectl:go_default_library",
"//pkg/kubectl/cmd/util:go_default_library",
"//pkg/runtime:go_default_library",
"//pkg/util/flag:go_default_library",

View File

@ -80,8 +80,15 @@ func NewKubeadmCommand(f cmdutil.Factory, in io.Reader, out, err io.Writer) *cob
cmds.AddCommand(NewCmdInit(out))
cmds.AddCommand(NewCmdJoin(out))
cmds.AddCommand(NewCmdReset(out))
cmds.AddCommand(NewCmdToken(out))
cmds.AddCommand(NewCmdVersion(out))
// Wrap not yet usable/supported commands in experimental sub-command:
experimentalCmd := &cobra.Command{
Use: "ex",
Short: "Experimental sub-commands not yet fully functional.",
}
experimentalCmd.AddCommand(NewCmdToken(out, err))
cmds.AddCommand(experimentalCmd)
return cmds
}

View File

@ -263,9 +263,13 @@ func (i *Init) Run(out io.Writer) error {
}
if i.cfg.Discovery.Token != nil {
fmt.Printf("[token-discovery] Using token: %s\n", kubeadmutil.BearerToken(i.cfg.Discovery.Token))
if err := kubemaster.CreateDiscoveryDeploymentAndSecret(i.cfg, client, caCert); err != nil {
return err
}
if err := kubeadmutil.UpdateOrCreateToken(client, i.cfg.Discovery.Token, kubeadmutil.DefaultTokenDuration); err != nil {
return err
}
}
if err := kubemaster.CreateEssentialAddons(i.cfg, client); err != nil {

View File

@ -20,19 +20,27 @@ import (
"errors"
"fmt"
"io"
"path"
"text/tabwriter"
"time"
"github.com/renstrom/dedent"
"github.com/spf13/cobra"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
"k8s.io/kubernetes/cmd/kubeadm/app/util"
kubemaster "k8s.io/kubernetes/cmd/kubeadm/app/master"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
"k8s.io/kubernetes/pkg/api"
v1 "k8s.io/kubernetes/pkg/api/v1"
"k8s.io/kubernetes/pkg/fields"
"k8s.io/kubernetes/pkg/kubectl"
)
func NewCmdToken(out io.Writer) *cobra.Command {
cmd := &cobra.Command{
func NewCmdToken(out io.Writer, errW io.Writer) *cobra.Command {
tokenCmd := &cobra.Command{
Use: "token",
Short: "Manage tokens used by init/join",
Short: "Manage bootstrap tokens.",
// Without this callback, if a user runs just the "token"
// command without a subcommand, or with an invalid subcommand,
@ -48,16 +56,55 @@ func NewCmdToken(out io.Writer) *cobra.Command {
},
}
cmd.AddCommand(NewCmdTokenGenerate(out))
return cmd
var token string
var tokenDuration time.Duration
createCmd := &cobra.Command{
Use: "create",
Short: "Create bootstrap tokens on the server.",
Run: func(tokenCmd *cobra.Command, args []string) {
err := RunCreateToken(out, tokenCmd, tokenDuration, token)
kubeadmutil.CheckErr(err)
},
}
createCmd.PersistentFlags().DurationVar(&tokenDuration,
"ttl", kubeadmutil.DefaultTokenDuration, "The duration before the token is automatically deleted.")
createCmd.PersistentFlags().StringVar(
&token, "token", "",
"Shared secret used to secure cluster bootstrap. If none is provided, one will be generated for you.",
)
tokenCmd.AddCommand(createCmd)
tokenCmd.AddCommand(NewCmdTokenGenerate(out))
listCmd := &cobra.Command{
Use: "list",
Short: "List bootstrap tokens on the server.",
Run: func(tokenCmd *cobra.Command, args []string) {
err := RunListTokens(out, errW, tokenCmd)
kubeadmutil.CheckErr(err)
},
}
tokenCmd.AddCommand(listCmd)
deleteCmd := &cobra.Command{
Use: "delete",
Short: "Delete bootstrap tokens on the server.",
Run: func(tokenCmd *cobra.Command, args []string) {
err := RunDeleteToken(out, tokenCmd, args[0])
kubeadmutil.CheckErr(err)
},
}
tokenCmd.AddCommand(deleteCmd)
return tokenCmd
}
func NewCmdTokenGenerate(out io.Writer) *cobra.Command {
return &cobra.Command{
Use: "generate",
Short: "Generate and print a token suitable for use with init/join",
Short: "Generate and print a bootstrap token, but do not create it on the server.",
Long: dedent.Dedent(`
This command will print out a randomly-generated token that you can use with
This command will print out a randomly-generated bootstrap token that can be used with
the "init" and "join" commands.
You don't have to use this command in order to generate a token, you can do so
@ -74,13 +121,114 @@ func NewCmdTokenGenerate(out io.Writer) *cobra.Command {
}
}
func RunGenerateToken(out io.Writer) error {
d := &kubeadmapi.TokenDiscovery{}
err := util.GenerateToken(d)
// RunCreateToken generates a new bootstrap token and stores it as a secret on the server.
func RunCreateToken(out io.Writer, cmd *cobra.Command, tokenDuration time.Duration, token string) error {
client, err := kubemaster.CreateClientFromFile(path.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, "admin.conf"))
if err != nil {
return err
}
fmt.Fprintln(out, util.BearerToken(d))
d := &kubeadmapi.TokenDiscovery{}
if token != "" {
parsedID, parsedSecret, err := kubeadmutil.ParseToken(token)
if err != nil {
return err
}
d.ID = parsedID
d.Secret = parsedSecret
}
err = kubeadmutil.GenerateTokenIfNeeded(d)
if err != nil {
return err
}
err = kubeadmutil.UpdateOrCreateToken(client, d, tokenDuration)
if err != nil {
return err
}
fmt.Fprintln(out, kubeadmutil.BearerToken(d))
return nil
}
func RunGenerateToken(out io.Writer) error {
d := &kubeadmapi.TokenDiscovery{}
err := kubeadmutil.GenerateToken(d)
if err != nil {
return err
}
fmt.Fprintln(out, kubeadmutil.BearerToken(d))
return nil
}
// RunListTokens lists details on all existing bootstrap tokens on the server.
func RunListTokens(out io.Writer, errW io.Writer, cmd *cobra.Command) error {
client, err := kubemaster.CreateClientFromFile(path.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, "admin.conf"))
if err != nil {
return err
}
tokenSelector := fields.SelectorFromSet(
map[string]string{
api.SecretTypeField: string(api.SecretTypeBootstrapToken),
},
)
listOptions := v1.ListOptions{
FieldSelector: tokenSelector.String(),
}
results, err := client.Secrets(api.NamespaceSystem).List(listOptions)
if err != nil {
return fmt.Errorf("failed to list bootstrap tokens [%v]", err)
}
w := tabwriter.NewWriter(out, 10, 4, 3, ' ', 0)
fmt.Fprintln(w, "ID\tTOKEN\tTTL")
for _, secret := range results.Items {
tokenId, ok := secret.Data["token-id"]
if !ok {
fmt.Fprintf(errW, "[token] bootstrap token has no token-id data: %s\n", secret.Name)
continue
}
tokenSecret, ok := secret.Data["token-secret"]
if !ok {
fmt.Fprintf(errW, "[token] bootstrap token has no token-secret data: %s\n", secret.Name)
continue
}
token := fmt.Sprintf("%s.%s", tokenId, tokenSecret)
// Expiration time is optional, if not specified this implies the token
// never expires.
expires := "<never>"
secretExpiration, ok := secret.Data["expiration"]
if ok {
expireTime, err := time.Parse(time.RFC3339, string(secretExpiration))
if err != nil {
return fmt.Errorf("error parsing expiry time [%v]", err)
}
expires = kubectl.ShortHumanDuration(expireTime.Sub(time.Now()))
}
fmt.Fprintf(w, "%s\t%s\t%s\n", tokenId, token, expires)
}
w.Flush()
return nil
}
// RunDeleteToken removes a bootstrap token from the server.
func RunDeleteToken(out io.Writer, cmd *cobra.Command, tokenId string) error {
client, err := kubemaster.CreateClientFromFile(path.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, "admin.conf"))
if err != nil {
return err
}
tokenSecretName := fmt.Sprintf("%s%s", kubeadmutil.BootstrapTokenSecretPrefix, tokenId)
if err := client.Secrets(api.NamespaceSystem).Delete(tokenSecretName, nil); err != nil {
return fmt.Errorf("failed to delete bootstrap token [%v]", err)
}
fmt.Fprintf(out, "[token] bootstrap token deleted: %s\n", tokenId)
return nil
}

View File

@ -60,6 +60,7 @@ go_test(
tags = ["automanaged"],
deps = [
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
"//cmd/kubeadm/app/util:go_default_library",
"//pkg/api/v1:go_default_library",
"//pkg/util/cert:go_default_library",
"//pkg/util/intstr:go_default_library",

View File

@ -36,9 +36,9 @@ import (
const apiCallRetryInterval = 500 * time.Millisecond
func CreateClientAndWaitForAPI(adminConfig *clientcmdapi.Config) (*clientset.Clientset, error) {
func createAPIClient(adminKubeconfig *clientcmdapi.Config) (*clientset.Clientset, error) {
adminClientConfig, err := clientcmd.NewDefaultClientConfig(
*adminConfig,
*adminKubeconfig,
&clientcmd.ConfigOverrides{},
).ClientConfig()
if err != nil {
@ -49,7 +49,22 @@ func CreateClientAndWaitForAPI(adminConfig *clientcmdapi.Config) (*clientset.Cli
if err != nil {
return nil, fmt.Errorf("failed to create API client [%v]", err)
}
return client, nil
}
func CreateClientFromFile(path string) (*clientset.Clientset, error) {
adminKubeconfig, err := clientcmd.LoadFromFile(path)
if err != nil {
return nil, fmt.Errorf("failed to load admin kubeconfig [%v]", err)
}
return createAPIClient(adminKubeconfig)
}
func CreateClientAndWaitForAPI(adminConfig *clientcmdapi.Config) (*clientset.Clientset, error) {
client, err := createAPIClient(adminConfig)
if err != nil {
return nil, err
}
fmt.Println("[apiclient] Created API client, waiting for the control plane to become ready")
start := time.Now()

View File

@ -31,22 +31,6 @@ import (
"k8s.io/kubernetes/pkg/util/uuid"
)
func generateTokenIfNeeded(d *kubeadmapi.TokenDiscovery) error {
ok, err := kubeadmutil.IsTokenValid(d)
if err != nil {
return err
}
if ok {
fmt.Println("[tokens] Accepted provided token")
return nil
}
if err := kubeadmutil.GenerateToken(d); err != nil {
return err
}
fmt.Printf("[tokens] Generated token: %q\n", kubeadmutil.BearerToken(d))
return nil
}
func PrepareTokenDiscovery(d *kubeadmapi.TokenDiscovery) error {
if len(d.Addresses) == 0 {
ip, err := netutil.ChooseHostInterface()
@ -55,7 +39,7 @@ func PrepareTokenDiscovery(d *kubeadmapi.TokenDiscovery) error {
}
d.Addresses = []string{ip.String() + ":" + strconv.Itoa(kubeadmapiext.DefaultDiscoveryBindPort)}
}
if err := generateTokenIfNeeded(d); err != nil {
if err := kubeadmutil.GenerateTokenIfNeeded(d); err != nil {
return fmt.Errorf("failed to generate token(s) [%v]", err)
}
return nil

View File

@ -20,6 +20,7 @@ import (
"testing"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
)
func TestValidTokenPopulatesSecrets(t *testing.T) {
@ -31,30 +32,30 @@ func TestValidTokenPopulatesSecrets(t *testing.T) {
Secret: expectedSecret,
}
err := generateTokenIfNeeded(s)
err := kubeadmutil.GenerateTokenIfNeeded(s)
if err != nil {
t.Errorf("generateTokenIfNeeded gave an error for a valid token: %v", err)
t.Errorf("GenerateTokenIfNeeded gave an error for a valid token: %v", err)
}
if s.ID != expectedID {
t.Errorf("generateTokenIfNeeded did not populate the TokenID correctly; expected [%s] but got [%s]", expectedID, s.ID)
t.Errorf("GenerateTokenIfNeeded did not populate the TokenID correctly; expected [%s] but got [%s]", expectedID, s.ID)
}
if s.Secret != expectedSecret {
t.Errorf("generateTokenIfNeeded did not populate the Token correctly; expected %v but got %v", expectedSecret, s.Secret)
t.Errorf("GenerateTokenIfNeeded did not populate the Token correctly; expected %v but got %v", expectedSecret, s.Secret)
}
})
t.Run("not provided", func(t *testing.T) {
s := &kubeadmapi.TokenDiscovery{}
err := generateTokenIfNeeded(s)
err := kubeadmutil.GenerateTokenIfNeeded(s)
if err != nil {
t.Errorf("generateTokenIfNeeded gave an error for a valid token: %v", err)
t.Errorf("GenerateTokenIfNeeded gave an error for a valid token: %v", err)
}
if s.ID == "" {
t.Errorf("generateTokenIfNeeded did not populate the TokenID correctly; expected ID to be non-empty")
t.Errorf("GenerateTokenIfNeeded did not populate the TokenID correctly; expected ID to be non-empty")
}
if s.Secret == "" {
t.Errorf("generateTokenIfNeeded did not populate the Token correctly; expected Secret to be non-empty")
t.Errorf("GenerateTokenIfNeeded did not populate the Token correctly; expected Secret to be non-empty")
}
})
}

View File

@ -21,6 +21,11 @@ go_library(
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
"//cmd/kubeadm/app/apis/kubeadm/v1alpha1:go_default_library",
"//cmd/kubeadm/app/preflight:go_default_library",
"//pkg/api:go_default_library",
"//pkg/api/errors:go_default_library",
"//pkg/api/v1:go_default_library",
"//pkg/apis/meta/v1:go_default_library",
"//pkg/client/clientset_generated/clientset:go_default_library",
"//pkg/client/unversioned/clientcmd:go_default_library",
"//pkg/client/unversioned/clientcmd/api:go_default_library",
],

View File

@ -23,14 +23,23 @@ import (
"regexp"
"strconv"
"strings"
"time"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1"
"k8s.io/kubernetes/pkg/api"
apierrors "k8s.io/kubernetes/pkg/api/errors"
v1 "k8s.io/kubernetes/pkg/api/v1"
metav1 "k8s.io/kubernetes/pkg/apis/meta/v1"
clientset "k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
)
const (
TokenIDBytes = 3
TokenBytes = 8
TokenIDBytes = 3
TokenBytes = 8
BootstrapTokenSecretPrefix = "bootstrap-token-"
DefaultTokenDuration = time.Duration(8) * time.Hour
tokenCreateRetries = 5
)
func RandBytes(length int) (string, error) {
@ -63,6 +72,21 @@ var (
tokenRegexp = regexp.MustCompile(tokenRegexpString)
)
func GenerateTokenIfNeeded(d *kubeadmapi.TokenDiscovery) error {
ok, err := IsTokenValid(d)
if err != nil {
return err
}
if ok {
return nil
}
if err := GenerateToken(d); err != nil {
return err
}
return nil
}
func ParseToken(s string) (string, string, error) {
split := tokenRegexp.FindStringSubmatch(s)
if len(split) != 3 {
@ -99,3 +123,63 @@ func DiscoveryPort(d *kubeadmapi.TokenDiscovery) int32 {
}
return kubeadmapiext.DefaultDiscoveryBindPort
}
// UpdateOrCreateToken attempts to update a token with the given ID, or create if it does
// not already exist.
func UpdateOrCreateToken(client *clientset.Clientset, d *kubeadmapi.TokenDiscovery, tokenDuration time.Duration) error {
secretName := fmt.Sprintf("%s%s", BootstrapTokenSecretPrefix, d.ID)
var lastErr error
for i := 0; i < tokenCreateRetries; i++ {
secret, err := client.Secrets(api.NamespaceSystem).Get(secretName, metav1.GetOptions{})
if err == nil {
// Secret with this ID already exists, update it:
secret.Data = encodeTokenSecretData(d, tokenDuration)
if _, err := client.Secrets(api.NamespaceSystem).Update(secret); err == nil {
return nil
} else {
lastErr = err
}
continue
}
// Secret does not already exist:
if apierrors.IsNotFound(err) {
secret = &v1.Secret{
ObjectMeta: v1.ObjectMeta{
Name: secretName,
},
Type: api.SecretTypeBootstrapToken,
Data: encodeTokenSecretData(d, tokenDuration),
}
if _, err := client.Secrets(api.NamespaceSystem).Create(secret); err == nil {
return nil
} else {
lastErr = err
}
continue
}
}
return fmt.Errorf("<util/tokens> unable to create bootstrap token after %d attempts [%v]", tokenCreateRetries, lastErr)
}
func encodeTokenSecretData(d *kubeadmapi.TokenDiscovery, duration time.Duration) map[string][]byte {
var (
data = map[string][]byte{}
)
data["token-id"] = []byte(d.ID)
data["token-secret"] = []byte(d.Secret)
data["usage-bootstrap-signing"] = []byte("true")
if duration > 0 {
t := time.Now()
t = t.Add(duration)
data["expiration"] = []byte(t.Format(time.RFC3339))
}
return data
}

View File

@ -35,17 +35,17 @@ func init() {
}
func TestCmdTokenGenerate(t *testing.T) {
stdout, _, err := RunCmd(kubeadmPath, "token", "generate")
stdout, _, err := RunCmd(kubeadmPath, "ex", "token", "generate")
if err != nil {
t.Errorf("'kubeadm token generate' exited uncleanly: %v", err)
t.Errorf("'kubeadm ex token generate' exited uncleanly: %v", err)
}
matched, err := regexp.MatchString(TokenExpectedRegex, stdout)
if err != nil {
t.Fatalf("encountered an error while trying to match 'kubeadm token generate' stdout: %v", err)
t.Fatalf("encountered an error while trying to match 'kubeadm ex token generate' stdout: %v", err)
}
if !matched {
t.Errorf("'kubeadm token generate' stdout did not match expected regex; wanted: [%s], got: [%s]", TokenExpectedRegex, stdout)
t.Errorf("'kubeadm ex token generate' stdout did not match expected regex; wanted: [%s], got: [%s]", TokenExpectedRegex, stdout)
}
}
@ -53,15 +53,15 @@ func TestCmdTokenGenerateTypoError(t *testing.T) {
/*
Since we expect users to do things like this:
$ TOKEN=$(kubeadm token generate)
$ TOKEN=$(kubeadm ex token generate)
we want to make sure that if they have a typo in their command, we exit
with a non-zero status code after showing the command's usage, so that
the usage itself isn't captured as a token without the user noticing.
*/
_, _, err := RunCmd(kubeadmPath, "token", "genorate") // subtle typo
_, _, err := RunCmd(kubeadmPath, "ex", "token", "genorate") // subtle typo
if err == nil {
t.Error("'kubeadm token genorate' (a deliberate typo) exited without an error when we expected non-zero exit status")
t.Error("'kubeadm ex token genorate' (a deliberate typo) exited without an error when we expected non-zero exit status")
}
}

View File

@ -3249,6 +3249,9 @@ const (
// - Secret.Data["token"] - a token that identifies the service account to the API
SecretTypeServiceAccountToken SecretType = "kubernetes.io/service-account-token"
// SecretTypeBootstrapToken is the key for tokens used by kubeadm to validate cluster info during discovery.
SecretTypeBootstrapToken = "bootstrap.kubernetes.io/token"
// ServiceAccountNameKey is the key of the required annotation for SecretTypeServiceAccountToken secrets
ServiceAccountNameKey = "kubernetes.io/service-account.name"
// ServiceAccountUIDKey is the key of the required annotation for SecretTypeServiceAccountToken secrets

View File

@ -657,7 +657,7 @@ func formatEndpoints(endpoints *api.Endpoints, ports sets.String) string {
return ret
}
func shortHumanDuration(d time.Duration) string {
func ShortHumanDuration(d time.Duration) string {
// Allow deviation no more than 2 seconds(excluded) to tolerate machine time
// inconsistence, it can be considered as almost now.
if seconds := int(d.Seconds()); seconds < -1 {
@ -682,7 +682,7 @@ func translateTimestamp(timestamp metav1.Time) string {
if timestamp.IsZero() {
return "<unknown>"
}
return shortHumanDuration(time.Now().Sub(timestamp.Time))
return ShortHumanDuration(time.Now().Sub(timestamp.Time))
}
func printPodBase(pod *api.Pod, w io.Writer, options PrintOptions) error {