Merge pull request #75082 from neolit123/reset-clean-status

kubeadm-reset: add means to clear the ClusterStatus

cmd/kubeadm/app/cmd/reset.go

@@ -38,6 +38,7 @@ import (
 	cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	etcdphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/etcd"
+	uploadconfig "k8s.io/kubernetes/cmd/kubeadm/app/phases/uploadconfig"
 	"k8s.io/kubernetes/cmd/kubeadm/app/preflight"
 	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
 	configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
@@ -140,10 +141,15 @@ func NewReset(in io.Reader, ignorePreflightErrors sets.String, forceReset bool,
 // Run reverts any changes made to this host by "kubeadm init" or "kubeadm join".
 func (r *Reset) Run(out io.Writer, client clientset.Interface, cfg *kubeadmapi.InitConfiguration) error {
 	var dirsToClean []string
-	// Only clear etcd data when using local etcd.
-	etcdManifestPath := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.ManifestsSubDirName, "etcd.yaml")
 
+	// Reset the ClusterStatus for a given control-plane node.
+	if isControlPlane() && cfg != nil {
+		uploadconfig.ResetClusterStatusForNode(cfg.NodeRegistration.Name, client)
+	}
+
+	// Only clear etcd data when using local etcd.
 	klog.V(1).Infoln("[reset] Checking for etcd config")
+	etcdManifestPath := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.ManifestsSubDirName, "etcd.yaml")
 	etcdDataDir, err := getEtcdDataDir(etcdManifestPath, cfg)
 	if err == nil {
 		dirsToClean = append(dirsToClean, etcdDataDir)
@@ -318,3 +324,13 @@ func resetDetectCRISocket(cfg *kubeadmapi.InitConfiguration) (string, error) {
 	// if this fails, try to detect it
 	return utilruntime.DetectCRISocket()
 }
+
+// isControlPlane checks if a node is a control-plane node by looking up
+// the kube-apiserver manifest file
+func isControlPlane() bool {
+	filepath := kubeadmconstants.GetStaticPodFilepath(kubeadmconstants.KubeAPIServer, kubeadmconstants.GetStaticPodDirectory())
+	if _, err := os.Stat(filepath); os.IsNotExist(err) {
+		return false
+	}
+	return true
+}

cmd/kubeadm/app/phases/uploadconfig/BUILD

@@ -20,6 +20,8 @@ go_library(
         "//staging/src/k8s.io/api/rbac/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
+        "//vendor/github.com/pkg/errors:go_default_library",
+        "//vendor/k8s.io/klog:go_default_library",
     ],
 )
 

cmd/kubeadm/app/phases/uploadconfig/uploadconfig.go

@@ -19,10 +19,12 @@ package uploadconfig
 import (
 	"fmt"
 
-	"k8s.io/api/core/v1"
+	"github.com/pkg/errors"
+	v1 "k8s.io/api/core/v1"
 	rbac "k8s.io/api/rbac/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/klog"
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
@@ -37,6 +39,69 @@ const (
 	NodesKubeadmConfigClusterRoleName = "kubeadm:nodes-kubeadm-config"
 )
 
+// ResetClusterStatusForNode removes the APIEndpoint of a given control-plane node
+// from the ClusterStatus and updates the kubeadm ConfigMap
+func ResetClusterStatusForNode(nodeName string, client clientset.Interface) error {
+	fmt.Printf("[reset] Removing info for node %q from the ConfigMap %q in the %q Namespace\n",
+		nodeName, kubeadmconstants.KubeadmConfigConfigMap, metav1.NamespaceSystem)
+
+	// Get the kubeadm ConfigMap
+	configMap, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(kubeadmconstants.KubeadmConfigConfigMap, metav1.GetOptions{})
+	if err != nil {
+		return errors.Wrap(err, "failed to get config map")
+	}
+
+	// Handle missing ClusterConfiguration in the ConfigMap. Should only happen if someone manually
+	// interacted with the ConfigMap.
+	clusterConfigurationYaml, ok := configMap.Data[kubeadmconstants.ClusterConfigurationConfigMapKey]
+	if !ok {
+		return errors.Errorf("cannot find key %q in ConfigMap %q in the %q Namespace",
+			kubeadmconstants.ClusterConfigurationConfigMapKey, kubeadmconstants.KubeadmConfigConfigMap, metav1.NamespaceSystem)
+	}
+
+	// Obtain the existing ClusterStatus object
+	clusterStatus, err := configutil.UnmarshalClusterStatus(configMap.Data)
+	if err != nil {
+		return err
+	}
+
+	// Handle a nil APIEndpoints map. Should only happen if someone manually
+	// interacted with the ConfigMap.
+	if clusterStatus.APIEndpoints == nil {
+		return errors.Errorf("APIEndpoints from ConfigMap %q in the %q Namespace is nil",
+			kubeadmconstants.KubeadmConfigConfigMap, metav1.NamespaceSystem)
+	}
+
+	// Check for existence of the nodeName key in the list of APIEndpoints.
+	// Return early if it's missing.
+	apiEndpoint, ok := clusterStatus.APIEndpoints[nodeName]
+	if !ok {
+		klog.Warningf("No APIEndpoint registered for node %q", nodeName)
+		return nil
+	}
+
+	klog.V(2).Infof("Removing APIEndpoint %#v for node %q", apiEndpoint, nodeName)
+	delete(clusterStatus.APIEndpoints, nodeName)
+
+	// Marshal the ClusterStatus back into YAML
+	clusterStatusYaml, err := configutil.MarshalKubeadmConfigObject(clusterStatus)
+	if err != nil {
+		return err
+	}
+
+	// Update the ClusterStatus in the ConfigMap
+	return apiclient.CreateOrUpdateConfigMap(client, &v1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      kubeadmconstants.KubeadmConfigConfigMap,
+			Namespace: metav1.NamespaceSystem,
+		},
+		Data: map[string]string{
+			kubeadmconstants.ClusterConfigurationConfigMapKey: clusterConfigurationYaml,
+			kubeadmconstants.ClusterStatusConfigMapKey:        string(clusterStatusYaml),
+		},
+	})
+}
+
 // UploadConfiguration saves the InitConfiguration used for later reference (when upgrading for instance)
 func UploadConfiguration(cfg *kubeadmapi.InitConfiguration, client clientset.Interface) error {
 	fmt.Printf("[upload-config] storing the configuration used in ConfigMap %q in the %q Namespace\n", kubeadmconstants.KubeadmConfigConfigMap, metav1.NamespaceSystem)

cmd/kubeadm/app/util/config/cluster.go

@@ -167,7 +167,7 @@ func getNodeNameFromKubeletConfig(kubeconfigDir string) (string, error) {
 // getAPIEndpoint returns the APIEndpoint for the current node
 func getAPIEndpoint(data map[string]string, nodeName string, apiEndpoint *kubeadmapi.APIEndpoint) error {
 	// gets the ClusterStatus from kubeadm-config
-	clusterStatus, err := unmarshalClusterStatus(data)
+	clusterStatus, err := UnmarshalClusterStatus(data)
 	if err != nil {
 		return err
 	}
@@ -210,7 +210,7 @@ func GetClusterStatus(client clientset.Interface) (*kubeadmapi.ClusterStatus, er
 		return nil, err
 	}
 
-	clusterStatus, err := unmarshalClusterStatus(configMap.Data)
+	clusterStatus, err := UnmarshalClusterStatus(configMap.Data)
 	if err != nil {
 		return nil, err
 	}
@@ -218,7 +218,8 @@ func GetClusterStatus(client clientset.Interface) (*kubeadmapi.ClusterStatus, er
 	return clusterStatus, nil
 }
 
-func unmarshalClusterStatus(data map[string]string) (*kubeadmapi.ClusterStatus, error) {
+// UnmarshalClusterStatus takes raw ConfigMap.Data and converts it to a ClusterStatus object
+func UnmarshalClusterStatus(data map[string]string) (*kubeadmapi.ClusterStatus, error) {
 	clusterStatusData, ok := data[constants.ClusterStatusConfigMapKey]
 	if !ok {
 		return nil, errors.Errorf("unexpected error when reading kubeadm-config ConfigMap: %s key value pair missing", constants.ClusterStatusConfigMapKey)