Merge pull request #37019 from pipejakob/e2e-anywhere-pr

Automatic merge from submit-queue

Add kubernetes-anywhere as a new e2e deployment option.

This change adds support for using `kubernetes-anywhere` as a deployment option for hack/e2e.go. This work is toward the larger goal of being able to run e2e tests against `kubeadm` clusters, which `kubernetes-anywhere` supports.

**Release note**:
```release-note
Add kubernetes-anywhere as a new e2e deployment option
```

The configuration in `getConfig()` comes mostly from the defaults in `kubernetes-anywhere`. In the future, we can add more plumbing to override them via CLI flags.

CC @mikedanese

hack/e2e.go

@@ -18,6 +18,7 @@ limitations under the License.
 package main
 
 import (
+	"bytes"
 	"encoding/xml"
 	"flag"
 	"fmt"
@@ -30,6 +31,7 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
+	"text/template"
 	"time"
 )
 
@@ -41,7 +43,7 @@ var (
 		"You can explicitly set to false if you're, e.g., testing client changes "+
 		"for which the server version doesn't make a difference.")
 	checkLeakedResources = flag.Bool("check_leaked_resources", false, "Ensure project ends with the same resources")
-	deployment           = flag.String("deployment", "bash", "up/down mechanism (defaults to cluster/kube-{up,down}.sh) (choices: bash/kops)")
+	deployment           = flag.String("deployment", "bash", "up/down mechanism (defaults to cluster/kube-{up,down}.sh) (choices: bash/kops/kubernetes-anywhere)")
 	down                 = flag.Bool("down", false, "If true, tear down the cluster before exiting.")
 	dump                 = flag.String("dump", "", "If set, dump cluster logs to this location on test or cluster-up failure")
 	kubemark             = flag.Bool("kubemark", false, "If true, run kubemark tests.")
@@ -62,12 +64,42 @@ var (
 	kopsNodes       = flag.Int("kops-nodes", 2, "(kops only) Number of nodes to create.")
 	kopsUpTimeout   = flag.Duration("kops-up-timeout", 20*time.Minute, "(kops only) Time limit between 'kops config / kops update' and a response from the Kubernetes API.")
 
+	// kubernetes-anywhere specific flags.
+	kubernetesAnywherePath           = flag.String("kubernetes-anywhere-path", "", "(kubernetes-anywhere only) Path to the kubernetes-anywhere directory. Must be set for kubernetes-anywhere.")
+	kubernetesAnywherePhase2Provider = flag.String("kubernetes-anywhere-phase2-provider", "ignition", "(kubernetes-anywhere only) Provider for phase2 bootstrapping. (Defaults to ignition).")
+	kubernetesAnywhereCluster        = flag.String("kubernetes-anywhere-cluster", "", "(kubernetes-anywhere only) Cluster name. Must be set for kubernetes-anywhere.")
+	kubernetesAnywhereUpTimeout      = flag.Duration("kubernetes-anywhere-up-timeout", 20*time.Minute, "(kubernetes-anywhere only) Time limit between starting a cluster and making a successful call to the Kubernetes API.")
+
 	// Deprecated flags.
 	deprecatedPush   = flag.Bool("push", false, "Deprecated. Does nothing.")
 	deprecatedPushup = flag.Bool("pushup", false, "Deprecated. Does nothing.")
 	deprecatedCtlCmd = flag.String("ctl", "", "Deprecated. Does nothing.")
 )
 
+const kubernetesAnywhereConfigTemplate = `
+.phase1.num_nodes=4
+.phase1.cluster_name="{{.Cluster}}"
+.phase1.cloud_provider="gce"
+
+.phase1.gce.os_image="ubuntu-1604-xenial-v20160420c"
+.phase1.gce.instance_type="n1-standard-2"
+.phase1.gce.project="{{.Project}}"
+.phase1.gce.region="us-central1"
+.phase1.gce.zone="us-central1-b"
+.phase1.gce.network="default"
+
+.phase2.installer_container="docker.io/colemickens/k8s-ignition:latest"
+.phase2.docker_registry="gcr.io/google-containers"
+.phase2.kubernetes_version="v1.4.1"
+.phase2.provider="{{.Phase2Provider}}"
+
+.phase3.run_addons=y
+.phase3.kube_proxy=y
+.phase3.dashboard=y
+.phase3.heapster=y
+.phase3.kube_dns=y
+`
+
 func appendError(errs []error, err error) []error {
 	if err != nil {
 		return append(errs, err)
@@ -426,6 +458,8 @@ func getDeployer() (deployer, error) {
 		return bash{}, nil
 	case "kops":
 		return NewKops()
+	case "kubernetes-anywhere":
+		return NewKubernetesAnywhere()
 	default:
 		return nil, fmt.Errorf("Unknown deployment strategy %q", *deployment)
 	}
@@ -542,30 +576,11 @@ func (k kops) Up() error {
 	// TODO(zmerlynn): More cluster validation. This should perhaps be
 	// added to kops and not here, but this is a fine place to loop
 	// for now.
-	for stop := time.Now().Add(*kopsUpTimeout); time.Now().Before(stop); time.Sleep(30 * time.Second) {
-		n, err := clusterSize(k)
-		if err != nil {
-			log.Printf("Can't get cluster size, sleeping: %v", err)
-			continue
-		}
-		if n < k.nodes+1 {
-			log.Printf("%d (current nodes) < %d (requested instances), sleeping", n, k.nodes+1)
-			continue
-		}
-		return nil
-	}
-	return fmt.Errorf("kops bringup timed out")
+	return waitForNodes(k, k.nodes+1, *kopsUpTimeout)
 }
 
 func (k kops) IsUp() error {
-	n, err := clusterSize(k)
-	if err != nil {
-		return err
-	}
-	if n <= 0 {
-		return fmt.Errorf("kops cluster found, but %d nodes reported", n)
-	}
-	return nil
+	return isUp(k)
 }
 
 func (k kops) SetupKubecfg() error {
@@ -595,6 +610,115 @@ func (k kops) Down() error {
 	return finishRunning("kops delete", exec.Command(k.path, "delete", "cluster", k.cluster, "--yes"))
 }
 
+type kubernetesAnywhere struct {
+	path string
+	// These are exported only because their use in the config template requires it.
+	Phase2Provider string
+	Project        string
+	Cluster        string
+}
+
+func NewKubernetesAnywhere() (*kubernetesAnywhere, error) {
+	if *kubernetesAnywherePath == "" {
+		return nil, fmt.Errorf("--kubernetes-anywhere-path is required")
+	}
+
+	if *kubernetesAnywhereCluster == "" {
+		return nil, fmt.Errorf("--kubernetes-anywhere-cluster is required")
+	}
+
+	project, ok := os.LookupEnv("PROJECT")
+	if !ok {
+		return nil, fmt.Errorf("The PROJECT environment variable is required to be set for kubernetes-anywhere")
+	}
+
+	// Set KUBERNETES_CONFORMANCE_TEST so the auth info is picked up
+	// from kubectl instead of bash inference.
+	if err := os.Setenv("KUBERNETES_CONFORMANCE_TEST", "yes"); err != nil {
+		return nil, err
+	}
+
+	k := &kubernetesAnywhere{
+		path:           *kubernetesAnywherePath,
+		Phase2Provider: *kubernetesAnywherePhase2Provider,
+		Project:        project,
+		Cluster:        *kubernetesAnywhereCluster,
+	}
+
+	if err := k.writeConfig(); err != nil {
+		return nil, err
+	}
+	return k, nil
+}
+
+func (k kubernetesAnywhere) getConfig() (string, error) {
+	// As needed, plumb through more CLI options to replace these defaults
+	tmpl, err := template.New("kubernetes-anywhere-config").Parse(kubernetesAnywhereConfigTemplate)
+
+	if err != nil {
+		return "", fmt.Errorf("Error creating template for KubernetesAnywhere config: %v", err)
+	}
+
+	var buf bytes.Buffer
+	if err = tmpl.Execute(&buf, k); err != nil {
+		return "", fmt.Errorf("Error executing template for KubernetesAnywhere config: %v", err)
+	}
+
+	return buf.String(), nil
+}
+
+func (k kubernetesAnywhere) writeConfig() error {
+	config, err := k.getConfig()
+	if err != nil {
+		return fmt.Errorf("Could not generate config: %v", err)
+	}
+
+	f, err := os.Create(k.path + "/.config")
+	if err != nil {
+		return fmt.Errorf("Could not create file: %v", err)
+	}
+	defer f.Close()
+
+	fmt.Fprint(f, config)
+	return nil
+}
+
+func (k kubernetesAnywhere) Up() error {
+	cmd := exec.Command("make", "-C", k.path, "WAIT_FOR_KUBECONFIG=y", "deploy-cluster")
+	if err := finishRunning("deploy-cluster", cmd); err != nil {
+		return err
+	}
+
+	nodes := 4 // For now, this is hardcoded in the config
+	return waitForNodes(k, nodes+1, *kubernetesAnywhereUpTimeout)
+}
+
+func (k kubernetesAnywhere) IsUp() error {
+	return isUp(k)
+}
+
+func (k kubernetesAnywhere) SetupKubecfg() error {
+	output, err := exec.Command("make", "--silent", "-C", k.path, "kubeconfig-path").Output()
+	if err != nil {
+		return fmt.Errorf("Could not get kubeconfig-path: %v", err)
+	}
+	kubecfg := strings.TrimSuffix(string(output), "\n")
+
+	if err = os.Setenv("KUBECONFIG", kubecfg); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (k kubernetesAnywhere) Down() error {
+	err := finishRunning("get kubeconfig-path", exec.Command("make", "-C", k.path, "kubeconfig-path"))
+	if err != nil {
+		// This is expected if the cluster doesn't exist.
+		return nil
+	}
+	return finishRunning("destroy-cluster", exec.Command("make", "-C", k.path, "FORCE_DESTROY=y", "destroy-cluster"))
+}
+
 func clusterSize(deploy deployer) (int, error) {
 	if err := deploy.SetupKubecfg(); err != nil {
 		return -1, err
@@ -638,6 +762,33 @@ func (e *CommandError) Error() string {
 	return fmt.Sprintf("%q: %q", exitErr.Error(), stderr)
 }
 
+func isUp(d deployer) error {
+	n, err := clusterSize(d)
+	if err != nil {
+		return err
+	}
+	if n <= 0 {
+		return fmt.Errorf("cluster found, but %d nodes reported", n)
+	}
+	return nil
+}
+
+func waitForNodes(d deployer, nodes int, timeout time.Duration) error {
+	for stop := time.Now().Add(timeout); time.Now().Before(stop); time.Sleep(30 * time.Second) {
+		n, err := clusterSize(d)
+		if err != nil {
+			log.Printf("Can't get cluster size, sleeping: %v", err)
+			continue
+		}
+		if n < nodes {
+			log.Printf("%d (current nodes) < %d (requested instances), sleeping", n, nodes)
+			continue
+		}
+		return nil
+	}
+	return fmt.Errorf("waiting for nodes timed out")
+}
+
 func DumpClusterLogs(location string) error {
 	log.Printf("Dumping cluster logs to: %v", location)
 	return finishRunning("dump cluster logs", exec.Command("./cluster/log-dump.sh", location))

hack/verify-flags/exceptions.txt

@@ -85,6 +85,9 @@ federation/deploy/config.json.sample:      "cluster_name": "cluster3-kubernetes"
 federation/deploy/config.json.sample:      "num_nodes": 3,
 federation/deploy/config.json.sample:      "num_nodes": 3,
 federation/deploy/config.json.sample:      "num_nodes": 3,
+hack/e2e.go:.phase1.cloud_provider="gce"
+hack/e2e.go:.phase1.cluster_name="{{.Cluster}}"
+hack/e2e.go:.phase1.num_nodes=4
 hack/local-up-cluster.sh:        advertise_address="--advertise_address=${API_HOST_IP}"
 hack/local-up-cluster.sh:      runtime_config="--runtime-config=${RUNTIME_CONFIG}"
 hack/local-up-cluster.sh:    advertise_address=""

hack/verify-flags/known-flags.txt

@@ -336,6 +336,10 @@ kubelet-read-only-port
 kubelet-root-dir
 kubelet-sync-frequency
 kubelet-timeout
+kubernetes-anywhere-cluster
+kubernetes-anywhere-path
+kubernetes-anywhere-phase2-provider
+kubernetes-anywhere-up-timeout
 kubernetes-service-node-port
 label-columns
 large-cluster-size-threshold