From 4f03532f47ae9c0cafb12b75bfed181a69920983 Mon Sep 17 00:00:00 2001 From: Shylaja Devadiga Date: Wed, 28 Jul 2021 09:28:26 -0700 Subject: [PATCH] Add nightly automation tests Signed-off-by: Shylaja Devadiga --- tests/e2e/Dockerfile.build | 26 + tests/e2e/amd64_resource_files/clusterip.yaml | 33 + tests/e2e/amd64_resource_files/daemonset.yaml | 18 + tests/e2e/amd64_resource_files/dnsutils.yaml | 14 + tests/e2e/amd64_resource_files/ingress.yaml | 48 ++ .../amd64_resource_files/loadbalancer.yaml | 36 ++ .../local-path-provisioner.yaml | 32 + tests/e2e/amd64_resource_files/nodeport.yaml | 35 + tests/e2e/arm_resource_files/clusterip.yaml | 33 + tests/e2e/arm_resource_files/daemonset.yaml | 18 + tests/e2e/arm_resource_files/ingress.yaml | 48 ++ .../e2e/arm_resource_files/loadbalancer.yaml | 36 ++ .../local-path-provisioner.yaml | 32 + tests/e2e/arm_resource_files/nodeport.yaml | 35 + tests/e2e/createcluster.go | 86 +++ tests/e2e/createcluster_test.go | 227 +++++++ tests/e2e/go.mod | 11 + tests/e2e/go.sum | 608 ++++++++++++++++++ tests/e2e/scripts/build.sh | 1 + .../modules/k3scluster/install_k3s_master.sh | 106 +++ .../modules/k3scluster/join_k3s_agent.sh | 31 + .../modules/k3scluster/join_k3s_master.sh | 44 ++ .../e2e/terraform/modules/k3scluster/main.tf | 53 ++ .../k3scluster/master/instances_server.tf | 285 ++++++++ .../modules/k3scluster/master/outputs.tf | 14 + .../modules/k3scluster/master/providers.tf | 3 + .../modules/k3scluster/master/variables.tf | 31 + .../terraform/modules/k3scluster/outputs.tf | 14 + .../terraform/modules/k3scluster/providers.tf | 3 + .../terraform/modules/k3scluster/variables.tf | 31 + .../k3scluster/worker/instances_worker.tf | 49 ++ .../modules/k3scluster/worker/outputs.tf | 12 + .../modules/k3scluster/worker/providers.tf | 3 + .../modules/k3scluster/worker/variables.tf | 22 + tests/e2e/testutils.go | 233 +++++++ tests/e2e/upgradecluster_test.go | 362 +++++++++++ 36 files changed, 2673 insertions(+) create mode 100644 tests/e2e/Dockerfile.build create mode 100644 tests/e2e/amd64_resource_files/clusterip.yaml create mode 100644 tests/e2e/amd64_resource_files/daemonset.yaml create mode 100644 tests/e2e/amd64_resource_files/dnsutils.yaml create mode 100644 tests/e2e/amd64_resource_files/ingress.yaml create mode 100644 tests/e2e/amd64_resource_files/loadbalancer.yaml create mode 100644 tests/e2e/amd64_resource_files/local-path-provisioner.yaml create mode 100644 tests/e2e/amd64_resource_files/nodeport.yaml create mode 100644 tests/e2e/arm_resource_files/clusterip.yaml create mode 100644 tests/e2e/arm_resource_files/daemonset.yaml create mode 100644 tests/e2e/arm_resource_files/ingress.yaml create mode 100644 tests/e2e/arm_resource_files/loadbalancer.yaml create mode 100644 tests/e2e/arm_resource_files/local-path-provisioner.yaml create mode 100644 tests/e2e/arm_resource_files/nodeport.yaml create mode 100644 tests/e2e/createcluster.go create mode 100644 tests/e2e/createcluster_test.go create mode 100644 tests/e2e/go.mod create mode 100644 tests/e2e/go.sum create mode 100755 tests/e2e/scripts/build.sh create mode 100644 tests/e2e/terraform/modules/k3scluster/install_k3s_master.sh create mode 100644 tests/e2e/terraform/modules/k3scluster/join_k3s_agent.sh create mode 100644 tests/e2e/terraform/modules/k3scluster/join_k3s_master.sh create mode 100644 tests/e2e/terraform/modules/k3scluster/main.tf create mode 100644 tests/e2e/terraform/modules/k3scluster/master/instances_server.tf create mode 100644 tests/e2e/terraform/modules/k3scluster/master/outputs.tf create mode 100644 tests/e2e/terraform/modules/k3scluster/master/providers.tf create mode 100644 tests/e2e/terraform/modules/k3scluster/master/variables.tf create mode 100644 tests/e2e/terraform/modules/k3scluster/outputs.tf create mode 100644 tests/e2e/terraform/modules/k3scluster/providers.tf create mode 100644 tests/e2e/terraform/modules/k3scluster/variables.tf create mode 100644 tests/e2e/terraform/modules/k3scluster/worker/instances_worker.tf create mode 100644 tests/e2e/terraform/modules/k3scluster/worker/outputs.tf create mode 100644 tests/e2e/terraform/modules/k3scluster/worker/providers.tf create mode 100644 tests/e2e/terraform/modules/k3scluster/worker/variables.tf create mode 100644 tests/e2e/testutils.go create mode 100644 tests/e2e/upgradecluster_test.go diff --git a/tests/e2e/Dockerfile.build b/tests/e2e/Dockerfile.build new file mode 100644 index 0000000000..97cfcc7db4 --- /dev/null +++ b/tests/e2e/Dockerfile.build @@ -0,0 +1,26 @@ +FROM golang:alpine + +ARG TERRAFORM_VERSION=0.12.10 +ENV TERRAFORM_VERSION=$TERRAFORM_VERSION + + +RUN apk update && \ + apk upgrade --update-cache --available && \ + apk add curl git jq bash openssh unzip gcc g++ make ca-certificates && \ + curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh + +RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl && \ + chmod +x ./kubectl && \ + mv ./kubectl /usr/local/bin +RUN mkdir tmp && \ + curl "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip" -o tmp/terraform.zip && \ + unzip tmp/terraform.zip -d /usr/local/bin && \ + chmod +x /usr/local/bin/terraform && \ + rm -rf tmp + +WORKDIR $GOPATH/src/github.com/rancher/k3s-io/k3s/tests/e2e + +COPY . . + +RUN go get -u github.com/onsi/gomega +RUN go get -u github.com/onsi/ginkgo diff --git a/tests/e2e/amd64_resource_files/clusterip.yaml b/tests/e2e/amd64_resource_files/clusterip.yaml new file mode 100644 index 0000000000..f1c501e8fa --- /dev/null +++ b/tests/e2e/amd64_resource_files/clusterip.yaml @@ -0,0 +1,33 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-clusterip +spec: + selector: + matchLabels: + k8s-app: nginx-app-clusterip + replicas: 2 + template: + metadata: + labels: + k8s-app: nginx-app-clusterip + spec: + containers: + - name: nginx + image: ranchertest/mytestcontainer + ports: + - containerPort: 80 +--- +apiVersion: v1 +kind: Service +metadata: + labels: + k8s-app: nginx-app-clusterip + name: nginx-clusterip-svc + namespace: default +spec: + type: ClusterIP + ports: + - port: 80 + selector: + k8s-app: nginx-app-clusterip diff --git a/tests/e2e/amd64_resource_files/daemonset.yaml b/tests/e2e/amd64_resource_files/daemonset.yaml new file mode 100644 index 0000000000..3360f35421 --- /dev/null +++ b/tests/e2e/amd64_resource_files/daemonset.yaml @@ -0,0 +1,18 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: test-daemonset +spec: + selector: + matchLabels: + k8s-app: test-daemonset + template: + metadata: + labels: + k8s-app: test-daemonset + spec: + containers: + - name: webserver + image: nginx + ports: + - containerPort: 80 diff --git a/tests/e2e/amd64_resource_files/dnsutils.yaml b/tests/e2e/amd64_resource_files/dnsutils.yaml new file mode 100644 index 0000000000..9192bf3502 --- /dev/null +++ b/tests/e2e/amd64_resource_files/dnsutils.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Pod +metadata: + name: dnsutils + namespace: default +spec: + containers: + - name: dnsutils + image: gcr.io/kubernetes-e2e-test-images/dnsutils:1.3 + command: + - sleep + - "3600" + imagePullPolicy: IfNotPresent + restartPolicy: Always diff --git a/tests/e2e/amd64_resource_files/ingress.yaml b/tests/e2e/amd64_resource_files/ingress.yaml new file mode 100644 index 0000000000..957e7ad005 --- /dev/null +++ b/tests/e2e/amd64_resource_files/ingress.yaml @@ -0,0 +1,48 @@ +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: ingress +spec: + rules: + - host: foo1.bar.com + http: + paths: + - path: /name.html + backend: + serviceName: nginx-ingress-svc + servicePort: 80 +--- +apiVersion: v1 +kind: Service +metadata: + name: nginx-ingress-svc + labels: + k8s-app: nginx-app-ingress +spec: + ports: + - port: 80 + targetPort: 80 + protocol: TCP + name: http + selector: + k8s-app: nginx-app-ingress +--- +apiVersion: v1 +kind: ReplicationController +metadata: + name: test-ingress +spec: + replicas: 2 + selector: + k8s-app: nginx-app-ingress + template: + metadata: + labels: + k8s-app: nginx-app-ingress + spec: + terminationGracePeriodSeconds: 60 + containers: + - name: testcontainer + image: ranchertest/mytestcontainer + ports: + - containerPort: 80 diff --git a/tests/e2e/amd64_resource_files/loadbalancer.yaml b/tests/e2e/amd64_resource_files/loadbalancer.yaml new file mode 100644 index 0000000000..0897d215ae --- /dev/null +++ b/tests/e2e/amd64_resource_files/loadbalancer.yaml @@ -0,0 +1,36 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-loadbalancer +spec: + selector: + matchLabels: + k8s-app: nginx-app-loadbalancer + replicas: 2 + template: + metadata: + labels: + k8s-app: nginx-app-loadbalancer + spec: + containers: + - name: nginx + image: ranchertest/mytestcontainer + ports: + - containerPort: 80 +--- +apiVersion: v1 +kind: Service +metadata: + name: nginx-loadbalancer-svc + labels: + k8s-app: nginx-app-loadbalancer +spec: + type: LoadBalancer + ports: + - port: 81 + targetPort: 80 + protocol: TCP + name: http + selector: + k8s-app: nginx-app-loadbalancer diff --git a/tests/e2e/amd64_resource_files/local-path-provisioner.yaml b/tests/e2e/amd64_resource_files/local-path-provisioner.yaml new file mode 100644 index 0000000000..8e2b01ab8f --- /dev/null +++ b/tests/e2e/amd64_resource_files/local-path-provisioner.yaml @@ -0,0 +1,32 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: local-path-pvc + namespace: default +spec: + accessModes: + - ReadWriteOnce + storageClassName: local-path + resources: + requests: + storage: 500Mi +--- +apiVersion: v1 +kind: Pod +metadata: + name: volume-test + namespace: default +spec: + containers: + - name: volume-test + image: nginx:stable-alpine + imagePullPolicy: IfNotPresent + volumeMounts: + - name: volv + mountPath: /data + ports: + - containerPort: 80 + volumes: + - name: volv + persistentVolumeClaim: + claimName: local-path-pvc diff --git a/tests/e2e/amd64_resource_files/nodeport.yaml b/tests/e2e/amd64_resource_files/nodeport.yaml new file mode 100644 index 0000000000..2187b732db --- /dev/null +++ b/tests/e2e/amd64_resource_files/nodeport.yaml @@ -0,0 +1,35 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-nodeport +spec: + selector: + matchLabels: + k8s-app: nginx-app-nodeport + replicas: 2 + template: + metadata: + labels: + k8s-app: nginx-app-nodeport + spec: + containers: + - name: nginx + image: ranchertest/mytestcontainer + ports: + - containerPort: 80 +--- +apiVersion: v1 +kind: Service +metadata: + labels: + k8s-app: nginx-app-nodeport + name: nginx-nodeport-svc + namespace: default +spec: + type: NodePort + ports: + - port: 80 + nodePort: 30096 + name: http + selector: + k8s-app: nginx-app-nodeport diff --git a/tests/e2e/arm_resource_files/clusterip.yaml b/tests/e2e/arm_resource_files/clusterip.yaml new file mode 100644 index 0000000000..ae5efe6a1e --- /dev/null +++ b/tests/e2e/arm_resource_files/clusterip.yaml @@ -0,0 +1,33 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-clusterip +spec: + selector: + matchLabels: + k8s-app: nginx-app-clusterip + replicas: 2 + template: + metadata: + labels: + k8s-app: nginx-app-clusterip + spec: + containers: + - name: nginx + image: shylajarancher19/shylajaarm64:v1.0 + ports: + - containerPort: 80 +--- +apiVersion: v1 +kind: Service +metadata: + labels: + k8s-app: nginx-app-clusterip + name: nginx-clusterip-svc + namespace: default +spec: + type: ClusterIP + ports: + - port: 80 + selector: + k8s-app: nginx-app-clusterip diff --git a/tests/e2e/arm_resource_files/daemonset.yaml b/tests/e2e/arm_resource_files/daemonset.yaml new file mode 100644 index 0000000000..3360f35421 --- /dev/null +++ b/tests/e2e/arm_resource_files/daemonset.yaml @@ -0,0 +1,18 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: test-daemonset +spec: + selector: + matchLabels: + k8s-app: test-daemonset + template: + metadata: + labels: + k8s-app: test-daemonset + spec: + containers: + - name: webserver + image: nginx + ports: + - containerPort: 80 diff --git a/tests/e2e/arm_resource_files/ingress.yaml b/tests/e2e/arm_resource_files/ingress.yaml new file mode 100644 index 0000000000..67adff28e9 --- /dev/null +++ b/tests/e2e/arm_resource_files/ingress.yaml @@ -0,0 +1,48 @@ +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: ingress +spec: + rules: + - host: foo1.bar.com + http: + paths: + - path: /name.html + backend: + serviceName: nginx-ingress-svc + servicePort: 80 +--- +apiVersion: v1 +kind: Service +metadata: + name: nginx-ingress-svc + labels: + k8s-app: nginx-app-ingress +spec: + ports: + - port: 80 + targetPort: 80 + protocol: TCP + name: http + selector: + k8s-app: nginx-app-ingress +--- +apiVersion: v1 +kind: ReplicationController +metadata: + name: test-ingress +spec: + replicas: 2 + selector: + k8s-app: nginx-app-ingress + template: + metadata: + labels: + k8s-app: nginx-app-ingress + spec: + terminationGracePeriodSeconds: 60 + containers: + - name: testcontainer + image: shylajarancher19/shylajaarm64:v1.0 + ports: + - containerPort: 80 diff --git a/tests/e2e/arm_resource_files/loadbalancer.yaml b/tests/e2e/arm_resource_files/loadbalancer.yaml new file mode 100644 index 0000000000..6cc02bd898 --- /dev/null +++ b/tests/e2e/arm_resource_files/loadbalancer.yaml @@ -0,0 +1,36 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-loadbalancer +spec: + selector: + matchLabels: + k8s-app: nginx-app-loadbalancer + replicas: 2 + template: + metadata: + labels: + k8s-app: nginx-app-loadbalancer + spec: + containers: + - name: nginx + image: shylajarancher19/shylajaarm64:v1.0 + ports: + - containerPort: 80 +--- +apiVersion: v1 +kind: Service +metadata: + name: nginx-loadbalancer-svc + labels: + k8s-app: nginx-app-loadbalancer +spec: + type: LoadBalancer + ports: + - port: 81 + targetPort: 80 + protocol: TCP + name: http + selector: + k8s-app: nginx-app-loadbalancer diff --git a/tests/e2e/arm_resource_files/local-path-provisioner.yaml b/tests/e2e/arm_resource_files/local-path-provisioner.yaml new file mode 100644 index 0000000000..8e2b01ab8f --- /dev/null +++ b/tests/e2e/arm_resource_files/local-path-provisioner.yaml @@ -0,0 +1,32 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: local-path-pvc + namespace: default +spec: + accessModes: + - ReadWriteOnce + storageClassName: local-path + resources: + requests: + storage: 500Mi +--- +apiVersion: v1 +kind: Pod +metadata: + name: volume-test + namespace: default +spec: + containers: + - name: volume-test + image: nginx:stable-alpine + imagePullPolicy: IfNotPresent + volumeMounts: + - name: volv + mountPath: /data + ports: + - containerPort: 80 + volumes: + - name: volv + persistentVolumeClaim: + claimName: local-path-pvc diff --git a/tests/e2e/arm_resource_files/nodeport.yaml b/tests/e2e/arm_resource_files/nodeport.yaml new file mode 100644 index 0000000000..8978efaebb --- /dev/null +++ b/tests/e2e/arm_resource_files/nodeport.yaml @@ -0,0 +1,35 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-nodeport +spec: + selector: + matchLabels: + k8s-app: nginx-app-nodeport + replicas: 2 + template: + metadata: + labels: + k8s-app: nginx-app-nodeport + spec: + containers: + - name: nginx + image: shylajarancher19/shylajaarm64:v1.0 + ports: + - containerPort: 80 +--- +apiVersion: v1 +kind: Service +metadata: + labels: + k8s-app: nginx-app-nodeport + name: nginx-nodeport-svc + namespace: default +spec: + type: NodePort + ports: + - port: 80 + nodePort: 30096 + name: http + selector: + k8s-app: nginx-app-nodeport diff --git a/tests/e2e/createcluster.go b/tests/e2e/createcluster.go new file mode 100644 index 0000000000..c418233d2f --- /dev/null +++ b/tests/e2e/createcluster.go @@ -0,0 +1,86 @@ +package e2e + +import ( + "flag" + "fmt" + "io/ioutil" + "log" + "path/filepath" + "testing" + + "github.com/gruntwork-io/terratest/modules/terraform" +) + +var destroy = flag.Bool("destroy", false, "a bool") +var nodeOs = flag.String("node_os", "centos8", "a string") +var externalDb = flag.String("external_db", "mysql", "a string") +var arch = flag.String("arch", "amd64", "a string") +var clusterType = flag.String("cluster_type", "etcd", "a string") +var resourceName = flag.String("resource_name", "etcd", "a string") +var sshuser = flag.String("sshuser", "ubuntu", "a string") +var sshkey = flag.String("sshkey", "", "a string") + +var ( + kubeconfig string + masterIPs string + workerIPs string +) + +func DeployWorkloads(arch, Kubeconfig string) { + resource_dir := "" + if arch == "amd64" { + resource_dir = "./amd64_resource_files" + } else { + resource_dir = "./arm_resource_files" + } + + files, err := ioutil.ReadDir(resource_dir) + if err != nil { + log.Fatal(err) + } + + for _, f := range files { + workload := filepath.Join(resource_dir, f.Name()) + _, _ = DeployWorkload(workload, Kubeconfig) + } +} + +// nodeOs: ubuntu centos7 centos8 sles15 +// clusterType arm, etcd externaldb, if external_db var is not "" picks database from the vars file, +// resourceName: name to resource created timestamp attached + +func BuildCluster(nodeOs, clusterType, externalDb, resourceName string, t *testing.T, destroy bool) (string, string, string) { + + tDir := "./terraform/modules/k3scluster" + vDir := "/config/" + nodeOs + clusterType + ".tfvars" + + if externalDb != "" { + vDir = "/config/" + nodeOs + externalDb + ".tfvars" + } + + tfDir, _ := filepath.Abs(tDir) + varDir, _ := filepath.Abs(vDir) + TerraformOptions := &terraform.Options{ + TerraformDir: tfDir, + VarFiles: []string{varDir}, + Vars: map[string]interface{}{ + "cluster_type": clusterType, + "resource_name": resourceName, + "external_db": externalDb, + }, + } + + if destroy { + fmt.Printf("Cluster is being deleted") + terraform.Destroy(t, TerraformOptions) + return "", "", "" + } + + fmt.Printf("Creating Cluster") + terraform.InitAndApply(t, TerraformOptions) + kubeconfig := terraform.Output(t, TerraformOptions, "kubeconfig") + "_kubeconfig" + masterIps := terraform.Output(t, TerraformOptions, "master_ips") + workerIps := terraform.Output(t, TerraformOptions, "worker_ips") + kubeconfigFile := "/config/" + kubeconfig + return kubeconfigFile, masterIps, workerIps +} diff --git a/tests/e2e/createcluster_test.go b/tests/e2e/createcluster_test.go new file mode 100644 index 0000000000..8166a416d2 --- /dev/null +++ b/tests/e2e/createcluster_test.go @@ -0,0 +1,227 @@ +package e2e + +import ( + "fmt" + + . "github.com/onsi/ginkgo" + "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" + + "strings" + "testing" + "time" +) + +func Test_E2EClusterCreateValidation(t *testing.T) { + junitReporter := reporters.NewJUnitReporter(fmt.Sprintf("/config/" + *resourceName + ".xml")) + RegisterFailHandler(Fail) + RunSpecsWithDefaultAndCustomReporters(t, "Test Suite", []Reporter{junitReporter}) + +} + +var _ = Describe("Test:", func() { + Context("Build Cluster:", func() { + Context("Cluster Configuration: OS: "+*nodeOs+" Cluster Type; "+*externalDb+" "+*clusterType, func() { + + kubeconfig, masterIPs, workerIPs = BuildCluster(*nodeOs, *clusterType, *externalDb, *resourceName, &testing.T{}, *destroy) + defer GinkgoRecover() + if *destroy { + fmt.Printf("\nCluster is being Deleted\n") + return + } + fmt.Println("\nCLUSTER CONFIG:\nOS", *nodeOs, "BACKEND", *clusterType, *externalDb) + fmt.Printf("\nIPs:\n") + fmt.Println("Master Node IPS:", masterIPs) + fmt.Println("Worker Node IPS:", workerIPs) + + fmt.Printf("\nFetching node status\n") + nodes := ParseNode(kubeconfig, true) + for _, config := range nodes { + Expect(config.Status).Should(Equal("Ready"), func() string { return config.Name }) + } + + fmt.Printf("\nFetching Pods status\n") + pods := ParsePod(kubeconfig, true) + for _, pod := range pods { + if strings.Contains(pod.Name, "helm-install") { + Expect(pod.Status).Should(Equal("Completed"), func() string { return pod.Name }) + } else { + Expect(pod.Status).Should(Equal("Running"), func() string { return pod.Name }) + } + } + }) + Context("Validate Rebooting nodes", func() { + if *destroy { + return + } + defer GinkgoRecover() + nodeExternalIP := FetchNodeExternalIP(kubeconfig) + for _, ip := range nodeExternalIP { + fmt.Println("\nRebooting node: ", ip) + cmd := "ssh -i " + *sshkey + " -o \"StrictHostKeyChecking no\" " + *sshuser + "@" + ip + " sudo reboot" + _, _ = RunCommand(cmd) + time.Sleep(3 * time.Minute) + fmt.Println("\nNode and Pod Status after rebooting node: ", ip) + nodes := ParseNode(kubeconfig, true) + for _, config := range nodes { + Expect(config.Status).Should(Equal("Ready"), func() string { return config.Name }) + } + + pods := ParsePod(kubeconfig, true) + for _, pod := range pods { + if strings.Contains(pod.Name, "helm-install") { + Expect(pod.Status).Should(Equal("Completed"), func() string { return pod.Name }) + } else { + Expect(pod.Status).Should(Equal("Running"), func() string { return pod.Name }) + } + } + } + }) + + Context("Deploy workloads ", func() { + if *destroy { + return + } + defer GinkgoRecover() + + It("Validate Cluster IP", func() { + if *destroy { + return + } + DeployWorkloads(*arch, kubeconfig) + fmt.Println("Validating ClusterIP") + clusterip := FetchClusterIP(kubeconfig, "nginx-clusterip-svc") + cmd := "curl -L --insecure http://" + clusterip + "/name.html" + fmt.Println(cmd) + //Fetch External IP to login to node and validate cluster ip + nodeExternalIP := FetchNodeExternalIP(kubeconfig) + + for _, ip := range nodeExternalIP { + res := RunCmdOnNode(cmd, ip, *sshuser, *sshkey) + fmt.Println(res) + Expect(res).Should(ContainSubstring("test-clusterip"), func() string { return res }) + } + }) + + It("Validate NodePort", func() { + if *destroy { + return + } + fmt.Println("Validating NodePort") + nodeExternalIP := FetchNodeExternalIP(kubeconfig) + cmd := "kubectl get service nginx-nodeport-svc --kubeconfig=" + kubeconfig + " --output jsonpath=\"{.spec.ports[0].nodePort}\"" + nodeport, _ := RunCommand(cmd) + for _, nodeExternalIp := range nodeExternalIP { + cmd := "curl -L --insecure http://" + nodeExternalIp + ":" + nodeport + "/name.html" + fmt.Println(cmd) + res, _ := RunCommand(cmd) + fmt.Println(res) + Expect(res).Should(ContainSubstring("test-nodeport"), func() string { return res }) + } + }) + + It("Validate LoadBalancer", func() { + if *destroy { + return + } + fmt.Println("Validating Service LoadBalancer") + nodeExternalIP := FetchNodeExternalIP(kubeconfig) + cmd := "kubectl get service nginx-loadbalancer-svc --kubeconfig=" + kubeconfig + " --output jsonpath=\"{.spec.ports[0].port}\"" + port, _ := RunCommand(cmd) + for _, ip := range nodeExternalIP { + cmd = "curl -L --insecure http://" + ip + ":" + port + "/name.html" + fmt.Println(cmd) + res, _ := RunCommand(cmd) + fmt.Println(res) + Expect(res).Should(ContainSubstring("test-loadbalancer"), func() string { return res }) + } + }) + + It("Validate Daemonset", func() { + if *destroy { + return + } + fmt.Println("Validating Daemonset") + nodes := ParseNode(kubeconfig, false) + pods := ParsePod(kubeconfig, false) + count := CountOfStringInSlice("test-daemonset", pods) + fmt.Println("POD COUNT") + fmt.Println(count) + fmt.Println("NODE COUNT") + fmt.Println(len(nodes)) + Eventually(len(nodes)).Should((Equal(count)), "120s", "60s") + }) + + It("Validate Ingress", func() { + if *destroy { + return + } + fmt.Println("Validating Ingress") + + ingressIps := FetchIngressIP(kubeconfig) + for _, ip := range ingressIps { + cmd := "curl --header host:foo1.bar.com" + " http://" + ip + "/name.html" + fmt.Println(cmd) + //Access path from outside node + res, _ := RunCommand(cmd) + fmt.Println(res) + Eventually(res).Should((ContainSubstring("test-ingress")), "120s", "60s", func() string { return res }) + } + }) + It("Validate Local Path Provisioner storage ", func() { + if *destroy { + return + } + fmt.Println("Validating Local Path Provisioner") + cmd := "kubectl get pvc local-path-pvc --kubeconfig=" + kubeconfig + res, _ := RunCommand(cmd) + fmt.Println(res) + Eventually(res).Should((ContainSubstring("local-path-pvc")), "120s", "60s") + Eventually(res).Should((ContainSubstring("Bound")), "120s", "60s") + + cmd = "kubectl get pod volume-test --kubeconfig=" + kubeconfig + res, _ = RunCommand(cmd) + fmt.Println(res) + + Eventually(res).Should((ContainSubstring("volume-test")), "120s", "60s", func() string { return res }) + + cmd = "kubectl --kubeconfig=" + kubeconfig + " exec volume-test -- sh -c 'echo local-path-test > /data/test'" + fmt.Println(cmd) + res, _ = RunCommand(cmd) + fmt.Println(res) + fmt.Println("Data stored", res) + + cmd = "kubectl delete pod volume-test --kubeconfig=" + kubeconfig + res, _ = RunCommand(cmd) + fmt.Println(res) + resource_dir := "./amd64_resource_files" + cmd = "kubectl apply -f " + resource_dir + "/local-path-provisioner.yaml --kubeconfig=" + kubeconfig + res, _ = RunCommand(cmd) + fmt.Println(res) + + time.Sleep(1 * time.Minute) + cmd = "kubectl exec volume-test cat /data/test --kubeconfig=" + kubeconfig + res, _ = RunCommand(cmd) + fmt.Println("Data after re-creation", res) + + Eventually(res).Should((ContainSubstring("local-path-test")), "120s", "60s", func() string { return res }) + }) + It("Validate DNS Resolution", func() { + if *destroy { + return + } + fmt.Println("Validating DNS Resolution") + cmd := "kubectl --kubeconfig=" + kubeconfig + " exec -i -t dnsutils -- nslookup kubernetes.default" + fmt.Println(cmd) + res, _ := RunCommand(cmd) + fmt.Println("Result", res) + Eventually(res).ShouldNot((ContainSubstring("nslookup")), "120s", "60s") + }) + }) + }) + +}) + +var _ = AfterSuite(func() { + kubeconfig, masterIPs, workerIPs = BuildCluster(*nodeOs, *clusterType, *externalDb, *resourceName, &testing.T{}, true) +}) diff --git a/tests/e2e/go.mod b/tests/e2e/go.mod new file mode 100644 index 0000000000..4a9a1e1d1e --- /dev/null +++ b/tests/e2e/go.mod @@ -0,0 +1,11 @@ +module github.com/rancher/k3s/tests/e2e + +go 1.15 + +require ( + github.com/gruntwork-io/terratest v0.30.23 + github.com/onsi/ginkgo v1.14.2 + github.com/onsi/gomega v1.10.3 + github.com/stretchr/testify v1.6.1 // indirect + golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a +) diff --git a/tests/e2e/go.sum b/tests/e2e/go.sum new file mode 100644 index 0000000000..c5f386190f --- /dev/null +++ b/tests/e2e/go.sum @@ -0,0 +1,608 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.51.0/go.mod h1:hWtGJ6gnXH+KgDv+V0zFGDvpi07n3z8ZNj3T1RW0Gcw= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/azure-sdk-for-go v35.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go v38.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go v46.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= +github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= +github.com/Azure/go-autorest/autorest v0.9.3/go.mod h1:GsRuLYvwzLjjjRoWEIyMUaYq8GNUx2nRB378IPt/1p0= +github.com/Azure/go-autorest/autorest v0.9.6/go.mod h1:/FALq9T/kS7b5J5qsQ+RSTUdAmGFqi0vUdVNNx8q630= +github.com/Azure/go-autorest/autorest v0.11.0/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw= +github.com/Azure/go-autorest/autorest v0.11.5/go.mod h1:foo3aIXRQ90zFve3r0QiDsrjGDUwWhKl0ZOQy1CT14k= +github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= +github.com/Azure/go-autorest/autorest/adal v0.8.0/go.mod h1:Z6vX6WXXuyieHAXwMj0S6HY6e6wcHn37qQMBQlvY3lc= +github.com/Azure/go-autorest/autorest/adal v0.8.1/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= +github.com/Azure/go-autorest/autorest/adal v0.8.2/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= +github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg= +github.com/Azure/go-autorest/autorest/adal v0.9.2/go.mod h1:/3SMAM86bP6wC9Ev35peQDUeqFZBMH07vvUOmg4z/fE= +github.com/Azure/go-autorest/autorest/azure/auth v0.5.1/go.mod h1:ea90/jvmnAwDrSooLH4sRIehEPtG/EPUXavDh31MnA4= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.0/go.mod h1:JljT387FplPzBA31vUcvsetLKF3pec5bdAxjVU4kI2s= +github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= +github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g= +github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= +github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/autorest/to v0.2.0/go.mod h1:GunWKJp1AEqgMaGLV+iocmRAJWqST1wQYhyyjXJ3SJc= +github.com/Azure/go-autorest/autorest/to v0.3.0/go.mod h1:MgwOyqaIuKdG4TL/2ywSsIWKAfJfgHDo8ObuUk3t5sA= +github.com/Azure/go-autorest/autorest/validation v0.1.0/go.mod h1:Ha3z/SqBeaalWQvokg3NZAlQTalVMtOIAs1aGK7G6u8= +github.com/Azure/go-autorest/autorest/validation v0.3.0/go.mod h1:yhLgjC0Wda5DYXl6JAsWyUe4KVNffhoDhG0zVzUMo3E= +github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= +github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= +github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/GoogleCloudPlatform/k8s-cloud-provider v0.0.0-20190822182118-27a4ced34534/go.mod h1:iroGtC8B3tQiqtds1l+mgk/BBOrxbqjH+eUfFQYRc14= +github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= +github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= +github.com/aws/aws-sdk-go v1.16.26/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.27.1/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/containerd/containerd v1.3.0/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= +github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= +github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/cli v0.0.0-20200109221225-a4f60165b7a3/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y= +github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= +github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= +github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/elazarl/goproxy v0.0.0-20190911111923-ecfe977594f1/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= +github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8= +github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/go-errors/errors v1.0.2-0.20180813162953-d98b870cc4e0/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= +github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= +github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= +github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= +github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= +github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= +github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= +github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-containerregistry v0.0.0-20200110202235-f4fb41bf00a3/go.mod h1:2wIuQute9+hhWqvL3vEI7YB0EKluF4WcPzI1eAliazk= +github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/googleapis/gnostic v0.2.2/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= +github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= +github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/gruntwork-io/gruntwork-cli v0.7.0/go.mod h1:jp6Z7NcLF2avpY8v71fBx6hds9eOFPELSuD/VPv7w00= +github.com/gruntwork-io/terratest v0.30.23 h1:9iGmn2kL9hnchriqtSm66BGK67pYJl6SU4iOIjfD9f8= +github.com/gruntwork-io/terratest v0.30.23/go.mod h1:EEgJie28gX/4AD71IFqgMj6e99KP5mi81hEtzmDjxTo= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.7/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a h1:zPPuIq2jAWWPTrGt70eK/BSch+gFAGrNzecsoENgu2o= +github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a/go.mod h1:yL958EeXv8Ylng6IfnvG4oflryUi3vgA3xPs9hmII1s= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/joefitzgerald/rainbow-reporter v0.1.0/go.mod h1:481CNgqmVHQZzdIbN52CupLJyoVwB10FQ/IQlF1pdL8= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= +github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-zglob v0.0.1/go.mod h1:9fxibJccNxU2cnpIKLRRFA7zX7qhkJIQWBb449FYHOo= +github.com/mattn/go-zglob v0.0.2-0.20190814121620-e3c945676326/go.mod h1:9fxibJccNxU2cnpIKLRRFA7zX7qhkJIQWBb449FYHOo= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY= +github.com/miekg/dns v1.1.31/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= +github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.14.2 h1:8mVmC9kjFFmA8H4pKMUhcblgifdkOIXPvbhN1T36q1M= +github.com/onsi/ginkgo v1.14.2/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.10.3 h1:gph6h/qe9GSUw1NhH1gp+qb+h8rXD8Cy60Z32Qw3ELA= +github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= +github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/oracle/oci-go-sdk v7.1.0+incompatible/go.mod h1:VQb79nF8Z2cwLkLS35ukwStZIg5F66tcBccjip/j888= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= +github.com/pquerna/otp v1.2.0/go.mod h1:dkJfzwRKNiegxyNb54X/3fLwhCynbMspSyWKnvi1AEg= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rubiojr/go-vhd v0.0.0-20160810183302-0bfd3b39853c/go.mod h1:DM5xW0nvfNNm2uytzsvhI3OnX8uzaRAg8UX/CnDqbto= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/vdemeester/k8s-pkg-credentialprovider v0.0.0-20200107171650-7c61ffa44238/go.mod h1:JwQJCMWpUDqjZrB5jpw0f5VbN7U95zxFy1ZDpoEarGo= +github.com/vmware/govmomi v0.20.3/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a h1:vclmkQCjlDX5OydZ9wv8rBCcS0QyQY66Mpf/7BZbInM= +golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0 h1:wBouT66WTYFXdxfVdz9sVWARVd/2vfGcmI45D2gj45M= +golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190706070813-72ffa07ba3db/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191205215504-7b8c8591a921/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200113040837-eac381796e91/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e/go.mod h1:kS+toOQn6AQKjmKJ7gzohV1XkqsFehRA2FbsbkopSuQ= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.6.1-0.20190607001116-5213b8090861/go.mod h1:btoxGiFvQNVUZQ8W08zLtrVS08CNpINPEfxXxgJL1Q4= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEGA= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/gcfg.v1 v1.2.0/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/warnings.v0 v0.1.1/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +k8s.io/api v0.17.0/go.mod h1:npsyOePkeP0CPwyGfXDHxvypiYMJxBWAMpQxCaJ4ZxI= +k8s.io/api v0.19.3/go.mod h1:VF+5FT1B74Pw3KxMdKyinLo+zynBaMBiAfGMuldcNDs= +k8s.io/apimachinery v0.17.0/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= +k8s.io/apimachinery v0.19.3/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA= +k8s.io/apiserver v0.17.0/go.mod h1:ABM+9x/prjINN6iiffRVNCBR2Wk7uY4z+EtEGZD48cg= +k8s.io/client-go v0.17.0/go.mod h1:TYgR6EUHs6k45hb6KWjVD6jFZvJV4gHDikv/It0xz+k= +k8s.io/client-go v0.19.3/go.mod h1:+eEMktZM+MG0KO+PTkci8xnbCZHvj9TqR6Q1XDUIJOM= +k8s.io/cloud-provider v0.17.0/go.mod h1:Ze4c3w2C0bRsjkBUoHpFi+qWe3ob1wI2/7cUn+YQIDE= +k8s.io/code-generator v0.0.0-20191121015212-c4c8f8345c7e/go.mod h1:DVmfPQgxQENqDIzVR2ddLXMH34qeszkKSdH/N+s+38s= +k8s.io/component-base v0.17.0/go.mod h1:rKuRAokNMY2nn2A6LP/MiwpoaMRHpfRnrPaUJJj1Yoc= +k8s.io/csi-translation-lib v0.17.0/go.mod h1:HEF7MEz7pOLJCnxabi45IPkhSsE/KmxPQksuCrHKWls= +k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/gengo v0.0.0-20190822140433-26a664648505/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= +k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= +k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= +k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= +k8s.io/legacy-cloud-providers v0.17.0/go.mod h1:DdzaepJ3RtRy+e5YhNtrCYwlgyK87j/5+Yfp0L9Syp8= +k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= +k8s.io/utils v0.0.0-20200729134348-d5654de09c73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw= +modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= +modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= +modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= +modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= +sigs.k8s.io/structured-merge-diff v1.0.1-0.20191108220359-b1b620dd3f06/go.mod h1:/ULNhyfzRopfcjskuui0cTITekDduZ7ycKN3oUT9R18= +sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/tests/e2e/scripts/build.sh b/tests/e2e/scripts/build.sh new file mode 100755 index 0000000000..7024fd3904 --- /dev/null +++ b/tests/e2e/scripts/build.sh @@ -0,0 +1 @@ +/usr/local/bin/docker build -f ../Dockerfile.build -t k3s_create_cluster:v1.0 . diff --git a/tests/e2e/terraform/modules/k3scluster/install_k3s_master.sh b/tests/e2e/terraform/modules/k3scluster/install_k3s_master.sh new file mode 100644 index 0000000000..af0ee3fda9 --- /dev/null +++ b/tests/e2e/terraform/modules/k3scluster/install_k3s_master.sh @@ -0,0 +1,106 @@ +#!/bin/bash + +mkdir -p /etc/rancher/k3s +cat << EOF >/etc/rancher/k3s/config.yaml +write-kubeconfig-mode: "0644" +tls-san: + - ${2} +EOF + +if [[ -n "$8" ]] && [[ "$8" == *":"* ]] +then + echo "$" + echo -e "$8" >> /etc/rancher/k3s/config.yaml + cat /etc/rancher/k3s/config.yaml +fi + +if [ "${1}" = "rhel" ] +then + subscription-manager register --auto-attach --username="${9}" --password="${10}" + subscription-manager repos --enable=rhel-7-server-extras-rpms +fi + +export "${3}"="${4}" + +if [ "${5}" = "etcd" ] +then + echo "CLUSTER TYPE is etcd" + if [[ "$4" == *"v1.18"* ]] || [["$4" == *"v1.17"* ]] && [[ -n "$8" ]] + then + echo "curl -sfL https://get.k3s.io | INSTALL_K3S_TYPE='server' sh -s - --cluster-init --node-external-ip=${6} $8" >/tmp/master_cmd + curl -sfL https://get.k3s.io | INSTALL_K3S_TYPE='server' sh -s - --cluster-init --node-external-ip="${6}" "$8" + else + echo "curl -sfL https://get.k3s.io | INSTALL_K3S_TYPE='server' sh -s - --cluster-init --node-external-ip=${6}" >/tmp/master_cmd + curl -sfL https://get.k3s.io | INSTALL_K3S_TYPE='server' sh -s - --cluster-init --node-external-ip="${6}" + fi +else + echo "CLUSTER TYPE is external db" + echo "$8" + if [[ "$4" == *"v1.18"* ]] || [[ "$4" == *"v1.17"* ]] && [[ -n "$8" ]] + then + echo "curl -sfL https://get.k3s.io | sh -s - server --node-external-ip=${6} --datastore-endpoint=\"${7}\" $8" >/tmp/master_cmd + curl -sfL https://get.k3s.io | sh -s - server --node-external-ip="${6}" --datastore-endpoint="${7}" "$8" + else + echo "curl -sfL https://get.k3s.io | sh -s - server --node-external-ip=${6} --datastore-endpoint=\"${7}\" " >/tmp/master_cmd + curl -sfL https://get.k3s.io | sh -s - server --node-external-ip="${6}" --datastore-endpoint="${7}" + fi +fi + +export PATH=$PATH:/usr/local/bin +timeElapsed=0 +while ! $(kubectl get nodes >/dev/null 2>&1) && [[ $timeElapsed -lt 300 ]] +do + sleep 5 + timeElapsed=$(expr $timeElapsed + 5) +done + +IFS=$'\n' +timeElapsed=0 +sleep 10 +while [[ $timeElapsed -lt 420 ]] +do + notready=false + for rec in $(kubectl get nodes) + do + if [[ "$rec" == *"NotReady"* ]] + then + notready=true + fi + done + if [[ $notready == false ]] + then + break + fi + sleep 20 + timeElapsed=$(expr $timeElapsed + 20) +done + +IFS=$'\n' +timeElapsed=0 +while [[ $timeElapsed -lt 420 ]] +do + helmPodsNR=false + systemPodsNR=false + for rec in $(kubectl get pods -A --no-headers) + do + if [[ "$rec" == *"helm-install"* ]] && [[ "$rec" != *"Completed"* ]] + then + helmPodsNR=true + elif [[ "$rec" != *"helm-install"* ]] && [[ "$rec" != *"Running"* ]] + then + systemPodsNR=true + else + echo "" + fi + done + + if [[ $systemPodsNR == false ]] && [[ $helmPodsNR == false ]] + then + break + fi + sleep 20 + timeElapsed=$(expr $timeElapsed + 20) +done +cat /etc/rancher/k3s/config.yaml> /tmp/joinflags +cat /var/lib/rancher/k3s/server/node-token >/tmp/nodetoken +cat /etc/rancher/k3s/k3s.yaml >/tmp/config diff --git a/tests/e2e/terraform/modules/k3scluster/join_k3s_agent.sh b/tests/e2e/terraform/modules/k3scluster/join_k3s_agent.sh new file mode 100644 index 0000000000..93c42fed0a --- /dev/null +++ b/tests/e2e/terraform/modules/k3scluster/join_k3s_agent.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# This script is used to join one or more nodes as agents + +mkdir -p /etc/rancher/k3s +cat <>/etc/rancher/k3s/config.yaml +server: https://${4}:6443 +token: "${5}" +EOF + +if [[ ! -z "$7" ]] && [[ "$7" == *":"* ]] +then + echo -e "$7" >> /etc/rancher/k3s/config.yaml + cat /etc/rancher/k3s/config.yaml +fi + +if [ ${1} = "rhel" ] +then + subscription-manager register --auto-attach --username=${8} --password=${9} + subscription-manager repos --enable=rhel-7-server-extras-rpms +fi + +export "${2}"="${3}" + if [[ "$3" == *"v1.18"* ]] || [["$3" == *"v1.17"* ]] && [[ -n "$7" ]] +then + echo "curl -sfL https://get.k3s.io | sh -s - agent --node-external-ip=${6} $7" >/tmp/agent_cmd +curl -sfL https://get.k3s.io | sh -s - agent --node-external-ip=${6} ${7} + else + +echo "curl -sfL https://get.k3s.io | sh -s - agent --node-external-ip=${6}" >/tmp/agent_cmd +curl -sfL https://get.k3s.io | sh -s - agent --node-external-ip=${6} +fi diff --git a/tests/e2e/terraform/modules/k3scluster/join_k3s_master.sh b/tests/e2e/terraform/modules/k3scluster/join_k3s_master.sh new file mode 100644 index 0000000000..30ac201d08 --- /dev/null +++ b/tests/e2e/terraform/modules/k3scluster/join_k3s_master.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# This script is used to join one or more nodes as masters + +mkdir -p /etc/rancher/k3s +cat <>/etc/rancher/k3s/config.yaml +write-kubeconfig-mode: "0644" +tls-san: + - ${2} +EOF + +if [[ -n "${10}" ]] && [[ "${10}" == *":"* ]] +then + echo -e "${10}" >> /etc/rancher/k3s/config.yaml + cat /etc/rancher/k3s/config.yaml +fi + +if [ "${1}" = "rhel" ] +then + subscription-manager register --auto-attach --username="${11}" --password="${12}" + subscription-manager repos --enable=rhel-7-server-extras-rpms +fi + +export "${3}"="${4}" + +if [ "${5}" = "etcd" ] +then + if [[ "$4" == *"v1.18"* ]] || [["$4" == *"v1.17"* ]] && [[ -n "$10" ]] + then + echo "curl -sfL https://get.k3s.io | INSTALL_K3S_TYPE='server' sh -s - --server https://\"${7}\":6443 --token \"${8}\" --node-external-ip=\"${6}\" ${10}" >/tmp/master_cmd + curl -sfL https://get.k3s.io | INSTALL_K3S_TYPE='server' sh -s - --server https://"${7}":6443 --token "${8}" --node-external-ip="${6} ${10}" + else + echo "curl -sfL https://get.k3s.io | INSTALL_K3S_TYPE='server' sh -s - --server https://\"${7}\":6443 --token \"${8}\" --node-external-ip=\"${6}\"" >/tmp/master_cmd + curl -sfL https://get.k3s.io | INSTALL_K3S_TYPE='server' sh -s - --server https://"${7}":6443 --token "${8}" --node-external-ip="${6}" + fi +else + if [[ "$4" == *"v1.18"* ]] || [["$4" == *"v1.17"* ]] && [[ -n "$10" ]] + then + echo "curl -sfL https://get.k3s.io | INSTALL_K3S_TYPE='server' sh -s - --node-external-ip=\"${6}\" --datastore-endpoint=\"${9}\" ${10}" >/tmp/master_cmd + curl -sfL https://get.k3s.io | INSTALL_K3S_TYPE='server' sh -s - --node-external-ip="${6}" --token="${8}" --datastore-endpoint="${9} ${10}" + else + echo "curl -sfL https://get.k3s.io | INSTALL_K3S_TYPE='server' sh -s - --node-external-ip=\"${6}\" --token \"${8}\" --datastore-endpoint=\"${9}\"" >/tmp/master_cmd + curl -sfL https://get.k3s.io | INSTALL_K3S_TYPE='server' sh -s - --node-external-ip="${6}" --token="${8}" --datastore-endpoint="${9}" + fi +fi diff --git a/tests/e2e/terraform/modules/k3scluster/main.tf b/tests/e2e/terraform/modules/k3scluster/main.tf new file mode 100644 index 0000000000..ecdd01f2af --- /dev/null +++ b/tests/e2e/terraform/modules/k3scluster/main.tf @@ -0,0 +1,53 @@ +module "master" { + source="./master" + aws_ami=var.aws_ami + aws_user=var.aws_user + key_name=var.key_name + no_of_server_nodes=var.no_of_server_nodes + k3s_version=var.k3s_version + install_mode=var.install_mode + region=var.region + vpc_id=var.vpc_id + subnets=var.subnets + qa_space=var.qa_space + ec2_instance_class=var.ec2_instance_class + access_key=var.access_key + cluster_type=var.cluster_type + server_flags=var.server_flags + availability_zone=var.availability_zone + sg_id=var.sg_id + resource_name=var.resource_name + node_os=var.node_os + username=var.username + password=var.password + db_username=var.db_username + db_password=var.db_password + db_group_name=var.db_group_name + external_db=var.external_db + instance_class=var.instance_class + external_db_version=var.external_db_version + engine_mode=var.engine_mode + environment=var.environment +} +module "worker" { + source="./worker" + dependency = module.master + aws_ami=var.aws_ami + aws_user=var.aws_user + key_name=var.key_name + no_of_worker_nodes=var.no_of_worker_nodes + k3s_version=var.k3s_version + install_mode=var.install_mode + region=var.region + vpc_id=var.vpc_id + subnets=var.subnets + ec2_instance_class=var.ec2_instance_class + access_key=var.access_key + worker_flags=var.worker_flags + availability_zone=var.availability_zone + sg_id=var.sg_id + resource_name=var.resource_name + node_os=var.node_os + username=var.username + password=var.password +} diff --git a/tests/e2e/terraform/modules/k3scluster/master/instances_server.tf b/tests/e2e/terraform/modules/k3scluster/master/instances_server.tf new file mode 100644 index 0000000000..0a52fe39ff --- /dev/null +++ b/tests/e2e/terraform/modules/k3scluster/master/instances_server.tf @@ -0,0 +1,285 @@ +resource "aws_db_instance" "db" { + count = (var.cluster_type == "etcd" ? 0 : (var.external_db != "aurora-mysql" ? 1 : 0)) + identifier = "${var.resource_name}-db" + allocated_storage = 20 + storage_type = "gp2" + engine = var.external_db + engine_version = var.external_db_version + instance_class = var.instance_class + name = "mydb" + parameter_group_name = var.db_group_name + username = var.db_username + password = var.db_password + availability_zone = var.availability_zone + tags = { + Environment = var.environment + } + skip_final_snapshot = true +} + +resource "aws_rds_cluster" "db" { + count = (var.external_db == "aurora-mysql" ? 1 : 0) + cluster_identifier = "${var.resource_name}-db" + engine = var.external_db + engine_version = var.external_db_version + availability_zones = [var.availability_zone] + database_name = "mydb" + master_username = var.db_username + master_password = var.db_password + engine_mode = var.engine_mode + tags = { + Environment = var.environment + } + skip_final_snapshot = true +} + +resource "aws_rds_cluster_instance" "db" { + count = (var.external_db == "aurora-mysql" ? 1 : 0) + cluster_identifier = "${aws_rds_cluster.db[0].id}" + identifier = "${var.resource_name}-instance1" + instance_class = var.instance_class + engine = aws_rds_cluster.db[0].engine + engine_version = aws_rds_cluster.db[0].engine_version +} + +resource "aws_instance" "master" { + ami = var.aws_ami + instance_type = var.ec2_instance_class + connection { + type = "ssh" + user = var.aws_user + host = self.public_ip + private_key = file(var.access_key) + } + root_block_device { + volume_size = "20" + volume_type = "standard" + } + subnet_id = var.subnets + availability_zone = var.availability_zone + vpc_security_group_ids = [var.sg_id] + key_name = var.key_name + tags = { + Name = "${var.resource_name}-server" + } + provisioner "file" { + source = "install_k3s_master.sh" + destination = "/tmp/install_k3s_master.sh" + } + provisioner "remote-exec" { + inline = [ + "chmod +x /tmp/install_k3s_master.sh", + "sudo /tmp/install_k3s_master.sh ${var.node_os} ${aws_route53_record.aws_route53.fqdn} ${var.install_mode} ${var.k3s_version} ${var.cluster_type} ${self.public_ip} \"${data.template_file.test.rendered}\" \"${var.server_flags}\" ${var.username} ${var.password}", + ] + } + provisioner "local-exec" { + command = "echo ${aws_instance.master.public_ip} >/tmp/${var.resource_name}_master_ip" + } + provisioner "local-exec" { + command = "scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i ${var.access_key} ${var.aws_user}@${aws_instance.master.public_ip}:/tmp/nodetoken /tmp/${var.resource_name}_nodetoken" + } + provisioner "local-exec" { + command = "scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i ${var.access_key} ${var.aws_user}@${aws_instance.master.public_ip}:/tmp/config /tmp/${var.resource_name}_config" + } + provisioner "local-exec" { + command = "scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i ${var.access_key} ${var.aws_user}@${aws_instance.master.public_ip}:/tmp/joinflags /tmp/${var.resource_name}_joinflags" + } + provisioner "local-exec" { + command = "scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i ${var.access_key} ${var.aws_user}@${aws_instance.master.public_ip}:/tmp/master_cmd /tmp/${var.resource_name}_master_cmd" + } + provisioner "local-exec" { + command = "sed s/127.0.0.1/\"${aws_route53_record.aws_route53.fqdn}\"/g /tmp/${var.resource_name}_config >/tmp/${var.resource_name}_kubeconfig" + } + provisioner "local-exec" { + command = "sed s/127.0.0.1/\"${aws_route53_record.aws_route53.fqdn}\"/g /tmp/${var.resource_name}_config >/config/${var.resource_name}_kubeconfig" + } +} + +data "template_file" "test" { + template = (var.cluster_type == "etcd" ? "NULL": (var.external_db == "postgres" ? "postgres://${aws_db_instance.db[0].username}:${aws_db_instance.db[0].password}@${aws_db_instance.db[0].endpoint}/${aws_db_instance.db[0].name}" : (var.external_db == "aurora-mysql" ? "mysql://${aws_rds_cluster.db[0].master_username}:${aws_rds_cluster.db[0].master_password}@tcp(${aws_rds_cluster.db[0].endpoint})/${aws_rds_cluster.db[0].database_name}" : "mysql://${aws_db_instance.db[0].username}:${aws_db_instance.db[0].password}@tcp(${aws_db_instance.db[0].endpoint})/${aws_db_instance.db[0].name}"))) + depends_on = [data.template_file.test_status] +} + +data "template_file" "test_status" { + template = (var.cluster_type == "etcd" ? "NULL": ((var.external_db == "postgres" ? aws_db_instance.db[0].endpoint : (var.external_db == "aurora-mysql" ? aws_rds_cluster_instance.db[0].endpoint : aws_db_instance.db[0].endpoint)))) +} + +data "local_file" "token" { + filename = "/tmp/${var.resource_name}_nodetoken" + depends_on = [aws_instance.master] +} + +locals { + node_token = trimspace("${data.local_file.token.content}") +} + +resource "aws_instance" "master2-ha" { + ami = var.aws_ami + instance_type = var.ec2_instance_class + count = var.no_of_server_nodes + connection { + type = "ssh" + user = var.aws_user + host = self.public_ip + private_key = file(var.access_key) + } + root_block_device { + volume_size = "20" + volume_type = "standard" + } + subnet_id = var.subnets + availability_zone = var.availability_zone + vpc_security_group_ids = [var.sg_id] + key_name = var.key_name + depends_on = [aws_instance.master] + tags = { + Name = "${var.resource_name}-servers" + } + provisioner "file" { + source = "join_k3s_master.sh" + destination = "/tmp/join_k3s_master.sh" + } + provisioner "remote-exec" { + inline = [ + "chmod +x /tmp/join_k3s_master.sh", + "sudo /tmp/join_k3s_master.sh ${var.node_os} ${aws_route53_record.aws_route53.fqdn} ${var.install_mode} ${var.k3s_version} ${var.cluster_type} ${self.public_ip} ${aws_instance.master.public_ip} ${local.node_token} \"${data.template_file.test.rendered}\" \"${var.server_flags}\" ${var.username} ${var.password}", + ] + } +} + +resource "aws_lb_target_group" "aws_tg_80" { + port = 80 + protocol = "TCP" + vpc_id = "${var.vpc_id}" + name = "${var.resource_name}-tg-80" + health_check { + protocol = "HTTP" + port = "traffic-port" + path = "/ping" + interval = 10 + timeout = 6 + healthy_threshold = 3 + unhealthy_threshold = 3 + matcher = "200-399" + } +} + +resource "aws_lb_target_group_attachment" "aws_tg_attachment_80" { + target_group_arn = "${aws_lb_target_group.aws_tg_80.arn}" + target_id = "${aws_instance.master.id}" + port = 80 + depends_on = ["aws_instance.master"] +} + +resource "aws_lb_target_group_attachment" "aws_tg_attachment_80_2" { + target_group_arn = "${aws_lb_target_group.aws_tg_80.arn}" + count = length(aws_instance.master2-ha) + target_id = "${aws_instance.master2-ha[count.index].id}" + port = 80 + depends_on = ["aws_instance.master"] +} + +resource "aws_lb_target_group" "aws_tg_443" { + port = 443 + protocol = "TCP" + vpc_id = "${var.vpc_id}" + name = "${var.resource_name}-tg-443" + health_check { + protocol = "HTTP" + port = 80 + path = "/ping" + interval = 10 + timeout = 6 + healthy_threshold = 3 + unhealthy_threshold = 3 + matcher = "200-399" + } +} + +resource "aws_lb_target_group_attachment" "aws_tg_attachment_443" { + target_group_arn = "${aws_lb_target_group.aws_tg_443.arn}" + target_id = "${aws_instance.master.id}" + port = 443 + depends_on = ["aws_instance.master"] +} + +resource "aws_lb_target_group_attachment" "aws_tg_attachment_443_2" { + target_group_arn = "${aws_lb_target_group.aws_tg_443.arn}" + count = length(aws_instance.master2-ha) + target_id = "${aws_instance.master2-ha[count.index].id}" + port = 443 + depends_on = ["aws_instance.master"] +} + +resource "aws_lb_target_group" "aws_tg_6443" { + port = 6443 + protocol = "TCP" + vpc_id = "${var.vpc_id}" + name = "${var.resource_name}-tg-6443" +} + +resource "aws_lb_target_group_attachment" "aws_tg_attachment_6443" { + target_group_arn = "${aws_lb_target_group.aws_tg_6443.arn}" + target_id = "${aws_instance.master.id}" + port = 6443 + depends_on = ["aws_instance.master"] +} + +resource "aws_lb_target_group_attachment" "aws_tg_attachment_6443_2" { + target_group_arn = "${aws_lb_target_group.aws_tg_6443.arn}" + count = length(aws_instance.master2-ha) + target_id = "${aws_instance.master2-ha[count.index].id}" + port = 6443 + depends_on = ["aws_instance.master"] +} + +resource "aws_lb" "aws_nlb" { + internal = false + load_balancer_type = "network" + subnets = ["${var.subnets}"] + name = "${var.resource_name}-nlb" +} + +resource "aws_lb_listener" "aws_nlb_listener_80" { + load_balancer_arn = "${aws_lb.aws_nlb.arn}" + port = "80" + protocol = "TCP" + default_action { + type = "forward" + target_group_arn = "${aws_lb_target_group.aws_tg_80.arn}" + } +} + +resource "aws_lb_listener" "aws_nlb_listener_443" { + load_balancer_arn = "${aws_lb.aws_nlb.arn}" + port = "443" + protocol = "TCP" + default_action { + type = "forward" + target_group_arn = "${aws_lb_target_group.aws_tg_443.arn}" + } +} + +resource "aws_lb_listener" "aws_nlb_listener_6443" { + load_balancer_arn = "${aws_lb.aws_nlb.arn}" + port = "6443" + protocol = "TCP" + default_action { + type = "forward" + target_group_arn = "${aws_lb_target_group.aws_tg_6443.arn}" + } +} + +resource "aws_route53_record" "aws_route53" { + zone_id = "${data.aws_route53_zone.selected.zone_id}" + name = "${var.resource_name}" + type = "CNAME" + ttl = "300" + records = ["${aws_lb.aws_nlb.dns_name}"] + depends_on = ["aws_lb_listener.aws_nlb_listener_6443"] +} + +data "aws_route53_zone" "selected" { + name = "${var.qa_space}" + private_zone = false +} diff --git a/tests/e2e/terraform/modules/k3scluster/master/outputs.tf b/tests/e2e/terraform/modules/k3scluster/master/outputs.tf new file mode 100644 index 0000000000..0f867e0ff3 --- /dev/null +++ b/tests/e2e/terraform/modules/k3scluster/master/outputs.tf @@ -0,0 +1,14 @@ +output "Route53_info" { + value = aws_route53_record.aws_route53.* + description = "List of DNS records" +} + +output "master_ips" { + value = join("," , aws_instance.master.*.public_ip,aws_instance.master2-ha.*.public_ip) + description = "The public IP of the AWS node" +} + +output "kubeconfig" { + value = var.resource_name + description = "kubeconfig of the cluster created" +} diff --git a/tests/e2e/terraform/modules/k3scluster/master/providers.tf b/tests/e2e/terraform/modules/k3scluster/master/providers.tf new file mode 100644 index 0000000000..0ced9d7521 --- /dev/null +++ b/tests/e2e/terraform/modules/k3scluster/master/providers.tf @@ -0,0 +1,3 @@ +provider "aws" { + region = "${var.region}" +} diff --git a/tests/e2e/terraform/modules/k3scluster/master/variables.tf b/tests/e2e/terraform/modules/k3scluster/master/variables.tf new file mode 100644 index 0000000000..098abfa841 --- /dev/null +++ b/tests/e2e/terraform/modules/k3scluster/master/variables.tf @@ -0,0 +1,31 @@ +variable "aws_ami" {} +variable "aws_user" {} +variable "region" {} +variable "access_key" {} +variable "vpc_id" {} +variable "subnets" {} +variable "availability_zone" {} +variable "sg_id" {} +variable "qa_space" {} +variable "ec2_instance_class" {} +variable "resource_name" {} +variable "key_name" {} + +variable "external_db" {} +variable "external_db_version" {} +variable "instance_class" {} + +variable "db_group_name" {} +variable "username" {} +variable "password" {} +variable "k3s_version" {} +variable "no_of_server_nodes" {} +variable "server_flags" {} + +variable "cluster_type" {} +variable "node_os" {} +variable "db_username" {} +variable "db_password" {} +variable "environment" {} +variable "engine_mode" {} +variable "install_mode" {} diff --git a/tests/e2e/terraform/modules/k3scluster/outputs.tf b/tests/e2e/terraform/modules/k3scluster/outputs.tf new file mode 100644 index 0000000000..6c5d8c9792 --- /dev/null +++ b/tests/e2e/terraform/modules/k3scluster/outputs.tf @@ -0,0 +1,14 @@ +output "master_ips" { + value = module.master.master_ips + description = "The public IP of the AWS node" +} + +output "worker_ips" { + value = module.worker.worker_ips + description = "The public IP of the AWS node" +} + +output "kubeconfig" { + value = module.master.kubeconfig + description = "kubeconfig of the cluster created" +} diff --git a/tests/e2e/terraform/modules/k3scluster/providers.tf b/tests/e2e/terraform/modules/k3scluster/providers.tf new file mode 100644 index 0000000000..0ced9d7521 --- /dev/null +++ b/tests/e2e/terraform/modules/k3scluster/providers.tf @@ -0,0 +1,3 @@ +provider "aws" { + region = "${var.region}" +} diff --git a/tests/e2e/terraform/modules/k3scluster/variables.tf b/tests/e2e/terraform/modules/k3scluster/variables.tf new file mode 100644 index 0000000000..03aa07597a --- /dev/null +++ b/tests/e2e/terraform/modules/k3scluster/variables.tf @@ -0,0 +1,31 @@ +#variable "db" {} +variable "no_of_worker_nodes" {} +variable "aws_ami" {} +variable "aws_user" {} +variable "region" {} +variable "access_key" {} +variable "vpc_id" {} +variable "subnets" {} +variable "qa_space" {} +variable "resource_name" {} +variable "key_name" {} +variable "external_db" {} +variable "external_db_version" {} +variable "instance_class" {} +variable "ec2_instance_class" {} +variable "db_group_name" {} +variable "username" {} +variable "password" {} +variable "k3s_version" {} +variable "no_of_server_nodes" {} +variable "server_flags" {} +variable "worker_flags" {} +variable "availability_zone" {} +variable "sg_id" {} +variable "cluster_type" {} +variable "node_os" {} +variable "db_username" {} +variable "db_password" {} +variable "environment" {} +variable "engine_mode" {} +variable "install_mode" {} diff --git a/tests/e2e/terraform/modules/k3scluster/worker/instances_worker.tf b/tests/e2e/terraform/modules/k3scluster/worker/instances_worker.tf new file mode 100644 index 0000000000..59b40fdc62 --- /dev/null +++ b/tests/e2e/terraform/modules/k3scluster/worker/instances_worker.tf @@ -0,0 +1,49 @@ +resource "aws_instance" "worker" { + depends_on = [ + var.dependency + ] + ami = var.aws_ami + instance_type = var.ec2_instance_class + count = var.no_of_worker_nodes + connection { + type = "ssh" + user = var.aws_user + host = self.public_ip + private_key = file(var.access_key) + } + subnet_id = var.subnets + availability_zone = var.availability_zone + vpc_security_group_ids = [var.sg_id] + key_name = var.key_name + tags = { + Name = "${var.resource_name}-worker" + } + provisioner "file" { + source = "join_k3s_agent.sh" + destination = "/tmp/join_k3s_agent.sh" + } + provisioner "remote-exec" { + inline = [ + "chmod +x /tmp/join_k3s_agent.sh", + "sudo /tmp/join_k3s_agent.sh ${var.node_os} ${var.install_mode} ${var.k3s_version} ${local.master_ip} ${local.node_token} ${self.public_ip} \"${var.worker_flags}\" ${var.username} ${var.password} ", + ] + } +} + +data "local_file" "master_ip" { + depends_on = [var.dependency] + filename = "/tmp/${var.resource_name}_master_ip" +} + +locals { + master_ip = trimspace(data.local_file.master_ip.content) +} + +data "local_file" "token" { + depends_on = [var.dependency] + filename = "/tmp/${var.resource_name}_nodetoken" +} + +locals { + node_token = trimspace(data.local_file.token.content) +} diff --git a/tests/e2e/terraform/modules/k3scluster/worker/outputs.tf b/tests/e2e/terraform/modules/k3scluster/worker/outputs.tf new file mode 100644 index 0000000000..145541c4c2 --- /dev/null +++ b/tests/e2e/terraform/modules/k3scluster/worker/outputs.tf @@ -0,0 +1,12 @@ +output "Registration_address" { + value = "${data.local_file.master_ip.content}" +} + +output "master_node_token" { + value = "${data.local_file.token.content}" +} + +output "worker_ips" { + value = join("," , aws_instance.worker.*.public_ip) + description = "The public IP of the AWS node" +} diff --git a/tests/e2e/terraform/modules/k3scluster/worker/providers.tf b/tests/e2e/terraform/modules/k3scluster/worker/providers.tf new file mode 100644 index 0000000000..0ced9d7521 --- /dev/null +++ b/tests/e2e/terraform/modules/k3scluster/worker/providers.tf @@ -0,0 +1,3 @@ +provider "aws" { + region = "${var.region}" +} diff --git a/tests/e2e/terraform/modules/k3scluster/worker/variables.tf b/tests/e2e/terraform/modules/k3scluster/worker/variables.tf new file mode 100644 index 0000000000..b5d6979324 --- /dev/null +++ b/tests/e2e/terraform/modules/k3scluster/worker/variables.tf @@ -0,0 +1,22 @@ +variable "dependency" { + type = any + default = null +} +variable "region" {} +variable "aws_ami" {} +variable "aws_user" {} +variable "vpc_id" {} +variable "subnets" {} +variable "resource_name" {} +variable "access_key" {} +variable "k3s_version" {} +variable "no_of_worker_nodes" {} +variable "worker_flags" {} +variable "ec2_instance_class" {} +variable "availability_zone" {} +variable "sg_id" {} +variable "username" {} +variable "password" {} +variable "node_os" {} +variable "install_mode" {} +variable "key_name" {} diff --git a/tests/e2e/testutils.go b/tests/e2e/testutils.go new file mode 100644 index 0000000000..91405a7ca6 --- /dev/null +++ b/tests/e2e/testutils.go @@ -0,0 +1,233 @@ +package e2e + +import ( + "bytes" + "errors" + "fmt" + "io/ioutil" + "log" + "os/exec" + "strings" + "time" + + "golang.org/x/crypto/ssh" +) + +type Node struct { + Name string + Status string + Roles string + InternalIP string + ExternalIP string +} + +type Pod struct { + NameSpace string + Name string + Ready string + Status string + Restarts string + NodeIP string + Node string +} + +var config *ssh.ClientConfig +var SSHKEY string +var SSHUSER string +var err error + +func checkError(e error) { + if e != nil { + log.Fatal(err) + panic(e) + } +} + +func publicKey(path string) ssh.AuthMethod { + key, err := ioutil.ReadFile(path) + if err != nil { + panic(err) + } + signer, err := ssh.ParsePrivateKey(key) + if err != nil { + panic(err) + } + return ssh.PublicKeys(signer) +} + +func ConfigureSSH(host string, SSHUser string, SSHKey string) *ssh.Client { + config = &ssh.ClientConfig{ + User: SSHUser, + Auth: []ssh.AuthMethod{ + publicKey(SSHKey), + }, + HostKeyCallback: ssh.InsecureIgnoreHostKey(), + } + conn, err := ssh.Dial("tcp", host, config) + checkError(err) + return conn +} + +func runsshCommand(cmd string, conn *ssh.Client) string { + session, err := conn.NewSession() + if err != nil { + panic(err) + } + defer session.Close() + var stdoutBuf bytes.Buffer + var stderrBuf bytes.Buffer + session.Stdout = &stdoutBuf + session.Stderr = &stderrBuf + + if err := session.Run(cmd); err != nil { + log.Println(session.Stdout) + log.Fatal("Error on command execution", err.Error()) + } + return fmt.Sprintf("%s", stdoutBuf.String()) +} + +//Runs command passed from within the node ServerIP +func RunCmdOnNode(cmd string, ServerIP string, SSHUser string, SSHKey string) string { + Server := ServerIP + ":22" + conn := ConfigureSSH(Server, SSHUser, SSHKey) + res := runsshCommand(cmd, conn) + res = strings.TrimSpace(res) + return res +} + +// RunCommand Runs command on the cluster accessing the cluster through kubeconfig file +func RunCommand(cmd string) (string, error) { + c := exec.Command("bash", "-c", cmd) + time.Sleep(10 * time.Second) + var out bytes.Buffer + c.Stdout = &out + err := c.Run() + if err != nil { + return "", errors.New(fmt.Sprintf("%s", err)) + } + return out.String(), nil +} + +//Used to count the pods using prefix passed in the list of pods +func CountOfStringInSlice(str string, pods []Pod) int { + count := 0 + for _, pod := range pods { + if strings.Contains(pod.Name, str) { + count++ + } + } + return count +} + +func DeployWorkload(workload string, kubeconfig string) (string, error) { + cmd := "kubectl apply -f " + workload + " --kubeconfig=" + kubeconfig + return RunCommand(cmd) +} + +func FetchClusterIP(kubeconfig string, servicename string) string { + cmd := "kubectl get svc " + servicename + " -o jsonpath='{.spec.clusterIP}' --kubeconfig=" + kubeconfig + res, _ := RunCommand(cmd) + return res +} + +func FetchNodeExternalIP(kubeconfig string) []string { + cmd := "kubectl get node --output=jsonpath='{range .items[*]} { .status.addresses[?(@.type==\"ExternalIP\")].address}' --kubeconfig=" + kubeconfig + time.Sleep(10 * time.Second) + res, _ := RunCommand(cmd) + nodeExternalIP := strings.Trim(res, " ") + nodeExternalIPs := strings.Split(nodeExternalIP, " ") + return nodeExternalIPs +} +func FetchIngressIP(kubeconfig string) []string { + cmd := "kubectl get ing ingress -o jsonpath='{.status.loadBalancer.ingress[*].ip}' --kubeconfig=" + kubeconfig + time.Sleep(10 * time.Second) + res, _ := RunCommand(cmd) + + ingressIp := strings.Trim(res, " ") + ingressIps := strings.Split(ingressIp, " ") + return ingressIps +} + +func ParseNode(kubeconfig string, printres bool) []Node { + nodes := make([]Node, 0, 10) + var node Node + timeElapsed := 0 + nodeList := "" + time.Sleep(60 * time.Second) + for timeElapsed < 420 { + notReady := false + cmd := "kubectl get nodes --no-headers -o wide -A --kubeconfig=" + kubeconfig + res, _ := RunCommand(cmd) + res = strings.TrimSpace(res) + nodeList = res + split := strings.Split(res, "\n") + for _, rec := range split { + fields := strings.Fields(string(rec)) + node.Name = fields[0] + node.Status = fields[1] + node.Roles = fields[2] + node.InternalIP = fields[5] + node.ExternalIP = fields[6] + nodes = append(nodes, node) + if node.Status != "Ready" { + notReady = true + break + } + } + if notReady == false { + break + } + time.Sleep(5 * time.Second) + timeElapsed = timeElapsed + 10 + } + if printres { + fmt.Println(nodeList) + } + return nodes +} + +func ParsePod(kubeconfig string, printres bool) []Pod { + pods := make([]Pod, 0, 10) + var pod Pod + timeElapsed := 0 + time.Sleep(60 * time.Second) + podList := "" + for timeElapsed < 420 { + helmPodsNR := false + systemPodsNR := false + cmd := "kubectl get pods -o wide --no-headers -A --kubeconfig=" + kubeconfig + res, _ := RunCommand(cmd) + res = strings.TrimSpace(res) + podList = res + + split := strings.Split(res, "\n") + for _, rec := range split { + fields := strings.Fields(string(rec)) + pod.NameSpace = fields[0] + pod.Name = fields[1] + pod.Ready = fields[2] + pod.Status = fields[3] + pod.Restarts = fields[4] + pod.NodeIP = fields[6] + pod.Node = fields[7] + pods = append(pods, pod) + if strings.HasPrefix(pod.Name, "helm-install") && pod.Status != "Completed" { + helmPodsNR = true + break + } else if !strings.HasPrefix(pod.Name, "helm-install") && pod.Status != "Running" { + + systemPodsNR = true + break + } + time.Sleep(10 * time.Second) + timeElapsed = timeElapsed + 10 + } + if systemPodsNR == false && helmPodsNR == false { + break + } + } + if printres { + fmt.Println(podList) + } + return pods +} diff --git a/tests/e2e/upgradecluster_test.go b/tests/e2e/upgradecluster_test.go new file mode 100644 index 0000000000..417276a26f --- /dev/null +++ b/tests/e2e/upgradecluster_test.go @@ -0,0 +1,362 @@ +package e2e +import ( + "flag" + "fmt" + . "github.com/onsi/ginkgo" + "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" + + "strings" + "testing" + "time" +) + +var upgradeVersion = flag.String("upgrade_version", "", "a string") + + +func Test_E2EClusterUpgradeValidation(t *testing.T) { + reporters := []Reporter{ + reporters.NewJUnitReporter("./" + *resourceName + "upgraderesults.xml"), + } + RegisterFailHandler(Fail) + RunSpecsWithCustomReporters(t, "Cluster Upgrade Validation", reporters) +} + +var _ = Describe("Test: ", func() { + + Context("Cluster Upgrade" + *nodeOs+ " " + *clusterType+ " " + *externalDb, func() { + + It("Verify Node and Pod Status", func() { + kubeconfig, masterIPs, workerIPs = BuildCluster(*nodeOs, *clusterType, *externalDb, *resourceName, &testing.T{}, *destroy) + if *destroy { + fmt.Printf("\nCluster is being Deleted\n") + return + } + fmt.Println("Cluster Version", *upgradeVersion) + fmt.Println("\nCluster Config:\nOS", *nodeOs, "Backend", *clusterType, *externalDb) + fmt.Printf("\nIPs:\n") + fmt.Println("Master Node IPS:", masterIPs) + fmt.Println("Worker Node IPS:", workerIPs) + + fmt.Printf("\nFetching node status\n") + nodes := ParseNode(kubeconfig, true) + for _, config := range nodes { + Expect(config.Status).Should(Equal("Ready"), func() string { return config.Name }) + } + + fmt.Printf("\nFetching Pods status\n") + pods := ParsePod(kubeconfig, true) + for _, pod := range pods { + if strings.Contains(pod.Name, "helm-install") { + Expect(pod.Status).Should(Equal("Completed"), func() string { return pod.Name }) + } else { + Expect(pod.Status).Should(Equal("Running"), func() string { return pod.Name }) + } + } + }) + + It("Validate ClusterIP", func() { + if *destroy { + return + } + DeployWorkloads(*arch, kubeconfig) + fmt.Println("Validating ClusterIP") + clusterip := FetchClusterIP(kubeconfig, "nginx-clusterip-svc") + cmd := "curl -L --insecure http://" + clusterip + "/name.html" + fmt.Println(cmd) + //Fetch External IP to login to node and validate cluster ip + node_external_ip := FetchNodeExternalIP(kubeconfig) + + for _, ip := range node_external_ip { + res := RunCmdOnNode(cmd, ip, *sshuser, *sshkey) + fmt.Println(res) + Expect(res).Should(ContainSubstring("test-clusterip"), func() string { return res }) + } + }) + + It("\nValidate NodePort", func() { + if *destroy { + return + } + fmt.Println("Validating NodePort") + node_external_ip := FetchNodeExternalIP(kubeconfig) + cmd := "kubectl get service nginx-nodeport-svc --kubeconfig=" + kubeconfig + " --output jsonpath=\"{.spec.ports[0].nodePort}\"" + nodeport,_ := RunCommand(cmd) + for _, nodeExternalIp := range node_external_ip { + cmd := "curl -L --insecure http://" + nodeExternalIp + ":" + nodeport + "/name.html" + fmt.Println(cmd) + res,_ := RunCommand(cmd) + fmt.Println(res) + Expect(res).Should(ContainSubstring("test-nodeport"), func() string { return res }) //Need to check of returned value is unique to node + } + }) + + It("\nValidate LoadBalancer", func() { + if *destroy { + return + } + fmt.Println("Validating Service LoaadBalancer") + node_external_ip := FetchNodeExternalIP(kubeconfig) + cmd := "kubectl get service nginx-loadbalancer-svc --kubeconfig=" + kubeconfig + " --output jsonpath=\"{.spec.ports[0].port}\"" + port, _:= RunCommand(cmd) + for _, ip := range node_external_ip { + cmd = "curl -L --insecure http://" + ip + ":" + port + "/name.html" + fmt.Println(cmd) + res,_ := RunCommand(cmd) + fmt.Println(res) + Expect(res).Should(ContainSubstring("test-loadbalancer"), func() string { return res }) + } + }) + + It("\nValidate Daemonset", func() { + if *destroy { + return + } + fmt.Println("Validating Daemonset") + nodes := ParseNode(kubeconfig, false) //nodes := + pods := ParsePod(kubeconfig, false) + fmt.Println("\nValidating Daemonset") + count := CountOfStringInSlice("test-daemonset", pods) + fmt.Println("POD COUNT") + fmt.Println(count) + fmt.Println("NODE COUNT") + fmt.Println(len(nodes)) + Eventually(len(nodes)).Should((Equal(count)), "120s", "60s") + }) + + It("Validate Ingress", func() { + if *destroy { + return + } + fmt.Println("Validating Ingress") + + ingressIps := FetchIngressIP(kubeconfig) + for _, ip := range ingressIps{ + cmd := "curl --header host:foo1.bar.com" + " http://" + ip + "/name.html" + fmt.Println(cmd) + //Access path from outside node + res, _ := RunCommand(cmd) + fmt.Println(res) + Eventually(res).Should((ContainSubstring("test-ingress")), "120s", "60s", func() string { return res }) + } + }) + + It("\nValidate Local Path Provisioner storage ", func() { + if *destroy { + return + } + fmt.Println("Validating Local Path Provisioner") + cmd := "kubectl get pvc local-path-pvc --kubeconfig=" + kubeconfig + res,_ := RunCommand(cmd) + fmt.Println(res) + Eventually(res).Should((ContainSubstring("local-path-pvc")), "120s", "60s") + Eventually(res).Should((ContainSubstring("Bound")), "120s", "60s") + + cmd = "kubectl get pod volume-test --kubeconfig=" + kubeconfig + res,_ = RunCommand(cmd) + fmt.Println(res) + + Eventually(res).Should((ContainSubstring("volume-test")), "120s", "60s", func() string { return res }) + + cmd = "kubectl --kubeconfig=" + kubeconfig + " exec volume-test -- sh -c 'echo local-path-test > /data/test'" + res, _= RunCommand(cmd) + fmt.Println(res) + fmt.Println("Data stored", res) + + cmd = "kubectl delete pod volume-test --kubeconfig=" + kubeconfig + res,_ = RunCommand(cmd) + fmt.Println(res) + resource_dir := "./amd64_resource_files" + cmd = "kubectl apply -f " + resource_dir + "/local-path-provisioner.yaml --kubeconfig=" + kubeconfig + res, _= RunCommand(cmd) + fmt.Println(res) + + time.Sleep(1 * time.Minute) + cmd = "kubectl exec volume-test cat /data/test --kubeconfig=" + kubeconfig + res,_ = RunCommand(cmd) + fmt.Println("Data after re-creation", res) + + Eventually(res).Should((ContainSubstring("local-path-test")), "120s", "60s", func() string { return res }) + }) + + It("\nVerify Cluster is upgraded and default pods running", func() { + if *destroy { + //fmt.Printf("\nCluster is Deleted\n") + return + } + MIPs := strings.Split(masterIPs,",") + + for _, ip := range MIPs { + cmd := "sudo sed -i \"s/|/| INSTALL_K3S_VERSION=" + *upgradeVersion + "/g\" /tmp/master_cmd" + fmt.Println(cmd) + _ = RunCmdOnNode(cmd,ip, *sshuser, *sshkey) + cmd = "sudo chmod u+x /tmp/master_cmd && sudo /tmp/master_cmd" + _ = RunCmdOnNode(cmd, ip, *sshuser, *sshkey) + } + + WIPs := strings.Split(workerIPs,",") + for i := 0; i < len(WIPs) && len(WIPs[0])>1; i++ { + ip := WIPs[i] + strings.TrimSpace(WIPs[i]) + cmd := "sudo sed -i \"s/|/| INSTALL_K3S_VERSION=" + *upgradeVersion + "/g\" /tmp/agent_cmd" + _ = RunCmdOnNode(cmd,ip, *sshuser, *sshkey) + By("Step4") + cmd = "sudo chmod u+x /tmp/agent_cmd && sudo /tmp/agent_cmd" + _ = RunCmdOnNode(cmd, ip, *sshuser, *sshkey) + } + + time.Sleep(5 * time.Second) + fmt.Println("After Upgrade") + nodes := ParseNode(kubeconfig, true) + for _, config := range nodes { + Expect(config.Status).Should(Equal("Ready")) + } + pods := ParsePod(kubeconfig, true) + for _, pod := range pods { + if strings.Contains(pod.Name, "helm-install") { + Expect(pod.Status).Should(Equal("Completed")) + } else { + Expect(pod.Status).Should(Equal("Running")) + } + } + }) + + + It("Validate ClusterIP after upgrade", func() { + if *destroy { + return + } + fmt.Println("Validating ClusterIP") + clusterip := FetchClusterIP(kubeconfig, "nginx-clusterip-svc") + cmd := "curl -L --insecure http://" + clusterip + "/name.html" + fmt.Println(cmd) + //Fetch External IP to login to node and validate cluster ip + node_external_ip := FetchNodeExternalIP(kubeconfig) + + for _, ip := range node_external_ip { + res := RunCmdOnNode(cmd, ip, *sshuser, *sshkey) + fmt.Println(res) + Expect(res).Should(ContainSubstring("test-clusterip"), func() string { return res }) + } + }) + + It("Validate NodePort after upgrade", func() { + if *destroy { + return + } + fmt.Println("Validating NodePort") + node_external_ip := FetchNodeExternalIP(kubeconfig) + cmd := "kubectl get service nginx-nodeport-svc --kubeconfig=" + kubeconfig + " --output jsonpath=\"{.spec.ports[0].nodePort}\"" + nodeport,_ := RunCommand(cmd) + for _, nodeExternalIp := range node_external_ip { + cmd := "curl -L --insecure http://" + nodeExternalIp + ":" + nodeport + "/name.html" + fmt.Println(cmd) + res,_ := RunCommand(cmd) + fmt.Println(res) + Expect(res).Should(ContainSubstring("test-nodeport"), func() string { return res }) //Need to check of returned value is unique to node + } + }) + + It("\nValidate Service LoadBalancer after upgrade", func() { + if *destroy { + return + } + fmt.Println("Validating Service LoaadBalancer") + node_external_ip := FetchNodeExternalIP(kubeconfig) + cmd := "kubectl get service nginx-loadbalancer-svc --kubeconfig=" + kubeconfig + " --output jsonpath=\"{.spec.ports[0].port}\"" + port,_ := RunCommand(cmd) + for _, ip := range node_external_ip { + cmd = "curl -L --insecure http://" + ip + ":" + port + "/name.html" + fmt.Println(cmd) + res,_ := RunCommand(cmd) + fmt.Println(res) + Expect(res).Should(ContainSubstring("test-loadbalancer"), func() string { return res }) + } + }) + + It("\nValidate Daemonset after upgrade", func() { + if *destroy { + return + } + fmt.Println("Validating Daemonset") + nodes := ParseNode(kubeconfig, false) //nodes := + pods := ParsePod(kubeconfig, false) + fmt.Println("\nValidating Daemonset") + count := CountOfStringInSlice("test-daemonset", pods) + fmt.Println("POD COUNT") + fmt.Println(count) + fmt.Println("NODE COUNT") + fmt.Println(len(nodes)) + Eventually(len(nodes)).Should((Equal(count)), "120s", "60s") + }) + It("nValidate Ingress after upgrade", func() { + if *destroy { + return + } + fmt.Println("Validating Ingress") + node_external_ip := FetchNodeExternalIP(kubeconfig) + + for _, ip := range node_external_ip { + cmd := "curl --header host:foo1.bar.com" + " http://" + ip + "/name.html" + fmt.Println(cmd) + //Access path from inside node + res := RunCmdOnNode(cmd, ip, *sshuser, *sshkey) + fmt.Println(res) + Eventually(res).Should(ContainSubstring("test-ingress"), "120s", "60s", func() string { return res }) + } + + for _, ip := range node_external_ip { + cmd := "curl --header host:foo1.bar.com" + " http://" + ip + "/name.html" + fmt.Println(cmd) + //Access path from outside node + res,_ := RunCommand(cmd) + fmt.Println(res) + Eventually(res).Should((ContainSubstring("test-ingress")), "120s", "60s", func() string { return res }) + } + }) + It("Validating Local Path Provisioner storage after upgrade", func() { + if *destroy { + return + } + fmt.Println("Validating Local Path Provisioner") + cmd := "kubectl get pvc local-path-pvc --kubeconfig=" + kubeconfig + res,_ := RunCommand(cmd) + fmt.Println(res) + Eventually(res).Should((ContainSubstring("local-path-pvc")), "120s", "60s") + Eventually(res).Should((ContainSubstring("Bound")), "120s", "60s") + + cmd = "kubectl get pod volume-test --kubeconfig=" + kubeconfig + res,_ = RunCommand(cmd) + fmt.Println(res) + + Eventually(res).Should((ContainSubstring("volume-test")), "120s", "60s", func() string { return res }) + + cmd = "kubectl --kubeconfig=" + kubeconfig + " exec volume-test -- sh -c 'echo local-path-test > /data/test'" + res,_ = RunCommand(cmd) + fmt.Println(res) + fmt.Println("Data stored", res) + + cmd = "kubectl delete pod volume-test --kubeconfig=" + kubeconfig + res,_ = RunCommand(cmd) + fmt.Println(res) + resource_dir := "./amd64_resource_files" + cmd = "kubectl apply -f " + resource_dir + "/local-path-provisioner.yaml --kubeconfig=" + kubeconfig + res,_ = RunCommand(cmd) + fmt.Println(res) + + time.Sleep(1 * time.Minute) + cmd = "kubectl exec volume-test cat /data/test --kubeconfig=" + kubeconfig + res, _ = RunCommand(cmd) + fmt.Println("Data after re-creation", res) + + Eventually(res).Should((ContainSubstring("local-path-test")), "120s", "60s", func() string { return res }) + }) + + It("Validating dns access after upgrade", func() { + cmd := "kubectl exec -i -t dnsutils -- nslookup kubernetes.default" + + res, _ := RunCommand(cmd) + fmt.Println(res) + }) + }) +})