[Release-1.22] Integration and E2E test improvements (#5685)

* Integration Test: Startup (#5630) * New startup integration test * Add testing section to PR template * Move helper functions to direct k8s client calls Signed-off-by: Derek Nola <derek.nola@suse.com> * E2E Improvements and groundwork for test-pad tool (#5593) * Add rancher install sript, taints to cp/etcd roles * Revert back to generic/ubuntu2004, libvirt networking is unreliable on opensuse * Added support for alpine * Rancher deployment script * Refactor installType into function * Cleanup splitserver test Signed-off-by: Derek Nola <derek.nola@suse.com> * E2E: Dualstack test (#5617) * E2E dualstack test * Improve testing documentation Signed-off-by: Derek Nola <derek.nola@suse.com> * Fix import Signed-off-by: Derek Nola <derek.nola@suse.com>
2022-06-14 17:51:20 -07:00 · 2022-06-14 17:51:20 -07:00 · 74c940ddc3
parent dd3485c202
commit 74c940ddc3
25 changed files with 1217 additions and 90 deletions
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@ -13,6 +13,11 @@

 <!-- How can the changes be verified? Please provide whatever additional information necessary to help verify the proposed changes. -->

+#### Testing ####
+
+<!-- Is this change covered by testing? If not, consider adding a Unit or Integration test. -->
+<!-- See https://github.com/k3s-io/k3s/blob/master/tests/TESTING.md for more info -->
+
 #### Linked Issues ####

 <!-- Link any related issues, pull-requests, or commit hashes that are relevant to this pull request. If you are opening a PR without a corresponding issue please consider creating one first, at https://github.com/k3s-io/k3s/issues . A functional example will greatly help QA with verifying/reproducing a bug or testing new features. -->
--- a/tests/e2e/README.md
+++ b/tests/e2e/README.md
@ -0,0 +1,51 @@
+# End-to-End (E2E) Tests
+
+E2E tests cover multi-node K3s configuration and administration: bringup, update, teardown etc. across a wide range of operating systems. E2E tests are run nightly as part of K3s quality assurance (QA).
+
+## Framework 
+End-to-end tests utilize [Ginkgo](https://onsi.github.io/ginkgo/) and [Gomega](https://onsi.github.io/gomega/) like the integration tests, but rely on [Vagrant](https://www.vagrantup.com/) to provide the underlying cluster configuration. 
+
+Currently tested operating systems are:
+- [Ubuntu 20.04](https://app.vagrantup.com/generic/boxes/ubuntu2004)
+- [Leap 15.3](https://app.vagrantup.com/opensuse/boxes/Leap-15.3.x86_64) (stand-in for SLE-Server)
+- [MicroOS](https://app.vagrantup.com/dweomer/boxes/microos.amd64) (stand-in for SLE-Micro)
+
+## Format
+
+All E2E tests should be placed under `tests/e2e/<TEST_NAME>`.  
+All E2E test functions should be named: `Test_E2E<TEST_NAME>`.  
+A E2E test consists of two parts:
+1. `Vagrantfile`: a vagrant file which describes and configures the VMs upon which the cluster and test will run
+2. `<TEST_NAME>.go`: A go test file which calls `vagrant up` and controls the actual testing of the cluster
+
+See the [validate cluster test](../tests/e2e/validatecluster/validatecluster_test.go) as an example.
+
+## Running
+
+Generally, E2E tests are run as a nightly Jenkins job for QA. They can still be run locally but additional setup may be required. By default, all E2E tests are designed with `libvirt` as the underlying VM provider. Instructions for installing libvirt and its associated vagrant plugin, `vagrant-libvirt` can be found [here.](https://github.com/vagrant-libvirt/vagrant-libvirt#installation) `VirtualBox` is also supported as a backup VM provider.
+
+Once setup is complete, all E2E tests can be run with:
+```bash
+go test -timeout=15m ./tests/e2e/... -run E2E
+```
+Tests can be run individually with:
+```bash
+go test -timeout=15m ./tests/e2e/validatecluster/... -run E2E
+#or
+go test -timeout=15m ./tests/e2e/... -run E2EClusterValidation
+```
+
+Additionally, to generate junit reporting for the tests, the Ginkgo CLI is used. Installation instructions can be found [here.](https://onsi.github.io/ginkgo/#getting-started)  
+
+To run the all E2E tests and generate JUnit testing reports:
+```
+ginkgo --junit-report=result.xml ./tests/e2e/...
+```
+
+Note: The `go test` default timeout is 10 minutes, thus the `-timeout` flag should be used. The `ginkgo` default timeout is 1 hour, no timeout flag is needed.
+
+# Debugging
+In the event of a test failure, the cluster and VMs are retained in their broken state. Startup logs are retained in `vagrant.log`.  
+To see a list of nodes: `vagrant status`    
+To ssh into a node: `vagrant ssh <NODE>`  
+Once you are done/ready to restart the test, use `vagrant destroy -f` to remove the broken cluster.  
--- a/tests/e2e/amd64_resource_files/dualstack_clusterip.yaml
+++ b/tests/e2e/amd64_resource_files/dualstack_clusterip.yaml
@ -0,0 +1,36 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ds-clusterip-pod
+spec:
+  selector:
+    matchLabels:
+      k8s-app: nginx-app-clusterip
+  replicas: 2
+  template:
+    metadata:
+      labels:
+        k8s-app: nginx-app-clusterip
+    spec:
+      containers:
+      - name: nginx
+        image: ranchertest/mytestcontainer
+        ports:
+        - containerPort: 80
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    k8s-app: nginx-app-clusterip
+  name: ds-clusterip-svc
+  namespace: default 
+spec:
+  type: ClusterIP
+  ipFamilyPolicy: PreferDualStack
+  ports:
+    - protocol: TCP
+      port: 80
+      targetPort: 80
+  selector:
+    k8s-app: nginx-app-clusterip
--- a/tests/e2e/amd64_resource_files/dualstack_ingress.yaml
+++ b/tests/e2e/amd64_resource_files/dualstack_ingress.yaml
@ -0,0 +1,16 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: ds-ingress
+spec:
+  rules:
+  - host: testds.com
+    http:
+      paths:
+      - backend:
+          service:
+            # Reliant on dualstack_clusterip.yaml
+            name: ds-clusterip-svc
+            port:
+              number: 80
+        pathType: ImplementationSpecific
--- a/tests/e2e/amd64_resource_files/dualstack_nodeport.yaml
+++ b/tests/e2e/amd64_resource_files/dualstack_nodeport.yaml
@ -0,0 +1,36 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ds-nodeport-pod
+spec:
+  selector:
+    matchLabels:
+      k8s-app: nginx-app-nodeport
+  replicas: 2
+  template:
+    metadata:
+      labels:
+        k8s-app: nginx-app-nodeport
+    spec:
+      containers:
+      - name: nginx
+        image: ranchertest/mytestcontainer
+        ports:
+        - containerPort: 80
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    k8s-app: nginx-app-nodeport
+  name: ds-nodeport-svc
+  namespace: default
+spec:
+  type: NodePort
+  ipFamilyPolicy: PreferDualStack
+  ports:
+    - port: 80
+      nodePort: 30096
+      name: http
+  selector:
+    k8s-app: nginx-app-nodeport
--- a/tests/e2e/dualstack/Vagrantfile
+++ b/tests/e2e/dualstack/Vagrantfile
@ -0,0 +1,111 @@
+ENV['VAGRANT_NO_PARALLEL'] = 'no'
+NODE_ROLES = (ENV['E2E_NODE_ROLES'] ||
+  ["server-0", "server-1", "server-2", "agent-0" ])
+NODE_BOXES = (ENV['E2E_NODE_BOXES'] ||
+  ['generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004'])
+GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master")
+RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "")
+NODE_CPUS = (ENV['E2E_NODE_CPUS'] || 2).to_i
+NODE_MEMORY = (ENV['E2E_NODE_MEMORY'] || 2048).to_i
+NETWORK4_PREFIX = "10.10.10"
+NETWORK6_PREFIX = "a11:decf:c0ff:ee"
+install_type = ""
+
+def provision(vm, roles, role_num, node_num)
+  vm.box = NODE_BOXES[node_num]
+  vm.hostname = "#{roles[0]}-#{role_num}"
+  node_ip4 = "#{NETWORK4_PREFIX}.#{100+node_num}"
+  node_ip6 = "#{NETWORK6_PREFIX}::#{10+node_num}"
+  # Only works with libvirt, which allows IPv4 + IPv6 on a single network/interface
+  vm.network "private_network", 
+    :ip => node_ip4,
+    :netmask => "255.255.255.0",
+    :libvirt__dhcp_enabled => false,
+    :libvirt__forward_mode => "none",
+    :libvirt__guest_ipv6 => "yes",
+    :libvirt__ipv6_address => "#{NETWORK6_PREFIX}::1",
+    :libvirt__ipv6_prefix => "64"
+    
+  vagrant_defaults = '../vagrantdefaults.rb'
+  load vagrant_defaults if File.exists?(vagrant_defaults)
+  
+  defaultOSConfigure(vm)
+
+  vm.provision "IPv6 Setup", type: "shell", path: "../scripts/ipv6.sh", args: [node_ip4, node_ip6, vm.box]
+  install_type = getInstallType(vm, RELEASE_VERSION, GITHUB_BRANCH)
+
+  vm.provision "Ping Check", type: "shell", inline: "ping -c 2 k3s.io"
+  
+  if roles.include?("server") && role_num == 0
+    vm.provision :k3s, run: 'once' do |k3s|
+      k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321
+      k3s.args = "server "
+      k3s.config = <<~YAML
+        node-external-ip: #{node_ip4},#{node_ip6}
+        node-ip: #{node_ip4},#{node_ip6}
+        cluster-init: true
+        token: vagrant
+        cluster-cidr: 10.42.0.0/16,2001:cafe:42:0::/56
+        service-cidr: 10.43.0.0/16,2001:cafe:42:1::/112
+        bind-address: #{NETWORK4_PREFIX}.100
+        flannel-iface: eth1
+      YAML
+      k3s.env = ["K3S_KUBECONFIG_MODE=0644", install_type]
+    end
+  elsif roles.include?("server") && role_num != 0
+    vm.provision :k3s, run: 'once' do |k3s|
+      k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321
+      k3s.args = "server "
+      k3s.config = <<~YAML
+        node-external-ip: #{node_ip4},#{node_ip6}
+        node-ip: #{node_ip4},#{node_ip6}
+        server: https://#{NETWORK4_PREFIX}.100:6443
+        token: vagrant
+        cluster-cidr: 10.42.0.0/16,2001:cafe:42:0::/56
+        service-cidr: 10.43.0.0/16,2001:cafe:42:1::/112
+        flannel-iface: eth1
+      YAML
+      k3s.env = ["K3S_KUBECONFIG_MODE=0644", install_type]
+    end
+  end
+  if roles.include?("agent")
+    vm.provision :k3s, run: 'once' do |k3s|
+      k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321
+      k3s.args = "agent "
+      k3s.config = <<~YAML
+        node-external-ip: #{node_ip4},#{node_ip6}
+        node-ip: #{node_ip4},#{node_ip6}
+        server: https://#{NETWORK4_PREFIX}.100:6443
+        token: vagrant
+        flannel-iface: eth1
+      YAML
+      k3s.env = ["K3S_KUBECONFIG_MODE=0644", install_type]
+    end
+  end
+end
+
+Vagrant.configure("2") do |config|
+  config.vagrant.plugins = ["vagrant-k3s", "vagrant-reload", "vagrant-libvirt"]
+  config.vm.provider "libvirt" do |v|
+    v.cpus = NODE_CPUS
+    v.memory = NODE_MEMORY
+  end
+  
+  if NODE_ROLES.kind_of?(String)
+    NODE_ROLES = NODE_ROLES.split(" ", -1)
+  end
+  if NODE_BOXES.kind_of?(String)
+    NODE_BOXES = NODE_BOXES.split(" ", -1)
+  end
+
+  # Must iterate on the index, vagrant does not understand iterating 
+  # over the node roles themselves
+  NODE_ROLES.length.times do |i|
+    name = NODE_ROLES[i]
+    config.vm.define name do |node|
+      roles = name.split("-", -1)
+      role_num = roles.pop.to_i
+      provision(node.vm, roles, role_num, i)
+    end
+  end
+end
--- a/tests/e2e/dualstack/dualstack_test.go
+++ b/tests/e2e/dualstack/dualstack_test.go
@ -0,0 +1,213 @@
+package validatecluster
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/k3s-io/k3s/tests/e2e"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+// Valid nodeOS: generic/ubuntu2004, opensuse/Leap-15.3.x86_64
+var nodeOS = flag.String("nodeOS", "generic/ubuntu2004", "VM operating system")
+var serverCount = flag.Int("serverCount", 3, "number of server nodes")
+var agentCount = flag.Int("agentCount", 0, "number of agent nodes")
+
+// Environment Variables Info:
+// E2E_RELEASE_VERSION=v1.23.1+k3s1 or nil for latest commit from master
+
+type objIP struct {
+	name string
+	ipv4 string
+	ipv6 string
+}
+
+func getPodIPs(kubeConfigFile string) ([]objIP, error) {
+	cmd := `kubectl get pods -A -o=jsonpath='{range .items[*]}{.metadata.name}{" "}{.status.podIPs[*].ip}{"\n"}{end}' --kubeconfig=` + kubeConfigFile
+	return getObjIPs(cmd)
+}
+func getNodeIPs(kubeConfigFile string) ([]objIP, error) {
+	cmd := `kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.name}{" "}{.status.addresses[?(@.type == "InternalIP")].address}{"\n"}{end}' --kubeconfig=` + kubeConfigFile
+	return getObjIPs(cmd)
+}
+
+func getObjIPs(cmd string) ([]objIP, error) {
+	var objIPs []objIP
+	res, err := e2e.RunCommand(cmd)
+	if err != nil {
+		return nil, err
+	}
+	objs := strings.Split(res, "\n")
+	objs = objs[:len(objs)-1]
+
+	for _, obj := range objs {
+		fields := strings.Fields(obj)
+		if len(fields) > 2 {
+			objIPs = append(objIPs, objIP{name: fields[0], ipv4: fields[1], ipv6: fields[2]})
+		} else if len(fields) > 1 {
+			objIPs = append(objIPs, objIP{name: fields[0], ipv4: fields[1]})
+		} else {
+			objIPs = append(objIPs, objIP{name: fields[0]})
+		}
+	}
+	return objIPs, nil
+}
+
+func Test_E2EDualStack(t *testing.T) {
+	flag.Parse()
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Validate DualStack Suite")
+}
+
+var (
+	kubeConfigFile  string
+	serverNodeNames []string
+	agentNodeNames  []string
+)
+
+var _ = Describe("Verify DualStack Configuration", func() {
+
+	It("Starts up with no issues", func() {
+		var err error
+		serverNodeNames, agentNodeNames, err = e2e.CreateCluster(*nodeOS, *serverCount, *agentCount)
+		Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog())
+		fmt.Println("CLUSTER CONFIG")
+		fmt.Println("OS:", *nodeOS)
+		fmt.Println("Server Nodes:", serverNodeNames)
+		fmt.Println("Agent Nodes:", agentNodeNames)
+		kubeConfigFile, err = e2e.GenKubeConfigFile(serverNodeNames[0])
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	It("Checks Node Status", func() {
+		Eventually(func(g Gomega) {
+			nodes, err := e2e.ParseNodes(kubeConfigFile, false)
+			g.Expect(err).NotTo(HaveOccurred())
+			for _, node := range nodes {
+				g.Expect(node.Status).Should(Equal("Ready"))
+			}
+		}, "420s", "5s").Should(Succeed())
+		_, err := e2e.ParseNodes(kubeConfigFile, true)
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	It("Checks Pod Status", func() {
+		Eventually(func(g Gomega) {
+			pods, err := e2e.ParsePods(kubeConfigFile, false)
+			g.Expect(err).NotTo(HaveOccurred())
+			for _, pod := range pods {
+				if strings.Contains(pod.Name, "helm-install") {
+					g.Expect(pod.Status).Should(Equal("Completed"), pod.Name)
+				} else {
+					g.Expect(pod.Status).Should(Equal("Running"), pod.Name)
+				}
+			}
+		}, "420s", "5s").Should(Succeed())
+		_, err := e2e.ParsePods(kubeConfigFile, true)
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	It("Verifies that each node has IPv4 and IPv6", func() {
+		nodeIPs, err := getNodeIPs(kubeConfigFile)
+		Expect(err).NotTo(HaveOccurred())
+		for _, node := range nodeIPs {
+			Expect(node.ipv4).Should(ContainSubstring("10.10.10"))
+			Expect(node.ipv6).Should(ContainSubstring("a11:decf:c0ff"))
+		}
+	})
+	It("Verifies that each pod has IPv4 and IPv6", func() {
+		podIPs, err := getPodIPs(kubeConfigFile)
+		Expect(err).NotTo(HaveOccurred())
+		for _, pod := range podIPs {
+			Expect(pod.ipv4).Should(Or(ContainSubstring("10.10.10"), ContainSubstring("10.42.")), pod.name)
+			Expect(pod.ipv6).Should(Or(ContainSubstring("a11:decf:c0ff"), ContainSubstring("2001:cafe:42")), pod.name)
+		}
+	})
+
+	It("Verifies ClusterIP Service", func() {
+		_, err := e2e.DeployWorkload("dualstack_clusterip.yaml", kubeConfigFile, false)
+		Expect(err).NotTo(HaveOccurred())
+		Eventually(func() (string, error) {
+			cmd := "kubectl get pods -o=name -l k8s-app=nginx-app-clusterip --field-selector=status.phase=Running --kubeconfig=" + kubeConfigFile
+			return e2e.RunCommand(cmd)
+		}, "120s", "5s").Should(ContainSubstring("ds-clusterip-pod"))
+
+		// Checks both IPv4 and IPv6
+		clusterips, err := e2e.FetchClusterIP(kubeConfigFile, "ds-clusterip-svc", true)
+		Expect(err).NotTo(HaveOccurred())
+		for _, ip := range strings.Split(clusterips, ",") {
+			if strings.Contains(ip, "::") {
+				ip = "[" + ip + "]"
+			}
+			pods, err := e2e.ParsePods(kubeConfigFile, false)
+			Expect(err).NotTo(HaveOccurred())
+			for _, pod := range pods {
+				if !strings.HasPrefix(pod.Name, "ds-clusterip-pod") {
+					continue
+				}
+				cmd := fmt.Sprintf("curl -L --insecure http://%s", ip)
+				Eventually(func() (string, error) {
+					return e2e.RunCmdOnNode(cmd, serverNodeNames[0])
+				}, "60s", "5s").Should(ContainSubstring("Welcome to nginx!"), "failed cmd: "+cmd)
+			}
+		}
+	})
+	It("Verifies Ingress", func() {
+		_, err := e2e.DeployWorkload("dualstack_ingress.yaml", kubeConfigFile, false)
+		Expect(err).NotTo(HaveOccurred(), "Ingress manifest not deployed")
+		cmd := "kubectl get ingress ds-ingress --kubeconfig=" + kubeConfigFile + " -o jsonpath=\"{.spec.rules[*].host}\""
+		hostName, err := e2e.RunCommand(cmd)
+		Expect(err).NotTo(HaveOccurred(), "failed cmd: "+cmd)
+		nodeIPs, err := getNodeIPs(kubeConfigFile)
+		Expect(err).NotTo(HaveOccurred(), "failed cmd: "+cmd)
+		for _, node := range nodeIPs {
+			cmd := fmt.Sprintf("curl  --header host:%s http://%s/name.html", hostName, node.ipv4)
+			Eventually(func() (string, error) {
+				return e2e.RunCommand(cmd)
+			}, "10s", "2s").Should(ContainSubstring("ds-clusterip-pod"), "failed cmd: "+cmd)
+			cmd = fmt.Sprintf("curl  --header host:%s http://[%s]/name.html", hostName, node.ipv6)
+			Eventually(func() (string, error) {
+				return e2e.RunCommand(cmd)
+			}, "5s", "1s").Should(ContainSubstring("ds-clusterip-pod"), "failed cmd: "+cmd)
+		}
+	})
+
+	It("Verifies NodePort Service", func() {
+		_, err := e2e.DeployWorkload("dualstack_nodeport.yaml", kubeConfigFile, false)
+		Expect(err).NotTo(HaveOccurred())
+		cmd := "kubectl get service ds-nodeport-svc --kubeconfig=" + kubeConfigFile + " --output jsonpath=\"{.spec.ports[0].nodePort}\""
+		nodeport, err := e2e.RunCommand(cmd)
+		Expect(err).NotTo(HaveOccurred(), "failed cmd: "+cmd)
+		nodeIPs, err := getNodeIPs(kubeConfigFile)
+		Expect(err).NotTo(HaveOccurred())
+		for _, node := range nodeIPs {
+			cmd = "curl -L --insecure http://" + node.ipv4 + ":" + nodeport + "/name.html"
+			Eventually(func() (string, error) {
+				return e2e.RunCommand(cmd)
+			}, "10s", "1s").Should(ContainSubstring("ds-nodeport-pod"), "failed cmd: "+cmd)
+			cmd = "curl -L --insecure http://[" + node.ipv6 + "]:" + nodeport + "/name.html"
+			Eventually(func() (string, error) {
+				return e2e.RunCommand(cmd)
+			}, "10s", "1s").Should(ContainSubstring("ds-nodeport-pod"), "failed cmd: "+cmd)
+		}
+	})
+
+})
+
+var failed bool
+var _ = AfterEach(func() {
+	failed = failed || CurrentGinkgoTestDescription().Failed
+})
+
+var _ = AfterSuite(func() {
+	if failed {
+		fmt.Println("FAILED!")
+	} else {
+		Expect(e2e.DestroyCluster()).To(Succeed())
+		Expect(os.Remove(kubeConfigFile)).To(Succeed())
+	}
+})
--- a/tests/e2e/scripts/ipv6.sh
+++ b/tests/e2e/scripts/ipv6.sh
@ -0,0 +1,16 @@
+#!/bin/bash
+ip4_addr=$1
+ip6_addr=$2
+os=$3
+
+sysctl -w net.ipv6.conf.all.disable_ipv6=0
+sysctl -w net.ipv6.conf.eth1.accept_dad=0
+
+if [ -z "${os##*ubuntu*}" ]; then
+  netplan set ethernets.eth1.accept-ra=false
+  netplan set ethernets.eth1.addresses=["$ip4_addr"/24,"$ip6_addr"/64]
+  netplan apply
+else
+  ip -6 addr add "$ip6_addr"/64 dev eth1
+fi
+ip addr show dev eth1
--- a/tests/e2e/scripts/rancher.sh
+++ b/tests/e2e/scripts/rancher.sh
@ -0,0 +1,63 @@
+#!/bin/bash
+node_ip=$1
+
+echo  "Give K3s time to startup"
+sleep 10
+kubectl -n kube-system rollout status deploy/coredns
+kubectl -n kube-system rollout status deploy/local-path-provisioner
+
+cat << EOF > /var/lib/rancher/k3s/server/manifests/rancher.yaml
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: cert-manager
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: cattle-system
+---
+apiVersion: helm.cattle.io/v1
+kind: HelmChart
+metadata:
+  namespace: kube-system
+  name: cert-manager
+spec:
+  targetNamespace: cert-manager
+  version: v1.6.1
+  chart: cert-manager
+  repo: https://charts.jetstack.io
+  set:
+    installCRDs: "true"
+---
+apiVersion: helm.cattle.io/v1
+kind: HelmChart
+metadata:
+  namespace: kube-system
+  name: rancher
+spec:
+  targetNamespace: cattle-system
+  version: 2.6.5
+  chart: rancher
+  repo: https://releases.rancher.com/server-charts/latest
+  set:
+    ingress.tls.source: "rancher"
+    hostname: "$node_ip.nip.io"
+    replicas: 1
+EOF
+
+
+echo "Give Rancher time to startup"
+sleep 20
+kubectl -n cert-manager rollout status deploy/cert-manager
+while ! kubectl get secret --namespace cattle-system bootstrap-secret -o go-template='{{.data.bootstrapPassword|base64decode}}' &> /dev/null; do
+    ((iterations++))
+    if [ "$iterations" -ge 8 ]; then
+        echo "Unable to find bootstrap-secret"
+        exit 1
+    fi
+    echo "waiting for bootstrap-secret..."
+    sleep 20
+done
+echo https://"$node_ip".nip.io/dashboard/?setup=$(kubectl get secret --namespace cattle-system bootstrap-secret -o go-template='{{.data.bootstrapPassword|base64decode}}')
--- a/tests/e2e/secretsencryption/Vagrantfile
+++ b/tests/e2e/secretsencryption/Vagrantfile
@ -21,14 +21,8 @@ def provision(vm, roles, role_num, node_num)
  load vagrant_defaults if File.exists?(vagrant_defaults)
  
  defaultOSConfigure(vm)
+  install_type = getInstallType(vm, RELEASE_VERSION, GITHUB_BRANCH)
  
-  if !RELEASE_VERSION.empty?
-    install_type = "INSTALL_K3S_VERSION=#{RELEASE_VERSION}"
-  else
-    # Grabs the last 5 commit SHA's from the given branch, then purges any commits that do not have a passing CI build
-    vm.provision "shell", path: "../scripts/latest_commit.sh", args: [GITHUB_BRANCH, "/tmp/k3s_commits"]
-    install_type = "INSTALL_K3S_COMMIT=$(head\ -n\ 1\ /tmp/k3s_commits)"
-  end
  vm.provision "shell", inline: "ping -c 2 k3s.io"
  
  if roles.include?("server") && role_num == 0
@ -44,7 +38,7 @@ def provision(vm, roles, role_num, node_num)
      k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321
    end
  end
-  if vm.box.include?("microos")
+  if vm.box.to_s.include?("microos")
    vm.provision 'k3s-reload', type: 'reload', run: 'once'
  end
 end
--- a/tests/e2e/splitserver/Vagrantfile
+++ b/tests/e2e/splitserver/Vagrantfile
@ -0,0 +1,127 @@
+ENV['VAGRANT_NO_PARALLEL'] = 'no'
+NODE_ROLES = (ENV['E2E_NODE_ROLES'] ||
+  ["server-etcd-0", "server-cp-0", "server-cp-1", "agent-0"])
+NODE_BOXES = (ENV['E2E_NODE_BOXES'] ||
+  ['generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004'])
+GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master")
+RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "")
+NODE_CPUS = (ENV['E2E_NODE_CPUS'] || 2).to_i
+NODE_MEMORY = (ENV['E2E_NODE_MEMORY'] || 1024).to_i
+# Virtualbox >= 6.1.28 require `/etc/vbox/network.conf` for expanded private networks 
+NETWORK_PREFIX = "10.10.10"
+
+def provision(vm, role, role_num, node_num)
+  vm.box = NODE_BOXES[node_num]
+  vm.hostname = role
+  # An expanded netmask is required to allow VM<-->VM communication, virtualbox defaults to /32
+  vm.network "private_network", ip: "#{NETWORK_PREFIX}.#{100+node_num}", netmask: "255.255.255.0"
+
+  vagrant_defaults = File.exists?("./vagrantdefaults.rb") ? "./vagrantdefaults.rb" : "../vagrantdefaults.rb"
+  load vagrant_defaults
+
+  defaultOSConfigure(vm)
+  
+  install_type = getInstallType(vm, RELEASE_VERSION, GITHUB_BRANCH)
+
+  vm.provision "ping k3s.io", type: "shell", inline: "ping -c 2 k3s.io"
+  
+  if node_num == 0 && !role.include?("server") && !role.include?("etcd")
+    puts "first node must be a etcd server"
+    abort
+  elsif role.include?("server") && role.include?("etcd") && role_num == 0
+    vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s|
+      k3s.args = "server"
+      k3s.config = <<~YAML
+        cluster-init: true
+        node-external-ip: #{NETWORK_PREFIX}.100
+        flannel-iface: eth1
+        disable-apiserver: true
+        disable-controller-manager: true
+        disable-scheduler: true
+        node-taint:
+          - node-role.kubernetes.io/etcd:NoExecute
+      YAML
+      k3s.env = %W[K3S_KUBECONFIG_MODE=0644 K3S_TOKEN=vagrant #{install_type}]
+      k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321
+    end
+  elsif role.include?("server") && role.include?("etcd") && role_num != 0
+    vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s|
+      k3s.args = "server"
+      k3s.config = <<~YAML
+        server: https://#{NETWORK_PREFIX}.100:6443
+        flannel-iface: eth1
+        disable-apiserver: true
+        disable-controller-manager: true
+        disable-scheduler: true
+        node-taint:
+          - node-role.kubernetes.io/etcd:NoExecute
+          
+      YAML
+      k3s.env = %W[K3S_KUBECONFIG_MODE=0644 K3S_TOKEN=vagrant #{install_type}]
+      k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321
+    end
+  elsif role.include?("server") && role.include?("cp")
+    vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s|
+      k3s.args = "server"
+      k3s.config = <<~YAML
+        server: https://#{NETWORK_PREFIX}.100:6443
+        flannel-iface: eth1
+        disable-etcd: true
+        node-taint:
+          - node-role.kubernetes.io/control-plane:NoSchedule
+      YAML
+      k3s.env = %W[K3S_KUBECONFIG_MODE=0644 K3S_TOKEN=vagrant #{install_type}]
+      k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321
+    end
+  elsif role.include?("server") && role.include?("all")
+    vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s|
+      k3s.args = "server"
+      k3s.config = <<~YAML
+        server: https://#{NETWORK_PREFIX}.100:6443
+        flannel-iface: eth1
+      YAML
+      k3s.env = %W[K3S_KUBECONFIG_MODE=0644 K3S_TOKEN=vagrant #{install_type}]
+      k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321
+    end
+  end
+  if role.include?("agent")
+    vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s|
+      k3s.args = %W[agent --server https://#{NETWORK_PREFIX}.101:6443 --flannel-iface=eth1]
+      k3s.env = %W[K3S_KUBECONFIG_MODE=0644 K3S_TOKEN=vagrant #{install_type}]
+      k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321
+    end
+  end
+  if vm.box.to_s.include?("microos")
+    vm.provision 'k3s-reload', type: 'reload', run: 'once'
+  end
+end
+
+Vagrant.configure("2") do |config|
+  config.vagrant.plugins = ["vagrant-k3s", "vagrant-reload"]
+  # Default provider is libvirt, virtualbox is only provided as a backup
+  config.vm.provider "libvirt" do |v|
+    v.cpus = NODE_CPUS
+    v.memory = NODE_MEMORY
+  end
+  config.vm.provider "virtualbox" do |v|
+    v.cpus = NODE_CPUS
+    v.memory = NODE_MEMORY
+  end
+  
+  if NODE_ROLES.kind_of?(String)
+    NODE_ROLES = NODE_ROLES.split(" ", -1)
+  end
+  if NODE_BOXES.kind_of?(String)
+    NODE_BOXES = NODE_BOXES.split(" ", -1)
+  end
+
+  # Must iterate on the index, vagrant does not understand iterating 
+  # over the node roles themselves
+  NODE_ROLES.length.times do |i|
+    name = NODE_ROLES[i]
+    role_num = name.split("-", -1).pop.to_i
+    config.vm.define name do |node|
+      provision(node.vm, name, role_num, i)
+    end
+  end
+end
--- a/tests/e2e/splitserver/splitserver_test.go
+++ b/tests/e2e/splitserver/splitserver_test.go
@ -0,0 +1,233 @@
+package validatecluster
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+	"testing"
+
+	"github.com/k3s-io/k3s/tests/e2e"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+// Valid nodeOS: generic/ubuntu2004, opensuse/Leap-15.3.x86_64, dweomer/microos.amd64
+var nodeOS = flag.String("nodeOS", "generic/ubuntu2004", "VM operating system")
+var etcdCount = flag.Int("etcdCount", 1, "number of server nodes only deploying etcd")
+var controlPlaneCount = flag.Int("controlPlaneCount", 1, "number of server nodes acting as control plane")
+var agentCount = flag.Int("agentCount", 1, "number of agent nodes")
+
+// Environment Variables Info:
+// E2E_RELEASE_VERSION=v1.23.1+k3s2 or nil for latest commit from master
+
+func createSplitCluster(nodeOS string, etcdCount, controlPlaneCount, agentCount int) ([]string, []string, []string, error) {
+	etcdNodeNames := make([]string, etcdCount)
+	for i := 0; i < etcdCount; i++ {
+		etcdNodeNames[i] = "server-etcd-" + strconv.Itoa(i)
+	}
+	cpNodeNames := make([]string, controlPlaneCount)
+	for i := 0; i < controlPlaneCount; i++ {
+		cpNodeNames[i] = "server-cp-" + strconv.Itoa(i)
+	}
+	agentNodeNames := make([]string, agentCount)
+	for i := 0; i < agentCount; i++ {
+		agentNodeNames[i] = "agent-" + strconv.Itoa(i)
+	}
+	nodeRoles := strings.Join(etcdNodeNames, " ") + " " + strings.Join(cpNodeNames, " ") + " " + strings.Join(agentNodeNames, " ")
+
+	nodeRoles = strings.TrimSpace(nodeRoles)
+	nodeBoxes := strings.Repeat(nodeOS+" ", etcdCount+controlPlaneCount+agentCount)
+	nodeBoxes = strings.TrimSpace(nodeBoxes)
+
+	var testOptions string
+	for _, env := range os.Environ() {
+		if strings.HasPrefix(env, "E2E_") {
+			testOptions += " " + env
+		}
+	}
+
+	cmd := fmt.Sprintf(`E2E_NODE_ROLES="%s" E2E_NODE_BOXES="%s" %s vagrant up &> vagrant.log`, nodeRoles, nodeBoxes, testOptions)
+	fmt.Println(cmd)
+	if _, err := e2e.RunCommand(cmd); err != nil {
+		fmt.Println("Error Creating Cluster", err)
+		return nil, nil, nil, err
+	}
+	return etcdNodeNames, cpNodeNames, agentNodeNames, nil
+}
+func Test_E2ESplitServer(t *testing.T) {
+	RegisterFailHandler(Fail)
+	flag.Parse()
+	RunSpecs(t, "Split Server Test Suite")
+}
+
+var (
+	kubeConfigFile string
+	etcdNodeNames  []string
+	cpNodeNames    []string
+	agentNodeNames []string
+)
+
+var _ = Describe("Verify Create", func() {
+	Context("Cluster :", func() {
+		It("Starts up with no issues", func() {
+			var err error
+			etcdNodeNames, cpNodeNames, agentNodeNames, err = createSplitCluster(*nodeOS, *etcdCount, *controlPlaneCount, *agentCount)
+			Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog())
+			fmt.Println("CLUSTER CONFIG")
+			fmt.Println("OS:", *nodeOS)
+			fmt.Println("Etcd Server Nodes:", etcdNodeNames)
+			fmt.Println("Control Plane Server Nodes:", cpNodeNames)
+			fmt.Println("Agent Nodes:", agentNodeNames)
+			kubeConfigFile, err = e2e.GenKubeConfigFile(cpNodeNames[0])
+			Expect(err).NotTo(HaveOccurred())
+		})
+
+		It("Checks Node and Pod Status", func() {
+			fmt.Printf("\nFetching node status\n")
+			Eventually(func(g Gomega) {
+				nodes, err := e2e.ParseNodes(kubeConfigFile, false)
+				g.Expect(err).NotTo(HaveOccurred())
+				for _, node := range nodes {
+					g.Expect(node.Status).Should(Equal("Ready"))
+				}
+			}, "420s", "5s").Should(Succeed())
+			_, _ = e2e.ParseNodes(kubeConfigFile, true)
+
+			fmt.Printf("\nFetching Pods status\n")
+			Eventually(func(g Gomega) {
+				pods, err := e2e.ParsePods(kubeConfigFile, false)
+				g.Expect(err).NotTo(HaveOccurred())
+				for _, pod := range pods {
+					if strings.Contains(pod.Name, "helm-install") {
+						g.Expect(pod.Status).Should(Equal("Completed"), pod.Name)
+					} else {
+						g.Expect(pod.Status).Should(Equal("Running"), pod.Name)
+					}
+				}
+			}, "420s", "5s").Should(Succeed())
+			_, _ = e2e.ParsePods(kubeConfigFile, true)
+		})
+
+		It("Verifies ClusterIP Service", func() {
+			_, err := e2e.DeployWorkload("clusterip.yaml", kubeConfigFile, false)
+			Expect(err).NotTo(HaveOccurred(), "Cluster IP manifest not deployed")
+
+			cmd := "kubectl get pods -o=name -l k8s-app=nginx-app-clusterip --field-selector=status.phase=Running --kubeconfig=" + kubeConfigFile
+			Eventually(func() (string, error) {
+				return e2e.RunCommand(cmd)
+			}, "240s", "5s").Should(ContainSubstring("test-clusterip"), "failed cmd: "+cmd)
+
+			clusterip, _ := e2e.FetchClusterIP(kubeConfigFile, "nginx-clusterip-svc", false)
+			cmd = "curl -L --insecure http://" + clusterip + "/name.html"
+			for _, nodeName := range cpNodeNames {
+				Eventually(func() (string, error) {
+					return e2e.RunCmdOnNode(cmd, nodeName)
+				}, "120s", "10s").Should(ContainSubstring("test-clusterip"), "failed cmd: "+cmd)
+			}
+		})
+
+		It("Verifies NodePort Service", func() {
+			_, err := e2e.DeployWorkload("nodeport.yaml", kubeConfigFile, false)
+			Expect(err).NotTo(HaveOccurred(), "NodePort manifest not deployed")
+
+			for _, nodeName := range cpNodeNames {
+				nodeExternalIP, _ := e2e.FetchNodeExternalIP(nodeName)
+				cmd := "kubectl get service nginx-nodeport-svc --kubeconfig=" + kubeConfigFile + " --output jsonpath=\"{.spec.ports[0].nodePort}\""
+				nodeport, err := e2e.RunCommand(cmd)
+				Expect(err).NotTo(HaveOccurred())
+
+				cmd = "kubectl get pods -o=name -l k8s-app=nginx-app-nodeport --field-selector=status.phase=Running --kubeconfig=" + kubeConfigFile
+				Eventually(func() (string, error) {
+					return e2e.RunCommand(cmd)
+				}, "240s", "5s").Should(ContainSubstring("test-nodeport"), "nodeport pod was not created")
+
+				cmd = "curl -L --insecure http://" + nodeExternalIP + ":" + nodeport + "/name.html"
+				Eventually(func() (string, error) {
+					return e2e.RunCommand(cmd)
+				}, "240s", "5s").Should(ContainSubstring("test-nodeport"), "failed cmd: "+cmd)
+			}
+		})
+
+		It("Verifies LoadBalancer Service", func() {
+			_, err := e2e.DeployWorkload("loadbalancer.yaml", kubeConfigFile, false)
+			Expect(err).NotTo(HaveOccurred(), "Loadbalancer manifest not deployed")
+
+			for _, nodeName := range cpNodeNames {
+				ip, _ := e2e.FetchNodeExternalIP(nodeName)
+
+				cmd := "kubectl get service nginx-loadbalancer-svc --kubeconfig=" + kubeConfigFile + " --output jsonpath=\"{.spec.ports[0].port}\""
+				port, err := e2e.RunCommand(cmd)
+				Expect(err).NotTo(HaveOccurred())
+
+				cmd = "kubectl get pods -o=name -l k8s-app=nginx-app-loadbalancer --field-selector=status.phase=Running --kubeconfig=" + kubeConfigFile
+				Eventually(func() (string, error) {
+					return e2e.RunCommand(cmd)
+				}, "240s", "5s").Should(ContainSubstring("test-loadbalancer"), "failed cmd: "+cmd)
+
+				cmd = "curl -L --insecure http://" + ip + ":" + port + "/name.html"
+				Eventually(func() (string, error) {
+					return e2e.RunCommand(cmd)
+				}, "240s", "5s").Should(ContainSubstring("test-loadbalancer"), "failed cmd: "+cmd)
+			}
+		})
+
+		It("Verifies Ingress", func() {
+			_, err := e2e.DeployWorkload("ingress.yaml", kubeConfigFile, false)
+			Expect(err).NotTo(HaveOccurred(), "Ingress manifest not deployed")
+
+			for _, nodeName := range cpNodeNames {
+				ip, _ := e2e.FetchNodeExternalIP(nodeName)
+				cmd := "curl  --header host:foo1.bar.com" + " http://" + ip + "/name.html"
+				Eventually(func() (string, error) {
+					return e2e.RunCommand(cmd)
+				}, "240s", "5s").Should(ContainSubstring("test-ingress"), "failed cmd: "+cmd)
+			}
+		})
+
+		It("Verifies Daemonset", func() {
+			_, err := e2e.DeployWorkload("daemonset.yaml", kubeConfigFile, false)
+			Expect(err).NotTo(HaveOccurred(), "Daemonset manifest not deployed")
+
+			Eventually(func(g Gomega) {
+				pods, _ := e2e.ParsePods(kubeConfigFile, false)
+				count := e2e.CountOfStringInSlice("test-daemonset", pods)
+				fmt.Println("POD COUNT")
+				fmt.Println(count)
+				fmt.Println("CP COUNT")
+				fmt.Println(len(cpNodeNames))
+				g.Expect(len(cpNodeNames)).Should((Equal(count)), "Daemonset pod count does not match cp node count")
+			}, "240s", "10s").Should(Succeed())
+		})
+
+		It("Verifies dns access", func() {
+			_, err := e2e.DeployWorkload("dnsutils.yaml", kubeConfigFile, false)
+			Expect(err).NotTo(HaveOccurred(), "dnsutils manifest not deployed")
+
+			cmd := "kubectl get pods dnsutils --kubeconfig=" + kubeConfigFile
+			Eventually(func() (string, error) {
+				return e2e.RunCommand(cmd)
+			}, "420s", "2s").Should(ContainSubstring("dnsutils"), "failed cmd: "+cmd)
+
+			cmd = "kubectl --kubeconfig=" + kubeConfigFile + " exec -i -t dnsutils -- nslookup kubernetes.default"
+			Eventually(func() (string, error) {
+				return e2e.RunCommand(cmd)
+			}, "420s", "2s").Should(ContainSubstring("kubernetes.default.svc.cluster.local"), "failed cmd: "+cmd)
+		})
+	})
+})
+
+var failed = false
+var _ = AfterEach(func() {
+	failed = failed || CurrentGinkgoTestDescription().Failed
+})
+
+var _ = AfterSuite(func() {
+	if failed {
+		fmt.Println("FAILED!")
+	} else {
+		Expect(e2e.DestroyCluster()).To(Succeed())
+		Expect(os.Remove(kubeConfigFile)).To(Succeed())
+	}
+})
--- a/tests/e2e/testutils.go
+++ b/tests/e2e/testutils.go
@ -97,7 +97,16 @@ func DestroyCluster() error {
 	return os.Remove("vagrant.log")
 }

-func FetchClusterIP(kubeconfig string, servicename string) (string, error) {
+func FetchClusterIP(kubeconfig string, servicename string, dualStack bool) (string, error) {
+	if dualStack {
+		cmd := "kubectl get svc " + servicename + " -o jsonpath='{.spec.clusterIPs}' --kubeconfig=" + kubeconfig
+		res, err := RunCommand(cmd)
+		if err != nil {
+			return res, err
+		}
+		res = strings.ReplaceAll(res, "\"", "")
+		return strings.Trim(res, "[]"), nil
+	}
 	cmd := "kubectl get svc " + servicename + " -o jsonpath='{.spec.clusterIP}' --kubeconfig=" + kubeconfig
 	return RunCommand(cmd)
 }
--- a/tests/e2e/upgradecluster/Vagrantfile
+++ b/tests/e2e/upgradecluster/Vagrantfile
@ -55,7 +55,7 @@ def provision(vm, roles, role_num, node_num)
      k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321
    end
  end
-  if vm.box.include?("microos")
+  if vm.box.to_s.include?("microos")
    vm.provision 'k3s-reload', type: 'reload', run: 'once'
  end
 end
--- a/tests/e2e/upgradecluster/upgradecluster_test.go
+++ b/tests/e2e/upgradecluster/upgradecluster_test.go
@ -82,7 +82,7 @@ var _ = Describe("Verify Upgrade", func() {
 				return e2e.RunCommand(cmd)
 			}, "240s", "5s").Should(ContainSubstring("test-clusterip"), "failed cmd: "+cmd)

-			clusterip, _ := e2e.FetchClusterIP(kubeConfigFile, "nginx-clusterip-svc")
+			clusterip, _ := e2e.FetchClusterIP(kubeConfigFile, "nginx-clusterip-svc", false)
 			cmd = "curl -L --insecure http://" + clusterip + "/name.html"
 			for _, nodeName := range serverNodeNames {
 				Eventually(func() (string, error) {
@ -278,7 +278,7 @@ var _ = Describe("Verify Upgrade", func() {
 				return e2e.RunCommand(cmd)
 			}, "420s", "5s").Should(ContainSubstring("test-clusterip"))

-			clusterip, _ := e2e.FetchClusterIP(kubeConfigFile, "nginx-clusterip-svc")
+			clusterip, _ := e2e.FetchClusterIP(kubeConfigFile, "nginx-clusterip-svc", false)
 			cmd := "curl -L --insecure http://" + clusterip + "/name.html"
 			fmt.Println(cmd)
 			for _, nodeName := range serverNodeNames {
--- a/tests/e2e/vagrantdefaults.rb
+++ b/tests/e2e/vagrantdefaults.rb
@ -1,13 +1,27 @@
 def defaultOSConfigure(vm)
-    if vm.box.include?("ubuntu2004")
-      vm.provision "shell", inline: "systemd-resolve --set-dns=8.8.8.8 --interface=eth0"
-      vm.provision "shell", inline: "apt install -y jq"
-    end
-    if vm.box.include?("Leap")
-      vm.provision "shell", inline: "zypper install -y jq"
-    end
-    if vm.box.include?("microos")
-      vm.provision "shell", inline: "transactional-update pkg install -y jq"
-      vm.provision 'reload', run: 'once'
-    end 
+  box = vm.box.to_s
+  if box.include?("generic/ubuntu")
+    vm.provision "Set DNS", type: "shell", inline: "netplan set ethernets.eth0.nameservers.addresses=[8.8.8.8,1.1.1.1]; netplan apply", run: 'once'
+    vm.provision "Install jq", type: "shell", inline: "apt install -y jq"
+  elsif box.include?("Leap") || box.include?("Tumbleweed")
+    vm.provision "Install jq", type: "shell", inline: "zypper install -y jq"
+  elsif box.include?("alpine")
+    vm.provision "Install tools", type: "shell", inline: "apk add jq coreutils"
+  elsif box.include?("microos")
+    vm.provision "Install jq", type: "shell", inline: "transactional-update pkg install -y jq"
+    vm.provision 'reload', run: 'once'
+  end 
+end
+
+def getInstallType(vm, release_version, branch)
+  if release_version == "skip"
+    install_type = "INSTALL_K3S_SKIP_DOWNLOAD=true"
+  elsif !release_version.empty?
+    return "INSTALL_K3S_VERSION=#{release_version}"
+  else
+    # Grabs the last 5 commit SHA's from the given branch, then purges any commits that do not have a passing CI build
+    # MicroOS requires it not be in a /tmp/ or other root system folder
+    vm.provision "Get latest commit", type: "shell", path: "../scripts/latest_commit.sh", args: [branch, "/tmp/k3s_commits"]
+    return "INSTALL_K3S_COMMIT=$(head\ -n\ 1\ /tmp/k3s_commits)"
+  end
 end
--- a/tests/e2e/validatecluster/Vagrantfile
+++ b/tests/e2e/validatecluster/Vagrantfile
@ -5,7 +5,9 @@ NODE_BOXES = (ENV['E2E_NODE_BOXES'] ||
  ['generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004'])
 GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master")
 RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "")
-EXTERNAL_DB = (ENV['E2E_EXTERNAL_DB'] || "mysql")
+EXTERNAL_DB = (ENV['E2E_EXTERNAL_DB'] || "etcd")
+HARDENED = (ENV['E2E_HARDENED'] || "")
+RANCHER = (ENV['E2E_RANCHER'] || "")
 NODE_CPUS = (ENV['E2E_NODE_CPUS'] || 2).to_i
 NODE_MEMORY = (ENV['E2E_NODE_MEMORY'] || 1024).to_i
 # Virtualbox >= 6.1.28 require `/etc/vbox/network.conf` for expanded private networks 
@ -23,38 +25,29 @@ def provision(vm, roles, role_num, node_num)
  load vagrant_defaults if File.exists?(vagrant_defaults)
  
  defaultOSConfigure(vm)
+  install_type = getInstallType(vm, RELEASE_VERSION, GITHUB_BRANCH)
  
-  if !RELEASE_VERSION.empty?
-    install_type = "INSTALL_K3S_VERSION=#{RELEASE_VERSION}"
-  else
-    # Grabs the last 5 commit SHA's from the given branch, then purges any commits that do not have a passing CI build
-    # MicroOS requires it not be in a /tmp/ or other root system folder
-    vm.provision "shell", path: "../scripts/latest_commit.sh", args: [GITHUB_BRANCH, "/home/vagrant/k3s_commits"]
-    install_type = "INSTALL_K3S_COMMIT=$(head\ -n\ 1\ /home/vagrant/k3s_commits)"
-  end
  vm.provision "shell", inline: "ping -c 2 k3s.io"
  
-  if roles.include?("server") && role_num == 0
-    if EXTERNAL_DB == "mysql"
-      dockerInstall(vm)
-      vm.provision "shell", inline: "docker run -d -p 3306:3306 --name #{EXTERNAL_DB} -e MYSQL_ROOT_PASSWORD=e2e mysql:5.7"
-      vm.provision "shell", inline: "echo \"Wait for mysql to startup\"; sleep 10"
-      db_type = "--datastore-endpoint='mysql://root:e2e@tcp(#{NETWORK_PREFIX}.100:3306)/k3s'"
-    elsif EXTERNAL_DB == "postgres"
-      dockerInstall(vm)
-      vm.provision "shell", inline: "docker run -d -p 5432:5432 --name #{EXTERNAL_DB} -e POSTGRES_PASSWORD=e2e  postgres:14-alpine"
-      vm.provision "shell", inline: "echo \"Wait for postgres to startup\"; sleep 10"
-      db_type = "--datastore-endpoint='postgres://postgres:e2e@#{NETWORK_PREFIX}.100:5432/k3s?sslmode=disable'"
-    elsif EXTERNAL_DB == "" || EXTERNAL_DB == "etcd"
-      db_type = "--cluster-init"
-    else
-      puts "Unknown EXTERNAL_DB: " + EXTERNAL_DB
-      abort
-    end
-    
-    vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s|
-      k3s.args = "server #{db_type} --node-external-ip=#{NETWORK_PREFIX}.100 --flannel-iface=eth1"
-      k3s.env = %W[K3S_KUBECONFIG_MODE=0644 K3S_TOKEN=vagrant #{install_type}]
+  db_type = getDBType(role, role_num, vm)
+
+  if !HARDENED.empty?
+    vm.provision "Set kernel parameters", type: "shell", path: scripts_location + "/harden.sh"
+    hardened_arg = "protect-kernel-defaults: true\nkube-apiserver-arg: \"enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount\""
+  end
+
+  if role.include?("server") && role_num == 0
+    vm.provision 'k3s-primary-server', type: 'k3s', run: 'once' do |k3s|
+      k3s.args = "server "
+      k3s.config = <<~YAML
+        token: vagrant
+        node-external-ip: #{NETWORK_PREFIX}.100
+        flannel-iface: eth1
+        tls-san: #{NETWORK_PREFIX}.100.nip.io
+        #{db_type}
+        #{hardened_arg}
+      YAML
+      k3s.env = %W[K3S_KUBECONFIG_MODE=0644 #{install_type}]
      k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321
    end
  elsif roles.include?("server") && role_num != 0
@ -77,12 +70,16 @@ def provision(vm, roles, role_num, node_num)
      k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321
    end
  end
-  if vm.box.include?("microos")
+  if vm.box.to_s.include?("microos")
    vm.provision 'k3s-reload', type: 'reload', run: 'once'
    if !EXTERNAL_DB.empty?
      vm.provision "shell", inline: "docker start #{EXTERNAL_DB}"
    end
  end
+  # This step does not run by default and is designed to be called by higher level tools
+  if !RANCHER.empty?
+    vm.provision "Install Rancher", type: "shell", run: "never", path: scripts_location + "/rancher.sh", args: node_ip
+  end
 end

 def dockerInstall(vm)
@ -92,14 +89,14 @@ def dockerInstall(vm)
  vm.provider "virtualbox" do |v|
    v.memory = NODE_MEMORY + 1024
  end
-  if vm.box.include?("ubuntu2004")
+  if vm.box.to_s.include?("ubuntu")
    vm.provision "shell", inline: "apt install -y docker.io"
  end
-  if vm.box.include?("Leap")
-    vm.provision "shell", inline: "zypper install -y docker"
+  if vm.box.to_s.include?("Leap")
+    vm.provision "shell", inline: "zypper install -y docker apparmor-parser"
  end
-  if vm.box.include?("microos")
-    vm.provision "shell", inline: "transactional-update pkg install -y docker"
+  if vm.box.to_s.include?("microos")
+    vm.provision "shell", inline: "transactional-update pkg install -y docker apparmor-parser"
    vm.provision 'docker-reload', type: 'reload', run: 'once'
    vm.provision "shell", inline: "systemctl enable --now docker"
  end
--- a/tests/e2e/validatecluster/validatecluster_test.go
+++ b/tests/e2e/validatecluster/validatecluster_test.go
@ -84,7 +84,7 @@ var _ = Describe("Verify Create", func() {
 				g.Expect(res).Should((ContainSubstring("test-clusterip")))
 			}, "240s", "5s").Should(Succeed())

-			clusterip, _ := e2e.FetchClusterIP(kubeConfigFile, "nginx-clusterip-svc")
+			clusterip, _ := e2e.FetchClusterIP(kubeConfigFile, "nginx-clusterip-svc", false)
 			cmd := "curl -L --insecure http://" + clusterip + "/name.html"
 			fmt.Println(cmd)
 			for _, nodeName := range serverNodeNames {
--- a/tests/integration/dualstack/dualstack_int_test.go
+++ b/tests/integration/dualstack/dualstack_int_test.go
@ -39,9 +39,9 @@ var _ = Describe("dual stack", func() {
 	})
 	When("a ipv4 and ipv6 cidr is present", func() {
 		It("starts up with no problems", func() {
-			Eventually(func() (string, error) {
-				return testutil.K3sCmd("kubectl", "get", "pods", "-A")
-			}, "180s", "5s").Should(MatchRegexp("kube-system.+traefik.+1\\/1.+Running"))
+			Eventually(func() error {
+				return testutil.K3sDefaultDeployments()
+			}, "180s", "10s").Should(Succeed())
 		})
 		It("creates pods with two IPs", func() {
 			podname, err := testutil.K3sCmd("kubectl", "get", "pods", "-n", "kube-system", "-o", "jsonpath={.items[?(@.metadata.labels.app\\.kubernetes\\.io/name==\"traefik\")].metadata.name}")
--- a/tests/integration/etcdrestore/etcd_restore_int_test.go
+++ b/tests/integration/etcdrestore/etcd_restore_int_test.go
@ -32,9 +32,9 @@ var _ = Describe("etcd snapshot restore", func() {
 	})
 	When("a snapshot is restored on existing node", func() {
 		It("etcd starts up with no problems", func() {
-			Eventually(func() (string, error) {
-				return testutil.K3sCmd("kubectl", "get", "pods", "-A")
-			}, "360s", "5s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
+			Eventually(func() error {
+				return testutil.K3sDefaultDeployments()
+			}, "180s", "5s").Should(Succeed())
 		})
 		It("create a workload", func() {
 			result, err := testutil.K3sCmd("kubectl", "create", "-f", "./testdata/temp_depl.yaml")
@ -79,9 +79,9 @@ var _ = Describe("etcd snapshot restore", func() {
 			Expect(err).ToNot(HaveOccurred())
 		})
 		It("starts up with no problems", func() {
-			Eventually(func() (string, error) {
-				return testutil.K3sCmd("kubectl", "get", "pods", "-A")
-			}, "360s", "5s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
+			Eventually(func() error {
+				return testutil.K3sDefaultDeployments()
+			}, "360s", "5s").Should(Succeed())
 		})
 		It("Make sure Workload 1 exists", func() {
 			Eventually(func() (string, error) {
--- a/tests/integration/etcdsnapshot/etcdsnapshot_int_test.go
+++ b/tests/integration/etcdsnapshot/etcdsnapshot_int_test.go
@ -33,9 +33,9 @@ var _ = Describe("etcd snapshots", func() {
 	})
 	When("a new etcd is created", func() {
 		It("starts up with no problems", func() {
-			Eventually(func() (string, error) {
-				return testutil.K3sCmd("kubectl", "get", "pods", "-A")
-			}, "90s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
+			Eventually(func() error {
+				return testutil.K3sDefaultDeployments()
+			}, "180s", "10s").Should(Succeed())
 		})
 		It("saves an etcd snapshot", func() {
 			Expect(testutil.K3sCmd("etcd-snapshot", "save")).
--- a/tests/integration/localstorage/localstorage_int_test.go
+++ b/tests/integration/localstorage/localstorage_int_test.go
@ -34,9 +34,9 @@ var _ = Describe("local storage", func() {
 	})
 	When("a new local storage is created", func() {
 		It("starts up with no problems", func() {
-			Eventually(func() (string, error) {
-				return testutil.K3sCmd("kubectl", "get", "pods", "-A")
-			}, "90s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
+			Eventually(func() error {
+				return testutil.K3sDefaultDeployments()
+			}, "120s", "5s").Should(Succeed())
 		})
 		It("creates a new pvc", func() {
 			result, err := testutil.K3sCmd("kubectl", "create", "-f", "./testdata/localstorage_pvc.yaml")
--- a/tests/integration/secretsencryption/secretsencryption_int_test.go
+++ b/tests/integration/secretsencryption/secretsencryption_int_test.go
@ -34,9 +34,9 @@ var _ = Describe("secrets encryption rotation", func() {
 	})
 	When("A server starts with secrets encryption", func() {
 		It("starts up with no problems", func() {
-			Eventually(func() (string, error) {
-				return testutil.K3sCmd("kubectl", "get", "pods", "-A")
-			}, "180s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
+			Eventually(func() error {
+				return testutil.K3sDefaultDeployments()
+			}, "180s", "5s").Should(Succeed())
 		})
 		It("it creates a encryption key", func() {
 			result, err := testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir)
@ -65,9 +65,9 @@ var _ = Describe("secrets encryption rotation", func() {
 			Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed())
 			secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...)
 			Expect(err).ToNot(HaveOccurred())
-			Eventually(func() (string, error) {
-				return testutil.K3sCmd("kubectl", "get", "pods", "-A")
-			}, "180s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
+			Eventually(func() error {
+				return testutil.K3sDefaultDeployments()
+			}, "180s", "5s").Should(Succeed())
 		})
 		It("rotates the keys", func() {
 			Eventually(func() (string, error) {
@ -89,9 +89,9 @@ var _ = Describe("secrets encryption rotation", func() {
 			Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed())
 			secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...)
 			Expect(err).ToNot(HaveOccurred())
-			Eventually(func() (string, error) {
-				return testutil.K3sCmd("kubectl", "get", "pods", "-A")
-			}, "180s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
+			Eventually(func() error {
+				return testutil.K3sDefaultDeployments()
+			}, "180s", "5s").Should(Succeed())
 			time.Sleep(10 * time.Second)
 		})
 		It("reencrypts the keys", func() {
@ -123,9 +123,9 @@ var _ = Describe("secrets encryption rotation", func() {
 			Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed())
 			secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...)
 			Expect(err).ToNot(HaveOccurred())
-			Eventually(func() (string, error) {
-				return testutil.K3sCmd("kubectl", "get", "pods", "-A")
-			}, "180s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
+			Eventually(func() error {
+				return testutil.K3sDefaultDeployments()
+			}, "180s", "5s").Should(Succeed())
 			time.Sleep(10 * time.Second)
 		})
 		It("reencrypts the keys", func() {
--- a/tests/integration/startup/startup_int_test.go
+++ b/tests/integration/startup/startup_int_test.go
@ -0,0 +1,123 @@
+package integration
+
+import (
+	"testing"
+
+	testutil "github.com/k3s-io/k3s/tests/util"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	v1 "k8s.io/api/core/v1"
+)
+
+var startupServer *testutil.K3sServer
+var startupServerArgs = []string{}
+var testLock int
+
+var _ = BeforeSuite(func() {
+	if testutil.IsExistingServer() {
+		Skip("Test does not support running on existing k3s servers")
+	}
+	var err error
+	testLock, err = testutil.K3sTestLock()
+	Expect(err).ToNot(HaveOccurred())
+})
+
+var _ = Describe("startup tests", func() {
+
+	When("a default server is created", func() {
+		It("is created with no arguments", func() {
+			var err error
+			startupServer, err = testutil.K3sStartServer(startupServerArgs...)
+			Expect(err).ToNot(HaveOccurred())
+		})
+		It("has the default pods deployed", func() {
+			Eventually(func() error {
+				return testutil.K3sDefaultDeployments()
+			}, "90s", "5s").Should(Succeed())
+		})
+		It("dies cleanly", func() {
+			Expect(testutil.K3sKillServer(startupServer)).To(Succeed())
+		})
+	})
+	When("a etcd backed server is created", func() {
+		It("is created with cluster-init arguments", func() {
+			var err error
+			startupServerArgs = []string{"--cluster-init"}
+			startupServer, err = testutil.K3sStartServer(startupServerArgs...)
+			Expect(err).ToNot(HaveOccurred())
+		})
+		It("has the default pods deployed", func() {
+			Eventually(func() error {
+				return testutil.K3sDefaultDeployments()
+			}, "90s", "5s").Should(Succeed())
+		})
+		It("dies cleanly", func() {
+			Expect(testutil.K3sKillServer(startupServer)).To(Succeed())
+		})
+	})
+	When("a server without traefik is created", func() {
+		It("is created with disable arguments", func() {
+			var err error
+			startupServerArgs = []string{"--disable", "traefik"}
+			startupServer, err = testutil.K3sStartServer(startupServerArgs...)
+			Expect(err).ToNot(HaveOccurred())
+		})
+		It("has the default pods without traefik deployed", func() {
+			Eventually(func() error {
+				return testutil.CheckDeployments([]string{"coredns", "local-path-provisioner", "metrics-server"})
+			}, "90s", "10s").Should(Succeed())
+		})
+		It("dies cleanly", func() {
+			Expect(testutil.K3sKillServer(startupServer)).To(Succeed())
+			Expect(testutil.K3sCleanup(-1, "")).To(Succeed())
+		})
+	})
+	When("a server with different IPs is created", func() {
+		It("creates dummy interfaces", func() {
+			Expect(testutil.RunCommand("ip link add dummy2 type dummy")).To(Equal(""))
+			Expect(testutil.RunCommand("ip link add dummy3 type dummy")).To(Equal(""))
+			Expect(testutil.RunCommand("ip addr add 11.22.33.44/24 dev dummy2")).To(Equal(""))
+			Expect(testutil.RunCommand("ip addr add 55.66.77.88/24 dev dummy3")).To(Equal(""))
+		})
+		It("is created with node-ip arguments", func() {
+			var err error
+			startupServerArgs = []string{"--node-ip", "11.22.33.44", "--node-external-ip", "55.66.77.88"}
+			startupServer, err = testutil.K3sStartServer(startupServerArgs...)
+			Expect(err).ToNot(HaveOccurred())
+		})
+		It("has the node deployed with correct IPs", func() {
+			Eventually(func() error {
+				return testutil.K3sDefaultDeployments()
+			}, "90s", "10s").Should(Succeed())
+
+			nodes, err := testutil.ParseNodes()
+			Expect(err).NotTo(HaveOccurred())
+			Expect(nodes).To(HaveLen(1))
+			Expect(nodes[0].Status.Addresses).To(ContainElements([]v1.NodeAddress{
+				{
+					Type:    "InternalIP",
+					Address: "11.22.33.44",
+				},
+				{
+					Type:    "ExternalIP",
+					Address: "55.66.77.88",
+				}}))
+		})
+		It("dies cleanly", func() {
+			Expect(testutil.K3sKillServer(startupServer)).To(Succeed())
+			Expect(testutil.RunCommand("ip link del dummy2")).To(Equal(""))
+			Expect(testutil.RunCommand("ip link del dummy3")).To(Equal(""))
+		})
+	})
+})
+
+var _ = AfterSuite(func() {
+	if !testutil.IsExistingServer() {
+		Expect(testutil.K3sCleanup(testLock, "")).To(Succeed())
+	}
+})
+
+func Test_IntegrationStartup(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Startup Suite")
+}
--- a/tests/util/cmd.go
+++ b/tests/util/cmd.go
@ -3,6 +3,7 @@ package util
 import (
 	"bufio"
 	"bytes"
+	"context"
 	"encoding/json"
 	"fmt"
 	"os"
@ -15,6 +16,10 @@ import (
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"github.com/vishvananda/netlink"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
 )

 // Compile-time variable
@ -116,6 +121,67 @@ func K3sServerArgs() []string {
 	return args
 }

+// K3sDefaultDeployments checks if the default deployments for K3s are ready, otherwise returns an error
+func K3sDefaultDeployments() error {
+	return CheckDeployments([]string{"coredns", "local-path-provisioner", "metrics-server", "traefik"})
+}
+
+// CheckDeployments checks if the provided list of deployments are ready, otherwise returns an error
+func CheckDeployments(deployments []string) error {
+
+	deploymentSet := make(map[string]bool)
+	for _, d := range deployments {
+		deploymentSet[d] = false
+	}
+
+	client, err := k8sClient()
+	if err != nil {
+		return err
+	}
+	deploymentList, err := client.AppsV1().Deployments("").List(context.Background(), metav1.ListOptions{})
+	if err != nil {
+		return err
+	}
+	for _, deployment := range deploymentList.Items {
+		if _, ok := deploymentSet[deployment.Name]; ok && deployment.Status.ReadyReplicas == deployment.Status.Replicas {
+			deploymentSet[deployment.Name] = true
+		}
+	}
+	for d, found := range deploymentSet {
+		if !found {
+			return fmt.Errorf("failed to deploy %s", d)
+		}
+	}
+
+	return nil
+}
+
+func ParsePods() ([]corev1.Pod, error) {
+	clientSet, err := k8sClient()
+	if err != nil {
+		return nil, err
+	}
+	pods, err := clientSet.CoreV1().Pods("").List(context.Background(), metav1.ListOptions{})
+	if err != nil {
+		return nil, err
+	}
+
+	return pods.Items, nil
+}
+
+func ParseNodes() ([]corev1.Node, error) {
+	clientSet, err := k8sClient()
+	if err != nil {
+		return nil, err
+	}
+	nodes, err := clientSet.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{})
+	if err != nil {
+		return nil, err
+	}
+
+	return nodes.Items, nil
+}
+
 func FindStringInCmdAsync(scanner *bufio.Scanner, target string) bool {
 	for scanner.Scan() {
 		if strings.Contains(scanner.Text(), target) {
@ -154,7 +220,6 @@ func K3sStartServer(inputArgs ...string) (*K3sServer, error) {
 }

 // K3sKillServer terminates the running K3s server and its children
-// and unlocks the file for other tests
 func K3sKillServer(server *K3sServer) error {
 	pgid, err := syscall.Getpgid(server.cmd.Process.Pid)
 	if err != nil {
@ -176,7 +241,10 @@ func K3sKillServer(server *K3sServer) error {
 	return nil
 }

-// K3sCleanup attempts to cleanup networking and files leftover from an integration test
+// K3sCleanup unlocks the test-lock and
+// attempts to cleanup networking and files leftover from an integration test.
+// This is similar to the k3s-killall.sh script, but we dynamically generate that on
+// install, so we don't have access to it during testing.
 func K3sCleanup(k3sTestLock int, dataDir string) error {
 	if cni0Link, err := netlink.LinkByName("cni0"); err == nil {
 		links, _ := netlink.LinkList()
@ -200,7 +268,10 @@ func K3sCleanup(k3sTestLock int, dataDir string) error {
 	if err := os.RemoveAll(dataDir); err != nil {
 		return err
 	}
-	return flock.Release(k3sTestLock)
+	if k3sTestLock != -1 {
+		return flock.Release(k3sTestLock)
+	}
+	return nil
 }

 // RunCommand Runs command on the cluster accessing the cluster through kubeconfig file
@ -214,3 +285,15 @@ func RunCommand(cmd string) (string, error) {
 	}
 	return out.String(), nil
 }
+
+func k8sClient() (*kubernetes.Clientset, error) {
+	config, err := clientcmd.BuildConfigFromFlags("", "/etc/rancher/k3s/k3s.yaml")
+	if err != nil {
+		return nil, err
+	}
+	clientSet, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		return nil, err
+	}
+	return clientSet, nil
+}