mirror of https://github.com/k3s-io/k3s
[Release-1.23] Backport missing E2E test commits (#6699)
* Backport PR 6131 Signed-off-by: Derek Nola <derek.nola@suse.com> * Add cluster reset test to nightly builds * Add journalctl logs to E2E tests * Expand nightly E2E tests (#6354) * Add snapshot restore e2e test (#6396) * Convert test output to JSON format (#6410) * Fix E2E test for prefer-bundled-bin * Fix external ip test Signed-off-by: Shylaja Devadiga <shylaja@rancher.com> Signed-off-by: Derek Nola <derek.nola@suse.com>pull/6749/head
Derek Nola
GitHub
parent
50cab3b326
commit
71a09aaba9
|
|
@ -0,0 +1,39 @@
|
|||
name: Build K3s
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
upload-repo:
|
||||
type: boolean
|
||||
required: false
|
||||
default: false
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: Build
|
||||
runs-on: ubuntu-20.04
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- name: Checkout K3s
|
||||
uses: actions/checkout@v3
|
||||
- name: Build K3s binary
|
||||
run: |
|
||||
DOCKER_BUILDKIT=1 SKIP_AIRGAP=1 SKIP_VALIDATE=1 make
|
||||
|
||||
- name: bundle repo
|
||||
if: inputs.upload-repo == true
|
||||
run: |
|
||||
tar -czvf ../k3s-repo.tar.gz .
|
||||
mv ../k3s-repo.tar.gz .
|
||||
- name: "Upload K3s directory"
|
||||
if: inputs.upload-repo == true
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: k3s-repo.tar.gz
|
||||
path: k3s-repo.tar.gz
|
||||
- name: "Upload K3s binary"
|
||||
if: inputs.upload-repo == false
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: k3s
|
||||
path: dist/artifacts/k3s
|
||||
|
|
@ -5,9 +5,8 @@ on:
|
|||
- "**.md"
|
||||
- "channel.yaml"
|
||||
- "install.sh"
|
||||
- "tests/snapshotter/**"
|
||||
- "tests/install/**"
|
||||
- "tests/cgroup/**"
|
||||
- "tests/**"
|
||||
- "!tests/integration**"
|
||||
- ".github/**"
|
||||
- "!.github/workflows/integration.yaml"
|
||||
pull_request:
|
||||
|
|
@ -15,29 +14,14 @@ on:
|
|||
- "**.md"
|
||||
- "channel.yaml"
|
||||
- "install.sh"
|
||||
- "tests/snapshotter/**"
|
||||
- "tests/install/**"
|
||||
- "tests/cgroup/**"
|
||||
- "tests/**"
|
||||
- "!tests/integration**"
|
||||
- ".github/**"
|
||||
- "!.github/workflows/integration.yaml"
|
||||
workflow_dispatch: {}
|
||||
jobs:
|
||||
build:
|
||||
name: Build
|
||||
runs-on: ubuntu-20.04
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- name: "Checkout"
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
fetch-depth: 1
|
||||
- name: "Make"
|
||||
run: DOCKER_BUILDKIT=1 SKIP_VALIDATE=1 make
|
||||
- name: "Upload k3s binary"
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: k3s
|
||||
path: dist/artifacts/k3s
|
||||
uses: ./.github/workflows/build-k3s.yaml
|
||||
test:
|
||||
needs: build
|
||||
name: Integration Tests
|
||||
|
|
|
|||
1
go.mod
1
go.mod
|
|
@ -91,6 +91,7 @@ require (
|
|||
github.com/gorilla/mux v1.8.0
|
||||
github.com/gorilla/websocket v1.4.2
|
||||
github.com/gruntwork-io/terratest v0.40.6
|
||||
github.com/json-iterator/go v1.1.12
|
||||
github.com/k3s-io/helm-controller v0.13.1
|
||||
github.com/k3s-io/kine v0.9.6
|
||||
github.com/klauspost/compress v1.15.12
|
||||
|
|
|
|||
|
|
@ -0,0 +1,136 @@
|
|||
ENV['VAGRANT_NO_PARALLEL'] = 'no'
|
||||
NODE_ROLES = (ENV['E2E_NODE_ROLES'] ||
|
||||
["server-0", "server-1", "server-2", "agent-0", "agent-1"])
|
||||
NODE_BOXES = (ENV['E2E_NODE_BOXES'] ||
|
||||
['generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004'])
|
||||
GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master")
|
||||
RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "")
|
||||
EXTERNAL_DB = (ENV['E2E_EXTERNAL_DB'] || "etcd")
|
||||
HARDENED = (ENV['E2E_HARDENED'] || "")
|
||||
NODE_CPUS = (ENV['E2E_NODE_CPUS'] || 2).to_i
|
||||
NODE_MEMORY = (ENV['E2E_NODE_MEMORY'] || 1024).to_i
|
||||
# Virtualbox >= 6.1.28 require `/etc/vbox/network.conf` for expanded private networks
|
||||
NETWORK_PREFIX = "10.10.10"
|
||||
install_type = ""
|
||||
hardened_arg = ""
|
||||
|
||||
def provision(vm, role, role_num, node_num)
|
||||
vm.box = NODE_BOXES[node_num]
|
||||
vm.hostname = role
|
||||
# An expanded netmask is required to allow VM<-->VM communication, virtualbox defaults to /32
|
||||
node_ip = "#{NETWORK_PREFIX}.#{100+node_num}"
|
||||
vm.network "private_network", ip: node_ip, netmask: "255.255.255.0"
|
||||
|
||||
scripts_location = Dir.exists?("./scripts") ? "./scripts" : "../scripts"
|
||||
vagrant_defaults = File.exists?("./vagrantdefaults.rb") ? "./vagrantdefaults.rb" : "../vagrantdefaults.rb"
|
||||
load vagrant_defaults
|
||||
|
||||
defaultOSConfigure(vm)
|
||||
install_type = getInstallType(vm, RELEASE_VERSION, GITHUB_BRANCH)
|
||||
|
||||
vm.provision "shell", inline: "ping -c 2 k3s.io"
|
||||
|
||||
db_type = getDBType(role, role_num, vm)
|
||||
|
||||
if !HARDENED.empty?
|
||||
vm.provision "Set kernel parameters", type: "shell", path: scripts_location + "/harden.sh"
|
||||
hardened_arg = "protect-kernel-defaults: true\nkube-apiserver-arg: \"enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount\""
|
||||
end
|
||||
|
||||
if role.include?("server") && role_num == 0
|
||||
vm.provision 'k3s-primary-server', type: 'k3s', run: 'once' do |k3s|
|
||||
k3s.args = "server "
|
||||
k3s.config = <<~YAML
|
||||
token: vagrant
|
||||
node-external-ip: #{NETWORK_PREFIX}.100
|
||||
flannel-iface: eth1
|
||||
tls-san: #{NETWORK_PREFIX}.100.nip.io
|
||||
#{db_type}
|
||||
#{hardened_arg}
|
||||
YAML
|
||||
k3s.env = %W[K3S_KUBECONFIG_MODE=0644 #{install_type}]
|
||||
k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321
|
||||
end
|
||||
|
||||
elsif role.include?("server") && role_num != 0
|
||||
vm.provision 'k3s-secondary-server', type: 'k3s', run: 'once' do |k3s|
|
||||
k3s.args = "server"
|
||||
k3s.config = <<~YAML
|
||||
server: "https://#{NETWORK_PREFIX}.100:6443"
|
||||
token: vagrant
|
||||
node-external-ip: #{node_ip}
|
||||
flannel-iface: eth1
|
||||
#{db_type}
|
||||
#{hardened_arg}
|
||||
YAML
|
||||
k3s.env = %W[K3S_KUBECONFIG_MODE=0644 K3S_TOKEN=vagrant #{install_type}]
|
||||
k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321
|
||||
end
|
||||
end
|
||||
|
||||
if role.include?("agent")
|
||||
vm.provision 'k3s-agent', type: 'k3s', run: 'once' do |k3s|
|
||||
k3s.args = "agent"
|
||||
k3s.config = <<~YAML
|
||||
server: "https://#{NETWORK_PREFIX}.100:6443"
|
||||
token: vagrant
|
||||
node-external-ip: #{node_ip}
|
||||
flannel-iface: eth1
|
||||
#{db_type}
|
||||
#{hardened_arg}
|
||||
YAML
|
||||
k3s.env = %W[K3S_KUBECONFIG_MODE=0644 #{install_type}]
|
||||
k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321
|
||||
end
|
||||
end
|
||||
if vm.box.to_s.include?("microos")
|
||||
vm.provision 'k3s-reload', type: 'reload', run: 'once'
|
||||
if !EXTERNAL_DB.empty?
|
||||
vm.provision "shell", inline: "docker start #{EXTERNAL_DB}"
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def getDBType(role, role_num, vm)
|
||||
if ( EXTERNAL_DB == "" || EXTERNAL_DB == "etcd" )
|
||||
if role.include?("server") && role_num == 0
|
||||
return "cluster-init: true"
|
||||
end
|
||||
elsif ( EXTERNAL_DB == "none" )
|
||||
# Use internal sqlite, only valid for single node clusters
|
||||
else
|
||||
puts "Unknown EXTERNAL_DB: " + EXTERNAL_DB
|
||||
abort
|
||||
end
|
||||
return ""
|
||||
end
|
||||
|
||||
Vagrant.configure("2") do |config|
|
||||
config.vagrant.plugins = ["vagrant-k3s", "vagrant-reload"]
|
||||
# Default provider is libvirt, virtualbox is only provided as a backup
|
||||
config.vm.provider "libvirt" do |v|
|
||||
v.cpus = NODE_CPUS
|
||||
v.memory = NODE_MEMORY
|
||||
end
|
||||
config.vm.provider "virtualbox" do |v|
|
||||
v.cpus = NODE_CPUS
|
||||
v.memory = NODE_MEMORY
|
||||
end
|
||||
|
||||
if NODE_ROLES.kind_of?(String)
|
||||
NODE_ROLES = NODE_ROLES.split(" ", -1)
|
||||
end
|
||||
if NODE_BOXES.kind_of?(String)
|
||||
NODE_BOXES = NODE_BOXES.split(" ", -1)
|
||||
end
|
||||
|
||||
# Must iterate on the index, vagrant does not understand iterating
|
||||
# over the node roles themselves
|
||||
NODE_ROLES.length.times do |i|
|
||||
name = NODE_ROLES[i]
|
||||
role_num = name.split("-", -1).pop.to_i
|
||||
config.vm.define name do |node|
|
||||
provision(node.vm, name, role_num, i)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
@ -0,0 +1,187 @@
|
|||
package clusterreset
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/k3s-io/k3s/tests/e2e"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
// Valid nodeOS:
|
||||
// generic/ubuntu2004, generic/centos7, generic/rocky8,
|
||||
// opensuse/Leap-15.3.x86_64, dweomer/microos.amd64
|
||||
var nodeOS = flag.String("nodeOS", "generic/ubuntu2004", "VM operating system")
|
||||
var serverCount = flag.Int("serverCount", 3, "number of server nodes")
|
||||
var agentCount = flag.Int("agentCount", 1, "number of agent nodes")
|
||||
var hardened = flag.Bool("hardened", false, "true or false")
|
||||
var ci = flag.Bool("ci", false, "running on CI")
|
||||
var local = flag.Bool("local", false, "deploy a locally built K3s binary")
|
||||
|
||||
// Environment Variables Info:
|
||||
// E2E_EXTERNAL_DB: mysql, postgres, etcd (default: etcd)
|
||||
// E2E_RELEASE_VERSION=v1.23.1+k3s2 (default: latest commit from master)
|
||||
|
||||
func Test_E2EClusterReset(t *testing.T) {
|
||||
RegisterFailHandler(Fail)
|
||||
flag.Parse()
|
||||
suiteConfig, reporterConfig := GinkgoConfiguration()
|
||||
RunSpecs(t, "ClusterReset Test Suite", suiteConfig, reporterConfig)
|
||||
}
|
||||
|
||||
var (
|
||||
kubeConfigFile string
|
||||
serverNodeNames []string
|
||||
agentNodeNames []string
|
||||
)
|
||||
|
||||
var _ = ReportAfterEach(e2e.GenReport)
|
||||
|
||||
var _ = Describe("Verify Create", Ordered, func() {
|
||||
Context("Cluster :", func() {
|
||||
It("Starts up with no issues", func() {
|
||||
var err error
|
||||
if *local {
|
||||
serverNodeNames, agentNodeNames, err = e2e.CreateLocalCluster(*nodeOS, *serverCount, *agentCount)
|
||||
} else {
|
||||
serverNodeNames, agentNodeNames, err = e2e.CreateCluster(*nodeOS, *serverCount, *agentCount)
|
||||
}
|
||||
Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err))
|
||||
fmt.Println("CLUSTER CONFIG")
|
||||
fmt.Println("OS:", *nodeOS)
|
||||
fmt.Println("Server Nodes:", serverNodeNames)
|
||||
fmt.Println("Agent Nodes:", agentNodeNames)
|
||||
kubeConfigFile, err = e2e.GenKubeConfigFile(serverNodeNames[0])
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
})
|
||||
|
||||
It("Checks Node and Pod Status", func() {
|
||||
fmt.Printf("\nFetching node status\n")
|
||||
Eventually(func(g Gomega) {
|
||||
nodes, err := e2e.ParseNodes(kubeConfigFile, false)
|
||||
g.Expect(err).NotTo(HaveOccurred())
|
||||
for _, node := range nodes {
|
||||
g.Expect(node.Status).Should(Equal("Ready"))
|
||||
}
|
||||
}, "420s", "5s").Should(Succeed())
|
||||
_, _ = e2e.ParseNodes(kubeConfigFile, true)
|
||||
|
||||
fmt.Printf("\nFetching Pods status\n")
|
||||
Eventually(func(g Gomega) {
|
||||
pods, err := e2e.ParsePods(kubeConfigFile, false)
|
||||
g.Expect(err).NotTo(HaveOccurred())
|
||||
for _, pod := range pods {
|
||||
if strings.Contains(pod.Name, "helm-install") {
|
||||
g.Expect(pod.Status).Should(Equal("Completed"), pod.Name)
|
||||
} else {
|
||||
g.Expect(pod.Status).Should(Equal("Running"), pod.Name)
|
||||
}
|
||||
}
|
||||
}, "420s", "5s").Should(Succeed())
|
||||
_, _ = e2e.ParsePods(kubeConfigFile, true)
|
||||
})
|
||||
|
||||
It("Verifies ClusterReset Functionality", func() {
|
||||
Eventually(func(g Gomega) {
|
||||
for _, nodeName := range serverNodeNames {
|
||||
if nodeName != "server-0" {
|
||||
cmd := "sudo systemctl stop k3s"
|
||||
_, err := e2e.RunCmdOnNode(cmd, nodeName)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
}
|
||||
}
|
||||
|
||||
cmd := "sudo systemctl stop k3s"
|
||||
_, err := e2e.RunCmdOnNode(cmd, "server-0")
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
cmd = "sudo k3s server --cluster-reset"
|
||||
res, err := e2e.RunCmdOnNode(cmd, "server-0")
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(res).Should(ContainSubstring("Managed etcd cluster membership has been reset, restart without --cluster-reset flag now"))
|
||||
|
||||
cmd = "sudo systemctl start k3s"
|
||||
_, err = e2e.RunCmdOnNode(cmd, "server-0")
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
fmt.Printf("\nFetching node status\n")
|
||||
Eventually(func(g Gomega) {
|
||||
nodes, err := e2e.ParseNodes(kubeConfigFile, false)
|
||||
g.Expect(err).NotTo(HaveOccurred())
|
||||
for _, node := range nodes {
|
||||
if strings.Contains(node.Name, "server-0") || strings.Contains(node.Name, "agent-") {
|
||||
g.Expect(node.Status).Should(Equal("Ready"))
|
||||
} else {
|
||||
g.Expect(node.Status).Should(Equal("NotReady"))
|
||||
}
|
||||
}
|
||||
}, "480s", "5s").Should(Succeed())
|
||||
_, _ = e2e.ParseNodes(kubeConfigFile, true)
|
||||
|
||||
fmt.Printf("\nFetching Pods status\n")
|
||||
Eventually(func(g Gomega) {
|
||||
pods, err := e2e.ParsePods(kubeConfigFile, false)
|
||||
g.Expect(err).NotTo(HaveOccurred())
|
||||
for _, pod := range pods {
|
||||
if strings.Contains(pod.Name, "helm-install") {
|
||||
g.Expect(pod.Status).Should(Equal("Completed"), pod.Name)
|
||||
} else {
|
||||
g.Expect(pod.Status).Should(Equal("Running"), pod.Name)
|
||||
}
|
||||
}
|
||||
}, "420s", "5s").Should(Succeed())
|
||||
_, _ = e2e.ParsePods(kubeConfigFile, true)
|
||||
for _, nodeName := range serverNodeNames {
|
||||
if nodeName != "server-0" {
|
||||
cmd := "sudo rm -rf /var/lib/rancher/k3s/server/db"
|
||||
_, err := e2e.RunCmdOnNode(cmd, nodeName)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
cmd = "sudo systemctl restart k3s"
|
||||
_, err = e2e.RunCmdOnNode(cmd, nodeName)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
}
|
||||
}
|
||||
Eventually(func(g Gomega) {
|
||||
nodes, err := e2e.ParseNodes(kubeConfigFile, false)
|
||||
g.Expect(err).NotTo(HaveOccurred())
|
||||
for _, node := range nodes {
|
||||
g.Expect(node.Status).Should(Equal("Ready"))
|
||||
}
|
||||
}, "420s", "5s").Should(Succeed())
|
||||
_, _ = e2e.ParseNodes(kubeConfigFile, true)
|
||||
|
||||
fmt.Printf("\nFetching Pods status\n")
|
||||
Eventually(func(g Gomega) {
|
||||
pods, err := e2e.ParsePods(kubeConfigFile, false)
|
||||
g.Expect(err).NotTo(HaveOccurred())
|
||||
for _, pod := range pods {
|
||||
if strings.Contains(pod.Name, "helm-install") {
|
||||
g.Expect(pod.Status).Should(Equal("Completed"), pod.Name)
|
||||
} else {
|
||||
g.Expect(pod.Status).Should(Equal("Running"), pod.Name)
|
||||
}
|
||||
}
|
||||
}, "420s", "5s").Should(Succeed())
|
||||
_, _ = e2e.ParsePods(kubeConfigFile, true)
|
||||
}, "240s", "5s").Should(Succeed())
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
var failed bool
|
||||
var _ = AfterEach(func() {
|
||||
failed = failed || CurrentSpecReport().Failed()
|
||||
})
|
||||
|
||||
var _ = AfterSuite(func() {
|
||||
if failed && !*ci {
|
||||
fmt.Println("FAILED!")
|
||||
} else {
|
||||
Expect(e2e.DestroyCluster()).To(Succeed())
|
||||
Expect(os.Remove(kubeConfigFile)).To(Succeed())
|
||||
}
|
||||
})
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package validatedualstack
|
||||
package dualstack
|
||||
|
||||
import (
|
||||
"flag"
|
||||
|
|
@ -14,14 +14,15 @@ import (
|
|||
|
||||
// Valid nodeOS: generic/ubuntu2004, opensuse/Leap-15.3.x86_64
|
||||
var nodeOS = flag.String("nodeOS", "generic/ubuntu2004", "VM operating system")
|
||||
var serverCount = flag.Int("serverCount", 1, "number of server nodes")
|
||||
var serverCount = flag.Int("serverCount", 3, "number of server nodes")
|
||||
var agentCount = flag.Int("agentCount", 1, "number of agent nodes")
|
||||
var hardened = flag.Bool("hardened", false, "true or false")
|
||||
|
||||
func Test_E2EDualStack(t *testing.T) {
|
||||
flag.Parse()
|
||||
RegisterFailHandler(Fail)
|
||||
RunSpecs(t, "Validate DualStack Suite")
|
||||
suiteConfig, reporterConfig := GinkgoConfiguration()
|
||||
RunSpecs(t, "DualStack Test Suite", suiteConfig, reporterConfig)
|
||||
}
|
||||
|
||||
var (
|
||||
|
|
@ -30,12 +31,14 @@ var (
|
|||
agentNodeNames []string
|
||||
)
|
||||
|
||||
var _ = Describe("Verify DualStack Configuration", func() {
|
||||
var _ = ReportAfterEach(e2e.GenReport)
|
||||
|
||||
var _ = Describe("Verify DualStack Configuration", Ordered, func() {
|
||||
|
||||
It("Starts up with no issues", func() {
|
||||
var err error
|
||||
serverNodeNames, agentNodeNames, err = e2e.CreateCluster(*nodeOS, *serverCount, *agentCount)
|
||||
Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog())
|
||||
Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err))
|
||||
fmt.Println("CLUSTER CONFIG")
|
||||
fmt.Println("OS:", *nodeOS)
|
||||
fmt.Println("Server Nodes:", serverNodeNames)
|
||||
|
|
@ -182,7 +185,7 @@ var _ = Describe("Verify DualStack Configuration", func() {
|
|||
|
||||
var failed bool
|
||||
var _ = AfterEach(func() {
|
||||
failed = failed || CurrentGinkgoTestDescription().Failed
|
||||
failed = failed || CurrentSpecReport().Failed()
|
||||
})
|
||||
|
||||
var _ = AfterSuite(func() {
|
||||
|
|
|
|||
|
|
@ -48,7 +48,9 @@ func getClientIPs(kubeConfigFile string) ([]e2e.ObjIP, error) {
|
|||
func Test_E2EExternalIP(t *testing.T) {
|
||||
flag.Parse()
|
||||
RegisterFailHandler(Fail)
|
||||
RunSpecs(t, "Validate External-IP config Suite")
|
||||
suiteConfig, reporterConfig := GinkgoConfiguration()
|
||||
RunSpecs(t, "External-IP config Suite", suiteConfig, reporterConfig)
|
||||
|
||||
}
|
||||
|
||||
var (
|
||||
|
|
@ -57,12 +59,14 @@ var (
|
|||
agentNodeNames []string
|
||||
)
|
||||
|
||||
var _ = ReportAfterEach(e2e.GenReport)
|
||||
|
||||
var _ = Describe("Verify External-IP config", Ordered, func() {
|
||||
|
||||
It("Starts up with no issues", func() {
|
||||
var err error
|
||||
serverNodeNames, agentNodeNames, err = e2e.CreateCluster(*nodeOS, *serverCount, *agentCount)
|
||||
Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog())
|
||||
Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err))
|
||||
fmt.Println("CLUSTER CONFIG")
|
||||
fmt.Println("OS:", *nodeOS)
|
||||
fmt.Println("Server Nodes:", serverNodeNames)
|
||||
|
|
@ -33,12 +33,14 @@ var (
|
|||
agentNodeNames []string
|
||||
)
|
||||
|
||||
var _ = ReportAfterEach(e2e.GenReport)
|
||||
|
||||
var _ = Describe("Verify prefer-bundled-bin flag", Ordered, func() {
|
||||
Context("Cluster :", func() {
|
||||
It("Starts up with no issues", func() {
|
||||
var err error
|
||||
serverNodeNames, agentNodeNames, err = e2e.CreateCluster(*nodeOS, *serverCount, *agentCount)
|
||||
Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog())
|
||||
Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err))
|
||||
fmt.Println("CLUSTER CONFIG")
|
||||
fmt.Println("OS:", *nodeOS)
|
||||
fmt.Println("Server Nodes:", serverNodeNames)
|
||||
|
|
|
|||
|
|
@ -7,13 +7,15 @@ curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stabl
|
|||
sudo mv kubectl /usr/local/bin/ && \
|
||||
chmod a+x /usr/local/bin/kubectl
|
||||
|
||||
echo 'Installing jq'
|
||||
sudo apt-get -y install jq
|
||||
echo 'Installing jq and docker'
|
||||
sudo apt-get -y install jq docker.io
|
||||
|
||||
echo 'Installing Go'
|
||||
curl -L https://dl.google.com/go/go1.16.10.linux-amd64.tar.gz | tar xz
|
||||
sudo mv go /usr/local
|
||||
/usr/local/go/bin/go version
|
||||
GO_VERSION=1.19.1
|
||||
wget --quiet https://dl.google.com/go/go$GO_VERSION.linux-amd64.tar.gz
|
||||
sudo rm -rf /usr/local/go && sudo tar -C /usr/local -xzf go$GO_VERSION.linux-amd64.tar.gz
|
||||
rm go$GO_VERSION.linux-amd64.tar.gz
|
||||
echo
|
||||
go version
|
||||
|
||||
echo 'Installing Virtualbox'
|
||||
|
|
@ -29,11 +31,18 @@ echo 'Installing vagrant'
|
|||
sudo apt-get -y install -f unzip
|
||||
curl -O https://releases.hashicorp.com/vagrant/2.2.19/vagrant_2.2.19_linux_amd64.zip
|
||||
unzip vagrant_2.2.19_linux_amd64.zip
|
||||
sudo cp vagrant /usr/local/bin/
|
||||
sudo mv vagrant /usr/local/bin/
|
||||
rm vagrant_2.2.19_linux_amd64.zip
|
||||
vagrant --version
|
||||
sudo apt-get -y install libarchive-tools
|
||||
vagrant plugin install vagrant-k3s
|
||||
vagrant plugin install vagrant-reload
|
||||
vagrant plugin install vagrant-k3s vagrant-reload vagrant-scp
|
||||
|
||||
echo 'Cloning repo'
|
||||
ls k3s 2>/dev/null || git clone https://github.com/k3s-io/k3s.git
|
||||
|
||||
# Use curl -X GET <IP_ADDR>:5000/v2/_catalog to see cached images
|
||||
echo 'Setting up docker registry as a cache'
|
||||
docker run -d -p 5000:5000 \
|
||||
-e REGISTRY_PROXY_REMOTEURL=https://registry-1.docker.io \
|
||||
--restart always \
|
||||
--name registry registry:2
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Script to to point k3s to the docker registry running on the host
|
||||
# This is used to avoid hitting dockerhub rate limits on E2E runners
|
||||
ip_addr=$1
|
||||
|
||||
mkdir -p /etc/rancher/k3s/
|
||||
echo "mirrors:
|
||||
docker.io:
|
||||
endpoint:
|
||||
- \"http://$ip_addr:5000\"" >> /etc/rancher/k3s/registries.yaml
|
||||
|
|
@ -1,32 +1,57 @@
|
|||
#!/bin/bash
|
||||
servercount=${5:-3}
|
||||
agentcount=${6:-2}
|
||||
agentcount=${6:-1}
|
||||
db=${7:-"etcd"}
|
||||
k3s_version=${k3s_version}
|
||||
k3s_channel=${k3s_channel:-"commit"}
|
||||
hardened=${8:-""}
|
||||
|
||||
E2E_EXTERNAL_DB=$db && export E2E_EXTERNAL_DB
|
||||
E2E_REGISTRY=true && export E2E_REGISTRY
|
||||
|
||||
eval openvpn --daemon --config external.ovpn &>/dev/null &
|
||||
sleep 10
|
||||
|
||||
ssh -i "$1" -o "StrictHostKeyChecking no" $2@$3 'cd k3s && git pull --rebase origin master'
|
||||
ssh -i "$1" -o "StrictHostKeyChecking no" $2@$3 '/usr/local/go/bin/go get github.com/onsi/ginkgo/v2'
|
||||
ssh -i "$1" -o "StrictHostKeyChecking no" $2@$3 '/usr/local/go/bin/go get github.com/onsi/gomega'
|
||||
ssh -i "$1" -o "StrictHostKeyChecking no" $2@$3 '/usr/local/go/bin/go get github.com/k3s-io/k3s/tests/e2e'
|
||||
ssh -i "$1" -o "StrictHostKeyChecking no" $2@$3 'cd k3s && /usr/local/go/bin/go mod tidy'
|
||||
|
||||
ssh -i "$1" -o "StrictHostKeyChecking no" $2@$3 "cd k3s/tests/e2e/dualstack && vagrant global-status | awk '/running/'|cut -c1-7| xargs -r -d '\n' -n 1 -- vagrant destroy -f"
|
||||
|
||||
echo 'RUNNING DUALSTACK VALIDATION TEST'
|
||||
ssh -i "$1" -o "StrictHostKeyChecking no" $2@$3 "cd k3s/tests/e2e && E2E_HARDENED="$hardened" /usr/local/go/bin/go test -v dualstack/dualstack_test.go -nodeOS="$4" -serverCount=$((servercount)) -agentCount=$((agentcount)) -timeout=30m -json" | tee -a testreport.log
|
||||
ssh -i "$1" -o "StrictHostKeyChecking no" $2@$3 'cd k3s/tests/e2e/dualstack && vagrant destroy -f'
|
||||
|
||||
echo 'RUNNING CLUSTER VALIDATION TEST'
|
||||
ssh -i "$1" -o "StrictHostKeyChecking no" $2@$3 "cd k3s/tests/e2e && E2E_REGISTRY=true E2E_HARDENED="$hardened" /usr/local/go/bin/go test -v validatecluster/validatecluster_test.go -nodeOS="$4" -serverCount=$((servercount)) -agentCount=$((agentcount)) -timeout=30m -json" | tee -a testreport.log
|
||||
ssh -i "$1" -o "StrictHostKeyChecking no" $2@$3 'cd k3s/tests/e2e/validatecluster && vagrant destroy -f'
|
||||
ssh -i "$1" -o "StrictHostKeyChecking no" $2@$3 "cd k3s/tests/e2e && E2E_HARDENED="$hardened" /usr/local/go/bin/go test -v validatecluster/validatecluster_test.go -nodeOS="$4" -serverCount=$((servercount)) -agentCount=$((agentcount)) -timeout=1h"
|
||||
|
||||
echo 'RUNNING SECRETS ENCRYPTION TEST'
|
||||
ssh -i "$1" -o "StrictHostKeyChecking no" $2@$3 "cd k3s/tests/e2e && /usr/local/go/bin/go test -v secretsencryption/secretsencryption_test.go -nodeOS="$4" -serverCount=$((servercount)) -timeout=30m -json" | tee -a testreport.log
|
||||
ssh -i "$1" -o "StrictHostKeyChecking no" $2@$3 'cd k3s/tests/e2e/secretsencryption && vagrant destroy -f'
|
||||
ssh -i "$1" -o "StrictHostKeyChecking no" $2@$3 "cd k3s/tests/e2e && /usr/local/go/bin/go test -v secretsencryption/secretsencryption_test.go -nodeOS="$4" -serverCount=$((servercount)) -timeout=1h"
|
||||
|
||||
echo 'RUNNING SNAPSHOT RESTORE TEST'
|
||||
ssh -i "$1" -o "StrictHostKeyChecking no" $2@$3 "cd k3s/tests/e2e && /usr/local/go/bin/go test -v snapshotrestore/snapshotrestore_test.go -nodeOS="$4" -serverCount=$((servercount)) -agentCount=$((agentcount)) -timeout=30m -json" | tee -a testreport.log
|
||||
ssh -i "$1" -o "StrictHostKeyChecking no" $2@$3 'cd k3s/tests/e2e/secretsencryption && vagrant destroy -f'
|
||||
|
||||
echo 'RUNNING SPLIT SERVER VALIDATION TEST'
|
||||
ssh -i "$1" -o "StrictHostKeyChecking no" $2@$3 "cd k3s/tests/e2e && E2E_HARDENED="$hardened" /usr/local/go/bin/go test -v splitserver/splitserver_test.go -nodeOS="$4" -timeout=30m -json" | tee -a testreport.log
|
||||
ssh -i "$1" -o "StrictHostKeyChecking no" $2@$3 'cd k3s/tests/e2e/splitserver && vagrant destroy -f'
|
||||
|
||||
E2E_RELEASE_VERSION=$k3s_version && export E2E_RELEASE_VERSION
|
||||
E2E_RELEASE_CHANNEL=$k3s_channel && export E2E_RELEASE_CHANNEL
|
||||
|
||||
echo 'RUNNING CLUSTER UPGRADE TEST'
|
||||
ssh -i "$1" -o "StrictHostKeyChecking no" $2@$3 "cd k3s/tests/e2e && E2E_REGISTRY=true /usr/local/go/bin/go test -v upgradecluster/upgradecluster_test.go -nodeOS="$4" -serverCount=$((servercount)) -agentCount=$((agentcount)) -timeout=30m -json" | tee -a testreport.log
|
||||
ssh -i "$1" -o "StrictHostKeyChecking no" $2@$3 'cd k3s/tests/e2e/upgradecluster && vagrant destroy -f'
|
||||
ssh -i "$1" -o "StrictHostKeyChecking no" $2@$3 "cd k3s/tests/e2e && /usr/local/go/bin/go test -v upgradecluster/upgradecluster_test.go -nodeOS="$4" -serverCount=$((servercount)) -agentCount=$((agentcount)) -timeout=1h"
|
||||
|
||||
echo 'RUN CLUSTER RESET TEST'
|
||||
ssh -i "$1" -o "StrictHostKeyChecking no" $2@$3 "cd k3s/tests/e2e && /usr/local/go/bin/go test -v clusterreset/clusterreset_test.go -nodeOS="$4" -serverCount=$((servercount)) -agentCount=$((agentcount)) -timeout=30m -json" | tee -a testreport.log
|
||||
ssh -i "$1" -o "StrictHostKeyChecking no" $2@$3 'cd k3s/tests/e2e/clusterreset && vagrant destroy -f'
|
||||
|
||||
echo 'RUNNING DOCKER CRI VALIDATION TEST'
|
||||
ssh -i "$1" -o "StrictHostKeyChecking no" $2@$3 "cd k3s/tests/e2e && /usr/local/go/bin/go test -v docker/docker_test.go -nodeOS="$4" -serverCount=1 -agentCount=1 -timeout=30m -json" | tee -a testreport.log
|
||||
ssh -i "$1" -o "StrictHostKeyChecking no" $2@$3 'cd k3s/tests/e2e/docker && vagrant destroy -f'
|
||||
|
||||
echo 'RUNNING EXTERNALIP VALIDATION TEST'
|
||||
ssh -i "$1" -o "StrictHostKeyChecking no" $2@$3 "cd k3s/tests/e2e && E2E_HARDENED="$hardened" /usr/local/go/bin/go test -v externalip/externalip_test.go -nodeOS="$4" -serverCount=1 -agentCount=1 -timeout=30m -json" | tee -a testreport.log
|
||||
ssh -i "$1" -o "StrictHostKeyChecking no" $2@$3 'cd k3s/tests/e2e/dualstack && vagrant destroy -f'
|
||||
|
|
|
|||
|
|
@ -23,7 +23,8 @@ var hardened = flag.Bool("hardened", false, "true or false")
|
|||
func Test_E2ESecretsEncryption(t *testing.T) {
|
||||
RegisterFailHandler(Fail)
|
||||
flag.Parse()
|
||||
RunSpecs(t, "Secrets Encryption Test Suite")
|
||||
suiteConfig, reporterConfig := GinkgoConfiguration()
|
||||
RunSpecs(t, "Secrets Encryption Test Suite", suiteConfig, reporterConfig)
|
||||
}
|
||||
|
||||
var (
|
||||
|
|
@ -31,7 +32,9 @@ var (
|
|||
serverNodeNames []string
|
||||
)
|
||||
|
||||
var _ = Describe("Verify Secrets Encryption Rotation", func() {
|
||||
var _ = ReportAfterEach(e2e.GenReport)
|
||||
|
||||
var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() {
|
||||
Context("Cluster :", func() {
|
||||
It("Starts up with no issues", func() {
|
||||
var err error
|
||||
|
|
@ -286,7 +289,7 @@ var _ = Describe("Verify Secrets Encryption Rotation", func() {
|
|||
|
||||
})
|
||||
|
||||
var failed = false
|
||||
var failed bool
|
||||
var _ = AfterEach(func() {
|
||||
failed = failed || CurrentSpecReport().Failed()
|
||||
})
|
||||
|
|
|
|||
|
|
@ -0,0 +1,125 @@
|
|||
ENV['VAGRANT_NO_PARALLEL'] = 'no'
|
||||
NODE_ROLES = (ENV['E2E_NODE_ROLES'] ||
|
||||
["server-0", "server-1", "server-2", "agent-0", "agent-1"])
|
||||
NODE_BOXES = (ENV['E2E_NODE_BOXES'] ||
|
||||
['generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004'])
|
||||
GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master")
|
||||
RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "")
|
||||
EXTERNAL_DB = (ENV['E2E_EXTERNAL_DB'] || "etcd")
|
||||
NODE_CPUS = (ENV['E2E_NODE_CPUS'] || 2).to_i
|
||||
NODE_MEMORY = (ENV['E2E_NODE_MEMORY'] || 1024).to_i
|
||||
# Virtualbox >= 6.1.28 require `/etc/vbox/network.conf` for expanded private networks
|
||||
NETWORK_PREFIX = "10.10.10"
|
||||
install_type = ""
|
||||
|
||||
def provision(vm, role, role_num, node_num)
|
||||
vm.box = NODE_BOXES[node_num]
|
||||
vm.hostname = role
|
||||
# An expanded netmask is required to allow VM<-->VM communication, virtualbox defaults to /32
|
||||
node_ip = "#{NETWORK_PREFIX}.#{100+node_num}"
|
||||
vm.network "private_network", ip: node_ip, netmask: "255.255.255.0"
|
||||
|
||||
scripts_location = Dir.exists?("./scripts") ? "./scripts" : "../scripts"
|
||||
vagrant_defaults = File.exists?("./vagrantdefaults.rb") ? "./vagrantdefaults.rb" : "../vagrantdefaults.rb"
|
||||
load vagrant_defaults
|
||||
|
||||
defaultOSConfigure(vm)
|
||||
install_type = getInstallType(vm, RELEASE_VERSION, GITHUB_BRANCH)
|
||||
|
||||
vm.provision "shell", inline: "ping -c 2 k3s.io"
|
||||
|
||||
db_type = getDBType(role, role_num, vm)
|
||||
|
||||
if role.include?("server") && role_num == 0
|
||||
vm.provision 'k3s-primary-server', type: 'k3s', run: 'once' do |k3s|
|
||||
k3s.args = "server "
|
||||
k3s.config = <<~YAML
|
||||
token: vagrant
|
||||
node-external-ip: #{NETWORK_PREFIX}.100
|
||||
flannel-iface: eth1
|
||||
tls-san: #{NETWORK_PREFIX}.100.nip.io
|
||||
#{db_type}
|
||||
YAML
|
||||
k3s.env = %W[K3S_KUBECONFIG_MODE=0644 #{install_type}]
|
||||
k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321
|
||||
end
|
||||
|
||||
elsif role.include?("server") && role_num != 0
|
||||
vm.provision 'k3s-secondary-server', type: 'k3s', run: 'once' do |k3s|
|
||||
k3s.args = "server"
|
||||
k3s.config = <<~YAML
|
||||
server: "https://#{NETWORK_PREFIX}.100:6443"
|
||||
token: vagrant
|
||||
node-external-ip: #{node_ip}
|
||||
flannel-iface: eth1
|
||||
#{db_type}
|
||||
YAML
|
||||
k3s.env = %W[K3S_KUBECONFIG_MODE=0644 K3S_TOKEN=vagrant #{install_type}]
|
||||
k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321
|
||||
end
|
||||
end
|
||||
|
||||
if role.include?("agent")
|
||||
vm.provision 'k3s-agent', type: 'k3s', run: 'once' do |k3s|
|
||||
k3s.args = "agent"
|
||||
k3s.config = <<~YAML
|
||||
server: "https://#{NETWORK_PREFIX}.100:6443"
|
||||
token: vagrant
|
||||
node-external-ip: #{node_ip}
|
||||
flannel-iface: eth1
|
||||
#{db_type}
|
||||
YAML
|
||||
k3s.env = %W[K3S_KUBECONFIG_MODE=0644 #{install_type}]
|
||||
k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321
|
||||
end
|
||||
end
|
||||
if vm.box.to_s.include?("microos")
|
||||
vm.provision 'k3s-reload', type: 'reload', run: 'once'
|
||||
if !EXTERNAL_DB.empty?
|
||||
vm.provision "shell", inline: "docker start #{EXTERNAL_DB}"
|
||||
end
|
||||
end
|
||||
# This step does not run by default and is designed to be called by higher level tools
|
||||
end
|
||||
|
||||
def getDBType(role, role_num, vm)
|
||||
if ( EXTERNAL_DB == "" || EXTERNAL_DB == "etcd" )
|
||||
if role.include?("server") && role_num == 0
|
||||
return "cluster-init: true"
|
||||
end
|
||||
else
|
||||
puts "Unknown EXTERNAL_DB: " + EXTERNAL_DB
|
||||
abort
|
||||
end
|
||||
return ""
|
||||
end
|
||||
|
||||
Vagrant.configure("2") do |config|
|
||||
config.vagrant.plugins = ["vagrant-k3s", "vagrant-reload"]
|
||||
# Default provider is libvirt, virtualbox is only provided as a backup
|
||||
config.vm.provider "libvirt" do |v|
|
||||
v.cpus = NODE_CPUS
|
||||
v.memory = NODE_MEMORY
|
||||
end
|
||||
config.vm.provider "virtualbox" do |v|
|
||||
v.cpus = NODE_CPUS
|
||||
v.memory = NODE_MEMORY
|
||||
end
|
||||
|
||||
if NODE_ROLES.kind_of?(String)
|
||||
NODE_ROLES = NODE_ROLES.split(" ", -1)
|
||||
end
|
||||
if NODE_BOXES.kind_of?(String)
|
||||
NODE_BOXES = NODE_BOXES.split(" ", -1)
|
||||
end
|
||||
|
||||
# Must iterate on the index, vagrant does not understand iterating
|
||||
# over the node roles themselves
|
||||
NODE_ROLES.length.times do |i|
|
||||
name = NODE_ROLES[i]
|
||||
role_num = name.split("-", -1).pop.to_i
|
||||
config.vm.define name do |node|
|
||||
provision(node.vm, name, role_num, i)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
@ -0,0 +1,192 @@
|
|||
package snapshotrestore
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/k3s-io/k3s/tests/e2e"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
// Valid nodeOS:
|
||||
// generic/ubuntu2004, generic/centos7, generic/rocky8,
|
||||
// opensuse/Leap-15.3.x86_64, dweomer/microos.amd64
|
||||
|
||||
var nodeOS = flag.String("nodeOS", "generic/ubuntu2004", "VM operating system")
|
||||
var serverCount = flag.Int("serverCount", 3, "number of server nodes")
|
||||
var agentCount = flag.Int("agentCount", 1, "number of agent nodes")
|
||||
var hardened = flag.Bool("hardened", false, "true or false")
|
||||
var ci = flag.Bool("ci", false, "running on CI")
|
||||
var local = flag.Bool("local", false, "deploy a locally built K3s binary")
|
||||
|
||||
// Environment Variables Info:
|
||||
// E2E_EXTERNAL_DB: mysql, postgres, etcd (default: etcd)
|
||||
// E2E_RELEASE_VERSION=v1.23.1+k3s2 (default: latest commit from master)
|
||||
|
||||
func Test_E2ESnapshotRestore(t *testing.T) {
|
||||
RegisterFailHandler(Fail)
|
||||
flag.Parse()
|
||||
suiteConfig, reporterConfig := GinkgoConfiguration()
|
||||
RunSpecs(t, "SnapshotRestore Test Suite", suiteConfig, reporterConfig)
|
||||
}
|
||||
|
||||
var (
|
||||
kubeConfigFile string
|
||||
serverNodeNames []string
|
||||
agentNodeNames []string
|
||||
snapshotname string
|
||||
)
|
||||
|
||||
var _ = ReportAfterEach(e2e.GenReport)
|
||||
|
||||
var _ = Describe("Verify Create", Ordered, func() {
|
||||
Context("Cluster :", func() {
|
||||
It("Starts up with no issues", func() {
|
||||
var err error
|
||||
if *local {
|
||||
serverNodeNames, agentNodeNames, err = e2e.CreateLocalCluster(*nodeOS, *serverCount, *agentCount)
|
||||
} else {
|
||||
serverNodeNames, agentNodeNames, err = e2e.CreateCluster(*nodeOS, *serverCount, *agentCount)
|
||||
}
|
||||
Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err))
|
||||
fmt.Println("CLUSTER CONFIG")
|
||||
fmt.Println("OS:", *nodeOS)
|
||||
fmt.Println("Server Nodes:", serverNodeNames)
|
||||
fmt.Println("Agent Nodes:", agentNodeNames)
|
||||
kubeConfigFile, err = e2e.GenKubeConfigFile(serverNodeNames[0])
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
})
|
||||
|
||||
It("Checks Node and Pod Status", func() {
|
||||
fmt.Printf("\nFetching node status\n")
|
||||
Eventually(func(g Gomega) {
|
||||
nodes, err := e2e.ParseNodes(kubeConfigFile, false)
|
||||
g.Expect(err).NotTo(HaveOccurred())
|
||||
for _, node := range nodes {
|
||||
g.Expect(node.Status).Should(Equal("Ready"))
|
||||
}
|
||||
}, "420s", "5s").Should(Succeed())
|
||||
_, _ = e2e.ParseNodes(kubeConfigFile, true)
|
||||
|
||||
fmt.Printf("\nFetching Pods status\n")
|
||||
Eventually(func(g Gomega) {
|
||||
pods, err := e2e.ParsePods(kubeConfigFile, false)
|
||||
g.Expect(err).NotTo(HaveOccurred())
|
||||
for _, pod := range pods {
|
||||
if strings.Contains(pod.Name, "helm-install") {
|
||||
g.Expect(pod.Status).Should(Equal("Completed"), pod.Name)
|
||||
} else {
|
||||
g.Expect(pod.Status).Should(Equal("Running"), pod.Name)
|
||||
}
|
||||
}
|
||||
}, "620s", "5s").Should(Succeed())
|
||||
_, _ = e2e.ParsePods(kubeConfigFile, true)
|
||||
})
|
||||
|
||||
It("Verifies test workload before snapshot is created", func() {
|
||||
res, err := e2e.DeployWorkload("clusterip.yaml", kubeConfigFile, *hardened)
|
||||
Expect(err).NotTo(HaveOccurred(), "Cluster IP manifest not deployed: "+res)
|
||||
|
||||
Eventually(func(g Gomega) {
|
||||
cmd := "kubectl get pods -o=name -l k8s-app=nginx-app-clusterip --field-selector=status.phase=Running --kubeconfig=" + kubeConfigFile
|
||||
res, err := e2e.RunCommand(cmd)
|
||||
g.Expect(err).NotTo(HaveOccurred())
|
||||
g.Expect(res).Should((ContainSubstring("test-clusterip")), "failed cmd: "+cmd+" result: "+res)
|
||||
}, "240s", "5s").Should(Succeed())
|
||||
})
|
||||
|
||||
It("Verifies Snapshot is created", func() {
|
||||
Eventually(func(g Gomega) {
|
||||
cmd := "sudo k3s etcd-snapshot"
|
||||
_, err := e2e.RunCmdOnNode(cmd, "server-0")
|
||||
g.Expect(err).NotTo(HaveOccurred())
|
||||
cmd = "sudo ls /var/lib/rancher/k3s/server/db/snapshots/"
|
||||
snapshotname, err = e2e.RunCmdOnNode(cmd, "server-0")
|
||||
fmt.Println("Snapshot Name", snapshotname)
|
||||
g.Expect(snapshotname).Should(ContainSubstring("on-demand-server-0"))
|
||||
}, "420s", "10s").Should(Succeed())
|
||||
})
|
||||
|
||||
It("Verifies another test workload after snapshot is created", func() {
|
||||
_, err := e2e.DeployWorkload("nodeport.yaml", kubeConfigFile, *hardened)
|
||||
Expect(err).NotTo(HaveOccurred(), "NodePort manifest not deployed")
|
||||
Eventually(func(g Gomega) {
|
||||
cmd := "kubectl get pods -o=name -l k8s-app=nginx-app-nodeport --field-selector=status.phase=Running --kubeconfig=" + kubeConfigFile
|
||||
res, err := e2e.RunCommand(cmd)
|
||||
g.Expect(err).NotTo(HaveOccurred())
|
||||
g.Expect(res).Should(ContainSubstring("test-nodeport"), "nodeport pod was not created")
|
||||
}, "240s", "5s").Should(Succeed())
|
||||
})
|
||||
|
||||
It("Verifies snapshot is restored successfully and validates only test workload1 is present", func() {
|
||||
//Stop k3s on all nodes
|
||||
for _, nodeName := range serverNodeNames {
|
||||
cmd := "sudo systemctl stop k3s"
|
||||
_, err := e2e.RunCmdOnNode(cmd, nodeName)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
}
|
||||
//Restores from snapshot on server-0
|
||||
for _, nodeName := range serverNodeNames {
|
||||
if nodeName == "server-0" {
|
||||
cmd := "sudo k3s server --cluster-init --cluster-reset --cluster-reset-restore-path=/var/lib/rancher/k3s/server/db/snapshots/" + snapshotname
|
||||
res, err := e2e.RunCmdOnNode(cmd, nodeName)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(res).Should(ContainSubstring("Managed etcd cluster membership has been reset, restart without --cluster-reset flag now"))
|
||||
|
||||
cmd = "sudo systemctl start k3s"
|
||||
_, err = e2e.RunCmdOnNode(cmd, nodeName)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
}
|
||||
}
|
||||
//Verifies node is up and pods running
|
||||
fmt.Printf("\nFetching node status\n")
|
||||
Eventually(func(g Gomega) {
|
||||
nodes, err := e2e.ParseNodes(kubeConfigFile, false)
|
||||
g.Expect(err).NotTo(HaveOccurred())
|
||||
for _, node := range nodes {
|
||||
g.Expect(node.Status).Should(Equal("Ready"))
|
||||
}
|
||||
}, "420s", "5s").Should(Succeed())
|
||||
_, _ = e2e.ParseNodes(kubeConfigFile, true)
|
||||
|
||||
fmt.Printf("\nFetching Pods status\n")
|
||||
Eventually(func(g Gomega) {
|
||||
pods, err := e2e.ParsePods(kubeConfigFile, false)
|
||||
g.Expect(err).NotTo(HaveOccurred())
|
||||
for _, pod := range pods {
|
||||
if strings.Contains(pod.Name, "helm-install") {
|
||||
g.Expect(pod.Status).Should(Equal("Completed"), pod.Name)
|
||||
} else {
|
||||
g.Expect(pod.Status).Should(Equal("Running"), pod.Name)
|
||||
}
|
||||
}
|
||||
}, "620s", "5s").Should(Succeed())
|
||||
_, _ = e2e.ParsePods(kubeConfigFile, true)
|
||||
//Verifies test workload1 is present
|
||||
//Verifies test workload2 is not present
|
||||
cmd := "kubectl get pods --kubeconfig=" + kubeConfigFile
|
||||
res, err := e2e.RunCommand(cmd)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(res).Should(ContainSubstring("test-clusterip"))
|
||||
Expect(res).ShouldNot(ContainSubstring("test-nodeport"))
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
var failed bool
|
||||
var _ = AfterEach(func() {
|
||||
failed = failed || CurrentSpecReport().Failed()
|
||||
})
|
||||
|
||||
var _ = AfterSuite(func() {
|
||||
if failed && !*ci {
|
||||
fmt.Println("FAILED!")
|
||||
} else {
|
||||
Expect(e2e.DestroyCluster()).To(Succeed())
|
||||
Expect(os.Remove(kubeConfigFile)).To(Succeed())
|
||||
}
|
||||
})
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package validatecluster
|
||||
package splitserver
|
||||
|
||||
import (
|
||||
"flag"
|
||||
|
|
@ -60,7 +60,8 @@ func createSplitCluster(nodeOS string, etcdCount, controlPlaneCount, agentCount
|
|||
func Test_E2ESplitServer(t *testing.T) {
|
||||
RegisterFailHandler(Fail)
|
||||
flag.Parse()
|
||||
RunSpecs(t, "Split Server Test Suite")
|
||||
suiteConfig, reporterConfig := GinkgoConfiguration()
|
||||
RunSpecs(t, "Split Server Test Suite", suiteConfig, reporterConfig)
|
||||
}
|
||||
|
||||
var (
|
||||
|
|
@ -70,12 +71,14 @@ var (
|
|||
agentNodeNames []string
|
||||
)
|
||||
|
||||
var _ = Describe("Verify Create", func() {
|
||||
var _ = ReportAfterEach(e2e.GenReport)
|
||||
|
||||
var _ = Describe("Verify Create", Ordered, func() {
|
||||
Context("Cluster :", func() {
|
||||
It("Starts up with no issues", func() {
|
||||
var err error
|
||||
etcdNodeNames, cpNodeNames, agentNodeNames, err = createSplitCluster(*nodeOS, *etcdCount, *controlPlaneCount, *agentCount)
|
||||
Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog())
|
||||
Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err))
|
||||
fmt.Println("CLUSTER CONFIG")
|
||||
fmt.Println("OS:", *nodeOS)
|
||||
fmt.Println("Etcd Server Nodes:", etcdNodeNames)
|
||||
|
|
@ -219,9 +222,9 @@ var _ = Describe("Verify Create", func() {
|
|||
})
|
||||
})
|
||||
|
||||
var failed = false
|
||||
var failed bool
|
||||
var _ = AfterEach(func() {
|
||||
failed = failed || CurrentGinkgoTestDescription().Failed
|
||||
failed = failed || CurrentSpecReport().Failed()
|
||||
})
|
||||
|
||||
var _ = AfterSuite(func() {
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
package e2e
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
|
|
@ -8,6 +10,11 @@ import (
|
|||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
json "github.com/json-iterator/go"
|
||||
ginkgo "github.com/onsi/ginkgo/v2"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
type Node struct {
|
||||
|
|
@ -34,6 +41,28 @@ type ObjIP struct {
|
|||
IPv6 string
|
||||
}
|
||||
|
||||
type NodeError struct {
|
||||
Node string
|
||||
Cmd string
|
||||
Err error
|
||||
}
|
||||
|
||||
func (ne *NodeError) Error() string {
|
||||
return fmt.Sprintf("failed creating cluster: %s: %v", ne.Cmd, ne.Err)
|
||||
}
|
||||
|
||||
func (ne *NodeError) Unwrap() error {
|
||||
return ne.Err
|
||||
}
|
||||
|
||||
func newNodeError(cmd, node string, err error) *NodeError {
|
||||
return &NodeError{
|
||||
Cmd: cmd,
|
||||
Node: node,
|
||||
Err: err,
|
||||
}
|
||||
}
|
||||
|
||||
func CountOfStringInSlice(str string, pods []Pod) int {
|
||||
count := 0
|
||||
for _, pod := range pods {
|
||||
|
|
@ -76,12 +105,30 @@ func CreateCluster(nodeOS string, serverCount, agentCount int) ([]string, []stri
|
|||
testOptions += " " + env
|
||||
}
|
||||
}
|
||||
// Bring up the first server node
|
||||
cmd := fmt.Sprintf(`%s %s vagrant up %s &> vagrant.log`, nodeEnvs, testOptions, serverNodeNames[0])
|
||||
|
||||
cmd := fmt.Sprintf(`%s %s vagrant up &> vagrant.log`, nodeEnvs, testOptions)
|
||||
fmt.Println(cmd)
|
||||
if _, err := RunCommand(cmd); err != nil {
|
||||
return nil, nil, fmt.Errorf("failed creating cluster: %s: %v", cmd, err)
|
||||
return nil, nil, newNodeError(cmd, serverNodeNames[0], err)
|
||||
}
|
||||
// Bring up the rest of the nodes in parallel
|
||||
errg, _ := errgroup.WithContext(context.Background())
|
||||
for _, node := range append(serverNodeNames[1:], agentNodeNames...) {
|
||||
cmd := fmt.Sprintf(`%s %s vagrant up %s &>> vagrant.log`, nodeEnvs, testOptions, node)
|
||||
errg.Go(func() error {
|
||||
if _, err := RunCommand(cmd); err != nil {
|
||||
return newNodeError(cmd, node, err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
// We must wait a bit between provisioning nodes to avoid too many learners attempting to join the cluster
|
||||
time.Sleep(20 * time.Second)
|
||||
}
|
||||
if err := errg.Wait(); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
return serverNodeNames, agentNodeNames, nil
|
||||
}
|
||||
|
||||
|
|
@ -93,6 +140,8 @@ func CreateLocalCluster(nodeOS string, serverCount, agentCount int) ([]string, [
|
|||
serverNodeNames, agentNodeNames, nodeEnvs := genNodeEnvs(nodeOS, serverCount, agentCount)
|
||||
|
||||
var testOptions string
|
||||
var cmd string
|
||||
|
||||
for _, env := range os.Environ() {
|
||||
if strings.HasPrefix(env, "E2E_") {
|
||||
testOptions += " " + env
|
||||
|
|
@ -100,14 +149,27 @@ func CreateLocalCluster(nodeOS string, serverCount, agentCount int) ([]string, [
|
|||
}
|
||||
testOptions += " E2E_RELEASE_VERSION=skip"
|
||||
|
||||
cmd := fmt.Sprintf(`%s vagrant up --no-provision &> vagrant.log`, nodeEnvs)
|
||||
if _, err := RunCommand(cmd); err != nil {
|
||||
return nil, nil, fmt.Errorf("failed creating nodes: %s: %v", cmd, err)
|
||||
// Bring up the all of the nodes in parallel
|
||||
errg, _ := errgroup.WithContext(context.Background())
|
||||
for i, node := range append(serverNodeNames, agentNodeNames...) {
|
||||
if i == 0 {
|
||||
cmd = fmt.Sprintf(`%s %s vagrant up --no-provision %s &> vagrant.log`, nodeEnvs, testOptions, node)
|
||||
} else {
|
||||
cmd = fmt.Sprintf(`%s %s vagrant up --no-provision %s &>> vagrant.log`, nodeEnvs, testOptions, node)
|
||||
}
|
||||
errg.Go(func() error {
|
||||
if _, err := RunCommand(cmd); err != nil {
|
||||
return fmt.Errorf("failed initializing nodes: %s: %v", cmd, err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
// libVirt/Virtualbox needs some time between provisioning nodes
|
||||
time.Sleep(10 * time.Second)
|
||||
}
|
||||
|
||||
nodeRoles := append(serverNodeNames, agentNodeNames...)
|
||||
|
||||
for _, node := range nodeRoles {
|
||||
if err := errg.Wait(); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
for _, node := range append(serverNodeNames, agentNodeNames...) {
|
||||
cmd = fmt.Sprintf(`vagrant scp ../../../dist/artifacts/k3s %s:/tmp/`, node)
|
||||
if _, err := RunCommand(cmd); err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to scp k3s binary to %s: %v", node, err)
|
||||
|
|
@ -117,9 +179,21 @@ func CreateLocalCluster(nodeOS string, serverCount, agentCount int) ([]string, [
|
|||
}
|
||||
}
|
||||
|
||||
cmd = fmt.Sprintf(`%s %s vagrant provision &>> vagrant.log`, nodeEnvs, testOptions)
|
||||
if _, err := RunCommand(cmd); err != nil {
|
||||
return nil, nil, fmt.Errorf("failed creating cluster: %s: %v", cmd, err)
|
||||
// Install K3s on all nodes in parallel
|
||||
errg, _ = errgroup.WithContext(context.Background())
|
||||
for _, node := range append(serverNodeNames, agentNodeNames...) {
|
||||
cmd = fmt.Sprintf(`%s %s vagrant provision %s &>> vagrant.log`, nodeEnvs, testOptions, node)
|
||||
errg.Go(func() error {
|
||||
if _, err := RunCommand(cmd); err != nil {
|
||||
return newNodeError(cmd, node, err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
// K3s needs some time between joining nodes to avoid learner issues
|
||||
time.Sleep(20 * time.Second)
|
||||
}
|
||||
if err := errg.Wait(); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
return serverNodeNames, agentNodeNames, nil
|
||||
|
|
@ -208,7 +282,32 @@ func GenKubeConfigFile(serverName string) (string, error) {
|
|||
return kubeConfigFile, nil
|
||||
}
|
||||
|
||||
func GetVagrantLog() string {
|
||||
func GenReport(specReport ginkgo.SpecReport) {
|
||||
state := struct {
|
||||
State string `json:"state"`
|
||||
Name string `json:"name"`
|
||||
Type string `json:"type"`
|
||||
Time time.Duration `json:"time"`
|
||||
}{
|
||||
State: specReport.State.String(),
|
||||
Name: specReport.LeafNodeText,
|
||||
Type: "k3s test",
|
||||
Time: specReport.RunTime,
|
||||
}
|
||||
status, _ := json.Marshal(state)
|
||||
fmt.Printf("%s", status)
|
||||
}
|
||||
|
||||
// GetVagrantLog returns the logs of on vagrant commands that initialize the nodes and provision K3s on each node.
|
||||
// It also attempts to fetch the systemctl logs of K3s on nodes where the k3s.service failed.
|
||||
func GetVagrantLog(cErr error) string {
|
||||
var nodeErr *NodeError
|
||||
nodeJournal := ""
|
||||
if errors.As(cErr, &nodeErr) {
|
||||
nodeJournal, _ = RunCommand("vagrant ssh " + nodeErr.Node + " -c \"sudo journalctl -u k3s* --no-pager\"")
|
||||
nodeJournal = "\nNode Journal Logs:\n" + nodeJournal
|
||||
}
|
||||
|
||||
log, err := os.Open("vagrant.log")
|
||||
if err != nil {
|
||||
return err.Error()
|
||||
|
|
@ -217,7 +316,7 @@ func GetVagrantLog() string {
|
|||
if err != nil {
|
||||
return err.Error()
|
||||
}
|
||||
return string(bytes)
|
||||
return string(bytes) + nodeJournal
|
||||
}
|
||||
|
||||
func ParseNodes(kubeConfig string, print bool) ([]Node, error) {
|
||||
|
|
@ -253,11 +352,11 @@ func ParseNodes(kubeConfig string, print bool) ([]Node, error) {
|
|||
return nodes, nil
|
||||
}
|
||||
|
||||
func ParsePods(kubeconfig string, print bool) ([]Pod, error) {
|
||||
func ParsePods(kubeConfig string, print bool) ([]Pod, error) {
|
||||
pods := make([]Pod, 0, 10)
|
||||
podList := ""
|
||||
|
||||
cmd := "kubectl get pods -o wide --no-headers -A --kubeconfig=" + kubeconfig
|
||||
cmd := "kubectl get pods -o wide --no-headers -A --kubeconfig=" + kubeConfig
|
||||
res, _ := RunCommand(cmd)
|
||||
res = strings.TrimSpace(res)
|
||||
podList = res
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ NODE_BOXES = (ENV['E2E_NODE_BOXES'] ||
|
|||
RELEASE_CHANNEL = (ENV['E2E_RELEASE_CHANNEL'] || "latest")
|
||||
RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "")
|
||||
EXTERNAL_DB = (ENV['E2E_EXTERNAL_DB'] || "etcd")
|
||||
REGISTRY = (ENV['E2E_REGISTRY'] || "")
|
||||
NODE_CPUS = (ENV['E2E_NODE_CPUS'] || 2).to_i
|
||||
NODE_MEMORY = (ENV['E2E_NODE_MEMORY'] || 1024).to_i
|
||||
# Virtualbox >= 6.1.28 require `/etc/vbox/network.conf` for expanded private networks
|
||||
|
|
@ -41,7 +42,9 @@ def provision(vm, role, role_num, node_num)
|
|||
vm.provision "shell", inline: "ping -c 2 k3s.io"
|
||||
db_type = getDBType(role, role_num, vm)
|
||||
|
||||
|
||||
if !REGISTRY.empty?
|
||||
vm.provision "Set private registry", type: "shell", path: scripts_location + "/registry.sh", args: [ "#{NETWORK_PREFIX}.1" ]
|
||||
end
|
||||
|
||||
if role.include?("server") && role_num == 0
|
||||
vm.provision 'k3s-primary-server', type: 'k3s', run: 'once' do |k3s|
|
||||
|
|
|
|||
|
|
@ -19,8 +19,11 @@ var nodeOS = flag.String("nodeOS", "generic/ubuntu2004", "VM operating system")
|
|||
var serverCount = flag.Int("serverCount", 3, "number of server nodes")
|
||||
var agentCount = flag.Int("agentCount", 2, "number of agent nodes")
|
||||
var hardened = flag.Bool("hardened", false, "true or false")
|
||||
var ci = flag.Bool("ci", false, "running on CI")
|
||||
|
||||
// Environment Variables Info:
|
||||
// E2E_REGISTRY: true/false (default: false)
|
||||
// Controls which K3s version is installed first, upgrade is always to latest commit
|
||||
// E2E_RELEASE_VERSION=v1.23.3+k3s1
|
||||
// OR
|
||||
// E2E_RELEASE_CHANNEL=(commit|latest|stable), commit pulls latest commit from master
|
||||
|
|
@ -28,7 +31,8 @@ var hardened = flag.Bool("hardened", false, "true or false")
|
|||
func Test_E2EUpgradeValidation(t *testing.T) {
|
||||
RegisterFailHandler(Fail)
|
||||
flag.Parse()
|
||||
RunSpecs(t, "Create Cluster Test Suite")
|
||||
suiteConfig, reporterConfig := GinkgoConfiguration()
|
||||
RunSpecs(t, "Upgrade Cluster Test Suite", suiteConfig, reporterConfig)
|
||||
}
|
||||
|
||||
var (
|
||||
|
|
@ -37,12 +41,14 @@ var (
|
|||
agentNodeNames []string
|
||||
)
|
||||
|
||||
var _ = Describe("Verify Upgrade", func() {
|
||||
var _ = ReportAfterEach(e2e.GenReport)
|
||||
|
||||
var _ = Describe("Verify Upgrade", Ordered, func() {
|
||||
Context("Cluster :", func() {
|
||||
It("Starts up with no issues", func() {
|
||||
var err error
|
||||
serverNodeNames, agentNodeNames, err = e2e.CreateCluster(*nodeOS, *serverCount, *agentCount)
|
||||
Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog())
|
||||
Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err))
|
||||
fmt.Println("CLUSTER CONFIG")
|
||||
fmt.Println("OS:", *nodeOS)
|
||||
fmt.Println("Server Nodes:", serverNodeNames)
|
||||
|
|
@ -373,13 +379,13 @@ var _ = Describe("Verify Upgrade", func() {
|
|||
})
|
||||
})
|
||||
|
||||
var failed = false
|
||||
var failed bool
|
||||
var _ = AfterEach(func() {
|
||||
failed = failed || CurrentGinkgoTestDescription().Failed
|
||||
failed = failed || CurrentSpecReport().Failed()
|
||||
})
|
||||
|
||||
var _ = AfterSuite(func() {
|
||||
if failed {
|
||||
if failed && !*ci {
|
||||
fmt.Println("FAILED!")
|
||||
} else {
|
||||
Expect(e2e.DestroyCluster()).To(Succeed())
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master")
|
|||
RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "")
|
||||
EXTERNAL_DB = (ENV['E2E_EXTERNAL_DB'] || "etcd")
|
||||
HARDENED = (ENV['E2E_HARDENED'] || "")
|
||||
REGISTRY = (ENV['E2E_REGISTRY'] || "")
|
||||
RANCHER = (ENV['E2E_RANCHER'] || "")
|
||||
NODE_CPUS = (ENV['E2E_NODE_CPUS'] || 2).to_i
|
||||
NODE_MEMORY = (ENV['E2E_NODE_MEMORY'] || 1024).to_i
|
||||
|
|
@ -37,6 +38,10 @@ def provision(vm, role, role_num, node_num)
|
|||
vm.provision "Set kernel parameters", type: "shell", path: scripts_location + "/harden.sh"
|
||||
hardened_arg = "protect-kernel-defaults: true\nkube-apiserver-arg: \"enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount\""
|
||||
end
|
||||
if !REGISTRY.empty?
|
||||
vm.provision "Set private registry", type: "shell", path: scripts_location + "/registry.sh", args: [ "#{NETWORK_PREFIX}.1" ]
|
||||
end
|
||||
|
||||
|
||||
if role.include?("server") && role_num == 0
|
||||
vm.provision 'k3s-primary-server', type: 'k3s', run: 'once' do |k3s|
|
||||
|
|
|
|||
|
|
@ -19,15 +19,19 @@ var nodeOS = flag.String("nodeOS", "generic/ubuntu2004", "VM operating system")
|
|||
var serverCount = flag.Int("serverCount", 3, "number of server nodes")
|
||||
var agentCount = flag.Int("agentCount", 2, "number of agent nodes")
|
||||
var hardened = flag.Bool("hardened", false, "true or false")
|
||||
var ci = flag.Bool("ci", false, "running on CI")
|
||||
var local = flag.Bool("local", false, "deploy a locally built K3s binary")
|
||||
|
||||
// Environment Variables Info:
|
||||
// E2E_EXTERNAL_DB: mysql, postgres, etcd (default: etcd)
|
||||
// E2E_RELEASE_VERSION=v1.23.1+k3s2 (default: latest commit from master)
|
||||
// E2E_REGISTRY: true/false (default: false)
|
||||
|
||||
func Test_E2EClusterValidation(t *testing.T) {
|
||||
RegisterFailHandler(Fail)
|
||||
flag.Parse()
|
||||
RunSpecs(t, "Create Cluster Test Suite")
|
||||
suiteConfig, reporterConfig := GinkgoConfiguration()
|
||||
RunSpecs(t, "Create Cluster Test Suite", suiteConfig, reporterConfig)
|
||||
}
|
||||
|
||||
var (
|
||||
|
|
@ -36,12 +40,18 @@ var (
|
|||
agentNodeNames []string
|
||||
)
|
||||
|
||||
var _ = Describe("Verify Create", func() {
|
||||
var _ = ReportAfterEach(e2e.GenReport)
|
||||
|
||||
var _ = Describe("Verify Create", Ordered, func() {
|
||||
Context("Cluster :", func() {
|
||||
It("Starts up with no issues", func() {
|
||||
var err error
|
||||
serverNodeNames, agentNodeNames, err = e2e.CreateCluster(*nodeOS, *serverCount, *agentCount)
|
||||
Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog())
|
||||
if *local {
|
||||
serverNodeNames, agentNodeNames, err = e2e.CreateLocalCluster(*nodeOS, *serverCount, *agentCount)
|
||||
} else {
|
||||
serverNodeNames, agentNodeNames, err = e2e.CreateCluster(*nodeOS, *serverCount, *agentCount)
|
||||
}
|
||||
Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err))
|
||||
fmt.Println("CLUSTER CONFIG")
|
||||
fmt.Println("OS:", *nodeOS)
|
||||
fmt.Println("Server Nodes:", serverNodeNames)
|
||||
|
|
@ -77,14 +87,14 @@ var _ = Describe("Verify Create", func() {
|
|||
})
|
||||
|
||||
It("Verifies ClusterIP Service", func() {
|
||||
_, err := e2e.DeployWorkload("clusterip.yaml", kubeConfigFile, *hardened)
|
||||
Expect(err).NotTo(HaveOccurred(), "Cluster IP manifest not deployed")
|
||||
res, err := e2e.DeployWorkload("clusterip.yaml", kubeConfigFile, *hardened)
|
||||
Expect(err).NotTo(HaveOccurred(), "Cluster IP manifest not deployed: "+res)
|
||||
|
||||
Eventually(func(g Gomega) {
|
||||
cmd := "kubectl get pods -o=name -l k8s-app=nginx-app-clusterip --field-selector=status.phase=Running --kubeconfig=" + kubeConfigFile
|
||||
res, err := e2e.RunCommand(cmd)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
g.Expect(res).Should((ContainSubstring("test-clusterip")))
|
||||
g.Expect(res).Should((ContainSubstring("test-clusterip")), "failed cmd: "+cmd+" result: "+res)
|
||||
}, "240s", "5s").Should(Succeed())
|
||||
|
||||
clusterip, _ := e2e.FetchClusterIP(kubeConfigFile, "nginx-clusterip-svc", false)
|
||||
|
|
@ -207,8 +217,8 @@ var _ = Describe("Verify Create", func() {
|
|||
})
|
||||
|
||||
It("Verifies Local Path Provisioner storage ", func() {
|
||||
_, err := e2e.DeployWorkload("local-path-provisioner.yaml", kubeConfigFile, *hardened)
|
||||
Expect(err).NotTo(HaveOccurred(), "local-path-provisioner manifest not deployed")
|
||||
res, err := e2e.DeployWorkload("local-path-provisioner.yaml", kubeConfigFile, *hardened)
|
||||
Expect(err).NotTo(HaveOccurred(), "local-path-provisioner manifest not deployed: "+res)
|
||||
|
||||
Eventually(func(g Gomega) {
|
||||
cmd := "kubectl get pvc local-path-pvc --kubeconfig=" + kubeConfigFile
|
||||
|
|
@ -231,7 +241,7 @@ var _ = Describe("Verify Create", func() {
|
|||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
cmd = "kubectl delete pod volume-test --kubeconfig=" + kubeConfigFile
|
||||
res, err := e2e.RunCommand(cmd)
|
||||
res, err = e2e.RunCommand(cmd)
|
||||
Expect(err).NotTo(HaveOccurred(), "failed cmd: "+cmd+" result: "+res)
|
||||
|
||||
_, err = e2e.DeployWorkload("local-path-provisioner.yaml", kubeConfigFile, *hardened)
|
||||
|
|
@ -263,13 +273,13 @@ var _ = Describe("Verify Create", func() {
|
|||
})
|
||||
})
|
||||
|
||||
var failed = false
|
||||
var failed bool
|
||||
var _ = AfterEach(func() {
|
||||
failed = failed || CurrentGinkgoTestDescription().Failed
|
||||
failed = failed || CurrentSpecReport().Failed()
|
||||
})
|
||||
|
||||
var _ = AfterSuite(func() {
|
||||
if failed {
|
||||
if failed && !*ci {
|
||||
fmt.Println("FAILED!")
|
||||
} else {
|
||||
Expect(e2e.DestroyCluster()).To(Succeed())
|
||||
|
|
|
|||
Loading…
Reference in New Issue