From d07edd72f61546bba733969919033b3fc0b607c8 Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Mon, 28 Feb 2022 12:10:40 -0800 Subject: [PATCH] [Release-1.22] Backport E2E Testing Improvements (#5124) * Fix cluster validation and add upgrade cluster test (#5020) Signed-off-by: Shylaja Devadiga Co-authored-by: Derek Nola Signed-off-by: Derek Nola * Migrate Ginkgo testing framework to V2, consolidate integration tests (#5097) * Upgrade and convert ginkgo from v1 to v2 * Move all integration tests into integration folder * Update TESTING.md Signed-off-by: Derek Nola * E2E Test Improvements (#5102) * Fix infinite while loop on failure, reduce upgradecluster * DRY code Signed-off-by: Derek Nola --- go.mod | 5 +- go.sum | 6 +- pkg/flock/flock_int_test.go | 7 +- tests/TESTING.md | 41 +- tests/e2e/Dockerfile.build | 2 - tests/e2e/amd64_resource_files/ingress.yaml | 23 +- tests/e2e/go.mod | 2 +- tests/e2e/go.sum | 8 + tests/e2e/scripts/latest_commit.sh | 10 +- tests/e2e/testutils.go | 184 +++++---- tests/e2e/upgradecluster/Vagrantfile | 25 +- .../e2e/upgradecluster/upgradecluster_test.go | 383 ++++++++++++++++++ tests/e2e/vagrantdefaults.rb | 13 + tests/e2e/validatecluster/Vagrantfile | 24 +- .../validatecluster/validatecluster_test.go | 253 ++++++++++-- .../custometcdargs/custometcdargs_int_test.go | 7 +- .../dualstack/dualstack_int_test.go | 7 +- .../etcdrestore/etcd_restore_int_test.go | 7 +- .../etcdsnapshot/etcdsnapshot_int_test.go | 7 +- .../localstorage/localstorage_int_test.go | 7 +- .../secretsencryption_int_test.go | 7 +- 21 files changed, 821 insertions(+), 207 deletions(-) mode change 100644 => 100755 tests/e2e/scripts/latest_commit.sh create mode 100644 tests/e2e/upgradecluster/upgradecluster_test.go create mode 100644 tests/e2e/vagrantdefaults.rb diff --git a/go.mod b/go.mod index 3e8b8e57b0..ed406a0c41 100644 --- a/go.mod +++ b/go.mod @@ -99,9 +99,8 @@ require ( github.com/mattn/go-sqlite3 v1.14.8 github.com/minio/minio-go/v7 v7.0.7 github.com/natefinch/lumberjack v2.0.0+incompatible - github.com/onsi/ginkgo v1.16.4 - github.com/onsi/gomega v1.11.0 - // LOOK TO scripts/download FOR THE VERSION OF runc THAT WE ARE BUILDING/SHIPPING + github.com/onsi/ginkgo/v2 v2.1.1 + github.com/onsi/gomega v1.17.0 github.com/opencontainers/runc v1.0.3 github.com/opencontainers/selinux v1.8.2 github.com/otiai10/copy v1.6.0 diff --git a/go.sum b/go.sum index fbeaf9dac1..5bbce9b1b5 100644 --- a/go.sum +++ b/go.sum @@ -491,6 +491,7 @@ github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= @@ -863,6 +864,8 @@ github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9k github.com/onsi/ginkgo v1.16.1/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E= github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/ginkgo/v2 v2.1.1 h1:LCnPB85AvFNr91s0B2aDzEiiIg6MUwLYbryC1NSlWi8= +github.com/onsi/ginkgo/v2 v2.1.1/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.3.0/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= @@ -870,8 +873,9 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J github.com/onsi/gomega v1.8.1/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= -github.com/onsi/gomega v1.11.0 h1:+CqWgvj0OZycCaqclBD1pxKHAU+tOkHmQIWvDHq2aug= github.com/onsi/gomega v1.11.0/go.mod h1:azGKhqFUon9Vuj0YmTfLSmx0FUwqXYSTl5re8lQLTUg= +github.com/onsi/gomega v1.17.0 h1:9Luw4uT5HTjHTN8+aNcSThgH1vdXnmdJ8xIfZ4wyTRE= +github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= diff --git a/pkg/flock/flock_int_test.go b/pkg/flock/flock_int_test.go index 2641d33880..ffd6c12432 100644 --- a/pkg/flock/flock_int_test.go +++ b/pkg/flock/flock_int_test.go @@ -3,8 +3,7 @@ package flock_test import ( "testing" - . "github.com/onsi/ginkgo" - "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/rancher/k3s/pkg/flock" ) @@ -31,7 +30,5 @@ var _ = Describe("file locks", func() { func TestFlock(t *testing.T) { RegisterFailHandler(Fail) - RunSpecsWithDefaultAndCustomReporters(t, "Flock Suite", []Reporter{ - reporters.NewJUnitReporter("/tmp/results/junit-flock.xml"), - }) + RunSpecs(t, "Flock Suite") } diff --git a/tests/TESTING.md b/tests/TESTING.md index c5ca3e51e8..8fa58ddf73 100644 --- a/tests/TESTING.md +++ b/tests/TESTING.md @@ -35,7 +35,7 @@ To facilitate unit test creation, see `tests/util/runtime.go` helper functions. All unit tests should be placed within the package of the file they test. All unit test files should be named: `_test.go`. All unit test functions should be named: `Test_Unit` or `Test_Unit_`. -See the [etcd unit test](https://github.com/k3s-io/k3s/blob/master/pkg/etcd/etcd_test.go) as an example. +See the [etcd unit test](../pkg/etcd/etcd_test.go) as an example. ### Running @@ -62,29 +62,27 @@ To facilitate K3s CLI testing, see `tests/util/cmd.go` helper functions. ### Format -Integration tests can be placed in two areas: - -1. Next to the go package they intend to test. -2. In `tests/integration/` for package agnostic testing. - -Package specific integration tests should use the `_test` package. -Package agnostic integration tests should use the `integration` package. +All integration tests should be placed under `tests/integration/`. All integration test files should be named: `_int_test.go`. All integration test functions should be named: `Test_Integration`. -See the [etcd snapshot test](https://github.com/k3s-io/k3s/blob/master/pkg/etcd/etcd_int_test.go) as a package specific example. -See the [local storage test](https://github.com/k3s-io/k3s/blob/master/tests/integration/localstorage/localstorage_int_test.go) as a package agnostic example. +See the [local storage test](../tests/integration/localstorage/localstorage_int_test.go) as an example. ### Running Integration tests can be run with no k3s cluster present, each test will spin up and kill the appropriate k3s server it needs. Note: Integration tests must be run as root, prefix the commands below with `sudo -E env "PATH=$PATH"` if a sudo user. ```bash -go test ./pkg/... ./tests/integration/... -run Integration +go test ./tests/integration/... -run Integration +``` + +Additionally, to generate JUnit reporting for the tests, the Ginkgo CLI is used +``` +ginkgo --junit-report=result.xml ./tests/integration/... ``` Integration tests can be run on an existing single-node cluster via compile time flag, tests will skip if the server is not configured correctly. ```bash -go test -ldflags "-X 'github.com/rancher/k3s/tests/util.existingServer=True'" ./pkg/... ./tests/integration/... -run Integration +go test -ldflags "-X 'github.com/rancher/k3s/tests/util.existingServer=True'" ./tests/integration/... -run Integration ``` Integration tests can also be run via a [Sonobuoy](https://sonobuoy.io/docs/v0.53.2/) plugin on an existing single-node cluster. @@ -191,10 +189,25 @@ See the [validate cluster test](../tests/e2e/validatecluster/validatecluster_tes Generally, E2E tests are run as a nightly Jenkins job for QA. They can still be run locally but additional setup may be required. By default, all E2E tests are designed with `libvirt` as the underlying VM provider. Instructions for installing libvirt and its associated vagrant plugin, `vagrant-libvirt` can be found [here.](https://github.com/vagrant-libvirt/vagrant-libvirt#installation) `VirtualBox` is also supported as a backup VM provider. -Once setup is complete, E2E tests can be run with: +Once setup is complete, all E2E tests can be run with: ```bash -go test ./tests/e2e/... -run E2E +go test -timeout=15m ./tests/e2e/... -run E2E ``` +Tests can be run individually with: +```bash +go test -timeout=15m ./tests/e2e/validatecluster/... -run E2E +#or +go test -timeout=15m ./tests/e2e/... -run E2EClusterValidation +``` + +Additionally, to generate junit reporting for the tests, the Ginkgo CLI is used. Installation instructions can be found [here.](https://onsi.github.io/ginkgo/#getting-started) + +To run the all E2E tests and generate JUnit testing reports: +``` +ginkgo --junit-report=result.xml ./tests/e2e/... +``` + +Note: The `go test` default timeout is 10 minutes, thus the `-timeout` flag should be used. The `ginkgo` default timeout is 1 hour, no timeout flag is needed. ## Contributing New Or Updated Tests diff --git a/tests/e2e/Dockerfile.build b/tests/e2e/Dockerfile.build index 97cfcc7db4..6a96df285e 100644 --- a/tests/e2e/Dockerfile.build +++ b/tests/e2e/Dockerfile.build @@ -22,5 +22,3 @@ WORKDIR $GOPATH/src/github.com/rancher/k3s-io/k3s/tests/e2e COPY . . -RUN go get -u github.com/onsi/gomega -RUN go get -u github.com/onsi/ginkgo diff --git a/tests/e2e/amd64_resource_files/ingress.yaml b/tests/e2e/amd64_resource_files/ingress.yaml index 957e7ad005..cf49a50640 100644 --- a/tests/e2e/amd64_resource_files/ingress.yaml +++ b/tests/e2e/amd64_resource_files/ingress.yaml @@ -1,16 +1,19 @@ -apiVersion: extensions/v1beta1 +apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: ingress + name: test-ingress spec: - rules: - - host: foo1.bar.com - http: - paths: - - path: /name.html - backend: - serviceName: nginx-ingress-svc - servicePort: 80 + rules: + - host: foo1.bar.com + http: + paths: + - backend: + service: + name: nginx-ingress-svc + port: + number: 80 + path: / + pathType: ImplementationSpecific --- apiVersion: v1 kind: Service diff --git a/tests/e2e/go.mod b/tests/e2e/go.mod index 5944afc603..8786b5c787 100644 --- a/tests/e2e/go.mod +++ b/tests/e2e/go.mod @@ -4,7 +4,7 @@ go 1.15 require ( github.com/kr/pretty v0.2.0 // indirect - github.com/onsi/ginkgo v1.16.4 + github.com/onsi/ginkgo/v2 v2.1.1 github.com/onsi/gomega v1.17.0 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect ) diff --git a/tests/e2e/go.sum b/tests/e2e/go.sum index 401aff7701..f9529a34fe 100644 --- a/tests/e2e/go.sum +++ b/tests/e2e/go.sum @@ -1,3 +1,6 @@ +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -19,7 +22,9 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -32,6 +37,8 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/ginkgo/v2 v2.1.1 h1:LCnPB85AvFNr91s0B2aDzEiiIg6MUwLYbryC1NSlWi8= +github.com/onsi/ginkgo/v2 v2.1.1/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0 h1:9Luw4uT5HTjHTN8+aNcSThgH1vdXnmdJ8xIfZ4wyTRE= @@ -60,6 +67,7 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/tests/e2e/scripts/latest_commit.sh b/tests/e2e/scripts/latest_commit.sh old mode 100644 new mode 100755 index 9f8780114d..db63eb6d70 --- a/tests/e2e/scripts/latest_commit.sh +++ b/tests/e2e/scripts/latest_commit.sh @@ -1,7 +1,15 @@ +#!/bin/bash # Grabs the last 5 commit SHA's from the given branch, then purges any commits that do not have a passing CI build +iterations=0 curl -s -H 'Accept: application/vnd.github.v3+json' "https://api.github.com/repos/k3s-io/k3s/commits?per_page=5&sha=$1" | jq -r '.[] | .sha' &> $2 curl -s --fail https://storage.googleapis.com/k3s-ci-builds/k3s-$(head -n 1 $2).sha256sum while [ $? -ne 0 ]; do - sed -i 1d $2 + ((iterations++)) + if [ "$iterations" -ge 6 ]; then + echo "No valid commits found" + exit 1 + fi + sed -i 1d "$2" + sleep 1 curl -s --fail https://storage.googleapis.com/k3s-ci-builds/k3s-$(head -n 1 $2).sha256sum done \ No newline at end of file diff --git a/tests/e2e/testutils.go b/tests/e2e/testutils.go index 77d16c0c76..cbf5db1f10 100644 --- a/tests/e2e/testutils.go +++ b/tests/e2e/testutils.go @@ -3,8 +3,10 @@ package e2e import ( "bytes" "fmt" + "io/ioutil" "os" "os/exec" + "path/filepath" "strconv" "strings" ) @@ -27,53 +29,58 @@ type Pod struct { Node string } -//Runs command passed from within the node ServerIP -func RunCmdOnNode(cmd string, nodename string) (string, error) { - runcmd := "vagrant ssh server-0 -c " + cmd - c := exec.Command("bash", "-c", runcmd) - - var out bytes.Buffer - var errOut bytes.Buffer - c.Stdout = &out - c.Stderr = &errOut - if err := c.Run(); err != nil { - return errOut.String(), err +func CountOfStringInSlice(str string, pods []Pod) int { + count := 0 + for _, pod := range pods { + if strings.Contains(pod.Name, str) { + count++ + } } - return out.String(), nil + return count } -// RunCommand Runs command on the cluster accessing the cluster through kubeconfig file -func RunCommand(cmd string) (string, error) { - c := exec.Command("bash", "-c", cmd) - var out bytes.Buffer - c.Stdout = &out - if err := c.Run(); err != nil { - return "", err - } - return out.String(), nil -} - -func CreateCluster(nodeos string, serverCount int, agentCount int) ([]string, []string, error) { - serverNodenames := make([]string, serverCount+1) +func CreateCluster(nodeOS string, serverCount int, agentCount int, installType string) ([]string, []string, error) { + serverNodeNames := make([]string, serverCount) for i := 0; i < serverCount; i++ { - serverNodenames[i] = "server-" + strconv.Itoa(i) + serverNodeNames[i] = "server-" + strconv.Itoa(i) } - - agentNodenames := make([]string, agentCount+1) + agentNodeNames := make([]string, agentCount) for i := 0; i < agentCount; i++ { - agentNodenames[i] = "agent-" + strconv.Itoa(i) + agentNodeNames[i] = "agent-" + strconv.Itoa(i) } - nodeRoles := strings.Join(serverNodenames, " ") + strings.Join(agentNodenames, " ") + nodeRoles := strings.Join(serverNodeNames, " ") + " " + strings.Join(agentNodeNames, " ") + nodeRoles = strings.TrimSpace(nodeRoles) - nodeBoxes := strings.Repeat(nodeos+" ", serverCount+agentCount) + nodeBoxes := strings.Repeat(nodeOS+" ", serverCount+agentCount) nodeBoxes = strings.TrimSpace(nodeBoxes) - cmd := fmt.Sprintf("NODE_ROLES=\"%s\" NODE_BOXES=\"%s\" vagrant up &> vagrant.log", nodeRoles, nodeBoxes) - if out, err := RunCommand(cmd); err != nil { - fmt.Println("Error Creating Cluster", out) + cmd := fmt.Sprintf("NODE_ROLES=\"%s\" NODE_BOXES=\"%s\" %s vagrant up &> vagrant.log", nodeRoles, nodeBoxes, installType) + fmt.Println(cmd) + if _, err := RunCommand(cmd); err != nil { + fmt.Println("Error Creating Cluster", err) return nil, nil, err } + return serverNodeNames, agentNodeNames, nil +} - return serverNodenames, agentNodenames, nil +func DeployWorkload(workload, kubeconfig string, arch bool) (string, error) { + resource_dir := "../amd64_resource_files" + if arch { + resource_dir = "../arm64_resource_files" + } + files, err := ioutil.ReadDir(resource_dir) + if err != nil { + err = fmt.Errorf("%s : Unable to read resource manifest file for %s", err, workload) + return "", err + } + fmt.Println("\nDeploying", workload) + for _, f := range files { + filename := filepath.Join(resource_dir, f.Name()) + if strings.TrimSpace(f.Name()) == workload { + cmd := "kubectl apply -f " + filename + " --kubeconfig=" + kubeconfig + return RunCommand(cmd) + } + } + return "", nil } func DestroyCluster() error { @@ -83,8 +90,36 @@ func DestroyCluster() error { return os.Remove("vagrant.log") } +func FetchClusterIP(kubeconfig string, servicename string) (string, error) { + cmd := "kubectl get svc " + servicename + " -o jsonpath='{.spec.clusterIP}' --kubeconfig=" + kubeconfig + return RunCommand(cmd) +} + +func FetchIngressIP(kubeconfig string) ([]string, error) { + cmd := "kubectl get ing ingress -o jsonpath='{.status.loadBalancer.ingress[*].ip}' --kubeconfig=" + kubeconfig + res, err := RunCommand(cmd) + if err != nil { + return nil, err + } + ingressIP := strings.Trim(res, " ") + ingressIPs := strings.Split(ingressIP, " ") + return ingressIPs, nil +} + +func FetchNodeExternalIP(nodename string) (string, error) { + cmd := "vagrant ssh " + nodename + " -c \"ip -f inet addr show eth1| awk '/inet / {print $2}'|cut -d/ -f1\"" + ipaddr, err := RunCommand(cmd) + if err != nil { + return "", err + } + ips := strings.Trim(ipaddr, "") + ip := strings.Split(ips, "inet") + nodeip := strings.TrimSpace(ip[1]) + return nodeip, nil +} + func GenKubeConfigFile(serverName string) (string, error) { - cmd := fmt.Sprintf("vagrant ssh %s -c \"cat /etc/rancher/k3s/k3s.yaml\"", serverName) + cmd := fmt.Sprintf("vagrant ssh %s -c \"sudo cat /etc/rancher/k3s/k3s.yaml\"", serverName) kubeConfig, err := RunCommand(cmd) if err != nil { return "", err @@ -101,44 +136,13 @@ func GenKubeConfigFile(serverName string) (string, error) { return kubeConfigFile, nil } -func DeployWorkload(workload string, kubeconfig string) (string, error) { - cmd := "kubectl apply -f " + workload + " --kubeconfig=" + kubeconfig - return RunCommand(cmd) -} - -func FetchClusterIP(kubeconfig string, servicename string) (string, error) { - cmd := "kubectl get svc " + servicename + " -o jsonpath='{.spec.clusterIP}' --kubeconfig=" + kubeconfig - return RunCommand(cmd) -} - -func FetchNodeExternalIP(nodename string) (string, error) { - cmd := "vagrant ssh " + nodename + " -c \"ip -f inet addr show eth1| awk '/inet / {print $2}'|cut -d/ -f1\"" - ipaddr, err := RunCommand(cmd) - if err != nil { - return "", err - } - ips := strings.Trim(ipaddr, "") - ip := strings.Split(ips, "inet") - nodeip := strings.TrimSpace(ip[1]) - return nodeip, nil -} -func FetchIngressIP(kubeconfig string) ([]string, error) { - cmd := "kubectl get ing ingress -o jsonpath='{.status.loadBalancer.ingress[*].ip}' --kubeconfig=" + kubeconfig - res, err := RunCommand(cmd) - if err != nil { - return nil, err - } - ingressIP := strings.Trim(res, " ") - ingressIPs := strings.Split(ingressIP, " ") - return ingressIPs, nil -} - -func ParseNodes(kubeConfig string, debug bool) ([]Node, error) { +func ParseNodes(kubeConfig string, print bool) ([]Node, error) { nodes := make([]Node, 0, 10) nodeList := "" cmd := "kubectl get nodes --no-headers -o wide -A --kubeconfig=" + kubeConfig res, err := RunCommand(cmd) + if err != nil { return nil, err } @@ -157,13 +161,13 @@ func ParseNodes(kubeConfig string, debug bool) ([]Node, error) { nodes = append(nodes, node) } } - if debug { + if print { fmt.Println(nodeList) } return nodes, nil } -func ParsePods(kubeconfig string, debug bool) ([]Pod, error) { +func ParsePods(kubeconfig string, print bool) ([]Pod, error) { pods := make([]Pod, 0, 10) podList := "" @@ -186,8 +190,44 @@ func ParsePods(kubeconfig string, debug bool) ([]Pod, error) { } pods = append(pods, pod) } - if debug { + if print { fmt.Println(podList) } return pods, nil } + +// RunCmdOnNode executes a command from within the given node +func RunCmdOnNode(cmd string, nodename string) (string, error) { + runcmd := "vagrant ssh -c " + cmd + " " + nodename + return RunCommand(runcmd) +} + +// RunCommand executes a command on the host +func RunCommand(cmd string) (string, error) { + c := exec.Command("bash", "-c", cmd) + var out bytes.Buffer + c.Stdout = &out + if err := c.Run(); err != nil { + return "", err + } + return out.String(), nil +} + +func UpgradeCluster(serverNodenames []string, agentNodenames []string) error { + for _, nodeName := range serverNodenames { + cmd := "RELEASE_CHANNEL=commit vagrant provision " + nodeName + fmt.Println(cmd) + if out, err := RunCommand(cmd); err != nil { + fmt.Println("Error Upgrading Cluster", out) + return err + } + } + for _, nodeName := range agentNodenames { + cmd := "RELEASE_CHANNEL=commit vagrant provision " + nodeName + if _, err := RunCommand(cmd); err != nil { + fmt.Println("Error Upgrading Cluster", err) + return err + } + } + return nil +} diff --git a/tests/e2e/upgradecluster/Vagrantfile b/tests/e2e/upgradecluster/Vagrantfile index 857afc906b..fb9e99aab4 100644 --- a/tests/e2e/upgradecluster/Vagrantfile +++ b/tests/e2e/upgradecluster/Vagrantfile @@ -17,9 +17,11 @@ def provision(vm, roles, role_num, node_num) # An expanded netmask is required to allow VM<-->VM communication, virtualbox defaults to /32 vm.network "private_network", ip: "#{NETWORK_PREFIX}.#{100+node_num}", netmask: "255.255.255.0" - osConfigure(vm) - + vagrant_defaults = '../vagrantdefaults.rb' + load vagrant_defaults if File.exists?(vagrant_defaults) + defaultOSConfigure(vm) + if !RELEASE_VERSION.empty? install_type = "INSTALL_K3S_VERSION=#{RELEASE_VERSION}" elsif RELEASE_CHANNEL == "commit" @@ -35,14 +37,12 @@ def provision(vm, roles, role_num, node_num) if roles.include?("server") && role_num == 0 vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s| - k3s.installer_url = 'https://get.k3s.io' k3s.args = %W[server --cluster-init --node-external-ip=#{NETWORK_PREFIX}.100 --flannel-iface=eth1] k3s.env = %W[K3S_KUBECONFIG_MODE=0644 K3S_TOKEN=vagrant #{install_type}] k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 end elsif roles.include?("server") && role_num != 0 vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s| - k3s.installer_url = 'https://get.k3s.io' k3s.args = %W[server --server https://#{NETWORK_PREFIX}.100:6443 --flannel-iface=eth1] k3s.env = %W[K3S_KUBECONFIG_MODE=0644 K3S_TOKEN=vagrant #{install_type}] k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 @@ -50,7 +50,6 @@ def provision(vm, roles, role_num, node_num) end if roles.include?("agent") vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s| - k3s.installer_url = 'https://get.k3s.io' k3s.args = %W[agent --server https://#{NETWORK_PREFIX}.100:6443 --flannel-iface=eth1] k3s.env = %W[K3S_KUBECONFIG_MODE=0644 K3S_TOKEN=vagrant #{install_type}] k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 @@ -61,22 +60,6 @@ def provision(vm, roles, role_num, node_num) end end -def osConfigure(vm) - - if vm.box.include?("ubuntu2004") - vm.provision "shell", inline: "systemd-resolve --set-dns=8.8.8.8 --interface=eth0", run: 'once' - vm.provision "shell", inline: "apt install -y jq", run: 'once' - end - if vm.box.include?("Leap") - vm.provision "shell", inline: "zypper install -y jq", run: 'once' - end - if vm.box.include?("microos") - vm.provision "shell", inline: "transactional-update pkg install -y jq", run: 'once' - vm.provision 'reload', run: 'once' - end - -end - Vagrant.configure("2") do |config| config.vagrant.plugins = ["vagrant-k3s", "vagrant-reload"] # Default provider is libvirt, virtualbox is only provided as a backup diff --git a/tests/e2e/upgradecluster/upgradecluster_test.go b/tests/e2e/upgradecluster/upgradecluster_test.go new file mode 100644 index 0000000000..cb7313fe08 --- /dev/null +++ b/tests/e2e/upgradecluster/upgradecluster_test.go @@ -0,0 +1,383 @@ +package upgradecluster + +import ( + "flag" + "fmt" + "os" + "strings" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/rancher/k3s/tests/e2e" +) + +// Valid nodeOS: generic/ubuntu2004, opensuse/Leap-15.3.x86_64, dweomer/microos.amd64 +var nodeOS = flag.String("nodeOS", "generic/ubuntu2004", "VM operating system") +var serverCount = flag.Int("serverCount", 3, "number of server nodes") +var agentCount = flag.Int("agentCount", 2, "number of agent nodes") + +//valid format: RELEASE_VERSION=v1.23.1+k3s2 or nil for latest commit from master +var installType = flag.String("installType", "", "version or nil to use latest commit") + +func Test_E2EUpgradeValidation(t *testing.T) { + RegisterFailHandler(Fail) + flag.Parse() + RunSpecs(t, "Create Cluster Test Suite") +} + +var ( + kubeConfigFile string + serverNodenames []string + agentNodenames []string +) + +var _ = Describe("Verify Upgrade", func() { + Context("Cluster :", func() { + It("Starts up with no issues", func() { + var err error + serverNodenames, agentNodenames, err = e2e.CreateCluster(*nodeOS, *serverCount, *agentCount, *installType) + Expect(err).NotTo(HaveOccurred()) + fmt.Println("CLUSTER CONFIG") + fmt.Println("OS:", *nodeOS) + fmt.Println("Server Nodes:", serverNodenames) + fmt.Println("Agent Nodes:", agentNodenames) + kubeConfigFile, err = e2e.GenKubeConfigFile(serverNodenames[0]) + Expect(err).NotTo(HaveOccurred()) + }) + + It("Checks Node and Pod Status", func() { + fmt.Printf("\nFetching node status\n") + Eventually(func(g Gomega) { + nodes, err := e2e.ParseNodes(kubeConfigFile, false) + g.Expect(err).NotTo(HaveOccurred()) + for _, node := range nodes { + g.Expect(node.Status).Should(Equal("Ready")) + } + }, "420s", "5s").Should(Succeed()) + _, _ = e2e.ParseNodes(kubeConfigFile, true) + + fmt.Printf("\nFetching Pods status\n") + Eventually(func(g Gomega) { + pods, err := e2e.ParsePods(kubeConfigFile, false) + g.Expect(err).NotTo(HaveOccurred()) + for _, pod := range pods { + if strings.Contains(pod.Name, "helm-install") { + g.Expect(pod.Status).Should(Equal("Completed"), pod.Name) + } else { + g.Expect(pod.Status).Should(Equal("Running"), pod.Name) + } + } + }, "420s", "5s").Should(Succeed()) + _, _ = e2e.ParsePods(kubeConfigFile, true) + }) + + It("Verifies ClusterIP Service", func() { + _, err := e2e.DeployWorkload("clusterip.yaml", kubeConfigFile, false) + + Expect(err).NotTo(HaveOccurred(), "Cluster IP manifest not deployed") + + cmd := "kubectl get pods -o=name -l k8s-app=nginx-app-clusterip --field-selector=status.phase=Running --kubeconfig=" + kubeConfigFile + Eventually(func() (string, error) { + return e2e.RunCommand(cmd) + }, "240s", "5s").Should(ContainSubstring("test-clusterip"), "failed cmd: "+cmd) + + clusterip, _ := e2e.FetchClusterIP(kubeConfigFile, "nginx-clusterip-svc") + cmd = "\"curl -L --insecure http://" + clusterip + "/name.html\"" + for _, nodeName := range serverNodenames { + Eventually(func() (string, error) { + return e2e.RunCmdOnNode(cmd, nodeName) + }, "120s", "10s").Should(ContainSubstring("test-clusterip"), "failed cmd: "+cmd) + } + }) + + It("Verifies NodePort Service", func() { + _, err := e2e.DeployWorkload("nodeport.yaml", kubeConfigFile, false) + Expect(err).NotTo(HaveOccurred(), "NodePort manifest not deployed") + + for _, nodeName := range serverNodenames { + node_external_ip, _ := e2e.FetchNodeExternalIP(nodeName) + cmd := "kubectl get service nginx-nodeport-svc --kubeconfig=" + kubeConfigFile + " --output jsonpath=\"{.spec.ports[0].nodePort}\"" + nodeport, err := e2e.RunCommand(cmd) + Expect(err).NotTo(HaveOccurred(), "failed cmd: "+cmd) + + cmd = "kubectl get pods -o=name -l k8s-app=nginx-app-nodeport --field-selector=status.phase=Running --kubeconfig=" + kubeConfigFile + Eventually(func() (string, error) { + return e2e.RunCommand(cmd) + }, "240s", "5s").Should(ContainSubstring("test-nodeport"), "nodeport pod was not created") + + cmd = "curl -L --insecure http://" + node_external_ip + ":" + nodeport + "/name.html" + fmt.Println(cmd) + Eventually(func() (string, error) { + return e2e.RunCommand(cmd) + }, "240s", "5s").Should(ContainSubstring("test-nodeport"), "failed cmd: "+cmd) + } + }) + + It("Verifies LoadBalancer Service", func() { + _, err := e2e.DeployWorkload("loadbalancer.yaml", kubeConfigFile, false) + Expect(err).NotTo(HaveOccurred(), "Loadbalancer manifest not deployed") + for _, nodeName := range serverNodenames { + ip, _ := e2e.FetchNodeExternalIP(nodeName) + cmd := "kubectl get service nginx-loadbalancer-svc --kubeconfig=" + kubeConfigFile + " --output jsonpath=\"{.spec.ports[0].port}\"" + port, err := e2e.RunCommand(cmd) + Expect(err).NotTo(HaveOccurred()) + + cmd = "kubectl get pods -o=name -l k8s-app=nginx-app-loadbalancer --field-selector=status.phase=Running --kubeconfig=" + kubeConfigFile + Eventually(func() (string, error) { + return e2e.RunCommand(cmd) + }, "240s", "5s").Should(ContainSubstring("test-loadbalancer")) + + cmd = "curl -L --insecure http://" + ip + ":" + port + "/name.html" + Eventually(func() (string, error) { + return e2e.RunCommand(cmd) + }, "240s", "5s").Should(ContainSubstring("test-loadbalancer"), "failed cmd: "+cmd) + } + }) + + It("Verifies Ingress", func() { + _, err := e2e.DeployWorkload("ingress.yaml", kubeConfigFile, false) + Expect(err).NotTo(HaveOccurred(), "Ingress manifest not deployed") + + for _, nodeName := range serverNodenames { + ip, _ := e2e.FetchNodeExternalIP(nodeName) + cmd := "curl --header host:foo1.bar.com" + " http://" + ip + "/name.html" + Eventually(func() (string, error) { + return e2e.RunCommand(cmd) + }, "240s", "5s").Should(ContainSubstring("test-ingress"), "failed cmd: "+cmd) + } + }) + + It("Verifies Daemonset", func() { + _, err := e2e.DeployWorkload("daemonset.yaml", kubeConfigFile, false) + Expect(err).NotTo(HaveOccurred(), "Daemonset manifest not deployed") + + nodes, _ := e2e.ParseNodes(kubeConfigFile, false) //nodes := + pods, _ := e2e.ParsePods(kubeConfigFile, false) + + Eventually(func(g Gomega) { + count := e2e.CountOfStringInSlice("test-daemonset", pods) + fmt.Println("POD COUNT") + fmt.Println(count) + fmt.Println("NODE COUNT") + fmt.Println(len(nodes)) + g.Expect(len(nodes)).Should((Equal(count)), "Daemonset pod count does not match node count") + }, "240s", "10s").Should(Succeed()) + }) + + It("Verifies dns access", func() { + _, err := e2e.DeployWorkload("dnsutils.yaml", kubeConfigFile, false) + Expect(err).NotTo(HaveOccurred(), "dnsutils manifest not deployed") + + Eventually(func() (string, error) { + cmd := "kubectl get pods dnsutils --kubeconfig=" + kubeConfigFile + return e2e.RunCommand(cmd) + }, "420s", "2s").Should(ContainSubstring("dnsutils")) + + cmd := "kubectl --kubeconfig=" + kubeConfigFile + " exec -i -t dnsutils -- nslookup kubernetes.default" + Eventually(func() (string, error) { + return e2e.RunCommand(cmd) + }, "420s", "2s").Should(ContainSubstring("kubernetes.default.svc.cluster.local")) + }) + + It("Verifies Local Path Provisioner storage ", func() { + _, err := e2e.DeployWorkload("local-path-provisioner.yaml", kubeConfigFile, false) + Expect(err).NotTo(HaveOccurred(), "local-path-provisioner manifest not deployed") + Eventually(func(g Gomega) { + cmd := "kubectl get pvc local-path-pvc --kubeconfig=" + kubeConfigFile + res, err := e2e.RunCommand(cmd) + g.Expect(err).NotTo(HaveOccurred()) + fmt.Println(res) + g.Expect(res).Should(ContainSubstring("local-path-pvc")) + g.Expect(res).Should(ContainSubstring("Bound")) + }, "240s", "2s").Should(Succeed()) + + Eventually(func(g Gomega) { + cmd := "kubectl get pod volume-test --kubeconfig=" + kubeConfigFile + res, err := e2e.RunCommand(cmd) + g.Expect(err).NotTo(HaveOccurred()) + fmt.Println(res) + + g.Expect(res).Should(ContainSubstring("volume-test")) + g.Expect(res).Should(ContainSubstring("Running")) + }, "420s", "2s").Should(Succeed()) + + cmd := "kubectl --kubeconfig=" + kubeConfigFile + " exec volume-test -- sh -c 'echo local-path-test > /data/test'" + _, err = e2e.RunCommand(cmd) + Expect(err).NotTo(HaveOccurred()) + fmt.Println("Data stored in pvc: local-path-test") + + cmd = "kubectl delete pod volume-test --kubeconfig=" + kubeConfigFile + res, err := e2e.RunCommand(cmd) + Expect(err).NotTo(HaveOccurred()) + fmt.Println(res) + + _, err = e2e.DeployWorkload("local-path-provisioner.yaml", kubeConfigFile, false) + Expect(err).NotTo(HaveOccurred(), "local-path-provisioner manifest not deployed") + + Eventually(func() (string, error) { + cmd := "kubectl get pods -o=name -l app=local-path-provisioner --field-selector=status.phase=Running -n kube-system --kubeconfig=" + kubeConfigFile + return e2e.RunCommand(cmd) + }, "420s", "2s").Should(ContainSubstring("local-path-provisioner")) + + Eventually(func(g Gomega) { + cmd := "kubectl get pod volume-test --kubeconfig=" + kubeConfigFile + res, err := e2e.RunCommand(cmd) + g.Expect(err).NotTo(HaveOccurred()) + fmt.Println(res) + g.Expect(res).Should(ContainSubstring("volume-test")) + g.Expect(res).Should(ContainSubstring("Running")) + }, "420s", "2s").Should(Succeed()) + + // Check data after re-creation + Eventually(func() (string, error) { + cmd = "kubectl exec volume-test cat /data/test --kubeconfig=" + kubeConfigFile + return e2e.RunCommand(cmd) + }, "180s", "2s").Should(ContainSubstring("local-path-test")) + }) + + It("Upgrades with no issues", func() { + var err error + err = e2e.UpgradeCluster(serverNodenames, agentNodenames) + fmt.Println(err) + Expect(err).NotTo(HaveOccurred()) + fmt.Println("CLUSTER UPGRADED") + kubeConfigFile, err = e2e.GenKubeConfigFile(serverNodenames[0]) + Expect(err).NotTo(HaveOccurred()) + }) + + It("After upgrade Checks Node and Pod Status", func() { + fmt.Printf("\nFetching node status\n") + Eventually(func(g Gomega) { + nodes, err := e2e.ParseNodes(kubeConfigFile, false) + g.Expect(err).NotTo(HaveOccurred()) + for _, node := range nodes { + g.Expect(node.Status).Should(Equal("Ready")) + } + }, "420s", "5s").Should(Succeed()) + e2e.ParseNodes(kubeConfigFile, true) + + fmt.Printf("\nFetching Pods status\n") + Eventually(func(g Gomega) { + pods, err := e2e.ParsePods(kubeConfigFile, false) + g.Expect(err).NotTo(HaveOccurred()) + for _, pod := range pods { + if strings.Contains(pod.Name, "helm-install") { + g.Expect(pod.Status).Should(Equal("Completed")) + } else { + g.Expect(pod.Status).Should(Equal("Running")) + } + } + }, "420s", "5s").Should(Succeed()) + e2e.ParsePods(kubeConfigFile, true) + }) + + It("After upgrade verifies ClusterIP Service", func() { + Eventually(func() (string, error) { + cmd := "kubectl get pods -o=name -l k8s-app=nginx-app-clusterip --field-selector=status.phase=Running --kubeconfig=" + kubeConfigFile + return e2e.RunCommand(cmd) + }, "420s", "5s").Should(ContainSubstring("test-clusterip")) + + clusterip, _ := e2e.FetchClusterIP(kubeConfigFile, "nginx-clusterip-svc") + cmd := "\"curl -L --insecure http://" + clusterip + "/name.html\"" + fmt.Println(cmd) + for _, nodeName := range serverNodenames { + Eventually(func() (string, error) { + return e2e.RunCmdOnNode(cmd, nodeName) + }, "120s", "10s").Should(ContainSubstring("test-clusterip"), "failed cmd: "+cmd) + } + }) + + It("After upgrade verifies NodePort Service", func() { + + for _, nodeName := range serverNodenames { + node_external_ip, _ := e2e.FetchNodeExternalIP(nodeName) + cmd := "kubectl get service nginx-nodeport-svc --kubeconfig=" + kubeConfigFile + " --output jsonpath=\"{.spec.ports[0].nodePort}\"" + nodeport, err := e2e.RunCommand(cmd) + Expect(err).NotTo(HaveOccurred()) + + Eventually(func() (string, error) { + cmd := "kubectl get pods -o=name -l k8s-app=nginx-app-nodeport --field-selector=status.phase=Running --kubeconfig=" + kubeConfigFile + return e2e.RunCommand(cmd) + }, "240s", "5s").Should(ContainSubstring("test-nodeport"), "nodeport pod was not created") + + cmd = "curl -L --insecure http://" + node_external_ip + ":" + nodeport + "/name.html" + fmt.Println(cmd) + Eventually(func() (string, error) { + return e2e.RunCommand(cmd) + }, "240s", "5s").Should(ContainSubstring("test-nodeport")) + } + }) + + It("After upgrade verifies LoadBalancer Service", func() { + for _, nodeName := range serverNodenames { + ip, _ := e2e.FetchNodeExternalIP(nodeName) + cmd := "kubectl get service nginx-loadbalancer-svc --kubeconfig=" + kubeConfigFile + " --output jsonpath=\"{.spec.ports[0].port}\"" + port, err := e2e.RunCommand(cmd) + Expect(err).NotTo(HaveOccurred()) + Eventually(func() (string, error) { + cmd := "curl -L --insecure http://" + ip + ":" + port + "/name.html" + return e2e.RunCommand(cmd) + }, "240s", "5s").Should(ContainSubstring("test-loadbalancer")) + + Eventually(func() (string, error) { + cmd := "kubectl get pods -o=name -l k8s-app=nginx-app-loadbalancer --field-selector=status.phase=Running --kubeconfig=" + kubeConfigFile + return e2e.RunCommand(cmd) + }, "240s", "5s").Should(ContainSubstring("test-loadbalancer")) + } + }) + + It("After upgrade verifies Ingress", func() { + for _, nodeName := range serverNodenames { + ip, _ := e2e.FetchNodeExternalIP(nodeName) + cmd := "curl --header host:foo1.bar.com" + " http://" + ip + "/name.html" + fmt.Println(cmd) + + Eventually(func() (string, error) { + return e2e.RunCommand(cmd) + }, "420s", "5s").Should(ContainSubstring("test-ingress")) + } + }) + + It("After upgrade verifies Daemonset", func() { + nodes, _ := e2e.ParseNodes(kubeConfigFile, false) //nodes := + pods, _ := e2e.ParsePods(kubeConfigFile, false) + + Eventually(func(g Gomega) { + count := e2e.CountOfStringInSlice("test-daemonset", pods) + fmt.Println("POD COUNT") + fmt.Println(count) + fmt.Println("NODE COUNT") + fmt.Println(len(nodes)) + g.Expect(len(nodes)).Should(Equal(count), "Daemonset pod count does not match node count") + }, "420s", "1s").Should(Succeed()) + }) + It("After upgrade verifies dns access", func() { + Eventually(func() (string, error) { + cmd := "kubectl --kubeconfig=" + kubeConfigFile + " exec -i -t dnsutils -- nslookup kubernetes.default" + return e2e.RunCommand(cmd) + }, "180s", "2s").Should((ContainSubstring("kubernetes.default.svc.cluster.local"))) + }) + + It("After upgrade verify Local Path Provisioner storage ", func() { + Eventually(func() (string, error) { + cmd := "kubectl exec volume-test cat /data/test --kubeconfig=" + kubeConfigFile + return e2e.RunCommand(cmd) + }, "180s", "2s").Should(ContainSubstring("local-path-test")) + }) + }) +}) + +var failed = false +var _ = AfterEach(func() { + failed = failed || CurrentGinkgoTestDescription().Failed +}) + +var _ = AfterSuite(func() { + if failed { + fmt.Println("FAILED!") + } else { + Expect(e2e.DestroyCluster()).To(Succeed()) + Expect(os.Remove(kubeConfigFile)).To(Succeed()) + } +}) diff --git a/tests/e2e/vagrantdefaults.rb b/tests/e2e/vagrantdefaults.rb new file mode 100644 index 0000000000..895fbd7852 --- /dev/null +++ b/tests/e2e/vagrantdefaults.rb @@ -0,0 +1,13 @@ +def defaultOSConfigure(vm) + if vm.box.include?("ubuntu2004") + vm.provision "shell", inline: "systemd-resolve --set-dns=8.8.8.8 --interface=eth0" + vm.provision "shell", inline: "apt install -y jq" + end + if vm.box.include?("Leap") + vm.provision "shell", inline: "zypper install -y jq" + end + if vm.box.include?("microos") + vm.provision "shell", inline: "transactional-update pkg install -y jq" + vm.provision 'reload', run: 'once' + end +end \ No newline at end of file diff --git a/tests/e2e/validatecluster/Vagrantfile b/tests/e2e/validatecluster/Vagrantfile index a8aae4b614..6ec1551441 100644 --- a/tests/e2e/validatecluster/Vagrantfile +++ b/tests/e2e/validatecluster/Vagrantfile @@ -17,7 +17,10 @@ def provision(vm, roles, role_num, node_num) # An expanded netmask is required to allow VM<-->VM communication, virtualbox defaults to /32 vm.network "private_network", ip: "#{NETWORK_PREFIX}.#{100+node_num}", netmask: "255.255.255.0" - osConfigure(vm) + vagrant_defaults = '../vagrantdefaults.rb' + load vagrant_defaults if File.exists?(vagrant_defaults) + + defaultOSConfigure(vm) if !RELEASE_VERSION.empty? install_type = "INSTALL_K3S_VERSION=#{RELEASE_VERSION}" @@ -30,14 +33,12 @@ def provision(vm, roles, role_num, node_num) if roles.include?("server") && role_num == 0 vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s| - k3s.installer_url = 'https://get.k3s.io' k3s.args = %W[server --cluster-init --node-external-ip=#{NETWORK_PREFIX}.100 --flannel-iface=eth1] k3s.env = %W[K3S_KUBECONFIG_MODE=0644 K3S_TOKEN=vagrant #{install_type}] k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 end elsif roles.include?("server") && role_num != 0 vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s| - k3s.installer_url = 'https://get.k3s.io' k3s.args = %W[server --server https://#{NETWORK_PREFIX}.100:6443 --flannel-iface=eth1] k3s.env = %W[K3S_KUBECONFIG_MODE=0644 K3S_TOKEN=vagrant #{install_type}] k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 @@ -45,7 +46,6 @@ def provision(vm, roles, role_num, node_num) end if roles.include?("agent") vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s| - k3s.installer_url = 'https://get.k3s.io' k3s.args = %W[agent --server https://#{NETWORK_PREFIX}.100:6443 --flannel-iface=eth1] k3s.env = %W[K3S_KUBECONFIG_MODE=0644 K3S_TOKEN=vagrant #{install_type}] k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 @@ -56,22 +56,6 @@ def provision(vm, roles, role_num, node_num) end end -def osConfigure(vm) - - if vm.box.include?("ubuntu2004") - vm.provision "shell", inline: "systemd-resolve --set-dns=8.8.8.8 --interface=eth0" - vm.provision "shell", inline: "apt install -y jq" - end - if vm.box.include?("Leap") - vm.provision "shell", inline: "zypper install -y jq" - end - if vm.box.include?("microos") - vm.provision "shell", inline: "transactional-update pkg install -y jq" - vm.provision 'reload', run: 'once' - end - -end - Vagrant.configure("2") do |config| config.vagrant.plugins = ["vagrant-k3s", "vagrant-reload"] # Default provider is libvirt, virtualbox is only provided as a backup diff --git a/tests/e2e/validatecluster/validatecluster_test.go b/tests/e2e/validatecluster/validatecluster_test.go index 2dcf78dc36..f241cf3a71 100644 --- a/tests/e2e/validatecluster/validatecluster_test.go +++ b/tests/e2e/validatecluster/validatecluster_test.go @@ -1,72 +1,271 @@ package validatecluster import ( + "flag" "fmt" "os" "strings" "testing" - . "github.com/onsi/ginkgo" - "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/rancher/k3s/tests/e2e" ) +// Valid nodeOS: generic/ubuntu2004, opensuse/Leap-15.3.x86_64, dweomer/microos.amd64 +var nodeOS = flag.String("nodeOS", "generic/ubuntu2004", "VM operating system") +var serverCount = flag.Int("serverCount", 3, "number of server nodes") +var agentCount = flag.Int("agentCount", 2, "number of agent nodes") + +//valid format: RELEASE_VERSION=v1.23.1+k3s2 or nil for latest commit from master +var installType = flag.String("installType", "", "version or nil to use latest commit") + func Test_E2EClusterValidation(t *testing.T) { RegisterFailHandler(Fail) - RunSpecsWithDefaultAndCustomReporters(t, "Create Cluster Test Suite", []Reporter{ - reporters.NewJUnitReporter("/tmp/results/junit-cc.xml"), - }) + flag.Parse() + RunSpecs(t, "Create Cluster Test Suite") } -const ( - // Valid nodeOS: generic/ubuntu2004, opensuse/Leap-15.3.x86_64, dweomer/microos.amd64 - nodeOs = "generic/ubuntu2004" - serverCount = 3 - agentCount = 2 -) - var ( kubeConfigFile string - serverNodeNames []string - agentNodeNames []string + serverNodenames []string + agentNodenames []string ) -var _ = Describe("Verify Cluster Creation", func() { - Context("Create the Cluster", func() { +var _ = Describe("Verify Create", func() { + Context("Cluster :", func() { It("Starts up with no issues", func() { var err error - serverNodeNames, agentNodeNames, err = e2e.CreateCluster(nodeOs, serverCount, agentCount) + serverNodenames, agentNodenames, err = e2e.CreateCluster(*nodeOS, *serverCount, *agentCount, *installType) Expect(err).NotTo(HaveOccurred()) fmt.Println("CLUSTER CONFIG") - fmt.Println("OS:", nodeOs) - fmt.Println("Server Nodes:", serverNodeNames) - fmt.Println("Agent Nodes:", agentNodeNames) - kubeConfigFile, err = e2e.GenKubeConfigFile(serverNodeNames[0]) + fmt.Println("OS:", *nodeOS) + fmt.Println("Server Nodes:", serverNodenames) + fmt.Println("Agent Nodes:", agentNodenames) + kubeConfigFile, err = e2e.GenKubeConfigFile(serverNodenames[0]) Expect(err).NotTo(HaveOccurred()) }) - It("Verify Node and Pod Status", func() { + + It("Checks Node and Pod Status", func() { fmt.Printf("\nFetching node status\n") Eventually(func(g Gomega) { - nodes, err := e2e.ParseNodes(kubeConfigFile, true) + nodes, err := e2e.ParseNodes(kubeConfigFile, false) g.Expect(err).NotTo(HaveOccurred()) for _, node := range nodes { - g.Expect(node.Status).Should(Equal("Ready"), func() string { return node.Name }) + g.Expect(node.Status).Should(Equal("Ready")) } }, "420s", "5s").Should(Succeed()) + _, _ = e2e.ParseNodes(kubeConfigFile, true) fmt.Printf("\nFetching Pods status\n") Eventually(func(g Gomega) { - pods, err := e2e.ParsePods(kubeConfigFile, true) + pods, err := e2e.ParsePods(kubeConfigFile, false) g.Expect(err).NotTo(HaveOccurred()) for _, pod := range pods { if strings.Contains(pod.Name, "helm-install") { - g.Expect(pod.Status).Should(Equal("Completed"), func() string { return pod.Name }) + g.Expect(pod.Status).Should(Equal("Completed"), pod.Name) } else { - g.Expect(pod.Status).Should(Equal("Running"), func() string { return pod.Name }) + g.Expect(pod.Status).Should(Equal("Running"), pod.Name) } } }, "420s", "5s").Should(Succeed()) + _, _ = e2e.ParsePods(kubeConfigFile, true) + }) + + It("Verifies ClusterIP Service", func() { + _, err := e2e.DeployWorkload("clusterip.yaml", kubeConfigFile, false) + Expect(err).NotTo(HaveOccurred(), "Cluster IP manifest not deployed") + + Eventually(func(g Gomega) { + cmd := "kubectl get pods -o=name -l k8s-app=nginx-app-clusterip --field-selector=status.phase=Running --kubeconfig=" + kubeConfigFile + res, err := e2e.RunCommand(cmd) + Expect(err).NotTo(HaveOccurred()) + g.Expect(res).Should((ContainSubstring("test-clusterip"))) + }, "240s", "5s").Should(Succeed()) + + clusterip, _ := e2e.FetchClusterIP(kubeConfigFile, "nginx-clusterip-svc") + cmd := "\"curl -L --insecure http://" + clusterip + "/name.html\"" + fmt.Println(cmd) + for _, nodeName := range serverNodenames { + Eventually(func(g Gomega) { + res, err := e2e.RunCmdOnNode(cmd, nodeName) + g.Expect(err).NotTo(HaveOccurred()) + fmt.Println(res) + Expect(res).Should(ContainSubstring("test-clusterip")) + }, "120s", "10s").Should(Succeed()) + } + }) + + It("Verifies NodePort Service", func() { + _, err := e2e.DeployWorkload("nodeport.yaml", kubeConfigFile, false) + Expect(err).NotTo(HaveOccurred(), "NodePort manifest not deployed") + + for _, nodeName := range serverNodenames { + node_external_ip, _ := e2e.FetchNodeExternalIP(nodeName) + cmd := "kubectl get service nginx-nodeport-svc --kubeconfig=" + kubeConfigFile + " --output jsonpath=\"{.spec.ports[0].nodePort}\"" + nodeport, err := e2e.RunCommand(cmd) + Expect(err).NotTo(HaveOccurred()) + + Eventually(func(g Gomega) { + cmd := "kubectl get pods -o=name -l k8s-app=nginx-app-nodeport --field-selector=status.phase=Running --kubeconfig=" + kubeConfigFile + res, err := e2e.RunCommand(cmd) + Expect(err).NotTo(HaveOccurred()) + g.Expect(res).Should(ContainSubstring("test-nodeport"), "nodeport pod was not created") + }, "240s", "5s").Should(Succeed()) + + cmd = "curl -L --insecure http://" + node_external_ip + ":" + nodeport + "/name.html" + fmt.Println(cmd) + Eventually(func(g Gomega) { + res, err := e2e.RunCommand(cmd) + Expect(err).NotTo(HaveOccurred()) + fmt.Println(res) + g.Expect(res).Should(ContainSubstring("test-nodeport")) + }, "240s", "5s").Should(Succeed()) + } + }) + + It("Verifies LoadBalancer Service", func() { + _, err := e2e.DeployWorkload("loadbalancer.yaml", kubeConfigFile, false) + Expect(err).NotTo(HaveOccurred(), "Loadbalancer manifest not deployed") + + for _, nodeName := range serverNodenames { + ip, _ := e2e.FetchNodeExternalIP(nodeName) + + cmd := "kubectl get service nginx-loadbalancer-svc --kubeconfig=" + kubeConfigFile + " --output jsonpath=\"{.spec.ports[0].port}\"" + port, err := e2e.RunCommand(cmd) + Expect(err).NotTo(HaveOccurred()) + + Eventually(func(g Gomega) { + cmd := "kubectl get pods -o=name -l k8s-app=nginx-app-loadbalancer --field-selector=status.phase=Running --kubeconfig=" + kubeConfigFile + res, err := e2e.RunCommand(cmd) + Expect(err).NotTo(HaveOccurred()) + g.Expect(res).Should(ContainSubstring("test-loadbalancer")) + }, "240s", "5s").Should(Succeed()) + + Eventually(func(g Gomega) { + cmd = "curl -L --insecure http://" + ip + ":" + port + "/name.html" + fmt.Println(cmd) + res, err := e2e.RunCommand(cmd) + Expect(err).NotTo(HaveOccurred()) + fmt.Println(res) + g.Expect(res).Should(ContainSubstring("test-loadbalancer")) + }, "240s", "5s").Should(Succeed()) + } + }) + + It("Verifies Ingress", func() { + _, err := e2e.DeployWorkload("ingress.yaml", kubeConfigFile, false) + Expect(err).NotTo(HaveOccurred(), "Ingress manifest not deployed") + + for _, nodeName := range serverNodenames { + ip, _ := e2e.FetchNodeExternalIP(nodeName) + cmd := "curl --header host:foo1.bar.com" + " http://" + ip + "/name.html" + fmt.Println(cmd) + + Eventually(func(g Gomega) { + res, err := e2e.RunCommand(cmd) + g.Expect(err).NotTo(HaveOccurred()) + fmt.Println(res) + g.Expect(res).Should(ContainSubstring("test-ingress")) + }, "240s", "5s").Should(Succeed()) + } + }) + + It("Verifies Daemonset", func() { + _, err := e2e.DeployWorkload("daemonset.yaml", kubeConfigFile, false) + Expect(err).NotTo(HaveOccurred(), "Daemonset manifest not deployed") + + nodes, _ := e2e.ParseNodes(kubeConfigFile, false) + pods, _ := e2e.ParsePods(kubeConfigFile, false) + + Eventually(func(g Gomega) { + count := e2e.CountOfStringInSlice("test-daemonset", pods) + fmt.Println("POD COUNT") + fmt.Println(count) + fmt.Println("NODE COUNT") + fmt.Println(len(nodes)) + g.Expect(len(nodes)).Should((Equal(count)), "Daemonset pod count does not match node count") + }, "420s", "10s").Should(Succeed()) + }) + + It("Verifies dns access", func() { + _, err := e2e.DeployWorkload("dnsutils.yaml", kubeConfigFile, false) + Expect(err).NotTo(HaveOccurred(), "dnsutils manifest not deployed") + + Eventually(func(g Gomega) { + cmd := "kubectl get pods dnsutils --kubeconfig=" + kubeConfigFile + res, _ := e2e.RunCommand(cmd) + fmt.Println(res) + g.Expect(res).Should(ContainSubstring("dnsutils")) + }, "420s", "2s").Should(Succeed()) + + Eventually(func(g Gomega) { + cmd := "kubectl --kubeconfig=" + kubeConfigFile + " exec -i -t dnsutils -- nslookup kubernetes.default" + fmt.Println(cmd) + res, _ := e2e.RunCommand(cmd) + fmt.Println(res) + g.Expect(res).Should(ContainSubstring("kubernetes.default.svc.cluster.local")) + }, "420s", "2s").Should(Succeed()) + }) + + It("Verifies Local Path Provisioner storage ", func() { + _, err := e2e.DeployWorkload("local-path-provisioner.yaml", kubeConfigFile, false) + Expect(err).NotTo(HaveOccurred(), "local-path-provisioner manifest not deployed") + + Eventually(func(g Gomega) { + cmd := "kubectl get pvc local-path-pvc --kubeconfig=" + kubeConfigFile + res, err := e2e.RunCommand(cmd) + g.Expect(err).NotTo(HaveOccurred()) + fmt.Println(res) + g.Expect(res).Should(ContainSubstring("local-path-pvc")) + g.Expect(res).Should(ContainSubstring("Bound")) + }, "420s", "2s").Should(Succeed()) + + Eventually(func(g Gomega) { + cmd := "kubectl get pod volume-test --kubeconfig=" + kubeConfigFile + res, err := e2e.RunCommand(cmd) + Expect(err).NotTo(HaveOccurred()) + fmt.Println(res) + g.Expect(res).Should(ContainSubstring("volume-test")) + g.Expect(res).Should(ContainSubstring("Running")) + }, "420s", "2s").Should(Succeed()) + + cmd := "kubectl --kubeconfig=" + kubeConfigFile + " exec volume-test -- sh -c 'echo local-path-test > /data/test'" + _, err = e2e.RunCommand(cmd) + Expect(err).NotTo(HaveOccurred()) + fmt.Println("Data stored in pvc: local-path-test") + + cmd = "kubectl delete pod volume-test --kubeconfig=" + kubeConfigFile + res, err := e2e.RunCommand(cmd) + Expect(err).NotTo(HaveOccurred()) + fmt.Println(res) + + _, err = e2e.DeployWorkload("local-path-provisioner.yaml", kubeConfigFile, false) + Expect(err).NotTo(HaveOccurred(), "local-path-provisioner manifest not deployed") + + Eventually(func(g Gomega) { + cmd := "kubectl get pods -o=name -l app=local-path-provisioner --field-selector=status.phase=Running -n kube-system --kubeconfig=" + kubeConfigFile + res, _ := e2e.RunCommand(cmd) + fmt.Println(res) + g.Expect(res).Should(ContainSubstring("local-path-provisioner")) + }, "420s", "2s").Should(Succeed()) + + Eventually(func(g Gomega) { + cmd := "kubectl get pod volume-test --kubeconfig=" + kubeConfigFile + res, err := e2e.RunCommand(cmd) + g.Expect(err).NotTo(HaveOccurred()) + fmt.Println(res) + g.Expect(res).Should(ContainSubstring("volume-test")) + g.Expect(res).Should(ContainSubstring("Running")) + }, "420s", "2s").Should(Succeed()) + + Eventually(func(g Gomega) { + cmd = "kubectl exec volume-test cat /data/test --kubeconfig=" + kubeConfigFile + res, err = e2e.RunCommand(cmd) + g.Expect(err).NotTo(HaveOccurred()) + fmt.Println("Data after re-creation", res) + g.Expect(res).Should(ContainSubstring("local-path-test")) + }, "180s", "2s").Should(Succeed()) }) }) }) diff --git a/tests/integration/custometcdargs/custometcdargs_int_test.go b/tests/integration/custometcdargs/custometcdargs_int_test.go index 9e978258f8..d686e9076f 100644 --- a/tests/integration/custometcdargs/custometcdargs_int_test.go +++ b/tests/integration/custometcdargs/custometcdargs_int_test.go @@ -5,8 +5,7 @@ import ( "strings" "testing" - . "github.com/onsi/ginkgo" - "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" testutil "github.com/rancher/k3s/tests/util" ) @@ -62,7 +61,5 @@ var _ = AfterSuite(func() { func Test_IntegrationCustomEtcdArgs(t *testing.T) { RegisterFailHandler(Fail) - RunSpecsWithDefaultAndCustomReporters(t, "Custom etcd Arguments", []Reporter{ - reporters.NewJUnitReporter("/tmp/results/junit-ls.xml"), - }) + RunSpecs(t, "Custom etcd Arguments") } diff --git a/tests/integration/dualstack/dualstack_int_test.go b/tests/integration/dualstack/dualstack_int_test.go index 94ed6dda61..4ec2fa91fa 100644 --- a/tests/integration/dualstack/dualstack_int_test.go +++ b/tests/integration/dualstack/dualstack_int_test.go @@ -5,8 +5,7 @@ import ( "strings" "testing" - . "github.com/onsi/ginkgo" - "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" testutil "github.com/rancher/k3s/tests/util" ) @@ -63,7 +62,5 @@ var _ = AfterSuite(func() { func Test_IntegrationDualStack(t *testing.T) { RegisterFailHandler(Fail) - RunSpecsWithDefaultAndCustomReporters(t, "Dual-Stack Suite", []Reporter{ - reporters.NewJUnitReporter("/tmp/results/junit-ls.xml"), - }) + RunSpecs(t, "Dual-Stack Suite") } diff --git a/tests/integration/etcdrestore/etcd_restore_int_test.go b/tests/integration/etcdrestore/etcd_restore_int_test.go index ff7a9fbbec..253f273929 100644 --- a/tests/integration/etcdrestore/etcd_restore_int_test.go +++ b/tests/integration/etcdrestore/etcd_restore_int_test.go @@ -4,8 +4,7 @@ import ( "strings" "testing" - . "github.com/onsi/ginkgo" - "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" testutil "github.com/rancher/k3s/tests/util" ) @@ -117,7 +116,5 @@ var _ = AfterSuite(func() { func Test_IntegrationEtcdRestoreSnapshot(t *testing.T) { RegisterFailHandler(Fail) - RunSpecsWithDefaultAndCustomReporters(t, "Etcd Suite", []Reporter{ - reporters.NewJUnitReporter("/tmp/results/junit-etcd-restore.xml"), - }) + RunSpecs(t, "Etcd Restore Suite") } diff --git a/tests/integration/etcdsnapshot/etcdsnapshot_int_test.go b/tests/integration/etcdsnapshot/etcdsnapshot_int_test.go index e73ffaaea3..1757b469f7 100644 --- a/tests/integration/etcdsnapshot/etcdsnapshot_int_test.go +++ b/tests/integration/etcdsnapshot/etcdsnapshot_int_test.go @@ -6,8 +6,7 @@ import ( "testing" "time" - . "github.com/onsi/ginkgo" - "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" testutil "github.com/rancher/k3s/tests/util" ) @@ -123,7 +122,5 @@ var _ = AfterSuite(func() { func Test_IntegrationEtcdSnapshot(t *testing.T) { RegisterFailHandler(Fail) - RunSpecsWithDefaultAndCustomReporters(t, "Etcd Suite", []Reporter{ - reporters.NewJUnitReporter("/tmp/results/junit-etcd.xml"), - }) + RunSpecs(t, "Etcd Snapshot Suite") } diff --git a/tests/integration/localstorage/localstorage_int_test.go b/tests/integration/localstorage/localstorage_int_test.go index b90e801a4b..6bdf515a2c 100644 --- a/tests/integration/localstorage/localstorage_int_test.go +++ b/tests/integration/localstorage/localstorage_int_test.go @@ -7,8 +7,7 @@ import ( "strings" "testing" - . "github.com/onsi/ginkgo" - "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" testutil "github.com/rancher/k3s/tests/util" ) @@ -92,7 +91,5 @@ var _ = AfterSuite(func() { func Test_IntegrationLocalStorage(t *testing.T) { RegisterFailHandler(Fail) - RunSpecsWithDefaultAndCustomReporters(t, "Local Storage Suite", []Reporter{ - reporters.NewJUnitReporter("/tmp/results/junit-ls.xml"), - }) + RunSpecs(t, "Local Storage Suite") } diff --git a/tests/integration/secretsencryption/secretsencryption_int_test.go b/tests/integration/secretsencryption/secretsencryption_int_test.go index 5b79fe09d0..abb333ef1f 100644 --- a/tests/integration/secretsencryption/secretsencryption_int_test.go +++ b/tests/integration/secretsencryption/secretsencryption_int_test.go @@ -6,8 +6,7 @@ import ( "testing" "time" - . "github.com/onsi/ginkgo" - "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" testutil "github.com/rancher/k3s/tests/util" ) @@ -150,7 +149,5 @@ var _ = AfterSuite(func() { func Test_IntegrationSecretsEncryption(t *testing.T) { RegisterFailHandler(Fail) - RunSpecsWithDefaultAndCustomReporters(t, "Secrets Encryption Suite", []Reporter{ - reporters.NewJUnitReporter("/tmp/results/junit-se.xml"), - }) + RunSpecs(t, "Secrets Encryption Suite") }