From 114b30277f9e4e75c0f0e4b90b5d9ebcfac72d0e Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Thu, 26 Aug 2021 16:26:19 -0700 Subject: [PATCH] Redux: Enable K3s integration test to run on existing cluster (#3905) * Made it possible to run int tests on existing cluster Signed-off-by: dereknola --- pkg/etcd/etcd_int_test.go | 38 ++++++++-------- tests/TESTING.md | 6 +++ tests/integration/localstorage_int_test.go | 20 ++++++--- tests/util/cmd.go | 51 ++++++++++++++++++++++ 4 files changed, 92 insertions(+), 23 deletions(-) diff --git a/pkg/etcd/etcd_int_test.go b/pkg/etcd/etcd_int_test.go index 761a3a6935..dba0b83005 100644 --- a/pkg/etcd/etcd_int_test.go +++ b/pkg/etcd/etcd_int_test.go @@ -12,13 +12,21 @@ import ( ) var server *testutil.K3sServer +var serverArgs = []string{"--cluster-init"} var _ = BeforeSuite(func() { - var err error - server, err = testutil.K3sStartServer("--cluster-init") - Expect(err).ToNot(HaveOccurred()) + if !testutil.IsExistingServer() { + var err error + server, err = testutil.K3sStartServer(serverArgs...) + Expect(err).ToNot(HaveOccurred()) + } }) var _ = Describe("etcd snapshots", func() { + BeforeEach(func() { + if testutil.IsExistingServer() && !testutil.ServerArgsPresent(serverArgs) { + Skip("Test needs k3s server with: " + strings.Join(serverArgs, " ")) + } + }) When("a new etcd is created", func() { It("starts up with no problems", func() { Eventually(func() (string, error) { @@ -44,9 +52,6 @@ var _ = Describe("etcd snapshots", func() { }) }) When("saving a custom name", func() { - It("starts with no snapshots", func() { - Expect(testutil.K3sCmd("etcd-snapshot", "ls")).To(BeEmpty()) - }) It("saves an etcd snapshot with a custom name", func() { Expect(testutil.K3sCmd("etcd-snapshot", "save", "--name", "ALIVEBEEF")). To(ContainSubstring("Saving etcd snapshot to /var/lib/rancher/k3s/server/db/snapshots/ALIVEBEEF")) @@ -62,9 +67,6 @@ var _ = Describe("etcd snapshots", func() { }) }) When("using etcd snapshot prune", func() { - It("starts with no snapshots", func() { - Expect(testutil.K3sCmd("etcd-snapshot", "ls")).To(BeEmpty()) - }) It("saves 3 different snapshots", func() { Expect(testutil.K3sCmd("etcd-snapshot", "save", "-name", "PRUNE_TEST")). To(ContainSubstring("Saving current etcd snapshot set to k3s-etcd-snapshots")) @@ -79,10 +81,9 @@ var _ = Describe("etcd snapshots", func() { It("lists all 3 snapshots", func() { lsResult, err := testutil.K3sCmd("etcd-snapshot", "ls") Expect(err).ToNot(HaveOccurred()) - sepLines := strings.FieldsFunc(lsResult, func(c rune) bool { - return c == '\n' - }) - Expect(lsResult).To(MatchRegexp(`:///var/lib/rancher/k3s/server/db/snapshots/PRUNE_TEST`)) + reg, err := regexp.Compile(`:///var/lib/rancher/k3s/server/db/snapshots/PRUNE_TEST`) + Expect(err).ToNot(HaveOccurred()) + sepLines := reg.FindAllString(lsResult, -1) Expect(sepLines).To(HaveLen(3)) }) It("prunes snapshots down to 2", func() { @@ -90,10 +91,9 @@ var _ = Describe("etcd snapshots", func() { To(BeEmpty()) lsResult, err := testutil.K3sCmd("etcd-snapshot", "ls") Expect(err).ToNot(HaveOccurred()) - sepLines := strings.FieldsFunc(lsResult, func(c rune) bool { - return c == '\n' - }) - Expect(lsResult).To(MatchRegexp(`:///var/lib/rancher/k3s/server/db/snapshots/PRUNE_TEST`)) + reg, err := regexp.Compile(`:///var/lib/rancher/k3s/server/db/snapshots/PRUNE_TEST`) + Expect(err).ToNot(HaveOccurred()) + sepLines := reg.FindAllString(lsResult, -1) Expect(sepLines).To(HaveLen(2)) }) It("cleans up remaining snapshots", func() { @@ -110,7 +110,9 @@ var _ = Describe("etcd snapshots", func() { }) var _ = AfterSuite(func() { - Expect(testutil.K3sKillServer(server)).To(Succeed()) + if !testutil.IsExistingServer() { + Expect(testutil.K3sKillServer(server)).To(Succeed()) + } }) func Test_IntegrationEtcd(t *testing.T) { diff --git a/tests/TESTING.md b/tests/TESTING.md index 892eed036f..d5a17c3273 100644 --- a/tests/TESTING.md +++ b/tests/TESTING.md @@ -71,10 +71,16 @@ See the [local storage test](https://github.com/k3s-io/k3s/blob/master/tests/int ### Running +Integration tests can be run with no k3s cluster present, each test will spin up and kill the appropriate k3s server it needs. ```bash go test ./pkg/... ./tests/... -run Integration ``` +Integration tests can also be run on an existing single-node cluster via compile time flag, tests will skip if the server is not configured correctly. +``` +go test -ldflags "-X 'github.com/rancher/k3s/tests/util.existingServer=True'" ./pkg/... ./tests/... -run Integration +``` + ___ ## End-to-End (E2E) Tests diff --git a/tests/integration/localstorage_int_test.go b/tests/integration/localstorage_int_test.go index a7127d59f0..1b664974b0 100644 --- a/tests/integration/localstorage_int_test.go +++ b/tests/integration/localstorage_int_test.go @@ -4,6 +4,7 @@ import ( "fmt" "os" "regexp" + "strings" "testing" . "github.com/onsi/ginkgo" @@ -12,13 +13,21 @@ import ( ) var server *testutil.K3sServer +var serverArgs = []string{"--cluster-init"} var _ = BeforeSuite(func() { - var err error - server, err = testutil.K3sStartServer("--cluster-init") - Expect(err).ToNot(HaveOccurred()) + if !testutil.IsExistingServer() { + var err error + server, err = testutil.K3sStartServer(serverArgs...) + Expect(err).ToNot(HaveOccurred()) + } }) var _ = Describe("local storage", func() { + BeforeEach(func() { + if testutil.IsExistingServer() && !testutil.ServerArgsPresent(serverArgs) { + Skip("Test needs k3s server with: " + strings.Join(serverArgs, " ")) + } + }) When("a new local storage is created", func() { It("starts up with no problems", func() { Eventually(func() (string, error) { @@ -66,11 +75,12 @@ var _ = Describe("local storage", func() { }) var _ = AfterSuite(func() { - Expect(testutil.K3sKillServer(server)).To(Succeed()) + if !testutil.IsExistingServer() { + Expect(testutil.K3sKillServer(server)).To(Succeed()) + } }) func Test_IntegrationLocalStorage(t *testing.T) { RegisterFailHandler(Fail) RunSpecs(t, "Local Storage Suite") - } diff --git a/tests/util/cmd.go b/tests/util/cmd.go index 1294b4ee62..49172e8700 100644 --- a/tests/util/cmd.go +++ b/tests/util/cmd.go @@ -2,6 +2,7 @@ package util import ( "bufio" + "encoding/json" "os" "os/exec" "os/user" @@ -11,7 +12,18 @@ import ( "github.com/sirupsen/logrus" ) +// Compile-time variable +var existingServer = "False" + func findK3sExecutable() string { + // if running on an existing cluster, it maybe installed via k3s.service + // or run manually from dist/artifacts/k3s + if IsExistingServer() { + k3sBin, err := exec.LookPath("k3s") + if err == nil { + return k3sBin + } + } k3sBin := "dist/artifacts/k3s" for { _, err := os.Stat(k3sBin) @@ -33,6 +45,10 @@ func IsRoot() bool { return currentUser.Uid == "0" } +func IsExistingServer() bool { + return existingServer == "True" +} + // K3sCmd launches the provided K3s command via exec. Command blocks until finished. // Command output from both Stderr and Stdout is provided via string. // cmdEx1, err := K3sCmd("etcd-snapshot", "ls") @@ -52,6 +68,41 @@ func K3sCmd(cmdName string, cmdArgs ...string) (string, error) { return string(byteOut), err } +func contains(source []string, target string) bool { + for _, s := range source { + if s == target { + return true + } + } + return false +} + +// ServerArgsPresent checks if the given arguments are found in the running k3s server +func ServerArgsPresent(neededArgs []string) bool { + currentArgs := K3sServerArgs() + for _, arg := range neededArgs { + if !contains(currentArgs, arg) { + return false + } + } + return true +} + +// K3sServerArgs returns the list of arguments that the k3s server launched with +func K3sServerArgs() []string { + results, err := K3sCmd("kubectl", "get", "nodes", "-o", `jsonpath='{.items[0].metadata.annotations.k3s\.io/node-args}'`) + if err != nil { + return nil + } + res := strings.ReplaceAll(results, "'", "") + var args []string + if err := json.Unmarshal([]byte(res), &args); err != nil { + logrus.Error(err) + return nil + } + return args +} + func FindStringInCmdAsync(scanner *bufio.Scanner, target string) bool { for scanner.Scan() { if strings.Contains(scanner.Text(), target) {