From 299ca60009e160a6ab0b0db47becd990d68de5fe Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Fri, 25 Feb 2022 08:42:21 -0800 Subject: [PATCH] Move testing lock from server creation to test start (#5155) Signed-off-by: Derek Nola --- .../custometcdargs/custometcdargs_int_test.go | 8 +++-- .../dualstack/dualstack_int_test.go | 8 +++-- .../etcdrestore/etcd_restore_int_test.go | 14 ++++---- .../etcdsnapshot/etcdsnapshot_int_test.go | 8 +++-- .../localstorage/localstorage_int_test.go | 8 +++-- .../secretsencryption_int_test.go | 17 +++++----- tests/util/integration.go | 33 +++++++------------ 7 files changed, 52 insertions(+), 44 deletions(-) diff --git a/tests/integration/custometcdargs/custometcdargs_int_test.go b/tests/integration/custometcdargs/custometcdargs_int_test.go index f966959811..d686e9076f 100644 --- a/tests/integration/custometcdargs/custometcdargs_int_test.go +++ b/tests/integration/custometcdargs/custometcdargs_int_test.go @@ -15,9 +15,13 @@ var customEtcdArgsServerArgs = []string{ "--cluster-init", "--etcd-arg quota-backend-bytes=858993459", } +var testLock int + var _ = BeforeSuite(func() { if !testutil.IsExistingServer() { var err error + testLock, err = testutil.K3sTestLock() + Expect(err).ToNot(HaveOccurred()) customEtcdArgsServer, err = testutil.K3sStartServer(customEtcdArgsServerArgs...) Expect(err).ToNot(HaveOccurred()) } @@ -50,8 +54,8 @@ var _ = Describe("custom etcd args", func() { var _ = AfterSuite(func() { if !testutil.IsExistingServer() { - Expect(testutil.K3sKillServer(customEtcdArgsServer, false)).To(Succeed()) - Expect(testutil.K3sCleanup(customEtcdArgsServer, true, "")).To(Succeed()) + Expect(testutil.K3sKillServer(customEtcdArgsServer)).To(Succeed()) + Expect(testutil.K3sCleanup(testLock, "")).To(Succeed()) } }) diff --git a/tests/integration/dualstack/dualstack_int_test.go b/tests/integration/dualstack/dualstack_int_test.go index 4350f30afd..4ec2fa91fa 100644 --- a/tests/integration/dualstack/dualstack_int_test.go +++ b/tests/integration/dualstack/dualstack_int_test.go @@ -17,9 +17,13 @@ var dualStackServerArgs = []string{ "--service-cidr 10.43.0.0/16,2001:cafe:42:1::/112", "--disable-network-policy", } +var testLock int + var _ = BeforeSuite(func() { if !testutil.IsExistingServer() && os.Getenv("CI") != "true" { var err error + testLock, err = testutil.K3sTestLock() + Expect(err).ToNot(HaveOccurred()) dualStackServer, err = testutil.K3sStartServer(dualStackServerArgs...) Expect(err).ToNot(HaveOccurred()) } @@ -51,8 +55,8 @@ var _ = Describe("dual stack", func() { var _ = AfterSuite(func() { if !testutil.IsExistingServer() && os.Getenv("CI") != "true" { - Expect(testutil.K3sKillServer(dualStackServer, false)).To(Succeed()) - Expect(testutil.K3sCleanup(dualStackServer, true, "")).To(Succeed()) + Expect(testutil.K3sKillServer(dualStackServer)).To(Succeed()) + Expect(testutil.K3sCleanup(testLock, "")).To(Succeed()) } }) diff --git a/tests/integration/etcdrestore/etcd_restore_int_test.go b/tests/integration/etcdrestore/etcd_restore_int_test.go index 7795a627ee..253f273929 100644 --- a/tests/integration/etcdrestore/etcd_restore_int_test.go +++ b/tests/integration/etcdrestore/etcd_restore_int_test.go @@ -12,10 +12,13 @@ import ( var server1, server2 *testutil.K3sServer var tmpdDataDir = "/tmp/restoredatadir" var clientCACertHash string +var testLock int var restoreServerArgs = []string{"--cluster-init", "-t", "test", "-d", tmpdDataDir} var _ = BeforeSuite(func() { if !testutil.IsExistingServer() { var err error + testLock, err = testutil.K3sTestLock() + Expect(err).ToNot(HaveOccurred()) server1, err = testutil.K3sStartServer(restoreServerArgs...) Expect(err).ToNot(HaveOccurred()) } @@ -59,7 +62,7 @@ var _ = Describe("etcd snapshot restore", func() { Expect(err).ToNot(HaveOccurred()) }) It("stop k3s", func() { - Expect(testutil.K3sKillServer(server1, true)).To(Succeed()) + Expect(testutil.K3sKillServer(server1)).To(Succeed()) }) It("restore the snapshot", func() { // get snapshot file @@ -98,17 +101,16 @@ var _ = Describe("etcd snapshot restore", func() { Expect(clientCACertHash2).To(Equal(clientCACertHash)) }) It("stop k3s", func() { - Expect(testutil.K3sKillServer(server2, false)).To(Succeed()) + Expect(testutil.K3sKillServer(server2)).To(Succeed()) }) }) }) var _ = AfterSuite(func() { if !testutil.IsExistingServer() { - Expect(testutil.K3sKillServer(server1, false)).To(Succeed()) - Expect(testutil.K3sCleanup(server1, true, tmpdDataDir)).To(Succeed()) - Expect(testutil.K3sKillServer(server2, false)).To(Succeed()) - Expect(testutil.K3sCleanup(server2, true, tmpdDataDir)).To(Succeed()) + Expect(testutil.K3sKillServer(server1)).To(Succeed()) + Expect(testutil.K3sKillServer(server2)).To(Succeed()) + Expect(testutil.K3sCleanup(testLock, tmpdDataDir)).To(Succeed()) } }) diff --git a/tests/integration/etcdsnapshot/etcdsnapshot_int_test.go b/tests/integration/etcdsnapshot/etcdsnapshot_int_test.go index 45b4aa9740..a7a1a36c8b 100644 --- a/tests/integration/etcdsnapshot/etcdsnapshot_int_test.go +++ b/tests/integration/etcdsnapshot/etcdsnapshot_int_test.go @@ -13,9 +13,13 @@ import ( var server *testutil.K3sServer var serverArgs = []string{"--cluster-init"} +var testLock int + var _ = BeforeSuite(func() { if !testutil.IsExistingServer() { var err error + testLock, err = testutil.K3sTestLock() + Expect(err).ToNot(HaveOccurred()) server, err = testutil.K3sStartServer(serverArgs...) Expect(err).ToNot(HaveOccurred()) } @@ -111,8 +115,8 @@ var _ = Describe("etcd snapshots", func() { var _ = AfterSuite(func() { if !testutil.IsExistingServer() { - Expect(testutil.K3sKillServer(server, false)).To(Succeed()) - Expect(testutil.K3sCleanup(server, true, "")).To(Succeed()) + Expect(testutil.K3sKillServer(server)).To(Succeed()) + Expect(testutil.K3sCleanup(testLock, "")).To(Succeed()) } }) diff --git a/tests/integration/localstorage/localstorage_int_test.go b/tests/integration/localstorage/localstorage_int_test.go index 08c0b682e2..192f76005e 100644 --- a/tests/integration/localstorage/localstorage_int_test.go +++ b/tests/integration/localstorage/localstorage_int_test.go @@ -14,9 +14,13 @@ import ( var localStorageServer *testutil.K3sServer var localStorageServerArgs = []string{"--cluster-init"} +var testLock int + var _ = BeforeSuite(func() { if !testutil.IsExistingServer() { var err error + testLock, err = testutil.K3sTestLock() + Expect(err).ToNot(HaveOccurred()) localStorageServer, err = testutil.K3sStartServer(localStorageServerArgs...) Expect(err).ToNot(HaveOccurred()) } @@ -80,8 +84,8 @@ var _ = Describe("local storage", func() { var _ = AfterSuite(func() { if !testutil.IsExistingServer() { - Expect(testutil.K3sKillServer(localStorageServer, false)).To(Succeed()) - Expect(testutil.K3sCleanup(localStorageServer, true, "")).To(Succeed()) + Expect(testutil.K3sKillServer(localStorageServer)).To(Succeed()) + Expect(testutil.K3sCleanup(testLock, "")).To(Succeed()) } }) diff --git a/tests/integration/secretsencryption/secretsencryption_int_test.go b/tests/integration/secretsencryption/secretsencryption_int_test.go index aae8f7e9c9..f73fc92ff1 100644 --- a/tests/integration/secretsencryption/secretsencryption_int_test.go +++ b/tests/integration/secretsencryption/secretsencryption_int_test.go @@ -2,7 +2,6 @@ package integration import ( "fmt" - "os" "regexp" "testing" "time" @@ -14,12 +13,14 @@ import ( var secretsEncryptionServer *testutil.K3sServer var secretsEncryptionDataDir = "/tmp/k3sse" - var secretsEncryptionServerArgs = []string{"--secrets-encryption", "-d", secretsEncryptionDataDir} +var testLock int + var _ = BeforeSuite(func() { if !testutil.IsExistingServer() { var err error - Expect(os.MkdirAll(secretsEncryptionDataDir, 0777)).To(Succeed()) + testLock, err = testutil.K3sTestLock() + Expect(err).ToNot(HaveOccurred()) secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...) Expect(err).ToNot(HaveOccurred()) } @@ -61,7 +62,7 @@ var _ = Describe("secrets encryption rotation", func() { }) It("restarts the server", func() { var err error - Expect(testutil.K3sKillServer(secretsEncryptionServer, true)).To(Succeed()) + Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed()) secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...) Expect(err).ToNot(HaveOccurred()) Eventually(func() (string, error) { @@ -85,7 +86,7 @@ var _ = Describe("secrets encryption rotation", func() { }) It("restarts the server", func() { var err error - Expect(testutil.K3sKillServer(secretsEncryptionServer, true)).To(Succeed()) + Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed()) secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...) Expect(err).ToNot(HaveOccurred()) Eventually(func() (string, error) { @@ -119,7 +120,7 @@ var _ = Describe("secrets encryption rotation", func() { }) It("restarts the server", func() { var err error - Expect(testutil.K3sKillServer(secretsEncryptionServer, true)).To(Succeed()) + Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed()) secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...) Expect(err).ToNot(HaveOccurred()) Eventually(func() (string, error) { @@ -141,8 +142,8 @@ var _ = Describe("secrets encryption rotation", func() { var _ = AfterSuite(func() { if !testutil.IsExistingServer() { - Expect(testutil.K3sKillServer(secretsEncryptionServer, true)).To(Succeed()) - Expect(os.RemoveAll(secretsEncryptionDataDir)).To(Succeed()) + Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed()) + Expect(testutil.K3sCleanup(testLock, secretsEncryptionDataDir)).To(Succeed()) } }) diff --git a/tests/util/integration.go b/tests/util/integration.go index 791a8479fd..025d268eb7 100644 --- a/tests/util/integration.go +++ b/tests/util/integration.go @@ -25,7 +25,6 @@ const lockFile = "/tmp/k3s-test.lock" type K3sServer struct { cmd *exec.Cmd scanner *bufio.Scanner - lock int } func findK3sExecutable() string { @@ -130,6 +129,11 @@ func FindStringInCmdAsync(scanner *bufio.Scanner, target string) bool { return false } +func K3sTestLock() (int, error) { + logrus.Info("waiting to get test lock") + return flock.Acquire(lockFile) +} + // K3sStartServer acquires an exclusive lock on a temporary file, then launches a k3s cluster // with the provided arguments. Subsequent/parallel calls to this function will block until // the original lock is cleared using K3sKillServer @@ -138,12 +142,6 @@ func K3sStartServer(inputArgs ...string) (*K3sServer, error) { return nil, errors.New("integration tests must be run as sudo/root") } - logrus.Info("waiting to get server lock") - k3sLock, err := flock.Acquire(lockFile) - if err != nil { - return nil, err - } - var cmdArgs []string for _, arg := range inputArgs { cmdArgs = append(cmdArgs, strings.Fields(arg)...) @@ -155,13 +153,13 @@ func K3sStartServer(inputArgs ...string) (*K3sServer, error) { cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} cmdOut, _ := cmd.StderrPipe() cmd.Stderr = os.Stderr - err = cmd.Start() - return &K3sServer{cmd, bufio.NewScanner(cmdOut), k3sLock}, err + err := cmd.Start() + return &K3sServer{cmd, bufio.NewScanner(cmdOut)}, err } // K3sKillServer terminates the running K3s server and its children // and unlocks the file for other tests -func K3sKillServer(server *K3sServer, releaseLock bool) error { +func K3sKillServer(server *K3sServer) error { pgid, err := syscall.Getpgid(server.cmd.Process.Pid) if err != nil { return err @@ -169,17 +167,11 @@ func K3sKillServer(server *K3sServer, releaseLock bool) error { if err := syscall.Kill(-pgid, syscall.SIGKILL); err != nil { return err } - if err := server.cmd.Process.Kill(); err != nil { - return err - } - if releaseLock { - return flock.Release(server.lock) - } - return nil + return server.cmd.Process.Kill() } // K3sCleanup attempts to cleanup networking and files leftover from an integration test -func K3sCleanup(server *K3sServer, releaseLock bool, dataDir string) error { +func K3sCleanup(k3sTestLock int, dataDir string) error { if cni0Link, err := netlink.LinkByName("cni0"); err == nil { links, _ := netlink.LinkList() for _, link := range links { @@ -202,10 +194,7 @@ func K3sCleanup(server *K3sServer, releaseLock bool, dataDir string) error { if err := os.RemoveAll(dataDir); err != nil { return err } - if releaseLock { - return flock.Release(server.lock) - } - return nil + return flock.Release(k3sTestLock) } // RunCommand Runs command on the cluster accessing the cluster through kubeconfig file