Integration tests utilities improvements (#4832)

* Remove sudo commands from integration tests

Signed-off-by: Derek Nola <derek.nola@suse.com>

* Added cleanup fucntion

Signed-off-by: Derek Nola <derek.nola@suse.com>

* Implement better int cleanup

Signed-off-by: Derek Nola <derek.nola@suse.com>

* Rename test utils

Signed-off-by: Derek Nola <derek.nola@suse.com>

* Enable K3sCmd to be a single string

Signed-off-by: Derek Nola <derek.nola@suse.com>

* Removed parsePod function

Signed-off-by: Derek Nola <derek.nola@suse.com>

* codespell

Signed-off-by: Derek Nola <derek.nola@suse.com>

* Revert startup timeout

Signed-off-by: Derek Nola <derek.nola@suse.com>

* Reorder sonobuoy tests, drop concurrent tests to 3

Signed-off-by: Derek Nola <derek.nola@suse.com>

* Disable etcd

Signed-off-by: Derek Nola <derek.nola@suse.com>

* Skip parallel testing for etcd

Signed-off-by: Derek Nola <derek.nola@suse.com>
pull/4866/head
Derek Nola 2022-01-06 08:05:56 -08:00 committed by GitHub
parent 644c30c8a6
commit 2ac8df3602
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 113 additions and 70 deletions

3
go.mod
View File

@ -89,7 +89,7 @@ require (
github.com/minio/minio-go/v7 v7.0.7
github.com/natefinch/lumberjack v2.0.0+incompatible
github.com/onsi/ginkgo v1.16.4
github.com/onsi/gomega v1.14.0
github.com/onsi/gomega v1.17.0
github.com/opencontainers/runc v1.0.3
github.com/opencontainers/selinux v1.8.3
github.com/otiai10/copy v1.6.0
@ -106,6 +106,7 @@ require (
github.com/stretchr/testify v1.7.0
github.com/tchap/go-patricia v2.3.0+incompatible // indirect
github.com/urfave/cli v1.22.4
github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852
go.etcd.io/etcd/api/v3 v3.5.0
go.etcd.io/etcd/client/v3 v3.5.0
go.etcd.io/etcd/etcdutl/v3 v3.5.0

3
go.sum
View File

@ -853,8 +853,9 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J
github.com/onsi/gomega v1.8.1/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA=
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc=
github.com/onsi/gomega v1.14.0 h1:ep6kpPVwmr/nTbklSx2nrLNSIO62DoYAhnPNIMhK8gI=
github.com/onsi/gomega v1.14.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0=
github.com/onsi/gomega v1.17.0 h1:9Luw4uT5HTjHTN8+aNcSThgH1vdXnmdJ8xIfZ4wyTRE=
github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=

View File

@ -31,7 +31,7 @@ var _ = Describe("etcd snapshots", func() {
When("a new etcd is created", func() {
It("starts up with no problems", func() {
Eventually(func() (string, error) {
return testutil.K3sCmd("kubectl", "get", "pods", "-A")
return testutil.K3sCmd("kubectl", "get pods -A")
}, "180s", "5s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
})
It("saves an etcd snapshot", func() {
@ -54,7 +54,7 @@ var _ = Describe("etcd snapshots", func() {
})
When("saving a custom name", func() {
It("saves an etcd snapshot with a custom name", func() {
Expect(testutil.K3sCmd("etcd-snapshot", "save", "--name", "ALIVEBEEF")).
Expect(testutil.K3sCmd("etcd-snapshot", "save --name ALIVEBEEF")).
To(ContainSubstring("Saving etcd snapshot to /var/lib/rancher/k3s/server/db/snapshots/ALIVEBEEF"))
})
It("deletes that snapshot", func() {
@ -69,13 +69,13 @@ var _ = Describe("etcd snapshots", func() {
})
When("using etcd snapshot prune", func() {
It("saves 3 different snapshots", func() {
Expect(testutil.K3sCmd("etcd-snapshot", "save", "-name", "PRUNE_TEST")).
Expect(testutil.K3sCmd("etcd-snapshot", "save -name PRUNE_TEST")).
To(ContainSubstring("saved"))
time.Sleep(1 * time.Second)
Expect(testutil.K3sCmd("etcd-snapshot", "save", "-name", "PRUNE_TEST")).
Expect(testutil.K3sCmd("etcd-snapshot", "save -name PRUNE_TEST")).
To(ContainSubstring("saved"))
time.Sleep(1 * time.Second)
Expect(testutil.K3sCmd("etcd-snapshot", "save", "-name", "PRUNE_TEST")).
Expect(testutil.K3sCmd("etcd-snapshot", "save -name PRUNE_TEST")).
To(ContainSubstring("saved"))
time.Sleep(1 * time.Second)
})
@ -88,7 +88,7 @@ var _ = Describe("etcd snapshots", func() {
Expect(sepLines).To(HaveLen(3))
})
It("prunes snapshots down to 2", func() {
Expect(testutil.K3sCmd("etcd-snapshot", "prune", "--snapshot-retention", "2", "--name", "PRUNE_TEST")).
Expect(testutil.K3sCmd("etcd-snapshot", "prune --snapshot-retention 2 --name PRUNE_TEST")).
To(ContainSubstring("Removing local snapshot"))
lsResult, err := testutil.K3sCmd("etcd-snapshot", "ls")
Expect(err).ToNot(HaveOccurred())
@ -112,7 +112,8 @@ var _ = Describe("etcd snapshots", func() {
var _ = AfterSuite(func() {
if !testutil.IsExistingServer() {
Expect(testutil.K3sKillServer(server)).To(Succeed())
Expect(testutil.K3sKillServer(server, false)).To(Succeed())
Expect(testutil.K3sCleanup(server, true)).To(Succeed())
}
})

View File

@ -38,8 +38,8 @@ echo "Did test-run-sonobuoy $?"
# ---
test-run-sonobuoy etcd
test-run-sonobuoy mysql
test-run-sonobuoy postgres
test-run-sonobuoy etcd skip-parallel
exit 0

View File

@ -554,7 +554,7 @@ run-test() {
local delay=15
(
set +x
while [ $(count-running-tests) -ge ${MAX_CONCURRENT_TESTS:-4} ]; do
while [ $(count-running-tests) -ge ${MAX_CONCURRENT_TESTS:-3} ]; do
sleep $delay
done
)
@ -619,21 +619,6 @@ e2e-test() {
# ---
run-e2e-tests() {
label=PARALLEL \
logName=e2e-STATUS-${ARCH}-parallel.log \
e2e-test ${sonobuoyParallelArgs[@]}
echo "Exit code $? for parallel start"
label=SERIAL \
logName=e2e-STATUS-${ARCH}-serial.log \
e2e-test ${sonobuoySerialArgs[@]}
echo "Exit code $? for serial start"
}
export -f run-e2e-tests
# ---
test-run-sonobuoy() {
local suffix
if [ "$1" ]; then
@ -643,7 +628,18 @@ test-run-sonobuoy() {
cleanup-test-env
. ./scripts/test-setup-sonobuoy$suffix
run-e2e-tests
if [ "$2" != "skip-parallel" ]; then
label=PARALLEL \
logName=e2e-STATUS-${ARCH}-parallel.log \
e2e-test ${sonobuoyParallelArgs[@]}
echo "Exit code $? for parallel start"
fi
label=SERIAL \
logName=e2e-STATUS-${ARCH}-serial.log \
e2e-test ${sonobuoySerialArgs[@]}
echo "Exit code $? for serial start"
}
export -f test-run-sonobuoy

View File

@ -2,8 +2,8 @@
. ./scripts/test-setup-sonobuoy
export NUM_SERVERS=2
export NUM_AGENTS=0
export NUM_SERVERS=1
export NUM_AGENTS=1
export SERVER_1_ARGS="--cluster-init"
server-post-hook() {

View File

@ -51,7 +51,7 @@ var _ = Describe("custom etcd args", func() {
var _ = AfterSuite(func() {
if !testutil.IsExistingServer() {
Expect(testutil.K3sKillServer(customEtcdArgsServer)).To(Succeed())
Expect(testutil.K3sKillServer(customEtcdArgsServer, true)).To(Succeed())
}
})

View File

@ -52,7 +52,8 @@ var _ = Describe("dual stack", func() {
var _ = AfterSuite(func() {
if !testutil.IsExistingServer() && os.Getenv("CI") != "true" {
Expect(testutil.K3sKillServer(dualStackServer)).To(Succeed())
Expect(testutil.K3sKillServer(dualStackServer, false)).To(Succeed())
Expect(testutil.K3sCleanup(dualStackServer, true)).To(Succeed())
}
})

View File

@ -32,27 +32,27 @@ var _ = Describe("local storage", func() {
When("a new local storage is created", func() {
It("starts up with no problems", func() {
Eventually(func() (string, error) {
return testutil.K3sCmd("kubectl", "get", "pods", "-A")
return testutil.K3sCmd("kubectl get pods -A")
}, "90s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
})
It("creates a new pvc", func() {
result, err := testutil.K3sCmd("kubectl", "create", "-f", "./testdata/localstorage_pvc.yaml")
result, err := testutil.K3sCmd("kubectl create -f ./testdata/localstorage_pvc.yaml")
Expect(result).To(ContainSubstring("persistentvolumeclaim/local-path-pvc created"))
Expect(err).NotTo(HaveOccurred())
})
It("creates a new pod", func() {
Expect(testutil.K3sCmd("kubectl", "create", "-f", "./testdata/localstorage_pod.yaml")).
Expect(testutil.K3sCmd("kubectl create -f ./testdata/localstorage_pod.yaml")).
To(ContainSubstring("pod/volume-test created"))
})
It("shows storage up in kubectl", func() {
Eventually(func() (string, error) {
return testutil.K3sCmd("kubectl", "get", "--namespace=default", "pvc")
return testutil.K3sCmd("kubectl get --namespace=default pvc")
}, "45s", "1s").Should(MatchRegexp(`local-path-pvc.+Bound`))
Eventually(func() (string, error) {
return testutil.K3sCmd("kubectl", "get", "--namespace=default", "pv")
return testutil.K3sCmd("kubectl get --namespace=default pv")
}, "10s", "1s").Should(MatchRegexp(`pvc.+1Gi.+Bound`))
Eventually(func() (string, error) {
return testutil.K3sCmd("kubectl", "get", "--namespace=default", "pod")
return testutil.K3sCmd("kubectl get --namespace=default pod")
}, "10s", "1s").Should(MatchRegexp(`volume-test.+Running`))
})
It("has proper folder permissions", func() {
@ -61,7 +61,7 @@ var _ = Describe("local storage", func() {
Expect(err).ToNot(HaveOccurred())
Expect(fmt.Sprintf("%04o", fileStat.Mode().Perm())).To(Equal("0701"))
pvResult, err := testutil.K3sCmd("kubectl", "get", "--namespace=default", "pv")
pvResult, err := testutil.K3sCmd("kubectl get --namespace=default pv")
Expect(err).ToNot(HaveOccurred())
reg, err := regexp.Compile(`pvc[^\s]+`)
Expect(err).ToNot(HaveOccurred())
@ -71,9 +71,9 @@ var _ = Describe("local storage", func() {
Expect(fmt.Sprintf("%04o", fileStat.Mode().Perm())).To(Equal("0777"))
})
It("deletes properly", func() {
Expect(testutil.K3sCmd("kubectl", "delete", "--namespace=default", "--force", "pod", "volume-test")).
Expect(testutil.K3sCmd("kubectl delete --namespace=default --force pod volume-test")).
To(ContainSubstring("pod \"volume-test\" force deleted"))
Expect(testutil.K3sCmd("kubectl", "delete", "--namespace=default", "pvc", "local-path-pvc")).
Expect(testutil.K3sCmd("kubectl delete --namespace=default pvc local-path-pvc")).
To(ContainSubstring("persistentvolumeclaim \"local-path-pvc\" deleted"))
})
})
@ -81,7 +81,8 @@ var _ = Describe("local storage", func() {
var _ = AfterSuite(func() {
if !testutil.IsExistingServer() {
Expect(testutil.K3sKillServer(localStorageServer)).To(Succeed())
Expect(testutil.K3sKillServer(localStorageServer, false)).To(Succeed())
Expect(testutil.K3sCleanup(localStorageServer, true)).To(Succeed())
}
})

View File

@ -35,11 +35,11 @@ var _ = Describe("secrets encryption rotation", func() {
When("A server starts with secrets encryption", func() {
It("starts up with no problems", func() {
Eventually(func() (string, error) {
return testutil.K3sCmd("kubectl", "get", "pods", "-A")
return testutil.K3sCmd("kubectl get pods -A")
}, "180s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
})
It("it creates a encryption key", func() {
result, err := testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir)
result, err := testutil.K3sCmd("secrets-encrypt status -d", secretsEncryptionDataDir)
Expect(err).NotTo(HaveOccurred())
Expect(result).To(ContainSubstring("Encryption Status: Enabled"))
Expect(result).To(ContainSubstring("Current Rotation Stage: start"))
@ -47,10 +47,10 @@ var _ = Describe("secrets encryption rotation", func() {
})
When("A server rotates encryption keys", func() {
It("it prepares to rotate", func() {
Expect(testutil.K3sCmd("secrets-encrypt", "prepare", "-d", secretsEncryptionDataDir)).
Expect(testutil.K3sCmd("secrets-encrypt prepare -d", secretsEncryptionDataDir)).
To(ContainSubstring("prepare completed successfully"))
result, err := testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir)
result, err := testutil.K3sCmd("secrets-encrypt status -d", secretsEncryptionDataDir)
Expect(err).NotTo(HaveOccurred())
Expect(result).To(ContainSubstring("Current Rotation Stage: prepare"))
reg, err := regexp.Compile(`AES-CBC.+aescbckey.*`)
@ -62,19 +62,19 @@ var _ = Describe("secrets encryption rotation", func() {
})
It("restarts the server", func() {
var err error
Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed())
Expect(testutil.K3sKillServer(secretsEncryptionServer, true)).To(Succeed())
secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...)
Expect(err).ToNot(HaveOccurred())
Eventually(func() (string, error) {
return testutil.K3sCmd("kubectl", "get", "pods", "-A")
return testutil.K3sCmd("kubectl get pods -A")
}, "180s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
})
It("rotates the keys", func() {
Eventually(func() (string, error) {
return testutil.K3sCmd("secrets-encrypt", "rotate", "-d", secretsEncryptionDataDir)
return testutil.K3sCmd("secrets-encrypt rotate -d", secretsEncryptionDataDir)
}, "10s", "2s").Should(ContainSubstring("rotate completed successfully"))
result, err := testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir)
result, err := testutil.K3sCmd("secrets-encrypt status -d", secretsEncryptionDataDir)
Expect(err).NotTo(HaveOccurred())
Expect(result).To(ContainSubstring("Current Rotation Stage: rotate"))
reg, err := regexp.Compile(`AES-CBC.+aescbckey.*`)
@ -86,21 +86,21 @@ var _ = Describe("secrets encryption rotation", func() {
})
It("restarts the server", func() {
var err error
Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed())
Expect(testutil.K3sKillServer(secretsEncryptionServer, true)).To(Succeed())
secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...)
Expect(err).ToNot(HaveOccurred())
Eventually(func() (string, error) {
return testutil.K3sCmd("kubectl", "get", "pods", "-A")
return testutil.K3sCmd("kubectl get pods -A")
}, "180s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
time.Sleep(10 * time.Second)
})
It("reencrypts the keys", func() {
Expect(testutil.K3sCmd("secrets-encrypt", "reencrypt", "-d", secretsEncryptionDataDir)).
Expect(testutil.K3sCmd("secrets-encrypt reencrypt -d", secretsEncryptionDataDir)).
To(ContainSubstring("reencryption started"))
Eventually(func() (string, error) {
return testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir)
return testutil.K3sCmd("secrets-encrypt status -d", secretsEncryptionDataDir)
}, "30s", "2s").Should(ContainSubstring("Current Rotation Stage: reencrypt_finished"))
result, err := testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir)
result, err := testutil.K3sCmd("secrets-encrypt status -d", secretsEncryptionDataDir)
Expect(err).NotTo(HaveOccurred())
reg, err := regexp.Compile(`AES-CBC.+aescbckey.*`)
Expect(err).ToNot(HaveOccurred())
@ -111,29 +111,29 @@ var _ = Describe("secrets encryption rotation", func() {
})
When("A server disables encryption", func() {
It("it triggers the disable", func() {
Expect(testutil.K3sCmd("secrets-encrypt", "disable", "-d", secretsEncryptionDataDir)).
Expect(testutil.K3sCmd("secrets-encrypt disable -d", secretsEncryptionDataDir)).
To(ContainSubstring("secrets-encryption disabled"))
result, err := testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir)
result, err := testutil.K3sCmd("secrets-encrypt status -d", secretsEncryptionDataDir)
Expect(err).NotTo(HaveOccurred())
Expect(result).To(ContainSubstring("Encryption Status: Disabled"))
})
It("restarts the server", func() {
var err error
Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed())
Expect(testutil.K3sKillServer(secretsEncryptionServer, true)).To(Succeed())
secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...)
Expect(err).ToNot(HaveOccurred())
Eventually(func() (string, error) {
return testutil.K3sCmd("kubectl", "get", "pods", "-A")
return testutil.K3sCmd("kubectl get pods -A")
}, "180s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
time.Sleep(10 * time.Second)
})
It("reencrypts the keys", func() {
result, err := testutil.K3sCmd("secrets-encrypt", "reencrypt", "-f", "--skip", "-d", secretsEncryptionDataDir)
result, err := testutil.K3sCmd("secrets-encrypt reencrypt -f --skip -d", secretsEncryptionDataDir)
Expect(err).NotTo(HaveOccurred())
Expect(result).To(ContainSubstring("reencryption started"))
result, err = testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir)
result, err = testutil.K3sCmd("secrets-encrypt status -d", secretsEncryptionDataDir)
Expect(err).NotTo(HaveOccurred())
Expect(result).To(ContainSubstring("Encryption Status: Disabled"))
})
@ -142,7 +142,7 @@ var _ = Describe("secrets encryption rotation", func() {
var _ = AfterSuite(func() {
if !testutil.IsExistingServer() {
Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed())
Expect(testutil.K3sKillServer(secretsEncryptionServer, true)).To(Succeed())
Expect(os.RemoveAll(secretsEncryptionDataDir)).To(Succeed())
}
})

View File

@ -8,9 +8,11 @@ import (
"os/exec"
"os/user"
"strings"
"syscall"
"github.com/rancher/k3s/pkg/flock"
"github.com/sirupsen/logrus"
"github.com/vishvananda/netlink"
)
// Compile-time variable
@ -63,16 +65,20 @@ func IsExistingServer() bool {
}
// K3sCmd launches the provided K3s command via exec. Command blocks until finished.
// Command output from both Stderr and Stdout is provided via string.
// Command output from both Stderr and Stdout is provided via string. Input can
// be a single string with space separated args, or multiple string args
// cmdEx1, err := K3sCmd("etcd-snapshot", "ls")
// cmdEx2, err := K3sCmd("kubectl get pods -A")
// cmdEx2, err := K3sCmd("kubectl", "get", "pods", "-A")
func K3sCmd(cmdName string, cmdArgs ...string) (string, error) {
func K3sCmd(inputArgs ...string) (string, error) {
if !IsRoot() {
return "", errors.New("integration tests must be run as sudo/root")
}
k3sBin := findK3sExecutable()
// Only run sudo if not root
k3sCmd := append([]string{cmdName}, cmdArgs...)
var k3sCmd []string
for _, arg := range inputArgs {
k3sCmd = append(k3sCmd, strings.Fields(arg)...)
}
cmd := exec.Command(k3sBin, k3sCmd...)
byteOut, err := cmd.CombinedOutput()
return string(byteOut), err
@ -145,17 +151,53 @@ func K3sStartServer(inputArgs ...string) (*K3sServer, error) {
k3sCmd := append([]string{"server"}, cmdArgs...)
cmd := exec.Command(k3sBin, k3sCmd...)
// Give the server a new group id so we can kill it and its children later
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
cmdOut, _ := cmd.StderrPipe()
cmd.Stderr = os.Stderr
err = cmd.Start()
return &K3sServer{cmd, bufio.NewScanner(cmdOut), k3sLock}, err
}
// K3sKillServer terminates the running K3s server and unlocks the file for
// other tests
func K3sKillServer(server *K3sServer) error {
if err := server.cmd.Process.Kill(); err != nil {
// K3sKillServer terminates the running K3s server and its children
// and unlocks the file for other tests
func K3sKillServer(server *K3sServer, releaseLock bool) error {
pgid, err := syscall.Getpgid(server.cmd.Process.Pid)
if err != nil {
return err
}
return flock.Release(server.lock)
if err := syscall.Kill(-pgid, syscall.SIGKILL); err != nil {
return err
}
if releaseLock {
return flock.Release(server.lock)
}
return nil
}
// K3sCleanup attempts to cleanup networking and files leftover from an integration test
func K3sCleanup(server *K3sServer, releaseLock bool) error {
if cni0Link, err := netlink.LinkByName("cni0"); err == nil {
links, _ := netlink.LinkList()
for _, link := range links {
if link.Attrs().MasterIndex == cni0Link.Attrs().Index {
netlink.LinkDel(link)
}
}
netlink.LinkDel(cni0Link)
}
if flannel1, err := netlink.LinkByName("flannel.1"); err == nil {
netlink.LinkDel(flannel1)
}
if flannelV6, err := netlink.LinkByName("flannel-v6.1"); err == nil {
netlink.LinkDel(flannelV6)
}
if err := os.RemoveAll("/var/lib/rancher/k3s"); err != nil {
return err
}
if releaseLock {
return flock.Release(server.lock)
}
return nil
}