mirror of https://github.com/k3s-io/k3s
Integration tests utilities improvements (#4832)
* Remove sudo commands from integration tests Signed-off-by: Derek Nola <derek.nola@suse.com> * Added cleanup fucntion Signed-off-by: Derek Nola <derek.nola@suse.com> * Implement better int cleanup Signed-off-by: Derek Nola <derek.nola@suse.com> * Rename test utils Signed-off-by: Derek Nola <derek.nola@suse.com> * Enable K3sCmd to be a single string Signed-off-by: Derek Nola <derek.nola@suse.com> * Removed parsePod function Signed-off-by: Derek Nola <derek.nola@suse.com> * codespell Signed-off-by: Derek Nola <derek.nola@suse.com> * Revert startup timeout Signed-off-by: Derek Nola <derek.nola@suse.com> * Reorder sonobuoy tests, drop concurrent tests to 3 Signed-off-by: Derek Nola <derek.nola@suse.com> * Disable etcd Signed-off-by: Derek Nola <derek.nola@suse.com> * Skip parallel testing for etcd Signed-off-by: Derek Nola <derek.nola@suse.com>pull/4866/head
parent
644c30c8a6
commit
2ac8df3602
3
go.mod
3
go.mod
|
@ -89,7 +89,7 @@ require (
|
|||
github.com/minio/minio-go/v7 v7.0.7
|
||||
github.com/natefinch/lumberjack v2.0.0+incompatible
|
||||
github.com/onsi/ginkgo v1.16.4
|
||||
github.com/onsi/gomega v1.14.0
|
||||
github.com/onsi/gomega v1.17.0
|
||||
github.com/opencontainers/runc v1.0.3
|
||||
github.com/opencontainers/selinux v1.8.3
|
||||
github.com/otiai10/copy v1.6.0
|
||||
|
@ -106,6 +106,7 @@ require (
|
|||
github.com/stretchr/testify v1.7.0
|
||||
github.com/tchap/go-patricia v2.3.0+incompatible // indirect
|
||||
github.com/urfave/cli v1.22.4
|
||||
github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852
|
||||
go.etcd.io/etcd/api/v3 v3.5.0
|
||||
go.etcd.io/etcd/client/v3 v3.5.0
|
||||
go.etcd.io/etcd/etcdutl/v3 v3.5.0
|
||||
|
|
3
go.sum
3
go.sum
|
@ -853,8 +853,9 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J
|
|||
github.com/onsi/gomega v1.8.1/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA=
|
||||
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
|
||||
github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc=
|
||||
github.com/onsi/gomega v1.14.0 h1:ep6kpPVwmr/nTbklSx2nrLNSIO62DoYAhnPNIMhK8gI=
|
||||
github.com/onsi/gomega v1.14.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0=
|
||||
github.com/onsi/gomega v1.17.0 h1:9Luw4uT5HTjHTN8+aNcSThgH1vdXnmdJ8xIfZ4wyTRE=
|
||||
github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
|
||||
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
|
||||
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
|
||||
github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
|
||||
|
|
|
@ -31,7 +31,7 @@ var _ = Describe("etcd snapshots", func() {
|
|||
When("a new etcd is created", func() {
|
||||
It("starts up with no problems", func() {
|
||||
Eventually(func() (string, error) {
|
||||
return testutil.K3sCmd("kubectl", "get", "pods", "-A")
|
||||
return testutil.K3sCmd("kubectl", "get pods -A")
|
||||
}, "180s", "5s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
|
||||
})
|
||||
It("saves an etcd snapshot", func() {
|
||||
|
@ -54,7 +54,7 @@ var _ = Describe("etcd snapshots", func() {
|
|||
})
|
||||
When("saving a custom name", func() {
|
||||
It("saves an etcd snapshot with a custom name", func() {
|
||||
Expect(testutil.K3sCmd("etcd-snapshot", "save", "--name", "ALIVEBEEF")).
|
||||
Expect(testutil.K3sCmd("etcd-snapshot", "save --name ALIVEBEEF")).
|
||||
To(ContainSubstring("Saving etcd snapshot to /var/lib/rancher/k3s/server/db/snapshots/ALIVEBEEF"))
|
||||
})
|
||||
It("deletes that snapshot", func() {
|
||||
|
@ -69,13 +69,13 @@ var _ = Describe("etcd snapshots", func() {
|
|||
})
|
||||
When("using etcd snapshot prune", func() {
|
||||
It("saves 3 different snapshots", func() {
|
||||
Expect(testutil.K3sCmd("etcd-snapshot", "save", "-name", "PRUNE_TEST")).
|
||||
Expect(testutil.K3sCmd("etcd-snapshot", "save -name PRUNE_TEST")).
|
||||
To(ContainSubstring("saved"))
|
||||
time.Sleep(1 * time.Second)
|
||||
Expect(testutil.K3sCmd("etcd-snapshot", "save", "-name", "PRUNE_TEST")).
|
||||
Expect(testutil.K3sCmd("etcd-snapshot", "save -name PRUNE_TEST")).
|
||||
To(ContainSubstring("saved"))
|
||||
time.Sleep(1 * time.Second)
|
||||
Expect(testutil.K3sCmd("etcd-snapshot", "save", "-name", "PRUNE_TEST")).
|
||||
Expect(testutil.K3sCmd("etcd-snapshot", "save -name PRUNE_TEST")).
|
||||
To(ContainSubstring("saved"))
|
||||
time.Sleep(1 * time.Second)
|
||||
})
|
||||
|
@ -88,7 +88,7 @@ var _ = Describe("etcd snapshots", func() {
|
|||
Expect(sepLines).To(HaveLen(3))
|
||||
})
|
||||
It("prunes snapshots down to 2", func() {
|
||||
Expect(testutil.K3sCmd("etcd-snapshot", "prune", "--snapshot-retention", "2", "--name", "PRUNE_TEST")).
|
||||
Expect(testutil.K3sCmd("etcd-snapshot", "prune --snapshot-retention 2 --name PRUNE_TEST")).
|
||||
To(ContainSubstring("Removing local snapshot"))
|
||||
lsResult, err := testutil.K3sCmd("etcd-snapshot", "ls")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
@ -112,7 +112,8 @@ var _ = Describe("etcd snapshots", func() {
|
|||
|
||||
var _ = AfterSuite(func() {
|
||||
if !testutil.IsExistingServer() {
|
||||
Expect(testutil.K3sKillServer(server)).To(Succeed())
|
||||
Expect(testutil.K3sKillServer(server, false)).To(Succeed())
|
||||
Expect(testutil.K3sCleanup(server, true)).To(Succeed())
|
||||
}
|
||||
})
|
||||
|
||||
|
|
|
@ -38,8 +38,8 @@ echo "Did test-run-sonobuoy $?"
|
|||
|
||||
# ---
|
||||
|
||||
test-run-sonobuoy etcd
|
||||
test-run-sonobuoy mysql
|
||||
test-run-sonobuoy postgres
|
||||
test-run-sonobuoy etcd skip-parallel
|
||||
|
||||
exit 0
|
||||
|
|
|
@ -554,7 +554,7 @@ run-test() {
|
|||
local delay=15
|
||||
(
|
||||
set +x
|
||||
while [ $(count-running-tests) -ge ${MAX_CONCURRENT_TESTS:-4} ]; do
|
||||
while [ $(count-running-tests) -ge ${MAX_CONCURRENT_TESTS:-3} ]; do
|
||||
sleep $delay
|
||||
done
|
||||
)
|
||||
|
@ -619,21 +619,6 @@ e2e-test() {
|
|||
|
||||
# ---
|
||||
|
||||
run-e2e-tests() {
|
||||
label=PARALLEL \
|
||||
logName=e2e-STATUS-${ARCH}-parallel.log \
|
||||
e2e-test ${sonobuoyParallelArgs[@]}
|
||||
echo "Exit code $? for parallel start"
|
||||
|
||||
label=SERIAL \
|
||||
logName=e2e-STATUS-${ARCH}-serial.log \
|
||||
e2e-test ${sonobuoySerialArgs[@]}
|
||||
echo "Exit code $? for serial start"
|
||||
}
|
||||
export -f run-e2e-tests
|
||||
|
||||
# ---
|
||||
|
||||
test-run-sonobuoy() {
|
||||
local suffix
|
||||
if [ "$1" ]; then
|
||||
|
@ -643,7 +628,18 @@ test-run-sonobuoy() {
|
|||
|
||||
cleanup-test-env
|
||||
. ./scripts/test-setup-sonobuoy$suffix
|
||||
run-e2e-tests
|
||||
|
||||
if [ "$2" != "skip-parallel" ]; then
|
||||
label=PARALLEL \
|
||||
logName=e2e-STATUS-${ARCH}-parallel.log \
|
||||
e2e-test ${sonobuoyParallelArgs[@]}
|
||||
echo "Exit code $? for parallel start"
|
||||
fi
|
||||
|
||||
label=SERIAL \
|
||||
logName=e2e-STATUS-${ARCH}-serial.log \
|
||||
e2e-test ${sonobuoySerialArgs[@]}
|
||||
echo "Exit code $? for serial start"
|
||||
}
|
||||
export -f test-run-sonobuoy
|
||||
|
||||
|
|
|
@ -2,8 +2,8 @@
|
|||
|
||||
. ./scripts/test-setup-sonobuoy
|
||||
|
||||
export NUM_SERVERS=2
|
||||
export NUM_AGENTS=0
|
||||
export NUM_SERVERS=1
|
||||
export NUM_AGENTS=1
|
||||
export SERVER_1_ARGS="--cluster-init"
|
||||
|
||||
server-post-hook() {
|
||||
|
|
|
@ -51,7 +51,7 @@ var _ = Describe("custom etcd args", func() {
|
|||
|
||||
var _ = AfterSuite(func() {
|
||||
if !testutil.IsExistingServer() {
|
||||
Expect(testutil.K3sKillServer(customEtcdArgsServer)).To(Succeed())
|
||||
Expect(testutil.K3sKillServer(customEtcdArgsServer, true)).To(Succeed())
|
||||
}
|
||||
})
|
||||
|
||||
|
|
|
@ -52,7 +52,8 @@ var _ = Describe("dual stack", func() {
|
|||
|
||||
var _ = AfterSuite(func() {
|
||||
if !testutil.IsExistingServer() && os.Getenv("CI") != "true" {
|
||||
Expect(testutil.K3sKillServer(dualStackServer)).To(Succeed())
|
||||
Expect(testutil.K3sKillServer(dualStackServer, false)).To(Succeed())
|
||||
Expect(testutil.K3sCleanup(dualStackServer, true)).To(Succeed())
|
||||
}
|
||||
})
|
||||
|
||||
|
|
|
@ -32,27 +32,27 @@ var _ = Describe("local storage", func() {
|
|||
When("a new local storage is created", func() {
|
||||
It("starts up with no problems", func() {
|
||||
Eventually(func() (string, error) {
|
||||
return testutil.K3sCmd("kubectl", "get", "pods", "-A")
|
||||
return testutil.K3sCmd("kubectl get pods -A")
|
||||
}, "90s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
|
||||
})
|
||||
It("creates a new pvc", func() {
|
||||
result, err := testutil.K3sCmd("kubectl", "create", "-f", "./testdata/localstorage_pvc.yaml")
|
||||
result, err := testutil.K3sCmd("kubectl create -f ./testdata/localstorage_pvc.yaml")
|
||||
Expect(result).To(ContainSubstring("persistentvolumeclaim/local-path-pvc created"))
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
})
|
||||
It("creates a new pod", func() {
|
||||
Expect(testutil.K3sCmd("kubectl", "create", "-f", "./testdata/localstorage_pod.yaml")).
|
||||
Expect(testutil.K3sCmd("kubectl create -f ./testdata/localstorage_pod.yaml")).
|
||||
To(ContainSubstring("pod/volume-test created"))
|
||||
})
|
||||
It("shows storage up in kubectl", func() {
|
||||
Eventually(func() (string, error) {
|
||||
return testutil.K3sCmd("kubectl", "get", "--namespace=default", "pvc")
|
||||
return testutil.K3sCmd("kubectl get --namespace=default pvc")
|
||||
}, "45s", "1s").Should(MatchRegexp(`local-path-pvc.+Bound`))
|
||||
Eventually(func() (string, error) {
|
||||
return testutil.K3sCmd("kubectl", "get", "--namespace=default", "pv")
|
||||
return testutil.K3sCmd("kubectl get --namespace=default pv")
|
||||
}, "10s", "1s").Should(MatchRegexp(`pvc.+1Gi.+Bound`))
|
||||
Eventually(func() (string, error) {
|
||||
return testutil.K3sCmd("kubectl", "get", "--namespace=default", "pod")
|
||||
return testutil.K3sCmd("kubectl get --namespace=default pod")
|
||||
}, "10s", "1s").Should(MatchRegexp(`volume-test.+Running`))
|
||||
})
|
||||
It("has proper folder permissions", func() {
|
||||
|
@ -61,7 +61,7 @@ var _ = Describe("local storage", func() {
|
|||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(fmt.Sprintf("%04o", fileStat.Mode().Perm())).To(Equal("0701"))
|
||||
|
||||
pvResult, err := testutil.K3sCmd("kubectl", "get", "--namespace=default", "pv")
|
||||
pvResult, err := testutil.K3sCmd("kubectl get --namespace=default pv")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
reg, err := regexp.Compile(`pvc[^\s]+`)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
@ -71,9 +71,9 @@ var _ = Describe("local storage", func() {
|
|||
Expect(fmt.Sprintf("%04o", fileStat.Mode().Perm())).To(Equal("0777"))
|
||||
})
|
||||
It("deletes properly", func() {
|
||||
Expect(testutil.K3sCmd("kubectl", "delete", "--namespace=default", "--force", "pod", "volume-test")).
|
||||
Expect(testutil.K3sCmd("kubectl delete --namespace=default --force pod volume-test")).
|
||||
To(ContainSubstring("pod \"volume-test\" force deleted"))
|
||||
Expect(testutil.K3sCmd("kubectl", "delete", "--namespace=default", "pvc", "local-path-pvc")).
|
||||
Expect(testutil.K3sCmd("kubectl delete --namespace=default pvc local-path-pvc")).
|
||||
To(ContainSubstring("persistentvolumeclaim \"local-path-pvc\" deleted"))
|
||||
})
|
||||
})
|
||||
|
@ -81,7 +81,8 @@ var _ = Describe("local storage", func() {
|
|||
|
||||
var _ = AfterSuite(func() {
|
||||
if !testutil.IsExistingServer() {
|
||||
Expect(testutil.K3sKillServer(localStorageServer)).To(Succeed())
|
||||
Expect(testutil.K3sKillServer(localStorageServer, false)).To(Succeed())
|
||||
Expect(testutil.K3sCleanup(localStorageServer, true)).To(Succeed())
|
||||
}
|
||||
})
|
||||
|
||||
|
|
|
@ -35,11 +35,11 @@ var _ = Describe("secrets encryption rotation", func() {
|
|||
When("A server starts with secrets encryption", func() {
|
||||
It("starts up with no problems", func() {
|
||||
Eventually(func() (string, error) {
|
||||
return testutil.K3sCmd("kubectl", "get", "pods", "-A")
|
||||
return testutil.K3sCmd("kubectl get pods -A")
|
||||
}, "180s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
|
||||
})
|
||||
It("it creates a encryption key", func() {
|
||||
result, err := testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir)
|
||||
result, err := testutil.K3sCmd("secrets-encrypt status -d", secretsEncryptionDataDir)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(result).To(ContainSubstring("Encryption Status: Enabled"))
|
||||
Expect(result).To(ContainSubstring("Current Rotation Stage: start"))
|
||||
|
@ -47,10 +47,10 @@ var _ = Describe("secrets encryption rotation", func() {
|
|||
})
|
||||
When("A server rotates encryption keys", func() {
|
||||
It("it prepares to rotate", func() {
|
||||
Expect(testutil.K3sCmd("secrets-encrypt", "prepare", "-d", secretsEncryptionDataDir)).
|
||||
Expect(testutil.K3sCmd("secrets-encrypt prepare -d", secretsEncryptionDataDir)).
|
||||
To(ContainSubstring("prepare completed successfully"))
|
||||
|
||||
result, err := testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir)
|
||||
result, err := testutil.K3sCmd("secrets-encrypt status -d", secretsEncryptionDataDir)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(result).To(ContainSubstring("Current Rotation Stage: prepare"))
|
||||
reg, err := regexp.Compile(`AES-CBC.+aescbckey.*`)
|
||||
|
@ -62,19 +62,19 @@ var _ = Describe("secrets encryption rotation", func() {
|
|||
})
|
||||
It("restarts the server", func() {
|
||||
var err error
|
||||
Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed())
|
||||
Expect(testutil.K3sKillServer(secretsEncryptionServer, true)).To(Succeed())
|
||||
secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Eventually(func() (string, error) {
|
||||
return testutil.K3sCmd("kubectl", "get", "pods", "-A")
|
||||
return testutil.K3sCmd("kubectl get pods -A")
|
||||
}, "180s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
|
||||
})
|
||||
It("rotates the keys", func() {
|
||||
Eventually(func() (string, error) {
|
||||
return testutil.K3sCmd("secrets-encrypt", "rotate", "-d", secretsEncryptionDataDir)
|
||||
return testutil.K3sCmd("secrets-encrypt rotate -d", secretsEncryptionDataDir)
|
||||
}, "10s", "2s").Should(ContainSubstring("rotate completed successfully"))
|
||||
|
||||
result, err := testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir)
|
||||
result, err := testutil.K3sCmd("secrets-encrypt status -d", secretsEncryptionDataDir)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(result).To(ContainSubstring("Current Rotation Stage: rotate"))
|
||||
reg, err := regexp.Compile(`AES-CBC.+aescbckey.*`)
|
||||
|
@ -86,21 +86,21 @@ var _ = Describe("secrets encryption rotation", func() {
|
|||
})
|
||||
It("restarts the server", func() {
|
||||
var err error
|
||||
Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed())
|
||||
Expect(testutil.K3sKillServer(secretsEncryptionServer, true)).To(Succeed())
|
||||
secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Eventually(func() (string, error) {
|
||||
return testutil.K3sCmd("kubectl", "get", "pods", "-A")
|
||||
return testutil.K3sCmd("kubectl get pods -A")
|
||||
}, "180s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
|
||||
time.Sleep(10 * time.Second)
|
||||
})
|
||||
It("reencrypts the keys", func() {
|
||||
Expect(testutil.K3sCmd("secrets-encrypt", "reencrypt", "-d", secretsEncryptionDataDir)).
|
||||
Expect(testutil.K3sCmd("secrets-encrypt reencrypt -d", secretsEncryptionDataDir)).
|
||||
To(ContainSubstring("reencryption started"))
|
||||
Eventually(func() (string, error) {
|
||||
return testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir)
|
||||
return testutil.K3sCmd("secrets-encrypt status -d", secretsEncryptionDataDir)
|
||||
}, "30s", "2s").Should(ContainSubstring("Current Rotation Stage: reencrypt_finished"))
|
||||
result, err := testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir)
|
||||
result, err := testutil.K3sCmd("secrets-encrypt status -d", secretsEncryptionDataDir)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
reg, err := regexp.Compile(`AES-CBC.+aescbckey.*`)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
@ -111,29 +111,29 @@ var _ = Describe("secrets encryption rotation", func() {
|
|||
})
|
||||
When("A server disables encryption", func() {
|
||||
It("it triggers the disable", func() {
|
||||
Expect(testutil.K3sCmd("secrets-encrypt", "disable", "-d", secretsEncryptionDataDir)).
|
||||
Expect(testutil.K3sCmd("secrets-encrypt disable -d", secretsEncryptionDataDir)).
|
||||
To(ContainSubstring("secrets-encryption disabled"))
|
||||
|
||||
result, err := testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir)
|
||||
result, err := testutil.K3sCmd("secrets-encrypt status -d", secretsEncryptionDataDir)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(result).To(ContainSubstring("Encryption Status: Disabled"))
|
||||
})
|
||||
It("restarts the server", func() {
|
||||
var err error
|
||||
Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed())
|
||||
Expect(testutil.K3sKillServer(secretsEncryptionServer, true)).To(Succeed())
|
||||
secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Eventually(func() (string, error) {
|
||||
return testutil.K3sCmd("kubectl", "get", "pods", "-A")
|
||||
return testutil.K3sCmd("kubectl get pods -A")
|
||||
}, "180s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
|
||||
time.Sleep(10 * time.Second)
|
||||
})
|
||||
It("reencrypts the keys", func() {
|
||||
result, err := testutil.K3sCmd("secrets-encrypt", "reencrypt", "-f", "--skip", "-d", secretsEncryptionDataDir)
|
||||
result, err := testutil.K3sCmd("secrets-encrypt reencrypt -f --skip -d", secretsEncryptionDataDir)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(result).To(ContainSubstring("reencryption started"))
|
||||
|
||||
result, err = testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir)
|
||||
result, err = testutil.K3sCmd("secrets-encrypt status -d", secretsEncryptionDataDir)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(result).To(ContainSubstring("Encryption Status: Disabled"))
|
||||
})
|
||||
|
@ -142,7 +142,7 @@ var _ = Describe("secrets encryption rotation", func() {
|
|||
|
||||
var _ = AfterSuite(func() {
|
||||
if !testutil.IsExistingServer() {
|
||||
Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed())
|
||||
Expect(testutil.K3sKillServer(secretsEncryptionServer, true)).To(Succeed())
|
||||
Expect(os.RemoveAll(secretsEncryptionDataDir)).To(Succeed())
|
||||
}
|
||||
})
|
||||
|
|
|
@ -8,9 +8,11 @@ import (
|
|||
"os/exec"
|
||||
"os/user"
|
||||
"strings"
|
||||
"syscall"
|
||||
|
||||
"github.com/rancher/k3s/pkg/flock"
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/vishvananda/netlink"
|
||||
)
|
||||
|
||||
// Compile-time variable
|
||||
|
@ -63,16 +65,20 @@ func IsExistingServer() bool {
|
|||
}
|
||||
|
||||
// K3sCmd launches the provided K3s command via exec. Command blocks until finished.
|
||||
// Command output from both Stderr and Stdout is provided via string.
|
||||
// Command output from both Stderr and Stdout is provided via string. Input can
|
||||
// be a single string with space separated args, or multiple string args
|
||||
// cmdEx1, err := K3sCmd("etcd-snapshot", "ls")
|
||||
// cmdEx2, err := K3sCmd("kubectl get pods -A")
|
||||
// cmdEx2, err := K3sCmd("kubectl", "get", "pods", "-A")
|
||||
func K3sCmd(cmdName string, cmdArgs ...string) (string, error) {
|
||||
func K3sCmd(inputArgs ...string) (string, error) {
|
||||
if !IsRoot() {
|
||||
return "", errors.New("integration tests must be run as sudo/root")
|
||||
}
|
||||
k3sBin := findK3sExecutable()
|
||||
// Only run sudo if not root
|
||||
k3sCmd := append([]string{cmdName}, cmdArgs...)
|
||||
var k3sCmd []string
|
||||
for _, arg := range inputArgs {
|
||||
k3sCmd = append(k3sCmd, strings.Fields(arg)...)
|
||||
}
|
||||
cmd := exec.Command(k3sBin, k3sCmd...)
|
||||
byteOut, err := cmd.CombinedOutput()
|
||||
return string(byteOut), err
|
||||
|
@ -145,17 +151,53 @@ func K3sStartServer(inputArgs ...string) (*K3sServer, error) {
|
|||
|
||||
k3sCmd := append([]string{"server"}, cmdArgs...)
|
||||
cmd := exec.Command(k3sBin, k3sCmd...)
|
||||
// Give the server a new group id so we can kill it and its children later
|
||||
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
|
||||
cmdOut, _ := cmd.StderrPipe()
|
||||
cmd.Stderr = os.Stderr
|
||||
err = cmd.Start()
|
||||
return &K3sServer{cmd, bufio.NewScanner(cmdOut), k3sLock}, err
|
||||
}
|
||||
|
||||
// K3sKillServer terminates the running K3s server and unlocks the file for
|
||||
// other tests
|
||||
func K3sKillServer(server *K3sServer) error {
|
||||
if err := server.cmd.Process.Kill(); err != nil {
|
||||
// K3sKillServer terminates the running K3s server and its children
|
||||
// and unlocks the file for other tests
|
||||
func K3sKillServer(server *K3sServer, releaseLock bool) error {
|
||||
pgid, err := syscall.Getpgid(server.cmd.Process.Pid)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return flock.Release(server.lock)
|
||||
if err := syscall.Kill(-pgid, syscall.SIGKILL); err != nil {
|
||||
return err
|
||||
}
|
||||
if releaseLock {
|
||||
return flock.Release(server.lock)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// K3sCleanup attempts to cleanup networking and files leftover from an integration test
|
||||
func K3sCleanup(server *K3sServer, releaseLock bool) error {
|
||||
if cni0Link, err := netlink.LinkByName("cni0"); err == nil {
|
||||
links, _ := netlink.LinkList()
|
||||
for _, link := range links {
|
||||
if link.Attrs().MasterIndex == cni0Link.Attrs().Index {
|
||||
netlink.LinkDel(link)
|
||||
}
|
||||
}
|
||||
netlink.LinkDel(cni0Link)
|
||||
}
|
||||
|
||||
if flannel1, err := netlink.LinkByName("flannel.1"); err == nil {
|
||||
netlink.LinkDel(flannel1)
|
||||
}
|
||||
if flannelV6, err := netlink.LinkByName("flannel-v6.1"); err == nil {
|
||||
netlink.LinkDel(flannelV6)
|
||||
}
|
||||
if err := os.RemoveAll("/var/lib/rancher/k3s"); err != nil {
|
||||
return err
|
||||
}
|
||||
if releaseLock {
|
||||
return flock.Release(server.lock)
|
||||
}
|
||||
return nil
|
||||
}
|
Loading…
Reference in New Issue