2022-02-08 19:24:34 +00:00
|
|
|
package restore_test
|
|
|
|
|
|
|
|
import (
|
|
|
|
"strings"
|
|
|
|
"testing"
|
|
|
|
|
2022-03-15 17:29:56 +00:00
|
|
|
testutil "github.com/k3s-io/k3s/tests/integration"
|
2022-02-09 16:22:53 +00:00
|
|
|
. "github.com/onsi/ginkgo/v2"
|
2022-02-08 19:24:34 +00:00
|
|
|
. "github.com/onsi/gomega"
|
|
|
|
)
|
|
|
|
|
|
|
|
var server1, server2 *testutil.K3sServer
|
|
|
|
var tmpdDataDir = "/tmp/restoredatadir"
|
|
|
|
var clientCACertHash string
|
2022-02-25 16:42:21 +00:00
|
|
|
var testLock int
|
2022-02-08 19:24:34 +00:00
|
|
|
var restoreServerArgs = []string{"--cluster-init", "-t", "test", "-d", tmpdDataDir}
|
|
|
|
var _ = BeforeSuite(func() {
|
|
|
|
if !testutil.IsExistingServer() {
|
|
|
|
var err error
|
2022-02-25 16:42:21 +00:00
|
|
|
testLock, err = testutil.K3sTestLock()
|
|
|
|
Expect(err).ToNot(HaveOccurred())
|
2022-02-08 19:24:34 +00:00
|
|
|
server1, err = testutil.K3sStartServer(restoreServerArgs...)
|
|
|
|
Expect(err).ToNot(HaveOccurred())
|
|
|
|
}
|
|
|
|
})
|
|
|
|
|
2022-08-31 22:40:17 +00:00
|
|
|
var _ = Describe("etcd snapshot restore", Ordered, func() {
|
2022-02-08 19:24:34 +00:00
|
|
|
BeforeEach(func() {
|
|
|
|
if testutil.IsExistingServer() && !testutil.ServerArgsPresent(restoreServerArgs) {
|
|
|
|
Skip("Test needs k3s server with: " + strings.Join(restoreServerArgs, " "))
|
|
|
|
}
|
|
|
|
})
|
|
|
|
When("a snapshot is restored on existing node", func() {
|
|
|
|
It("etcd starts up with no problems", func() {
|
2022-06-13 20:32:13 +00:00
|
|
|
Eventually(func() error {
|
|
|
|
return testutil.K3sDefaultDeployments()
|
|
|
|
}, "180s", "5s").Should(Succeed())
|
2022-02-08 19:24:34 +00:00
|
|
|
})
|
|
|
|
It("create a workload", func() {
|
|
|
|
result, err := testutil.K3sCmd("kubectl", "create", "-f", "./testdata/temp_depl.yaml")
|
|
|
|
Expect(result).To(ContainSubstring("deployment.apps/nginx-deployment created"))
|
|
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
})
|
2023-10-10 17:03:11 +00:00
|
|
|
It("make sure workload exists", func() {
|
|
|
|
res, err := testutil.K3sCmd("kubectl", "rollout", "status", "deployment", "nginx-deployment", "--watch=true", "--timeout=360s")
|
|
|
|
Expect(res).To(ContainSubstring("successfully rolled out"))
|
|
|
|
Expect(err).ToNot(HaveOccurred())
|
|
|
|
})
|
2022-02-08 19:24:34 +00:00
|
|
|
It("saves an etcd snapshot", func() {
|
|
|
|
Expect(testutil.K3sCmd("etcd-snapshot", "save", "-d", tmpdDataDir, "--name", "snapshot-to-restore")).
|
|
|
|
To(ContainSubstring("saved"))
|
|
|
|
})
|
|
|
|
It("list snapshots", func() {
|
|
|
|
Expect(testutil.K3sCmd("etcd-snapshot", "ls", "-d", tmpdDataDir)).
|
|
|
|
To(MatchRegexp(`://` + tmpdDataDir + `/server/db/snapshots/snapshot-to-restore`))
|
|
|
|
})
|
|
|
|
// create another workload
|
|
|
|
It("create a workload 2", func() {
|
|
|
|
result, err := testutil.K3sCmd("kubectl", "create", "-f", "./testdata/temp_depl2.yaml")
|
|
|
|
Expect(result).To(ContainSubstring("deployment.apps/nginx-deployment-post-snapshot created"))
|
|
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
})
|
|
|
|
It("get Client CA cert hash", func() {
|
|
|
|
// get md5sum of the CA certs
|
|
|
|
var err error
|
|
|
|
clientCACertHash, err = testutil.RunCommand("md5sum " + tmpdDataDir + "/server/tls/client-ca.crt | cut -f 1 -d' '")
|
|
|
|
Expect(err).ToNot(HaveOccurred())
|
|
|
|
})
|
|
|
|
It("stop k3s", func() {
|
2022-02-25 16:42:21 +00:00
|
|
|
Expect(testutil.K3sKillServer(server1)).To(Succeed())
|
2022-02-08 19:24:34 +00:00
|
|
|
})
|
|
|
|
It("restore the snapshot", func() {
|
|
|
|
// get snapshot file
|
|
|
|
filePath, err := testutil.RunCommand(`sudo find ` + tmpdDataDir + `/server -name "*snapshot-to-restore*"`)
|
|
|
|
Expect(err).ToNot(HaveOccurred())
|
|
|
|
filePath = strings.TrimSuffix(filePath, "\n")
|
|
|
|
Eventually(func() (string, error) {
|
|
|
|
return testutil.K3sCmd("server", "-d", tmpdDataDir, "--cluster-reset", "--token", "test", "--cluster-reset-restore-path", filePath)
|
2022-04-27 20:44:15 +00:00
|
|
|
}, "360s", "5s").Should(ContainSubstring(`restart without --cluster-reset flag now`))
|
2022-02-08 19:24:34 +00:00
|
|
|
})
|
|
|
|
It("start k3s server", func() {
|
|
|
|
var err error
|
|
|
|
server2, err = testutil.K3sStartServer(restoreServerArgs...)
|
|
|
|
Expect(err).ToNot(HaveOccurred())
|
|
|
|
})
|
|
|
|
It("starts up with no problems", func() {
|
2022-06-13 20:32:13 +00:00
|
|
|
Eventually(func() error {
|
|
|
|
return testutil.K3sDefaultDeployments()
|
|
|
|
}, "360s", "5s").Should(Succeed())
|
2022-02-08 19:24:34 +00:00
|
|
|
})
|
2023-10-10 17:03:11 +00:00
|
|
|
It("make sure workload 1 exists", func() {
|
|
|
|
res, err := testutil.K3sCmd("kubectl", "rollout", "status", "deployment", "nginx-deployment", "--watch=true", "--timeout=360s")
|
|
|
|
Expect(res).To(ContainSubstring("successfully rolled out"))
|
|
|
|
Expect(err).ToNot(HaveOccurred())
|
2022-02-08 19:24:34 +00:00
|
|
|
})
|
2023-10-10 17:03:11 +00:00
|
|
|
It("make sure workload 2 does not exists", func() {
|
2022-02-08 19:24:34 +00:00
|
|
|
res, err := testutil.K3sCmd("kubectl", "get", "deployment", "nginx-deployment-post-snapshot")
|
|
|
|
Expect(res).To(ContainSubstring("not found"))
|
2023-10-10 17:03:11 +00:00
|
|
|
Expect(err).To(HaveOccurred())
|
2022-02-08 19:24:34 +00:00
|
|
|
})
|
|
|
|
It("check if CA cert hash matches", func() {
|
|
|
|
// get md5sum of the CA certs
|
|
|
|
var err error
|
|
|
|
clientCACertHash2, err := testutil.RunCommand("md5sum " + tmpdDataDir + "/server/tls/client-ca.crt | cut -f 1 -d' '")
|
|
|
|
Expect(err).ToNot(HaveOccurred())
|
|
|
|
Expect(clientCACertHash2).To(Equal(clientCACertHash))
|
|
|
|
})
|
|
|
|
It("stop k3s", func() {
|
2022-02-25 16:42:21 +00:00
|
|
|
Expect(testutil.K3sKillServer(server2)).To(Succeed())
|
2022-02-08 19:24:34 +00:00
|
|
|
})
|
|
|
|
})
|
|
|
|
})
|
|
|
|
|
2023-01-12 21:43:31 +00:00
|
|
|
var failed bool
|
|
|
|
var _ = AfterEach(func() {
|
|
|
|
failed = failed || CurrentSpecReport().Failed()
|
|
|
|
})
|
|
|
|
|
2022-02-08 19:24:34 +00:00
|
|
|
var _ = AfterSuite(func() {
|
|
|
|
if !testutil.IsExistingServer() {
|
2023-01-12 21:43:31 +00:00
|
|
|
if failed {
|
|
|
|
testutil.K3sSaveLog(server1, false)
|
2022-08-31 22:40:17 +00:00
|
|
|
}
|
2022-02-25 16:42:21 +00:00
|
|
|
Expect(testutil.K3sKillServer(server1)).To(Succeed())
|
|
|
|
Expect(testutil.K3sKillServer(server2)).To(Succeed())
|
|
|
|
Expect(testutil.K3sCleanup(testLock, tmpdDataDir)).To(Succeed())
|
2022-02-08 19:24:34 +00:00
|
|
|
}
|
|
|
|
})
|
|
|
|
|
2022-02-09 16:22:53 +00:00
|
|
|
func Test_IntegrationEtcdRestoreSnapshot(t *testing.T) {
|
2022-02-08 19:24:34 +00:00
|
|
|
RegisterFailHandler(Fail)
|
2022-02-09 16:22:53 +00:00
|
|
|
RunSpecs(t, "Etcd Restore Suite")
|
2022-02-08 19:24:34 +00:00
|
|
|
}
|