github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Add support for restarting k3s docker nodes 

Signed-off-by: Derek Nola <derek.nola@suse.com>
pull/11673/head
Derek Nola 2025-01-16 14:58:43 -08:00
parent 419dd5b03c
commit ca165cacbc
2 changed files with 106 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
tests/docker/basics/basics_test.go
View File

@ -27,6 +27,7 @@ var _ = Describe("Basic Tests", Ordered, func() {
It("should provision servers and agents", func() {
var err error
config, err = tester.NewTestConfig(*k3sImage)
config.NeedRestart = true
Expect(err).NotTo(HaveOccurred())
Expect(config.ProvisionServers(1)).To(Succeed())
Expect(config.ProvisionAgents(1)).To(Succeed())

81
tests/docker/test-helpers.go
View File

@ -30,18 +30,21 @@ type TestConfig struct {
K3sImage string
NumServers int
NumAgents int
Servers []ServerConfig
Agents []AgentConfig
NeedRestart bool
Servers []Server
Agents []Agent
ServerYaml string
AgentYaml string
}
type ServerConfig struct {
type Server struct {
Name string
Port int
IP string
URL string
}
type AgentConfig struct {
type Agent struct {
Name string
IP string
}
@ -117,6 +120,15 @@ func (config *TestConfig) ProvisionServers(numOfServers int) error {
serverImage := getEnvOrDefault("K3S_IMAGE_SERVER", config.K3sImage)
// Write the server yaml to a tmp file and mount it into the container
var yamlMount string
if config.ServerYaml != "" {
if err := os.WriteFile(filepath.Join(config.TestDir, fmt.Sprintf("server-%d.yaml", i)), []byte(config.ServerYaml), 0644); err != nil {
return fmt.Errorf("failed to write server yaml: %v", err)
}
yamlMount = fmt.Sprintf("--mount type=bind,src=%s,dst=/etc/rancher/k3s/config.yaml", filepath.Join(config.TestDir, fmt.Sprintf("server-%d.yaml", i)))
}
var joinOrStart string
if numOfServers > 0 {
if i == 0 {
@ -129,6 +141,36 @@ func (config *TestConfig) ProvisionServers(numOfServers int) error {
}
}
// If we need restarts, we use the systemd-node container, volume mount the k3s binary
// and start the server using the install script
if config.NeedRestart {
dRun := strings.Join([]string{"docker run -d",
"--name", name,
"--hostname", name,
"--privileged",
"-p", fmt.Sprintf("127.0.0.1:%d:6443", port),
"--memory", "2048m",
"-e", fmt.Sprintf("K3S_TOKEN=%s", config.Secret),
"-e", "K3S_DEBUG=true",
"-v", "/sys/fs/bpf:/sys/fs/bpf",
"-v", "/lib/modules:/lib/modules",
"-v", "/var/run/docker.sock:/var/run/docker.sock",
"-v", "/var/lib/docker:/var/lib/docker",
yamlMount,
"--mount", "type=bind,source=$(pwd)/../../../dist/artifacts/k3s,target=/usr/local/bin/k3s",
"rancher/systemd-node:v0.0.5",
"/usr/lib/systemd/systemd --unit=noop.target --show-status=true"}, " ")
if out, err := RunCommand(dRun); err != nil {
return fmt.Errorf("failed to start systemd container: %s: %v", out, err)
}
time.Sleep(5 * time.Second)
// The pipe requires that we use sh -c with "" to run the command
sCmd := fmt.Sprintf("/bin/sh -c \"curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC='%s' INSTALL_K3S_SKIP_DOWNLOAD=true sh -\"",
joinOrStart+" "+os.Getenv(fmt.Sprintf("SERVER_%d_ARGS", i)))
if out, err := RunCmdOnDocker(name, sCmd); err != nil {
return fmt.Errorf("failed to start server: %s: %v", out, err)
}
} else {
// Assemble all the Docker args
dRun := strings.Join([]string{"docker run -d",
"--name", name,
@ -146,6 +188,7 @@ func (config *TestConfig) ProvisionServers(numOfServers int) error {
if out, err := RunCommand(dRun); err != nil {
return fmt.Errorf("failed to run server container: %s: %v", out, err)
}
}
// Get the IP address of the container
ipOutput, err := RunCommand("docker inspect --format \"{{ .NetworkSettings.IPAddress }}\" " + name)
@ -156,7 +199,7 @@ func (config *TestConfig) ProvisionServers(numOfServers int) error {
url := fmt.Sprintf("https://%s:6443", ip)
config.Servers = append(config.Servers, ServerConfig{
config.Servers = append(config.Servers, Server{
Name: name,
Port: port,
IP: ip,
@ -192,6 +235,31 @@ func (config *TestConfig) ProvisionAgents(numOfAgents int) error {
agentInstanceArgs := fmt.Sprintf("AGENT_%d_ARGS", i)
if config.NeedRestart {
dRun := strings.Join([]string{"docker run -d",
"--name", name,
"--hostname", name,
"--privileged",
"-e", fmt.Sprintf("K3S_TOKEN=%s", config.Secret),
"-e", fmt.Sprintf("K3S_URL=%s", k3sURL),
"-v", "/sys/fs/bpf:/sys/fs/bpf",
"-v", "/lib/modules:/lib/modules",
"-v", "/var/run/docker.sock:/var/run/docker.sock",
"-v", "/var/lib/docker:/var/lib/docker",
"--mount", "type=bind,source=$(pwd)/../../../dist/artifacts/k3s,target=/usr/local/bin/k3s",
"rancher/systemd-node:v0.0.5",
"/usr/lib/systemd/systemd --unit=noop.target --show-status=true"}, " ")
if out, err := RunCommand(dRun); err != nil {
return fmt.Errorf("failed to start systemd container: %s: %v", out, err)
}
time.Sleep(5 * time.Second)
// The pipe requires that we use sh -c with "" to run the command
sCmd := fmt.Sprintf("/bin/sh -c \"curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC='agent %s' INSTALL_K3S_SKIP_DOWNLOAD=true sh -\"",
os.Getenv(agentInstanceArgs))
if out, err := RunCmdOnDocker(name, sCmd); err != nil {
return fmt.Errorf("failed to start server: %s: %v", out, err)
}
} else {
// Assemble all the Docker args
dRun := strings.Join([]string{"docker run -d",
"--name", name,
@ -208,6 +276,7 @@ func (config *TestConfig) ProvisionAgents(numOfAgents int) error {
if out, err := RunCommand(dRun); err != nil {
return fmt.Errorf("failed to run agent container: %s: %v", out, err)
}
}
// Get the IP address of the container
ipOutput, err := RunCommand("docker inspect --format \"{{ .NetworkSettings.IPAddress }}\" " + name)
@ -216,7 +285,7 @@ func (config *TestConfig) ProvisionAgents(numOfAgents int) error {
}
ip := strings.TrimSpace(ipOutput)
config.Agents = append(config.Agents, AgentConfig{
config.Agents = append(config.Agents, Agent{
Name: name,
IP: ip,
})