mirror of https://github.com/k3s-io/k3s
Merge pull request #52574 from vmware/fix-kubelet-restart-tests
Automatic merge from submit-queue (batch tested with PRs 52469, 52574, 52330, 52689, 52829). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.. Fixing E2E Test - After restarting kubelet test expects node's status to be NotReady **What this PR does / why we need it**: This PR is fixing the e2e tests involves restarting the kubelets. After the kubelet is restarted, test expect the desired state to be NotReady. After restarting the kubelet we should wait for some time and then check nodes status to be Ready. Node should not be checked for NotReady state, after restarting kubelet. **Which issue this PR fixes** fixes # https://github.com/vmware/kubernetes/issues/285 **Special notes for your reviewer**: @BaluDontu @rohitjogvmw @tusharnt Test logs before fix ----- STEP: Restarting kubelet Sep 15 11:26:32.768: INFO: Attempting sudo systemctl restart kubelet Sep 15 11:26:33.001: INFO: ssh root@10.162.22.205:22: command: sudo systemctl restart kubelet Sep 15 11:26:33.001: INFO: ssh root@10.162.22.205:22: stdout: "" Sep 15 11:26:33.001: INFO: ssh root@10.162.22.205:22: stderr: "" Sep 15 11:26:33.001: INFO: ssh root@10.162.22.205:22: exit code: 0 Sep 15 11:26:33.002: INFO: Waiting up to 1m0s for node kubernetes-node2 condition Ready to be false Sep 15 11:26:33.012: INFO: Condition Ready of node kubernetes-node2 is true instead of false. Reason: KubeletReady, message: kubelet is posting ready status Sep 15 11:26:35.023: INFO: Condition Ready of node kubernetes-node2 is true instead of false. Reason: KubeletReady, message: kubelet is posting ready status Sep 15 11:26:37.032: INFO: Condition Ready of node kubernetes-node2 is true instead of false. Reason: KubeletReady, message: kubelet is posting ready status Sep 15 11:26:39.041: INFO: Condition Ready of node kubernetes-node2 is true instead of false. Reason: KubeletReady, message: kubelet is posting ready status Sep 15 11:26:41.051: INFO: Condition Ready of node kubernetes-node2 is true instead of false. Reason: KubeletReady, message: kubelet is posting ready status Sep 15 11:26:43.061: INFO: Condition Ready of node kubernetes-node2 is true instead of false. Reason: KubeletReady, message: kubelet is posting ready status Sep 15 11:26:45.070: INFO: Condition Ready of node kubernetes-node2 is true instead of false. Reason: KubeletReady, message: kubelet is posting ready status Sep 15 11:26:47.080: INFO: Condition Ready of node kubernetes-node2 is true instead of false. Reason: KubeletReady, message: kubelet is posting ready status Sep 15 11:26:49.093: INFO: Condition Ready of node kubernetes-node2 is true instead of false. Reason: KubeletReady, message: kubelet is posting ready status Sep 15 11:26:51.105: INFO: Condition Ready of node kubernetes-node2 is true instead of false. Reason: KubeletReady, message: kubelet is posting ready status Sep 15 11:26:53.117: INFO: Condition Ready of node kubernetes-node2 is true instead of false. Reason: KubeletReady, message: kubelet is posting ready status Sep 15 11:26:55.128: INFO: Condition Ready of node kubernetes-node2 is true instead of false. Reason: KubeletReady, message: kubelet is posting ready status Sep 15 11:26:57.140: INFO: Condition Ready of node kubernetes-node2 is true instead of false. Reason: KubeletReady, message: kubelet is posting ready status Sep 15 11:26:59.151: INFO: Condition Ready of node kubernetes-node2 is true instead of false. Reason: KubeletReady, message: kubelet is posting ready status Sep 15 11:27:01.158: INFO: Condition Ready of node kubernetes-node2 is true instead of false. Reason: KubeletReady, message: kubelet is posting ready status Sep 15 11:27:03.167: INFO: Condition Ready of node kubernetes-node2 is true instead of false. Reason: KubeletReady, message: kubelet is posting ready status Sep 15 11:27:05.180: INFO: Condition Ready of node kubernetes-node2 is true instead of false. Reason: KubeletReady, message: kubelet is posting ready status Sep 15 11:27:07.188: INFO: Condition Ready of node kubernetes-node2 is true instead of false. Reason: KubeletReady, message: kubelet is posting ready status Sep 15 11:27:09.210: INFO: Condition Ready of node kubernetes-node2 is true instead of false. Reason: KubeletReady, message: kubelet is posting ready status Sep 15 11:27:11.221: INFO: Condition Ready of node kubernetes-node2 is true instead of false. Reason: KubeletReady, message: kubelet is posting ready status Sep 15 11:27:13.231: INFO: Condition Ready of node kubernetes-node2 is true instead of false. Reason: KubeletReady, message: kubelet is posting ready status Sep 15 11:27:15.240: INFO: Condition Ready of node kubernetes-node2 is true instead of false. Reason: KubeletReady, message: kubelet is posting ready status Sep 15 11:27:17.249: INFO: Condition Ready of node kubernetes-node2 is true instead of false. Reason: KubeletReady, message: kubelet is posting ready status Sep 15 11:27:19.263: INFO: Condition Ready of node kubernetes-node2 is true instead of false. Reason: KubeletReady, message: kubelet is posting ready status Sep 15 11:27:21.272: INFO: Condition Ready of node kubernetes-node2 is true instead of false. Reason: KubeletReady, message: kubelet is posting ready status Sep 15 11:27:23.283: INFO: Condition Ready of node kubernetes-node2 is true instead of false. Reason: KubeletReady, message: kubelet is posting ready status Sep 15 11:27:25.309: INFO: Condition Ready of node kubernetes-node2 is true instead of false. Reason: KubeletReady, message: kubelet is posting ready status Sep 15 11:27:27.317: INFO: Condition Ready of node kubernetes-node2 is true instead of false. Reason: KubeletReady, message: kubelet is posting ready status Sep 15 11:27:29.327: INFO: Condition Ready of node kubernetes-node2 is true instead of false. Reason: KubeletReady, message: kubelet is posting ready status Sep 15 11:27:31.342: INFO: Condition Ready of node kubernetes-node2 is true instead of false. Reason: KubeletReady, message: kubelet is posting ready status Sep 15 11:27:33.343: INFO: Node kubernetes-node2 didn't reach desired Ready condition status (false) within 1m0s Sep 15 11:27:33.343: INFO: Node kubernetes-node2 failed to enter NotReady state [AfterEach] [sig-storage] PersistentVolumes:vsphere Test logs after fix ----- STEP: Restarting kubelet Sep 18 15:40:49.066: INFO: Checking if sudo command is present Sep 18 15:40:49.342: INFO: Checking if systemctl command is present Sep 18 15:40:49.573: INFO: Attempting `sudo systemctl status kubelet | grep 'Main PID'` Sep 18 15:40:49.733: INFO: ssh root@10.162.16.97:22: command: sudo systemctl status kubelet | grep 'Main PID' Sep 18 15:40:49.733: INFO: ssh root@10.162.16.97:22: stdout: " Main PID: 19715 (docker)\n" Sep 18 15:40:49.733: INFO: ssh root@10.162.16.97:22: stderr: "" Sep 18 15:40:49.733: INFO: ssh root@10.162.16.97:22: exit code: 0 Sep 18 15:40:49.733: INFO: Attempting `sudo systemctl restart kubelet` Sep 18 15:40:49.986: INFO: ssh root@10.162.16.97:22: command: sudo systemctl restart kubelet Sep 18 15:40:49.986: INFO: ssh root@10.162.16.97:22: stdout: "" Sep 18 15:40:49.986: INFO: ssh root@10.162.16.97:22: stderr: "" Sep 18 15:40:49.986: INFO: ssh root@10.162.16.97:22: exit code: 0 Sep 18 15:40:49.988: INFO: Attempting `sudo systemctl status kubelet | grep 'Main PID'` Sep 18 15:40:50.158: INFO: ssh root@10.162.16.97:22: command: sudo systemctl status kubelet | grep 'Main PID' Sep 18 15:40:50.158: INFO: ssh root@10.162.16.97:22: stdout: " Main PID: 25021 (docker)\n" Sep 18 15:40:50.158: INFO: ssh root@10.162.16.97:22: stderr: "" Sep 18 15:40:50.158: INFO: ssh root@10.162.16.97:22: exit code: 0 Sep 18 15:40:50.158: INFO: Noticed that kubelet PID is changed. Waiting for 30 Seconds for Kubelet to come back Sep 18 15:41:20.159: INFO: Waiting up to 1m0s for node kubernetes-node4 condition Ready to be true STEP: Testing that written file is accessible. Sep 18 15:41:20.191: INFO: Running '/Users/divyenp/github/vmware/kubernetes/_output/dockerized/bin/darwin/amd64/kubectl --server=https://10.162.0.45 --kubeconfig=/Users/divyenp/.kube/config exec --namespace=e2e-tests-pv-9j8j0 pvc-tester-3t9ds -- /bin/sh -c cat /mnt/_SUCCESS' Sep 18 15:41:20.855: INFO: stderr: "" Sep 18 15:41:20.855: INFO: Sep 18 15:41:20.855: INFO: Volume mount detected on pod pvc-tester-3t9ds and written file /mnt/_SUCCESS is readable post-restart. **Release note**: ```release-note NONE ```pull/6/head
Kubernetes Submit Queue
GitHub
commit
c2a781428d
|
|
@ -347,36 +347,92 @@ func tearDownTestCase(c clientset.Interface, f *framework.Framework, ns string,
|
|||
// - If `service` also returns stderr "command not found", the test is aborted.
|
||||
// Allowed kubeletOps are `kStart`, `kStop`, and `kRestart`
|
||||
func kubeletCommand(kOp kubeletOpt, c clientset.Interface, pod *v1.Pod) {
|
||||
command := ""
|
||||
sudoPresent := false
|
||||
systemctlPresent := false
|
||||
kubeletPid := ""
|
||||
|
||||
nodeIP, err := framework.GetHostExternalAddress(c, pod)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
nodeIP = nodeIP + ":22"
|
||||
systemctlCmd := fmt.Sprintf("sudo systemctl %s kubelet", string(kOp))
|
||||
framework.Logf("Attempting `%s`", systemctlCmd)
|
||||
sshResult, err := framework.SSH(systemctlCmd, nodeIP, framework.TestContext.Provider)
|
||||
|
||||
framework.Logf("Checking if sudo command is present")
|
||||
sshResult, err := framework.SSH("sudo --version", nodeIP, framework.TestContext.Provider)
|
||||
Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("SSH to Node %q errored.", pod.Spec.NodeName))
|
||||
framework.LogSSHResult(sshResult)
|
||||
if strings.Contains(sshResult.Stderr, "command not found") {
|
||||
serviceCmd := fmt.Sprintf("sudo service kubelet %s", string(kOp))
|
||||
framework.Logf("Attempting `%s`", serviceCmd)
|
||||
sshResult, err = framework.SSH(serviceCmd, nodeIP, framework.TestContext.Provider)
|
||||
Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("SSH to Node %q errored.", pod.Spec.NodeName))
|
||||
framework.LogSSHResult(sshResult)
|
||||
if !strings.Contains(sshResult.Stderr, "command not found") {
|
||||
sudoPresent = true
|
||||
}
|
||||
|
||||
framework.Logf("Checking if systemctl command is present")
|
||||
sshResult, err = framework.SSH("systemctl --version", nodeIP, framework.TestContext.Provider)
|
||||
Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("SSH to Node %q errored.", pod.Spec.NodeName))
|
||||
if !strings.Contains(sshResult.Stderr, "command not found") {
|
||||
command = fmt.Sprintf("systemctl %s kubelet", string(kOp))
|
||||
systemctlPresent = true
|
||||
} else {
|
||||
command = fmt.Sprintf("service kubelet %s", string(kOp))
|
||||
}
|
||||
if sudoPresent {
|
||||
command = fmt.Sprintf("sudo %s", command)
|
||||
}
|
||||
|
||||
if kOp == kRestart {
|
||||
kubeletPid = getKubeletMainPid(nodeIP, sudoPresent, systemctlPresent)
|
||||
}
|
||||
|
||||
framework.Logf("Attempting `%s`", command)
|
||||
sshResult, err = framework.SSH(command, nodeIP, framework.TestContext.Provider)
|
||||
Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("SSH to Node %q errored.", pod.Spec.NodeName))
|
||||
framework.LogSSHResult(sshResult)
|
||||
Expect(sshResult.Code).To(BeZero(), "Failed to [%s] kubelet:\n%#v", string(kOp), sshResult)
|
||||
// On restart, waiting for node NotReady prevents a race condition where the node takes a few moments to leave the
|
||||
// Ready state which in turn short circuits WaitForNodeToBeReady()
|
||||
if kOp == kStop || kOp == kRestart {
|
||||
|
||||
if kOp == kStop {
|
||||
if ok := framework.WaitForNodeToBeNotReady(c, pod.Spec.NodeName, NodeStateTimeout); !ok {
|
||||
framework.Failf("Node %s failed to enter NotReady state", pod.Spec.NodeName)
|
||||
}
|
||||
}
|
||||
if kOp == kRestart {
|
||||
// Wait for a minute to check if kubelet Pid is getting changed
|
||||
isPidChanged := false
|
||||
for start := time.Now(); time.Since(start) < 1*time.Minute; time.Sleep(2 * time.Second) {
|
||||
kubeletPidAfterRestart := getKubeletMainPid(nodeIP, sudoPresent, systemctlPresent)
|
||||
if kubeletPid != kubeletPidAfterRestart {
|
||||
isPidChanged = true
|
||||
break
|
||||
}
|
||||
}
|
||||
Expect(isPidChanged).To(BeTrue(), "Kubelet PID remained unchanged after restarting Kubelet")
|
||||
framework.Logf("Noticed that kubelet PID is changed. Waiting for 30 Seconds for Kubelet to come back")
|
||||
time.Sleep(30 * time.Second)
|
||||
}
|
||||
if kOp == kStart || kOp == kRestart {
|
||||
// For kubelet start and restart operations, Wait until Node becomes Ready
|
||||
if ok := framework.WaitForNodeToBeReady(c, pod.Spec.NodeName, NodeStateTimeout); !ok {
|
||||
framework.Failf("Node %s failed to enter Ready state", pod.Spec.NodeName)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// return the Main PID of the Kubelet Process
|
||||
func getKubeletMainPid(nodeIP string, sudoPresent bool, systemctlPresent bool) string {
|
||||
command := ""
|
||||
if systemctlPresent {
|
||||
command = "systemctl status kubelet | grep 'Main PID'"
|
||||
} else {
|
||||
command = "service kubelet status | grep 'Main PID'"
|
||||
}
|
||||
if sudoPresent {
|
||||
command = fmt.Sprintf("sudo %s", command)
|
||||
}
|
||||
framework.Logf("Attempting `%s`", command)
|
||||
sshResult, err := framework.SSH(command, nodeIP, framework.TestContext.Provider)
|
||||
Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("SSH to Node %q errored.", nodeIP))
|
||||
framework.LogSSHResult(sshResult)
|
||||
Expect(sshResult.Code).To(BeZero(), "Failed to get kubelet PID")
|
||||
Expect(sshResult.Stdout).NotTo(BeEmpty(), "Kubelet Main PID should not be Empty")
|
||||
return sshResult.Stdout
|
||||
}
|
||||
|
||||
// podExec wraps RunKubectl to execute a bash cmd in target pod
|
||||
func podExec(pod *v1.Pod, bashExec string) (string, error) {
|
||||
return framework.RunKubectl("exec", fmt.Sprintf("--namespace=%s", pod.Namespace), pod.Name, "--", "/bin/sh", "-c", bashExec)
|
||||
|
|
|
|||
Loading…
Reference in New Issue