mirror of https://github.com/k3s-io/k3s
Merge pull request #35868 from Random-Liu/cleanup-node-e2e-output-dir
Automatic merge from submit-queue Node E2E: Reorganize node e2e output directories. Fixes https://github.com/kubernetes/kubernetes/issues/35074. This PR cleans up the result directory and workspace directory of node e2e test. Local result directory: ``` /tmp/_artifacts/ |----- build-log.txt (build log) |----- *.xml (junit xml file) |----- local/ (local run *.log) |----- hostname1/ (remote run *.log) |----- hostname2/ ``` Workspace directory on test node: ``` /tmp/node-e2e-yyyy-mm-ddThh-mm-ss/ |----- cluster/ (gci mounter) |----- cni/ (cni binary) |----- e2e_node.test (test binary) |----- e2e_node_test.tar.gz (test tar) |----- etcd060429031/ (etcd data directory) |----- ginkgo (ginkgo binary) |----- kubelet (kubelet binary) |----- pod-manifest365096781/ (mirror pod directory) |----- results/ (test result directory) ``` @mtaufen /cc @kubernetes/sig-nodepull/6/head
Kubernetes Submit Queue
GitHub
commit
467a1cd23b
|
|
@ -26,8 +26,7 @@ skip=${SKIP:-""}
|
|||
# Currently, parallelism only affects when REMOTE=true. For local test,
|
||||
# ginkgo default parallelism (cores - 1) is used.
|
||||
parallelism=${PARALLELISM:-8}
|
||||
report=${REPORT:-"/tmp/"}
|
||||
artifacts=${ARTIFACTS:-"/tmp/_artifacts"}
|
||||
artifacts=${ARTIFACTS:-"/tmp/_artifacts/`date +%y%m%dT%H%M%S`"}
|
||||
remote=${REMOTE:-"false"}
|
||||
run_until_failure=${RUN_UNTIL_FAILURE:-"false"}
|
||||
test_args=${TEST_ARGS:-""}
|
||||
|
|
@ -39,17 +38,23 @@ if [[ $parallelism > 1 ]]; then
|
|||
fi
|
||||
|
||||
if [[ $focus != "" ]]; then
|
||||
ginkgoflags="$ginkgoflags -focus=$focus "
|
||||
ginkgoflags="$ginkgoflags -focus='$focus' "
|
||||
fi
|
||||
|
||||
if [[ $skip != "" ]]; then
|
||||
ginkgoflags="$ginkgoflags -skip=$skip "
|
||||
ginkgoflags="$ginkgoflags -skip='$skip' "
|
||||
fi
|
||||
|
||||
if [[ $run_until_failure != "" ]]; then
|
||||
ginkgoflags="$ginkgoflags -untilItFails=$run_until_failure "
|
||||
fi
|
||||
|
||||
# Setup the directory to copy test artifacts (logs, junit.xml, etc) from remote host to local host
|
||||
if [ ! -d "${artifacts}" ]; then
|
||||
echo "Creating artifacts directory at ${artifacts}"
|
||||
mkdir -p ${artifacts}
|
||||
fi
|
||||
echo "Test artifacts will be written to ${artifacts}"
|
||||
|
||||
if [ $remote = true ] ; then
|
||||
# The following options are only valid in remote run.
|
||||
|
|
@ -74,18 +79,6 @@ if [ $remote = true ] ; then
|
|||
cleanup=${CLEANUP:-"true"}
|
||||
delete_instances=${DELETE_INSTANCES:-"false"}
|
||||
|
||||
# Setup the directory to copy test artifacts (logs, junit.xml, etc) from remote host to local host
|
||||
if [[ $gubernator = true && -d "${artifacts}" ]]; then
|
||||
echo "Removing artifacts directory at ${artifacts}"
|
||||
rm -r ${artifacts}
|
||||
fi
|
||||
|
||||
if [ ! -d "${artifacts}" ]; then
|
||||
echo "Creating artifacts directory at ${artifacts}"
|
||||
mkdir -p ${artifacts}
|
||||
fi
|
||||
echo "Test artifacts will be written to ${artifacts}"
|
||||
|
||||
# Get the compute zone
|
||||
zone=$(gcloud info --format='value(config.properties.compute.zone)')
|
||||
if [[ $zone == "" ]]; then
|
||||
|
|
@ -137,7 +130,7 @@ if [ $remote = true ] ; then
|
|||
--results-dir="$artifacts" --ginkgo-flags="$ginkgoflags" \
|
||||
--image-project="$image_project" --instance-name-prefix="$instance_prefix" --setup-node="true" \
|
||||
--delete-instances="$delete_instances" --test_args="$test_args" --instance-metadata="$metadata" \
|
||||
2>&1 | tee "${artifacts}/build-log.txt"
|
||||
2>&1 | tee -i "${artifacts}/build-log.txt"
|
||||
exit $?
|
||||
|
||||
else
|
||||
|
|
@ -155,7 +148,7 @@ else
|
|||
# Test using the host the script was run on
|
||||
# Provided for backwards compatibility
|
||||
go run test/e2e_node/runner/local/run_local.go --ginkgo-flags="$ginkgoflags" \
|
||||
--test-flags="--alsologtostderr --v 4 --report-dir=${report} --node-name $(hostname) \
|
||||
$test_args" --build-dependencies=true
|
||||
--test-flags="--alsologtostderr --v 4 --report-dir=${artifacts} --node-name $(hostname) \
|
||||
$test_args" --build-dependencies=true 2>&1 | tee -i "${artifacts}/build-log.txt"
|
||||
exit $?
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -20,7 +20,6 @@ import (
|
|||
"flag"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"math/rand"
|
||||
"os"
|
||||
"os/exec"
|
||||
"os/user"
|
||||
|
|
@ -167,26 +166,24 @@ func RunRemote(archive string, host string, cleanup bool, junitFilePrefix string
|
|||
|
||||
// Create the temp staging directory
|
||||
glog.Infof("Staging test binaries on %s", host)
|
||||
dirName := fmt.Sprintf("gcloud-e2e-%d", rand.Int31())
|
||||
tmp := fmt.Sprintf("/tmp/%s", dirName)
|
||||
|
||||
workspace := fmt.Sprintf("/tmp/node-e2e-%s", getTimestamp())
|
||||
// Do not sudo here, so that we can use scp to copy test archive to the directdory.
|
||||
_, err := SSHNoSudo(host, "mkdir", tmp)
|
||||
_, err := SSHNoSudo(host, "mkdir", workspace)
|
||||
if err != nil {
|
||||
// Exit failure with the error
|
||||
return "", false, err
|
||||
}
|
||||
if cleanup {
|
||||
defer func() {
|
||||
output, err := SSH(host, "rm", "-rf", tmp)
|
||||
output, err := SSH(host, "rm", "-rf", workspace)
|
||||
if err != nil {
|
||||
glog.Errorf("failed to cleanup tmp directory %s on host %v. Output:\n%s", tmp, err, output)
|
||||
glog.Errorf("failed to cleanup workspace %s on host %v. Output:\n%s", workspace, err, output)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// Install the cni plugin.
|
||||
cniPath := filepath.Join(tmp, CNIDirectory)
|
||||
cniPath := filepath.Join(workspace, CNIDirectory)
|
||||
cmd := getSSHCommand(" ; ",
|
||||
fmt.Sprintf("mkdir -p %s", cniPath),
|
||||
fmt.Sprintf("wget -O - %s | tar -xz -C %s", CNIURL, cniPath),
|
||||
|
|
@ -218,7 +215,7 @@ func RunRemote(archive string, host string, cleanup bool, junitFilePrefix string
|
|||
}
|
||||
|
||||
// Copy the archive to the staging directory
|
||||
_, err = runSSHCommand("scp", archive, fmt.Sprintf("%s:%s/", GetHostnameOrIp(host), tmp))
|
||||
_, err = runSSHCommand("scp", archive, fmt.Sprintf("%s:%s/", GetHostnameOrIp(host), workspace))
|
||||
if err != nil {
|
||||
// Exit failure with the error
|
||||
return "", false, err
|
||||
|
|
@ -238,7 +235,7 @@ func RunRemote(archive string, host string, cleanup bool, junitFilePrefix string
|
|||
|
||||
// Extract the archive
|
||||
cmd = getSSHCommand(" && ",
|
||||
fmt.Sprintf("cd %s", tmp),
|
||||
fmt.Sprintf("cd %s", workspace),
|
||||
fmt.Sprintf("tar -xzvf ./%s", archiveName),
|
||||
)
|
||||
glog.Infof("Extracting tar on %s", host)
|
||||
|
|
@ -249,7 +246,7 @@ func RunRemote(archive string, host string, cleanup bool, junitFilePrefix string
|
|||
}
|
||||
|
||||
// If we are testing on a GCI node, we chmod 544 the mounter and specify a different mounter path in the test args.
|
||||
// We do this here because the local var `tmp` tells us which /tmp/gcloud-e2e-%d is relevant to the current test run.
|
||||
// We do this here because the local var `workspace` tells us which /tmp/node-e2e-%d is relevant to the current test run.
|
||||
|
||||
// Determine if the GCI mounter script exists locally.
|
||||
k8sDir, err := builder.GetK8sRootDir()
|
||||
|
|
@ -274,7 +271,7 @@ func RunRemote(archive string, host string, cleanup bool, junitFilePrefix string
|
|||
glog.Infof("GCI node and GCI mounter both detected, modifying --experimental-mounter-path accordingly")
|
||||
// Note this implicitly requires the script to be where we expect in the tarball, so if that location changes the error
|
||||
// here will tell us to update the remote test runner.
|
||||
mounterPath := filepath.Join(tmp, "cluster/gce/gci/mounter/mounter")
|
||||
mounterPath := filepath.Join(workspace, "cluster/gce/gci/mounter/mounter")
|
||||
output, err = SSH(host, "sh", "-c", fmt.Sprintf("'chmod 544 %s'", mounterPath))
|
||||
if err != nil {
|
||||
glog.Errorf("Unable to chmod 544 GCI mounter script. Err: %v, Output:\n%s", err, output)
|
||||
|
|
@ -286,9 +283,9 @@ func RunRemote(archive string, host string, cleanup bool, junitFilePrefix string
|
|||
|
||||
// Run the tests
|
||||
cmd = getSSHCommand(" && ",
|
||||
fmt.Sprintf("cd %s", tmp),
|
||||
fmt.Sprintf("cd %s", workspace),
|
||||
fmt.Sprintf("timeout -k 30s %fs ./ginkgo %s ./e2e_node.test -- --logtostderr --v 4 --node-name=%s --report-dir=%s/results --report-prefix=%s %s",
|
||||
testTimeoutSeconds.Seconds(), ginkgoFlags, host, tmp, junitFilePrefix, testArgs),
|
||||
testTimeoutSeconds.Seconds(), ginkgoFlags, host, workspace, junitFilePrefix, testArgs),
|
||||
)
|
||||
aggErrs := []error{}
|
||||
|
||||
|
|
@ -306,8 +303,8 @@ func RunRemote(archive string, host string, cleanup bool, junitFilePrefix string
|
|||
// TODO: This is a best-effort, temporary hack that only works for
|
||||
// journald nodes. We should have a more robust way to collect logs.
|
||||
var (
|
||||
logName = fmt.Sprintf("%s-system.log", dirName)
|
||||
logPath = fmt.Sprintf("/tmp/%s-system.log", dirName)
|
||||
logName = "system.log"
|
||||
logPath = filepath.Join(workspace, logName)
|
||||
destPath = fmt.Sprintf("%s/%s-%s", *resultsDir, host, logName)
|
||||
)
|
||||
glog.Infof("Test failed unexpectedly. Attempting to retreiving system logs (only works for nodes with journald)")
|
||||
|
|
@ -326,7 +323,7 @@ func RunRemote(archive string, host string, cleanup bool, junitFilePrefix string
|
|||
}
|
||||
|
||||
glog.Infof("Copying test artifacts from %s", host)
|
||||
scpErr := getTestArtifacts(host, tmp)
|
||||
scpErr := getTestArtifacts(host, workspace)
|
||||
if scpErr != nil {
|
||||
aggErrs = append(aggErrs, scpErr)
|
||||
}
|
||||
|
|
@ -334,14 +331,25 @@ func RunRemote(archive string, host string, cleanup bool, junitFilePrefix string
|
|||
return output, len(aggErrs) == 0, utilerrors.NewAggregate(aggErrs)
|
||||
}
|
||||
|
||||
// timestampFormat is the timestamp format used in the node e2e directory name.
|
||||
const timestampFormat = "20060102T150405"
|
||||
|
||||
func getTimestamp() string {
|
||||
return fmt.Sprintf(time.Now().Format(timestampFormat))
|
||||
}
|
||||
|
||||
func getTestArtifacts(host, testDir string) error {
|
||||
_, err := runSSHCommand("scp", "-r", fmt.Sprintf("%s:%s/results/", GetHostnameOrIp(host), testDir), fmt.Sprintf("%s/%s", *resultsDir, host))
|
||||
logPath := filepath.Join(*resultsDir, host)
|
||||
if err := os.MkdirAll(logPath, 0755); err != nil {
|
||||
return fmt.Errorf("failed to create log directory %q: %v", logPath, err)
|
||||
}
|
||||
// Copy logs to artifacts/hostname
|
||||
_, err := runSSHCommand("scp", "-r", fmt.Sprintf("%s:%s/results/*.log", GetHostnameOrIp(host), testDir), logPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Copy junit to the top of artifacts
|
||||
_, err = runSSHCommand("scp", fmt.Sprintf("%s:%s/results/junit*", GetHostnameOrIp(host), testDir), fmt.Sprintf("%s/", *resultsDir))
|
||||
_, err = runSSHCommand("scp", fmt.Sprintf("%s:%s/results/junit*", GetHostnameOrIp(host), testDir), *resultsDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -115,7 +115,8 @@ func (es *e2eServices) stop() {
|
|||
// startEtcd starts the embedded etcd instance or returns an error.
|
||||
func (es *e2eServices) startEtcd() error {
|
||||
glog.Info("Starting etcd")
|
||||
dataDir, err := ioutil.TempDir("", "node-e2e")
|
||||
// Create data directory in current working space.
|
||||
dataDir, err := ioutil.TempDir(".", "etcd")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -82,7 +82,11 @@ func (e *E2EServices) Start() error {
|
|||
// Start kubelet
|
||||
// Create the manifest path for kubelet.
|
||||
// TODO(random-liu): Remove related logic when we move kubelet starting logic out of the test.
|
||||
framework.TestContext.ManifestPath, err = ioutil.TempDir("", "node-e2e-pod")
|
||||
cwd, err := os.Getwd()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get current working directory: %v", err)
|
||||
}
|
||||
framework.TestContext.ManifestPath, err = ioutil.TempDir(cwd, "pod-manifest")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create static pod manifest directory: %v", err)
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue