From 7814865b406c4ff90e4d096d0f236cad1654293c Mon Sep 17 00:00:00 2001 From: Jiatong Wang Date: Wed, 10 Apr 2019 11:37:06 -0700 Subject: [PATCH] Move gpu_util.go to e2e/framework/gpu --- test/e2e/framework/BUILD | 2 +- test/e2e/framework/gpu/BUILD | 29 +++++++++++++++++++ test/e2e/framework/{ => gpu}/gpu_util.go | 9 +++--- test/e2e/instrumentation/monitoring/BUILD | 1 + .../instrumentation/monitoring/accelerator.go | 5 ++-- test/e2e/scheduling/BUILD | 1 + test/e2e/scheduling/nvidia-gpus.go | 5 ++-- test/e2e/upgrades/BUILD | 1 + test/e2e/upgrades/nvidia-gpu.go | 3 +- test/e2e_node/BUILD | 1 + test/e2e_node/gpu_device_plugin.go | 17 ++++++----- test/e2e_node/image_list.go | 3 +- 12 files changed, 58 insertions(+), 19 deletions(-) create mode 100644 test/e2e/framework/gpu/BUILD rename test/e2e/framework/{ => gpu}/gpu_util.go (92%) diff --git a/test/e2e/framework/BUILD b/test/e2e/framework/BUILD index 7b3142f6bc..28aa7819b7 100644 --- a/test/e2e/framework/BUILD +++ b/test/e2e/framework/BUILD @@ -14,7 +14,6 @@ go_library( "framework.go", "get-kubemark-resource-usage.go", "google_compute.go", - "gpu_util.go", "jobs_util.go", "kubelet_stats.go", "log_size_monitoring.go", @@ -156,6 +155,7 @@ filegroup( ":package-srcs", "//test/e2e/framework/config:all-srcs", "//test/e2e/framework/ginkgowrapper:all-srcs", + "//test/e2e/framework/gpu:all-srcs", "//test/e2e/framework/ingress:all-srcs", "//test/e2e/framework/metrics:all-srcs", "//test/e2e/framework/podlogs:all-srcs", diff --git a/test/e2e/framework/gpu/BUILD b/test/e2e/framework/gpu/BUILD new file mode 100644 index 0000000000..340b4bc2ed --- /dev/null +++ b/test/e2e/framework/gpu/BUILD @@ -0,0 +1,29 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["gpu_util.go"], + importpath = "k8s.io/kubernetes/test/e2e/framework/gpu", + visibility = ["//visibility:public"], + deps = [ + "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library", + "//test/e2e/framework:go_default_library", + "//vendor/k8s.io/klog:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/test/e2e/framework/gpu_util.go b/test/e2e/framework/gpu/gpu_util.go similarity index 92% rename from test/e2e/framework/gpu_util.go rename to test/e2e/framework/gpu/gpu_util.go index 8a5aac81d0..415d4b1a93 100644 --- a/test/e2e/framework/gpu_util.go +++ b/test/e2e/framework/gpu/gpu_util.go @@ -14,13 +14,14 @@ See the License for the specific language governing permissions and limitations under the License. */ -package framework +package gpu import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/klog" + "k8s.io/kubernetes/test/e2e/framework" ) const ( @@ -48,8 +49,8 @@ func NumberOfNVIDIAGPUs(node *v1.Node) int64 { // NVIDIADevicePlugin returns the official Google Device Plugin pod for NVIDIA GPU in GKE func NVIDIADevicePlugin() *v1.Pod { - ds, err := DsFromManifest(GPUDevicePluginDSYAML) - ExpectNoError(err) + ds, err := framework.DsFromManifest(GPUDevicePluginDSYAML) + framework.ExpectNoError(err) p := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "device-plugin-nvidia-gpu-" + string(uuid.NewUUID()), @@ -64,7 +65,7 @@ func NVIDIADevicePlugin() *v1.Pod { // GetGPUDevicePluginImage returns the image of GPU device plugin. func GetGPUDevicePluginImage() string { - ds, err := DsFromManifest(GPUDevicePluginDSYAML) + ds, err := framework.DsFromManifest(GPUDevicePluginDSYAML) if err != nil { klog.Errorf("Failed to parse the device plugin image: %v", err) return "" diff --git a/test/e2e/instrumentation/monitoring/BUILD b/test/e2e/instrumentation/monitoring/BUILD index 0b635c1455..0add13cd5c 100644 --- a/test/e2e/instrumentation/monitoring/BUILD +++ b/test/e2e/instrumentation/monitoring/BUILD @@ -37,6 +37,7 @@ go_library( "//test/e2e/common:go_default_library", "//test/e2e/framework:go_default_library", "//test/e2e/framework/config:go_default_library", + "//test/e2e/framework/gpu:go_default_library", "//test/e2e/framework/metrics:go_default_library", "//test/e2e/instrumentation/common:go_default_library", "//test/e2e/scheduling:go_default_library", diff --git a/test/e2e/instrumentation/monitoring/accelerator.go b/test/e2e/instrumentation/monitoring/accelerator.go index 9654d3ef63..502fb2d560 100644 --- a/test/e2e/instrumentation/monitoring/accelerator.go +++ b/test/e2e/instrumentation/monitoring/accelerator.go @@ -24,11 +24,12 @@ import ( "github.com/onsi/ginkgo" "golang.org/x/oauth2/google" gcm "google.golang.org/api/monitoring/v3" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/framework/gpu" instrumentation "k8s.io/kubernetes/test/e2e/instrumentation/common" "k8s.io/kubernetes/test/e2e/scheduling" "k8s.io/kubernetes/test/utils/image" @@ -88,7 +89,7 @@ func testStackdriverAcceleratorMonitoring(f *framework.Framework) { Args: []string{"nvidia-smi && sleep infinity"}, Resources: v1.ResourceRequirements{ Limits: v1.ResourceList{ - framework.NVIDIAGPUResourceName: *resource.NewQuantity(1, resource.DecimalSI), + gpu.NVIDIAGPUResourceName: *resource.NewQuantity(1, resource.DecimalSI), }, }, }, diff --git a/test/e2e/scheduling/BUILD b/test/e2e/scheduling/BUILD index 8f934fb2ff..452733eb8e 100644 --- a/test/e2e/scheduling/BUILD +++ b/test/e2e/scheduling/BUILD @@ -44,6 +44,7 @@ go_library( "//staging/src/k8s.io/client-go/tools/cache:go_default_library", "//test/e2e/common:go_default_library", "//test/e2e/framework:go_default_library", + "//test/e2e/framework/gpu:go_default_library", "//test/e2e/framework/providers/gce:go_default_library", "//test/utils:go_default_library", "//test/utils/image:go_default_library", diff --git a/test/e2e/scheduling/nvidia-gpus.go b/test/e2e/scheduling/nvidia-gpus.go index 8dc0a3873d..37d50f7672 100644 --- a/test/e2e/scheduling/nvidia-gpus.go +++ b/test/e2e/scheduling/nvidia-gpus.go @@ -20,12 +20,13 @@ import ( "os" "time" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/uuid" extensionsinternal "k8s.io/kubernetes/pkg/apis/extensions" "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/framework/gpu" imageutils "k8s.io/kubernetes/test/utils/image" . "github.com/onsi/ginkgo" @@ -114,7 +115,7 @@ func SetupNVIDIAGPUNode(f *framework.Framework, setupResourceGatherer bool) *fra } else { dsYamlUrl = "https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/master/daemonset.yaml" } - gpuResourceName = framework.NVIDIAGPUResourceName + gpuResourceName = gpu.NVIDIAGPUResourceName framework.Logf("Using %v", dsYamlUrl) // Creates the DaemonSet that installs Nvidia Drivers. diff --git a/test/e2e/upgrades/BUILD b/test/e2e/upgrades/BUILD index 4bd5c3f5d7..a2465882ed 100644 --- a/test/e2e/upgrades/BUILD +++ b/test/e2e/upgrades/BUILD @@ -37,6 +37,7 @@ go_library( "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//test/e2e/common:go_default_library", "//test/e2e/framework:go_default_library", + "//test/e2e/framework/gpu:go_default_library", "//test/e2e/framework/testfiles:go_default_library", "//test/e2e/scheduling:go_default_library", "//test/utils/image:go_default_library", diff --git a/test/e2e/upgrades/nvidia-gpu.go b/test/e2e/upgrades/nvidia-gpu.go index 3b52989bad..459de01bcf 100644 --- a/test/e2e/upgrades/nvidia-gpu.go +++ b/test/e2e/upgrades/nvidia-gpu.go @@ -23,6 +23,7 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/framework/gpu" "k8s.io/kubernetes/test/e2e/scheduling" imageutils "k8s.io/kubernetes/test/utils/image" @@ -78,7 +79,7 @@ func (t *NvidiaGPUUpgradeTest) startJob(f *framework.Framework) { Command: []string{"/bin/sh", "-c", "./vectorAdd && sleep 60"}, Resources: v1.ResourceRequirements{ Limits: v1.ResourceList{ - framework.NVIDIAGPUResourceName: *resource.NewQuantity(1, resource.DecimalSI), + gpu.NVIDIAGPUResourceName: *resource.NewQuantity(1, resource.DecimalSI), }, }, }, diff --git a/test/e2e_node/BUILD b/test/e2e_node/BUILD index c8fe18d976..d4c423a0bb 100644 --- a/test/e2e_node/BUILD +++ b/test/e2e_node/BUILD @@ -47,6 +47,7 @@ go_library( "//staging/src/k8s.io/kubelet/config/v1beta1:go_default_library", "//test/e2e/common:go_default_library", "//test/e2e/framework:go_default_library", + "//test/e2e/framework/gpu:go_default_library", "//test/e2e/framework/metrics:go_default_library", "//test/utils/image:go_default_library", "//vendor/github.com/blang/semver:go_default_library", diff --git a/test/e2e_node/gpu_device_plugin.go b/test/e2e_node/gpu_device_plugin.go index 5ea0b089b2..30b9ee2c45 100644 --- a/test/e2e_node/gpu_device_plugin.go +++ b/test/e2e_node/gpu_device_plugin.go @@ -21,10 +21,11 @@ import ( "strconv" "time" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kubeletmetrics "k8s.io/kubernetes/pkg/kubelet/metrics" "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/framework/gpu" "k8s.io/kubernetes/test/e2e/framework/metrics" . "github.com/onsi/ginkgo" @@ -46,15 +47,15 @@ var _ = framework.KubeDescribe("NVIDIA GPU Device Plugin [Feature:GPUDevicePlugi } By("Creating the Google Device Plugin pod for NVIDIA GPU in GKE") - devicePluginPod, err = f.ClientSet.CoreV1().Pods(metav1.NamespaceSystem).Create(framework.NVIDIADevicePlugin()) + devicePluginPod, err = f.ClientSet.CoreV1().Pods(metav1.NamespaceSystem).Create(gpu.NVIDIADevicePlugin()) framework.ExpectNoError(err) By("Waiting for GPUs to become available on the local node") Eventually(func() bool { - return framework.NumberOfNVIDIAGPUs(getLocalNode(f)) > 0 + return gpu.NumberOfNVIDIAGPUs(getLocalNode(f)) > 0 }, 5*time.Minute, framework.Poll).Should(BeTrue()) - if framework.NumberOfNVIDIAGPUs(getLocalNode(f)) < 2 { + if gpu.NumberOfNVIDIAGPUs(getLocalNode(f)) < 2 { Skip("Not enough GPUs to execute this test (at least two needed)") } }) @@ -75,7 +76,7 @@ var _ = framework.KubeDescribe("NVIDIA GPU Device Plugin [Feature:GPUDevicePlugi It("checks that when Kubelet restarts exclusive GPU assignation to pods is kept.", func() { By("Creating one GPU pod on a node with at least two GPUs") podRECMD := "devs=$(ls /dev/ | egrep '^nvidia[0-9]+$') && echo gpu devices: $devs" - p1 := f.PodClient().CreateSync(makeBusyboxPod(framework.NVIDIAGPUResourceName, podRECMD)) + p1 := f.PodClient().CreateSync(makeBusyboxPod(gpu.NVIDIAGPUResourceName, podRECMD)) deviceIDRE := "gpu devices: (nvidia[0-9]+)" devId1 := parseLog(f, p1.Name, p1.Name, deviceIDRE) @@ -94,9 +95,9 @@ var _ = framework.KubeDescribe("NVIDIA GPU Device Plugin [Feature:GPUDevicePlugi restartKubelet() framework.WaitForAllNodesSchedulable(f.ClientSet, framework.TestContext.NodeSchedulableTimeout) Eventually(func() bool { - return framework.NumberOfNVIDIAGPUs(getLocalNode(f)) > 0 + return gpu.NumberOfNVIDIAGPUs(getLocalNode(f)) > 0 }, 5*time.Minute, framework.Poll).Should(BeTrue()) - p2 := f.PodClient().CreateSync(makeBusyboxPod(framework.NVIDIAGPUResourceName, podRECMD)) + p2 := f.PodClient().CreateSync(makeBusyboxPod(gpu.NVIDIAGPUResourceName, podRECMD)) By("Checking that pods got a different GPU") devId2 := parseLog(f, p2.Name, p2.Name, deviceIDRE) @@ -109,7 +110,7 @@ var _ = framework.KubeDescribe("NVIDIA GPU Device Plugin [Feature:GPUDevicePlugi Eventually(func() bool { node, err := f.ClientSet.CoreV1().Nodes().Get(framework.TestContext.NodeName, metav1.GetOptions{}) framework.ExpectNoError(err) - return framework.NumberOfNVIDIAGPUs(node) <= 0 + return gpu.NumberOfNVIDIAGPUs(node) <= 0 }, 10*time.Minute, framework.Poll).Should(BeTrue()) By("Checking that scheduled pods can continue to run even after we delete device plugin.") ensurePodContainerRestart(f, p1.Name, p1.Name) diff --git a/test/e2e_node/image_list.go b/test/e2e_node/image_list.go index f6c1726179..34c548029b 100644 --- a/test/e2e_node/image_list.go +++ b/test/e2e_node/image_list.go @@ -30,6 +30,7 @@ import ( runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" commontest "k8s.io/kubernetes/test/e2e/common" "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/framework/gpu" imageutils "k8s.io/kubernetes/test/utils/image" ) @@ -52,7 +53,7 @@ var NodeImageWhiteList = sets.NewString( imageutils.GetE2EImage(imageutils.Netexec), imageutils.GetE2EImage(imageutils.Nonewprivs), imageutils.GetPauseImageName(), - framework.GetGPUDevicePluginImage(), + gpu.GetGPUDevicePluginImage(), "gcr.io/kubernetes-e2e-test-images/node-perf/npb-is:1.0", "gcr.io/kubernetes-e2e-test-images/node-perf/npb-ep:1.0", "gcr.io/kubernetes-e2e-test-images/node-perf/tf-wide-deep-amd64:1.0",