From 14be65c74b3258d883fb0b879803e6f6aa851e48 Mon Sep 17 00:00:00 2001 From: enxebre Date: Tue, 30 May 2017 12:46:06 +0200 Subject: [PATCH] Improving test coverage for kubelet/kuberuntime. --- pkg/kubelet/kuberuntime/BUILD | 4 + pkg/kubelet/kuberuntime/helpers_test.go | 83 ++++++++++++++ .../kuberuntime/instrumented_services_test.go | 61 ++++++++++ .../kuberuntime/kuberuntime_container_test.go | 91 +++++++++++++++ .../kuberuntime/security_context_test.go | 104 ++++++++++++++++++ 5 files changed, 343 insertions(+) create mode 100644 pkg/kubelet/kuberuntime/instrumented_services_test.go create mode 100644 pkg/kubelet/kuberuntime/security_context_test.go diff --git a/pkg/kubelet/kuberuntime/BUILD b/pkg/kubelet/kuberuntime/BUILD index 5a78beb568..045ca924e3 100644 --- a/pkg/kubelet/kuberuntime/BUILD +++ b/pkg/kubelet/kuberuntime/BUILD @@ -69,6 +69,7 @@ go_test( name = "go_default_test", srcs = [ "helpers_test.go", + "instrumented_services_test.go", "kuberuntime_container_test.go", "kuberuntime_gc_test.go", "kuberuntime_image_test.go", @@ -77,6 +78,7 @@ go_test( "kuberuntime_sandbox_test.go", "labels_test.go", "legacy_test.go", + "security_context_test.go", ], library = ":go_default_library", tags = ["automanaged"], @@ -87,8 +89,10 @@ go_test( "//pkg/kubelet/apis/cri/v1alpha1:go_default_library", "//pkg/kubelet/container:go_default_library", "//pkg/kubelet/container/testing:go_default_library", + "//pkg/kubelet/metrics:go_default_library", "//vendor/github.com/golang/mock/gomock:go_default_library", "//vendor/github.com/google/cadvisor/info/v1:go_default_library", + "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", "//vendor/github.com/stretchr/testify/assert:go_default_library", "//vendor/github.com/stretchr/testify/require:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", diff --git a/pkg/kubelet/kuberuntime/helpers_test.go b/pkg/kubelet/kuberuntime/helpers_test.go index 687f25193e..f2d198d17b 100644 --- a/pkg/kubelet/kuberuntime/helpers_test.go +++ b/pkg/kubelet/kuberuntime/helpers_test.go @@ -122,3 +122,86 @@ func TestToKubeContainer(t *testing.T) { assert.NoError(t, err) assert.Equal(t, expect, got) } + +func TestGetImageUser(t *testing.T) { + _, i, m, err := createTestRuntimeManager() + assert.NoError(t, err) + + type image struct { + name string + uid *runtimeapi.Int64Value + username string + } + + type imageUserValues struct { + // getImageUser can return (*int64)(nil) so comparing with *uid will break + // type cannot be *int64 as Golang does not allow to take the address of a numeric constant" + uid interface{} + username string + err error + } + + tests := []struct { + description string + originalImage image + expectedImageUserValues imageUserValues + }{ + { + "image without username and uid should return (new(int64), \"\", nil)", + image{ + name: "test-image-ref1", + uid: (*runtimeapi.Int64Value)(nil), + username: "", + }, + imageUserValues{ + uid: int64(0), + username: "", + err: nil, + }, + }, + { + "image with username and no uid should return ((*int64)nil, imageStatus.Username, nil)", + image{ + name: "test-image-ref2", + uid: (*runtimeapi.Int64Value)(nil), + username: "testUser", + }, + imageUserValues{ + uid: (*int64)(nil), + username: "testUser", + err: nil, + }, + }, + { + "image with uid should return (*int64, \"\", nil)", + image{ + name: "test-image-ref3", + uid: &runtimeapi.Int64Value{ + Value: 2, + }, + username: "whatever", + }, + imageUserValues{ + uid: int64(2), + username: "", + err: nil, + }, + }, + } + + i.SetFakeImages([]string{"test-image-ref1", "test-image-ref2", "test-image-ref3"}) + for j, test := range tests { + i.Images[test.originalImage.name].Username = test.originalImage.username + i.Images[test.originalImage.name].Uid = test.originalImage.uid + + uid, username, error := m.getImageUser(test.originalImage.name) + assert.NoError(t, error, "TestCase[%d]", j) + + if test.expectedImageUserValues.uid == (*int64)(nil) { + assert.Equal(t, test.expectedImageUserValues.uid, uid, "TestCase[%d]", j) + } else { + assert.Equal(t, test.expectedImageUserValues.uid, *uid, "TestCase[%d]", j) + } + assert.Equal(t, test.expectedImageUserValues.username, username, "TestCase[%d]", j) + } +} diff --git a/pkg/kubelet/kuberuntime/instrumented_services_test.go b/pkg/kubelet/kuberuntime/instrumented_services_test.go new file mode 100644 index 0000000000..fab1470070 --- /dev/null +++ b/pkg/kubelet/kuberuntime/instrumented_services_test.go @@ -0,0 +1,61 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kuberuntime + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/stretchr/testify/assert" + "k8s.io/kubernetes/pkg/kubelet/metrics" + "net" + "net/http" + "testing" + "time" +) + +func TestRecordOperation(t *testing.T) { + prometheus.MustRegister(metrics.RuntimeOperations) + prometheus.MustRegister(metrics.RuntimeOperationsLatency) + prometheus.MustRegister(metrics.RuntimeOperationsErrors) + + temporalServer := "127.0.0.1:1234" + l, err := net.Listen("tcp", temporalServer) + assert.NoError(t, err) + defer l.Close() + + prometheusUrl := "http://" + temporalServer + "/metrics" + mux := http.NewServeMux() + mux.Handle("/metrics", prometheus.Handler()) + server := &http.Server{ + Addr: temporalServer, + Handler: mux, + } + go func() { + server.Serve(l) + }() + + recordOperation("create_container", time.Now()) + runtimeOperationsCounterExpected := "kubelet_runtime_operations{operation_type=\"create_container\"} 1" + runtimeOperationsLatencyExpected := "kubelet_runtime_operations_latency_microseconds_count{operation_type=\"create_container\"} 1" + + assert.HTTPBodyContains(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mux.ServeHTTP(w, r) + }), "GET", prometheusUrl, nil, runtimeOperationsCounterExpected) + + assert.HTTPBodyContains(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mux.ServeHTTP(w, r) + }), "GET", prometheusUrl, nil, runtimeOperationsLatencyExpected) +} diff --git a/pkg/kubelet/kuberuntime/kuberuntime_container_test.go b/pkg/kubelet/kuberuntime/kuberuntime_container_test.go index 3f0d99ffcc..0f02ddc2ec 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_container_test.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_container_test.go @@ -24,6 +24,7 @@ import ( "github.com/stretchr/testify/assert" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/kubernetes/pkg/api/v1" runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" @@ -165,3 +166,93 @@ func TestToKubeContainerStatus(t *testing.T) { assert.Equal(t, test.expected, actual, desc) } } + +func makeExpetectedConfig(m *kubeGenericRuntimeManager, pod *v1.Pod, containerIndex int) *runtimeapi.ContainerConfig { + container := &pod.Spec.Containers[containerIndex] + podIP := "" + restartCount := 0 + opts, _, _ := m.runtimeHelper.GenerateRunContainerOptions(pod, container, podIP) + containerLogsPath := buildContainerLogsPath(container.Name, restartCount) + restartCountUint32 := uint32(restartCount) + envs := make([]*runtimeapi.KeyValue, len(opts.Envs)) + + expectedConfig := &runtimeapi.ContainerConfig{ + Metadata: &runtimeapi.ContainerMetadata{ + Name: container.Name, + Attempt: restartCountUint32, + }, + Image: &runtimeapi.ImageSpec{Image: container.Image}, + Command: container.Command, + Args: []string(nil), + WorkingDir: container.WorkingDir, + Labels: newContainerLabels(container, pod), + Annotations: newContainerAnnotations(container, pod, restartCount), + Devices: makeDevices(opts), + Mounts: m.makeMounts(opts, container), + LogPath: containerLogsPath, + Stdin: container.Stdin, + StdinOnce: container.StdinOnce, + Tty: container.TTY, + Linux: m.generateLinuxContainerConfig(container, pod, new(int64), ""), + Envs: envs, + } + return expectedConfig +} + +func TestGenerateContainerConfig(t *testing.T) { + _, _, m, err := createTestRuntimeManager() + assert.NoError(t, err) + + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + UID: "12345678", + Name: "bar", + Namespace: "new", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "foo", + Image: "busybox", + ImagePullPolicy: v1.PullIfNotPresent, + Command: []string{"testCommand"}, + WorkingDir: "testWorkingDir", + }, + }, + }, + } + + expectedConfig := makeExpetectedConfig(m, pod, 0) + containerConfig, err := m.generateContainerConfig(&pod.Spec.Containers[0], pod, 0, "", pod.Spec.Containers[0].Image) + assert.NoError(t, err) + assert.Equal(t, expectedConfig, containerConfig, "generate container config for kubelet runtime v1.") + + runAsUser := types.UnixUserID(0) + RunAsNonRoot := false + podWithContainerSecurityContext := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + UID: "12345678", + Name: "bar", + Namespace: "new", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "foo", + Image: "busybox", + ImagePullPolicy: v1.PullIfNotPresent, + Command: []string{"testCommand"}, + WorkingDir: "testWorkingDir", + SecurityContext: &v1.SecurityContext{ + RunAsNonRoot: &RunAsNonRoot, + RunAsUser: &runAsUser, + }, + }, + }, + }, + } + + expectedConfig = makeExpetectedConfig(m, podWithContainerSecurityContext, 0) + containerConfig, err = m.generateContainerConfig(&podWithContainerSecurityContext.Spec.Containers[0], podWithContainerSecurityContext, 0, "", podWithContainerSecurityContext.Spec.Containers[0].Image) + assert.Error(t, err) +} diff --git a/pkg/kubelet/kuberuntime/security_context_test.go b/pkg/kubelet/kuberuntime/security_context_test.go new file mode 100644 index 0000000000..414d2e711d --- /dev/null +++ b/pkg/kubelet/kuberuntime/security_context_test.go @@ -0,0 +1,104 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kuberuntime + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/kubernetes/pkg/api/v1" + + "github.com/stretchr/testify/assert" + "testing" +) + +func TestVerifyRunAsNonRoot(t *testing.T) { + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + UID: "12345678", + Name: "bar", + Namespace: "new", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "foo", + Image: "busybox", + ImagePullPolicy: v1.PullIfNotPresent, + Command: []string{"testCommand"}, + WorkingDir: "testWorkingDir", + }, + }, + }, + } + + err := verifyRunAsNonRoot(pod, &pod.Spec.Containers[0], int64(0)) + assert.NoError(t, err) + + runAsUser := types.UnixUserID(0) + RunAsNonRoot := false + podWithContainerSecurityContext := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + UID: "12345678", + Name: "bar", + Namespace: "new", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "foo", + Image: "busybox", + ImagePullPolicy: v1.PullIfNotPresent, + Command: []string{"testCommand"}, + WorkingDir: "testWorkingDir", + SecurityContext: &v1.SecurityContext{ + RunAsNonRoot: &RunAsNonRoot, + RunAsUser: &runAsUser, + }, + }, + }, + }, + } + + err2 := verifyRunAsNonRoot(podWithContainerSecurityContext, &podWithContainerSecurityContext.Spec.Containers[0], int64(0)) + assert.EqualError(t, err2, "container's runAsUser breaks non-root policy") + + RunAsNonRoot = false + podWithContainerSecurityContext = &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + UID: "12345678", + Name: "bar", + Namespace: "new", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "foo", + Image: "busybox", + ImagePullPolicy: v1.PullIfNotPresent, + Command: []string{"testCommand"}, + WorkingDir: "testWorkingDir", + SecurityContext: &v1.SecurityContext{ + RunAsNonRoot: &RunAsNonRoot, + }, + }, + }, + }, + } + + err3 := verifyRunAsNonRoot(podWithContainerSecurityContext, &podWithContainerSecurityContext.Spec.Containers[0], int64(0)) + assert.EqualError(t, err3, "container has runAsNonRoot and image will run as root") +}