Merge pull request #46620 from enxebre/kuberuntime-test-coverage

Automatic merge from submit-queue (batch tested with PRs 46620, 46732, 46773, 46772, 46725)

Improving test coverage for kubelet/kuberuntime.

**What this PR does / why we need it**:
Increases test coverage for kubelet/kuberuntime 
https://github.com/kubernetes/kubernetes/issues/46123

**Which issue this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close that issue when PR gets merged)*: fixes #
https://github.com/kubernetes/kubernetes/issues/46123

**Special notes for your reviewer**:

**Release note**:

```release-note
NONE
```

pkg/kubelet/kuberuntime/BUILD

@@ -69,6 +69,7 @@ go_test(
     name = "go_default_test",
     srcs = [
         "helpers_test.go",
+        "instrumented_services_test.go",
         "kuberuntime_container_test.go",
         "kuberuntime_gc_test.go",
         "kuberuntime_image_test.go",
@@ -77,6 +78,7 @@ go_test(
         "kuberuntime_sandbox_test.go",
         "labels_test.go",
         "legacy_test.go",
+        "security_context_test.go",
     ],
     library = ":go_default_library",
     tags = ["automanaged"],
@@ -87,8 +89,10 @@ go_test(
         "//pkg/kubelet/apis/cri/v1alpha1:go_default_library",
         "//pkg/kubelet/container:go_default_library",
         "//pkg/kubelet/container/testing:go_default_library",
+        "//pkg/kubelet/metrics:go_default_library",
         "//vendor/github.com/golang/mock/gomock:go_default_library",
         "//vendor/github.com/google/cadvisor/info/v1:go_default_library",
+        "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
         "//vendor/github.com/stretchr/testify/assert:go_default_library",
         "//vendor/github.com/stretchr/testify/require:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",

pkg/kubelet/kuberuntime/helpers_test.go

@@ -122,3 +122,86 @@ func TestToKubeContainer(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Equal(t, expect, got)
 }
+
+func TestGetImageUser(t *testing.T) {
+	_, i, m, err := createTestRuntimeManager()
+	assert.NoError(t, err)
+
+	type image struct {
+		name     string
+		uid      *runtimeapi.Int64Value
+		username string
+	}
+
+	type imageUserValues struct {
+		// getImageUser can return (*int64)(nil) so comparing with *uid will break
+		// type cannot be *int64 as Golang does not allow to take the address of a numeric constant"
+		uid      interface{}
+		username string
+		err      error
+	}
+
+	tests := []struct {
+		description             string
+		originalImage           image
+		expectedImageUserValues imageUserValues
+	}{
+		{
+			"image without username and uid should return (new(int64), \"\", nil)",
+			image{
+				name:     "test-image-ref1",
+				uid:      (*runtimeapi.Int64Value)(nil),
+				username: "",
+			},
+			imageUserValues{
+				uid:      int64(0),
+				username: "",
+				err:      nil,
+			},
+		},
+		{
+			"image with username and no uid should return ((*int64)nil, imageStatus.Username, nil)",
+			image{
+				name:     "test-image-ref2",
+				uid:      (*runtimeapi.Int64Value)(nil),
+				username: "testUser",
+			},
+			imageUserValues{
+				uid:      (*int64)(nil),
+				username: "testUser",
+				err:      nil,
+			},
+		},
+		{
+			"image with uid should return (*int64, \"\", nil)",
+			image{
+				name: "test-image-ref3",
+				uid: &runtimeapi.Int64Value{
+					Value: 2,
+				},
+				username: "whatever",
+			},
+			imageUserValues{
+				uid:      int64(2),
+				username: "",
+				err:      nil,
+			},
+		},
+	}
+
+	i.SetFakeImages([]string{"test-image-ref1", "test-image-ref2", "test-image-ref3"})
+	for j, test := range tests {
+		i.Images[test.originalImage.name].Username = test.originalImage.username
+		i.Images[test.originalImage.name].Uid = test.originalImage.uid
+
+		uid, username, error := m.getImageUser(test.originalImage.name)
+		assert.NoError(t, error, "TestCase[%d]", j)
+
+		if test.expectedImageUserValues.uid == (*int64)(nil) {
+			assert.Equal(t, test.expectedImageUserValues.uid, uid, "TestCase[%d]", j)
+		} else {
+			assert.Equal(t, test.expectedImageUserValues.uid, *uid, "TestCase[%d]", j)
+		}
+		assert.Equal(t, test.expectedImageUserValues.username, username, "TestCase[%d]", j)
+	}
+}

pkg/kubelet/kuberuntime/instrumented_services_test.go

@@ -0,0 +1,61 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kuberuntime
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/stretchr/testify/assert"
+	"k8s.io/kubernetes/pkg/kubelet/metrics"
+	"net"
+	"net/http"
+	"testing"
+	"time"
+)
+
+func TestRecordOperation(t *testing.T) {
+	prometheus.MustRegister(metrics.RuntimeOperations)
+	prometheus.MustRegister(metrics.RuntimeOperationsLatency)
+	prometheus.MustRegister(metrics.RuntimeOperationsErrors)
+
+	temporalServer := "127.0.0.1:1234"
+	l, err := net.Listen("tcp", temporalServer)
+	assert.NoError(t, err)
+	defer l.Close()
+
+	prometheusUrl := "http://" + temporalServer + "/metrics"
+	mux := http.NewServeMux()
+	mux.Handle("/metrics", prometheus.Handler())
+	server := &http.Server{
+		Addr:    temporalServer,
+		Handler: mux,
+	}
+	go func() {
+		server.Serve(l)
+	}()
+
+	recordOperation("create_container", time.Now())
+	runtimeOperationsCounterExpected := "kubelet_runtime_operations{operation_type=\"create_container\"} 1"
+	runtimeOperationsLatencyExpected := "kubelet_runtime_operations_latency_microseconds_count{operation_type=\"create_container\"} 1"
+
+	assert.HTTPBodyContains(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		mux.ServeHTTP(w, r)
+	}), "GET", prometheusUrl, nil, runtimeOperationsCounterExpected)
+
+	assert.HTTPBodyContains(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		mux.ServeHTTP(w, r)
+	}), "GET", prometheusUrl, nil, runtimeOperationsLatencyExpected)
+}

pkg/kubelet/kuberuntime/kuberuntime_container_test.go

@@ -24,6 +24,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/kubernetes/pkg/api/v1"
 	runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1"
 	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
@@ -165,3 +166,93 @@ func TestToKubeContainerStatus(t *testing.T) {
 		assert.Equal(t, test.expected, actual, desc)
 	}
 }
+
+func makeExpetectedConfig(m *kubeGenericRuntimeManager, pod *v1.Pod, containerIndex int) *runtimeapi.ContainerConfig {
+	container := &pod.Spec.Containers[containerIndex]
+	podIP := ""
+	restartCount := 0
+	opts, _, _ := m.runtimeHelper.GenerateRunContainerOptions(pod, container, podIP)
+	containerLogsPath := buildContainerLogsPath(container.Name, restartCount)
+	restartCountUint32 := uint32(restartCount)
+	envs := make([]*runtimeapi.KeyValue, len(opts.Envs))
+
+	expectedConfig := &runtimeapi.ContainerConfig{
+		Metadata: &runtimeapi.ContainerMetadata{
+			Name:    container.Name,
+			Attempt: restartCountUint32,
+		},
+		Image:       &runtimeapi.ImageSpec{Image: container.Image},
+		Command:     container.Command,
+		Args:        []string(nil),
+		WorkingDir:  container.WorkingDir,
+		Labels:      newContainerLabels(container, pod),
+		Annotations: newContainerAnnotations(container, pod, restartCount),
+		Devices:     makeDevices(opts),
+		Mounts:      m.makeMounts(opts, container),
+		LogPath:     containerLogsPath,
+		Stdin:       container.Stdin,
+		StdinOnce:   container.StdinOnce,
+		Tty:         container.TTY,
+		Linux:       m.generateLinuxContainerConfig(container, pod, new(int64), ""),
+		Envs:        envs,
+	}
+	return expectedConfig
+}
+
+func TestGenerateContainerConfig(t *testing.T) {
+	_, _, m, err := createTestRuntimeManager()
+	assert.NoError(t, err)
+
+	pod := &v1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			UID:       "12345678",
+			Name:      "bar",
+			Namespace: "new",
+		},
+		Spec: v1.PodSpec{
+			Containers: []v1.Container{
+				{
+					Name:            "foo",
+					Image:           "busybox",
+					ImagePullPolicy: v1.PullIfNotPresent,
+					Command:         []string{"testCommand"},
+					WorkingDir:      "testWorkingDir",
+				},
+			},
+		},
+	}
+
+	expectedConfig := makeExpetectedConfig(m, pod, 0)
+	containerConfig, err := m.generateContainerConfig(&pod.Spec.Containers[0], pod, 0, "", pod.Spec.Containers[0].Image)
+	assert.NoError(t, err)
+	assert.Equal(t, expectedConfig, containerConfig, "generate container config for kubelet runtime v1.")
+
+	runAsUser := types.UnixUserID(0)
+	RunAsNonRoot := false
+	podWithContainerSecurityContext := &v1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			UID:       "12345678",
+			Name:      "bar",
+			Namespace: "new",
+		},
+		Spec: v1.PodSpec{
+			Containers: []v1.Container{
+				{
+					Name:            "foo",
+					Image:           "busybox",
+					ImagePullPolicy: v1.PullIfNotPresent,
+					Command:         []string{"testCommand"},
+					WorkingDir:      "testWorkingDir",
+					SecurityContext: &v1.SecurityContext{
+						RunAsNonRoot: &RunAsNonRoot,
+						RunAsUser:    &runAsUser,
+					},
+				},
+			},
+		},
+	}
+
+	expectedConfig = makeExpetectedConfig(m, podWithContainerSecurityContext, 0)
+	containerConfig, err = m.generateContainerConfig(&podWithContainerSecurityContext.Spec.Containers[0], podWithContainerSecurityContext, 0, "", podWithContainerSecurityContext.Spec.Containers[0].Image)
+	assert.Error(t, err)
+}

pkg/kubelet/kuberuntime/security_context_test.go

@@ -0,0 +1,104 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kuberuntime
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/kubernetes/pkg/api/v1"
+
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+func TestVerifyRunAsNonRoot(t *testing.T) {
+	pod := &v1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			UID:       "12345678",
+			Name:      "bar",
+			Namespace: "new",
+		},
+		Spec: v1.PodSpec{
+			Containers: []v1.Container{
+				{
+					Name:            "foo",
+					Image:           "busybox",
+					ImagePullPolicy: v1.PullIfNotPresent,
+					Command:         []string{"testCommand"},
+					WorkingDir:      "testWorkingDir",
+				},
+			},
+		},
+	}
+
+	err := verifyRunAsNonRoot(pod, &pod.Spec.Containers[0], int64(0))
+	assert.NoError(t, err)
+
+	runAsUser := types.UnixUserID(0)
+	RunAsNonRoot := false
+	podWithContainerSecurityContext := &v1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			UID:       "12345678",
+			Name:      "bar",
+			Namespace: "new",
+		},
+		Spec: v1.PodSpec{
+			Containers: []v1.Container{
+				{
+					Name:            "foo",
+					Image:           "busybox",
+					ImagePullPolicy: v1.PullIfNotPresent,
+					Command:         []string{"testCommand"},
+					WorkingDir:      "testWorkingDir",
+					SecurityContext: &v1.SecurityContext{
+						RunAsNonRoot: &RunAsNonRoot,
+						RunAsUser:    &runAsUser,
+					},
+				},
+			},
+		},
+	}
+
+	err2 := verifyRunAsNonRoot(podWithContainerSecurityContext, &podWithContainerSecurityContext.Spec.Containers[0], int64(0))
+	assert.EqualError(t, err2, "container's runAsUser breaks non-root policy")
+
+	RunAsNonRoot = false
+	podWithContainerSecurityContext = &v1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			UID:       "12345678",
+			Name:      "bar",
+			Namespace: "new",
+		},
+		Spec: v1.PodSpec{
+			Containers: []v1.Container{
+				{
+					Name:            "foo",
+					Image:           "busybox",
+					ImagePullPolicy: v1.PullIfNotPresent,
+					Command:         []string{"testCommand"},
+					WorkingDir:      "testWorkingDir",
+					SecurityContext: &v1.SecurityContext{
+						RunAsNonRoot: &RunAsNonRoot,
+					},
+				},
+			},
+		},
+	}
+
+	err3 := verifyRunAsNonRoot(podWithContainerSecurityContext, &podWithContainerSecurityContext.Spec.Containers[0], int64(0))
+	assert.EqualError(t, err3, "container has runAsNonRoot and image will run as root")
+}