github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

adds dynamic audit e2e test

pull/564/head
Patrick Barker 2018-08-15 08:07:45 -06:00 committed by Patrick Barker
parent 98fa2c7d32
commit 45d715cdc6
7 changed files with 398 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cluster/gce/config-test.sh
View File

@ -380,6 +380,8 @@ else
ADMISSION_CONTROL=${KUBE_ADMISSION_CONTROL}
fi
ENABLE_APISERVER_DYNAMIC_AUDIT="${ENABLE_APISERVER_DYNAMIC_AUDIT:-false}"
# Optional: if set to true kube-up will automatically check for existing resources and clean them up.
KUBE_UP_AUTOMATIC_CLEANUP=${KUBE_UP_AUTOMATIC_CLEANUP:-false}

6
cluster/gce/gci/configure-helper.sh
View File

@ -1630,7 +1630,6 @@ function start-kube-apiserver {
create-master-audit-policy "${audit_policy_file}" "${ADVANCED_AUDIT_POLICY:-}"
audit_policy_config_mount="{\"name\": \"auditpolicyconfigmount\",\"mountPath\": \"${audit_policy_file}\", \"readOnly\": true},"
audit_policy_config_volume="{\"name\": \"auditpolicyconfigmount\",\"hostPath\": {\"path\": \"${audit_policy_file}\", \"type\": \"FileOrCreate\"}},"
if [[ "${ADVANCED_AUDIT_BACKEND:-log}" == *"log"* ]]; then
# The advanced audit log backend config matches the basic audit log config.
params+=" --audit-log-path=/var/log/kube-apiserver-audit.log"
@ -1709,6 +1708,11 @@ function start-kube-apiserver {
fi
fi
if [[ "${ENABLE_APISERVER_DYNAMIC_AUDIT:-}" == "true" ]]; then
params+=" --audit-dynamic-configuration"
RUNTIME_CONFIG="${RUNTIME_CONFIG},auditconfiguration.k8s.io/v1alpha1=true"
fi
if [[ "${ENABLE_APISERVER_LOGS_HANDLER:-}" == "false" ]]; then
params+=" --enable-logs-handler=false"
fi

1
cluster/gce/util.sh
View File

@ -1110,6 +1110,7 @@ MULTIZONE: $(yaml-quote ${MULTIZONE:-})
NON_MASQUERADE_CIDR: $(yaml-quote ${NON_MASQUERADE_CIDR:-})
ENABLE_DEFAULT_STORAGE_CLASS: $(yaml-quote ${ENABLE_DEFAULT_STORAGE_CLASS:-})
ENABLE_APISERVER_ADVANCED_AUDIT: $(yaml-quote ${ENABLE_APISERVER_ADVANCED_AUDIT:-})
ENABLE_APISERVER_DYNAMIC_AUDIT: $(yaml-quote ${ENABLE_APISERVER_DYNAMIC_AUDIT:-})
ENABLE_CACHE_MUTATION_DETECTOR: $(yaml-quote ${ENABLE_CACHE_MUTATION_DETECTOR:-false})
ENABLE_PATCH_CONVERSION_DETECTOR: $(yaml-quote ${ENABLE_PATCH_CONVERSION_DETECTOR:-false})
ADVANCED_AUDIT_POLICY: $(yaml-quote ${ADVANCED_AUDIT_POLICY:-})

3
test/e2e/auth/BUILD
View File

@ -9,6 +9,7 @@ go_library(
name = "go_default_library",
srcs = [
"audit.go",
"audit_dynamic.go",
"certificates.go",
"framework.go",
"metadata_concealment.go",
@ -26,6 +27,7 @@ go_library(
"//pkg/security/podsecuritypolicy/util:go_default_library",
"//plugin/pkg/admission/serviceaccount:go_default_library",
"//staging/src/k8s.io/api/apps/v1:go_default_library",
"//staging/src/k8s.io/api/auditregistration/v1alpha1:go_default_library",
"//staging/src/k8s.io/api/authentication/v1:go_default_library",
"//staging/src/k8s.io/api/batch/v1:go_default_library",
"//staging/src/k8s.io/api/certificates/v1beta1:go_default_library",
@ -39,6 +41,7 @@ go_library(
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",

383
test/e2e/auth/audit_dynamic.go Normal file
View File

@ -0,0 +1,383 @@
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package auth
import (
"fmt"
"strings"
"time"
. "github.com/onsi/ginkgo"
auditregv1alpha1 "k8s.io/api/auditregistration/v1alpha1"
apiv1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/apimachinery/pkg/util/wait"
auditinternal "k8s.io/apiserver/pkg/apis/audit"
auditv1 "k8s.io/apiserver/pkg/apis/audit/v1"
clientset "k8s.io/client-go/kubernetes"
restclient "k8s.io/client-go/rest"
"k8s.io/kubernetes/test/e2e/framework"
"k8s.io/kubernetes/test/utils"
imageutils "k8s.io/kubernetes/test/utils/image"
)
var _ = SIGDescribe("[Feature:DynamicAudit]", func() {
f := framework.NewDefaultFramework("audit")
BeforeEach(func() {
framework.SkipUnlessProviderIs("gce")
})
// TODO: Get rid of [DisabledForLargeClusters] when feature request #53455 is ready.
It("should dynamically audit API calls [DisabledForLargeClusters]", func() {
namespace := f.Namespace.Name
By("Creating a kubernetes client that impersonates an unauthorized anonymous user")
config, err := framework.LoadConfig()
framework.ExpectNoError(err, "failed to fetch config")
config.Impersonate = restclient.ImpersonationConfig{
UserName: "system:anonymous",
Groups: []string{"system:unauthenticated"},
}
anonymousClient, err := clientset.NewForConfig(config)
framework.ExpectNoError(err, "failed to create the anonymous client")
_, err = f.ClientSet.CoreV1().Namespaces().Create(&apiv1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: "audit",
},
})
framework.ExpectNoError(err, "failed to create namespace")
_, err = f.ClientSet.CoreV1().Pods(namespace).Create(&apiv1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "audit-proxy",
Labels: map[string]string{
"app": "audit",
},
},
Spec: apiv1.PodSpec{
Containers: []apiv1.Container{
{
Name: "proxy",
Image: imageutils.GetE2EImage(imageutils.AuditProxy),
Ports: []apiv1.ContainerPort{
{
ContainerPort: 8080,
},
},
},
},
},
})
framework.ExpectNoError(err, "failed to create proxy pod")
_, err = f.ClientSet.CoreV1().Services(namespace).Create(&apiv1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "audit",
},
Spec: apiv1.ServiceSpec{
Ports: []apiv1.ServicePort{
{
Port: 80,
TargetPort: intstr.IntOrString{Type: intstr.Int, IntVal: 8080},
},
},
Selector: map[string]string{
"app": "audit",
},
},
})
framework.ExpectNoError(err, "failed to create proxy service")
config, err = framework.LoadConfig()
framework.ExpectNoError(err, "failed to load config")
var podIP string
// get pod ip
err = wait.Poll(100*time.Millisecond, 10*time.Second, func() (done bool, err error) {
p, err := f.ClientSet.CoreV1().Pods(namespace).Get("audit-proxy", metav1.GetOptions{})
if errors.IsNotFound(err) {
return false, nil
} else if err != nil {
return false, err
}
podIP = p.Status.PodIP
if podIP == "" {
return false, nil
}
return true, nil
})
framework.ExpectNoError(err, "could not look up pod ip")
podURL := fmt.Sprintf("http://%s:8080", podIP)
// create audit sink
sink := auditregv1alpha1.AuditSink{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
},
Spec: auditregv1alpha1.AuditSinkSpec{
Policy: auditregv1alpha1.Policy{
Level: auditregv1alpha1.LevelRequestResponse,
Stages: []auditregv1alpha1.Stage{
auditregv1alpha1.StageRequestReceived,
auditregv1alpha1.StageResponseStarted,
auditregv1alpha1.StageResponseComplete,
auditregv1alpha1.StagePanic,
},
},
Webhook: auditregv1alpha1.Webhook{
ClientConfig: auditregv1alpha1.WebhookClientConfig{
URL: &podURL,
},
},
},
}
_, err = f.ClientSet.AuditregistrationV1alpha1().AuditSinks().Create(&sink)
framework.ExpectNoError(err, "failed to create audit sink")
framework.Logf("created audit sink")
fmt.Println("created audit sink")
// check that we are receiving logs in the proxy
err = wait.Poll(100*time.Millisecond, 10*time.Second, func() (done bool, err error) {
logs, err := framework.GetPodLogs(f.ClientSet, namespace, "audit-proxy", "proxy")
if err != nil {
fmt.Println("could not get pod logs: ", err)
return false, nil
}
if logs == "" {
return false, nil
}
return true, nil
})
framework.ExpectNoError(err, "failed to get logs from sink")
auditTestUser = "kubernetes-admin"
testCases := []struct {
action func()
events []utils.AuditEvent
}{
// Create, get, update, patch, delete, list, watch pods.
// TODO: dedup this with the main audit test once policy functionality is available
// https://github.com/kubernetes/kubernetes/issues/70818
{
func() {
pod := &apiv1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "audit-pod",
},
Spec: apiv1.PodSpec{
Containers: []apiv1.Container{{
Name: "pause",
Image: imageutils.GetPauseImageName(),
}},
},
}
updatePod := func(pod *apiv1.Pod) {}
f.PodClient().CreateSync(pod)
_, err := f.PodClient().Get(pod.Name, metav1.GetOptions{})
framework.ExpectNoError(err, "failed to get audit-pod")
podChan, err := f.PodClient().Watch(watchOptions)
framework.ExpectNoError(err, "failed to create watch for pods")
for range podChan.ResultChan() {
}
f.PodClient().Update(pod.Name, updatePod)
_, err = f.PodClient().List(metav1.ListOptions{})
framework.ExpectNoError(err, "failed to list pods")
_, err = f.PodClient().Patch(pod.Name, types.JSONPatchType, patch)
framework.ExpectNoError(err, "failed to patch pod")
f.PodClient().DeleteSync(pod.Name, &metav1.DeleteOptions{}, framework.DefaultPodDeletionTimeout)
},
[]utils.AuditEvent{
{
Level: auditinternal.LevelRequestResponse,
Stage: auditinternal.StageResponseComplete,
RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/pods", namespace),
Verb: "create",
Code: 201,
User: auditTestUser,
Resource: "pods",
Namespace: namespace,
RequestObject: true,
ResponseObject: true,
AuthorizeDecision: "allow",
}, {
Level: auditinternal.LevelRequestResponse,
Stage: auditinternal.StageResponseComplete,
RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/pods/audit-pod", namespace),
Verb: "get",
Code: 200,
User: auditTestUser,
Resource: "pods",
Namespace: namespace,
RequestObject: false,
ResponseObject: true,
AuthorizeDecision: "allow",
}, {
Level: auditinternal.LevelRequestResponse,
Stage: auditinternal.StageResponseComplete,
RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/pods", namespace),
Verb: "list",
Code: 200,
User: auditTestUser,
Resource: "pods",
Namespace: namespace,
RequestObject: false,
ResponseObject: true,
AuthorizeDecision: "allow",
}, {
Level: auditinternal.LevelRequestResponse,
Stage: auditinternal.StageResponseStarted,
RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/pods?timeout=%ds&timeoutSeconds=%d&watch=true", namespace, watchTestTimeout, watchTestTimeout),
Verb: "watch",
Code: 200,
User: auditTestUser,
Resource: "pods",
Namespace: namespace,
RequestObject: false,
ResponseObject: false,
AuthorizeDecision: "allow",
}, {
Level: auditinternal.LevelRequestResponse,
Stage: auditinternal.StageResponseComplete,
RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/pods?timeout=%ds&timeoutSeconds=%d&watch=true", namespace, watchTestTimeout, watchTestTimeout),
Verb: "watch",
Code: 200,
User: auditTestUser,
Resource: "pods",
Namespace: namespace,
RequestObject: false,
ResponseObject: false,
AuthorizeDecision: "allow",
}, {
Level: auditinternal.LevelRequestResponse,
Stage: auditinternal.StageResponseComplete,
RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/pods/audit-pod", namespace),
Verb: "update",
Code: 200,
User: auditTestUser,
Resource: "pods",
Namespace: namespace,
RequestObject: true,
ResponseObject: true,
AuthorizeDecision: "allow",
}, {
Level: auditinternal.LevelRequestResponse,
Stage: auditinternal.StageResponseComplete,
RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/pods/audit-pod", namespace),
Verb: "patch",
Code: 200,
User: auditTestUser,
Resource: "pods",
Namespace: namespace,
RequestObject: true,
ResponseObject: true,
AuthorizeDecision: "allow",
}, {
Level: auditinternal.LevelRequestResponse,
Stage: auditinternal.StageResponseComplete,
RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/pods/audit-pod", namespace),
Verb: "delete",
Code: 200,
User: auditTestUser,
Resource: "pods",
Namespace: namespace,
RequestObject: true,
ResponseObject: true,
AuthorizeDecision: "allow",
},
},
},
}
// test authorizer annotations, RBAC is required.
annotationTestCases := []struct {
action func()
events []utils.AuditEvent
}{
// get a pod with unauthorized user
{
func() {
_, err := anonymousClient.CoreV1().Pods(namespace).Get("another-audit-pod", metav1.GetOptions{})
expectForbidden(err)
},
[]utils.AuditEvent{
{
Level: auditinternal.LevelRequestResponse,
Stage: auditinternal.StageResponseComplete,
RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/pods/another-audit-pod", namespace),
Verb: "get",
Code: 403,
User: auditTestUser,
ImpersonatedUser: "system:anonymous",
ImpersonatedGroups: "system:unauthenticated",
Resource: "pods",
Namespace: namespace,
RequestObject: false,
ResponseObject: true,
AuthorizeDecision: "forbid",
},
},
},
}
if framework.IsRBACEnabled(f) {
testCases = append(testCases, annotationTestCases...)
}
expectedEvents := []utils.AuditEvent{}
for _, t := range testCases {
t.action()
expectedEvents = append(expectedEvents, t.events...)
}
// The default flush timeout is 30 seconds, therefore it should be enough to retry once
// to find all expected events. However, we're waiting for 5 minutes to avoid flakes.
pollingInterval := 30 * time.Second
pollingTimeout := 5 * time.Minute
err = wait.Poll(pollingInterval, pollingTimeout, func() (bool, error) {
// Fetch the logs
logs, err := framework.GetPodLogs(f.ClientSet, namespace, "audit-proxy", "proxy")
if err != nil {
return false, err
}
reader := strings.NewReader(logs)
missing, err := utils.CheckAuditLines(reader, expectedEvents, auditv1.SchemeGroupVersion)
if err != nil {
framework.Logf("Failed to observe audit events: %v", err)
} else if len(missing) > 0 {
framework.Logf("Events %#v not found!", missing)
}
return len(missing) == 0, nil
})
framework.ExpectNoError(err, "after %v failed to observe audit events", pollingTimeout)
err = f.ClientSet.AuditregistrationV1alpha1().AuditSinks().Delete("test", &metav1.DeleteOptions{})
framework.ExpectNoError(err, "could not delete audit configuration")
})
})

1
test/e2e/common/util.go
View File

@ -48,6 +48,7 @@ var CurrentSuite Suite
// only used by node e2e test.
// TODO(random-liu): Change the image puller pod to use similar mechanism.
var CommonImageWhiteList = sets.NewString(
imageutils.GetE2EImage(imageutils.AuditProxy),
imageutils.GetE2EImage(imageutils.BusyBox),
imageutils.GetE2EImage(imageutils.EntrypointTester),
imageutils.GetE2EImage(imageutils.IpcUtils),

3
test/utils/image/manifest.go
View File

@ -105,6 +105,8 @@ const (
APIServer
// AppArmorLoader image
AppArmorLoader
// AuditProxy image
AuditProxy
// BusyBox image
BusyBox
// CheckMetadataConcealment image
@ -196,6 +198,7 @@ func initImageConfigs() map[int]Config {
configs[AdmissionWebhook] = Config{e2eRegistry, "webhook", "1.14v1"}
configs[APIServer] = Config{e2eRegistry, "sample-apiserver", "1.10"}
configs[AppArmorLoader] = Config{e2eRegistry, "apparmor-loader", "1.0"}
configs[AuditProxy] = Config{e2eRegistry, "audit-proxy", "1.0"}
configs[BusyBox] = Config{dockerLibraryRegistry, "busybox", "1.29"}
configs[CheckMetadataConcealment] = Config{e2eRegistry, "metadata-concealment", "1.2"}
configs[CudaVectorAdd] = Config{e2eRegistry, "cuda-vector-add", "1.0"}