Merge pull request #27087 from soltysh/audit_log

Automatic merge from submit-queue

Basic audit log

Fixes #2203 by introducing simple audit logging, including the information about impersonation. We currently have something identical in openshift, but I'm open to any suggestions. Sample logs look like that:

as `<self>`:
```
AUDIT: id="75114bb5-970a-47d5-a5f1-1e99cea0574c" ip="127.0.0.1" method="GET" user="test-admin" as="<self>" namespace="openshift" uri="/api/v1/namespaces/openshift/pods/python"
AUDIT: id="75114bb5-970a-47d5-a5f1-1e99cea0574c" response=200
```

as user:
```
AUDIT: id="b0a443ae-f7d8-408c-a355-eb9501fd5c59" ip="192.168.121.118" method="GET" user="system:admin" as="test-admin" namespace="openshift" uri="/api/v1/namespaces/openshift/pods/python"
AUDIT: id="b0a443ae-f7d8-408c-a355-eb9501fd5c59" response=200
```

```release-note
* Add basic audit logging
```

@ericchiang @smarterclayton @roberthbailey @erictune @ghodss 

[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/.github/PULL_REQUEST_TEMPLATE.md?pixel)]()

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.kubernetes.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.kubernetes.io/reviews/kubernetes/kubernetes/27087)
<!-- Reviewable:end -->

hack/.linted_packages

@@ -64,6 +64,7 @@ pkg/apis/autoscaling/install
 pkg/apis/batch/install
 pkg/apis/certificates/install
 pkg/apis/componentconfig/install
+pkg/apiserver/audit
 pkg/api/service
 pkg/apis/extensions/install
 pkg/apis/extensions/v1beta1

hack/verify-flags/known-flags.txt

@@ -16,6 +16,10 @@ api-servers
 api-token
 api-version
 apiserver-count
+audit-log-maxage
+audit-log-maxbackup
+audit-log-maxsize
+audit-log-path
 auth-path
 auth-provider
 auth-provider-arg

pkg/apiserver/audit/audit.go

@@ -0,0 +1,114 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package audit
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"time"
+
+	"github.com/pborman/uuid"
+
+	"k8s.io/kubernetes/pkg/api"
+	utilnet "k8s.io/kubernetes/pkg/util/net"
+)
+
+var _ http.ResponseWriter = &auditResponseWriter{}
+
+type auditResponseWriter struct {
+	http.ResponseWriter
+	out io.Writer
+	id  string
+}
+
+func (a *auditResponseWriter) WriteHeader(code int) {
+	fmt.Fprintf(a.out, "%s AUDIT: id=%q response=\"%d\"\n", time.Now().Format(time.RFC3339Nano), a.id, code)
+	a.ResponseWriter.WriteHeader(code)
+}
+
+// fancyResponseWriterDelegator implements http.CloseNotifier, http.Flusher and
+// http.Hijacker which are needed to make certain http operation (eg. watch, rsh, etc)
+// working.
+type fancyResponseWriterDelegator struct {
+	*auditResponseWriter
+}
+
+func (f *fancyResponseWriterDelegator) CloseNotify() <-chan bool {
+	return f.ResponseWriter.(http.CloseNotifier).CloseNotify()
+}
+
+func (f *fancyResponseWriterDelegator) Flush() {
+	f.ResponseWriter.(http.Flusher).Flush()
+}
+
+func (f *fancyResponseWriterDelegator) Hijack() (net.Conn, *bufio.ReadWriter, error) {
+	return f.ResponseWriter.(http.Hijacker).Hijack()
+}
+
+var _ http.CloseNotifier = &fancyResponseWriterDelegator{}
+var _ http.Flusher = &fancyResponseWriterDelegator{}
+var _ http.Hijacker = &fancyResponseWriterDelegator{}
+
+// WithAudit decorates a http.Handler with audit logging information for all the
+// requests coming to the server. Each audit log contains two entries:
+// 1. the request line containing:
+//    - unique id allowing to match the response line (see 2)
+//    - source ip of the request
+//    - HTTP method being invoked
+//    - original user invoking the operation
+//    - impersonated user for the operation
+//    - namespace of the request or <none>
+//    - uri is the full URI as requested
+// 2. the response line containing:
+//    - the unique id from 1
+//    - response code
+func WithAudit(handler http.Handler, requestContextMapper api.RequestContextMapper, out io.Writer) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		ctx, _ := requestContextMapper.Get(req)
+		user, _ := api.UserFrom(ctx)
+		asuser := req.Header.Get("Impersonate-User")
+		if len(asuser) == 0 {
+			asuser = "<self>"
+		}
+		namespace := api.NamespaceValue(ctx)
+		if len(namespace) == 0 {
+			namespace = "<none>"
+		}
+		id := uuid.NewRandom().String()
+
+		fmt.Fprintf(out, "%s AUDIT: id=%q ip=%q method=%q user=%q as=%q namespace=%q uri=%q\n",
+			time.Now().Format(time.RFC3339Nano), id, utilnet.GetClientIP(req), req.Method, user.GetName(), asuser, namespace, req.URL)
+		respWriter := decorateResponseWriter(w, out, id)
+		handler.ServeHTTP(respWriter, req)
+	})
+}
+
+func decorateResponseWriter(responseWriter http.ResponseWriter, out io.Writer, id string) http.ResponseWriter {
+	delegate := &auditResponseWriter{ResponseWriter: responseWriter, out: out, id: id}
+	// check if the ResponseWriter we're wrapping is the fancy one we need
+	// or if the basic is sufficient
+	_, cn := responseWriter.(http.CloseNotifier)
+	_, fl := responseWriter.(http.Flusher)
+	_, hj := responseWriter.(http.Hijacker)
+	if cn && fl && hj {
+		return &fancyResponseWriterDelegator{delegate}
+	}
+	return delegate
+}

pkg/apiserver/audit/audit_test.go

@@ -0,0 +1,58 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package audit
+
+import (
+	"bufio"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"reflect"
+	"testing"
+)
+
+type simpleResponseWriter struct {
+	http.ResponseWriter
+}
+
+func (*simpleResponseWriter) WriteHeader(code int) {}
+
+type fancyResponseWriter struct {
+	simpleResponseWriter
+}
+
+func (*fancyResponseWriter) CloseNotify() <-chan bool { return nil }
+
+func (*fancyResponseWriter) Flush() {}
+
+func (*fancyResponseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) { return nil, nil, nil }
+
+func TestConstructResponseWriter(t *testing.T) {
+	actual := decorateResponseWriter(&simpleResponseWriter{}, ioutil.Discard, "")
+	switch v := actual.(type) {
+	case *auditResponseWriter:
+	default:
+		t.Errorf("Expected auditResponseWriter, got %v", reflect.TypeOf(v))
+	}
+
+	actual = decorateResponseWriter(&fancyResponseWriter{}, ioutil.Discard, "")
+	switch v := actual.(type) {
+	case *fancyResponseWriterDelegator:
+	default:
+		t.Errorf("Expected fancyResponseWriterDelegator, got %v", reflect.TypeOf(v))
+	}
+}

pkg/genericapiserver/genericapiserver.go

@@ -30,6 +30,12 @@ import (
 	"strings"
 	"time"
 
+	systemd "github.com/coreos/go-systemd/daemon"
+	"github.com/emicklei/go-restful"
+	"github.com/emicklei/go-restful/swagger"
+	"github.com/golang/glog"
+	"gopkg.in/natefinch/lumberjack.v2"
+
 	"k8s.io/kubernetes/pkg/admission"
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/api/rest"
@@ -37,6 +43,7 @@ import (
 	"k8s.io/kubernetes/pkg/apimachinery"
 	"k8s.io/kubernetes/pkg/apimachinery/registered"
 	"k8s.io/kubernetes/pkg/apiserver"
+	"k8s.io/kubernetes/pkg/apiserver/audit"
 	"k8s.io/kubernetes/pkg/auth/authenticator"
 	"k8s.io/kubernetes/pkg/auth/authorizer"
 	"k8s.io/kubernetes/pkg/auth/handlers"
@@ -54,11 +61,6 @@ import (
 	utilnet "k8s.io/kubernetes/pkg/util/net"
 	utilruntime "k8s.io/kubernetes/pkg/util/runtime"
 	"k8s.io/kubernetes/pkg/util/sets"
-
-	systemd "github.com/coreos/go-systemd/daemon"
-	"github.com/emicklei/go-restful"
-	"github.com/emicklei/go-restful/swagger"
-	"github.com/golang/glog"
 )
 
 const globalTimeout = time.Minute
@@ -96,7 +98,11 @@ type APIGroupInfo struct {
 // Config is a structure used to configure a GenericAPIServer.
 type Config struct {
 	// The storage factory for other objects
-	StorageFactory StorageFactory
+	StorageFactory     StorageFactory
+	AuditLogPath       string
+	AuditLogMaxAge     int
+	AuditLogMaxBackups int
+	AuditLogMaxSize    int
 	// allow downstream consumers to disable the core controller loops
 	EnableLogsSupport bool
 	EnableUISupport   bool
@@ -475,6 +481,17 @@ func (s *GenericAPIServer) init(c *Config) {
 
 	attributeGetter := apiserver.NewRequestAttributeGetter(s.RequestContextMapper, s.NewRequestInfoResolver())
 	handler = apiserver.WithAuthorizationCheck(handler, attributeGetter, s.authorizer)
+	if len(c.AuditLogPath) != 0 {
+		// audit handler must comes before the impersonationFilter to read the original user
+		writer := &lumberjack.Logger{
+			Filename:   c.AuditLogPath,
+			MaxAge:     c.AuditLogMaxAge,
+			MaxBackups: c.AuditLogMaxBackups,
+			MaxSize:    c.AuditLogMaxSize,
+		}
+		handler = audit.WithAudit(handler, s.RequestContextMapper, writer)
+		defer writer.Close()
+	}
 	handler = apiserver.WithImpersonation(handler, s.RequestContextMapper, s.authorizer)
 
 	// Install Authenticator
@@ -542,6 +559,10 @@ func NewConfig(options *options.ServerRunOptions) *Config {
 		APIGroupPrefix:            options.APIGroupPrefix,
 		APIPrefix:                 options.APIPrefix,
 		CorsAllowedOriginList:     options.CorsAllowedOriginList,
+		AuditLogPath:              options.AuditLogPath,
+		AuditLogMaxAge:            options.AuditLogMaxAge,
+		AuditLogMaxBackups:        options.AuditLogMaxBackups,
+		AuditLogMaxSize:           options.AuditLogMaxSize,
 		EnableIndex:               true,
 		EnableLogsSupport:         options.EnableLogsSupport,
 		EnableProfiling:           options.EnableProfiling,

pkg/genericapiserver/options/server_run_options.go

@@ -67,6 +67,10 @@ type ServerRunOptions struct {
 	DeleteCollectionWorkers    int
 	// Used to specify the storage version that should be used for the legacy v1 api group.
 	DeprecatedStorageVersion  string
+	AuditLogPath              string
+	AuditLogMaxAge            int
+	AuditLogMaxBackups        int
+	AuditLogMaxSize           int
 	EnableLogsSupport         bool
 	EnableProfiling           bool
 	EnableSwaggerUI           bool
@@ -294,6 +298,15 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) {
 	fs.IntVar(&s.DeleteCollectionWorkers, "delete-collection-workers", s.DeleteCollectionWorkers,
 		"Number of workers spawned for DeleteCollection call. These are used to speed up namespace cleanup.")
 
+	fs.StringVar(&s.AuditLogPath, "audit-log-path", s.AuditLogPath,
+		"If set, all requests coming to the apiserver will be logged to this file.")
+	fs.IntVar(&s.AuditLogMaxAge, "audit-log-maxage", s.AuditLogMaxBackups,
+		"The maximum number of days to retain old audit log files based on the timestamp encoded in their filename.")
+	fs.IntVar(&s.AuditLogMaxBackups, "audit-log-maxbackup", s.AuditLogMaxBackups,
+		"The maximum number of old audit log files to retain.")
+	fs.IntVar(&s.AuditLogMaxSize, "audit-log-maxsize", s.AuditLogMaxSize,
+		"The maximum size in megabytes of the audit log file before it gets rotated. Defaults to 100MB.")
+
 	fs.BoolVar(&s.EnableProfiling, "profiling", s.EnableProfiling,
 		"Enable profiling via web interface host:port/debug/pprof/")