Merge pull request #22091 from a-robinson/multiline

Fix parsing of kube logs to handle logs split across lines.

cluster/addons/fluentd-elasticsearch/fluentd-es-image/td-agent.conf

@@ -162,11 +162,17 @@
   tag etcd
 </source>
 
+# Multi-line parsing is required for all the kube logs because very large log
+# statements, such as those that include entire object bodies, get split into
+# multiple lines by glog.
+
 # Example:
 # I0204 07:32:30.020537    3368 server.go:1048] POST /stats/container/: (13.972191ms) 200 [[Go-http-client/1.1] 10.244.1.3:40537]
 <source>
   type tail
-  format /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)$/
+  format multiline
+  format_firstline /^\w\d{4}/
+  format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
   time_format %m%d %H:%M:%S.%N
   path /var/log/kubelet.log
   pos_file /var/log/es-kubelet.log.pos
@@ -177,7 +183,9 @@
 # I0204 07:00:19.604280       5 handlers.go:131] GET /api/v1/nodes: (1.624207ms) 200 [[kube-controller-manager/v1.1.3 (linux/amd64) kubernetes/6a81b50] 127.0.0.1:38266]
 <source>
   type tail
-  format /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)$/
+  format multiline
+  format_firstline /^\w\d{4}/
+  format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
   time_format %m%d %H:%M:%S.%N
   path /var/log/kube-apiserver.log
   pos_file /var/log/es-kube-apiserver.log.pos
@@ -188,7 +196,9 @@
 # I0204 06:55:31.872680       5 servicecontroller.go:277] LB already exists and doesn't need update for service kube-system/kube-ui
 <source>
   type tail
-  format /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)$/
+  format multiline
+  format_firstline /^\w\d{4}/
+  format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
   time_format %m%d %H:%M:%S.%N
   path /var/log/kube-controller-manager.log
   pos_file /var/log/es-kube-controller-manager.log.pos
@@ -199,7 +209,9 @@
 # W0204 06:49:18.239674       7 reflector.go:245] pkg/scheduler/factory/factory.go:193: watch of *api.Service ended with: 401: The event in requested index is outdated and cleared (the requested history has been cleared [2578313/2577886]) [2579312]
 <source>
   type tail
-  format /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)$/
+  format multiline
+  format_firstline /^\w\d{4}/
+  format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
   time_format %m%d %H:%M:%S.%N
   path /var/log/kube-scheduler.log
   pos_file /var/log/es-kube-scheduler.log.pos

cluster/addons/fluentd-gcp/fluentd-gcp-image/google-fluentd.conf

@@ -111,11 +111,17 @@
   tag etcd
 </source>
 
+# Multi-line parsing is required for all the kube logs because very large log
+# statements, such as those that include entire object bodies, get split into
+# multiple lines by glog.
+
 # Example:
 # I0204 07:32:30.020537    3368 server.go:1048] POST /stats/container/: (13.972191ms) 200 [[Go-http-client/1.1] 10.244.1.3:40537]
 <source>
   type tail
-  format /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)$/
+  format multiline
+  format_firstline /^\w\d{4}/
+  format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
   time_format %m%d %H:%M:%S.%N
   path /var/log/kubelet.log
   pos_file /var/log/gcp-kubelet.log.pos
@@ -126,7 +132,9 @@
 # I0204 07:00:19.604280       5 handlers.go:131] GET /api/v1/nodes: (1.624207ms) 200 [[kube-controller-manager/v1.1.3 (linux/amd64) kubernetes/6a81b50] 127.0.0.1:38266]
 <source>
   type tail
-  format /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)$/
+  format multiline
+  format_firstline /^\w\d{4}/
+  format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
   time_format %m%d %H:%M:%S.%N
   path /var/log/kube-apiserver.log
   pos_file /var/log/gcp-kube-apiserver.log.pos
@@ -137,7 +145,9 @@
 # I0204 06:55:31.872680       5 servicecontroller.go:277] LB already exists and doesn't need update for service kube-system/kube-ui
 <source>
   type tail
-  format /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)$/
+  format multiline
+  format_firstline /^\w\d{4}/
+  format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
   time_format %m%d %H:%M:%S.%N
   path /var/log/kube-controller-manager.log
   pos_file /var/log/gcp-kube-controller-manager.log.pos
@@ -148,7 +158,9 @@
 # W0204 06:49:18.239674       7 reflector.go:245] pkg/scheduler/factory/factory.go:193: watch of *api.Service ended with: 401: The event in requested index is outdated and cleared (the requested history has been cleared [2578313/2577886]) [2579312]
 <source>
   type tail
-  format /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)$/
+  format multiline
+  format_firstline /^\w\d{4}/
+  format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
   time_format %m%d %H:%M:%S.%N
   path /var/log/kube-scheduler.log
   pos_file /var/log/gcp-kube-scheduler.log.pos