diff --git a/cluster/addons/fluentd-elasticsearch/fluentd-es-configmap.yaml b/cluster/addons/fluentd-elasticsearch/fluentd-es-configmap.yaml index 9f71b174ce..d6b74199a8 100644 --- a/cluster/addons/fluentd-elasticsearch/fluentd-es-configmap.yaml +++ b/cluster/addons/fluentd-elasticsearch/fluentd-es-configmap.yaml @@ -1,7 +1,7 @@ kind: ConfigMap apiVersion: v1 metadata: - name: fluentd-es-config-v0.1.6 + name: fluentd-es-config-v0.2.0 namespace: kube-system labels: addonmanager.kubernetes.io/mode: Reconcile @@ -143,6 +143,39 @@ data: max_lines 1000 + # Concatenate multi-line logs + + @id filter_concat + @type concat + key message + multiline_end_regexp /\n$/ + separator "" + + + # Enriches records with Kubernetes metadata + + @id filter_kubernetes_metadata + @type kubernetes_metadata + + + # Fixes json fields in Elasticsearch + + @id filter_parser + @type parser + key_name log + reserve_data true + remove_key_name_field true + + @type multi_format + + format json + + + format none + + + + system.input.conf: |- # Example: # 2015-12-21 23:17:22,066 [salt.state ][INFO ] Completed state [net.ipv4.ip_forward] at time 23:17:22.066081 @@ -376,6 +409,7 @@ data: forward.input.conf: |- # Takes the messages sent over TCP + @id forward @type forward @@ -383,15 +417,18 @@ data: # Prometheus Exporter Plugin # input plugin that exports metrics + @id prometheus @type prometheus + @id monitor_agent @type monitor_agent # input plugin that collects metrics from MonitorAgent + @id prometheus_monitor @type prometheus_monitor host ${hostname} @@ -400,6 +437,7 @@ data: # input plugin that collects metrics for output plugin + @id prometheus_output_monitor @type prometheus_output_monitor host ${hostname} @@ -408,6 +446,7 @@ data: # input plugin that collects metrics for in_tail plugin + @id prometheus_tail_monitor @type prometheus_tail_monitor host ${hostname} @@ -415,24 +454,11 @@ data: output.conf: |- - # Enriches records with Kubernetes metadata - - @type kubernetes_metadata - - - # Concatenate multi-line logs - - @type concat - key message - multiline_end_regexp /\n$/ - separator "" - - @id elasticsearch @type elasticsearch @log_level info - type_name fluentd + type_name _doc include_tag_key true host elasticsearch-logging port 9200 diff --git a/cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml b/cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml index d804801b66..c3df2ac253 100644 --- a/cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml +++ b/cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml @@ -48,24 +48,24 @@ roleRef: apiVersion: apps/v1 kind: DaemonSet metadata: - name: fluentd-es-v2.2.1 + name: fluentd-es-v2.4.0 namespace: kube-system labels: k8s-app: fluentd-es - version: v2.2.1 + version: v2.4.0 kubernetes.io/cluster-service: "true" addonmanager.kubernetes.io/mode: Reconcile spec: selector: matchLabels: k8s-app: fluentd-es - version: v2.2.1 + version: v2.4.0 template: metadata: labels: k8s-app: fluentd-es kubernetes.io/cluster-service: "true" - version: v2.2.1 + version: v2.4.0 # This annotation ensures that fluentd does not get evicted if the node # supports critical pod annotation based priority scheme. # Note that this does not guarantee admission on the nodes (#40573). @@ -77,7 +77,7 @@ spec: serviceAccountName: fluentd-es containers: - name: fluentd-es - image: k8s.gcr.io/fluentd-elasticsearch:v2.2.0 + image: k8s.gcr.io/fluentd-elasticsearch:v2.4.0 env: - name: FLUENTD_ARGS value: --no-supervisor -q @@ -107,4 +107,4 @@ spec: path: /var/lib/docker/containers - name: config-volume configMap: - name: fluentd-es-config-v0.1.6 + name: fluentd-es-config-v0.2.0 diff --git a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Gemfile b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Gemfile index 145b15fc73..9bcdff4cb6 100644 --- a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Gemfile +++ b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Gemfile @@ -1,12 +1,12 @@ source 'https://rubygems.org' -gem 'fluentd', '<=1.2.4' -gem 'activesupport', '~>5.2.1' +gem 'activesupport', '~>5.2.2' +gem 'fluentd', '<=1.3.3' gem 'fluent-plugin-concat', '~>2.3.0' gem 'fluent-plugin-detect-exceptions', '~>0.0.11' -gem 'fluent-plugin-elasticsearch', '~>2.11.5' -gem 'fluent-plugin-kubernetes_metadata_filter', '~>2.0.0' +gem 'fluent-plugin-elasticsearch', '~>3.0.1' +gem 'fluent-plugin-kubernetes_metadata_filter', '~>2.1.6' gem 'fluent-plugin-multi-format-parser', '~>1.0.0' -gem 'fluent-plugin-prometheus', '~>1.0.1' +gem 'fluent-plugin-prometheus', '~>1.3.0' gem 'fluent-plugin-systemd', '~>1.0.1' -gem 'oj', '~>3.6.5' +gem 'oj', '~>3.7.6' diff --git a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Makefile b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Makefile index 064c5260dd..b3b8c4c896 100644 --- a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Makefile +++ b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Makefile @@ -16,7 +16,7 @@ PREFIX = staging-k8s.gcr.io IMAGE = fluentd-elasticsearch -TAG = v2.3.1 +TAG = v2.4.0 build: docker build --pull -t $(PREFIX)/$(IMAGE):$(TAG) .