From 303c6ce6386ba029cc1692dde727d8a727833e00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Bauer?= Date: Sun, 18 Nov 2018 11:37:54 +0100 Subject: [PATCH 1/7] - updated fluentd to 1.3.0 - added filter_parse_json_field to restore removed functionality of kubernetes_metadata (https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter/issues/145) - updated all gems to newest version --- .../fluentd-elasticsearch/fluentd-es-configmap.yaml | 7 ++++++- .../addons/fluentd-elasticsearch/fluentd-es-ds.yaml | 12 ++++++------ .../fluentd-es-image/Dockerfile | 4 +++- .../fluentd-elasticsearch/fluentd-es-image/Gemfile | 10 +++++----- .../fluentd-elasticsearch/fluentd-es-image/Makefile | 2 +- 5 files changed, 21 insertions(+), 14 deletions(-) diff --git a/cluster/addons/fluentd-elasticsearch/fluentd-es-configmap.yaml b/cluster/addons/fluentd-elasticsearch/fluentd-es-configmap.yaml index db3f5af254..74bb65b5a1 100644 --- a/cluster/addons/fluentd-elasticsearch/fluentd-es-configmap.yaml +++ b/cluster/addons/fluentd-elasticsearch/fluentd-es-configmap.yaml @@ -1,7 +1,7 @@ kind: ConfigMap apiVersion: v1 metadata: - name: fluentd-es-config-v0.1.6 + name: fluentd-es-config-v0.1.7 namespace: kube-system labels: addonmanager.kubernetes.io/mode: Reconcile @@ -420,6 +420,11 @@ data: @type kubernetes_metadata + # Adds the json parsing which was removed in kubernetes_metadata filter > 2.1.x + + @type parse_json_field + + # Concatenate multi-line logs @type concat diff --git a/cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml b/cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml index d804801b66..dbf9b92081 100644 --- a/cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml +++ b/cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml @@ -48,24 +48,24 @@ roleRef: apiVersion: apps/v1 kind: DaemonSet metadata: - name: fluentd-es-v2.2.1 + name: fluentd-es-v2.4.0 namespace: kube-system labels: k8s-app: fluentd-es - version: v2.2.1 + version: v2.4.0 kubernetes.io/cluster-service: "true" addonmanager.kubernetes.io/mode: Reconcile spec: selector: matchLabels: k8s-app: fluentd-es - version: v2.2.1 + version: v2.4.0 template: metadata: labels: k8s-app: fluentd-es kubernetes.io/cluster-service: "true" - version: v2.2.1 + version: v2.4.0 # This annotation ensures that fluentd does not get evicted if the node # supports critical pod annotation based priority scheme. # Note that this does not guarantee admission on the nodes (#40573). @@ -77,7 +77,7 @@ spec: serviceAccountName: fluentd-es containers: - name: fluentd-es - image: k8s.gcr.io/fluentd-elasticsearch:v2.2.0 + image: k8s.gcr.io/fluentd-elasticsearch:v2.4.0 env: - name: FLUENTD_ARGS value: --no-supervisor -q @@ -107,4 +107,4 @@ spec: path: /var/lib/docker/containers - name: config-volume configMap: - name: fluentd-es-config-v0.1.6 + name: fluentd-es-config-v0.1.7 diff --git a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Dockerfile b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Dockerfile index 7f84870bd5..408e496282 100644 --- a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Dockerfile +++ b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Dockerfile @@ -31,13 +31,15 @@ COPY Gemfile /Gemfile # 2. Install fluentd via ruby. # 3. Remove build dependencies. # 4. Cleanup leftover caches & files. -RUN BUILD_DEPS="make gcc g++ libc6-dev ruby-dev libffi-dev" \ +RUN BUILD_DEPS="make gcc g++ libc6-dev ruby-dev libffi-dev curl" \ && clean-install $BUILD_DEPS \ ca-certificates \ libjemalloc1 \ ruby \ && echo 'gem: --no-document' >> /etc/gemrc \ && gem install --file Gemfile \ + && mkdir -p /etc/fluent/plugin \ + && curl -s https://raw.githubusercontent.com/openshift/origin-aggregated-logging/master/fluentd/lib/filter_parse_json_field/lib/filter_parse_json_field.rb -o /etc/fluent/plugin/filter_parse_json_field.rb \ && apt-get purge -y --auto-remove \ -o APT::AutoRemove::RecommendsImportant=false \ $BUILD_DEPS \ diff --git a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Gemfile b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Gemfile index 145b15fc73..1f8ddff200 100644 --- a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Gemfile +++ b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Gemfile @@ -1,12 +1,12 @@ source 'https://rubygems.org' -gem 'fluentd', '<=1.2.4' gem 'activesupport', '~>5.2.1' +gem 'fluentd', '<=1.3.0' gem 'fluent-plugin-concat', '~>2.3.0' gem 'fluent-plugin-detect-exceptions', '~>0.0.11' -gem 'fluent-plugin-elasticsearch', '~>2.11.5' -gem 'fluent-plugin-kubernetes_metadata_filter', '~>2.0.0' +gem 'fluent-plugin-elasticsearch', '~>2.12.0' +gem 'fluent-plugin-kubernetes_metadata_filter', '~>2.1.4' gem 'fluent-plugin-multi-format-parser', '~>1.0.0' -gem 'fluent-plugin-prometheus', '~>1.0.1' +gem 'fluent-plugin-prometheus', '~>1.2.1' gem 'fluent-plugin-systemd', '~>1.0.1' -gem 'oj', '~>3.6.5' +gem 'oj', '~>3.7.1' diff --git a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Makefile b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Makefile index 064c5260dd..b3b8c4c896 100644 --- a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Makefile +++ b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Makefile @@ -16,7 +16,7 @@ PREFIX = staging-k8s.gcr.io IMAGE = fluentd-elasticsearch -TAG = v2.3.1 +TAG = v2.4.0 build: docker build --pull -t $(PREFIX)/$(IMAGE):$(TAG) . From 14b208db26e0179dd9a7eb56a4ffe71d222de36b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Bauer?= Date: Mon, 19 Nov 2018 19:30:21 +0100 Subject: [PATCH 2/7] prepared elasticsearch output plugin type_name for es 7.0.x --- cluster/addons/fluentd-elasticsearch/fluentd-es-configmap.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster/addons/fluentd-elasticsearch/fluentd-es-configmap.yaml b/cluster/addons/fluentd-elasticsearch/fluentd-es-configmap.yaml index 74bb65b5a1..f035b34914 100644 --- a/cluster/addons/fluentd-elasticsearch/fluentd-es-configmap.yaml +++ b/cluster/addons/fluentd-elasticsearch/fluentd-es-configmap.yaml @@ -437,7 +437,7 @@ data: @id elasticsearch @type elasticsearch @log_level info - type_name fluentd + type_name _doc include_tag_key true host elasticsearch-logging port 9200 From dd50d00c51099e2010ff81a0cf7554be6a41400d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Bauer?= Date: Mon, 17 Dec 2018 23:44:26 +0100 Subject: [PATCH 3/7] updated fluentd to 1.3.2 --- .../fluentd-elasticsearch/fluentd-es-image/Gemfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Gemfile b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Gemfile index 1f8ddff200..ca1454d74a 100644 --- a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Gemfile +++ b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Gemfile @@ -1,12 +1,12 @@ source 'https://rubygems.org' -gem 'activesupport', '~>5.2.1' -gem 'fluentd', '<=1.3.0' +gem 'activesupport', '~>5.2.2' +gem 'fluentd', '<=1.3.2' gem 'fluent-plugin-concat', '~>2.3.0' gem 'fluent-plugin-detect-exceptions', '~>0.0.11' -gem 'fluent-plugin-elasticsearch', '~>2.12.0' -gem 'fluent-plugin-kubernetes_metadata_filter', '~>2.1.4' +gem 'fluent-plugin-elasticsearch', '~>2.12.5' +gem 'fluent-plugin-kubernetes_metadata_filter', '~>2.1.6' gem 'fluent-plugin-multi-format-parser', '~>1.0.0' gem 'fluent-plugin-prometheus', '~>1.2.1' gem 'fluent-plugin-systemd', '~>1.0.1' -gem 'oj', '~>3.7.1' +gem 'oj', '~>3.7.4' From c33b5b60d19636449d731ee1efb94d26f3841ffd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Bauer?= Date: Fri, 21 Dec 2018 00:07:56 +0100 Subject: [PATCH 4/7] removed fabric8 json plugin & replaced with fluetnds build in filter_parser MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: André Bauer --- .../fluentd-es-configmap.yaml | 59 +++++++++++++------ .../fluentd-elasticsearch/fluentd-es-ds.yaml | 2 +- .../fluentd-es-image/Dockerfile | 1 - 3 files changed, 41 insertions(+), 21 deletions(-) diff --git a/cluster/addons/fluentd-elasticsearch/fluentd-es-configmap.yaml b/cluster/addons/fluentd-elasticsearch/fluentd-es-configmap.yaml index 656c92d384..d6b74199a8 100644 --- a/cluster/addons/fluentd-elasticsearch/fluentd-es-configmap.yaml +++ b/cluster/addons/fluentd-elasticsearch/fluentd-es-configmap.yaml @@ -1,7 +1,7 @@ kind: ConfigMap apiVersion: v1 metadata: - name: fluentd-es-config-v0.1.7 + name: fluentd-es-config-v0.2.0 namespace: kube-system labels: addonmanager.kubernetes.io/mode: Reconcile @@ -143,6 +143,39 @@ data: max_lines 1000 + # Concatenate multi-line logs + + @id filter_concat + @type concat + key message + multiline_end_regexp /\n$/ + separator "" + + + # Enriches records with Kubernetes metadata + + @id filter_kubernetes_metadata + @type kubernetes_metadata + + + # Fixes json fields in Elasticsearch + + @id filter_parser + @type parser + key_name log + reserve_data true + remove_key_name_field true + + @type multi_format + + format json + + + format none + + + + system.input.conf: |- # Example: # 2015-12-21 23:17:22,066 [salt.state ][INFO ] Completed state [net.ipv4.ip_forward] at time 23:17:22.066081 @@ -376,6 +409,7 @@ data: forward.input.conf: |- # Takes the messages sent over TCP + @id forward @type forward @@ -383,15 +417,18 @@ data: # Prometheus Exporter Plugin # input plugin that exports metrics + @id prometheus @type prometheus + @id monitor_agent @type monitor_agent # input plugin that collects metrics from MonitorAgent + @id prometheus_monitor @type prometheus_monitor host ${hostname} @@ -400,6 +437,7 @@ data: # input plugin that collects metrics for output plugin + @id prometheus_output_monitor @type prometheus_output_monitor host ${hostname} @@ -408,6 +446,7 @@ data: # input plugin that collects metrics for in_tail plugin + @id prometheus_tail_monitor @type prometheus_tail_monitor host ${hostname} @@ -415,24 +454,6 @@ data: output.conf: |- - # Enriches records with Kubernetes metadata - - @type kubernetes_metadata - - - # Adds the json parsing which was removed in kubernetes_metadata filter > 2.1.x - - @type parse_json_field - - - # Concatenate multi-line logs - - @type concat - key message - multiline_end_regexp /\n$/ - separator "" - - @id elasticsearch @type elasticsearch diff --git a/cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml b/cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml index dbf9b92081..c3df2ac253 100644 --- a/cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml +++ b/cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml @@ -107,4 +107,4 @@ spec: path: /var/lib/docker/containers - name: config-volume configMap: - name: fluentd-es-config-v0.1.7 + name: fluentd-es-config-v0.2.0 diff --git a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Dockerfile b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Dockerfile index 408e496282..650966a852 100644 --- a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Dockerfile +++ b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Dockerfile @@ -39,7 +39,6 @@ RUN BUILD_DEPS="make gcc g++ libc6-dev ruby-dev libffi-dev curl" \ && echo 'gem: --no-document' >> /etc/gemrc \ && gem install --file Gemfile \ && mkdir -p /etc/fluent/plugin \ - && curl -s https://raw.githubusercontent.com/openshift/origin-aggregated-logging/master/fluentd/lib/filter_parse_json_field/lib/filter_parse_json_field.rb -o /etc/fluent/plugin/filter_parse_json_field.rb \ && apt-get purge -y --auto-remove \ -o APT::AutoRemove::RecommendsImportant=false \ $BUILD_DEPS \ From 7b2651429a0af8545727db3cb694e699c873c560 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Bauer?= Date: Fri, 21 Dec 2018 00:20:49 +0100 Subject: [PATCH 5/7] removed curl instalation as its not needed anymore --- .../addons/fluentd-elasticsearch/fluentd-es-image/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Dockerfile b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Dockerfile index 650966a852..b6c58f00c8 100644 --- a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Dockerfile +++ b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Dockerfile @@ -31,7 +31,7 @@ COPY Gemfile /Gemfile # 2. Install fluentd via ruby. # 3. Remove build dependencies. # 4. Cleanup leftover caches & files. -RUN BUILD_DEPS="make gcc g++ libc6-dev ruby-dev libffi-dev curl" \ +RUN BUILD_DEPS="make gcc g++ libc6-dev ruby-dev libffi-dev" \ && clean-install $BUILD_DEPS \ ca-certificates \ libjemalloc1 \ From dea164fda86bbe5398d9e102b59d1eec7b654078 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Bauer?= Date: Fri, 21 Dec 2018 00:21:21 +0100 Subject: [PATCH 6/7] removed plugin dir creation as no plugin is installed --- cluster/addons/fluentd-elasticsearch/fluentd-es-image/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Dockerfile b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Dockerfile index b6c58f00c8..7f84870bd5 100644 --- a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Dockerfile +++ b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Dockerfile @@ -38,7 +38,6 @@ RUN BUILD_DEPS="make gcc g++ libc6-dev ruby-dev libffi-dev" \ ruby \ && echo 'gem: --no-document' >> /etc/gemrc \ && gem install --file Gemfile \ - && mkdir -p /etc/fluent/plugin \ && apt-get purge -y --auto-remove \ -o APT::AutoRemove::RecommendsImportant=false \ $BUILD_DEPS \ From 273fa998f7699c9d12cdaf1417d66e61770aacd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Bauer?= Date: Tue, 8 Jan 2019 20:53:26 +0100 Subject: [PATCH 7/7] updated fluentd to 1.3.3 & gem updates --- .../addons/fluentd-elasticsearch/fluentd-es-image/Gemfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Gemfile b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Gemfile index ca1454d74a..9bcdff4cb6 100644 --- a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Gemfile +++ b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Gemfile @@ -1,12 +1,12 @@ source 'https://rubygems.org' gem 'activesupport', '~>5.2.2' -gem 'fluentd', '<=1.3.2' +gem 'fluentd', '<=1.3.3' gem 'fluent-plugin-concat', '~>2.3.0' gem 'fluent-plugin-detect-exceptions', '~>0.0.11' -gem 'fluent-plugin-elasticsearch', '~>2.12.5' +gem 'fluent-plugin-elasticsearch', '~>3.0.1' gem 'fluent-plugin-kubernetes_metadata_filter', '~>2.1.6' gem 'fluent-plugin-multi-format-parser', '~>1.0.0' -gem 'fluent-plugin-prometheus', '~>1.2.1' +gem 'fluent-plugin-prometheus', '~>1.3.0' gem 'fluent-plugin-systemd', '~>1.0.1' -gem 'oj', '~>3.7.4' +gem 'oj', '~>3.7.6'