From 535a26412f18f65898efe3ef92694c3895e3604c Mon Sep 17 00:00:00 2001 From: andres-portainer <91705312+andres-portainer@users.noreply.github.com> Date: Thu, 20 Oct 2022 11:33:54 -0300 Subject: [PATCH] fix(logging): default to pretty logging [EE-4371] (#7847) * fix(logging): default to pretty logging EE-4371 * feat(app/logs): prettify stack traces in JSON logs * feat(nomad/logs): prettify JSON logs in log viewer * feat(kubernetes/logs): prettigy JSON logs in log viewers * feat(app/logs): format and color zerolog prettified logs * fix(app/logs): pre-parse logs when they are double serialized Co-authored-by: andres-portainer Co-authored-by: LP B --- api/cli/cli.go | 1 + api/cmd/portainer/log.go | 22 ++ api/cmd/portainer/main.go | 2 + api/portainer.go | 1 + .../components/log-viewer/logViewer.html | 2 +- .../log-viewer/logViewerController.js | 8 +- app/docker/helpers/logHelper.js | 223 --------------- app/docker/helpers/logHelper/colors/colors.ts | 63 +++++ app/docker/helpers/logHelper/colors/index.ts | 7 + .../helpers/logHelper/colors/rawColors.json | 258 ++++++++++++++++++ .../helpers/logHelper/concatLogsToString.ts | 15 + .../helpers/logHelper/formatJSONLogs.ts | 55 ++++ app/docker/helpers/logHelper/formatLogs.ts | 141 ++++++++++ .../helpers/logHelper/formatZerologLogs.ts | 119 ++++++++ app/docker/helpers/logHelper/formatters.ts | 154 +++++++++++ app/docker/helpers/logHelper/index.ts | 2 + app/docker/helpers/logHelper/types.ts | 53 ++++ app/docker/services/containerService.js | 5 +- app/docker/services/serviceService.js | 9 +- app/docker/services/taskService.js | 6 +- app/kubernetes/pod/service.js | 2 +- .../views/applications/logs/logs.html | 7 +- .../views/applications/logs/logsController.js | 18 +- app/kubernetes/views/stacks/logs/logs.html | 8 +- .../views/stacks/logs/logsController.js | 27 +- package.json | 1 - yarn.lock | 5 - 27 files changed, 935 insertions(+), 279 deletions(-) delete mode 100644 app/docker/helpers/logHelper.js create mode 100644 app/docker/helpers/logHelper/colors/colors.ts create mode 100644 app/docker/helpers/logHelper/colors/index.ts create mode 100644 app/docker/helpers/logHelper/colors/rawColors.json create mode 100644 app/docker/helpers/logHelper/concatLogsToString.ts create mode 100644 app/docker/helpers/logHelper/formatJSONLogs.ts create mode 100644 app/docker/helpers/logHelper/formatLogs.ts create mode 100644 app/docker/helpers/logHelper/formatZerologLogs.ts create mode 100644 app/docker/helpers/logHelper/formatters.ts create mode 100644 app/docker/helpers/logHelper/index.ts create mode 100644 app/docker/helpers/logHelper/types.ts diff --git a/api/cli/cli.go b/api/cli/cli.go index 83ced70ea..69dd61b75 100644 --- a/api/cli/cli.go +++ b/api/cli/cli.go @@ -62,6 +62,7 @@ func (*Service) ParseFlags(version string) (*portainer.CLIFlags, error) { MaxBatchDelay: kingpin.Flag("max-batch-delay", "Maximum delay before a batch starts").Duration(), SecretKeyName: kingpin.Flag("secret-key-name", "Secret key name for encryption and will be used as /run/secrets/.").Default(defaultSecretKeyName).String(), LogLevel: kingpin.Flag("log-level", "Set the minimum logging level to show").Default("INFO").Enum("DEBUG", "INFO", "WARN", "ERROR"), + LogMode: kingpin.Flag("log-mode", "Set the logging output mode").Default("PRETTY").Enum("PRETTY", "JSON"), } kingpin.Parse() diff --git a/api/cmd/portainer/log.go b/api/cmd/portainer/log.go index 1623b4fd7..ef95b26b0 100644 --- a/api/cmd/portainer/log.go +++ b/api/cmd/portainer/log.go @@ -1,7 +1,9 @@ package main import ( + "fmt" stdlog "log" + "os" "github.com/rs/zerolog" "github.com/rs/zerolog/log" @@ -31,3 +33,23 @@ func setLoggingLevel(level string) { zerolog.SetGlobalLevel(zerolog.DebugLevel) } } + +func setLoggingMode(mode string) { + switch mode { + case "PRETTY": + log.Logger = log.Output(zerolog.ConsoleWriter{ + Out: os.Stderr, + NoColor: true, + TimeFormat: "2006/01/02 03:04PM", + FormatMessage: formatMessage}) + case "JSON": + log.Logger = log.Output(os.Stderr) + } +} + +func formatMessage(i interface{}) string { + if i == nil { + return "" + } + return fmt.Sprintf("%s |", i) +} diff --git a/api/cmd/portainer/main.go b/api/cmd/portainer/main.go index 360e3b663..031482b8d 100644 --- a/api/cmd/portainer/main.go +++ b/api/cmd/portainer/main.go @@ -758,10 +758,12 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server { func main() { configureLogger() + setLoggingMode("PRETTY") flags := initCLI() setLoggingLevel(*flags.LogLevel) + setLoggingMode(*flags.LogMode) for { server := buildServer(flags) diff --git a/api/portainer.go b/api/portainer.go index 0277de258..d1df3c2be 100644 --- a/api/portainer.go +++ b/api/portainer.go @@ -130,6 +130,7 @@ type ( MaxBatchDelay *time.Duration SecretKeyName *string LogLevel *string + LogMode *string } // CustomTemplateVariableDefinition diff --git a/app/docker/components/log-viewer/logViewer.html b/app/docker/components/log-viewer/logViewer.html index 61b0ccb24..2f3ac1f15 100644 --- a/app/docker/components/log-viewer/logViewer.html +++ b/app/docker/components/log-viewer/logViewer.html @@ -86,7 +86,7 @@ 
-      
{{ span.text }}

+      
{{ span.text }}

       
No log line matching the '{{ $ctrl.state.search }}' filter

       
No logs available
diff --git a/app/docker/components/log-viewer/logViewerController.js b/app/docker/components/log-viewer/logViewerController.js index 4cee6ec92..07b47267e 100644 --- a/app/docker/components/log-viewer/logViewerController.js +++ b/app/docker/components/log-viewer/logViewerController.js @@ -1,6 +1,7 @@ import moment from 'moment'; -import _ from 'lodash-es'; + import { NEW_LINE_BREAKER } from '@/constants'; +import { concatLogsToString } from '@/docker/helpers/logHelper'; angular.module('portainer.docker').controller('LogViewerController', [ '$scope', @@ -74,9 +75,8 @@ angular.module('portainer.docker').controller('LogViewerController', [ }; this.downloadLogs = function () { - // To make the feature of downloading container logs working both on Windows and Linux, - // we need to use correct new line breakers on corresponding OS. - const data = new Blob([_.reduce(this.state.filteredLogs, (acc, log) => acc + log.line + NEW_LINE_BREAKER, '')]); + const logsAsString = concatLogsToString(this.state.filteredLogs); + const data = new Blob([logsAsString]); FileSaver.saveAs(data, this.resourceName + '_logs.txt'); }; }, diff --git a/app/docker/helpers/logHelper.js b/app/docker/helpers/logHelper.js deleted file mode 100644 index d2214b3c2..000000000 --- a/app/docker/helpers/logHelper.js +++ /dev/null @@ -1,223 +0,0 @@ -import tokenize from '@nxmix/tokenize-ansi'; -import x256 from 'x256'; -import { takeRight, without } from 'lodash'; -import { format } from 'date-fns'; - -const FOREGROUND_COLORS_BY_ANSI = { - black: x256.colors[0], - red: x256.colors[1], - green: x256.colors[2], - yellow: x256.colors[3], - blue: x256.colors[4], - magenta: x256.colors[5], - cyan: x256.colors[6], - white: x256.colors[7], - brightBlack: x256.colors[8], - brightRed: x256.colors[9], - brightGreen: x256.colors[10], - brightYellow: x256.colors[11], - brightBlue: x256.colors[12], - brightMagenta: x256.colors[13], - brightCyan: x256.colors[14], - brightWhite: x256.colors[15], -}; - -const BACKGROUND_COLORS_BY_ANSI = { - bgBlack: x256.colors[0], - bgRed: x256.colors[1], - bgGreen: x256.colors[2], - bgYellow: x256.colors[3], - bgBlue: x256.colors[4], - bgMagenta: x256.colors[5], - bgCyan: x256.colors[6], - bgWhite: x256.colors[7], - bgBrightBlack: x256.colors[8], - bgBrightRed: x256.colors[9], - bgBrightGreen: x256.colors[10], - bgBrightYellow: x256.colors[11], - bgBrightBlue: x256.colors[12], - bgBrightMagenta: x256.colors[13], - bgBrightCyan: x256.colors[14], - bgBrightWhite: x256.colors[15], -}; - -const TIMESTAMP_LENGTH = 31; // 30 for timestamp + 1 for trailing space - -angular.module('portainer.docker').factory('LogHelper', [ - function LogHelperFactory() { - 'use strict'; - var helper = {}; - - function stripHeaders(logs) { - logs = logs.substring(8); - logs = logs.replace(/\r?\n(.{8})/g, '\n'); - - return logs; - } - - function stripEscapeCodes(logs) { - return logs.replace(/[\u001b\u009b][[()#;?]*(?:[0-9]{1,4}(?:;[0-9]{0,4})*)?[0-9A-ORZcf-nqry=><]/g, ''); - } - - function cssColorFromRgb(rgb) { - const [r, g, b] = rgb; - - return `rgb(${r}, ${g}, ${b})`; - } - - function extendedColorForToken(token) { - const colorMode = token[1]; - - if (colorMode === 2) { - return cssColorFromRgb(token.slice(2)); - } - - if (colorMode === 5 && x256.colors[token[2]]) { - return cssColorFromRgb(x256.colors[token[2]]); - } - - return ''; - } - - // Return an array with each log including a line and styled spans for each entry. - // If the stripHeaders param is specified, it will strip the 8 first characters of each line. - // withTimestamps param is needed to find the start of JSON for Zerolog logs parsing - helper.formatLogs = function (logs, { stripHeaders: skipHeaders, withTimestamps }) { - if (skipHeaders) { - logs = stripHeaders(logs); - } - - const tokens = tokenize(logs); - const formattedLogs = []; - - let foregroundColor = null; - let backgroundColor = null; - let line = ''; - let spans = []; - - for (const token of tokens) { - const type = token[0]; - - if (FOREGROUND_COLORS_BY_ANSI[type]) { - foregroundColor = cssColorFromRgb(FOREGROUND_COLORS_BY_ANSI[type]); - } else if (type === 'moreColor') { - foregroundColor = extendedColorForToken(token); - } else if (type === 'fgDefault') { - foregroundColor = null; - } else if (BACKGROUND_COLORS_BY_ANSI[type]) { - backgroundColor = cssColorFromRgb(BACKGROUND_COLORS_BY_ANSI[type]); - } else if (type === 'bgMoreColor') { - backgroundColor = extendedColorForToken(token); - } else if (type === 'bgDefault') { - backgroundColor = null; - } else if (type === 'reset') { - foregroundColor = null; - backgroundColor = null; - } else if (type === 'text') { - const tokenLines = token[1].split('\n'); - - for (let i = 0; i < tokenLines.length; i++) { - if (i !== 0) { - formattedLogs.push({ line, spans }); - - line = ''; - spans = []; - } - - const text = stripEscapeCodes(tokenLines[i]); - if ((!withTimestamps && text.startsWith('{')) || (withTimestamps && text.substring(TIMESTAMP_LENGTH).startsWith('{'))) { - line += JSONToFormattedLine(text, spans, withTimestamps); - } else { - spans.push({ foregroundColor, backgroundColor, text }); - line += text; - } - } - } - } - - if (line) { - formattedLogs.push({ line, spans }); - } - - return formattedLogs; - }; - - return helper; - }, -]); - -const JSONColors = { - Grey: 'var(--text-log-viewer-color-json-grey)', - Magenta: 'var(--text-log-viewer-color-json-magenta)', - Yellow: 'var(--text-log-viewer-color-json-yellow)', - Green: 'var(--text-log-viewer-color-json-green)', - Red: 'var(--text-log-viewer-color-json-red)', - Blue: 'var(--text-log-viewer-color-json-blue)', -}; - -const spaceSpan = { text: ' ' }; - -function logLevelToSpan(level) { - switch (level) { - case 'debug': - return { foregroundColor: JSONColors.Grey, text: 'DBG', fontWeight: 'bold' }; - case 'info': - return { foregroundColor: JSONColors.Green, text: 'INF', fontWeight: 'bold' }; - case 'warn': - return { foregroundColor: JSONColors.Yellow, text: 'WRN', fontWeight: 'bold' }; - case 'error': - return { foregroundColor: JSONColors.Red, text: 'ERR', fontWeight: 'bold' }; - default: - return { text: level }; - } -} - -function JSONToFormattedLine(rawText, spans, withTimestamps) { - const text = withTimestamps ? rawText.substring(TIMESTAMP_LENGTH) : rawText; - const json = JSON.parse(text); - const { level, caller, message, time } = json; - let line = ''; - - if (withTimestamps) { - const timestamp = rawText.substring(0, TIMESTAMP_LENGTH); - spans.push({ text: timestamp }); - line += `${timestamp}`; - } - if (time) { - const date = format(new Date(time * 1000), 'Y/MM/dd hh:mmaa'); - spans.push({ foregroundColor: JSONColors.Grey, text: date }, spaceSpan); - line += `${date} `; - } - if (level) { - const levelSpan = logLevelToSpan(level); - spans.push(levelSpan, spaceSpan); - line += `${levelSpan.text} `; - } - if (caller) { - const trimmedCaller = takeRight(caller.split('/'), 2).join('/'); - spans.push({ foregroundColor: JSONColors.Magenta, text: trimmedCaller, fontWeight: 'bold' }, spaceSpan); - spans.push({ foregroundColor: JSONColors.Blue, text: '>' }, spaceSpan); - line += `${trimmedCaller} > `; - } - - const keys = without(Object.keys(json), 'time', 'level', 'caller', 'message'); - if (message) { - spans.push({ foregroundColor: JSONColors.Magenta, text: `${message}` }, spaceSpan); - line += `${message} `; - - if (keys.length) { - spans.push({ foregroundColor: JSONColors.Magenta, text: `|` }, spaceSpan); - line += '| '; - } - } - - keys.forEach((key) => { - const value = json[key]; - spans.push({ foregroundColor: JSONColors.Blue, text: `${key}=` }); - spans.push({ foregroundColor: key === 'error' ? JSONColors.Red : JSONColors.Magenta, text: value }); - spans.push(spaceSpan); - line += `${key}=${value} `; - }); - - return line; -} diff --git a/app/docker/helpers/logHelper/colors/colors.ts b/app/docker/helpers/logHelper/colors/colors.ts new file mode 100644 index 000000000..f9183c2b6 --- /dev/null +++ b/app/docker/helpers/logHelper/colors/colors.ts @@ -0,0 +1,63 @@ +// original code comes from https://www.npmjs.com/package/x256 +// only picking the used parts as there is no type definition +// package is unmaintained and repository doesn't exist anymore + +// colors scraped from +// http://www.calmar.ws/vim/256-xterm-24bit-rgb-color-chart.html +// %s/ *\d\+ \+#\([^ ]\+\)/\1\r/g + +import rawColors from './rawColors.json'; + +export type RGBColor = [number, number, number]; +export type TextColor = string | undefined; + +function hexToRGB(hex: string): RGBColor { + const r = parseInt(hex.slice(0, 2), 16); + const g = parseInt(hex.slice(2, 4), 16); + const b = parseInt(hex.slice(4, 6), 16); + return [r, g, b]; +} + +export const colors = rawColors.map(hexToRGB); + +export const FOREGROUND_COLORS_BY_ANSI: { + [k: string]: RGBColor; +} = { + black: colors[0], + red: colors[1], + green: colors[2], + yellow: colors[3], + blue: colors[4], + magenta: colors[5], + cyan: colors[6], + white: colors[7], + brightBlack: colors[8], + brightRed: colors[9], + brightGreen: colors[10], + brightYellow: colors[11], + brightBlue: colors[12], + brightMagenta: colors[13], + brightCyan: colors[14], + brightWhite: colors[15], +}; + +export const BACKGROUND_COLORS_BY_ANSI: { + [k: string]: RGBColor; +} = { + bgBlack: colors[0], + bgRed: colors[1], + bgGreen: colors[2], + bgYellow: colors[3], + bgBlue: colors[4], + bgMagenta: colors[5], + bgCyan: colors[6], + bgWhite: colors[7], + bgBrightBlack: colors[8], + bgBrightRed: colors[9], + bgBrightGreen: colors[10], + bgBrightYellow: colors[11], + bgBrightBlue: colors[12], + bgBrightMagenta: colors[13], + bgBrightCyan: colors[14], + bgBrightWhite: colors[15], +}; diff --git a/app/docker/helpers/logHelper/colors/index.ts b/app/docker/helpers/logHelper/colors/index.ts new file mode 100644 index 000000000..25eed2280 --- /dev/null +++ b/app/docker/helpers/logHelper/colors/index.ts @@ -0,0 +1,7 @@ +export { + type RGBColor, + type TextColor, + colors, + FOREGROUND_COLORS_BY_ANSI, + BACKGROUND_COLORS_BY_ANSI, +} from './colors'; diff --git a/app/docker/helpers/logHelper/colors/rawColors.json b/app/docker/helpers/logHelper/colors/rawColors.json new file mode 100644 index 000000000..b9bba82d4 --- /dev/null +++ b/app/docker/helpers/logHelper/colors/rawColors.json @@ -0,0 +1,258 @@ +[ + "000000", + "800000", + "008000", + "808000", + "000080", + "800080", + "008080", + "c0c0c0", + "808080", + "ff0000", + "00ff00", + "ffff00", + "0000ff", + "ff00ff", + "00ffff", + "ffffff", + "000000", + "00005f", + "000087", + "0000af", + "0000d7", + "0000ff", + "005f00", + "005f5f", + "005f87", + "005faf", + "005fd7", + "005fff", + "008700", + "00875f", + "008787", + "0087af", + "0087d7", + "0087ff", + "00af00", + "00af5f", + "00af87", + "00afaf", + "00afd7", + "00afff", + "00d700", + "00d75f", + "00d787", + "00d7af", + "00d7d7", + "00d7ff", + "00ff00", + "00ff5f", + "00ff87", + "00ffaf", + "00ffd7", + "00ffff", + "5f0000", + "5f005f", + "5f0087", + "5f00af", + "5f00d7", + "5f00ff", + "5f5f00", + "5f5f5f", + "5f5f87", + "5f5faf", + "5f5fd7", + "5f5fff", + "5f8700", + "5f875f", + "5f8787", + "5f87af", + "5f87d7", + "5f87ff", + "5faf00", + "5faf5f", + "5faf87", + "5fafaf", + "5fafd7", + "5fafff", + "5fd700", + "5fd75f", + "5fd787", + "5fd7af", + "5fd7d7", + "5fd7ff", + "5fff00", + "5fff5f", + "5fff87", + "5fffaf", + "5fffd7", + "5fffff", + "870000", + "87005f", + "870087", + "8700af", + "8700d7", + "8700ff", + "875f00", + "875f5f", + "875f87", + "875faf", + "875fd7", + "875fff", + "878700", + "87875f", + "878787", + "8787af", + "8787d7", + "8787ff", + "87af00", + "87af5f", + "87af87", + "87afaf", + "87afd7", + "87afff", + "87d700", + "87d75f", + "87d787", + "87d7af", + "87d7d7", + "87d7ff", + "87ff00", + "87ff5f", + "87ff87", + "87ffaf", + "87ffd7", + "87ffff", + "af0000", + "af005f", + "af0087", + "af00af", + "af00d7", + "af00ff", + "af5f00", + "af5f5f", + "af5f87", + "af5faf", + "af5fd7", + "af5fff", + "af8700", + "af875f", + "af8787", + "af87af", + "af87d7", + "af87ff", + "afaf00", + "afaf5f", + "afaf87", + "afafaf", + "afafd7", + "afafff", + "afd700", + "afd75f", + "afd787", + "afd7af", + "afd7d7", + "afd7ff", + "afff00", + "afff5f", + "afff87", + "afffaf", + "afffd7", + "afffff", + "d70000", + "d7005f", + "d70087", + "d700af", + "d700d7", + "d700ff", + "d75f00", + "d75f5f", + "d75f87", + "d75faf", + "d75fd7", + "d75fff", + "d78700", + "d7875f", + "d78787", + "d787af", + "d787d7", + "d787ff", + "d7af00", + "d7af5f", + "d7af87", + "d7afaf", + "d7afd7", + "d7afff", + "d7d700", + "d7d75f", + "d7d787", + "d7d7af", + "d7d7d7", + "d7d7ff", + "d7ff00", + "d7ff5f", + "d7ff87", + "d7ffaf", + "d7ffd7", + "d7ffff", + "ff0000", + "ff005f", + "ff0087", + "ff00af", + "ff00d7", + "ff00ff", + "ff5f00", + "ff5f5f", + "ff5f87", + "ff5faf", + "ff5fd7", + "ff5fff", + "ff8700", + "ff875f", + "ff8787", + "ff87af", + "ff87d7", + "ff87ff", + "ffaf00", + "ffaf5f", + "ffaf87", + "ffafaf", + "ffafd7", + "ffafff", + "ffd700", + "ffd75f", + "ffd787", + "ffd7af", + "ffd7d7", + "ffd7ff", + "ffff00", + "ffff5f", + "ffff87", + "ffffaf", + "ffffd7", + "ffffff", + "080808", + "121212", + "1c1c1c", + "262626", + "303030", + "3a3a3a", + "444444", + "4e4e4e", + "585858", + "606060", + "666666", + "767676", + "808080", + "8a8a8a", + "949494", + "9e9e9e", + "a8a8a8", + "b2b2b2", + "bcbcbc", + "c6c6c6", + "d0d0d0", + "dadada", + "e4e4e4", + "eeeeee" +] diff --git a/app/docker/helpers/logHelper/concatLogsToString.ts b/app/docker/helpers/logHelper/concatLogsToString.ts new file mode 100644 index 000000000..b67d4312e --- /dev/null +++ b/app/docker/helpers/logHelper/concatLogsToString.ts @@ -0,0 +1,15 @@ +import { NEW_LINE_BREAKER } from '@/constants'; + +import { FormattedLine } from './types'; + +type FormatFunc = (line: FormattedLine) => string; + +export function concatLogsToString( + logs: FormattedLine[], + formatFunc: FormatFunc = (line) => line.line +) { + return logs.reduce( + (acc, formattedLine) => acc + formatFunc(formattedLine) + NEW_LINE_BREAKER, + '' + ); +} diff --git a/app/docker/helpers/logHelper/formatJSONLogs.ts b/app/docker/helpers/logHelper/formatJSONLogs.ts new file mode 100644 index 000000000..5a08cc360 --- /dev/null +++ b/app/docker/helpers/logHelper/formatJSONLogs.ts @@ -0,0 +1,55 @@ +import { without } from 'lodash'; + +import { FormattedLine, Span, JSONLogs, TIMESTAMP_LENGTH } from './types'; +import { + formatCaller, + formatKeyValuePair, + formatLevel, + formatMessage, + formatStackTrace, + formatTime, +} from './formatters'; + +function removeKnownKeys(keys: string[]) { + return without(keys, 'time', 'level', 'caller', 'message', 'stack_trace'); +} + +export function formatJSONLine( + rawText: string, + withTimestamps?: boolean +): FormattedLine[] { + const spans: Span[] = []; + const lines: FormattedLine[] = []; + let line = ''; + + const text = withTimestamps ? rawText.substring(TIMESTAMP_LENGTH) : rawText; + + const json: JSONLogs = JSON.parse(text); + const { time, level, caller, message, stack_trace: stackTrace } = json; + const keys = removeKnownKeys(Object.keys(json)); + + if (withTimestamps) { + const timestamp = rawText.substring(0, TIMESTAMP_LENGTH); + spans.push({ text: timestamp }); + line += `${timestamp}`; + } + line += formatTime(time, spans, line); + line += formatLevel(level, spans, line); + line += formatCaller(caller, spans, line); + line += formatMessage(message, spans, line, !!keys.length); + + keys.forEach((key, idx) => { + line += formatKeyValuePair( + key, + json[key], + spans, + line, + idx === keys.length - 1 + ); + }); + + lines.push({ line, spans }); + formatStackTrace(stackTrace, lines); + + return lines; +} diff --git a/app/docker/helpers/logHelper/formatLogs.ts b/app/docker/helpers/logHelper/formatLogs.ts new file mode 100644 index 000000000..e0af64375 --- /dev/null +++ b/app/docker/helpers/logHelper/formatLogs.ts @@ -0,0 +1,141 @@ +import tokenize from '@nxmix/tokenize-ansi'; +import { FontWeight } from 'xterm'; + +import { + colors, + BACKGROUND_COLORS_BY_ANSI, + FOREGROUND_COLORS_BY_ANSI, + RGBColor, +} from './colors'; +import { formatJSONLine } from './formatJSONLogs'; +import { formatZerologLogs, ZerologRegex } from './formatZerologLogs'; +import { Token, Span, TIMESTAMP_LENGTH, FormattedLine } from './types'; + +type FormatOptions = { + stripHeaders?: boolean; + withTimestamps?: boolean; + splitter?: string; +}; + +const defaultOptions: FormatOptions = { + splitter: '\n', +}; + +export function formatLogs( + rawLogs: string, + { + stripHeaders, + withTimestamps, + splitter = '\n', + }: FormatOptions = defaultOptions +) { + let logs = rawLogs; + if (stripHeaders) { + logs = stripHeadersFunc(logs); + } + if (logs.includes('\\n')) { + logs = JSON.parse(logs); + } + + const tokens: Token[][] = tokenize(logs); + const formattedLogs: FormattedLine[] = []; + + let fgColor: string | undefined; + let bgColor: string | undefined; + let fontWeight: FontWeight | undefined; + let line = ''; + let spans: Span[] = []; + + tokens.forEach((token) => { + const [type] = token; + + const fgAnsi = FOREGROUND_COLORS_BY_ANSI[type]; + const bgAnsi = BACKGROUND_COLORS_BY_ANSI[type]; + + if (fgAnsi) { + fgColor = cssColorFromRgb(fgAnsi); + } else if (type === 'moreColor') { + fgColor = extendedColorForToken(token); + } else if (type === 'fgDefault') { + fgColor = undefined; + } else if (bgAnsi) { + bgColor = cssColorFromRgb(bgAnsi); + } else if (type === 'bgMoreColor') { + bgColor = extendedColorForToken(token); + } else if (type === 'bgDefault') { + bgColor = undefined; + } else if (type === 'reset') { + fgColor = undefined; + bgColor = undefined; + fontWeight = undefined; + } else if (type === 'bold') { + fontWeight = 'bold'; + } else if (type === 'normal') { + fontWeight = 'normal'; + } else if (type === 'text') { + const tokenLines = (token[1] as string).split(splitter); + + tokenLines.forEach((tokenLine, idx) => { + if (idx && line) { + formattedLogs.push({ line, spans }); + line = ''; + spans = []; + } + + const text = stripEscapeCodes(tokenLine); + if ( + (!withTimestamps && text.startsWith('{')) || + (withTimestamps && text.substring(TIMESTAMP_LENGTH).startsWith('{')) + ) { + const lines = formatJSONLine(text, withTimestamps); + formattedLogs.push(...lines); + } else if (ZerologRegex.test(text)) { + const lines = formatZerologLogs(text, withTimestamps); + formattedLogs.push(...lines); + } else { + spans.push({ fgColor, bgColor, text, fontWeight }); + line += text; + } + }); + } + }); + if (line) { + formattedLogs.push({ line, spans }); + } + + return formattedLogs; +} + +function stripHeadersFunc(logs: string) { + return logs.substring(8).replace(/\r?\n(.{8})/g, '\n'); +} + +function stripEscapeCodes(logs: string) { + return logs.replace( + // eslint-disable-next-line no-control-regex + /[\u001b\u009b][[()#;?]*(?:[0-9]{1,4}(?:;[0-9]{0,4})*)?[0-9A-ORZcf-nqry=><]/g, + '' + ); +} + +function cssColorFromRgb(rgb: RGBColor) { + const [r, g, b] = rgb; + + return `rgb(${r}, ${g}, ${b})`; +} + +// assuming types based on original JS implementation +// there is not much type definitions for the tokenize library +function extendedColorForToken(token: Token[]) { + const [, colorMode, colorRef] = token as [undefined, number, number]; + + if (colorMode === 2) { + return cssColorFromRgb(token.slice(2) as RGBColor); + } + + if (colorMode === 5 && colors[colorRef]) { + return cssColorFromRgb(colors[colorRef]); + } + + return ''; +} diff --git a/app/docker/helpers/logHelper/formatZerologLogs.ts b/app/docker/helpers/logHelper/formatZerologLogs.ts new file mode 100644 index 000000000..e983e3141 --- /dev/null +++ b/app/docker/helpers/logHelper/formatZerologLogs.ts @@ -0,0 +1,119 @@ +import { + formatCaller, + formatKeyValuePair, + formatLevel, + formatMessage, + formatStackTrace, + formatTime, +} from './formatters'; +import { + FormattedLine, + JSONStackTrace, + Level, + Span, + TIMESTAMP_LENGTH, +} from './types'; + +const dateRegex = /(\d{4}\/\d{2}\/\d{2} \d{2}:\d{2}[AP]M) /; // "2022/02/01 04:30AM " +const levelRegex = /(\w{3}) /; // "INF " or "ERR " +const callerRegex = /(.+?.go:\d+) /; // "path/to/file.go:line " +const chevRegex = /> /; // "> " +const messageAndPairsRegex = /(.*)/; // include the rest of the string in a separate group + +const keyRegex = /(\S+=)/g; // "" + +export const ZerologRegex = concatRegex( + dateRegex, + levelRegex, + callerRegex, + chevRegex, + messageAndPairsRegex +); + +function concatRegex(...regs: RegExp[]) { + const flags = Array.from( + new Set( + regs + .map((r) => r.flags) + .join('') + .split('') + ) + ).join(''); + const source = regs.map((r) => r.source).join(''); + return new RegExp(source, flags); +} + +type Pair = { + key: string; + value: string; +}; + +export function formatZerologLogs(rawText: string, withTimestamps?: boolean) { + const spans: Span[] = []; + const lines: FormattedLine[] = []; + let line = ''; + + const text = withTimestamps ? rawText.substring(TIMESTAMP_LENGTH) : rawText; + + const [, date, level, caller, messageAndPairs] = + text.match(ZerologRegex) || []; + + const [message, pairs] = extractPairs(messageAndPairs); + + line += formatTime(date, spans, line); + line += formatLevel(level as Level, spans, line); + line += formatCaller(caller, spans, line); + line += formatMessage(message, spans, line, !!pairs.length); + + let stackTrace: JSONStackTrace | undefined; + const stackTraceIndex = pairs.findIndex((p) => p.key === 'stack_trace'); + + if (stackTraceIndex !== -1) { + stackTrace = JSON.parse(pairs[stackTraceIndex].value); + pairs.splice(stackTraceIndex); + } + + pairs.forEach(({ key, value }, idx) => { + line += formatKeyValuePair( + key, + value, + spans, + line, + idx === pairs.length - 1 + ); + }); + lines.push({ line, spans }); + + formatStackTrace(stackTrace, lines); + + return lines; +} + +function extractPairs(messageAndPairs: string): [string, Pair[]] { + const pairs: Pair[] = []; + let [message, rawPairs] = messageAndPairs.split('|'); + + if (!messageAndPairs.includes('|') && !rawPairs) { + rawPairs = message; + message = ''; + } + message = message.trim(); + rawPairs = rawPairs.trim(); + + const matches = [...rawPairs.matchAll(keyRegex)]; + + matches.forEach((m, idx) => { + const rawKey = m[0]; + const key = rawKey.slice(0, -1); + const start = m.index || 0; + const end = idx !== matches.length - 1 ? matches[idx + 1].index : undefined; + const value = ( + end + ? rawPairs.slice(start + rawKey.length, end) + : rawPairs.slice(start + rawKey.length) + ).trim(); + pairs.push({ key, value }); + }); + + return [message, pairs]; +} diff --git a/app/docker/helpers/logHelper/formatters.ts b/app/docker/helpers/logHelper/formatters.ts new file mode 100644 index 000000000..10c5b284f --- /dev/null +++ b/app/docker/helpers/logHelper/formatters.ts @@ -0,0 +1,154 @@ +import { format } from 'date-fns'; +import { takeRight } from 'lodash'; + +import { Span, Level, Colors, JSONStackTrace, FormattedLine } from './types'; + +const spaceSpan: Span = { text: ' ' }; + +function logLevelToSpan(level: Level): Span { + switch (level) { + case 'debug': + case 'DBG': + return { + fgColor: Colors.Grey, + text: 'DBG', + fontWeight: 'bold', + }; + case 'info': + case 'INF': + return { + fgColor: Colors.Green, + text: 'INF', + fontWeight: 'bold', + }; + case 'warn': + case 'WRN': + return { + fgColor: Colors.Yellow, + text: 'WRN', + fontWeight: 'bold', + }; + case 'error': + case 'ERR': + return { + fgColor: Colors.Red, + text: 'ERR', + fontWeight: 'bold', + }; + default: + return { text: level }; + } +} + +export function formatTime( + time: number | string | undefined, + spans: Span[], + line: string +) { + let nl = line; + if (time) { + let date = ''; + if (typeof time === 'number') { + date = format(new Date(time * 1000), 'Y/MM/dd hh:mmaa'); + } else { + date = time; + } + spans.push({ fgColor: Colors.Grey, text: date }, spaceSpan); + nl += `${date} `; + } + return nl; +} + +export function formatLevel( + level: Level | undefined, + spans: Span[], + line: string +) { + let nl = line; + if (level) { + const levelSpan = logLevelToSpan(level); + spans.push(levelSpan, spaceSpan); + nl += `${levelSpan.text} `; + } + return nl; +} + +export function formatCaller( + caller: string | undefined, + spans: Span[], + line: string +) { + let nl = line; + if (caller) { + const trim = takeRight(caller.split('/'), 2).join('/'); + spans.push( + { fgColor: Colors.Magenta, text: trim, fontWeight: 'bold' }, + spaceSpan + ); + spans.push({ fgColor: Colors.Blue, text: '>' }, spaceSpan); + nl += `${trim} > `; + } + return nl; +} + +export function formatMessage( + message: string, + spans: Span[], + line: string, + hasKeys: boolean +) { + let nl = line; + if (message) { + spans.push({ fgColor: Colors.Magenta, text: `${message}` }, spaceSpan); + nl += `${message} `; + + if (hasKeys) { + spans.push({ fgColor: Colors.Magenta, text: `|` }, spaceSpan); + nl += '| '; + } + } + return nl; +} + +export function formatKeyValuePair( + key: string, + value: unknown, + spans: Span[], + line: string, + isLastKey: boolean +) { + let nl = line; + + spans.push( + { fgColor: Colors.Blue, text: `${key}=` }, + { + fgColor: key === 'error' || key === 'ERR' ? Colors.Red : Colors.Magenta, + text: value as string, + } + ); + if (!isLastKey) spans.push(spaceSpan); + nl += `${key}=${value}${!isLastKey ? ' ' : ''}`; + + return nl; +} + +export function formatStackTrace( + stackTrace: JSONStackTrace | undefined, + lines: FormattedLine[] +) { + if (stackTrace) { + stackTrace.forEach(({ func, line: lineNumber, source }) => { + const line = ` at ${func} (${source}:${lineNumber})`; + const spans: Span[] = [ + spaceSpan, + spaceSpan, + spaceSpan, + spaceSpan, + { text: 'at ', fgColor: Colors.Grey }, + { text: func, fgColor: Colors.Red }, + { text: `(${source}:${lineNumber})`, fgColor: Colors.Grey }, + ]; + lines.push({ line, spans }); + }); + } +} diff --git a/app/docker/helpers/logHelper/index.ts b/app/docker/helpers/logHelper/index.ts new file mode 100644 index 000000000..91fe97ed7 --- /dev/null +++ b/app/docker/helpers/logHelper/index.ts @@ -0,0 +1,2 @@ +export { formatLogs } from './formatLogs'; +export { concatLogsToString } from './concatLogsToString'; diff --git a/app/docker/helpers/logHelper/types.ts b/app/docker/helpers/logHelper/types.ts new file mode 100644 index 000000000..7d467e296 --- /dev/null +++ b/app/docker/helpers/logHelper/types.ts @@ -0,0 +1,53 @@ +import { FontWeight } from 'xterm'; + +import { type TextColor } from './colors'; + +export type Token = string | number; + +export type Level = + | 'debug' + | 'info' + | 'warn' + | 'error' + | 'DBG' + | 'INF' + | 'WRN' + | 'ERR'; + +export type JSONStackTrace = { + func: string; + line: string; + source: string; +}[]; + +export type JSONLogs = { + [k: string]: unknown; + time: number; + level: Level; + caller: string; + message: string; + stack_trace?: JSONStackTrace; +}; + +export type Span = { + fgColor?: TextColor; + bgColor?: TextColor; + text: string; + fontWeight?: FontWeight; +}; + +export type FormattedLine = { + spans: Span[]; + line: string; +}; + +export const TIMESTAMP_LENGTH = 31; // 30 for timestamp + 1 for trailing space + +export const Colors = { + Grey: 'var(--text-log-viewer-color-json-grey)', + Magenta: 'var(--text-log-viewer-color-json-magenta)', + Yellow: 'var(--text-log-viewer-color-json-yellow)', + Green: 'var(--text-log-viewer-color-json-green)', + Red: 'var(--text-log-viewer-color-json-red)', + Blue: 'var(--text-log-viewer-color-json-blue)', +}; diff --git a/app/docker/services/containerService.js b/app/docker/services/containerService.js index fc1514b6f..175720b09 100644 --- a/app/docker/services/containerService.js +++ b/app/docker/services/containerService.js @@ -10,11 +10,12 @@ import { stopContainer, } from '@/react/docker/containers/containers.service'; import { ContainerDetailsViewModel, ContainerStatsViewModel, ContainerViewModel } from '../models/container'; +import { formatLogs } from '../helpers/logHelper'; angular.module('portainer.docker').factory('ContainerService', ContainerServiceFactory); /* @ngInject */ -function ContainerServiceFactory($q, Container, LogHelper, $timeout, EndpointProvider) { +function ContainerServiceFactory($q, Container, $timeout, EndpointProvider) { const service = { killContainer: withEndpointId(killContainer), pauseContainer: withEndpointId(pauseContainer), @@ -159,7 +160,7 @@ function ContainerServiceFactory($q, Container, LogHelper, $timeout, EndpointPro Container.logs(parameters) .$promise.then(function success(data) { - var logs = LogHelper.formatLogs(data.logs, { stripHeaders, withTimestamps: !!timestamps }); + var logs = formatLogs(data.logs, { stripHeaders, withTimestamps: !!timestamps }); deferred.resolve(logs); }) .catch(function error(err) { diff --git a/app/docker/services/serviceService.js b/app/docker/services/serviceService.js index cab03024b..749bc91c8 100644 --- a/app/docker/services/serviceService.js +++ b/app/docker/services/serviceService.js @@ -1,13 +1,10 @@ +import { formatLogs } from '../helpers/logHelper'; import { ServiceViewModel } from '../models/service'; angular.module('portainer.docker').factory('ServiceService', [ '$q', 'Service', - 'ServiceHelper', - 'TaskService', - 'ResourceControlService', - 'LogHelper', - function ServiceServiceFactory($q, Service, ServiceHelper, TaskService, ResourceControlService, LogHelper) { + function ServiceServiceFactory($q, Service) { 'use strict'; var service = {}; @@ -88,7 +85,7 @@ angular.module('portainer.docker').factory('ServiceService', [ Service.logs(parameters) .$promise.then(function success(data) { - var logs = LogHelper.formatLogs(data.logs, { stripHeaders: true, withTimestamps: !!timestamps }); + var logs = formatLogs(data.logs, { stripHeaders: true, withTimestamps: !!timestamps }); deferred.resolve(logs); }) .catch(function error(err) { diff --git a/app/docker/services/taskService.js b/app/docker/services/taskService.js index 5c38a316f..0f68183a0 100644 --- a/app/docker/services/taskService.js +++ b/app/docker/services/taskService.js @@ -1,10 +1,10 @@ +import { formatLogs } from '../helpers/logHelper'; import { TaskViewModel } from '../models/task'; angular.module('portainer.docker').factory('TaskService', [ '$q', 'Task', - 'LogHelper', - function TaskServiceFactory($q, Task, LogHelper) { + function TaskServiceFactory($q, Task) { 'use strict'; var service = {}; @@ -54,7 +54,7 @@ angular.module('portainer.docker').factory('TaskService', [ Task.logs(parameters) .$promise.then(function success(data) { - var logs = LogHelper.formatLogs(data.logs, { stripHeaders: true, withTimestamps: !!timestamps }); + var logs = formatLogs(data.logs, { stripHeaders: true, withTimestamps: !!timestamps }); deferred.resolve(logs); }) .catch(function error(err) { diff --git a/app/kubernetes/pod/service.js b/app/kubernetes/pod/service.js index 925b09797..45d735700 100644 --- a/app/kubernetes/pod/service.js +++ b/app/kubernetes/pod/service.js @@ -67,7 +67,7 @@ class KubernetesPodService { params.container = containerName; } const data = await this.KubernetesPods(namespace).logs(params).$promise; - return data.logs.length === 0 ? [] : data.logs.split('\n'); + return data.logs; } catch (err) { throw new PortainerError('Unable to retrieve pod logs', err); } diff --git a/app/kubernetes/views/applications/logs/logs.html b/app/kubernetes/views/applications/logs/logs.html index 833f1670c..42f35524c 100644 --- a/app/kubernetes/views/applications/logs/logs.html +++ b/app/kubernetes/views/applications/logs/logs.html @@ -77,9 +77,10 @@
- 
{{ line }}
No log line matching the '{{ ctrl.state.search }}' filter
No logs available
+ 
+        
{{ span.text }}

+        
No log line matching the '{{ ctrl.state.search }}' filter

+        
No logs available
diff --git a/app/kubernetes/views/applications/logs/logsController.js b/app/kubernetes/views/applications/logs/logsController.js index a72e6acb1..66601d98b 100644 --- a/app/kubernetes/views/applications/logs/logsController.js +++ b/app/kubernetes/views/applications/logs/logsController.js @@ -1,5 +1,6 @@ import angular from 'angular'; -import _ from 'lodash-es'; + +import { concatLogsToString, formatLogs } from '@/docker/helpers/logHelper'; class KubernetesApplicationLogsController { /* @ngInject */ @@ -39,13 +40,15 @@ class KubernetesApplicationLogsController { } downloadLogs() { - const data = new this.Blob([_.reduce(this.applicationLogs, (acc, log) => acc + '\n' + log, '')]); + const logsAsString = concatLogsToString(this.applicationLogs); + const data = new this.Blob([logsAsString]); this.FileSaver.saveAs(data, this.podName + '_logs.txt'); } async getApplicationLogsAsync() { try { - this.applicationLogs = await this.KubernetesPodService.logs(this.application.ResourcePool, this.podName, this.containerName); + const rawLogs = await this.KubernetesPodService.logs(this.application.ResourcePool, this.podName, this.containerName); + this.applicationLogs = formatLogs(rawLogs); } catch (err) { this.stopRepeater(); this.Notifications.error('Failure', err, 'Unable to retrieve application logs'); @@ -70,13 +73,8 @@ class KubernetesApplicationLogsController { this.containerName = containerName; try { - const [application, applicationLogs] = await Promise.all([ - this.KubernetesApplicationService.get(namespace, applicationName), - this.KubernetesPodService.logs(namespace, podName, containerName), - ]); - - this.application = application; - this.applicationLogs = applicationLogs; + this.application = await this.KubernetesApplicationService.get(namespace, applicationName); + await this.getApplicationLogsAsync(); } catch (err) { this.Notifications.error('Failure', err, 'Unable to retrieve application logs'); } finally { diff --git a/app/kubernetes/views/stacks/logs/logs.html b/app/kubernetes/views/stacks/logs/logs.html index 5ec715af5..3847c5a96 100644 --- a/app/kubernetes/views/stacks/logs/logs.html +++ b/app/kubernetes/views/stacks/logs/logs.html @@ -69,9 +69,11 @@ ctrl.state.transition.name,
- 
{{ line.AppName }} {{ line.Line }}
No log line matching the '{{ ctrl.state.search }}' filter
No logs available
+ 
+        
{{ log.appName }} {{ span.text }}

+        
No log line matching the '{{ ctrl.state.search }}' filter

+        
No logs available

+
diff --git a/app/kubernetes/views/stacks/logs/logsController.js b/app/kubernetes/views/stacks/logs/logsController.js index 547ba9e6c..536ea2ae4 100644 --- a/app/kubernetes/views/stacks/logs/logsController.js +++ b/app/kubernetes/views/stacks/logs/logsController.js @@ -1,6 +1,7 @@ -import _ from 'lodash-es'; +import { filter, flatMap, map } from 'lodash'; import angular from 'angular'; import $allSettled from 'Portainer/services/allSettled'; +import { concatLogsToString, formatLogs } from '@/docker/helpers/logHelper'; const colors = ['red', 'orange', 'lime', 'green', 'darkgreen', 'cyan', 'turquoise', 'teal', 'deepskyblue', 'blue', 'darkblue', 'slateblue', 'magenta', 'darkviolet']; @@ -58,7 +59,7 @@ class KubernetesStackLogsController { Pods: [], }; - const promises = _.flatMap(_.map(app.Pods, (pod) => _.map(pod.Containers, (container) => this.generateLogsPromise(pod, container)))); + const promises = flatMap(map(app.Pods, (pod) => map(pod.Containers, (container) => this.generateLogsPromise(pod, container)))); const result = await $allSettled(promises); res.Pods = result.fulfilled; return res; @@ -67,21 +68,12 @@ class KubernetesStackLogsController { async getStackLogsAsync() { try { const applications = await this.KubernetesApplicationService.get(this.state.transition.namespace); - const filteredApplications = _.filter(applications, (app) => app.StackName === this.state.transition.name); - const logsPromises = _.map(filteredApplications, this.generateAppPromise); + const filteredApplications = filter(applications, (app) => app.StackName === this.state.transition.name); + const logsPromises = map(filteredApplications, this.generateAppPromise); const data = await Promise.all(logsPromises); - const logs = _.flatMap(data, (app, index) => { - return _.flatMap(app.Pods, (pod) => { - return _.map(pod.Logs, (line) => { - const res = { - Color: colors[index % colors.length], - Line: line, - AppName: pod.Pod.Name, - }; - return res; - }); - }); - }); + const logs = flatMap(data, (app, index) => + flatMap(app.Pods, (pod) => formatLogs(pod.Logs).map((line) => ({ ...line, appColor: colors[index % colors.length], appName: pod.Pod.Name }))) + ); this.stackLogs = logs; } catch (err) { this.stopRepeater(); @@ -90,7 +82,8 @@ class KubernetesStackLogsController { } downloadLogs() { - const data = new this.Blob([(this.dataLogs = _.reduce(this.stackLogs, (acc, log) => acc + '\n' + log.AppName + ' ' + log.Line, ''))]); + const logsAsString = concatLogsToString(this.state.filteredLogs, (line) => `${line.appName} ${line.line}`); + const data = new this.Blob([logsAsString]); this.FileSaver.saveAs(data, this.state.transition.name + '_logs.txt'); } diff --git a/package.json b/package.json index 113234da6..244cdfa6d 100644 --- a/package.json +++ b/package.json @@ -140,7 +140,6 @@ "strip-ansi": "^6.0.0", "toastr": "^2.1.4", "uuid": "^3.3.2", - "x256": "^0.0.2", "xterm": "^3.8.0", "yaml": "^1.10.2", "yup": "^0.32.11", diff --git a/yarn.lock b/yarn.lock index 3854e115f..022e55f14 100644 --- a/yarn.lock +++ b/yarn.lock @@ -18981,11 +18981,6 @@ x-default-browser@^0.4.0: optionalDependencies: default-browser-id "^1.0.4" -x256@^0.0.2: - version "0.0.2" - resolved "https://registry.yarnpkg.com/x256/-/x256-0.0.2.tgz#c9af18876f7a175801d564fe70ad9e8317784934" - integrity sha1-ya8Yh296F1gB1WT+cK2egxd4STQ= - xml-name-validator@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-3.0.0.tgz#6ae73e06de4d8c6e47f9fb181f78d648ad457c6a"