fix(logging): default to pretty logging [EE-4371] (#7847)

* fix(logging): default to pretty logging EE-4371

* feat(app/logs): prettify stack traces in JSON logs

* feat(nomad/logs): prettify JSON logs in log viewer

* feat(kubernetes/logs): prettigy JSON logs in log viewers

* feat(app/logs): format and color zerolog prettified logs

* fix(app/logs): pre-parse logs when they are double serialized

Co-authored-by: andres-portainer <andres-portainer@users.noreply.github.com>
Co-authored-by: LP B <xAt0mZ@users.noreply.github.com>
pull/7917/head
andres-portainer 2022-10-20 11:33:54 -03:00 committed by GitHub
parent ee5600b6af
commit 535a26412f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
27 changed files with 935 additions and 279 deletions

View File

@ -62,6 +62,7 @@ func (*Service) ParseFlags(version string) (*portainer.CLIFlags, error) {
MaxBatchDelay: kingpin.Flag("max-batch-delay", "Maximum delay before a batch starts").Duration(),
SecretKeyName: kingpin.Flag("secret-key-name", "Secret key name for encryption and will be used as /run/secrets/<secret-key-name>.").Default(defaultSecretKeyName).String(),
LogLevel: kingpin.Flag("log-level", "Set the minimum logging level to show").Default("INFO").Enum("DEBUG", "INFO", "WARN", "ERROR"),
LogMode: kingpin.Flag("log-mode", "Set the logging output mode").Default("PRETTY").Enum("PRETTY", "JSON"),
}
kingpin.Parse()

View File

@ -1,7 +1,9 @@
package main
import (
"fmt"
stdlog "log"
"os"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
@ -31,3 +33,23 @@ func setLoggingLevel(level string) {
zerolog.SetGlobalLevel(zerolog.DebugLevel)
}
}
func setLoggingMode(mode string) {
switch mode {
case "PRETTY":
log.Logger = log.Output(zerolog.ConsoleWriter{
Out: os.Stderr,
NoColor: true,
TimeFormat: "2006/01/02 03:04PM",
FormatMessage: formatMessage})
case "JSON":
log.Logger = log.Output(os.Stderr)
}
}
func formatMessage(i interface{}) string {
if i == nil {
return ""
}
return fmt.Sprintf("%s |", i)
}

View File

@ -758,10 +758,12 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
func main() {
configureLogger()
setLoggingMode("PRETTY")
flags := initCLI()
setLoggingLevel(*flags.LogLevel)
setLoggingMode(*flags.LogMode)
for {
server := buildServer(flags)

View File

@ -130,6 +130,7 @@ type (
MaxBatchDelay *time.Duration
SecretKeyName *string
LogLevel *string
LogMode *string
}
// CustomTemplateVariableDefinition

View File

@ -86,7 +86,7 @@
<div class="row" style="height: 54%">
<div class="col-sm-12" style="height: 100%">
<pre ng-class="{ wrap_lines: $ctrl.state.wrapLines }" class="log_viewer" scroll-glue="$ctrl.state.autoScroll" force-glue>
<div ng-repeat="log in $ctrl.state.filteredLogs = ($ctrl.data | filter:{ 'line': $ctrl.state.search }) track by $index" class="line" ng-if="log.line"><p class="inner_line" ng-click="$ctrl.selectLine(log.line)" ng-class="{ 'line_selected': $ctrl.state.selectedLines.indexOf(log.line) > -1 }"><span ng-repeat="span in log.spans track by $index" ng-style="{ 'color': span.foregroundColor, 'background-color': span.backgroundColor, 'font-weight': span.fontWeight }">{{ span.text }}</span></p></div>
<div ng-repeat="log in $ctrl.state.filteredLogs = ($ctrl.data | filter:{ 'line': $ctrl.state.search }) track by $index" class="line" ng-if="log.line"><p class="inner_line" ng-click="$ctrl.selectLine(log.line)" ng-class="{ 'line_selected': $ctrl.state.selectedLines.indexOf(log.line) > -1 }"><span ng-repeat="span in log.spans track by $index" ng-style="{ 'color': span.fgColor, 'background-color': span.bgColor, 'font-weight': span.fontWeight }">{{ span.text }}</span></p></div>
<div ng-if="!$ctrl.state.filteredLogs.length" class="line"><p class="inner_line">No log line matching the '{{ $ctrl.state.search }}' filter</p></div>
<div ng-if="$ctrl.state.filteredLogs.length === 1 && !$ctrl.state.filteredLogs[0].line" class="line"><p class="inner_line">No logs available</p></div>
</pre>

View File

@ -1,6 +1,7 @@
import moment from 'moment';
import _ from 'lodash-es';
import { NEW_LINE_BREAKER } from '@/constants';
import { concatLogsToString } from '@/docker/helpers/logHelper';
angular.module('portainer.docker').controller('LogViewerController', [
'$scope',
@ -74,9 +75,8 @@ angular.module('portainer.docker').controller('LogViewerController', [
};
this.downloadLogs = function () {
// To make the feature of downloading container logs working both on Windows and Linux,
// we need to use correct new line breakers on corresponding OS.
const data = new Blob([_.reduce(this.state.filteredLogs, (acc, log) => acc + log.line + NEW_LINE_BREAKER, '')]);
const logsAsString = concatLogsToString(this.state.filteredLogs);
const data = new Blob([logsAsString]);
FileSaver.saveAs(data, this.resourceName + '_logs.txt');
};
},

View File

@ -1,223 +0,0 @@
import tokenize from '@nxmix/tokenize-ansi';
import x256 from 'x256';
import { takeRight, without } from 'lodash';
import { format } from 'date-fns';
const FOREGROUND_COLORS_BY_ANSI = {
black: x256.colors[0],
red: x256.colors[1],
green: x256.colors[2],
yellow: x256.colors[3],
blue: x256.colors[4],
magenta: x256.colors[5],
cyan: x256.colors[6],
white: x256.colors[7],
brightBlack: x256.colors[8],
brightRed: x256.colors[9],
brightGreen: x256.colors[10],
brightYellow: x256.colors[11],
brightBlue: x256.colors[12],
brightMagenta: x256.colors[13],
brightCyan: x256.colors[14],
brightWhite: x256.colors[15],
};
const BACKGROUND_COLORS_BY_ANSI = {
bgBlack: x256.colors[0],
bgRed: x256.colors[1],
bgGreen: x256.colors[2],
bgYellow: x256.colors[3],
bgBlue: x256.colors[4],
bgMagenta: x256.colors[5],
bgCyan: x256.colors[6],
bgWhite: x256.colors[7],
bgBrightBlack: x256.colors[8],
bgBrightRed: x256.colors[9],
bgBrightGreen: x256.colors[10],
bgBrightYellow: x256.colors[11],
bgBrightBlue: x256.colors[12],
bgBrightMagenta: x256.colors[13],
bgBrightCyan: x256.colors[14],
bgBrightWhite: x256.colors[15],
};
const TIMESTAMP_LENGTH = 31; // 30 for timestamp + 1 for trailing space
angular.module('portainer.docker').factory('LogHelper', [
function LogHelperFactory() {
'use strict';
var helper = {};
function stripHeaders(logs) {
logs = logs.substring(8);
logs = logs.replace(/\r?\n(.{8})/g, '\n');
return logs;
}
function stripEscapeCodes(logs) {
return logs.replace(/[\u001b\u009b][[()#;?]*(?:[0-9]{1,4}(?:;[0-9]{0,4})*)?[0-9A-ORZcf-nqry=><]/g, '');
}
function cssColorFromRgb(rgb) {
const [r, g, b] = rgb;
return `rgb(${r}, ${g}, ${b})`;
}
function extendedColorForToken(token) {
const colorMode = token[1];
if (colorMode === 2) {
return cssColorFromRgb(token.slice(2));
}
if (colorMode === 5 && x256.colors[token[2]]) {
return cssColorFromRgb(x256.colors[token[2]]);
}
return '';
}
// Return an array with each log including a line and styled spans for each entry.
// If the stripHeaders param is specified, it will strip the 8 first characters of each line.
// withTimestamps param is needed to find the start of JSON for Zerolog logs parsing
helper.formatLogs = function (logs, { stripHeaders: skipHeaders, withTimestamps }) {
if (skipHeaders) {
logs = stripHeaders(logs);
}
const tokens = tokenize(logs);
const formattedLogs = [];
let foregroundColor = null;
let backgroundColor = null;
let line = '';
let spans = [];
for (const token of tokens) {
const type = token[0];
if (FOREGROUND_COLORS_BY_ANSI[type]) {
foregroundColor = cssColorFromRgb(FOREGROUND_COLORS_BY_ANSI[type]);
} else if (type === 'moreColor') {
foregroundColor = extendedColorForToken(token);
} else if (type === 'fgDefault') {
foregroundColor = null;
} else if (BACKGROUND_COLORS_BY_ANSI[type]) {
backgroundColor = cssColorFromRgb(BACKGROUND_COLORS_BY_ANSI[type]);
} else if (type === 'bgMoreColor') {
backgroundColor = extendedColorForToken(token);
} else if (type === 'bgDefault') {
backgroundColor = null;
} else if (type === 'reset') {
foregroundColor = null;
backgroundColor = null;
} else if (type === 'text') {
const tokenLines = token[1].split('\n');
for (let i = 0; i < tokenLines.length; i++) {
if (i !== 0) {
formattedLogs.push({ line, spans });
line = '';
spans = [];
}
const text = stripEscapeCodes(tokenLines[i]);
if ((!withTimestamps && text.startsWith('{')) || (withTimestamps && text.substring(TIMESTAMP_LENGTH).startsWith('{'))) {
line += JSONToFormattedLine(text, spans, withTimestamps);
} else {
spans.push({ foregroundColor, backgroundColor, text });
line += text;
}
}
}
}
if (line) {
formattedLogs.push({ line, spans });
}
return formattedLogs;
};
return helper;
},
]);
const JSONColors = {
Grey: 'var(--text-log-viewer-color-json-grey)',
Magenta: 'var(--text-log-viewer-color-json-magenta)',
Yellow: 'var(--text-log-viewer-color-json-yellow)',
Green: 'var(--text-log-viewer-color-json-green)',
Red: 'var(--text-log-viewer-color-json-red)',
Blue: 'var(--text-log-viewer-color-json-blue)',
};
const spaceSpan = { text: ' ' };
function logLevelToSpan(level) {
switch (level) {
case 'debug':
return { foregroundColor: JSONColors.Grey, text: 'DBG', fontWeight: 'bold' };
case 'info':
return { foregroundColor: JSONColors.Green, text: 'INF', fontWeight: 'bold' };
case 'warn':
return { foregroundColor: JSONColors.Yellow, text: 'WRN', fontWeight: 'bold' };
case 'error':
return { foregroundColor: JSONColors.Red, text: 'ERR', fontWeight: 'bold' };
default:
return { text: level };
}
}
function JSONToFormattedLine(rawText, spans, withTimestamps) {
const text = withTimestamps ? rawText.substring(TIMESTAMP_LENGTH) : rawText;
const json = JSON.parse(text);
const { level, caller, message, time } = json;
let line = '';
if (withTimestamps) {
const timestamp = rawText.substring(0, TIMESTAMP_LENGTH);
spans.push({ text: timestamp });
line += `${timestamp}`;
}
if (time) {
const date = format(new Date(time * 1000), 'Y/MM/dd hh:mmaa');
spans.push({ foregroundColor: JSONColors.Grey, text: date }, spaceSpan);
line += `${date} `;
}
if (level) {
const levelSpan = logLevelToSpan(level);
spans.push(levelSpan, spaceSpan);
line += `${levelSpan.text} `;
}
if (caller) {
const trimmedCaller = takeRight(caller.split('/'), 2).join('/');
spans.push({ foregroundColor: JSONColors.Magenta, text: trimmedCaller, fontWeight: 'bold' }, spaceSpan);
spans.push({ foregroundColor: JSONColors.Blue, text: '>' }, spaceSpan);
line += `${trimmedCaller} > `;
}
const keys = without(Object.keys(json), 'time', 'level', 'caller', 'message');
if (message) {
spans.push({ foregroundColor: JSONColors.Magenta, text: `${message}` }, spaceSpan);
line += `${message} `;
if (keys.length) {
spans.push({ foregroundColor: JSONColors.Magenta, text: `|` }, spaceSpan);
line += '| ';
}
}
keys.forEach((key) => {
const value = json[key];
spans.push({ foregroundColor: JSONColors.Blue, text: `${key}=` });
spans.push({ foregroundColor: key === 'error' ? JSONColors.Red : JSONColors.Magenta, text: value });
spans.push(spaceSpan);
line += `${key}=${value} `;
});
return line;
}

View File

@ -0,0 +1,63 @@
// original code comes from https://www.npmjs.com/package/x256
// only picking the used parts as there is no type definition
// package is unmaintained and repository doesn't exist anymore
// colors scraped from
// http://www.calmar.ws/vim/256-xterm-24bit-rgb-color-chart.html
// %s/ *\d\+ \+#\([^ ]\+\)/\1\r/g
import rawColors from './rawColors.json';
export type RGBColor = [number, number, number];
export type TextColor = string | undefined;
function hexToRGB(hex: string): RGBColor {
const r = parseInt(hex.slice(0, 2), 16);
const g = parseInt(hex.slice(2, 4), 16);
const b = parseInt(hex.slice(4, 6), 16);
return [r, g, b];
}
export const colors = rawColors.map(hexToRGB);
export const FOREGROUND_COLORS_BY_ANSI: {
[k: string]: RGBColor;
} = {
black: colors[0],
red: colors[1],
green: colors[2],
yellow: colors[3],
blue: colors[4],
magenta: colors[5],
cyan: colors[6],
white: colors[7],
brightBlack: colors[8],
brightRed: colors[9],
brightGreen: colors[10],
brightYellow: colors[11],
brightBlue: colors[12],
brightMagenta: colors[13],
brightCyan: colors[14],
brightWhite: colors[15],
};
export const BACKGROUND_COLORS_BY_ANSI: {
[k: string]: RGBColor;
} = {
bgBlack: colors[0],
bgRed: colors[1],
bgGreen: colors[2],
bgYellow: colors[3],
bgBlue: colors[4],
bgMagenta: colors[5],
bgCyan: colors[6],
bgWhite: colors[7],
bgBrightBlack: colors[8],
bgBrightRed: colors[9],
bgBrightGreen: colors[10],
bgBrightYellow: colors[11],
bgBrightBlue: colors[12],
bgBrightMagenta: colors[13],
bgBrightCyan: colors[14],
bgBrightWhite: colors[15],
};

View File

@ -0,0 +1,7 @@
export {
type RGBColor,
type TextColor,
colors,
FOREGROUND_COLORS_BY_ANSI,
BACKGROUND_COLORS_BY_ANSI,
} from './colors';

View File

@ -0,0 +1,258 @@
[
"000000",
"800000",
"008000",
"808000",
"000080",
"800080",
"008080",
"c0c0c0",
"808080",
"ff0000",
"00ff00",
"ffff00",
"0000ff",
"ff00ff",
"00ffff",
"ffffff",
"000000",
"00005f",
"000087",
"0000af",
"0000d7",
"0000ff",
"005f00",
"005f5f",
"005f87",
"005faf",
"005fd7",
"005fff",
"008700",
"00875f",
"008787",
"0087af",
"0087d7",
"0087ff",
"00af00",
"00af5f",
"00af87",
"00afaf",
"00afd7",
"00afff",
"00d700",
"00d75f",
"00d787",
"00d7af",
"00d7d7",
"00d7ff",
"00ff00",
"00ff5f",
"00ff87",
"00ffaf",
"00ffd7",
"00ffff",
"5f0000",
"5f005f",
"5f0087",
"5f00af",
"5f00d7",
"5f00ff",
"5f5f00",
"5f5f5f",
"5f5f87",
"5f5faf",
"5f5fd7",
"5f5fff",
"5f8700",
"5f875f",
"5f8787",
"5f87af",
"5f87d7",
"5f87ff",
"5faf00",
"5faf5f",
"5faf87",
"5fafaf",
"5fafd7",
"5fafff",
"5fd700",
"5fd75f",
"5fd787",
"5fd7af",
"5fd7d7",
"5fd7ff",
"5fff00",
"5fff5f",
"5fff87",
"5fffaf",
"5fffd7",
"5fffff",
"870000",
"87005f",
"870087",
"8700af",
"8700d7",
"8700ff",
"875f00",
"875f5f",
"875f87",
"875faf",
"875fd7",
"875fff",
"878700",
"87875f",
"878787",
"8787af",
"8787d7",
"8787ff",
"87af00",
"87af5f",
"87af87",
"87afaf",
"87afd7",
"87afff",
"87d700",
"87d75f",
"87d787",
"87d7af",
"87d7d7",
"87d7ff",
"87ff00",
"87ff5f",
"87ff87",
"87ffaf",
"87ffd7",
"87ffff",
"af0000",
"af005f",
"af0087",
"af00af",
"af00d7",
"af00ff",
"af5f00",
"af5f5f",
"af5f87",
"af5faf",
"af5fd7",
"af5fff",
"af8700",
"af875f",
"af8787",
"af87af",
"af87d7",
"af87ff",
"afaf00",
"afaf5f",
"afaf87",
"afafaf",
"afafd7",
"afafff",
"afd700",
"afd75f",
"afd787",
"afd7af",
"afd7d7",
"afd7ff",
"afff00",
"afff5f",
"afff87",
"afffaf",
"afffd7",
"afffff",
"d70000",
"d7005f",
"d70087",
"d700af",
"d700d7",
"d700ff",
"d75f00",
"d75f5f",
"d75f87",
"d75faf",
"d75fd7",
"d75fff",
"d78700",
"d7875f",
"d78787",
"d787af",
"d787d7",
"d787ff",
"d7af00",
"d7af5f",
"d7af87",
"d7afaf",
"d7afd7",
"d7afff",
"d7d700",
"d7d75f",
"d7d787",
"d7d7af",
"d7d7d7",
"d7d7ff",
"d7ff00",
"d7ff5f",
"d7ff87",
"d7ffaf",
"d7ffd7",
"d7ffff",
"ff0000",
"ff005f",
"ff0087",
"ff00af",
"ff00d7",
"ff00ff",
"ff5f00",
"ff5f5f",
"ff5f87",
"ff5faf",
"ff5fd7",
"ff5fff",
"ff8700",
"ff875f",
"ff8787",
"ff87af",
"ff87d7",
"ff87ff",
"ffaf00",
"ffaf5f",
"ffaf87",
"ffafaf",
"ffafd7",
"ffafff",
"ffd700",
"ffd75f",
"ffd787",
"ffd7af",
"ffd7d7",
"ffd7ff",
"ffff00",
"ffff5f",
"ffff87",
"ffffaf",
"ffffd7",
"ffffff",
"080808",
"121212",
"1c1c1c",
"262626",
"303030",
"3a3a3a",
"444444",
"4e4e4e",
"585858",
"606060",
"666666",
"767676",
"808080",
"8a8a8a",
"949494",
"9e9e9e",
"a8a8a8",
"b2b2b2",
"bcbcbc",
"c6c6c6",
"d0d0d0",
"dadada",
"e4e4e4",
"eeeeee"
]

View File

@ -0,0 +1,15 @@
import { NEW_LINE_BREAKER } from '@/constants';
import { FormattedLine } from './types';
type FormatFunc = (line: FormattedLine) => string;
export function concatLogsToString(
logs: FormattedLine[],
formatFunc: FormatFunc = (line) => line.line
) {
return logs.reduce(
(acc, formattedLine) => acc + formatFunc(formattedLine) + NEW_LINE_BREAKER,
''
);
}

View File

@ -0,0 +1,55 @@
import { without } from 'lodash';
import { FormattedLine, Span, JSONLogs, TIMESTAMP_LENGTH } from './types';
import {
formatCaller,
formatKeyValuePair,
formatLevel,
formatMessage,
formatStackTrace,
formatTime,
} from './formatters';
function removeKnownKeys(keys: string[]) {
return without(keys, 'time', 'level', 'caller', 'message', 'stack_trace');
}
export function formatJSONLine(
rawText: string,
withTimestamps?: boolean
): FormattedLine[] {
const spans: Span[] = [];
const lines: FormattedLine[] = [];
let line = '';
const text = withTimestamps ? rawText.substring(TIMESTAMP_LENGTH) : rawText;
const json: JSONLogs = JSON.parse(text);
const { time, level, caller, message, stack_trace: stackTrace } = json;
const keys = removeKnownKeys(Object.keys(json));
if (withTimestamps) {
const timestamp = rawText.substring(0, TIMESTAMP_LENGTH);
spans.push({ text: timestamp });
line += `${timestamp}`;
}
line += formatTime(time, spans, line);
line += formatLevel(level, spans, line);
line += formatCaller(caller, spans, line);
line += formatMessage(message, spans, line, !!keys.length);
keys.forEach((key, idx) => {
line += formatKeyValuePair(
key,
json[key],
spans,
line,
idx === keys.length - 1
);
});
lines.push({ line, spans });
formatStackTrace(stackTrace, lines);
return lines;
}

View File

@ -0,0 +1,141 @@
import tokenize from '@nxmix/tokenize-ansi';
import { FontWeight } from 'xterm';
import {
colors,
BACKGROUND_COLORS_BY_ANSI,
FOREGROUND_COLORS_BY_ANSI,
RGBColor,
} from './colors';
import { formatJSONLine } from './formatJSONLogs';
import { formatZerologLogs, ZerologRegex } from './formatZerologLogs';
import { Token, Span, TIMESTAMP_LENGTH, FormattedLine } from './types';
type FormatOptions = {
stripHeaders?: boolean;
withTimestamps?: boolean;
splitter?: string;
};
const defaultOptions: FormatOptions = {
splitter: '\n',
};
export function formatLogs(
rawLogs: string,
{
stripHeaders,
withTimestamps,
splitter = '\n',
}: FormatOptions = defaultOptions
) {
let logs = rawLogs;
if (stripHeaders) {
logs = stripHeadersFunc(logs);
}
if (logs.includes('\\n')) {
logs = JSON.parse(logs);
}
const tokens: Token[][] = tokenize(logs);
const formattedLogs: FormattedLine[] = [];
let fgColor: string | undefined;
let bgColor: string | undefined;
let fontWeight: FontWeight | undefined;
let line = '';
let spans: Span[] = [];
tokens.forEach((token) => {
const [type] = token;
const fgAnsi = FOREGROUND_COLORS_BY_ANSI[type];
const bgAnsi = BACKGROUND_COLORS_BY_ANSI[type];
if (fgAnsi) {
fgColor = cssColorFromRgb(fgAnsi);
} else if (type === 'moreColor') {
fgColor = extendedColorForToken(token);
} else if (type === 'fgDefault') {
fgColor = undefined;
} else if (bgAnsi) {
bgColor = cssColorFromRgb(bgAnsi);
} else if (type === 'bgMoreColor') {
bgColor = extendedColorForToken(token);
} else if (type === 'bgDefault') {
bgColor = undefined;
} else if (type === 'reset') {
fgColor = undefined;
bgColor = undefined;
fontWeight = undefined;
} else if (type === 'bold') {
fontWeight = 'bold';
} else if (type === 'normal') {
fontWeight = 'normal';
} else if (type === 'text') {
const tokenLines = (token[1] as string).split(splitter);
tokenLines.forEach((tokenLine, idx) => {
if (idx && line) {
formattedLogs.push({ line, spans });
line = '';
spans = [];
}
const text = stripEscapeCodes(tokenLine);
if (
(!withTimestamps && text.startsWith('{')) ||
(withTimestamps && text.substring(TIMESTAMP_LENGTH).startsWith('{'))
) {
const lines = formatJSONLine(text, withTimestamps);
formattedLogs.push(...lines);
} else if (ZerologRegex.test(text)) {
const lines = formatZerologLogs(text, withTimestamps);
formattedLogs.push(...lines);
} else {
spans.push({ fgColor, bgColor, text, fontWeight });
line += text;
}
});
}
});
if (line) {
formattedLogs.push({ line, spans });
}
return formattedLogs;
}
function stripHeadersFunc(logs: string) {
return logs.substring(8).replace(/\r?\n(.{8})/g, '\n');
}
function stripEscapeCodes(logs: string) {
return logs.replace(
// eslint-disable-next-line no-control-regex
/[\u001b\u009b][[()#;?]*(?:[0-9]{1,4}(?:;[0-9]{0,4})*)?[0-9A-ORZcf-nqry=><]/g,
''
);
}
function cssColorFromRgb(rgb: RGBColor) {
const [r, g, b] = rgb;
return `rgb(${r}, ${g}, ${b})`;
}
// assuming types based on original JS implementation
// there is not much type definitions for the tokenize library
function extendedColorForToken(token: Token[]) {
const [, colorMode, colorRef] = token as [undefined, number, number];
if (colorMode === 2) {
return cssColorFromRgb(token.slice(2) as RGBColor);
}
if (colorMode === 5 && colors[colorRef]) {
return cssColorFromRgb(colors[colorRef]);
}
return '';
}

View File

@ -0,0 +1,119 @@
import {
formatCaller,
formatKeyValuePair,
formatLevel,
formatMessage,
formatStackTrace,
formatTime,
} from './formatters';
import {
FormattedLine,
JSONStackTrace,
Level,
Span,
TIMESTAMP_LENGTH,
} from './types';
const dateRegex = /(\d{4}\/\d{2}\/\d{2} \d{2}:\d{2}[AP]M) /; // "2022/02/01 04:30AM "
const levelRegex = /(\w{3}) /; // "INF " or "ERR "
const callerRegex = /(.+?.go:\d+) /; // "path/to/file.go:line "
const chevRegex = /> /; // "> "
const messageAndPairsRegex = /(.*)/; // include the rest of the string in a separate group
const keyRegex = /(\S+=)/g; // ""
export const ZerologRegex = concatRegex(
dateRegex,
levelRegex,
callerRegex,
chevRegex,
messageAndPairsRegex
);
function concatRegex(...regs: RegExp[]) {
const flags = Array.from(
new Set(
regs
.map((r) => r.flags)
.join('')
.split('')
)
).join('');
const source = regs.map((r) => r.source).join('');
return new RegExp(source, flags);
}
type Pair = {
key: string;
value: string;
};
export function formatZerologLogs(rawText: string, withTimestamps?: boolean) {
const spans: Span[] = [];
const lines: FormattedLine[] = [];
let line = '';
const text = withTimestamps ? rawText.substring(TIMESTAMP_LENGTH) : rawText;
const [, date, level, caller, messageAndPairs] =
text.match(ZerologRegex) || [];
const [message, pairs] = extractPairs(messageAndPairs);
line += formatTime(date, spans, line);
line += formatLevel(level as Level, spans, line);
line += formatCaller(caller, spans, line);
line += formatMessage(message, spans, line, !!pairs.length);
let stackTrace: JSONStackTrace | undefined;
const stackTraceIndex = pairs.findIndex((p) => p.key === 'stack_trace');
if (stackTraceIndex !== -1) {
stackTrace = JSON.parse(pairs[stackTraceIndex].value);
pairs.splice(stackTraceIndex);
}
pairs.forEach(({ key, value }, idx) => {
line += formatKeyValuePair(
key,
value,
spans,
line,
idx === pairs.length - 1
);
});
lines.push({ line, spans });
formatStackTrace(stackTrace, lines);
return lines;
}
function extractPairs(messageAndPairs: string): [string, Pair[]] {
const pairs: Pair[] = [];
let [message, rawPairs] = messageAndPairs.split('|');
if (!messageAndPairs.includes('|') && !rawPairs) {
rawPairs = message;
message = '';
}
message = message.trim();
rawPairs = rawPairs.trim();
const matches = [...rawPairs.matchAll(keyRegex)];
matches.forEach((m, idx) => {
const rawKey = m[0];
const key = rawKey.slice(0, -1);
const start = m.index || 0;
const end = idx !== matches.length - 1 ? matches[idx + 1].index : undefined;
const value = (
end
? rawPairs.slice(start + rawKey.length, end)
: rawPairs.slice(start + rawKey.length)
).trim();
pairs.push({ key, value });
});
return [message, pairs];
}

View File

@ -0,0 +1,154 @@
import { format } from 'date-fns';
import { takeRight } from 'lodash';
import { Span, Level, Colors, JSONStackTrace, FormattedLine } from './types';
const spaceSpan: Span = { text: ' ' };
function logLevelToSpan(level: Level): Span {
switch (level) {
case 'debug':
case 'DBG':
return {
fgColor: Colors.Grey,
text: 'DBG',
fontWeight: 'bold',
};
case 'info':
case 'INF':
return {
fgColor: Colors.Green,
text: 'INF',
fontWeight: 'bold',
};
case 'warn':
case 'WRN':
return {
fgColor: Colors.Yellow,
text: 'WRN',
fontWeight: 'bold',
};
case 'error':
case 'ERR':
return {
fgColor: Colors.Red,
text: 'ERR',
fontWeight: 'bold',
};
default:
return { text: level };
}
}
export function formatTime(
time: number | string | undefined,
spans: Span[],
line: string
) {
let nl = line;
if (time) {
let date = '';
if (typeof time === 'number') {
date = format(new Date(time * 1000), 'Y/MM/dd hh:mmaa');
} else {
date = time;
}
spans.push({ fgColor: Colors.Grey, text: date }, spaceSpan);
nl += `${date} `;
}
return nl;
}
export function formatLevel(
level: Level | undefined,
spans: Span[],
line: string
) {
let nl = line;
if (level) {
const levelSpan = logLevelToSpan(level);
spans.push(levelSpan, spaceSpan);
nl += `${levelSpan.text} `;
}
return nl;
}
export function formatCaller(
caller: string | undefined,
spans: Span[],
line: string
) {
let nl = line;
if (caller) {
const trim = takeRight(caller.split('/'), 2).join('/');
spans.push(
{ fgColor: Colors.Magenta, text: trim, fontWeight: 'bold' },
spaceSpan
);
spans.push({ fgColor: Colors.Blue, text: '>' }, spaceSpan);
nl += `${trim} > `;
}
return nl;
}
export function formatMessage(
message: string,
spans: Span[],
line: string,
hasKeys: boolean
) {
let nl = line;
if (message) {
spans.push({ fgColor: Colors.Magenta, text: `${message}` }, spaceSpan);
nl += `${message} `;
if (hasKeys) {
spans.push({ fgColor: Colors.Magenta, text: `|` }, spaceSpan);
nl += '| ';
}
}
return nl;
}
export function formatKeyValuePair(
key: string,
value: unknown,
spans: Span[],
line: string,
isLastKey: boolean
) {
let nl = line;
spans.push(
{ fgColor: Colors.Blue, text: `${key}=` },
{
fgColor: key === 'error' || key === 'ERR' ? Colors.Red : Colors.Magenta,
text: value as string,
}
);
if (!isLastKey) spans.push(spaceSpan);
nl += `${key}=${value}${!isLastKey ? ' ' : ''}`;
return nl;
}
export function formatStackTrace(
stackTrace: JSONStackTrace | undefined,
lines: FormattedLine[]
) {
if (stackTrace) {
stackTrace.forEach(({ func, line: lineNumber, source }) => {
const line = ` at ${func} (${source}:${lineNumber})`;
const spans: Span[] = [
spaceSpan,
spaceSpan,
spaceSpan,
spaceSpan,
{ text: 'at ', fgColor: Colors.Grey },
{ text: func, fgColor: Colors.Red },
{ text: `(${source}:${lineNumber})`, fgColor: Colors.Grey },
];
lines.push({ line, spans });
});
}
}

View File

@ -0,0 +1,2 @@
export { formatLogs } from './formatLogs';
export { concatLogsToString } from './concatLogsToString';

View File

@ -0,0 +1,53 @@
import { FontWeight } from 'xterm';
import { type TextColor } from './colors';
export type Token = string | number;
export type Level =
| 'debug'
| 'info'
| 'warn'
| 'error'
| 'DBG'
| 'INF'
| 'WRN'
| 'ERR';
export type JSONStackTrace = {
func: string;
line: string;
source: string;
}[];
export type JSONLogs = {
[k: string]: unknown;
time: number;
level: Level;
caller: string;
message: string;
stack_trace?: JSONStackTrace;
};
export type Span = {
fgColor?: TextColor;
bgColor?: TextColor;
text: string;
fontWeight?: FontWeight;
};
export type FormattedLine = {
spans: Span[];
line: string;
};
export const TIMESTAMP_LENGTH = 31; // 30 for timestamp + 1 for trailing space
export const Colors = {
Grey: 'var(--text-log-viewer-color-json-grey)',
Magenta: 'var(--text-log-viewer-color-json-magenta)',
Yellow: 'var(--text-log-viewer-color-json-yellow)',
Green: 'var(--text-log-viewer-color-json-green)',
Red: 'var(--text-log-viewer-color-json-red)',
Blue: 'var(--text-log-viewer-color-json-blue)',
};

View File

@ -10,11 +10,12 @@ import {
stopContainer,
} from '@/react/docker/containers/containers.service';
import { ContainerDetailsViewModel, ContainerStatsViewModel, ContainerViewModel } from '../models/container';
import { formatLogs } from '../helpers/logHelper';
angular.module('portainer.docker').factory('ContainerService', ContainerServiceFactory);
/* @ngInject */
function ContainerServiceFactory($q, Container, LogHelper, $timeout, EndpointProvider) {
function ContainerServiceFactory($q, Container, $timeout, EndpointProvider) {
const service = {
killContainer: withEndpointId(killContainer),
pauseContainer: withEndpointId(pauseContainer),
@ -159,7 +160,7 @@ function ContainerServiceFactory($q, Container, LogHelper, $timeout, EndpointPro
Container.logs(parameters)
.$promise.then(function success(data) {
var logs = LogHelper.formatLogs(data.logs, { stripHeaders, withTimestamps: !!timestamps });
var logs = formatLogs(data.logs, { stripHeaders, withTimestamps: !!timestamps });
deferred.resolve(logs);
})
.catch(function error(err) {

View File

@ -1,13 +1,10 @@
import { formatLogs } from '../helpers/logHelper';
import { ServiceViewModel } from '../models/service';
angular.module('portainer.docker').factory('ServiceService', [
'$q',
'Service',
'ServiceHelper',
'TaskService',
'ResourceControlService',
'LogHelper',
function ServiceServiceFactory($q, Service, ServiceHelper, TaskService, ResourceControlService, LogHelper) {
function ServiceServiceFactory($q, Service) {
'use strict';
var service = {};
@ -88,7 +85,7 @@ angular.module('portainer.docker').factory('ServiceService', [
Service.logs(parameters)
.$promise.then(function success(data) {
var logs = LogHelper.formatLogs(data.logs, { stripHeaders: true, withTimestamps: !!timestamps });
var logs = formatLogs(data.logs, { stripHeaders: true, withTimestamps: !!timestamps });
deferred.resolve(logs);
})
.catch(function error(err) {

View File

@ -1,10 +1,10 @@
import { formatLogs } from '../helpers/logHelper';
import { TaskViewModel } from '../models/task';
angular.module('portainer.docker').factory('TaskService', [
'$q',
'Task',
'LogHelper',
function TaskServiceFactory($q, Task, LogHelper) {
function TaskServiceFactory($q, Task) {
'use strict';
var service = {};
@ -54,7 +54,7 @@ angular.module('portainer.docker').factory('TaskService', [
Task.logs(parameters)
.$promise.then(function success(data) {
var logs = LogHelper.formatLogs(data.logs, { stripHeaders: true, withTimestamps: !!timestamps });
var logs = formatLogs(data.logs, { stripHeaders: true, withTimestamps: !!timestamps });
deferred.resolve(logs);
})
.catch(function error(err) {

View File

@ -67,7 +67,7 @@ class KubernetesPodService {
params.container = containerName;
}
const data = await this.KubernetesPods(namespace).logs(params).$promise;
return data.logs.length === 0 ? [] : data.logs.split('\n');
return data.logs;
} catch (err) {
throw new PortainerError('Unable to retrieve pod logs', err);
}

View File

@ -77,9 +77,10 @@
<div class="row">
<div class="col-sm-12 h-[max(400px,calc(100vh-380px))]">
<pre
class="log_viewer widget"
><div ng-repeat="line in ctrl.state.filteredLogs = (ctrl.applicationLogs | filter:ctrl.state.search) track by $index" class="line" ng-if="line"><p class="inner_line">{{ line }}</p></div><div ng-if="ctrl.applicationLogs.length && !ctrl.state.filteredLogs.length" class="line"><p class="inner_line">No log line matching the '{{ ctrl.state.search }}' filter</p></div><div ng-if="ctrl.applicationLogs.length === 0" class="line"><p class="inner_line">No logs available</p></div></pre>
<pre class="log_viewer widget">
<div ng-repeat="log in ctrl.state.filteredLogs = (ctrl.applicationLogs | filter:{ 'line': ctrl.state.search }) track by $index" class="line" ng-if="log.line"><p class="inner_line"><span ng-repeat="span in log.spans track by $index" ng-style="{ 'color': span.fgColor, 'background-color': span.bgColor, 'font-weight': span.fontWeight }">{{ span.text }}</span></p></div>
<div ng-if="ctrl.applicationLogs.length && !ctrl.state.filteredLogs.length" class="line"><p class="inner_line">No log line matching the '{{ ctrl.state.search }}' filter</p></div>
<div ng-if="ctrl.applicationLogs.length === 0" class="line"><p class="inner_line">No logs available</p></div></pre>
</div>
</div>
</div>

View File

@ -1,5 +1,6 @@
import angular from 'angular';
import _ from 'lodash-es';
import { concatLogsToString, formatLogs } from '@/docker/helpers/logHelper';
class KubernetesApplicationLogsController {
/* @ngInject */
@ -39,13 +40,15 @@ class KubernetesApplicationLogsController {
}
downloadLogs() {
const data = new this.Blob([_.reduce(this.applicationLogs, (acc, log) => acc + '\n' + log, '')]);
const logsAsString = concatLogsToString(this.applicationLogs);
const data = new this.Blob([logsAsString]);
this.FileSaver.saveAs(data, this.podName + '_logs.txt');
}
async getApplicationLogsAsync() {
try {
this.applicationLogs = await this.KubernetesPodService.logs(this.application.ResourcePool, this.podName, this.containerName);
const rawLogs = await this.KubernetesPodService.logs(this.application.ResourcePool, this.podName, this.containerName);
this.applicationLogs = formatLogs(rawLogs);
} catch (err) {
this.stopRepeater();
this.Notifications.error('Failure', err, 'Unable to retrieve application logs');
@ -70,13 +73,8 @@ class KubernetesApplicationLogsController {
this.containerName = containerName;
try {
const [application, applicationLogs] = await Promise.all([
this.KubernetesApplicationService.get(namespace, applicationName),
this.KubernetesPodService.logs(namespace, podName, containerName),
]);
this.application = application;
this.applicationLogs = applicationLogs;
this.application = await this.KubernetesApplicationService.get(namespace, applicationName);
await this.getApplicationLogsAsync();
} catch (err) {
this.Notifications.error('Failure', err, 'Unable to retrieve application logs');
} finally {

View File

@ -69,9 +69,11 @@ ctrl.state.transition.name,
<div class="row">
<div class="col-sm-12 h-[max(400px,calc(100vh-380px))]">
<pre
class="log_viewer"
><div ng-repeat="line in ctrl.state.filteredLogs = (ctrl.stackLogs | filter:ctrl.state.search) track by $index" class="line" ng-if="line"><p class="inner_line"><span ng-style="{'color': line.Color, 'font-weight': 'bold'};">{{ line.AppName }}</span> {{ line.Line }}</p></div><div ng-if="ctrl.stackLogs.length && !ctrl.state.filteredLogs.length" class="line"><p class="inner_line">No log line matching the '{{ ctrl.state.search }}' filter</p></div><div ng-if="ctrl.stackLogs.length === 0" class="line"><p class="inner_line">No logs available</p></div></pre>
<pre class="log_viewer">
<div ng-repeat="log in ctrl.state.filteredLogs = (ctrl.stackLogs | filter:{ 'line': ctrl.state.search }) track by $index" class="line" ng-if="log.line"><p class="inner_line"><span ng-style="{'color': log.appColor, 'font-weight': 'bold'};">{{ log.appName }}</span> <span ng-repeat="span in log.spans track by $index" ng-style="{ 'color': span.fgColor, 'background-color': span.bgColor, 'font-weight': span.fontWeight }">{{ span.text }}</span></p></div>
<div ng-if="ctrl.stackLogs.length && !ctrl.state.filteredLogs.length" class="line"><p class="inner_line">No log line matching the '{{ ctrl.state.search }}' filter</p></div>
<div ng-if="ctrl.stackLogs.length === 0" class="line"><p class="inner_line">No logs available</p></div>
</pre>
</div>
</div>
</div>

View File

@ -1,6 +1,7 @@
import _ from 'lodash-es';
import { filter, flatMap, map } from 'lodash';
import angular from 'angular';
import $allSettled from 'Portainer/services/allSettled';
import { concatLogsToString, formatLogs } from '@/docker/helpers/logHelper';
const colors = ['red', 'orange', 'lime', 'green', 'darkgreen', 'cyan', 'turquoise', 'teal', 'deepskyblue', 'blue', 'darkblue', 'slateblue', 'magenta', 'darkviolet'];
@ -58,7 +59,7 @@ class KubernetesStackLogsController {
Pods: [],
};
const promises = _.flatMap(_.map(app.Pods, (pod) => _.map(pod.Containers, (container) => this.generateLogsPromise(pod, container))));
const promises = flatMap(map(app.Pods, (pod) => map(pod.Containers, (container) => this.generateLogsPromise(pod, container))));
const result = await $allSettled(promises);
res.Pods = result.fulfilled;
return res;
@ -67,21 +68,12 @@ class KubernetesStackLogsController {
async getStackLogsAsync() {
try {
const applications = await this.KubernetesApplicationService.get(this.state.transition.namespace);
const filteredApplications = _.filter(applications, (app) => app.StackName === this.state.transition.name);
const logsPromises = _.map(filteredApplications, this.generateAppPromise);
const filteredApplications = filter(applications, (app) => app.StackName === this.state.transition.name);
const logsPromises = map(filteredApplications, this.generateAppPromise);
const data = await Promise.all(logsPromises);
const logs = _.flatMap(data, (app, index) => {
return _.flatMap(app.Pods, (pod) => {
return _.map(pod.Logs, (line) => {
const res = {
Color: colors[index % colors.length],
Line: line,
AppName: pod.Pod.Name,
};
return res;
});
});
});
const logs = flatMap(data, (app, index) =>
flatMap(app.Pods, (pod) => formatLogs(pod.Logs).map((line) => ({ ...line, appColor: colors[index % colors.length], appName: pod.Pod.Name })))
);
this.stackLogs = logs;
} catch (err) {
this.stopRepeater();
@ -90,7 +82,8 @@ class KubernetesStackLogsController {
}
downloadLogs() {
const data = new this.Blob([(this.dataLogs = _.reduce(this.stackLogs, (acc, log) => acc + '\n' + log.AppName + ' ' + log.Line, ''))]);
const logsAsString = concatLogsToString(this.state.filteredLogs, (line) => `${line.appName} ${line.line}`);
const data = new this.Blob([logsAsString]);
this.FileSaver.saveAs(data, this.state.transition.name + '_logs.txt');
}

View File

@ -140,7 +140,6 @@
"strip-ansi": "^6.0.0",
"toastr": "^2.1.4",
"uuid": "^3.3.2",
"x256": "^0.0.2",
"xterm": "^3.8.0",
"yaml": "^1.10.2",
"yup": "^0.32.11",

View File

@ -18981,11 +18981,6 @@ x-default-browser@^0.4.0:
optionalDependencies:
default-browser-id "^1.0.4"
x256@^0.0.2:
version "0.0.2"
resolved "https://registry.yarnpkg.com/x256/-/x256-0.0.2.tgz#c9af18876f7a175801d564fe70ad9e8317784934"
integrity sha1-ya8Yh296F1gB1WT+cK2egxd4STQ=
xml-name-validator@^3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-3.0.0.tgz#6ae73e06de4d8c6e47f9fb181f78d648ad457c6a"