From 3dfdd3c31cba2999504d83c291d5c0e196c4e432 Mon Sep 17 00:00:00 2001 From: xiaojunnuo Date: Fri, 28 Oct 2022 16:08:12 +0800 Subject: [PATCH] refactor: pipeline run log --- packages/basic/acme-client | 2 +- packages/core/pipeline/package.json | 2 +- packages/core/pipeline/src/core/executor.ts | 129 ++++++++++++------ .../core/pipeline/src/core/run-history.ts | 54 +++++--- packages/core/pipeline/src/core/storage.ts | 13 +- packages/core/pipeline/src/d.ts/pipeline.ts | 23 +++- .../src/dns-provider/abstract-dns-provider.ts | 9 +- .../core/pipeline/src/dns-provider/api.ts | 11 +- .../providers/aliyun-dns-provider.ts | 1 - .../pipeline/src/plugin/abstract-plugin.ts | 6 +- .../src/plugin/plugins/cert-plugin/acme.ts | 35 ++--- .../src/plugin/plugins/cert-plugin/index.ts | 24 ++-- .../src/plugin/plugins/deploy-to-cdn/index.ts | 22 ++- .../src/plugin/plugins/echo-plugin.ts | 6 +- .../core/pipeline/src/registry/registry.ts | 25 +++- packages/core/pipeline/src/utils/util.log.ts | 36 ++++- packages/core/pipeline/test/cert.fake.test.ts | 58 ++++++++ .../test/pipeline/access-service-test.ts | 2 +- .../core/pipeline/test/pipeline/init.test.ts | 15 ++ .../pipeline/test/pipeline/pipeline.define.ts | 9 +- .../pipeline/test/pipeline/pipeline.test.ts | 4 +- .../test/pipeline/plugins/cert-apply.test.ts | 23 ++++ .../pipeline/plugins/deploy-to-cdn.test.ts | 23 ++++ packages/server/certd-client | 2 +- 24 files changed, 381 insertions(+), 153 deletions(-) create mode 100644 packages/core/pipeline/test/cert.fake.test.ts create mode 100644 packages/core/pipeline/test/pipeline/init.test.ts create mode 100644 packages/core/pipeline/test/pipeline/plugins/cert-apply.test.ts create mode 100644 packages/core/pipeline/test/pipeline/plugins/deploy-to-cdn.test.ts diff --git a/packages/basic/acme-client b/packages/basic/acme-client index a78bb43f..1c0e55a5 160000 --- a/packages/basic/acme-client +++ b/packages/basic/acme-client @@ -1 +1 @@ -Subproject commit a78bb43f6b65877b4f0aad25995d7cbf3215a3bc +Subproject commit 1c0e55a5339ef0ac92f96726c4c61e6918908730 diff --git a/packages/core/pipeline/package.json b/packages/core/pipeline/package.json index 4b98520e..50003f1e 100644 --- a/packages/core/pipeline/package.json +++ b/packages/core/pipeline/package.json @@ -39,7 +39,7 @@ "log4js": "^6.3.0", "mocha": "^10.1.0", "ts-node": "^10.9.1", - "typescript": "^4.6.4", + "typescript": "^4.8.4", "vite": "^3.1.0" } } diff --git a/packages/core/pipeline/src/core/executor.ts b/packages/core/pipeline/src/core/executor.ts index edbd60cb..9dba70b2 100644 --- a/packages/core/pipeline/src/core/executor.ts +++ b/packages/core/pipeline/src/core/executor.ts @@ -1,94 +1,136 @@ -import { ConcurrencyStrategy, Pipeline, Runnable, Stage, Step, Task } from "../d.ts/pipeline"; +import { ConcurrencyStrategy, HistoryResult, HistoryResultGroup, Pipeline, ResultType, Runnable, RunStrategy, Stage, Step, Task } from "../d.ts"; import _ from "lodash"; import { RunHistory } from "./run-history"; import { pluginRegistry, TaskPlugin } from "../plugin"; import { IAccessService } from "../access/access-service"; -import { ContextFactory } from "./context"; +import { ContextFactory, IContext } from "./context"; import { IStorage } from "./storage"; import { logger } from "../utils/util.log"; +import { Logger } from "log4js"; export class Executor { userId: any; pipeline: Pipeline; - runtime: RunHistory = new RunHistory(); - lastSuccessHistory: RunHistory; + runtime!: RunHistory; accessService: IAccessService; contextFactory: ContextFactory; + logger: Logger; + pipelineContext: IContext; onChanged: (history: RunHistory) => void; - constructor(options: { - userId: any; - pipeline: Pipeline; - storage: IStorage; - onChanged: (history: RunHistory) => void; - lastSuccessHistory?: RunHistory; - accessService: IAccessService; - }) { + constructor(options: { userId: any; pipeline: Pipeline; storage: IStorage; onChanged: (history: RunHistory) => void; accessService: IAccessService }) { this.pipeline = options.pipeline; - this.lastSuccessHistory = options.lastSuccessHistory ?? new RunHistory(); this.onChanged = options.onChanged; this.accessService = options.accessService; this.userId = options.userId; - this.contextFactory = new ContextFactory(options.storage); + this.logger = logger; + this.pipelineContext = this.contextFactory.getContext("pipeline", this.pipeline.id); } - async run() { - await this.runWithHistory(this.pipeline, async () => { - return await this.runStages(); - }); - } - - async runWithHistory(runnable: Runnable, run: () => Promise) { - this.runtime.start(runnable); - this.onChanged(this.runtime); + async run(runtimeId: any = 0) { try { - await run(); + this.runtime = new RunHistory(runtimeId); + this.logger.info(`pipeline.${this.pipeline.id} start`); + await this.runWithHistory(this.pipeline, "pipeline", async () => { + return await this.runStages(); + }); + } finally { + this.logger.info(`pipeline.${this.pipeline.id} end`); + } + } + + async runWithHistory(runnable: Runnable, runnableType: string, run: (result: HistoryResult) => Promise) { + const result = this.runtime.start(runnable, runnableType); + this.onChanged(this.runtime); + const contextKey = `status.${runnable.id}`; + if (runnable.strategy?.runStrategy === RunStrategy.SkipWhenSucceed) { + const lastResult = await this.pipelineContext.get(contextKey); + if (lastResult != null && lastResult === ResultType.success) { + this.runtime.log(runnable, result, "跳过"); + return ResultType.skip; + } + } + try { + await run(result); this.runtime.success(runnable); this.onChanged(this.runtime); + await this.pipelineContext.set(contextKey, ResultType.success); + return ResultType.success; } catch (e: any) { - logger.error(e); + this.logger.error(e); this.runtime.error(runnable, e); this.onChanged(this.runtime); + await this.pipelineContext.set(contextKey, ResultType.error); + throw e; + } finally { + this.runtime.finally(runnable); } } private async runStages() { + const resList: ResultType[] = []; for (const stage of this.pipeline.stages) { - await this.runWithHistory(stage, async () => { - return await this.runStage(stage); + const res: ResultType = await this.runWithHistory(stage, "stage", async (result) => { + return await this.runStage(stage, result); }); + resList.push(res); } + return this.compositionResultType(resList); } - async runStage(stage: Stage) { + async runStage(stage: Stage, result: HistoryResult) { const runnerList = []; for (const task of stage.tasks) { - const runner = this.runWithHistory(task, async () => { - return await this.runTask(task); + const runner = this.runWithHistory(task, "task", async (result) => { + return await this.runTask(task, result); }); runnerList.push(runner); } + + let resList: ResultType[] = []; if (stage.concurrency === ConcurrencyStrategy.Parallel) { - await Promise.all(runnerList); + resList = await Promise.all(runnerList); } else { - for (const runner of runnerList) { - await runner; + for (let i = 0; i < runnerList.length; i++) { + const runner = runnerList[i]; + resList[i] = await runner; } } + return this.compositionResultType(resList); } - private async runTask(task: Task) { - for (const step of task.steps) { - await this.runWithHistory(step, async () => { - return await this.runStep(step); - }); + compositionResultType(resList: ResultType[]) { + let hasSuccess = false; + for (const type of resList) { + if (type === ResultType.error) { + return ResultType.error; + } + if (type === ResultType.success) { + hasSuccess = true; + } } + if (hasSuccess) { + return ResultType.success; + } + return ResultType.error; } - private async runStep(step: Step) { - //执行任务 - const taskPlugin: TaskPlugin = await this.getPlugin(step.type); + private async runTask(task: Task, result: HistoryResult) { + const resList: ResultType[] = []; + for (const step of task.steps) { + const res: ResultType = await this.runWithHistory(step, "step", async (result) => { + await this.runStep(step, result); + }); + resList.push(res); + } + return this.compositionResultType(resList); + } + private async runStep(step: Step, result: HistoryResult) { + //执行任务 + const taskPlugin: TaskPlugin = await this.getPlugin(step.type, result.logger); + // @ts-ignore + delete taskPlugin.define; const define = taskPlugin.getDefine(); //从outputContext读取输入参数 _.forEach(define.input, (item, key) => { @@ -109,14 +151,15 @@ export class Executor { }); } - private async getPlugin(type: string): Promise { + private async getPlugin(type: string, logger: Logger): Promise { const pluginClass = pluginRegistry.get(type); // @ts-ignore const plugin = new pluginClass(); await plugin.doInit({ accessService: this.accessService, - pipelineContext: this.contextFactory.getContext("pipeline", this.pipeline.id), + pipelineContext: this.pipelineContext, userContext: this.contextFactory.getContext("user", this.userId), + logger, }); return plugin; } diff --git a/packages/core/pipeline/src/core/run-history.ts b/packages/core/pipeline/src/core/run-history.ts index 2d3713c9..a4987082 100644 --- a/packages/core/pipeline/src/core/run-history.ts +++ b/packages/core/pipeline/src/core/run-history.ts @@ -1,62 +1,74 @@ -import { Context, HistoryResult, Log, Runnable } from "../d.ts/pipeline"; +import { Context, HistoryResult, Log, Runnable } from "../d.ts"; import _ from "lodash"; +import { buildLogger, logger } from "../utils/util.log"; export class RunHistory { + constructor(runtimeId: any) { + this.id = runtimeId; + } + id: any; logs: Log[] = []; context: Context = {}; results: { [key: string]: HistoryResult; } = {}; + logger = logger; - start(runnable: Runnable) { - const status = "ing"; + start(runnable: Runnable, runnableType: string) { + const status = "start"; const now = new Date().getTime(); _.merge(runnable, { status, lastTime: now }); - this.results[runnable.id] = { + const result: HistoryResult = { + type: runnableType, startTime: new Date().getTime(), title: runnable.title, status, + logs: [], + logger: buildLogger((text) => { + result.logs.push(text); + }), }; - this.log(runnable, `${runnable.title}<${runnable.id}> 开始执行`); + this.results[runnable.id] = result; + this.log(runnable, result, `${runnable.title} 开始执行`); + return result; } - success(runnable: Runnable, result?: any) { + success(runnable: Runnable, res?: any) { const status = "success"; const now = new Date().getTime(); _.merge(runnable, { status, lastTime: now }); - _.merge(this.results[runnable.id], { status, endTime: now }, result); - this.log( - runnable, - `${this.results[runnable.id].title}<${runnable.id}> 执行成功` - ); + const result = this.results[runnable.id]; + _.merge(result, { status, endTime: now }, res); + this.log(runnable, result, `${result.title} 执行成功`); } error(runnable: Runnable, e: Error) { const status = "error"; const now = new Date().getTime(); _.merge(runnable, { status, lastTime: now }); - _.merge(this.results[runnable.id], { + const result = this.results[runnable.id]; + _.merge(result, { status, endTime: now, errorMessage: e.message, }); - this.log( - runnable, - `${this.results[runnable.id].title}<${runnable.id}> 执行异常：${ - e.message - }`, - status - ); + this.log(runnable, result, `${result.title} 执行异常：${e.message}`, status); } - log(runnable: Runnable, text: string, level = "info") { + log(runnable: Runnable, result: HistoryResult, text: string, level = "info") { + const now = new Date().getTime(); + result.logger.info(`[${runnable.title}] [${result.type}]`, text); this.logs.push({ - time: new Date().getTime(), + time: now, level, title: runnable.title, text, }); } + + finally(runnable: Runnable) { + // + } } diff --git a/packages/core/pipeline/src/core/storage.ts b/packages/core/pipeline/src/core/storage.ts index 863f62ee..3cafb503 100644 --- a/packages/core/pipeline/src/core/storage.ts +++ b/packages/core/pipeline/src/core/storage.ts @@ -7,11 +7,6 @@ export interface IStorage { } export class FileStorage implements IStorage { - /** - * 范围： user / pipeline / runtime / task - */ - scope: any; - namespace: any; root: string; constructor(rootDir?: string) { if (rootDir == null) { @@ -42,17 +37,17 @@ export class FileStorage implements IStorage { } async get(scope: string, namespace: string, key: string): Promise { - const path = `${this.root}/${this.scope}/${namespace}/${key}`; + const path = this.buildPath(scope, namespace, key); return this.readFile(path); } async set(scope: string, namespace: string, key: string, value: string): Promise { - const path = this.buildPath(namespace, key); + const path = this.buildPath(scope, namespace, key); this.writeFile(path, value); } - private buildPath(namespace: string, key: string) { - return `${this.root}/${this.scope}/${namespace}/${key}`; + private buildPath(scope: string, namespace: string, key: string) { + return `${this.root}/${scope}/${namespace}/${key}`; } } diff --git a/packages/core/pipeline/src/d.ts/pipeline.ts b/packages/core/pipeline/src/d.ts/pipeline.ts index e6457158..3ee8eec8 100644 --- a/packages/core/pipeline/src/d.ts/pipeline.ts +++ b/packages/core/pipeline/src/d.ts/pipeline.ts @@ -1,3 +1,5 @@ +import { Logger } from "log4js"; + export enum RunStrategy { AlwaysRun, SkipWhenSucceed, @@ -27,9 +29,9 @@ export type EventHandler = { }; export type RunnableStrategy = { - runStrategy: RunStrategy; - onSuccess: EventHandler[]; - onError: EventHandler[]; + runStrategy?: RunStrategy; + onSuccess?: EventHandler[]; + onError?: EventHandler[]; }; export type Step = Runnable & { @@ -79,7 +81,20 @@ export type Log = { text: string; }; +export enum ResultType { + success, + error, + skip, +} + +export type HistoryResultGroup = { + [key: string]: { + runnable: Runnable; + res: HistoryResult; + }; +}; export type HistoryResult = { + type: string; title: string; /** * 任务状态 @@ -92,4 +107,6 @@ export type HistoryResult = { */ result?: string; errorMessage?: string; + logs: string[]; + logger: Logger; }; diff --git a/packages/core/pipeline/src/dns-provider/abstract-dns-provider.ts b/packages/core/pipeline/src/dns-provider/abstract-dns-provider.ts index 1eddbae9..84a4d272 100644 --- a/packages/core/pipeline/src/dns-provider/abstract-dns-provider.ts +++ b/packages/core/pipeline/src/dns-provider/abstract-dns-provider.ts @@ -1,12 +1,13 @@ import { AbstractRegistrable } from "../registry"; import { CreateRecordOptions, IDnsProvider, DnsProviderDefine, RemoveRecordOptions } from "./api"; import { AbstractAccess } from "../access"; +import { Logger } from "log4js"; export abstract class AbstractDnsProvider extends AbstractRegistrable implements IDnsProvider { - // @ts-ignore - access: AbstractAccess; - - doInit(options: { access: AbstractAccess }) { + access!: AbstractAccess; + logger!: Logger; + doInit(options: { access: AbstractAccess; logger: Logger }) { this.access = options.access; + this.logger = options.logger; this.onInit(); } diff --git a/packages/core/pipeline/src/dns-provider/api.ts b/packages/core/pipeline/src/dns-provider/api.ts index ab3e18d3..ed3e511f 100644 --- a/packages/core/pipeline/src/dns-provider/api.ts +++ b/packages/core/pipeline/src/dns-provider/api.ts @@ -1,5 +1,6 @@ import { Registrable } from "../registry"; import { dnsProviderRegistry } from "./registry"; +import { AbstractDnsProvider } from "./abstract-dns-provider"; export type DnsProviderDefine = Registrable & { accessType: string; @@ -21,9 +22,13 @@ export interface IDnsProvider { removeRecord(options: RemoveRecordOptions): Promise; } -export function IsDnsProvider(define: DnsProviderDefine) { - return function (target: any) { - target.prototype.define = define; +export function IsDnsProvider(define: (() => DnsProviderDefine) | DnsProviderDefine) { + return function (target: typeof AbstractDnsProvider) { + if (define instanceof Function) { + target.prototype.define = define(); + } else { + target.prototype.define = define; + } dnsProviderRegistry.install(target); }; } diff --git a/packages/core/pipeline/src/dns-provider/providers/aliyun-dns-provider.ts b/packages/core/pipeline/src/dns-provider/providers/aliyun-dns-provider.ts index 33cd0a75..0dafb087 100644 --- a/packages/core/pipeline/src/dns-provider/providers/aliyun-dns-provider.ts +++ b/packages/core/pipeline/src/dns-provider/providers/aliyun-dns-provider.ts @@ -14,7 +14,6 @@ export class AliyunDnsProvider extends AbstractDnsProvider implements IDnsProvid constructor() { super(); } - async onInit() { const access: any = this.access; this.client = new Core({ diff --git a/packages/core/pipeline/src/plugin/abstract-plugin.ts b/packages/core/pipeline/src/plugin/abstract-plugin.ts index 125ecc96..db6ed9de 100644 --- a/packages/core/pipeline/src/plugin/abstract-plugin.ts +++ b/packages/core/pipeline/src/plugin/abstract-plugin.ts @@ -1,12 +1,11 @@ import { AbstractRegistrable } from "../registry"; import { Logger } from "log4js"; -import { logger } from "../utils/util.log"; import { IAccessService } from "../access/access-service"; import { IContext } from "../core/context"; import { PluginDefine, TaskInput, TaskOutput, TaskPlugin } from "./api"; export abstract class AbstractPlugin extends AbstractRegistrable implements TaskPlugin { - logger: Logger = logger; + logger!: Logger; // @ts-ignore accessService: IAccessService; // @ts-ignore @@ -14,10 +13,11 @@ export abstract class AbstractPlugin extends AbstractRegistrable i // @ts-ignore userContext: IContext; - async doInit(options: { accessService: IAccessService; pipelineContext: IContext; userContext: IContext }) { + async doInit(options: { accessService: IAccessService; pipelineContext: IContext; userContext: IContext; logger: Logger }) { this.accessService = options.accessService; this.pipelineContext = options.pipelineContext; this.userContext = options.userContext; + this.logger = options.logger; await this.onInit(); } diff --git a/packages/core/pipeline/src/plugin/plugins/cert-plugin/acme.ts b/packages/core/pipeline/src/plugin/plugins/cert-plugin/acme.ts index a882cd62..9fb95685 100644 --- a/packages/core/pipeline/src/plugin/plugins/cert-plugin/acme.ts +++ b/packages/core/pipeline/src/plugin/plugins/cert-plugin/acme.ts @@ -1,18 +1,19 @@ // @ts-ignore import * as acme from "@certd/acme-client"; import _ from "lodash"; -import { logger } from "../../../utils/util.log"; import { AbstractDnsProvider } from "../../../dns-provider/abstract-dns-provider"; import { IContext } from "../../../core/context"; import { IDnsProvider } from "../../../dns-provider"; import { Challenge } from "@certd/acme-client/types/rfc8555"; -console.log("acme", acme); +import { Logger } from "log4js"; export class AcmeService { userContext: IContext; - constructor(options: { userContext: IContext }) { + logger: Logger; + constructor(options: { userContext: IContext; logger: Logger }) { this.userContext = options.userContext; + this.logger = options.logger; acme.setLogger((text: string) => { - logger.info(text); + this.logger.info(text); }); } @@ -64,27 +65,27 @@ export class AcmeService { } async challengeCreateFn(authz: any, challenge: any, keyAuthorization: string, dnsProvider: IDnsProvider) { - logger.info("Triggered challengeCreateFn()"); + this.logger.info("Triggered challengeCreateFn()"); /* http-01 */ if (challenge.type === "http-01") { const filePath = `/var/www/html/.well-known/acme-challenge/${challenge.token}`; const fileContents = keyAuthorization; - logger.info(`Creating challenge response for ${authz.identifier.value} at path: ${filePath}`); + this.logger.info(`Creating challenge response for ${authz.identifier.value} at path: ${filePath}`); /* Replace this */ - logger.info(`Would write "${fileContents}" to path "${filePath}"`); + this.logger.info(`Would write "${fileContents}" to path "${filePath}"`); // await fs.writeFileAsync(filePath, fileContents); } else if (challenge.type === "dns-01") { /* dns-01 */ const dnsRecord = `_acme-challenge.${authz.identifier.value}`; const recordValue = keyAuthorization; - logger.info(`Creating TXT record for ${authz.identifier.value}: ${dnsRecord}`); + this.logger.info(`Creating TXT record for ${authz.identifier.value}: ${dnsRecord}`); /* Replace this */ - logger.info(`Would create TXT record "${dnsRecord}" with value "${recordValue}"`); + this.logger.info(`Would create TXT record "${dnsRecord}" with value "${recordValue}"`); return await dnsProvider.createRecord({ fullRecord: dnsRecord, @@ -106,25 +107,25 @@ export class AcmeService { */ async challengeRemoveFn(authz: any, challenge: any, keyAuthorization: string, recordItem: any, dnsProvider: IDnsProvider) { - logger.info("Triggered challengeRemoveFn()"); + this.logger.info("Triggered challengeRemoveFn()"); /* http-01 */ if (challenge.type === "http-01") { const filePath = `/var/www/html/.well-known/acme-challenge/${challenge.token}`; - logger.info(`Removing challenge response for ${authz.identifier.value} at path: ${filePath}`); + this.logger.info(`Removing challenge response for ${authz.identifier.value} at path: ${filePath}`); /* Replace this */ - logger.info(`Would remove file on path "${filePath}"`); + this.logger.info(`Would remove file on path "${filePath}"`); // await fs.unlinkAsync(filePath); } else if (challenge.type === "dns-01") { const dnsRecord = `_acme-challenge.${authz.identifier.value}`; const recordValue = keyAuthorization; - logger.info(`Removing TXT record for ${authz.identifier.value}: ${dnsRecord}`); + this.logger.info(`Removing TXT record for ${authz.identifier.value}: ${dnsRecord}`); /* Replace this */ - logger.info(`Would remove TXT record "${dnsRecord}" with value "${recordValue}"`); + this.logger.info(`Would remove TXT record "${dnsRecord}" with value "${recordValue}"`); await dnsProvider.removeRecord({ fullRecord: dnsRecord, type: "TXT", @@ -169,9 +170,9 @@ export class AcmeService { csr: csr.toString(), }; /* Done */ - logger.debug(`CSR:\n${cert.csr}`); - logger.debug(`Certificate:\n${cert.crt}`); - logger.info("证书申请成功"); + this.logger.debug(`CSR:\n${cert.csr}`); + this.logger.debug(`Certificate:\n${cert.crt}`); + this.logger.info("证书申请成功"); return cert; } diff --git a/packages/core/pipeline/src/plugin/plugins/cert-plugin/index.ts b/packages/core/pipeline/src/plugin/plugins/cert-plugin/index.ts index 5dd721fc..18854417 100644 --- a/packages/core/pipeline/src/plugin/plugins/cert-plugin/index.ts +++ b/packages/core/pipeline/src/plugin/plugins/cert-plugin/index.ts @@ -1,6 +1,6 @@ import { AbstractPlugin } from "../../abstract-plugin"; import forge from "node-forge"; -import { ContextScope, IsTask, TaskInput, TaskOutput, TaskPlugin } from "../../api"; +import { IsTask, TaskInput, TaskOutput, TaskPlugin } from "../../api"; import dayjs from "dayjs"; import { dnsProviderRegistry } from "../../../dns-provider"; import { AbstractDnsProvider } from "../../../dns-provider/abstract-dns-provider"; @@ -39,53 +39,49 @@ export type CertInfo = { dnsProviderType: { title: "DNS提供商", component: { - name: "a-select", + name: "pi-dns-provider-selector", }, helper: "请选择dns解析提供商", }, dnsProviderAccess: { title: "DNS解析授权", component: { - name: "access-selector", + name: "pi-access-selector", }, helper: "请选择dns解析提供商授权", }, renewDays: { title: "更新天数", + value: 20, component: { name: "a-input-number", - value: 20, }, helper: "到期前多少天后更新证书", }, forceUpdate: { title: "强制更新", + value: false, component: { name: "a-switch", vModel: "checked", - value: false, }, - helper: "强制重新申请证书", + helper: "是否强制重新申请证书", }, }, output: { cert: { key: "cert", type: "CertInfo", - title: "证书", - scope: ContextScope.pipeline, + title: "域名证书", }, }, }; }) -export class CertPlugin extends AbstractPlugin implements TaskPlugin { +export class CertApplyPlugin extends AbstractPlugin implements TaskPlugin { // @ts-ignore acme: AcmeService; - constructor() { - super(); - } protected async onInit() { - this.acme = new AcmeService({ userContext: this.userContext }); + this.acme = new AcmeService({ userContext: this.userContext, logger: this.logger }); } async execute(input: TaskInput): Promise { @@ -139,7 +135,7 @@ export class CertPlugin extends AbstractPlugin implements TaskPlugin { const access = await this.accessService.getById(dnsProviderAccessId); // @ts-ignore const dnsProvider: AbstractDnsProvider = new dnsProviderClass(); - dnsProvider.doInit({ access }); + dnsProvider.doInit({ access, logger: this.logger }); const cert = await this.acme.order({ email, diff --git a/packages/core/pipeline/src/plugin/plugins/deploy-to-cdn/index.ts b/packages/core/pipeline/src/plugin/plugins/deploy-to-cdn/index.ts index c8eeeaf4..bbce352b 100644 --- a/packages/core/pipeline/src/plugin/plugins/deploy-to-cdn/index.ts +++ b/packages/core/pipeline/src/plugin/plugins/deploy-to-cdn/index.ts @@ -10,33 +10,34 @@ import { CertInfo } from "../cert-plugin"; return { name: "DeployCertToAliyunCDN", title: "部署证书至阿里云CDN", + desc: "依赖证书申请前置任务，自动部署域名证书至阿里云CDN", input: { domainName: { - title: "cdn加速域名", + title: "CDN加速域名", component: { - placeholder: "cdn加速域名", + placeholder: "你在阿里云上配置的CDN加速域名，比如certd.docmirror.cn", }, required: true, }, certName: { title: "证书名称", component: { - placeholder: "上传后将以此名称作为前缀", + placeholder: "上传后将以此名称作为前缀备注", }, }, cert: { title: "域名证书", helper: "请选择前置任务输出的域名证书", component: { - name: "output-selector", + name: "pi-output-selector", }, required: true, }, accessId: { - title: "Access提供者", - helper: "access授权", + title: "Access授权", + helper: "阿里云授权AccessKeyId、AccessKeySecret", component: { - name: "access-selector", + name: "pi-access-selector", type: "aliyun", }, required: true, @@ -46,16 +47,13 @@ import { CertInfo } from "../cert-plugin"; }; }) export class DeployCertToAliyunCDN extends AbstractPlugin implements TaskPlugin { - constructor() { - super(); - } - async execute(input: TaskInput): Promise { console.log("开始部署证书到阿里云cdn"); const access = (await this.accessService.getById(input.accessId)) as AliyunAccess; const client = this.getClient(access); const params = await this.buildParams(input); await this.doRequest(client, params); + console.log("部署完成"); return {}; } @@ -70,7 +68,7 @@ export class DeployCertToAliyunCDN extends AbstractPlugin implements TaskPlugin async buildParams(input: TaskInput) { const { certName, domainName } = input; - const CertName = certName + "-" + dayjs().format("YYYYMMDDHHmmss"); + const CertName = (certName ?? "certd") + "-" + dayjs().format("YYYYMMDDHHmmss"); const cert = input.cert as CertInfo; return { RegionId: "cn-hangzhou", diff --git a/packages/core/pipeline/src/plugin/plugins/echo-plugin.ts b/packages/core/pipeline/src/plugin/plugins/echo-plugin.ts index 11670df8..a6b92ee6 100644 --- a/packages/core/pipeline/src/plugin/plugins/echo-plugin.ts +++ b/packages/core/pipeline/src/plugin/plugins/echo-plugin.ts @@ -4,12 +4,12 @@ import { IsTask, TaskInput, TaskOutput, TaskPlugin } from "../api"; @IsTask(() => { return { name: "EchoPlugin", - title: "测试插件回声", + title: "测试插件【echo】", input: { cert: { title: "cert", component: { - name: "output-selector", + name: "pi-output-selector", }, helper: "输出选择", }, @@ -20,7 +20,7 @@ import { IsTask, TaskInput, TaskOutput, TaskPlugin } from "../api"; export class EchoPlugin extends AbstractPlugin implements TaskPlugin { async execute(input: TaskInput): Promise { for (const key in input) { - console.log("input :", key, input[key]); + this.logger.info("input :", key, input[key]); } return input; } diff --git a/packages/core/pipeline/src/registry/registry.ts b/packages/core/pipeline/src/registry/registry.ts index 245f065b..08dff317 100644 --- a/packages/core/pipeline/src/registry/registry.ts +++ b/packages/core/pipeline/src/registry/registry.ts @@ -1,5 +1,4 @@ import { Logger } from "log4js"; -import { logger } from "../utils/util.log"; export type Registrable = { name: string; @@ -8,7 +7,6 @@ export type Registrable = { }; export abstract class AbstractRegistrable { - logger: Logger = logger; // @ts-ignore define: T; @@ -21,18 +19,20 @@ export class Registry { [key: string]: T; } = {}; - install(target: T) { - if (target == null) { + install(pluginClass: T) { + if (pluginClass == null) { return; } // @ts-ignore - const define = new target().define; + const plugin = new pluginClass(); + delete plugin.define; + const define = plugin.getDefine(); let defineName = define.name; if (defineName == null) { - defineName = target.name; + defineName = plugin.name; } - this.register(defineName, target); + this.register(defineName, pluginClass); } register(key: string, value: T) { @@ -57,4 +57,15 @@ export class Registry { getStorage() { return this.storage; } + + getDefineList() { + const list = []; + for (const key in this.storage) { + const PluginClass = this.storage[key]; + // @ts-ignore + const plugin = new PluginClass(); + list.push({ ...plugin.define, key }); + } + return list; + } } diff --git a/packages/core/pipeline/src/utils/util.log.ts b/packages/core/pipeline/src/utils/util.log.ts index e6783ef5..6e48c676 100644 --- a/packages/core/pipeline/src/utils/util.log.ts +++ b/packages/core/pipeline/src/utils/util.log.ts @@ -1,6 +1,34 @@ -import log4js from "log4js"; +import log4js, { Appender, Logger, LoggingEvent } from "log4js"; + +const OutputAppender = { + configure: (config: any, layouts: any, findAppender: any, levels: any) => { + let layout = layouts.colouredLayout; + if (config.layout) { + layout = layouts.layout(config.layout.type, config.layout); + } + function customAppender(layout: any, timezoneOffset: any) { + return (loggingEvent: LoggingEvent) => { + if (loggingEvent.context.outputHandler?.write) { + const text = `${layout(loggingEvent, timezoneOffset)}\n`; + loggingEvent.context.outputHandler.write(text); + } + }; + } + return customAppender(layout, config.timezoneOffset); + }, +}; + +// @ts-ignore log4js.configure({ - appenders: { std: { type: "stdout" } }, - categories: { default: { appenders: ["std"], level: "info" } }, + appenders: { std: { type: "stdout" }, output: { type: OutputAppender } }, + categories: { default: { appenders: ["std"], level: "info" }, pipeline: { appenders: ["std", "output"], level: "info" } }, }); -export const logger = log4js.getLogger("pipeline"); +export const logger = log4js.getLogger("default"); + +export function buildLogger(write: (text: string) => void) { + const logger = log4js.getLogger("pipeline"); + logger.addContext("outputHandler", { + write, + }); + return logger; +} diff --git a/packages/core/pipeline/test/cert.fake.test.ts b/packages/core/pipeline/test/cert.fake.test.ts new file mode 100644 index 00000000..f6a0fb0a --- /dev/null +++ b/packages/core/pipeline/test/cert.fake.test.ts @@ -0,0 +1,58 @@ +export const fakeCrt = `-----BEGIN CERTIFICATE----- +MIIFSTCCBDGgAwIBAgITAPoZZk/LhVIyXoic2NnJyxubezANBgkqhkiG9w0BAQsF +ADAiMSAwHgYDVQQDDBdGYWtlIExFIEludGVybWVkaWF0ZSBYMTAeFw0yMDEyMTQx +NjA1NTFaFw0yMTAzMTQxNjA1NTFaMBsxGTAXBgNVBAMMECouZG9jbWlycm9yLmNs +dWIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC75tGrYjly+RpcZehQ +my1EpaXElT4L60pINKV2YDKnBrcSSo1c6rO7nFh12eC/ju4WwYUep0RVmBDF8xD0 +I1Sd1uuDTQWP0UT1X9yqdXtjvxpUqoCHAzG633f3sJRFul7mDLuC9tRCuae9o7qP +EZ827XOmjBR35dso9I2GEE4828J3YE3tSKtobZlM+30jozLEcsO0PTyM5mq5PPjP +VI3fGLcEaBmLZf5ixz4XkcY9IAhyAMYf03cT2wRoYPBaDdXblgCYL6sFtIMbzl3M +Di94PB8NyoNSsC2nmBdWi54wFOgBvY/4ljsX/q7X3EqlSvcA0/M6/c/J9kJ3eupv +jV8nAgMBAAGjggJ9MIICeTAOBgNVHQ8BAf8EBAMCBaAwHQYDVR0lBBYwFAYIKwYB +BQUHAwEGCCsGAQUFBwMCMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFAkdTjSCV3KD +x28sf98MrwVfyFYgMB8GA1UdIwQYMBaAFMDMA0a5WCDMXHJw8+EuyyCm9Wg6MHcG +CCsGAQUFBwEBBGswaTAyBggrBgEFBQcwAYYmaHR0cDovL29jc3Auc3RnLWludC14 +MS5sZXRzZW5jcnlwdC5vcmcwMwYIKwYBBQUHMAKGJ2h0dHA6Ly9jZXJ0LnN0Zy1p +bnQteDEubGV0c2VuY3J5cHQub3JnLzArBgNVHREEJDAighAqLmRvY21pcnJvci5j +bHVigg5kb2NtaXJyb3IuY2x1YjBMBgNVHSAERTBDMAgGBmeBDAECATA3BgsrBgEE +AYLfEwEBATAoMCYGCCsGAQUFBwIBFhpodHRwOi8vY3BzLmxldHNlbmNyeXB0Lm9y +ZzCCAQQGCisGAQQB1nkCBAIEgfUEgfIA8AB1ABboacHRlerXw/iXGuPwdgH3jOG2 +nTGoUhi2g38xqBUIAAABdmI3LM4AAAQDAEYwRAIgaiNqXSEq+sxp8eqlJXp/KFdO +so5mT50MoRsLF8Inu0ACIDP46+ekng7I0BlmyIPmbqFcZgnZFVWLLCdLYijhVyOL +AHcA3Zk0/KXnJIDJVmh9gTSZCEmySfe1adjHvKs/XMHzbmQAAAF2YjcuxwAABAMA +SDBGAiEAxpeB8/w4YkHZ62nH20h128VtuTSmYDCnF7EK2fQyeZYCIQDbJlF2wehZ +sF1BeE7qnYYqCTP0dYIrQ9HWtBa/MbGOKTANBgkqhkiG9w0BAQsFAAOCAQEAL2di +HKh6XcZtGk0BFxJa51sCZ3MLu9+Zy90kCRD4ooP5x932WxVM25+LBRd+xSzx+TRL +UVrlKp9GdMYX1JXL4Vf2NwzuFO3snPDe/qizD/3+D6yo8eKJ/LD82t5kLWAD2rto +YfVSTKwfNIBBJwHUnjviBPJmheHHCKmz8Ct6/6QxFAeta9TAMn0sFeVCQnmAq7HL +jrunq0tNHR/EKG0ITPLf+6P7MxbmpYNnq918766l0tKsW8oo8ZSGEwKU2LMaSiAa +hasyl/2gMnYXjtKOjDcnR8oLpbrOg0qpVbynmJin1HP835oHPPAZ1gLsqYTTizNz +AHxTaXliTVvS83dogw== +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIEqzCCApOgAwIBAgIRAIvhKg5ZRO08VGQx8JdhT+UwDQYJKoZIhvcNAQELBQAw +GjEYMBYGA1UEAwwPRmFrZSBMRSBSb290IFgxMB4XDTE2MDUyMzIyMDc1OVoXDTM2 +MDUyMzIyMDc1OVowIjEgMB4GA1UEAwwXRmFrZSBMRSBJbnRlcm1lZGlhdGUgWDEw +ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDtWKySDn7rWZc5ggjz3ZB0 +8jO4xti3uzINfD5sQ7Lj7hzetUT+wQob+iXSZkhnvx+IvdbXF5/yt8aWPpUKnPym +oLxsYiI5gQBLxNDzIec0OIaflWqAr29m7J8+NNtApEN8nZFnf3bhehZW7AxmS1m0 +ZnSsdHw0Fw+bgixPg2MQ9k9oefFeqa+7Kqdlz5bbrUYV2volxhDFtnI4Mh8BiWCN +xDH1Hizq+GKCcHsinDZWurCqder/afJBnQs+SBSL6MVApHt+d35zjBD92fO2Je56 +dhMfzCgOKXeJ340WhW3TjD1zqLZXeaCyUNRnfOmWZV8nEhtHOFbUCU7r/KkjMZO9 +AgMBAAGjgeMwgeAwDgYDVR0PAQH/BAQDAgGGMBIGA1UdEwEB/wQIMAYBAf8CAQAw +HQYDVR0OBBYEFMDMA0a5WCDMXHJw8+EuyyCm9Wg6MHoGCCsGAQUFBwEBBG4wbDA0 +BggrBgEFBQcwAYYoaHR0cDovL29jc3Auc3RnLXJvb3QteDEubGV0c2VuY3J5cHQu +b3JnLzA0BggrBgEFBQcwAoYoaHR0cDovL2NlcnQuc3RnLXJvb3QteDEubGV0c2Vu +Y3J5cHQub3JnLzAfBgNVHSMEGDAWgBTBJnSkikSg5vogKNhcI5pFiBh54DANBgkq +hkiG9w0BAQsFAAOCAgEABYSu4Il+fI0MYU42OTmEj+1HqQ5DvyAeyCA6sGuZdwjF +UGeVOv3NnLyfofuUOjEbY5irFCDtnv+0ckukUZN9lz4Q2YjWGUpW4TTu3ieTsaC9 +AFvCSgNHJyWSVtWvB5XDxsqawl1KzHzzwr132bF2rtGtazSqVqK9E07sGHMCf+zp +DQVDVVGtqZPHwX3KqUtefE621b8RI6VCl4oD30Olf8pjuzG4JKBFRFclzLRjo/h7 +IkkfjZ8wDa7faOjVXx6n+eUQ29cIMCzr8/rNWHS9pYGGQKJiY2xmVC9h12H99Xyf +zWE9vb5zKP3MVG6neX1hSdo7PEAb9fqRhHkqVsqUvJlIRmvXvVKTwNCP3eCjRCCI +PTAvjV+4ni786iXwwFYNz8l3PmPLCyQXWGohnJ8iBm+5nk7O2ynaPVW0U2W+pt2w +SVuvdDM5zGv2f9ltNWUiYZHJ1mmO97jSY/6YfdOUH66iRtQtDkHBRdkNBsMbD+Em +2TgBldtHNSJBfB3pm9FblgOcJ0FSWcUDWJ7vO0+NTXlgrRofRT6pVywzxVo6dND0 +WzYlTWeUVsO40xJqhgUQRER9YLOLxJ0O6C8i0xFxAMKOtSdodMB3RIwt7RFQ0uyt +n5Z5MqkYhlMI3J1tPRTp1nEt9fyGspBOO05gi148Qasp+3N+svqKomoQglNoAxU= +-----END CERTIFICATE-----`; diff --git a/packages/core/pipeline/test/pipeline/access-service-test.ts b/packages/core/pipeline/test/pipeline/access-service-test.ts index 68dcb9d9..2b8c2915 100644 --- a/packages/core/pipeline/test/pipeline/access-service-test.ts +++ b/packages/core/pipeline/test/pipeline/access-service-test.ts @@ -2,7 +2,7 @@ import { IAccessService } from "../../src/access/access-service"; import { AbstractAccess, AliyunAccess } from "../../src"; import { aliyunSecret } from "../user.secret"; export class AccessServiceTest implements IAccessService { - getById(id: any): AbstractAccess { + async getById(id: any): Promise { return { ...aliyunSecret, } as AliyunAccess; diff --git a/packages/core/pipeline/test/pipeline/init.test.ts b/packages/core/pipeline/test/pipeline/init.test.ts new file mode 100644 index 00000000..d1e28763 --- /dev/null +++ b/packages/core/pipeline/test/pipeline/init.test.ts @@ -0,0 +1,15 @@ +import { ContextFactory } from "../../src/core/context"; +import { FileStorage } from "../../src/core/storage"; +import { AccessServiceTest } from "./access-service-test"; +import { logger } from "../../src/utils/util.log"; + +const contextFactory = new ContextFactory(new FileStorage()); + +const userContext = contextFactory.getContext("user", "test"); +const pipelineContext = contextFactory.getContext("pipeline", "test"); +export const pluginInitProps = { + accessService: new AccessServiceTest(), + pipelineContext: pipelineContext, + userContext: userContext, + logger: logger, +}; diff --git a/packages/core/pipeline/test/pipeline/pipeline.define.ts b/packages/core/pipeline/test/pipeline/pipeline.define.ts index 3cbf43c2..649f4229 100644 --- a/packages/core/pipeline/test/pipeline/pipeline.define.ts +++ b/packages/core/pipeline/test/pipeline/pipeline.define.ts @@ -1,4 +1,4 @@ -import { ConcurrencyStrategy, NextStrategy, Pipeline } from "../../src"; +import { ConcurrencyStrategy, NextStrategy, Pipeline, RunStrategy } from "../../src"; let idIndex = 0; function generateId() { @@ -44,15 +44,18 @@ export const pipeline: Pipeline = { tasks: [ { id: generateId(), - title: "测试输出参数", + title: "测试输出参数任务", steps: [ { id: generateId(), - title: "输出参数", + title: "输出参数（echo插件）", type: "EchoPlugin", input: { cert: "cert", }, + strategy: { + runStrategy: RunStrategy.SkipWhenSucceed, + }, }, ], }, diff --git a/packages/core/pipeline/test/pipeline/pipeline.test.ts b/packages/core/pipeline/test/pipeline/pipeline.test.ts index be06c4fd..7a0baee9 100644 --- a/packages/core/pipeline/test/pipeline/pipeline.test.ts +++ b/packages/core/pipeline/test/pipeline/pipeline.test.ts @@ -1,4 +1,4 @@ -import { expect } from "chai"; +//import { expect } from "chai"; import "mocha"; import { Executor, RunHistory } from "../../src"; import { pipeline } from "./pipeline.define"; @@ -11,7 +11,7 @@ describe("pipeline", function () { console.log("changed:"); } - const executor = new Executor({ userId: 1, pipeline, onChanged, accessService: new AccessServiceTest(), storage: new FileStorage() }); + const executor = new Executor({ userId: "test", pipeline, onChanged, accessService: new AccessServiceTest(), storage: new FileStorage() }); await executor.run(); // expect(define.name).eq("EchoPlugin"); }); diff --git a/packages/core/pipeline/test/pipeline/plugins/cert-apply.test.ts b/packages/core/pipeline/test/pipeline/plugins/cert-apply.test.ts new file mode 100644 index 00000000..5304eb89 --- /dev/null +++ b/packages/core/pipeline/test/pipeline/plugins/cert-apply.test.ts @@ -0,0 +1,23 @@ +import { expect } from "chai"; +import "mocha"; +import { CertApplyPlugin } from "../../../src/plugin/plugins"; +import { pluginInitProps } from "../init.test"; +describe("CertApply", function () { + it("#execute", async function () { + this.timeout(120000); + const plugin = new CertApplyPlugin(); + // @ts-ignore + delete plugin.define; + await plugin.doInit(pluginInitProps); + const output = await plugin.execute({ + domains: ["*.docmirror.cn", "docmirror.cn"], + email: "xiaojunnuo@qq.com", + dnsProviderType: "aliyun", + accessId: "111", + forceUpdate: true, + }); + const cert = output.cert; + expect(plugin.getDefine().name).eq("CertApply"); + expect(cert.crt != null).eq(true); + }); +}); diff --git a/packages/core/pipeline/test/pipeline/plugins/deploy-to-cdn.test.ts b/packages/core/pipeline/test/pipeline/plugins/deploy-to-cdn.test.ts new file mode 100644 index 00000000..7d9706ba --- /dev/null +++ b/packages/core/pipeline/test/pipeline/plugins/deploy-to-cdn.test.ts @@ -0,0 +1,23 @@ +import { expect } from "chai"; +import "mocha"; +import { DeployCertToAliyunCDN } from "../../../src/plugin/plugins"; +import { pluginInitProps } from "../init.test"; + +describe("DeployToAliyunCDN", function () { + it("#execute", async function () { + this.timeout(120000); + const plugin = new DeployCertToAliyunCDN(); + // @ts-ignore + delete plugin.define; + + await plugin.doInit(pluginInitProps); + + const cert = await pluginInitProps.pipelineContext.get("cert"); + + await plugin.execute({ + cert, + domainName: "certd-cdn-upload.docmirror.cn", + }); + expect(plugin.getDefine().name).eq("DeployCertToAliyunCDN"); + }); +}); diff --git a/packages/server/certd-client b/packages/server/certd-client index ae0676e9..51dd7756 160000 --- a/packages/server/certd-client +++ b/packages/server/certd-client @@ -1 +1 @@ -Subproject commit ae0676e9e3e32a9359195a1f0ab4b83c941698c1 +Subproject commit 51dd77563b83ccc541067037e1125dcd8e46f7da