github
/
certd
mirror of https://github.com/certd/certd
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

perf: 优化定时器

pull/21/merge
xiaojunnuo 2023-05-24 15:41:35 +08:00
parent 6f6606d76d
commit 3751fcd4c9
29 changed files with 381 additions and 163 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
packages/core/pipeline/src/core/context.ts
View File

@ -1,10 +1,13 @@
import { IStorage, MemoryStorage } from "./storage";
const CONTEXT_VERSION_KEY = "contextVersion";
export interface IContext {
getInt(key: string): Promise<number>;
get(key: string): Promise<string | null>;
set(key: string, value: string): Promise<void>;
getObj(key: string): Promise<any>;
setObj(key: string, value: any): Promise<void>;
getObj<T = any>(key: string): Promise<T | null>;
setObj<T = any>(key: string, value: T): Promise<void>;
updateVersion(): Promise<void>;
initVersion(): Promise<void>;
}
export class ContextFactory {
@ -17,11 +20,13 @@ export class ContextFactory {
}
getContext(scope: string, namespace: string): IContext {
return new StorageContext(scope, namespace, this.storage);
const context = new StorageContext(scope, namespace, this.storage);
return context;
}
getMemoryContext(scope: string, namespace: string): IContext {
return new StorageContext(scope, namespace, this.memoryStorage);
const context = new StorageContext(scope, namespace, this.memoryStorage);
return context;
}
}
@ -29,19 +34,46 @@ export class StorageContext implements IContext {
storage: IStorage;
namespace: string;
scope: string;
_version = 0;
_initialVersion = 0;
constructor(scope: string, namespace: string, storage: IStorage) {
this.storage = storage;
this.scope = scope;
this.namespace = namespace;
}
async initVersion() {
const version = await this.getInt(CONTEXT_VERSION_KEY);
this._initialVersion = version;
this._version = version;
}
async updateVersion() {
if (this._version === this._initialVersion) {
this._version++;
}
await this.set(CONTEXT_VERSION_KEY, this._version.toString());
}
async get(key: string) {
return await this.storage.get(this.scope, this.namespace, key);
const version = key === CONTEXT_VERSION_KEY ? 0 : this._version;
return await this.storage.get(this.scope, this.namespace, version.toString(), key);
}
async set(key: string, value: string) {
return await this.storage.set(this.scope, this.namespace, key, value);
const version = key === CONTEXT_VERSION_KEY ? 0 : this._version;
return await this.storage.set(this.scope, this.namespace, version.toString(), key, value);
}
async getObj(key: string): Promise<any> {
async getInt(key: string): Promise<number> {
const str = await this.get(key);
if (str) {
return parseInt(str);
}
return 0;
}
async getObj<T = any>(key: string): Promise<T | null> {
const str = await this.get(key);
if (str) {
const store = JSON.parse(str);
@ -50,7 +82,7 @@ export class StorageContext implements IContext {
return null;
}
async setObj(key: string, value: any) {
async setObj<T = any>(key: string, value: T) {
await this.set(key, JSON.stringify({ value }));
}
}

75
packages/core/pipeline/src/core/executor.ts
View File

@ -1,7 +1,7 @@
import { ConcurrencyStrategy, Pipeline, ResultType, Runnable, RunStrategy, Stage, Step, Task } from "../d.ts";
import _ from "lodash";
import { RunHistory } from "./run-history";
import { PluginDefine, pluginRegistry } from "../plugin";
import { RunHistory, RunnableCollection } from "./run-history";
import { AbstractTaskPlugin, PluginDefine, pluginRegistry } from "../plugin";
import { ContextFactory, IContext } from "./context";
import { IStorage } from "./storage";
import { logger } from "../utils/util.log";
@ -18,11 +18,20 @@ export class Executor {
accessService: IAccessService;
contextFactory: ContextFactory;
logger: Logger;
pipelineContext: IContext;
pipelineContext!: IContext;
lastStatusMap!: RunnableCollection;
onChanged: (history: RunHistory) => void;
constructor(options: { userId: any; pipeline: Pipeline; storage: IStorage; onChanged: (history: RunHistory) => void; accessService: IAccessService }) {
constructor(options: {
userId: any;
pipeline: Pipeline;
storage: IStorage;
onChanged: (history: RunHistory) => Promise<void>;
accessService: IAccessService;
}) {
this.pipeline = _.cloneDeep(options.pipeline);
this.onChanged = options.onChanged;
this.onChanged = async (history: RunHistory) => {
await options.onChanged(history);
};
this.accessService = options.accessService;
this.userId = options.userId;
this.pipeline.userId = this.userId;
@ -31,17 +40,25 @@ export class Executor {
this.pipelineContext = this.contextFactory.getContext("pipeline", this.pipeline.id);
}
async init() {
const lastRuntime = await this.pipelineContext.getObj(`lastRuntime`);
this.lastStatusMap = new RunnableCollection(lastRuntime?.pipeline);
}
async run(runtimeId: any = 0, triggerType: string) {
try {
await this.init();
const trigger = { type: triggerType };
// 读取last
this.runtime = new RunHistory(runtimeId, trigger, this.pipeline);
this.logger.info(`pipeline.${this.pipeline.id} start`);
await this.runWithHistory(this.pipeline, "pipeline", async () => {
await this.runStages();
await this.runStages(this.pipeline);
});
} catch (e) {
this.logger.error("pipeline 执行失败", e);
} finally {
await this.pipelineContext.setObj("lastRuntime", this.runtime);
this.logger.info(`pipeline.${this.pipeline.id} end`);
}
}
@ -50,20 +67,18 @@ export class Executor {
runnable.runnableType = runnableType;
this.runtime.start(runnable);
await this.onChanged(this.runtime);
const contextKey = `status.${runnable.id}`;
const inputKey = `input.${runnable.id}`;
if (runnable.strategy?.runStrategy === RunStrategy.SkipWhenSucceed) {
//如果是成功后跳过策略
const lastResult = await this.pipelineContext.getObj(contextKey);
const lastInput = await this.pipelineContext.get(inputKey);
const lastNode = this.lastStatusMap.get(runnable.id);
const lastResult = lastNode?.status?.status;
const lastInput = JSON.stringify(lastNode?.status?.input);
let inputChanged = false;
//TODO 参数不变
if (runnableType === "step") {
const step = runnable as Step;
const input = JSON.stringify(step.input);
await this.pipelineContext.set(inputKey, input);
if (input != null && lastInput !== input) {
//参数有变化
inputChanged = true;
}
}
@ -76,12 +91,10 @@ export class Executor {
try {
await run();
this.runtime.success(runnable);
await this.pipelineContext.setObj(contextKey, ResultType.success);
await this.onChanged(this.runtime);
return ResultType.success;
} catch (e: any) {
this.runtime.error(runnable, e);
await this.pipelineContext.setObj(contextKey, ResultType.error);
await this.onChanged(this.runtime);
throw e;
} finally {
@ -89,9 +102,9 @@ export class Executor {
}
}
private async runStages() {
private async runStages(pipeline: Pipeline) {
const resList: ResultType[] = [];
for (const stage of this.pipeline.stages) {
for (const stage of pipeline.stages) {
const res: ResultType = await this.runWithHistory(stage, "stage", async () => {
await this.runStage(stage);
});
@ -150,17 +163,12 @@ export class Executor {
}
private async runStep(step: Step) {
const lastStatus = this.lastStatusMap.get(step.id);
//执行任务
const plugin: RegistryItem = pluginRegistry.get(step.type);
const context: any = {
logger: this.runtime.loggers[step.id],
accessService: this.accessService,
pipelineContext: this.pipelineContext,
userContext: this.contextFactory.getContext("user", this.userId),
http: request,
};
const plugin: RegistryItem<AbstractTaskPlugin> = pluginRegistry.get(step.type);
// @ts-ignore
const instance = new plugin.target();
const instance: ITaskPlugin = new plugin.target();
// @ts-ignore
const define: PluginDefine = plugin.define;
//从outputContext读取输入参数
@ -173,14 +181,27 @@ export class Executor {
}
});
const context: any = {
logger: this.runtime._loggers[step.id],
accessService: this.accessService,
pipelineContext: this.pipelineContext,
lastStatus,
userContext: this.contextFactory.getContext("user", this.userId),
http: request,
};
Decorator.inject(define.autowire, instance, context);
await instance.onInstance();
await instance.execute();
if (instance.result.clearLastStatus) {
this.lastStatusMap.clear();
}
//输出到output context
_.forEach(define.output, (item, key) => {
const contextKey = `step.${step.id}.${key}`;
this.runtime.context[contextKey] = instance[key];
step!.status!.output[key] = instance[key];
const stepOutputKey = `step.${step.id}.${key}`;
this.runtime.context[stepOutputKey] = instance[key];
});
}
}

86
packages/core/pipeline/src/core/run-history.ts
View File

@ -1,4 +1,4 @@
import { Context, HistoryResult, Pipeline, Runnable } from "../d.ts";
import { Context, HistoryResult, Pipeline, ResultType, Runnable, RunnableMap, Stage, Step, Task } from "../d.ts";
import _ from "lodash";
import { buildLogger } from "../utils/util.log";
import { Logger } from "log4js";
@ -11,18 +11,27 @@ export type HistoryStatus = {
export type RunTrigger = {
type: string; // user , timer
};
export function NewRunHistory(obj: any) {
const history = new RunHistory(obj.id, obj.trigger, obj.pipeline);
history.context = obj.context;
history.logs = obj.logs;
history._loggers = obj.loggers;
return history;
}
export class RunHistory {
id: string;
id!: string;
//运行时上下文变量
context: Context = {};
pipeline: Pipeline;
pipeline!: Pipeline;
logs: {
[runnableId: string]: string[];
} = {};
loggers: {
_loggers: {
[runnableId: string]: Logger;
} = {};
trigger: RunTrigger;
trigger!: RunTrigger;
constructor(runtimeId: any, trigger: RunTrigger, pipeline: Pipeline) {
this.id = runtimeId;
this.pipeline = pipeline;
@ -32,13 +41,16 @@ export class RunHistory {
start(runnable: Runnable): HistoryResult {
const now = new Date().getTime();
this.logs[runnable.id] = [];
this.loggers[runnable.id] = buildLogger((text) => {
this._loggers[runnable.id] = buildLogger((text) => {
this.logs[runnable.id].push(text);
});
const input = (runnable as Step).input;
const status: HistoryResult = {
status: "start",
output: {},
input: _.cloneDeep(input),
status: ResultType.start,
startTime: now,
result: "start",
result: ResultType.start,
};
runnable.status = status;
this.log(runnable, `开始执行`);
@ -71,9 +83,9 @@ export class RunHistory {
const now = new Date().getTime();
const status = runnable.status;
_.merge(status, {
status: "error",
status: ResultType.error,
endTime: now,
result: "error",
result: ResultType.error,
message: e.message,
});
@ -82,15 +94,65 @@ export class RunHistory {
log(runnable: Runnable, text: string) {
// @ts-ignore
this.loggers[runnable.id].info(`[${runnable.title}]<id:${runnable.id}> [${runnable.runnableType}]`, text);
this._loggers[runnable.id].info(`[${runnable.title}]<id:${runnable.id}> [${runnable.runnableType}]`, text);
}
logError(runnable: Runnable, e: Error) {
// @ts-ignore
this.loggers[runnable.id].error(`[${runnable.title}]<id:${runnable.id}> [${runnable.runnableType}]`, e);
this._loggers[runnable.id].error(`[${runnable.title}]<id:${runnable.id}> [${runnable.runnableType}]`, e);
}
finally(runnable: Runnable) {
//
}
}
export class RunnableCollection {
private collection: RunnableMap = {};
private pipeline!: Pipeline;
constructor(pipeline?: Pipeline) {
if (!pipeline) {
return;
}
this.pipeline = pipeline;
const map = this.toMap(pipeline);
this.collection = map;
}
private each<T extends Runnable>(list: T[], exec: (item: Runnable) => void) {
list.forEach((item) => {
exec(item);
if (item.runnableType === "pipeline") {
// @ts-ignore
this.each<Stage>(item.stages, exec);
} else if (item.runnableType === "stage") {
// @ts-ignore
this.each<Task>(item.tasks, exec);
} else if (item.runnableType === "task") {
// @ts-ignore
this.each<Step>(item.steps, exec);
}
});
}
private toMap(pipeline: Pipeline) {
const map: RunnableMap = {};
this.each(pipeline.stages, (item) => {
map[item.id] = item;
});
return map;
}
get(id: string): Runnable | undefined {
return this.collection[id];
}
clear() {
if (!this.pipeline) {
return;
}
this.each(this.pipeline.stages, (item) => {
item.status = undefined;
});
}
}

93
packages/core/pipeline/src/core/storage.ts
View File

@ -1,9 +1,12 @@
import fs from "fs";
import path from "path";
export interface IStorage {
get(scope: string, namespace: string, key: string): Promise<string | null>;
set(scope: string, namespace: string, key: string, value: string): Promise<void>;
get(scope: string, namespace: string, version: string, key: string): Promise<string | null>;
set(scope: string, namespace: string, version: string, key: string, value: string): Promise<void>;
remove(scope: string, namespace: string, version: string, key: string): Promise<void>;
}
export class FileStorage implements IStorage {
@ -20,6 +23,18 @@ export class FileStorage implements IStorage {
}
}
async remove(scope: string, namespace: string, version: string, key: string): Promise<void> {
if (key) {
fs.unlinkSync(this.buildPath(scope, namespace, version, key));
} else if (version) {
fs.rmdirSync(this.buildPath(scope, namespace, version));
} else if (namespace) {
fs.rmdirSync(this.buildPath(scope, namespace));
} else {
fs.rmdirSync(this.buildPath(scope));
}
}
writeFile(filePath: string, value: string) {
const dir = path.dirname(filePath);
if (!fs.existsSync(dir)) {
@ -36,18 +51,26 @@ export class FileStorage implements IStorage {
return fs.readFileSync(filePath).toString();
}
async get(scope: string, namespace: string, key: string): Promise<string | null> {
const path = this.buildPath(scope, namespace, key);
async get(scope: string, namespace: string, version: string, key: string): Promise<string | null> {
const path = this.buildPath(scope, namespace, version, key);
return this.readFile(path);
}
async set(scope: string, namespace: string, key: string, value: string): Promise<void> {
const path = this.buildPath(scope, namespace, key);
async set(scope: string, namespace: string, version: string, key: string, value: string): Promise<void> {
const path = this.buildPath(scope, namespace, version, key);
this.writeFile(path, value);
}
private buildPath(scope: string, namespace: string, key: string) {
return `${this.root}/${scope}/${namespace}/${key}`;
private buildPath(scope: string, namespace?: string, version?: string, key?: string) {
if (key) {
return `${this.root}/${scope}/${namespace}/${version}/${key}`;
} else if (version) {
return `${this.root}/${scope}/${namespace}/${version}`;
} else if (namespace) {
return `${this.root}/${scope}/${namespace}`;
} else {
return `${this.root}/${scope}`;
}
}
}
@ -63,19 +86,55 @@ export class MemoryStorage implements IStorage {
};
} = {};
async get(scope: string, namespace: string, key: string): Promise<string | null> {
const context = this.context[scope];
if (context == null) {
async get(scope: string, namespace: string, version: string, key: string): Promise<string | null> {
const scopeContext = this.context[scope];
if (scopeContext == null) {
return null;
}
return context[namespace + "." + key];
const nsContext = scopeContext[namespace];
if (nsContext == null) {
return null;
}
const versionContext = nsContext[version];
if (versionContext == null) {
return null;
}
return versionContext[key];
}
async set(scope: string, namespace: string, key: string, value: string): Promise<void> {
let context = this.context[scope];
if (context == null) {
context = context[scope];
async set(scope: string, namespace: string, version: string, key: string, value: string): Promise<void> {
let scopeContext = this.context[scope];
if (scopeContext == null) {
scopeContext = scopeContext[scope];
}
let nsContext = scopeContext[namespace];
if (nsContext == null) {
nsContext = {};
scopeContext[namespace] = nsContext;
}
let versionContext = nsContext[version];
if (versionContext == null) {
versionContext = {};
nsContext[version] = versionContext;
}
versionContext[key] = value;
}
async remove(scope: string, namespace = "", version = "", key = "") {
if (key) {
if (this.context[scope] && this.context[scope][namespace] && this.context[scope][namespace][version]) {
delete this.context[scope][namespace][version][key];
}
} else if (version) {
if (this.context[scope] && this.context[scope][namespace]) {
delete this.context[scope][namespace][version];
}
} else if (namespace) {
if (this.context[scope]) {
delete this.context[scope][namespace];
}
} else {
delete this.context[scope];
}
context[namespace + "." + key] = value;
}
}

18
packages/core/pipeline/src/d.ts/pipeline.ts
View File

@ -85,9 +85,11 @@ export type Log = {
};
export enum ResultType {
success,
error,
skip,
start = "start",
success = "success",
error = "error",
skip = "skip",
none = "none",
}
export type HistoryResultGroup = {
@ -97,15 +99,21 @@ export type HistoryResultGroup = {
};
};
export type HistoryResult = {
input: any;
output: any;
/**
* 任务状态
*/
status: string;
status: ResultType;
startTime: number;
endTime?: number;
/**
* 处理结果
*/
result?: string; //success, error,skip
result?: ResultType; //success, error,skip
message?: string;
};
export type RunnableMap = {
[id: string]: Runnable;
};

17
packages/core/pipeline/src/plugin/api.ts
View File

@ -33,9 +33,24 @@ export type PluginDefine = Registrable & {
};
};
export interface ITaskPlugin {
export type ITaskPlugin = {
onInstance(): Promise<void>;
execute(): Promise<void>;
[key: string]: any;
};
export type TaskResult = {
clearLastStatus?: boolean;
};
export abstract class AbstractTaskPlugin implements ITaskPlugin {
result: TaskResult = {};
clearLastStatus() {
this.result.clearLastStatus = true;
}
async onInstance(): Promise<void> {
return;
}
abstract execute(): Promise<void>;
}
export type OutputVO = {

4
packages/core/pipeline/src/plugin/registry.ts
View File

@ -1,4 +1,4 @@
import { Registry } from "../registry";
import { AbstractTaskPlugin } from "./api";
// @ts-ignore
export const pluginRegistry = new Registry();
export const pluginRegistry = new Registry<AbstractTaskPlugin>();

12
packages/core/pipeline/src/registry/registry.ts
View File

@ -4,23 +4,23 @@ export type Registrable = {
desc?: string;
};
export type RegistryItem = {
export type RegistryItem<T> = {
define: Registrable;
target: any;
target: T;
};
export class Registry {
export class Registry<T> {
storage: {
[key: string]: RegistryItem;
[key: string]: RegistryItem<T>;
} = {};
register(key: string, value: RegistryItem) {
register(key: string, value: RegistryItem<T>) {
if (!key || value == null) {
return;
}
this.storage[key] = value;
}
get(name: string): RegistryItem {
get(name: string): RegistryItem<T> {
if (!name) {
throw new Error("插件名称不能为空");
}

6
packages/plugins/plugin-aliyun/src/plugin/deploy-to-ack-ingress/index.ts
View File

@ -1,4 +1,4 @@
import { Autowire, IAccessService, IsTaskPlugin, ITaskPlugin, ILogger, RunStrategy, TaskInput, utils } from "@certd/pipeline";
import { AbstractTaskPlugin, Autowire, IAccessService, ILogger, IsTaskPlugin, RunStrategy, TaskInput, utils } from "@certd/pipeline";
// @ts-ignore
import { ROAClient } from "@alicloud/pop-core";
import { AliyunAccess } from "../../access";
@ -17,7 +17,7 @@ import { CertInfo } from "@certd/plugin-cert";
},
},
})
export class DeployCertToAliyunAckIngressPlugin implements ITaskPlugin {
export class DeployCertToAliyunAckIngressPlugin extends AbstractTaskPlugin {
@TaskInput({
title: "集群id",
component: {
@ -108,8 +108,6 @@ export class DeployCertToAliyunAckIngressPlugin implements ITaskPlugin {
@Autowire()
logger!: ILogger;
// eslint-disable-next-line @typescript-eslint/no-empty-function
async onInstance(): Promise<void> {}
async execute(): Promise<void> {
console.log("开始部署证书到阿里云cdn");
const { regionId, ingressClass, clusterId, isPrivateIpAddress, cert } = this;

5
packages/plugins/plugin-aliyun/src/plugin/deploy-to-cdn/index.ts
View File

@ -1,10 +1,9 @@
import { Autowire, IAccessService, ILogger, IsTaskPlugin, ITaskPlugin, RunStrategy, TaskInput } from "@certd/pipeline";
import { AbstractTaskPlugin, Autowire, IAccessService, ILogger, IsTaskPlugin, RunStrategy, TaskInput } from "@certd/pipeline";
import dayjs from "dayjs";
import Core from "@alicloud/pop-core";
import RPCClient from "@alicloud/pop-core";
import { AliyunAccess } from "../../access";
@IsTaskPlugin({
name: "DeployCertToAliyunCDN",
title: "部署证书至阿里云CDN",
@ -15,7 +14,7 @@ import { AliyunAccess } from "../../access";
},
},
})
export class DeployCertToAliyunCDN implements ITaskPlugin {
export class DeployCertToAliyunCDN extends AbstractTaskPlugin {
@TaskInput({
title: "CDN加速域名",
helper: "你在阿里云上配置的CDN加速域名，比如certd.docmirror.cn",

4
packages/plugins/plugin-aliyun/src/plugin/upload-to-aliyun/index.ts
View File

@ -1,4 +1,4 @@
import { Autowire, IAccessService, IsTaskPlugin, ITaskPlugin, RunStrategy, TaskInput, TaskOutput } from "@certd/pipeline";
import { AbstractTaskPlugin, Autowire, IAccessService, IsTaskPlugin, RunStrategy, TaskInput, TaskOutput } from "@certd/pipeline";
import Core from "@alicloud/pop-core";
import { AliyunAccess } from "../../access";
import { appendTimeSuffix, checkRet, ZoneOptions } from "../../utils";
@ -14,7 +14,7 @@ import { Logger } from "log4js";
},
},
})
export class UploadCertToAliyun implements ITaskPlugin {
export class UploadCertToAliyun extends AbstractTaskPlugin {
@TaskInput({
title: "证书名称",
helper: "证书上传后将以此参数作为名称前缀",

32
packages/plugins/plugin-cert/src/plugin/cert-plugin/index.ts
View File

@ -1,4 +1,4 @@
import { Autowire, HttpClient, IAccessService, IContext, IsTaskPlugin, ITaskPlugin, RunStrategy, TaskInput, TaskOutput } from "@certd/pipeline";
import { AbstractTaskPlugin, Autowire, HttpClient, IAccessService, IContext, IsTaskPlugin, RunStrategy, Step, TaskInput, TaskOutput } from "@certd/pipeline";
import dayjs from "dayjs";
import { AcmeService, CertInfo } from "./acme";
import _ from "lodash";
@ -24,7 +24,7 @@ export type { CertInfo };
},
},
})
export class CertApplyPlugin implements ITaskPlugin {
export class CertApplyPlugin extends AbstractTaskPlugin {
@TaskInput({
title: "域名",
component: {
@ -118,7 +118,7 @@ export class CertApplyPlugin implements ITaskPlugin {
http!: HttpClient;
@Autowire()
pipelineContext!: IContext;
lastStatus!: Step;
@TaskOutput({
title: "域名证书",
@ -137,6 +137,8 @@ export class CertApplyPlugin implements ITaskPlugin {
const cert = await this.doCertApply();
if (cert != null) {
this.output(cert.toCertInfo());
//清空后续任务的状态，让后续任务能够重新执行
this.clearLastStatus();
} else {
throw new Error("申请证书失败");
}
@ -156,10 +158,7 @@ export class CertApplyPlugin implements ITaskPlugin {
}
let inputChanged = false;
const inputCacheKey = "input.domains";
const oldInputStr = await this.pipelineContext.getObj(inputCacheKey);
await this.pipelineContext.setObj(inputCacheKey, this.domains);
const oldInput = JSON.stringify(oldInputStr);
const oldInput = JSON.stringify(this.lastStatus?.input?.domains);
const thisInput = JSON.stringify(this.domains);
if (oldInput !== thisInput) {
inputChanged = true;
@ -167,7 +166,7 @@ export class CertApplyPlugin implements ITaskPlugin {
let oldCert: CertReader | undefined = undefined;
try {
oldCert = await this.readCurrentCert();
oldCert = await this.readLastCert();
} catch (e) {
this.logger.warn("读取cert失败：", e);
}
@ -227,10 +226,8 @@ export class CertApplyPlugin implements ITaskPlugin {
isTest: false,
});
await this.writeCert(cert);
const ret = await this.readCurrentCert();
return ret;
const certInfo = this.formatCerts(cert);
return new CertReader(certInfo);
}
formatCert(pem: string) {
@ -240,20 +237,17 @@ export class CertApplyPlugin implements ITaskPlugin {
return pem;
}
async writeCert(cert: { crt: string; key: string; csr: string }) {
formatCerts(cert: { crt: string; key: string; csr: string }) {
const newCert: CertInfo = {
crt: this.formatCert(cert.crt),
key: this.formatCert(cert.key),
csr: this.formatCert(cert.csr),
};
await this.pipelineContext.setObj("cert", newCert);
await this.pipelineContext.set("cert.crt", newCert.crt);
await this.pipelineContext.set("cert.key", newCert.key);
await this.pipelineContext.set("cert.csr", newCert.csr);
return newCert;
}
async readCurrentCert(): Promise<CertReader | undefined> {
const cert: CertInfo = await this.pipelineContext.getObj("cert");
async readLastCert(): Promise<CertReader | undefined> {
const cert = this.lastStatus?.status?.output?.cert;
if (cert == null) {
return undefined;
}

4
packages/plugins/plugin-host/src/plugin/host-shell-execute/index.ts
View File

@ -1,4 +1,4 @@
import { Autowire, IAccessService, ILogger, IsTaskPlugin, ITaskPlugin, RunStrategy, TaskInput } from "@certd/pipeline";
import { AbstractTaskPlugin, Autowire, IAccessService, ILogger, IsTaskPlugin, RunStrategy, TaskInput } from "@certd/pipeline";
import { SshClient } from "../../lib/ssh";
@IsTaskPlugin({
@ -12,7 +12,7 @@ import { SshClient } from "../../lib/ssh";
},
output: {},
})
export class HostShellExecutePlugin implements ITaskPlugin {
export class HostShellExecutePlugin extends AbstractTaskPlugin {
@TaskInput({
title: "主机登录配置",
helper: "登录",

4
packages/plugins/plugin-host/src/plugin/upload-to-host/index.ts
View File

@ -1,4 +1,4 @@
import { Autowire, IAccessService, IsTaskPlugin, ITaskPlugin, ILogger, RunStrategy, TaskInput, TaskOutput } from "@certd/pipeline";
import { AbstractTaskPlugin, Autowire, IAccessService, ILogger, IsTaskPlugin, RunStrategy, TaskInput, TaskOutput } from "@certd/pipeline";
import { SshClient } from "../../lib/ssh";
import { CertInfo, CertReader } from "@certd/plugin-cert";
import * as fs from "fs";
@ -12,7 +12,7 @@ import * as fs from "fs";
},
},
})
export class UploadCertToHostPlugin implements ITaskPlugin {
export class UploadCertToHostPlugin extends AbstractTaskPlugin {
@TaskInput({
title: "证书保存路径",
})

4
packages/plugins/plugin-tencent/src/plugin/deploy-to-cdn/index.ts
View File

@ -1,4 +1,4 @@
import { Autowire, IAccessService, IsTaskPlugin, ITaskPlugin, ILogger, RunStrategy, TaskInput } from "@certd/pipeline";
import { AbstractTaskPlugin, Autowire, IAccessService, ILogger, IsTaskPlugin, RunStrategy, TaskInput } from "@certd/pipeline";
import tencentcloud from "tencentcloud-sdk-nodejs/index";
import { TencentAccess } from "../../access";
import { CertInfo } from "@certd/plugin-cert";
@ -12,7 +12,7 @@ import { CertInfo } from "@certd/plugin-cert";
},
},
})
export class DeployToCdnPlugin implements ITaskPlugin {
export class DeployToCdnPlugin extends AbstractTaskPlugin {
@TaskInput({
title: "域名证书",
helper: "请选择前置任务输出的域名证书",

4
packages/plugins/plugin-tencent/src/plugin/deploy-to-clb/index.ts
View File

@ -1,4 +1,4 @@
import { Autowire, IAccessService, IsTaskPlugin, ITaskPlugin, ILogger, RunStrategy, TaskInput, utils } from "@certd/pipeline";
import { AbstractTaskPlugin, Autowire, IAccessService, ILogger, IsTaskPlugin, RunStrategy, TaskInput, utils } from "@certd/pipeline";
import tencentcloud from "tencentcloud-sdk-nodejs/index";
import { TencentAccess } from "../../access";
import dayjs from "dayjs";
@ -13,7 +13,7 @@ import dayjs from "dayjs";
},
},
})
export class DeployToClbPlugin implements ITaskPlugin {
export class DeployToClbPlugin extends AbstractTaskPlugin {
@TaskInput({
title: "大区",
value: "ap-guangzhou",

4
packages/plugins/plugin-tencent/src/plugin/deploy-to-tke-ingress/index.ts
View File

@ -1,4 +1,4 @@
import { Autowire, IAccessService, IsTaskPlugin, ITaskPlugin, RunStrategy, TaskInput, utils } from "@certd/pipeline";
import { AbstractTaskPlugin, Autowire, IAccessService, IsTaskPlugin, RunStrategy, TaskInput, utils } from "@certd/pipeline";
import tencentcloud from "tencentcloud-sdk-nodejs/index";
import { K8sClient } from "@certd/plugin-util";
import dayjs from "dayjs";
@ -14,7 +14,7 @@ import { Logger } from "log4js";
},
},
})
export class DeployCertToTencentTKEIngressPlugin implements ITaskPlugin {
export class DeployCertToTencentTKEIngressPlugin extends AbstractTaskPlugin {
@TaskInput({ title: "大区", value: "ap-guangzhou", required: true })
region!: string;

4
packages/plugins/plugin-tencent/src/plugin/upload-to-tencent/index.ts
View File

@ -1,4 +1,4 @@
import { Autowire, IAccessService, IsTaskPlugin, ITaskPlugin, RunStrategy, TaskInput, TaskOutput, ILogger } from "@certd/pipeline";
import { AbstractTaskPlugin, Autowire, IAccessService, ILogger, IsTaskPlugin, RunStrategy, TaskInput, TaskOutput } from "@certd/pipeline";
import tencentcloud from "tencentcloud-sdk-nodejs/index";
import dayjs from "dayjs";
@ -12,7 +12,7 @@ import dayjs from "dayjs";
},
},
})
export class UploadToTencentPlugin implements ITaskPlugin {
export class UploadToTencentPlugin extends AbstractTaskPlugin {
@TaskInput({ title: "证书名称" })
name!: string;

18
packages/ui/certd-client/src/views/certd/pipeline/crud.tsx
View File

@ -2,28 +2,28 @@ import * as api from "./api";
import { useI18n } from "vue-i18n";
import { ref, shallowRef } from "vue";
import { useRouter } from "vue-router";
import { dict } from "@fast-crud/fast-crud";
import { AddReq, CreateCrudOptionsProps, CreateCrudOptionsRet, DelReq, DialogOpenOption, dict, EditReq, UserPageQuery, UserPageRes } from "@fast-crud/fast-crud";
import { statusUtil } from "/@/views/certd/pipeline/pipeline/utils/util.status";
import { nanoid } from "nanoid";
import { message } from "ant-design-vue";
export default function ({ expose, certdFormRef }) {
export default function ({ crudExpose, context: { certdFormRef } }: CreateCrudOptionsProps): CreateCrudOptionsRet {
const router = useRouter();
const { t } = useI18n();
const lastResRef = ref();
const pageRequest = async (query) => {
const pageRequest = async (query: UserPageQuery): Promise<UserPageRes> => {
return await api.GetList(query);
};
const editRequest = async ({ form, row }) => {
const editRequest = async ({ form, row }: EditReq) => {
form.id = row.id;
const res = await api.UpdateObj(form);
lastResRef.value = res;
return res;
};
const delRequest = async ({ row }) => {
const delRequest = async ({ row }: DelReq) => {
return await api.DelObj(row.id);
};
const addRequest = async ({ form }) => {
const addRequest = async ({ form }: AddReq) => {
form.content = JSON.stringify({
title: form.title
});
@ -32,7 +32,7 @@ export default function ({ expose, certdFormRef }) {
return res;
};
function addCertdPipeline() {
certdFormRef.value.open(async ({ form }) => {
certdFormRef.value.open(async ({ form }: any) => {
// 添加certd pipeline
const pipeline = {
title: form.domains[0] + "证书自动化",
@ -176,8 +176,8 @@ export default function ({ expose, certdFormRef }) {
type: "dict-switch",
dict: dict({
data: [
{ value: true, label: "禁用" },
{ value: false, label: "启用" }
{ value: false, label: "启用" },
{ value: true, label: "禁用" }
]
}),
form: {

25
packages/ui/certd-client/src/views/certd/pipeline/index.vue
View File

@ -9,9 +9,9 @@
</fs-page>
</template>
<script>
<script lang="ts">
import { defineComponent, ref, onMounted } from "vue";
import { useCrud } from "@fast-crud/fast-crud";
import { useCrud, useFs } from "@fast-crud/fast-crud";
import createCrudOptions from "./crud";
import { useExpose } from "@fast-crud/fast-crud";
import PiCertdForm from "./certd-form/index.vue";
@ -20,25 +20,14 @@ export default defineComponent({
components: { PiCertdForm },
setup() {
const certdFormRef = ref();
// crud组件的ref
const crudRef = ref();
// crud 配置的ref
const crudBinding = ref();
// 暴露的方法
const { expose } = useExpose({ crudRef, crudBinding });
// 你的crud配置
const { crudOptions } = createCrudOptions({ expose, certdFormRef });
// 初始化crud配置
// eslint-disable-next-line @typescript-eslint/no-unused-vars,no-unused-vars
const { resetCrudOptions } = useCrud({ expose, crudOptions });
// 你可以调用此方法，重新初始化crud配置
// resetCrudOptions(options)
const context: any = {
certdFormRef
};
const { crudBinding, crudRef, crudExpose } = useFs({ createCrudOptions, context });
// 页面打开后获取列表数据
onMounted(() => {
expose.doRefresh();
crudExpose.doRefresh();
});
return {

2
packages/ui/certd-client/src/views/certd/pipeline/pipeline/component/status-show.vue
View File

@ -16,7 +16,7 @@ export default defineComponent({
name: "PiStatusShow",
props: {
status: {
type: String,
type: [String, Number],
default: ""
},
type: {

13
packages/ui/certd-client/src/views/certd/pipeline/pipeline/utils/util.status.ts
View File

@ -1,4 +1,15 @@
const StatusEnum = {
export type StatusEnumItem = {
value: string;
label: string;
color: string;
icon: string;
spin?: boolean;
};
export type StatusEnumType = {
[key: string]: StatusEnumItem;
};
const StatusEnum: StatusEnumType = {
success: {
value: "success",
label: "成功",

2
packages/ui/certd-server/db/migration/v10001__certdv2.sql
View File

@ -4,4 +4,4 @@ CREATE TABLE "pi_history_log" ("id" integer PRIMARY KEY AUTOINCREMENT NOT NULL,
CREATE TABLE "pi_pipeline" ("id" integer PRIMARY KEY AUTOINCREMENT NOT NULL, "user_id" integer NOT NULL, "title" integer NOT NULL, "content" varchar(40960) NOT NULL, "keep_history_count" integer, "remark" varchar(100), "status" varchar(100), "disabled" boolean DEFAULT (0), "last_history_time" integer, "create_time" datetime NOT NULL DEFAULT (CURRENT_TIMESTAMP), "update_time" datetime NOT NULL DEFAULT (CURRENT_TIMESTAMP))
CREATE TABLE "pi_storage" ("id" integer PRIMARY KEY AUTOINCREMENT NOT NULL, "user_id" integer NOT NULL, "scope" varchar NOT NULL, "namespace" varchar NOT NULL, "key" varchar(100), "value" varchar(40960), "create_time" datetime NOT NULL DEFAULT (CURRENT_TIMESTAMP), "update_time" datetime NOT NULL DEFAULT (CURRENT_TIMESTAMP))
CREATE TABLE "pi_storage" ("id" integer PRIMARY KEY AUTOINCREMENT NOT NULL, "user_id" integer NOT NULL, "scope" varchar NOT NULL, "namespace" varchar NOT NULL, "version" varchar(100),"key" varchar(100), "value" varchar(40960), "create_time" datetime NOT NULL DEFAULT (CURRENT_TIMESTAMP), "update_time" datetime NOT NULL DEFAULT (CURRENT_TIMESTAMP))

1
packages/ui/certd-server/src/modules/pipeline/controller/pipeline-controller.ts
View File

@ -52,6 +52,7 @@ export class PipelineController extends CrudController<PipelineService> {
await this.service.checkUserId(bean.id, this.ctx.user.id);
}
await this.service.save(bean);
await this.service.registerTriggerById(bean.id);
return this.ok(bean.id);
}

3
packages/ui/certd-server/src/modules/pipeline/entity/storage.ts
View File

@ -14,6 +14,9 @@ export class StorageEntity {
@Column({ name: 'namespace', comment: '命名空间' })
namespace: string;
@Column({ comment: 'version', length: 100, nullable: true })
version: string;
@Column({ comment: 'key', length: 100, nullable: true })
key: string;

13
packages/ui/certd-server/src/modules/pipeline/service/db-storage.ts
View File

@ -12,15 +12,26 @@ export class DbStorage implements IStorage {
this.storageService = storageService;
}
remove(
scope: string,
namespace: string,
version: string,
key: string
): Promise<void> {
throw new Error('Method not implemented.');
}
async get(
scope: string,
namespace: string,
version: string,
key: string
): Promise<string | null> {
const storageEntity = await this.storageService.get({
userId: this.userId,
scope: scope,
namespace: namespace,
version,
key,
});
@ -33,6 +44,7 @@ export class DbStorage implements IStorage {
async set(
scope: string,
namespace: string,
version: string,
key: string,
value: string
): Promise<void> {
@ -40,6 +52,7 @@ export class DbStorage implements IStorage {
userId: this.userId,
scope: scope,
namespace: namespace,
version,
key,
value,
});

28
packages/ui/certd-server/src/modules/pipeline/service/pipeline-service.ts
View File

@ -42,11 +42,9 @@ export class PipelineService extends BaseService<PipelineEntity> {
async update(entity) {
await super.update(entity);
await this.registerTriggerById(entity.id);
}
private async registerTriggerById(pipelineId) {
public async registerTriggerById(pipelineId) {
if (pipelineId == null) {
return;
}
@ -70,13 +68,13 @@ export class PipelineService extends BaseService<PipelineEntity> {
const pipeline = JSON.parse(bean.content);
bean.title = pipeline.title;
await this.addOrUpdate(bean);
await this.registerTriggerById(bean.id);
}
/**
* 应用启动后初始加载记录
*/
async onStartup() {
logger.info('加载定时trigger开始');
const idEntityList = await this.repository.find({
select: {
id: true,
@ -111,7 +109,7 @@ export class PipelineService extends BaseService<PipelineEntity> {
this.registerTriggers(pipeline);
}
}
logger.info('定时器数量：', this.cron.getList());
logger.info('定时器数量：', this.cron.getListSize());
}
registerTriggers(pipeline?: Pipeline) {
@ -121,6 +119,7 @@ export class PipelineService extends BaseService<PipelineEntity> {
for (const trigger of pipeline.triggers) {
this.registerCron(pipeline.id, trigger);
}
logger.info('当前定时器数量：', this.cron.getListSize());
}
async trigger(id) {
@ -128,10 +127,11 @@ export class PipelineService extends BaseService<PipelineEntity> {
name: `pipeline.${id}.trigger.once`,
cron: null,
job: async () => {
logger.info('job准备启动，当前定时器数量：', this.cron.getListSize());
await this.run(id, null);
},
});
logger.info('定时器数量：', this.cron.getList());
logger.info('定时器数量：', this.cron.getListSize());
}
registerCron(pipelineId, trigger) {
@ -141,10 +141,11 @@ export class PipelineService extends BaseService<PipelineEntity> {
}
if (cron.startsWith('*')) {
cron = '0' + cron.substring(1, cron.length);
return;
}
const name = this.buildCronKey(pipelineId, trigger.id);
this.cron.remove(name);
this.cron.register({
name: this.buildCronKey(pipelineId, trigger.id),
name,
cron: cron,
job: async () => {
logger.info('定时任务触发：', pipelineId, trigger.id);
@ -191,8 +192,13 @@ export class PipelineService extends BaseService<PipelineEntity> {
accessService: this.accessService,
storage: new DbStorage(userId, this.storageService),
});
await executor.run(historyId, triggerType);
try {
await executor.init();
await executor.run(historyId, triggerType);
} catch (e) {
logger.error('执行失败：', e);
throw e;
}
}
private getTriggerType(triggerId, pipeline) {
@ -230,7 +236,7 @@ export class PipelineService extends BaseService<PipelineEntity> {
//修改pipeline状态
const pipelineEntity = new PipelineEntity();
pipelineEntity.id = parseInt(history.pipeline.id);
pipelineEntity.status = history.pipeline.status.status;
pipelineEntity.status = history.pipeline.status.status + '';
pipelineEntity.lastHistoryTime = history.pipeline.status.startTime;
await this.update(pipelineEntity);

4
packages/ui/certd-server/src/modules/pipeline/service/storage-service.ts
View File

@ -1,4 +1,4 @@
import { Provide, Scope, ScopeEnum } from "@midwayjs/decorator";
import { Provide, Scope, ScopeEnum } from '@midwayjs/decorator';
import { InjectEntityModel } from '@midwayjs/typeorm';
import { Repository } from 'typeorm';
import { BaseService } from '../../../basic/base-service';
@ -20,6 +20,7 @@ export class StorageService extends BaseService<StorageEntity> {
scope: any;
namespace: any;
userId: number;
version: string;
key: string;
}) {
if (where.userId == null) {
@ -35,6 +36,7 @@ export class StorageService extends BaseService<StorageEntity> {
scope: any;
namespace: any;
userId: number;
version: string;
value: string;
key: string;
}) {

9
packages/ui/certd-server/src/plugins/cron/cron.ts
View File

@ -23,15 +23,20 @@ export class Cron {
cron.schedule(task.cron, task.job, {
name: task.name,
});
this.logger.info('当前定时任务数量：', this.getListSize());
}
remove(taskName: string) {
this.logger.info(`[cron] remove : [${taskName}]`);
const tasks = cron.getTasks() as Map<any, any>;
tasks.delete(taskName);
const node = tasks.get(taskName);
if (node) {
node.stop();
tasks.delete(taskName);
}
}
getList() {
getListSize() {
const tasks = cron.getTasks();
return tasks.size;
}