From afb694361a20b49a7f81c44cd50a887c371cdadb Mon Sep 17 00:00:00 2001 From: Doflatango Date: Thu, 7 Dec 2017 15:37:01 +0800 Subject: [PATCH] New feature: log cleaner. --- bin/web/server.go | 11 +++++- conf/conf.go | 15 +++++++- conf/files/web.json.sample | 6 ++++ event/event.go | 4 +++ job.go | 6 ++++ job_log.go | 13 +++++++ version.go | 2 +- web/configuration.go | 15 +++++--- web/log_cleaner.go | 50 +++++++++++++++++++++++++++ web/ui/src/components/JobEditForm.vue | 7 ++++ web/ui/src/i18n/languages/en.js | 1 + web/ui/src/i18n/languages/zh-CN.js | 1 + 12 files changed, 124 insertions(+), 7 deletions(-) create mode 100644 web/log_cleaner.go diff --git a/bin/web/server.go b/bin/web/server.go index 5000969..794d555 100644 --- a/bin/web/server.go +++ b/bin/web/server.go @@ -69,6 +69,15 @@ func main() { go cronsun.StartNoticer(noticer) } + period := int64(conf.Config.Web.LogCleaner.EveryMinute) + var stopCleaner func(interface{}) + if period > 0 { + closeChan := web.RunLogCleaner(time.Duration(period)*time.Minute, time.Duration(conf.Config.Web.LogCleaner.ExpirationDays)*time.Hour*24) + stopCleaner = func(i interface{}) { + close(closeChan) + } + } + go func() { err := httpServer.Serve(httpL) if err != nil { @@ -80,7 +89,7 @@ func main() { log.Infof("cronsun web server started on %s, Ctrl+C or send kill sign to exit", conf.Config.Web.BindAddr) // 注册退出事件 - event.On(event.EXIT, conf.Exit) + event.On(event.EXIT, conf.Exit, stopCleaner) // 监听退出信号 event.Wait() event.Emit(event.EXIT, nil) diff --git a/conf/conf.go b/conf/conf.go index b274dc9..06db5a7 100644 --- a/conf/conf.go +++ b/conf/conf.go @@ -76,7 +76,11 @@ type webConfig struct { Auth struct { Enabled bool } - Session SessionConfig + Session SessionConfig + LogCleaner struct { + EveryMinute int + ExpirationDays int + } } type SessionConfig struct { @@ -144,6 +148,15 @@ func (c *Conf) parse() error { c.Mgo.Timeout *= time.Second } + if c.Web != nil { + if c.Web.LogCleaner.EveryMinute < 0 { + c.Web.LogCleaner.EveryMinute = 30 + } + if c.Web.LogCleaner.ExpirationDays <= 0 { + c.Web.LogCleaner.ExpirationDays = 1 + } + } + c.Node = cleanKeyPrefix(c.Node) c.Proc = cleanKeyPrefix(c.Proc) c.Cmd = cleanKeyPrefix(c.Cmd) diff --git a/conf/files/web.json.sample b/conf/files/web.json.sample index e36d37d..d4a5366 100644 --- a/conf/files/web.json.sample +++ b/conf/files/web.json.sample @@ -7,5 +7,11 @@ "StorePrefixPath": "/cronsun/sess/", "CookieName": "uid", "Expiration": 8640000 + }, + "#comment": "Delete the expired log (which store in mongodb) periodically", + "LogCleaner": { + "#comment": "if EveryMinute is 0, the LogCleaner will not run", + "EveryMinute": 30, + "ExpirationDays": 3 } } diff --git a/event/event.go b/event/event.go index bfc554b..8f23c63 100644 --- a/event/event.go +++ b/event/event.go @@ -24,6 +24,10 @@ func On(name string, fs ...func(interface{})) error { } for _, f := range fs { + if fs == nil { + continue + } + fp := reflect.ValueOf(f).Pointer() for i := 0; i < len(evs); i++ { if reflect.ValueOf(evs[i]).Pointer() == fp { diff --git a/job.go b/job.go index 6edf0cf..0cb4b8b 100644 --- a/job.go +++ b/job.go @@ -65,6 +65,8 @@ type Job struct { FailNotify bool `json:"fail_notify"` // 发送通知地址 To []string `json:"to"` + // 单独对任务指定日志清除时间 + LogExpiration int `json:"log_expiration"` // 执行任务的结点，用于记录 job log runOn string @@ -526,6 +528,10 @@ func (j *Job) Check() error { return ErrIllegalJobGroupName } + if j.LogExpiration < 0 { + j.LogExpiration = 0 + } + j.User = strings.TrimSpace(j.User) for i := range j.Rules { diff --git a/job_log.go b/job_log.go index 4bb6f0a..f1eb594 100644 --- a/job_log.go +++ b/job_log.go @@ -6,6 +6,7 @@ import ( "gopkg.in/mgo.v2" "gopkg.in/mgo.v2/bson" + "github.com/shunfei/cronsun/conf" "github.com/shunfei/cronsun/log" ) @@ -28,6 +29,7 @@ type JobLog struct { Success bool `bson:"success" json:"success"` // 是否执行成功 BeginTime time.Time `bson:"beginTime" json:"beginTime"` // 任务开始执行时间，精确到毫秒，索引 EndTime time.Time `bson:"endTime" json:"endTime"` // 任务执行完毕时间，精确到毫秒 + Cleanup time.Time `bson:"cleanup,omitempty" json:"-"` // 日志清除时间标志 } type JobLatestLog struct { @@ -102,6 +104,17 @@ func CreateJobLog(j *Job, t time.Time, rs string, success bool) { BeginTime: t, EndTime: et, } + + if conf.Config.Web.LogCleaner.EveryMinute > 0 { + var expiration int + if j.LogExpiration > 0 { + expiration = j.LogExpiration + } else { + expiration = conf.Config.Web.LogCleaner.ExpirationDays + } + jl.Cleanup = jl.EndTime.Add(time.Duration(expiration) * time.Hour * 24) + } + if err := mgoDB.Insert(Coll_JobLog, jl); err != nil { log.Errorf(err.Error()) } diff --git a/version.go b/version.go index 8347f4a..16ba735 100644 --- a/version.go +++ b/version.go @@ -5,7 +5,7 @@ import ( "runtime" ) -const Binary = "v0.2.2" +const Binary = "v0.2.3" var ( Version = fmt.Sprintf("%s (build %s)", Binary, runtime.Version()) diff --git a/web/configuration.go b/web/configuration.go index 395cd61..5d2fc4c 100644 --- a/web/configuration.go +++ b/web/configuration.go @@ -5,11 +5,18 @@ import "github.com/shunfei/cronsun/conf" type Configuration struct{} func (cnf *Configuration) Configuratios(ctx *Context) { - outJSON(ctx.W, struct { - Security *conf.Security `json:"security"` - Alarm bool `json:"alarm"` + r := struct { + Security *conf.Security `json:"security"` + Alarm bool `json:"alarm"` + LogExpirationDays int `json:"log_expiration_days"` }{ Security: conf.Config.Security, Alarm: conf.Config.Mail.Enable, - }) + } + + if conf.Config.Web.LogCleaner.EveryMinute > 0 { + r.LogExpirationDays = conf.Config.Web.LogCleaner.ExpirationDays + } + + outJSON(ctx.W, r) } diff --git a/web/log_cleaner.go b/web/log_cleaner.go new file mode 100644 index 0000000..84ac632 --- /dev/null +++ b/web/log_cleaner.go @@ -0,0 +1,50 @@ +package web + +import ( + "time" + + "github.com/shunfei/cronsun" + "github.com/shunfei/cronsun/log" + mgo "gopkg.in/mgo.v2" + "gopkg.in/mgo.v2/bson" +) + +func RunLogCleaner(cleanPeriod, expiration time.Duration) (close chan struct{}) { + t := time.NewTicker(cleanPeriod) + close = make(chan struct{}) + go func() { + for { + select { + case <-t.C: + cleanupLogs(expiration) + case <-close: + return + } + } + }() + + return +} + +func cleanupLogs(expiration time.Duration) { + err := cronsun.GetDb().WithC(cronsun.Coll_JobLog, func(c *mgo.Collection) error { + _, err := c.RemoveAll(bson.M{"$or": []bson.M{ + bson.M{"$and": []bson.M{ + bson.M{"cleanup": bson.M{"$exists": true}}, + bson.M{"cleanup": bson.M{"$lte": time.Now()}}, + }}, + bson.M{"$and": []bson.M{ + bson.M{"cleanup": bson.M{"$exists": false}}, + bson.M{"endTime": bson.M{"$lte": time.Now().Add(-expiration)}}, + }}, + }}) + + return err + }) + + if err != nil { + log.Errorf("[Cleaner] Failed to remove expired logs: %s", err.Error()) + return + } + +} diff --git a/web/ui/src/components/JobEditForm.vue b/web/ui/src/components/JobEditForm.vue index daef570..8266fb6 100644 --- a/web/ui/src/components/JobEditForm.vue +++ b/web/ui/src/components/JobEditForm.vue @@ -79,6 +79,12 @@ +
+
+ + +
+
{{$L('the job dose not have a timer currently, please click the button below to add a timer')}}
@@ -123,6 +129,7 @@ export default { retry: 0, rules: [], fail_notify: false, + log_expiration: 0, to: [] } } diff --git a/web/ui/src/i18n/languages/en.js b/web/ui/src/i18n/languages/en.js index d4d48f9..01a84c3 100644 --- a/web/ui/src/i18n/languages/en.js +++ b/web/ui/src/i18n/languages/en.js @@ -102,6 +102,7 @@ var language = { 'retry interval(in seconds)': 'Retry interval(in seconds)', 'parallel number in one node(0 for no limits)': 'Parallel number in one node(0 for no limits)', 'timeout(in seconds, 0 for no limits)': 'Timeout(in seconds, 0 for no limits)', + 'log expiration(log expired after N days, 0 will use default setting: {n} days)': 'Log expiration(log expired after N days, 0 will use default setting: {0} days)', '
, rule is same with Cron': '
, rule is same with Cron', 'and please running on those nodes': 'And please running on those nodes', 'do not running on those nodes': 'Do not running on those nodes', diff --git a/web/ui/src/i18n/languages/zh-CN.js b/web/ui/src/i18n/languages/zh-CN.js index 7ee8dd6..93f23a9 100644 --- a/web/ui/src/i18n/languages/zh-CN.js +++ b/web/ui/src/i18n/languages/zh-CN.js @@ -104,6 +104,7 @@ var language = { 'retry interval(in seconds)': '失败重试间隔时间（秒）', 'parallel number in one node(0 for no limits)': '一个节点上面该任务并行数（0 表示不限制）', 'timeout(in seconds, 0 for no limits)': '超时设置（单位“秒”，0 表示不限制）', + 'log expiration(log expired after N days, 0 will use default setting: {n} days)': '日志过期（日志保存天数，0 表示使用默认设置：{0} 天）', '
, rule is same with Cron': '<秒> <分> <时> <日> <月> <周>，规则与 Cron 一样', 'and please running on those nodes': '同时在这些节点上面运行', 'do not running on those nodes': '不要在这些节点上面运行',