New feature: log cleaner.

pull/49/head
Doflatango 2017-12-07 15:37:01 +08:00
parent d54e1b0bc0
commit afb694361a
12 changed files with 124 additions and 7 deletions

View File

@ -69,6 +69,15 @@ func main() {
go cronsun.StartNoticer(noticer)
}
period := int64(conf.Config.Web.LogCleaner.EveryMinute)
var stopCleaner func(interface{})
if period > 0 {
closeChan := web.RunLogCleaner(time.Duration(period)*time.Minute, time.Duration(conf.Config.Web.LogCleaner.ExpirationDays)*time.Hour*24)
stopCleaner = func(i interface{}) {
close(closeChan)
}
}
go func() {
err := httpServer.Serve(httpL)
if err != nil {
@ -80,7 +89,7 @@ func main() {
log.Infof("cronsun web server started on %s, Ctrl+C or send kill sign to exit", conf.Config.Web.BindAddr)
// 注册退出事件
event.On(event.EXIT, conf.Exit)
event.On(event.EXIT, conf.Exit, stopCleaner)
// 监听退出信号
event.Wait()
event.Emit(event.EXIT, nil)

View File

@ -76,7 +76,11 @@ type webConfig struct {
Auth struct {
Enabled bool
}
Session SessionConfig
Session SessionConfig
LogCleaner struct {
EveryMinute int
ExpirationDays int
}
}
type SessionConfig struct {
@ -144,6 +148,15 @@ func (c *Conf) parse() error {
c.Mgo.Timeout *= time.Second
}
if c.Web != nil {
if c.Web.LogCleaner.EveryMinute < 0 {
c.Web.LogCleaner.EveryMinute = 30
}
if c.Web.LogCleaner.ExpirationDays <= 0 {
c.Web.LogCleaner.ExpirationDays = 1
}
}
c.Node = cleanKeyPrefix(c.Node)
c.Proc = cleanKeyPrefix(c.Proc)
c.Cmd = cleanKeyPrefix(c.Cmd)

View File

@ -7,5 +7,11 @@
"StorePrefixPath": "/cronsun/sess/",
"CookieName": "uid",
"Expiration": 8640000
},
"#comment": "Delete the expired log (which store in mongodb) periodically",
"LogCleaner": {
"#comment": "if EveryMinute is 0, the LogCleaner will not run",
"EveryMinute": 30,
"ExpirationDays": 3
}
}

View File

@ -24,6 +24,10 @@ func On(name string, fs ...func(interface{})) error {
}
for _, f := range fs {
if fs == nil {
continue
}
fp := reflect.ValueOf(f).Pointer()
for i := 0; i < len(evs); i++ {
if reflect.ValueOf(evs[i]).Pointer() == fp {

6
job.go
View File

@ -65,6 +65,8 @@ type Job struct {
FailNotify bool `json:"fail_notify"`
// 发送通知地址
To []string `json:"to"`
// 单独对任务指定日志清除时间
LogExpiration int `json:"log_expiration"`
// 执行任务的结点,用于记录 job log
runOn string
@ -526,6 +528,10 @@ func (j *Job) Check() error {
return ErrIllegalJobGroupName
}
if j.LogExpiration < 0 {
j.LogExpiration = 0
}
j.User = strings.TrimSpace(j.User)
for i := range j.Rules {

View File

@ -6,6 +6,7 @@ import (
"gopkg.in/mgo.v2"
"gopkg.in/mgo.v2/bson"
"github.com/shunfei/cronsun/conf"
"github.com/shunfei/cronsun/log"
)
@ -28,6 +29,7 @@ type JobLog struct {
Success bool `bson:"success" json:"success"` // 是否执行成功
BeginTime time.Time `bson:"beginTime" json:"beginTime"` // 任务开始执行时间,精确到毫秒,索引
EndTime time.Time `bson:"endTime" json:"endTime"` // 任务执行完毕时间,精确到毫秒
Cleanup time.Time `bson:"cleanup,omitempty" json:"-"` // 日志清除时间标志
}
type JobLatestLog struct {
@ -102,6 +104,17 @@ func CreateJobLog(j *Job, t time.Time, rs string, success bool) {
BeginTime: t,
EndTime: et,
}
if conf.Config.Web.LogCleaner.EveryMinute > 0 {
var expiration int
if j.LogExpiration > 0 {
expiration = j.LogExpiration
} else {
expiration = conf.Config.Web.LogCleaner.ExpirationDays
}
jl.Cleanup = jl.EndTime.Add(time.Duration(expiration) * time.Hour * 24)
}
if err := mgoDB.Insert(Coll_JobLog, jl); err != nil {
log.Errorf(err.Error())
}

View File

@ -5,7 +5,7 @@ import (
"runtime"
)
const Binary = "v0.2.2"
const Binary = "v0.2.3"
var (
Version = fmt.Sprintf("%s (build %s)", Binary, runtime.Version())

View File

@ -5,11 +5,18 @@ import "github.com/shunfei/cronsun/conf"
type Configuration struct{}
func (cnf *Configuration) Configuratios(ctx *Context) {
outJSON(ctx.W, struct {
Security *conf.Security `json:"security"`
Alarm bool `json:"alarm"`
r := struct {
Security *conf.Security `json:"security"`
Alarm bool `json:"alarm"`
LogExpirationDays int `json:"log_expiration_days"`
}{
Security: conf.Config.Security,
Alarm: conf.Config.Mail.Enable,
})
}
if conf.Config.Web.LogCleaner.EveryMinute > 0 {
r.LogExpirationDays = conf.Config.Web.LogCleaner.ExpirationDays
}
outJSON(ctx.W, r)
}

50
web/log_cleaner.go Normal file
View File

@ -0,0 +1,50 @@
package web
import (
"time"
"github.com/shunfei/cronsun"
"github.com/shunfei/cronsun/log"
mgo "gopkg.in/mgo.v2"
"gopkg.in/mgo.v2/bson"
)
func RunLogCleaner(cleanPeriod, expiration time.Duration) (close chan struct{}) {
t := time.NewTicker(cleanPeriod)
close = make(chan struct{})
go func() {
for {
select {
case <-t.C:
cleanupLogs(expiration)
case <-close:
return
}
}
}()
return
}
func cleanupLogs(expiration time.Duration) {
err := cronsun.GetDb().WithC(cronsun.Coll_JobLog, func(c *mgo.Collection) error {
_, err := c.RemoveAll(bson.M{"$or": []bson.M{
bson.M{"$and": []bson.M{
bson.M{"cleanup": bson.M{"$exists": true}},
bson.M{"cleanup": bson.M{"$lte": time.Now()}},
}},
bson.M{"$and": []bson.M{
bson.M{"cleanup": bson.M{"$exists": false}},
bson.M{"endTime": bson.M{"$lte": time.Now().Add(-expiration)}},
}},
}})
return err
})
if err != nil {
log.Errorf("[Cleaner] Failed to remove expired logs: %s", err.Error())
return
}
}

View File

@ -79,6 +79,12 @@
<input type="number" ref="interval" v-model.number="job.interval">
</div>
</div>
<div class="two fields" v-if="$appConfig.log_expiration_days>0">
<div class="field">
<label>{{$L('log expiration(log expired after N days, 0 will use default setting: {n} days)', $appConfig.log_expiration_days)}}</label>
<input type="number" ref="log_expiration" v-model.number="job.log_expiration">
</div>
</div>
<div class="field">
<span v-if="!job.rules || job.rules.length == 0"><i class="warning circle icon"></i>{{$L('the job dose not have a timer currently, please click the button below to add a timer')}}</span>
</div>
@ -123,6 +129,7 @@ export default {
retry: 0,
rules: [],
fail_notify: false,
log_expiration: 0,
to: []
}
}

View File

@ -102,6 +102,7 @@ var language = {
'retry interval(in seconds)': 'Retry interval(in seconds)',
'parallel number in one node(0 for no limits)': 'Parallel number in one node(0 for no limits)',
'timeout(in seconds, 0 for no limits)': 'Timeout(in seconds, 0 for no limits)',
'log expiration(log expired after N days, 0 will use default setting: {n} days)': 'Log expiration(log expired after N days, 0 will use default setting: {0} days)',
'<sec> <min> <hr> <day> <month> <week>, rule is same with Cron': '<sec> <min> <hr> <day> <month> <week>, rule is same with Cron',
'and please running on those nodes': 'And please running on those nodes',
'do not running on those nodes': 'Do not running on those nodes',

View File

@ -104,6 +104,7 @@ var language = {
'retry interval(in seconds)': '失败重试间隔时间(秒)',
'parallel number in one node(0 for no limits)': '一个节点上面该任务并行数0 表示不限制)',
'timeout(in seconds, 0 for no limits)': '超时设置单位“秒”0 表示不限制)',
'log expiration(log expired after N days, 0 will use default setting: {n} days)': '日志过期日志保存天数0 表示使用默认设置:{0} 天)',
'<sec> <min> <hr> <day> <month> <week>, rule is same with Cron': '<秒> <分> <时> <日> <月> <周>,规则与 Cron 一样',
'and please running on those nodes': '同时在这些节点上面运行',
'do not running on those nodes': '不要在这些节点上面运行',