github
/
portainer
mirror of https://github.com/portainer/portainer
1
0
Fork 0
Code Issues Releases Wiki Activity

feat(schedules): add retry policy to script schedules (#2445)

pull/2452/head
Anthony Lapenna 2018-11-09 15:22:08 +13:00 committed by GitHub
parent e7ab057c81
commit a2d9f591a7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 160 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
api/cron/job_script_execution.go
View File

@ -2,6 +2,7 @@ package cron
import (
"log"
"time"
"github.com/portainer/portainer"
)
@ -46,6 +47,7 @@ func (runner *ScriptExecutionJobRunner) Run() {
return
}
targets := make([]*portainer.Endpoint, 0)
for _, endpointID := range runner.job.Endpoints {
endpoint, err := runner.context.endpointService.Endpoint(endpointID)
if err != nil {
@ -53,11 +55,32 @@ func (runner *ScriptExecutionJobRunner) Run() {
return
}
err = runner.context.jobService.Execute(endpoint, "", runner.job.Image, scriptFile)
if err != nil {
log.Printf("scheduled job error (script execution). Unable to execute scrtip (endpoint=%s) (err=%s)\n", endpoint.Name, err)
targets = append(targets, endpoint)
}
runner.executeAndRetry(targets, scriptFile, 0)
}
func (runner *ScriptExecutionJobRunner) executeAndRetry(endpoints []*portainer.Endpoint, script []byte, retryCount int) {
retryTargets := make([]*portainer.Endpoint, 0)
for _, endpoint := range endpoints {
err := runner.context.jobService.Execute(endpoint, "", runner.job.Image, script)
if err == portainer.ErrUnableToPingEndpoint {
retryTargets = append(retryTargets, endpoint)
} else if err != nil {
log.Printf("scheduled job error (script execution). Unable to execute script (endpoint=%s) (err=%s)\n", endpoint.Name, err)
}
}
retryCount++
if retryCount >= runner.job.RetryCount {
return
}
time.Sleep(time.Duration(runner.job.RetryInterval) * time.Second)
runner.executeAndRetry(retryTargets, script, retryCount)
}
// GetScheduleID returns the schedule identifier associated to the runner

5
api/docker/job.go
View File

@ -41,6 +41,11 @@ func (service *JobService) Execute(endpoint *portainer.Endpoint, nodeName, image
}
defer cli.Close()
_, err = cli.Ping(context.Background())
if err != nil {
return portainer.ErrUnableToPingEndpoint
}
err = pullImage(cli, image)
if err != nil {
return err

5
api/errors.go
View File

@ -88,6 +88,11 @@ const (
ErrUndefinedTLSFileType = Error("Undefined TLS file type")
)
// Docker errors.
const (
ErrUnableToPingEndpoint = Error("Unable to communicate with the endpoint")
)
// Error represents an application error.
type Error string

103
api/http/handler/schedules/schedule_create.go
View File

@ -14,23 +14,27 @@ import (
"github.com/portainer/portainer/cron"
)
type scheduleFromFilePayload struct {
type scheduleCreateFromFilePayload struct {
Name string
Image string
CronExpression string
Endpoints []portainer.EndpointID
File []byte
RetryCount int
RetryInterval int
}
type scheduleFromFileContentPayload struct {
type scheduleCreateFromFileContentPayload struct {
Name string
CronExpression string
Image string
Endpoints []portainer.EndpointID
FileContent string
RetryCount int
RetryInterval int
}
func (payload *scheduleFromFilePayload) Validate(r *http.Request) error {
func (payload *scheduleCreateFromFilePayload) Validate(r *http.Request) error {
name, err := request.RetrieveMultiPartFormValue(r, "Name", false)
if err != nil {
return errors.New("Invalid name")
@ -62,10 +66,16 @@ func (payload *scheduleFromFilePayload) Validate(r *http.Request) error {
}
payload.File = file
retryCount, _ := request.RetrieveNumericMultiPartFormValue(r, "RetryCount", true)
payload.RetryCount = retryCount
retryInterval, _ := request.RetrieveNumericMultiPartFormValue(r, "RetryInterval", true)
payload.RetryInterval = retryInterval
return nil
}
func (payload *scheduleFromFileContentPayload) Validate(r *http.Request) error {
func (payload *scheduleCreateFromFileContentPayload) Validate(r *http.Request) error {
if govalidator.IsNull(payload.Name) {
return portainer.Error("Invalid schedule name")
}
@ -86,6 +96,10 @@ func (payload *scheduleFromFileContentPayload) Validate(r *http.Request) error {
return portainer.Error("Invalid script file content")
}
if payload.RetryCount != 0 && payload.RetryInterval == 0 {
return portainer.Error("RetryInterval must be set")
}
return nil
}
@ -107,71 +121,100 @@ func (handler *Handler) scheduleCreate(w http.ResponseWriter, r *http.Request) *
}
func (handler *Handler) createScheduleFromFileContent(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
var payload scheduleFromFileContentPayload
var payload scheduleCreateFromFileContentPayload
err := request.DecodeAndValidateJSONPayload(r, &payload)
if err != nil {
return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
}
schedule, err := handler.createSchedule(payload.Name, payload.Image, payload.CronExpression, payload.Endpoints, []byte(payload.FileContent))
schedule := handler.createScheduleObjectFromFileContentPayload(&payload)
err = handler.addAndPersistSchedule(schedule, []byte(payload.FileContent))
if err != nil {
return &httperror.HandlerError{http.StatusInternalServerError, "Failed executing job", err}
return &httperror.HandlerError{http.StatusInternalServerError, "Unable to schedule script job", err}
}
return response.JSON(w, schedule)
}
func (handler *Handler) createScheduleFromFile(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
payload := &scheduleFromFilePayload{}
payload := &scheduleCreateFromFilePayload{}
err := payload.Validate(r)
if err != nil {
return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
}
schedule, err := handler.createSchedule(payload.Name, payload.Image, payload.CronExpression, payload.Endpoints, payload.File)
schedule := handler.createScheduleObjectFromFilePayload(payload)
err = handler.addAndPersistSchedule(schedule, payload.File)
if err != nil {
return &httperror.HandlerError{http.StatusInternalServerError, "Failed executing job", err}
return &httperror.HandlerError{http.StatusInternalServerError, "Unable to schedule script job", err}
}
return response.JSON(w, schedule)
}
func (handler *Handler) createSchedule(name, image, cronExpression string, endpoints []portainer.EndpointID, file []byte) (*portainer.Schedule, error) {
func (handler *Handler) createScheduleObjectFromFilePayload(payload *scheduleCreateFromFilePayload) *portainer.Schedule {
scheduleIdentifier := portainer.ScheduleID(handler.ScheduleService.GetNextIdentifier())
scriptPath, err := handler.FileService.StoreScheduledJobFileFromBytes(strconv.Itoa(int(scheduleIdentifier)), file)
if err != nil {
return nil, err
}
job := &portainer.ScriptExecutionJob{
Endpoints: endpoints,
Image: image,
ScriptPath: scriptPath,
ScheduleID: scheduleIdentifier,
Endpoints: payload.Endpoints,
Image: payload.Image,
ScheduleID: scheduleIdentifier,
RetryCount: payload.RetryCount,
RetryInterval: payload.RetryInterval,
}
schedule := &portainer.Schedule{
ID: scheduleIdentifier,
Name: name,
CronExpression: cronExpression,
Name: payload.Name,
CronExpression: payload.CronExpression,
JobType: portainer.ScriptExecutionJobType,
ScriptExecutionJob: job,
Created: time.Now().Unix(),
}
return schedule
}
func (handler *Handler) createScheduleObjectFromFileContentPayload(payload *scheduleCreateFromFileContentPayload) *portainer.Schedule {
scheduleIdentifier := portainer.ScheduleID(handler.ScheduleService.GetNextIdentifier())
job := &portainer.ScriptExecutionJob{
Endpoints: payload.Endpoints,
Image: payload.Image,
ScheduleID: scheduleIdentifier,
RetryCount: payload.RetryCount,
RetryInterval: payload.RetryInterval,
}
schedule := &portainer.Schedule{
ID: scheduleIdentifier,
Name: payload.Name,
CronExpression: payload.CronExpression,
JobType: portainer.ScriptExecutionJobType,
ScriptExecutionJob: job,
Created: time.Now().Unix(),
}
return schedule
}
func (handler *Handler) addAndPersistSchedule(schedule *portainer.Schedule, file []byte) error {
scriptPath, err := handler.FileService.StoreScheduledJobFileFromBytes(strconv.Itoa(int(schedule.ID)), file)
if err != nil {
return err
}
schedule.ScriptExecutionJob.ScriptPath = scriptPath
jobContext := cron.NewScriptExecutionJobContext(handler.JobService, handler.EndpointService, handler.FileService)
jobRunner := cron.NewScriptExecutionJobRunner(job, jobContext)
jobRunner := cron.NewScriptExecutionJobRunner(schedule.ScriptExecutionJob, jobContext)
err = handler.JobScheduler.CreateSchedule(schedule, jobRunner)
if err != nil {
return nil, err
return err
}
err = handler.ScheduleService.CreateSchedule(schedule)
if err != nil {
return nil, err
}
return schedule, nil
return handler.ScheduleService.CreateSchedule(schedule)
}

12
api/http/handler/schedules/schedule_update.go
View File

@ -17,6 +17,8 @@ type scheduleUpdatePayload struct {
CronExpression *string
Endpoints []portainer.EndpointID
FileContent *string
RetryCount *int
RetryInterval *int
}
func (payload *scheduleUpdatePayload) Validate(r *http.Request) error {
@ -91,5 +93,15 @@ func updateSchedule(schedule *portainer.Schedule, payload *scheduleUpdatePayload
updateJobSchedule = true
}
if payload.RetryCount != nil {
schedule.ScriptExecutionJob.RetryCount = *payload.RetryCount
updateJobSchedule = true
}
if payload.RetryInterval != nil {
schedule.ScriptExecutionJob.RetryInterval = *payload.RetryInterval
updateJobSchedule = true
}
return updateJobSchedule
}

10
api/portainer.go
View File

@ -228,10 +228,12 @@ type (
// ScriptExecutionJob represents a scheduled job that can execute a script via a privileged container
ScriptExecutionJob struct {
ScheduleID ScheduleID `json:"ScheduleId"`
Endpoints []EndpointID
Image string
ScriptPath string
ScheduleID ScheduleID `json:"ScheduleId"`
Endpoints []EndpointID
Image string
ScriptPath string
RetryCount int
RetryInterval int
}
// SnapshotJob represents a scheduled job that can create endpoint snapshots

30
app/portainer/components/forms/schedule-form/scheduleForm.html
View File

@ -42,8 +42,8 @@
</div>
<!-- image-input -->
<div class="form-group">
<label for="schedule_image" class="col-sm-1 control-label text-left">Image</label>
<div class="col-sm-11">
<label for="schedule_image" class="col-sm-2 control-label text-left">Image</label>
<div class="col-sm-10">
<input type="text" class="form-control" ng-model="$ctrl.model.Job.Image" id="schedule_image" name="schedule_image" placeholder="e.g. ubuntu:latest" required>
</div>
</div>
@ -55,12 +55,24 @@
</div>
</div>
<!-- !image-input -->
<!-- retry-policy -->
<div class="form-group">
<span class="col-sm-12 text-muted small">
This schedule will be executed via a privileged container on the target hosts. You can access the host filesystem under the
<code>/host</code> folder.
</span>
<label for="retrycount" class="col-sm-2 control-label text-left">
Retry count
<portainer-tooltip position="bottom" message="Number of retries when it's not possible to reach the endpoint."></portainer-tooltip>
</label>
<div class="col-sm-10 col-md-4">
<input type="number" class="form-control" ng-model="$ctrl.model.Job.RetryCount" id="retrycount" name="retrycount" placeholder="3">
</div>
<label for="retryinterval" class="col-sm-2 control-label text-left">
Retry interval
<portainer-tooltip position="bottom" message="Retry interval in seconds."></portainer-tooltip>
</label>
<div class="col-sm-10 col-md-4">
<input type="number" class="form-control" ng-model="$ctrl.model.Job.RetryInterval" id="retryinterval" name="retryinterval" placeholder="30">
</div>
</div>
<!-- !retry-policy -->
<!-- execution-method -->
<div ng-if="!$ctrl.model.Id">
<div class="col-sm-12 form-section-title">
@ -98,6 +110,12 @@
<div class="col-sm-12 form-section-title">
Web editor
</div>
<div class="form-group">
<span class="col-sm-12 text-muted small">
This schedule will be executed via a privileged container on the target hosts. You can access the host filesystem under the
<code>/host</code> folder.
</span>
</div>
<div class="form-group">
<div class="col-sm-12">
<code-editor

6
app/portainer/models/schedule.js
View File

@ -29,6 +29,8 @@ function ScriptExecutionJobModel(data) {
this.Endpoints = data.Endpoints;
this.FileContent = '';
this.Method = 'editor';
this.RetryCount = data.RetryCount;
this.RetryInterval = data.RetryInterval;
}
function ScheduleCreateRequest(model) {
@ -37,6 +39,8 @@ function ScheduleCreateRequest(model) {
this.Image = model.Job.Image;
this.Endpoints = model.Job.Endpoints;
this.FileContent = model.Job.FileContent;
this.RetryCount = model.Job.RetryCount;
this.RetryInterval = model.Job.RetryInterval;
this.File = model.Job.File;
}
@ -47,4 +51,6 @@ function ScheduleUpdateRequest(model) {
this.Image = model.Job.Image;
this.Endpoints = model.Job.Endpoints;
this.FileContent = model.Job.FileContent;
this.RetryCount = model.Job.RetryCount;
this.RetryInterval = model.Job.RetryInterval;
}

4
app/portainer/services/fileUpload.js
View File

@ -47,7 +47,9 @@ angular.module('portainer.app')
Name: payload.Name,
CronExpression: payload.CronExpression,
Image: payload.Image,
Endpoints: Upload.json(payload.Endpoints)
Endpoints: Upload.json(payload.Endpoints),
RetryCount: payload.RetryCount,
RetryInterval: payload.RetryInterval
}
});
};