github
/
portainer
mirror of https://github.com/portainer/portainer
1
0
Fork 0
Code Issues Releases Wiki Activity

feat(edge): EE-4570 allow pre-pull images with edge stack deployment (#8210) 

Co-authored-by: Matt Hook <hookenz@gmail.com>
pull/8230/head
cmeng 2022-12-21 13:18:51 +13:00 committed by GitHub
parent 7fe0712b61
commit 919a854d93
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 234 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
api/datastore/migrate_data.go
View File

@ -81,6 +81,7 @@ func (store *Store) newMigratorParameters(version *models.Version) *migrator.Mig
FileService: store.fileService,
DockerhubService: store.DockerHubService,
AuthorizationService: authorization.NewService(store),
EdgeStackService: store.EdgeStackService,
}
}

1
api/datastore/migrate_legacyversion.go
View File

@ -40,6 +40,7 @@ var dbVerToSemVerMap = map[int]string{
60: "2.15",
61: "2.15.1",
70: "2.16",
80: "2.17",
}
func dbVersionToSemanticVersion(dbVersion int) string {

26
api/datastore/migrator/migrate_ce.go
View File

@ -36,29 +36,29 @@ func (m *Migrator) Migrate() error {
if schemaVersion.Equal(apiVersion) {
// detect and run migrations when the versions are the same.
// e.g. development builds
latestMigrations := m.latestMigrations()
if latestMigrations.version.Equal(schemaVersion) &&
version.MigratorCount != len(latestMigrations.migrationFuncs) {
err := runMigrations(latestMigrations.migrationFuncs)
latestMigrations := m.LatestMigrations()
if latestMigrations.Version.Equal(schemaVersion) &&
version.MigratorCount != len(latestMigrations.MigrationFuncs) {
err := runMigrations(latestMigrations.MigrationFuncs)
if err != nil {
return err
}
newMigratorCount = len(latestMigrations.migrationFuncs)
newMigratorCount = len(latestMigrations.MigrationFuncs)
}
} else {
// regular path when major/minor/patch versions differ
for _, migration := range m.migrations {
if schemaVersion.LessThan(migration.version) {
if schemaVersion.LessThan(migration.Version) {
log.Info().Msgf("migrating data to %s", migration.version.String())
err := runMigrations(migration.migrationFuncs)
log.Info().Msgf("migrating data to %s", migration.Version.String())
err := runMigrations(migration.MigrationFuncs)
if err != nil {
return err
}
}
if apiVersion.Equal(migration.version) {
newMigratorCount = len(migration.migrationFuncs)
if apiVersion.Equal(migration.Version) {
newMigratorCount = len(migration.MigrationFuncs)
}
}
}
@ -107,9 +107,9 @@ func (m *Migrator) NeedsMigration() bool {
}
// Check if we have any migrations for the current version
latestMigrations := m.latestMigrations()
if latestMigrations.version.Equal(semver.MustParse(portainer.APIVersion)) {
if m.currentDBVersion.MigratorCount != len(latestMigrations.migrationFuncs) {
latestMigrations := m.LatestMigrations()
if latestMigrations.Version.Equal(semver.MustParse(portainer.APIVersion)) {
if m.currentDBVersion.MigratorCount != len(latestMigrations.MigrationFuncs) {
return true
}
} else {

47
api/datastore/migrator/migrate_dbversion80.go Normal file
View File

@ -0,0 +1,47 @@
package migrator
import (
portainer "github.com/portainer/portainer/api"
"github.com/rs/zerolog/log"
)
func (m *Migrator) migrateDBVersionToDB80() error {
return m.updateEdgeStackStatusForDB80()
}
func (m *Migrator) updateEdgeStackStatusForDB80() error {
log.Info().Msg("transfer type field to details field for edge stack status")
edgeStacks, err := m.edgeStackService.EdgeStacks()
if err != nil {
return err
}
for _, edgeStack := range edgeStacks {
for endpointId, status := range edgeStack.Status {
switch status.Type {
case portainer.EdgeStackStatusPending:
status.Details.Pending = true
case portainer.EdgeStackStatusOk:
status.Details.Ok = true
case portainer.EdgeStackStatusError:
status.Details.Error = true
case portainer.EdgeStackStatusAcknowledged:
status.Details.Acknowledged = true
case portainer.EdgeStackStatusRemove:
status.Details.Remove = true
case portainer.EdgeStackStatusRemoteUpdateSuccess:
status.Details.RemoteUpdateSuccess = true
}
edgeStack.Status[endpointId] = status
}
err = m.edgeStackService.UpdateEdgeStack(edgeStack.ID, &edgeStack)
if err != nil {
return err
}
}
return nil
}

16
api/datastore/migrator/migrator.go
View File

@ -3,6 +3,8 @@ package migrator
import (
"errors"
"github.com/portainer/portainer/api/dataservices/edgestack"
"github.com/Masterminds/semver"
"github.com/rs/zerolog/log"
@ -53,6 +55,7 @@ type (
fileService portainer.FileService
authorizationService *authorization.Service
dockerhubService *dockerhub.Service
edgeStackService *edgestack.Service
}
// MigratorParameters represents the required parameters to create a new Migrator instance.
@ -77,6 +80,7 @@ type (
FileService portainer.FileService
AuthorizationService *authorization.Service
DockerhubService *dockerhub.Service
EdgeStackService *edgestack.Service
}
)
@ -103,6 +107,7 @@ func NewMigrator(parameters *MigratorParameters) *Migrator {
fileService: parameters.FileService,
authorizationService: parameters.AuthorizationService,
dockerhubService: parameters.DockerhubService,
edgeStackService: parameters.EdgeStackService,
}
migrator.initMigrations()
@ -128,12 +133,12 @@ func (m *Migrator) CurrentSemanticDBVersion() *semver.Version {
func (m *Migrator) addMigrations(v string, funcs ...func() error) {
m.migrations = append(m.migrations, Migrations{
version: semver.MustParse(v),
migrationFuncs: funcs,
Version: semver.MustParse(v),
MigrationFuncs: funcs,
})
}
func (m *Migrator) latestMigrations() Migrations {
func (m *Migrator) LatestMigrations() Migrations {
return m.migrations[len(m.migrations)-1]
}
@ -146,8 +151,8 @@ func (m *Migrator) latestMigrations() Migrations {
// ! This increases the migration funcs count and so they all run again.
type Migrations struct {
version *semver.Version
migrationFuncs MigrationFuncs
Version *semver.Version
MigrationFuncs MigrationFuncs
}
type MigrationFuncs []func() error
@ -199,6 +204,7 @@ func (m *Migrator) initMigrations() {
m.addMigrations("2.15", m.migrateDBVersionToDB60)
m.addMigrations("2.16", m.migrateDBVersionToDB70)
m.addMigrations("2.16.1", m.migrateDBVersionToDB71)
m.addMigrations("2.17", m.migrateDBVersionToDB80)
// Add new migrations below...
// One function per migration, each versions migration funcs in the same file.

2
api/datastore/test_data/output_24_to_latest.json
View File

@ -931,6 +931,6 @@
}
],
"version": {
"VERSION": "{\"SchemaVersion\":\"2.17.0\",\"MigratorCount\":0,\"Edition\":1,\"InstanceID\":\"463d5c47-0ea5-4aca-85b1-405ceefee254\"}"
"VERSION": "{\"SchemaVersion\":\"2.17.0\",\"MigratorCount\":1,\"Edition\":1,\"InstanceID\":\"463d5c47-0ea5-4aca-85b1-405ceefee254\"}"
}
}

20
api/http/handler/edgestacks/edgestack_status_update.go
View File

@ -27,7 +27,7 @@ func (payload *updateStatusPayload) Validate(r *http.Request) error {
return errors.New("Invalid EnvironmentID")
}
if *payload.Status == portainer.StatusError && govalidator.IsNull(payload.Error) {
if *payload.Status == portainer.EdgeStackStatusError && govalidator.IsNull(payload.Error) {
return errors.New("Error message is mandatory when status is error")
}
@ -74,8 +74,24 @@ func (handler *Handler) edgeStackStatusUpdate(w http.ResponseWriter, r *http.Req
var stack portainer.EdgeStack
err = handler.DataStore.EdgeStack().UpdateEdgeStackFunc(portainer.EdgeStackID(stackID), func(edgeStack *portainer.EdgeStack) {
details := edgeStack.Status[payload.EndpointID].Details
details.Pending = false
switch *payload.Status {
case portainer.EdgeStackStatusOk:
details.Ok = true
case portainer.EdgeStackStatusError:
details.Error = true
case portainer.EdgeStackStatusAcknowledged:
details.Acknowledged = true
case portainer.EdgeStackStatusRemove:
details.Remove = true
case portainer.EdgeStackStatusImagesPulled:
details.ImagesPulled = true
}
edgeStack.Status[payload.EndpointID] = portainer.EdgeStackStatus{
Type: *payload.Status,
Details: details,
Error: payload.Error,
EndpointID: payload.EndpointID,
}

12
api/http/handler/edgestacks/edgestack_test.go
View File

@ -149,7 +149,7 @@ func createEdgeStack(t *testing.T, store dataservices.DataStore, endpointID port
ID: edgeStackID,
Name: "test-edge-stack-" + strconv.Itoa(int(edgeStackID)),
Status: map[portainer.EndpointID]portainer.EdgeStackStatus{
endpointID: {Type: portainer.StatusOk, Error: "", EndpointID: endpointID},
endpointID: {Details: portainer.EdgeStackStatusDetails{Ok: true}, Error: "", EndpointID: endpointID},
},
CreationDate: time.Now().Unix(),
EdgeGroups: []portainer.EdgeGroupID{edgeGroup.ID},
@ -775,7 +775,7 @@ func TestUpdateStatusAndInspect(t *testing.T) {
edgeStack := createEdgeStack(t, handler.DataStore, endpoint.ID)
// Update edge stack status
newStatus := portainer.StatusError
newStatus := portainer.EdgeStackStatusError
payload := updateStatusPayload{
Error: "test-error",
Status: &newStatus,
@ -821,8 +821,8 @@ func TestUpdateStatusAndInspect(t *testing.T) {
t.Fatal("error decoding response:", err)
}
if data.Status[endpoint.ID].Type != *payload.Status {
t.Fatalf("expected EdgeStackStatusType %d, found %d", payload.Status, data.Status[endpoint.ID].Type)
if !data.Status[endpoint.ID].Details.Error {
t.Fatalf("expected EdgeStackStatusType %d, found %t", payload.Status, data.Status[endpoint.ID].Details.Error)
}
if data.Status[endpoint.ID].Error != payload.Error {
@ -841,8 +841,8 @@ func TestUpdateStatusWithInvalidPayload(t *testing.T) {
edgeStack := createEdgeStack(t, handler.DataStore, endpoint.ID)
// Update edge stack status
statusError := portainer.StatusError
statusOk := portainer.StatusOk
statusError := portainer.EdgeStackStatusError
statusOk := portainer.EdgeStackStatusOk
cases := []struct {
Name string
Payload updateStatusPayload

3
api/http/handler/edgestacks/edgestack_update.go
View File

@ -193,6 +193,9 @@ func (handler *Handler) edgeStackUpdate(w http.ResponseWriter, r *http.Request)
stack.Status = map[portainer.EndpointID]portainer.EdgeStackStatus{}
}
stack.NumDeployments = len(relatedEndpointIds)
stack.Status = make(map[portainer.EndpointID]portainer.EdgeStackStatus)
err = handler.DataStore.EdgeStack().UpdateEdgeStack(stack.ID, stack)
if err != nil {
return httperror.InternalServerError("Unable to persist the stack changes inside the database", err)

2
api/http/handler/endpointedge/endpointedge_status_inspect_test.go
View File

@ -324,7 +324,7 @@ func TestEdgeStackStatus(t *testing.T) {
ID: edgeStackID,
Name: "test-edge-stack-17",
Status: map[portainer.EndpointID]portainer.EdgeStackStatus{
endpointID: {Type: portainer.StatusOk, Error: "", EndpointID: endpoint.ID},
endpointID: {Details: portainer.EdgeStackStatusDetails{Ok: true}, Error: "", EndpointID: endpoint.ID},
},
CreationDate: time.Now().Unix(),
EdgeGroups: []portainer.EdgeGroupID{1, 2},

1
api/internal/edge/edgestacks/service.go
View File

@ -92,6 +92,7 @@ func (service *Service) PersistEdgeStack(
stack.ManifestPath = manifestPath
stack.ProjectPath = projectPath
stack.EntryPoint = composePath
stack.NumDeployments = len(relatedEndpointIds)
err = service.updateEndpointRelations(stack.ID, relatedEndpointIds)
if err != nil {

37
api/portainer.go
View File

@ -283,6 +283,7 @@ type (
ProjectPath string `json:"ProjectPath"`
EntryPoint string `json:"EntryPoint"`
Version int `json:"Version"`
NumDeployments int `json:"NumDeployments"`
ManifestPath string
DeploymentType EdgeStackDeploymentType
// Uses the manifest's namespaces instead of the default one
@ -297,11 +298,24 @@ type (
//EdgeStackID represents an edge stack id
EdgeStackID int
EdgeStackStatusDetails struct {
Pending bool
Ok bool
Error bool
Acknowledged bool
Remove bool
RemoteUpdateSuccess bool
ImagesPulled bool
}
//EdgeStackStatus represents an edge stack status
EdgeStackStatus struct {
Type EdgeStackStatusType `json:"Type"`
Details EdgeStackStatusDetails `json:"Details"`
Error string `json:"Error"`
EndpointID EndpointID `json:"EndpointID"`
// Deprecated
Type EdgeStackStatusType `json:"Type"`
}
//EdgeStackStatusType represents an edge stack status type
@ -1558,13 +1572,20 @@ const (
)
const (
_ EdgeStackStatusType = iota
//StatusOk represents a successfully deployed edge stack
StatusOk
//StatusError represents an edge environment(endpoint) which failed to deploy its edge stack
StatusError
//StatusAcknowledged represents an acknowledged edge stack
StatusAcknowledged
// EdgeStackStatusPending represents a pending edge stack
EdgeStackStatusPending EdgeStackStatusType = iota
//EdgeStackStatusOk represents a successfully deployed edge stack
EdgeStackStatusOk
//EdgeStackStatusError represents an edge environment(endpoint) which failed to deploy its edge stack
EdgeStackStatusError
//EdgeStackStatusAcknowledged represents an acknowledged edge stack
EdgeStackStatusAcknowledged
//EdgeStackStatusRemove represents a removed edge stack (status isn't persisted)
EdgeStackStatusRemove
// StatusRemoteUpdateSuccess represents a successfully updated edge stack
EdgeStackStatusRemoteUpdateSuccess
// EdgeStackStatusImagesPulled represents a successfully images-pulling
EdgeStackStatusImagesPulled
)
const (

9
app/edge/components/edge-stack-endpoints-datatable/edgeStackEndpointsDatatableController.js
View File

@ -20,12 +20,6 @@ export class EdgeStackEndpointsDatatableController {
this.onPageChange = this.onPageChange.bind(this);
this.paginationChanged = this.paginationChanged.bind(this);
this.paginationChangedAsync = this.paginationChangedAsync.bind(this);
this.statusMap = {
1: 'OK',
2: 'Error',
3: 'Acknowledged',
};
}
extendGenericController($controller, $scope) {
@ -45,8 +39,9 @@ export class EdgeStackEndpointsDatatableController {
endpointStatusLabel(endpointId) {
const status = this.getEndpointStatus(endpointId);
const details = (status && status.Details) || {};
return status ? this.statusMap[status.Type] : 'Pending';
return (details.Error && 'Error') || (details.Ok && 'Ok') || (details.ImagesPulled && 'Images pre-pulled') || (details.Acknowledged && 'Acknowledged') || 'Pending';
}
endpointStatusError(endpointId) {

28
app/edge/components/edge-stacks-datatable/edgeStackDatatable.css Normal file
View File

@ -0,0 +1,28 @@
.edge-stack-status {
padding: 2px 10px;
border-radius: 10px;
}
.edge-stack-status.status-acknowledged {
color: #337ab7;
background-color: rgba(51, 122, 183, 0.1);
}
.edge-stack-status.status-images-pulled {
color: #e1a800;
background-color: rgba(238, 192, 32, 0.1);
}
.edge-stack-status.status-ok {
color: #23ae89;
background-color: rgba(35, 174, 137, 0.1);
}
.edge-stack-status.status-error {
color: #ae2323;
background-color: rgba(174, 35, 35, 0.1);
}
.edge-stack-status.status-total {
background-color: rgba(168, 167, 167, 0.1);
}

33
app/edge/components/edge-stacks-datatable/edgeStacksDatatable.html
View File

@ -87,7 +87,10 @@
></table-column-header>
</div>
</th>
<th> Status </th>
<th class="text-center"> Acknowledged </th>
<th class="text-center"> Deployed </th>
<th class="text-center"> Failed </th>
<th class="text-center"> Total deployments </th>
<th>
<table-column-header
col-title="'Creation Date'"
@ -113,14 +116,36 @@
{{ item.Name }}
</a>
</td>
<td><edge-stack-status stack-status="item.Status"></edge-stack-status></td>
<td class="text-center">
<span class="edge-stack-status status-acknowledged">
&bull;
{{ item.aggregateStatus.acknowledged }}
</span>
</td>
<td class="text-center">
<span class="edge-stack-status status-ok">
&bull;
{{ item.aggregateStatus.ok }}
</span>
</td>
<td class="text-center">
<span class="edge-stack-status status-error">
&bull;
{{ item.aggregateStatus.error }}
</span>
</td>
<td class="text-center">
<span class="edge-stack-status status-total">
{{ item.NumDeployments }}
</span>
</td>
<td>{{ item.CreationDate | getisodatefromtimestamp }}</td>
</tr>
<tr ng-if="!$ctrl.dataset" data-cy="edgeStack-loadingRow">
<td colspan="4" class="text-center text-muted">Loading...</td>
<td colspan="6" class="text-center text-muted">Loading...</td>
</tr>
<tr ng-if="$ctrl.state.filteredDataSet.length === 0" data-cy="edgeStack-noStackRow">
<td colspan="4" class="text-center text-muted"> No stack available. </td>
<td colspan="6" class="text-center text-muted"> No stack available. </td>
</tr>
</tbody>
</table>

1
app/edge/components/edge-stacks-datatable/index.js
View File

@ -1,4 +1,5 @@
import angular from 'angular';
import './edgeStackDatatable.css';
angular.module('portainer.edge').component('edgeStacksDatatable', {
templateUrl: './edgeStacksDatatable.html',

4
app/edge/components/edit-edge-stack-form/editEdgeStackForm.html
View File

@ -32,7 +32,7 @@
</div>
<web-editor-form
ng-if="$ctrl.model.DeploymentType === 0"
ng-if="$ctrl.model.DeploymentType === $ctrl.EditorType.Compose"
value="$ctrl.model.StackFileContent"
yml="true"
identifier="compose-editor"
@ -48,7 +48,7 @@
</editor-description>
</web-editor-form>
<div ng-if="$ctrl.model.DeploymentType === 1">
<div ng-if="$ctrl.model.DeploymentType === $ctrl.EditorType.Kubernetes">
<div class="form-group">
<div class="col-sm-12">
<por-switch-field

3
app/edge/components/edit-edge-stack-form/editEdgeStackFormController.js
View File

@ -1,4 +1,5 @@
import { PortainerEndpointTypes } from '@/portainer/models/endpoint/models';
import { EditorType } from '@/react/edge/edge-stacks/types';
export class EditEdgeStackFormController {
/* @ngInject */
@ -13,6 +14,8 @@ export class EditEdgeStackFormController {
1: '',
};
this.EditorType = EditorType;
this.onChangeGroups = this.onChangeGroups.bind(this);
this.onChangeFileContent = this.onChangeFileContent.bind(this);
this.onChangeComposeConfig = this.onChangeComposeConfig.bind(this);

4
app/edge/views/edge-stacks/createEdgeStackView/create-edge-stack-view.controller.js
View File

@ -1,3 +1,5 @@
import { EditorType } from '@/react/edge/edge-stacks/types';
export default class CreateEdgeStackViewController {
/* @ngInject */
constructor($state, $window, ModalService, EdgeStackService, EdgeGroupService, EdgeTemplateService, Notifications, FormHelper, $async, $scope) {
@ -19,6 +21,8 @@ export default class CreateEdgeStackViewController {
UseManifestNamespaces: false,
};
this.EditorType = EditorType;
this.state = {
Method: 'editor',
formValidationError: '',

12
app/edge/views/edge-stacks/createEdgeStackView/create-edge-stack-view.html
View File

@ -57,9 +57,17 @@
</div>
</div>
<edge-stacks-docker-compose-form ng-if="$ctrl.formValues.DeploymentType == 0" form-values="$ctrl.formValues" state="$ctrl.state"></edge-stacks-docker-compose-form>
<edge-stacks-docker-compose-form
ng-if="$ctrl.formValues.DeploymentType == $ctrl.EditorType.Compose"
form-values="$ctrl.formValues"
state="$ctrl.state"
></edge-stacks-docker-compose-form>
<edge-stacks-kube-manifest-form ng-if="$ctrl.formValues.DeploymentType == 1" form-values="$ctrl.formValues" state="$ctrl.state"></edge-stacks-kube-manifest-form>
<edge-stacks-kube-manifest-form
ng-if="$ctrl.formValues.DeploymentType == $ctrl.EditorType.Kubernetes"
form-values="$ctrl.formValues"
state="$ctrl.state"
></edge-stacks-kube-manifest-form>
<!-- actions -->
<div class="col-sm-12 form-section-title"> Actions </div>

16
app/edge/views/edge-stacks/edgeStacksView/edgeStacksViewController.js
View File

@ -38,9 +38,25 @@ export class EdgeStacksViewController {
this.$state.reload();
}
aggregateStatus() {
if (this.stacks) {
this.stacks.forEach((stack) => {
const aggregateStatus = { ok: 0, error: 0, acknowledged: 0 };
for (let endpointId in stack.Status) {
const { Details } = stack.Status[endpointId];
aggregateStatus.ok += Number(Details.Ok);
aggregateStatus.error += Number(Details.Error);
aggregateStatus.acknowledged += Number(Details.Acknowledged);
}
stack.aggregateStatus = aggregateStatus;
});
}
}
async getStacks() {
try {
this.stacks = await this.EdgeStackService.stacks();
this.aggregateStatus();
} catch (err) {
this.stacks = [];
this.Notifications.error('Failure', err, 'Unable to retrieve stacks');

5
app/react/edge/edge-stacks/types.ts Normal file
View File

@ -0,0 +1,5 @@
export enum EditorType {
Compose,
Kubernetes,
Nomad,
}