diff --git a/api/bolt/endpoint_service.go b/api/bolt/endpoint_service.go index f8bbdd795..df086bea7 100644 --- a/api/bolt/endpoint_service.go +++ b/api/bolt/endpoint_service.go @@ -67,6 +67,49 @@ func (service *EndpointService) Endpoints() ([]portainer.Endpoint, error) { return endpoints, nil } +// Synchronize creates, updates and deletes endpoints inside a single transaction. +func (service *EndpointService) Synchronize(toCreate, toUpdate, toDelete []*portainer.Endpoint) error { + return service.store.db.Update(func(tx *bolt.Tx) error { + bucket := tx.Bucket([]byte(endpointBucketName)) + + for _, endpoint := range toCreate { + id, _ := bucket.NextSequence() + endpoint.ID = portainer.EndpointID(id) + + data, err := internal.MarshalEndpoint(endpoint) + if err != nil { + return err + } + + err = bucket.Put(internal.Itob(int(endpoint.ID)), data) + if err != nil { + return err + } + } + + for _, endpoint := range toUpdate { + data, err := internal.MarshalEndpoint(endpoint) + if err != nil { + return err + } + + err = bucket.Put(internal.Itob(int(endpoint.ID)), data) + if err != nil { + return err + } + } + + for _, endpoint := range toDelete { + err := bucket.Delete(internal.Itob(int(endpoint.ID))) + if err != nil { + return err + } + } + + return nil + }) +} + // CreateEndpoint assign an ID to a new endpoint and saves it. func (service *EndpointService) CreateEndpoint(endpoint *portainer.Endpoint) error { return service.store.db.Update(func(tx *bolt.Tx) error { diff --git a/api/cli/cli.go b/api/cli/cli.go index acaaa8ea7..36c459225 100644 --- a/api/cli/cli.go +++ b/api/cli/cli.go @@ -1,6 +1,8 @@ package cli import ( + "time" + "github.com/portainer/portainer" "os" @@ -15,6 +17,8 @@ type Service struct{} const ( errInvalidEnpointProtocol = portainer.Error("Invalid endpoint protocol: Portainer only supports unix:// or tcp://") errSocketNotFound = portainer.Error("Unable to locate Unix socket") + errEndpointsFileNotFound = portainer.Error("Unable to locate external endpoints file") + errInvalidSyncInterval = portainer.Error("Invalid synchronization interval") ) // ParseFlags parse the CLI flags and return a portainer.Flags struct @@ -22,18 +26,20 @@ func (*Service) ParseFlags(version string) (*portainer.CLIFlags, error) { kingpin.Version(version) flags := &portainer.CLIFlags{ - Endpoint: kingpin.Flag("host", "Dockerd endpoint").Short('H').String(), - Logo: kingpin.Flag("logo", "URL for the logo displayed in the UI").String(), - Labels: pairs(kingpin.Flag("hide-label", "Hide containers with a specific label in the UI").Short('l')), - Addr: kingpin.Flag("bind", "Address and port to serve Portainer").Default(defaultBindAddress).Short('p').String(), - Assets: kingpin.Flag("assets", "Path to the assets").Default(defaultAssetsDirectory).Short('a').String(), - Data: kingpin.Flag("data", "Path to the folder where the data is stored").Default(defaultDataDirectory).Short('d').String(), - Templates: kingpin.Flag("templates", "URL to the templates (apps) definitions").Default(defaultTemplatesURL).Short('t').String(), - NoAuth: kingpin.Flag("no-auth", "Disable authentication").Default(defaultNoAuth).Bool(), - TLSVerify: kingpin.Flag("tlsverify", "TLS support").Default(defaultTLSVerify).Bool(), - TLSCacert: kingpin.Flag("tlscacert", "Path to the CA").Default(defaultTLSCACertPath).String(), - TLSCert: kingpin.Flag("tlscert", "Path to the TLS certificate file").Default(defaultTLSCertPath).String(), - TLSKey: kingpin.Flag("tlskey", "Path to the TLS key").Default(defaultTLSKeyPath).String(), + Endpoint: kingpin.Flag("host", "Dockerd endpoint").Short('H').String(), + Logo: kingpin.Flag("logo", "URL for the logo displayed in the UI").String(), + ExternalEndpoints: kingpin.Flag("external-endpoints", "Path to a file defining available endpoints").String(), + SyncInterval: kingpin.Flag("sync-interval", "Duration between each synchronization via the external endpoints source").Default(defaultSyncInterval).String(), + Labels: pairs(kingpin.Flag("hide-label", "Hide containers with a specific label in the UI").Short('l')), + Addr: kingpin.Flag("bind", "Address and port to serve Portainer").Default(defaultBindAddress).Short('p').String(), + Assets: kingpin.Flag("assets", "Path to the assets").Default(defaultAssetsDirectory).Short('a').String(), + Data: kingpin.Flag("data", "Path to the folder where the data is stored").Default(defaultDataDirectory).Short('d').String(), + Templates: kingpin.Flag("templates", "URL to the templates (apps) definitions").Default(defaultTemplatesURL).Short('t').String(), + NoAuth: kingpin.Flag("no-auth", "Disable authentication").Default(defaultNoAuth).Bool(), + TLSVerify: kingpin.Flag("tlsverify", "TLS support").Default(defaultTLSVerify).Bool(), + TLSCacert: kingpin.Flag("tlscacert", "Path to the CA").Default(defaultTLSCACertPath).String(), + TLSCert: kingpin.Flag("tlscert", "Path to the TLS certificate file").Default(defaultTLSCertPath).String(), + TLSKey: kingpin.Flag("tlskey", "Path to the TLS key").Default(defaultTLSKeyPath).String(), } kingpin.Parse() @@ -58,5 +64,21 @@ func (*Service) ValidateFlags(flags *portainer.CLIFlags) error { } } + if *flags.ExternalEndpoints != "" { + if _, err := os.Stat(*flags.ExternalEndpoints); err != nil { + if os.IsNotExist(err) { + return errEndpointsFileNotFound + } + return err + } + } + + if *flags.SyncInterval != defaultSyncInterval { + _, err := time.ParseDuration(*flags.SyncInterval) + if err != nil { + return errInvalidSyncInterval + } + } + return nil } diff --git a/api/cli/defaults.go b/api/cli/defaults.go index 57c755ea0..4545ecf79 100644 --- a/api/cli/defaults.go +++ b/api/cli/defaults.go @@ -12,4 +12,5 @@ const ( defaultTLSCACertPath = "/certs/ca.pem" defaultTLSCertPath = "/certs/cert.pem" defaultTLSKeyPath = "/certs/key.pem" + defaultSyncInterval = "60s" ) diff --git a/api/cli/defaults_windows.go b/api/cli/defaults_windows.go index 6ffc1f331..17c6c3776 100644 --- a/api/cli/defaults_windows.go +++ b/api/cli/defaults_windows.go @@ -10,4 +10,5 @@ const ( defaultTLSCACertPath = "C:\\certs\\ca.pem" defaultTLSCertPath = "C:\\certs\\cert.pem" defaultTLSKeyPath = "C:\\certs\\key.pem" + defaultSyncInterval = "60s" ) diff --git a/api/cmd/portainer/main.go b/api/cmd/portainer/main.go index 4e12d2091..9b56f3238 100644 --- a/api/cmd/portainer/main.go +++ b/api/cmd/portainer/main.go @@ -4,6 +4,7 @@ import ( "github.com/portainer/portainer" "github.com/portainer/portainer/bolt" "github.com/portainer/portainer/cli" + "github.com/portainer/portainer/cron" "github.com/portainer/portainer/crypto" "github.com/portainer/portainer/file" "github.com/portainer/portainer/http" @@ -24,12 +25,6 @@ func main() { log.Fatal(err) } - settings := &portainer.Settings{ - HiddenLabels: *flags.Labels, - Logo: *flags.Logo, - Authentication: !*flags.NoAuth, - } - fileService, err := file.NewService(*flags.Data, "") if err != nil { log.Fatal(err) @@ -52,6 +47,25 @@ func main() { var cryptoService portainer.CryptoService = &crypto.Service{} + var endpointWatcher portainer.EndpointWatcher + authorizeEndpointMgmt := true + if *flags.ExternalEndpoints != "" { + log.Println("Using external endpoint definition. Disabling endpoint management via API.") + authorizeEndpointMgmt = false + endpointWatcher = cron.NewWatcher(store.EndpointService, *flags.SyncInterval) + err = endpointWatcher.WatchEndpointFile(*flags.ExternalEndpoints) + if err != nil { + log.Fatal(err) + } + } + + settings := &portainer.Settings{ + HiddenLabels: *flags.Labels, + Logo: *flags.Logo, + Authentication: !*flags.NoAuth, + EndpointManagement: authorizeEndpointMgmt, + } + // Initialize the active endpoint from the CLI only if there is no // active endpoint defined yet. var activeEndpoint *portainer.Endpoint @@ -74,19 +88,36 @@ func main() { log.Fatal(err) } } + if *flags.ExternalEndpoints != "" { + activeEndpoint, err = store.EndpointService.GetActive() + if err == portainer.ErrEndpointNotFound { + var endpoints []portainer.Endpoint + endpoints, err = store.EndpointService.Endpoints() + if err != nil { + log.Fatal(err) + } + err = store.EndpointService.SetActive(&endpoints[0]) + if err != nil { + log.Fatal(err) + } + } else if err != nil { + log.Fatal(err) + } + } var server portainer.Server = &http.Server{ - BindAddress: *flags.Addr, - AssetsPath: *flags.Assets, - Settings: settings, - TemplatesURL: *flags.Templates, - AuthDisabled: *flags.NoAuth, - UserService: store.UserService, - EndpointService: store.EndpointService, - CryptoService: cryptoService, - JWTService: jwtService, - FileService: fileService, - ActiveEndpoint: activeEndpoint, + BindAddress: *flags.Addr, + AssetsPath: *flags.Assets, + Settings: settings, + TemplatesURL: *flags.Templates, + AuthDisabled: *flags.NoAuth, + EndpointManagement: authorizeEndpointMgmt, + UserService: store.UserService, + EndpointService: store.EndpointService, + CryptoService: cryptoService, + JWTService: jwtService, + FileService: fileService, + ActiveEndpoint: activeEndpoint, } log.Printf("Starting Portainer on %s", *flags.Addr) diff --git a/api/cron/endpoint_sync.go b/api/cron/endpoint_sync.go new file mode 100644 index 000000000..221143087 --- /dev/null +++ b/api/cron/endpoint_sync.go @@ -0,0 +1,166 @@ +package cron + +import ( + "encoding/json" + "io/ioutil" + "log" + "os" + + "github.com/portainer/portainer" +) + +type ( + endpointSyncJob struct { + logger *log.Logger + endpointService portainer.EndpointService + endpointFilePath string + } + + synchronization struct { + endpointsToCreate []*portainer.Endpoint + endpointsToUpdate []*portainer.Endpoint + endpointsToDelete []*portainer.Endpoint + } +) + +const ( + // ErrEmptyEndpointArray is an error raised when the external endpoint source array is empty. + ErrEmptyEndpointArray = portainer.Error("External endpoint source is empty") +) + +func newEndpointSyncJob(endpointFilePath string, endpointService portainer.EndpointService) endpointSyncJob { + return endpointSyncJob{ + logger: log.New(os.Stderr, "", log.LstdFlags), + endpointService: endpointService, + endpointFilePath: endpointFilePath, + } +} + +func endpointSyncError(err error, logger *log.Logger) bool { + if err != nil { + logger.Printf("Endpoint synchronization error: %s", err) + return true + } + return false +} + +func isValidEndpoint(endpoint *portainer.Endpoint) bool { + if endpoint.Name != "" && endpoint.URL != "" { + return true + } + return false +} + +func endpointExists(endpoint *portainer.Endpoint, endpoints []portainer.Endpoint) int { + for idx, v := range endpoints { + if endpoint.Name == v.Name && isValidEndpoint(&v) { + return idx + } + } + return -1 +} + +func mergeEndpointIfRequired(original, updated *portainer.Endpoint) *portainer.Endpoint { + var endpoint *portainer.Endpoint + if original.URL != updated.URL || original.TLS != updated.TLS { + endpoint = original + endpoint.URL = updated.URL + if updated.TLS { + endpoint.TLS = true + endpoint.TLSCACertPath = updated.TLSCACertPath + endpoint.TLSCertPath = updated.TLSCertPath + endpoint.TLSKeyPath = updated.TLSKeyPath + } else { + endpoint.TLS = false + endpoint.TLSCACertPath = "" + endpoint.TLSCertPath = "" + endpoint.TLSKeyPath = "" + } + } + return endpoint +} + +func (sync synchronization) requireSync() bool { + if len(sync.endpointsToCreate) != 0 || len(sync.endpointsToUpdate) != 0 || len(sync.endpointsToDelete) != 0 { + return true + } + return false +} + +// TMP: endpointSyncJob method to access logger, should be generic +func (job endpointSyncJob) prepareSyncData(storedEndpoints, fileEndpoints []portainer.Endpoint) *synchronization { + endpointsToCreate := make([]*portainer.Endpoint, 0) + endpointsToUpdate := make([]*portainer.Endpoint, 0) + endpointsToDelete := make([]*portainer.Endpoint, 0) + + for idx := range storedEndpoints { + fidx := endpointExists(&storedEndpoints[idx], fileEndpoints) + if fidx != -1 { + endpoint := mergeEndpointIfRequired(&storedEndpoints[idx], &fileEndpoints[fidx]) + if endpoint != nil { + job.logger.Printf("New definition for a stored endpoint found in file, updating database. [name: %v] [url: %v]\n", endpoint.Name, endpoint.URL) + endpointsToUpdate = append(endpointsToUpdate, endpoint) + } else { + job.logger.Printf("No change detected for a stored endpoint. [name: %v] [url: %v]\n", storedEndpoints[idx].Name, storedEndpoints[idx].URL) + } + } else { + job.logger.Printf("Stored endpoint not found in file (definition might be invalid), removing from database. [name: %v] [url: %v]", storedEndpoints[idx].Name, storedEndpoints[idx].URL) + endpointsToDelete = append(endpointsToDelete, &storedEndpoints[idx]) + } + } + + for idx, endpoint := range fileEndpoints { + if endpoint.Name == "" || endpoint.URL == "" { + job.logger.Printf("Invalid file endpoint definition, skipping. [name: %v] [url: %v]", endpoint.Name, endpoint.URL) + continue + } + sidx := endpointExists(&fileEndpoints[idx], storedEndpoints) + if sidx == -1 { + job.logger.Printf("File endpoint not found in database, adding to database. [name: %v] [url: %v]", fileEndpoints[idx].Name, fileEndpoints[idx].URL) + endpointsToCreate = append(endpointsToCreate, &fileEndpoints[idx]) + } + } + + return &synchronization{ + endpointsToCreate: endpointsToCreate, + endpointsToUpdate: endpointsToUpdate, + endpointsToDelete: endpointsToDelete, + } +} + +func (job endpointSyncJob) Sync() error { + data, err := ioutil.ReadFile(job.endpointFilePath) + if endpointSyncError(err, job.logger) { + return err + } + + var fileEndpoints []portainer.Endpoint + err = json.Unmarshal(data, &fileEndpoints) + if endpointSyncError(err, job.logger) { + return err + } + + if len(fileEndpoints) == 0 { + return ErrEmptyEndpointArray + } + + storedEndpoints, err := job.endpointService.Endpoints() + if endpointSyncError(err, job.logger) { + return err + } + + sync := job.prepareSyncData(storedEndpoints, fileEndpoints) + if sync.requireSync() { + err = job.endpointService.Synchronize(sync.endpointsToCreate, sync.endpointsToUpdate, sync.endpointsToDelete) + if endpointSyncError(err, job.logger) { + return err + } + } + return nil +} + +func (job endpointSyncJob) Run() { + job.logger.Printf("Endpoint synchronization job started") + err := job.Sync() + endpointSyncError(err, job.logger) +} diff --git a/api/cron/watcher.go b/api/cron/watcher.go new file mode 100644 index 000000000..01852edb7 --- /dev/null +++ b/api/cron/watcher.go @@ -0,0 +1,34 @@ +package cron + +import ( + "github.com/portainer/portainer" + "github.com/robfig/cron" +) + +// Watcher represents a service for managing crons. +type Watcher struct { + Cron *cron.Cron + EndpointService portainer.EndpointService + syncInterval string +} + +// NewWatcher initializes a new service. +func NewWatcher(endpointService portainer.EndpointService, syncInterval string) *Watcher { + return &Watcher{ + Cron: cron.New(), + EndpointService: endpointService, + syncInterval: syncInterval, + } +} + +// WatchEndpointFile starts a cron job to synchronize the endpoints from a file +func (watcher *Watcher) WatchEndpointFile(endpointFilePath string) error { + job := newEndpointSyncJob(endpointFilePath, watcher.EndpointService) + err := job.Sync() + if err != nil { + return err + } + watcher.Cron.AddJob("@every "+watcher.syncInterval, job) + watcher.Cron.Start() + return nil +} diff --git a/api/http/endpoint_handler.go b/api/http/endpoint_handler.go index 9f5ca31c0..7c7d0f930 100644 --- a/api/http/endpoint_handler.go +++ b/api/http/endpoint_handler.go @@ -16,13 +16,20 @@ import ( // EndpointHandler represents an HTTP API handler for managing Docker endpoints. type EndpointHandler struct { *mux.Router - Logger *log.Logger - EndpointService portainer.EndpointService - FileService portainer.FileService - server *Server - middleWareService *middleWareService + Logger *log.Logger + authorizeEndpointManagement bool + EndpointService portainer.EndpointService + FileService portainer.FileService + server *Server + middleWareService *middleWareService } +const ( + // ErrEndpointManagementDisabled is an error raised when trying to access the endpoints management endpoints + // when the server has been started with the --external-endpoints flag + ErrEndpointManagementDisabled = portainer.Error("Endpoint management is disabled") +) + // NewEndpointHandler returns a new instance of EndpointHandler. func NewEndpointHandler(middleWareService *middleWareService) *EndpointHandler { h := &EndpointHandler{ @@ -65,6 +72,11 @@ func (handler *EndpointHandler) handleGetEndpoints(w http.ResponseWriter, r *htt // if the active URL parameter is specified, will also define the new endpoint as the active endpoint. // /endpoints(?active=true|false) func (handler *EndpointHandler) handlePostEndpoints(w http.ResponseWriter, r *http.Request) { + if !handler.authorizeEndpointManagement { + Error(w, ErrEndpointManagementDisabled, http.StatusServiceUnavailable, handler.Logger) + return + } + var req postEndpointsRequest if err := json.NewDecoder(r.Body).Decode(&req); err != nil { Error(w, ErrInvalidJSON, http.StatusBadRequest, handler.Logger) @@ -203,6 +215,11 @@ func (handler *EndpointHandler) handlePostEndpoint(w http.ResponseWriter, r *htt // handlePutEndpoint handles PUT requests on /endpoints/:id func (handler *EndpointHandler) handlePutEndpoint(w http.ResponseWriter, r *http.Request) { + if !handler.authorizeEndpointManagement { + Error(w, ErrEndpointManagementDisabled, http.StatusServiceUnavailable, handler.Logger) + return + } + vars := mux.Vars(r) id := vars["id"] @@ -262,6 +279,11 @@ type putEndpointsRequest struct { // handleDeleteEndpoint handles DELETE requests on /endpoints/:id // DELETE /endpoints/0 deletes the active endpoint func (handler *EndpointHandler) handleDeleteEndpoint(w http.ResponseWriter, r *http.Request) { + if !handler.authorizeEndpointManagement { + Error(w, ErrEndpointManagementDisabled, http.StatusServiceUnavailable, handler.Logger) + return + } + vars := mux.Vars(r) id := vars["id"] diff --git a/api/http/server.go b/api/http/server.go index 784edc5dc..dc09db888 100644 --- a/api/http/server.go +++ b/api/http/server.go @@ -8,18 +8,19 @@ import ( // Server implements the portainer.Server interface type Server struct { - BindAddress string - AssetsPath string - AuthDisabled bool - UserService portainer.UserService - EndpointService portainer.EndpointService - CryptoService portainer.CryptoService - JWTService portainer.JWTService - FileService portainer.FileService - Settings *portainer.Settings - TemplatesURL string - ActiveEndpoint *portainer.Endpoint - Handler *Handler + BindAddress string + AssetsPath string + AuthDisabled bool + EndpointManagement bool + UserService portainer.UserService + EndpointService portainer.EndpointService + CryptoService portainer.CryptoService + JWTService portainer.JWTService + FileService portainer.FileService + Settings *portainer.Settings + TemplatesURL string + ActiveEndpoint *portainer.Endpoint + Handler *Handler } func (server *Server) updateActiveEndpoint(endpoint *portainer.Endpoint) error { @@ -61,6 +62,7 @@ func (server *Server) Start() error { var websocketHandler = NewWebSocketHandler() // EndpointHandler requires a reference to the server to be able to update the active endpoint. var endpointHandler = NewEndpointHandler(middleWareService) + endpointHandler.authorizeEndpointManagement = server.EndpointManagement endpointHandler.EndpointService = server.EndpointService endpointHandler.FileService = server.FileService endpointHandler.server = server diff --git a/api/portainer.go b/api/portainer.go index bc5af38ac..a3e8b76bf 100644 --- a/api/portainer.go +++ b/api/portainer.go @@ -13,25 +13,28 @@ type ( // CLIFlags represents the available flags on the CLI. CLIFlags struct { - Addr *string - Assets *string - Data *string - Endpoint *string - Labels *[]Pair - Logo *string - Templates *string - NoAuth *bool - TLSVerify *bool - TLSCacert *string - TLSCert *string - TLSKey *string + Addr *string + Assets *string + Data *string + ExternalEndpoints *string + SyncInterval *string + Endpoint *string + Labels *[]Pair + Logo *string + Templates *string + NoAuth *bool + TLSVerify *bool + TLSCacert *string + TLSCert *string + TLSKey *string } // Settings represents Portainer settings. Settings struct { - HiddenLabels []Pair `json:"hiddenLabels"` - Logo string `json:"logo"` - Authentication bool `json:"authentication"` + HiddenLabels []Pair `json:"hiddenLabels"` + Logo string `json:"logo"` + Authentication bool `json:"authentication"` + EndpointManagement bool `json:"endpointManagement"` } // User represent a user account. @@ -97,6 +100,7 @@ type ( GetActive() (*Endpoint, error) SetActive(endpoint *Endpoint) error DeleteActive() error + Synchronize(toCreate, toUpdate, toDelete []*Endpoint) error } // CryptoService represents a service for encrypting/hashing data. @@ -117,6 +121,11 @@ type ( GetPathForTLSFile(endpointID EndpointID, fileType TLSFileType) (string, error) DeleteTLSFiles(endpointID EndpointID) error } + + // EndpointWatcher represents a service to synchronize the endpoints via an external source. + EndpointWatcher interface { + WatchEndpointFile(endpointFilePath string) error + } ) const ( diff --git a/app/components/endpoint/endpointController.js b/app/components/endpoint/endpointController.js index 0c254b3df..e6e4eb526 100644 --- a/app/components/endpoint/endpointController.js +++ b/app/components/endpoint/endpointController.js @@ -1,6 +1,11 @@ angular.module('endpoint', []) .controller('EndpointController', ['$scope', '$state', '$stateParams', '$filter', 'EndpointService', 'Messages', function ($scope, $state, $stateParams, $filter, EndpointService, Messages) { + + if (!$scope.applicationState.application.endpointManagement) { + $state.go('endpoints'); + } + $scope.state = { error: '', uploadInProgress: false diff --git a/app/components/endpoints/endpoints.html b/app/components/endpoints/endpoints.html index 7c6acea3e..41501eaa5 100644 --- a/app/components/endpoints/endpoints.html +++ b/app/components/endpoints/endpoints.html @@ -8,7 +8,19 @@ Endpoint management -
+
+
+ + + + + Portainer has been started using the --external-endpoints flag. Endpoint management via the UI is disabled. + + +
+
+ +
@@ -113,7 +125,7 @@
-
+
@@ -125,7 +137,7 @@ - + - + - + -
Name @@ -147,16 +159,16 @@
{{ endpoint.Name }} {{ endpoint.URL | stripprotocol }} + Edit diff --git a/app/services/stateManager.js b/app/services/stateManager.js index ee71fee4c..573747984 100644 --- a/app/services/stateManager.js +++ b/app/services/stateManager.js @@ -24,6 +24,7 @@ angular.module('portainer.services') } else { Config.$promise.then(function success(data) { state.application.authentication = data.authentication; + state.application.endpointManagement = data.endpointManagement; state.application.logo = data.logo; LocalStorage.storeApplicationState(state.application); state.loading = false;