From d570aee5546854312c0426cd23b7b8c5f753d6ea Mon Sep 17 00:00:00 2001 From: Chao Geng <93526589+chaogeng77977@users.noreply.github.com> Date: Wed, 14 Sep 2022 14:47:24 +0800 Subject: [PATCH] feat(image): upload local files for building image EE-3021 (#7507) * support to make multiple files in archive buffer * upload files by multipart --- api/archive/tar.go | 42 ++++++++++ api/http/proxy/factory/docker/build.go | 82 ++++++++++++++++--- app/docker/services/buildService.js | 18 ++++ .../images/build/buildImageController.js | 12 ++- app/docker/views/images/build/buildimage.html | 14 ++++ app/portainer/services/fileUpload.js | 17 ++++ 6 files changed, 171 insertions(+), 14 deletions(-) diff --git a/api/archive/tar.go b/api/archive/tar.go index 3beccec8a..b14448da1 100644 --- a/api/archive/tar.go +++ b/api/archive/tar.go @@ -34,3 +34,45 @@ func TarFileInBuffer(fileContent []byte, fileName string, mode int64) ([]byte, e return buffer.Bytes(), nil } + +// tarFileInBuffer represents a tar archive buffer. +type tarFileInBuffer struct { + b *bytes.Buffer + w *tar.Writer +} + +func NewTarFileInBuffer() *tarFileInBuffer { + var b bytes.Buffer + return &tarFileInBuffer{ + b: &b, + w: tar.NewWriter(&b), + } +} + +// Put puts a single file to tar archive buffer. +func (t *tarFileInBuffer) Put(fileContent []byte, fileName string, mode int64) error { + hdr := &tar.Header{ + Name: fileName, + Mode: mode, + Size: int64(len(fileContent)), + } + + if err := t.w.WriteHeader(hdr); err != nil { + return err + } + + if _, err := t.w.Write(fileContent); err != nil { + return err + } + + return nil +} + +// Bytes returns the archive as a byte array. +func (t *tarFileInBuffer) Bytes() []byte { + return t.b.Bytes() +} + +func (t *tarFileInBuffer) Close() error { + return t.w.Close() +} diff --git a/api/http/proxy/factory/docker/build.go b/api/http/proxy/factory/docker/build.go index 4bec82ec0..5c2da2650 100644 --- a/api/http/proxy/factory/docker/build.go +++ b/api/http/proxy/factory/docker/build.go @@ -3,13 +3,17 @@ package docker import ( "bytes" "encoding/json" + "errors" "io/ioutil" + "log" + "mime" "net/http" - "strings" "github.com/portainer/portainer/api/archive" ) +const OneMegabyte = 1024768 + type postDockerfileRequest struct { Content string } @@ -23,29 +27,81 @@ type postDockerfileRequest struct { // In any other case, it will leave the request unaltered. func buildOperation(request *http.Request) error { contentTypeHeader := request.Header.Get("Content-Type") - if contentTypeHeader != "" && !strings.Contains(contentTypeHeader, "application/json") { - return nil + + mediaType, _, err := mime.ParseMediaType(contentTypeHeader) + if err != nil { + return err } - var dockerfileContent []byte - - if contentTypeHeader == "" { + var buffer []byte + switch mediaType { + case "": body, err := ioutil.ReadAll(request.Body) if err != nil { return err } - dockerfileContent = body - } else { + + buffer, err = archive.TarFileInBuffer(body, "Dockerfile", 0600) + if err != nil { + return err + } + + case "application/json": var req postDockerfileRequest if err := json.NewDecoder(request.Body).Decode(&req); err != nil { return err } - dockerfileContent = []byte(req.Content) - } - buffer, err := archive.TarFileInBuffer(dockerfileContent, "Dockerfile", 0600) - if err != nil { - return err + buffer, err = archive.TarFileInBuffer([]byte(req.Content), "Dockerfile", 0600) + if err != nil { + return err + } + + case "multipart/form-data": + err := request.ParseMultipartForm(32 * OneMegabyte) // limit parser memory to 32MB + if err != nil { + return err + } + + if request.MultipartForm == nil || request.MultipartForm.File == nil { + return errors.New("uploaded files not found to build image") + } + + tfb := archive.NewTarFileInBuffer() + defer tfb.Close() + + for k := range request.MultipartForm.File { + f, hdr, err := request.FormFile(k) + if err != nil { + return err + } + + defer f.Close() + + log.Printf("[INFO] [http,proxy,docker] [message: upload the file to build image] [filename: %s] [size: %d]", hdr.Filename, hdr.Size) + + content, err := ioutil.ReadAll(f) + if err != nil { + return err + } + + filename := hdr.Filename + if hdr.Filename == "blob" { + filename = "Dockerfile" + } + + if err := tfb.Put(content, filename, 0600); err != nil { + return err + } + } + + buffer = tfb.Bytes() + request.Form = nil + request.PostForm = nil + request.MultipartForm = nil + + default: + return nil } request.Body = ioutil.NopCloser(bytes.NewReader(buffer)) diff --git a/app/docker/services/buildService.js b/app/docker/services/buildService.js index e9e5fc433..99d7ef746 100644 --- a/app/docker/services/buildService.js +++ b/app/docker/services/buildService.js @@ -66,6 +66,24 @@ angular.module('portainer.docker').factory('BuildService', [ return deferred.promise; }; + service.buildImageFromDockerfileContentAndFiles = function (names, content, files) { + var dockerfile = new Blob([content], { type: 'text/plain' }); + var uploadFiles = [dockerfile].concat(files); + + var deferred = $q.defer(); + + FileUploadService.buildImageFromFiles(names, uploadFiles) + .then(function success(response) { + var model = new ImageBuildModel(response.data); + deferred.resolve(model); + }) + .catch(function error(err) { + deferred.reject(err); + }); + + return deferred.promise; + }; + return service; }, ]); diff --git a/app/docker/views/images/build/buildImageController.js b/app/docker/views/images/build/buildImageController.js index 951b77bb5..fcd2a6aef 100644 --- a/app/docker/views/images/build/buildImageController.js +++ b/app/docker/views/images/build/buildImageController.js @@ -13,6 +13,7 @@ function BuildImageController($scope, $async, $window, ModalService, BuildServic ImageNames: [{ Name: '', Valid: false, Unique: true }], UploadFile: null, DockerFileContent: '', + AdditionalFiles: [], URL: '', Path: 'Dockerfile', NodeName: null, @@ -74,7 +75,12 @@ function BuildImageController($scope, $async, $window, ModalService, BuildServic return BuildService.buildImageFromURL(names, URL, dockerfilePath); } else { var dockerfileContent = $scope.formValues.DockerFileContent; - return BuildService.buildImageFromDockerfileContent(names, dockerfileContent); + if ($scope.formValues.AdditionalFiles.length === 0) { + return BuildService.buildImageFromDockerfileContent(names, dockerfileContent); + } else { + var additionalFiles = $scope.formValues.AdditionalFiles; + return BuildService.buildImageFromDockerfileContentAndFiles(names, dockerfileContent, additionalFiles); + } } } @@ -140,4 +146,8 @@ function BuildImageController($scope, $async, $window, ModalService, BuildServic return ModalService.confirmWebEditorDiscard(); } }; + + $scope.selectAdditionalFiles = function (files) { + $scope.formValues.AdditionalFiles = files; + }; } diff --git a/app/docker/views/images/build/buildimage.html b/app/docker/views/images/build/buildimage.html index 463bbed45..7ef27b56b 100644 --- a/app/docker/views/images/build/buildimage.html +++ b/app/docker/views/images/build/buildimage.html @@ -132,6 +132,20 @@ > +
Upload
+
+
+
+ + You can upload files from your local computer for referencing in your Dockerfile (using ADD filename) so they are included in your built image. + +
+ + + {{ item.name }} + +
+
diff --git a/app/portainer/services/fileUpload.js b/app/portainer/services/fileUpload.js index 128e0804a..0eba97333 100644 --- a/app/portainer/services/fileUpload.js +++ b/app/portainer/services/fileUpload.js @@ -33,6 +33,23 @@ angular.module('portainer.app').factory('FileUploadService', [ }); }; + service.buildImageFromFiles = function (names, files) { + var endpointID = EndpointProvider.endpointID(); + return Upload.upload({ + url: 'api/endpoints/' + endpointID + '/docker/build', + headers: { + 'Content-Type': 'multipart/form-data', + }, + data: { file: files }, + params: { + t: names, + }, + transformResponse: function (data) { + return jsonObjectsToArrayHandler(data); + }, + }); + }; + service.loadImages = function (file) { var endpointID = EndpointProvider.endpointID(); return Upload.http({