feat(image): upload local files for building image EE-3021 (#7507)

* support to make multiple files in archive buffer

* upload files by multipart

api/archive/tar.go

@@ -34,3 +34,45 @@ func TarFileInBuffer(fileContent []byte, fileName string, mode int64) ([]byte, e
 
 	return buffer.Bytes(), nil
 }
+
+// tarFileInBuffer represents a tar archive buffer.
+type tarFileInBuffer struct {
+	b *bytes.Buffer
+	w *tar.Writer
+}
+
+func NewTarFileInBuffer() *tarFileInBuffer {
+	var b bytes.Buffer
+	return &tarFileInBuffer{
+		b: &b,
+		w: tar.NewWriter(&b),
+	}
+}
+
+// Put puts a single file to tar archive buffer.
+func (t *tarFileInBuffer) Put(fileContent []byte, fileName string, mode int64) error {
+	hdr := &tar.Header{
+		Name: fileName,
+		Mode: mode,
+		Size: int64(len(fileContent)),
+	}
+
+	if err := t.w.WriteHeader(hdr); err != nil {
+		return err
+	}
+
+	if _, err := t.w.Write(fileContent); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Bytes returns the archive as a byte array.
+func (t *tarFileInBuffer) Bytes() []byte {
+	return t.b.Bytes()
+}
+
+func (t *tarFileInBuffer) Close() error {
+	return t.w.Close()
+}

api/http/proxy/factory/docker/build.go

@@ -3,13 +3,17 @@ package docker
 import (
 	"bytes"
 	"encoding/json"
+	"errors"
 	"io/ioutil"
+	"log"
+	"mime"
 	"net/http"
-	"strings"
 
 	"github.com/portainer/portainer/api/archive"
 )
 
+const OneMegabyte = 1024768
+
 type postDockerfileRequest struct {
 	Content string
 }
@@ -23,29 +27,81 @@ type postDockerfileRequest struct {
 // In any other case, it will leave the request unaltered.
 func buildOperation(request *http.Request) error {
 	contentTypeHeader := request.Header.Get("Content-Type")
-	if contentTypeHeader != "" && !strings.Contains(contentTypeHeader, "application/json") {
-		return nil
+
+	mediaType, _, err := mime.ParseMediaType(contentTypeHeader)
+	if err != nil {
+		return err
 	}
 
-	var dockerfileContent []byte
-
-	if contentTypeHeader == "" {
+	var buffer []byte
+	switch mediaType {
+	case "":
 		body, err := ioutil.ReadAll(request.Body)
 		if err != nil {
 			return err
 		}
-		dockerfileContent = body
-	} else {
+
+		buffer, err = archive.TarFileInBuffer(body, "Dockerfile", 0600)
+		if err != nil {
+			return err
+		}
+
+	case "application/json":
 		var req postDockerfileRequest
 		if err := json.NewDecoder(request.Body).Decode(&req); err != nil {
 			return err
 		}
-		dockerfileContent = []byte(req.Content)
-	}
 
-	buffer, err := archive.TarFileInBuffer(dockerfileContent, "Dockerfile", 0600)
-	if err != nil {
-		return err
+		buffer, err = archive.TarFileInBuffer([]byte(req.Content), "Dockerfile", 0600)
+		if err != nil {
+			return err
+		}
+
+	case "multipart/form-data":
+		err := request.ParseMultipartForm(32 * OneMegabyte) // limit parser memory to 32MB
+		if err != nil {
+			return err
+		}
+
+		if request.MultipartForm == nil || request.MultipartForm.File == nil {
+			return errors.New("uploaded files not found to build image")
+		}
+
+		tfb := archive.NewTarFileInBuffer()
+		defer tfb.Close()
+
+		for k := range request.MultipartForm.File {
+			f, hdr, err := request.FormFile(k)
+			if err != nil {
+				return err
+			}
+
+			defer f.Close()
+
+			log.Printf("[INFO] [http,proxy,docker] [message: upload the file to build image] [filename: %s] [size: %d]", hdr.Filename, hdr.Size)
+
+			content, err := ioutil.ReadAll(f)
+			if err != nil {
+				return err
+			}
+
+			filename := hdr.Filename
+			if hdr.Filename == "blob" {
+				filename = "Dockerfile"
+			}
+
+			if err := tfb.Put(content, filename, 0600); err != nil {
+				return err
+			}
+		}
+
+		buffer = tfb.Bytes()
+		request.Form = nil
+		request.PostForm = nil
+		request.MultipartForm = nil
+
+	default:
+		return nil
 	}
 
 	request.Body = ioutil.NopCloser(bytes.NewReader(buffer))

app/docker/services/buildService.js

@@ -66,6 +66,24 @@ angular.module('portainer.docker').factory('BuildService', [
       return deferred.promise;
     };
 
+    service.buildImageFromDockerfileContentAndFiles = function (names, content, files) {
+      var dockerfile = new Blob([content], { type: 'text/plain' });
+      var uploadFiles = [dockerfile].concat(files);
+
+      var deferred = $q.defer();
+
+      FileUploadService.buildImageFromFiles(names, uploadFiles)
+        .then(function success(response) {
+          var model = new ImageBuildModel(response.data);
+          deferred.resolve(model);
+        })
+        .catch(function error(err) {
+          deferred.reject(err);
+        });
+
+      return deferred.promise;
+    };
+
     return service;
   },
 ]);

app/docker/views/images/build/buildImageController.js

@@ -13,6 +13,7 @@ function BuildImageController($scope, $async, $window, ModalService, BuildServic
     ImageNames: [{ Name: '', Valid: false, Unique: true }],
     UploadFile: null,
     DockerFileContent: '',
+    AdditionalFiles: [],
     URL: '',
     Path: 'Dockerfile',
     NodeName: null,
@@ -74,7 +75,12 @@ function BuildImageController($scope, $async, $window, ModalService, BuildServic
       return BuildService.buildImageFromURL(names, URL, dockerfilePath);
     } else {
       var dockerfileContent = $scope.formValues.DockerFileContent;
-      return BuildService.buildImageFromDockerfileContent(names, dockerfileContent);
+      if ($scope.formValues.AdditionalFiles.length === 0) {
+        return BuildService.buildImageFromDockerfileContent(names, dockerfileContent);
+      } else {
+        var additionalFiles = $scope.formValues.AdditionalFiles;
+        return BuildService.buildImageFromDockerfileContentAndFiles(names, dockerfileContent, additionalFiles);
+      }
     }
   }
 
@@ -140,4 +146,8 @@ function BuildImageController($scope, $async, $window, ModalService, BuildServic
       return ModalService.confirmWebEditorDiscard();
     }
   };
+
+  $scope.selectAdditionalFiles = function (files) {
+    $scope.formValues.AdditionalFiles = files;
+  };
 }

app/docker/views/images/build/buildimage.html

@@ -132,6 +132,20 @@
                     ></code-editor>
                   </div>
                 </div>
+                <div class="col-sm-12 form-section-title"> Upload </div>
+                <div class="form-group">
+                  <div class="col-sm-12">
+                    <div class="form-group">
+                      <span class="col-sm-12 text-muted small">
+                        You can upload files from your local computer for referencing in your Dockerfile (using ADD filename) so they are included in your built image.
+                      </span>
+                    </div>
+                    <button class="btn btn-sm btn-primary" ngf-select="selectAdditionalFiles($files)" ngf-multiple="true">Select files</button>
+                    <span ng-repeat="item in formValues.AdditionalFiles track by $index" class="mx-2">
+                      {{ item.name }}
+                    </span>
+                  </div>
+                </div>
               </div>
               <!-- !web-editor -->
               <!-- upload -->

app/portainer/services/fileUpload.js

@@ -33,6 +33,23 @@ angular.module('portainer.app').factory('FileUploadService', [
       });
     };
 
+    service.buildImageFromFiles = function (names, files) {
+      var endpointID = EndpointProvider.endpointID();
+      return Upload.upload({
+        url: 'api/endpoints/' + endpointID + '/docker/build',
+        headers: {
+          'Content-Type': 'multipart/form-data',
+        },
+        data: { file: files },
+        params: {
+          t: names,
+        },
+        transformResponse: function (data) {
+          return jsonObjectsToArrayHandler(data);
+        },
+      });
+    };
+
     service.loadImages = function (file) {
       var endpointID = EndpointProvider.endpointID();
       return Upload.http({