From e8a8b71daa36ad1b17735a2a6d1de9bde5eaeec0 Mon Sep 17 00:00:00 2001 From: Matt Hook <hookenz@gmail.com> Date: Fri, 3 Jun 2022 13:50:37 +1200 Subject: [PATCH] feat(compose): upgrade to docker compose v2 EE-2096 (#6994) Upgrade to compose v2 + new helm + new kubectl --- api/exec/compose_stack_integration_test.go | 3 + api/go.mod | 2 +- api/go.sum | 6 +- build/download_docker_binary.sh | 35 +++++++---- build/download_docker_compose_binary.sh | 71 +++++++--------------- build/download_helm_binary.sh | 17 +++--- build/download_kompose_binary.sh | 11 +++- build/download_kubectl_binary.sh | 11 +++- gruntfile.js | 51 +++++----------- 9 files changed, 93 insertions(+), 114 deletions(-) diff --git a/api/exec/compose_stack_integration_test.go b/api/exec/compose_stack_integration_test.go index 5557b5173..68ff18da8 100644 --- a/api/exec/compose_stack_integration_test.go +++ b/api/exec/compose_stack_integration_test.go @@ -11,6 +11,7 @@ import ( "testing" portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/internal/testhelpers" ) const composeFile = `version: "3.9" @@ -41,6 +42,8 @@ func setup(t *testing.T) (*portainer.Stack, *portainer.Endpoint) { func Test_UpAndDown(t *testing.T) { + testhelpers.IntegrationTest(t) + stack, endpoint := setup(t) w, err := NewComposeStackManager("", "", nil) diff --git a/api/go.mod b/api/go.mod index ab2e7168b..0757f5ed6 100644 --- a/api/go.mod +++ b/api/go.mod @@ -32,7 +32,7 @@ require ( github.com/koding/websocketproxy v0.0.0-20181220232114-7ed82d81a28c github.com/orcaman/concurrent-map v0.0.0-20190826125027-8c72a8bb44f6 github.com/pkg/errors v0.9.1 - github.com/portainer/docker-compose-wrapper v0.0.0-20220407011010-3c7408969ad3 + github.com/portainer/docker-compose-wrapper v0.0.0-20220531190153-c597b853e410 github.com/portainer/libcrypto v0.0.0-20210422035235-c652195c5c3a github.com/portainer/libhelm v0.0.0-20210929000907-825e93d62108 github.com/portainer/libhttp v0.0.0-20211208103139-07a5f798eb3f diff --git a/api/go.sum b/api/go.sum index ce36b6bbc..da4624f2c 100644 --- a/api/go.sum +++ b/api/go.sum @@ -807,8 +807,10 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/portainer/docker-compose-wrapper v0.0.0-20220407011010-3c7408969ad3 h1:dg/uvltrR++AVDjjVkXKrinZ/T8YlaKeUAOAmQ1i+dk= -github.com/portainer/docker-compose-wrapper v0.0.0-20220407011010-3c7408969ad3/go.mod h1:WxDlJWZxCnicdLCPnLNEv7/gRhjeIVuCGmsv+iOPH3c= +github.com/portainer/docker-compose-wrapper v0.0.0-20220526210722-e1574867298e h1:gW1Ooaj7RZ9YkwHxesnNEyOB5nUD71FlZ7cdb5h63vw= +github.com/portainer/docker-compose-wrapper v0.0.0-20220526210722-e1574867298e/go.mod h1:WxDlJWZxCnicdLCPnLNEv7/gRhjeIVuCGmsv+iOPH3c= +github.com/portainer/docker-compose-wrapper v0.0.0-20220531190153-c597b853e410 h1:LjxLd8UGR8ae73ov/vLrt/0jedj/nh98XnONkr8DJj8= +github.com/portainer/docker-compose-wrapper v0.0.0-20220531190153-c597b853e410/go.mod h1:WxDlJWZxCnicdLCPnLNEv7/gRhjeIVuCGmsv+iOPH3c= github.com/portainer/libcrypto v0.0.0-20210422035235-c652195c5c3a h1:qY8TbocN75n5PDl16o0uVr5MevtM5IhdwSelXEd4nFM= github.com/portainer/libcrypto v0.0.0-20210422035235-c652195c5c3a/go.mod h1:n54EEIq+MM0NNtqLeCby8ljL+l275VpolXO0ibHegLE= github.com/portainer/libhelm v0.0.0-20210929000907-825e93d62108 h1:5e8KAnDa2G3cEHK7aV/ue8lOaoQwBZUzoALslwWkR04= diff --git a/build/download_docker_binary.sh b/build/download_docker_binary.sh index ec2d2098b..4c90c185c 100755 --- a/build/download_docker_binary.sh +++ b/build/download_docker_binary.sh @@ -1,23 +1,34 @@ #!/usr/bin/env bash +set -euo pipefail + +if [[ $# -ne 3 ]]; then + echo "Illegal number of parameters" >&2 + exit 1 +fi PLATFORM=$1 ARCH=$2 -DOCKER_VERSION=$3 - +DOCKER_VERSION=${3:1} DOWNLOAD_FOLDER=".tmp/download" + +if [[ ${ARCH} == "amd64" ]]; then + ARCH="x86_64" +elif [[ ${ARCH} == "arm" ]]; then + ARCH="armhf" +elif [[ ${ARCH} == "arm64" ]]; then + ARCH="aarch64" +fi + rm -rf "${DOWNLOAD_FOLDER}" mkdir -pv "${DOWNLOAD_FOLDER}" -if [ "${PLATFORM}" == 'win' ]; then - wget -O "${DOWNLOAD_FOLDER}/docker-binaries.zip" "https://dockermsft.azureedge.net/dockercontainer/docker-${DOCKER_VERSION}.zip" - unzip "${DOWNLOAD_FOLDER}/docker-binaries.zip" -d "${DOWNLOAD_FOLDER}" - mv "${DOWNLOAD_FOLDER}/docker/docker.exe" dist/ - mv ${DOWNLOAD_FOLDER}/docker/*.dll dist/ +if [[ ${PLATFORM} == "windows" ]]; then + wget -O "${DOWNLOAD_FOLDER}/docker-binaries.zip" "https://download.docker.com/win/static/stable/${ARCH}/docker-${DOCKER_VERSION}.zip" + unzip "${DOWNLOAD_FOLDER}/docker-binaries.zip" -d "${DOWNLOAD_FOLDER}" + mv "${DOWNLOAD_FOLDER}/docker/docker.exe" dist/ else - wget -O "${DOWNLOAD_FOLDER}/docker-binaries.tgz" "https://download.docker.com/${PLATFORM}/static/stable/${ARCH}/docker-${DOCKER_VERSION}.tgz" - tar -xf "${DOWNLOAD_FOLDER}/docker-binaries.tgz" -C "${DOWNLOAD_FOLDER}" - mv "${DOWNLOAD_FOLDER}/docker/docker" dist/ + wget -O "${DOWNLOAD_FOLDER}/docker-binaries.tgz" "https://download.docker.com/${PLATFORM}/static/stable/${ARCH}/docker-${DOCKER_VERSION}.tgz" + tar -xf "${DOWNLOAD_FOLDER}/docker-binaries.tgz" -C "${DOWNLOAD_FOLDER}" + mv "${DOWNLOAD_FOLDER}/docker/docker" dist/ fi - -exit 0 diff --git a/build/download_docker_compose_binary.sh b/build/download_docker_compose_binary.sh index 63ea5b005..f12dd729e 100755 --- a/build/download_docker_compose_binary.sh +++ b/build/download_docker_compose_binary.sh @@ -1,57 +1,30 @@ -#!/bin/bash +#!/usr/bin/env bash set -euo pipefail -IFS=$'\n\t' + +if [[ $# -ne 3 ]]; then + echo "Illegal number of parameters" >&2 + exit 1 +fi PLATFORM=$1 ARCH=$2 -DOCKER_COMPOSE_VERSION=$3 +COMPOSE_VERSION=$3 -function download_binary() { - local PLATFORM=$1 - local ARCH=$2 - local BINARY_VERSION=$3 - - if [ "${PLATFORM}" == 'linux' ] && [ "${ARCH}" == 'amd64' ]; then - wget -O "dist/docker-compose" "https://github.com/portainer/docker-compose-linux-amd64-static-binary/releases/download/${BINARY_VERSION}/docker-compose" - chmod +x "dist/docker-compose" - return - fi - - if [ "${PLATFORM}" == 'mac' ]; then - wget -O "dist/docker-compose" "https://github.com/docker/compose/releases/download/${BINARY_VERSION}/docker-compose-Darwin-x86_64" - chmod +x "dist/docker-compose" - return - fi - - if [ "${PLATFORM}" == 'win' ]; then - wget -O "dist/docker-compose.exe" "https://github.com/docker/compose/releases/download/${BINARY_VERSION}/docker-compose-Windows-x86_64.exe" - chmod +x "dist/docker-compose.exe" - return - fi -} -function download_plugin() { - local PLATFORM=$1 - local ARCH=$2 - local PLUGIN_VERSION=$3 - - if [ "${PLATFORM}" == 'mac' ]; then - PLATFORM="darwin" - fi - - FILENAME="docker-compose-${PLATFORM}-${ARCH}" - TARGET_FILENAME="docker-compose.plugin" - if [[ "$PLATFORM" == "windows" ]]; then - FILENAME="$FILENAME.exe" - TARGET_FILENAME="$TARGET_FILENAME.exe" - fi - - wget -O "dist/$TARGET_FILENAME" "https://github.com/docker/compose-cli/releases/download/v$PLUGIN_VERSION/$FILENAME" - chmod +x "dist/$TARGET_FILENAME" -} - -if [ "${PLATFORM}" == 'linux' ] && [ "${ARCH}" != 'amd64' ]; then - download_plugin "$PLATFORM" "$ARCH" "$DOCKER_COMPOSE_VERSION" +if [[ ${ARCH} == "amd64" ]]; then + ARCH="x86_64" +elif [[ ${ARCH} == "arm" ]]; then + ARCH="armv7" +elif [[ ${ARCH} == "arm64" ]]; then + ARCH="aarch64" +fi + + +if [[ "$PLATFORM" == "windows" ]]; then + wget -O "dist/docker-compose.plugin.exe" "https://github.com/docker/compose/releases/download/$COMPOSE_VERSION/docker-compose-windows-${ARCH}.exe" + chmod +x "dist/docker-compose.plugin.exe" +else + wget -O "dist/docker-compose.plugin" "https://github.com/docker/compose/releases/download/$COMPOSE_VERSION/docker-compose-${PLATFORM}-${ARCH}" + chmod +x "dist/docker-compose.plugin" fi -download_binary "$PLATFORM" "$ARCH" "$DOCKER_COMPOSE_VERSION" \ No newline at end of file diff --git a/build/download_helm_binary.sh b/build/download_helm_binary.sh index 4aa1f896b..1b21bc5d1 100755 --- a/build/download_helm_binary.sh +++ b/build/download_helm_binary.sh @@ -1,21 +1,24 @@ #!/usr/bin/env bash +set -euo pipefail + +if [[ $# -ne 3 ]]; then + echo "Illegal number of parameters" >&2 + exit 1 +fi PLATFORM=$1 ARCH=$2 HELM_VERSION=$3 - HELM_DIST="helm-$HELM_VERSION-$PLATFORM-$ARCH" -if [ "${PLATFORM}" == 'linux' ]; then +if [[ ${PLATFORM} == "linux" ]]; then wget -qO- "https://get.helm.sh/${HELM_DIST}.tar.gz" | tar -x -z --strip-components 1 "${PLATFORM}-${ARCH}/helm" mv "helm" "dist/helm" chmod +x "dist/helm" -elif [ "${PLATFORM}" == 'darwin' ]; then +elif [[ ${PLATFORM} == "darwin" ]]; then wget -qO- "https://get.helm.sh/helm-canary-darwin-amd64.tar.gz" | tar -x -z --strip-components 1 "darwin-amd64/helm" mv "helm" "dist/helm" chmod +x "dist/helm" -elif [ "${PLATFORM}" == 'windows' ]; then - wget -q -O tmp.zip "https://get.helm.sh/${HELM_DIST}.zip" && unzip -o -j tmp.zip "${PLATFORM}-${ARCH}/helm.exe" -d dist && rm -f tmp.zip +elif [[ ${PLATFORM} == "windows" ]]; then + wget -O tmp.zip "https://get.helm.sh/${HELM_DIST}.zip" && unzip -o -j tmp.zip "${PLATFORM}-${ARCH}/helm.exe" -d dist && rm -f tmp.zip fi - -exit 0 diff --git a/build/download_kompose_binary.sh b/build/download_kompose_binary.sh index 8b7ec5f2d..84bb74faa 100755 --- a/build/download_kompose_binary.sh +++ b/build/download_kompose_binary.sh @@ -1,15 +1,20 @@ #!/usr/bin/env bash +set -euo pipefail + +if [[ $# -ne 3 ]]; then + echo "Illegal number of parameters" >&2 + exit 1 +fi PLATFORM=$1 ARCH=$2 KOMPOSE_VERSION=$3 -if [ "${PLATFORM}" == 'windows' ]; then + +if [[ ${PLATFORM} == "windows" ]]; then wget -O "dist/kompose.exe" "https://github.com/kubernetes/kompose/releases/download/${KOMPOSE_VERSION}/kompose-windows-amd64.exe" chmod +x "dist/kompose.exe" else wget -O "dist/kompose" "https://github.com/kubernetes/kompose/releases/download/${KOMPOSE_VERSION}/kompose-${PLATFORM}-${ARCH}" chmod +x "dist/kompose" fi - -exit 0 diff --git a/build/download_kubectl_binary.sh b/build/download_kubectl_binary.sh index 3e4069613..8b492a781 100755 --- a/build/download_kubectl_binary.sh +++ b/build/download_kubectl_binary.sh @@ -1,15 +1,20 @@ #!/usr/bin/env bash +set -euo pipefail + +if [[ $# -ne 3 ]]; then + echo "Illegal number of parameters" >&2 + exit 1 +fi PLATFORM=$1 ARCH=$2 KUBECTL_VERSION=$3 -if [ "${PLATFORM}" == 'windows' ]; then + +if [[ ${PLATFORM} == "windows" ]]; then wget -O "dist/kubectl.exe" "https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/windows/amd64/kubectl.exe" chmod +x "dist/kubectl.exe" else wget -O "dist/kubectl" "https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/${PLATFORM}/${ARCH}/kubectl" chmod +x "dist/kubectl" fi - -exit 0 diff --git a/gruntfile.js b/gruntfile.js index c32ec2baf..fce287064 100644 --- a/gruntfile.js +++ b/gruntfile.js @@ -18,14 +18,11 @@ module.exports = function (grunt) { root: 'dist', distdir: 'dist/public', binaries: { - dockerLinuxVersion: '19.03.13', - dockerWindowsVersion: '19-03-12', - dockerLinuxComposeVersion: '1.27.4', - dockerWindowsComposeVersion: '1.28.0', - dockerComposePluginVersion: '2.0.0-rc.2', - helmVersion: 'v3.8.0', + dockerVersion: 'v20.10.9', + dockerComposePluginVersion: 'v2.5.1', + helmVersion: 'v3.9.0', komposeVersion: 'v1.22.0', - kubectlVersion: 'v1.18.0', + kubectlVersion: 'v1.24.1', }, env: gruntConfig.env, clean: gruntConfig.clean, @@ -109,9 +106,9 @@ gruntConfig.shell = { build_binary: { command: shell_build_binary }, build_binary_azuredevops: { command: shell_build_binary_azuredevops }, download_docker_binary: { command: shell_download_docker_binary }, + download_helm_binary: { command: shell_download_helm_binary }, download_kompose_binary: { command: shell_download_kompose_binary }, download_kubectl_binary: { command: shell_download_kubectl_binary }, - download_helm_binary: { command: shell_download_helm_binary }, download_docker_compose_binary: { command: shell_download_docker_compose_binary }, run_container: { command: shell_run_container }, run_localserver: { command: shell_run_localserver, options: { async: true } }, @@ -186,47 +183,27 @@ function shell_install_yarndeps() { } function shell_download_docker_binary(platform, arch) { - const ps = { windows: 'win', darwin: 'mac' }; - const as = { amd64: 'x86_64', arm: 'armhf', arm64: 'aarch64' }; - - const ip = ps[platform] === undefined ? platform : ps[platform]; - const ia = as[arch] === undefined ? arch : as[arch]; - const binaryVersion = platform === 'windows' ? '<%= binaries.dockerWindowsVersion %>' : '<%= binaries.dockerLinuxVersion %>'; + const binaryVersion = '<%= binaries.dockerVersion %>'; return ` if [ -f dist/docker ] || [ -f dist/docker.exe ]; then echo "docker binary exists"; else - build/download_docker_binary.sh ${ip} ${ia} ${binaryVersion}; + build/download_docker_binary.sh ${platform} ${arch} ${binaryVersion}; fi `; } -function shell_download_docker_compose_binary(p, a) { - var ps = { windows: 'win', darwin: 'mac' }; - var as = { arm: 'armhf', arm64: 'aarch64' }; - var ip = ps[p] || p; - var ia = as[a] || a; - var binaryVersion = p === 'windows' ? '<%= binaries.dockerWindowsComposeVersion %>' : '<%= binaries.dockerLinuxComposeVersion %>'; - - // plugin - if (p === 'linux' && a !== 'amd64') { - if (a === 'arm64') { - ia = 'arm64'; - } - - if (a === 'arm') { - ia = 'armv7'; - } - binaryVersion = '<%= binaries.dockerComposePluginVersion %>'; - } +function shell_download_docker_compose_binary(platform, arch) { + var binaryVersion = '<%= binaries.dockerComposePluginVersion %>'; return ` - if [ -f dist/docker-compose ] || [ -f dist/docker-compose.exe ] || [ -f dist/docker-compose.plugin ] || [ -f dist/docker-compose.plugin.exe ]; then - echo "Docker Compose binary exists"; + if [ -f dist/docker-compose.plugin ] || [ -f dist/docker-compose.plugin.exe ]; then + echo "docker compose binary binary exists"; else - build/download_docker_compose_binary.sh ${ip} ${ia} ${binaryVersion}; - fi`; + build/download_docker_compose_binary.sh ${platform} ${arch} ${binaryVersion}; + fi + `; } function shell_download_helm_binary(platform, arch) {