From 08fdebfbd9333fcc8ba4c964eff6f10848dbaf68 Mon Sep 17 00:00:00 2001 From: Steven Kang Date: Tue, 24 Oct 2023 19:30:33 +0900 Subject: [PATCH] feat(ci): introduce GH Actions for Portainer CE (#10419) Co-authored-by: Chaim Lev-Ari --- .github/workflows/ci.yaml | 154 ++++++++++++++++++++++++ binary-version.json | 3 +- build/build_binary.sh | 12 +- build/download_binaries.sh | 4 +- build/download_docker_binary.sh | 4 +- build/download_docker_compose_binary.sh | 4 +- build/download_helm_binary.sh | 2 +- build/download_kubectl_binary.sh | 4 +- build/download_mingit_binary.sh | 18 +++ build/windows/Dockerfile | 11 +- pkg/libstack/compose/download.sh | 4 +- 11 files changed, 194 insertions(+), 26 deletions(-) create mode 100644 .github/workflows/ci.yaml create mode 100755 build/download_mingit_binary.sh diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 000000000..fe6ce2ca2 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,154 @@ +name: ci + +on: + workflow_dispatch: + push: + branches: + - "develop" + - "!release/*" + pull_request: + branches: + - "develop" + - "release/*" + - "feat/*" + - "fix/*" + - "refactor/*" + +env: + DOCKER_HUB_REPO: portainerci/portainer + NODE_ENV: testing + GO_VERSION: 1.21.0 + NODE_VERSION: 18.x + +jobs: + build_images: + strategy: + matrix: + config: + - { platform: linux, arch: amd64 } + - { platform: linux, arch: arm64 } + - { platform: windows, arch: amd64, version: 1809 } + - { platform: windows, arch: amd64, version: ltsc2022 } + runs-on: arc-runner-set + steps: + - name: "[preparation] checkout the current branch" + uses: actions/checkout@v3.5.3 + with: + ref: ${{ github.event.inputs.branch }} + - name: "[preparation] set up golang" + uses: actions/setup-go@v4.0.1 + with: + go-version: ${{ env.GO_VERSION }} + cache: false + - name: "[preparation] cache paths" + id: cache-dir-path + run: | + echo "yarn-cache-dir=$(yarn cache dir)" >> "$GITHUB_OUTPUT" + echo "go-build-dir=$(go env GOCACHE)" >> "$GITHUB_OUTPUT" + echo "go-mod-dir=$(go env GOMODCACHE)" >> "$GITHUB_OUTPUT" + - name: "[preparation] cache go" + uses: actions/cache@v3 + with: + path: | + ${{ steps.cache-dir-path.outputs.go-build-dir }} + ${{ steps.cache-dir-path.outputs.go-mod-dir }} + key: ${{ matrix.config.platform }}-${{ matrix.config.arch }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ matrix.config.platform }}-${{ matrix.config.arch }}-go- + enableCrossOsArchive: true + - name: "[preparation] set up node.js" + uses: actions/setup-node@v3 + with: + node-version: ${{ env.NODE_VERSION }} + cache: '' + - name: "[preparation] cache yarn" + uses: actions/cache@v3 + with: + path: | + **/node_modules + ${{ steps.cache-dir-path.outputs.yarn-cache-dir }} + key: ${{ matrix.config.platform }}-${{ matrix.config.arch }}-yarn-${{ hashFiles('**/yarn.lock') }} + restore-keys: | + ${{ matrix.config.platform }}-${{ matrix.config.arch }}-yarn- + enableCrossOsArchive: true + - name: "[preparation] execute client test" + run: | + make client-deps && make test-client + - name: "[preparation] execute server test" + run: | + make test-server + - name: "[preparation] set up qemu" + uses: docker/setup-qemu-action@v2 + - name: "[preparation] set up docker context for buildx" + run: docker context create builders + - name: "[preparation] set up docker buildx" + uses: docker/setup-buildx-action@v2 + with: + endpoint: builders + - name: "[preparation] docker login" + uses: docker/login-action@v2.2.0 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_PASSWORD }} + - name: "[preparation] set the container image tag" + run: | + if [ "${GITHUB_EVENT_NAME}" == "pull_request" ]; then + CONTAINER_IMAGE_TAG="pr${{ github.event.number }}" + else + CONTAINER_IMAGE_TAG=$(echo $GITHUB_REF_NAME | sed 's/\//-/g') + fi + + if [ "${{ matrix.config.platform }}" == "windows" ]; then + CONTAINER_IMAGE_TAG="${CONTAINER_IMAGE_TAG}-${{ matrix.config.platform }}${{ matrix.config.version }}-${{ matrix.config.arch }}" + else + CONTAINER_IMAGE_TAG="${CONTAINER_IMAGE_TAG}-${{ matrix.config.platform }}-${{ matrix.config.arch }}" + fi + + echo "CONTAINER_IMAGE_TAG=${CONTAINER_IMAGE_TAG}" >> $GITHUB_ENV + - name: "[execution] build linux & windows portainer binaries" + run: | + export YARN_VERSION=$(yarn --version) + export WEBPACK_VERSION=$(yarn list webpack --depth=0 | grep webpack | awk -F@ '{print $2}') + export BUILDNUMBER=${GITHUB_RUN_NUMBER} + make build-all PLATFORM=${{ matrix.config.platform }} ARCH=${{ matrix.config.arch }} ENV=${NODE_ENV} + env: + CONTAINER_IMAGE_TAG: ${{ env.CONTAINER_IMAGE_TAG }} + - name: "[execution] build and push docker images" + run: | + if [ "${{ matrix.config.platform }}" == "windows" ]; then + mv dist/portainer dist/portainer.exe + docker buildx build --output=type=registry --platform ${{ matrix.config.platform }}/${{ matrix.config.arch }} --build-arg OSVERSION=${{ matrix.config.version }} -t "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}" -f build/${{ matrix.config.platform }}/Dockerfile . + else + docker buildx build --output=type=registry --platform ${{ matrix.config.platform }}/${{ matrix.config.arch }} -t "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}" -f build/${{ matrix.config.platform }}/Dockerfile . + docker buildx build --output=type=registry --platform ${{ matrix.config.platform }}/${{ matrix.config.arch }} -t "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-alpine" -f build/${{ matrix.config.platform }}/alpine.Dockerfile . + fi + env: + CONTAINER_IMAGE_TAG: ${{ env.CONTAINER_IMAGE_TAG }} + build_manifests: + runs-on: arc-runner-set + needs: [build_images] + steps: + - name: "[preparation] docker login" + uses: docker/login-action@v2.2.0 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_PASSWORD }} + - name: "[preparation] set up docker context for buildx" + run: docker version && docker context create builders + - name: "[preparation] set up docker buildx" + uses: docker/setup-buildx-action@v2 + with: + endpoint: builders + - name: "[execution] build and push manifests" + run: | + if [ "${GITHUB_EVENT_NAME}" == "pull_request" ]; then + CONTAINER_IMAGE_TAG="pr${{ github.event.number }}" + else + CONTAINER_IMAGE_TAG=$(echo $GITHUB_REF_NAME | sed 's/\//-/g') + fi + + docker buildx imagetools create -t "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}" \ + "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-amd64" \ + "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-arm64" \ + "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-windows1809-amd64" \ + "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-windowsltsc2022-amd64" \ No newline at end of file diff --git a/binary-version.json b/binary-version.json index 622b543ef..e33762c6c 100644 --- a/binary-version.json +++ b/binary-version.json @@ -2,5 +2,6 @@ "docker": "v20.10.21", "dockerCompose": "v2.20.3", "helm": "v3.12.3", - "kubectl": "v1.28.1" + "kubectl": "v1.28.1", + "mingit": "2.42.0.2" } diff --git a/build/build_binary.sh b/build/build_binary.sh index dc17396f9..393c61c74 100755 --- a/build/build_binary.sh +++ b/build/build_binary.sh @@ -4,12 +4,12 @@ set -euo pipefail mkdir -p dist # populate tool versions -BUILDNUMBER="N/A" -CONTAINER_IMAGE_TAG="N/A" -NODE_VERSION="0" -YARN_VERSION="0" -WEBPACK_VERSION="0" -GO_VERSION="0" +BUILDNUMBER=${BUILDNUMBER:-"N/A"} +CONTAINER_IMAGE_TAG=${CONTAINER_IMAGE_TAG:-"N/A"} +NODE_VERSION=${NODE_VERSION:-"N/A"} +YARN_VERSION=${YARN_VERSION:-"N/A"} +WEBPACK_VERSION=${WEBPACK_VERSION:-"N/A"} +GO_VERSION=${GO_VERSION:-"N/A"} # copy templates cp -r "./mustache-templates" "./dist" diff --git a/build/download_binaries.sh b/build/download_binaries.sh index ed57c70be..f54340f46 100755 --- a/build/download_binaries.sh +++ b/build/download_binaries.sh @@ -11,13 +11,15 @@ dockerVersion=$(jq -r '.docker' < "${BINARY_VERSION_FILE}") dockerComposeVersion=$(jq -r '.dockerCompose' < "${BINARY_VERSION_FILE}") helmVersion=$(jq -r '.helm' < "${BINARY_VERSION_FILE}") kubectlVersion=$(jq -r '.kubectl' < "${BINARY_VERSION_FILE}") +mingitVersion=$(jq -r '.mingit' < "${BINARY_VERSION_FILE}") mkdir -p dist -echo "Downloading binaries for docker ${dockerVersion}, docker-compose ${dockerComposeVersion}, helm ${helmVersion}, kubectl ${kubectlVersion}" +echo "Downloading binaries for docker ${dockerVersion}, docker-compose ${dockerComposeVersion}, helm ${helmVersion}, kubectl ${kubectlVersion}, and mingit ${mingitVersion}" ./build/download_docker_binary.sh "$PLATFORM" "$ARCH" "$dockerVersion" & ./build/download_docker_compose_binary.sh "$PLATFORM" "$ARCH" "$dockerComposeVersion" & ./build/download_helm_binary.sh "$PLATFORM" "$ARCH" "$helmVersion" & ./build/download_kubectl_binary.sh "$PLATFORM" "$ARCH" "$kubectlVersion" & +./build/download_mingit_binary.sh "$PLATFORM" "$ARCH" "$mingitVersion" & wait diff --git a/build/download_docker_binary.sh b/build/download_docker_binary.sh index 368417fae..eed8fcb31 100755 --- a/build/download_docker_binary.sh +++ b/build/download_docker_binary.sh @@ -31,11 +31,11 @@ rm -rf "${DOWNLOAD_FOLDER}" mkdir -pv "${DOWNLOAD_FOLDER}" if [[ ${PLATFORM} == "windows" ]]; then - wget -O "${DOWNLOAD_FOLDER}/docker-binaries.zip" "https://download.docker.com/win/static/stable/${ARCH}/docker-${DOCKER_VERSION}.zip" + wget --tries=3 --waitretry=30 --quiet -O "${DOWNLOAD_FOLDER}/docker-binaries.zip" "https://download.docker.com/win/static/stable/${ARCH}/docker-${DOCKER_VERSION}.zip" unzip "${DOWNLOAD_FOLDER}/docker-binaries.zip" -d "${DOWNLOAD_FOLDER}" mv "${DOWNLOAD_FOLDER}/docker/docker.exe" dist/ else - wget -O "${DOWNLOAD_FOLDER}/docker-binaries.tgz" "https://download.docker.com/${PLATFORM}/static/stable/${ARCH}/docker-${DOCKER_VERSION}.tgz" + wget --tries=3 --waitretry=30 --quiet -O "${DOWNLOAD_FOLDER}/docker-binaries.tgz" "https://download.docker.com/${PLATFORM}/static/stable/${ARCH}/docker-${DOCKER_VERSION}.tgz" tar -xf "${DOWNLOAD_FOLDER}/docker-binaries.tgz" -C "${DOWNLOAD_FOLDER}" mv "${DOWNLOAD_FOLDER}/docker/docker" dist/ fi diff --git a/build/download_docker_compose_binary.sh b/build/download_docker_compose_binary.sh index 8a6955815..e9255b41a 100755 --- a/build/download_docker_compose_binary.sh +++ b/build/download_docker_compose_binary.sh @@ -21,10 +21,10 @@ fi if [[ "$PLATFORM" == "windows" ]]; then - wget -O "dist/docker-compose.exe" "https://github.com/docker/compose/releases/download/$COMPOSE_VERSION/docker-compose-windows-${ARCH}.exe" + wget --tries=3 --waitretry=30 --quiet -O "dist/docker-compose.exe" "https://github.com/docker/compose/releases/download/$COMPOSE_VERSION/docker-compose-windows-${ARCH}.exe" chmod +x "dist/docker-compose.exe" else - wget -O "dist/docker-compose" "https://github.com/docker/compose/releases/download/$COMPOSE_VERSION/docker-compose-${PLATFORM}-${ARCH}" + wget --tries=3 --waitretry=30 --quiet -O "dist/docker-compose" "https://github.com/docker/compose/releases/download/$COMPOSE_VERSION/docker-compose-${PLATFORM}-${ARCH}" chmod +x "dist/docker-compose" fi diff --git a/build/download_helm_binary.sh b/build/download_helm_binary.sh index b464918c8..ae3de93e7 100755 --- a/build/download_helm_binary.sh +++ b/build/download_helm_binary.sh @@ -13,7 +13,7 @@ HELM_DIST="helm-$HELM_VERSION-$PLATFORM-$ARCH" if [[ ${PLATFORM} == "windows" ]]; then - wget -O tmp.zip "https://get.helm.sh/${HELM_DIST}.zip" && unzip -o -j tmp.zip "${PLATFORM}-${ARCH}/helm.exe" -d dist && rm -f tmp.zip + wget --tries=3 --waitretry=30 --quiet -O tmp.zip "https://get.helm.sh/${HELM_DIST}.zip" && unzip -o -j tmp.zip "${PLATFORM}-${ARCH}/helm.exe" -d dist && rm -f tmp.zip else wget -qO- "https://get.helm.sh/${HELM_DIST}.tar.gz" | tar -x -z --strip-components 1 "${PLATFORM}-${ARCH}/helm" mv "helm" "dist/helm" diff --git a/build/download_kubectl_binary.sh b/build/download_kubectl_binary.sh index 8b492a781..785912c4e 100755 --- a/build/download_kubectl_binary.sh +++ b/build/download_kubectl_binary.sh @@ -12,9 +12,9 @@ KUBECTL_VERSION=$3 if [[ ${PLATFORM} == "windows" ]]; then - wget -O "dist/kubectl.exe" "https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/windows/amd64/kubectl.exe" + wget --tries=3 --waitretry=30 --quiet -O "dist/kubectl.exe" "https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/windows/amd64/kubectl.exe" chmod +x "dist/kubectl.exe" else - wget -O "dist/kubectl" "https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/${PLATFORM}/${ARCH}/kubectl" + wget --tries=3 --waitretry=30 --quiet -O "dist/kubectl" "https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/${PLATFORM}/${ARCH}/kubectl" chmod +x "dist/kubectl" fi diff --git a/build/download_mingit_binary.sh b/build/download_mingit_binary.sh new file mode 100755 index 000000000..7f3a7d0cf --- /dev/null +++ b/build/download_mingit_binary.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -ne 3 ]]; then + echo "Illegal number of parameters" >&2 + exit 1 +fi + +PLATFORM=$1 +MINGIT_VERSION=$3 + +if [[ ${PLATFORM} == "windows" ]]; then + GIT_VERSION=$(echo $MINGIT_VERSION | cut -d "." -f 1-3) + GIT_PATCH_VERSION=$(echo $MINGIT_VERSION | cut -d "." -f 4) + + wget --tries=3 --waitretry=30 --quiet "https://github.com/git-for-windows/git/releases/download/v$GIT_VERSION.windows.$GIT_PATCH_VERSION/MinGit-$GIT_VERSION.$GIT_PATCH_VERSION-busybox-64-bit.zip" + unzip "MinGit-$GIT_VERSION.$GIT_PATCH_VERSION-busybox-64-bit.zip" -d dist/mingit +fi \ No newline at end of file diff --git a/build/windows/Dockerfile b/build/windows/Dockerfile index ab27d7eb9..cda8e5586 100644 --- a/build/windows/Dockerfile +++ b/build/windows/Dockerfile @@ -1,21 +1,14 @@ ARG OSVERSION FROM --platform=linux/amd64 gcr.io/k8s-staging-e2e-test-images/windows-servercore-cache:1.0-linux-amd64-${OSVERSION} as core -FROM --platform=linux/amd64 alpine:3.14 as downloader -ENV GIT_VERSION 2.30.0 -ENV GIT_PATCH_VERSION 2 - -RUN mkdir mingit/ \ - && wget https://github.com/git-for-windows/git/releases/download/v$GIT_VERSION.windows.$GIT_PATCH_VERSION/MinGit-$GIT_VERSION.$GIT_PATCH_VERSION-busybox-64-bit.zip \ - && unzip MinGit-$GIT_VERSION.$GIT_PATCH_VERSION-busybox-64-bit.zip -d mingit/ FROM mcr.microsoft.com/windows/nanoserver:${OSVERSION} as production ENV PATH "C:\mingit\cmd;C:\Windows\system32;C:\Windows;" -COPY --from=downloader /mingit mingit/ COPY --from=core /Windows/System32/netapi32.dll /Windows/System32/netapi32.dll USER ContainerAdministrator +COPY dist/mingit/ mingit/ COPY dist/docker.exe / COPY dist/docker-compose.exe / COPY dist/helm.exe / @@ -33,4 +26,4 @@ EXPOSE 8000 LABEL io.portainer.server true -ENTRYPOINT ["/portainer.exe"] +ENTRYPOINT ["/portainer.exe"] \ No newline at end of file diff --git a/pkg/libstack/compose/download.sh b/pkg/libstack/compose/download.sh index 6441854ce..7fe19cb0b 100755 --- a/pkg/libstack/compose/download.sh +++ b/pkg/libstack/compose/download.sh @@ -16,10 +16,10 @@ fi if [[ "$PLATFORM" == "windows" ]]; then - wget -O "docker-compose.exe" "https://github.com/docker/compose/releases/download/$COMPOSE_VERSION/docker-compose-windows-${ARCH}.exe" + wget --tries=3 --waitretry=30 --quiet -O "docker-compose.exe" "https://github.com/docker/compose/releases/download/$COMPOSE_VERSION/docker-compose-windows-${ARCH}.exe" chmod +x "docker-compose.exe" else - wget -O "docker-compose" "https://github.com/docker/compose/releases/download/$COMPOSE_VERSION/docker-compose-${PLATFORM}-${ARCH}" + wget --tries=3 --waitretry=30 --quiet -O "docker-compose" "https://github.com/docker/compose/releases/download/$COMPOSE_VERSION/docker-compose-${PLATFORM}-${ARCH}" chmod +x "docker-compose" fi