Merge pull request #1239 from erikwilson/upload-builds-cloud-storage

Upload build to google cloud bucket

.drone.yml

@@ -9,6 +9,10 @@ platform:
 steps:
 - name: build
   image: rancher/dapper:v0.4.2
+  secrets: [ gcloud_auth ]
+  environment:
+    GCLOUD_AUTH:
+      from_secret: gcloud_auth
   commands:
   - dapper ci
   volumes:
@@ -100,6 +104,10 @@ platform:
 steps:
 - name: build
   image: rancher/dapper:v0.4.2
+  secrets: [ gcloud_auth ]
+  environment:
+    GCLOUD_AUTH:
+      from_secret: gcloud_auth
   commands:
   - dapper ci
   volumes:
@@ -174,6 +182,10 @@ platform:
 steps:
 - name: build
   image: rancher/dapper:v0.4.2
+  secrets: [ gcloud_auth ]
+  environment:
+    GCLOUD_AUTH:
+      from_secret: gcloud_auth
   commands:
   - dapper ci
   volumes:

Dockerfile.dapper

@@ -1,7 +1,7 @@
 FROM golang:1.13.5-alpine3.10
 
 RUN apk -U --no-cache add bash git gcc musl-dev docker vim less file curl wget ca-certificates jq linux-headers zlib-dev tar zip squashfs-tools npm coreutils \
-    python3 py3-pip python3-dev openssl-dev libffi-dev libseccomp libseccomp-dev make libuv-static
+    python2 python3 py3-pip python3-dev openssl-dev libffi-dev libseccomp libseccomp-dev make libuv-static
 RUN pip3 install 'tox==3.6.0'
 RUN apk -U --no-cache --repository http://dl-3.alpinelinux.org/alpine/edge/main/ add sqlite-dev sqlite-static
 RUN mkdir -p /go/src/golang.org/x && \
@@ -27,7 +27,7 @@ RUN if [ "$DQLITE" = true ]; then \
 
 ENV GO111MODULE off
 ENV DAPPER_RUN_ARGS --privileged -v k3s-cache:/go/src/github.com/rancher/k3s/.cache
-ENV DAPPER_ENV REPO TAG DRONE_TAG IMAGE_NAME SKIP_VALIDATE
+ENV DAPPER_ENV REPO TAG DRONE_TAG IMAGE_NAME SKIP_VALIDATE GCLOUD_AUTH
 ENV DAPPER_SOURCE /go/src/github.com/rancher/k3s/
 ENV DAPPER_OUTPUT ./bin ./dist ./build/out
 ENV DAPPER_DOCKER_SOCKET true

install.sh

@@ -33,6 +33,10 @@ set -e
 #     Version of k3s to download from github. Will attempt to download the
 #     latest version if not specified.
 #
+#   - INSTALL_K3S_COMMIT
+#     Commit of k3s to download from temporary cloud storage.
+#     * (for developer & QA use)
+#
 #   - INSTALL_K3S_BIN_DIR
 #     Directory to install k3s binary, links, and uninstall script to, or use
 #     /usr/local/bin as the default
@@ -67,6 +71,7 @@ set -e
 #     if not specified.
 
 GITHUB_URL=https://github.com/rancher/k3s/releases
+STORAGE_URL=https://storage.googleapis.com/k3s-ci-builds
 DOWNLOADER=
 
 # --- helper functions for logs ---
@@ -297,7 +302,9 @@ setup_tmp() {
 
 # --- use desired k3s version if defined or find latest ---
 get_release_version() {
-    if [ -n "${INSTALL_K3S_VERSION}" ]; then
+    if [ -n "${INSTALL_K3S_COMMIT}" ]; then
+        VERSION_K3S="commit ${INSTALL_K3S_COMMIT}"
+    elif [ -n "${INSTALL_K3S_VERSION}" ]; then
         VERSION_K3S=${INSTALL_K3S_VERSION}
     else
         info "Finding latest release"
@@ -338,7 +345,11 @@ download() {
 
 # --- download hash from github url ---
 download_hash() {
-    HASH_URL=${GITHUB_URL}/download/${VERSION_K3S}/sha256sum-${ARCH}.txt
+    if [ -n "${INSTALL_K3S_COMMIT}" ]; then
+        HASH_URL=${STORAGE_URL}/k3s${SUFFIX}-${INSTALL_K3S_COMMIT}.sha256sum
+    else
+        HASH_URL=${GITHUB_URL}/download/${VERSION_K3S}/sha256sum-${ARCH}.txt
+    fi
     info "Downloading hash ${HASH_URL}"
     download ${TMP_HASH} ${HASH_URL}
     HASH_EXPECTED=$(grep " k3s${SUFFIX}$" ${TMP_HASH})
@@ -359,7 +370,11 @@ installed_hash_matches() {
 
 # --- download binary from github url ---
 download_binary() {
-    BIN_URL=${GITHUB_URL}/download/${VERSION_K3S}/k3s${SUFFIX}
+    if [ -n "${INSTALL_K3S_COMMIT}" ]; then
+        BIN_URL=${STORAGE_URL}/k3s${SUFFIX}-${INSTALL_K3S_COMMIT}
+    else
+        BIN_URL=${GITHUB_URL}/download/${VERSION_K3S}/k3s${SUFFIX}
+    fi
     info "Downloading binary ${BIN_URL}"
     download ${TMP_BIN} ${BIN_URL}
 }

scripts/build

@@ -18,12 +18,12 @@ VERSIONFLAGS="
     -X ${PKG}/pkg/version.GitCommit=${COMMIT:0:8}
 
     -X ${VENDOR_PREFIX}k8s.io/client-go/pkg/version.gitVersion=${VERSION}
-    -X ${VENDOR_PREFIX}k8s.io/client-go/pkg/version.gitCommit=${LONG_COMMIT}
+    -X ${VENDOR_PREFIX}k8s.io/client-go/pkg/version.gitCommit=${COMMIT}
     -X ${VENDOR_PREFIX}k8s.io/client-go/pkg/version.gitTreeState=${TREE_STATE}
     -X ${VENDOR_PREFIX}k8s.io/client-go/pkg/version.buildDate=${buildDate}
 
     -X ${VENDOR_PREFIX}k8s.io/component-base/version.gitVersion=${VERSION}
-    -X ${VENDOR_PREFIX}k8s.io/component-base/version.gitCommit=${LONG_COMMIT}
+    -X ${VENDOR_PREFIX}k8s.io/component-base/version.gitCommit=${COMMIT}
     -X ${VENDOR_PREFIX}k8s.io/component-base/version.gitTreeState=${TREE_STATE}
     -X ${VENDOR_PREFIX}k8s.io/component-base/version.buildDate=${buildDate}
 

scripts/build-upload

@@ -0,0 +1,57 @@
+#!/bin/bash
+
+[ -n "$GCLOUD_AUTH" ] || {
+  exit 0
+}
+[ -x "$1" ] || {
+  echo "First argument should be an executable" >&2
+  exit 1
+}
+[ -n "$2" ] || {
+  echo "Second argument should be a commit hash" >&2
+  exit 1
+}
+
+umask 077
+
+TMPDIR=$(mktemp -d)
+cleanup() {
+  exit_code=$?
+  trap - EXIT INT
+  rm -rf ${TMPDIR}
+  exit ${exit_code}
+}
+trap cleanup EXIT INT
+
+GCLOUD_JSON=${TMPDIR}/.gcloud.json
+[ -z "${GCLOUD_AUTH}" ] || echo "${GCLOUD_AUTH}" >${GCLOUD_JSON}
+[ -s "${GCLOUD_JSON}" ] || {
+  echo "gcloud auth not defined" >&2
+  exit 1
+}
+
+BOTO_CONF=${TMPDIR}/.boto
+[ -s "${BOTO_CONF}" ] || cat >${BOTO_CONF} <<END
+[Credentials]
+gs_service_key_file = ${GCLOUD_JSON}
+[Boto]
+https_validate_certificates = True
+[GSUtil]
+content_language = en
+default_api_version = 2
+default_project_id = rancher-dev
+END
+
+BUILD_NAME=$(basename $1)-$2
+(cd $(dirname $1) && sha256sum $(basename $1)) >${TMPDIR}/${BUILD_NAME}.sha256sum
+cp $1 ${TMPDIR}/${BUILD_NAME}
+
+[ -d "${TMPDIR}/gsutil" ] || curl -sfL https://storage.googleapis.com/pub/gsutil.tar.gz | tar xz -C ${TMPDIR}
+
+HOME=${TMPDIR}
+PATH=${PATH}:${HOME}/gsutil
+
+gsutil cp ${TMPDIR}/${BUILD_NAME}* gs://k3s-ci-builds || exit 1
+
+echo "Build uploaded" >&2
+echo "https://storage.googleapis.com/k3s-ci-builds/${BUILD_NAME}"

scripts/package-cli

@@ -60,3 +60,5 @@ if [ "$DQLITE" = "true" ]; then
     DQLITE_TAGS="dqlite"
 fi
 CGO_ENABLED=0 go build -tags "$DQLITE_TAGS" -ldflags "$LDFLAGS $STATIC" -o ${CMD_NAME} ./cmd/k3s/main.go
+
+./scripts/build-upload ${CMD_NAME} ${COMMIT}

scripts/version.sh

@@ -6,8 +6,10 @@ if [ -n "$(git status --porcelain --untracked-files=no)" ]; then
     TREE_STATE=dirty
 fi
 
-COMMIT=$(git rev-parse --short HEAD)
+COMMIT=$(git log -n3 --pretty=format:"%H %ae" | grep -v ' drone@localhost$' | cut -f1 -d\  | head -1)
-LONG_COMMIT=$(git rev-parse HEAD)
+if [ -z "${COMMIT}" ]; then
+  COMMIT=$(git rev-parse HEAD)
+fi
 
 GIT_TAG=${DRONE_TAG:-$(git tag -l --contains HEAD | head -n 1)}
 
@@ -34,6 +36,6 @@ VERSION_CNIPLUGINS="v0.7.6-k3s1"
 if [[ -n "$GIT_TAG" ]]; then
     VERSION=$GIT_TAG
 else
-    VERSION="$(sed -e 's/[-+].*//' <<< "$VERSION_K8S")+$COMMIT$DIRTY"
+    VERSION="$(sed -e 's/[-+].*//' <<< "$VERSION_K8S")+${COMMIT:0:8}$DIRTY"
 fi
 VERSION_TAG="$(sed -e 's/+/-/g' <<< "$VERSION")"