From dd9219f304fa0dff8d8b1ac9d08342821759535c Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Thu, 23 Feb 2017 18:13:15 +0100 Subject: [PATCH] update-staging-{client-go,godeps}.sh: no godep-restore, pin godep, check workdir --- hack/lib/util.sh | 70 +++++++++++++++ hack/update-staging-client-go.sh | 16 +++- hack/update-staging-godeps.sh | 148 ++++++++++++++----------------- hack/verify-staging-client-go.sh | 61 +------------ hack/verify-staging-godeps.sh | 29 +----- staging/copy.sh | 46 ++++++---- 6 files changed, 184 insertions(+), 186 deletions(-) diff --git a/hack/lib/util.sh b/hack/lib/util.sh index dec32e8a21..dd3de6d49b 100755 --- a/hack/lib/util.sh +++ b/hack/lib/util.sh @@ -411,6 +411,76 @@ kube::util::git_upstream_remote_name() { head -n 1 | awk '{print $1}' } +# Checks whether godep restore was run in the current GOPATH, i.e. that all referenced repos exist +# and are checked out to the referenced rev. +kube::util::godep_restored() { + local -r godeps_json=${1:-Godeps/Godeps.json} + local -r gopath=${2:-${GOPATH%:*}} + if ! which jq &>/dev/null; then + echo "jq not found. Please install." 1>&2 + return 1 + fi + local root + local old_rev="" + while read path rev; do + rev=$(echo "${rev}" | sed "s/['\"]//g") # remove quotes which are around revs sometimes + + if [[ "${rev}" == "${old_rev}" ]] && [[ "${path}" == "${root}"* ]]; then + # avoid checking the same git/hg root again + continue + fi + + root="${path}" + while [ "${root}" != "." -a ! -d "${gopath}/src/${root}/.git" -a ! -d "${gopath}/src/${root}/.hg" ]; do + root=$(dirname "${root}") + done + if [ "${root}" == "." ]; then + echo "No checkout of ${path} found in GOPATH \"${gopath}\"." 1>&2 + return 1 + fi + local head + if [ -d "${gopath}/src/${root}/.git" ]; then + head="$(cd "${gopath}/src/${root}" && git rev-parse HEAD)" + else + head="$(cd "${gopath}/src/${root}" && hg parent --template '{node}')" + fi + if [ "${head}" != "${rev}" ]; then + echo "Unexpected HEAD '${head}' at ${gopath}/src/${root}, expected '${rev}'." 1>&2 + return 1 + fi + old_rev="${rev}" + done < <(jq '.Deps|.[]|.ImportPath + " " + .Rev' -r < "${godeps_json}") + return 0 +} + +# Exits script if working directory is dirty. +kube::util::ensure_clean_working_dir() { + if ! git diff --exit-code; then + echo -e "\nUnexpected dirty working directory." + exit 1 + fi +} + +# Ensure that the given godep version is installed and in the path +kube::util::ensure_godep_version() { + if [[ "$(godep version)" == *"godep ${1}"* ]]; then + return + fi + + kube::util::ensure-temp-dir + mkdir -p "${KUBE_TEMP}/go/src" + + GOPATH="${KUBE_TEMP}/go" go get -u github.com/tools/godep 2>/dev/null + pushd "${KUBE_TEMP}/go/src/github.com/tools/godep" >/dev/null + git checkout "${1:-v74}" + GOPATH="${KUBE_TEMP}/go" go install . + popd >/dev/null + + PATH="${KUBE_TEMP}/go/bin:${PATH}" + hash -r # force bash to clear PATH cache + godep version +} + # Checks whether there are any files matching pattern $2 changed between the # current branch and upstream branch named by $1. # Returns 1 (false) if there are no changes, 0 (true) if there are changes diff --git a/hack/update-staging-client-go.sh b/hack/update-staging-client-go.sh index dc72a8c29c..a723c37b81 100755 --- a/hack/update-staging-client-go.sh +++ b/hack/update-staging-client-go.sh @@ -19,4 +19,18 @@ set -o nounset set -o pipefail KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. -"${KUBE_ROOT}"/staging/copy.sh +source "${KUBE_ROOT}/hack/lib/util.sh" + +kube::util::ensure_clean_working_dir +kube::util::ensure_godep_version v74 + +cd ${KUBE_ROOT} + +echo "Checking whether godeps are restored" +if ! kube::util::godep_restored 2>&1 | sed 's/^/ /'; then + echo -e '\nRun 'godep restore' to download dependencies.' 1>&2 + exit 1 +fi + +echo "Running staging/copy.sh" +staging/copy.sh -u "$@" 2>&1 | sed 's/^/ /' diff --git a/hack/update-staging-godeps.sh b/hack/update-staging-godeps.sh index 3e21feec4f..6a0e9e547f 100755 --- a/hack/update-staging-godeps.sh +++ b/hack/update-staging-godeps.sh @@ -23,102 +23,84 @@ set -o errexit set -o nounset set -o pipefail +V="" +DRY_RUN=false +FAIL_ON_DIFF=false +while getopts ":vdf" opt; do + case $opt in + v) # increase verbosity + V="-v" + ;; + d) # do not godep-restore into a temporary directory, but use the existing GOPATH + DRY_RUN=true + ;; + f) # fail if something in the Godeps.json files changed + FAIL_ON_DIFF=true + ;; + \?) + echo "Invalid option: -$OPTARG" >&2 + exit 1 + ;; + esac +done +readonly V IN_PLACE KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. source "${KUBE_ROOT}/hack/lib/init.sh" +source "${KUBE_ROOT}/hack/lib/util.sh" -kube::golang::setup_env - -ORIGINAL_GOPATH="${GOPATH}" - -godepBinDir=${TMPDIR:-/tmp/}/kube-godep-bin -mkdir -p "${godepBinDir}" -godep=${godepBinDir}/bin/godep -pushd "${godepBinDir}" 2>&1 > /dev/null - # Build the godep tool - GOPATH="${godepBinDir}" - rm -rf * - go get -u github.com/tools/godep 2>/dev/null - export GODEP="${GOPATH}/bin/godep" - pin-godep() { - pushd "${GOPATH}/src/github.com/tools/godep" > /dev/null - git checkout "$1" - "${GODEP}" go install - popd > /dev/null - } - # Use to following if we ever need to pin godep to a specific version again - pin-godep 'v74' - "${godep}" version -popd 2>&1 > /dev/null - -# keep the godep restore path reasonably stable to avoid unnecessary restores -godepRestoreDir=${TMPDIR:-/tmp/}/kube-godep-restore - -TARGET_DIR=${TARGET_DIR:-${KUBE_ROOT}/staging} -echo "working in ${TARGET_DIR}" - -SKIP_RESTORE=${SKIP_RESTORE:-} -if [ "${SKIP_RESTORE}" != "true" ]; then - echo "starting godep restore" - mkdir -p "${godepRestoreDir}" - - # add the vendor folder so that we don't redownload things during restore - GOPATH="${godepRestoreDir}:${KUBE_ROOT}/staging:${ORIGINAL_GOPATH}" - # restore from kubernetes godeps to ensure we get the correct levels - # you get errors about the staging repos not using a known version control system - ${godep} restore > ${godepRestoreDir}/godep-restore.log - echo "finished godep restore" +echo "Checking whether godeps are restored" +if ! kube::util::godep_restored 2>&1 | sed 's/^/ /'; then + echo -e '\nRun 'godep restore' to download dependencies.' 1>&2 + exit 1 fi -echo "forcing fake godep folders to match the current state of master in tmp" -for stagingRepo in $(ls ${KUBE_ROOT}/staging/src/k8s.io); do - echo " creating ${stagingRepo}" - rm -rf ${godepRestoreDir}/src/k8s.io/${stagingRepo} - cp -a ${KUBE_ROOT}/staging/src/k8s.io/${stagingRepo} ${godepRestoreDir}/src/k8s.io +kube::util::ensure_clean_working_dir +kube::util::ensure-temp-dir +kube::util::ensure_godep_version v74 - # we need a git commit in the godep folder, otherwise godep won't save - pushd ${godepRestoreDir}/src/k8s.io/${stagingRepo} - git init > /dev/null - # we need this so later commands work, but nothing should ever actually include these commits - # these are local only, not global - git config user.email "you@example.com" - git config user.name "Your Name" - git add . > /dev/null - git commit -qm "fake commit" - popd -done +TMP_GOPATH="${KUBE_TEMP}/go" function updateGodepManifest() { - local repo=${1} + local repo="${1}" + pushd "${TMP_GOPATH}/src/k8s.io/${repo}" >/dev/null + echo "Updating godeps for k8s.io/${repo}" + rm -rf Godeps # remove the current Godeps.json so we always rebuild it + GOPATH="${TMP_GOPATH}:${GOPATH}:${KUBE_ROOT}/staging" godep save ${V} ./... 2>&1 | sed 's/^/ /' - echo "starting ${repo} save" - mkdir -p ${TARGET_DIR}/src/k8s.io - # if target_dir isn't the same as source dir, you need copy - test "${KUBE_ROOT}/staging" = "${TARGET_DIR}" || cp -a ${KUBE_ROOT}/staging/src/${repo} ${TARGET_DIR}/src/k8s.io - # remove the current Godeps.json so we always rebuild it - rm -rf ${TARGET_DIR}/src/${repo}/Godeps - GOPATH="${godepRestoreDir}:${TARGET_DIR}" - pushd ${TARGET_DIR}/src/${repo} - ${godep} save ./... + echo "Rewriting Godeps.json to remove commits that don't really exist because we haven't pushed the prereqs yet" + go run "${KUBE_ROOT}/staging/godeps-json-updater.go" --godeps-file="${TMP_GOPATH}/src/k8s.io/${repo}/Godeps/Godeps.json" --client-go-import-path="k8s.io/${repo}" - # now remove all the go files. We'll re-run a restore, go get, godep save cycle in the sync scripts - # to get the commits for other staging k8s.io repos anyway, so we don't need the added files - rm -rf vendor - - echo "rewriting Godeps.json to remove commits that don't really exist because we haven't pushed the prereqs yet" - GOPATH="${ORIGINAL_GOPATH}" - go run "${KUBE_ROOT}/staging/godeps-json-updater.go" --godeps-file="${TARGET_DIR}/src/${repo}/Godeps/Godeps.json" --client-go-import-path="${repo}" - - popd - echo "finished ${repo} save" + # commit so that following repos do not see this repo as dirty + git add vendor >/dev/null + git commit -a -m "Updated Godeps.json" >/dev/null + popd >/dev/null } # move into staging and save the dependencies for everything in order -for stagingRepo in $(ls ${KUBE_ROOT}/staging/src/k8s.io); do - # we have to skip client-go because it does unusual manipulation of its godeps - if [ "${stagingRepo}" == "client-go" ]; then - continue - fi +mkdir -p "${TMP_GOPATH}/src/k8s.io" +for repo in $(ls ${KUBE_ROOT}/staging/src/k8s.io); do + # we have to skip client-go because it does unusual manipulation of its godeps + if [ "${repo}" == "client-go" ]; then + continue + fi - updateGodepManifest "k8s.io/${stagingRepo}" + cp -a "${KUBE_ROOT}/staging/src/k8s.io/${repo}" "${TMP_GOPATH}/src/k8s.io/" + + pushd "${TMP_GOPATH}/src/k8s.io/${repo}" >/dev/null + git init >/dev/null + git config --local user.email "nobody@k8s.io" + git config --local user.name "$0" + git add . >/dev/null + git commit -q -m "Snapshot" >/dev/null + popd >/dev/null + + updateGodepManifest "${repo}" + + if [ "${FAIL_ON_DIFF}" == true ]; then + diff --ignore-matching-lines='^\s*\"Comment\"' -u "${KUBE_ROOT}/staging/src/k8s.io/${repo}/Godeps" "${TMP_GOPATH}/src/k8s.io/${repo}/Godeps/Godeps.json" + fi + if [ "${DRY_RUN}" != true ]; then + cp "${TMP_GOPATH}/src/k8s.io/${repo}/Godeps/Godeps.json" "${KUBE_ROOT}/staging/src/k8s.io/${repo}/Godeps" + fi done diff --git a/hack/verify-staging-client-go.sh b/hack/verify-staging-client-go.sh index 39d86ca6ed..a1428a3141 100755 --- a/hack/verify-staging-client-go.sh +++ b/hack/verify-staging-client-go.sh @@ -18,65 +18,12 @@ set -o errexit set -o nounset set -o pipefail -V="" -while getopts ":v" opt; do - case $opt in - v) # increase verbosity - V="-v" - ;; - \?) - echo "Invalid option: -$OPTARG" >&2 - ;; - esac -done -readonly V - KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. cd ${KUBE_ROOT} -if ! git diff --exit-code; then - echo "ERROR: $0 cannot be run on a dirty working directory." - exit 1 -fi - # Smoke test client-go examples -go install ./staging/src/k8s.io/client-go/examples/... +echo "Smoke testing client-go examples" +go install ./staging/src/k8s.io/client-go/examples/... 2>&1 | sed 's/^/ /' -# Create a temporary GOPATH for apimachinery and client-go, copy the current HEAD into each and turn them -# into a git repo. Then we can run copy.sh with this additional GOPATH. The Godeps.json will -# have invalid git SHA1s, but it's good enough as a smoke test of copy.sh. -if [ "${USE_TEMP_DIR:-1}" = 1 ]; then - TEMP_STAGING_GOPATH=$(mktemp -d -t verify-staging-client-go.XXXXX) - echo "Creating the temporary staging GOPATH directory: ${TEMP_STAGING_GOPATH}" - cleanup() { - if [ "${KEEP_TEMP_DIR:-0}" != 1 ]; then - rm -rf "${TEMP_STAGING_GOPATH}" - fi - } - trap cleanup EXIT SIGINT - mkdir -p "${TEMP_STAGING_GOPATH}/src/k8s.io" - ln -s "${PWD}" "${TEMP_STAGING_GOPATH}/src/k8s.io" -else - TEMP_STAGING_GOPATH="${GOPATH}" -fi -for PACKAGE in apimachinery client-go; do - PACKAGE_PATH="${TEMP_STAGING_GOPATH}/src/k8s.io/${PACKAGE}" - echo "Creating a temporary ${PACKAGE} repo with a snapshot of HEAD" - rm -rf "${PACKAGE_PATH}" - mkdir -p "${PACKAGE_PATH}" - git archive --format=tar "HEAD:staging/src/k8s.io/${PACKAGE}" | tar -xf - -C "${PACKAGE_PATH}" - pushd "${PACKAGE_PATH}" >/dev/null - git init >/dev/null - git add * - git -c user.email="nobody@k8s.io" -c user.name="verify-staging-client-go.sh" commit -q -m "Snapshot" - popd >/dev/null -done - -echo "Running godep restore" -pushd "${TEMP_STAGING_GOPATH}/src/k8s.io/kubernetes" >/dev/null -export GOPATH="${TEMP_STAGING_GOPATH}" -godep restore ${V} 2>&1 | sed 's/^/ /' - -echo "Testing staging/copy.sh" -staging/copy.sh -d 2>&1 | sed 's/^/ /' -popd +# Run update-staging-client-go.sh in dry-run mode, copy nothing into the staging dir +hack/update-staging-client-go.sh -d "$@" diff --git a/hack/verify-staging-godeps.sh b/hack/verify-staging-godeps.sh index 2cfa1b91ca..a81ffa5349 100755 --- a/hack/verify-staging-godeps.sh +++ b/hack/verify-staging-godeps.sh @@ -14,36 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -# TODO this does not address client-go, since it takes a different approach to vendoring -# TODO client-go should probably be made consistent - set -o errexit set -o nounset set -o pipefail - KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. -source "${KUBE_ROOT}/hack/lib/init.sh" - - -TARGET_DIR=$(mktemp -d "${TMPDIR:-/tmp/}$(basename 0).XXXXXXXXXXXX") -# Register function to be called on EXIT to remove folder. -function cleanup { - SKIP_CLEANUP=${SKIP_CLEANUP:-} - if [ "${SKIP_CLEANUP}" != "true" ]; then - rm -rf "${TARGET_DIR}" - fi -} -trap cleanup EXIT - -TARGET_DIR=${TARGET_DIR} ${KUBE_ROOT}/hack/update-staging-godeps.sh - -# check each staging repo to make sure its Godeps.json is correct -for stagingRepo in $(ls ${KUBE_ROOT}/staging/src/k8s.io); do - # we have to skip client-go because it does unusual manipulation of its godeps - if [ "${stagingRepo}" == "client-go" ]; then - continue - fi - - diff --ignore-matching-lines='^\s*\"Comment\"' ${KUBE_ROOT}/staging/src/k8s.io/${stagingRepo}/Godeps/Godeps.json ${TARGET_DIR}/src/k8s.io/${stagingRepo}/Godeps/Godeps.json -done +${KUBE_ROOT}/hack/update-staging-godeps.sh -d -f "$@" diff --git a/staging/copy.sh b/staging/copy.sh index 4816179dbf..817e7f3623 100755 --- a/staging/copy.sh +++ b/staging/copy.sh @@ -18,9 +18,15 @@ set -o errexit set -o nounset set -o pipefail + +KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. +source "${KUBE_ROOT}/hack/lib/init.sh" + + FAIL_ON_CHANGES=false DRY_RUN=false -while getopts ":fd" opt; do +RUN_FROM_UPDATE_SCRIPT=false +while getopts ":fdu" opt; do case $opt in f) FAIL_ON_CHANGES=true @@ -28,25 +34,31 @@ while getopts ":fd" opt; do d) DRY_RUN=true ;; + u) + RUN_FROM_UPDATE_SCRIPT=true + ;; \?) echo "Invalid option: -$OPTARG" >&2 + exit 1 ;; esac done readonly FAIL_ON_CHANGES DRY_RUN -echo "**PLEASE** run \"godep restore\" before running this script" +if [ "${RUN_FROM_UPDATE_SCRIPT}" != true ]; then + echo "Do not run this script directly, but via hack/update-staging-client-go.sh." + exit 1 +fi + # PREREQUISITES: run `godep restore` in the main repo before calling this script. CLIENTSET="clientset" MAIN_REPO_FROM_SRC="k8s.io/kubernetes" -MAIN_REPO="${GOPATH%:*}/src/${MAIN_REPO_FROM_SRC}" +MAIN_REPO="$(cd "${KUBE_ROOT}"; pwd)" # absolute path CLIENT_REPO_FROM_SRC="k8s.io/client-go" CLIENT_REPO_TEMP_FROM_SRC="k8s.io/_tmp" CLIENT_REPO="${MAIN_REPO}/staging/src/${CLIENT_REPO_FROM_SRC}" CLIENT_REPO_TEMP="${MAIN_REPO}/staging/src/${CLIENT_REPO_TEMP_FROM_SRC}" -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - if LANG=C sed --help 2>&1 | grep -q GNU; then SED="sed" elif which gsed &>/dev/null; then @@ -127,7 +139,7 @@ rm -rf "${CLIENT_REPO_TEMP}"/vendor/k8s.io/kubernetes mv "${CLIENT_REPO_TEMP}"/vendor "${CLIENT_REPO_TEMP}"/_vendor echo "rewriting Godeps.json" -go run "${DIR}/godeps-json-updater.go" --godeps-file="${CLIENT_REPO_TEMP}/Godeps/Godeps.json" --client-go-import-path="${CLIENT_REPO_FROM_SRC}" +go run "${KUBE_ROOT}/staging/godeps-json-updater.go" --godeps-file="${CLIENT_REPO_TEMP}/Godeps/Godeps.json" --client-go-import-path="${CLIENT_REPO_FROM_SRC}" echo "rewriting imports" grep -Rl "\"${MAIN_REPO_FROM_SRC}" "${CLIENT_REPO_TEMP}" | \ @@ -204,17 +216,17 @@ rm -rf "${CLIENT_REPO_TEMP}/_vendor/k8s.io/client-go" rm -rf "${CLIENT_REPO_TEMP}/staging" if [ "${FAIL_ON_CHANGES}" = true ]; then - echo "running FAIL_ON_CHANGES" - ret=0 - if diff -NauprB -I "GoVersion.*\|GodepVersion.*" "${CLIENT_REPO}" "${CLIENT_REPO_TEMP}"; then - echo "${CLIENT_REPO} up to date." - cleanup - exit 0 - else - echo "${CLIENT_REPO} is out of date. Please run hack/update-client-go.sh" - cleanup - exit 1 - fi + echo "running FAIL_ON_CHANGES" + ret=0 + if diff --ignore-matching-lines='^\s*\"Comment\"' -NauprB -I "GoVersion.*\|GodepVersion.*" "${CLIENT_REPO}" "${CLIENT_REPO_TEMP}"; then + echo "${CLIENT_REPO} up to date." + cleanup + exit 0 + else + echo "${CLIENT_REPO} is out of date. Please run hack/update-staging-client-go.sh" + cleanup + exit 1 + fi fi # clean the ${CLIENT_REPO}