Merge pull request #50717 from cblecker/git-util-func

Automatic merge from submit-queue

Fix rsync issue when maintaining data containers

**What this PR does / why we need it**:
When we stopped syncing the full .git dir into the container and created a dummy git tree, an issue arose if you're maintaining a data dir volume. The git tree created in the container wouldn't be updated if we make more commits and sync them in, and would cause a dirty tree.

This PR changes the rsync flags so that "H" filtered files are hidden from rsync on the source side, and if they exist on the destination side, they will be deleted.

**Which issue this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close that issue when PR gets merged)*: fixes a bug introduced in #50417

**Special notes for your reviewer**:
/assign @sttts @dims @ixdy

**Release note**:

```release-note
NONE
```

build/common.sh

@@ -654,7 +654,6 @@ function kube::build::stop_rsyncd_container() {
 function kube::build::rsync {
   local -a rsync_opts=(
     --archive
-    --prune-empty-dirs
     --password-file="${LOCAL_OUTPUT_BUILD_CONTEXT}/rsyncd.password"
   )
   if (( ${KUBE_VERBOSE} >= 6 )); then
@@ -679,16 +678,19 @@ function kube::build::sync_to_container() {
   # directory and generated files. The '- /' filter prevents rsync
   # from trying to set the uid/gid/perms on the root of the sync tree.
   # As an exception, we need to sync generated files in staging/, because
-  # they will not be re-generated by 'make'.
+  # they will not be re-generated by 'make'. Note that the 'H' filtered files
+  # are hidden from rsync so they will be deleted in the target container if
+  # they exist. This will allow them to be re-created in the container if
+  # necessary.
   kube::build::rsync \
     --delete \
-    --filter='- /.git/' \
+    --filter='H /.git/' \
     --filter='- /.make/' \
     --filter='- /_tmp/' \
     --filter='- /_output/' \
     --filter='- /' \
-    --filter='- zz_generated.*' \
-    --filter='- generated.proto' \
+    --filter='H zz_generated.*' \
+    --filter='H generated.proto' \
     "${KUBE_ROOT}/" "rsync://k8s@${KUBE_RSYNC_ADDR}/k8s/"
 
   kube::build::stop_rsyncd_container
@@ -714,6 +716,7 @@ function kube::build::copy_output() {
   # We are looking to copy out all of the built binaries along with various
   # generated files.
   kube::build::rsync \
+    --prune-empty-dirs \
     --filter='- /_temp/' \
     --filter='+ /vendor/' \
     --filter='+ /Godeps/' \

build/run.sh

@@ -25,10 +25,19 @@ set -o pipefail
 KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
 source "$KUBE_ROOT/build/common.sh"
 
+KUBE_RUN_COPY_OUTPUT="${KUBE_RUN_COPY_OUTPUT:-y}"
+
 kube::build::verify_prereqs
 kube::build::build_image
+
+if [[ ${KUBE_RUN_COPY_OUTPUT} =~ ^[yY]$ ]]; then
+  kube::log::status "Output from this container will be rsynced out upon completion. Set KUBE_RUN_COPY_OUTPUT=n to disable."
+else
+  kube::log::status "Output from this container will NOT be rsynced out upon completion. Set KUBE_RUN_COPY_OUTPUT=y to enable."
+fi
+
 kube::build::run_build_command "$@"
 
-if [[ ${KUBE_RUN_COPY_OUTPUT:-y} =~ ^[yY]$ ]]; then
+if [[ ${KUBE_RUN_COPY_OUTPUT} =~ ^[yY]$ ]]; then
   kube::build::copy_output
 fi

build/shell.sh

@@ -24,8 +24,5 @@ set -o pipefail
 
 KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
 source "${KUBE_ROOT}/build/common.sh"
-source "${KUBE_ROOT}/build/lib/release.sh"
 
-kube::build::verify_prereqs
-kube::build::build_image
-kube::build::run_build_command bash || true
+KUBE_RUN_COPY_OUTPUT="${KUBE_RUN_COPY_OUTPUT:-n}" "${KUBE_ROOT}/build/run.sh" bash "$@"

hack/lib/util.sh

@@ -331,7 +331,7 @@ kube::util::group-version-to-pkg-path() {
     return
   fi
 
-  # "v1" is the API GroupVersion 
+  # "v1" is the API GroupVersion
   if [[ "${group_version}" == "v1" ]]; then
     echo "vendor/k8s.io/api/core/v1"
     return
@@ -437,6 +437,23 @@ kube::util::git_upstream_remote_name() {
     head -n 1 | awk '{print $1}'
 }
 
+# Ensures the current directory is a git tree for doing things like restoring or
+# validating godeps
+kube::util::create-fake-git-tree() {
+  local -r target_dir=${1:-$(pwd)}
+
+  pushd "${target_dir}" >/dev/null
+    git init >/dev/null
+    git config --local user.email "nobody@k8s.io"
+    git config --local user.name "$0"
+    git add . >/dev/null
+    git commit -q -m "Snapshot" >/dev/null
+    if (( ${KUBE_VERBOSE:-5} >= 6 )); then
+      kube::log::status "${target_dir} is now a git tree."
+    fi
+  popd >/dev/null
+}
+
 # Checks whether godep restore was run in the current GOPATH, i.e. that all referenced repos exist
 # and are checked out to the referenced rev.
 kube::util::godep_restored() {

hack/update-staging-godeps-dockerized.sh

@@ -50,11 +50,16 @@ while getopts ":df" opt; do
   esac
 done
 
+# Confirm this is running inside a docker container, as this will modify the git tree (unsafe to run outside of container)
 kube::util::ensure_dockerized
 kube::golang::setup_env
+# Ensure we have a simple gopath so that we can modify it, and that no staging repos have made their way in
 kube::util::ensure_single_dir_gopath
 kube::util::ensure_no_staging_repos_in_gopath
+# Confirm we have the right godep version installed
 kube::util::ensure_godep_version v79
+# Create a fake git repo the root of the repo to prevent godeps from complaining
+kube::util::create-fake-git-tree "${KUBE_ROOT}"
 
 kube::log::status "Checking whether godeps are restored"
 if ! kube::util::godep_restored 2>&1 | sed 's/^/  /'; then
@@ -87,27 +92,13 @@ function diffGodepManifest() {
   fi
 }
 
-# Create a fake git repo for staging to prevent godeps from complaining
-pushd "${KUBE_ROOT}" >/dev/null
-  git init >/dev/null
-  git config --local user.email "nobody@k8s.io"
-  git config --local user.name "$0"
-  git add . >/dev/null
-  git commit -q -m "Snapshot" >/dev/null
-popd >/dev/null
-
 # move into staging and save the dependencies for everything in order
 mkdir -p "${TMP_GOPATH}/src/k8s.io"
 for repo in $(ls ${KUBE_ROOT}/staging/src/k8s.io); do
   cp -a "${KUBE_ROOT}/staging/src/k8s.io/${repo}" "${TMP_GOPATH}/src/k8s.io/"
 
-  pushd "${TMP_GOPATH}/src/k8s.io/${repo}" >/dev/null
-    git init >/dev/null
-    git config --local user.email "nobody@k8s.io"
-    git config --local user.name "$0"
-    git add . >/dev/null
-    git commit -q -m "Snapshot" >/dev/null
-  popd >/dev/null
+  # Create a fake git tree for the staging repo to prevent godeps from complaining
+  kube::util::create-fake-git-tree "${TMP_GOPATH}/src/k8s.io/${repo}"
 
   updateGodepManifest
   diffGodepManifest