added copy.sh which cuts the client repo; made necessary changes to scripts in

the repo to cope with the client repo staging area; added symlink
vendor/k8s.io/client-go which pointed to the staging area.

hack/godep-save.sh

@@ -19,8 +19,11 @@ set -o nounset
 set -o pipefail
 
 KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
+source "${KUBE_ROOT}/hack/lib/init.sh"
+export GOPATH=${GOPATH}:${KUBE_ROOT}/staging
 GODEP="${GODEP:-godep}"
 
+
 # Some things we want in godeps aren't code dependencies, so ./...
 # won't pick them up.
 REQUIRED_BINS=(
@@ -31,4 +34,9 @@ REQUIRED_BINS=(
 
 pushd "${KUBE_ROOT}" > /dev/null
   GO15VENDOREXPERIMENT=1 ${GODEP} save "${REQUIRED_BINS[@]}"
+  # create a symlink in vendor directory pointing to the staging client. This
+  # let other packages use the staging client as if it were vendored.
+  if [ ! -e "vendor/k8s.io/client-go" ]; then
+    ln -s ../../staging/src/k8s.io/client-go vendor/k8s.io/client-go
+  fi
 popd > /dev/null

hack/make-rules/helpers/cache_go_dirs.sh

@@ -38,6 +38,7 @@ function kfind() {
         -not \(                    \
             \(                     \
                 -path ./vendor -o  \
+                -path ./staging -o \
                 -path ./_\* -o     \
                 -path ./.\* -o     \
                 -path ./docs -o    \

hack/make-rules/test.sh

@@ -26,7 +26,7 @@ kube::golang::setup_env
 kube::test::find_dirs() {
   (
     cd ${KUBE_ROOT}
-    find . -not \( \
+    find -L . -not \( \
         \( \
           -path './_artifacts/*' \
           -o -path './_output/*' \
@@ -39,10 +39,16 @@ kube::test::find_dirs() {
           -o -path './test/e2e_node/*' \
           -o -path './test/integration/*' \
           -o -path './test/component/scheduler/perf/*' \
-          -o -path './third_party/*'\
+          -o -path './third_party/*' \
-          -o -path './vendor/*'\
+          -o -path './staging/*' \
+          -o -path './vendor/*' \
         \) -prune \
       \) -name '*_test.go' -print0 | xargs -0n1 dirname | sed 's|^\./||' | sort -u
+
+    find -L . \
+        -path './_output' -prune \
+        -o -path './vendor/k8s.io/client-go/*' \
+      -name '*_test.go' -print0 | xargs -0n1 dirname | sed 's|^\./||' | sort -u
   )
 }
 

hack/make-rules/vet.sh

@@ -43,7 +43,7 @@ done
 
 if [[ ${#targets[@]} -eq 0 ]]; then
   # Do not run on third_party directories.
-  targets=$(go list ./... | egrep -v "/(third_party|vendor)/")
+  targets=$(go list -e ./... | egrep -v "/(third_party|vendor|staging)/")
 fi
 
 go vet "${goflags[@]:+${goflags[@]}}" ${targets[@]}

hack/update-codecgen.sh

@@ -31,6 +31,7 @@ generated_files=($(
       \( \
         -wholename './output' \
         -o -wholename './_output' \
+        -o -wholename './staging' \
         -o -wholename './release' \
         -o -wholename './target' \
         -o -wholename '*/third_party/*' \

staging/src/k8s.io/client-go/README.md

@@ -0,0 +1,9 @@
+This _staging/src/k8s.io/client-go directory is the staging area of the client repo. It contains a versioned client, tools built around the client like the reflector, and all the client dependencies. The content will be periodically published to k8s.io/client-go repo.
+The staged content is copied from the main repo, i.e., k8s.io/kubernetes, with directory rearrangement and necessary rewritings. To sync the content with the latest code in your local k8s.io/kubernetes, you need to run `godep restore` in k8s root directory, then run _staging/src/k8s.io/client-go/copy.sh.
+vendor/k8s.io/client-go is a symlink pointing to this staging area, so to use the packages in the staging area, you can import it as "vendor/client-go/<package-name>", as if the client were vendored. The client will be vendored from k8s.io/client-go for real after the test matrix is converted to vendor k8s components.
+
+
+[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/_staging/src/k8s.io/client-go/README.md?pixel)]()
+
+
+[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/staging/src/k8s.io/client-go/README.md?pixel)]()

staging/src/k8s.io/client-go/copy.sh

@@ -0,0 +1,158 @@
+#!/bin/bash
+
+# Copyright 2016 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+# PREREQUISITES: run `godep restore` in the main repo before calling this script.
+RELEASE="1.4"
+MAIN_REPO_FROM_SRC="${1:-"k8s.io/kubernetes"}"
+MAIN_REPO="${GOPATH%:*}/src/${MAIN_REPO_FROM_SRC}"
+CLIENT_REPO_FROM_SRC="${2:-"k8s.io/client-go/${RELEASE}"}"
+CLIENT_REPO="${MAIN_REPO}/staging/src/${CLIENT_REPO_FROM_SRC}"
+CLIENT_REPO_TEMP="${CLIENT_REPO}"/_tmp
+
+# working in the ${CLIENT_REPO_TEMP} so 'godep save' won't complain about dirty working tree.
+echo "creating the _tmp directory"
+mkdir -p "${CLIENT_REPO_TEMP}"
+cd "${CLIENT_REPO}"
+
+# mkcp copies file from the main repo to the client repo, it creates the directory if it doesn't exist in the client repo.
+function mkcp() {
+    mkdir -p "${CLIENT_REPO_TEMP}/$2" && cp -r "${MAIN_REPO}/$1" "${CLIENT_REPO_TEMP}/$2"
+}
+
+echo "copying client packages"
+mkcp "pkg/client/clientset_generated/release_1_4" "pkg/client/clientset_generated"
+mkcp "/pkg/client/record/" "/pkg/client"
+mkcp "/pkg/client/cache/" "/pkg/client"
+# TODO: make this test file not depending on pkg/client/unversioned
+rm "${CLIENT_REPO_TEMP}"/pkg/client/cache/listwatch_test.go
+mkcp "/pkg/client/restclient" "/pkg/client"
+mkcp "/pkg/client/testing" "/pkg/client"
+# remove this test because it imports the internal clientset
+rm "${CLIENT_REPO_TEMP}"/pkg/client/testing/core/fake_test.go
+mkcp "/pkg/client/transport" "/pkg/client"
+mkcp "/pkg/client/typed" "/pkg/client"
+
+mkcp "/pkg/client/unversioned/auth" "/pkg/client/unversioned"
+mkcp "/pkg/client/unversioned/clientcmd" "/pkg/client/unversioned"
+mkcp "/pkg/client/unversioned/portforward" "/pkg/client/unversioned"
+# remove this test because it imports the internal clientset
+rm "${CLIENT_REPO_TEMP}"/pkg/client/unversioned/portforward/portforward_test.go
+
+pushd "${CLIENT_REPO_TEMP}"
+echo "generating vendor/"
+GO15VENDOREXPERIMENT=1 godep save ./...
+popd
+
+echo "move to the client repo"
+# clean the ${CLIENT_REPO}
+ls "${CLIENT_REPO}" | grep -v '_tmp' | xargs rm -r
+mv "${CLIENT_REPO_TEMP}"/* "${CLIENT_REPO}"
+rm -r "${CLIENT_REPO_TEMP}"
+rm -r "${CLIENT_REPO}"/Godeps
+
+echo "moving vendor/k8s.io/kuberentes"
+cp -rn "${CLIENT_REPO}"/vendor/k8s.io/kubernetes/. "${CLIENT_REPO}"/
+# client-go will share the vendor of the main repo for now. When client-go
+# becomes a standalone repo, it will have its own vendor
+rm -r "${CLIENT_REPO}"/vendor
+# remove the pkg/util/net/sets/README.md to silent hack/verify-munge-docs.sh
+# TODO: probably we should convert the README.md a doc.go
+find ./ -name "README.md" -delete
+
+echo "rewriting imports"
+grep -Rl "\"${MAIN_REPO_FROM_SRC}" ./ | grep ".go" | grep -v "vendor/" | xargs sed -i "s|\"${MAIN_REPO_FROM_SRC}|\"${CLIENT_REPO_FROM_SRC}|g"
+
+echo "converting pkg/client/record to v1"
+# need a v1 version of ref.go
+cp "${CLIENT_REPO}"/pkg/api/ref.go "${CLIENT_REPO}"/pkg/api/v1/ref.go
+gofmt -w -r 'api.a -> v1.a' "${CLIENT_REPO}"/pkg/api/v1/ref.go
+gofmt -w -r 'Scheme -> api.Scheme' "${CLIENT_REPO}"/pkg/api/v1/ref.go
+# rewriting package name to v1
+sed -i 's/package api/package v1/g' "${CLIENT_REPO}"/pkg/api/v1/ref.go
+# ref.go refers api.Scheme, so manually import /pkg/api
+sed -i "s,import (,import (\n\"${CLIENT_REPO_FROM_SRC}/pkg/api\",g" "${CLIENT_REPO}"/pkg/api/v1/ref.go
+gofmt -w "${CLIENT_REPO}"/pkg/api/v1/ref.go 
+# rewrite pkg/client/record to v1
+gofmt -w -r 'api.a -> v1.a' "${CLIENT_REPO}"/pkg/client/record
+# need to call sed to rewrite the strings in test cases...
+find "${CLIENT_REPO}"/pkg/client/record -type f -name "*.go" -print0 | xargs -0 sed -i "s/api.ObjectReference/v1.ObjectReference/g"
+# rewrite the imports
+find "${CLIENT_REPO}"/pkg/client/record -type f -name "*.go" -print0 | xargs -0 sed -i 's,pkg/api",pkg/api/v1",g'
+# gofmt the changed files
+
+echo "rewrite conflicting Prometheus registration"
+sed -i "s/request_latency_microseconds/request_latency_microseconds_copy/g" "${CLIENT_REPO}"/pkg/client/metrics/metrics.go
+sed -i "s/request_status_codes/request_status_codes_copy/g" "${CLIENT_REPO}"/pkg/client/metrics/metrics.go
+sed -i "s/kubernetes_build_info/kubernetes_build_info_copy/g" "${CLIENT_REPO}"/pkg/version/version.go
+
+echo "rewrite proto names in proto.RegisterType"
+find "${CLIENT_REPO}" -type f -name "generated.pb.go" -print0 | xargs -0 sed -i "s/k8s\.io\.kubernetes/k8s.io.client-go.1.4/g"
+
+echo "rearranging directory layout"
+# $1 and $2 are relative to ${CLIENT_REPO}
+function mvfolder {
+    local src=${1%/#/}
+    local dst=${2%/#/}
+    # create the parent directory of dst
+    if [ "${dst%/*}" != "${dst}" ]; then
+        mkdir -p "${CLIENT_REPO}/${dst%/*}"
+    fi
+    # move
+    mv "${CLIENT_REPO}/${src}" "${CLIENT_REPO}/${dst}"
+    # rewrite package
+    local src_package="${src##*/}"
+    local dst_package="${dst##*/}"
+    find "${CLIENT_REPO}" -type f -name "*.go" -print0 | xargs -0 sed -i "s,package ${src_package},package ${dst_package},g"
+    # rewrite imports
+    # the first rule is to convert import lines like `restclient "k8s.io/client-go/pkg/client/restclient"`,
+    # where a package alias is the same the package name.
+    find "${CLIENT_REPO}" -type f -name "*.go" -print0 | \
+        xargs -0 sed -i "s,${src_package} \"${CLIENT_REPO_FROM_SRC}/${src},${dst_package} \"${CLIENT_REPO_FROM_SRC}/${dst},g"
+    find "${CLIENT_REPO}" -type f -name "*.go" -print0 | \
+        xargs -0 sed -i "s,\"${CLIENT_REPO_FROM_SRC}/${src},\"${CLIENT_REPO_FROM_SRC}/${dst},g"
+    # rewrite import invocation
+    if [ "${src_package}" != "${dst_package}" ]; then
+        find "${CLIENT_REPO}" -type f -name "*.go" -print0 | xargs -0 sed -i "s,\<${src_package}\.\([a-zA-Z]\),${dst_package}\.\1,g"
+    fi
+}
+
+mvfolder pkg/client/clientset_generated/release_1_4 kubernetes
+mvfolder pkg/client/typed/discovery discovery
+mvfolder pkg/client/typed/dynamic dynamic
+mvfolder pkg/client/transport transport
+mvfolder pkg/client/record tools/record
+mvfolder pkg/client/restclient rest
+mvfolder pkg/client/cache tools/cache
+mvfolder pkg/client/unversioned/auth tools/auth
+mvfolder pkg/client/unversioned/clientcmd tools/clientcmd
+mvfolder pkg/client/unversioned/portforward tools/portforward
+mvfolder pkg/client/metrics tools/metrics
+mvfolder pkg/client/testing/core testing
+if [ "$(find "${CLIENT_REPO}"/pkg/client -type f -name "*.go")" ]; then
+    echo "${CLIENT_REPO}/pkg/client is expected to be empty"
+    exit 1
+else 
+    rm -r "${CLIENT_REPO}"/pkg/client
+fi
+mvfolder third_party pkg/third_party
+mvfolder federation pkg/federation
+
+echo "running gofmt"
+find "${CLIENT_REPO}" -type f -name "*.go" -print0 | xargs -0 gofmt -w

vendor/k8s.io/client-go

@@ -0,0 +1 @@
+../../staging/src/k8s.io/client-go