Merge pull request #39898 from ixdy/bazel-release-tars

Automatic merge from submit-queue

Build release tars using bazel

**What this PR does / why we need it**: builds equivalents of the various kubernetes release tarballs, solely using bazel.

For example, you can now do
```console
$ make bazel-release
$ hack/e2e.go -v -up -test -down
```

**Special notes for your reviewer**: this is currently dependent on 3b29803eb5, which I have yet to turn into a pull request, since I'm still trying to figure out if this is the best approach.

Basically, the issue comes up with the way we generate the various server docker image tarfiles and load them on nodes:
* we `md5sum` the binary being encapsulated (e.g. kube-proxy) and save that to `$binary.docker_tag` in the server tarball
* we then build the docker image and tag using that md5sum (e.g. `gcr.io/google_containers/kube-proxy:$MD5SUM`)
* we `docker save` this image, which embeds the full tag in the `$binary.tar` file.
* on cluster startup, we `docker load` these tarballs, which are loaded with the tag that we'd created at build time. the nodes then use the `$binary.docker_tag` file to find the right image.

With the current bazel `docker_build` rule, the tag isn't saved in the docker image tar, so the node is unable to find the image after `docker load`ing it.

My changes to the rule save the tag in the docker image tar, though I don't know if there are subtle issues with it. (Maybe we want to only tag when `--stamp` is given?)

Also, the docker images produced by bazel have the timestamp set to the unix epoch, which is not great for debugging. Might be another thing to change with a `--stamp`.

Long story short, we probably need to follow up with bazel folks on the best way to solve this problem.

**Release note**:

```release-note
NONE
```

BUILD.bazel

@@ -12,6 +12,7 @@ gcs_upload(
     name = "ci-artifacts",
     data = [
         "//build/debs",
+        "//build/release-tars",
     ],
 )
 
@@ -51,13 +52,3 @@ filegroup(
     ],
     tags = ["automanaged"],
 )
-
-pkg_tar(
-    name = "kubernetes-src",
-    extension = "tar.gz",
-    files = [
-        ":all-srcs",
-    ],
-    package_dir = "kubernetes",
-    strip_prefix = "//",
-)

Makefile

@@ -470,7 +470,7 @@ help:
 endif
 
 # Non-dockerized bazel rules.
-.PHONY: bazel-build bazel-test
+.PHONY: bazel-build bazel-test bazel-release
 
 ifeq ($(PRINT_HELP),y)
 define BAZEL_BUILD_HELP_INFO
@@ -500,3 +500,17 @@ else
 bazel-test:
 	bazel test  --test_output=errors //cmd/... //pkg/... //federation/... //plugin/... //build/... //third_party/... //hack/...
 endif
+
+ifeq ($(PRINT_HELP),y)
+define BAZEL_BUILD_HELP_INFO
+# Build release tars with bazel
+#
+# Example:
+# make bazel-release
+endef
+bazel-release:
+	@echo "$$BAZEL_BUILD_HELP_INFO"
+else
+bazel-release:
+	bazel build //build/release-tars
+endif

WORKSPACE

@@ -10,6 +10,12 @@ git_repository(
     remote = "https://github.com/kubernetes/release.git",
 )
 
+git_repository(
+    name = "io_bazel",
+    commit = "3b29803eb528ff525c7024190ffbf4b08c598cf2",
+    remote = "https://github.com/ixdy/bazel.git",
+)
+
 load("@io_bazel_rules_go//go:def.bzl", "go_repositories")
 
 go_repositories()

build/BUILD

@@ -1,6 +1,22 @@
 package(default_visibility = ["//visibility:public"])
 
-load("@bazel_tools//tools/build_defs/docker:docker.bzl", "docker_build")
+load("@io_bazel//tools/build_defs/docker:docker.bzl", "docker_build")
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//build/debs:all-srcs",
+        "//build/release-tars:all-srcs",
+    ],
+    tags = ["automanaged"],
+)
 
 docker_build(
     name = "busybox",
@@ -34,43 +50,59 @@ docker_build(
     ],
 )
 
+DOCKERIZED_BINARIES = {
+    "kube-apiserver": {
+        "base": ":busybox-libc",
+        "target": "//cmd/kube-apiserver:kube-apiserver",
+    },
+    "kube-controller-manager": {
+        "base": ":busybox-libc",
+        "target": "//cmd/kube-controller-manager:kube-controller-manager",
+    },
+    "kube-scheduler": {
+        "base": ":busybox-libc",
+        "target": "//plugin/cmd/kube-scheduler:kube-scheduler",
+    },
+    "kube-aggregator": {
+        "base": ":busybox-libc",
+        "target": "//cmd/kube-aggregator:kube-aggregator",
+    },
+    "kube-proxy": {
+        "base": ":busybox-net",
+        "target": "//cmd/kube-proxy:kube-proxy",
+    },
+}
+
+[genrule(
+    name = binary + "_docker_tag",
+    srcs = [meta["target"]],
+    outs = [binary + ".docker_tag"],
+    # Currently each target has two outputs, the binary and its library, so hash only the first item (the binary).
+    # This can be made less hacky when we have static linking working.
+    cmd = "md5sum $(locations " + meta["target"] + ") | grep '" + binary + "'$$ | cut -f1 -d' ' | tr -d '\n' > $@",
+) for binary, meta in DOCKERIZED_BINARIES.items()]
+
 [docker_build(
     name = binary,
-    base = ":busybox-libc",
+    base = meta["base"],
     cmd = ["/usr/bin/" + binary],
     debs = [
         "//build/debs:%s.deb" % binary,
     ],
-    repository = "gcr.io/google-containers",
+    image_tags = [
-) for binary in [
+        "@%s.docker_tag" % binary,
-    "kube-apiserver",
-    "kube-controller-manager",
-    "kube-scheduler",
-    "kube-aggregator",
-]]
-
-docker_build(
-    name = "kube-proxy",
-    base = ":busybox-net",
-    cmd = ["/usr/bin/kube-proxy"],
-    debs = [
-        "//build/debs:kube-proxy.deb",
     ],
-    repository = "gcr.io/google-containers",
+    repository = "gcr.io/google_containers/" + binary,
-)
+    repository_append_package = False,
+    symlinks = {
+        # Some cluster startup scripts expect to find the binaries in /usr/local/bin,
+        # but the debs install the binaries into /usr/bin.
+        "/usr/local/bin/" + binary: "/usr/bin/" + binary,
+    },
+) for binary, meta in DOCKERIZED_BINARIES.items()]
 
 filegroup(
-    name = "package-srcs",
+    name = "docker-artifacts",
-    srcs = glob(["**"]),
+    srcs = [":%s.tar" % binary for binary in DOCKERIZED_BINARIES.keys()] +
-    tags = ["automanaged"],
+           [":%s.docker_tag" % binary for binary in DOCKERIZED_BINARIES.keys()],
-    visibility = ["//visibility:private"],
-)
-
-filegroup(
-    name = "all-srcs",
-    srcs = [
-        ":package-srcs",
-        "//build/debs:all-srcs",
-    ],
-    tags = ["automanaged"],
 )

build/lib/release.sh

@@ -356,34 +356,37 @@ function kube::release::package_salt_tarball() {
 function kube::release::package_kube_manifests_tarball() {
   kube::log::status "Building tarball: manifests"
 
+  local salt_dir="${KUBE_ROOT}/cluster/saltbase/salt"
+
   local release_stage="${RELEASE_STAGE}/manifests/kubernetes"
   rm -rf "${release_stage}"
-  local dst_dir="${release_stage}/gci-trusty"
-  mkdir -p "${dst_dir}"
 
-  local salt_dir="${KUBE_ROOT}/cluster/saltbase/salt"
+  mkdir -p "${release_stage}"
-  cp "${salt_dir}/cluster-autoscaler/cluster-autoscaler.manifest" "${dst_dir}/"
   cp "${salt_dir}/fluentd-gcp/fluentd-gcp.yaml" "${release_stage}/"
   cp "${salt_dir}/kube-registry-proxy/kube-registry-proxy.yaml" "${release_stage}/"
   cp "${salt_dir}/kube-proxy/kube-proxy.manifest" "${release_stage}/"
-  cp "${salt_dir}/etcd/etcd.manifest" "${dst_dir}"
+
-  cp "${salt_dir}/kube-scheduler/kube-scheduler.manifest" "${dst_dir}"
+  local gci_dst_dir="${release_stage}/gci-trusty"
-  cp "${salt_dir}/kube-apiserver/kube-apiserver.manifest" "${dst_dir}"
+  mkdir -p "${gci_dst_dir}"
-  cp "${salt_dir}/kube-apiserver/abac-authz-policy.jsonl" "${dst_dir}"
+  cp "${salt_dir}/cluster-autoscaler/cluster-autoscaler.manifest" "${gci_dst_dir}/"
-  cp "${salt_dir}/kube-controller-manager/kube-controller-manager.manifest" "${dst_dir}"
+  cp "${salt_dir}/etcd/etcd.manifest" "${gci_dst_dir}"
-  cp "${salt_dir}/kube-addons/kube-addon-manager.yaml" "${dst_dir}"
+  cp "${salt_dir}/kube-scheduler/kube-scheduler.manifest" "${gci_dst_dir}"
-  cp "${salt_dir}/l7-gcp/glbc.manifest" "${dst_dir}"
+  cp "${salt_dir}/kube-apiserver/kube-apiserver.manifest" "${gci_dst_dir}"
-  cp "${salt_dir}/rescheduler/rescheduler.manifest" "${dst_dir}/"
+  cp "${salt_dir}/kube-apiserver/abac-authz-policy.jsonl" "${gci_dst_dir}"
-  cp "${salt_dir}/e2e-image-puller/e2e-image-puller.manifest" "${dst_dir}/"
+  cp "${salt_dir}/kube-controller-manager/kube-controller-manager.manifest" "${gci_dst_dir}"
-  cp "${KUBE_ROOT}/cluster/gce/trusty/configure-helper.sh" "${dst_dir}/trusty-configure-helper.sh"
+  cp "${salt_dir}/kube-addons/kube-addon-manager.yaml" "${gci_dst_dir}"
-  cp "${KUBE_ROOT}/cluster/gce/gci/configure-helper.sh" "${dst_dir}/gci-configure-helper.sh"
+  cp "${salt_dir}/l7-gcp/glbc.manifest" "${gci_dst_dir}"
-  cp "${KUBE_ROOT}/cluster/gce/gci/mounter/mounter" "${dst_dir}/gci-mounter"
+  cp "${salt_dir}/rescheduler/rescheduler.manifest" "${gci_dst_dir}/"
-  cp "${KUBE_ROOT}/cluster/gce/gci/health-monitor.sh" "${dst_dir}/health-monitor.sh"
+  cp "${salt_dir}/e2e-image-puller/e2e-image-puller.manifest" "${gci_dst_dir}/"
-  cp "${KUBE_ROOT}/cluster/gce/container-linux/configure-helper.sh" "${dst_dir}/container-linux-configure-helper.sh"
+  cp "${KUBE_ROOT}/cluster/gce/trusty/configure-helper.sh" "${gci_dst_dir}/trusty-configure-helper.sh"
-  cp -r "${salt_dir}/kube-admission-controls/limit-range" "${dst_dir}"
+  cp "${KUBE_ROOT}/cluster/gce/gci/configure-helper.sh" "${gci_dst_dir}/gci-configure-helper.sh"
+  cp "${KUBE_ROOT}/cluster/gce/gci/mounter/mounter" "${gci_dst_dir}/gci-mounter"
+  cp "${KUBE_ROOT}/cluster/gce/gci/health-monitor.sh" "${gci_dst_dir}/health-monitor.sh"
+  cp "${KUBE_ROOT}/cluster/gce/container-linux/configure-helper.sh" "${gci_dst_dir}/container-linux-configure-helper.sh"
+  cp -r "${salt_dir}/kube-admission-controls/limit-range" "${gci_dst_dir}"
   local objects
   objects=$(cd "${KUBE_ROOT}/cluster/addons" && find . \( -name \*.yaml -or -name \*.yaml.in -or -name \*.json \) | grep -v demo)
-  tar c -C "${KUBE_ROOT}/cluster/addons" ${objects} | tar x -C "${dst_dir}"
+  tar c -C "${KUBE_ROOT}/cluster/addons" ${objects} | tar x -C "${gci_dst_dir}"
 
   kube::release::clean_cruft
 

build/release-tars/BUILD

@@ -0,0 +1,244 @@
+package(default_visibility = ["//visibility:public"])
+
+load("@bazel_tools//tools/build_defs/pkg:pkg.bzl", "pkg_tar")
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)
+
+pkg_tar(
+    name = "kubernetes-src",
+    extension = "tar.gz",
+    files = [
+        "//:all-srcs",
+    ],
+    package_dir = "kubernetes",
+    strip_prefix = "//",
+)
+
+# FIXME: this should be configurable/auto-detected
+PLATFORM_ARCH_STRING = "linux-amd64"
+
+# KUBE_CLIENT_TARGETS
+CLIENT_TARGETS = [
+    "//cmd/kubectl",
+    "//federation/cmd/kubefed",
+]
+
+# KUBE_NODE_TARGETS
+NODE_TARGETS = [
+    "//cmd/kube-proxy",
+    "//cmd/kubelet",
+]
+
+# KUBE_SERVER_TARGETS
+# No need to duplicate CLIENT_TARGETS or NODE_TARGETS here,
+# since we include them in the actual build rule.
+SERVER_TARGETS = [
+    "//cmd/hyperkube",
+    "//cmd/kube-aggregator",
+    "//cmd/kube-apiserver",
+    "//cmd/kube-controller-manager",
+    "//cmd/kube-discovery",
+    "//cmd/kubeadm",
+    "//plugin/cmd/kube-scheduler",
+]
+
+# kube::golang::test_targets
+TEST_BINARY_TARGETS = [
+    "//cmd/gendocs",
+    "//cmd/genkubedocs",
+    "//cmd/genman",
+    "//cmd/genswaggertypedocs",
+    "//cmd/genyaml",
+    "//cmd/linkcheck",
+    "//cmd/mungedocs",
+    "//examples/k8petstore/web-server/src",
+    "//federation/cmd/genfeddocs",
+    "//test/e2e:e2e.test",
+    "//vendor:github.com/onsi/ginkgo/ginkgo_bin",
+    "//cmd/kubemark",  # TODO: server platforms only
+    "//test/e2e_node:e2e_node.test",  # TODO: server platforms only
+]
+
+TEST_PORTABLE_TARGETS = [
+    "//federation/develop:all-srcs",
+    "//hack:e2e.go",
+    "//hack:federated-ginkgo-e2e.sh",
+    "//hack:get-build.sh",
+    "//hack:ginkgo-e2e.sh",
+    "//hack/e2e-internal:all-srcs",
+    "//hack/lib:all-srcs",
+    "//test/e2e/testing-manifests:all-srcs",
+    "//test/kubemark:all-srcs",
+]
+
+# Included in node and server tarballs.
+LICENSE_TARGETS = [
+    "//:Godeps/LICENSES",
+    ":kubernetes-src.tar.gz",
+]
+
+pkg_tar(
+    name = "_client-bin",
+    files = CLIENT_TARGETS,
+    mode = "0755",
+    package_dir = "client/bin",
+    visibility = ["//visibility:private"],
+)
+
+pkg_tar(
+    name = "kubernetes-client-%s" % PLATFORM_ARCH_STRING,
+    extension = "tar.gz",
+    package_dir = "kubernetes",
+    deps = [
+        ":_client-bin",
+    ],
+)
+
+pkg_tar(
+    name = "_node-bin",
+    files = NODE_TARGETS + CLIENT_TARGETS,
+    mode = "0755",
+    package_dir = "node/bin",
+    visibility = ["//visibility:private"],
+)
+
+pkg_tar(
+    name = "kubernetes-node-%s" % PLATFORM_ARCH_STRING,
+    extension = "tar.gz",
+    files = LICENSE_TARGETS,
+    mode = "0644",
+    package_dir = "kubernetes",
+    deps = [
+        ":_node-bin",
+    ],
+)
+
+pkg_tar(
+    name = "_server-bin",
+    files = SERVER_TARGETS + NODE_TARGETS + CLIENT_TARGETS + [
+        "//build:docker-artifacts",
+    ],
+    mode = "0755",
+    package_dir = "server/bin",
+    visibility = ["//visibility:private"],
+)
+
+genrule(
+    name = "dummy",
+    outs = [".dummy"],
+    cmd = "touch $@",
+)
+
+# Some of the startup scripts fail if there isn't an addons/ directory in the server tarball.
+pkg_tar(
+    name = "_server-addons",
+    files = [
+        ":.dummy",
+    ],
+    package_dir = "addons",
+    visibility = ["//visibility:private"],
+)
+
+pkg_tar(
+    name = "kubernetes-server-%s" % PLATFORM_ARCH_STRING,
+    extension = "tar.gz",
+    files = LICENSE_TARGETS,
+    mode = "0644",
+    package_dir = "kubernetes",
+    deps = [
+        ":_server-addons",
+        ":_server-bin",
+    ],
+)
+
+pkg_tar(
+    name = "_test-bin",
+    files = TEST_BINARY_TARGETS,
+    mode = "0755",
+    package_dir = "platforms/" + PLATFORM_ARCH_STRING.replace("-", "/"),
+    # TODO: how to make this multiplatform?
+    visibility = ["//visibility:private"],
+)
+
+pkg_tar(
+    name = "kubernetes-test",
+    extension = "tar.gz",
+    files = TEST_PORTABLE_TARGETS,
+    package_dir = "kubernetes",
+    strip_prefix = "//",
+    deps = [
+        # TODO: how to make this multiplatform?
+        ":_test-bin",
+    ],
+)
+
+pkg_tar(
+    name = "_full_server",
+    files = [
+        ":kubernetes-manifests.tar.gz",
+        ":kubernetes-salt.tar.gz",
+    ],
+    package_dir = "server",
+    visibility = ["//visibility:private"],
+)
+
+pkg_tar(
+    name = "kubernetes",
+    extension = "tar.gz",
+    files = [
+        "//:Godeps/LICENSES",
+        "//:README.md",
+        "//:Vagrantfile",
+        "//cluster:all-srcs",
+        "//docs:all-srcs",
+        "//examples:all-srcs",
+        "//third_party/htpasswd:all-srcs",
+    ],
+    package_dir = "kubernetes",
+    strip_prefix = "//",
+    deps = [
+        ":_full_server",
+        "//federation:release",
+    ],
+)
+
+pkg_tar(
+    name = "kubernetes-manifests",
+    extension = "tar.gz",
+    deps = [
+        "//cluster:manifests",
+    ],
+)
+
+pkg_tar(
+    name = "kubernetes-salt",
+    extension = "tar.gz",
+    deps = [
+        "//cluster/saltbase:salt",
+    ],
+)
+
+filegroup(
+    name = "release-tars",
+    srcs = [
+        ":kubernetes.tar.gz",
+        ":kubernetes-client-%s.tar.gz" % PLATFORM_ARCH_STRING,
+        ":kubernetes-node-%s.tar.gz" % PLATFORM_ARCH_STRING,
+        ":kubernetes-server-%s.tar.gz" % PLATFORM_ARCH_STRING,
+        ":kubernetes-manifests.tar.gz",
+        ":kubernetes-salt.tar.gz",
+        ":kubernetes-src.tar.gz",
+        ":kubernetes-test.tar.gz",
+    ],
+)

cluster/BUILD

@@ -1,5 +1,7 @@
 package(default_visibility = ["//visibility:public"])
 
+load("@bazel_tools//tools/build_defs/pkg:pkg.bzl", "pkg_tar")
+
 licenses(["notice"])
 
 filegroup(
@@ -14,6 +16,31 @@ filegroup(
     srcs = [
         ":package-srcs",
         "//cluster/addons:all-srcs",
+        "//cluster/gce:all-srcs",
+        "//cluster/saltbase:all-srcs",
     ],
     tags = ["automanaged"],
 )
+
+# All of the manifests that are expected to be in a "gci-trusty"
+# subdir of the manifests tarball.
+pkg_tar(
+    name = "_manifests-gci-trusty",
+    package_dir = "gci-trusty",
+    visibility = ["//visibility:private"],
+    deps = [
+        "//cluster/addons",
+        "//cluster/gce:gci-trusty-manifests",
+        "//cluster/saltbase:gci-trusty-salt-manifests",
+    ],
+)
+
+pkg_tar(
+    name = "manifests",
+    mode = "0644",
+    package_dir = "kubernetes",
+    deps = [
+        ":_manifests-gci-trusty",
+        "//cluster/saltbase:salt-manifests",
+    ],
+)

cluster/addons/BUILD

@@ -4,21 +4,14 @@ load("@bazel_tools//tools/build_defs/pkg:pkg.bzl", "pkg_tar")
 
 filegroup(
     name = "addon-srcs",
-    srcs = glob([
+    srcs = glob(
-        "calico-policy-controller/*",
+        [
-        "cluster-loadbalancing/*",
+            "**/*.json",
-        "cluster-monitoring/*",
+            "**/*.yaml",
-        "dashboard/*",
+            "**/*.yaml.in",
-        "dns/*",
+        ],
-        "etcd-empty-dir-cleanup/*",
+        exclude = ["**/*demo*/**"],
-        "fluentd-elasticsearch/*",
+    ),
-        "fluentd-gcp/*",
-        "gci/*",
-        "node-problem-detector/*",
-        "podsecuritypolicies/*",
-        "python-image/*",
-        "registry/*",
-    ]),
 )
 
 pkg_tar(
@@ -27,6 +20,7 @@ pkg_tar(
     files = [
         ":addon-srcs",
     ],
+    mode = "0644",
     strip_prefix = ".",
 )
 

cluster/common.sh

@@ -412,6 +412,30 @@ function tars_from_version() {
   fi
 }
 
+# Search for the specified tarball in the various known output locations,
+# echoing the location if found.
+#
+# Assumed vars:
+#   KUBE_ROOT
+#
+# Args:
+#   $1 name of tarball to search for
+function find-tar() {
+  local -r tarball=$1
+  locations=(
+    "${KUBE_ROOT}/server/${tarball}"
+    "${KUBE_ROOT}/_output/release-tars/${tarball}"
+    "${KUBE_ROOT}/bazel-bin/build/release-tars/${tarball}"
+  )
+  location=$( (ls -t "${locations[@]}" 2>/dev/null || true) | head -1 )
+
+  if [[ ! -f "${location}" ]]; then
+    echo "!!! Cannot find ${tarball}" >&2
+    exit 1
+  fi
+  echo "${location}"
+}
+
 # Verify and find the various tar files that we are going to use on the server.
 #
 # Assumed vars:
@@ -421,36 +445,14 @@ function tars_from_version() {
 #   SALT_TAR
 #   KUBE_MANIFESTS_TAR
 function find-release-tars() {
-  SERVER_BINARY_TAR="${KUBE_ROOT}/server/kubernetes-server-linux-amd64.tar.gz"
+  SERVER_BINARY_TAR=$(find-tar kubernetes-server-linux-amd64.tar.gz)
-  if [[ ! -f "${SERVER_BINARY_TAR}" ]]; then
+  SALT_TAR=$(find-tar kubernetes-salt.tar.gz)
-    SERVER_BINARY_TAR="${KUBE_ROOT}/_output/release-tars/kubernetes-server-linux-amd64.tar.gz"
-  fi
-  if [[ ! -f "${SERVER_BINARY_TAR}" ]]; then
-    echo "!!! Cannot find kubernetes-server-linux-amd64.tar.gz" >&2
-    exit 1
-  fi
-
-  SALT_TAR="${KUBE_ROOT}/server/kubernetes-salt.tar.gz"
-  if [[ ! -f "${SALT_TAR}" ]]; then
-    SALT_TAR="${KUBE_ROOT}/_output/release-tars/kubernetes-salt.tar.gz"
-  fi
-  if [[ ! -f "${SALT_TAR}" ]]; then
-    echo "!!! Cannot find kubernetes-salt.tar.gz" >&2
-    exit 1
-  fi
 
   # This tarball is used by GCI, Ubuntu Trusty, and Container Linux.
   KUBE_MANIFESTS_TAR=
   if [[ "${MASTER_OS_DISTRIBUTION:-}" == "trusty" || "${MASTER_OS_DISTRIBUTION:-}" == "gci" || "${MASTER_OS_DISTRIBUTION:-}" == "container-linux" ]] || \
      [[ "${NODE_OS_DISTRIBUTION:-}" == "trusty" || "${NODE_OS_DISTRIBUTION:-}" == "gci" || "${NODE_OS_DISTRIBUTION:-}" == "container-linux" ]] ; then
-    KUBE_MANIFESTS_TAR="${KUBE_ROOT}/server/kubernetes-manifests.tar.gz"
+    KUBE_MANIFESTS_TAR=$(find-tar kubernetes-manifests.tar.gz)
-    if [[ ! -f "${KUBE_MANIFESTS_TAR}" ]]; then
-      KUBE_MANIFESTS_TAR="${KUBE_ROOT}/_output/release-tars/kubernetes-manifests.tar.gz"
-    fi
-    if [[ ! -f "${KUBE_MANIFESTS_TAR}" ]]; then
-      echo "!!! Cannot find kubernetes-manifests.tar.gz" >&2
-      exit 1
-    fi
   fi
 }
 

cluster/gce/BUILD

@@ -0,0 +1,37 @@
+package(default_visibility = ["//visibility:public"])
+
+load("@bazel_tools//tools/build_defs/pkg:pkg.bzl", "pkg_tar")
+
+pkg_tar(
+    name = "gci-trusty-manifests",
+    files = [
+        "container-linux/configure-helper.sh",
+        "gci/configure-helper.sh",
+        "gci/health-monitor.sh",
+        "gci/mounter/mounter",
+        "trusty/configure-helper.sh",
+    ],
+    mode = "0755",
+    strip_prefix = ".",
+    # pkg_tar doesn't support renaming the files we add, so instead create symlinks.
+    symlinks = {
+        "container-linux-configure-helper.sh": "container-linux/configure-helper.sh",
+        "gci-configure-helper.sh": "gci/configure-helper.sh",
+        "health-monitor.sh": "gci/health-monitor.sh",
+        "gci-mounter": "gci/mounter/mounter",
+        "trusty-configure-helper.sh": "trusty/configure-helper.sh",
+    },
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

cluster/kubeadm.sh

@@ -76,6 +76,7 @@ if [[ -z "${KUBEADM_PATH:-}" ]]; then
     "${KUBE_ROOT}/_output/bin/kubeadm"
     "${KUBE_ROOT}/_output/dockerized/bin/${host_os}/${host_arch}/kubeadm"
     "${KUBE_ROOT}/_output/local/bin/${host_os}/${host_arch}/kubeadm"
+    "${KUBE_ROOT}/bazel-bin/cmd/kubectl/kubeadm"
     "${KUBE_ROOT}/platforms/${host_os}/${host_arch}/kubeadm"
   )
   kubeadm=$( (ls -t "${locations[@]}" 2>/dev/null || true) | head -1 )

cluster/kubectl.sh

@@ -88,6 +88,7 @@ if [[ -z "${KUBECTL_PATH:-}" ]]; then
     "${KUBE_ROOT}/_output/bin/kubectl"
     "${KUBE_ROOT}/_output/dockerized/bin/${host_os}/${host_arch}/kubectl"
     "${KUBE_ROOT}/_output/local/bin/${host_os}/${host_arch}/kubectl"
+    "${KUBE_ROOT}/bazel-bin/cmd/kubectl/kubectl"
     "${KUBE_ROOT}/platforms/${host_os}/${host_arch}/kubectl"
   )
   kubectl=$( (ls -t "${locations[@]}" 2>/dev/null || true) | head -1 )

cluster/saltbase/BUILD

@@ -0,0 +1,87 @@
+package(default_visibility = ["//visibility:public"])
+
+load("@bazel_tools//tools/build_defs/pkg:pkg.bzl", "pkg_tar")
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)
+
+# TODO(#3579): This is a temporary hack. It gathers up the yaml,
+# yaml.in, json files in cluster/addons (minus any demos) and overlays
+# them into kube-addons, where we expect them.
+# These files are expected in a salt/kube-addons subdirectory.
+pkg_tar(
+    name = "_salt_kube-addons",
+    package_dir = "salt/kube-addons",
+    strip_prefix = "/cluster/addons",
+    visibility = ["//visibility:private"],
+    deps = [
+        "//cluster/addons",
+    ],
+)
+
+pkg_tar(
+    name = "salt",
+    files = glob(
+        ["**"],
+        exclude = ["BUILD"],
+    ),
+    mode = "0644",
+    modes = {
+        "install.sh": "0755",
+    },
+    package_dir = "kubernetes/saltbase",
+    strip_prefix = ".",
+    deps = [
+        ":_salt_kube-addons",
+    ],
+)
+
+# The following are used in the kubernetes salt tarball.
+pkg_tar(
+    name = "salt-manifests",
+    files = [
+        "salt/fluentd-gcp/fluentd-gcp.yaml",
+        "salt/kube-proxy/kube-proxy.manifest",
+        "salt/kube-registry-proxy/kube-registry-proxy.yaml",
+    ],
+    mode = "0644",
+)
+
+pkg_tar(
+    name = "_kube-admission-controls",
+    files = glob(["salt/kube-admission-controls/limit-range/**"]),
+    mode = "0644",
+    # Maintain limit-range/ subdirectory in tarball
+    strip_prefix = "./salt/kube-admission-controls/",
+    visibility = ["//visibility:private"],
+)
+
+pkg_tar(
+    name = "gci-trusty-salt-manifests",
+    files = [
+        "salt/cluster-autoscaler/cluster-autoscaler.manifest",
+        "salt/e2e-image-puller/e2e-image-puller.manifest",
+        "salt/etcd/etcd.manifest",
+        "salt/kube-addons/kube-addon-manager.yaml",
+        "salt/kube-apiserver/abac-authz-policy.jsonl",
+        "salt/kube-apiserver/kube-apiserver.manifest",
+        "salt/kube-controller-manager/kube-controller-manager.manifest",
+        "salt/kube-scheduler/kube-scheduler.manifest",
+        "salt/l7-gcp/glbc.manifest",
+        "salt/rescheduler/rescheduler.manifest",
+    ],
+    mode = "0644",
+    deps = [
+        "_kube-admission-controls",
+    ],
+)

federation/BUILD

@@ -2,6 +2,8 @@ package(default_visibility = ["//visibility:public"])
 
 licenses(["notice"])
 
+load("@bazel_tools//tools/build_defs/pkg:pkg.bzl", "pkg_tar")
+
 filegroup(
     name = "package-srcs",
     srcs = glob(["**"]),
@@ -23,6 +25,7 @@ filegroup(
         "//federation/cmd/federation-controller-manager:all-srcs",
         "//federation/cmd/genfeddocs:all-srcs",
         "//federation/cmd/kubefed:all-srcs",
+        "//federation/develop:all-srcs",
         "//federation/pkg/dnsprovider:all-srcs",
         "//federation/pkg/federation-controller:all-srcs",
         "//federation/pkg/kubefed:all-srcs",
@@ -30,3 +33,12 @@ filegroup(
     ],
     tags = ["automanaged"],
 )
+
+pkg_tar(
+    name = "release",
+    files = glob([
+        "deploy/**",
+        "manifests/**",
+    ]) + ["//federation/cluster:all-srcs"],
+    package_dir = "federation",
+)

federation/develop/BUILD

@@ -0,0 +1,16 @@
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

hack/BUILD

@@ -14,6 +14,8 @@ filegroup(
     srcs = [
         ":package-srcs",
         "//hack/boilerplate:all-srcs",
+        "//hack/e2e-internal:all-srcs",
+        "//hack/lib:all-srcs",
     ],
     tags = ["automanaged"],
 )

hack/e2e-internal/BUILD

@@ -0,0 +1,16 @@
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

hack/lib/BUILD

@@ -0,0 +1,16 @@
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

hack/lib/golang.sh

@@ -21,6 +21,7 @@ readonly KUBE_GOPATH="${KUBE_OUTPUT}/go"
 # The set of server targets that we are only building for Linux
 # Note: if you are adding something here, you might need to add it to
 # kube::build::source_targets in build/common.sh as well.
+# If you update this list, please also update build/release-tars/BUILD.
 kube::golang::server_targets() {
   local targets=(
     cmd/kube-proxy
@@ -40,6 +41,7 @@ readonly KUBE_SERVER_TARGETS=($(kube::golang::server_targets))
 readonly KUBE_SERVER_BINARIES=("${KUBE_SERVER_TARGETS[@]##*/}")
 
 # The set of server targets that we are only building for Kubernetes nodes 
+# If you update this list, please also update build/release-tars/BUILD.
 kube::golang::node_targets() {
   local targets=(
     cmd/kube-proxy
@@ -122,6 +124,7 @@ else
 fi
 
 # The set of client targets that we are building for all platforms
+# If you update this list, please also update build/release-tars/BUILD.
 readonly KUBE_CLIENT_TARGETS=(
   cmd/kubectl
   federation/cmd/kubefed
@@ -130,6 +133,7 @@ readonly KUBE_CLIENT_BINARIES=("${KUBE_CLIENT_TARGETS[@]##*/}")
 readonly KUBE_CLIENT_BINARIES_WIN=("${KUBE_CLIENT_BINARIES[@]/%/.exe}")
 
 # The set of test targets that we are building for all platforms
+# If you update this list, please also update build/release-tars/BUILD.
 kube::golang::test_targets() {
   local targets=(
     cmd/gendocs
@@ -149,6 +153,7 @@ kube::golang::test_targets() {
 readonly KUBE_TEST_TARGETS=($(kube::golang::test_targets))
 readonly KUBE_TEST_BINARIES=("${KUBE_TEST_TARGETS[@]##*/}")
 readonly KUBE_TEST_BINARIES_WIN=("${KUBE_TEST_BINARIES[@]/%/.exe}")
+# If you update this list, please also update build/release-tars/BUILD.
 readonly KUBE_TEST_PORTABLE=(
   test/e2e/testing-manifests
   test/kubemark
@@ -164,6 +169,7 @@ readonly KUBE_TEST_PORTABLE=(
 # Test targets which run on the Kubernetes clusters directly, so we only
 # need to target server platforms.
 # These binaries will be distributed in the kubernetes-test tarball.
+# If you update this list, please also update build/release-tars/BUILD.
 readonly KUBE_TEST_SERVER_TARGETS=(
   cmd/kubemark
   vendor/github.com/onsi/ginkgo/ginkgo

hack/lib/util.sh

@@ -167,12 +167,19 @@ kube::util::host_platform() {
 kube::util::find-binary-for-platform() {
   local -r lookfor="$1"
   local -r platform="$2"
-  local -r locations=(
+  local locations=(
     "${KUBE_ROOT}/_output/bin/${lookfor}"
     "${KUBE_ROOT}/_output/dockerized/bin/${platform}/${lookfor}"
     "${KUBE_ROOT}/_output/local/bin/${platform}/${lookfor}"
     "${KUBE_ROOT}/platforms/${platform}/${lookfor}"
   )
+  # Also search for binary in bazel build tree.
+  # In some cases we have to name the binary $BINARY_bin, since there was a
+  # directory named $BINARY next to it.
+  locations+=($(find "${KUBE_ROOT}/bazel-bin/" -type f -executable \
+    \( -name "${lookfor}" -o -name "${lookfor}_bin" \) 2>/dev/null || true) )
+
+  # List most recently-updated location.
   local -r bin=$( (ls -t "${locations[@]}" 2>/dev/null || true) | head -1 )
   echo -n "${bin}"
 }

test/BUILD

@@ -18,6 +18,7 @@ filegroup(
         "//test/fixtures:all-srcs",
         "//test/images:all-srcs",
         "//test/integration:all-srcs",
+        "//test/kubemark:all-srcs",
         "//test/list:all-srcs",
         "//test/soak/cauldron:all-srcs",
         "//test/soak/serve_hostnames:all-srcs",

test/e2e_node/BUILD

@@ -71,7 +71,10 @@ go_test(
         "volume_manager_test.go",
     ],
     library = ":go_default_library",
-    tags = ["automanaged"],
+    tags = [
+        "automanaged",
+        "integration",
+    ],
     deps = [
         "//pkg/api/resource:go_default_library",
         "//pkg/api/v1:go_default_library",
@@ -114,6 +117,15 @@ go_test(
     ],
 )
 
+genrule(
+    name = "gen_e2e_node.test",
+    testonly = 1,
+    srcs = [":go_default_test"],
+    outs = ["e2e_node.test"],
+    cmd = "srcs=($(SRCS)); cp $$(dirname $${srcs[0]})/go_default_test $@;",
+    output_to_bindir = 1,
+)
+
 filegroup(
     name = "package-srcs",
     srcs = glob(["**"]),

test/kubemark/BUILD

@@ -0,0 +1,16 @@
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

third_party/BUILD

@@ -20,6 +20,7 @@ filegroup(
         "//third_party/forked/golang/netutil:all-srcs",
         "//third_party/forked/golang/reflect:all-srcs",
         "//third_party/forked/golang/template:all-srcs",
+        "//third_party/htpasswd:all-srcs",
     ],
     tags = ["automanaged"],
 )

third_party/htpasswd/BUILD

@@ -0,0 +1,16 @@
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)