Merge pull request #62495 from rmmh/bzl-cache

Automatic merge from submit-queue (batch tested with PRs 62495, 63003, 62829, 62151, 62002). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Add a GCS mirror to WORKSPACE URLs, along with a script to update them.

**What this PR does / why we need it**:
We get occasional flakes when WORKSPACE downloads fail. This should make that much more unlikely, by having two independent endpoints to try.

**Release note**:
```release-note
NONE
```

build/root/WORKSPACE

@@ -1,21 +1,23 @@
+load("//build:workspace_mirror.bzl", "mirror")
+
 http_archive(
     name = "io_bazel_rules_go",
     sha256 = "4b2c61795ac2eefcb28f3eb8e1cb2d8fb3c2eafa0f6712473bc5f93728f38758",
-    urls = ["https://github.com/bazelbuild/rules_go/releases/download/0.10.2/rules_go-0.10.2.tar.gz"],
+    urls = mirror("https://github.com/bazelbuild/rules_go/releases/download/0.10.2/rules_go-0.10.2.tar.gz"),
 )
 
 http_archive(
     name = "io_kubernetes_build",
     sha256 = "007774f06536059f3f782d1a092bddc625d88c17f20bbe731cea844a52485b11",
     strip_prefix = "repo-infra-97099dccc8807e9159dc28f374a8f0602cab07e1",
-    urls = ["https://github.com/kubernetes/repo-infra/archive/97099dccc8807e9159dc28f374a8f0602cab07e1.tar.gz"],
+    urls = mirror("https://github.com/kubernetes/repo-infra/archive/97099dccc8807e9159dc28f374a8f0602cab07e1.tar.gz"),
 )
 
 http_archive(
     name = "bazel_skylib",
     sha256 = "bbccf674aa441c266df9894182d80de104cabd19be98be002f6d478aaa31574d",
     strip_prefix = "bazel-skylib-2169ae1c374aab4a09aa90e65efe1a3aad4e279b",
-    urls = ["https://github.com/bazelbuild/bazel-skylib/archive/2169ae1c374aab4a09aa90e65efe1a3aad4e279b.tar.gz"],
+    urls = mirror("https://github.com/bazelbuild/bazel-skylib/archive/2169ae1c374aab4a09aa90e65efe1a3aad4e279b.tar.gz"),
 )
 
 ETCD_VERSION = "3.2.18"
@@ -25,14 +27,14 @@ new_http_archive(
     build_file = "third_party/etcd.BUILD",
     sha256 = "b729db0732448064271ea6fdcb901773c4fe917763ca07776f22d0e5e0bd4097",
     strip_prefix = "etcd-v%s-linux-amd64" % ETCD_VERSION,
-    urls = ["https://github.com/coreos/etcd/releases/download/v%s/etcd-v%s-linux-amd64.tar.gz" % (ETCD_VERSION, ETCD_VERSION)],
+    urls = mirror("https://github.com/coreos/etcd/releases/download/v%s/etcd-v%s-linux-amd64.tar.gz" % (ETCD_VERSION, ETCD_VERSION)),
 )
 
 http_archive(
     name = "io_bazel_rules_docker",
     sha256 = "c440717ee9b1b2f4a1e9bf5622539feb5aef9db83fc1fa1517818f13c041b0be",
     strip_prefix = "rules_docker-8bbe2a8abd382641e65ff7127a3700a8530f02ce",
-    urls = ["https://github.com/bazelbuild/rules_docker/archive/8bbe2a8abd382641e65ff7127a3700a8530f02ce.tar.gz"],
+    urls = mirror("https://github.com/bazelbuild/rules_docker/archive/8bbe2a8abd382641e65ff7127a3700a8530f02ce.tar.gz"),
 )
 
 load("@bazel_skylib//:lib.bzl", "versions")
@@ -53,7 +55,7 @@ docker_repositories()
 http_file(
     name = "kubernetes_cni",
     sha256 = "f04339a21b8edf76d415e7f17b620e63b8f37a76b2f706671587ab6464411f2d",
-    url = "https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.6.0.tgz",
+    urls = mirror("https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.6.0.tgz"),
 )
 
 docker_pull(
@@ -79,3 +81,6 @@ docker_pull(
     repository = "library/busybox",
     tag = "latest",  # ignored, but kept here for documentation
 )
+
+load("//build:workspace_mirror.bzl", "export_urls")
+export_urls("workspace_urls")

build/workspace_mirror.bzl

@@ -0,0 +1,53 @@
+# Copyright 2018 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+prefix = "https://storage.googleapis.com/k8s-bazel-cache/"
+
+def mirror(url):
+  """Try downloading a URL from a GCS mirror first, then from the original.
+
+  Update the GCS bucket using bazel run //hack:update-mirror"""
+  return [prefix + url, url]
+
+# This function only gives proper results when executed from WORKSPACE,
+# but the data is needed in sh_binary, which can only be in a BUILD file.
+# Thus, it is be exported by a repository_rule (which executes in WORKSPACE)
+# to be used by the sh_binary.
+def mirror_urls():
+  urls = []
+  for k, v in native.existing_rules().items():
+    us = list(v.get('urls', []))
+    if 'url' in v:
+      us.append(v['url'])
+    for u in us:
+      if u and not u.startswith(prefix):
+        urls.append(u)
+  return sorted(urls)
+
+def export_urls_impl(repo_ctx):
+  repo_ctx.file(repo_ctx.path("BUILD.bazel"), """
+exports_files(glob(["**"]), visibility=["//visibility:public"])
+""")
+  repo_ctx.file(repo_ctx.path("urls.txt"), content="\n".join(repo_ctx.attr.urls))
+
+_export_urls = repository_rule(
+    attrs = {
+        "urls": attr.string_list(mandatory = True),
+    },
+    local = True,
+    implementation = export_urls_impl,
+)
+
+def export_urls(name):
+  return _export_urls(name=name, urls=mirror_urls())

hack/BUILD

@@ -72,3 +72,10 @@ go_library(
     srcs = ["e2e.go"],
     importpath = "k8s.io/kubernetes/hack",
 )
+
+sh_binary(
+    name = "update-mirror",
+    srcs = ["update-workspace-mirror.sh"],
+    args = ["$(location @workspace_urls//:urls.txt)"],
+    data = ["@workspace_urls//:urls.txt"],
+)

hack/update-workspace-mirror.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+# Copyright 2018 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+if [[ $# -ne 1 ]]; then
+    echo 'use "bazel run //build:update-mirror"'
+    echo "(usage: $0 <file with list of URLs to mirror>)"
+    exit 1
+fi
+
+BUCKET="gs://k8s-bazel-cache"
+
+gsutil acl get "${BUCKET}" > /dev/null
+
+tmpfile=$(mktemp bazel_workspace_mirror.XXXXXX)
+trap "rm ${tmpfile}" EXIT
+cat "$1" | while read url; do
+  echo "${url}"
+  if gsutil ls "${BUCKET}/${url}" &> /dev/null; then
+    echo present
+  else
+    echo missing
+    if curl -fLag "${url}" > "${tmpfile}"; then
+        gsutil cp -a public-read "${tmpfile}" "${BUCKET}/${url}"
+    fi
+  fi
+done