Merge pull request #41106 from spxtr/gen3

Automatic merge from submit-queue

Don't check in zz_generated.openapi.go.

`zz_generated.openapi.go` is the file that causes the most merge conflicts of all. In #33440, @thockin updated the makefile to support generating these files on demand, but that didn't play well with bazel/gazel.

In this PR, I add a new build macro that will generate this file with a `go_genrule`. I added support for keeping the BUILD file up to date in mikedanese/gazel#34.

**Release note**:
```release-note
NONE
```

.gazelcfg.json

@@ -15,5 +15,6 @@
 		"_vendor"
 	],
 	"AddSourcesRules": true,
-	"VendorMultipleBuildFiles": true
+	"VendorMultipleBuildFiles": true,
+	"K8sOpenAPIGen": true
 }

.gitignore

@@ -110,6 +110,7 @@ kubernetes.tar.gz
 # generated files in any directory
 # TODO(thockin): uncomment this when we stop committing the generated files.
 #zz_generated.*
+zz_generated.openapi.go
 
 # make-related metadata
 /.make/

Makefile

@@ -287,6 +287,8 @@ clean: clean_meta
 	build/make-clean.sh
 	rm -rf $(OUT_DIR)
 	rm -rf Godeps/_workspace # Just until we are sure it is gone
+	# TODO(thockin): Remove this when we call clean_generated.
+	rm -f pkg/generated/openapi/zz_generated.openapi.go
 endif
 
 define CLEAN_META_HELP_INFO

Makefile.generated_files

@@ -40,8 +40,7 @@ generated_files: gen_deepcopy gen_defaulter gen_conversion gen_openapi
 .PHONY: verify_generated_files
 verify_generated_files: verify_gen_deepcopy    \
                         verify_gen_defaulter   \
-                        verify_gen_conversion  \
-                        verify_gen_openapi
+                        verify_gen_conversion
 
 # Code-generation logic.
 #
@@ -480,27 +479,10 @@ OPENAPI_DIRS := $(shell                                             \
 
 OPENAPI_OUTFILE := $(OPENAPI_OUTPUT_PKG)/$(OPENAPI_FILENAME)
 
-# Shell function for reuse in rules.
-RUN_GEN_OPENAPI =                                                                       \
-    function run_gen_openapi() {                                                        \
-        ./hack/run-in-gopath.sh $(OPENAPI_GEN)                                          \
-            --v $(KUBE_VERBOSE)                                                         \
-            --logtostderr                                                               \
-            -i $$(echo $(addprefix $(PRJ_SRC_PATH)/, $(OPENAPI_DIRS)) | sed 's/ /,/g')  \
-            -p $(PRJ_SRC_PATH)/$(OPENAPI_OUTPUT_PKG)                                    \
-            -O $(OPENAPI_BASENAME)                                                      \
-            "$$@";                                                                      \
-    };                                                                                  \
-    run_gen_openapi
-
 # This rule is the user-friendly entrypoint for openapi generation.
 .PHONY: gen_openapi
 gen_openapi: $(OPENAPI_OUTFILE) $(OPENAPI_GEN)
 
-.PHONY: verify_gen_openapi
-verify_gen_openapi: $(OPENAPI_GEN)
-	$(RUN_GEN_OPENAPI) --verify-only
-
 # For each dir in OPENAPI_DIRS, this establishes a dependency between the
 # output file and the input files that should trigger a rebuild.
 #
@@ -520,7 +502,16 @@ $(foreach dir, $(OPENAPI_DIRS), $(eval                                     \
 
 # How to regenerate open-api code.  This emits a single file for all results.
 $(OPENAPI_OUTFILE): $(OPENAPI_GEN) $(OPENAPI_GEN)
-	$(RUN_GEN_OPENAPI)
+	function run_gen_openapi() {                                                        \
+        ./hack/run-in-gopath.sh $(OPENAPI_GEN)                                          \
+            --v $(KUBE_VERBOSE)                                                         \
+            --logtostderr                                                               \
+            -i $$(echo $(addprefix $(PRJ_SRC_PATH)/, $(OPENAPI_DIRS)) | sed 's/ /,/g')  \
+            -p $(PRJ_SRC_PATH)/$(OPENAPI_OUTPUT_PKG)                                    \
+            -O $(OPENAPI_BASENAME)                                                      \
+            "$$@";                                                                      \
+    };                                                                                  \
+    run_gen_openapi
 
 # This calculates the dependencies for the generator tool, so we only rebuild
 # it when needed.  It is PHONY so that it always runs, but it only updates the

hack/update-bazel.sh

@@ -20,7 +20,11 @@ set -o pipefail
 export KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
 source "${KUBE_ROOT}/hack/lib/init.sh"
 
-go get gopkg.in/mikedanese/gazel.v16/gazel
+# Remove generated files prior to running gazel.
+# TODO(spxtr): Remove this line once Bazel is the only way to build.
+rm -f "${KUBE_ROOT}/pkg/generated/openapi/zz_generated.openapi.go"
+
+go get -u gopkg.in/mikedanese/gazel.v16/gazel
 
 for path in ${GOPATH//:/ }; do
   if [[ -e "${path}/bin/gazel" ]]; then

hack/verify-bazel.sh

@@ -22,6 +22,10 @@ source "${KUBE_ROOT}/hack/lib/init.sh"
 
 go get gopkg.in/mikedanese/gazel.v16/gazel
 
+# Remove generated files prior to running gazel.
+# TODO(spxtr): Remove this line once Bazel is the only way to build.
+rm -f "${KUBE_ROOT}/pkg/generated/openapi/zz_generated.openapi.go"
+
 for path in ${GOPATH//:/ }; do
   if [[ -e "${path}/bin/gazel" ]]; then
     gazel="${path}/bin/gazel"
@@ -33,6 +37,7 @@ if [[ -z "${gazel:-}" ]]; then
   exit 1
 fi
 
+
 if ! "${gazel}" -validate -print-diff -root="$(kube::realpath ${KUBE_ROOT})" ; then
   echo
   echo "Run ./hack/update-bazel.sh"

pkg/generated/openapi/BUILD

@@ -1,25 +1,50 @@
 package(default_visibility = ["//visibility:public"])
 
+load("//pkg/generated/openapi:def.bzl", "openapi_library")
+
 licenses(["notice"])
 
-load(
-    "@io_bazel_rules_go//go:def.bzl",
-    "go_library",
-)
-
-go_library(
+openapi_library(
     name = "go_default_library",
-    srcs = [
-        "doc.go",
-        "zz_generated.openapi.go",
+    srcs = ["doc.go"],
+    openapi_targets = [
+        "cmd/libs/go2idl/client-gen/test_apis/testgroup/v1",
+        "federation/apis/federation/v1beta1",
+        "pkg/api/v1",
+        "pkg/apis/abac/v0",
+        "pkg/apis/abac/v1beta1",
+        "pkg/apis/apps/v1beta1",
+        "pkg/apis/authentication/v1",
+        "pkg/apis/authentication/v1beta1",
+        "pkg/apis/authorization/v1",
+        "pkg/apis/authorization/v1beta1",
+        "pkg/apis/autoscaling/v1",
+        "pkg/apis/autoscaling/v2alpha1",
+        "pkg/apis/batch/v1",
+        "pkg/apis/batch/v2alpha1",
+        "pkg/apis/certificates/v1beta1",
+        "pkg/apis/componentconfig/v1alpha1",
+        "pkg/apis/extensions/v1beta1",
+        "pkg/apis/imagepolicy/v1alpha1",
+        "pkg/apis/policy/v1beta1",
+        "pkg/apis/rbac/v1alpha1",
+        "pkg/apis/rbac/v1beta1",
+        "pkg/apis/settings/v1alpha1",
+        "pkg/apis/storage/v1",
+        "pkg/apis/storage/v1beta1",
+        "pkg/version",
+        "vendor/k8s.io/metrics/pkg/apis/custom_metrics/v1alpha1",
+        "vendor/k8s.io/metrics/pkg/apis/metrics/v1alpha1",
     ],
     tags = ["automanaged"],
-    deps = [
-        "//vendor/github.com/go-openapi/spec:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/openapi:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
+    vendor_targets = [
+        "k8s.io/apimachinery/pkg/api/resource",
+        "k8s.io/apimachinery/pkg/apis/meta/v1",
+        "k8s.io/apimachinery/pkg/runtime",
+        "k8s.io/apimachinery/pkg/util/intstr",
+        "k8s.io/apimachinery/pkg/version",
+        "k8s.io/apiserver/pkg/apis/example/v1",
+        "k8s.io/client-go/pkg/api/v1",
     ],
 )
 

pkg/generated/openapi/def.bzl

@@ -0,0 +1,31 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+load("@io_kubernetes_build//defs:go.bzl", "go_genrule")
+
+def openapi_library(name, tags, srcs, openapi_targets=[], vendor_targets=[]):
+  deps = [
+      "//vendor/github.com/go-openapi/spec:go_default_library",
+      "//vendor/k8s.io/apimachinery/pkg/openapi:go_default_library",
+  ] + ["//%s:go_default_library" % target for target in openapi_targets] + ["//vendor/%s:go_default_library" % target for target in vendor_targets]
+  go_library(
+      name=name,
+      tags=tags,
+      srcs=srcs + [":zz_generated.openapi"],
+      deps=deps,
+  )
+  go_genrule(
+      name = "zz_generated.openapi",
+      srcs = srcs + ["//hack/boilerplate:boilerplate.go.txt"],
+      outs = ["zz_generated.openapi.go"],
+      cmd = " ".join([
+        "$(location //cmd/libs/go2idl/openapi-gen)",
+        "--v 1",
+        "--logtostderr",
+        "--go-header-file $(location //hack/boilerplate:boilerplate.go.txt)",
+        "--output-file-base zz_generated.openapi",
+        "--output-package k8s.io/kubernetes/pkg/generated/openapi",
+        "--input-dirs " + ",".join(["k8s.io/kubernetes/" + target for target in openapi_targets] + ["k8s.io/kubernetes/vendor/" + target for target in vendor_targets]),
+        "&& cp pkg/generated/openapi/zz_generated.openapi.go $(GENDIR)/pkg/generated/openapi",
+      ]),
+      go_deps = deps,
+      tools = ["//cmd/libs/go2idl/openapi-gen"],
+)