Fix tooling for apis/experimental's new home

* fix package name * add a script to auto-gofmt everything, useful after grep/sed incantations * update conversion/deep copy generation * doc update
2015-09-09 16:01:08 -07:00 · 2015-09-09 16:01:08 -07:00 · 7a0fc719fb
parent 4c2adabf42
commit 7a0fc719fb
13 changed files with 203 additions and 70 deletions
--- a/cmd/genconversion/conversion.go
+++ b/cmd/genconversion/conversion.go
@ -44,6 +44,24 @@ var (
 	groupVersion = flag.StringP("version", "v", "api/v1", "groupPath/version for conversion.")
 )

+// We're moving to pkg/apis/group/version. This handles new and legacy packages.
+func pkgPath(group, version string) string {
+	if group == "" {
+		group = "api"
+	}
+	gv := group
+	if version != "" {
+		gv = path.Join(group, version)
+	}
+	switch {
+	case group == "api":
+		// TODO(lavalamp): remove this special case when we move api to apis/api
+		return path.Join(pkgBase, gv)
+	default:
+		return path.Join(pkgBase, "apis", gv)
+	}
+}
+
 func main() {
 	runtime.GOMAXPROCS(runtime.NumCPU())
 	flag.Parse()
@ -70,14 +88,14 @@ func main() {
 		glog.Fatalf("error writing package line: %v", err)
 	}

-	versionPath := path.Join(pkgBase, group, version)
+	versionPath := pkgPath(group, version)
 	generator := pkg_runtime.NewConversionGenerator(api.Scheme.Raw(), versionPath)
 	apiShort := generator.AddImport(path.Join(pkgBase, "api"))
 	generator.AddImport(path.Join(pkgBase, "api/resource"))
 	// TODO(wojtek-t): Change the overwrites to a flag.
 	generator.OverwritePackage(version, "")
 	for _, knownType := range api.Scheme.KnownTypes(version) {
-		if !strings.HasPrefix(knownType.PkgPath(), versionPath) {
+		if knownType.PkgPath() != versionPath {
 			continue
 		}
 		if err := generator.GenerateConversionsForType(version, knownType); err != nil {
--- a/cmd/gendeepcopy/deep_copy.go
+++ b/cmd/gendeepcopy/deep_copy.go
@ -45,6 +45,32 @@ var (
 	overwrites   = flag.StringP("overwrites", "o", "", "Comma-separated overwrites for package names")
 )

+// types inside the api package don't need to say "api.Scheme"; all others do.
+func destScheme(group, version string) string {
+	if group == "api" && version == "" {
+		return "Scheme"
+	}
+	return "api.Scheme"
+}
+
+// We're moving to pkg/apis/group/version. This handles new and legacy packages.
+func pkgPath(group, version string) string {
+	if group == "" {
+		group = "api"
+	}
+	gv := group
+	if version != "" {
+		gv = path.Join(group, version)
+	}
+	switch {
+	case group == "api":
+		// TODO(lavalamp): remove this special case when we move api to apis/api
+		return path.Join(pkgBase, gv)
+	default:
+		return path.Join(pkgBase, "apis", gv)
+	}
+}
+
 func main() {
 	runtime.GOMAXPROCS(runtime.NumCPU())
 	flag.Parse()
@ -65,10 +91,7 @@ func main() {

 	group, version := path.Split(*groupVersion)
 	group = strings.TrimRight(group, "/")
-	registerTo := "api.Scheme"
-	if *groupVersion == "api/" {
-		registerTo = "Scheme"
-	}
+	registerTo := destScheme(group, version)
 	pkgname := group
 	if len(version) != 0 {
 		pkgname = version
@ -79,7 +102,7 @@ func main() {
 		glog.Fatalf("error writing package line: %v", err)
 	}

-	versionPath := path.Join(pkgBase, group, version)
+	versionPath := pkgPath(group, version)
 	generator := pkg_runtime.NewDeepCopyGenerator(api.Scheme.Raw(), versionPath, sets.NewString("k8s.io/kubernetes"))
 	generator.AddImport(path.Join(pkgBase, "api"))

@ -93,7 +116,7 @@ func main() {
 		}
 	}
 	for _, knownType := range api.Scheme.KnownTypes(version) {
-		if !strings.HasPrefix(knownType.PkgPath(), versionPath) {
+		if knownType.PkgPath() != versionPath {
 			continue
 		}
 		if err := generator.AddType(knownType); err != nil {
--- a/docs/devel/api_changes.md
+++ b/docs/devel/api_changes.md
@ -38,7 +38,7 @@ with a number of existing API types and with the [API
 conventions](api-conventions.md).  If creating a new API
 type/resource, we also recommend that you first send a PR containing
 just a proposal for the new API types, and that you initially target
-the experimental API (pkg/expapi).
+the experimental API (pkg/apis/experimental).

 The Kubernetes API has two major components - the internal structures and
 the versioned APIs.  The versioned APIs are intended to be stable, while the
@ -399,10 +399,10 @@ The conversion code resides with each versioned API. There are two files:
     functions
   - `pkg/api/<version>/conversion_generated.go` containing auto-generated
     conversion functions
-   - `pkg/expapi/<version>/conversion.go` containing manually written conversion
-     functions
-   - `pkg/expapi/<version>/conversion_generated.go` containing auto-generated
+   - `pkg/apis/experimental/<version>/conversion.go` containing manually written
     conversion functions
+   - `pkg/apis/experimental/<version>/conversion_generated.go` containing
+     auto-generated conversion functions

 Since auto-generated conversion functions are using manually written ones,
 those manually written should be named with a defined convention, i.e. a function
@ -437,7 +437,7 @@ of your versioned api objects.

 The deep copy code resides with each versioned API:
   - `pkg/api/<version>/deep_copy_generated.go` containing auto-generated copy functions
-   - `pkg/expapi/<version>/deep_copy_generated.go` containing auto-generated copy functions
+   - `pkg/apis/experimental/<version>/deep_copy_generated.go` containing auto-generated copy functions

 To regenerate them:
   - run
@ -446,6 +446,23 @@ To regenerate them:
 hack/update-generated-deep-copies.sh
 ```

+## Making a new API Group
+
+This section is under construction, as we make the tooling completely generic.
+
+At the moment, you'll have to make a new directory under pkg/apis/; copy the
+directory structure from pkg/apis/experimental. Add the new group/version to all
+of the hack/{verify,update}-generated-{deep-copy,conversions,swagger}.sh files
+in the appropriate places--it should just require adding your new group/version
+to a bash array. You will also need to make sure your new types are imported by
+the generation commands (cmd/gendeepcopy/ & cmd/genconversion). These
+instructions may not be complete and will be updated as we gain experience.
+
+Adding API groups outside of the pkg/apis/ directory is not currently supported,
+but is clearly desirable. The deep copy & conversion generators need to work by
+parsing go files instead of by reflection; then they will be easy to point at
+arbitrary directories: see issue [#13775](http://issue.k8s.io/13775).
+
 ## Update the fuzzer

 Part of our testing regimen for APIs is to "fuzz" (fill with random values) API
--- a/hack/after-build/update-generated-conversions.sh
+++ b/hack/after-build/update-generated-conversions.sh
@ -26,26 +26,27 @@ kube::golang::setup_env
 genconversion=$(kube::util::find-binary "genconversion")

 function generate_version() {
-	local version=$1
-	local TMPFILE="/tmp/conversion_generated.$(date +%s).go"
+  local group_version=$1
+  local TMPFILE="/tmp/conversion_generated.$(date +%s).go"

-	echo "Generating for ${version}"
+  echo "Generating for ${group_version}"

-	sed 's/YEAR/2015/' hack/boilerplate/boilerplate.go.txt > "$TMPFILE"
-	cat >> "$TMPFILE" <<EOF
+  sed 's/YEAR/2015/' hack/boilerplate/boilerplate.go.txt > "$TMPFILE"
+  cat >> "$TMPFILE" <<EOF
 // DO NOT EDIT. THIS FILE IS AUTO-GENERATED BY \$KUBEROOT/hack/update-generated-conversions.sh

 EOF

-  "${genconversion}" -v "${version}" -f - >>  "$TMPFILE"
+  "${genconversion}" -v "${group_version}" -f - >>  "$TMPFILE"

-	mv "$TMPFILE" "pkg/${version}/conversion_generated.go"
+  mv "$TMPFILE" "pkg/$(kube::util::group-version-to-pkg-path "${group_version}")/conversion_generated.go"
 }

-DEFAULT_VERSIONS="api/v1 expapi/v1"
-VERSIONS=${VERSIONS:-$DEFAULT_VERSIONS}
+# TODO(lavalamp): get this list by listing the pkg/apis/ directory?
+DEFAULT_GROUP_VERSIONS="api/v1 experimental/v1"
+VERSIONS=${VERSIONS:-$DEFAULT_GROUP_VERSIONS}
 for ver in $VERSIONS; do
-	# Ensure that the version being processed is registered by setting
-	# KUBE_API_VERSIONS.
-	KUBE_API_VERSIONS="${ver##*/}" generate_version "${ver}"
+  # Ensure that the version being processed is registered by setting
+  # KUBE_API_VERSIONS.
+  KUBE_API_VERSIONS="${ver##*/}" generate_version "${ver}"
 done
--- a/hack/after-build/update-generated-deep-copies.sh
+++ b/hack/after-build/update-generated-deep-copies.sh
@ -25,42 +25,35 @@ kube::golang::setup_env

 gendeepcopy=$(kube::util::find-binary "gendeepcopy")

-function result_file_name() {
-	local version=$1
-	echo "pkg/${version}/deep_copy_generated.go"
-}
-
 function generate_version() {
-	local version=$1
-	local TMPFILE="/tmp/deep_copy_generated.$(date +%s).go"
+  local group_version=$1
+  local TMPFILE="/tmp/deep_copy_generated.$(date +%s).go"

-	echo "Generating for ${version}"
+  echo "Generating for ${group_version}"

-	sed 's/YEAR/2015/' hack/boilerplate/boilerplate.go.txt > $TMPFILE
-	cat >> $TMPFILE <<EOF
+  sed 's/YEAR/2015/' hack/boilerplate/boilerplate.go.txt > $TMPFILE
+  cat >> $TMPFILE <<EOF
 // DO NOT EDIT. THIS FILE IS AUTO-GENERATED BY \$KUBEROOT/hack/update-generated-deep-copies.sh.

 EOF

-	"${gendeepcopy}" -v "${version}" -f - -o "${version}=" >>  "$TMPFILE"
+  "${gendeepcopy}" -v "${group_version}" -f - -o "${group_version}=" >>  "$TMPFILE"

-	mv "$TMPFILE" `result_file_name ${version}`
+  local dest="pkg/$(kube::util::group-version-to-pkg-path "${group_version}")/deep_copy_generated.go"
+  rm -f "${dest}"
+  mv "${TMPFILE}" "${dest}"
 }

 function generate_deep_copies() {
-	local versions="$@"
-	# To avoid compile errors, remove the currently existing files.
-	for ver in ${versions}; do
-		rm -f `result_file_name ${ver}`
-	done
-	for ver in ${versions}; do
-		# Ensure that the version being processed is registered by setting
-		# KUBE_API_VERSIONS.
-		apiVersions="${ver##*/}"
-		KUBE_API_VERSIONS="${apiVersions}" generate_version "${ver}"
-	done
+  local group_versions="$@"
+  for ver in ${group_versions}; do
+    # Ensure that the version being processed is registered by setting
+    # KUBE_API_VERSIONS.
+    apiVersions="${ver##*/}"
+    KUBE_API_VERSIONS="${apiVersions}" generate_version "${ver}"
+  done
 }

-DEFAULT_VERSIONS="api/ api/v1 expapi/ expapi/v1"
+DEFAULT_VERSIONS="api/ api/v1 experimental/ experimental/v1"
 VERSIONS=${VERSIONS:-$DEFAULT_VERSIONS}
 generate_deep_copies "$VERSIONS"
--- a/hack/after-build/verify-description.sh
+++ b/hack/after-build/verify-description.sh
@ -40,7 +40,7 @@ find_files() {
      \) -prune \
    \) \
    \( -wholename '*pkg/api/v*/types.go' \
-       -o -wholename '*pkg/expapi/v*/types.go' \
+       -o -wholename '*pkg/apis/*/v*/types.go' \
    \)
 }

@ -61,7 +61,7 @@ for file in $versioned_api_files; do
  fi
 done

-internal_types_files="${KUBE_ROOT}/pkg/api/types.go ${KUBE_ROOT}/pkg/expapi/types.go"
+internal_types_files="${KUBE_ROOT}/pkg/api/types.go ${KUBE_ROOT}/pkg/apis/experimental/types.go"
 for internal_types_file in $internal_types_files; do
  if grep json: "${internal_types_file}" | grep -v // | grep description: ; then
    echo "Internal API types should not contain descriptions"
--- a/hack/after-build/verify-generated-conversions.sh
+++ b/hack/after-build/verify-generated-conversions.sh
@ -23,7 +23,7 @@ source "${KUBE_ROOT}/hack/lib/init.sh"

 kube::golang::setup_env

-APIROOTS=${APIROOTS:-pkg/api pkg/expapi}
+APIROOTS=${APIROOTS:-pkg/api pkg/apis/experimental}
 _tmp="${KUBE_ROOT}/_tmp"

 cleanup() {
--- a/hack/after-build/verify-generated-deep-copies.sh
+++ b/hack/after-build/verify-generated-deep-copies.sh
@ -25,7 +25,7 @@ kube::golang::setup_env

 gendeepcopy=$(kube::util::find-binary "gendeepcopy")

-APIROOTS=${APIROOTS:-pkg/api pkg/expapi}
+APIROOTS=${APIROOTS:-pkg/api pkg/apis/experimental}
 _tmp="${KUBE_ROOT}/_tmp"

 cleanup() {
--- a/hack/lib/util.sh
+++ b/hack/lib/util.sh
@ -220,4 +220,35 @@ kube::util::analytics-link() {
  echo "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/${path}?pixel)]()"
 }

+# Takes a group/version and returns the path to its location on disk, sans
+# "pkg". E.g.:
+# * default behavior: experimental/v1 -> apis/experimental/v1
+# * legacy behavior: api/v1 -> api/v1
+# * Special handling for only a group: experimental -> apis/experimental
+# * Special handling for only "api" group: api -> api
+# * Very special handling for "v1": v1 -> api/v1
+kube::util::group-version-to-pkg-path() {
+  local group_version="$1"
+  # Special cases first.
+  # TODO(lavalamp): Simplify this by moving pkg/api/v1 and splitting pkg/api,
+  # moving the results to pkg/apis/api.
+  case "${group_version}" in
+    v1)
+      echo "api/v1"
+      ;;
+    api)
+      echo "api/v1"
+      ;;
+    api/*)
+      echo "${group_version}"
+      ;;
+    api/*)
+      echo "${group_version}"
+      ;;
+    *)
+      echo "apis/${group_version}"
+      ;;
+  esac
+}
+
 # ex: ts=2 sw=2 et filetype=sh
--- a/hack/update-generated-swagger-docs.sh
+++ b/hack/update-generated-swagger-docs.sh
@ -24,14 +24,14 @@ source "${KUBE_ROOT}/hack/lib/init.sh"
 kube::golang::setup_env

 function generate_version() {
-	local groupVersion=$1
-	local TMPFILE="/tmp/types_swagger_doc_generated.$(date +%s).go"
+  local group_version=$1
+  local TMPFILE="/tmp/types_swagger_doc_generated.$(date +%s).go"

-	echo "Generating swagger type docs for ${groupVersion}"
+  echo "Generating swagger type docs for ${group_version}"

-	sed 's/YEAR/2015/' hack/boilerplate/boilerplate.go.txt > $TMPFILE
-  echo "package ${groupVersion##*/}" >> $TMPFILE
-	cat >> $TMPFILE <<EOF
+  sed 's/YEAR/2015/' hack/boilerplate/boilerplate.go.txt > $TMPFILE
+  echo "package ${group_version##*/}" >> $TMPFILE
+  cat >> $TMPFILE <<EOF

 // This file contains a collection of methods that can be used from go-resful to
 // generate Swagger API documentation for its models. Please read this PR for more
@ -46,21 +46,23 @@ function generate_version() {
 // AUTO-GENERATED FUNCTIONS START HERE
 EOF

-	GOPATH=$(godep path):$GOPATH go run cmd/genswaggertypedocs/swagger_type_docs.go -s "pkg/${groupVersion}/types.go" -f - >>  $TMPFILE
+  GOPATH=$(godep path):$GOPATH go run cmd/genswaggertypedocs/swagger_type_docs.go -s \
+    "pkg/$(kube::util::group-version-to-pkg-path "${group_version}")/types.go" -f - \
+    >>  $TMPFILE

-	echo "// AUTO-GENERATED FUNCTIONS END HERE" >> $TMPFILE
+  echo "// AUTO-GENERATED FUNCTIONS END HERE" >> $TMPFILE

-	gofmt -w -s $TMPFILE
-	mv $TMPFILE "pkg/${groupVersion}/types_swagger_doc_generated.go"
+  gofmt -w -s $TMPFILE
+  mv $TMPFILE "pkg/$(kube::util::group-version-to-pkg-path "${group_version}")/types_swagger_doc_generated.go"
 }

-GROUP_VERSIONS="api/v1 expapi/v1"
+GROUP_VERSIONS="api/v1 experimental/v1"
 # To avoid compile errors, remove the currently existing files.
-for groupVersion in $GROUP_VERSIONS; do
-	rm -f "pkg/${groupVersion}/types_swagger_doc_generated.go"
+for group_version in $GROUP_VERSIONS; do
+  rm -f "pkg/$(kube::util::group-version-to-pkg-path "${group_version}")/types_swagger_doc_generated.go"
 done
-for groupVersion in $GROUP_VERSIONS; do
-	generate_version "${groupVersion}"
+for group_version in $GROUP_VERSIONS; do
+  generate_version "${group_version}"
 done

 "${KUBE_ROOT}/hack/update-swagger-spec.sh"
--- a/hack/update-gofmt.sh
+++ b/hack/update-gofmt.sh
@ -0,0 +1,48 @@
+#!/bin/bash
+
+# Copyright 2014 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# GoFmt apparently is changing @ head...
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
+
+GO_VERSION=($(go version))
+
+if [[ -z $(echo "${GO_VERSION[2]}" | grep -E 'go1.2|go1.3|go1.4|go1.5') ]]; then
+  echo "Unknown go version '${GO_VERSION}', skipping gofmt."
+  exit 0
+fi
+
+cd "${KUBE_ROOT}"
+
+find_files() {
+  find . -not \( \
+      \( \
+        -wholename './output' \
+        -o -wholename './_output' \
+        -o -wholename './release' \
+        -o -wholename './target' \
+        -o -wholename '*/third_party/*' \
+        -o -wholename '*/Godeps/*' \
+      \) -prune \
+    \) -name '*.go'
+}
+
+GOFMT="gofmt -s -w"
+find_files | xargs $GOFMT
--- a/pkg/apis/experimental/deep_copy_generated.go
+++ b/pkg/apis/experimental/deep_copy_generated.go
@ -16,7 +16,7 @@ limitations under the License.

 // DO NOT EDIT. THIS FILE IS AUTO-GENERATED BY $KUBEROOT/hack/update-generated-deep-copies.sh.

-package expapi
+package experimental

 import (
 	time "time"
--- a/pkg/apis/experimental/register.go
+++ b/pkg/apis/experimental/register.go
@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */

-package expapi
+package experimental

 import (
 	"k8s.io/kubernetes/pkg/api"