github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #71192 from roycaihw/crd-publish-openapi 

Publish CRD openapi
pull/564/head
Kubernetes Prow Robot 2019-03-07 03:54:58 -08:00 committed by GitHub
parent bf4e93da57 54b99413b2
commit 0b48018a39
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 1680 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
build/root/Makefile.generated_files
View File

@ -409,8 +409,10 @@ $(CONVERSION_GEN): $(k8s.io/kubernetes/vendor/k8s.io/code-generator/cmd/conversi
OPENAPI_BASENAME := $(GENERATED_FILE_PREFIX)openapi
OPENAPI_FILENAME := $(OPENAPI_BASENAME).go
OPENAPI_OUTPUT_PKG := pkg/generated/openapi
CRD_OPENAPI_OUTPUT_PKG := staging/src/k8s.io/apiextensions-apiserver/pkg/generated/openapi
BOILERPLATE_FILENAME := vendor/k8s.io/code-generator/hack/boilerplate.go.txt
REPORT_FILENAME := $(OUT_DIR)/violations.report
IGNORED_REPORT_FILENAME := $(OUT_DIR)/ignored_violations.report
KNOWN_VIOLATION_FILENAME := api/api-rules/violation_exceptions.list
# When UPDATE_API_KNOWN_VIOLATIONS is set to be true, let the generator to write
# updated API violations to the known API violation exceptions list.
@ -436,10 +438,11 @@ OPENAPI_DIRS := $(shell \
)
OPENAPI_OUTFILE := $(OPENAPI_OUTPUT_PKG)/$(OPENAPI_FILENAME)
CRD_OPENAPI_OUTFILE := $(CRD_OPENAPI_OUTPUT_PKG)/$(OPENAPI_FILENAME)
# This rule is the user-friendly entrypoint for openapi generation.
.PHONY: gen_openapi
gen_openapi: $(OPENAPI_OUTFILE) $(OPENAPI_GEN)
gen_openapi: $(OPENAPI_OUTFILE) $(OPENAPI_GEN) $(CRD_OPENAPI_OUTFILE)
# For each dir in OPENAPI_DIRS, this establishes a dependency between the
# output file and the input files that should trigger a rebuild.
@ -469,6 +472,17 @@ $(OPENAPI_OUTFILE): $(OPENAPI_GEN) $(KNOWN_VIOLATION_FILENAME)
diff $(REPORT_FILENAME) $(KNOWN_VIOLATION_FILENAME) || \
(echo -e $(API_RULE_CHECK_FAILURE_MESSAGE); exit 1)
# TODO(roycaihw): move the automation to apiextensions-apiserver
$(CRD_OPENAPI_OUTFILE): $(OPENAPI_GEN)
./hack/run-in-gopath.sh $(OPENAPI_GEN) \
--v $(KUBE_VERBOSE) \
--logtostderr \
-i "k8s.io/apimachinery/pkg/apis/meta/v1,k8s.io/api/autoscaling/v1" \
-p $(PRJ_SRC_PATH)/$(CRD_OPENAPI_OUTPUT_PKG) \
-O $(OPENAPI_BASENAME) \
-h $(BOILERPLATE_FILENAME) \
-r $(IGNORED_REPORT_FILENAME) \
"$$@"
# How to build the generator tool. The deps for this are defined in
# the $(GO_PKGDEPS_FILE), above.

1
pkg/features/kube_features.go
View File

@ -510,6 +510,7 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS
apiextensionsfeatures.CustomResourceValidation: {Default: true, PreRelease: utilfeature.Beta},
apiextensionsfeatures.CustomResourceSubresources: {Default: true, PreRelease: utilfeature.Beta},
apiextensionsfeatures.CustomResourceWebhookConversion: {Default: false, PreRelease: utilfeature.Alpha},
apiextensionsfeatures.CustomResourcePublishOpenAPI: {Default: false, PreRelease: utilfeature.Alpha},
// features that enable backwards compatibility but are scheduled to be removed
// ...

1
staging/src/k8s.io/apiextensions-apiserver/BUILD
View File

@ -51,6 +51,7 @@ filegroup(
"//staging/src/k8s.io/apiextensions-apiserver/pkg/controller/status:all-srcs",
"//staging/src/k8s.io/apiextensions-apiserver/pkg/crdserverscheme:all-srcs",
"//staging/src/k8s.io/apiextensions-apiserver/pkg/features:all-srcs",
"//staging/src/k8s.io/apiextensions-apiserver/pkg/generated/openapi:all-srcs",
"//staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresource:all-srcs",
"//staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition:all-srcs",
"//staging/src/k8s.io/apiextensions-apiserver/test/integration:all-srcs",

8
staging/src/k8s.io/apiextensions-apiserver/Godeps/Godeps.json generated
View File

@ -2426,6 +2426,10 @@
"ImportPath": "k8s.io/apiserver/pkg/authorization/authorizer",
"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
},
{
"ImportPath": "k8s.io/apiserver/pkg/endpoints",
"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
},
{
"ImportPath": "k8s.io/apiserver/pkg/endpoints/discovery",
"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
@ -2446,6 +2450,10 @@
"ImportPath": "k8s.io/apiserver/pkg/endpoints/metrics",
"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
},
{
"ImportPath": "k8s.io/apiserver/pkg/endpoints/openapi",
"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
},
{
"ImportPath": "k8s.io/apiserver/pkg/endpoints/request",
"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"

1
staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/BUILD
View File

@ -32,6 +32,7 @@ go_library(
"//staging/src/k8s.io/apiextensions-apiserver/pkg/client/listers/apiextensions/internalversion:go_default_library",
"//staging/src/k8s.io/apiextensions-apiserver/pkg/controller/establish:go_default_library",
"//staging/src/k8s.io/apiextensions-apiserver/pkg/controller/finalizer:go_default_library",
"//staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi:go_default_library",
"//staging/src/k8s.io/apiextensions-apiserver/pkg/controller/status:go_default_library",
"//staging/src/k8s.io/apiextensions-apiserver/pkg/crdserverscheme:go_default_library",
"//staging/src/k8s.io/apiextensions-apiserver/pkg/features:go_default_library",

11
staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go
View File

@ -31,7 +31,9 @@ import (
internalinformers "k8s.io/apiextensions-apiserver/pkg/client/informers/internalversion"
"k8s.io/apiextensions-apiserver/pkg/controller/establish"
"k8s.io/apiextensions-apiserver/pkg/controller/finalizer"
openapicontroller "k8s.io/apiextensions-apiserver/pkg/controller/openapi"
"k8s.io/apiextensions-apiserver/pkg/controller/status"
apiextensionsfeatures "k8s.io/apiextensions-apiserver/pkg/features"
"k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
@ -44,6 +46,7 @@ import (
"k8s.io/apiserver/pkg/registry/rest"
genericapiserver "k8s.io/apiserver/pkg/server"
serverstorage "k8s.io/apiserver/pkg/server/storage"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/apiserver/pkg/util/webhook"
)
@ -198,12 +201,20 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget)
crdClient.Apiextensions(),
crdHandler,
)
var openapiController *openapicontroller.Controller
if utilfeature.DefaultFeatureGate.Enabled(apiextensionsfeatures.CustomResourcePublishOpenAPI) {
openapiController = openapicontroller.NewController(s.Informers.Apiextensions().InternalVersion().CustomResourceDefinitions())
}
s.GenericAPIServer.AddPostStartHookOrDie("start-apiextensions-informers", func(context genericapiserver.PostStartHookContext) error {
s.Informers.Start(context.StopCh)
return nil
})
s.GenericAPIServer.AddPostStartHookOrDie("start-apiextensions-controllers", func(context genericapiserver.PostStartHookContext) error {
if utilfeature.DefaultFeatureGate.Enabled(apiextensionsfeatures.CustomResourcePublishOpenAPI) {
go openapiController.Run(s.GenericAPIServer.StaticOpenAPISpec, s.GenericAPIServer.OpenAPIVersionedService, context.StopCh)
}
go crdController.Run(context.StopCh)
go namingController.Run(context.StopCh)
go establishingController.Run(context.StopCh)

34
staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/BUILD
View File

@ -2,25 +2,55 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
go_library(
name = "go_default_library",
srcs = ["conversion.go"],
srcs = [
"aggregator.go",
"builder.go",
"controller.go",
"conversion.go",
],
importmap = "k8s.io/kubernetes/vendor/k8s.io/apiextensions-apiserver/pkg/controller/openapi",
importpath = "k8s.io/apiextensions-apiserver/pkg/controller/openapi",
visibility = ["//visibility:public"],
deps = [
"//staging/src/k8s.io/api/autoscaling/v1:go_default_library",
"//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions:go_default_library",
"//staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/validation:go_default_library",
"//staging/src/k8s.io/apiextensions-apiserver/pkg/client/informers/internalversion/apiextensions/internalversion:go_default_library",
"//staging/src/k8s.io/apiextensions-apiserver/pkg/client/listers/apiextensions/internalversion:go_default_library",
"//staging/src/k8s.io/apiextensions-apiserver/pkg/generated/openapi:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1beta1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/endpoints:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/endpoints/openapi:go_default_library",
"//staging/src/k8s.io/client-go/tools/cache:go_default_library",
"//staging/src/k8s.io/client-go/util/workqueue:go_default_library",
"//vendor/github.com/emicklei/go-restful:go_default_library",
"//vendor/github.com/go-openapi/spec:go_default_library",
"//vendor/k8s.io/klog:go_default_library",
"//vendor/k8s.io/kube-openapi/pkg/builder:go_default_library",
"//vendor/k8s.io/kube-openapi/pkg/common:go_default_library",
"//vendor/k8s.io/kube-openapi/pkg/handler:go_default_library",
"//vendor/k8s.io/kube-openapi/pkg/util:go_default_library",
],
)
go_test(
name = "go_default_test",
srcs = ["conversion_test.go"],
srcs = [
"builder_test.go",
"conversion_test.go",
],
embed = [":go_default_library"],
deps = [
"//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions:go_default_library",
"//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/json:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//vendor/github.com/go-openapi/spec:go_default_library",
"//vendor/github.com/google/gofuzz:go_default_library",
"//vendor/github.com/googleapis/gnostic/OpenAPIv2:go_default_library",

70
staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/aggregator.go Normal file
View File

@ -0,0 +1,70 @@
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package openapi
import (
"github.com/go-openapi/spec"
)
// mergeSpecs aggregates all OpenAPI specs, reusing the metadata of the first, static spec as the basis.
func mergeSpecs(staticSpec *spec.Swagger, crdSpecs ...*spec.Swagger) *spec.Swagger {
// create shallow copy of staticSpec, but replace paths and definitions because we modify them.
specToReturn := *staticSpec
if staticSpec.Definitions != nil {
specToReturn.Definitions = spec.Definitions{}
for k, s := range staticSpec.Definitions {
specToReturn.Definitions[k] = s
}
}
if staticSpec.Paths != nil {
specToReturn.Paths = &spec.Paths{
Paths: map[string]spec.PathItem{},
}
for k, p := range staticSpec.Paths.Paths {
specToReturn.Paths.Paths[k] = p
}
}
for _, s := range crdSpecs {
mergeSpec(&specToReturn, s)
}
return &specToReturn
}
// mergeSpec copies paths and definitions from source to dest, mutating dest, but not source.
// We assume that conflicts do not matter.
func mergeSpec(dest, source *spec.Swagger) {
if source == nil || source.Paths == nil {
return
}
if dest.Paths == nil {
dest.Paths = &spec.Paths{}
}
for k, v := range source.Definitions {
if dest.Definitions == nil {
dest.Definitions = spec.Definitions{}
}
dest.Definitions[k] = v
}
for k, v := range source.Paths.Paths {
if dest.Paths.Paths == nil {
dest.Paths.Paths = map[string]spec.PathItem{}
}
dest.Paths.Paths[k] = v
}
}

417
staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/builder.go Normal file
View File

@ -0,0 +1,417 @@
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package openapi
import (
"fmt"
"net/http"
"strings"
"sync"
restful "github.com/emicklei/go-restful"
"github.com/go-openapi/spec"
v1 "k8s.io/api/autoscaling/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
metav1beta1 "k8s.io/apimachinery/pkg/apis/meta/v1beta1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apiserver/pkg/endpoints"
"k8s.io/apiserver/pkg/endpoints/openapi"
openapibuilder "k8s.io/kube-openapi/pkg/builder"
"k8s.io/kube-openapi/pkg/common"
"k8s.io/kube-openapi/pkg/util"
"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions"
generatedopenapi "k8s.io/apiextensions-apiserver/pkg/generated/openapi"
)
const (
// Reference and Go types for built-in metadata
objectMetaSchemaRef = "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta"
listMetaType = "k8s.io/apimachinery/pkg/apis/meta/v1.ListMeta"
typeMetaType = "k8s.io/apimachinery/pkg/apis/meta/v1.TypeMeta"
definitionPrefix = "#/definitions/"
)
var (
swaggerPartialObjectMetadataDescriptions = metav1beta1.PartialObjectMetadata{}.SwaggerDoc()
)
var definitions map[string]common.OpenAPIDefinition
var buildDefinitions sync.Once
var namer *openapi.DefinitionNamer
// BuildSwagger builds swagger for the given crd in the given version
func BuildSwagger(crd *apiextensions.CustomResourceDefinition, version string) (*spec.Swagger, error) {
var schema *spec.Schema
s, err := apiextensions.GetSchemaForVersion(crd, version)
if err != nil {
return nil, err
}
if s != nil && s.OpenAPIV3Schema != nil {
schema, err = ConvertJSONSchemaPropsToOpenAPIv2Schema(s.OpenAPIV3Schema)
if err != nil {
return nil, err
}
}
// TODO(roycaihw): remove the WebService templating below. The following logic
// comes from function registerResourceHandlers() in k8s.io/apiserver.
// Alternatives are either (ideally) refactoring registerResourceHandlers() to
// reuse the code, or faking an APIInstaller for CR to feed to registerResourceHandlers().
b := newBuilder(crd, version, schema)
// Sample response types for building web service
sample := &CRDCanonicalTypeNamer{
group: b.group,
version: b.version,
kind: b.kind,
}
sampleList := &CRDCanonicalTypeNamer{
group: b.group,
version: b.version,
kind: b.listKind,
}
status := &metav1.Status{}
patch := &metav1.Patch{}
scale := &v1.Scale{}
routes := make([]*restful.RouteBuilder, 0)
root := fmt.Sprintf("/apis/%s/%s/%s", b.group, b.version, b.plural)
if b.namespaced {
routes = append(routes, b.buildRoute(root, "", "GET", "list", sampleList).
Operation("list"+b.kind+"ForAllNamespaces"))
root = fmt.Sprintf("/apis/%s/%s/namespaces/{namespace}/%s", b.group, b.version, b.plural)
}
routes = append(routes, b.buildRoute(root, "", "GET", "list", sampleList))
routes = append(routes, b.buildRoute(root, "", "POST", "create", sample).Reads(sample))
routes = append(routes, b.buildRoute(root, "", "DELETE", "deletecollection", status))
routes = append(routes, b.buildRoute(root, "/{name}", "GET", "read", sample))
routes = append(routes, b.buildRoute(root, "/{name}", "PUT", "replace", sample).Reads(sample))
routes = append(routes, b.buildRoute(root, "/{name}", "DELETE", "delete", status))
routes = append(routes, b.buildRoute(root, "/{name}", "PATCH", "patch", sample).Reads(patch))
subresources, err := apiextensions.GetSubresourcesForVersion(crd, version)
if err != nil {
return nil, err
}
if subresources != nil && subresources.Status != nil {
routes = append(routes, b.buildRoute(root, "/{name}/status", "GET", "read", sample))
routes = append(routes, b.buildRoute(root, "/{name}/status", "PUT", "replace", sample).Reads(sample))
routes = append(routes, b.buildRoute(root, "/{name}/status", "PATCH", "patch", sample).Reads(patch))
}
if subresources != nil && subresources.Scale != nil {
routes = append(routes, b.buildRoute(root, "/{name}/scale", "GET", "read", scale))
routes = append(routes, b.buildRoute(root, "/{name}/scale", "PUT", "replace", scale).Reads(scale))
routes = append(routes, b.buildRoute(root, "/{name}/scale", "PATCH", "patch", scale).Reads(patch))
}
for _, route := range routes {
b.ws.Route(route)
}
openAPISpec, err := openapibuilder.BuildOpenAPISpec([]*restful.WebService{b.ws}, b.getOpenAPIConfig())
if err != nil {
return nil, err
}
return openAPISpec, nil
}
// Implements CanonicalTypeNamer
var _ = util.OpenAPICanonicalTypeNamer(&CRDCanonicalTypeNamer{})
// CRDCanonicalTypeNamer implements CanonicalTypeNamer interface for CRDs to
// seed kube-openapi canonical type name without Go types
type CRDCanonicalTypeNamer struct {
group string
version string
kind string
}
// OpenAPICanonicalTypeName returns canonical type name for given CRD
func (c *CRDCanonicalTypeNamer) OpenAPICanonicalTypeName() string {
return fmt.Sprintf("%s/%s.%s", c.group, c.version, c.kind)
}
// builder contains validation schema and basic naming information for a CRD in
// one version. The builder works to build a WebService that kube-openapi can
// consume.
type builder struct {
schema *spec.Schema
listSchema *spec.Schema
ws *restful.WebService
group string
version string
kind string
listKind string
plural string
namespaced bool
}
// subresource is a handy method to get subresource name. Valid inputs are:
// input output
// "" ""
// "/" ""
// "/{name}" ""
// "/{name}/scale" "scale"
// "/{name}/scale/foo" invalid input
func subresource(path string) string {
parts := strings.Split(path, "/")
if len(parts) <= 2 {
return ""
}
if len(parts) == 3 {
return parts[2]
}
// panic to alert on programming error
panic("failed to parse subresource; invalid path")
}
func (b *builder) descriptionFor(path, verb string) string {
var article string
switch verb {
case "list":
article = " objects of kind "
case "read", "replace":
article = " the specified "
case "patch":
article = " the specified "
case "create", "delete":
article = endpoints.GetArticleForNoun(b.kind, " ")
default:
article = ""
}
var description string
sub := subresource(path)
if len(sub) > 0 {
sub = " " + sub + " of"
}
switch verb {
case "patch":
description = "partially update" + sub + article + b.kind
case "deletecollection":
// to match the text for built-in APIs
if len(sub) > 0 {
sub = sub + " a"
}
description = "delete collection of" + sub + " " + b.kind
default:
description = verb + sub + article + b.kind
}
return description
}
// buildRoute returns a RouteBuilder for WebService to consume and builds path in swagger
// action can be one of: GET, PUT, PATCH, POST, DELETE;
// verb can be one of: list, read, replace, patch, create, delete, deletecollection;
// sample is the sample Go type for response type.
func (b *builder) buildRoute(root, path, action, verb string, sample interface{}) *restful.RouteBuilder {
var namespaced string
if b.namespaced {
namespaced = "Namespaced"
}
route := b.ws.Method(action).
Path(root+path).
To(func(req *restful.Request, res *restful.Response) {}).
Doc(b.descriptionFor(path, verb)).
Param(b.ws.QueryParameter("pretty", "If 'true', then the output is pretty printed.")).
Operation(verb+namespaced+b.kind+strings.Title(subresource(path))).
Metadata(endpoints.ROUTE_META_GVK, metav1.GroupVersionKind{
Group: b.group,
Version: b.version,
Kind: b.kind,
}).
Metadata(endpoints.ROUTE_META_ACTION, strings.ToLower(action)).
Produces("application/json", "application/yaml").
Returns(http.StatusOK, "OK", sample).
Writes(sample)
// Build consume media types
if action == "PATCH" {
route.Consumes("application/json-patch+json",
"application/merge-patch+json",
"application/strategic-merge-patch+json")
} else {
route.Consumes("*/*")
}
// Build option parameters
switch verb {
case "get":
// TODO: CRD support for export is still under consideration
endpoints.AddObjectParams(b.ws, route, &metav1.GetOptions{})
case "list", "deletecollection":
endpoints.AddObjectParams(b.ws, route, &metav1.ListOptions{})
case "replace", "patch":
// TODO: PatchOption added in feature branch but not in master yet
endpoints.AddObjectParams(b.ws, route, &metav1.UpdateOptions{})
case "create":
endpoints.AddObjectParams(b.ws, route, &metav1.CreateOptions{})
case "delete":
endpoints.AddObjectParams(b.ws, route, &metav1.DeleteOptions{})
route.Reads(&metav1.DeleteOptions{}).ParameterNamed("body").Required(false)
}
// Build responses
switch verb {
case "create":
route.Returns(http.StatusAccepted, "Accepted", sample)
route.Returns(http.StatusCreated, "Created", sample)
case "delete":
route.Returns(http.StatusAccepted, "Accepted", sample)
case "replace":
route.Returns(http.StatusCreated, "Created", sample)
}
return route
}
// buildKubeNative builds input schema with Kubernetes' native object meta, type meta and
// extensions
func (b *builder) buildKubeNative(schema *spec.Schema) *spec.Schema {
// only add properties if we have a schema. Otherwise, kubectl would (wrongly) assume additionalProperties=false
// and forbid anything outside of apiVersion, kind and metadata. We have to fix kubectl to stop doing this, e.g. by
// adding additionalProperties=true support to explicitly allow additional fields.
// TODO: fix kubectl to understand additionalProperties=true
if schema == nil {
schema = &spec.Schema{
SchemaProps: spec.SchemaProps{Type: []string{"object"}},
}
// no, we cannot add more properties here, not even TypeMeta/ObjectMeta because kubectl will complain about
// unknown fields for anything else.
} else {
schema.SetProperty("metadata", *spec.RefSchema(objectMetaSchemaRef).
WithDescription(swaggerPartialObjectMetadataDescriptions["metadata"]))
addTypeMetaProperties(schema)
}
schema.AddExtension(endpoints.ROUTE_META_GVK, []interface{}{
map[string]interface{}{
"group": b.group,
"version": b.version,
"kind": b.kind,
},
})
return schema
}
// getDefinition gets definition for given Kubernetes type. This function is extracted from
// kube-openapi builder logic
func getDefinition(name string) spec.Schema {
buildDefinitions.Do(buildDefinitionsFunc)
return definitions[name].Schema
}
func buildDefinitionsFunc() {
namer = openapi.NewDefinitionNamer(runtime.NewScheme())
definitions = generatedopenapi.GetOpenAPIDefinitions(func(name string) spec.Ref {
defName, _ := namer.GetDefinitionName(name)
return spec.MustCreateRef(definitionPrefix + common.EscapeJsonPointer(defName))
})
}
// addTypeMetaProperties adds Kubernetes-specific type meta properties to input schema:
// apiVersion and kind
func addTypeMetaProperties(s *spec.Schema) {
s.SetProperty("apiVersion", getDefinition(typeMetaType).SchemaProps.Properties["apiVersion"])
s.SetProperty("kind", getDefinition(typeMetaType).SchemaProps.Properties["kind"])
}
// buildListSchema builds the list kind schema for the CRD
func (b *builder) buildListSchema() *spec.Schema {
name := definitionPrefix + util.ToRESTFriendlyName(fmt.Sprintf("%s/%s/%s", b.group, b.version, b.kind))
doc := fmt.Sprintf("List of %s. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md", b.plural)
s := new(spec.Schema).WithDescription(fmt.Sprintf("%s is a list of %s", b.listKind, b.kind)).
WithRequired("items").
SetProperty("items", *spec.ArrayProperty(spec.RefSchema(name)).WithDescription(doc)).
SetProperty("metadata", getDefinition(listMetaType))
addTypeMetaProperties(s)
s.AddExtension(endpoints.ROUTE_META_GVK, []map[string]string{
{
"group": b.group,
"version": b.version,
"kind": b.listKind,
},
})
return s
}
// getOpenAPIConfig builds config which wires up generated definitions for kube-openapi to consume
func (b *builder) getOpenAPIConfig() *common.Config {
return &common.Config{
ProtocolList: []string{"https"},
Info: &spec.Info{
InfoProps: spec.InfoProps{
Title: "Kubernetes CRD Swagger",
Version: "v0.1.0",
},
},
CommonResponses: map[int]spec.Response{
401: {
ResponseProps: spec.ResponseProps{
Description: "Unauthorized",
},
},
},
GetOperationIDAndTags: openapi.GetOperationIDAndTags,
GetDefinitionName: func(name string) (string, spec.Extensions) {
buildDefinitions.Do(buildDefinitionsFunc)
return namer.GetDefinitionName(name)
},
GetDefinitions: func(ref common.ReferenceCallback) map[string]common.OpenAPIDefinition {
def := generatedopenapi.GetOpenAPIDefinitions(ref)
def[fmt.Sprintf("%s/%s.%s", b.group, b.version, b.kind)] = common.OpenAPIDefinition{
Schema: *b.schema,
}
def[fmt.Sprintf("%s/%s.%s", b.group, b.version, b.listKind)] = common.OpenAPIDefinition{
Schema: *b.listSchema,
}
return def
},
}
}
func newBuilder(crd *apiextensions.CustomResourceDefinition, version string, schema *spec.Schema) *builder {
b := &builder{
schema: &spec.Schema{
SchemaProps: spec.SchemaProps{Type: []string{"object"}},
},
listSchema: &spec.Schema{},
ws: &restful.WebService{},
group: crd.Spec.Group,
version: version,
kind: crd.Spec.Names.Kind,
listKind: crd.Spec.Names.ListKind,
plural: crd.Spec.Names.Plural,
}
if crd.Spec.Scope == apiextensions.NamespaceScoped {
b.namespaced = true
}
// Pre-build schema with Kubernetes native properties
b.schema = b.buildKubeNative(schema)
b.listSchema = b.buildListSchema()
return b
}

139
staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/builder_test.go Normal file
View File

@ -0,0 +1,139 @@
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package openapi
import (
"reflect"
"testing"
"github.com/go-openapi/spec"
"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions"
"k8s.io/apimachinery/pkg/util/diff"
"k8s.io/apimachinery/pkg/util/json"
"k8s.io/apimachinery/pkg/util/sets"
)
func TestNewBuilder(t *testing.T) {
type args struct {
}
tests := []struct {
name string
schema string
wantedSchema string
wantedItemsSchema string
}{
{
"nil",
"",
`{"type":"object","x-kubernetes-group-version-kind":[{"group":"bar.k8s.io","kind":"Foo","version":"v1"}]}`, `{"$ref":"#/definitions/io.k8s.bar.v1.Foo"}`,
},
{"empty",
"{}",
`{"properties":{"apiVersion":{},"kind":{},"metadata":{}},"x-kubernetes-group-version-kind":[{"group":"bar.k8s.io","kind":"Foo","version":"v1"}]}`,
`{"$ref":"#/definitions/io.k8s.bar.v1.Foo"}`,
},
{"empty properties",
`{"properties":{"spec":{},"status":{}}}`,
`{"properties":{"apiVersion":{},"kind":{},"metadata":{},"spec":{},"status":{}},"x-kubernetes-group-version-kind":[{"group":"bar.k8s.io","kind":"Foo","version":"v1"}]}`,
`{"$ref":"#/definitions/io.k8s.bar.v1.Foo"}`,
},
{"filled properties",
`{"properties":{"spec":{"type":"object"},"status":{"type":"object"}}}`,
`{"properties":{"apiVersion":{},"kind":{},"metadata":{},"spec":{"type":"object"},"status":{"type":"object"}},"x-kubernetes-group-version-kind":[{"group":"bar.k8s.io","kind":"Foo","version":"v1"}]}`,
`{"$ref":"#/definitions/io.k8s.bar.v1.Foo"}`,
},
{"type",
`{"type":"object"}`,
`{"properties":{"apiVersion":{},"kind":{},"metadata":{}},"type":"object","x-kubernetes-group-version-kind":[{"group":"bar.k8s.io","kind":"Foo","version":"v1"}]}`,
`{"$ref":"#/definitions/io.k8s.bar.v1.Foo"}`,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
var schema *spec.Schema
if len(tt.schema) > 0 {
schema = &spec.Schema{}
if err := json.Unmarshal([]byte(tt.schema), schema); err != nil {
t.Fatal(err)
}
}
got := newBuilder(&apiextensions.CustomResourceDefinition{
Spec: apiextensions.CustomResourceDefinitionSpec{
Group: "bar.k8s.io",
Version: "v1",
Names: apiextensions.CustomResourceDefinitionNames{
Plural: "foos",
Singular: "foo",
Kind: "Foo",
ListKind: "FooList",
},
Scope: apiextensions.NamespaceScoped,
},
}, "v1", schema)
var wantedSchema, wantedItemsSchema spec.Schema
if err := json.Unmarshal([]byte(tt.wantedSchema), &wantedSchema); err != nil {
t.Fatal(err)
}
if err := json.Unmarshal([]byte(tt.wantedItemsSchema), &wantedItemsSchema); err != nil {
t.Fatal(err)
}
gotProperties := properties(got.schema.Properties)
wantedProperties := properties(wantedSchema.Properties)
if !gotProperties.Equal(wantedProperties) {
t.Fatalf("unexpected properties, got: %s, expected: %s", gotProperties.List(), wantedProperties.List())
}
// wipe out TypeMeta/ObjectMeta content, with those many lines of descriptions. We trust that they match here.
if _, found := got.schema.Properties["kind"]; found {
got.schema.Properties["kind"] = spec.Schema{}
}
if _, found := got.schema.Properties["apiVersion"]; found {
got.schema.Properties["apiVersion"] = spec.Schema{}
}
if _, found := got.schema.Properties["metadata"]; found {
got.schema.Properties["metadata"] = spec.Schema{}
}
if !reflect.DeepEqual(&wantedSchema, got.schema) {
t.Errorf("unexpected schema: %s\nwant = %#v\ngot = %#v", diff.ObjectDiff(&wantedSchema, got.schema), &wantedSchema, got.schema)
}
gotListProperties := properties(got.listSchema.Properties)
if want := sets.NewString("apiVersion", "kind", "metadata", "items"); !gotListProperties.Equal(want) {
t.Fatalf("unexpected list properties, got: %s, expected: %s", gotListProperties.List(), want.List())
}
gotListSchema := got.listSchema.Properties["items"].Items.Schema
if !reflect.DeepEqual(&wantedItemsSchema, gotListSchema) {
t.Errorf("unexpected list schema: %s (want/got)", diff.ObjectDiff(&wantedItemsSchema, &gotListSchema))
}
})
}
}
func properties(p map[string]spec.Schema) sets.String {
ret := sets.NewString()
for k := range p {
ret.Insert(k)
}
return ret
}

218
staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/controller.go Normal file
View File

@ -0,0 +1,218 @@
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package openapi
import (
"fmt"
"reflect"
"sync"
"time"
"github.com/go-openapi/spec"
"k8s.io/apimachinery/pkg/api/errors"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/util/workqueue"
"k8s.io/klog"
"k8s.io/kube-openapi/pkg/handler"
"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions"
informers "k8s.io/apiextensions-apiserver/pkg/client/informers/internalversion/apiextensions/internalversion"
listers "k8s.io/apiextensions-apiserver/pkg/client/listers/apiextensions/internalversion"
)
// Controller watches CustomResourceDefinitions and publishes validation schema
type Controller struct {
crdLister listers.CustomResourceDefinitionLister
crdsSynced cache.InformerSynced
// To allow injection for testing.
syncFn func(string) error
queue workqueue.RateLimitingInterface
staticSpec *spec.Swagger
openAPIService *handler.OpenAPIService
// specs per version and per CRD name
lock sync.Mutex
crdSpecs map[string]map[string]*spec.Swagger
}
// NewController creates a new Controller with input CustomResourceDefinition informer
func NewController(crdInformer informers.CustomResourceDefinitionInformer) *Controller {
c := &Controller{
crdLister: crdInformer.Lister(),
crdsSynced: crdInformer.Informer().HasSynced,
queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "crd_openapi_controller"),
crdSpecs: map[string]map[string]*spec.Swagger{},
}
crdInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc: c.addCustomResourceDefinition,
UpdateFunc: c.updateCustomResourceDefinition,
DeleteFunc: c.deleteCustomResourceDefinition,
})
c.syncFn = c.sync
return c
}
// Run sets openAPIAggregationManager and starts workers
func (c *Controller) Run(staticSpec *spec.Swagger, openAPIService *handler.OpenAPIService, stopCh <-chan struct{}) {
defer utilruntime.HandleCrash()
defer c.queue.ShutDown()
defer klog.Infof("Shutting down OpenAPI controller")
klog.Infof("Starting OpenAPI controller")
c.staticSpec = staticSpec
c.openAPIService = openAPIService
if !cache.WaitForCacheSync(stopCh, c.crdsSynced) {
utilruntime.HandleError(fmt.Errorf("timed out waiting for caches to sync"))
return
}
// only start one worker thread since its a slow moving API
go wait.Until(c.runWorker, time.Second, stopCh)
<-stopCh
}
func (c *Controller) runWorker() {
for c.processNextWorkItem() {
}
}
func (c *Controller) processNextWorkItem() bool {
key, quit := c.queue.Get()
if quit {
return false
}
defer c.queue.Done(key)
// log slow aggregations
start := time.Now()
defer func() {
elapsed := time.Since(start)
if elapsed > time.Second {
klog.Warningf("slow openapi aggregation of %q: %s", key.(string), elapsed)
}
}()
err := c.syncFn(key.(string))
if err == nil {
c.queue.Forget(key)
return true
}
utilruntime.HandleError(fmt.Errorf("%v failed with: %v", key, err))
c.queue.AddRateLimited(key)
return true
}
func (c *Controller) sync(name string) error {
c.lock.Lock()
defer c.lock.Unlock()
crd, err := c.crdLister.Get(name)
if err != nil && !errors.IsNotFound(err) {
return err
}
// do we have to remove all specs of this CRD?
if errors.IsNotFound(err) || !apiextensions.IsCRDConditionTrue(crd, apiextensions.Established) {
if _, found := c.crdSpecs[name]; !found {
return nil
}
delete(c.crdSpecs, name)
return c.updateSpecLocked()
}
// compute CRD spec and see whether it changed
oldSpecs := c.crdSpecs[crd.Name]
newSpecs := map[string]*spec.Swagger{}
anyChanged := false
for _, v := range crd.Spec.Versions {
if !v.Served {
continue
}
spec, err := BuildSwagger(crd, v.Name)
if err != nil {
return err
}
newSpecs[v.Name] = spec
if oldSpecs[v.Name] == nil || !reflect.DeepEqual(oldSpecs[v.Name], spec) {
anyChanged = true
}
}
if !anyChanged && len(oldSpecs) == len(newSpecs) {
return nil
}
// update specs of this CRD
c.crdSpecs[crd.Name] = newSpecs
return c.updateSpecLocked()
}
// updateSpecLocked aggregates all OpenAPI specs and updates openAPIService.
// It is not thread-safe. The caller is responsible to hold proper lock (Controller.lock).
func (c *Controller) updateSpecLocked() error {
crdSpecs := []*spec.Swagger{}
for _, versionSpecs := range c.crdSpecs {
for _, s := range versionSpecs {
crdSpecs = append(crdSpecs, s)
}
}
return c.openAPIService.UpdateSpec(mergeSpecs(c.staticSpec, crdSpecs...))
}
func (c *Controller) addCustomResourceDefinition(obj interface{}) {
castObj := obj.(*apiextensions.CustomResourceDefinition)
klog.V(4).Infof("Adding customresourcedefinition %s", castObj.Name)
c.enqueue(castObj)
}
func (c *Controller) updateCustomResourceDefinition(oldObj, newObj interface{}) {
castNewObj := newObj.(*apiextensions.CustomResourceDefinition)
klog.V(4).Infof("Updating customresourcedefinition %s", castNewObj.Name)
c.enqueue(castNewObj)
}
func (c *Controller) deleteCustomResourceDefinition(obj interface{}) {
castObj, ok := obj.(*apiextensions.CustomResourceDefinition)
if !ok {
tombstone, ok := obj.(cache.DeletedFinalStateUnknown)
if !ok {
klog.Errorf("Couldn't get object from tombstone %#v", obj)
return
}
castObj, ok = tombstone.Obj.(*apiextensions.CustomResourceDefinition)
if !ok {
klog.Errorf("Tombstone contained object that is not expected %#v", obj)
return
}
}
klog.V(4).Infof("Deleting customresourcedefinition %q", castObj.Name)
c.enqueue(castObj)
}
func (c *Controller) enqueue(obj *apiextensions.CustomResourceDefinition) {
c.queue.Add(obj.Name)
}

7
staging/src/k8s.io/apiextensions-apiserver/pkg/features/kube_features.go
View File

@ -34,6 +34,12 @@ const (
// CustomResourceValidation is a list of validation methods for CustomResources
CustomResourceValidation utilfeature.Feature = "CustomResourceValidation"
// owner: @roycaihw, @sttts
// alpha: v1.14
//
// CustomResourcePublishOpenAPI enables publishing of CRD OpenAPI specs.
CustomResourcePublishOpenAPI utilfeature.Feature = "CustomResourcePublishOpenAPI"
// owner: @sttts, @nikhita
// alpha: v1.10
// beta: v1.11
@ -59,4 +65,5 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS
CustomResourceValidation: {Default: true, PreRelease: utilfeature.Beta},
CustomResourceSubresources: {Default: true, PreRelease: utilfeature.Beta},
CustomResourceWebhookConversion: {Default: false, PreRelease: utilfeature.Alpha},
CustomResourcePublishOpenAPI: {Default: false, PreRelease: utilfeature.Alpha},
}

33
staging/src/k8s.io/apiextensions-apiserver/pkg/generated/openapi/BUILD Normal file
View File

@ -0,0 +1,33 @@
package(default_visibility = ["//visibility:public"])
load("//build:code_generation.bzl", "gen_openapi", "openapi_deps")
load("@io_bazel_rules_go//go:def.bzl", "go_library")
gen_openapi(
outs = ["zz_generated.openapi.go"],
output_pkg = "k8s.io/apiextensions-apiserver/pkg/generated/openapi",
)
go_library(
name = "go_default_library",
srcs = [
"doc.go",
"zz_generated.openapi.go",
],
importmap = "k8s.io/kubernetes/vendor/k8s.io/apiextensions-apiserver/pkg/generated/openapi",
importpath = "k8s.io/apiextensions-apiserver/pkg/generated/openapi",
deps = openapi_deps(), # keep
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
)

18
staging/src/k8s.io/apiextensions-apiserver/pkg/generated/openapi/doc.go Normal file
View File

@ -0,0 +1,18 @@
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// openapi generated definitions.
package openapi

34
staging/src/k8s.io/apiserver/pkg/endpoints/installer.go
View File

@ -611,12 +611,12 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
Returns(http.StatusOK, "OK", producedObject).
Writes(producedObject)
if isGetterWithOptions {
if err := addObjectParams(ws, route, versionedGetOptions); err != nil {
if err := AddObjectParams(ws, route, versionedGetOptions); err != nil {
return nil, err
}
}
if isExporter {
if err := addObjectParams(ws, route, versionedExportOptions); err != nil {
if err := AddObjectParams(ws, route, versionedExportOptions); err != nil {
return nil, err
}
}
@ -638,7 +638,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
Produces(append(storageMeta.ProducesMIMETypes(action.Verb), allMediaTypes...)...).
Returns(http.StatusOK, "OK", versionedList).
Writes(versionedList)
if err := addObjectParams(ws, route, versionedListOptions); err != nil {
if err := AddObjectParams(ws, route, versionedListOptions); err != nil {
return nil, err
}
switch {
@ -674,7 +674,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
Returns(http.StatusCreated, "Created", producedObject).
Reads(defaultVersionedObject).
Writes(producedObject)
if err := addObjectParams(ws, route, versionedUpdateOptions); err != nil {
if err := AddObjectParams(ws, route, versionedUpdateOptions); err != nil {
return nil, err
}
addParams(route, action.Params)
@ -702,7 +702,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
Returns(http.StatusOK, "OK", producedObject).
Reads(metav1.Patch{}).
Writes(producedObject)
if err := addObjectParams(ws, route, versionedPatchOptions); err != nil {
if err := AddObjectParams(ws, route, versionedPatchOptions); err != nil {
return nil, err
}
addParams(route, action.Params)
@ -715,7 +715,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
handler = restfulCreateResource(creater, reqScope, admit)
}
handler = metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, handler)
article := getArticleForNoun(kind, " ")
article := GetArticleForNoun(kind, " ")
doc := "create" + article + kind
if isSubresource {
doc = "create " + subresource + " of" + article + kind
@ -732,13 +732,13 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
Returns(http.StatusAccepted, "Accepted", producedObject).
Reads(defaultVersionedObject).
Writes(producedObject)
if err := addObjectParams(ws, route, versionedCreateOptions); err != nil {
if err := AddObjectParams(ws, route, versionedCreateOptions); err != nil {
return nil, err
}
addParams(route, action.Params)
routes = append(routes, route)
case "DELETE": // Delete a resource.
article := getArticleForNoun(kind, " ")
article := GetArticleForNoun(kind, " ")
doc := "delete" + article + kind
if isSubresource {
doc = "delete " + subresource + " of" + article + kind
@ -755,7 +755,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
if isGracefulDeleter {
route.Reads(versionedDeleterObject)
route.ParameterNamed("body").Required(false)
if err := addObjectParams(ws, route, versionedDeleteOptions); err != nil {
if err := AddObjectParams(ws, route, versionedDeleteOptions); err != nil {
return nil, err
}
}
@ -774,7 +774,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
Produces(append(storageMeta.ProducesMIMETypes(action.Verb), mediaTypes...)...).
Writes(versionedStatus).
Returns(http.StatusOK, "OK", versionedStatus)
if err := addObjectParams(ws, route, versionedListOptions); err != nil {
if err := AddObjectParams(ws, route, versionedListOptions); err != nil {
return nil, err
}
addParams(route, action.Params)
@ -794,7 +794,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
Produces(allMediaTypes...).
Returns(http.StatusOK, "OK", versionedWatchEvent).
Writes(versionedWatchEvent)
if err := addObjectParams(ws, route, versionedListOptions); err != nil {
if err := AddObjectParams(ws, route, versionedListOptions); err != nil {
return nil, err
}
addParams(route, action.Params)
@ -814,7 +814,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
Produces(allMediaTypes...).
Returns(http.StatusOK, "OK", versionedWatchEvent).
Writes(versionedWatchEvent)
if err := addObjectParams(ws, route, versionedListOptions); err != nil {
if err := AddObjectParams(ws, route, versionedListOptions); err != nil {
return nil, err
}
addParams(route, action.Params)
@ -838,7 +838,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
Consumes("*/*").
Writes(connectProducedObject)
if versionedConnectOptions != nil {
if err := addObjectParams(ws, route, versionedConnectOptions); err != nil {
if err := AddObjectParams(ws, route, versionedConnectOptions); err != nil {
return nil, err
}
}
@ -907,13 +907,13 @@ func addParams(route *restful.RouteBuilder, params []*restful.Parameter) {
}
}
// addObjectParams converts a runtime.Object into a set of go-restful Param() definitions on the route.
// AddObjectParams converts a runtime.Object into a set of go-restful Param() definitions on the route.
// The object must be a pointer to a struct; only fields at the top level of the struct that are not
// themselves interfaces or structs are used; only fields with a json tag that is non empty (the standard
// Go JSON behavior for omitting a field) become query parameters. The name of the query parameter is
// the JSON field name. If a description struct tag is set on the field, that description is used on the
// query parameter. In essence, it converts a standard JSON top level object into a query param schema.
func addObjectParams(ws *restful.WebService, route *restful.RouteBuilder, obj interface{}) error {
func AddObjectParams(ws *restful.WebService, route *restful.RouteBuilder, obj interface{}) error {
sv, err := conversion.EnforcePtr(obj)
if err != nil {
return err
@ -1015,8 +1015,8 @@ func splitSubresource(path string) (string, string, error) {
return resource, subresource, nil
}
// getArticleForNoun returns the article needed for the given noun.
func getArticleForNoun(noun string, padding string) string {
// GetArticleForNoun returns the article needed for the given noun.
func GetArticleForNoun(noun string, padding string) string {
if noun[len(noun)-2:] != "ss" && noun[len(noun)-1:] == "s" {
// Plurals don't have an article.
// Don't catch words like class

2
staging/src/k8s.io/apiserver/pkg/endpoints/installer_test.go
View File

@ -82,7 +82,7 @@ func TestGetArticleForNoun(t *testing.T) {
},
}
for _, tt := range tests {
if got := getArticleForNoun(tt.noun, tt.padding); got != tt.want {
if got := GetArticleForNoun(tt.noun, tt.padding); got != tt.want {
t.Errorf("%q. GetArticleForNoun() = %v, want %v", tt.noun, got, tt.want)
}
}

29
staging/src/k8s.io/kube-aggregator/pkg/controllers/openapi/aggregator/aggregator.go
View File

@ -19,6 +19,7 @@ package aggregator
import (
"fmt"
"net/http"
"strings"
"sync"
"time"
@ -40,17 +41,35 @@ type SpecAggregator interface {
UpdateAPIServiceSpec(apiServiceName string, spec *spec.Swagger, etag string) error
RemoveAPIServiceSpec(apiServiceName string) error
GetAPIServiceInfo(apiServiceName string) (handler http.Handler, etag string, exists bool)
GetAPIServiceNames() []string
}
const (
aggregatorUser = "system:aggregator"
specDownloadTimeout = 60 * time.Second
localDelegateChainNamePattern = "k8s_internal_local_delegation_chain_%010d"
localDelegateChainNamePrefix = "k8s_internal_local_delegation_chain_"
localDelegateChainNamePattern = localDelegateChainNamePrefix + "%010d"
// A randomly generated UUID to differentiate local and remote eTags.
locallyGeneratedEtagPrefix = "\"6E8F849B434D4B98A569B9D7718876E9-"
)
// IsLocalAPIService returns true for local specs from delegates.
func IsLocalAPIService(apiServiceName string) bool {
return strings.HasPrefix(apiServiceName, localDelegateChainNamePrefix)
}
// GetAPIServicesName returns the names of APIServices recorded in specAggregator.openAPISpecs.
// We use this function to pass the names of local APIServices to the controller in this package,
// so that the controller can periodically sync the OpenAPI spec from delegation API servers.
func (s *specAggregator) GetAPIServiceNames() []string {
names := make([]string, len(s.openAPISpecs))
for key := range s.openAPISpecs {
names = append(names, key)
}
return names
}
// BuildAndRegisterAggregator registered OpenAPI aggregator handler. This function is not thread safe as it only being called on startup.
func BuildAndRegisterAggregator(downloader *Downloader, delegationTarget server.DelegationTarget, webServices []*restful.WebService,
config *common.Config, pathHandler common.PathHandler) (SpecAggregator, error) {
@ -161,6 +180,7 @@ func (s *specAggregator) buildOpenAPISpec() (specToReturn *spec.Swagger, err err
return nil, err
}
}
return specToReturn, nil
}
@ -179,7 +199,14 @@ func (s *specAggregator) updateOpenAPISpec() error {
// tryUpdatingServiceSpecs tries updating openAPISpecs map with specified specInfo, and keeps the map intact
// if the update fails.
func (s *specAggregator) tryUpdatingServiceSpecs(specInfo *openAPISpecInfo) error {
if specInfo == nil {
return fmt.Errorf("invalid input: specInfo must be non-nil")
}
orgSpecInfo, exists := s.openAPISpecs[specInfo.apiService.Name]
// Skip aggregation if OpenAPI spec didn't change
if exists && orgSpecInfo != nil && orgSpecInfo.etag == specInfo.etag {
return nil
}
s.openAPISpecs[specInfo.apiService.Name] = specInfo
if err := s.updateOpenAPISpec(); err != nil {
if exists {

19
staging/src/k8s.io/kube-aggregator/pkg/controllers/openapi/controller.go
View File

@ -30,8 +30,9 @@ import (
)
const (
successfulUpdateDelay = time.Minute
failedUpdateMaxExpDelay = time.Hour
successfulUpdateDelay = time.Minute
successfulUpdateDelayLocal = time.Second
failedUpdateMaxExpDelay = time.Hour
)
type syncAction int
@ -64,6 +65,11 @@ func NewAggregationController(downloader *aggregator.Downloader, openAPIAggregat
c.syncHandler = c.sync
// update each service at least once, also those which are not coming from APIServices, namely local services
for _, name := range openAPIAggregationManager.GetAPIServiceNames() {
c.queue.AddAfter(name, time.Second)
}
return c
}
@ -104,8 +110,13 @@ func (c *AggregationController) processNextWorkItem() bool {
switch action {
case syncRequeue:
klog.Infof("OpenAPI AggregationController: action for item %s: Requeue.", key)
c.queue.AddAfter(key, successfulUpdateDelay)
if aggregator.IsLocalAPIService(key.(string)) {
klog.V(7).Infof("OpenAPI AggregationController: action for local item %s: Requeue after %s.", key, successfulUpdateDelayLocal)
c.queue.AddAfter(key, successfulUpdateDelayLocal)
} else {
klog.V(7).Infof("OpenAPI AggregationController: action for item %s: Requeue.", key)
c.queue.AddAfter(key, successfulUpdateDelay)
}
case syncRequeueRateLimited:
klog.Infof("OpenAPI AggregationController: action for item %s: Rate Limited Requeue.", key)
c.queue.AddRateLimited(key)

7
test/e2e/apimachinery/BUILD
View File

@ -12,6 +12,7 @@ go_library(
"certs.go",
"chunking.go",
"crd_conversion_webhook.go",
"crd_publish_openapi.go",
"crd_watch.go",
"custom_resource_definition.go",
"etcd_failure.go",
@ -40,7 +41,9 @@ go_library(
"//staging/src/k8s.io/api/rbac/v1:go_default_library",
"//staging/src/k8s.io/api/rbac/v1beta1:go_default_library",
"//staging/src/k8s.io/api/scheduling/v1:go_default_library",
"//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions:go_default_library",
"//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library",
"//staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/validation:go_default_library",
"//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset:go_default_library",
"//staging/src/k8s.io/apiextensions-apiserver/test/integration:go_default_library",
"//staging/src/k8s.io/apiextensions-apiserver/test/integration/fixtures:go_default_library",
@ -60,6 +63,7 @@ go_library(
"//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/version:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/yaml:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/storage/names:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/storage/storagebackend:go_default_library",
@ -77,9 +81,12 @@ go_library(
"//test/e2e/framework/metrics:go_default_library",
"//test/utils:go_default_library",
"//test/utils/image:go_default_library",
"//vendor/github.com/go-openapi/spec:go_default_library",
"//vendor/github.com/onsi/ginkgo:go_default_library",
"//vendor/github.com/onsi/gomega:go_default_library",
"//vendor/github.com/stretchr/testify/assert:go_default_library",
"//vendor/k8s.io/kube-openapi/pkg/util:go_default_library",
"//vendor/sigs.k8s.io/yaml:go_default_library",
],
)

552
test/e2e/apimachinery/crd_publish_openapi.go Normal file
View File

@ -0,0 +1,552 @@
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package apimachinery
import (
"encoding/json"
"fmt"
"regexp"
"strings"
"time"
"github.com/go-openapi/spec"
. "github.com/onsi/ginkgo"
"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions"
"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
"k8s.io/apiextensions-apiserver/pkg/apiserver/validation"
apiequality "k8s.io/apimachinery/pkg/api/equality"
"k8s.io/apimachinery/pkg/types"
utilversion "k8s.io/apimachinery/pkg/util/version"
"k8s.io/apimachinery/pkg/util/wait"
utilyaml "k8s.io/apimachinery/pkg/util/yaml"
k8sclientset "k8s.io/client-go/kubernetes"
openapiutil "k8s.io/kube-openapi/pkg/util"
"k8s.io/kubernetes/test/e2e/framework"
"sigs.k8s.io/yaml"
)
var (
crdPublishOpenAPIVersion = utilversion.MustParseSemantic("v1.14.0")
metaPattern = `"kind":"%s","apiVersion":"%s/%s","metadata":{"name":"%s"}`
)
var _ = SIGDescribe("CustomResourcePublishOpenAPI [Feature:CustomResourcePublishOpenAPI]", func() {
f := framework.NewDefaultFramework("crd-publish-openapi")
BeforeEach(func() {
framework.SkipUnlessServerVersionGTE(crdPublishOpenAPIVersion, f.ClientSet.Discovery())
})
It("works for CRD with validation schema", func() {
crd, err := setupCRD(f, schemaFoo, "foo", "v1")
if err != nil {
framework.Failf("%v", err)
}
meta := fmt.Sprintf(metaPattern, crd.Kind, crd.ApiGroup, crd.Versions[0].Name, "test-foo")
ns := fmt.Sprintf("--namespace=%v", f.Namespace.Name)
By("client-side validation (kubectl create and apply) allows request with known and required properties")
validCR := fmt.Sprintf(`{%s,"spec":{"bars":[{"name":"test-bar"}]}}`, meta)
if _, err := framework.RunKubectlInput(validCR, ns, "create", "-f", "-"); err != nil {
framework.Failf("failed to create valid CR %s: %v", validCR, err)
}
if _, err := framework.RunKubectl(ns, "delete", crd.GetPluralName(), "test-foo"); err != nil {
framework.Failf("failed to delete valid CR: %v", err)
}
if _, err := framework.RunKubectlInput(validCR, ns, "apply", "-f", "-"); err != nil {
framework.Failf("failed to apply valid CR %s: %v", validCR, err)
}
if _, err := framework.RunKubectl(ns, "delete", crd.GetPluralName(), "test-foo"); err != nil {
framework.Failf("failed to delete valid CR: %v", err)
}
By("client-side validation (kubectl create and apply) rejects request with unknown properties when disallowed by the schema")
unknownCR := fmt.Sprintf(`{%s,"spec":{"foo":true}}`, meta)
if _, err := framework.RunKubectlInput(unknownCR, ns, "create", "-f", "-"); err == nil || !strings.Contains(err.Error(), `unknown field "foo"`) {
framework.Failf("unexpected no error when creating CR with unknown field: %v", err)
}
if _, err := framework.RunKubectlInput(unknownCR, ns, "apply", "-f", "-"); err == nil || !strings.Contains(err.Error(), `unknown field "foo"`) {
framework.Failf("unexpected no error when applying CR with unknown field: %v", err)
}
By("client-side validation (kubectl create and apply) rejects request without required properties")
noRequireCR := fmt.Sprintf(`{%s,"spec":{"bars":[{"age":"10"}]}}`, meta)
if _, err := framework.RunKubectlInput(noRequireCR, ns, "create", "-f", "-"); err == nil || !strings.Contains(err.Error(), `missing required field "name"`) {
framework.Failf("unexpected no error when creating CR without required field: %v", err)
}
if _, err := framework.RunKubectlInput(noRequireCR, ns, "apply", "-f", "-"); err == nil || !strings.Contains(err.Error(), `missing required field "name"`) {
framework.Failf("unexpected no error when applying CR without required field: %v", err)
}
By("kubectl explain works to explain CR properties")
if err := verifyKubectlExplain(crd.GetPluralName(), `(?s)DESCRIPTION:.*Foo CRD for Testing.*FIELDS:.*apiVersion.*<string>.*APIVersion defines.*spec.*<Object>.*Specification of Foo`); err != nil {
framework.Failf("%v", err)
}
By("kubectl explain works to explain CR properties recursively")
if err := verifyKubectlExplain(crd.GetPluralName()+".metadata", `(?s)DESCRIPTION:.*Standard object's metadata.*FIELDS:.*creationTimestamp.*<string>.*CreationTimestamp is a timestamp`); err != nil {
framework.Failf("%v", err)
}
if err := verifyKubectlExplain(crd.GetPluralName()+".spec", `(?s)DESCRIPTION:.*Specification of Foo.*FIELDS:.*bars.*<\[\]Object>.*List of Bars and their specs`); err != nil {
framework.Failf("%v", err)
}
if err := verifyKubectlExplain(crd.GetPluralName()+".spec.bars", `(?s)RESOURCE:.*bars.*<\[\]Object>.*DESCRIPTION:.*List of Bars and their specs.*FIELDS:.*bazs.*<\[\]string>.*List of Bazs.*name.*<string>.*Name of Bar`); err != nil {
framework.Failf("%v", err)
}
By("kubectl explain works to return error when explain is called on property that doesn't exist")
if _, err := framework.RunKubectl("explain", crd.GetPluralName()+".spec.bars2"); err == nil || !strings.Contains(err.Error(), `field "bars2" does not exist`) {
framework.Failf("unexpected no error when explaining property that doesn't exist: %v", err)
}
if err := cleanupCRD(f, crd); err != nil {
framework.Failf("%v", err)
}
})
It("works for CRD without validation schema", func() {
crd, err := setupCRD(f, nil, "empty", "v1")
if err != nil {
framework.Failf("%v", err)
}
meta := fmt.Sprintf(metaPattern, crd.Kind, crd.ApiGroup, crd.Versions[0].Name, "test-cr")
ns := fmt.Sprintf("--namespace=%v", f.Namespace.Name)
By("client-side validation (kubectl create and apply) allows request with any unknown properties")
randomCR := fmt.Sprintf(`{%s,"a":{"b":[{"c":"d"}]}}`, meta)
if _, err := framework.RunKubectlInput(randomCR, ns, "create", "-f", "-"); err != nil {
framework.Failf("failed to create random CR %s for CRD without schema: %v", randomCR, err)
}
if _, err := framework.RunKubectl(ns, "delete", crd.GetPluralName(), "test-cr"); err != nil {
framework.Failf("failed to delete random CR: %v", err)
}
if _, err := framework.RunKubectlInput(randomCR, ns, "apply", "-f", "-"); err != nil {
framework.Failf("failed to apply random CR %s for CRD without schema: %v", randomCR, err)
}
if _, err := framework.RunKubectl(ns, "delete", crd.GetPluralName(), "test-cr"); err != nil {
framework.Failf("failed to delete random CR: %v", err)
}
By("kubectl explain works to explain CR without validation schema")
if err := verifyKubectlExplain(crd.GetPluralName(), `(?s)DESCRIPTION:.*<empty>`); err != nil {
framework.Failf("%v", err)
}
if err := cleanupCRD(f, crd); err != nil {
framework.Failf("%v", err)
}
})
It("works for multiple CRDs of different groups", func() {
By("CRs in different groups (two CRDs) show up in OpenAPI documentation")
crdFoo, err := setupCRD(f, schemaFoo, "foo", "v1")
if err != nil {
framework.Failf("%v", err)
}
crdWaldo, err := setupCRD(f, schemaWaldo, "waldo", "v1beta1")
if err != nil {
framework.Failf("%v", err)
}
if crdFoo.ApiGroup == crdWaldo.ApiGroup {
framework.Failf("unexpected: CRDs should be of different group %v, %v", crdFoo.ApiGroup, crdWaldo.ApiGroup)
}
if err := waitForDefinition(f.ClientSet, definitionName(crdWaldo, "v1beta1"), schemaWaldo); err != nil {
framework.Failf("%v", err)
}
if err := waitForDefinition(f.ClientSet, definitionName(crdFoo, "v1"), schemaFoo); err != nil {
framework.Failf("%v", err)
}
if err := cleanupCRD(f, crdFoo); err != nil {
framework.Failf("%v", err)
}
if err := cleanupCRD(f, crdWaldo); err != nil {
framework.Failf("%v", err)
}
})
It("works for multiple CRDs of same group but different versions", func() {
By("CRs in the same group but different versions (one multiversion CRD) show up in OpenAPI documentation")
crdMultiVer, err := setupCRD(f, schemaFoo, "multi-ver", "v2", "v3")
if err != nil {
framework.Failf("%v", err)
}
if err := waitForDefinition(f.ClientSet, definitionName(crdMultiVer, "v3"), schemaFoo); err != nil {
framework.Failf("%v", err)
}
if err := waitForDefinition(f.ClientSet, definitionName(crdMultiVer, "v2"), schemaFoo); err != nil {
framework.Failf("%v", err)
}
if err := cleanupCRD(f, crdMultiVer); err != nil {
framework.Failf("%v", err)
}
By("CRs in the same group but different versions (two CRDs) show up in OpenAPI documentation")
crdFoo, err := setupCRD(f, schemaFoo, "common-group", "v4")
if err != nil {
framework.Failf("%v", err)
}
crdWaldo, err := setupCRD(f, schemaWaldo, "common-group", "v5")
if err != nil {
framework.Failf("%v", err)
}
if crdFoo.ApiGroup != crdWaldo.ApiGroup {
framework.Failf("unexpected: CRDs should be of the same group %v, %v", crdFoo.ApiGroup, crdWaldo.ApiGroup)
}
if err := waitForDefinition(f.ClientSet, definitionName(crdWaldo, "v5"), schemaWaldo); err != nil {
framework.Failf("%v", err)
}
if err := waitForDefinition(f.ClientSet, definitionName(crdFoo, "v4"), schemaFoo); err != nil {
framework.Failf("%v", err)
}
if err := cleanupCRD(f, crdFoo); err != nil {
framework.Failf("%v", err)
}
if err := cleanupCRD(f, crdWaldo); err != nil {
framework.Failf("%v", err)
}
})
It("works for multiple CRDs of same group and version but different kinds", func() {
By("CRs in the same group and version but different kinds (two CRDs) show up in OpenAPI documentation")
crdFoo, err := setupCRD(f, schemaFoo, "common-group", "v6")
if err != nil {
framework.Failf("%v", err)
}
crdWaldo, err := setupCRD(f, schemaWaldo, "common-group", "v6")
if err != nil {
framework.Failf("%v", err)
}
if crdFoo.ApiGroup != crdWaldo.ApiGroup {
framework.Failf("unexpected: CRDs should be of the same group %v, %v", crdFoo.ApiGroup, crdWaldo.ApiGroup)
}
if err := waitForDefinition(f.ClientSet, definitionName(crdWaldo, "v6"), schemaWaldo); err != nil {
framework.Failf("%v", err)
}
if err := waitForDefinition(f.ClientSet, definitionName(crdFoo, "v6"), schemaFoo); err != nil {
framework.Failf("%v", err)
}
if err := cleanupCRD(f, crdFoo); err != nil {
framework.Failf("%v", err)
}
if err := cleanupCRD(f, crdWaldo); err != nil {
framework.Failf("%v", err)
}
})
It("updates the published spec when one versin gets renamed", func() {
By("set up a multi version CRD")
crdMultiVer, err := setupCRD(f, schemaFoo, "multi-ver", "v2", "v3")
if err != nil {
framework.Failf("%v", err)
}
if err := waitForDefinition(f.ClientSet, definitionName(crdMultiVer, "v3"), schemaFoo); err != nil {
framework.Failf("%v", err)
}
if err := waitForDefinition(f.ClientSet, definitionName(crdMultiVer, "v2"), schemaFoo); err != nil {
framework.Failf("%v", err)
}
By("rename a version")
patch := []byte(`{"spec":{"versions":[{"name":"v2","served":true,"storage":true},{"name":"v4","served":true,"storage":false}]}}`)
crdMultiVer.Crd, err = crdMultiVer.ApiExtensionClient.ApiextensionsV1beta1().CustomResourceDefinitions().Patch(crdMultiVer.GetMetaName(), types.MergePatchType, patch)
if err != nil {
framework.Failf("%v", err)
}
By("check the new version name is served")
if err := waitForDefinition(f.ClientSet, definitionName(crdMultiVer, "v4"), schemaFoo); err != nil {
framework.Failf("%v", err)
}
By("check the old version name is removed")
if err := waitForDefinitionCleanup(f.ClientSet, definitionName(crdMultiVer, "v3")); err != nil {
framework.Failf("%v", err)
}
By("check the other version is not changed")
if err := waitForDefinition(f.ClientSet, definitionName(crdMultiVer, "v2"), schemaFoo); err != nil {
framework.Failf("%v", err)
}
// TestCrd.Versions is different from TestCrd.Crd.Versions, we have to manually
// update the name there. Used by cleanupCRD
crdMultiVer.Versions[1].Name = "v4"
if err := cleanupCRD(f, crdMultiVer); err != nil {
framework.Failf("%v", err)
}
})
It("removes definition from spec when one versin gets changed to not be served", func() {
By("set up a multi version CRD")
crd, err := setupCRD(f, schemaFoo, "multi-to-single-ver", "v5", "v6alpha1")
if err != nil {
framework.Failf("%v", err)
}
// just double check. setupCRD() checked this for us already
if err := waitForDefinition(f.ClientSet, definitionName(crd, "v6alpha1"), schemaFoo); err != nil {
framework.Failf("%v", err)
}
if err := waitForDefinition(f.ClientSet, definitionName(crd, "v5"), schemaFoo); err != nil {
framework.Failf("%v", err)
}
By("mark a version not serverd")
crd.Crd.Spec.Versions[1].Served = false
crd.Crd, err = crd.ApiExtensionClient.ApiextensionsV1beta1().CustomResourceDefinitions().Update(crd.Crd)
if err != nil {
framework.Failf("%v", err)
}
By("check the unserved version gets removed")
if err := waitForDefinitionCleanup(f.ClientSet, definitionName(crd, "v6alpha1")); err != nil {
framework.Failf("%v", err)
}
By("check the other version is not changed")
if err := waitForDefinition(f.ClientSet, definitionName(crd, "v5"), schemaFoo); err != nil {
framework.Failf("%v", err)
}
if err := cleanupCRD(f, crd); err != nil {
framework.Failf("%v", err)
}
})
})
func setupCRD(f *framework.Framework, schema []byte, groupSuffix string, versions ...string) (*framework.TestCrd, error) {
group := fmt.Sprintf("%s-test-%s.k8s.io", f.BaseName, groupSuffix)
if len(versions) == 0 {
return nil, fmt.Errorf("require at least one version for CRD")
}
apiVersions := []v1beta1.CustomResourceDefinitionVersion{}
for _, version := range versions {
v := v1beta1.CustomResourceDefinitionVersion{
Name: version,
Served: true,
Storage: false,
}
apiVersions = append(apiVersions, v)
}
apiVersions[0].Storage = true
crd, err := framework.CreateMultiVersionTestCRD(f, group, apiVersions, nil)
if err != nil {
return nil, fmt.Errorf("failed to create CRD: %v", err)
}
if schema != nil {
// patch validation schema for all versions
if err := patchSchema(schema, crd); err != nil {
return nil, fmt.Errorf("failed to patch schema: %v", err)
}
} else {
// change expectation if CRD doesn't have schema
schema = []byte(`type: object`)
}
for _, v := range crd.Versions {
if err := waitForDefinition(f.ClientSet, definitionName(crd, v.Name), schema); err != nil {
return nil, fmt.Errorf("%v", err)
}
}
return crd, nil
}
func cleanupCRD(f *framework.Framework, crd *framework.TestCrd) error {
crd.CleanUp()
for _, v := range crd.Versions {
name := definitionName(crd, v.Name)
if err := waitForDefinitionCleanup(f.ClientSet, name); err != nil {
return fmt.Errorf("%v", err)
}
}
return nil
}
// patchSchema takes schema in YAML and patches it to given CRD in given version
func patchSchema(schema []byte, crd *framework.TestCrd) error {
s, err := utilyaml.ToJSON(schema)
if err != nil {
return fmt.Errorf("failed to create json patch: %v", err)
}
patch := []byte(fmt.Sprintf(`{"spec":{"validation":{"openAPIV3Schema":%s}}}`, string(s)))
crd.Crd, err = crd.ApiExtensionClient.ApiextensionsV1beta1().CustomResourceDefinitions().Patch(crd.GetMetaName(), types.MergePatchType, patch)
return err
}
// waitForDefinition waits for given definition showing up in swagger with given schema
func waitForDefinition(c k8sclientset.Interface, name string, schema []byte) error {
expect := spec.Schema{}
if err := convertJSONSchemaProps(schema, &expect); err != nil {
return err
}
lastMsg := ""
if err := wait.Poll(500*time.Millisecond, 10*time.Second, func() (bool, error) {
bs, err := c.CoreV1().RESTClient().Get().AbsPath("openapi", "v2").DoRaw()
if err != nil {
return false, err
}
spec := spec.Swagger{}
if err := json.Unmarshal(bs, &spec); err != nil {
return false, err
}
d, ok := spec.SwaggerProps.Definitions[name]
if !ok {
lastMsg = fmt.Sprintf("spec.SwaggerProps.Definitions[\"%s\"] not found", name)
return false, nil
}
// drop properties and extension that we added
dropDefaults(&d)
if !apiequality.Semantic.DeepEqual(expect, d) {
lastMsg = fmt.Sprintf("spec.SwaggerProps.Definitions[\"%s\"] not match; expect: %v, actual: %v", name, expect, d)
return false, nil
}
return true, nil
}); err != nil {
return fmt.Errorf("failed to wait for definition %s to be served: %v; lastMsg: %s", name, err, lastMsg)
}
return nil
}
// waitForDefinitionCleanup waits for given definition to be removed from swagger
func waitForDefinitionCleanup(c k8sclientset.Interface, name string) error {
lastMsg := ""
if err := wait.Poll(500*time.Millisecond, 10*time.Second, func() (bool, error) {
bs, err := c.CoreV1().RESTClient().Get().AbsPath("openapi", "v2").DoRaw()
if err != nil {
return false, err
}
spec := spec.Swagger{}
if err := json.Unmarshal(bs, &spec); err != nil {
return false, err
}
_, ok := spec.SwaggerProps.Definitions[name]
if ok {
lastMsg = fmt.Sprintf("spec.SwaggerProps.Definitions[\"%s\"] still exists", name)
return false, nil
}
return true, nil
}); err != nil {
return fmt.Errorf("failed to wait for definition %s to be removed: %v; lastMsg: %s", name, err, lastMsg)
}
return nil
}
// convertJSONSchemaProps converts JSONSchemaProps in YAML to spec.Schema
func convertJSONSchemaProps(in []byte, out *spec.Schema) error {
external := v1beta1.JSONSchemaProps{}
if err := yaml.UnmarshalStrict(in, &external); err != nil {
return err
}
internal := apiextensions.JSONSchemaProps{}
if err := v1beta1.Convert_v1beta1_JSONSchemaProps_To_apiextensions_JSONSchemaProps(&external, &internal, nil); err != nil {
return err
}
if err := validation.ConvertJSONSchemaProps(&internal, out); err != nil {
return err
}
return nil
}
// dropDefaults drops properties and extension that we added to a schema
func dropDefaults(s *spec.Schema) {
delete(s.Properties, "metadata")
delete(s.Properties, "apiVersion")
delete(s.Properties, "kind")
delete(s.Extensions, "x-kubernetes-group-version-kind")
}
func verifyKubectlExplain(name, pattern string) error {
result, err := framework.RunKubectl("explain", name)
if err != nil {
return fmt.Errorf("failed to explain %s: %v", name, err)
}
r := regexp.MustCompile(pattern)
if !r.Match([]byte(result)) {
return fmt.Errorf("kubectl explain %s result {%s} doesn't match pattern {%s}", name, result, pattern)
}
return nil
}
// definitionName returns the openapi definition name for given CRD in given version
func definitionName(crd *framework.TestCrd, version string) string {
return openapiutil.ToRESTFriendlyName(fmt.Sprintf("%s/%s/%s", crd.ApiGroup, version, crd.Kind))
}
var schemaFoo = []byte(`description: Foo CRD for Testing
type: object
properties:
spec:
type: object
description: Specification of Foo
properties:
bars:
description: List of Bars and their specs.
type: array
items:
type: object
required:
- name
properties:
name:
description: Name of Bar.
type: string
age:
description: Age of Bar.
type: string
bazs:
description: List of Bazs.
items:
type: string
type: array
status:
description: Status of Foo
type: object
properties:
bars:
description: List of Bars and their statuses.
type: array
items:
type: object
properties:
name:
description: Name of Bar.
type: string
available:
description: Whether the Bar is installed.
type: boolean
quxType:
description: Indicates to external qux type.
pattern: in-tree|out-of-tree
type: string`)
var schemaWaldo = []byte(`description: Waldo CRD for Testing
type: object
properties:
spec:
description: Specification of Waldo
type: object
properties:
dummy:
description: Dummy property.
status:
description: Status of Waldo
type: object
properties:
bars:
description: List of Bars and their statuses.`)

1
test/integration/master/BUILD
View File

@ -66,6 +66,7 @@ go_test(
"//test/integration/framework:go_default_library",
"//test/utils:go_default_library",
"//vendor/github.com/evanphx/json-patch:go_default_library",
"//vendor/github.com/go-openapi/spec:go_default_library",
"//vendor/github.com/stretchr/testify/require:go_default_library",
"//vendor/sigs.k8s.io/yaml:go_default_library",
] + select({

88
test/integration/master/crd_test.go
View File

@ -18,15 +18,19 @@ package master
import (
"encoding/json"
"fmt"
"testing"
"time"
"github.com/go-openapi/spec"
networkingv1 "k8s.io/api/networking/v1"
apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/kubernetes"
kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
@ -130,6 +134,90 @@ func TestCRD(t *testing.T) {
}
}
func TestCRDOpenAPI(t *testing.T) {
result := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--feature-gates=CustomResourcePublishOpenAPI=true"}, framework.SharedEtcd())
defer result.TearDownFn()
kubeclient, err := kubernetes.NewForConfig(result.ClientConfig)
if err != nil {
t.Fatalf("Unexpected error: %v", err)
}
apiextensionsclient, err := apiextensionsclientset.NewForConfig(result.ClientConfig)
if err != nil {
t.Fatalf("Unexpected error: %v", err)
}
t.Logf("Trying to create a custom resource without conflict")
crd := &apiextensionsv1beta1.CustomResourceDefinition{
ObjectMeta: metav1.ObjectMeta{
Name: "foos.cr.bar.com",
},
Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
Group: "cr.bar.com",
Version: "v1",
Scope: apiextensionsv1beta1.NamespaceScoped,
Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
Plural: "foos",
Kind: "Foo",
},
Validation: &apiextensionsv1beta1.CustomResourceValidation{
OpenAPIV3Schema: &apiextensionsv1beta1.JSONSchemaProps{
Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{
"foo": {Type: "string"},
},
},
},
},
}
etcd.CreateTestCRDs(t, apiextensionsclient, false, crd)
waitForSpec := func(expectedType string) {
t.Logf(`Waiting for {properties: {"foo": {"type":"%s"}}} to show up in schema`, expectedType)
lastMsg := ""
if err := wait.PollImmediate(500*time.Millisecond, 10*time.Second, func() (bool, error) {
lastMsg = ""
bs, err := kubeclient.RESTClient().Get().AbsPath("openapi", "v2").DoRaw()
if err != nil {
return false, err
}
spec := spec.Swagger{}
if err := json.Unmarshal(bs, &spec); err != nil {
return false, err
}
if spec.SwaggerProps.Paths == nil {
lastMsg = "spec.SwaggerProps.Paths is nil"
return false, nil
}
d, ok := spec.SwaggerProps.Definitions["com.bar.cr.v1.Foo"]
if !ok {
lastMsg = `spec.SwaggerProps.Definitions["com.bar.cr.v1.Foo"] not found`
return false, nil
}
p, ok := d.Properties["foo"]
if !ok {
lastMsg = `spec.SwaggerProps.Definitions["com.bar.cr.v1.Foo"].Properties["foo"] not found`
return false, nil
}
if !p.Type.Contains(expectedType) {
lastMsg = fmt.Sprintf(`spec.SwaggerProps.Definitions["com.bar.cr.v1.Foo"].Properties["foo"].Type should be %q, but got: %q`, expectedType, p.Type)
return false, nil
}
return true, nil
}); err != nil {
t.Fatalf("Failed to see %s OpenAPI spec in discovery: %v, last message: %s", crd.Name, err, lastMsg)
}
}
waitForSpec("string")
crd, err = apiextensionsclient.ApiextensionsV1beta1().CustomResourceDefinitions().Get(crd.Name, metav1.GetOptions{})
if err != nil {
t.Fatal(err)
}
prop := crd.Spec.Validation.OpenAPIV3Schema.Properties["foo"]
prop.Type = "boolean"
crd.Spec.Validation.OpenAPIV3Schema.Properties["foo"] = prop
if _, err = apiextensionsclient.ApiextensionsV1beta1().CustomResourceDefinitions().Update(crd); err != nil {
t.Fatal(err)
}
waitForSpec("boolean")
}
type Foo struct {
metav1.TypeMeta `json:",inline"`
metav1.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`