diff --git a/build/root/Makefile.generated_files b/build/root/Makefile.generated_files index 3c4327ad61..6f354e25e7 100644 --- a/build/root/Makefile.generated_files +++ b/build/root/Makefile.generated_files @@ -409,8 +409,10 @@ $(CONVERSION_GEN): $(k8s.io/kubernetes/vendor/k8s.io/code-generator/cmd/conversi OPENAPI_BASENAME := $(GENERATED_FILE_PREFIX)openapi OPENAPI_FILENAME := $(OPENAPI_BASENAME).go OPENAPI_OUTPUT_PKG := pkg/generated/openapi +CRD_OPENAPI_OUTPUT_PKG := staging/src/k8s.io/apiextensions-apiserver/pkg/generated/openapi BOILERPLATE_FILENAME := vendor/k8s.io/code-generator/hack/boilerplate.go.txt REPORT_FILENAME := $(OUT_DIR)/violations.report +IGNORED_REPORT_FILENAME := $(OUT_DIR)/ignored_violations.report KNOWN_VIOLATION_FILENAME := api/api-rules/violation_exceptions.list # When UPDATE_API_KNOWN_VIOLATIONS is set to be true, let the generator to write # updated API violations to the known API violation exceptions list. @@ -436,10 +438,11 @@ OPENAPI_DIRS := $(shell \ ) OPENAPI_OUTFILE := $(OPENAPI_OUTPUT_PKG)/$(OPENAPI_FILENAME) +CRD_OPENAPI_OUTFILE := $(CRD_OPENAPI_OUTPUT_PKG)/$(OPENAPI_FILENAME) # This rule is the user-friendly entrypoint for openapi generation. .PHONY: gen_openapi -gen_openapi: $(OPENAPI_OUTFILE) $(OPENAPI_GEN) +gen_openapi: $(OPENAPI_OUTFILE) $(OPENAPI_GEN) $(CRD_OPENAPI_OUTFILE) # For each dir in OPENAPI_DIRS, this establishes a dependency between the # output file and the input files that should trigger a rebuild. @@ -469,6 +472,17 @@ $(OPENAPI_OUTFILE): $(OPENAPI_GEN) $(KNOWN_VIOLATION_FILENAME) diff $(REPORT_FILENAME) $(KNOWN_VIOLATION_FILENAME) || \ (echo -e $(API_RULE_CHECK_FAILURE_MESSAGE); exit 1) +# TODO(roycaihw): move the automation to apiextensions-apiserver +$(CRD_OPENAPI_OUTFILE): $(OPENAPI_GEN) + ./hack/run-in-gopath.sh $(OPENAPI_GEN) \ + --v $(KUBE_VERBOSE) \ + --logtostderr \ + -i "k8s.io/apimachinery/pkg/apis/meta/v1,k8s.io/api/autoscaling/v1" \ + -p $(PRJ_SRC_PATH)/$(CRD_OPENAPI_OUTPUT_PKG) \ + -O $(OPENAPI_BASENAME) \ + -h $(BOILERPLATE_FILENAME) \ + -r $(IGNORED_REPORT_FILENAME) \ + "$$@" # How to build the generator tool. The deps for this are defined in # the $(GO_PKGDEPS_FILE), above. diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index aecad9bedd..c70a83a37a 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -510,6 +510,7 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS apiextensionsfeatures.CustomResourceValidation: {Default: true, PreRelease: utilfeature.Beta}, apiextensionsfeatures.CustomResourceSubresources: {Default: true, PreRelease: utilfeature.Beta}, apiextensionsfeatures.CustomResourceWebhookConversion: {Default: false, PreRelease: utilfeature.Alpha}, + apiextensionsfeatures.CustomResourcePublishOpenAPI: {Default: false, PreRelease: utilfeature.Alpha}, // features that enable backwards compatibility but are scheduled to be removed // ... diff --git a/staging/src/k8s.io/apiextensions-apiserver/BUILD b/staging/src/k8s.io/apiextensions-apiserver/BUILD index 911ac1469a..a52a7a2b2f 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/BUILD +++ b/staging/src/k8s.io/apiextensions-apiserver/BUILD @@ -51,6 +51,7 @@ filegroup( "//staging/src/k8s.io/apiextensions-apiserver/pkg/controller/status:all-srcs", "//staging/src/k8s.io/apiextensions-apiserver/pkg/crdserverscheme:all-srcs", "//staging/src/k8s.io/apiextensions-apiserver/pkg/features:all-srcs", + "//staging/src/k8s.io/apiextensions-apiserver/pkg/generated/openapi:all-srcs", "//staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresource:all-srcs", "//staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition:all-srcs", "//staging/src/k8s.io/apiextensions-apiserver/test/integration:all-srcs", diff --git a/staging/src/k8s.io/apiextensions-apiserver/Godeps/Godeps.json b/staging/src/k8s.io/apiextensions-apiserver/Godeps/Godeps.json index 0e6e6d8195..654a9bd431 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/Godeps/Godeps.json +++ b/staging/src/k8s.io/apiextensions-apiserver/Godeps/Godeps.json @@ -2426,6 +2426,10 @@ "ImportPath": "k8s.io/apiserver/pkg/authorization/authorizer", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, + { + "ImportPath": "k8s.io/apiserver/pkg/endpoints", + "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, { "ImportPath": "k8s.io/apiserver/pkg/endpoints/discovery", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" @@ -2446,6 +2450,10 @@ "ImportPath": "k8s.io/apiserver/pkg/endpoints/metrics", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, + { + "ImportPath": "k8s.io/apiserver/pkg/endpoints/openapi", + "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, { "ImportPath": "k8s.io/apiserver/pkg/endpoints/request", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/BUILD b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/BUILD index 3304465ac1..c7378143b9 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/BUILD +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/BUILD @@ -32,6 +32,7 @@ go_library( "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/listers/apiextensions/internalversion:go_default_library", "//staging/src/k8s.io/apiextensions-apiserver/pkg/controller/establish:go_default_library", "//staging/src/k8s.io/apiextensions-apiserver/pkg/controller/finalizer:go_default_library", + "//staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi:go_default_library", "//staging/src/k8s.io/apiextensions-apiserver/pkg/controller/status:go_default_library", "//staging/src/k8s.io/apiextensions-apiserver/pkg/crdserverscheme:go_default_library", "//staging/src/k8s.io/apiextensions-apiserver/pkg/features:go_default_library", diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go index 1ba74a1933..a223d41ace 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go @@ -31,7 +31,9 @@ import ( internalinformers "k8s.io/apiextensions-apiserver/pkg/client/informers/internalversion" "k8s.io/apiextensions-apiserver/pkg/controller/establish" "k8s.io/apiextensions-apiserver/pkg/controller/finalizer" + openapicontroller "k8s.io/apiextensions-apiserver/pkg/controller/openapi" "k8s.io/apiextensions-apiserver/pkg/controller/status" + apiextensionsfeatures "k8s.io/apiextensions-apiserver/pkg/features" "k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -44,6 +46,7 @@ import ( "k8s.io/apiserver/pkg/registry/rest" genericapiserver "k8s.io/apiserver/pkg/server" serverstorage "k8s.io/apiserver/pkg/server/storage" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/apiserver/pkg/util/webhook" ) @@ -198,12 +201,20 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget) crdClient.Apiextensions(), crdHandler, ) + var openapiController *openapicontroller.Controller + if utilfeature.DefaultFeatureGate.Enabled(apiextensionsfeatures.CustomResourcePublishOpenAPI) { + openapiController = openapicontroller.NewController(s.Informers.Apiextensions().InternalVersion().CustomResourceDefinitions()) + } s.GenericAPIServer.AddPostStartHookOrDie("start-apiextensions-informers", func(context genericapiserver.PostStartHookContext) error { s.Informers.Start(context.StopCh) return nil }) s.GenericAPIServer.AddPostStartHookOrDie("start-apiextensions-controllers", func(context genericapiserver.PostStartHookContext) error { + if utilfeature.DefaultFeatureGate.Enabled(apiextensionsfeatures.CustomResourcePublishOpenAPI) { + go openapiController.Run(s.GenericAPIServer.StaticOpenAPISpec, s.GenericAPIServer.OpenAPIVersionedService, context.StopCh) + } + go crdController.Run(context.StopCh) go namingController.Run(context.StopCh) go establishingController.Run(context.StopCh) diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/BUILD b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/BUILD index 557398c071..38b71df1d0 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/BUILD +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/BUILD @@ -2,25 +2,55 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", - srcs = ["conversion.go"], + srcs = [ + "aggregator.go", + "builder.go", + "controller.go", + "conversion.go", + ], importmap = "k8s.io/kubernetes/vendor/k8s.io/apiextensions-apiserver/pkg/controller/openapi", importpath = "k8s.io/apiextensions-apiserver/pkg/controller/openapi", visibility = ["//visibility:public"], deps = [ + "//staging/src/k8s.io/api/autoscaling/v1:go_default_library", "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions:go_default_library", "//staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/validation:go_default_library", + "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/informers/internalversion/apiextensions/internalversion:go_default_library", + "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/listers/apiextensions/internalversion:go_default_library", + "//staging/src/k8s.io/apiextensions-apiserver/pkg/generated/openapi:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1beta1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/endpoints:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/endpoints/openapi:go_default_library", + "//staging/src/k8s.io/client-go/tools/cache:go_default_library", + "//staging/src/k8s.io/client-go/util/workqueue:go_default_library", + "//vendor/github.com/emicklei/go-restful:go_default_library", "//vendor/github.com/go-openapi/spec:go_default_library", + "//vendor/k8s.io/klog:go_default_library", + "//vendor/k8s.io/kube-openapi/pkg/builder:go_default_library", + "//vendor/k8s.io/kube-openapi/pkg/common:go_default_library", + "//vendor/k8s.io/kube-openapi/pkg/handler:go_default_library", + "//vendor/k8s.io/kube-openapi/pkg/util:go_default_library", ], ) go_test( name = "go_default_test", - srcs = ["conversion_test.go"], + srcs = [ + "builder_test.go", + "conversion_test.go", + ], embed = [":go_default_library"], deps = [ "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions:go_default_library", "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/json:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//vendor/github.com/go-openapi/spec:go_default_library", "//vendor/github.com/google/gofuzz:go_default_library", "//vendor/github.com/googleapis/gnostic/OpenAPIv2:go_default_library", diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/aggregator.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/aggregator.go new file mode 100644 index 0000000000..1dc355fd94 --- /dev/null +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/aggregator.go @@ -0,0 +1,70 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package openapi + +import ( + "github.com/go-openapi/spec" +) + +// mergeSpecs aggregates all OpenAPI specs, reusing the metadata of the first, static spec as the basis. +func mergeSpecs(staticSpec *spec.Swagger, crdSpecs ...*spec.Swagger) *spec.Swagger { + // create shallow copy of staticSpec, but replace paths and definitions because we modify them. + specToReturn := *staticSpec + if staticSpec.Definitions != nil { + specToReturn.Definitions = spec.Definitions{} + for k, s := range staticSpec.Definitions { + specToReturn.Definitions[k] = s + } + } + if staticSpec.Paths != nil { + specToReturn.Paths = &spec.Paths{ + Paths: map[string]spec.PathItem{}, + } + for k, p := range staticSpec.Paths.Paths { + specToReturn.Paths.Paths[k] = p + } + } + + for _, s := range crdSpecs { + mergeSpec(&specToReturn, s) + } + + return &specToReturn +} + +// mergeSpec copies paths and definitions from source to dest, mutating dest, but not source. +// We assume that conflicts do not matter. +func mergeSpec(dest, source *spec.Swagger) { + if source == nil || source.Paths == nil { + return + } + if dest.Paths == nil { + dest.Paths = &spec.Paths{} + } + for k, v := range source.Definitions { + if dest.Definitions == nil { + dest.Definitions = spec.Definitions{} + } + dest.Definitions[k] = v + } + for k, v := range source.Paths.Paths { + if dest.Paths.Paths == nil { + dest.Paths.Paths = map[string]spec.PathItem{} + } + dest.Paths.Paths[k] = v + } +} diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/builder.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/builder.go new file mode 100644 index 0000000000..2bbffda174 --- /dev/null +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/builder.go @@ -0,0 +1,417 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package openapi + +import ( + "fmt" + "net/http" + "strings" + "sync" + + restful "github.com/emicklei/go-restful" + "github.com/go-openapi/spec" + + v1 "k8s.io/api/autoscaling/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + metav1beta1 "k8s.io/apimachinery/pkg/apis/meta/v1beta1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apiserver/pkg/endpoints" + "k8s.io/apiserver/pkg/endpoints/openapi" + openapibuilder "k8s.io/kube-openapi/pkg/builder" + "k8s.io/kube-openapi/pkg/common" + "k8s.io/kube-openapi/pkg/util" + + "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" + generatedopenapi "k8s.io/apiextensions-apiserver/pkg/generated/openapi" +) + +const ( + // Reference and Go types for built-in metadata + objectMetaSchemaRef = "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta" + listMetaType = "k8s.io/apimachinery/pkg/apis/meta/v1.ListMeta" + typeMetaType = "k8s.io/apimachinery/pkg/apis/meta/v1.TypeMeta" + + definitionPrefix = "#/definitions/" +) + +var ( + swaggerPartialObjectMetadataDescriptions = metav1beta1.PartialObjectMetadata{}.SwaggerDoc() +) + +var definitions map[string]common.OpenAPIDefinition +var buildDefinitions sync.Once +var namer *openapi.DefinitionNamer + +// BuildSwagger builds swagger for the given crd in the given version +func BuildSwagger(crd *apiextensions.CustomResourceDefinition, version string) (*spec.Swagger, error) { + var schema *spec.Schema + s, err := apiextensions.GetSchemaForVersion(crd, version) + if err != nil { + return nil, err + } + if s != nil && s.OpenAPIV3Schema != nil { + schema, err = ConvertJSONSchemaPropsToOpenAPIv2Schema(s.OpenAPIV3Schema) + if err != nil { + return nil, err + } + } + // TODO(roycaihw): remove the WebService templating below. The following logic + // comes from function registerResourceHandlers() in k8s.io/apiserver. + // Alternatives are either (ideally) refactoring registerResourceHandlers() to + // reuse the code, or faking an APIInstaller for CR to feed to registerResourceHandlers(). + b := newBuilder(crd, version, schema) + + // Sample response types for building web service + sample := &CRDCanonicalTypeNamer{ + group: b.group, + version: b.version, + kind: b.kind, + } + sampleList := &CRDCanonicalTypeNamer{ + group: b.group, + version: b.version, + kind: b.listKind, + } + status := &metav1.Status{} + patch := &metav1.Patch{} + scale := &v1.Scale{} + + routes := make([]*restful.RouteBuilder, 0) + root := fmt.Sprintf("/apis/%s/%s/%s", b.group, b.version, b.plural) + if b.namespaced { + routes = append(routes, b.buildRoute(root, "", "GET", "list", sampleList). + Operation("list"+b.kind+"ForAllNamespaces")) + root = fmt.Sprintf("/apis/%s/%s/namespaces/{namespace}/%s", b.group, b.version, b.plural) + } + routes = append(routes, b.buildRoute(root, "", "GET", "list", sampleList)) + routes = append(routes, b.buildRoute(root, "", "POST", "create", sample).Reads(sample)) + routes = append(routes, b.buildRoute(root, "", "DELETE", "deletecollection", status)) + + routes = append(routes, b.buildRoute(root, "/{name}", "GET", "read", sample)) + routes = append(routes, b.buildRoute(root, "/{name}", "PUT", "replace", sample).Reads(sample)) + routes = append(routes, b.buildRoute(root, "/{name}", "DELETE", "delete", status)) + routes = append(routes, b.buildRoute(root, "/{name}", "PATCH", "patch", sample).Reads(patch)) + + subresources, err := apiextensions.GetSubresourcesForVersion(crd, version) + if err != nil { + return nil, err + } + if subresources != nil && subresources.Status != nil { + routes = append(routes, b.buildRoute(root, "/{name}/status", "GET", "read", sample)) + routes = append(routes, b.buildRoute(root, "/{name}/status", "PUT", "replace", sample).Reads(sample)) + routes = append(routes, b.buildRoute(root, "/{name}/status", "PATCH", "patch", sample).Reads(patch)) + } + if subresources != nil && subresources.Scale != nil { + routes = append(routes, b.buildRoute(root, "/{name}/scale", "GET", "read", scale)) + routes = append(routes, b.buildRoute(root, "/{name}/scale", "PUT", "replace", scale).Reads(scale)) + routes = append(routes, b.buildRoute(root, "/{name}/scale", "PATCH", "patch", scale).Reads(patch)) + } + + for _, route := range routes { + b.ws.Route(route) + } + + openAPISpec, err := openapibuilder.BuildOpenAPISpec([]*restful.WebService{b.ws}, b.getOpenAPIConfig()) + if err != nil { + return nil, err + } + + return openAPISpec, nil +} + +// Implements CanonicalTypeNamer +var _ = util.OpenAPICanonicalTypeNamer(&CRDCanonicalTypeNamer{}) + +// CRDCanonicalTypeNamer implements CanonicalTypeNamer interface for CRDs to +// seed kube-openapi canonical type name without Go types +type CRDCanonicalTypeNamer struct { + group string + version string + kind string +} + +// OpenAPICanonicalTypeName returns canonical type name for given CRD +func (c *CRDCanonicalTypeNamer) OpenAPICanonicalTypeName() string { + return fmt.Sprintf("%s/%s.%s", c.group, c.version, c.kind) +} + +// builder contains validation schema and basic naming information for a CRD in +// one version. The builder works to build a WebService that kube-openapi can +// consume. +type builder struct { + schema *spec.Schema + listSchema *spec.Schema + ws *restful.WebService + + group string + version string + kind string + listKind string + plural string + + namespaced bool +} + +// subresource is a handy method to get subresource name. Valid inputs are: +// input output +// "" "" +// "/" "" +// "/{name}" "" +// "/{name}/scale" "scale" +// "/{name}/scale/foo" invalid input +func subresource(path string) string { + parts := strings.Split(path, "/") + if len(parts) <= 2 { + return "" + } + if len(parts) == 3 { + return parts[2] + } + // panic to alert on programming error + panic("failed to parse subresource; invalid path") +} + +func (b *builder) descriptionFor(path, verb string) string { + var article string + switch verb { + case "list": + article = " objects of kind " + case "read", "replace": + article = " the specified " + case "patch": + article = " the specified " + case "create", "delete": + article = endpoints.GetArticleForNoun(b.kind, " ") + default: + article = "" + } + + var description string + sub := subresource(path) + if len(sub) > 0 { + sub = " " + sub + " of" + } + switch verb { + case "patch": + description = "partially update" + sub + article + b.kind + case "deletecollection": + // to match the text for built-in APIs + if len(sub) > 0 { + sub = sub + " a" + } + description = "delete collection of" + sub + " " + b.kind + default: + description = verb + sub + article + b.kind + } + + return description +} + +// buildRoute returns a RouteBuilder for WebService to consume and builds path in swagger +// action can be one of: GET, PUT, PATCH, POST, DELETE; +// verb can be one of: list, read, replace, patch, create, delete, deletecollection; +// sample is the sample Go type for response type. +func (b *builder) buildRoute(root, path, action, verb string, sample interface{}) *restful.RouteBuilder { + var namespaced string + if b.namespaced { + namespaced = "Namespaced" + } + route := b.ws.Method(action). + Path(root+path). + To(func(req *restful.Request, res *restful.Response) {}). + Doc(b.descriptionFor(path, verb)). + Param(b.ws.QueryParameter("pretty", "If 'true', then the output is pretty printed.")). + Operation(verb+namespaced+b.kind+strings.Title(subresource(path))). + Metadata(endpoints.ROUTE_META_GVK, metav1.GroupVersionKind{ + Group: b.group, + Version: b.version, + Kind: b.kind, + }). + Metadata(endpoints.ROUTE_META_ACTION, strings.ToLower(action)). + Produces("application/json", "application/yaml"). + Returns(http.StatusOK, "OK", sample). + Writes(sample) + + // Build consume media types + if action == "PATCH" { + route.Consumes("application/json-patch+json", + "application/merge-patch+json", + "application/strategic-merge-patch+json") + } else { + route.Consumes("*/*") + } + + // Build option parameters + switch verb { + case "get": + // TODO: CRD support for export is still under consideration + endpoints.AddObjectParams(b.ws, route, &metav1.GetOptions{}) + case "list", "deletecollection": + endpoints.AddObjectParams(b.ws, route, &metav1.ListOptions{}) + case "replace", "patch": + // TODO: PatchOption added in feature branch but not in master yet + endpoints.AddObjectParams(b.ws, route, &metav1.UpdateOptions{}) + case "create": + endpoints.AddObjectParams(b.ws, route, &metav1.CreateOptions{}) + case "delete": + endpoints.AddObjectParams(b.ws, route, &metav1.DeleteOptions{}) + route.Reads(&metav1.DeleteOptions{}).ParameterNamed("body").Required(false) + } + + // Build responses + switch verb { + case "create": + route.Returns(http.StatusAccepted, "Accepted", sample) + route.Returns(http.StatusCreated, "Created", sample) + case "delete": + route.Returns(http.StatusAccepted, "Accepted", sample) + case "replace": + route.Returns(http.StatusCreated, "Created", sample) + } + + return route +} + +// buildKubeNative builds input schema with Kubernetes' native object meta, type meta and +// extensions +func (b *builder) buildKubeNative(schema *spec.Schema) *spec.Schema { + // only add properties if we have a schema. Otherwise, kubectl would (wrongly) assume additionalProperties=false + // and forbid anything outside of apiVersion, kind and metadata. We have to fix kubectl to stop doing this, e.g. by + // adding additionalProperties=true support to explicitly allow additional fields. + // TODO: fix kubectl to understand additionalProperties=true + if schema == nil { + schema = &spec.Schema{ + SchemaProps: spec.SchemaProps{Type: []string{"object"}}, + } + // no, we cannot add more properties here, not even TypeMeta/ObjectMeta because kubectl will complain about + // unknown fields for anything else. + } else { + schema.SetProperty("metadata", *spec.RefSchema(objectMetaSchemaRef). + WithDescription(swaggerPartialObjectMetadataDescriptions["metadata"])) + addTypeMetaProperties(schema) + } + schema.AddExtension(endpoints.ROUTE_META_GVK, []interface{}{ + map[string]interface{}{ + "group": b.group, + "version": b.version, + "kind": b.kind, + }, + }) + + return schema +} + +// getDefinition gets definition for given Kubernetes type. This function is extracted from +// kube-openapi builder logic +func getDefinition(name string) spec.Schema { + buildDefinitions.Do(buildDefinitionsFunc) + return definitions[name].Schema +} + +func buildDefinitionsFunc() { + namer = openapi.NewDefinitionNamer(runtime.NewScheme()) + definitions = generatedopenapi.GetOpenAPIDefinitions(func(name string) spec.Ref { + defName, _ := namer.GetDefinitionName(name) + return spec.MustCreateRef(definitionPrefix + common.EscapeJsonPointer(defName)) + }) +} + +// addTypeMetaProperties adds Kubernetes-specific type meta properties to input schema: +// apiVersion and kind +func addTypeMetaProperties(s *spec.Schema) { + s.SetProperty("apiVersion", getDefinition(typeMetaType).SchemaProps.Properties["apiVersion"]) + s.SetProperty("kind", getDefinition(typeMetaType).SchemaProps.Properties["kind"]) +} + +// buildListSchema builds the list kind schema for the CRD +func (b *builder) buildListSchema() *spec.Schema { + name := definitionPrefix + util.ToRESTFriendlyName(fmt.Sprintf("%s/%s/%s", b.group, b.version, b.kind)) + doc := fmt.Sprintf("List of %s. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md", b.plural) + s := new(spec.Schema).WithDescription(fmt.Sprintf("%s is a list of %s", b.listKind, b.kind)). + WithRequired("items"). + SetProperty("items", *spec.ArrayProperty(spec.RefSchema(name)).WithDescription(doc)). + SetProperty("metadata", getDefinition(listMetaType)) + addTypeMetaProperties(s) + s.AddExtension(endpoints.ROUTE_META_GVK, []map[string]string{ + { + "group": b.group, + "version": b.version, + "kind": b.listKind, + }, + }) + return s +} + +// getOpenAPIConfig builds config which wires up generated definitions for kube-openapi to consume +func (b *builder) getOpenAPIConfig() *common.Config { + return &common.Config{ + ProtocolList: []string{"https"}, + Info: &spec.Info{ + InfoProps: spec.InfoProps{ + Title: "Kubernetes CRD Swagger", + Version: "v0.1.0", + }, + }, + CommonResponses: map[int]spec.Response{ + 401: { + ResponseProps: spec.ResponseProps{ + Description: "Unauthorized", + }, + }, + }, + GetOperationIDAndTags: openapi.GetOperationIDAndTags, + GetDefinitionName: func(name string) (string, spec.Extensions) { + buildDefinitions.Do(buildDefinitionsFunc) + return namer.GetDefinitionName(name) + }, + GetDefinitions: func(ref common.ReferenceCallback) map[string]common.OpenAPIDefinition { + def := generatedopenapi.GetOpenAPIDefinitions(ref) + def[fmt.Sprintf("%s/%s.%s", b.group, b.version, b.kind)] = common.OpenAPIDefinition{ + Schema: *b.schema, + } + def[fmt.Sprintf("%s/%s.%s", b.group, b.version, b.listKind)] = common.OpenAPIDefinition{ + Schema: *b.listSchema, + } + return def + }, + } +} + +func newBuilder(crd *apiextensions.CustomResourceDefinition, version string, schema *spec.Schema) *builder { + b := &builder{ + schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{Type: []string{"object"}}, + }, + listSchema: &spec.Schema{}, + ws: &restful.WebService{}, + + group: crd.Spec.Group, + version: version, + kind: crd.Spec.Names.Kind, + listKind: crd.Spec.Names.ListKind, + plural: crd.Spec.Names.Plural, + } + if crd.Spec.Scope == apiextensions.NamespaceScoped { + b.namespaced = true + } + + // Pre-build schema with Kubernetes native properties + b.schema = b.buildKubeNative(schema) + b.listSchema = b.buildListSchema() + + return b +} diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/builder_test.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/builder_test.go new file mode 100644 index 0000000000..2738137aa5 --- /dev/null +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/builder_test.go @@ -0,0 +1,139 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package openapi + +import ( + "reflect" + "testing" + + "github.com/go-openapi/spec" + "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" + "k8s.io/apimachinery/pkg/util/diff" + "k8s.io/apimachinery/pkg/util/json" + "k8s.io/apimachinery/pkg/util/sets" +) + +func TestNewBuilder(t *testing.T) { + type args struct { + } + tests := []struct { + name string + + schema string + + wantedSchema string + wantedItemsSchema string + }{ + { + "nil", + "", + `{"type":"object","x-kubernetes-group-version-kind":[{"group":"bar.k8s.io","kind":"Foo","version":"v1"}]}`, `{"$ref":"#/definitions/io.k8s.bar.v1.Foo"}`, + }, + {"empty", + "{}", + `{"properties":{"apiVersion":{},"kind":{},"metadata":{}},"x-kubernetes-group-version-kind":[{"group":"bar.k8s.io","kind":"Foo","version":"v1"}]}`, + `{"$ref":"#/definitions/io.k8s.bar.v1.Foo"}`, + }, + {"empty properties", + `{"properties":{"spec":{},"status":{}}}`, + `{"properties":{"apiVersion":{},"kind":{},"metadata":{},"spec":{},"status":{}},"x-kubernetes-group-version-kind":[{"group":"bar.k8s.io","kind":"Foo","version":"v1"}]}`, + `{"$ref":"#/definitions/io.k8s.bar.v1.Foo"}`, + }, + {"filled properties", + `{"properties":{"spec":{"type":"object"},"status":{"type":"object"}}}`, + `{"properties":{"apiVersion":{},"kind":{},"metadata":{},"spec":{"type":"object"},"status":{"type":"object"}},"x-kubernetes-group-version-kind":[{"group":"bar.k8s.io","kind":"Foo","version":"v1"}]}`, + `{"$ref":"#/definitions/io.k8s.bar.v1.Foo"}`, + }, + {"type", + `{"type":"object"}`, + `{"properties":{"apiVersion":{},"kind":{},"metadata":{}},"type":"object","x-kubernetes-group-version-kind":[{"group":"bar.k8s.io","kind":"Foo","version":"v1"}]}`, + `{"$ref":"#/definitions/io.k8s.bar.v1.Foo"}`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var schema *spec.Schema + if len(tt.schema) > 0 { + schema = &spec.Schema{} + if err := json.Unmarshal([]byte(tt.schema), schema); err != nil { + t.Fatal(err) + } + } + + got := newBuilder(&apiextensions.CustomResourceDefinition{ + Spec: apiextensions.CustomResourceDefinitionSpec{ + Group: "bar.k8s.io", + Version: "v1", + Names: apiextensions.CustomResourceDefinitionNames{ + Plural: "foos", + Singular: "foo", + Kind: "Foo", + ListKind: "FooList", + }, + Scope: apiextensions.NamespaceScoped, + }, + }, "v1", schema) + + var wantedSchema, wantedItemsSchema spec.Schema + if err := json.Unmarshal([]byte(tt.wantedSchema), &wantedSchema); err != nil { + t.Fatal(err) + } + if err := json.Unmarshal([]byte(tt.wantedItemsSchema), &wantedItemsSchema); err != nil { + t.Fatal(err) + } + + gotProperties := properties(got.schema.Properties) + wantedProperties := properties(wantedSchema.Properties) + if !gotProperties.Equal(wantedProperties) { + t.Fatalf("unexpected properties, got: %s, expected: %s", gotProperties.List(), wantedProperties.List()) + } + + // wipe out TypeMeta/ObjectMeta content, with those many lines of descriptions. We trust that they match here. + if _, found := got.schema.Properties["kind"]; found { + got.schema.Properties["kind"] = spec.Schema{} + } + if _, found := got.schema.Properties["apiVersion"]; found { + got.schema.Properties["apiVersion"] = spec.Schema{} + } + if _, found := got.schema.Properties["metadata"]; found { + got.schema.Properties["metadata"] = spec.Schema{} + } + + if !reflect.DeepEqual(&wantedSchema, got.schema) { + t.Errorf("unexpected schema: %s\nwant = %#v\ngot = %#v", diff.ObjectDiff(&wantedSchema, got.schema), &wantedSchema, got.schema) + } + + gotListProperties := properties(got.listSchema.Properties) + if want := sets.NewString("apiVersion", "kind", "metadata", "items"); !gotListProperties.Equal(want) { + t.Fatalf("unexpected list properties, got: %s, expected: %s", gotListProperties.List(), want.List()) + } + + gotListSchema := got.listSchema.Properties["items"].Items.Schema + if !reflect.DeepEqual(&wantedItemsSchema, gotListSchema) { + t.Errorf("unexpected list schema: %s (want/got)", diff.ObjectDiff(&wantedItemsSchema, &gotListSchema)) + } + }) + } +} + +func properties(p map[string]spec.Schema) sets.String { + ret := sets.NewString() + for k := range p { + ret.Insert(k) + } + return ret +} diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/controller.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/controller.go new file mode 100644 index 0000000000..0e083f9c18 --- /dev/null +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/controller.go @@ -0,0 +1,218 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package openapi + +import ( + "fmt" + "reflect" + "sync" + "time" + + "github.com/go-openapi/spec" + "k8s.io/apimachinery/pkg/api/errors" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/util/workqueue" + "k8s.io/klog" + "k8s.io/kube-openapi/pkg/handler" + + "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" + informers "k8s.io/apiextensions-apiserver/pkg/client/informers/internalversion/apiextensions/internalversion" + listers "k8s.io/apiextensions-apiserver/pkg/client/listers/apiextensions/internalversion" +) + +// Controller watches CustomResourceDefinitions and publishes validation schema +type Controller struct { + crdLister listers.CustomResourceDefinitionLister + crdsSynced cache.InformerSynced + + // To allow injection for testing. + syncFn func(string) error + + queue workqueue.RateLimitingInterface + + staticSpec *spec.Swagger + openAPIService *handler.OpenAPIService + + // specs per version and per CRD name + lock sync.Mutex + crdSpecs map[string]map[string]*spec.Swagger +} + +// NewController creates a new Controller with input CustomResourceDefinition informer +func NewController(crdInformer informers.CustomResourceDefinitionInformer) *Controller { + c := &Controller{ + crdLister: crdInformer.Lister(), + crdsSynced: crdInformer.Informer().HasSynced, + queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "crd_openapi_controller"), + crdSpecs: map[string]map[string]*spec.Swagger{}, + } + + crdInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: c.addCustomResourceDefinition, + UpdateFunc: c.updateCustomResourceDefinition, + DeleteFunc: c.deleteCustomResourceDefinition, + }) + + c.syncFn = c.sync + return c +} + +// Run sets openAPIAggregationManager and starts workers +func (c *Controller) Run(staticSpec *spec.Swagger, openAPIService *handler.OpenAPIService, stopCh <-chan struct{}) { + defer utilruntime.HandleCrash() + defer c.queue.ShutDown() + defer klog.Infof("Shutting down OpenAPI controller") + + klog.Infof("Starting OpenAPI controller") + + c.staticSpec = staticSpec + c.openAPIService = openAPIService + + if !cache.WaitForCacheSync(stopCh, c.crdsSynced) { + utilruntime.HandleError(fmt.Errorf("timed out waiting for caches to sync")) + return + } + + // only start one worker thread since its a slow moving API + go wait.Until(c.runWorker, time.Second, stopCh) + + <-stopCh +} + +func (c *Controller) runWorker() { + for c.processNextWorkItem() { + } +} + +func (c *Controller) processNextWorkItem() bool { + key, quit := c.queue.Get() + if quit { + return false + } + defer c.queue.Done(key) + + // log slow aggregations + start := time.Now() + defer func() { + elapsed := time.Since(start) + if elapsed > time.Second { + klog.Warningf("slow openapi aggregation of %q: %s", key.(string), elapsed) + } + }() + + err := c.syncFn(key.(string)) + if err == nil { + c.queue.Forget(key) + return true + } + + utilruntime.HandleError(fmt.Errorf("%v failed with: %v", key, err)) + c.queue.AddRateLimited(key) + return true +} + +func (c *Controller) sync(name string) error { + c.lock.Lock() + defer c.lock.Unlock() + + crd, err := c.crdLister.Get(name) + if err != nil && !errors.IsNotFound(err) { + return err + } + + // do we have to remove all specs of this CRD? + if errors.IsNotFound(err) || !apiextensions.IsCRDConditionTrue(crd, apiextensions.Established) { + if _, found := c.crdSpecs[name]; !found { + return nil + } + delete(c.crdSpecs, name) + return c.updateSpecLocked() + } + + // compute CRD spec and see whether it changed + oldSpecs := c.crdSpecs[crd.Name] + newSpecs := map[string]*spec.Swagger{} + anyChanged := false + for _, v := range crd.Spec.Versions { + if !v.Served { + continue + } + spec, err := BuildSwagger(crd, v.Name) + if err != nil { + return err + } + newSpecs[v.Name] = spec + if oldSpecs[v.Name] == nil || !reflect.DeepEqual(oldSpecs[v.Name], spec) { + anyChanged = true + } + } + if !anyChanged && len(oldSpecs) == len(newSpecs) { + return nil + } + + // update specs of this CRD + c.crdSpecs[crd.Name] = newSpecs + return c.updateSpecLocked() +} + +// updateSpecLocked aggregates all OpenAPI specs and updates openAPIService. +// It is not thread-safe. The caller is responsible to hold proper lock (Controller.lock). +func (c *Controller) updateSpecLocked() error { + crdSpecs := []*spec.Swagger{} + for _, versionSpecs := range c.crdSpecs { + for _, s := range versionSpecs { + crdSpecs = append(crdSpecs, s) + } + } + return c.openAPIService.UpdateSpec(mergeSpecs(c.staticSpec, crdSpecs...)) +} + +func (c *Controller) addCustomResourceDefinition(obj interface{}) { + castObj := obj.(*apiextensions.CustomResourceDefinition) + klog.V(4).Infof("Adding customresourcedefinition %s", castObj.Name) + c.enqueue(castObj) +} + +func (c *Controller) updateCustomResourceDefinition(oldObj, newObj interface{}) { + castNewObj := newObj.(*apiextensions.CustomResourceDefinition) + klog.V(4).Infof("Updating customresourcedefinition %s", castNewObj.Name) + c.enqueue(castNewObj) +} + +func (c *Controller) deleteCustomResourceDefinition(obj interface{}) { + castObj, ok := obj.(*apiextensions.CustomResourceDefinition) + if !ok { + tombstone, ok := obj.(cache.DeletedFinalStateUnknown) + if !ok { + klog.Errorf("Couldn't get object from tombstone %#v", obj) + return + } + castObj, ok = tombstone.Obj.(*apiextensions.CustomResourceDefinition) + if !ok { + klog.Errorf("Tombstone contained object that is not expected %#v", obj) + return + } + } + klog.V(4).Infof("Deleting customresourcedefinition %q", castObj.Name) + c.enqueue(castObj) +} + +func (c *Controller) enqueue(obj *apiextensions.CustomResourceDefinition) { + c.queue.Add(obj.Name) +} diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/features/kube_features.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/features/kube_features.go index 0ed34211ea..f2b7453733 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/features/kube_features.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/features/kube_features.go @@ -34,6 +34,12 @@ const ( // CustomResourceValidation is a list of validation methods for CustomResources CustomResourceValidation utilfeature.Feature = "CustomResourceValidation" + // owner: @roycaihw, @sttts + // alpha: v1.14 + // + // CustomResourcePublishOpenAPI enables publishing of CRD OpenAPI specs. + CustomResourcePublishOpenAPI utilfeature.Feature = "CustomResourcePublishOpenAPI" + // owner: @sttts, @nikhita // alpha: v1.10 // beta: v1.11 @@ -59,4 +65,5 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS CustomResourceValidation: {Default: true, PreRelease: utilfeature.Beta}, CustomResourceSubresources: {Default: true, PreRelease: utilfeature.Beta}, CustomResourceWebhookConversion: {Default: false, PreRelease: utilfeature.Alpha}, + CustomResourcePublishOpenAPI: {Default: false, PreRelease: utilfeature.Alpha}, } diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/generated/openapi/BUILD b/staging/src/k8s.io/apiextensions-apiserver/pkg/generated/openapi/BUILD new file mode 100644 index 0000000000..473b65dc55 --- /dev/null +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/generated/openapi/BUILD @@ -0,0 +1,33 @@ +package(default_visibility = ["//visibility:public"]) + +load("//build:code_generation.bzl", "gen_openapi", "openapi_deps") +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +gen_openapi( + outs = ["zz_generated.openapi.go"], + output_pkg = "k8s.io/apiextensions-apiserver/pkg/generated/openapi", +) + +go_library( + name = "go_default_library", + srcs = [ + "doc.go", + "zz_generated.openapi.go", + ], + importmap = "k8s.io/kubernetes/vendor/k8s.io/apiextensions-apiserver/pkg/generated/openapi", + importpath = "k8s.io/apiextensions-apiserver/pkg/generated/openapi", + deps = openapi_deps(), # keep +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], +) diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/generated/openapi/doc.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/generated/openapi/doc.go new file mode 100644 index 0000000000..5e6851f547 --- /dev/null +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/generated/openapi/doc.go @@ -0,0 +1,18 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// openapi generated definitions. +package openapi diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/installer.go b/staging/src/k8s.io/apiserver/pkg/endpoints/installer.go index 2b64cdc93a..8440624329 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/installer.go +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/installer.go @@ -611,12 +611,12 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag Returns(http.StatusOK, "OK", producedObject). Writes(producedObject) if isGetterWithOptions { - if err := addObjectParams(ws, route, versionedGetOptions); err != nil { + if err := AddObjectParams(ws, route, versionedGetOptions); err != nil { return nil, err } } if isExporter { - if err := addObjectParams(ws, route, versionedExportOptions); err != nil { + if err := AddObjectParams(ws, route, versionedExportOptions); err != nil { return nil, err } } @@ -638,7 +638,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag Produces(append(storageMeta.ProducesMIMETypes(action.Verb), allMediaTypes...)...). Returns(http.StatusOK, "OK", versionedList). Writes(versionedList) - if err := addObjectParams(ws, route, versionedListOptions); err != nil { + if err := AddObjectParams(ws, route, versionedListOptions); err != nil { return nil, err } switch { @@ -674,7 +674,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag Returns(http.StatusCreated, "Created", producedObject). Reads(defaultVersionedObject). Writes(producedObject) - if err := addObjectParams(ws, route, versionedUpdateOptions); err != nil { + if err := AddObjectParams(ws, route, versionedUpdateOptions); err != nil { return nil, err } addParams(route, action.Params) @@ -702,7 +702,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag Returns(http.StatusOK, "OK", producedObject). Reads(metav1.Patch{}). Writes(producedObject) - if err := addObjectParams(ws, route, versionedPatchOptions); err != nil { + if err := AddObjectParams(ws, route, versionedPatchOptions); err != nil { return nil, err } addParams(route, action.Params) @@ -715,7 +715,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag handler = restfulCreateResource(creater, reqScope, admit) } handler = metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, handler) - article := getArticleForNoun(kind, " ") + article := GetArticleForNoun(kind, " ") doc := "create" + article + kind if isSubresource { doc = "create " + subresource + " of" + article + kind @@ -732,13 +732,13 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag Returns(http.StatusAccepted, "Accepted", producedObject). Reads(defaultVersionedObject). Writes(producedObject) - if err := addObjectParams(ws, route, versionedCreateOptions); err != nil { + if err := AddObjectParams(ws, route, versionedCreateOptions); err != nil { return nil, err } addParams(route, action.Params) routes = append(routes, route) case "DELETE": // Delete a resource. - article := getArticleForNoun(kind, " ") + article := GetArticleForNoun(kind, " ") doc := "delete" + article + kind if isSubresource { doc = "delete " + subresource + " of" + article + kind @@ -755,7 +755,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag if isGracefulDeleter { route.Reads(versionedDeleterObject) route.ParameterNamed("body").Required(false) - if err := addObjectParams(ws, route, versionedDeleteOptions); err != nil { + if err := AddObjectParams(ws, route, versionedDeleteOptions); err != nil { return nil, err } } @@ -774,7 +774,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag Produces(append(storageMeta.ProducesMIMETypes(action.Verb), mediaTypes...)...). Writes(versionedStatus). Returns(http.StatusOK, "OK", versionedStatus) - if err := addObjectParams(ws, route, versionedListOptions); err != nil { + if err := AddObjectParams(ws, route, versionedListOptions); err != nil { return nil, err } addParams(route, action.Params) @@ -794,7 +794,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag Produces(allMediaTypes...). Returns(http.StatusOK, "OK", versionedWatchEvent). Writes(versionedWatchEvent) - if err := addObjectParams(ws, route, versionedListOptions); err != nil { + if err := AddObjectParams(ws, route, versionedListOptions); err != nil { return nil, err } addParams(route, action.Params) @@ -814,7 +814,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag Produces(allMediaTypes...). Returns(http.StatusOK, "OK", versionedWatchEvent). Writes(versionedWatchEvent) - if err := addObjectParams(ws, route, versionedListOptions); err != nil { + if err := AddObjectParams(ws, route, versionedListOptions); err != nil { return nil, err } addParams(route, action.Params) @@ -838,7 +838,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag Consumes("*/*"). Writes(connectProducedObject) if versionedConnectOptions != nil { - if err := addObjectParams(ws, route, versionedConnectOptions); err != nil { + if err := AddObjectParams(ws, route, versionedConnectOptions); err != nil { return nil, err } } @@ -907,13 +907,13 @@ func addParams(route *restful.RouteBuilder, params []*restful.Parameter) { } } -// addObjectParams converts a runtime.Object into a set of go-restful Param() definitions on the route. +// AddObjectParams converts a runtime.Object into a set of go-restful Param() definitions on the route. // The object must be a pointer to a struct; only fields at the top level of the struct that are not // themselves interfaces or structs are used; only fields with a json tag that is non empty (the standard // Go JSON behavior for omitting a field) become query parameters. The name of the query parameter is // the JSON field name. If a description struct tag is set on the field, that description is used on the // query parameter. In essence, it converts a standard JSON top level object into a query param schema. -func addObjectParams(ws *restful.WebService, route *restful.RouteBuilder, obj interface{}) error { +func AddObjectParams(ws *restful.WebService, route *restful.RouteBuilder, obj interface{}) error { sv, err := conversion.EnforcePtr(obj) if err != nil { return err @@ -1015,8 +1015,8 @@ func splitSubresource(path string) (string, string, error) { return resource, subresource, nil } -// getArticleForNoun returns the article needed for the given noun. -func getArticleForNoun(noun string, padding string) string { +// GetArticleForNoun returns the article needed for the given noun. +func GetArticleForNoun(noun string, padding string) string { if noun[len(noun)-2:] != "ss" && noun[len(noun)-1:] == "s" { // Plurals don't have an article. // Don't catch words like class diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/installer_test.go b/staging/src/k8s.io/apiserver/pkg/endpoints/installer_test.go index 898d3b3858..c661bd6fc0 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/installer_test.go +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/installer_test.go @@ -82,7 +82,7 @@ func TestGetArticleForNoun(t *testing.T) { }, } for _, tt := range tests { - if got := getArticleForNoun(tt.noun, tt.padding); got != tt.want { + if got := GetArticleForNoun(tt.noun, tt.padding); got != tt.want { t.Errorf("%q. GetArticleForNoun() = %v, want %v", tt.noun, got, tt.want) } } diff --git a/staging/src/k8s.io/kube-aggregator/pkg/controllers/openapi/aggregator/aggregator.go b/staging/src/k8s.io/kube-aggregator/pkg/controllers/openapi/aggregator/aggregator.go index ca36b8e790..c360e4b0f5 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/controllers/openapi/aggregator/aggregator.go +++ b/staging/src/k8s.io/kube-aggregator/pkg/controllers/openapi/aggregator/aggregator.go @@ -19,6 +19,7 @@ package aggregator import ( "fmt" "net/http" + "strings" "sync" "time" @@ -40,17 +41,35 @@ type SpecAggregator interface { UpdateAPIServiceSpec(apiServiceName string, spec *spec.Swagger, etag string) error RemoveAPIServiceSpec(apiServiceName string) error GetAPIServiceInfo(apiServiceName string) (handler http.Handler, etag string, exists bool) + GetAPIServiceNames() []string } const ( aggregatorUser = "system:aggregator" specDownloadTimeout = 60 * time.Second - localDelegateChainNamePattern = "k8s_internal_local_delegation_chain_%010d" + localDelegateChainNamePrefix = "k8s_internal_local_delegation_chain_" + localDelegateChainNamePattern = localDelegateChainNamePrefix + "%010d" // A randomly generated UUID to differentiate local and remote eTags. locallyGeneratedEtagPrefix = "\"6E8F849B434D4B98A569B9D7718876E9-" ) +// IsLocalAPIService returns true for local specs from delegates. +func IsLocalAPIService(apiServiceName string) bool { + return strings.HasPrefix(apiServiceName, localDelegateChainNamePrefix) +} + +// GetAPIServicesName returns the names of APIServices recorded in specAggregator.openAPISpecs. +// We use this function to pass the names of local APIServices to the controller in this package, +// so that the controller can periodically sync the OpenAPI spec from delegation API servers. +func (s *specAggregator) GetAPIServiceNames() []string { + names := make([]string, len(s.openAPISpecs)) + for key := range s.openAPISpecs { + names = append(names, key) + } + return names +} + // BuildAndRegisterAggregator registered OpenAPI aggregator handler. This function is not thread safe as it only being called on startup. func BuildAndRegisterAggregator(downloader *Downloader, delegationTarget server.DelegationTarget, webServices []*restful.WebService, config *common.Config, pathHandler common.PathHandler) (SpecAggregator, error) { @@ -161,6 +180,7 @@ func (s *specAggregator) buildOpenAPISpec() (specToReturn *spec.Swagger, err err return nil, err } } + return specToReturn, nil } @@ -179,7 +199,14 @@ func (s *specAggregator) updateOpenAPISpec() error { // tryUpdatingServiceSpecs tries updating openAPISpecs map with specified specInfo, and keeps the map intact // if the update fails. func (s *specAggregator) tryUpdatingServiceSpecs(specInfo *openAPISpecInfo) error { + if specInfo == nil { + return fmt.Errorf("invalid input: specInfo must be non-nil") + } orgSpecInfo, exists := s.openAPISpecs[specInfo.apiService.Name] + // Skip aggregation if OpenAPI spec didn't change + if exists && orgSpecInfo != nil && orgSpecInfo.etag == specInfo.etag { + return nil + } s.openAPISpecs[specInfo.apiService.Name] = specInfo if err := s.updateOpenAPISpec(); err != nil { if exists { diff --git a/staging/src/k8s.io/kube-aggregator/pkg/controllers/openapi/controller.go b/staging/src/k8s.io/kube-aggregator/pkg/controllers/openapi/controller.go index 764d7b8f3a..ea8de12151 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/controllers/openapi/controller.go +++ b/staging/src/k8s.io/kube-aggregator/pkg/controllers/openapi/controller.go @@ -30,8 +30,9 @@ import ( ) const ( - successfulUpdateDelay = time.Minute - failedUpdateMaxExpDelay = time.Hour + successfulUpdateDelay = time.Minute + successfulUpdateDelayLocal = time.Second + failedUpdateMaxExpDelay = time.Hour ) type syncAction int @@ -64,6 +65,11 @@ func NewAggregationController(downloader *aggregator.Downloader, openAPIAggregat c.syncHandler = c.sync + // update each service at least once, also those which are not coming from APIServices, namely local services + for _, name := range openAPIAggregationManager.GetAPIServiceNames() { + c.queue.AddAfter(name, time.Second) + } + return c } @@ -104,8 +110,13 @@ func (c *AggregationController) processNextWorkItem() bool { switch action { case syncRequeue: - klog.Infof("OpenAPI AggregationController: action for item %s: Requeue.", key) - c.queue.AddAfter(key, successfulUpdateDelay) + if aggregator.IsLocalAPIService(key.(string)) { + klog.V(7).Infof("OpenAPI AggregationController: action for local item %s: Requeue after %s.", key, successfulUpdateDelayLocal) + c.queue.AddAfter(key, successfulUpdateDelayLocal) + } else { + klog.V(7).Infof("OpenAPI AggregationController: action for item %s: Requeue.", key) + c.queue.AddAfter(key, successfulUpdateDelay) + } case syncRequeueRateLimited: klog.Infof("OpenAPI AggregationController: action for item %s: Rate Limited Requeue.", key) c.queue.AddRateLimited(key) diff --git a/test/e2e/apimachinery/BUILD b/test/e2e/apimachinery/BUILD index 18b1dbc8c9..136dd18f70 100644 --- a/test/e2e/apimachinery/BUILD +++ b/test/e2e/apimachinery/BUILD @@ -12,6 +12,7 @@ go_library( "certs.go", "chunking.go", "crd_conversion_webhook.go", + "crd_publish_openapi.go", "crd_watch.go", "custom_resource_definition.go", "etcd_failure.go", @@ -40,7 +41,9 @@ go_library( "//staging/src/k8s.io/api/rbac/v1:go_default_library", "//staging/src/k8s.io/api/rbac/v1beta1:go_default_library", "//staging/src/k8s.io/api/scheduling/v1:go_default_library", + "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions:go_default_library", "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library", + "//staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/validation:go_default_library", "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset:go_default_library", "//staging/src/k8s.io/apiextensions-apiserver/test/integration:go_default_library", "//staging/src/k8s.io/apiextensions-apiserver/test/integration/fixtures:go_default_library", @@ -60,6 +63,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/version:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/yaml:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/names:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/storagebackend:go_default_library", @@ -77,9 +81,12 @@ go_library( "//test/e2e/framework/metrics:go_default_library", "//test/utils:go_default_library", "//test/utils/image:go_default_library", + "//vendor/github.com/go-openapi/spec:go_default_library", "//vendor/github.com/onsi/ginkgo:go_default_library", "//vendor/github.com/onsi/gomega:go_default_library", "//vendor/github.com/stretchr/testify/assert:go_default_library", + "//vendor/k8s.io/kube-openapi/pkg/util:go_default_library", + "//vendor/sigs.k8s.io/yaml:go_default_library", ], ) diff --git a/test/e2e/apimachinery/crd_publish_openapi.go b/test/e2e/apimachinery/crd_publish_openapi.go new file mode 100644 index 0000000000..591e40ba22 --- /dev/null +++ b/test/e2e/apimachinery/crd_publish_openapi.go @@ -0,0 +1,552 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apimachinery + +import ( + "encoding/json" + "fmt" + "regexp" + "strings" + "time" + + "github.com/go-openapi/spec" + . "github.com/onsi/ginkgo" + + "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" + "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + "k8s.io/apiextensions-apiserver/pkg/apiserver/validation" + apiequality "k8s.io/apimachinery/pkg/api/equality" + "k8s.io/apimachinery/pkg/types" + utilversion "k8s.io/apimachinery/pkg/util/version" + "k8s.io/apimachinery/pkg/util/wait" + utilyaml "k8s.io/apimachinery/pkg/util/yaml" + k8sclientset "k8s.io/client-go/kubernetes" + openapiutil "k8s.io/kube-openapi/pkg/util" + "k8s.io/kubernetes/test/e2e/framework" + "sigs.k8s.io/yaml" +) + +var ( + crdPublishOpenAPIVersion = utilversion.MustParseSemantic("v1.14.0") + metaPattern = `"kind":"%s","apiVersion":"%s/%s","metadata":{"name":"%s"}` +) + +var _ = SIGDescribe("CustomResourcePublishOpenAPI [Feature:CustomResourcePublishOpenAPI]", func() { + f := framework.NewDefaultFramework("crd-publish-openapi") + + BeforeEach(func() { + framework.SkipUnlessServerVersionGTE(crdPublishOpenAPIVersion, f.ClientSet.Discovery()) + }) + + It("works for CRD with validation schema", func() { + crd, err := setupCRD(f, schemaFoo, "foo", "v1") + if err != nil { + framework.Failf("%v", err) + } + + meta := fmt.Sprintf(metaPattern, crd.Kind, crd.ApiGroup, crd.Versions[0].Name, "test-foo") + ns := fmt.Sprintf("--namespace=%v", f.Namespace.Name) + + By("client-side validation (kubectl create and apply) allows request with known and required properties") + validCR := fmt.Sprintf(`{%s,"spec":{"bars":[{"name":"test-bar"}]}}`, meta) + if _, err := framework.RunKubectlInput(validCR, ns, "create", "-f", "-"); err != nil { + framework.Failf("failed to create valid CR %s: %v", validCR, err) + } + if _, err := framework.RunKubectl(ns, "delete", crd.GetPluralName(), "test-foo"); err != nil { + framework.Failf("failed to delete valid CR: %v", err) + } + if _, err := framework.RunKubectlInput(validCR, ns, "apply", "-f", "-"); err != nil { + framework.Failf("failed to apply valid CR %s: %v", validCR, err) + } + if _, err := framework.RunKubectl(ns, "delete", crd.GetPluralName(), "test-foo"); err != nil { + framework.Failf("failed to delete valid CR: %v", err) + } + + By("client-side validation (kubectl create and apply) rejects request with unknown properties when disallowed by the schema") + unknownCR := fmt.Sprintf(`{%s,"spec":{"foo":true}}`, meta) + if _, err := framework.RunKubectlInput(unknownCR, ns, "create", "-f", "-"); err == nil || !strings.Contains(err.Error(), `unknown field "foo"`) { + framework.Failf("unexpected no error when creating CR with unknown field: %v", err) + } + if _, err := framework.RunKubectlInput(unknownCR, ns, "apply", "-f", "-"); err == nil || !strings.Contains(err.Error(), `unknown field "foo"`) { + framework.Failf("unexpected no error when applying CR with unknown field: %v", err) + } + + By("client-side validation (kubectl create and apply) rejects request without required properties") + noRequireCR := fmt.Sprintf(`{%s,"spec":{"bars":[{"age":"10"}]}}`, meta) + if _, err := framework.RunKubectlInput(noRequireCR, ns, "create", "-f", "-"); err == nil || !strings.Contains(err.Error(), `missing required field "name"`) { + framework.Failf("unexpected no error when creating CR without required field: %v", err) + } + if _, err := framework.RunKubectlInput(noRequireCR, ns, "apply", "-f", "-"); err == nil || !strings.Contains(err.Error(), `missing required field "name"`) { + framework.Failf("unexpected no error when applying CR without required field: %v", err) + } + + By("kubectl explain works to explain CR properties") + if err := verifyKubectlExplain(crd.GetPluralName(), `(?s)DESCRIPTION:.*Foo CRD for Testing.*FIELDS:.*apiVersion.*.*APIVersion defines.*spec.*.*Specification of Foo`); err != nil { + framework.Failf("%v", err) + } + + By("kubectl explain works to explain CR properties recursively") + if err := verifyKubectlExplain(crd.GetPluralName()+".metadata", `(?s)DESCRIPTION:.*Standard object's metadata.*FIELDS:.*creationTimestamp.*.*CreationTimestamp is a timestamp`); err != nil { + framework.Failf("%v", err) + } + if err := verifyKubectlExplain(crd.GetPluralName()+".spec", `(?s)DESCRIPTION:.*Specification of Foo.*FIELDS:.*bars.*<\[\]Object>.*List of Bars and their specs`); err != nil { + framework.Failf("%v", err) + } + if err := verifyKubectlExplain(crd.GetPluralName()+".spec.bars", `(?s)RESOURCE:.*bars.*<\[\]Object>.*DESCRIPTION:.*List of Bars and their specs.*FIELDS:.*bazs.*<\[\]string>.*List of Bazs.*name.*.*Name of Bar`); err != nil { + framework.Failf("%v", err) + } + + By("kubectl explain works to return error when explain is called on property that doesn't exist") + if _, err := framework.RunKubectl("explain", crd.GetPluralName()+".spec.bars2"); err == nil || !strings.Contains(err.Error(), `field "bars2" does not exist`) { + framework.Failf("unexpected no error when explaining property that doesn't exist: %v", err) + } + + if err := cleanupCRD(f, crd); err != nil { + framework.Failf("%v", err) + } + }) + + It("works for CRD without validation schema", func() { + crd, err := setupCRD(f, nil, "empty", "v1") + if err != nil { + framework.Failf("%v", err) + } + + meta := fmt.Sprintf(metaPattern, crd.Kind, crd.ApiGroup, crd.Versions[0].Name, "test-cr") + ns := fmt.Sprintf("--namespace=%v", f.Namespace.Name) + + By("client-side validation (kubectl create and apply) allows request with any unknown properties") + randomCR := fmt.Sprintf(`{%s,"a":{"b":[{"c":"d"}]}}`, meta) + if _, err := framework.RunKubectlInput(randomCR, ns, "create", "-f", "-"); err != nil { + framework.Failf("failed to create random CR %s for CRD without schema: %v", randomCR, err) + } + if _, err := framework.RunKubectl(ns, "delete", crd.GetPluralName(), "test-cr"); err != nil { + framework.Failf("failed to delete random CR: %v", err) + } + if _, err := framework.RunKubectlInput(randomCR, ns, "apply", "-f", "-"); err != nil { + framework.Failf("failed to apply random CR %s for CRD without schema: %v", randomCR, err) + } + if _, err := framework.RunKubectl(ns, "delete", crd.GetPluralName(), "test-cr"); err != nil { + framework.Failf("failed to delete random CR: %v", err) + } + + By("kubectl explain works to explain CR without validation schema") + if err := verifyKubectlExplain(crd.GetPluralName(), `(?s)DESCRIPTION:.*`); err != nil { + framework.Failf("%v", err) + } + + if err := cleanupCRD(f, crd); err != nil { + framework.Failf("%v", err) + } + }) + + It("works for multiple CRDs of different groups", func() { + By("CRs in different groups (two CRDs) show up in OpenAPI documentation") + crdFoo, err := setupCRD(f, schemaFoo, "foo", "v1") + if err != nil { + framework.Failf("%v", err) + } + crdWaldo, err := setupCRD(f, schemaWaldo, "waldo", "v1beta1") + if err != nil { + framework.Failf("%v", err) + } + if crdFoo.ApiGroup == crdWaldo.ApiGroup { + framework.Failf("unexpected: CRDs should be of different group %v, %v", crdFoo.ApiGroup, crdWaldo.ApiGroup) + } + if err := waitForDefinition(f.ClientSet, definitionName(crdWaldo, "v1beta1"), schemaWaldo); err != nil { + framework.Failf("%v", err) + } + if err := waitForDefinition(f.ClientSet, definitionName(crdFoo, "v1"), schemaFoo); err != nil { + framework.Failf("%v", err) + } + if err := cleanupCRD(f, crdFoo); err != nil { + framework.Failf("%v", err) + } + if err := cleanupCRD(f, crdWaldo); err != nil { + framework.Failf("%v", err) + } + }) + + It("works for multiple CRDs of same group but different versions", func() { + By("CRs in the same group but different versions (one multiversion CRD) show up in OpenAPI documentation") + crdMultiVer, err := setupCRD(f, schemaFoo, "multi-ver", "v2", "v3") + if err != nil { + framework.Failf("%v", err) + } + if err := waitForDefinition(f.ClientSet, definitionName(crdMultiVer, "v3"), schemaFoo); err != nil { + framework.Failf("%v", err) + } + if err := waitForDefinition(f.ClientSet, definitionName(crdMultiVer, "v2"), schemaFoo); err != nil { + framework.Failf("%v", err) + } + if err := cleanupCRD(f, crdMultiVer); err != nil { + framework.Failf("%v", err) + } + + By("CRs in the same group but different versions (two CRDs) show up in OpenAPI documentation") + crdFoo, err := setupCRD(f, schemaFoo, "common-group", "v4") + if err != nil { + framework.Failf("%v", err) + } + crdWaldo, err := setupCRD(f, schemaWaldo, "common-group", "v5") + if err != nil { + framework.Failf("%v", err) + } + if crdFoo.ApiGroup != crdWaldo.ApiGroup { + framework.Failf("unexpected: CRDs should be of the same group %v, %v", crdFoo.ApiGroup, crdWaldo.ApiGroup) + } + if err := waitForDefinition(f.ClientSet, definitionName(crdWaldo, "v5"), schemaWaldo); err != nil { + framework.Failf("%v", err) + } + if err := waitForDefinition(f.ClientSet, definitionName(crdFoo, "v4"), schemaFoo); err != nil { + framework.Failf("%v", err) + } + if err := cleanupCRD(f, crdFoo); err != nil { + framework.Failf("%v", err) + } + if err := cleanupCRD(f, crdWaldo); err != nil { + framework.Failf("%v", err) + } + }) + + It("works for multiple CRDs of same group and version but different kinds", func() { + By("CRs in the same group and version but different kinds (two CRDs) show up in OpenAPI documentation") + crdFoo, err := setupCRD(f, schemaFoo, "common-group", "v6") + if err != nil { + framework.Failf("%v", err) + } + crdWaldo, err := setupCRD(f, schemaWaldo, "common-group", "v6") + if err != nil { + framework.Failf("%v", err) + } + if crdFoo.ApiGroup != crdWaldo.ApiGroup { + framework.Failf("unexpected: CRDs should be of the same group %v, %v", crdFoo.ApiGroup, crdWaldo.ApiGroup) + } + if err := waitForDefinition(f.ClientSet, definitionName(crdWaldo, "v6"), schemaWaldo); err != nil { + framework.Failf("%v", err) + } + if err := waitForDefinition(f.ClientSet, definitionName(crdFoo, "v6"), schemaFoo); err != nil { + framework.Failf("%v", err) + } + if err := cleanupCRD(f, crdFoo); err != nil { + framework.Failf("%v", err) + } + if err := cleanupCRD(f, crdWaldo); err != nil { + framework.Failf("%v", err) + } + }) + + It("updates the published spec when one versin gets renamed", func() { + By("set up a multi version CRD") + crdMultiVer, err := setupCRD(f, schemaFoo, "multi-ver", "v2", "v3") + if err != nil { + framework.Failf("%v", err) + } + if err := waitForDefinition(f.ClientSet, definitionName(crdMultiVer, "v3"), schemaFoo); err != nil { + framework.Failf("%v", err) + } + if err := waitForDefinition(f.ClientSet, definitionName(crdMultiVer, "v2"), schemaFoo); err != nil { + framework.Failf("%v", err) + } + + By("rename a version") + patch := []byte(`{"spec":{"versions":[{"name":"v2","served":true,"storage":true},{"name":"v4","served":true,"storage":false}]}}`) + crdMultiVer.Crd, err = crdMultiVer.ApiExtensionClient.ApiextensionsV1beta1().CustomResourceDefinitions().Patch(crdMultiVer.GetMetaName(), types.MergePatchType, patch) + if err != nil { + framework.Failf("%v", err) + } + + By("check the new version name is served") + if err := waitForDefinition(f.ClientSet, definitionName(crdMultiVer, "v4"), schemaFoo); err != nil { + framework.Failf("%v", err) + } + By("check the old version name is removed") + if err := waitForDefinitionCleanup(f.ClientSet, definitionName(crdMultiVer, "v3")); err != nil { + framework.Failf("%v", err) + } + By("check the other version is not changed") + if err := waitForDefinition(f.ClientSet, definitionName(crdMultiVer, "v2"), schemaFoo); err != nil { + framework.Failf("%v", err) + } + + // TestCrd.Versions is different from TestCrd.Crd.Versions, we have to manually + // update the name there. Used by cleanupCRD + crdMultiVer.Versions[1].Name = "v4" + if err := cleanupCRD(f, crdMultiVer); err != nil { + framework.Failf("%v", err) + } + }) + + It("removes definition from spec when one versin gets changed to not be served", func() { + By("set up a multi version CRD") + crd, err := setupCRD(f, schemaFoo, "multi-to-single-ver", "v5", "v6alpha1") + if err != nil { + framework.Failf("%v", err) + } + // just double check. setupCRD() checked this for us already + if err := waitForDefinition(f.ClientSet, definitionName(crd, "v6alpha1"), schemaFoo); err != nil { + framework.Failf("%v", err) + } + if err := waitForDefinition(f.ClientSet, definitionName(crd, "v5"), schemaFoo); err != nil { + framework.Failf("%v", err) + } + + By("mark a version not serverd") + crd.Crd.Spec.Versions[1].Served = false + crd.Crd, err = crd.ApiExtensionClient.ApiextensionsV1beta1().CustomResourceDefinitions().Update(crd.Crd) + if err != nil { + framework.Failf("%v", err) + } + + By("check the unserved version gets removed") + if err := waitForDefinitionCleanup(f.ClientSet, definitionName(crd, "v6alpha1")); err != nil { + framework.Failf("%v", err) + } + By("check the other version is not changed") + if err := waitForDefinition(f.ClientSet, definitionName(crd, "v5"), schemaFoo); err != nil { + framework.Failf("%v", err) + } + + if err := cleanupCRD(f, crd); err != nil { + framework.Failf("%v", err) + } + }) +}) + +func setupCRD(f *framework.Framework, schema []byte, groupSuffix string, versions ...string) (*framework.TestCrd, error) { + group := fmt.Sprintf("%s-test-%s.k8s.io", f.BaseName, groupSuffix) + if len(versions) == 0 { + return nil, fmt.Errorf("require at least one version for CRD") + } + apiVersions := []v1beta1.CustomResourceDefinitionVersion{} + for _, version := range versions { + v := v1beta1.CustomResourceDefinitionVersion{ + Name: version, + Served: true, + Storage: false, + } + apiVersions = append(apiVersions, v) + } + apiVersions[0].Storage = true + + crd, err := framework.CreateMultiVersionTestCRD(f, group, apiVersions, nil) + if err != nil { + return nil, fmt.Errorf("failed to create CRD: %v", err) + } + + if schema != nil { + // patch validation schema for all versions + if err := patchSchema(schema, crd); err != nil { + return nil, fmt.Errorf("failed to patch schema: %v", err) + } + } else { + // change expectation if CRD doesn't have schema + schema = []byte(`type: object`) + } + + for _, v := range crd.Versions { + if err := waitForDefinition(f.ClientSet, definitionName(crd, v.Name), schema); err != nil { + return nil, fmt.Errorf("%v", err) + } + } + return crd, nil +} + +func cleanupCRD(f *framework.Framework, crd *framework.TestCrd) error { + crd.CleanUp() + for _, v := range crd.Versions { + name := definitionName(crd, v.Name) + if err := waitForDefinitionCleanup(f.ClientSet, name); err != nil { + return fmt.Errorf("%v", err) + } + } + return nil +} + +// patchSchema takes schema in YAML and patches it to given CRD in given version +func patchSchema(schema []byte, crd *framework.TestCrd) error { + s, err := utilyaml.ToJSON(schema) + if err != nil { + return fmt.Errorf("failed to create json patch: %v", err) + } + patch := []byte(fmt.Sprintf(`{"spec":{"validation":{"openAPIV3Schema":%s}}}`, string(s))) + crd.Crd, err = crd.ApiExtensionClient.ApiextensionsV1beta1().CustomResourceDefinitions().Patch(crd.GetMetaName(), types.MergePatchType, patch) + return err +} + +// waitForDefinition waits for given definition showing up in swagger with given schema +func waitForDefinition(c k8sclientset.Interface, name string, schema []byte) error { + expect := spec.Schema{} + if err := convertJSONSchemaProps(schema, &expect); err != nil { + return err + } + + lastMsg := "" + if err := wait.Poll(500*time.Millisecond, 10*time.Second, func() (bool, error) { + bs, err := c.CoreV1().RESTClient().Get().AbsPath("openapi", "v2").DoRaw() + if err != nil { + return false, err + } + spec := spec.Swagger{} + if err := json.Unmarshal(bs, &spec); err != nil { + return false, err + } + d, ok := spec.SwaggerProps.Definitions[name] + if !ok { + lastMsg = fmt.Sprintf("spec.SwaggerProps.Definitions[\"%s\"] not found", name) + return false, nil + } + // drop properties and extension that we added + dropDefaults(&d) + if !apiequality.Semantic.DeepEqual(expect, d) { + lastMsg = fmt.Sprintf("spec.SwaggerProps.Definitions[\"%s\"] not match; expect: %v, actual: %v", name, expect, d) + return false, nil + } + return true, nil + }); err != nil { + return fmt.Errorf("failed to wait for definition %s to be served: %v; lastMsg: %s", name, err, lastMsg) + } + return nil +} + +// waitForDefinitionCleanup waits for given definition to be removed from swagger +func waitForDefinitionCleanup(c k8sclientset.Interface, name string) error { + lastMsg := "" + if err := wait.Poll(500*time.Millisecond, 10*time.Second, func() (bool, error) { + bs, err := c.CoreV1().RESTClient().Get().AbsPath("openapi", "v2").DoRaw() + if err != nil { + return false, err + } + spec := spec.Swagger{} + if err := json.Unmarshal(bs, &spec); err != nil { + return false, err + } + _, ok := spec.SwaggerProps.Definitions[name] + if ok { + lastMsg = fmt.Sprintf("spec.SwaggerProps.Definitions[\"%s\"] still exists", name) + return false, nil + } + return true, nil + }); err != nil { + return fmt.Errorf("failed to wait for definition %s to be removed: %v; lastMsg: %s", name, err, lastMsg) + } + return nil +} + +// convertJSONSchemaProps converts JSONSchemaProps in YAML to spec.Schema +func convertJSONSchemaProps(in []byte, out *spec.Schema) error { + external := v1beta1.JSONSchemaProps{} + if err := yaml.UnmarshalStrict(in, &external); err != nil { + return err + } + internal := apiextensions.JSONSchemaProps{} + if err := v1beta1.Convert_v1beta1_JSONSchemaProps_To_apiextensions_JSONSchemaProps(&external, &internal, nil); err != nil { + return err + } + if err := validation.ConvertJSONSchemaProps(&internal, out); err != nil { + return err + } + return nil +} + +// dropDefaults drops properties and extension that we added to a schema +func dropDefaults(s *spec.Schema) { + delete(s.Properties, "metadata") + delete(s.Properties, "apiVersion") + delete(s.Properties, "kind") + delete(s.Extensions, "x-kubernetes-group-version-kind") +} + +func verifyKubectlExplain(name, pattern string) error { + result, err := framework.RunKubectl("explain", name) + if err != nil { + return fmt.Errorf("failed to explain %s: %v", name, err) + } + r := regexp.MustCompile(pattern) + if !r.Match([]byte(result)) { + return fmt.Errorf("kubectl explain %s result {%s} doesn't match pattern {%s}", name, result, pattern) + } + return nil +} + +// definitionName returns the openapi definition name for given CRD in given version +func definitionName(crd *framework.TestCrd, version string) string { + return openapiutil.ToRESTFriendlyName(fmt.Sprintf("%s/%s/%s", crd.ApiGroup, version, crd.Kind)) +} + +var schemaFoo = []byte(`description: Foo CRD for Testing +type: object +properties: + spec: + type: object + description: Specification of Foo + properties: + bars: + description: List of Bars and their specs. + type: array + items: + type: object + required: + - name + properties: + name: + description: Name of Bar. + type: string + age: + description: Age of Bar. + type: string + bazs: + description: List of Bazs. + items: + type: string + type: array + status: + description: Status of Foo + type: object + properties: + bars: + description: List of Bars and their statuses. + type: array + items: + type: object + properties: + name: + description: Name of Bar. + type: string + available: + description: Whether the Bar is installed. + type: boolean + quxType: + description: Indicates to external qux type. + pattern: in-tree|out-of-tree + type: string`) + +var schemaWaldo = []byte(`description: Waldo CRD for Testing +type: object +properties: + spec: + description: Specification of Waldo + type: object + properties: + dummy: + description: Dummy property. + status: + description: Status of Waldo + type: object + properties: + bars: + description: List of Bars and their statuses.`) diff --git a/test/integration/master/BUILD b/test/integration/master/BUILD index b59ce9479a..cad6f6810b 100644 --- a/test/integration/master/BUILD +++ b/test/integration/master/BUILD @@ -66,6 +66,7 @@ go_test( "//test/integration/framework:go_default_library", "//test/utils:go_default_library", "//vendor/github.com/evanphx/json-patch:go_default_library", + "//vendor/github.com/go-openapi/spec:go_default_library", "//vendor/github.com/stretchr/testify/require:go_default_library", "//vendor/sigs.k8s.io/yaml:go_default_library", ] + select({ diff --git a/test/integration/master/crd_test.go b/test/integration/master/crd_test.go index 47f664f6b9..2ed3784d12 100644 --- a/test/integration/master/crd_test.go +++ b/test/integration/master/crd_test.go @@ -18,15 +18,19 @@ package master import ( "encoding/json" + "fmt" "testing" "time" + "github.com/go-openapi/spec" + networkingv1 "k8s.io/api/networking/v1" apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" @@ -130,6 +134,90 @@ func TestCRD(t *testing.T) { } } +func TestCRDOpenAPI(t *testing.T) { + result := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--feature-gates=CustomResourcePublishOpenAPI=true"}, framework.SharedEtcd()) + defer result.TearDownFn() + kubeclient, err := kubernetes.NewForConfig(result.ClientConfig) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + apiextensionsclient, err := apiextensionsclientset.NewForConfig(result.ClientConfig) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + t.Logf("Trying to create a custom resource without conflict") + crd := &apiextensionsv1beta1.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foos.cr.bar.com", + }, + Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{ + Group: "cr.bar.com", + Version: "v1", + Scope: apiextensionsv1beta1.NamespaceScoped, + Names: apiextensionsv1beta1.CustomResourceDefinitionNames{ + Plural: "foos", + Kind: "Foo", + }, + Validation: &apiextensionsv1beta1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensionsv1beta1.JSONSchemaProps{ + Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{ + "foo": {Type: "string"}, + }, + }, + }, + }, + } + etcd.CreateTestCRDs(t, apiextensionsclient, false, crd) + waitForSpec := func(expectedType string) { + t.Logf(`Waiting for {properties: {"foo": {"type":"%s"}}} to show up in schema`, expectedType) + lastMsg := "" + if err := wait.PollImmediate(500*time.Millisecond, 10*time.Second, func() (bool, error) { + lastMsg = "" + bs, err := kubeclient.RESTClient().Get().AbsPath("openapi", "v2").DoRaw() + if err != nil { + return false, err + } + spec := spec.Swagger{} + if err := json.Unmarshal(bs, &spec); err != nil { + return false, err + } + if spec.SwaggerProps.Paths == nil { + lastMsg = "spec.SwaggerProps.Paths is nil" + return false, nil + } + d, ok := spec.SwaggerProps.Definitions["com.bar.cr.v1.Foo"] + if !ok { + lastMsg = `spec.SwaggerProps.Definitions["com.bar.cr.v1.Foo"] not found` + return false, nil + } + p, ok := d.Properties["foo"] + if !ok { + lastMsg = `spec.SwaggerProps.Definitions["com.bar.cr.v1.Foo"].Properties["foo"] not found` + return false, nil + } + if !p.Type.Contains(expectedType) { + lastMsg = fmt.Sprintf(`spec.SwaggerProps.Definitions["com.bar.cr.v1.Foo"].Properties["foo"].Type should be %q, but got: %q`, expectedType, p.Type) + return false, nil + } + return true, nil + }); err != nil { + t.Fatalf("Failed to see %s OpenAPI spec in discovery: %v, last message: %s", crd.Name, err, lastMsg) + } + } + waitForSpec("string") + crd, err = apiextensionsclient.ApiextensionsV1beta1().CustomResourceDefinitions().Get(crd.Name, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + prop := crd.Spec.Validation.OpenAPIV3Schema.Properties["foo"] + prop.Type = "boolean" + crd.Spec.Validation.OpenAPIV3Schema.Properties["foo"] = prop + if _, err = apiextensionsclient.ApiextensionsV1beta1().CustomResourceDefinitions().Update(crd); err != nil { + t.Fatal(err) + } + waitForSpec("boolean") +} + type Foo struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`