Add verb support for discovery client

pull/6/head
Dr. Stefan Schimanski 2016-11-17 14:19:03 +01:00 committed by Dr. Stefan Schimanski
parent 4d1d98c49a
commit 458d2b2fe4
18 changed files with 580 additions and 208 deletions

View File

@ -393,25 +393,40 @@ func StartControllers(s *options.CMServer, rootClientBuilder, clientBuilder cont
namespaceKubeClient := clientBuilder.ClientOrDie("namespace-controller")
namespaceClientPool := dynamic.NewClientPool(rootClientBuilder.ConfigOrDie("namespace-controller"), restMapper, dynamic.LegacyAPIPathResolverFunc)
// TODO: consider using a list-watch + cache here rather than polling
var gvrFn func() ([]schema.GroupVersionResource, error)
gvrFn := func() (map[schema.GroupVersionResource]struct{}, error) {
resources, err := namespaceKubeClient.Discovery().ServerPreferredNamespacedResources()
if err != nil {
// best effort extraction
gvrs, _ := discovery.GroupVersionResources(resources)
return gvrs, fmt.Errorf("failed to get supported namespaced resources: %v", err)
}
gvrs, err := discovery.GroupVersionResources(resources)
if err != nil {
return gvrs, fmt.Errorf("failed to parse supported namespaced resources: %v", err)
}
return gvrs, nil
}
rsrcs, err := namespaceKubeClient.Discovery().ServerResources()
if err != nil {
return fmt.Errorf("failed to get group version resources: %v", err)
}
tprFound := false
searchThirdPartyResource:
for _, rsrcList := range rsrcs {
for ix := range rsrcList.APIResources {
rsrc := &rsrcList.APIResources[ix]
if rsrc.Kind == "ThirdPartyResource" {
gvrFn = namespaceKubeClient.Discovery().ServerPreferredNamespacedResources
tprFound = true
break searchThirdPartyResource
}
}
}
if gvrFn == nil {
gvr, err := namespaceKubeClient.Discovery().ServerPreferredNamespacedResources()
if !tprFound {
gvr, err := gvrFn()
if err != nil {
return fmt.Errorf("failed to get resources: %v", err)
}
gvrFn = func() ([]schema.GroupVersionResource, error) {
gvrFn = func() (map[schema.GroupVersionResource]struct{}, error) {
return gvr, nil
}
}
@ -548,10 +563,14 @@ func StartControllers(s *options.CMServer, rootClientBuilder, clientBuilder cont
if s.EnableGarbageCollector {
gcClientset := clientBuilder.ClientOrDie("generic-garbage-collector")
groupVersionResources, err := gcClientset.Discovery().ServerPreferredResources()
preferredResources, err := gcClientset.Discovery().ServerPreferredResources()
if err != nil {
return fmt.Errorf("failed to get supported resources from server: %v", err)
}
groupVersionResources, err := discovery.GroupVersionResources(preferredResources)
if err != nil {
glog.Fatalf("Failed to parse supported resources from server: %v", err)
}
config := rootClientBuilder.ConfigOrDie("generic-garbage-collector")
config.ContentConfig.NegotiatedSerializer = serializer.DirectCodecFactory{CodecFactory: metaonly.NewMetadataCodecFactory()}

View File

@ -45,7 +45,7 @@ type Fake struct {
// for every request in the order they are tried.
ProxyReactionChain []ProxyReactor
Resources map[string]*metav1.APIResourceList
Resources []*metav1.APIResourceList
}
// Reactor is an interface to allow the composition of reaction functions.
@ -225,10 +225,16 @@ func (c *FakeDiscovery) ServerResourcesForGroupVersion(groupVersion string) (*me
Resource: schema.GroupVersionResource{Resource: "resource"},
}
c.Invokes(action, nil)
return c.Resources[groupVersion], nil
for _, rl := range c.Resources {
if rl.GroupVersion == groupVersion {
return rl, nil
}
}
return nil, fmt.Errorf("GroupVersion %q not found", groupVersion)
}
func (c *FakeDiscovery) ServerResources() (map[string]*metav1.APIResourceList, error) {
func (c *FakeDiscovery) ServerResources() ([]*metav1.APIResourceList, error) {
action := ActionImpl{
Verb: "get",
Resource: schema.GroupVersionResource{Resource: "resource"},

View File

@ -36,6 +36,9 @@ import (
"k8s.io/kubernetes/pkg/version"
)
// defaultRetries is the number of times a resource discovery is repeated if an api group disappears on the fly (e.g. ThirdPartyResources).
const defaultRetries = 2
// DiscoveryInterface holds the methods that discover server-supported API groups,
// versions and resources.
type DiscoveryInterface interface {
@ -67,13 +70,13 @@ type ServerResourcesInterface interface {
// ServerResourcesForGroupVersion returns the supported resources for a group and version.
ServerResourcesForGroupVersion(groupVersion string) (*metav1.APIResourceList, error)
// ServerResources returns the supported resources for all groups and versions.
ServerResources() (map[string]*metav1.APIResourceList, error)
ServerResources() ([]*metav1.APIResourceList, error)
// ServerPreferredResources returns the supported resources with the version preferred by the
// server.
ServerPreferredResources() ([]schema.GroupVersionResource, error)
ServerPreferredResources() ([]*metav1.APIResourceList, error)
// ServerPreferredNamespacedResources returns the supported namespaced resources with the
// version preferred by the server.
ServerPreferredNamespacedResources() ([]schema.GroupVersionResource, error)
ServerPreferredNamespacedResources() ([]*metav1.APIResourceList, error)
}
// ServerVersionInterface has a method for retrieving the server's version.
@ -154,7 +157,9 @@ func (d *DiscoveryClient) ServerResourcesForGroupVersion(groupVersion string) (r
} else {
url.Path = "/apis/" + groupVersion
}
resources = &metav1.APIResourceList{}
resources = &metav1.APIResourceList{
GroupVersion: groupVersion,
}
err = d.restClient.Get().AbsPath(url.String()).Do().Into(resources)
if err != nil {
// ignore 403 or 404 error to be compatible with an v1.0 server.
@ -166,22 +171,43 @@ func (d *DiscoveryClient) ServerResourcesForGroupVersion(groupVersion string) (r
return resources, nil
}
// ServerResources returns the supported resources for all groups and versions.
func (d *DiscoveryClient) ServerResources() (map[string]*metav1.APIResourceList, error) {
// serverResources returns the supported resources for all groups and versions.
func (d *DiscoveryClient) serverResources(failEarly bool) ([]*metav1.APIResourceList, error) {
apiGroups, err := d.ServerGroups()
if err != nil {
return nil, err
}
groupVersions := metav1.ExtractGroupVersions(apiGroups)
result := map[string]*metav1.APIResourceList{}
for _, groupVersion := range groupVersions {
resources, err := d.ServerResourcesForGroupVersion(groupVersion)
if err != nil {
return nil, err
result := []*metav1.APIResourceList{}
failedGroups := make(map[schema.GroupVersion]error)
for _, apiGroup := range apiGroups.Groups {
for _, version := range apiGroup.Versions {
gv := schema.GroupVersion{Group: apiGroup.Name, Version: version.Version}
resources, err := d.ServerResourcesForGroupVersion(version.GroupVersion)
if err != nil {
// TODO: maybe restrict this to NotFound errors
failedGroups[gv] = err
if failEarly {
return nil, &ErrGroupDiscoveryFailed{Groups: failedGroups}
}
continue
}
result = append(result, resources)
}
result[groupVersion] = resources
}
return result, nil
if len(failedGroups) == 0 {
return result, nil
}
return result, &ErrGroupDiscoveryFailed{Groups: failedGroups}
}
// ServerResources returns the supported resources for all groups and versions.
func (d *DiscoveryClient) ServerResources() ([]*metav1.APIResourceList, error) {
return withRetries(defaultRetries, d.serverResources)
}
// ErrGroupDiscoveryFailed is returned if one or more API groups fail to load.
@ -207,78 +233,86 @@ func IsGroupDiscoveryFailedError(err error) bool {
return err != nil && ok
}
// serverPreferredResources returns the supported resources with the version preferred by the
// server. If namespaced is true, only namespaced resources will be returned.
func (d *DiscoveryClient) serverPreferredResources(namespaced bool) ([]schema.GroupVersionResource, error) {
// retry in case the groups supported by the server change after ServerGroup() returns.
const maxRetries = 2
var failedGroups map[schema.GroupVersion]error
var results []schema.GroupVersionResource
var resources map[schema.GroupResource]string
RetrieveGroups:
for i := 0; i < maxRetries; i++ {
results = []schema.GroupVersionResource{}
resources = map[schema.GroupResource]string{}
failedGroups = make(map[schema.GroupVersion]error)
serverGroupList, err := d.ServerGroups()
if err != nil {
return results, err
}
// serverPreferredResources returns the supported resources with the version preferred by the server.
func (d *DiscoveryClient) serverPreferredResources(failEarly bool) ([]*metav1.APIResourceList, error) {
serverGroupList, err := d.ServerGroups()
if err != nil {
return nil, err
}
for _, apiGroup := range serverGroupList.Groups {
versions := apiGroup.Versions
for _, version := range versions {
groupVersion := schema.GroupVersion{Group: apiGroup.Name, Version: version.Version}
apiResourceList, err := d.ServerResourcesForGroupVersion(version.GroupVersion)
if err != nil {
if i < maxRetries-1 {
continue RetrieveGroups
}
failedGroups[groupVersion] = err
result := []*metav1.APIResourceList{}
failedGroups := make(map[schema.GroupVersion]error)
grVersions := map[schema.GroupResource]string{} // selected version of a GroupResource
grApiResources := map[schema.GroupResource]*metav1.APIResource{} // selected APIResource for a GroupResource
gvApiResourceLists := map[schema.GroupVersion]*metav1.APIResourceList{} // blueprint for a APIResourceList for later grouping
for _, apiGroup := range serverGroupList.Groups {
for _, version := range apiGroup.Versions {
groupVersion := schema.GroupVersion{Group: apiGroup.Name, Version: version.Version}
apiResourceList, err := d.ServerResourcesForGroupVersion(version.GroupVersion)
if err != nil {
// TODO: maybe restrict this to NotFound errors
failedGroups[groupVersion] = err
if failEarly {
return nil, &ErrGroupDiscoveryFailed{Groups: failedGroups}
}
continue
}
// create empty list which is filled later in another loop
emptyApiResourceList := metav1.APIResourceList{
GroupVersion: version.GroupVersion,
}
gvApiResourceLists[groupVersion] = &emptyApiResourceList
result = append(result, &emptyApiResourceList)
for i := range apiResourceList.APIResources {
apiResource := &apiResourceList.APIResources[i]
if strings.Contains(apiResource.Name, "/") {
continue
}
for _, apiResource := range apiResourceList.APIResources {
// ignore the root scoped resources if "namespaced" is true.
if namespaced && !apiResource.Namespaced {
continue
}
if strings.Contains(apiResource.Name, "/") {
continue
}
gvr := groupVersion.WithResource(apiResource.Name)
if _, ok := resources[gvr.GroupResource()]; ok {
if gvr.Version != apiGroup.PreferredVersion.Version {
continue
}
// remove previous entry, because it will be replaced with a preferred one
for i := range results {
if results[i].GroupResource() == gvr.GroupResource() {
results = append(results[:i], results[i+1:]...)
}
}
}
resources[gvr.GroupResource()] = gvr.Version
results = append(results, gvr)
gv := schema.GroupResource{Group: apiGroup.Name, Resource: apiResource.Name}
if _, ok := grApiResources[gv]; ok && version.Version != apiGroup.PreferredVersion.Version {
// only override with preferred version
continue
}
grVersions[gv] = version.Version
grApiResources[gv] = apiResource
}
}
if len(failedGroups) == 0 {
return results, nil
}
}
return results, &ErrGroupDiscoveryFailed{Groups: failedGroups}
// group selected APIResources according to GroupVersion into APIResourceLists
for groupResource, apiResource := range grApiResources {
version := grVersions[groupResource]
groupVersion := schema.GroupVersion{Group: groupResource.Group, Version: version}
apiResourceList := gvApiResourceLists[groupVersion]
apiResourceList.APIResources = append(apiResourceList.APIResources, *apiResource)
}
if len(failedGroups) == 0 {
return result, nil
}
return result, &ErrGroupDiscoveryFailed{Groups: failedGroups}
}
// ServerPreferredResources returns the supported resources with the version preferred by the
// server.
func (d *DiscoveryClient) ServerPreferredResources() ([]schema.GroupVersionResource, error) {
return d.serverPreferredResources(false)
func (d *DiscoveryClient) ServerPreferredResources() ([]*metav1.APIResourceList, error) {
return withRetries(defaultRetries, func(retryEarly bool) ([]*metav1.APIResourceList, error) {
return d.serverPreferredResources(retryEarly)
})
}
// ServerPreferredNamespacedResources returns the supported namespaced resources with the
// version preferred by the server.
func (d *DiscoveryClient) ServerPreferredNamespacedResources() ([]schema.GroupVersionResource, error) {
return d.serverPreferredResources(true)
func (d *DiscoveryClient) ServerPreferredNamespacedResources() ([]*metav1.APIResourceList, error) {
all, err := d.ServerPreferredResources()
return FilteredBy(ResourcePredicateFunc(func(groupVersion string, r *metav1.APIResource) bool {
return r.Namespaced
}), all), err
}
// ServerVersion retrieves and parses the server's version (git version).
@ -329,6 +363,23 @@ func (d *DiscoveryClient) SwaggerSchema(version schema.GroupVersion) (*swagger.A
return &schema, nil
}
// withRetries retries the given recovery function in case the groups supported by the server change after ServerGroup() returns.
func withRetries(maxRetries int, f func(failEarly bool) ([]*metav1.APIResourceList, error)) ([]*metav1.APIResourceList, error) {
var result []*metav1.APIResourceList
var err error
for i := 0; i < maxRetries; i++ {
failEarly := i < maxRetries-1
result, err = f(failEarly)
if err == nil {
return result, nil
}
if _, ok := err.(*ErrGroupDiscoveryFailed); !ok {
return nil, err
}
}
return result, err
}
func setDiscoveryDefaults(config *restclient.Config) error {
config.APIPath = ""
config.GroupVersion = nil

View File

@ -29,6 +29,7 @@ import (
metav1 "k8s.io/kubernetes/pkg/apis/meta/v1"
"k8s.io/kubernetes/pkg/client/restclient"
"k8s.io/kubernetes/pkg/runtime/schema"
"k8s.io/kubernetes/pkg/util/sets"
"k8s.io/kubernetes/pkg/version"
)
@ -141,14 +142,14 @@ func TestGetServerResourcesWithV1Server(t *testing.T) {
defer server.Close()
client := NewDiscoveryClientForConfigOrDie(&restclient.Config{Host: server.URL})
// ServerResources should not return an error even if server returns error at /api/v1.
resourceMap, err := client.ServerResources()
serverResources, err := client.ServerResources()
if err != nil {
t.Errorf("unexpected error: %v", err)
}
if _, found := resourceMap["v1"]; !found {
t.Errorf("missing v1 in resource map")
gvs := groupVersions(serverResources)
if !sets.NewString(gvs...).Has("v1") {
t.Errorf("missing v1 in resource list: %v", serverResources)
}
}
func TestGetServerResources(t *testing.T) {
@ -161,7 +162,7 @@ func TestGetServerResources(t *testing.T) {
},
}
beta := metav1.APIResourceList{
GroupVersion: "extensions/v1",
GroupVersion: "extensions/v1beta1",
APIResources: []metav1.APIResource{
{Name: "deployments", Namespaced: true, Kind: "Deployment"},
{Name: "ingresses", Namespaced: true, Kind: "Ingress"},
@ -249,13 +250,14 @@ func TestGetServerResources(t *testing.T) {
}
}
resourceMap, err := client.ServerResources()
serverResources, err := client.ServerResources()
if err != nil {
t.Errorf("unexpected error: %v", err)
}
serverGroupVersions := sets.NewString(groupVersions(serverResources)...)
for _, api := range []string{"v1", "extensions/v1beta1"} {
if _, found := resourceMap[api]; !found {
t.Errorf("missing expected api: %s", api)
if !serverGroupVersions.Has(api) {
t.Errorf("missing expected api %q in %v", api, serverResources)
}
}
}
@ -332,12 +334,12 @@ func TestServerPreferredResources(t *testing.T) {
},
}
tests := []struct {
resourcesList *metav1.APIResourceList
resourcesList []*metav1.APIResourceList
response func(w http.ResponseWriter, req *http.Request)
expectErr func(err error) bool
}{
{
resourcesList: &stable,
resourcesList: []*metav1.APIResourceList{&stable},
expectErr: IsGroupDiscoveryFailedError,
response: func(w http.ResponseWriter, req *http.Request) {
var list interface{}
@ -426,7 +428,7 @@ func TestServerPreferredResources(t *testing.T) {
defer server.Close()
client := NewDiscoveryClientForConfigOrDie(&restclient.Config{Host: server.URL})
got, err := client.ServerPreferredResources()
resources, err := client.ServerPreferredResources()
if test.expectErr != nil {
if err == nil {
t.Error("unexpected non-error")
@ -438,7 +440,13 @@ func TestServerPreferredResources(t *testing.T) {
t.Errorf("unexpected error: %v", err)
continue
}
if !reflect.DeepEqual(got, test.resourcesList) {
got, err := GroupVersionResources(resources)
if err != nil {
t.Errorf("unexpected error: %v", err)
continue
}
expected, _ := GroupVersionResources(test.resourcesList)
if !reflect.DeepEqual(got, expected) {
t.Errorf("expected:\n%v\ngot:\n%v\n", test.resourcesList, got)
}
server.Close()
@ -533,10 +541,14 @@ func TestServerPreferredResourcesRetries(t *testing.T) {
defer server.Close()
client := NewDiscoveryClientForConfigOrDie(&restclient.Config{Host: server.URL})
got, err := client.ServerPreferredResources()
resources, err := client.ServerPreferredResources()
if !tc.expectedError(err) {
t.Errorf("case %d: unexpected error: %v", i, err)
}
got, err := GroupVersionResources(resources)
if err != nil {
t.Errorf("case %d: unexpected error: %v", i, err)
}
if len(got) != tc.expectResources {
t.Errorf("case %d: expect %d resources, got %#v", i, tc.expectResources, got)
}
@ -575,7 +587,7 @@ func TestServerPreferredNamespacedResources(t *testing.T) {
}
tests := []struct {
response func(w http.ResponseWriter, req *http.Request)
expected []schema.GroupVersionResource
expected map[schema.GroupVersionResource]struct{}
}{
{
response: func(w http.ResponseWriter, req *http.Request) {
@ -603,9 +615,9 @@ func TestServerPreferredNamespacedResources(t *testing.T) {
w.WriteHeader(http.StatusOK)
w.Write(output)
},
expected: []schema.GroupVersionResource{
{Group: "", Version: "v1", Resource: "pods"},
{Group: "", Version: "v1", Resource: "services"},
expected: map[schema.GroupVersionResource]struct{}{
schema.GroupVersionResource{Group: "", Version: "v1", Resource: "pods"}: {},
schema.GroupVersionResource{Group: "", Version: "v1", Resource: "services"}: {},
},
},
{
@ -646,9 +658,9 @@ func TestServerPreferredNamespacedResources(t *testing.T) {
w.WriteHeader(http.StatusOK)
w.Write(output)
},
expected: []schema.GroupVersionResource{
{Group: "batch", Version: "v1", Resource: "jobs"},
{Group: "batch", Version: "v2alpha1", Resource: "cronjobs"},
expected: map[schema.GroupVersionResource]struct{}{
schema.GroupVersionResource{Group: "batch", Version: "v1", Resource: "jobs"}: {},
schema.GroupVersionResource{Group: "batch", Version: "v2alpha1", Resource: "cronjobs"}: {},
},
},
{
@ -689,27 +701,39 @@ func TestServerPreferredNamespacedResources(t *testing.T) {
w.WriteHeader(http.StatusOK)
w.Write(output)
},
expected: []schema.GroupVersionResource{
{Group: "batch", Version: "v2alpha1", Resource: "jobs"},
{Group: "batch", Version: "v2alpha1", Resource: "cronjobs"},
expected: map[schema.GroupVersionResource]struct{}{
schema.GroupVersionResource{Group: "batch", Version: "v2alpha1", Resource: "jobs"}: {},
schema.GroupVersionResource{Group: "batch", Version: "v2alpha1", Resource: "cronjobs"}: {},
},
},
}
for _, test := range tests {
for i, test := range tests {
server := httptest.NewServer(http.HandlerFunc(test.response))
defer server.Close()
client := NewDiscoveryClientForConfigOrDie(&restclient.Config{Host: server.URL})
got, err := client.ServerPreferredNamespacedResources()
resources, err := client.ServerPreferredNamespacedResources()
if err != nil {
t.Errorf("unexpected error: %v", err)
t.Errorf("[%d] unexpected error: %v", i, err)
continue
}
// we need deterministic order and since during processing in ServerPreferredNamespacedResources
// a map comes into play the result needs sorting
got, err := GroupVersionResources(resources)
if err != nil {
t.Errorf("[%d] unexpected error: %v", i, err)
continue
}
if !reflect.DeepEqual(got, test.expected) {
t.Errorf("expected:\n%v\ngot:\n%v\n", test.expected, got)
t.Errorf("[%d] expected:\n%v\ngot:\n%v\n", i, test.expected, got)
}
server.Close()
}
}
func groupVersions(resources []*metav1.APIResourceList) []string {
result := []string{}
for _, resourceList := range resources {
result = append(result, resourceList.GroupVersion)
}
return result
}

View File

@ -17,7 +17,10 @@ limitations under the License.
package fake
import (
"fmt"
"github.com/emicklei/go-restful/swagger"
"k8s.io/kubernetes/pkg/api/v1"
metav1 "k8s.io/kubernetes/pkg/apis/meta/v1"
"k8s.io/kubernetes/pkg/client/restclient"
@ -36,10 +39,15 @@ func (c *FakeDiscovery) ServerResourcesForGroupVersion(groupVersion string) (*me
Resource: schema.GroupVersionResource{Resource: "resource"},
}
c.Invokes(action, nil)
return c.Resources[groupVersion], nil
for _, resourceList := range c.Resources {
if resourceList.GroupVersion == groupVersion {
return resourceList, nil
}
}
return nil, fmt.Errorf("GroupVersion %q not found", groupVersion)
}
func (c *FakeDiscovery) ServerResources() (map[string]*metav1.APIResourceList, error) {
func (c *FakeDiscovery) ServerResources() ([]*metav1.APIResourceList, error) {
action := core.ActionImpl{
Verb: "get",
Resource: schema.GroupVersionResource{Resource: "resource"},
@ -48,11 +56,11 @@ func (c *FakeDiscovery) ServerResources() (map[string]*metav1.APIResourceList, e
return c.Resources, nil
}
func (c *FakeDiscovery) ServerPreferredResources() ([]schema.GroupVersionResource, error) {
func (c *FakeDiscovery) ServerPreferredResources() ([]*metav1.APIResourceList, error) {
return nil, nil
}
func (c *FakeDiscovery) ServerPreferredNamespacedResources() ([]schema.GroupVersionResource, error) {
func (c *FakeDiscovery) ServerPreferredNamespacedResources() ([]*metav1.APIResourceList, error) {
return nil, nil
}

View File

@ -108,3 +108,55 @@ func NegotiateVersion(client DiscoveryInterface, requiredGV *schema.GroupVersion
return nil, fmt.Errorf("failed to negotiate an api version; server supports: %v, client supports: %v",
serverVersions, clientVersions)
}
// GroupVersionResources converts APIResourceLists to the GroupVersionResources.
func GroupVersionResources(rls []*metav1.APIResourceList) (map[schema.GroupVersionResource]struct{}, error) {
gvrs := map[schema.GroupVersionResource]struct{}{}
for _, rl := range rls {
gv, err := schema.ParseGroupVersion(rl.GroupVersion)
if err != nil {
return nil, err
}
for i := range rl.APIResources {
gvrs[schema.GroupVersionResource{Group: gv.Group, Version: gv.Version, Resource: rl.APIResources[i].Name}] = struct{}{}
}
}
return gvrs, nil
}
// FilteredBy filters by the given predicate. Empty APIResourceLists are dropped.
func FilteredBy(pred ResourcePredicate, rls []*metav1.APIResourceList) []*metav1.APIResourceList {
result := []*metav1.APIResourceList{}
for _, rl := range rls {
filtered := *rl
filtered.APIResources = nil
for i := range rl.APIResources {
if pred.Match(rl.GroupVersion, &rl.APIResources[i]) {
filtered.APIResources = append(filtered.APIResources, rl.APIResources[i])
}
}
if filtered.APIResources != nil {
result = append(result, &filtered)
}
}
return result
}
type ResourcePredicate interface {
Match(groupVersion string, r *metav1.APIResource) bool
}
type ResourcePredicateFunc func(groupVersion string, r *metav1.APIResource) bool
func (fn ResourcePredicateFunc) Match(groupVersion string, r *metav1.APIResource) bool {
return fn(groupVersion, r)
}
// SupportsAllVerbs is a predicate matching a resource iff all given verbs are supported.
type SupportsAllVerbs struct {
Verbs []string
}
func (p SupportsAllVerbs) Match(groupVersion string, r *metav1.APIResource) bool {
return sets.NewString([]string(r.Verbs)...).HasAll(p.Verbs...)
}

View File

@ -29,12 +29,14 @@ import (
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/api/testapi"
"k8s.io/kubernetes/pkg/apimachinery/registered"
metav1 "k8s.io/kubernetes/pkg/apis/meta/v1"
uapi "k8s.io/kubernetes/pkg/apis/meta/v1"
"k8s.io/kubernetes/pkg/client/restclient"
"k8s.io/kubernetes/pkg/client/restclient/fake"
"k8s.io/kubernetes/pkg/client/typed/discovery"
"k8s.io/kubernetes/pkg/runtime"
"k8s.io/kubernetes/pkg/runtime/schema"
"k8s.io/kubernetes/pkg/util/sets"
)
func objBody(object interface{}) io.ReadCloser {
@ -155,3 +157,74 @@ func TestNegotiateVersion(t *testing.T) {
}
}
}
func TestFilteredBy(t *testing.T) {
all := discovery.ResourcePredicateFunc(func(gv string, r *metav1.APIResource) bool {
return true
})
none := discovery.ResourcePredicateFunc(func(gv string, r *metav1.APIResource) bool {
return false
})
onlyV2 := discovery.ResourcePredicateFunc(func(gv string, r *metav1.APIResource) bool {
return strings.HasSuffix(gv, "/v2") || gv == "v2"
})
onlyBar := discovery.ResourcePredicateFunc(func(gv string, r *metav1.APIResource) bool {
return r.Kind == "Bar"
})
foo := []*metav1.APIResourceList{
{
GroupVersion: "foo/v1",
APIResources: []metav1.APIResource{
{Name: "bar", Kind: "Bar"},
{Name: "test", Kind: "Test"},
},
},
{
GroupVersion: "foo/v2",
APIResources: []metav1.APIResource{
{Name: "bar", Kind: "Bar"},
{Name: "test", Kind: "Test"},
},
},
{
GroupVersion: "foo/v3",
APIResources: []metav1.APIResource{},
},
}
tests := []struct {
input []*metav1.APIResourceList
pred discovery.ResourcePredicate
expectedResources []string
}{
{nil, all, []string{}},
{[]*metav1.APIResourceList{
{GroupVersion: "foo/v1"},
}, all, []string{}},
{foo, all, []string{"foo/v1.bar", "foo/v1.test", "foo/v2.bar", "foo/v2.test"}},
{foo, onlyV2, []string{"foo/v2.bar", "foo/v2.test"}},
{foo, onlyBar, []string{"foo/v1.bar", "foo/v2.bar"}},
{foo, none, []string{}},
}
for i, test := range tests {
filtered := discovery.FilteredBy(test.pred, test.input)
if expected, got := sets.NewString(test.expectedResources...), sets.NewString(stringify(filtered)...); !expected.Equal(got) {
t.Errorf("[%d] unexpected group versions: expected=%v, got=%v", i, test.expectedResources, stringify(filtered))
}
}
}
func stringify(rls []*metav1.APIResourceList) []string {
result := []string{}
for _, rl := range rls {
for _, r := range rl.APIResources {
result = append(result, rl.GroupVersion+"."+r.Name)
}
if len(rl.APIResources) == 0 {
result = append(result, rl.GroupVersion)
}
}
return result
}

View File

@ -290,31 +290,34 @@ func (c *fakeCachedDiscoveryInterface) ServerResourcesForGroupVersion(groupVersi
return nil, errors.NewNotFound(schema.GroupResource{}, "")
}
func (c *fakeCachedDiscoveryInterface) ServerResources() (map[string]*metav1.APIResourceList, error) {
func (c *fakeCachedDiscoveryInterface) ServerResources() ([]*metav1.APIResourceList, error) {
if c.enabledA {
av1, _ := c.ServerResourcesForGroupVersion("a/v1")
return map[string]*metav1.APIResourceList{
"a/v1": av1,
}, nil
return []*metav1.APIResourceList{av1}, nil
}
return map[string]*metav1.APIResourceList{}, nil
return []*metav1.APIResourceList{}, nil
}
func (c *fakeCachedDiscoveryInterface) ServerPreferredResources() ([]schema.GroupVersionResource, error) {
func (c *fakeCachedDiscoveryInterface) ServerPreferredResources() ([]*metav1.APIResourceList, error) {
if c.enabledA {
return []schema.GroupVersionResource{
return []*metav1.APIResourceList{
{
Group: "a",
Version: "v1",
Resource: "foo",
GroupVersion: "a/v1",
APIResources: []metav1.APIResource{
{
Name: "foo",
Kind: "Foo",
Verbs: []string{},
},
},
},
}, nil
}
return []schema.GroupVersionResource{}, nil
return nil, nil
}
func (c *fakeCachedDiscoveryInterface) ServerPreferredNamespacedResources() ([]schema.GroupVersionResource, error) {
return []schema.GroupVersionResource{}, nil
func (c *fakeCachedDiscoveryInterface) ServerPreferredNamespacedResources() ([]*metav1.APIResourceList, error) {
return nil, nil
}
func (c *fakeCachedDiscoveryInterface) ServerVersion() (*version.Info, error) {

View File

@ -537,7 +537,7 @@ var ignoredResources = map[schema.GroupVersionResource]struct{}{
schema.GroupVersionResource{Group: "authorization.k8s.io", Version: "v1beta1", Resource: "localsubjectaccessreviews"}: {},
}
func NewGarbageCollector(metaOnlyClientPool dynamic.ClientPool, clientPool dynamic.ClientPool, mapper meta.RESTMapper, resources []schema.GroupVersionResource) (*GarbageCollector, error) {
func NewGarbageCollector(metaOnlyClientPool dynamic.ClientPool, clientPool dynamic.ClientPool, mapper meta.RESTMapper, resources map[schema.GroupVersionResource]struct{}) (*GarbageCollector, error) {
gc := &GarbageCollector{
metaOnlyClientPool: metaOnlyClientPool,
clientPool: clientPool,
@ -557,7 +557,7 @@ func NewGarbageCollector(metaOnlyClientPool dynamic.ClientPool, clientPool dynam
},
gc: gc,
}
for _, resource := range resources {
for resource := range resources {
if _, ok := ignoredResources[resource]; ok {
glog.V(6).Infof("ignore resource %#v", resource)
continue

View File

@ -35,9 +35,9 @@ type RegisteredRateLimiter struct {
// NewRegisteredRateLimiter returns a new RegisteredRateLimiater.
// TODO: NewRegisteredRateLimiter is not dynamic. We need to find a better way
// when GC dynamically change the resources it monitors.
func NewRegisteredRateLimiter(resources []schema.GroupVersionResource) *RegisteredRateLimiter {
func NewRegisteredRateLimiter(resources map[schema.GroupVersionResource]struct{}) *RegisteredRateLimiter {
rateLimiters := make(map[schema.GroupVersion]*sync.Once)
for _, resource := range resources {
for resource := range resources {
gv := resource.GroupVersion()
if _, found := rateLimiters[gv]; !found {
rateLimiters[gv] = &sync.Once{}

View File

@ -58,7 +58,7 @@ type NamespaceController struct {
// namespaces that have been queued up for processing by workers
queue workqueue.RateLimitingInterface
// function to list of preferred group versions and their corresponding resource set for namespace deletion
groupVersionResourcesFn func() ([]schema.GroupVersionResource, error)
groupVersionResourcesFn func() (map[schema.GroupVersionResource]struct{}, error)
// opCache is a cache to remember if a particular operation is not supported to aid dynamic client.
opCache *operationNotSupportedCache
// finalizerToken is the finalizer token managed by this controller
@ -69,7 +69,7 @@ type NamespaceController struct {
func NewNamespaceController(
kubeClient clientset.Interface,
clientPool dynamic.ClientPool,
groupVersionResourcesFn func() ([]schema.GroupVersionResource, error),
groupVersionResourcesFn func() (map[schema.GroupVersionResource]struct{}, error),
resyncPeriod time.Duration,
finalizerToken v1.FinalizerName) *NamespaceController {

View File

@ -343,13 +343,13 @@ func deleteAllContent(
kubeClient clientset.Interface,
clientPool dynamic.ClientPool,
opCache *operationNotSupportedCache,
groupVersionResources []schema.GroupVersionResource,
groupVersionResources map[schema.GroupVersionResource]struct{},
namespace string,
namespaceDeletedAt metav1.Time,
) (int64, error) {
estimate := int64(0)
glog.V(4).Infof("namespace controller - deleteAllContent - namespace: %s, gvrs: %v", namespace, groupVersionResources)
for _, gvr := range groupVersionResources {
for gvr := range groupVersionResources {
gvrEstimate, err := deleteAllContentForGroupVersionResource(kubeClient, clientPool, opCache, gvr, namespace, namespaceDeletedAt)
if err != nil {
return estimate, err
@ -367,7 +367,7 @@ func syncNamespace(
kubeClient clientset.Interface,
clientPool dynamic.ClientPool,
opCache *operationNotSupportedCache,
groupVersionResourcesFn func() ([]schema.GroupVersionResource, error),
groupVersionResourcesFn func() (map[schema.GroupVersionResource]struct{}, error),
namespace *v1.Namespace,
finalizerToken v1.FinalizerName,
) error {

View File

@ -28,14 +28,18 @@ import (
"testing"
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/api/meta"
"k8s.io/kubernetes/pkg/api/rest"
"k8s.io/kubernetes/pkg/api/testapi"
"k8s.io/kubernetes/pkg/apimachinery/registered"
"k8s.io/kubernetes/pkg/api/v1"
"k8s.io/kubernetes/pkg/apimachinery"
"k8s.io/kubernetes/pkg/apis/extensions"
metav1 "k8s.io/kubernetes/pkg/apis/meta/v1"
"k8s.io/kubernetes/pkg/auth/authorizer"
"k8s.io/kubernetes/pkg/auth/user"
openapigen "k8s.io/kubernetes/pkg/generated/openapi"
"k8s.io/kubernetes/pkg/runtime"
"k8s.io/kubernetes/pkg/runtime/schema"
etcdtesting "k8s.io/kubernetes/pkg/storage/etcd/testing"
utilnet "k8s.io/kubernetes/pkg/util/net"
"k8s.io/kubernetes/pkg/util/sets"
@ -99,52 +103,165 @@ func TestInstallAPIGroups(t *testing.T) {
defer etcdserver.Terminate(t)
config.LegacyAPIGroupPrefixes = sets.NewString("/apiPrefix")
config.DiscoveryAddresses = DefaultDiscoveryAddresses{DefaultAddress: "ExternalAddress"}
s, err := config.SkipComplete().New()
if err != nil {
t.Fatalf("Error in bringing up the server: %v", err)
}
apiGroupMeta := registered.GroupOrDie(api.GroupName)
extensionsGroupMeta := registered.GroupOrDie(extensions.GroupName)
s.InstallLegacyAPIGroup("/apiPrefix", &APIGroupInfo{
// legacy group version
GroupMeta: *apiGroupMeta,
VersionedResourcesStorageMap: map[string]map[string]rest.Storage{},
ParameterCodec: api.ParameterCodec,
NegotiatedSerializer: api.Codecs,
})
testAPI := func(gv schema.GroupVersion) APIGroupInfo {
getter, noVerbs := testGetterStorage{}, testNoVerbsStorage{}
apiGroupsInfo := []APIGroupInfo{
{
// extensions group version
GroupMeta: *extensionsGroupMeta,
VersionedResourcesStorageMap: map[string]map[string]rest.Storage{},
OptionsExternalVersion: &apiGroupMeta.GroupVersion,
ParameterCodec: api.ParameterCodec,
NegotiatedSerializer: api.Codecs,
},
scheme := runtime.NewScheme()
scheme.AddKnownTypeWithName(gv.WithKind("Getter"), getter.New())
scheme.AddKnownTypeWithName(gv.WithKind("NoVerb"), noVerbs.New())
scheme.AddKnownTypes(v1.SchemeGroupVersion,
&v1.ListOptions{},
&v1.DeleteOptions{},
&metav1.ExportOptions{},
&metav1.Status{},
)
interfacesFor := func(version schema.GroupVersion) (*meta.VersionInterfaces, error) {
return &meta.VersionInterfaces{
ObjectConvertor: scheme,
MetadataAccessor: meta.NewAccessor(),
}, nil
}
mapper := api.NewDefaultRESTMapperFromScheme([]schema.GroupVersion{gv}, interfacesFor, "", sets.NewString(), sets.NewString(), scheme)
groupMeta := apimachinery.GroupMeta{
GroupVersion: gv,
GroupVersions: []schema.GroupVersion{gv},
RESTMapper: mapper,
InterfacesFor: interfacesFor,
}
return APIGroupInfo{
GroupMeta: groupMeta,
VersionedResourcesStorageMap: map[string]map[string]rest.Storage{
gv.Version: {
"getter": &testGetterStorage{Version: gv.Version},
"noverbs": &testNoVerbsStorage{Version: gv.Version},
},
},
OptionsExternalVersion: &schema.GroupVersion{Version: "v1"},
ParameterCodec: api.ParameterCodec,
NegotiatedSerializer: api.Codecs,
Scheme: scheme,
}
}
for i := range apiGroupsInfo {
s.InstallAPIGroup(&apiGroupsInfo[i])
apis := []APIGroupInfo{
testAPI(schema.GroupVersion{Group: "", Version: "v1"}),
testAPI(schema.GroupVersion{Group: "extensions", Version: "v1"}),
testAPI(schema.GroupVersion{Group: "batch", Version: "v1"}),
}
err = s.InstallLegacyAPIGroup("/apiPrefix", &apis[0])
assert.NoError(err)
groupPaths := []string{
config.LegacyAPIGroupPrefixes.List()[0], // /apiPrefix
}
for _, api := range apis[1:] {
err = s.InstallAPIGroup(&api)
assert.NoError(err)
groupPaths = append(groupPaths, APIGroupPrefix+"/"+api.GroupMeta.GroupVersion.Group) // /apis/<group>
}
server := httptest.NewServer(s.InsecureHandler)
defer server.Close()
validPaths := []string{
// "/api"
config.LegacyAPIGroupPrefixes.List()[0],
// "/api/v1"
config.LegacyAPIGroupPrefixes.List()[0] + "/" + apiGroupMeta.GroupVersion.Version,
// "/apis/extensions"
APIGroupPrefix + "/" + extensionsGroupMeta.GroupVersion.Group,
// "/apis/extensions/v1beta1"
APIGroupPrefix + "/" + extensionsGroupMeta.GroupVersion.String(),
}
for _, path := range validPaths {
_, err := http.Get(server.URL + path)
if !assert.NoError(err) {
t.Errorf("unexpected error: %v, for path: %s", err, path)
for i := range apis {
// should serve APIGroup at group path
info := &apis[i]
path := groupPaths[i]
resp, err := http.Get(server.URL + path)
if err != nil {
t.Errorf("[%d] unexpected error getting path %q path: %v", i, path, err)
continue
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
t.Errorf("[%d] unexpected error reading body at path %q: %v", i, path, err)
continue
}
t.Logf("[%d] json at %s: %s", i, path, string(body))
if i == 0 {
// legacy API returns APIVersions
group := metav1.APIVersions{}
err = json.Unmarshal(body, &group)
if err != nil {
t.Errorf("[%d] unexpected error parsing json body at path %q: %v", i, path, err)
continue
}
} else {
// API groups return APIGroup
group := metav1.APIGroup{}
err = json.Unmarshal(body, &group)
if err != nil {
t.Errorf("[%d] unexpected error parsing json body at path %q: %v", i, path, err)
continue
}
if got, expected := group.Name, info.GroupMeta.GroupVersion.Group; got != expected {
t.Errorf("[%d] unexpected group name at path %q: got=%q expected=%q", i, path, got, expected)
continue
}
if got, expected := group.PreferredVersion.Version, info.GroupMeta.GroupVersion.Version; got != expected {
t.Errorf("[%d] unexpected group version at path %q: got=%q expected=%q", i, path, got, expected)
continue
}
}
// should serve APIResourceList at group path + /<group-version>
path = path + "/" + info.GroupMeta.GroupVersion.Version
resp, err = http.Get(server.URL + path)
if err != nil {
t.Errorf("[%d] unexpected error getting path %q path: %v", i, path, err)
continue
}
body, err = ioutil.ReadAll(resp.Body)
if err != nil {
t.Errorf("[%d] unexpected error reading body at path %q: %v", i, path, err)
continue
}
t.Logf("[%d] json at %s: %s", i, path, string(body))
resources := metav1.APIResourceList{}
err = json.Unmarshal(body, &resources)
if err != nil {
t.Errorf("[%d] unexpected error parsing json body at path %q: %v", i, path, err)
continue
}
if got, expected := resources.GroupVersion, info.GroupMeta.GroupVersion.String(); got != expected {
t.Errorf("[%d] unexpected groupVersion at path %q: got=%q expected=%q", i, path, got, expected)
continue
}
// the verbs should match the features of resources
for _, r := range resources.APIResources {
switch r.Name {
case "getter":
if got, expected := sets.NewString([]string(r.Verbs)...), sets.NewString("get"); !got.Equal(expected) {
t.Errorf("[%d] unexpected verbs for resource %s/%s: got=%v expected=%v", i, resources.GroupVersion, r.Name, got, expected)
}
case "noverbs":
if r.Verbs == nil {
t.Errorf("[%d] unexpected nil verbs slice. Expected: []string{}", i)
}
if got, expected := sets.NewString([]string(r.Verbs)...), sets.NewString(); !got.Equal(expected) {
t.Errorf("[%d] unexpected verbs for resource %s/%s: got=%v expected=%v", i, resources.GroupVersion, r.Name, got, expected)
}
}
}
}
}
@ -462,3 +579,33 @@ func TestGetServerAddressByClientCIDRs(t *testing.T) {
}
}
}
type testGetterStorage struct {
Version string
}
func (p *testGetterStorage) New() runtime.Object {
return &metav1.APIGroup{
TypeMeta: metav1.TypeMeta{
Kind: "Getter",
APIVersion: p.Version,
},
}
}
func (p *testGetterStorage) Get(ctx api.Context, name string) (runtime.Object, error) {
return nil, nil
}
type testNoVerbsStorage struct {
Version string
}
func (p *testNoVerbsStorage) New() runtime.Object {
return &metav1.APIGroup{
TypeMeta: metav1.TypeMeta{
Kind: "NoVerbs",
APIVersion: p.Version,
},
}
}

View File

@ -33,6 +33,7 @@ import (
"k8s.io/kubernetes/pkg/apis/extensions/v1beta1"
metav1 "k8s.io/kubernetes/pkg/apis/meta/v1"
coreclient "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/core/internalversion"
"k8s.io/kubernetes/pkg/client/typed/discovery"
conditions "k8s.io/kubernetes/pkg/client/unversioned"
"k8s.io/kubernetes/pkg/kubectl"
"k8s.io/kubernetes/pkg/kubectl/cmd/templates"
@ -367,20 +368,11 @@ func Run(f cmdutil.Factory, cmdIn io.Reader, cmdOut, cmdErr io.Writer, cmd *cobr
}
// TODO turn this into reusable method checking available resources
func contains(resourcesList map[string]*metav1.APIResourceList, resource schema.GroupVersionResource) bool {
if resourcesList == nil {
return false
}
resourcesGroup, ok := resourcesList[resource.GroupVersion().String()]
if !ok {
return false
}
for _, item := range resourcesGroup.APIResources {
if resource.Resource == item.Name {
return true
}
}
return false
func contains(resourcesList []*metav1.APIResourceList, resource schema.GroupVersionResource) bool {
resources := discovery.FilteredBy(discovery.ResourcePredicateFunc(func(gv string, r *metav1.APIResource) bool {
return resource.GroupVersion().String() == gv && resource.Resource == r.Name
}), resourcesList)
return len(resources) != 0
}
// waitForPod watches the given pod until the exitCondition is true. Each two seconds

View File

@ -86,19 +86,19 @@ func (d *CachedDiscoveryClient) ServerResourcesForGroupVersion(groupVersion stri
}
// ServerResources returns the supported resources for all groups and versions.
func (d *CachedDiscoveryClient) ServerResources() (map[string]*metav1.APIResourceList, error) {
func (d *CachedDiscoveryClient) ServerResources() ([]*metav1.APIResourceList, error) {
apiGroups, err := d.ServerGroups()
if err != nil {
return nil, err
}
groupVersions := metav1.ExtractGroupVersions(apiGroups)
result := map[string]*metav1.APIResourceList{}
result := []*metav1.APIResourceList{}
for _, groupVersion := range groupVersions {
resources, err := d.ServerResourcesForGroupVersion(groupVersion)
if err != nil {
return nil, err
}
result[groupVersion] = resources
result = append(result, resources)
}
return result, nil
}
@ -209,11 +209,11 @@ func (d *CachedDiscoveryClient) RESTClient() restclient.Interface {
return d.delegate.RESTClient()
}
func (d *CachedDiscoveryClient) ServerPreferredResources() ([]schema.GroupVersionResource, error) {
func (d *CachedDiscoveryClient) ServerPreferredResources() ([]*metav1.APIResourceList, error) {
return d.delegate.ServerPreferredResources()
}
func (d *CachedDiscoveryClient) ServerPreferredNamespacedResources() ([]schema.GroupVersionResource, error) {
func (d *CachedDiscoveryClient) ServerPreferredNamespacedResources() ([]*metav1.APIResourceList, error) {
return d.delegate.ServerPreferredNamespacedResources()
}

View File

@ -139,19 +139,19 @@ func (c *fakeDiscoveryClient) ServerResourcesForGroupVersion(groupVersion string
return nil, errors.NewNotFound(schema.GroupResource{}, "")
}
func (c *fakeDiscoveryClient) ServerResources() (map[string]*metav1.APIResourceList, error) {
func (c *fakeDiscoveryClient) ServerResources() ([]*metav1.APIResourceList, error) {
c.resourceCalls = c.resourceCalls + 1
return map[string]*metav1.APIResourceList{}, nil
return []*metav1.APIResourceList{}, nil
}
func (c *fakeDiscoveryClient) ServerPreferredResources() ([]schema.GroupVersionResource, error) {
func (c *fakeDiscoveryClient) ServerPreferredResources() ([]*metav1.APIResourceList, error) {
c.resourceCalls = c.resourceCalls + 1
return []schema.GroupVersionResource{}, nil
return nil, nil
}
func (c *fakeDiscoveryClient) ServerPreferredNamespacedResources() ([]schema.GroupVersionResource, error) {
func (c *fakeDiscoveryClient) ServerPreferredNamespacedResources() ([]*metav1.APIResourceList, error) {
c.resourceCalls = c.resourceCalls + 1
return []schema.GroupVersionResource{}, nil
return nil, nil
}
func (c *fakeDiscoveryClient) ServerVersion() (*version.Info, error) {

View File

@ -51,21 +51,14 @@ func (e ShortcutExpander) getAll() []schema.GroupResource {
return e.All
}
availableResources := []schema.GroupVersionResource{}
for groupVersionString, resourceList := range apiResources {
currVersion, err := schema.ParseGroupVersion(groupVersionString)
if err != nil {
return e.All
}
for _, resource := range resourceList.APIResources {
availableResources = append(availableResources, currVersion.WithResource(resource.Name))
}
availableResources, err := discovery.GroupVersionResources(apiResources)
if err != nil {
return e.All
}
availableAll := []schema.GroupResource{}
for _, requestedResource := range e.All {
for _, availableResource := range availableResources {
for availableResource := range availableResources {
if requestedResource.Group == availableResource.Group &&
requestedResource.Resource == availableResource.Resource {
availableAll = append(availableAll, requestedResource)

View File

@ -1084,7 +1084,11 @@ func hasRemainingContent(c clientset.Interface, clientPool dynamic.ClientPool, n
}
// find out what content is supported on the server
groupVersionResources, err := c.Discovery().ServerPreferredNamespacedResources()
resources, err := c.Discovery().ServerPreferredNamespacedResources()
if err != nil {
return false, err
}
groupVersionResources, err := discovery.GroupVersionResources(resources)
if err != nil {
return false, err
}
@ -1095,7 +1099,7 @@ func hasRemainingContent(c clientset.Interface, clientPool dynamic.ClientPool, n
contentRemaining := false
// dump how many of resource type is on the server in a log.
for _, gvr := range groupVersionResources {
for gvr := range groupVersionResources {
// get a client for this group version...
dynamicClient, err := clientPool.ClientForGroupVersionResource(gvr)
if err != nil {