From 7a06299f4affcfe625d977e2d08a6aff697fc2f7 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Mon, 8 May 2017 17:24:15 +0200 Subject: [PATCH 1/3] apitesting: external serialization roundtrip test --- .../apimachinery/pkg/api/testing/roundtrip.go | 49 +++++++++++++++---- 1 file changed, 40 insertions(+), 9 deletions(-) diff --git a/staging/src/k8s.io/apimachinery/pkg/api/testing/roundtrip.go b/staging/src/k8s.io/apimachinery/pkg/api/testing/roundtrip.go index 217614ce7b..e6daf8c181 100644 --- a/staging/src/k8s.io/apimachinery/pkg/api/testing/roundtrip.go +++ b/staging/src/k8s.io/apimachinery/pkg/api/testing/roundtrip.go @@ -36,6 +36,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" runtimeserializer "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/apimachinery/pkg/runtime/serializer/json" "k8s.io/apimachinery/pkg/util/diff" "k8s.io/apimachinery/pkg/util/sets" ) @@ -125,24 +126,28 @@ func roundTripTypes(t *testing.T, scheme *runtime.Scheme, codecFactory runtimese } } -func RoundTripSpecificKindWithoutProtobuf(t *testing.T, internalGVK schema.GroupVersionKind, scheme *runtime.Scheme, codecFactory runtimeserializer.CodecFactory, fuzzer *fuzz.Fuzzer, nonRoundTrippableTypes map[schema.GroupVersionKind]bool) { - roundTripSpecificKind(t, internalGVK, scheme, codecFactory, fuzzer, nonRoundTrippableTypes, true) +func RoundTripSpecificKindWithoutProtobuf(t *testing.T, gvk schema.GroupVersionKind, scheme *runtime.Scheme, codecFactory runtimeserializer.CodecFactory, fuzzer *fuzz.Fuzzer, nonRoundTrippableTypes map[schema.GroupVersionKind]bool) { + roundTripSpecificKind(t, gvk, scheme, codecFactory, fuzzer, nonRoundTrippableTypes, true) } -func RoundTripSpecificKind(t *testing.T, internalGVK schema.GroupVersionKind, scheme *runtime.Scheme, codecFactory runtimeserializer.CodecFactory, fuzzer *fuzz.Fuzzer, nonRoundTrippableTypes map[schema.GroupVersionKind]bool) { - roundTripSpecificKind(t, internalGVK, scheme, codecFactory, fuzzer, nonRoundTrippableTypes, false) +func RoundTripSpecificKind(t *testing.T, gvk schema.GroupVersionKind, scheme *runtime.Scheme, codecFactory runtimeserializer.CodecFactory, fuzzer *fuzz.Fuzzer, nonRoundTrippableTypes map[schema.GroupVersionKind]bool) { + roundTripSpecificKind(t, gvk, scheme, codecFactory, fuzzer, nonRoundTrippableTypes, false) } -func roundTripSpecificKind(t *testing.T, internalGVK schema.GroupVersionKind, scheme *runtime.Scheme, codecFactory runtimeserializer.CodecFactory, fuzzer *fuzz.Fuzzer, nonRoundTrippableTypes map[schema.GroupVersionKind]bool, skipProtobuf bool) { - if nonRoundTrippableTypes[internalGVK] { - t.Logf("skipping %v", internalGVK) +func roundTripSpecificKind(t *testing.T, gvk schema.GroupVersionKind, scheme *runtime.Scheme, codecFactory runtimeserializer.CodecFactory, fuzzer *fuzz.Fuzzer, nonRoundTrippableTypes map[schema.GroupVersionKind]bool, skipProtobuf bool) { + if nonRoundTrippableTypes[gvk] { + t.Logf("skipping %v", gvk) return } - t.Logf("round tripping %v", internalGVK) + t.Logf("round tripping %v", gvk) // Try a few times, since runTest uses random values. for i := 0; i < *FuzzIters; i++ { - roundTripToAllExternalVersions(t, scheme, codecFactory, fuzzer, internalGVK, nonRoundTrippableTypes, skipProtobuf) + if gvk.Version == runtime.APIVersionInternal { + roundTripToAllExternalVersions(t, scheme, codecFactory, fuzzer, gvk, nonRoundTrippableTypes, skipProtobuf) + } else { + roundTripOfExternalType(t, scheme, codecFactory, fuzzer, gvk, skipProtobuf) + } if t.Failed() { break } @@ -209,6 +214,32 @@ func roundTripToAllExternalVersions(t *testing.T, scheme *runtime.Scheme, codecF } } +func roundTripOfExternalType(t *testing.T, scheme *runtime.Scheme, codecFactory runtimeserializer.CodecFactory, fuzzer *fuzz.Fuzzer, externalGVK schema.GroupVersionKind, skipProtobuf bool) { + object, err := scheme.New(externalGVK) + if err != nil { + t.Fatalf("Couldn't make a %v? %v", externalGVK, err) + } + typeAcc, err := meta.TypeAccessor(object) + if err != nil { + t.Fatalf("%q is not a TypeMeta and cannot be tested - add it to nonRoundTrippableInternalTypes: %v", externalGVK, err) + } + + fuzzInternalObject(t, fuzzer, object) + + externalGoType := reflect.TypeOf(object).PkgPath() + t.Logf("\tround tripping external type %v %v", externalGVK, externalGoType) + + typeAcc.SetKind(externalGVK.Kind) + typeAcc.SetAPIVersion(externalGVK.GroupVersion().String()) + + roundTrip(t, scheme, json.NewSerializer(json.DefaultMetaFactory, scheme, scheme, false), object) + + // TODO remove this hack after we're past the intermediate steps + if !skipProtobuf { + roundTrip(t, scheme, protobuf.NewSerializer(scheme, scheme, "application/protobuf"), object) + } +} + // roundTrip applies a single round-trip test to the given runtime object // using the given codec. The round-trip test ensures that an object can be // deep-copied and converted from internal -> versioned -> internal without From fb2298de18eee7cb3d677143d3dd2cfde440aa6a Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Mon, 8 May 2017 17:25:31 +0200 Subject: [PATCH 2/3] client-go tpr example: round trip external tpr types --- .../apimachinery/pkg/api/testing/roundtrip.go | 11 ++++-- .../apis/tpr/v1/types_test.go | 35 +++++++++++++++++++ 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/staging/src/k8s.io/apimachinery/pkg/api/testing/roundtrip.go b/staging/src/k8s.io/apimachinery/pkg/api/testing/roundtrip.go index e6daf8c181..8e665d86f8 100644 --- a/staging/src/k8s.io/apimachinery/pkg/api/testing/roundtrip.go +++ b/staging/src/k8s.io/apimachinery/pkg/api/testing/roundtrip.go @@ -242,8 +242,15 @@ func roundTripOfExternalType(t *testing.T, scheme *runtime.Scheme, codecFactory // roundTrip applies a single round-trip test to the given runtime object // using the given codec. The round-trip test ensures that an object can be -// deep-copied and converted from internal -> versioned -> internal without -// loss of data. +// deep-copied, converted, marshaled and back without loss of data. +// +// For internal types this means +// +// internal -> external -> json/protobuf -> external -> internal. +// +// For external types this means +// +// external -> json/protobuf -> external. func roundTrip(t *testing.T, scheme *runtime.Scheme, codec runtime.Codec, object runtime.Object) { printer := spew.ConfigState{DisableMethods: true} original := object diff --git a/staging/src/k8s.io/client-go/examples/third-party-resources/apis/tpr/v1/types_test.go b/staging/src/k8s.io/client-go/examples/third-party-resources/apis/tpr/v1/types_test.go index ebe0bd4852..38f612d890 100644 --- a/staging/src/k8s.io/client-go/examples/third-party-resources/apis/tpr/v1/types_test.go +++ b/staging/src/k8s.io/client-go/examples/third-party-resources/apis/tpr/v1/types_test.go @@ -17,8 +17,15 @@ limitations under the License. package v1 import ( + "math/rand" + "testing" + + "github.com/google/gofuzz" + + apitesting "k8s.io/apimachinery/pkg/api/testing" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" ) var _ runtime.Object = &Example{} @@ -26,3 +33,31 @@ var _ metav1.ObjectMetaAccessor = &Example{} var _ runtime.Object = &ExampleList{} var _ metav1.ListMetaAccessor = &ExampleList{} + +func exampleFuzzerFuncs(t apitesting.TestingCommon) []interface{} { + return []interface{}{ + func(obj *ExampleList, c fuzz.Continue) { + c.FuzzNoCustom(obj) + obj.Items = make([]Example, c.Intn(10)) + for i := range obj.Items { + c.Fuzz(&obj.Items[i]) + } + }, + } +} + +// TestRoundTrip tests that the third-party kinds can be marshaled and unmarshaled correctly to/from JSON +// without the loss of information. Moreover, deep copy is tested. +func TestRoundTrip(t *testing.T) { + scheme := runtime.NewScheme() + codecs := serializer.NewCodecFactory(scheme) + + AddToScheme(scheme) + + seed := rand.Int63() + fuzzerFuncs := apitesting.MergeFuzzerFuncs(t, apitesting.GenericFuzzerFuncs(t, codecs), exampleFuzzerFuncs(t)) + fuzzer := apitesting.FuzzerFor(fuzzerFuncs, rand.NewSource(seed)) + + apitesting.RoundTripSpecificKindWithoutProtobuf(t, SchemeGroupVersion.WithKind("Example"), scheme, codecs, fuzzer, nil) + apitesting.RoundTripSpecificKindWithoutProtobuf(t, SchemeGroupVersion.WithKind("ExampleList"), scheme, codecs, fuzzer, nil) +} From b498019f2f7aebe1c0e5c0d7a825069e25650fc6 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Wed, 10 May 2017 12:33:58 +0200 Subject: [PATCH 3/3] Update generated files --- staging/src/k8s.io/apimachinery/pkg/api/testing/BUILD | 1 + staging/src/k8s.io/client-go/Godeps/Godeps.json | 8 ++++++++ .../examples/third-party-resources/apis/tpr/v1/BUILD | 3 +++ 3 files changed, 12 insertions(+) diff --git a/staging/src/k8s.io/apimachinery/pkg/api/testing/BUILD b/staging/src/k8s.io/apimachinery/pkg/api/testing/BUILD index 993e12b18a..657cc57a91 100644 --- a/staging/src/k8s.io/apimachinery/pkg/api/testing/BUILD +++ b/staging/src/k8s.io/apimachinery/pkg/api/testing/BUILD @@ -38,6 +38,7 @@ go_library( "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/runtime/serializer/json:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/serializer/protobuf:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/serializer/recognizer:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", diff --git a/staging/src/k8s.io/client-go/Godeps/Godeps.json b/staging/src/k8s.io/client-go/Godeps/Godeps.json index 970e9d68d5..49dd81a79f 100644 --- a/staging/src/k8s.io/client-go/Godeps/Godeps.json +++ b/staging/src/k8s.io/client-go/Godeps/Godeps.json @@ -122,6 +122,10 @@ "ImportPath": "github.com/golang/groupcache/lru", "Rev": "02826c3e79038b59d737d3b1c0a1d937f71a4433" }, + { + "ImportPath": "github.com/golang/protobuf/proto", + "Rev": "4bd1920723d7b7c925de087aa32e2187708897f7" + }, { "ImportPath": "github.com/google/gofuzz", "Rev": "44d81051d367757e1c7c6a5a86423ece9afcf63c" @@ -286,6 +290,10 @@ "ImportPath": "k8s.io/apimachinery/pkg/api/resource", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, + { + "ImportPath": "k8s.io/apimachinery/pkg/api/testing", + "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, { "ImportPath": "k8s.io/apimachinery/pkg/apimachinery", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" diff --git a/staging/src/k8s.io/client-go/examples/third-party-resources/apis/tpr/v1/BUILD b/staging/src/k8s.io/client-go/examples/third-party-resources/apis/tpr/v1/BUILD index 8db010a125..f00e0752f9 100644 --- a/staging/src/k8s.io/client-go/examples/third-party-resources/apis/tpr/v1/BUILD +++ b/staging/src/k8s.io/client-go/examples/third-party-resources/apis/tpr/v1/BUILD @@ -14,8 +14,11 @@ go_test( library = ":go_default_library", tags = ["automanaged"], deps = [ + "//vendor/github.com/google/gofuzz:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/api/testing:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library", ], )