From 973a3c723350a07fbcf29655a4c0584f35b457da Mon Sep 17 00:00:00 2001 From: Bin Lu Date: Fri, 1 Mar 2019 17:25:04 +0800 Subject: [PATCH 01/21] Add bazel-test-integration for Arm64 Signed-off-by: Bin Lu --- build/root/WORKSPACE | 10 ---------- build/workspace.bzl | 20 +++++++++++++++++++- test/integration/framework/BUILD | 21 ++++++++++++++++++--- test/integration/framework/etcd.go | 4 +++- 4 files changed, 40 insertions(+), 15 deletions(-) diff --git a/build/root/WORKSPACE b/build/root/WORKSPACE index 43e2868bda..e0b66991b2 100644 --- a/build/root/WORKSPACE +++ b/build/root/WORKSPACE @@ -19,16 +19,6 @@ http_archive( urls = mirror("https://github.com/kubernetes/repo-infra/archive/b461270ab6ccfb94ff2d78df96d26f669376d660.tar.gz"), ) -ETCD_VERSION = "3.3.10" - -http_archive( - name = "com_coreos_etcd", - build_file = "@//third_party:etcd.BUILD", - sha256 = "1620a59150ec0a0124a65540e23891243feb2d9a628092fb1edcc23974724a45", - strip_prefix = "etcd-v%s-linux-amd64" % ETCD_VERSION, - urls = mirror("https://github.com/coreos/etcd/releases/download/v%s/etcd-v%s-linux-amd64.tar.gz" % (ETCD_VERSION, ETCD_VERSION)), -) - http_archive( name = "io_bazel_rules_go", sha256 = "492c3ac68ed9dcf527a07e6a1b2dcbf199c6bf8b35517951467ac32e421c06c1", diff --git a/build/workspace.bzl b/build/workspace.bzl index 6a0252e7c4..0b281235a6 100644 --- a/build/workspace.bzl +++ b/build/workspace.bzl @@ -14,7 +14,7 @@ load("//build:platforms.bzl", "SERVER_PLATFORMS") load("//build:workspace_mirror.bzl", "mirror") -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file") +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_file") load("@io_bazel_rules_docker//container:container.bzl", "container_pull") CNI_VERSION = "0.6.0" @@ -35,6 +35,13 @@ _CRI_TARBALL_ARCH_SHA256 = { "s390x": "814aa9cd496be416612c2653097a1c9eb5784e38aa4889034b44ebf888709057", } +ETCD_VERSION = "3.3.10" +_ETCD_TARBALL_ARCH_SHA256 = { + "amd64": "1620a59150ec0a0124a65540e23891243feb2d9a628092fb1edcc23974724a45", + "arm64": "5ec97b0b872adce275b8130d19db314f7f2b803aeb24c4aae17a19e2d66853c4", + "ppc64le": "148fe96f0ec1813c5db9916199e96a913174304546bc8447a2d2f9fee4b8f6c2", +} + # Note that these are digests for the manifest list. We resolve the manifest # list to each of its platform-specific images in # debian_image_dependencies(). @@ -48,6 +55,7 @@ def release_dependencies(): cni_tarballs() cri_tarballs() debian_image_dependencies() + etcd_tarballs() def cni_tarballs(): for arch, sha in _CNI_TARBALL_ARCH_SHA256.items(): @@ -92,3 +100,13 @@ def debian_image_dependencies(): registry = "k8s.gcr.io", repository = "debian-hyperkube-base", ) + +def etcd_tarballs(): + for arch, sha in _ETCD_TARBALL_ARCH_SHA256.items(): + http_archive( + name = "com_coreos_etcd_%s" % arch, + build_file = "@//third_party:etcd.BUILD", + sha256 = sha, + strip_prefix = "etcd-v%s-linux-%s" % (ETCD_VERSION, arch), + urls = mirror("https://github.com/coreos/etcd/releases/download/v%s/etcd-v%s-linux-%s.tar.gz" % (ETCD_VERSION, ETCD_VERSION, arch)), + ) diff --git a/test/integration/framework/BUILD b/test/integration/framework/BUILD index d8f30fa7c5..5cf2a832f0 100644 --- a/test/integration/framework/BUILD +++ b/test/integration/framework/BUILD @@ -4,6 +4,7 @@ load( "@io_bazel_rules_go//go:def.bzl", "go_library", ) +load("//build:platforms.bzl", "go_platform_constraint") go_library( name = "go_default_library", @@ -15,9 +16,23 @@ go_library( "test_server.go", "util.go", ], - data = [ - "@com_coreos_etcd//:etcd", - ], + data = select({ + go_platform_constraint( + arch = "arm64", + os = "linux", + ): [ + "@com_coreos_etcd_arm64//:etcd", + ], + go_platform_constraint( + arch = "ppc64le", + os = "linux", + ): [ + "@com_coreos_etcd_ppc64le//:etcd", + ], + "//conditions:default": [ + "@com_coreos_etcd_amd64//:etcd", + ], + }), importpath = "k8s.io/kubernetes/test/integration/framework", deps = [ "//cmd/kube-apiserver/app:go_default_library", diff --git a/test/integration/framework/etcd.go b/test/integration/framework/etcd.go index 97aa267f4b..f66469a370 100644 --- a/test/integration/framework/etcd.go +++ b/test/integration/framework/etcd.go @@ -24,6 +24,7 @@ import ( "os" "os/exec" "path/filepath" + "runtime" "strings" "k8s.io/klog" @@ -43,7 +44,8 @@ You can use 'hack/install-etcd.sh' to install a copy in third_party/. // getEtcdPath returns a path to an etcd executable. func getEtcdPath() (string, error) { - bazelPath := filepath.Join(os.Getenv("RUNFILES_DIR"), "com_coreos_etcd/etcd") + bazelPath := filepath.Join(os.Getenv("RUNFILES_DIR"), fmt.Sprintf("com_coreos_etcd_%s", runtime.GOARCH), "etcd") + p, err := exec.LookPath(bazelPath) if err == nil { return p, nil From 1065a473da3f49929668082cfc5925c3506c7f59 Mon Sep 17 00:00:00 2001 From: Minhan Xia Date: Wed, 6 Mar 2019 17:02:29 -0800 Subject: [PATCH 02/21] beef up NEG tests --- test/e2e/framework/ingress/ingress_utils.go | 14 +++++++++ test/e2e/network/ingress.go | 32 ++++++++++++++------- 2 files changed, 36 insertions(+), 10 deletions(-) diff --git a/test/e2e/framework/ingress/ingress_utils.go b/test/e2e/framework/ingress/ingress_utils.go index 6292aa9b24..5fa377d304 100644 --- a/test/e2e/framework/ingress/ingress_utils.go +++ b/test/e2e/framework/ingress/ingress_utils.go @@ -667,12 +667,26 @@ func (j *TestJig) pollIngressWithCert(ing *extensions.Ingress, address string, k } // WaitForIngress waits for the Ingress to get an address. +// WaitForIngress returns when it gets the first 200 response func (j *TestJig) WaitForIngress(waitForNodePort bool) { if err := j.WaitForGivenIngressWithTimeout(j.Ingress, waitForNodePort, framework.LoadBalancerPollTimeout); err != nil { framework.Failf("error in waiting for ingress to get an address: %s", err) } } +// WaitForIngressToStable waits for the LB return 100 consecutive 200 responses. +func (j *TestJig) WaitForIngressToStable() { + if err := wait.Poll(10*time.Second, framework.LoadBalancerCreateTimeoutDefault, func() (bool, error) { + _, err := j.GetDistinctResponseFromIngress() + if err != nil { + return false, nil + } + return true, nil + }); err != nil { + framework.Failf("error in waiting for ingress to stablize: %v", err) + } +} + // WaitForGivenIngressWithTimeout waits till the ingress acquires an IP, // then waits for its hosts/urls to respond to a protocol check (either // http or https). If waitForNodePort is true, the NodePort of the Service diff --git a/test/e2e/network/ingress.go b/test/e2e/network/ingress.go index 1fc1180a08..2b4470f0bb 100644 --- a/test/e2e/network/ingress.go +++ b/test/e2e/network/ingress.go @@ -529,9 +529,10 @@ var _ = SIGDescribe("Loadbalancing: L7", func() { _, err = f.ClientSet.CoreV1().Services(ns).Update(&svc) Expect(err).NotTo(HaveOccurred()) } - wait.Poll(5*time.Second, framework.LoadBalancerPollTimeout, func() (bool, error) { - return gceController.BackendServiceUsingIG(jig.GetServicePorts(true)) + err = wait.Poll(5*time.Second, framework.LoadBalancerPollTimeout, func() (bool, error) { + return gceController.BackendServiceUsingIG(jig.GetServicePorts(false)) }) + Expect(err).NotTo(HaveOccurred(), "Expect backend service to target IG, but failed to observe") jig.WaitForIngress(true) By("Switch backend service to use NEG") @@ -542,9 +543,10 @@ var _ = SIGDescribe("Loadbalancing: L7", func() { _, err = f.ClientSet.CoreV1().Services(ns).Update(&svc) Expect(err).NotTo(HaveOccurred()) } - wait.Poll(5*time.Second, framework.LoadBalancerPollTimeout, func() (bool, error) { + err = wait.Poll(5*time.Second, framework.LoadBalancerPollTimeout, func() (bool, error) { return gceController.BackendServiceUsingNEG(jig.GetServicePorts(false)) }) + Expect(err).NotTo(HaveOccurred(), "Expect backend service to target NEG, but failed to observe") jig.WaitForIngress(true) }) @@ -556,7 +558,7 @@ var _ = SIGDescribe("Loadbalancing: L7", func() { svcPorts := jig.GetServicePorts(false) usingNEG, err := gceController.BackendServiceUsingNEG(svcPorts) Expect(err).NotTo(HaveOccurred()) - Expect(usingNEG).To(BeTrue()) + Expect(usingNEG).To(BeTrue(), "Expect backend service to be using NEG. But not.") // ClusterIP ServicePorts have no NodePort for _, sp := range svcPorts { @@ -574,18 +576,21 @@ var _ = SIGDescribe("Loadbalancing: L7", func() { _, err = f.ClientSet.AppsV1().Deployments(ns).UpdateScale(name, scale) Expect(err).NotTo(HaveOccurred()) } - wait.Poll(10*time.Second, negUpdateTimeout, func() (bool, error) { + err = wait.Poll(10*time.Second, negUpdateTimeout, func() (bool, error) { res, err := jig.GetDistinctResponseFromIngress() if err != nil { return false, nil } + framework.Logf("Expecting %d backends, got %d", num, res.Len()) return res.Len() == num, nil }) + Expect(err).NotTo(HaveOccurred()) } By("Create a basic HTTP ingress using NEG") jig.CreateIngress(filepath.Join(ingress.IngressManifestPath, "neg"), ns, map[string]string{}, map[string]string{}) jig.WaitForIngress(true) + jig.WaitForIngressToStable() usingNEG, err := gceController.BackendServiceUsingNEG(jig.GetServicePorts(false)) Expect(err).NotTo(HaveOccurred()) Expect(usingNEG).To(BeTrue()) @@ -611,6 +616,7 @@ var _ = SIGDescribe("Loadbalancing: L7", func() { By("Create a basic HTTP ingress using NEG") jig.CreateIngress(filepath.Join(ingress.IngressManifestPath, "neg"), ns, map[string]string{}, map[string]string{}) jig.WaitForIngress(true) + jig.WaitForIngressToStable() usingNEG, err := gceController.BackendServiceUsingNEG(jig.GetServicePorts(false)) Expect(err).NotTo(HaveOccurred()) Expect(usingNEG).To(BeTrue()) @@ -621,13 +627,15 @@ var _ = SIGDescribe("Loadbalancing: L7", func() { scale.Spec.Replicas = int32(replicas) _, err = f.ClientSet.AppsV1().Deployments(ns).UpdateScale(name, scale) Expect(err).NotTo(HaveOccurred()) - wait.Poll(10*time.Second, framework.LoadBalancerPollTimeout, func() (bool, error) { + + err = wait.Poll(10*time.Second, framework.LoadBalancerPollTimeout, func() (bool, error) { res, err := jig.GetDistinctResponseFromIngress() if err != nil { return false, nil } return res.Len() == replicas, nil }) + Expect(err).NotTo(HaveOccurred()) By("Trigger rolling update and observe service disruption") deploy, err := f.ClientSet.AppsV1().Deployments(ns).Get(name, metav1.GetOptions{}) @@ -637,7 +645,7 @@ var _ = SIGDescribe("Loadbalancing: L7", func() { deploy.Spec.Template.Spec.TerminationGracePeriodSeconds = &gracePeriod _, err = f.ClientSet.AppsV1().Deployments(ns).Update(deploy) Expect(err).NotTo(HaveOccurred()) - wait.Poll(10*time.Second, framework.LoadBalancerPollTimeout, func() (bool, error) { + err = wait.Poll(10*time.Second, framework.LoadBalancerPollTimeout, func() (bool, error) { res, err := jig.GetDistinctResponseFromIngress() Expect(err).NotTo(HaveOccurred()) deploy, err := f.ClientSet.AppsV1().Deployments(ns).Get(name, metav1.GetOptions{}) @@ -655,6 +663,7 @@ var _ = SIGDescribe("Loadbalancing: L7", func() { return false, nil } }) + Expect(err).NotTo(HaveOccurred()) }) It("should sync endpoints for both Ingress-referenced NEG and standalone NEG", func() { @@ -669,7 +678,7 @@ var _ = SIGDescribe("Loadbalancing: L7", func() { _, err = f.ClientSet.AppsV1().Deployments(ns).UpdateScale(name, scale) Expect(err).NotTo(HaveOccurred()) } - wait.Poll(10*time.Second, negUpdateTimeout, func() (bool, error) { + err = wait.Poll(10*time.Second, negUpdateTimeout, func() (bool, error) { svc, err := f.ClientSet.CoreV1().Services(ns).Get(name, metav1.GetOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -716,6 +725,7 @@ var _ = SIGDescribe("Loadbalancing: L7", func() { return true, nil }) + Expect(err).NotTo(HaveOccurred()) } By("Create a basic HTTP ingress using NEG") @@ -1108,7 +1118,7 @@ func detectHttpVersionAndSchemeTest(f *framework.Framework, jig *ingress.TestJig } func detectNegAnnotation(f *framework.Framework, jig *ingress.TestJig, gceController *gce.GCEIngressController, ns, name string, negs int) { - wait.Poll(5*time.Second, framework.LoadBalancerPollTimeout, func() (bool, error) { + if err := wait.Poll(5*time.Second, negUpdateTimeout, func() (bool, error) { svc, err := f.ClientSet.CoreV1().Services(ns).Get(name, metav1.GetOptions{}) if err != nil { return false, nil @@ -1150,5 +1160,7 @@ func detectNegAnnotation(f *framework.Framework, jig *ingress.TestJig, gceContro } return gceController.BackendServiceUsingNEG(jig.GetServicePorts(false)) - }) + }); err != nil { + Expect(err).NotTo(HaveOccurred()) + } } From 88907da9a4ac1f9684cdefde9ca0efb4db7dbae2 Mon Sep 17 00:00:00 2001 From: Pengfei Ni Date: Mon, 11 Mar 2019 16:40:23 +0800 Subject: [PATCH 03/21] Ensure Azure load balancer cleaned up on 404 or 403 --- .../providers/azure/azure_loadbalancer.go | 32 ++++++++-- .../azure/azure_loadbalancer_test.go | 60 +++++++++++++++++++ .../providers/azure/azure_test.go | 7 +++ .../providers/azure/azure_wrap.go | 13 ++++ 4 files changed, 106 insertions(+), 6 deletions(-) diff --git a/pkg/cloudprovider/providers/azure/azure_loadbalancer.go b/pkg/cloudprovider/providers/azure/azure_loadbalancer.go index 222d92d519..9a114d5ec0 100644 --- a/pkg/cloudprovider/providers/azure/azure_loadbalancer.go +++ b/pkg/cloudprovider/providers/azure/azure_loadbalancer.go @@ -169,27 +169,47 @@ func (az *Cloud) UpdateLoadBalancer(ctx context.Context, clusterName string, ser func (az *Cloud) EnsureLoadBalancerDeleted(ctx context.Context, clusterName string, service *v1.Service) error { isInternal := requiresInternalLoadBalancer(service) serviceName := getServiceName(service) - klog.V(5).Infof("delete(%s): START clusterName=%q", serviceName, clusterName) + klog.V(5).Infof("Delete service (%s): START clusterName=%q", serviceName, clusterName) + + ignoreErrors := func(err error) error { + if ignoreStatusNotFoundFromError(err) == nil { + klog.V(5).Infof("EnsureLoadBalancerDeleted: ignoring StatusNotFound error because the resource doesn't exist (%v)", err) + return nil + } + + if ignoreStatusForbiddenFromError(err) == nil { + klog.V(5).Infof("EnsureLoadBalancerDeleted: ignoring StatusForbidden error (%v). This may be caused by wrong configuration via service annotations", err) + return nil + } + + return err + } serviceIPToCleanup, err := az.findServiceIPAddress(ctx, clusterName, service, isInternal) - if err != nil { + if ignoreErrors(err) != nil { return err } klog.V(2).Infof("EnsureLoadBalancerDeleted: reconciling security group for service %q with IP %q, wantLb = false", serviceName, serviceIPToCleanup) if _, err := az.reconcileSecurityGroup(clusterName, service, &serviceIPToCleanup, false /* wantLb */); err != nil { - return err + if ignoreErrors(err) != nil { + return err + } } if _, err := az.reconcileLoadBalancer(clusterName, service, nil, false /* wantLb */); err != nil { - return err + if ignoreErrors(err) != nil { + return err + } } if _, err := az.reconcilePublicIP(clusterName, service, nil, false /* wantLb */); err != nil { - return err + if ignoreErrors(err) != nil { + return err + } } - klog.V(2).Infof("delete(%s): FINISH", serviceName) + klog.V(2).Infof("Delete service (%s): FINISH", serviceName) return nil } diff --git a/pkg/cloudprovider/providers/azure/azure_loadbalancer_test.go b/pkg/cloudprovider/providers/azure/azure_loadbalancer_test.go index 7dda564e7d..f1368575b6 100644 --- a/pkg/cloudprovider/providers/azure/azure_loadbalancer_test.go +++ b/pkg/cloudprovider/providers/azure/azure_loadbalancer_test.go @@ -17,6 +17,7 @@ limitations under the License. package azure import ( + "context" "fmt" "reflect" "testing" @@ -308,3 +309,62 @@ func TestSubnet(t *testing.T) { assert.Equal(t, c.expected, real, fmt.Sprintf("TestCase[%d]: %s", i, c.desc)) } } + +func TestEnsureLoadBalancerDeleted(t *testing.T) { + const vmCount = 8 + const availabilitySetCount = 4 + const serviceCount = 9 + + tests := []struct { + desc string + service v1.Service + expectCreateError bool + }{ + { + desc: "external service should be created and deleted successfully", + service: getTestService("test1", v1.ProtocolTCP, 80), + }, + { + desc: "internal service should be created and deleted successfully", + service: getInternalTestService("test2", 80), + }, + { + desc: "annotated service with same resourceGroup should be created and deleted successfully", + service: getResourceGroupTestService("test3", "rg", "", 80), + }, + { + desc: "annotated service with different resourceGroup shouldn't be created but should be deleted successfully", + service: getResourceGroupTestService("test4", "random-rg", "1.2.3.4", 80), + expectCreateError: true, + }, + } + + az := getTestCloud() + for i, c := range tests { + clusterResources := getClusterResources(az, vmCount, availabilitySetCount) + getTestSecurityGroup(az) + if c.service.Annotations[ServiceAnnotationLoadBalancerInternal] == "true" { + addTestSubnet(t, az, &c.service) + } + + // create the service first. + lbStatus, err := az.EnsureLoadBalancer(context.TODO(), testClusterName, &c.service, clusterResources.nodes) + if c.expectCreateError { + assert.NotNil(t, err, "TestCase[%d]: %s", i, c.desc) + } else { + assert.Nil(t, err, "TestCase[%d]: %s", i, c.desc) + assert.NotNil(t, lbStatus, "TestCase[%d]: %s", i, c.desc) + result, err := az.LoadBalancerClient.List(context.TODO(), az.Config.ResourceGroup) + assert.Nil(t, err, "TestCase[%d]: %s", i, c.desc) + assert.Equal(t, len(result), 1, "TestCase[%d]: %s", i, c.desc) + assert.Equal(t, len(*result[0].LoadBalancingRules), 1, "TestCase[%d]: %s", i, c.desc) + } + + // finally, delete it. + err = az.EnsureLoadBalancerDeleted(context.TODO(), testClusterName, &c.service) + assert.Nil(t, err, "TestCase[%d]: %s", i, c.desc) + result, err := az.LoadBalancerClient.List(context.Background(), az.Config.ResourceGroup) + assert.Nil(t, err, "TestCase[%d]: %s", i, c.desc) + assert.Equal(t, len(result), 0, "TestCase[%d]: %s", i, c.desc) + } +} diff --git a/pkg/cloudprovider/providers/azure/azure_test.go b/pkg/cloudprovider/providers/azure/azure_test.go index 523809f3b3..e30b0d9297 100644 --- a/pkg/cloudprovider/providers/azure/azure_test.go +++ b/pkg/cloudprovider/providers/azure/azure_test.go @@ -1137,6 +1137,13 @@ func getInternalTestService(identifier string, requestedPorts ...int32) v1.Servi return svc } +func getResourceGroupTestService(identifier, resourceGroup, loadBalancerIP string, requestedPorts ...int32) v1.Service { + svc := getTestService(identifier, v1.ProtocolTCP, requestedPorts...) + svc.Spec.LoadBalancerIP = loadBalancerIP + svc.Annotations[ServiceAnnotationLoadBalancerResourceGroup] = resourceGroup + return svc +} + func setLoadBalancerModeAnnotation(service *v1.Service, lbMode string) { service.Annotations[ServiceAnnotationLoadBalancerMode] = lbMode } diff --git a/pkg/cloudprovider/providers/azure/azure_wrap.go b/pkg/cloudprovider/providers/azure/azure_wrap.go index 95e8c7dac8..277c6debbd 100644 --- a/pkg/cloudprovider/providers/azure/azure_wrap.go +++ b/pkg/cloudprovider/providers/azure/azure_wrap.go @@ -71,6 +71,19 @@ func ignoreStatusNotFoundFromError(err error) error { return err } +// ignoreStatusForbiddenFromError returns nil if the status code is StatusForbidden. +// This happens when AuthorizationFailed is reported from Azure API. +func ignoreStatusForbiddenFromError(err error) error { + if err == nil { + return nil + } + v, ok := err.(autorest.DetailedError) + if ok && v.StatusCode == http.StatusForbidden { + return nil + } + return err +} + /// getVirtualMachine calls 'VirtualMachinesClient.Get' with a timed cache /// The service side has throttling control that delays responses if there're multiple requests onto certain vm /// resource request in short period. From 19e333b5cc4ce36a1bae2c4d4ed2b97a837afcab Mon Sep 17 00:00:00 2001 From: Tim Hockin Date: Thu, 7 Mar 2019 17:14:18 -0800 Subject: [PATCH 04/21] Fix small race in e2e Occasionally we get spurious errors about "no route to host" when we race with kube-proxy. This should reduce that. It's mostly just log noise. --- test/e2e/framework/service_util.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/e2e/framework/service_util.go b/test/e2e/framework/service_util.go index 7ef9d711cd..a0d65f00d2 100644 --- a/test/e2e/framework/service_util.go +++ b/test/e2e/framework/service_util.go @@ -623,6 +623,7 @@ func (j *ServiceTestJig) waitForConditionOrFail(namespace, name string, timeout // name as the jig and runs the "netexec" container. func (j *ServiceTestJig) newRCTemplate(namespace string) *v1.ReplicationController { var replicas int32 = 1 + var grace int64 = 3 // so we don't race with kube-proxy when scaling up/down rc := &v1.ReplicationController{ ObjectMeta: metav1.ObjectMeta{ @@ -654,7 +655,7 @@ func (j *ServiceTestJig) newRCTemplate(namespace string) *v1.ReplicationControll }, }, }, - TerminationGracePeriodSeconds: new(int64), + TerminationGracePeriodSeconds: &grace, }, }, }, From cc6163225514afa81d0ad11fcc6b364994c89712 Mon Sep 17 00:00:00 2001 From: Hemant Kumar Date: Mon, 11 Mar 2019 17:11:56 -0400 Subject: [PATCH 05/21] Add pod information when max volume condition is missing --- test/e2e/storage/csi_mock_volume.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/storage/csi_mock_volume.go b/test/e2e/storage/csi_mock_volume.go index 7a41fa3020..9bc15116ca 100644 --- a/test/e2e/storage/csi_mock_volume.go +++ b/test/e2e/storage/csi_mock_volume.go @@ -379,7 +379,7 @@ var _ = utils.SIGDescribe("CSI mock volume", func() { _, _, pod3 := createPod() Expect(pod3).NotTo(BeNil(), "while creating third pod") err = waitForMaxVolumeCondition(pod3, m.cs) - Expect(err).NotTo(HaveOccurred(), "while waiting for max volume condition") + Expect(err).NotTo(HaveOccurred(), "while waiting for max volume condition on pod : %+v", pod3) }) }) From 3718f4e0ed89e95e1d6419ab7a7e157e7edc6c48 Mon Sep 17 00:00:00 2001 From: Jing Xu Date: Fri, 8 Mar 2019 15:28:16 -0800 Subject: [PATCH 06/21] Add bracket to the sig-window tag Need to add bracket in the tag for sig-windows. Also fix an issue: for current testing structure, it first init driver and then set up the framework. So when initialize the driver, it does not know what OS is and we can not set up the capabilities correctly. Instead we have to add all the capabilities and supported fs types including both linux and windows. Later in the code, we will check the Node OS and decide how to run the test. --- test/e2e/storage/drivers/csi.go | 4 +-- test/e2e/storage/drivers/in_tree.go | 31 +++++++++----------- test/e2e/storage/testpatterns/testpattern.go | 6 ++-- test/e2e/storage/testsuites/volume_io.go | 2 +- test/e2e/storage/testsuites/volumes.go | 9 ++++-- 5 files changed, 27 insertions(+), 25 deletions(-) diff --git a/test/e2e/storage/drivers/csi.go b/test/e2e/storage/drivers/csi.go index d802bfe45a..6c96924d14 100644 --- a/test/e2e/storage/drivers/csi.go +++ b/test/e2e/storage/drivers/csi.go @@ -347,7 +347,7 @@ func (g *gcePDCSIDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) { if pattern.FsType == "xfs" { framework.SkipUnlessNodeOSDistroIs("ubuntu", "custom") } - if pattern.FeatureTag == "sig-windows" { + if pattern.FeatureTag == "[sig-windows]" { framework.Skipf("Skipping tests for windows since CSI does not support it yet") } } @@ -464,7 +464,7 @@ func (g *gcePDExternalCSIDriver) SkipUnsupportedTest(pattern testpatterns.TestPa if pattern.FsType == "xfs" { framework.SkipUnlessNodeOSDistroIs("ubuntu", "custom") } - if pattern.FeatureTag == "sig-windows" { + if pattern.FeatureTag == "[sig-windows]" { framework.Skipf("Skipping tests for windows since CSI does not support it yet") } } diff --git a/test/e2e/storage/drivers/in_tree.go b/test/e2e/storage/drivers/in_tree.go index 886a00a470..76465ea1fb 100644 --- a/test/e2e/storage/drivers/in_tree.go +++ b/test/e2e/storage/drivers/in_tree.go @@ -1106,21 +1106,18 @@ var _ testsuites.DynamicPVTestDriver = &gcePdDriver{} // InitGcePdDriver returns gcePdDriver that implements TestDriver interface func InitGcePdDriver() testsuites.TestDriver { - var supportedTypes sets.String - var capFsGroup bool - if framework.NodeOSDistroIs("windows") { - supportedTypes = sets.NewString("ntfs") - capFsGroup = false - } else { - supportedTypes = sets.NewString( - "", // Default fsType - "ext2", - "ext3", - "ext4", - "xfs", - ) - capFsGroup = true - } + // In current test structure, it first initialize the driver and then set up + // the new framework, so we cannot get the correct OS here. So here set to + // support all fs types including both linux and windows. We have code to check Node OS later + // during test. + supportedTypes := sets.NewString( + "", // Default fsType + "ext2", + "ext3", + "ext4", + "xfs", + "ntfs", + ) return &gcePdDriver{ driverInfo: testsuites.DriverInfo{ Name: "gcepd", @@ -1129,7 +1126,7 @@ func InitGcePdDriver() testsuites.TestDriver { SupportedMountOption: sets.NewString("debug", "nouid32"), Capabilities: map[testsuites.Capability]bool{ testsuites.CapPersistence: true, - testsuites.CapFsGroup: capFsGroup, + testsuites.CapFsGroup: true, testsuites.CapBlock: true, testsuites.CapExec: true, }, @@ -1143,7 +1140,7 @@ func (g *gcePdDriver) GetDriverInfo() *testsuites.DriverInfo { func (g *gcePdDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) { framework.SkipUnlessProviderIs("gce", "gke") - if pattern.FeatureTag == "sig-windows" { + if pattern.FeatureTag == "[sig-windows]" { framework.SkipUnlessNodeOSDistroIs("windows") } } diff --git a/test/e2e/storage/testpatterns/testpattern.go b/test/e2e/storage/testpatterns/testpattern.go index 410ab85f4f..415ce45d56 100644 --- a/test/e2e/storage/testpatterns/testpattern.go +++ b/test/e2e/storage/testpatterns/testpattern.go @@ -152,21 +152,21 @@ var ( Name: "Inline-volume (ntfs)", VolType: InlineVolume, FsType: "ntfs", - FeatureTag: "sig-windows", + FeatureTag: "[sig-windows]", } // NtfsPreprovisionedPV is TestPattern for "Pre-provisioned PV (ntfs)" NtfsPreprovisionedPV = TestPattern{ Name: "Pre-provisioned PV (ntfs)", VolType: PreprovisionedPV, FsType: "ntfs", - FeatureTag: "sig-windows", + FeatureTag: "[sig-windows]", } // NtfsDynamicPV is TestPattern for "Dynamic PV (xfs)" NtfsDynamicPV = TestPattern{ Name: "Dynamic PV (ntfs)", VolType: DynamicPV, FsType: "ntfs", - FeatureTag: "sig-windows", + FeatureTag: "[sig-windows]", } // Definitions for Filesystem volume mode diff --git a/test/e2e/storage/testsuites/volume_io.go b/test/e2e/storage/testsuites/volume_io.go index 1ec29ba5a0..7f6227a258 100644 --- a/test/e2e/storage/testsuites/volume_io.go +++ b/test/e2e/storage/testsuites/volume_io.go @@ -125,7 +125,7 @@ func (t *volumeIOTestSuite) defineTests(driver TestDriver, pattern testpatterns. fileSizes := createFileSizes(dInfo.MaxFileSize) testFile := fmt.Sprintf("%s_io_test_%s", dInfo.Name, f.Namespace.Name) var fsGroup *int64 - if dInfo.Capabilities[CapFsGroup] { + if !framework.NodeOSDistroIs("windows") && dInfo.Capabilities[CapFsGroup] { fsGroupVal := int64(1234) fsGroup = &fsGroupVal } diff --git a/test/e2e/storage/testsuites/volumes.go b/test/e2e/storage/testsuites/volumes.go index 957b4e5cd1..5ffadae835 100644 --- a/test/e2e/storage/testsuites/volumes.go +++ b/test/e2e/storage/testsuites/volumes.go @@ -152,7 +152,7 @@ func (t *volumesTestSuite) defineTests(driver TestDriver, pattern testpatterns.T } config := convertTestConfig(l.config) var fsGroup *int64 - if dInfo.Capabilities[CapFsGroup] { + if framework.NodeOSDistroIs("windows") && dInfo.Capabilities[CapFsGroup] { fsGroupVal := int64(1234) fsGroup = &fsGroupVal } @@ -185,7 +185,12 @@ func testScriptInPod( ) suffix := generateSuffixForPodName(volumeType) fileName := fmt.Sprintf("test-%s", suffix) - content := fmt.Sprintf("ls %s", volPath) + var content string + if framework.NodeOSDistroIs("windows") { + content = fmt.Sprintf("ls -n %s", volPath) + } else { + content = fmt.Sprintf("ls %s", volPath) + } command := framework.GenerateWriteandExecuteScriptFileCmd(content, fileName, volPath) pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ From 382f5c83c0d02bdb981d0be0597ae4ea6de64f46 Mon Sep 17 00:00:00 2001 From: Tim Hockin Date: Thu, 7 Mar 2019 17:08:44 -0800 Subject: [PATCH 07/21] Retool HTTP and UDP e2e utils This is a prefactoring for followup changes that need to use very similar but subtly different test. Now it is more generic, though it pushes a little logic up the stack. That makes sense to me. --- test/e2e/framework/networking_utils.go | 382 +++++++++++++++---------- test/e2e/framework/service_util.go | 68 +++-- test/e2e/network/firewall.go | 8 +- test/e2e/network/service.go | 20 +- 4 files changed, 290 insertions(+), 188 deletions(-) diff --git a/test/e2e/framework/networking_utils.go b/test/e2e/framework/networking_utils.go index 7d43570aa9..acaf74553f 100644 --- a/test/e2e/framework/networking_utils.go +++ b/test/e2e/framework/networking_utils.go @@ -17,7 +17,6 @@ limitations under the License. package framework import ( - "bytes" "encoding/json" "fmt" "io/ioutil" @@ -708,13 +707,131 @@ func CheckReachabilityFromPod(expectToBeReachable bool, timeout time.Duration, n ExpectNoError(err) } -// Does an HTTP GET, but does not reuse TCP connections -// This masks problems where the iptables rule has changed, but we don't see it -// This is intended for relatively quick requests (status checks), so we set a short (5 seconds) timeout -func httpGetNoConnectionPool(url string) (*http.Response, error) { - return httpGetNoConnectionPoolTimeout(url, 5*time.Second) +type HTTPPokeParams struct { + Timeout time.Duration + ExpectCode int // default = 200 + BodyContains string + RetriableCodes []int } +type HTTPPokeResult struct { + Status HTTPPokeStatus + Code int // HTTP code: 0 if the connection was not made + Error error // if there was any error + Body []byte // if code != 0 +} + +type HTTPPokeStatus string + +const ( + HTTPSuccess HTTPPokeStatus = "Success" + HTTPError HTTPPokeStatus = "UnknownError" + // Any time we add new errors, we should audit all callers of this. + HTTPTimeout HTTPPokeStatus = "TimedOut" + HTTPRefused HTTPPokeStatus = "ConnectionRefused" + HTTPRetryCode HTTPPokeStatus = "RetryCode" + HTTPWrongCode HTTPPokeStatus = "WrongCode" + HTTPBadResponse HTTPPokeStatus = "BadResponse" +) + +// PokeHTTP tries to connect to a host on a port for a given URL path. Callers +// can specify additional success parameters, if desired. +// +// The result status will be characterized as precisely as possible, given the +// known users of this. +// +// The result code will be zero in case of any failure to connect, or non-zero +// if the HTTP transaction completed (even if the other test params make this a +// failure). +// +// The result error will be populated for any status other than Success. +// +// The result body will be populated if the HTTP transaction was completed, even +// if the other test params make this a failure). +func PokeHTTP(host string, port int, path string, params *HTTPPokeParams) HTTPPokeResult { + hostPort := net.JoinHostPort(host, strconv.Itoa(port)) + url := fmt.Sprintf("http://%s%s", hostPort, path) + + ret := HTTPPokeResult{} + + // Sanity check inputs, because it has happened. These are the only things + // that should hard fail the test - they are basically ASSERT()s. + if host == "" { + Failf("Got empty host for HTTP poke (%s)", url) + return ret + } + if port == 0 { + Failf("Got port==0 for HTTP poke (%s)", url) + return ret + } + + // Set default params. + if params == nil { + params = &HTTPPokeParams{} + } + if params.ExpectCode == 0 { + params.ExpectCode = http.StatusOK + } + + Logf("Poking %q", url) + + resp, err := httpGetNoConnectionPoolTimeout(url, params.Timeout) + if err != nil { + ret.Error = err + neterr, ok := err.(net.Error) + if ok && neterr.Timeout() { + ret.Status = HTTPTimeout + } else if strings.Contains(err.Error(), "connection refused") { + ret.Status = HTTPRefused + } else { + ret.Status = HTTPError + } + Logf("Poke(%q): %v", url, err) + return ret + } + + ret.Code = resp.StatusCode + + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + ret.Status = HTTPError + ret.Error = fmt.Errorf("error reading HTTP body: %v", err) + Logf("Poke(%q): %v", url, ret.Error) + return ret + } + ret.Body = make([]byte, len(body)) + copy(ret.Body, body) + + if resp.StatusCode != params.ExpectCode { + for _, code := range params.RetriableCodes { + if resp.StatusCode == code { + ret.Error = fmt.Errorf("retriable status code: %d", resp.StatusCode) + ret.Status = HTTPRetryCode + Logf("Poke(%q): %v", url, ret.Error) + return ret + } + } + ret.Status = HTTPWrongCode + ret.Error = fmt.Errorf("bad status code: %d", resp.StatusCode) + Logf("Poke(%q): %v", url, ret.Error) + return ret + } + + if params.BodyContains != "" && !strings.Contains(string(body), params.BodyContains) { + ret.Status = HTTPBadResponse + ret.Error = fmt.Errorf("response does not contain expected substring: %q", string(body)) + Logf("Poke(%q): %v", url, ret.Error) + return ret + } + + ret.Status = HTTPSuccess + Logf("Poke(%q): success", url) + return ret +} + +// Does an HTTP GET, but does not reuse TCP connections +// This masks problems where the iptables rule has changed, but we don't see it func httpGetNoConnectionPoolTimeout(url string, timeout time.Duration) (*http.Response, error) { tr := utilnet.SetTransportDefaults(&http.Transport{ DisableKeepAlives: true, @@ -727,178 +844,126 @@ func httpGetNoConnectionPoolTimeout(url string, timeout time.Duration) (*http.Re return client.Get(url) } -func TestReachableHTTP(ip string, port int, request string, expect string) (bool, error) { - return TestReachableHTTPWithContent(ip, port, request, expect, nil) +type UDPPokeParams struct { + Timeout time.Duration + Response string } -func TestReachableHTTPWithRetriableErrorCodes(ip string, port int, request string, expect string, retriableErrCodes []int) (bool, error) { - return TestReachableHTTPWithContentTimeoutWithRetriableErrorCodes(ip, port, request, expect, nil, retriableErrCodes, time.Second*5) +type UDPPokeResult struct { + Status UDPPokeStatus + Error error // if there was any error + Response []byte // if code != 0 } -func TestReachableHTTPWithContent(ip string, port int, request string, expect string, content *bytes.Buffer) (bool, error) { - return TestReachableHTTPWithContentTimeout(ip, port, request, expect, content, 5*time.Second) -} +type UDPPokeStatus string -func TestReachableHTTPWithContentTimeout(ip string, port int, request string, expect string, content *bytes.Buffer, timeout time.Duration) (bool, error) { - return TestReachableHTTPWithContentTimeoutWithRetriableErrorCodes(ip, port, request, expect, content, []int{}, timeout) -} +const ( + UDPSuccess UDPPokeStatus = "Success" + UDPError UDPPokeStatus = "UnknownError" + // Any time we add new errors, we should audit all callers of this. + UDPTimeout UDPPokeStatus = "TimedOut" + UDPRefused UDPPokeStatus = "ConnectionRefused" + UDPBadResponse UDPPokeStatus = "BadResponse" +) -func TestReachableHTTPWithContentTimeoutWithRetriableErrorCodes(ip string, port int, request string, expect string, content *bytes.Buffer, retriableErrCodes []int, timeout time.Duration) (bool, error) { +// PokeUDP tries to connect to a host on a port and send the given request. Callers +// can specify additional success parameters, if desired. +// +// The result status will be characterized as precisely as possible, given the +// known users of this. +// +// The result error will be populated for any status other than Success. +// +// The result response will be populated if the UDP transaction was completed, even +// if the other test params make this a failure). +func PokeUDP(host string, port int, request string, params *UDPPokeParams) UDPPokeResult { + hostPort := net.JoinHostPort(host, strconv.Itoa(port)) + url := fmt.Sprintf("udp://%s", hostPort) - ipPort := net.JoinHostPort(ip, strconv.Itoa(port)) - url := fmt.Sprintf("http://%s%s", ipPort, request) - if ip == "" { - Failf("Got empty IP for reachability check (%s)", url) - return false, nil + ret := UDPPokeResult{} + + // Sanity check inputs, because it has happened. These are the only things + // that should hard fail the test - they are basically ASSERT()s. + if host == "" { + Failf("Got empty host for UDP poke (%s)", url) + return ret } if port == 0 { - Failf("Got port==0 for reachability check (%s)", url) - return false, nil + Failf("Got port==0 for UDP poke (%s)", url) + return ret } - Logf("Testing HTTP reachability of %v", url) + // Set default params. + if params == nil { + params = &UDPPokeParams{} + } - resp, err := httpGetNoConnectionPoolTimeout(url, timeout) + Logf("Poking %v", url) + + con, err := net.Dial("udp", hostPort) if err != nil { - Logf("Got error testing for reachability of %s: %v", url, err) - return false, nil + ret.Status = UDPError + ret.Error = err + Logf("Poke(%q): %v", url, err) + return ret } - defer resp.Body.Close() - body, err := ioutil.ReadAll(resp.Body) + + _, err = con.Write([]byte(fmt.Sprintf("%s\n", request))) if err != nil { - Logf("Got error reading response from %s: %v", url, err) - return false, nil - } - if resp.StatusCode != 200 { - for _, code := range retriableErrCodes { - if resp.StatusCode == code { - Logf("Got non-success status %q when trying to access %s, but the error code is retriable", resp.Status, url) - return false, nil - } + ret.Error = err + neterr, ok := err.(net.Error) + if ok && neterr.Timeout() { + ret.Status = UDPTimeout + } else if strings.Contains(err.Error(), "connection refused") { + ret.Status = UDPRefused + } else { + ret.Status = UDPError } - return false, fmt.Errorf("received non-success return status %q trying to access %s; got body: %s", - resp.Status, url, string(body)) - } - if !strings.Contains(string(body), expect) { - return false, fmt.Errorf("received response body without expected substring %q: %s", expect, string(body)) - } - if content != nil { - content.Write(body) - } - return true, nil -} - -func TestNotReachableHTTP(ip string, port int) (bool, error) { - return TestNotReachableHTTPTimeout(ip, port, 5*time.Second) -} - -func TestNotReachableHTTPTimeout(ip string, port int, timeout time.Duration) (bool, error) { - ipPort := net.JoinHostPort(ip, strconv.Itoa(port)) - url := fmt.Sprintf("http://%s", ipPort) - if ip == "" { - Failf("Got empty IP for non-reachability check (%s)", url) - return false, nil - } - if port == 0 { - Failf("Got port==0 for non-reachability check (%s)", url) - return false, nil + Logf("Poke(%q): %v", url, err) + return ret } - Logf("Testing HTTP non-reachability of %v", url) + if params.Timeout != 0 { + err = con.SetDeadline(time.Now().Add(params.Timeout)) + if err != nil { + ret.Status = UDPError + ret.Error = err + Logf("Poke(%q): %v", url, err) + return ret + } + } - resp, err := httpGetNoConnectionPoolTimeout(url, timeout) + bufsize := len(params.Response) + 1 + if bufsize == 0 { + bufsize = 4096 + } + var buf []byte = make([]byte, bufsize) + n, err := con.Read(buf) if err != nil { - Logf("Confirmed that %s is not reachable", url) - return true, nil + ret.Error = err + neterr, ok := err.(net.Error) + if ok && neterr.Timeout() { + ret.Status = UDPTimeout + } else if strings.Contains(err.Error(), "connection refused") { + ret.Status = UDPRefused + } else { + ret.Status = UDPError + } + Logf("Poke(%q): %v", url, err) + return ret } - resp.Body.Close() - return false, nil -} + ret.Response = buf[0:n] -func TestReachableUDP(ip string, port int, request string, expect string) (bool, error) { - ipPort := net.JoinHostPort(ip, strconv.Itoa(port)) - uri := fmt.Sprintf("udp://%s", ipPort) - if ip == "" { - Failf("Got empty IP for reachability check (%s)", uri) - return false, nil - } - if port == 0 { - Failf("Got port==0 for reachability check (%s)", uri) - return false, nil + if params.Response != "" && string(ret.Response) != params.Response { + ret.Status = UDPBadResponse + ret.Error = fmt.Errorf("response does not match expected string: %q", string(ret.Response)) + Logf("Poke(%q): %v", url, ret.Error) + return ret } - Logf("Testing UDP reachability of %v", uri) - - con, err := net.Dial("udp", ipPort) - if err != nil { - return false, fmt.Errorf("Failed to dial %s: %v", ipPort, err) - } - - _, err = con.Write([]byte(fmt.Sprintf("%s\n", request))) - if err != nil { - return false, fmt.Errorf("Failed to send request: %v", err) - } - - var buf []byte = make([]byte, len(expect)+1) - - err = con.SetDeadline(time.Now().Add(3 * time.Second)) - if err != nil { - return false, fmt.Errorf("Failed to set deadline: %v", err) - } - - _, err = con.Read(buf) - if err != nil { - return false, nil - } - - if !strings.Contains(string(buf), expect) { - return false, fmt.Errorf("Failed to retrieve %q, got %q", expect, string(buf)) - } - - Logf("Successfully reached %v", uri) - return true, nil -} - -func TestNotReachableUDP(ip string, port int, request string) (bool, error) { - ipPort := net.JoinHostPort(ip, strconv.Itoa(port)) - uri := fmt.Sprintf("udp://%s", ipPort) - if ip == "" { - Failf("Got empty IP for reachability check (%s)", uri) - return false, nil - } - if port == 0 { - Failf("Got port==0 for reachability check (%s)", uri) - return false, nil - } - - Logf("Testing UDP non-reachability of %v", uri) - - con, err := net.Dial("udp", ipPort) - if err != nil { - Logf("Confirmed that %s is not reachable", uri) - return true, nil - } - - _, err = con.Write([]byte(fmt.Sprintf("%s\n", request))) - if err != nil { - Logf("Confirmed that %s is not reachable", uri) - return true, nil - } - - var buf []byte = make([]byte, 1) - - err = con.SetDeadline(time.Now().Add(3 * time.Second)) - if err != nil { - return false, fmt.Errorf("Failed to set deadline: %v", err) - } - - _, err = con.Read(buf) - if err != nil { - Logf("Confirmed that %s is not reachable", uri) - return true, nil - } - - return false, nil + ret.Status = UDPSuccess + Logf("Poke(%q): success", url) + return ret } func TestHitNodesFromOutside(externalIP string, httpPort int32, timeout time.Duration, expectedHosts sets.String) error { @@ -911,13 +976,12 @@ func TestHitNodesFromOutsideWithCount(externalIP string, httpPort int32, timeout hittedHosts := sets.NewString() count := 0 condition := func() (bool, error) { - var respBody bytes.Buffer - reached, err := TestReachableHTTPWithContentTimeout(externalIP, int(httpPort), "/hostname", "", &respBody, - 1*time.Second) - if err != nil || !reached { + result := PokeHTTP(externalIP, int(httpPort), "/hostname", &HTTPPokeParams{Timeout: 1 * time.Second}) + if result.Status != HTTPSuccess { return false, nil } - hittedHost := strings.TrimSpace(respBody.String()) + + hittedHost := strings.TrimSpace(string(result.Body)) if !expectedHosts.Has(hittedHost) { Logf("Error hitting unexpected host: %v, reset counter: %v", hittedHost, count) count = 0 diff --git a/test/e2e/framework/service_util.go b/test/e2e/framework/service_util.go index a0d65f00d2..172f52bd89 100644 --- a/test/e2e/framework/service_util.go +++ b/test/e2e/framework/service_util.go @@ -876,9 +876,19 @@ func (j *ServiceTestJig) TestReachableHTTP(host string, port int, timeout time.D } func (j *ServiceTestJig) TestReachableHTTPWithRetriableErrorCodes(host string, port int, retriableErrCodes []int, timeout time.Duration) { - if err := wait.PollImmediate(Poll, timeout, func() (bool, error) { - return TestReachableHTTPWithRetriableErrorCodes(host, port, "/echo?msg=hello", "hello", retriableErrCodes) - }); err != nil { + pollfn := func() (bool, error) { + result := PokeHTTP(host, port, "/echo?msg=hello", + &HTTPPokeParams{ + BodyContains: "hello", + RetriableCodes: retriableErrCodes, + }) + if result.Status == HTTPSuccess { + return true, nil + } + return false, nil // caller can retry + } + + if err := wait.PollImmediate(Poll, timeout, pollfn); err != nil { if err == wait.ErrWaitTimeout { Failf("Could not reach HTTP service through %v:%v after %v", host, port, timeout) } else { @@ -888,36 +898,60 @@ func (j *ServiceTestJig) TestReachableHTTPWithRetriableErrorCodes(host string, p } func (j *ServiceTestJig) TestNotReachableHTTP(host string, port int, timeout time.Duration) { - if err := wait.PollImmediate(Poll, timeout, func() (bool, error) { return TestNotReachableHTTP(host, port) }); err != nil { - Failf("Could still reach HTTP service through %v:%v after %v: %v", host, port, timeout, err) + pollfn := func() (bool, error) { + result := PokeHTTP(host, port, "/", nil) + if result.Code == 0 { + return true, nil + } + return false, nil // caller can retry + } + + if err := wait.PollImmediate(Poll, timeout, pollfn); err != nil { + Failf("HTTP service %v:%v reachable after %v: %v", host, port, timeout, err) } } func (j *ServiceTestJig) TestReachableUDP(host string, port int, timeout time.Duration) { - if err := wait.PollImmediate(Poll, timeout, func() (bool, error) { return TestReachableUDP(host, port, "echo hello", "hello") }); err != nil { + pollfn := func() (bool, error) { + result := PokeUDP(host, port, "echo hello", &UDPPokeParams{ + Timeout: 3 * time.Second, + Response: "hello", + }) + if result.Status == UDPSuccess { + return true, nil + } + return false, nil // caller can retry + } + + if err := wait.PollImmediate(Poll, timeout, pollfn); err != nil { Failf("Could not reach UDP service through %v:%v after %v: %v", host, port, timeout, err) } } func (j *ServiceTestJig) TestNotReachableUDP(host string, port int, timeout time.Duration) { - if err := wait.PollImmediate(Poll, timeout, func() (bool, error) { return TestNotReachableUDP(host, port, "echo hello") }); err != nil { - Failf("Could still reach UDP service through %v:%v after %v: %v", host, port, timeout, err) + pollfn := func() (bool, error) { + result := PokeUDP(host, port, "echo hello", &UDPPokeParams{Timeout: 3 * time.Second}) + if result.Status != UDPSuccess && result.Status != UDPError { + return true, nil + } + return false, nil // caller can retry + } + if err := wait.PollImmediate(Poll, timeout, pollfn); err != nil { + Failf("UDP service %v:%v reachable after %v: %v", host, port, timeout, err) } } func (j *ServiceTestJig) GetHTTPContent(host string, port int, timeout time.Duration, url string) bytes.Buffer { var body bytes.Buffer - var err error if pollErr := wait.PollImmediate(Poll, timeout, func() (bool, error) { - var result bool - result, err = TestReachableHTTPWithContent(host, port, url, "", &body) - if err != nil { - Logf("Error hitting %v:%v%v, retrying: %v", host, port, url, err) - return false, nil + result := PokeHTTP(host, port, url, nil) + if result.Status == HTTPSuccess { + body.Write(result.Body) + return true, nil } - return result, nil + return false, nil }); pollErr != nil { - Failf("Could not reach HTTP service through %v:%v%v after %v: %v", host, port, url, timeout, err) + Failf("Could not reach HTTP service through %v:%v%v after %v: %v", host, port, url, timeout, pollErr) } return body } @@ -930,7 +964,7 @@ func testHTTPHealthCheckNodePort(ip string, port int, request string) (bool, err return false, fmt.Errorf("Invalid input ip or port") } Logf("Testing HTTP health check on %v", url) - resp, err := httpGetNoConnectionPool(url) + resp, err := httpGetNoConnectionPoolTimeout(url, 5*time.Second) if err != nil { Logf("Got error testing for reachability of %s: %v", url, err) return false, err diff --git a/test/e2e/network/firewall.go b/test/e2e/network/firewall.go index db43181ab7..cfa3ffcaef 100644 --- a/test/e2e/network/firewall.go +++ b/test/e2e/network/firewall.go @@ -188,11 +188,11 @@ var _ = SIGDescribe("Firewall rule", func() { }) func assertNotReachableHTTPTimeout(ip string, port int, timeout time.Duration) { - unreachable, err := framework.TestNotReachableHTTPTimeout(ip, port, timeout) - if err != nil { - framework.Failf("Unexpected error checking for reachability of %s:%d: %v", ip, port, err) + result := framework.PokeHTTP(ip, port, "/", &framework.HTTPPokeParams{Timeout: timeout}) + if result.Status == framework.HTTPError { + framework.Failf("Unexpected error checking for reachability of %s:%d: %v", ip, port, result.Error) } - if !unreachable { + if result.Code != 0 { framework.Failf("Was unexpectedly able to reach %s:%d", ip, port) } } diff --git a/test/e2e/network/service.go b/test/e2e/network/service.go index 3df7628386..a60fdd4138 100644 --- a/test/e2e/network/service.go +++ b/test/e2e/network/service.go @@ -2063,14 +2063,18 @@ var _ = SIGDescribe("ESIPP [Slow] [DisabledForLargeClusters]", func() { for nodeName, nodeIPs := range endpointNodeMap { By(fmt.Sprintf("checking kube-proxy health check fails on node with endpoint (%s), public IP %s", nodeName, nodeIPs[0])) var body bytes.Buffer - var result bool - var err error - if pollErr := wait.PollImmediate(framework.Poll, framework.ServiceTestTimeout, func() (bool, error) { - result, err = framework.TestReachableHTTPWithContent(nodeIPs[0], healthCheckNodePort, "/healthz", "", &body) - return !result, nil - }); pollErr != nil { - framework.Failf("Kube-proxy still exposing health check on node %v:%v, after ESIPP was turned off. Last err %v, last body %v", - nodeName, healthCheckNodePort, err, body.String()) + pollfn := func() (bool, error) { + result := framework.PokeHTTP(nodeIPs[0], healthCheckNodePort, "/healthz", nil) + if result.Code == 0 { + return true, nil + } + body.Reset() + body.Write(result.Body) + return false, nil + } + if pollErr := wait.PollImmediate(framework.Poll, framework.ServiceTestTimeout, pollfn); pollErr != nil { + framework.Failf("Kube-proxy still exposing health check on node %v:%v, after ESIPP was turned off. body %s", + nodeName, healthCheckNodePort, body.String()) } } From de25d6cb9577b00b24ca2baf0187b34f483a0d6c Mon Sep 17 00:00:00 2001 From: Tim Hockin Date: Mon, 18 Feb 2019 23:52:24 -0800 Subject: [PATCH 08/21] Kube-proxy: REJECT LB IPs with no endpoints We REJECT every other case. Close this FIXME. To get this to work in all cases, we have to process service in filter.INPUT, since LB IPS might be manged as local addresses. --- pkg/proxy/iptables/proxier.go | 23 +++++++++++--- test/e2e/framework/service_util.go | 49 ++++++++++++++++++++++++++++++ test/e2e/network/service.go | 40 ++++++++++++++++++++++-- 3 files changed, 105 insertions(+), 7 deletions(-) diff --git a/pkg/proxy/iptables/proxier.go b/pkg/proxy/iptables/proxier.go index a4f8dc8ef7..7633c6c9dc 100644 --- a/pkg/proxy/iptables/proxier.go +++ b/pkg/proxy/iptables/proxier.go @@ -369,6 +369,7 @@ var iptablesJumpChains = []iptablesJumpChain{ {utiliptables.TableFilter, kubeExternalServicesChain, utiliptables.ChainInput, "kubernetes externally-visible service portals", []string{"-m", "conntrack", "--ctstate", "NEW"}}, {utiliptables.TableFilter, kubeServicesChain, utiliptables.ChainForward, "kubernetes service portals", []string{"-m", "conntrack", "--ctstate", "NEW"}}, {utiliptables.TableFilter, kubeServicesChain, utiliptables.ChainOutput, "kubernetes service portals", []string{"-m", "conntrack", "--ctstate", "NEW"}}, + {utiliptables.TableFilter, kubeServicesChain, utiliptables.ChainInput, "kubernetes service portals", []string{"-m", "conntrack", "--ctstate", "NEW"}}, {utiliptables.TableFilter, kubeForwardChain, utiliptables.ChainForward, "kubernetes forwarding rules", nil}, {utiliptables.TableNAT, kubeServicesChain, utiliptables.ChainOutput, "kubernetes service portals", nil}, {utiliptables.TableNAT, kubeServicesChain, utiliptables.ChainPrerouting, "kubernetes service portals", nil}, @@ -847,6 +848,7 @@ func (proxier *Proxier) syncProxyRules() { } writeLine(proxier.natRules, append(args, "-j", string(svcChain))...) } else { + // No endpoints. writeLine(proxier.filterRules, "-A", string(kubeServicesChain), "-m", "comment", "--comment", fmt.Sprintf(`"%s has no endpoints"`, svcNameString), @@ -917,6 +919,7 @@ func (proxier *Proxier) syncProxyRules() { // This covers cases like GCE load-balancers which get added to the local routing table. writeLine(proxier.natRules, append(dstLocalOnlyArgs, "-j", string(svcChain))...) } else { + // No endpoints. writeLine(proxier.filterRules, "-A", string(kubeExternalServicesChain), "-m", "comment", "--comment", fmt.Sprintf(`"%s has no endpoints"`, svcNameString), @@ -929,10 +932,10 @@ func (proxier *Proxier) syncProxyRules() { } // Capture load-balancer ingress. - if hasEndpoints { - fwChain := svcInfo.serviceFirewallChainName - for _, ingress := range svcInfo.LoadBalancerStatus.Ingress { - if ingress.IP != "" { + fwChain := svcInfo.serviceFirewallChainName + for _, ingress := range svcInfo.LoadBalancerStatus.Ingress { + if ingress.IP != "" { + if hasEndpoints { // create service firewall chain if chain, ok := existingNATChains[fwChain]; ok { writeBytesLine(proxier.natChains, chain) @@ -993,10 +996,19 @@ func (proxier *Proxier) syncProxyRules() { // If the packet was able to reach the end of firewall chain, then it did not get DNATed. // It means the packet cannot go thru the firewall, then mark it for DROP writeLine(proxier.natRules, append(args, "-j", string(KubeMarkDropChain))...) + } else { + // No endpoints. + writeLine(proxier.filterRules, + "-A", string(kubeServicesChain), + "-m", "comment", "--comment", fmt.Sprintf(`"%s has no endpoints"`, svcNameString), + "-m", protocol, "-p", protocol, + "-d", utilproxy.ToCIDR(net.ParseIP(ingress.IP)), + "--dport", strconv.Itoa(svcInfo.Port), + "-j", "REJECT", + ) } } } - // FIXME: do we need REJECT rules for load-balancer ingress if !hasEndpoints? // Capture nodeports. If we had more than 2 rules it might be // worthwhile to make a new per-service chain for nodeport rules, but @@ -1078,6 +1090,7 @@ func (proxier *Proxier) syncProxyRules() { writeLine(proxier.natRules, append(args, "-j", string(svcXlbChain))...) } } else { + // No endpoints. writeLine(proxier.filterRules, "-A", string(kubeExternalServicesChain), "-m", "comment", "--comment", fmt.Sprintf(`"%s has no endpoints"`, svcNameString), diff --git a/test/e2e/framework/service_util.go b/test/e2e/framework/service_util.go index 172f52bd89..b98ddba72e 100644 --- a/test/e2e/framework/service_util.go +++ b/test/e2e/framework/service_util.go @@ -738,6 +738,28 @@ func (j *ServiceTestJig) RunOrFail(namespace string, tweak func(rc *v1.Replicati return result } +func (j *ServiceTestJig) Scale(namespace string, replicas int) { + rc := j.Name + scale, err := j.Client.CoreV1().ReplicationControllers(namespace).GetScale(rc, metav1.GetOptions{}) + if err != nil { + Failf("Failed to get scale for RC %q: %v", rc, err) + } + + scale.Spec.Replicas = int32(replicas) + _, err = j.Client.CoreV1().ReplicationControllers(namespace).UpdateScale(rc, scale) + if err != nil { + Failf("Failed to scale RC %q: %v", rc, err) + } + pods, err := j.waitForPodsCreated(namespace, replicas) + if err != nil { + Failf("Failed waiting for pods: %v", err) + } + if err := j.waitForPodsReady(namespace, pods); err != nil { + Failf("Failed waiting for pods to be running: %v", err) + } + return +} + func (j *ServiceTestJig) waitForPdbReady(namespace string) error { timeout := 2 * time.Minute for start := time.Now(); time.Since(start) < timeout; time.Sleep(2 * time.Second) { @@ -911,6 +933,20 @@ func (j *ServiceTestJig) TestNotReachableHTTP(host string, port int, timeout tim } } +func (j *ServiceTestJig) TestRejectedHTTP(host string, port int, timeout time.Duration) { + pollfn := func() (bool, error) { + result := PokeHTTP(host, port, "/", nil) + if result.Status == HTTPRefused { + return true, nil + } + return false, nil // caller can retry + } + + if err := wait.PollImmediate(Poll, timeout, pollfn); err != nil { + Failf("HTTP service %v:%v not rejected: %v", host, port, err) + } +} + func (j *ServiceTestJig) TestReachableUDP(host string, port int, timeout time.Duration) { pollfn := func() (bool, error) { result := PokeUDP(host, port, "echo hello", &UDPPokeParams{ @@ -941,6 +977,19 @@ func (j *ServiceTestJig) TestNotReachableUDP(host string, port int, timeout time } } +func (j *ServiceTestJig) TestRejectedUDP(host string, port int, timeout time.Duration) { + pollfn := func() (bool, error) { + result := PokeUDP(host, port, "echo hello", &UDPPokeParams{Timeout: 3 * time.Second}) + if result.Status == UDPRefused { + return true, nil + } + return false, nil // caller can retry + } + if err := wait.PollImmediate(Poll, timeout, pollfn); err != nil { + Failf("UDP service %v:%v not rejected: %v", host, port, err) + } +} + func (j *ServiceTestJig) GetHTTPContent(host string, port int, timeout time.Duration, url string) bytes.Buffer { var body bytes.Buffer if pollErr := wait.PollImmediate(Poll, timeout, func() (bool, error) { diff --git a/test/e2e/network/service.go b/test/e2e/network/service.go index a60fdd4138..55f743992d 100644 --- a/test/e2e/network/service.go +++ b/test/e2e/network/service.go @@ -791,11 +791,47 @@ var _ = SIGDescribe("Services", func() { jig.TestReachableUDP(nodeIP, udpNodePort, framework.KubeProxyLagTimeout) By("hitting the TCP service's LoadBalancer") - jig.TestReachableHTTP(tcpIngressIP, svcPort, loadBalancerCreateTimeout) // this may actually recreate the LB + jig.TestReachableHTTP(tcpIngressIP, svcPort, loadBalancerCreateTimeout) if loadBalancerSupportsUDP { By("hitting the UDP service's LoadBalancer") - jig.TestReachableUDP(udpIngressIP, svcPort, loadBalancerCreateTimeout) // this may actually recreate the LB) + jig.TestReachableUDP(udpIngressIP, svcPort, loadBalancerCreateTimeout) + } + + By("Scaling the pods to 0") + jig.Scale(ns1, 0) + jig.Scale(ns2, 0) + + By("looking for ICMP REJECT on the TCP service's NodePort") + jig.TestRejectedHTTP(nodeIP, tcpNodePort, framework.KubeProxyLagTimeout) + + By("looking for ICMP REJECT on the UDP service's NodePort") + jig.TestRejectedUDP(nodeIP, udpNodePort, framework.KubeProxyLagTimeout) + + By("looking for ICMP REJECT on the TCP service's LoadBalancer") + jig.TestRejectedHTTP(tcpIngressIP, svcPort, loadBalancerCreateTimeout) + + if loadBalancerSupportsUDP { + By("looking for ICMP REJECT on the UDP service's LoadBalancer") + jig.TestRejectedUDP(udpIngressIP, svcPort, loadBalancerCreateTimeout) + } + + By("Scaling the pods to 1") + jig.Scale(ns1, 1) + jig.Scale(ns2, 1) + + By("hitting the TCP service's NodePort") + jig.TestReachableHTTP(nodeIP, tcpNodePort, framework.KubeProxyLagTimeout) + + By("hitting the UDP service's NodePort") + jig.TestReachableUDP(nodeIP, udpNodePort, framework.KubeProxyLagTimeout) + + By("hitting the TCP service's LoadBalancer") + jig.TestReachableHTTP(tcpIngressIP, svcPort, loadBalancerCreateTimeout) + + if loadBalancerSupportsUDP { + By("hitting the UDP service's LoadBalancer") + jig.TestReachableUDP(udpIngressIP, svcPort, loadBalancerCreateTimeout) } // Change the services back to ClusterIP. From d5bd17cda0c134e5ef5c03c3eac79a9ce4e18003 Mon Sep 17 00:00:00 2001 From: Aaron Prindle Date: Thu, 7 Mar 2019 16:52:26 -0800 Subject: [PATCH 09/21] Added version check between patch and live object in server side apply What is the problem being solved? https://github.com/kubernetes/kubernetes/pull/75135 Currently version compatibility is not being checked in server side apply between the patch object and the live object. This is causing a merge that will error to be run and the apiserver returns a 500 error. The request should fail if the apiVersion provided in the object is different from the apiVersion in the url, but it should fail before trying to merge, and be a 4xx error. Probably a bad request error. Why is this the best approach? The approach of serializing the patch byte array and then checking for version equality with the already serialized live object is the simplest and most straightforward solution. --- .../pkg/endpoints/handlers/fieldmanager/BUILD | 4 ++ .../handlers/fieldmanager/fieldmanager.go | 20 +++++++-- .../fieldmanager/fieldmanager_test.go | 41 +++++++++++++++++-- 3 files changed, 59 insertions(+), 6 deletions(-) diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/BUILD b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/BUILD index 1c2892a40e..10379edd55 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/BUILD @@ -7,14 +7,17 @@ go_library( importpath = "k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager", visibility = ["//visibility:public"], deps = [ + "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/meta:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal:go_default_library", "//vendor/k8s.io/kube-openapi/pkg/util/proto:go_default_library", "//vendor/sigs.k8s.io/structured-merge-diff/fieldpath:go_default_library", "//vendor/sigs.k8s.io/structured-merge-diff/merge:go_default_library", + "//vendor/sigs.k8s.io/yaml:go_default_library", ], ) @@ -41,6 +44,7 @@ go_test( embed = [":go_default_library"], deps = [ "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/meta:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/fieldmanager.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/fieldmanager.go index 7879dacb47..212bcadfe2 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/fieldmanager.go +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/fieldmanager.go @@ -20,14 +20,17 @@ import ( "fmt" "time" + "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal" openapiproto "k8s.io/kube-openapi/pkg/util/proto" "sigs.k8s.io/structured-merge-diff/fieldpath" "sigs.k8s.io/structured-merge-diff/merge" + "sigs.k8s.io/yaml" ) // FieldManager updates the managed fields and merge applied @@ -149,9 +152,20 @@ func (f *FieldManager) Apply(liveObj runtime.Object, patch []byte, fieldManager if err != nil { return nil, fmt.Errorf("failed to decode managed fields: %v", err) } - // We can assume that patchObj is already on the proper version: - // it shouldn't have to be converted so that it's not defaulted. - // TODO (jennybuckley): Explicitly checkt that patchObj is in the proper version. + // Check that the patch object has the same version as the live object + patchObj := &unstructured.Unstructured{Object: map[string]interface{}{}} + + if err := yaml.Unmarshal(patch, &patchObj.Object); err != nil { + return nil, fmt.Errorf("error decoding YAML: %v", err) + } + if patchObj.GetAPIVersion() != f.groupVersion.String() { + return nil, + errors.NewBadRequest( + fmt.Sprintf("Incorrect version specified in apply patch. "+ + "Specified patch version: %s, expected: %s", + patchObj.GetAPIVersion(), f.groupVersion.String())) + } + liveObjVersioned, err := f.toVersioned(liveObj) if err != nil { return nil, fmt.Errorf("failed to convert live object to proper version: %v", err) diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/fieldmanager_test.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/fieldmanager_test.go index f512c98b76..776dfd0a1e 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/fieldmanager_test.go +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/fieldmanager_test.go @@ -18,9 +18,11 @@ package fieldmanager_test import ( "errors" + "net/http" "testing" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" @@ -72,7 +74,7 @@ func TestApplyStripsFields(t *testing.T) { obj := &corev1.Pod{} newObj, err := f.Apply(obj, []byte(`{ - "apiVersion": "v1", + "apiVersion": "apps/v1", "kind": "Deployment", "metadata": { "name": "b", @@ -85,7 +87,7 @@ func TestApplyStripsFields(t *testing.T) { "managedFields": [{ "manager": "apply", "operation": "Apply", - "apiVersion": "v1", + "apiVersion": "apps/v1", "fields": { "f:metadata": { "f:labels": { @@ -111,13 +113,46 @@ func TestApplyStripsFields(t *testing.T) { } } +func TestVersionCheck(t *testing.T) { + f := NewTestFieldManager(t) + + obj := &corev1.Pod{} + + // patch has 'apiVersion: apps/v1' and live version is apps/v1 -> no errors + _, err := f.Apply(obj, []byte(`{ + "apiVersion": "apps/v1", + "kind": "Deployment", + }`), "fieldmanager_test", false) + if err != nil { + t.Fatalf("failed to apply object: %v", err) + } + + // patch has 'apiVersion: apps/v2' but live version is apps/v1 -> error + _, err = f.Apply(obj, []byte(`{ + "apiVersion": "apps/v2", + "kind": "Deployment", + }`), "fieldmanager_test", false) + if err == nil { + t.Fatalf("expected an error from mismatched patch and live versions") + } + switch typ := err.(type) { + default: + t.Fatalf("expected error to be of type %T was %T", apierrors.StatusError{}, typ) + case apierrors.APIStatus: + if typ.Status().Code != http.StatusBadRequest { + t.Fatalf("expected status code to be %d but was %d", + http.StatusBadRequest, typ.Status().Code) + } + } +} + func TestApplyDoesNotStripLabels(t *testing.T) { f := NewTestFieldManager(t) obj := &corev1.Pod{} newObj, err := f.Apply(obj, []byte(`{ - "apiVersion": "v1", + "apiVersion": "apps/v1", "kind": "Pod", "metadata": { "labels": { From 10259c959e9f20668659f7e20251a217b51c6355 Mon Sep 17 00:00:00 2001 From: jennybuckley Date: Mon, 11 Mar 2019 23:07:21 -0700 Subject: [PATCH 10/21] Fix int/float apply bug --- .../handlers/fieldmanager/internal/typeconverter.go | 7 ++++++- .../fieldmanager/internal/typeconverter_test.go | 2 -- test/integration/apiserver/apply/apply_test.go | 13 +++++++++++++ 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/typeconverter.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/typeconverter.go index 80ac84128f..e80e5634bc 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/typeconverter.go +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/typeconverter.go @@ -105,7 +105,12 @@ func (c *typeConverter) YAMLToTyped(from []byte) (typed.TypedValue, error) { return nil, fmt.Errorf("error decoding YAML: %v", err) } - return c.ObjectToTyped(unstructured) + gvk := unstructured.GetObjectKind().GroupVersionKind() + t := c.parser.Type(gvk) + if t == nil { + return nil, fmt.Errorf("no corresponding type for %v", gvk) + } + return t.FromYAML(typed.YAMLObject(string(from))) } func (c *typeConverter) TypedToObject(value typed.TypedValue) (runtime.Object, error) { diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/typeconverter_test.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/typeconverter_test.go index c36d686545..8e41f07960 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/typeconverter_test.go +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/typeconverter_test.go @@ -68,7 +68,6 @@ metadata: labels: app: nginx spec: - replicas: 3 selector: matchLabels: app: nginx @@ -91,7 +90,6 @@ metadata: labels: app: nginx spec: - replicas: 3 selector: matchLabels: app: nginx diff --git a/test/integration/apiserver/apply/apply_test.go b/test/integration/apiserver/apply/apply_test.go index d0787190fa..e0858940c8 100644 --- a/test/integration/apiserver/apply/apply_test.go +++ b/test/integration/apiserver/apply/apply_test.go @@ -119,6 +119,19 @@ func TestApplyAlsoCreates(t *testing.T) { if err != nil { t.Fatalf("Failed to retrieve object: %v", err) } + + // Test that we can re apply with a different field manager and don't get conflicts + _, err = client.CoreV1().RESTClient().Patch(types.ApplyPatchType). + Namespace("default"). + Resource(tc.resource). + Name(tc.name). + Param("fieldManager", "apply_test_2"). + Body([]byte(tc.body)). + Do(). + Get() + if err != nil { + t.Fatalf("Failed to re-apply object using Apply patch: %v", err) + } } } From 8f48d762717dfe1f479cfabaabc4fb9261a1bc8f Mon Sep 17 00:00:00 2001 From: ajatprabha Date: Thu, 14 Feb 2019 04:09:12 +0530 Subject: [PATCH 11/21] add ResourceVersion to DeleteOptions.Preconditions --- .../core/namespace/storage/storage.go | 10 ++++- .../registry/customresourcedefinition/etcd.go | 10 ++++- .../apimachinery/pkg/apis/meta/v1/helpers.go | 6 +++ .../apimachinery/pkg/apis/meta/v1/types.go | 3 ++ .../pkg/registry/generic/registry/store.go | 2 + .../apiserver/pkg/registry/rest/delete.go | 9 ++++- .../pkg/registry/rest/resttest/resttest.go | 37 +++++++++++++++++++ .../apiserver/pkg/storage/interfaces.go | 3 ++ 8 files changed, 76 insertions(+), 4 deletions(-) diff --git a/pkg/registry/core/namespace/storage/storage.go b/pkg/registry/core/namespace/storage/storage.go index 2fe25f1ee5..82f54b72a1 100644 --- a/pkg/registry/core/namespace/storage/storage.go +++ b/pkg/registry/core/namespace/storage/storage.go @@ -139,6 +139,7 @@ func (r *REST) Delete(ctx context.Context, name string, options *metav1.DeleteOp } if options.Preconditions.UID == nil { options.Preconditions.UID = &namespace.UID + options.Preconditions.ResourceVersion = &namespace.ResourceVersion } else if *options.Preconditions.UID != namespace.UID { err = apierrors.NewConflict( api.Resource("namespaces"), @@ -146,6 +147,13 @@ func (r *REST) Delete(ctx context.Context, name string, options *metav1.DeleteOp fmt.Errorf("Precondition failed: UID in precondition: %v, UID in object meta: %v", *options.Preconditions.UID, namespace.UID), ) return nil, false, err + } else if *options.Preconditions.ResourceVersion != namespace.ResourceVersion { + err = apierrors.NewConflict( + api.Resource("namespaces"), + name, + fmt.Errorf("Precondition failed: ResourceVersion in precondition: %v, ResourceVersion in object meta: %v", *options.Preconditions.ResourceVersion, namespace.ResourceVersion), + ) + return nil, false, err } // upon first request to delete, we switch the phase to start namespace termination @@ -156,7 +164,7 @@ func (r *REST) Delete(ctx context.Context, name string, options *metav1.DeleteOp return nil, false, err } - preconditions := storage.Preconditions{UID: options.Preconditions.UID} + preconditions := storage.Preconditions{UID: options.Preconditions.UID, ResourceVersion: options.Preconditions.ResourceVersion} out := r.store.NewFunc() err = r.store.Storage.GuaranteedUpdate( diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition/etcd.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition/etcd.go index c6b50915f1..fe7c3bafd6 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition/etcd.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition/etcd.go @@ -84,6 +84,7 @@ func (r *REST) Delete(ctx context.Context, name string, options *metav1.DeleteOp } if options.Preconditions.UID == nil { options.Preconditions.UID = &crd.UID + options.Preconditions.ResourceVersion = &crd.ResourceVersion } else if *options.Preconditions.UID != crd.UID { err = apierrors.NewConflict( apiextensions.Resource("customresourcedefinitions"), @@ -91,6 +92,13 @@ func (r *REST) Delete(ctx context.Context, name string, options *metav1.DeleteOp fmt.Errorf("Precondition failed: UID in precondition: %v, UID in object meta: %v", *options.Preconditions.UID, crd.UID), ) return nil, false, err + } else if *options.Preconditions.ResourceVersion != crd.ResourceVersion { + err = apierrors.NewConflict( + apiextensions.Resource("customresourcedefinitions"), + name, + fmt.Errorf("Precondition failed: ResourceVersion in precondition: %v, ResourceVersion in object meta: %v", *options.Preconditions.ResourceVersion, crd.ResourceVersion), + ) + return nil, false, err } // upon first request to delete, add our finalizer and then delegate @@ -100,7 +108,7 @@ func (r *REST) Delete(ctx context.Context, name string, options *metav1.DeleteOp return nil, false, err } - preconditions := storage.Preconditions{UID: options.Preconditions.UID} + preconditions := storage.Preconditions{UID: options.Preconditions.UID, ResourceVersion: options.Preconditions.ResourceVersion} out := r.Store.NewFunc() err = r.Store.Storage.GuaranteedUpdate( diff --git a/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/helpers.go b/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/helpers.go index b41d549a25..9da8008331 100644 --- a/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/helpers.go +++ b/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/helpers.go @@ -228,6 +228,12 @@ func NewUIDPreconditions(uid string) *Preconditions { return &Preconditions{UID: &u} } +// NewPreconditionDeleteOptionsWithResourceVersion returns a DeleteOptions with a UID precondition set. +func NewPreconditionDeleteOptionsWithResourceVersion(rv string) *DeleteOptions { + p := Preconditions{ResourceVersion: &rv} + return &DeleteOptions{Preconditions: &p} +} + // HasObjectMetaSystemFieldValues returns true if fields that are managed by the system on ObjectMeta have values. func HasObjectMetaSystemFieldValues(meta Object) bool { return !meta.GetCreationTimestamp().Time.IsZero() || diff --git a/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/types.go b/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/types.go index 23cbc2c6df..fd6395256a 100644 --- a/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/types.go +++ b/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/types.go @@ -575,6 +575,9 @@ type Preconditions struct { // Specifies the target UID. // +optional UID *types.UID `json:"uid,omitempty" protobuf:"bytes,1,opt,name=uid,casttype=k8s.io/apimachinery/pkg/types.UID"` + // Specifies the target ResourceVersion + // +optional + ResourceVersion *string `json:"resourceVersion,omitempty" protobuf:"bytes,2,opt,name=resourceVersion"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/staging/src/k8s.io/apiserver/pkg/registry/generic/registry/store.go b/staging/src/k8s.io/apiserver/pkg/registry/generic/registry/store.go index c84a3276c4..be15e2bb3b 100644 --- a/staging/src/k8s.io/apiserver/pkg/registry/generic/registry/store.go +++ b/staging/src/k8s.io/apiserver/pkg/registry/generic/registry/store.go @@ -448,6 +448,7 @@ func (e *Store) Update(ctx context.Context, name string, objInfo rest.UpdatedObj storagePreconditions := &storage.Preconditions{} if preconditions := objInfo.Preconditions(); preconditions != nil { storagePreconditions.UID = preconditions.UID + storagePreconditions.ResourceVersion = preconditions.ResourceVersion } out := e.NewFunc() @@ -879,6 +880,7 @@ func (e *Store) Delete(ctx context.Context, name string, options *metav1.DeleteO var preconditions storage.Preconditions if options.Preconditions != nil { preconditions.UID = options.Preconditions.UID + preconditions.ResourceVersion = options.Preconditions.ResourceVersion } graceful, pendingGraceful, err := rest.BeforeDelete(e.DeleteStrategy, ctx, obj, options) if err != nil { diff --git a/staging/src/k8s.io/apiserver/pkg/registry/rest/delete.go b/staging/src/k8s.io/apiserver/pkg/registry/rest/delete.go index 7c39f6be10..0713464e00 100644 --- a/staging/src/k8s.io/apiserver/pkg/registry/rest/delete.go +++ b/staging/src/k8s.io/apiserver/pkg/registry/rest/delete.go @@ -77,8 +77,13 @@ func BeforeDelete(strategy RESTDeleteStrategy, ctx context.Context, obj runtime. return false, false, errors.NewInvalid(schema.GroupKind{Group: metav1.GroupName, Kind: "DeleteOptions"}, "", errs) } // Checking the Preconditions here to fail early. They'll be enforced later on when we actually do the deletion, too. - if options.Preconditions != nil && options.Preconditions.UID != nil && *options.Preconditions.UID != objectMeta.GetUID() { - return false, false, errors.NewConflict(schema.GroupResource{Group: gvk.Group, Resource: gvk.Kind}, objectMeta.GetName(), fmt.Errorf("the UID in the precondition (%s) does not match the UID in record (%s). The object might have been deleted and then recreated", *options.Preconditions.UID, objectMeta.GetUID())) + if options.Preconditions != nil { + if options.Preconditions.UID != nil && *options.Preconditions.UID != objectMeta.GetUID() { + return false, false, errors.NewConflict(schema.GroupResource{Group: gvk.Group, Resource: gvk.Kind}, objectMeta.GetName(), fmt.Errorf("the UID in the precondition (%s) does not match the UID in record (%s). The object might have been deleted and then recreated", *options.Preconditions.UID, objectMeta.GetUID())) + } + if options.Preconditions.ResourceVersion != nil && *options.Preconditions.ResourceVersion != objectMeta.GetResourceVersion() { + return false, false, errors.NewConflict(schema.GroupResource{Group: gvk.Group, Resource: gvk.Kind}, objectMeta.GetName(), fmt.Errorf("the ResourceVersion in the precondition (%s) does not match the ResourceVersion in record (%s). The object might have been deleted and then recreated", *options.Preconditions.ResourceVersion, objectMeta.GetResourceVersion())) + } } gracefulStrategy, ok := strategy.(RESTGracefulDeleteStrategy) if !ok { diff --git a/staging/src/k8s.io/apiserver/pkg/registry/rest/resttest/resttest.go b/staging/src/k8s.io/apiserver/pkg/registry/rest/resttest/resttest.go index 8e6b555426..4316592f46 100644 --- a/staging/src/k8s.io/apiserver/pkg/registry/rest/resttest/resttest.go +++ b/staging/src/k8s.io/apiserver/pkg/registry/rest/resttest/resttest.go @@ -909,6 +909,43 @@ func (t *Tester) testDeleteWithUID(obj runtime.Object, createFn CreateFunc, getF } } +// This test the fast-fail path. We test that the precondition gets verified +// again before deleting the object in tests of pkg/storage/etcd. +func (t *Tester) testDeleteWithResourceVersion(obj runtime.Object, createFn CreateFunc, getFn GetFunc, isNotFoundFn IsErrorFunc, opts metav1.DeleteOptions) { + ctx := t.TestContext() + + foo := obj.DeepCopyObject() + t.setObjectMeta(foo, t.namer(1)) + objectMeta := t.getObjectMetaOrFail(foo) + objectMeta.SetResourceVersion("RV0000") + if err := createFn(ctx, foo); err != nil { + t.Errorf("unexpected error: %v", err) + } + opts.Preconditions = metav1.NewPreconditionDeleteOptionsWithResourceVersion("RV1111").Preconditions + obj, _, err := t.storage.(rest.GracefulDeleter).Delete(ctx, objectMeta.GetName(), &opts) + if err == nil || !errors.IsConflict(err) { + t.Errorf("unexpected error: %v", err) + } + + obj, _, err = t.storage.(rest.GracefulDeleter).Delete(ctx, objectMeta.GetName(), metav1.NewPreconditionDeleteOptionsWithResourceVersion("RV0000")) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + + if !t.returnDeletedObject { + if status, ok := obj.(*metav1.Status); !ok { + t.Errorf("expected status of delete, got %v", status) + } else if status.Status != metav1.StatusSuccess { + t.Errorf("expected success, got: %v", status.Status) + } + } + + _, err = getFn(ctx, foo) + if err == nil || !isNotFoundFn(err) { + t.Errorf("unexpected error: %v", err) + } +} + // ============================================================================= // Graceful Deletion tests. diff --git a/staging/src/k8s.io/apiserver/pkg/storage/interfaces.go b/staging/src/k8s.io/apiserver/pkg/storage/interfaces.go index 56dd6dbdc9..93b3f5deb8 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/interfaces.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/interfaces.go @@ -98,6 +98,9 @@ type Preconditions struct { // Specifies the target UID. // +optional UID *types.UID `json:"uid,omitempty"` + // Specifies the target ResourceVersion + // +optional + ResourceVersion *string `json:"resourceVersion,omitempty"` } // NewUIDPreconditions returns a Preconditions with UID set. From 42f0a36f44bac33f4230fdea9f3dcc4bfb645a0a Mon Sep 17 00:00:00 2001 From: ajatprabha Date: Fri, 15 Feb 2019 04:37:18 +0530 Subject: [PATCH 12/21] check for ResourceVersion conflict in separate if block --- pkg/registry/core/namespace/storage/storage.go | 4 +++- .../pkg/registry/customresourcedefinition/etcd.go | 4 +++- staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/helpers.go | 2 +- staging/src/k8s.io/apiserver/pkg/registry/rest/delete.go | 2 +- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/pkg/registry/core/namespace/storage/storage.go b/pkg/registry/core/namespace/storage/storage.go index 82f54b72a1..6b7bf0ec5c 100644 --- a/pkg/registry/core/namespace/storage/storage.go +++ b/pkg/registry/core/namespace/storage/storage.go @@ -139,7 +139,6 @@ func (r *REST) Delete(ctx context.Context, name string, options *metav1.DeleteOp } if options.Preconditions.UID == nil { options.Preconditions.UID = &namespace.UID - options.Preconditions.ResourceVersion = &namespace.ResourceVersion } else if *options.Preconditions.UID != namespace.UID { err = apierrors.NewConflict( api.Resource("namespaces"), @@ -147,6 +146,9 @@ func (r *REST) Delete(ctx context.Context, name string, options *metav1.DeleteOp fmt.Errorf("Precondition failed: UID in precondition: %v, UID in object meta: %v", *options.Preconditions.UID, namespace.UID), ) return nil, false, err + } + if options.Preconditions.ResourceVersion == nil { + options.Preconditions.ResourceVersion = &namespace.ResourceVersion } else if *options.Preconditions.ResourceVersion != namespace.ResourceVersion { err = apierrors.NewConflict( api.Resource("namespaces"), diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition/etcd.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition/etcd.go index fe7c3bafd6..f44a521b21 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition/etcd.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition/etcd.go @@ -84,7 +84,6 @@ func (r *REST) Delete(ctx context.Context, name string, options *metav1.DeleteOp } if options.Preconditions.UID == nil { options.Preconditions.UID = &crd.UID - options.Preconditions.ResourceVersion = &crd.ResourceVersion } else if *options.Preconditions.UID != crd.UID { err = apierrors.NewConflict( apiextensions.Resource("customresourcedefinitions"), @@ -92,6 +91,9 @@ func (r *REST) Delete(ctx context.Context, name string, options *metav1.DeleteOp fmt.Errorf("Precondition failed: UID in precondition: %v, UID in object meta: %v", *options.Preconditions.UID, crd.UID), ) return nil, false, err + } + if options.Preconditions.ResourceVersion == nil { + options.Preconditions.ResourceVersion = &crd.ResourceVersion } else if *options.Preconditions.ResourceVersion != crd.ResourceVersion { err = apierrors.NewConflict( apiextensions.Resource("customresourcedefinitions"), diff --git a/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/helpers.go b/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/helpers.go index 9da8008331..910f9bd273 100644 --- a/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/helpers.go +++ b/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/helpers.go @@ -228,7 +228,7 @@ func NewUIDPreconditions(uid string) *Preconditions { return &Preconditions{UID: &u} } -// NewPreconditionDeleteOptionsWithResourceVersion returns a DeleteOptions with a UID precondition set. +// NewPreconditionDeleteOptionsWithResourceVersion returns a DeleteOptions with a ResourceVersion precondition set. func NewPreconditionDeleteOptionsWithResourceVersion(rv string) *DeleteOptions { p := Preconditions{ResourceVersion: &rv} return &DeleteOptions{Preconditions: &p} diff --git a/staging/src/k8s.io/apiserver/pkg/registry/rest/delete.go b/staging/src/k8s.io/apiserver/pkg/registry/rest/delete.go index 0713464e00..16c9e1b40d 100644 --- a/staging/src/k8s.io/apiserver/pkg/registry/rest/delete.go +++ b/staging/src/k8s.io/apiserver/pkg/registry/rest/delete.go @@ -82,7 +82,7 @@ func BeforeDelete(strategy RESTDeleteStrategy, ctx context.Context, obj runtime. return false, false, errors.NewConflict(schema.GroupResource{Group: gvk.Group, Resource: gvk.Kind}, objectMeta.GetName(), fmt.Errorf("the UID in the precondition (%s) does not match the UID in record (%s). The object might have been deleted and then recreated", *options.Preconditions.UID, objectMeta.GetUID())) } if options.Preconditions.ResourceVersion != nil && *options.Preconditions.ResourceVersion != objectMeta.GetResourceVersion() { - return false, false, errors.NewConflict(schema.GroupResource{Group: gvk.Group, Resource: gvk.Kind}, objectMeta.GetName(), fmt.Errorf("the ResourceVersion in the precondition (%s) does not match the ResourceVersion in record (%s). The object might have been deleted and then recreated", *options.Preconditions.ResourceVersion, objectMeta.GetResourceVersion())) + return false, false, errors.NewConflict(schema.GroupResource{Group: gvk.Group, Resource: gvk.Kind}, objectMeta.GetName(), fmt.Errorf("the ResourceVersion in the precondition (%s) does not match the ResourceVersion in record (%s). The object might have been modified", *options.Preconditions.ResourceVersion, objectMeta.GetResourceVersion())) } } gracefulStrategy, ok := strategy.(RESTGracefulDeleteStrategy) From 64ecf9de7f1e6661c295ea87aeb5c18d6157aa14 Mon Sep 17 00:00:00 2001 From: ajatprabha Date: Fri, 15 Feb 2019 04:58:03 +0530 Subject: [PATCH 13/21] update deepcopy function --- .../apimachinery/pkg/apis/meta/v1/zz_generated.deepcopy.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/zz_generated.deepcopy.go b/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/zz_generated.deepcopy.go index 5b09ac54c4..68498b8d20 100644 --- a/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/zz_generated.deepcopy.go +++ b/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/zz_generated.deepcopy.go @@ -831,6 +831,11 @@ func (in *Preconditions) DeepCopyInto(out *Preconditions) { *out = new(types.UID) **out = **in } + if in.ResourceVersion != nil { + in, out := &in.ResourceVersion, &out.ResourceVersion + *out = new(string) + **out = **in + } return } From 3135cea2cce02bc8c6796e97f4579d538f357f74 Mon Sep 17 00:00:00 2001 From: ajatprabha Date: Fri, 15 Feb 2019 12:30:34 +0530 Subject: [PATCH 14/21] add ResourceVersion check to Preconditions#Check --- staging/src/k8s.io/apiserver/pkg/storage/interfaces.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/staging/src/k8s.io/apiserver/pkg/storage/interfaces.go b/staging/src/k8s.io/apiserver/pkg/storage/interfaces.go index 93b3f5deb8..cff425f3ab 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/interfaces.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/interfaces.go @@ -128,8 +128,14 @@ func (p *Preconditions) Check(key string, obj runtime.Object) error { objMeta.GetUID()) return NewInvalidObjError(key, err) } + if p.ResourceVersion != nil && *p.ResourceVersion != objMeta.GetResourceVersion() { + err := fmt.Sprintf( + "Precondition failed: ResourceVersion in precondition: %v, ResourceVersion in object meta: %v", + *p.ResourceVersion, + objMeta.GetResourceVersion()) + return NewInvalidObjError(key, err) + } return nil - } // Interface offers a common interface for object marshaling/unmarshaling operations and From 63f4827fca0a2fc8491a8340091fda3a0c952804 Mon Sep 17 00:00:00 2001 From: ajatprabha Date: Fri, 15 Feb 2019 12:31:20 +0530 Subject: [PATCH 15/21] update openapi-spec --- api/openapi-spec/swagger.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json index c98abc7cd3..572be2e0f6 100644 --- a/api/openapi-spec/swagger.json +++ b/api/openapi-spec/swagger.json @@ -17546,6 +17546,10 @@ "io.k8s.apimachinery.pkg.apis.meta.v1.Preconditions": { "description": "Preconditions must be fulfilled before an operation (update, delete, etc.) is carried out.", "properties": { + "resourceVersion": { + "description": "Specifies the target ResourceVersion", + "type": "string" + }, "uid": { "description": "Specifies the target UID.", "type": "string" From 4ddc198c39cc8894067df452b0bba6b4c7c0b65e Mon Sep 17 00:00:00 2001 From: ajatprabha Date: Sat, 16 Feb 2019 12:32:57 +0530 Subject: [PATCH 16/21] pin ResourceVersion precondition only when specified --- pkg/registry/core/namespace/storage/storage.go | 4 +--- .../pkg/registry/customresourcedefinition/etcd.go | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/pkg/registry/core/namespace/storage/storage.go b/pkg/registry/core/namespace/storage/storage.go index 6b7bf0ec5c..b9d0558ab7 100644 --- a/pkg/registry/core/namespace/storage/storage.go +++ b/pkg/registry/core/namespace/storage/storage.go @@ -147,9 +147,7 @@ func (r *REST) Delete(ctx context.Context, name string, options *metav1.DeleteOp ) return nil, false, err } - if options.Preconditions.ResourceVersion == nil { - options.Preconditions.ResourceVersion = &namespace.ResourceVersion - } else if *options.Preconditions.ResourceVersion != namespace.ResourceVersion { + if options.Preconditions.ResourceVersion != nil && *options.Preconditions.ResourceVersion != namespace.ResourceVersion { err = apierrors.NewConflict( api.Resource("namespaces"), name, diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition/etcd.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition/etcd.go index f44a521b21..0bc9f64ddc 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition/etcd.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition/etcd.go @@ -92,9 +92,7 @@ func (r *REST) Delete(ctx context.Context, name string, options *metav1.DeleteOp ) return nil, false, err } - if options.Preconditions.ResourceVersion == nil { - options.Preconditions.ResourceVersion = &crd.ResourceVersion - } else if *options.Preconditions.ResourceVersion != crd.ResourceVersion { + if options.Preconditions.ResourceVersion != nil && *options.Preconditions.ResourceVersion != crd.ResourceVersion { err = apierrors.NewConflict( apiextensions.Resource("customresourcedefinitions"), name, From fe3b9f486fe988cf6b6cd83c54c2be77892fa123 Mon Sep 17 00:00:00 2001 From: ajatprabha Date: Mon, 25 Feb 2019 22:43:28 +0530 Subject: [PATCH 17/21] update testDeleteWithResourceVersion --- .../k8s.io/apimachinery/pkg/apis/meta/v1/helpers.go | 4 ++-- .../apiserver/pkg/registry/rest/resttest/resttest.go | 10 ++++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/helpers.go b/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/helpers.go index 910f9bd273..b4dc78b3ea 100644 --- a/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/helpers.go +++ b/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/helpers.go @@ -228,8 +228,8 @@ func NewUIDPreconditions(uid string) *Preconditions { return &Preconditions{UID: &u} } -// NewPreconditionDeleteOptionsWithResourceVersion returns a DeleteOptions with a ResourceVersion precondition set. -func NewPreconditionDeleteOptionsWithResourceVersion(rv string) *DeleteOptions { +// NewRVDeletionPrecondition returns a DeleteOptions with a ResourceVersion precondition set. +func NewRVDeletionPrecondition(rv string) *DeleteOptions { p := Preconditions{ResourceVersion: &rv} return &DeleteOptions{Preconditions: &p} } diff --git a/staging/src/k8s.io/apiserver/pkg/registry/rest/resttest/resttest.go b/staging/src/k8s.io/apiserver/pkg/registry/rest/resttest/resttest.go index 4316592f46..f29f4325b6 100644 --- a/staging/src/k8s.io/apiserver/pkg/registry/rest/resttest/resttest.go +++ b/staging/src/k8s.io/apiserver/pkg/registry/rest/resttest/resttest.go @@ -921,13 +921,15 @@ func (t *Tester) testDeleteWithResourceVersion(obj runtime.Object, createFn Crea if err := createFn(ctx, foo); err != nil { t.Errorf("unexpected error: %v", err) } - opts.Preconditions = metav1.NewPreconditionDeleteOptionsWithResourceVersion("RV1111").Preconditions - obj, _, err := t.storage.(rest.GracefulDeleter).Delete(ctx, objectMeta.GetName(), &opts) + opts.Preconditions = metav1.NewRVDeletionPrecondition("RV1111").Preconditions + obj, wasDeleted, err := t.storage.(rest.GracefulDeleter).Delete(ctx, objectMeta.GetName(), &opts) if err == nil || !errors.IsConflict(err) { t.Errorf("unexpected error: %v", err) } - - obj, _, err = t.storage.(rest.GracefulDeleter).Delete(ctx, objectMeta.GetName(), metav1.NewPreconditionDeleteOptionsWithResourceVersion("RV0000")) + if wasDeleted { + t.Errorf("unexpected, object %s should not have been deleted immediately", objectMeta.GetName()) + } + obj, _, err = t.storage.(rest.GracefulDeleter).Delete(ctx, objectMeta.GetName(), metav1.NewRVDeletionPrecondition("RV0000")) if err != nil { t.Errorf("unexpected error: %v", err) } From c2755f66851cc4893482b42cba3896b3533cc335 Mon Sep 17 00:00:00 2001 From: ajatprabha Date: Tue, 26 Feb 2019 00:17:47 +0530 Subject: [PATCH 18/21] update generated-swagger-docs --- .../pkg/apis/meta/v1/types_swagger_doc_generated.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/types_swagger_doc_generated.go b/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/types_swagger_doc_generated.go index 34b8b4f4ee..3b1a09e57f 100644 --- a/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/types_swagger_doc_generated.go +++ b/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/types_swagger_doc_generated.go @@ -294,8 +294,9 @@ func (PatchOptions) SwaggerDoc() map[string]string { } var map_Preconditions = map[string]string{ - "": "Preconditions must be fulfilled before an operation (update, delete, etc.) is carried out.", - "uid": "Specifies the target UID.", + "": "Preconditions must be fulfilled before an operation (update, delete, etc.) is carried out.", + "uid": "Specifies the target UID.", + "resourceVersion": "Specifies the target ResourceVersion", } func (Preconditions) SwaggerDoc() map[string]string { From 4bd0c4ad56f95685266ce092297fdb7bb1b88a02 Mon Sep 17 00:00:00 2001 From: ajatprabha Date: Mon, 11 Mar 2019 23:40:21 +0530 Subject: [PATCH 19/21] update generated-protobuf --- .../pkg/apis/meta/v1/generated.pb.go | 360 ++++++++++-------- .../pkg/apis/meta/v1/generated.proto | 4 + 2 files changed, 205 insertions(+), 159 deletions(-) diff --git a/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/generated.pb.go b/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/generated.pb.go index ed252d3415..69e6509932 100644 --- a/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/generated.pb.go +++ b/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/generated.pb.go @@ -1591,6 +1591,12 @@ func (m *Preconditions) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintGenerated(dAtA, i, uint64(len(*m.UID))) i += copy(dAtA[i:], *m.UID) } + if m.ResourceVersion != nil { + dAtA[i] = 0x12 + i++ + i = encodeVarintGenerated(dAtA, i, uint64(len(*m.ResourceVersion))) + i += copy(dAtA[i:], *m.ResourceVersion) + } return i, nil } @@ -2428,6 +2434,10 @@ func (m *Preconditions) Size() (n int) { l = len(*m.UID) n += 1 + l + sovGenerated(uint64(l)) } + if m.ResourceVersion != nil { + l = len(*m.ResourceVersion) + n += 1 + l + sovGenerated(uint64(l)) + } return n } @@ -2907,6 +2917,7 @@ func (this *Preconditions) String() string { } s := strings.Join([]string{`&Preconditions{`, `UID:` + valueToStringGenerated(this.UID) + `,`, + `ResourceVersion:` + valueToStringGenerated(this.ResourceVersion) + `,`, `}`, }, "") return s @@ -7607,6 +7618,36 @@ func (m *Preconditions) Unmarshal(dAtA []byte) error { s := k8s_io_apimachinery_pkg_types.UID(dAtA[iNdEx:postIndex]) m.UID = &s iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ResourceVersion", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + s := string(dAtA[iNdEx:postIndex]) + m.ResourceVersion = &s + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipGenerated(dAtA[iNdEx:]) @@ -8989,172 +9030,173 @@ func init() { } var fileDescriptorGenerated = []byte{ - // 2664 bytes of a gzipped FileDescriptorProto + // 2674 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xcc, 0x19, 0x4d, 0x6c, 0x23, 0x57, 0x39, 0x63, 0xc7, 0x8e, 0xfd, 0x39, 0xce, 0xcf, 0xeb, 0x16, 0x5c, 0x4b, 0xc4, 0xe9, 0x14, 0x55, - 0x29, 0x6c, 0x6d, 0x92, 0xd2, 0x6a, 0x59, 0xa0, 0x10, 0xc7, 0xc9, 0x36, 0x74, 0xd3, 0x44, 0x2f, - 0xbb, 0x8b, 0x58, 0x56, 0x88, 0x89, 0xe7, 0xc5, 0x19, 0x62, 0xcf, 0x4c, 0xdf, 0x1b, 0x67, 0x37, - 0x70, 0xa0, 0x07, 0x10, 0x20, 0x01, 0xda, 0x23, 0x27, 0xd4, 0x15, 0x5c, 0xb8, 0x72, 0xe2, 0xc4, - 0xa9, 0x12, 0x7b, 0xac, 0xc4, 0xa5, 0x07, 0x64, 0x75, 0x03, 0x12, 0xdc, 0xb8, 0xe7, 0x80, 0xd0, - 0xfb, 0x99, 0x99, 0x37, 0x76, 0xbc, 0x19, 0xb3, 0x05, 0x71, 0xf2, 0xcc, 0xf7, 0x3f, 0xdf, 0xf7, - 0xbd, 0xef, 0xfb, 0xde, 0x67, 0xd8, 0x39, 0xbe, 0xc6, 0xea, 0x8e, 0xd7, 0x38, 0xee, 0x1f, 0x10, - 0xea, 0x92, 0x80, 0xb0, 0xc6, 0x09, 0x71, 0x6d, 0x8f, 0x36, 0x14, 0xc2, 0xf2, 0x9d, 0x9e, 0xd5, - 0x3e, 0x72, 0x5c, 0x42, 0x4f, 0x1b, 0xfe, 0x71, 0x87, 0x03, 0x58, 0xa3, 0x47, 0x02, 0xab, 0x71, - 0xb2, 0xda, 0xe8, 0x10, 0x97, 0x50, 0x2b, 0x20, 0x76, 0xdd, 0xa7, 0x5e, 0xe0, 0xa1, 0xcf, 0x4a, - 0xae, 0xba, 0xce, 0x55, 0xf7, 0x8f, 0x3b, 0x1c, 0xc0, 0xea, 0x9c, 0xab, 0x7e, 0xb2, 0x5a, 0x7d, - 0xb5, 0xe3, 0x04, 0x47, 0xfd, 0x83, 0x7a, 0xdb, 0xeb, 0x35, 0x3a, 0x5e, 0xc7, 0x6b, 0x08, 0xe6, - 0x83, 0xfe, 0xa1, 0x78, 0x13, 0x2f, 0xe2, 0x49, 0x0a, 0xad, 0x8e, 0x35, 0x85, 0xf6, 0xdd, 0xc0, - 0xe9, 0x91, 0x61, 0x2b, 0xaa, 0x6f, 0x5c, 0xc6, 0xc0, 0xda, 0x47, 0xa4, 0x67, 0x0d, 0xf3, 0x99, - 0x7f, 0xca, 0x42, 0x61, 0x7d, 0x6f, 0xfb, 0x06, 0xf5, 0xfa, 0x3e, 0x5a, 0x86, 0x69, 0xd7, 0xea, - 0x91, 0x8a, 0xb1, 0x6c, 0xac, 0x14, 0x9b, 0xb3, 0x8f, 0x07, 0xb5, 0xa9, 0xb3, 0x41, 0x6d, 0xfa, - 0x1d, 0xab, 0x47, 0xb0, 0xc0, 0xa0, 0x2e, 0x14, 0x4e, 0x08, 0x65, 0x8e, 0xe7, 0xb2, 0x4a, 0x66, - 0x39, 0xbb, 0x52, 0x5a, 0x7b, 0xb3, 0x9e, 0xe6, 0xfb, 0xeb, 0x42, 0xc1, 0x1d, 0xc9, 0xba, 0xe5, - 0xd1, 0x96, 0xc3, 0xda, 0xde, 0x09, 0xa1, 0xa7, 0xcd, 0x05, 0xa5, 0xa5, 0xa0, 0x90, 0x0c, 0x47, - 0x1a, 0xd0, 0x8f, 0x0c, 0x58, 0xf0, 0x29, 0x39, 0x24, 0x94, 0x12, 0x5b, 0xe1, 0x2b, 0xd9, 0x65, - 0xe3, 0x13, 0x50, 0x5b, 0x51, 0x6a, 0x17, 0xf6, 0x86, 0xe4, 0xe3, 0x11, 0x8d, 0xe8, 0x37, 0x06, - 0x54, 0x19, 0xa1, 0x27, 0x84, 0xae, 0xdb, 0x36, 0x25, 0x8c, 0x35, 0x4f, 0x37, 0xba, 0x0e, 0x71, - 0x83, 0x8d, 0xed, 0x16, 0x66, 0x95, 0x69, 0xe1, 0x87, 0xaf, 0xa5, 0x33, 0x68, 0x7f, 0x9c, 0x9c, - 0xa6, 0xa9, 0x2c, 0xaa, 0x8e, 0x25, 0x61, 0xf8, 0x29, 0x66, 0x98, 0x87, 0x30, 0x1b, 0x06, 0xf2, - 0xa6, 0xc3, 0x02, 0x74, 0x07, 0xf2, 0x1d, 0xfe, 0xc2, 0x2a, 0x86, 0x30, 0xb0, 0x9e, 0xce, 0xc0, - 0x50, 0x46, 0x73, 0x4e, 0xd9, 0x93, 0x17, 0xaf, 0x0c, 0x2b, 0x69, 0xe6, 0xcf, 0xa6, 0xa1, 0xb4, - 0xbe, 0xb7, 0x8d, 0x09, 0xf3, 0xfa, 0xb4, 0x4d, 0x52, 0x24, 0xcd, 0x1a, 0x00, 0xff, 0x65, 0xbe, - 0xd5, 0x26, 0x76, 0x25, 0xb3, 0x6c, 0xac, 0x14, 0x9a, 0x48, 0xd1, 0xc1, 0x3b, 0x11, 0x06, 0x6b, - 0x54, 0x5c, 0xea, 0xb1, 0xe3, 0xda, 0x22, 0xda, 0x9a, 0xd4, 0xb7, 0x1d, 0xd7, 0xc6, 0x02, 0x83, - 0x6e, 0x42, 0xee, 0x84, 0xd0, 0x03, 0xee, 0x7f, 0x9e, 0x10, 0x9f, 0x4f, 0xf7, 0x79, 0x77, 0x38, - 0x4b, 0xb3, 0x78, 0x36, 0xa8, 0xe5, 0xc4, 0x23, 0x96, 0x42, 0x50, 0x1d, 0x80, 0x1d, 0x79, 0x34, - 0x10, 0xe6, 0x54, 0x72, 0xcb, 0xd9, 0x95, 0x62, 0x73, 0x8e, 0xdb, 0xb7, 0x1f, 0x41, 0xb1, 0x46, - 0x81, 0xae, 0xc1, 0x2c, 0x73, 0xdc, 0x4e, 0xbf, 0x6b, 0x51, 0x0e, 0xa8, 0xe4, 0x85, 0x9d, 0x57, - 0x94, 0x9d, 0xb3, 0xfb, 0x1a, 0x0e, 0x27, 0x28, 0xb9, 0xa6, 0xb6, 0x15, 0x90, 0x8e, 0x47, 0x1d, - 0xc2, 0x2a, 0x33, 0xb1, 0xa6, 0x8d, 0x08, 0x8a, 0x35, 0x0a, 0xf4, 0x12, 0xe4, 0x84, 0xe7, 0x2b, - 0x05, 0xa1, 0xa2, 0xac, 0x54, 0xe4, 0x44, 0x58, 0xb0, 0xc4, 0xa1, 0x57, 0x60, 0x46, 0x9d, 0x9a, - 0x4a, 0x51, 0x90, 0xcd, 0x2b, 0xb2, 0x99, 0x30, 0xad, 0x43, 0x3c, 0xfa, 0x06, 0x20, 0x16, 0x78, - 0xd4, 0xea, 0x10, 0x85, 0x7a, 0xcb, 0x62, 0x47, 0x15, 0x10, 0x5c, 0x55, 0xc5, 0x85, 0xf6, 0x47, - 0x28, 0xf0, 0x05, 0x5c, 0xe6, 0xef, 0x0d, 0x98, 0xd7, 0x72, 0x41, 0xe4, 0xdd, 0x35, 0x98, 0xed, - 0x68, 0xa7, 0x4e, 0xe5, 0x45, 0xe4, 0x19, 0xfd, 0x44, 0xe2, 0x04, 0x25, 0x22, 0x50, 0xa4, 0x4a, - 0x52, 0x58, 0x5d, 0x56, 0x53, 0x27, 0x6d, 0x68, 0x43, 0xac, 0x49, 0x03, 0x32, 0x1c, 0x4b, 0x36, - 0xff, 0x6e, 0x88, 0x04, 0x0e, 0xeb, 0x0d, 0x5a, 0xd1, 0x6a, 0x9a, 0x21, 0xc2, 0x31, 0x3b, 0xa6, - 0x1e, 0x5d, 0x52, 0x08, 0x32, 0xff, 0x17, 0x85, 0xe0, 0x7a, 0xe1, 0x57, 0xef, 0xd7, 0xa6, 0xde, - 0xfb, 0xcb, 0xf2, 0x94, 0xd9, 0x83, 0xf2, 0x06, 0x25, 0x56, 0x40, 0x76, 0xfd, 0x40, 0x7c, 0x80, - 0x09, 0x79, 0x9b, 0x9e, 0xe2, 0xbe, 0xab, 0x3e, 0x14, 0xf8, 0xf9, 0x6e, 0x09, 0x08, 0x56, 0x18, - 0x1e, 0xbf, 0x43, 0x87, 0x74, 0xed, 0x1d, 0xcb, 0xb5, 0x3a, 0x84, 0xaa, 0x13, 0x18, 0x79, 0x75, - 0x4b, 0xc3, 0xe1, 0x04, 0xa5, 0xf9, 0x93, 0x2c, 0x94, 0x5b, 0xa4, 0x4b, 0x62, 0x7d, 0x5b, 0x80, + 0x29, 0x6c, 0x6d, 0x92, 0xd2, 0x6a, 0x59, 0x68, 0x21, 0x8e, 0x93, 0x6d, 0xe8, 0xa6, 0x89, 0x5e, + 0xba, 0x8b, 0x58, 0x56, 0x88, 0x89, 0xe7, 0xc5, 0x19, 0x62, 0xcf, 0x4c, 0xdf, 0x1b, 0x67, 0x37, + 0x70, 0xa0, 0x07, 0x10, 0x20, 0x41, 0xb5, 0x47, 0x4e, 0xa8, 0x2b, 0xb8, 0x70, 0xe5, 0xc4, 0x89, + 0x53, 0x25, 0xf6, 0x58, 0x89, 0x4b, 0x0f, 0xc8, 0xea, 0x06, 0x24, 0xb8, 0x71, 0xcf, 0x01, 0xa1, + 0xf7, 0x33, 0x33, 0x6f, 0xec, 0x78, 0x33, 0x66, 0x0b, 0xe2, 0x14, 0xcf, 0xf7, 0xff, 0xbe, 0xef, + 0x7b, 0xdf, 0xf7, 0xbd, 0x2f, 0xb0, 0x73, 0x7c, 0x8d, 0xd5, 0x1d, 0xaf, 0x71, 0xdc, 0x3f, 0x20, + 0xd4, 0x25, 0x01, 0x61, 0x8d, 0x13, 0xe2, 0xda, 0x1e, 0x6d, 0x28, 0x84, 0xe5, 0x3b, 0x3d, 0xab, + 0x7d, 0xe4, 0xb8, 0x84, 0x9e, 0x36, 0xfc, 0xe3, 0x0e, 0x07, 0xb0, 0x46, 0x8f, 0x04, 0x56, 0xe3, + 0x64, 0xb5, 0xd1, 0x21, 0x2e, 0xa1, 0x56, 0x40, 0xec, 0xba, 0x4f, 0xbd, 0xc0, 0x43, 0x9f, 0x97, + 0x5c, 0x75, 0x9d, 0xab, 0xee, 0x1f, 0x77, 0x38, 0x80, 0xd5, 0x39, 0x57, 0xfd, 0x64, 0xb5, 0xfa, + 0x72, 0xc7, 0x09, 0x8e, 0xfa, 0x07, 0xf5, 0xb6, 0xd7, 0x6b, 0x74, 0xbc, 0x8e, 0xd7, 0x10, 0xcc, + 0x07, 0xfd, 0x43, 0xf1, 0x25, 0x3e, 0xc4, 0x2f, 0x29, 0xb4, 0x3a, 0xd6, 0x14, 0xda, 0x77, 0x03, + 0xa7, 0x47, 0x86, 0xad, 0xa8, 0xbe, 0x76, 0x19, 0x03, 0x6b, 0x1f, 0x91, 0x9e, 0x35, 0xcc, 0x67, + 0xfe, 0x29, 0x0b, 0x85, 0xf5, 0xbd, 0xed, 0x1b, 0xd4, 0xeb, 0xfb, 0x68, 0x19, 0xa6, 0x5d, 0xab, + 0x47, 0x2a, 0xc6, 0xb2, 0xb1, 0x52, 0x6c, 0xce, 0x3e, 0x1a, 0xd4, 0xa6, 0xce, 0x06, 0xb5, 0xe9, + 0xb7, 0xad, 0x1e, 0xc1, 0x02, 0x83, 0xba, 0x50, 0x38, 0x21, 0x94, 0x39, 0x9e, 0xcb, 0x2a, 0x99, + 0xe5, 0xec, 0x4a, 0x69, 0xed, 0x8d, 0x7a, 0x9a, 0xf3, 0xd7, 0x85, 0x82, 0xdb, 0x92, 0x75, 0xcb, + 0xa3, 0x2d, 0x87, 0xb5, 0xbd, 0x13, 0x42, 0x4f, 0x9b, 0x0b, 0x4a, 0x4b, 0x41, 0x21, 0x19, 0x8e, + 0x34, 0xa0, 0x1f, 0x1b, 0xb0, 0xe0, 0x53, 0x72, 0x48, 0x28, 0x25, 0xb6, 0xc2, 0x57, 0xb2, 0xcb, + 0xc6, 0xa7, 0xa0, 0xb6, 0xa2, 0xd4, 0x2e, 0xec, 0x0d, 0xc9, 0xc7, 0x23, 0x1a, 0xd1, 0x6f, 0x0c, + 0xa8, 0x32, 0x42, 0x4f, 0x08, 0x5d, 0xb7, 0x6d, 0x4a, 0x18, 0x6b, 0x9e, 0x6e, 0x74, 0x1d, 0xe2, + 0x06, 0x1b, 0xdb, 0x2d, 0xcc, 0x2a, 0xd3, 0xc2, 0x0f, 0x5f, 0x4f, 0x67, 0xd0, 0xfe, 0x38, 0x39, + 0x4d, 0x53, 0x59, 0x54, 0x1d, 0x4b, 0xc2, 0xf0, 0x13, 0xcc, 0x30, 0x0f, 0x61, 0x36, 0x0c, 0xe4, + 0x4d, 0x87, 0x05, 0xe8, 0x36, 0xe4, 0x3b, 0xfc, 0x83, 0x55, 0x0c, 0x61, 0x60, 0x3d, 0x9d, 0x81, + 0xa1, 0x8c, 0xe6, 0x9c, 0xb2, 0x27, 0x2f, 0x3e, 0x19, 0x56, 0xd2, 0xcc, 0x9f, 0x4f, 0x43, 0x69, + 0x7d, 0x6f, 0x1b, 0x13, 0xe6, 0xf5, 0x69, 0x9b, 0xa4, 0x48, 0x9a, 0x35, 0x00, 0xfe, 0x97, 0xf9, + 0x56, 0x9b, 0xd8, 0x95, 0xcc, 0xb2, 0xb1, 0x52, 0x68, 0x22, 0x45, 0x07, 0x6f, 0x47, 0x18, 0xac, + 0x51, 0x71, 0xa9, 0xc7, 0x8e, 0x6b, 0x8b, 0x68, 0x6b, 0x52, 0xdf, 0x72, 0x5c, 0x1b, 0x0b, 0x0c, + 0xba, 0x09, 0xb9, 0x13, 0x42, 0x0f, 0xb8, 0xff, 0x79, 0x42, 0x7c, 0x31, 0xdd, 0xf1, 0x6e, 0x73, + 0x96, 0x66, 0xf1, 0x6c, 0x50, 0xcb, 0x89, 0x9f, 0x58, 0x0a, 0x41, 0x75, 0x00, 0x76, 0xe4, 0xd1, + 0x40, 0x98, 0x53, 0xc9, 0x2d, 0x67, 0x57, 0x8a, 0xcd, 0x39, 0x6e, 0xdf, 0x7e, 0x04, 0xc5, 0x1a, + 0x05, 0xba, 0x06, 0xb3, 0xcc, 0x71, 0x3b, 0xfd, 0xae, 0x45, 0x39, 0xa0, 0x92, 0x17, 0x76, 0x5e, + 0x51, 0x76, 0xce, 0xee, 0x6b, 0x38, 0x9c, 0xa0, 0xe4, 0x9a, 0xda, 0x56, 0x40, 0x3a, 0x1e, 0x75, + 0x08, 0xab, 0xcc, 0xc4, 0x9a, 0x36, 0x22, 0x28, 0xd6, 0x28, 0xd0, 0x0b, 0x90, 0x13, 0x9e, 0xaf, + 0x14, 0x84, 0x8a, 0xb2, 0x52, 0x91, 0x13, 0x61, 0xc1, 0x12, 0x87, 0x5e, 0x82, 0x19, 0x75, 0x6b, + 0x2a, 0x45, 0x41, 0x36, 0xaf, 0xc8, 0x66, 0xc2, 0xb4, 0x0e, 0xf1, 0xe8, 0x9b, 0x80, 0x58, 0xe0, + 0x51, 0xab, 0x43, 0x14, 0xea, 0x4d, 0x8b, 0x1d, 0x55, 0x40, 0x70, 0x55, 0x15, 0x17, 0xda, 0x1f, + 0xa1, 0xc0, 0x17, 0x70, 0x99, 0xbf, 0x37, 0x60, 0x5e, 0xcb, 0x05, 0x91, 0x77, 0xd7, 0x60, 0xb6, + 0xa3, 0xdd, 0x3a, 0x95, 0x17, 0x91, 0x67, 0xf4, 0x1b, 0x89, 0x13, 0x94, 0x88, 0x40, 0x91, 0x2a, + 0x49, 0x61, 0x75, 0x59, 0x4d, 0x9d, 0xb4, 0xa1, 0x0d, 0xb1, 0x26, 0x0d, 0xc8, 0x70, 0x2c, 0xd9, + 0xfc, 0xbb, 0x21, 0x12, 0x38, 0xac, 0x37, 0x68, 0x45, 0xab, 0x69, 0x86, 0x08, 0xc7, 0xec, 0x98, + 0x7a, 0x74, 0x49, 0x21, 0xc8, 0xfc, 0x5f, 0x14, 0x82, 0xeb, 0x85, 0x5f, 0x7d, 0x50, 0x9b, 0x7a, + 0xef, 0x2f, 0xcb, 0x53, 0x66, 0x0f, 0xca, 0x1b, 0x94, 0x58, 0x01, 0xd9, 0xf5, 0x03, 0x71, 0x00, + 0x13, 0xf2, 0x36, 0x3d, 0xc5, 0x7d, 0x57, 0x1d, 0x14, 0xf8, 0xfd, 0x6e, 0x09, 0x08, 0x56, 0x18, + 0x1e, 0xbf, 0x43, 0x87, 0x74, 0xed, 0x1d, 0xcb, 0xb5, 0x3a, 0x84, 0xaa, 0x1b, 0x18, 0x79, 0x75, + 0x4b, 0xc3, 0xe1, 0x04, 0xa5, 0xf9, 0xd3, 0x2c, 0x94, 0x5b, 0xa4, 0x4b, 0x62, 0x7d, 0x5b, 0x80, 0x3a, 0xd4, 0x6a, 0x93, 0x3d, 0x42, 0x1d, 0xcf, 0xde, 0x27, 0x6d, 0xcf, 0xb5, 0x99, 0xc8, 0x88, - 0x6c, 0xf3, 0x53, 0x3c, 0xcf, 0x6e, 0x8c, 0x60, 0xf1, 0x05, 0x1c, 0xa8, 0x0b, 0x65, 0x9f, 0x8a, - 0x67, 0x27, 0x50, 0xbd, 0x87, 0x9f, 0xf9, 0xd7, 0xd2, 0xb9, 0x7a, 0x4f, 0x67, 0x6d, 0x2e, 0x9e, - 0x0d, 0x6a, 0xe5, 0x04, 0x08, 0x27, 0x85, 0xa3, 0xaf, 0xc3, 0x82, 0x47, 0xfd, 0x23, 0xcb, 0x6d, - 0x11, 0x9f, 0xb8, 0x36, 0x71, 0x03, 0x26, 0xbc, 0x50, 0x68, 0x5e, 0xe1, 0x1d, 0x63, 0x77, 0x08, - 0x87, 0x47, 0xa8, 0xd1, 0x5d, 0x58, 0xf4, 0xa9, 0xe7, 0x5b, 0x1d, 0x8b, 0x4b, 0xdc, 0xf3, 0xba, - 0x4e, 0xfb, 0x54, 0xd4, 0xa9, 0x62, 0xf3, 0xea, 0xd9, 0xa0, 0xb6, 0xb8, 0x37, 0x8c, 0x3c, 0x1f, - 0xd4, 0x9e, 0x13, 0xae, 0xe3, 0x90, 0x18, 0x89, 0x47, 0xc5, 0x68, 0x31, 0xcc, 0x8d, 0x8b, 0xa1, - 0xb9, 0x0d, 0x85, 0x56, 0x9f, 0x0a, 0x2e, 0xf4, 0x55, 0x28, 0xd8, 0xea, 0x59, 0x79, 0xfe, 0xc5, - 0xb0, 0xe5, 0x86, 0x34, 0xe7, 0x83, 0x5a, 0x99, 0x0f, 0x09, 0xf5, 0x10, 0x80, 0x23, 0x16, 0xf3, - 0x1e, 0x94, 0x37, 0x1f, 0xf8, 0x1e, 0x0d, 0xc2, 0x98, 0xbe, 0x0c, 0x79, 0x22, 0x00, 0x42, 0x5a, - 0x21, 0xee, 0x13, 0x92, 0x0c, 0x2b, 0x2c, 0xaf, 0x5b, 0xe4, 0x81, 0xd5, 0x0e, 0x54, 0xc1, 0x8f, - 0xea, 0xd6, 0x26, 0x07, 0x62, 0x89, 0x33, 0x3f, 0x30, 0x20, 0x2f, 0x32, 0x8a, 0xa1, 0x5b, 0x90, - 0xed, 0x59, 0xbe, 0x6a, 0x56, 0xaf, 0xa7, 0x8b, 0xac, 0x64, 0xad, 0xef, 0x58, 0xfe, 0xa6, 0x1b, - 0xd0, 0xd3, 0x66, 0x49, 0x29, 0xc9, 0xee, 0x58, 0x3e, 0xe6, 0xe2, 0xaa, 0x36, 0x14, 0x42, 0x2c, - 0x5a, 0x80, 0xec, 0x31, 0x39, 0x95, 0x05, 0x09, 0xf3, 0x47, 0xd4, 0x84, 0xdc, 0x89, 0xd5, 0xed, - 0x13, 0x95, 0x4f, 0x57, 0x27, 0xd1, 0x8a, 0x25, 0xeb, 0xf5, 0xcc, 0x35, 0xc3, 0xdc, 0x05, 0xb8, - 0x41, 0x22, 0x0f, 0xad, 0xc3, 0x7c, 0x58, 0x6d, 0x92, 0x45, 0xf0, 0xd3, 0xca, 0xbc, 0x79, 0x9c, - 0x44, 0xe3, 0x61, 0x7a, 0xf3, 0x1e, 0x14, 0x45, 0xa1, 0xe4, 0xfd, 0x2e, 0xee, 0x00, 0xc6, 0x53, - 0x3a, 0x40, 0xd8, 0x30, 0x33, 0xe3, 0x1a, 0xa6, 0x56, 0x17, 0xba, 0x50, 0x96, 0xbc, 0x61, 0x0f, - 0x4f, 0xa5, 0xe1, 0x2a, 0x14, 0x42, 0x33, 0x95, 0x96, 0x68, 0x76, 0x0b, 0x05, 0xe1, 0x88, 0x42, - 0xd3, 0x76, 0x04, 0x89, 0xa2, 0x9f, 0x4e, 0x99, 0xd6, 0xd0, 0x32, 0x4f, 0x6f, 0x68, 0x9a, 0xa6, - 0x1f, 0x42, 0x65, 0xdc, 0xc0, 0xf7, 0x0c, 0x6d, 0x29, 0xbd, 0x29, 0xe6, 0x2f, 0x0d, 0x58, 0xd0, - 0x25, 0xa5, 0x0f, 0x5f, 0x7a, 0x25, 0x97, 0x8f, 0x46, 0x9a, 0x47, 0x7e, 0x6d, 0xc0, 0x95, 0xc4, - 0xa7, 0x4d, 0x14, 0xf1, 0x09, 0x8c, 0xd2, 0x93, 0x23, 0x3b, 0x41, 0x72, 0x34, 0xa0, 0xb4, 0xed, - 0x3a, 0x81, 0x63, 0x75, 0x9d, 0xef, 0x13, 0x7a, 0xf9, 0x30, 0x69, 0xfe, 0xd1, 0x80, 0x59, 0x8d, - 0x83, 0xa1, 0x7b, 0x30, 0xc3, 0xeb, 0xae, 0xe3, 0x76, 0x54, 0xed, 0x48, 0x39, 0x33, 0x68, 0x42, - 0xe2, 0xef, 0xda, 0x93, 0x92, 0x70, 0x28, 0x12, 0xed, 0x41, 0x9e, 0x12, 0xd6, 0xef, 0x06, 0x93, + 0x6c, 0xf3, 0x33, 0x3c, 0xcf, 0x6e, 0x8c, 0x60, 0xf1, 0x05, 0x1c, 0xa8, 0x0b, 0x65, 0x9f, 0x8a, + 0xdf, 0x4e, 0xa0, 0x7a, 0x0f, 0xbf, 0xf3, 0xaf, 0xa4, 0x73, 0xf5, 0x9e, 0xce, 0xda, 0x5c, 0x3c, + 0x1b, 0xd4, 0xca, 0x09, 0x10, 0x4e, 0x0a, 0x47, 0xdf, 0x80, 0x05, 0x8f, 0xfa, 0x47, 0x96, 0xdb, + 0x22, 0x3e, 0x71, 0x6d, 0xe2, 0x06, 0x4c, 0x78, 0xa1, 0xd0, 0xbc, 0xc2, 0x3b, 0xc6, 0xee, 0x10, + 0x0e, 0x8f, 0x50, 0xa3, 0x3b, 0xb0, 0xe8, 0x53, 0xcf, 0xb7, 0x3a, 0x16, 0x97, 0xb8, 0xe7, 0x75, + 0x9d, 0xf6, 0xa9, 0xa8, 0x53, 0xc5, 0xe6, 0xd5, 0xb3, 0x41, 0x6d, 0x71, 0x6f, 0x18, 0x79, 0x3e, + 0xa8, 0x3d, 0x23, 0x5c, 0xc7, 0x21, 0x31, 0x12, 0x8f, 0x8a, 0xd1, 0x62, 0x98, 0x1b, 0x17, 0x43, + 0x73, 0x1b, 0x0a, 0xad, 0x3e, 0x15, 0x5c, 0xe8, 0x75, 0x28, 0xd8, 0xea, 0xb7, 0xf2, 0xfc, 0xf3, + 0x61, 0xcb, 0x0d, 0x69, 0xce, 0x07, 0xb5, 0x32, 0x1f, 0x12, 0xea, 0x21, 0x00, 0x47, 0x2c, 0xe6, + 0x5d, 0x28, 0x6f, 0xde, 0xf7, 0x3d, 0x1a, 0x84, 0x31, 0x7d, 0x11, 0xf2, 0x44, 0x00, 0x84, 0xb4, + 0x42, 0xdc, 0x27, 0x24, 0x19, 0x56, 0x58, 0x5e, 0xb7, 0xc8, 0x7d, 0xab, 0x1d, 0xa8, 0x82, 0x1f, + 0xd5, 0xad, 0x4d, 0x0e, 0xc4, 0x12, 0x67, 0x7e, 0x68, 0x40, 0x5e, 0x64, 0x14, 0x43, 0xef, 0x40, + 0xb6, 0x67, 0xf9, 0xaa, 0x59, 0xbd, 0x9a, 0x2e, 0xb2, 0x92, 0xb5, 0xbe, 0x63, 0xf9, 0x9b, 0x6e, + 0x40, 0x4f, 0x9b, 0x25, 0xa5, 0x24, 0xbb, 0x63, 0xf9, 0x98, 0x8b, 0xab, 0xda, 0x50, 0x08, 0xb1, + 0x68, 0x01, 0xb2, 0xc7, 0xe4, 0x54, 0x16, 0x24, 0xcc, 0x7f, 0xa2, 0x26, 0xe4, 0x4e, 0xac, 0x6e, + 0x9f, 0xa8, 0x7c, 0xba, 0x3a, 0x89, 0x56, 0x2c, 0x59, 0xaf, 0x67, 0xae, 0x19, 0xe6, 0x2e, 0xc0, + 0x0d, 0x12, 0x79, 0x68, 0x1d, 0xe6, 0xc3, 0x6a, 0x93, 0x2c, 0x82, 0x9f, 0x55, 0xe6, 0xcd, 0xe3, + 0x24, 0x1a, 0x0f, 0xd3, 0x9b, 0x77, 0xa1, 0x28, 0x0a, 0x25, 0xef, 0x77, 0x71, 0x07, 0x30, 0x9e, + 0xd0, 0x01, 0xc2, 0x86, 0x99, 0x19, 0xd7, 0x30, 0xb5, 0xba, 0xd0, 0x85, 0xb2, 0xe4, 0x0d, 0x7b, + 0x78, 0x2a, 0x0d, 0x57, 0xa1, 0x10, 0x9a, 0xa9, 0xb4, 0x44, 0xb3, 0x5b, 0x28, 0x08, 0x47, 0x14, + 0x9a, 0xb6, 0x23, 0x48, 0x14, 0xfd, 0x74, 0xca, 0xb4, 0x86, 0x96, 0x79, 0x72, 0x43, 0xd3, 0x34, + 0xfd, 0x08, 0x2a, 0xe3, 0x06, 0xbe, 0xa7, 0x68, 0x4b, 0xe9, 0x4d, 0x31, 0xdf, 0x37, 0x60, 0x41, + 0x97, 0x94, 0x3e, 0x7c, 0xe9, 0x95, 0x5c, 0x3e, 0x1a, 0x69, 0x1e, 0xf9, 0xb5, 0x01, 0x57, 0x12, + 0x47, 0x9b, 0x28, 0xe2, 0x13, 0x18, 0xa5, 0x27, 0x47, 0x76, 0x82, 0xe4, 0x68, 0x40, 0x69, 0xdb, + 0x75, 0x02, 0xc7, 0xea, 0x3a, 0x3f, 0x20, 0xf4, 0xf2, 0x61, 0xd2, 0xfc, 0xa3, 0x01, 0xb3, 0x1a, + 0x07, 0x43, 0x77, 0x61, 0x86, 0xd7, 0x5d, 0xc7, 0xed, 0xa8, 0xda, 0x91, 0x72, 0x66, 0xd0, 0x84, + 0xc4, 0xe7, 0xda, 0x93, 0x92, 0x70, 0x28, 0x12, 0xed, 0x41, 0x9e, 0x12, 0xd6, 0xef, 0x06, 0x93, 0x95, 0x88, 0xfd, 0xc0, 0x0a, 0xfa, 0x4c, 0xd6, 0x66, 0x2c, 0xf8, 0xb1, 0x92, 0x63, 0xfe, 0x39, - 0x03, 0xe5, 0x9b, 0xd6, 0x01, 0xe9, 0xee, 0x93, 0x2e, 0x69, 0x07, 0x1e, 0x45, 0x3f, 0x80, 0x52, + 0x03, 0xe5, 0x9b, 0xd6, 0x01, 0xe9, 0xee, 0x93, 0x2e, 0x69, 0x07, 0x1e, 0x45, 0x3f, 0x84, 0x52, 0xcf, 0x0a, 0xda, 0x47, 0x02, 0x1a, 0x8e, 0xeb, 0xad, 0x74, 0x8a, 0x12, 0x92, 0xea, 0x3b, 0xb1, - 0x18, 0x59, 0x10, 0x9f, 0x53, 0x1f, 0x56, 0xd2, 0x30, 0x58, 0xd7, 0x26, 0xee, 0x58, 0xe2, 0x7d, - 0xf3, 0x81, 0xcf, 0x67, 0x89, 0xc9, 0xaf, 0x76, 0x09, 0x13, 0x30, 0x79, 0xb7, 0xef, 0x50, 0xd2, - 0x23, 0x6e, 0x10, 0xdf, 0xb1, 0x76, 0x86, 0xe4, 0xe3, 0x11, 0x8d, 0xd5, 0x37, 0x61, 0x61, 0xd8, - 0xf8, 0x0b, 0xea, 0xf5, 0x15, 0xbd, 0x5e, 0x17, 0xf5, 0x0a, 0xfc, 0x5b, 0x03, 0x2a, 0xe3, 0x0c, - 0x41, 0x9f, 0xd1, 0x04, 0xc5, 0x3d, 0xe2, 0x6d, 0x72, 0x2a, 0xa5, 0x6e, 0x42, 0xc1, 0xf3, 0xf9, - 0xad, 0xd8, 0xa3, 0x2a, 0xcf, 0x5f, 0x09, 0x73, 0x77, 0x57, 0xc1, 0xcf, 0x07, 0xb5, 0xe7, 0x13, - 0xe2, 0x43, 0x04, 0x8e, 0x58, 0x79, 0x63, 0x16, 0xf6, 0xf0, 0x61, 0x21, 0x6a, 0xcc, 0x77, 0x04, - 0x04, 0x2b, 0x8c, 0xf9, 0x07, 0x03, 0xa6, 0xc5, 0x94, 0x7c, 0x0f, 0x0a, 0xdc, 0x7f, 0xb6, 0x15, - 0x58, 0xc2, 0xae, 0xd4, 0xf7, 0x33, 0xce, 0xbd, 0x43, 0x02, 0x2b, 0x3e, 0x5f, 0x21, 0x04, 0x47, - 0x12, 0x11, 0x86, 0x9c, 0x13, 0x90, 0x5e, 0x18, 0xc8, 0x57, 0xc7, 0x8a, 0x56, 0xdb, 0x81, 0x3a, - 0xb6, 0xee, 0x6f, 0x3e, 0x08, 0x88, 0xcb, 0x83, 0x11, 0x17, 0x83, 0x6d, 0x2e, 0x03, 0x4b, 0x51, - 0xe6, 0xef, 0x0c, 0x88, 0x54, 0xf1, 0xe3, 0xce, 0x48, 0xf7, 0xf0, 0xa6, 0xe3, 0x1e, 0x2b, 0xb7, - 0x46, 0xe6, 0xec, 0x2b, 0x38, 0x8e, 0x28, 0x2e, 0x6a, 0x88, 0x99, 0xc9, 0x1a, 0x22, 0x57, 0xd8, - 0xf6, 0xdc, 0xc0, 0x71, 0xfb, 0x23, 0xf5, 0x65, 0x43, 0xc1, 0x71, 0x44, 0x61, 0xfe, 0x2b, 0x03, - 0x25, 0x6e, 0x6b, 0xd8, 0x91, 0xbf, 0x0c, 0xe5, 0xae, 0x1e, 0x3d, 0x65, 0xf3, 0xf3, 0x4a, 0x44, - 0xf2, 0x3c, 0xe2, 0x24, 0x2d, 0x67, 0x16, 0x63, 0x6e, 0xc4, 0x9c, 0x49, 0x32, 0x6f, 0xe9, 0x48, - 0x9c, 0xa4, 0xe5, 0x75, 0xf6, 0x3e, 0xcf, 0x6b, 0x35, 0x40, 0x46, 0xae, 0xfd, 0x26, 0x07, 0x62, - 0x89, 0xbb, 0xc8, 0x3f, 0xd3, 0x13, 0xfa, 0xe7, 0x3a, 0xcc, 0xf1, 0x40, 0x7a, 0xfd, 0x20, 0x9c, - 0xb2, 0x73, 0x62, 0xd6, 0x43, 0x67, 0x83, 0xda, 0xdc, 0xad, 0x04, 0x06, 0x0f, 0x51, 0x72, 0x1b, + 0x18, 0x59, 0x10, 0x9f, 0x51, 0x07, 0x2b, 0x69, 0x18, 0xac, 0x6b, 0x13, 0x6f, 0x2c, 0xf1, 0xbd, + 0x79, 0xdf, 0xe7, 0xb3, 0xc4, 0xe4, 0x4f, 0xbb, 0x84, 0x09, 0x98, 0xbc, 0xdb, 0x77, 0x28, 0xe9, + 0x11, 0x37, 0x88, 0xdf, 0x58, 0x3b, 0x43, 0xf2, 0xf1, 0x88, 0xc6, 0xea, 0x1b, 0xb0, 0x30, 0x6c, + 0xfc, 0x05, 0xf5, 0xfa, 0x8a, 0x5e, 0xaf, 0x8b, 0x7a, 0x05, 0xfe, 0xad, 0x01, 0x95, 0x71, 0x86, + 0xa0, 0xcf, 0x69, 0x82, 0xe2, 0x1e, 0xf1, 0x16, 0x39, 0x95, 0x52, 0x37, 0xa1, 0xe0, 0xf9, 0xfc, + 0x55, 0xec, 0x51, 0x95, 0xe7, 0x2f, 0x85, 0xb9, 0xbb, 0xab, 0xe0, 0xe7, 0x83, 0xda, 0xb3, 0x09, + 0xf1, 0x21, 0x02, 0x47, 0xac, 0xbc, 0x31, 0x0b, 0x7b, 0xf8, 0xb0, 0x10, 0x35, 0xe6, 0xdb, 0x02, + 0x82, 0x15, 0xc6, 0xfc, 0x83, 0x01, 0xd3, 0x62, 0x4a, 0xbe, 0x0b, 0x05, 0xee, 0x3f, 0xdb, 0x0a, + 0x2c, 0x61, 0x57, 0xea, 0xf7, 0x19, 0xe7, 0xde, 0x21, 0x81, 0x15, 0xdf, 0xaf, 0x10, 0x82, 0x23, + 0x89, 0x08, 0x43, 0xce, 0x09, 0x48, 0x2f, 0x0c, 0xe4, 0xcb, 0x63, 0x45, 0xab, 0xed, 0x40, 0x1d, + 0x5b, 0xf7, 0x36, 0xef, 0x07, 0xc4, 0xe5, 0xc1, 0x88, 0x8b, 0xc1, 0x36, 0x97, 0x81, 0xa5, 0x28, + 0xf3, 0x77, 0x06, 0x44, 0xaa, 0xf8, 0x75, 0x67, 0xa4, 0x7b, 0x78, 0xd3, 0x71, 0x8f, 0x95, 0x5b, + 0x23, 0x73, 0xf6, 0x15, 0x1c, 0x47, 0x14, 0x17, 0x35, 0xc4, 0xcc, 0x64, 0x0d, 0x91, 0x2b, 0x6c, + 0x7b, 0x6e, 0xe0, 0xb8, 0xfd, 0x91, 0xfa, 0xb2, 0xa1, 0xe0, 0x38, 0xa2, 0x30, 0xff, 0x95, 0x81, + 0x12, 0xb7, 0x35, 0xec, 0xc8, 0x5f, 0x85, 0x72, 0x57, 0x8f, 0x9e, 0xb2, 0xf9, 0x59, 0x25, 0x22, + 0x79, 0x1f, 0x71, 0x92, 0x96, 0x33, 0x8b, 0x31, 0x37, 0x62, 0xce, 0x24, 0x99, 0xb7, 0x74, 0x24, + 0x4e, 0xd2, 0xf2, 0x3a, 0x7b, 0x8f, 0xe7, 0xb5, 0x1a, 0x20, 0x23, 0xd7, 0x7e, 0x8b, 0x03, 0xb1, + 0xc4, 0x5d, 0xe4, 0x9f, 0xe9, 0x09, 0xfd, 0x73, 0x1d, 0xe6, 0x78, 0x20, 0xbd, 0x7e, 0x10, 0x4e, + 0xd9, 0x39, 0x31, 0xeb, 0xa1, 0xb3, 0x41, 0x6d, 0xee, 0x9d, 0x04, 0x06, 0x0f, 0x51, 0x72, 0x1b, 0xbb, 0x4e, 0xcf, 0x09, 0x2a, 0x33, 0x82, 0x25, 0xb2, 0xf1, 0x26, 0x07, 0x62, 0x89, 0x4b, 0x04, - 0xa0, 0x70, 0x69, 0x00, 0xfe, 0x91, 0x01, 0x24, 0xaf, 0x05, 0xb6, 0x9c, 0x96, 0xe4, 0x89, 0x7e, - 0x05, 0x66, 0x7a, 0xea, 0x5a, 0x61, 0x24, 0x1b, 0x4a, 0x78, 0xa3, 0x08, 0xf1, 0x68, 0x07, 0x8a, - 0xf2, 0x64, 0xc5, 0xd9, 0xd2, 0x50, 0xc4, 0xc5, 0xdd, 0x10, 0x71, 0x3e, 0xa8, 0x55, 0x13, 0x6a, - 0x22, 0xcc, 0xad, 0x53, 0x9f, 0xe0, 0x58, 0x02, 0x5a, 0x03, 0xb0, 0x7c, 0x47, 0xdf, 0x21, 0x15, - 0xe3, 0x1d, 0x44, 0x7c, 0x1b, 0xc4, 0x1a, 0x15, 0x7a, 0x0b, 0xa6, 0xb9, 0xa7, 0xd4, 0x82, 0xe1, - 0x73, 0xe9, 0xce, 0x27, 0xf7, 0x75, 0xb3, 0xc0, 0x9b, 0x16, 0x7f, 0xc2, 0x42, 0x02, 0xba, 0x0b, - 0x79, 0x91, 0x16, 0x32, 0x2a, 0x13, 0x0e, 0x9a, 0xe2, 0xd6, 0xa1, 0xa6, 0xe4, 0xf3, 0xe8, 0x09, - 0x2b, 0x89, 0xe6, 0xbb, 0x50, 0xdc, 0x71, 0xda, 0xd4, 0xe3, 0xea, 0xb8, 0x83, 0x59, 0xe2, 0x96, - 0x15, 0x39, 0x38, 0x0c, 0x7e, 0x88, 0xe7, 0x51, 0x77, 0x2d, 0xd7, 0x93, 0x77, 0xa9, 0x5c, 0x1c, - 0xf5, 0x77, 0x38, 0x10, 0x4b, 0xdc, 0xf5, 0x2b, 0xbc, 0x51, 0xff, 0xf4, 0x51, 0x6d, 0xea, 0xe1, - 0xa3, 0xda, 0xd4, 0xfb, 0x8f, 0x54, 0xd3, 0xfe, 0x5b, 0x09, 0x60, 0xf7, 0xe0, 0x7b, 0xa4, 0x2d, - 0x8b, 0xc1, 0xe5, 0x1b, 0x20, 0x3e, 0x7c, 0xa9, 0xc5, 0xa3, 0xd8, 0x96, 0x64, 0x86, 0x86, 0x2f, - 0x0d, 0x87, 0x13, 0x94, 0xa8, 0x01, 0xc5, 0x68, 0x2b, 0xa4, 0xc2, 0xb6, 0x18, 0xa6, 0x41, 0xb4, - 0x3a, 0xc2, 0x31, 0x4d, 0xa2, 0x32, 0x4d, 0x5f, 0x5a, 0x99, 0x9a, 0x90, 0xed, 0x3b, 0xb6, 0x88, - 0x4a, 0xb1, 0xf9, 0x85, 0xb0, 0x33, 0xdc, 0xde, 0x6e, 0x9d, 0x0f, 0x6a, 0x2f, 0x8e, 0x5b, 0xa9, - 0x06, 0xa7, 0x3e, 0x61, 0xf5, 0xdb, 0xdb, 0x2d, 0xcc, 0x99, 0x2f, 0x3a, 0xbd, 0xf9, 0x09, 0x4f, - 0xef, 0x1a, 0x80, 0xfa, 0x6a, 0xce, 0x2d, 0x8f, 0x61, 0x94, 0x9d, 0x37, 0x22, 0x0c, 0xd6, 0xa8, - 0x10, 0x83, 0xc5, 0x36, 0xbf, 0xdc, 0xf3, 0x64, 0x77, 0x7a, 0x84, 0x05, 0x56, 0x4f, 0xee, 0x88, - 0x26, 0x4b, 0xd5, 0x17, 0x94, 0x9a, 0xc5, 0x8d, 0x61, 0x61, 0x78, 0x54, 0x3e, 0xf2, 0x60, 0xd1, - 0x56, 0xd7, 0xd4, 0x58, 0x69, 0x71, 0x62, 0xa5, 0xcf, 0x73, 0x85, 0xad, 0x61, 0x41, 0x78, 0x54, - 0x36, 0xfa, 0x0e, 0x54, 0x43, 0xe0, 0xe8, 0xae, 0x40, 0x6c, 0xad, 0xb2, 0xcd, 0xa5, 0xb3, 0x41, - 0xad, 0xda, 0x1a, 0x4b, 0x85, 0x9f, 0x22, 0x01, 0xd9, 0x90, 0xef, 0xca, 0xb1, 0xab, 0x24, 0x5a, - 0xe5, 0x57, 0xd2, 0x7d, 0x45, 0x9c, 0xfd, 0x75, 0x7d, 0xdc, 0x8a, 0xee, 0xc2, 0x6a, 0xd2, 0x52, - 0xb2, 0xd1, 0x03, 0x28, 0x59, 0xae, 0xeb, 0x05, 0x96, 0xdc, 0x5e, 0xcc, 0x0a, 0x55, 0xeb, 0x13, - 0xab, 0x5a, 0x8f, 0x65, 0x0c, 0x8d, 0x77, 0x1a, 0x06, 0xeb, 0xaa, 0xd0, 0x7d, 0x98, 0xf7, 0xee, - 0xbb, 0x84, 0x62, 0x72, 0x48, 0x28, 0x71, 0xdb, 0x84, 0x55, 0xca, 0x42, 0xfb, 0x17, 0x53, 0x6a, - 0x4f, 0x30, 0xc7, 0x29, 0x9d, 0x84, 0x33, 0x3c, 0xac, 0x05, 0xd5, 0x01, 0x0e, 0x1d, 0x57, 0x0d, - 0xe9, 0x95, 0xb9, 0x78, 0xcd, 0xb9, 0x15, 0x41, 0xb1, 0x46, 0x81, 0x5e, 0x87, 0x52, 0xbb, 0xdb, - 0x67, 0x01, 0x91, 0xfb, 0xd4, 0x79, 0x71, 0x82, 0xa2, 0xef, 0xdb, 0x88, 0x51, 0x58, 0xa7, 0x43, - 0x47, 0x30, 0xeb, 0x68, 0xb7, 0x81, 0xca, 0x82, 0xc8, 0xc5, 0xb5, 0x89, 0xaf, 0x00, 0xac, 0xb9, + 0xa0, 0x70, 0x69, 0x00, 0xfe, 0x91, 0x01, 0x24, 0x9f, 0x05, 0xb6, 0x9c, 0x96, 0xe4, 0x8d, 0x7e, + 0x09, 0x66, 0x7a, 0xea, 0x59, 0x61, 0x24, 0x1b, 0x4a, 0xf8, 0xa2, 0x08, 0xf1, 0x68, 0x07, 0x8a, + 0xf2, 0x66, 0xc5, 0xd9, 0xd2, 0x50, 0xc4, 0xc5, 0xdd, 0x10, 0x71, 0x3e, 0xa8, 0x55, 0x13, 0x6a, + 0x22, 0xcc, 0x3b, 0xa7, 0x3e, 0xc1, 0xb1, 0x04, 0xb4, 0x06, 0x60, 0xf9, 0x8e, 0xbe, 0x43, 0x2a, + 0xc6, 0x3b, 0x88, 0xf8, 0x35, 0x88, 0x35, 0x2a, 0xf4, 0x26, 0x4c, 0x73, 0x4f, 0xa9, 0x05, 0xc3, + 0x17, 0xd2, 0xdd, 0x4f, 0xee, 0xeb, 0x66, 0x81, 0x37, 0x2d, 0xfe, 0x0b, 0x0b, 0x09, 0xe8, 0x0e, + 0xe4, 0x45, 0x5a, 0xc8, 0xa8, 0x4c, 0x38, 0x68, 0x8a, 0x57, 0x87, 0x9a, 0x92, 0xcf, 0xa3, 0x5f, + 0x58, 0x49, 0x34, 0xdf, 0x85, 0xe2, 0x8e, 0xd3, 0xa6, 0x1e, 0x57, 0xc7, 0x1d, 0xcc, 0x12, 0xaf, + 0xac, 0xc8, 0xc1, 0x61, 0xf0, 0x43, 0x3c, 0x8f, 0xba, 0x6b, 0xb9, 0x9e, 0x7c, 0x4b, 0xe5, 0xe2, + 0xa8, 0xbf, 0xcd, 0x81, 0x58, 0xe2, 0xae, 0x5f, 0xe1, 0x8d, 0xfa, 0x67, 0x0f, 0x6b, 0x53, 0x0f, + 0x1e, 0xd6, 0xa6, 0x3e, 0x78, 0xa8, 0x9a, 0xf6, 0xdf, 0x4a, 0x00, 0xbb, 0x07, 0xdf, 0x27, 0x6d, + 0x59, 0x0c, 0x2e, 0xdf, 0x00, 0xf1, 0xe1, 0x4b, 0x2d, 0x1e, 0xc5, 0xb6, 0x24, 0x33, 0x34, 0x7c, + 0x69, 0x38, 0x9c, 0xa0, 0x44, 0x0d, 0x28, 0x46, 0x5b, 0x21, 0x15, 0xb6, 0xc5, 0x30, 0x0d, 0xa2, + 0xd5, 0x11, 0x8e, 0x69, 0x12, 0x95, 0x69, 0xfa, 0xd2, 0xca, 0xd4, 0x84, 0x6c, 0xdf, 0xb1, 0x45, + 0x54, 0x8a, 0xcd, 0x2f, 0x85, 0x9d, 0xe1, 0xd6, 0x76, 0xeb, 0x7c, 0x50, 0x7b, 0x7e, 0xdc, 0x4a, + 0x35, 0x38, 0xf5, 0x09, 0xab, 0xdf, 0xda, 0x6e, 0x61, 0xce, 0x7c, 0xd1, 0xed, 0xcd, 0x4f, 0x78, + 0x7b, 0xd7, 0x00, 0xd4, 0xa9, 0x39, 0xb7, 0xbc, 0x86, 0x51, 0x76, 0xde, 0x88, 0x30, 0x58, 0xa3, + 0x42, 0x0c, 0x16, 0xdb, 0xfc, 0x71, 0xcf, 0x93, 0xdd, 0xe9, 0x11, 0x16, 0x58, 0x3d, 0xb9, 0x23, + 0x9a, 0x2c, 0x55, 0x9f, 0x53, 0x6a, 0x16, 0x37, 0x86, 0x85, 0xe1, 0x51, 0xf9, 0xc8, 0x83, 0x45, + 0x5b, 0x3d, 0x53, 0x63, 0xa5, 0xc5, 0x89, 0x95, 0x3e, 0xcb, 0x15, 0xb6, 0x86, 0x05, 0xe1, 0x51, + 0xd9, 0xe8, 0xbb, 0x50, 0x0d, 0x81, 0xa3, 0xbb, 0x02, 0xb1, 0xb5, 0xca, 0x36, 0x97, 0xce, 0x06, + 0xb5, 0x6a, 0x6b, 0x2c, 0x15, 0x7e, 0x82, 0x04, 0x64, 0x43, 0xbe, 0x2b, 0xc7, 0xae, 0x92, 0x68, + 0x95, 0x5f, 0x4b, 0x77, 0x8a, 0x38, 0xfb, 0xeb, 0xfa, 0xb8, 0x15, 0xbd, 0x85, 0xd5, 0xa4, 0xa5, + 0x64, 0xa3, 0xfb, 0x50, 0xb2, 0x5c, 0xd7, 0x0b, 0x2c, 0xb9, 0xbd, 0x98, 0x15, 0xaa, 0xd6, 0x27, + 0x56, 0xb5, 0x1e, 0xcb, 0x18, 0x1a, 0xef, 0x34, 0x0c, 0xd6, 0x55, 0xa1, 0x7b, 0x30, 0xef, 0xdd, + 0x73, 0x09, 0xc5, 0xe4, 0x90, 0x50, 0xe2, 0xb6, 0x09, 0xab, 0x94, 0x85, 0xf6, 0x2f, 0xa7, 0xd4, + 0x9e, 0x60, 0x8e, 0x53, 0x3a, 0x09, 0x67, 0x78, 0x58, 0x0b, 0xaa, 0x03, 0x1c, 0x3a, 0xae, 0x1a, + 0xd2, 0x2b, 0x73, 0xf1, 0x9a, 0x73, 0x2b, 0x82, 0x62, 0x8d, 0x02, 0xbd, 0x0a, 0xa5, 0x76, 0xb7, + 0xcf, 0x02, 0x22, 0xf7, 0xa9, 0xf3, 0xe2, 0x06, 0x45, 0xe7, 0xdb, 0x88, 0x51, 0x58, 0xa7, 0x43, + 0x47, 0x30, 0xeb, 0x68, 0xaf, 0x81, 0xca, 0x82, 0xc8, 0xc5, 0xb5, 0x89, 0x9f, 0x00, 0xac, 0xb9, 0xc0, 0x2b, 0x91, 0x0e, 0xc1, 0x09, 0xc9, 0xa8, 0x0f, 0xe5, 0x9e, 0xde, 0x6a, 0x2a, 0x8b, 0xc2, - 0x8f, 0xd7, 0xd2, 0xa9, 0x1a, 0x6d, 0x86, 0xf1, 0x00, 0x91, 0xc0, 0xe1, 0xa4, 0x96, 0xea, 0x97, - 0xa0, 0xf4, 0x1f, 0xce, 0xc4, 0x7c, 0xa6, 0x1e, 0xce, 0x98, 0x89, 0x66, 0xea, 0x0f, 0x32, 0x30, - 0x97, 0x8c, 0x73, 0x74, 0xf7, 0x34, 0xc6, 0xae, 0xe5, 0xc3, 0x66, 0x90, 0x1d, 0xdb, 0x0c, 0x54, - 0xcd, 0x9d, 0x7e, 0x96, 0x9a, 0x9b, 0x6c, 0xe7, 0xb9, 0x54, 0xed, 0xbc, 0x0e, 0xc0, 0xe7, 0x13, + 0x8f, 0xd7, 0xd2, 0xa9, 0x1a, 0x6d, 0x86, 0xf1, 0x00, 0x91, 0xc0, 0xe1, 0xa4, 0x96, 0xea, 0x57, + 0xa0, 0xf4, 0x1f, 0xce, 0xc4, 0x7c, 0xa6, 0x1e, 0xce, 0x98, 0x89, 0x66, 0xea, 0x0f, 0x33, 0x30, + 0x97, 0x8c, 0x73, 0xf4, 0xf6, 0x34, 0xc6, 0xae, 0xe5, 0xc3, 0x66, 0x90, 0x1d, 0xdb, 0x0c, 0x54, + 0xcd, 0x9d, 0x7e, 0x9a, 0x9a, 0x9b, 0x6c, 0xe7, 0xb9, 0x54, 0xed, 0xbc, 0x0e, 0xc0, 0xe7, 0x13, 0xea, 0x75, 0xbb, 0x84, 0x8a, 0x12, 0x5d, 0x50, 0x8b, 0xf7, 0x08, 0x8a, 0x35, 0x0a, 0xb4, 0x05, 0xe8, 0xa0, 0xeb, 0xb5, 0x8f, 0x85, 0x0b, 0xc2, 0xf2, 0x22, 0x8a, 0x73, 0x41, 0x2e, 0x2f, 0x9b, - 0x23, 0x58, 0x7c, 0x01, 0x87, 0x39, 0x03, 0xb9, 0x3d, 0x3e, 0xe6, 0x99, 0xbf, 0x30, 0x60, 0x56, - 0x3c, 0x4d, 0xb2, 0x8e, 0xad, 0x41, 0xee, 0xd0, 0x0b, 0x57, 0x2e, 0x05, 0xf9, 0xcf, 0xc5, 0x16, - 0x07, 0x60, 0x09, 0x7f, 0x86, 0x7d, 0xed, 0x2e, 0x24, 0xf7, 0xa0, 0xe8, 0x4d, 0x19, 0x19, 0x23, - 0x5a, 0x54, 0x4e, 0x16, 0x15, 0xf3, 0x2a, 0x14, 0xb1, 0xe7, 0x05, 0x7b, 0x56, 0x70, 0xc4, 0xb8, - 0xe1, 0x3e, 0x7f, 0x50, 0xdf, 0x26, 0x0c, 0x17, 0x18, 0x2c, 0xe1, 0xe6, 0xcf, 0x0d, 0x78, 0x61, - 0xec, 0x8a, 0x9b, 0x47, 0xb8, 0x1d, 0xbd, 0x29, 0x93, 0xa2, 0x08, 0xc7, 0x74, 0x58, 0xa3, 0xe2, - 0x93, 0x7a, 0x62, 0x2f, 0x3e, 0x3c, 0xa9, 0x27, 0xb4, 0xe1, 0x24, 0xad, 0xf9, 0xcf, 0x0c, 0xe4, - 0xe5, 0xb5, 0xfd, 0xbf, 0x7c, 0x39, 0x7b, 0x19, 0xf2, 0x4c, 0xe8, 0x51, 0xe6, 0x45, 0x4d, 0x43, - 0x6a, 0xc7, 0x0a, 0x2b, 0x86, 0x65, 0xc2, 0x98, 0xd5, 0x09, 0x0f, 0x53, 0x3c, 0x2c, 0x4b, 0x30, - 0x0e, 0xf1, 0xe8, 0x0d, 0xc8, 0x53, 0x62, 0xb1, 0xe8, 0xde, 0xb0, 0x14, 0x8a, 0xc4, 0x02, 0x7a, - 0x3e, 0xa8, 0xcd, 0x2a, 0xe1, 0xe2, 0x1d, 0x2b, 0x6a, 0x74, 0x17, 0x66, 0x6c, 0x12, 0x58, 0x4e, - 0x37, 0x1c, 0x4c, 0x5f, 0x9b, 0x64, 0xbd, 0xd1, 0x92, 0xac, 0xcd, 0x12, 0xb7, 0x49, 0xbd, 0xe0, - 0x50, 0x20, 0x2f, 0x04, 0x6d, 0xcf, 0x96, 0xff, 0x8c, 0xe5, 0xe2, 0x42, 0xb0, 0xe1, 0xd9, 0x04, - 0x0b, 0x8c, 0xf9, 0xd0, 0x80, 0x92, 0x94, 0xb4, 0x61, 0xf5, 0x19, 0x41, 0xab, 0xd1, 0x57, 0xc8, - 0x70, 0x87, 0xa3, 0xc9, 0x34, 0x1f, 0xe6, 0xcf, 0x07, 0xb5, 0xa2, 0x20, 0x13, 0x93, 0x7d, 0xf8, - 0x01, 0x9a, 0x8f, 0x32, 0x97, 0xf8, 0xe8, 0x25, 0xc8, 0x89, 0xec, 0x57, 0xce, 0x8c, 0xe6, 0x5d, - 0x71, 0x40, 0xb0, 0xc4, 0x99, 0x1f, 0x67, 0xa0, 0x9c, 0xf8, 0xb8, 0x14, 0xc3, 0x6d, 0xb4, 0x4a, - 0xcb, 0xa4, 0x58, 0xcf, 0x8e, 0xff, 0x3f, 0xf3, 0x5b, 0x90, 0x6f, 0xf3, 0xef, 0x0b, 0xff, 0x50, - 0x5e, 0x9d, 0x24, 0x14, 0xc2, 0x33, 0x71, 0x26, 0x89, 0x57, 0x86, 0x95, 0x40, 0x74, 0x03, 0x16, - 0x29, 0x09, 0xe8, 0xe9, 0xfa, 0x61, 0x40, 0xa8, 0x7e, 0x3f, 0xcc, 0xc5, 0xe3, 0x1f, 0x1e, 0x26, - 0xc0, 0xa3, 0x3c, 0x61, 0xe9, 0xce, 0x3f, 0x43, 0xe9, 0x36, 0xbb, 0x30, 0xfd, 0x3f, 0xbc, 0xaa, - 0x7c, 0x1b, 0x8a, 0xf1, 0x30, 0xf9, 0x09, 0xab, 0x34, 0xbf, 0x0b, 0x05, 0x9e, 0x8d, 0xe1, 0x25, - 0xe8, 0x92, 0xce, 0x98, 0xec, 0x59, 0x99, 0x34, 0x3d, 0xcb, 0xec, 0x41, 0xf9, 0xb6, 0x6f, 0x3f, - 0xe3, 0x3f, 0x78, 0x99, 0xd4, 0x1d, 0x61, 0x0d, 0xe4, 0xbf, 0xe2, 0xbc, 0x78, 0xcb, 0x05, 0x92, - 0x56, 0xbc, 0xf5, 0x6d, 0x90, 0xb6, 0xc1, 0xfd, 0xb1, 0x01, 0x20, 0xb6, 0x19, 0x9b, 0x27, 0xc4, - 0x0d, 0xb8, 0x1f, 0x78, 0xc0, 0x87, 0xfd, 0x20, 0x4e, 0xad, 0xc0, 0xa0, 0xdb, 0x90, 0xf7, 0xc4, - 0x4c, 0xab, 0x56, 0xaa, 0x13, 0x6e, 0xa7, 0xa2, 0x24, 0x97, 0x83, 0x31, 0x56, 0xc2, 0x9a, 0x2b, - 0x8f, 0x9f, 0x2c, 0x4d, 0x7d, 0xf8, 0x64, 0x69, 0xea, 0xa3, 0x27, 0x4b, 0x53, 0xef, 0x9d, 0x2d, - 0x19, 0x8f, 0xcf, 0x96, 0x8c, 0x0f, 0xcf, 0x96, 0x8c, 0x8f, 0xce, 0x96, 0x8c, 0x8f, 0xcf, 0x96, - 0x8c, 0x87, 0x7f, 0x5d, 0x9a, 0xba, 0x9b, 0x39, 0x59, 0xfd, 0x77, 0x00, 0x00, 0x00, 0xff, 0xff, - 0x1a, 0xd0, 0x3c, 0x48, 0x01, 0x24, 0x00, 0x00, + 0x23, 0x58, 0x7c, 0x01, 0x87, 0x39, 0x03, 0xb9, 0x3d, 0x3e, 0xe6, 0x99, 0xbf, 0x34, 0x60, 0x56, + 0xfc, 0x9a, 0x64, 0x1d, 0x5b, 0x83, 0xdc, 0xa1, 0x17, 0xae, 0x5c, 0x0a, 0xf2, 0x3f, 0x17, 0x5b, + 0x1c, 0x80, 0x25, 0xfc, 0x29, 0xf6, 0xb5, 0xef, 0x1b, 0x90, 0x5c, 0x84, 0xa2, 0x37, 0x64, 0x68, + 0x8c, 0x68, 0x53, 0x39, 0x61, 0x58, 0x5e, 0x1f, 0x37, 0xe8, 0x3f, 0x93, 0x6a, 0xeb, 0x75, 0x15, + 0x8a, 0xd8, 0xf3, 0x82, 0x3d, 0x2b, 0x38, 0x62, 0xfc, 0xe0, 0x3e, 0xff, 0xa1, 0x7c, 0x23, 0x0e, + 0x2e, 0x30, 0x58, 0xc2, 0xcd, 0x5f, 0x18, 0xf0, 0xdc, 0xd8, 0x15, 0x39, 0xcf, 0x90, 0x76, 0xf4, + 0xa5, 0x4e, 0x14, 0x65, 0x48, 0x4c, 0x87, 0x35, 0x2a, 0x3e, 0xe9, 0x27, 0xf6, 0xea, 0xc3, 0x93, + 0x7e, 0x42, 0x1b, 0x4e, 0xd2, 0x9a, 0xff, 0xcc, 0x40, 0x5e, 0x3e, 0xfb, 0xff, 0xcb, 0x8f, 0xbb, + 0x17, 0x21, 0xcf, 0x84, 0x1e, 0x65, 0x5e, 0xd4, 0x74, 0xa4, 0x76, 0xac, 0xb0, 0x62, 0xd8, 0x26, + 0x8c, 0x59, 0x9d, 0xf0, 0x32, 0xc6, 0xc3, 0xb6, 0x04, 0xe3, 0x10, 0x8f, 0x5e, 0x83, 0x3c, 0x25, + 0x16, 0x8b, 0xde, 0x1d, 0x4b, 0xa1, 0x48, 0x2c, 0xa0, 0xe7, 0x83, 0xda, 0xac, 0x12, 0x2e, 0xbe, + 0xb1, 0xa2, 0x46, 0x77, 0x60, 0xc6, 0x26, 0x81, 0xe5, 0x74, 0xc3, 0xc1, 0xf6, 0x95, 0x49, 0xd6, + 0x23, 0x2d, 0xc9, 0xda, 0x2c, 0x71, 0x9b, 0xd4, 0x07, 0x0e, 0x05, 0xf2, 0x42, 0xd2, 0xf6, 0x6c, + 0xf9, 0x9f, 0xb5, 0x5c, 0x5c, 0x48, 0x36, 0x3c, 0x9b, 0x60, 0x81, 0x31, 0x1f, 0x18, 0x50, 0x92, + 0x92, 0x36, 0xac, 0x3e, 0x23, 0x68, 0x35, 0x3a, 0x85, 0x0c, 0x77, 0x38, 0xda, 0x4c, 0xf3, 0xc7, + 0xc0, 0xf9, 0xa0, 0x56, 0x14, 0x64, 0xe2, 0x65, 0x10, 0x1e, 0x40, 0xf3, 0x51, 0xe6, 0x12, 0x1f, + 0xbd, 0x00, 0x39, 0x71, 0x7b, 0x94, 0x33, 0xa3, 0x79, 0x59, 0x5c, 0x30, 0x2c, 0x71, 0xe6, 0x27, + 0x19, 0x28, 0x27, 0x0e, 0x97, 0x62, 0x38, 0x8e, 0x56, 0x71, 0x99, 0x14, 0xeb, 0xdd, 0xf1, 0xff, + 0x0f, 0xfd, 0x36, 0xe4, 0xdb, 0xfc, 0x7c, 0xe1, 0x3f, 0xa4, 0x57, 0x27, 0x09, 0x85, 0xf0, 0x4c, + 0x9c, 0x49, 0xe2, 0x93, 0x61, 0x25, 0x10, 0xdd, 0x80, 0x45, 0x4a, 0x02, 0x7a, 0xba, 0x7e, 0x18, + 0x10, 0xaa, 0xbf, 0x2f, 0x73, 0xf1, 0xf8, 0x88, 0x87, 0x09, 0xf0, 0x28, 0x4f, 0x58, 0xfa, 0xf3, + 0x4f, 0x51, 0xfa, 0xcd, 0x2e, 0x4c, 0xff, 0x0f, 0x9f, 0x3a, 0xdf, 0x81, 0x62, 0x3c, 0x8c, 0x7e, + 0xca, 0x2a, 0xcd, 0xef, 0x41, 0x81, 0x67, 0x63, 0xf8, 0x88, 0xba, 0xa4, 0xb3, 0x26, 0x7b, 0x5e, + 0x26, 0x4d, 0xcf, 0x33, 0x7b, 0x50, 0xbe, 0xe5, 0xdb, 0x4f, 0xf9, 0x1f, 0xc0, 0x4c, 0xea, 0x8e, + 0xb2, 0x06, 0xf2, 0xbf, 0xea, 0xbc, 0x78, 0xcb, 0x05, 0x94, 0x56, 0xbc, 0xf5, 0x6d, 0x92, 0xb6, + 0x01, 0xfe, 0x89, 0x01, 0x20, 0xb6, 0x21, 0x9b, 0x27, 0xc4, 0x0d, 0xb8, 0x1f, 0x78, 0xc0, 0x87, + 0xfd, 0x20, 0x6e, 0xad, 0xc0, 0xa0, 0x5b, 0x90, 0xf7, 0xc4, 0x4c, 0xac, 0x56, 0xb2, 0x13, 0x6e, + 0xb7, 0xa2, 0x24, 0x97, 0x83, 0x35, 0x56, 0xc2, 0x9a, 0x2b, 0x8f, 0x1e, 0x2f, 0x4d, 0x7d, 0xf4, + 0x78, 0x69, 0xea, 0xe3, 0xc7, 0x4b, 0x53, 0xef, 0x9d, 0x2d, 0x19, 0x8f, 0xce, 0x96, 0x8c, 0x8f, + 0xce, 0x96, 0x8c, 0x8f, 0xcf, 0x96, 0x8c, 0x4f, 0xce, 0x96, 0x8c, 0x07, 0x7f, 0x5d, 0x9a, 0xba, + 0x93, 0x39, 0x59, 0xfd, 0x77, 0x00, 0x00, 0x00, 0xff, 0xff, 0xc3, 0x66, 0x55, 0x2c, 0x41, 0x24, + 0x00, 0x00, } diff --git a/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/generated.proto b/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/generated.proto index d07903ca77..36bda1fe59 100644 --- a/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/generated.proto +++ b/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/generated.proto @@ -754,6 +754,10 @@ message Preconditions { // Specifies the target UID. // +optional optional string uid = 1; + + // Specifies the target ResourceVersion + // +optional + optional string resourceVersion = 2; } // RootPaths lists the paths available at root. From ac56bd502ab96696682c66ebdff94b6e52471aa3 Mon Sep 17 00:00:00 2001 From: Anago GCB Date: Tue, 12 Mar 2019 18:44:37 +0000 Subject: [PATCH 20/21] Update CHANGELOG-1.14.md for v1.14.0-beta.2. --- CHANGELOG-1.14.md | 197 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 185 insertions(+), 12 deletions(-) diff --git a/CHANGELOG-1.14.md b/CHANGELOG-1.14.md index 90c81af905..997e379e38 100644 --- a/CHANGELOG-1.14.md +++ b/CHANGELOG-1.14.md @@ -1,41 +1,214 @@ -- [v1.14.0-beta.1](#v1140-beta1) - - [Downloads for v1.14.0-beta.1](#downloads-for-v1140-beta1) +- [v1.14.0-beta.2](#v1140-beta2) + - [Downloads for v1.14.0-beta.2](#downloads-for-v1140-beta2) - [Client Binaries](#client-binaries) - [Server Binaries](#server-binaries) - [Node Binaries](#node-binaries) - - [Changelog since v1.14.0-alpha.3](#changelog-since-v1140-alpha3) + - [Changelog since v1.14.0-beta.1](#changelog-since-v1140-beta1) - [Action Required](#action-required) - [Other notable changes](#other-notable-changes) -- [v1.14.0-alpha.3](#v1140-alpha3) - - [Downloads for v1.14.0-alpha.3](#downloads-for-v1140-alpha3) +- [v1.14.0-beta.1](#v1140-beta1) + - [Downloads for v1.14.0-beta.1](#downloads-for-v1140-beta1) - [Client Binaries](#client-binaries-1) - [Server Binaries](#server-binaries-1) - [Node Binaries](#node-binaries-1) - - [Changelog since v1.14.0-alpha.2](#changelog-since-v1140-alpha2) + - [Changelog since v1.14.0-alpha.3](#changelog-since-v1140-alpha3) - [Action Required](#action-required-1) - [Other notable changes](#other-notable-changes-1) -- [v1.14.0-alpha.2](#v1140-alpha2) - - [Downloads for v1.14.0-alpha.2](#downloads-for-v1140-alpha2) +- [v1.14.0-alpha.3](#v1140-alpha3) + - [Downloads for v1.14.0-alpha.3](#downloads-for-v1140-alpha3) - [Client Binaries](#client-binaries-2) - [Server Binaries](#server-binaries-2) - [Node Binaries](#node-binaries-2) - - [Changelog since v1.14.0-alpha.1](#changelog-since-v1140-alpha1) + - [Changelog since v1.14.0-alpha.2](#changelog-since-v1140-alpha2) - [Action Required](#action-required-2) - [Other notable changes](#other-notable-changes-2) -- [v1.14.0-alpha.1](#v1140-alpha1) - - [Downloads for v1.14.0-alpha.1](#downloads-for-v1140-alpha1) +- [v1.14.0-alpha.2](#v1140-alpha2) + - [Downloads for v1.14.0-alpha.2](#downloads-for-v1140-alpha2) - [Client Binaries](#client-binaries-3) - [Server Binaries](#server-binaries-3) - [Node Binaries](#node-binaries-3) - - [Changelog since v1.13.0](#changelog-since-v1130) + - [Changelog since v1.14.0-alpha.1](#changelog-since-v1140-alpha1) - [Action Required](#action-required-3) - [Other notable changes](#other-notable-changes-3) +- [v1.14.0-alpha.1](#v1140-alpha1) + - [Downloads for v1.14.0-alpha.1](#downloads-for-v1140-alpha1) + - [Client Binaries](#client-binaries-4) + - [Server Binaries](#server-binaries-4) + - [Node Binaries](#node-binaries-4) + - [Changelog since v1.13.0](#changelog-since-v1130) + - [Action Required](#action-required-4) + - [Other notable changes](#other-notable-changes-4) +# v1.14.0-beta.2 + +[Documentation](https://docs.k8s.io) + +## Downloads for v1.14.0-beta.2 + + +filename | sha512 hash +-------- | ----------- +[kubernetes.tar.gz](https://dl.k8s.io/v1.14.0-beta.2/kubernetes.tar.gz) | `c1d5f2615c3319fc167c577f40f385abe6652bf4fd3bdb04617b36029dc3000b190c18b4b3a29827da75c680979697d61fffb45b86ba6226f880b98b2f308f4f` +[kubernetes-src.tar.gz](https://dl.k8s.io/v1.14.0-beta.2/kubernetes-src.tar.gz) | `0a8d8ed208bc0bf424060126c76fcd8dbbd53a9b9695647314a4097f7013f548b76850438933760ff76835867676cddddf65134ad79f977ecdb98632fc2edda3` + +### Client Binaries + +filename | sha512 hash +-------- | ----------- +[kubernetes-client-darwin-386.tar.gz](https://dl.k8s.io/v1.14.0-beta.2/kubernetes-client-darwin-386.tar.gz) | `c919d030255c5d3879926d8aaa53939cd5aa37084799748452166ca6668bd1d10edf063d633682cddafaaed43dd1b991f4ad09139c5e4f519bd69f581b3fe0aa` +[kubernetes-client-darwin-amd64.tar.gz](https://dl.k8s.io/v1.14.0-beta.2/kubernetes-client-darwin-amd64.tar.gz) | `ec14d4a1d720890065211544b099be17315265534cfd20435194dc842cc807c20b5fae78f5b95ba7d05f3d921d522017f50861760d195ce1bf5b1acfdb2dbb29` +[kubernetes-client-linux-386.tar.gz](https://dl.k8s.io/v1.14.0-beta.2/kubernetes-client-linux-386.tar.gz) | `6cee12be5b855600ee80f15d1e0511099941b099bd5b252549abdc2a65c077f10ca4d53ab9804a0ce8d51f3b9cbab829cb551733cd4aed37c0d91238b82a8fe4` +[kubernetes-client-linux-amd64.tar.gz](https://dl.k8s.io/v1.14.0-beta.2/kubernetes-client-linux-amd64.tar.gz) | `27d8cd48c1f3259055965b85a6b973ecb5b8a36894f94c232d773f89539e28e6c270bfe35427c70b4ad4800e42c869851981cf88f586b7d488efa538e6c88126` +[kubernetes-client-linux-arm.tar.gz](https://dl.k8s.io/v1.14.0-beta.2/kubernetes-client-linux-arm.tar.gz) | `7f98230569c61fdf2b141e519f042b2e27ff37aeda746dc30bb7ce226b5d6b0c0bb85c6070b9ffc8d38c2441feb5bd8736c67708a59552e86a2c30cc02ecdece` +[kubernetes-client-linux-arm64.tar.gz](https://dl.k8s.io/v1.14.0-beta.2/kubernetes-client-linux-arm64.tar.gz) | `159da67010af38d87c5318b7ad594120afd6a9b780d11d6e607e7214862cd6514b00da673cce72574771dbc780ab435dbba0a3267f051a20155c05ee0729ded0` +[kubernetes-client-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.14.0-beta.2/kubernetes-client-linux-ppc64le.tar.gz) | `d8ca7871d3d40947db69061284cb31c4d072d4da56fbb11a4485f6853f041835d9605cfc5dcea88d58c7f484cce13dbca485e80891c845291b9b28c574df310b` +[kubernetes-client-linux-s390x.tar.gz](https://dl.k8s.io/v1.14.0-beta.2/kubernetes-client-linux-s390x.tar.gz) | `1c58db90b6e09b8d8f956a00263cb20271b8403f7fb6c5b20d76cca9ed973c35d2f5c910a6d42980ec9159480682d3786a59e9e05ce356a7e3b4181c848ad122` +[kubernetes-client-windows-386.tar.gz](https://dl.k8s.io/v1.14.0-beta.2/kubernetes-client-windows-386.tar.gz) | `00fb87dd4899208dd6607c22828f3985ebfd5e1f97cb24e3b2c69c249a4887d5c26c603b3bb4c21f9e2b737c917ddf95a1818d9de5c9ec97d3f5faa0c3dada52` +[kubernetes-client-windows-amd64.tar.gz](https://dl.k8s.io/v1.14.0-beta.2/kubernetes-client-windows-amd64.tar.gz) | `7afdf637d62dde480162ad1521360b2bc78e0d4d20f6e6201e2f19b55b8e9bbd69c1ce8d03101c750ef389c65a1bc0a94dfc9a2d501d6840fd31eacbd3582028` + +### Server Binaries + +filename | sha512 hash +-------- | ----------- +[kubernetes-server-linux-amd64.tar.gz](https://dl.k8s.io/v1.14.0-beta.2/kubernetes-server-linux-amd64.tar.gz) | `2ac3c4910cd36f02a62304d78fe144b821edf445c522028e6b57d2dc3bcc7355159a58815d5a6991b3f2c33bb0ef23e07134c8bbf93b34be7452f80c9a9e6edf` +[kubernetes-server-linux-arm.tar.gz](https://dl.k8s.io/v1.14.0-beta.2/kubernetes-server-linux-arm.tar.gz) | `db06b5f1a83ca4ec82428ab771eae2858b188dc23780fb9b146494c06aa6175421090b200c58b670e2d4253a7e0d4b07172a632e0754c35ccdee7e264c636f17` +[kubernetes-server-linux-arm64.tar.gz](https://dl.k8s.io/v1.14.0-beta.2/kubernetes-server-linux-arm64.tar.gz) | `b40e1745d1ecfcc95f3a750990244fa128381d6d74246798a62aecb8cec9c77cdcd470e79334eb5c670e1e3a288080b4e26a080c64481ba608e3156c72df474b` +[kubernetes-server-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.14.0-beta.2/kubernetes-server-linux-ppc64le.tar.gz) | `c84297fd2b18b6bdade5a135a3da929e286bbba5c8dd66778091bad4eea1ac4b97a32ab3b146a88f0716bebcce9a4a85a7cd421cb185a3df864dcfa77312b3a9` +[kubernetes-server-linux-s390x.tar.gz](https://dl.k8s.io/v1.14.0-beta.2/kubernetes-server-linux-s390x.tar.gz) | `39c8f6e7f52bec155b11652b4e80c2c52acf8754dbdf80a9d5bab5370d1debf4f4783c1a6968d41822b00ff744c72947df6cbc4623578e7679b9ce9a98f64ff3` + +### Node Binaries + +filename | sha512 hash +-------- | ----------- +[kubernetes-node-linux-amd64.tar.gz](https://dl.k8s.io/v1.14.0-beta.2/kubernetes-node-linux-amd64.tar.gz) | `bf19ae7140836aea1b6f414532eff886e3b91e0746b9224ce46e60e0b83fa90a8c3df1ff8e01ff340a1e1874ce15da28e98224024ed3139589474e89befa19f2` +[kubernetes-node-linux-arm.tar.gz](https://dl.k8s.io/v1.14.0-beta.2/kubernetes-node-linux-arm.tar.gz) | `4b8194340a8675107da3969845173fb34ed2b0a38745ec0ac395ebf2116ee84d55be6e22ff84fafbaf4ca60a05f6debf6e95957a2261ac8a587eea32e5803fd5` +[kubernetes-node-linux-arm64.tar.gz](https://dl.k8s.io/v1.14.0-beta.2/kubernetes-node-linux-arm64.tar.gz) | `8494dae5225f3b543afd575003fe0f30eb0f3cf9bc9dfbae72d6bda8f17c5446165433c28842a114af66ac3ae8fca9f92d780d1eb93e9bcb6b5dcc4fe8cd2a7f` +[kubernetes-node-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.14.0-beta.2/kubernetes-node-linux-ppc64le.tar.gz) | `0bd41d95f0a76c1b057a8913a8b2cffbf6d48c47aef1d9beed0de205b8010e8071e8f527eeaa003730ef97a017083278cb2036cf22a1abfb2f4669b935823cdc` +[kubernetes-node-linux-s390x.tar.gz](https://dl.k8s.io/v1.14.0-beta.2/kubernetes-node-linux-s390x.tar.gz) | `169b16512df42a6cb5b000a3d6d6da5ae48a733c5d11b034eaec6b3816b86ec97b92e4075872900188bc296427037299841224e552ccf079097d5cf333627cbc` +[kubernetes-node-windows-amd64.tar.gz](https://dl.k8s.io/v1.14.0-beta.2/kubernetes-node-windows-amd64.tar.gz) | `ccfebde5176cb28529552889250bf706add04df3c3f9aee5b8bdd9ebdb5bce334add8351cdbeebb9bd4b17b31d994b7739d2c494aa4c64bcb3ceba1a6ef53447` + +## Changelog since v1.14.0-beta.1 + +### Action Required + +* ACTION REQUIRED: The node.k8s.io API group and runtimeclasses.node.k8s.io resource have been migrated to a built-in API. If you were using RuntimeClasses (an default-disabled alpha feature, as of Kubernetes v1.12), then you must recreate all RuntimeClasses after upgrading, and the runtimeclasses.node.k8s.io CRD should be manually deleted. RuntimeClasses can no longer be created without a defined handler. ([#74433](https://github.com/kubernetes/kubernetes/pull/74433), [@tallclair](https://github.com/tallclair)) + * Introduce a RuntimeClass v1beta1 API. This new beta API renames `runtimeHandler` to `handler`, makes it a required field, and cuts out the spec (handler is a top-level field). +* Transition CSINodeInfo and CSIDriver alpha CRDs to in-tree CSINode and CSIDriver core storage v1beta1 APIs. ([#74283](https://github.com/kubernetes/kubernetes/pull/74283), [@xing-yang](https://github.com/xing-yang)) + * ACTION REQUIRED: the alpha CRDs are no longer used and drivers will need to be updated to use the beta APIs. + * The support for `_` in the CSI driver name will be dropped as the CSI Spec does not allow that. + +### Other notable changes + +* Support collecting pod logs under /var/log/pods/NAMESPACE_NAME_UID to stackdriver with `k8s_pod` resource type. ([#74502](https://github.com/kubernetes/kubernetes/pull/74502), [@Random-Liu](https://github.com/Random-Liu)) +* --make-symlinks for hyperkube was marked hidden for a while, This flag is now deprecated and will be removed in a future release. ([#74975](https://github.com/kubernetes/kubernetes/pull/74975), [@dims](https://github.com/dims)) +* fix smb unmount issue on Windows ([#75087](https://github.com/kubernetes/kubernetes/pull/75087), [@andyzhangx](https://github.com/andyzhangx)) +* Kubelet no longer watches configmaps and secrets for terminated pods, in worst scenario causing it to not be able to send other requests to kube-apiserver ([#74809](https://github.com/kubernetes/kubernetes/pull/74809), [@oxddr](https://github.com/oxddr)) +* - Fixes a bug concerning Quobyte volumes where user mappings only worked if the hosts Kubernetes plugin mount was provided via an external configuration using the _allow-usermapping-in-volumename_ option. ([#74520](https://github.com/kubernetes/kubernetes/pull/74520), [@casusbelli](https://github.com/casusbelli)) +* Change CRI pod log directory from `/var/log/pods/UID` to `/var/log/pods/NAMESPACE_NAME_UID`. ([#74441](https://github.com/kubernetes/kubernetes/pull/74441), [@Random-Liu](https://github.com/Random-Liu)) + * It is recommended to drain the node before upgrade, or reboot the node after upgrade. +* Promote RuntimeClass to beta, and enable by default. ([#75003](https://github.com/kubernetes/kubernetes/pull/75003), [@tallclair](https://github.com/tallclair)) +* New "dry_run" metric label (indicating the value of the dryRun query parameter) into the metrics: ([#74997](https://github.com/kubernetes/kubernetes/pull/74997), [@jennybuckley](https://github.com/jennybuckley)) + * apiserver_request_total + * apiserver_request_duration_seconds + * New "APPLY" value for the "verb" metric label which indicates a PATCH with "Content-Type: apply-patch+yaml". This value is experimental and will only be present if the ServerSideApply alpha feature is enabled. +* GCE: bump COS image version to cos-beta-73-11647-64-0 ([#75149](https://github.com/kubernetes/kubernetes/pull/75149), [@yguo0905](https://github.com/yguo0905)) +* - Add duration metric for CRD webhook converters ([#74376](https://github.com/kubernetes/kubernetes/pull/74376), [@mbohlool](https://github.com/mbohlool)) +* Alpha support for ephemeral CSI inline volumes that are embedded in pod specs. ([#74086](https://github.com/kubernetes/kubernetes/pull/74086), [@vladimirvivien](https://github.com/vladimirvivien)) +* Add support for node side CSI volume expansion ([#74863](https://github.com/kubernetes/kubernetes/pull/74863), [@gnufied](https://github.com/gnufied)) +* - Add mechanism for Admission Webhooks to specify which version of AdmissionReview they support ([#74998](https://github.com/kubernetes/kubernetes/pull/74998), [@mbohlool](https://github.com/mbohlool)) + * - Add mechanism for CRD Conversion Webhooks to specify which version of ConversionReview they support +* Add a new kubelet endpoint for serving first-class resource metrics ([#73946](https://github.com/kubernetes/kubernetes/pull/73946), [@dashpole](https://github.com/dashpole)) +* Deprecate AWS, Azure, GCE and Cinder specific volume limit predicates. ([#74544](https://github.com/kubernetes/kubernetes/pull/74544), [@gnufied](https://github.com/gnufied)) +* PodReadinessGate feature is now GA. The feature gate will not allow disabling it. ([#74434](https://github.com/kubernetes/kubernetes/pull/74434), [@freehan](https://github.com/freehan)) +* If CSINodeInfo and CSIMigration feature flags are active in the cluster, Kubelet will post NotReady until CSINode is initialized with basic volume plugin mechanism information for well-known drivers ([#74835](https://github.com/kubernetes/kubernetes/pull/74835), [@davidz627](https://github.com/davidz627)) +* Add network stats for Windows nodes and containers ([#74788](https://github.com/kubernetes/kubernetes/pull/74788), [@feiskyer](https://github.com/feiskyer)) +* kubeadm: when calling "reset" on a control-plane node, remove the APIEndpoint information for this node from the ClusterStatus in the kubeadm ConfigMap. ([#75082](https://github.com/kubernetes/kubernetes/pull/75082), [@neolit123](https://github.com/neolit123)) +* kube-apiserver now serves OpenAPI specs for registered CRDs with defined ([#71192](https://github.com/kubernetes/kubernetes/pull/71192), [@roycaihw](https://github.com/roycaihw)) + * validation schemata as an alpha feature, to be enabled via the "CustomResourcePublishOpenAPI" feature gate. Kubectl will validate client-side using those. Note that in + * future, client-side validation in 1.14 kubectl against a 1.15 cluster will reject + * unknown fields for CRDs with validation schema defined. +* Fix kubelet start failure issue on Azure Stack due to InstanceMetadata setting ([#74936](https://github.com/kubernetes/kubernetes/pull/74936), [@rjaini](https://github.com/rjaini)) +* add subcommand `kubectl create cronjob` ([#71651](https://github.com/kubernetes/kubernetes/pull/71651), [@Pingan2017](https://github.com/Pingan2017)) +* The CSIBlockVolume feature gate is now beta, and defaults to enabled. ([#74909](https://github.com/kubernetes/kubernetes/pull/74909), [@bswartz](https://github.com/bswartz)) +* Pre-existing log files are now opened with O_APPEND, instead of O_TRUNC. This helps prevent losing logs when components crash-loop, and also enables external log rotation utilities to truncate log files in-place without components extending log files to their pre-truncation sizes on subsequent writes. ([#74837](https://github.com/kubernetes/kubernetes/pull/74837), [@mtaufen](https://github.com/mtaufen)) +* the test/e2e/e2e.test binary can test arbitrary storage drivers, see the `-storage.testdriver` parameter ([#72836](https://github.com/kubernetes/kubernetes/pull/72836), [@pohly](https://github.com/pohly)) +* Fix panic in kubectl cp command ([#75037](https://github.com/kubernetes/kubernetes/pull/75037), [@soltysh](https://github.com/soltysh)) +* iscsi modules haven't even been loaded /sys/class/iscsi_host directory won't exist ([#74787](https://github.com/kubernetes/kubernetes/pull/74787), [@jianglingxia](https://github.com/jianglingxia)) +* the fluentd addon daemonset will now target all nodes. ([#74424](https://github.com/kubernetes/kubernetes/pull/74424), [@liggitt](https://github.com/liggitt)) + * setting `ENABLE_METADATA_CONCEALMENT=true` in kube-up will now set a `cloud.google.com/metadata-proxy-ready=true` label on new nodes. In v1.16, the metadata proxy add-on will switch to using that label as a node selector. + * setting `KUBE_PROXY_DAEMONSET=true` in kube-up will now set a `node.kubernetes.io/kube-proxy-ds-ready=true` label on new nodes. In v1.16, the kube-proxy daemonset add-on will switch to using that label as a node selector. + * In 1.16, the masq-agent daemonset add-on will switch to using `node.kubernetes.io/masq-agent-ds-ready` as a node selector. +* - Kubelet: replace `du` and `find` with a golang implementation ([#74675](https://github.com/kubernetes/kubernetes/pull/74675), [@dashpole](https://github.com/dashpole)) + * - Kubelet: periodically update machine info to support hot-add/remove +* kubeadm: add certificate-key and skip-certificate-key-print flags to kubeadm init ([#74671](https://github.com/kubernetes/kubernetes/pull/74671), [@yagonobre](https://github.com/yagonobre)) +* Admission webhooks rules can now limit scope to only match namespaced, or only cluster-scoped resources with a `scope: "Cluster" | "Namespaced" | "*"` field. ([#74477](https://github.com/kubernetes/kubernetes/pull/74477), [@liggitt](https://github.com/liggitt)) +* The CSIPersistentVolume and KubeletPluginWatcher feature gates cannot be disabled, and will be removed in Kubernetes v1.16 ([#74830](https://github.com/kubernetes/kubernetes/pull/74830), [@msau42](https://github.com/msau42)) +* Kubelet won't evict a static pod with priority `system-node-critical` upon resource pressure. ([#74222](https://github.com/kubernetes/kubernetes/pull/74222), [@Huang-Wei](https://github.com/Huang-Wei)) +* Fixes panic if a kubelet is run against an older kube-apiserver ([#74529](https://github.com/kubernetes/kubernetes/pull/74529), [@liggitt](https://github.com/liggitt)) +* The resource group name in Azure providerID is not converted to lower cases. ([#74882](https://github.com/kubernetes/kubernetes/pull/74882), [@feiskyer](https://github.com/feiskyer)) +* Remove the out-of-tree PersistentVolumeLabel controller because it cannot run without Initializers (removed in v1.14). If you are using AWS EBS, GCE PD, Azure Disk, Cinder Disk or vSphere volumes and rely on zone labels, then enable the `PersistentVolumeLabel` admission controller in the `kube-apiserver` in the `--enable-admission-plugins` flag. ([#74615](https://github.com/kubernetes/kubernetes/pull/74615), [@andrewsykim](https://github.com/andrewsykim)) +* kubeadm: improved RequiredIPVSKernelModulesAvailable warning message ([#74033](https://github.com/kubernetes/kubernetes/pull/74033), [@bart0sh](https://github.com/bart0sh)) +* Add `nullable` support to CustomResourceDefinition OpenAPI validation schemata. ([#74804](https://github.com/kubernetes/kubernetes/pull/74804), [@sttts](https://github.com/sttts)) +* Fix kube-apiserver not to create default/kubernetes service endpoints before it reports readiness via the /healthz and therefore is ready to serve requests. Also early during startup old endpoints are remove which might be left over from a previously crashed kube-apiserver. ([#74668](https://github.com/kubernetes/kubernetes/pull/74668), [@sttts](https://github.com/sttts)) +* kubeadm: fix a bug where standard kubeconfig paths were searched even if the user provided /etc/kubernetes/admin.conf explicitly for commands that accept --kubeconfig, like kubeadm token. ([#71874](https://github.com/kubernetes/kubernetes/pull/71874), [@neolit123](https://github.com/neolit123)) + * kubeadm: use the default kubeconfig (/etc/kubernetes/admin.conf) for "kubeadm reset" and "kubeadm upgrade" commands. +* Increase api server client certificate expiration histogram resolution to accommodate short-lived (< 6h) client certificates. ([#74806](https://github.com/kubernetes/kubernetes/pull/74806), [@mxinden](https://github.com/mxinden)) +* Default RBAC policy no longer grants access to discovery and permission-checking APIs (used by `kubectl auth can-i`) to *unauthenticated* users. Upgraded clusters preserve prior behavior, but cluster administrators wishing to grant unauthenticated users access in new clusters will need to explicitly opt-in to expose the discovery and/or permission-checking APIs: ([#73807](https://github.com/kubernetes/kubernetes/pull/73807), [@dekkagaijin](https://github.com/dekkagaijin)) + * `kubectl create clusterrolebinding anonymous-discovery --clusterrole=system:discovery --group=system:unauthenticated` + * `kubectl create clusterrolebinding anonymous-access-review --clusterrole=system:basic-user --group=system:unauthenticated` +* The PersistentLocalVolumes feature is GA. The feature gate cannot be disabled and will be removed in Kubernetes 1.17 ([#74769](https://github.com/kubernetes/kubernetes/pull/74769), [@msau42](https://github.com/msau42)) +* kubelet: resolved hang/timeout issues when running large numbers of pods with unique configmap/secret references by reverting to 1.11 configmap/secret lookup behavior ([#74755](https://github.com/kubernetes/kubernetes/pull/74755), [@liggitt](https://github.com/liggitt)) +* Convert `latency`/`latencies` in metrics name to `duration`. ([#74418](https://github.com/kubernetes/kubernetes/pull/74418), [@danielqsj](https://github.com/danielqsj)) + * The following metrics are changed and mark previous metrics as deprecated: + * `rest_client_request_latency_seconds` -> `rest_client_request_duration_seconds` + * `apiserver_proxy_tunnel_sync_latency_secs` -> `apiserver_proxy_tunnel_sync_duration_seconds` + * `scheduler_scheduling_latency_seconds` -> `scheduler_scheduling_duration_seconds ` +* Fix help message for --container-runtime-endpoint: only unix socket is support on Linux. ([#74712](https://github.com/kubernetes/kubernetes/pull/74712), [@feiskyer](https://github.com/feiskyer)) +* Update to use golang 1.12 ([#74632](https://github.com/kubernetes/kubernetes/pull/74632), [@cblecker](https://github.com/cblecker)) +* The `RunAsGroup` feature has been promoted to beta and enabled by default. PodSpec and PodSecurityPolicy objects can be used to control the primary GID of containers on supported container runtimes. ([#73007](https://github.com/kubernetes/kubernetes/pull/73007), [@krmayankk](https://github.com/krmayankk)) +* fix Azure Container Registry anonymous repo image pull error ([#74715](https://github.com/kubernetes/kubernetes/pull/74715), [@andyzhangx](https://github.com/andyzhangx)) +* Adds the same information to an init container as a standard container in a pod when using PodPresets. ([#71479](https://github.com/kubernetes/kubernetes/pull/71479), [@soggiest](https://github.com/soggiest)) +* fix the flake in scheduling_queue_test.go ([#74611](https://github.com/kubernetes/kubernetes/pull/74611), [@denkensk](https://github.com/denkensk)) +* The kube-apiserver OpenAPI definitions with the prefix "io.k8s.kubernetes.pkg" (deprecated since 1.9) have been removed. ([#74596](https://github.com/kubernetes/kubernetes/pull/74596), [@sttts](https://github.com/sttts)) +* kube-conformance image will now run ginkgo with the --dryRun flag if the container is run with the environment variable E2E_DRYRUN set. ([#74731](https://github.com/kubernetes/kubernetes/pull/74731), [@johnSchnake](https://github.com/johnSchnake)) +* The deprecated `MountPropagation` feature gate has been removed, and the feature is now unconditionally enabled. ([#74720](https://github.com/kubernetes/kubernetes/pull/74720), [@bertinatto](https://github.com/bertinatto)) +* Introduce dynamic volume provisioning shim for CSI migration ([#73653](https://github.com/kubernetes/kubernetes/pull/73653), [@ddebroy](https://github.com/ddebroy)) +* Fix --help flag parsing ([#74682](https://github.com/kubernetes/kubernetes/pull/74682), [@soltysh](https://github.com/soltysh)) +* This PR removes the following metrics: ([#74636](https://github.com/kubernetes/kubernetes/pull/74636), [@logicalhan](https://github.com/logicalhan)) + * reflector_items_per_list + * reflector_items_per_watch + * reflector_last_resource_version + * reflector_list_duration_seconds + * reflector_lists_total + * reflector_short_watches_total + * reflector_watch_duration_seconds + * reflector_watches_total + * While this is a backwards-incompatible change, it would have been impossible to setup reliable monitoring around these metrics since the labels were not stable. +* Add a configuration field to shorten the timeout of validating/mutating admission webhook call. The timeout value must be between 1 and 30 seconds. Default to 30 seconds when unspecified. ([#74562](https://github.com/kubernetes/kubernetes/pull/74562), [@roycaihw](https://github.com/roycaihw)) +* client-go: PortForwarder.GetPorts() now contain correct local port if no local port was initially specified when setting up the port forwarder ([#73676](https://github.com/kubernetes/kubernetes/pull/73676), [@martin-helmich](https://github.com/martin-helmich)) +* # Apply resources from a directory containing kustomization.yaml ([#74140](https://github.com/kubernetes/kubernetes/pull/74140), [@Liujingfang1](https://github.com/Liujingfang1)) + * kubectl apply -k dir + * # Delete resources from a directory containing kustomization.yaml. + * kubectl delete -k dir + * # List resources from a directory containing kustomization.yaml + * kubectl get -k dir +* kubeadm: Allow to download certificate secrets uploaded by `init` or `upload-certs` phase, allowing to transfer certificate secrets (certificates and keys) from the cluster to other master machines when creating HA deployments. ([#74168](https://github.com/kubernetes/kubernetes/pull/74168), [@ereslibre](https://github.com/ereslibre)) +* Fixes an issue with missing apiVersion/kind in object data sent to admission webhooks ([#74448](https://github.com/kubernetes/kubernetes/pull/74448), [@liggitt](https://github.com/liggitt)) +* client-go: the deprecated versionless API group accessors (like `clientset.Apps()` have been removed). Use an explicit version instead (like `clientset.AppsV1()`) ([#74422](https://github.com/kubernetes/kubernetes/pull/74422), [@liggitt](https://github.com/liggitt)) +* The `--quiet` option to `kubectl run` now suppresses resource deletion messages emitted when the `--rm` option is specified. ([#73266](https://github.com/kubernetes/kubernetes/pull/73266), [@awh](https://github.com/awh)) +* Add Custom Resource support to "kubectl autoscale" ([#72678](https://github.com/kubernetes/kubernetes/pull/72678), [@rmohr](https://github.com/rmohr)) +* Image garbage collection no longer fails for images with only one tag but more than one repository associated. ([#70647](https://github.com/kubernetes/kubernetes/pull/70647), [@corvus-ch](https://github.com/corvus-ch)) +* - Fix liveness probe in fluentd-gcp cluster addon ([#74522](https://github.com/kubernetes/kubernetes/pull/74522), [@Pluies](https://github.com/Pluies)) +* The new test ``[sig-network] DNS should provide /etc/hosts entries for the cluster [LinuxOnly] [Conformance]`` will validate the host entries set in the ``/etc/hosts`` file (pod's FQDN and hostname), which should be managed by Kubelet. ([#72729](https://github.com/kubernetes/kubernetes/pull/72729), [@bclau](https://github.com/bclau)) + * The test has the tag ``[LinuxOnly]`` because individual files cannot be mounted in Windows Containers, which means that it cannot pass using Windows nodes. + + + # v1.14.0-beta.1 [Documentation](https://docs.k8s.io) From e64c08f86971905560fec7761e37fcffa5a213d0 Mon Sep 17 00:00:00 2001 From: David Ashpole Date: Tue, 12 Mar 2019 12:58:23 -0700 Subject: [PATCH 21/21] remove DevicePlugin test for feature gate that can no longer be set --- test/e2e_node/device_plugin.go | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/test/e2e_node/device_plugin.go b/test/e2e_node/device_plugin.go index 138100a815..f0c7bdbe19 100644 --- a/test/e2e_node/device_plugin.go +++ b/test/e2e_node/device_plugin.go @@ -45,17 +45,12 @@ const ( ) // Serial because the test restarts Kubelet -var _ = framework.KubeDescribe("Device Plugin [Feature:DevicePlugin][NodeFeature:DevicePlugin][Serial]", func() { - f := framework.NewDefaultFramework("device-plugin-errors") - testDevicePlugin(f, false, pluginapi.DevicePluginPath) -}) - var _ = framework.KubeDescribe("Device Plugin [Feature:DevicePluginProbe][NodeFeature:DevicePluginProbe][Serial]", func() { f := framework.NewDefaultFramework("device-plugin-errors") - testDevicePlugin(f, true, "/var/lib/kubelet/plugins_registry") + testDevicePlugin(f, "/var/lib/kubelet/plugins_registry") }) -func testDevicePlugin(f *framework.Framework, enablePluginWatcher bool, pluginSockDir string) { +func testDevicePlugin(f *framework.Framework, pluginSockDir string) { pluginSockDir = filepath.Join(pluginSockDir) + "/" Context("DevicePlugin", func() { By("Enabling support for Kubelet Plugins Watcher") @@ -63,7 +58,6 @@ func testDevicePlugin(f *framework.Framework, enablePluginWatcher bool, pluginSo if initialConfig.FeatureGates == nil { initialConfig.FeatureGates = map[string]bool{} } - initialConfig.FeatureGates[string(features.KubeletPluginsWatcher)] = enablePluginWatcher initialConfig.FeatureGates[string(features.KubeletPodResources)] = true }) It("Verifies the Kubelet device plugin functionality.", func() {