diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json index 7d5005c7fe..2de9fa5235 100644 --- a/api/openapi-spec/swagger.json +++ b/api/openapi-spec/swagger.json @@ -17546,6 +17546,10 @@ "io.k8s.apimachinery.pkg.apis.meta.v1.Preconditions": { "description": "Preconditions must be fulfilled before an operation (update, delete, etc.) is carried out.", "properties": { + "resourceVersion": { + "description": "Specifies the target ResourceVersion", + "type": "string" + }, "uid": { "description": "Specifies the target UID.", "type": "string" diff --git a/build/root/WORKSPACE b/build/root/WORKSPACE index a6eff8996d..f0e6cbada8 100644 --- a/build/root/WORKSPACE +++ b/build/root/WORKSPACE @@ -19,16 +19,6 @@ http_archive( urls = mirror("https://github.com/kubernetes/repo-infra/archive/b461270ab6ccfb94ff2d78df96d26f669376d660.tar.gz"), ) -ETCD_VERSION = "3.3.10" - -http_archive( - name = "com_coreos_etcd", - build_file = "@//third_party:etcd.BUILD", - sha256 = "1620a59150ec0a0124a65540e23891243feb2d9a628092fb1edcc23974724a45", - strip_prefix = "etcd-v%s-linux-amd64" % ETCD_VERSION, - urls = mirror("https://github.com/coreos/etcd/releases/download/v%s/etcd-v%s-linux-amd64.tar.gz" % (ETCD_VERSION, ETCD_VERSION)), -) - http_archive( name = "io_bazel_rules_go", sha256 = "6776d68ebb897625dead17ae510eac3d5f6342367327875210df44dbe2aeeb19", diff --git a/build/workspace.bzl b/build/workspace.bzl index 6a0252e7c4..0b281235a6 100644 --- a/build/workspace.bzl +++ b/build/workspace.bzl @@ -14,7 +14,7 @@ load("//build:platforms.bzl", "SERVER_PLATFORMS") load("//build:workspace_mirror.bzl", "mirror") -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file") +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_file") load("@io_bazel_rules_docker//container:container.bzl", "container_pull") CNI_VERSION = "0.6.0" @@ -35,6 +35,13 @@ _CRI_TARBALL_ARCH_SHA256 = { "s390x": "814aa9cd496be416612c2653097a1c9eb5784e38aa4889034b44ebf888709057", } +ETCD_VERSION = "3.3.10" +_ETCD_TARBALL_ARCH_SHA256 = { + "amd64": "1620a59150ec0a0124a65540e23891243feb2d9a628092fb1edcc23974724a45", + "arm64": "5ec97b0b872adce275b8130d19db314f7f2b803aeb24c4aae17a19e2d66853c4", + "ppc64le": "148fe96f0ec1813c5db9916199e96a913174304546bc8447a2d2f9fee4b8f6c2", +} + # Note that these are digests for the manifest list. We resolve the manifest # list to each of its platform-specific images in # debian_image_dependencies(). @@ -48,6 +55,7 @@ def release_dependencies(): cni_tarballs() cri_tarballs() debian_image_dependencies() + etcd_tarballs() def cni_tarballs(): for arch, sha in _CNI_TARBALL_ARCH_SHA256.items(): @@ -92,3 +100,13 @@ def debian_image_dependencies(): registry = "k8s.gcr.io", repository = "debian-hyperkube-base", ) + +def etcd_tarballs(): + for arch, sha in _ETCD_TARBALL_ARCH_SHA256.items(): + http_archive( + name = "com_coreos_etcd_%s" % arch, + build_file = "@//third_party:etcd.BUILD", + sha256 = sha, + strip_prefix = "etcd-v%s-linux-%s" % (ETCD_VERSION, arch), + urls = mirror("https://github.com/coreos/etcd/releases/download/v%s/etcd-v%s-linux-%s.tar.gz" % (ETCD_VERSION, ETCD_VERSION, arch)), + ) diff --git a/pkg/cloudprovider/providers/azure/azure_loadbalancer.go b/pkg/cloudprovider/providers/azure/azure_loadbalancer.go index 222d92d519..9a114d5ec0 100644 --- a/pkg/cloudprovider/providers/azure/azure_loadbalancer.go +++ b/pkg/cloudprovider/providers/azure/azure_loadbalancer.go @@ -169,27 +169,47 @@ func (az *Cloud) UpdateLoadBalancer(ctx context.Context, clusterName string, ser func (az *Cloud) EnsureLoadBalancerDeleted(ctx context.Context, clusterName string, service *v1.Service) error { isInternal := requiresInternalLoadBalancer(service) serviceName := getServiceName(service) - klog.V(5).Infof("delete(%s): START clusterName=%q", serviceName, clusterName) + klog.V(5).Infof("Delete service (%s): START clusterName=%q", serviceName, clusterName) + + ignoreErrors := func(err error) error { + if ignoreStatusNotFoundFromError(err) == nil { + klog.V(5).Infof("EnsureLoadBalancerDeleted: ignoring StatusNotFound error because the resource doesn't exist (%v)", err) + return nil + } + + if ignoreStatusForbiddenFromError(err) == nil { + klog.V(5).Infof("EnsureLoadBalancerDeleted: ignoring StatusForbidden error (%v). This may be caused by wrong configuration via service annotations", err) + return nil + } + + return err + } serviceIPToCleanup, err := az.findServiceIPAddress(ctx, clusterName, service, isInternal) - if err != nil { + if ignoreErrors(err) != nil { return err } klog.V(2).Infof("EnsureLoadBalancerDeleted: reconciling security group for service %q with IP %q, wantLb = false", serviceName, serviceIPToCleanup) if _, err := az.reconcileSecurityGroup(clusterName, service, &serviceIPToCleanup, false /* wantLb */); err != nil { - return err + if ignoreErrors(err) != nil { + return err + } } if _, err := az.reconcileLoadBalancer(clusterName, service, nil, false /* wantLb */); err != nil { - return err + if ignoreErrors(err) != nil { + return err + } } if _, err := az.reconcilePublicIP(clusterName, service, nil, false /* wantLb */); err != nil { - return err + if ignoreErrors(err) != nil { + return err + } } - klog.V(2).Infof("delete(%s): FINISH", serviceName) + klog.V(2).Infof("Delete service (%s): FINISH", serviceName) return nil } diff --git a/pkg/cloudprovider/providers/azure/azure_loadbalancer_test.go b/pkg/cloudprovider/providers/azure/azure_loadbalancer_test.go index 7dda564e7d..f1368575b6 100644 --- a/pkg/cloudprovider/providers/azure/azure_loadbalancer_test.go +++ b/pkg/cloudprovider/providers/azure/azure_loadbalancer_test.go @@ -17,6 +17,7 @@ limitations under the License. package azure import ( + "context" "fmt" "reflect" "testing" @@ -308,3 +309,62 @@ func TestSubnet(t *testing.T) { assert.Equal(t, c.expected, real, fmt.Sprintf("TestCase[%d]: %s", i, c.desc)) } } + +func TestEnsureLoadBalancerDeleted(t *testing.T) { + const vmCount = 8 + const availabilitySetCount = 4 + const serviceCount = 9 + + tests := []struct { + desc string + service v1.Service + expectCreateError bool + }{ + { + desc: "external service should be created and deleted successfully", + service: getTestService("test1", v1.ProtocolTCP, 80), + }, + { + desc: "internal service should be created and deleted successfully", + service: getInternalTestService("test2", 80), + }, + { + desc: "annotated service with same resourceGroup should be created and deleted successfully", + service: getResourceGroupTestService("test3", "rg", "", 80), + }, + { + desc: "annotated service with different resourceGroup shouldn't be created but should be deleted successfully", + service: getResourceGroupTestService("test4", "random-rg", "1.2.3.4", 80), + expectCreateError: true, + }, + } + + az := getTestCloud() + for i, c := range tests { + clusterResources := getClusterResources(az, vmCount, availabilitySetCount) + getTestSecurityGroup(az) + if c.service.Annotations[ServiceAnnotationLoadBalancerInternal] == "true" { + addTestSubnet(t, az, &c.service) + } + + // create the service first. + lbStatus, err := az.EnsureLoadBalancer(context.TODO(), testClusterName, &c.service, clusterResources.nodes) + if c.expectCreateError { + assert.NotNil(t, err, "TestCase[%d]: %s", i, c.desc) + } else { + assert.Nil(t, err, "TestCase[%d]: %s", i, c.desc) + assert.NotNil(t, lbStatus, "TestCase[%d]: %s", i, c.desc) + result, err := az.LoadBalancerClient.List(context.TODO(), az.Config.ResourceGroup) + assert.Nil(t, err, "TestCase[%d]: %s", i, c.desc) + assert.Equal(t, len(result), 1, "TestCase[%d]: %s", i, c.desc) + assert.Equal(t, len(*result[0].LoadBalancingRules), 1, "TestCase[%d]: %s", i, c.desc) + } + + // finally, delete it. + err = az.EnsureLoadBalancerDeleted(context.TODO(), testClusterName, &c.service) + assert.Nil(t, err, "TestCase[%d]: %s", i, c.desc) + result, err := az.LoadBalancerClient.List(context.Background(), az.Config.ResourceGroup) + assert.Nil(t, err, "TestCase[%d]: %s", i, c.desc) + assert.Equal(t, len(result), 0, "TestCase[%d]: %s", i, c.desc) + } +} diff --git a/pkg/cloudprovider/providers/azure/azure_test.go b/pkg/cloudprovider/providers/azure/azure_test.go index 523809f3b3..e30b0d9297 100644 --- a/pkg/cloudprovider/providers/azure/azure_test.go +++ b/pkg/cloudprovider/providers/azure/azure_test.go @@ -1137,6 +1137,13 @@ func getInternalTestService(identifier string, requestedPorts ...int32) v1.Servi return svc } +func getResourceGroupTestService(identifier, resourceGroup, loadBalancerIP string, requestedPorts ...int32) v1.Service { + svc := getTestService(identifier, v1.ProtocolTCP, requestedPorts...) + svc.Spec.LoadBalancerIP = loadBalancerIP + svc.Annotations[ServiceAnnotationLoadBalancerResourceGroup] = resourceGroup + return svc +} + func setLoadBalancerModeAnnotation(service *v1.Service, lbMode string) { service.Annotations[ServiceAnnotationLoadBalancerMode] = lbMode } diff --git a/pkg/cloudprovider/providers/azure/azure_wrap.go b/pkg/cloudprovider/providers/azure/azure_wrap.go index 95e8c7dac8..277c6debbd 100644 --- a/pkg/cloudprovider/providers/azure/azure_wrap.go +++ b/pkg/cloudprovider/providers/azure/azure_wrap.go @@ -71,6 +71,19 @@ func ignoreStatusNotFoundFromError(err error) error { return err } +// ignoreStatusForbiddenFromError returns nil if the status code is StatusForbidden. +// This happens when AuthorizationFailed is reported from Azure API. +func ignoreStatusForbiddenFromError(err error) error { + if err == nil { + return nil + } + v, ok := err.(autorest.DetailedError) + if ok && v.StatusCode == http.StatusForbidden { + return nil + } + return err +} + /// getVirtualMachine calls 'VirtualMachinesClient.Get' with a timed cache /// The service side has throttling control that delays responses if there're multiple requests onto certain vm /// resource request in short period. diff --git a/pkg/proxy/iptables/proxier.go b/pkg/proxy/iptables/proxier.go index a4f8dc8ef7..7633c6c9dc 100644 --- a/pkg/proxy/iptables/proxier.go +++ b/pkg/proxy/iptables/proxier.go @@ -369,6 +369,7 @@ var iptablesJumpChains = []iptablesJumpChain{ {utiliptables.TableFilter, kubeExternalServicesChain, utiliptables.ChainInput, "kubernetes externally-visible service portals", []string{"-m", "conntrack", "--ctstate", "NEW"}}, {utiliptables.TableFilter, kubeServicesChain, utiliptables.ChainForward, "kubernetes service portals", []string{"-m", "conntrack", "--ctstate", "NEW"}}, {utiliptables.TableFilter, kubeServicesChain, utiliptables.ChainOutput, "kubernetes service portals", []string{"-m", "conntrack", "--ctstate", "NEW"}}, + {utiliptables.TableFilter, kubeServicesChain, utiliptables.ChainInput, "kubernetes service portals", []string{"-m", "conntrack", "--ctstate", "NEW"}}, {utiliptables.TableFilter, kubeForwardChain, utiliptables.ChainForward, "kubernetes forwarding rules", nil}, {utiliptables.TableNAT, kubeServicesChain, utiliptables.ChainOutput, "kubernetes service portals", nil}, {utiliptables.TableNAT, kubeServicesChain, utiliptables.ChainPrerouting, "kubernetes service portals", nil}, @@ -847,6 +848,7 @@ func (proxier *Proxier) syncProxyRules() { } writeLine(proxier.natRules, append(args, "-j", string(svcChain))...) } else { + // No endpoints. writeLine(proxier.filterRules, "-A", string(kubeServicesChain), "-m", "comment", "--comment", fmt.Sprintf(`"%s has no endpoints"`, svcNameString), @@ -917,6 +919,7 @@ func (proxier *Proxier) syncProxyRules() { // This covers cases like GCE load-balancers which get added to the local routing table. writeLine(proxier.natRules, append(dstLocalOnlyArgs, "-j", string(svcChain))...) } else { + // No endpoints. writeLine(proxier.filterRules, "-A", string(kubeExternalServicesChain), "-m", "comment", "--comment", fmt.Sprintf(`"%s has no endpoints"`, svcNameString), @@ -929,10 +932,10 @@ func (proxier *Proxier) syncProxyRules() { } // Capture load-balancer ingress. - if hasEndpoints { - fwChain := svcInfo.serviceFirewallChainName - for _, ingress := range svcInfo.LoadBalancerStatus.Ingress { - if ingress.IP != "" { + fwChain := svcInfo.serviceFirewallChainName + for _, ingress := range svcInfo.LoadBalancerStatus.Ingress { + if ingress.IP != "" { + if hasEndpoints { // create service firewall chain if chain, ok := existingNATChains[fwChain]; ok { writeBytesLine(proxier.natChains, chain) @@ -993,10 +996,19 @@ func (proxier *Proxier) syncProxyRules() { // If the packet was able to reach the end of firewall chain, then it did not get DNATed. // It means the packet cannot go thru the firewall, then mark it for DROP writeLine(proxier.natRules, append(args, "-j", string(KubeMarkDropChain))...) + } else { + // No endpoints. + writeLine(proxier.filterRules, + "-A", string(kubeServicesChain), + "-m", "comment", "--comment", fmt.Sprintf(`"%s has no endpoints"`, svcNameString), + "-m", protocol, "-p", protocol, + "-d", utilproxy.ToCIDR(net.ParseIP(ingress.IP)), + "--dport", strconv.Itoa(svcInfo.Port), + "-j", "REJECT", + ) } } } - // FIXME: do we need REJECT rules for load-balancer ingress if !hasEndpoints? // Capture nodeports. If we had more than 2 rules it might be // worthwhile to make a new per-service chain for nodeport rules, but @@ -1078,6 +1090,7 @@ func (proxier *Proxier) syncProxyRules() { writeLine(proxier.natRules, append(args, "-j", string(svcXlbChain))...) } } else { + // No endpoints. writeLine(proxier.filterRules, "-A", string(kubeExternalServicesChain), "-m", "comment", "--comment", fmt.Sprintf(`"%s has no endpoints"`, svcNameString), diff --git a/pkg/registry/core/namespace/storage/storage.go b/pkg/registry/core/namespace/storage/storage.go index 2fe25f1ee5..b9d0558ab7 100644 --- a/pkg/registry/core/namespace/storage/storage.go +++ b/pkg/registry/core/namespace/storage/storage.go @@ -147,6 +147,14 @@ func (r *REST) Delete(ctx context.Context, name string, options *metav1.DeleteOp ) return nil, false, err } + if options.Preconditions.ResourceVersion != nil && *options.Preconditions.ResourceVersion != namespace.ResourceVersion { + err = apierrors.NewConflict( + api.Resource("namespaces"), + name, + fmt.Errorf("Precondition failed: ResourceVersion in precondition: %v, ResourceVersion in object meta: %v", *options.Preconditions.ResourceVersion, namespace.ResourceVersion), + ) + return nil, false, err + } // upon first request to delete, we switch the phase to start namespace termination // TODO: enhance graceful deletion's calls to DeleteStrategy to allow phase change and finalizer patterns @@ -156,7 +164,7 @@ func (r *REST) Delete(ctx context.Context, name string, options *metav1.DeleteOp return nil, false, err } - preconditions := storage.Preconditions{UID: options.Preconditions.UID} + preconditions := storage.Preconditions{UID: options.Preconditions.UID, ResourceVersion: options.Preconditions.ResourceVersion} out := r.store.NewFunc() err = r.store.Storage.GuaranteedUpdate( diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition/etcd.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition/etcd.go index c6b50915f1..0bc9f64ddc 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition/etcd.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition/etcd.go @@ -92,6 +92,14 @@ func (r *REST) Delete(ctx context.Context, name string, options *metav1.DeleteOp ) return nil, false, err } + if options.Preconditions.ResourceVersion != nil && *options.Preconditions.ResourceVersion != crd.ResourceVersion { + err = apierrors.NewConflict( + apiextensions.Resource("customresourcedefinitions"), + name, + fmt.Errorf("Precondition failed: ResourceVersion in precondition: %v, ResourceVersion in object meta: %v", *options.Preconditions.ResourceVersion, crd.ResourceVersion), + ) + return nil, false, err + } // upon first request to delete, add our finalizer and then delegate if crd.DeletionTimestamp.IsZero() { @@ -100,7 +108,7 @@ func (r *REST) Delete(ctx context.Context, name string, options *metav1.DeleteOp return nil, false, err } - preconditions := storage.Preconditions{UID: options.Preconditions.UID} + preconditions := storage.Preconditions{UID: options.Preconditions.UID, ResourceVersion: options.Preconditions.ResourceVersion} out := r.Store.NewFunc() err = r.Store.Storage.GuaranteedUpdate( diff --git a/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/generated.pb.go b/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/generated.pb.go index ed252d3415..69e6509932 100644 --- a/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/generated.pb.go +++ b/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/generated.pb.go @@ -1591,6 +1591,12 @@ func (m *Preconditions) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintGenerated(dAtA, i, uint64(len(*m.UID))) i += copy(dAtA[i:], *m.UID) } + if m.ResourceVersion != nil { + dAtA[i] = 0x12 + i++ + i = encodeVarintGenerated(dAtA, i, uint64(len(*m.ResourceVersion))) + i += copy(dAtA[i:], *m.ResourceVersion) + } return i, nil } @@ -2428,6 +2434,10 @@ func (m *Preconditions) Size() (n int) { l = len(*m.UID) n += 1 + l + sovGenerated(uint64(l)) } + if m.ResourceVersion != nil { + l = len(*m.ResourceVersion) + n += 1 + l + sovGenerated(uint64(l)) + } return n } @@ -2907,6 +2917,7 @@ func (this *Preconditions) String() string { } s := strings.Join([]string{`&Preconditions{`, `UID:` + valueToStringGenerated(this.UID) + `,`, + `ResourceVersion:` + valueToStringGenerated(this.ResourceVersion) + `,`, `}`, }, "") return s @@ -7607,6 +7618,36 @@ func (m *Preconditions) Unmarshal(dAtA []byte) error { s := k8s_io_apimachinery_pkg_types.UID(dAtA[iNdEx:postIndex]) m.UID = &s iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ResourceVersion", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + s := string(dAtA[iNdEx:postIndex]) + m.ResourceVersion = &s + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipGenerated(dAtA[iNdEx:]) @@ -8989,172 +9030,173 @@ func init() { } var fileDescriptorGenerated = []byte{ - // 2664 bytes of a gzipped FileDescriptorProto + // 2674 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xcc, 0x19, 0x4d, 0x6c, 0x23, 0x57, 0x39, 0x63, 0xc7, 0x8e, 0xfd, 0x39, 0xce, 0xcf, 0xeb, 0x16, 0x5c, 0x4b, 0xc4, 0xe9, 0x14, 0x55, - 0x29, 0x6c, 0x6d, 0x92, 0xd2, 0x6a, 0x59, 0xa0, 0x10, 0xc7, 0xc9, 0x36, 0x74, 0xd3, 0x44, 0x2f, - 0xbb, 0x8b, 0x58, 0x56, 0x88, 0x89, 0xe7, 0xc5, 0x19, 0x62, 0xcf, 0x4c, 0xdf, 0x1b, 0x67, 0x37, - 0x70, 0xa0, 0x07, 0x10, 0x20, 0x01, 0xda, 0x23, 0x27, 0xd4, 0x15, 0x5c, 0xb8, 0x72, 0xe2, 0xc4, - 0xa9, 0x12, 0x7b, 0xac, 0xc4, 0xa5, 0x07, 0x64, 0x75, 0x03, 0x12, 0xdc, 0xb8, 0xe7, 0x80, 0xd0, - 0xfb, 0x99, 0x99, 0x37, 0x76, 0xbc, 0x19, 0xb3, 0x05, 0x71, 0xf2, 0xcc, 0xf7, 0x3f, 0xdf, 0xf7, - 0xbd, 0xef, 0xfb, 0xde, 0x67, 0xd8, 0x39, 0xbe, 0xc6, 0xea, 0x8e, 0xd7, 0x38, 0xee, 0x1f, 0x10, - 0xea, 0x92, 0x80, 0xb0, 0xc6, 0x09, 0x71, 0x6d, 0x8f, 0x36, 0x14, 0xc2, 0xf2, 0x9d, 0x9e, 0xd5, - 0x3e, 0x72, 0x5c, 0x42, 0x4f, 0x1b, 0xfe, 0x71, 0x87, 0x03, 0x58, 0xa3, 0x47, 0x02, 0xab, 0x71, - 0xb2, 0xda, 0xe8, 0x10, 0x97, 0x50, 0x2b, 0x20, 0x76, 0xdd, 0xa7, 0x5e, 0xe0, 0xa1, 0xcf, 0x4a, - 0xae, 0xba, 0xce, 0x55, 0xf7, 0x8f, 0x3b, 0x1c, 0xc0, 0xea, 0x9c, 0xab, 0x7e, 0xb2, 0x5a, 0x7d, - 0xb5, 0xe3, 0x04, 0x47, 0xfd, 0x83, 0x7a, 0xdb, 0xeb, 0x35, 0x3a, 0x5e, 0xc7, 0x6b, 0x08, 0xe6, - 0x83, 0xfe, 0xa1, 0x78, 0x13, 0x2f, 0xe2, 0x49, 0x0a, 0xad, 0x8e, 0x35, 0x85, 0xf6, 0xdd, 0xc0, - 0xe9, 0x91, 0x61, 0x2b, 0xaa, 0x6f, 0x5c, 0xc6, 0xc0, 0xda, 0x47, 0xa4, 0x67, 0x0d, 0xf3, 0x99, - 0x7f, 0xca, 0x42, 0x61, 0x7d, 0x6f, 0xfb, 0x06, 0xf5, 0xfa, 0x3e, 0x5a, 0x86, 0x69, 0xd7, 0xea, - 0x91, 0x8a, 0xb1, 0x6c, 0xac, 0x14, 0x9b, 0xb3, 0x8f, 0x07, 0xb5, 0xa9, 0xb3, 0x41, 0x6d, 0xfa, - 0x1d, 0xab, 0x47, 0xb0, 0xc0, 0xa0, 0x2e, 0x14, 0x4e, 0x08, 0x65, 0x8e, 0xe7, 0xb2, 0x4a, 0x66, - 0x39, 0xbb, 0x52, 0x5a, 0x7b, 0xb3, 0x9e, 0xe6, 0xfb, 0xeb, 0x42, 0xc1, 0x1d, 0xc9, 0xba, 0xe5, - 0xd1, 0x96, 0xc3, 0xda, 0xde, 0x09, 0xa1, 0xa7, 0xcd, 0x05, 0xa5, 0xa5, 0xa0, 0x90, 0x0c, 0x47, - 0x1a, 0xd0, 0x8f, 0x0c, 0x58, 0xf0, 0x29, 0x39, 0x24, 0x94, 0x12, 0x5b, 0xe1, 0x2b, 0xd9, 0x65, - 0xe3, 0x13, 0x50, 0x5b, 0x51, 0x6a, 0x17, 0xf6, 0x86, 0xe4, 0xe3, 0x11, 0x8d, 0xe8, 0x37, 0x06, - 0x54, 0x19, 0xa1, 0x27, 0x84, 0xae, 0xdb, 0x36, 0x25, 0x8c, 0x35, 0x4f, 0x37, 0xba, 0x0e, 0x71, - 0x83, 0x8d, 0xed, 0x16, 0x66, 0x95, 0x69, 0xe1, 0x87, 0xaf, 0xa5, 0x33, 0x68, 0x7f, 0x9c, 0x9c, - 0xa6, 0xa9, 0x2c, 0xaa, 0x8e, 0x25, 0x61, 0xf8, 0x29, 0x66, 0x98, 0x87, 0x30, 0x1b, 0x06, 0xf2, - 0xa6, 0xc3, 0x02, 0x74, 0x07, 0xf2, 0x1d, 0xfe, 0xc2, 0x2a, 0x86, 0x30, 0xb0, 0x9e, 0xce, 0xc0, - 0x50, 0x46, 0x73, 0x4e, 0xd9, 0x93, 0x17, 0xaf, 0x0c, 0x2b, 0x69, 0xe6, 0xcf, 0xa6, 0xa1, 0xb4, - 0xbe, 0xb7, 0x8d, 0x09, 0xf3, 0xfa, 0xb4, 0x4d, 0x52, 0x24, 0xcd, 0x1a, 0x00, 0xff, 0x65, 0xbe, - 0xd5, 0x26, 0x76, 0x25, 0xb3, 0x6c, 0xac, 0x14, 0x9a, 0x48, 0xd1, 0xc1, 0x3b, 0x11, 0x06, 0x6b, - 0x54, 0x5c, 0xea, 0xb1, 0xe3, 0xda, 0x22, 0xda, 0x9a, 0xd4, 0xb7, 0x1d, 0xd7, 0xc6, 0x02, 0x83, - 0x6e, 0x42, 0xee, 0x84, 0xd0, 0x03, 0xee, 0x7f, 0x9e, 0x10, 0x9f, 0x4f, 0xf7, 0x79, 0x77, 0x38, - 0x4b, 0xb3, 0x78, 0x36, 0xa8, 0xe5, 0xc4, 0x23, 0x96, 0x42, 0x50, 0x1d, 0x80, 0x1d, 0x79, 0x34, - 0x10, 0xe6, 0x54, 0x72, 0xcb, 0xd9, 0x95, 0x62, 0x73, 0x8e, 0xdb, 0xb7, 0x1f, 0x41, 0xb1, 0x46, - 0x81, 0xae, 0xc1, 0x2c, 0x73, 0xdc, 0x4e, 0xbf, 0x6b, 0x51, 0x0e, 0xa8, 0xe4, 0x85, 0x9d, 0x57, - 0x94, 0x9d, 0xb3, 0xfb, 0x1a, 0x0e, 0x27, 0x28, 0xb9, 0xa6, 0xb6, 0x15, 0x90, 0x8e, 0x47, 0x1d, - 0xc2, 0x2a, 0x33, 0xb1, 0xa6, 0x8d, 0x08, 0x8a, 0x35, 0x0a, 0xf4, 0x12, 0xe4, 0x84, 0xe7, 0x2b, - 0x05, 0xa1, 0xa2, 0xac, 0x54, 0xe4, 0x44, 0x58, 0xb0, 0xc4, 0xa1, 0x57, 0x60, 0x46, 0x9d, 0x9a, - 0x4a, 0x51, 0x90, 0xcd, 0x2b, 0xb2, 0x99, 0x30, 0xad, 0x43, 0x3c, 0xfa, 0x06, 0x20, 0x16, 0x78, - 0xd4, 0xea, 0x10, 0x85, 0x7a, 0xcb, 0x62, 0x47, 0x15, 0x10, 0x5c, 0x55, 0xc5, 0x85, 0xf6, 0x47, - 0x28, 0xf0, 0x05, 0x5c, 0xe6, 0xef, 0x0d, 0x98, 0xd7, 0x72, 0x41, 0xe4, 0xdd, 0x35, 0x98, 0xed, - 0x68, 0xa7, 0x4e, 0xe5, 0x45, 0xe4, 0x19, 0xfd, 0x44, 0xe2, 0x04, 0x25, 0x22, 0x50, 0xa4, 0x4a, - 0x52, 0x58, 0x5d, 0x56, 0x53, 0x27, 0x6d, 0x68, 0x43, 0xac, 0x49, 0x03, 0x32, 0x1c, 0x4b, 0x36, - 0xff, 0x6e, 0x88, 0x04, 0x0e, 0xeb, 0x0d, 0x5a, 0xd1, 0x6a, 0x9a, 0x21, 0xc2, 0x31, 0x3b, 0xa6, - 0x1e, 0x5d, 0x52, 0x08, 0x32, 0xff, 0x17, 0x85, 0xe0, 0x7a, 0xe1, 0x57, 0xef, 0xd7, 0xa6, 0xde, - 0xfb, 0xcb, 0xf2, 0x94, 0xd9, 0x83, 0xf2, 0x06, 0x25, 0x56, 0x40, 0x76, 0xfd, 0x40, 0x7c, 0x80, - 0x09, 0x79, 0x9b, 0x9e, 0xe2, 0xbe, 0xab, 0x3e, 0x14, 0xf8, 0xf9, 0x6e, 0x09, 0x08, 0x56, 0x18, - 0x1e, 0xbf, 0x43, 0x87, 0x74, 0xed, 0x1d, 0xcb, 0xb5, 0x3a, 0x84, 0xaa, 0x13, 0x18, 0x79, 0x75, - 0x4b, 0xc3, 0xe1, 0x04, 0xa5, 0xf9, 0x93, 0x2c, 0x94, 0x5b, 0xa4, 0x4b, 0x62, 0x7d, 0x5b, 0x80, + 0x29, 0x6c, 0x6d, 0x92, 0xd2, 0x6a, 0x59, 0x68, 0x21, 0x8e, 0x93, 0x6d, 0xe8, 0xa6, 0x89, 0x5e, + 0xba, 0x8b, 0x58, 0x56, 0x88, 0x89, 0xe7, 0xc5, 0x19, 0x62, 0xcf, 0x4c, 0xdf, 0x1b, 0x67, 0x37, + 0x70, 0xa0, 0x07, 0x10, 0x20, 0x41, 0xb5, 0x47, 0x4e, 0xa8, 0x2b, 0xb8, 0x70, 0xe5, 0xc4, 0x89, + 0x53, 0x25, 0xf6, 0x58, 0x89, 0x4b, 0x0f, 0xc8, 0xea, 0x06, 0x24, 0xb8, 0x71, 0xcf, 0x01, 0xa1, + 0xf7, 0x33, 0x33, 0x6f, 0xec, 0x78, 0x33, 0x66, 0x0b, 0xe2, 0x14, 0xcf, 0xf7, 0xff, 0xbe, 0xef, + 0x7b, 0xdf, 0xf7, 0xbd, 0x2f, 0xb0, 0x73, 0x7c, 0x8d, 0xd5, 0x1d, 0xaf, 0x71, 0xdc, 0x3f, 0x20, + 0xd4, 0x25, 0x01, 0x61, 0x8d, 0x13, 0xe2, 0xda, 0x1e, 0x6d, 0x28, 0x84, 0xe5, 0x3b, 0x3d, 0xab, + 0x7d, 0xe4, 0xb8, 0x84, 0x9e, 0x36, 0xfc, 0xe3, 0x0e, 0x07, 0xb0, 0x46, 0x8f, 0x04, 0x56, 0xe3, + 0x64, 0xb5, 0xd1, 0x21, 0x2e, 0xa1, 0x56, 0x40, 0xec, 0xba, 0x4f, 0xbd, 0xc0, 0x43, 0x9f, 0x97, + 0x5c, 0x75, 0x9d, 0xab, 0xee, 0x1f, 0x77, 0x38, 0x80, 0xd5, 0x39, 0x57, 0xfd, 0x64, 0xb5, 0xfa, + 0x72, 0xc7, 0x09, 0x8e, 0xfa, 0x07, 0xf5, 0xb6, 0xd7, 0x6b, 0x74, 0xbc, 0x8e, 0xd7, 0x10, 0xcc, + 0x07, 0xfd, 0x43, 0xf1, 0x25, 0x3e, 0xc4, 0x2f, 0x29, 0xb4, 0x3a, 0xd6, 0x14, 0xda, 0x77, 0x03, + 0xa7, 0x47, 0x86, 0xad, 0xa8, 0xbe, 0x76, 0x19, 0x03, 0x6b, 0x1f, 0x91, 0x9e, 0x35, 0xcc, 0x67, + 0xfe, 0x29, 0x0b, 0x85, 0xf5, 0xbd, 0xed, 0x1b, 0xd4, 0xeb, 0xfb, 0x68, 0x19, 0xa6, 0x5d, 0xab, + 0x47, 0x2a, 0xc6, 0xb2, 0xb1, 0x52, 0x6c, 0xce, 0x3e, 0x1a, 0xd4, 0xa6, 0xce, 0x06, 0xb5, 0xe9, + 0xb7, 0xad, 0x1e, 0xc1, 0x02, 0x83, 0xba, 0x50, 0x38, 0x21, 0x94, 0x39, 0x9e, 0xcb, 0x2a, 0x99, + 0xe5, 0xec, 0x4a, 0x69, 0xed, 0x8d, 0x7a, 0x9a, 0xf3, 0xd7, 0x85, 0x82, 0xdb, 0x92, 0x75, 0xcb, + 0xa3, 0x2d, 0x87, 0xb5, 0xbd, 0x13, 0x42, 0x4f, 0x9b, 0x0b, 0x4a, 0x4b, 0x41, 0x21, 0x19, 0x8e, + 0x34, 0xa0, 0x1f, 0x1b, 0xb0, 0xe0, 0x53, 0x72, 0x48, 0x28, 0x25, 0xb6, 0xc2, 0x57, 0xb2, 0xcb, + 0xc6, 0xa7, 0xa0, 0xb6, 0xa2, 0xd4, 0x2e, 0xec, 0x0d, 0xc9, 0xc7, 0x23, 0x1a, 0xd1, 0x6f, 0x0c, + 0xa8, 0x32, 0x42, 0x4f, 0x08, 0x5d, 0xb7, 0x6d, 0x4a, 0x18, 0x6b, 0x9e, 0x6e, 0x74, 0x1d, 0xe2, + 0x06, 0x1b, 0xdb, 0x2d, 0xcc, 0x2a, 0xd3, 0xc2, 0x0f, 0x5f, 0x4f, 0x67, 0xd0, 0xfe, 0x38, 0x39, + 0x4d, 0x53, 0x59, 0x54, 0x1d, 0x4b, 0xc2, 0xf0, 0x13, 0xcc, 0x30, 0x0f, 0x61, 0x36, 0x0c, 0xe4, + 0x4d, 0x87, 0x05, 0xe8, 0x36, 0xe4, 0x3b, 0xfc, 0x83, 0x55, 0x0c, 0x61, 0x60, 0x3d, 0x9d, 0x81, + 0xa1, 0x8c, 0xe6, 0x9c, 0xb2, 0x27, 0x2f, 0x3e, 0x19, 0x56, 0xd2, 0xcc, 0x9f, 0x4f, 0x43, 0x69, + 0x7d, 0x6f, 0x1b, 0x13, 0xe6, 0xf5, 0x69, 0x9b, 0xa4, 0x48, 0x9a, 0x35, 0x00, 0xfe, 0x97, 0xf9, + 0x56, 0x9b, 0xd8, 0x95, 0xcc, 0xb2, 0xb1, 0x52, 0x68, 0x22, 0x45, 0x07, 0x6f, 0x47, 0x18, 0xac, + 0x51, 0x71, 0xa9, 0xc7, 0x8e, 0x6b, 0x8b, 0x68, 0x6b, 0x52, 0xdf, 0x72, 0x5c, 0x1b, 0x0b, 0x0c, + 0xba, 0x09, 0xb9, 0x13, 0x42, 0x0f, 0xb8, 0xff, 0x79, 0x42, 0x7c, 0x31, 0xdd, 0xf1, 0x6e, 0x73, + 0x96, 0x66, 0xf1, 0x6c, 0x50, 0xcb, 0x89, 0x9f, 0x58, 0x0a, 0x41, 0x75, 0x00, 0x76, 0xe4, 0xd1, + 0x40, 0x98, 0x53, 0xc9, 0x2d, 0x67, 0x57, 0x8a, 0xcd, 0x39, 0x6e, 0xdf, 0x7e, 0x04, 0xc5, 0x1a, + 0x05, 0xba, 0x06, 0xb3, 0xcc, 0x71, 0x3b, 0xfd, 0xae, 0x45, 0x39, 0xa0, 0x92, 0x17, 0x76, 0x5e, + 0x51, 0x76, 0xce, 0xee, 0x6b, 0x38, 0x9c, 0xa0, 0xe4, 0x9a, 0xda, 0x56, 0x40, 0x3a, 0x1e, 0x75, + 0x08, 0xab, 0xcc, 0xc4, 0x9a, 0x36, 0x22, 0x28, 0xd6, 0x28, 0xd0, 0x0b, 0x90, 0x13, 0x9e, 0xaf, + 0x14, 0x84, 0x8a, 0xb2, 0x52, 0x91, 0x13, 0x61, 0xc1, 0x12, 0x87, 0x5e, 0x82, 0x19, 0x75, 0x6b, + 0x2a, 0x45, 0x41, 0x36, 0xaf, 0xc8, 0x66, 0xc2, 0xb4, 0x0e, 0xf1, 0xe8, 0x9b, 0x80, 0x58, 0xe0, + 0x51, 0xab, 0x43, 0x14, 0xea, 0x4d, 0x8b, 0x1d, 0x55, 0x40, 0x70, 0x55, 0x15, 0x17, 0xda, 0x1f, + 0xa1, 0xc0, 0x17, 0x70, 0x99, 0xbf, 0x37, 0x60, 0x5e, 0xcb, 0x05, 0x91, 0x77, 0xd7, 0x60, 0xb6, + 0xa3, 0xdd, 0x3a, 0x95, 0x17, 0x91, 0x67, 0xf4, 0x1b, 0x89, 0x13, 0x94, 0x88, 0x40, 0x91, 0x2a, + 0x49, 0x61, 0x75, 0x59, 0x4d, 0x9d, 0xb4, 0xa1, 0x0d, 0xb1, 0x26, 0x0d, 0xc8, 0x70, 0x2c, 0xd9, + 0xfc, 0xbb, 0x21, 0x12, 0x38, 0xac, 0x37, 0x68, 0x45, 0xab, 0x69, 0x86, 0x08, 0xc7, 0xec, 0x98, + 0x7a, 0x74, 0x49, 0x21, 0xc8, 0xfc, 0x5f, 0x14, 0x82, 0xeb, 0x85, 0x5f, 0x7d, 0x50, 0x9b, 0x7a, + 0xef, 0x2f, 0xcb, 0x53, 0x66, 0x0f, 0xca, 0x1b, 0x94, 0x58, 0x01, 0xd9, 0xf5, 0x03, 0x71, 0x00, + 0x13, 0xf2, 0x36, 0x3d, 0xc5, 0x7d, 0x57, 0x1d, 0x14, 0xf8, 0xfd, 0x6e, 0x09, 0x08, 0x56, 0x18, + 0x1e, 0xbf, 0x43, 0x87, 0x74, 0xed, 0x1d, 0xcb, 0xb5, 0x3a, 0x84, 0xaa, 0x1b, 0x18, 0x79, 0x75, + 0x4b, 0xc3, 0xe1, 0x04, 0xa5, 0xf9, 0xd3, 0x2c, 0x94, 0x5b, 0xa4, 0x4b, 0x62, 0x7d, 0x5b, 0x80, 0x3a, 0xd4, 0x6a, 0x93, 0x3d, 0x42, 0x1d, 0xcf, 0xde, 0x27, 0x6d, 0xcf, 0xb5, 0x99, 0xc8, 0x88, - 0x6c, 0xf3, 0x53, 0x3c, 0xcf, 0x6e, 0x8c, 0x60, 0xf1, 0x05, 0x1c, 0xa8, 0x0b, 0x65, 0x9f, 0x8a, - 0x67, 0x27, 0x50, 0xbd, 0x87, 0x9f, 0xf9, 0xd7, 0xd2, 0xb9, 0x7a, 0x4f, 0x67, 0x6d, 0x2e, 0x9e, - 0x0d, 0x6a, 0xe5, 0x04, 0x08, 0x27, 0x85, 0xa3, 0xaf, 0xc3, 0x82, 0x47, 0xfd, 0x23, 0xcb, 0x6d, - 0x11, 0x9f, 0xb8, 0x36, 0x71, 0x03, 0x26, 0xbc, 0x50, 0x68, 0x5e, 0xe1, 0x1d, 0x63, 0x77, 0x08, - 0x87, 0x47, 0xa8, 0xd1, 0x5d, 0x58, 0xf4, 0xa9, 0xe7, 0x5b, 0x1d, 0x8b, 0x4b, 0xdc, 0xf3, 0xba, - 0x4e, 0xfb, 0x54, 0xd4, 0xa9, 0x62, 0xf3, 0xea, 0xd9, 0xa0, 0xb6, 0xb8, 0x37, 0x8c, 0x3c, 0x1f, - 0xd4, 0x9e, 0x13, 0xae, 0xe3, 0x90, 0x18, 0x89, 0x47, 0xc5, 0x68, 0x31, 0xcc, 0x8d, 0x8b, 0xa1, - 0xb9, 0x0d, 0x85, 0x56, 0x9f, 0x0a, 0x2e, 0xf4, 0x55, 0x28, 0xd8, 0xea, 0x59, 0x79, 0xfe, 0xc5, - 0xb0, 0xe5, 0x86, 0x34, 0xe7, 0x83, 0x5a, 0x99, 0x0f, 0x09, 0xf5, 0x10, 0x80, 0x23, 0x16, 0xf3, - 0x1e, 0x94, 0x37, 0x1f, 0xf8, 0x1e, 0x0d, 0xc2, 0x98, 0xbe, 0x0c, 0x79, 0x22, 0x00, 0x42, 0x5a, - 0x21, 0xee, 0x13, 0x92, 0x0c, 0x2b, 0x2c, 0xaf, 0x5b, 0xe4, 0x81, 0xd5, 0x0e, 0x54, 0xc1, 0x8f, - 0xea, 0xd6, 0x26, 0x07, 0x62, 0x89, 0x33, 0x3f, 0x30, 0x20, 0x2f, 0x32, 0x8a, 0xa1, 0x5b, 0x90, - 0xed, 0x59, 0xbe, 0x6a, 0x56, 0xaf, 0xa7, 0x8b, 0xac, 0x64, 0xad, 0xef, 0x58, 0xfe, 0xa6, 0x1b, - 0xd0, 0xd3, 0x66, 0x49, 0x29, 0xc9, 0xee, 0x58, 0x3e, 0xe6, 0xe2, 0xaa, 0x36, 0x14, 0x42, 0x2c, - 0x5a, 0x80, 0xec, 0x31, 0x39, 0x95, 0x05, 0x09, 0xf3, 0x47, 0xd4, 0x84, 0xdc, 0x89, 0xd5, 0xed, - 0x13, 0x95, 0x4f, 0x57, 0x27, 0xd1, 0x8a, 0x25, 0xeb, 0xf5, 0xcc, 0x35, 0xc3, 0xdc, 0x05, 0xb8, - 0x41, 0x22, 0x0f, 0xad, 0xc3, 0x7c, 0x58, 0x6d, 0x92, 0x45, 0xf0, 0xd3, 0xca, 0xbc, 0x79, 0x9c, - 0x44, 0xe3, 0x61, 0x7a, 0xf3, 0x1e, 0x14, 0x45, 0xa1, 0xe4, 0xfd, 0x2e, 0xee, 0x00, 0xc6, 0x53, - 0x3a, 0x40, 0xd8, 0x30, 0x33, 0xe3, 0x1a, 0xa6, 0x56, 0x17, 0xba, 0x50, 0x96, 0xbc, 0x61, 0x0f, - 0x4f, 0xa5, 0xe1, 0x2a, 0x14, 0x42, 0x33, 0x95, 0x96, 0x68, 0x76, 0x0b, 0x05, 0xe1, 0x88, 0x42, - 0xd3, 0x76, 0x04, 0x89, 0xa2, 0x9f, 0x4e, 0x99, 0xd6, 0xd0, 0x32, 0x4f, 0x6f, 0x68, 0x9a, 0xa6, - 0x1f, 0x42, 0x65, 0xdc, 0xc0, 0xf7, 0x0c, 0x6d, 0x29, 0xbd, 0x29, 0xe6, 0x2f, 0x0d, 0x58, 0xd0, - 0x25, 0xa5, 0x0f, 0x5f, 0x7a, 0x25, 0x97, 0x8f, 0x46, 0x9a, 0x47, 0x7e, 0x6d, 0xc0, 0x95, 0xc4, - 0xa7, 0x4d, 0x14, 0xf1, 0x09, 0x8c, 0xd2, 0x93, 0x23, 0x3b, 0x41, 0x72, 0x34, 0xa0, 0xb4, 0xed, - 0x3a, 0x81, 0x63, 0x75, 0x9d, 0xef, 0x13, 0x7a, 0xf9, 0x30, 0x69, 0xfe, 0xd1, 0x80, 0x59, 0x8d, - 0x83, 0xa1, 0x7b, 0x30, 0xc3, 0xeb, 0xae, 0xe3, 0x76, 0x54, 0xed, 0x48, 0x39, 0x33, 0x68, 0x42, - 0xe2, 0xef, 0xda, 0x93, 0x92, 0x70, 0x28, 0x12, 0xed, 0x41, 0x9e, 0x12, 0xd6, 0xef, 0x06, 0x93, + 0x6c, 0xf3, 0x33, 0x3c, 0xcf, 0x6e, 0x8c, 0x60, 0xf1, 0x05, 0x1c, 0xa8, 0x0b, 0x65, 0x9f, 0x8a, + 0xdf, 0x4e, 0xa0, 0x7a, 0x0f, 0xbf, 0xf3, 0xaf, 0xa4, 0x73, 0xf5, 0x9e, 0xce, 0xda, 0x5c, 0x3c, + 0x1b, 0xd4, 0xca, 0x09, 0x10, 0x4e, 0x0a, 0x47, 0xdf, 0x80, 0x05, 0x8f, 0xfa, 0x47, 0x96, 0xdb, + 0x22, 0x3e, 0x71, 0x6d, 0xe2, 0x06, 0x4c, 0x78, 0xa1, 0xd0, 0xbc, 0xc2, 0x3b, 0xc6, 0xee, 0x10, + 0x0e, 0x8f, 0x50, 0xa3, 0x3b, 0xb0, 0xe8, 0x53, 0xcf, 0xb7, 0x3a, 0x16, 0x97, 0xb8, 0xe7, 0x75, + 0x9d, 0xf6, 0xa9, 0xa8, 0x53, 0xc5, 0xe6, 0xd5, 0xb3, 0x41, 0x6d, 0x71, 0x6f, 0x18, 0x79, 0x3e, + 0xa8, 0x3d, 0x23, 0x5c, 0xc7, 0x21, 0x31, 0x12, 0x8f, 0x8a, 0xd1, 0x62, 0x98, 0x1b, 0x17, 0x43, + 0x73, 0x1b, 0x0a, 0xad, 0x3e, 0x15, 0x5c, 0xe8, 0x75, 0x28, 0xd8, 0xea, 0xb7, 0xf2, 0xfc, 0xf3, + 0x61, 0xcb, 0x0d, 0x69, 0xce, 0x07, 0xb5, 0x32, 0x1f, 0x12, 0xea, 0x21, 0x00, 0x47, 0x2c, 0xe6, + 0x5d, 0x28, 0x6f, 0xde, 0xf7, 0x3d, 0x1a, 0x84, 0x31, 0x7d, 0x11, 0xf2, 0x44, 0x00, 0x84, 0xb4, + 0x42, 0xdc, 0x27, 0x24, 0x19, 0x56, 0x58, 0x5e, 0xb7, 0xc8, 0x7d, 0xab, 0x1d, 0xa8, 0x82, 0x1f, + 0xd5, 0xad, 0x4d, 0x0e, 0xc4, 0x12, 0x67, 0x7e, 0x68, 0x40, 0x5e, 0x64, 0x14, 0x43, 0xef, 0x40, + 0xb6, 0x67, 0xf9, 0xaa, 0x59, 0xbd, 0x9a, 0x2e, 0xb2, 0x92, 0xb5, 0xbe, 0x63, 0xf9, 0x9b, 0x6e, + 0x40, 0x4f, 0x9b, 0x25, 0xa5, 0x24, 0xbb, 0x63, 0xf9, 0x98, 0x8b, 0xab, 0xda, 0x50, 0x08, 0xb1, + 0x68, 0x01, 0xb2, 0xc7, 0xe4, 0x54, 0x16, 0x24, 0xcc, 0x7f, 0xa2, 0x26, 0xe4, 0x4e, 0xac, 0x6e, + 0x9f, 0xa8, 0x7c, 0xba, 0x3a, 0x89, 0x56, 0x2c, 0x59, 0xaf, 0x67, 0xae, 0x19, 0xe6, 0x2e, 0xc0, + 0x0d, 0x12, 0x79, 0x68, 0x1d, 0xe6, 0xc3, 0x6a, 0x93, 0x2c, 0x82, 0x9f, 0x55, 0xe6, 0xcd, 0xe3, + 0x24, 0x1a, 0x0f, 0xd3, 0x9b, 0x77, 0xa1, 0x28, 0x0a, 0x25, 0xef, 0x77, 0x71, 0x07, 0x30, 0x9e, + 0xd0, 0x01, 0xc2, 0x86, 0x99, 0x19, 0xd7, 0x30, 0xb5, 0xba, 0xd0, 0x85, 0xb2, 0xe4, 0x0d, 0x7b, + 0x78, 0x2a, 0x0d, 0x57, 0xa1, 0x10, 0x9a, 0xa9, 0xb4, 0x44, 0xb3, 0x5b, 0x28, 0x08, 0x47, 0x14, + 0x9a, 0xb6, 0x23, 0x48, 0x14, 0xfd, 0x74, 0xca, 0xb4, 0x86, 0x96, 0x79, 0x72, 0x43, 0xd3, 0x34, + 0xfd, 0x08, 0x2a, 0xe3, 0x06, 0xbe, 0xa7, 0x68, 0x4b, 0xe9, 0x4d, 0x31, 0xdf, 0x37, 0x60, 0x41, + 0x97, 0x94, 0x3e, 0x7c, 0xe9, 0x95, 0x5c, 0x3e, 0x1a, 0x69, 0x1e, 0xf9, 0xb5, 0x01, 0x57, 0x12, + 0x47, 0x9b, 0x28, 0xe2, 0x13, 0x18, 0xa5, 0x27, 0x47, 0x76, 0x82, 0xe4, 0x68, 0x40, 0x69, 0xdb, + 0x75, 0x02, 0xc7, 0xea, 0x3a, 0x3f, 0x20, 0xf4, 0xf2, 0x61, 0xd2, 0xfc, 0xa3, 0x01, 0xb3, 0x1a, + 0x07, 0x43, 0x77, 0x61, 0x86, 0xd7, 0x5d, 0xc7, 0xed, 0xa8, 0xda, 0x91, 0x72, 0x66, 0xd0, 0x84, + 0xc4, 0xe7, 0xda, 0x93, 0x92, 0x70, 0x28, 0x12, 0xed, 0x41, 0x9e, 0x12, 0xd6, 0xef, 0x06, 0x93, 0x95, 0x88, 0xfd, 0xc0, 0x0a, 0xfa, 0x4c, 0xd6, 0x66, 0x2c, 0xf8, 0xb1, 0x92, 0x63, 0xfe, 0x39, - 0x03, 0xe5, 0x9b, 0xd6, 0x01, 0xe9, 0xee, 0x93, 0x2e, 0x69, 0x07, 0x1e, 0x45, 0x3f, 0x80, 0x52, + 0x03, 0xe5, 0x9b, 0xd6, 0x01, 0xe9, 0xee, 0x93, 0x2e, 0x69, 0x07, 0x1e, 0x45, 0x3f, 0x84, 0x52, 0xcf, 0x0a, 0xda, 0x47, 0x02, 0x1a, 0x8e, 0xeb, 0xad, 0x74, 0x8a, 0x12, 0x92, 0xea, 0x3b, 0xb1, - 0x18, 0x59, 0x10, 0x9f, 0x53, 0x1f, 0x56, 0xd2, 0x30, 0x58, 0xd7, 0x26, 0xee, 0x58, 0xe2, 0x7d, - 0xf3, 0x81, 0xcf, 0x67, 0x89, 0xc9, 0xaf, 0x76, 0x09, 0x13, 0x30, 0x79, 0xb7, 0xef, 0x50, 0xd2, - 0x23, 0x6e, 0x10, 0xdf, 0xb1, 0x76, 0x86, 0xe4, 0xe3, 0x11, 0x8d, 0xd5, 0x37, 0x61, 0x61, 0xd8, - 0xf8, 0x0b, 0xea, 0xf5, 0x15, 0xbd, 0x5e, 0x17, 0xf5, 0x0a, 0xfc, 0x5b, 0x03, 0x2a, 0xe3, 0x0c, - 0x41, 0x9f, 0xd1, 0x04, 0xc5, 0x3d, 0xe2, 0x6d, 0x72, 0x2a, 0xa5, 0x6e, 0x42, 0xc1, 0xf3, 0xf9, - 0xad, 0xd8, 0xa3, 0x2a, 0xcf, 0x5f, 0x09, 0x73, 0x77, 0x57, 0xc1, 0xcf, 0x07, 0xb5, 0xe7, 0x13, - 0xe2, 0x43, 0x04, 0x8e, 0x58, 0x79, 0x63, 0x16, 0xf6, 0xf0, 0x61, 0x21, 0x6a, 0xcc, 0x77, 0x04, - 0x04, 0x2b, 0x8c, 0xf9, 0x07, 0x03, 0xa6, 0xc5, 0x94, 0x7c, 0x0f, 0x0a, 0xdc, 0x7f, 0xb6, 0x15, - 0x58, 0xc2, 0xae, 0xd4, 0xf7, 0x33, 0xce, 0xbd, 0x43, 0x02, 0x2b, 0x3e, 0x5f, 0x21, 0x04, 0x47, - 0x12, 0x11, 0x86, 0x9c, 0x13, 0x90, 0x5e, 0x18, 0xc8, 0x57, 0xc7, 0x8a, 0x56, 0xdb, 0x81, 0x3a, - 0xb6, 0xee, 0x6f, 0x3e, 0x08, 0x88, 0xcb, 0x83, 0x11, 0x17, 0x83, 0x6d, 0x2e, 0x03, 0x4b, 0x51, - 0xe6, 0xef, 0x0c, 0x88, 0x54, 0xf1, 0xe3, 0xce, 0x48, 0xf7, 0xf0, 0xa6, 0xe3, 0x1e, 0x2b, 0xb7, - 0x46, 0xe6, 0xec, 0x2b, 0x38, 0x8e, 0x28, 0x2e, 0x6a, 0x88, 0x99, 0xc9, 0x1a, 0x22, 0x57, 0xd8, - 0xf6, 0xdc, 0xc0, 0x71, 0xfb, 0x23, 0xf5, 0x65, 0x43, 0xc1, 0x71, 0x44, 0x61, 0xfe, 0x2b, 0x03, - 0x25, 0x6e, 0x6b, 0xd8, 0x91, 0xbf, 0x0c, 0xe5, 0xae, 0x1e, 0x3d, 0x65, 0xf3, 0xf3, 0x4a, 0x44, - 0xf2, 0x3c, 0xe2, 0x24, 0x2d, 0x67, 0x16, 0x63, 0x6e, 0xc4, 0x9c, 0x49, 0x32, 0x6f, 0xe9, 0x48, - 0x9c, 0xa4, 0xe5, 0x75, 0xf6, 0x3e, 0xcf, 0x6b, 0x35, 0x40, 0x46, 0xae, 0xfd, 0x26, 0x07, 0x62, - 0x89, 0xbb, 0xc8, 0x3f, 0xd3, 0x13, 0xfa, 0xe7, 0x3a, 0xcc, 0xf1, 0x40, 0x7a, 0xfd, 0x20, 0x9c, - 0xb2, 0x73, 0x62, 0xd6, 0x43, 0x67, 0x83, 0xda, 0xdc, 0xad, 0x04, 0x06, 0x0f, 0x51, 0x72, 0x1b, + 0x18, 0x59, 0x10, 0x9f, 0x51, 0x07, 0x2b, 0x69, 0x18, 0xac, 0x6b, 0x13, 0x6f, 0x2c, 0xf1, 0xbd, + 0x79, 0xdf, 0xe7, 0xb3, 0xc4, 0xe4, 0x4f, 0xbb, 0x84, 0x09, 0x98, 0xbc, 0xdb, 0x77, 0x28, 0xe9, + 0x11, 0x37, 0x88, 0xdf, 0x58, 0x3b, 0x43, 0xf2, 0xf1, 0x88, 0xc6, 0xea, 0x1b, 0xb0, 0x30, 0x6c, + 0xfc, 0x05, 0xf5, 0xfa, 0x8a, 0x5e, 0xaf, 0x8b, 0x7a, 0x05, 0xfe, 0xad, 0x01, 0x95, 0x71, 0x86, + 0xa0, 0xcf, 0x69, 0x82, 0xe2, 0x1e, 0xf1, 0x16, 0x39, 0x95, 0x52, 0x37, 0xa1, 0xe0, 0xf9, 0xfc, + 0x55, 0xec, 0x51, 0x95, 0xe7, 0x2f, 0x85, 0xb9, 0xbb, 0xab, 0xe0, 0xe7, 0x83, 0xda, 0xb3, 0x09, + 0xf1, 0x21, 0x02, 0x47, 0xac, 0xbc, 0x31, 0x0b, 0x7b, 0xf8, 0xb0, 0x10, 0x35, 0xe6, 0xdb, 0x02, + 0x82, 0x15, 0xc6, 0xfc, 0x83, 0x01, 0xd3, 0x62, 0x4a, 0xbe, 0x0b, 0x05, 0xee, 0x3f, 0xdb, 0x0a, + 0x2c, 0x61, 0x57, 0xea, 0xf7, 0x19, 0xe7, 0xde, 0x21, 0x81, 0x15, 0xdf, 0xaf, 0x10, 0x82, 0x23, + 0x89, 0x08, 0x43, 0xce, 0x09, 0x48, 0x2f, 0x0c, 0xe4, 0xcb, 0x63, 0x45, 0xab, 0xed, 0x40, 0x1d, + 0x5b, 0xf7, 0x36, 0xef, 0x07, 0xc4, 0xe5, 0xc1, 0x88, 0x8b, 0xc1, 0x36, 0x97, 0x81, 0xa5, 0x28, + 0xf3, 0x77, 0x06, 0x44, 0xaa, 0xf8, 0x75, 0x67, 0xa4, 0x7b, 0x78, 0xd3, 0x71, 0x8f, 0x95, 0x5b, + 0x23, 0x73, 0xf6, 0x15, 0x1c, 0x47, 0x14, 0x17, 0x35, 0xc4, 0xcc, 0x64, 0x0d, 0x91, 0x2b, 0x6c, + 0x7b, 0x6e, 0xe0, 0xb8, 0xfd, 0x91, 0xfa, 0xb2, 0xa1, 0xe0, 0x38, 0xa2, 0x30, 0xff, 0x95, 0x81, + 0x12, 0xb7, 0x35, 0xec, 0xc8, 0x5f, 0x85, 0x72, 0x57, 0x8f, 0x9e, 0xb2, 0xf9, 0x59, 0x25, 0x22, + 0x79, 0x1f, 0x71, 0x92, 0x96, 0x33, 0x8b, 0x31, 0x37, 0x62, 0xce, 0x24, 0x99, 0xb7, 0x74, 0x24, + 0x4e, 0xd2, 0xf2, 0x3a, 0x7b, 0x8f, 0xe7, 0xb5, 0x1a, 0x20, 0x23, 0xd7, 0x7e, 0x8b, 0x03, 0xb1, + 0xc4, 0x5d, 0xe4, 0x9f, 0xe9, 0x09, 0xfd, 0x73, 0x1d, 0xe6, 0x78, 0x20, 0xbd, 0x7e, 0x10, 0x4e, + 0xd9, 0x39, 0x31, 0xeb, 0xa1, 0xb3, 0x41, 0x6d, 0xee, 0x9d, 0x04, 0x06, 0x0f, 0x51, 0x72, 0x1b, 0xbb, 0x4e, 0xcf, 0x09, 0x2a, 0x33, 0x82, 0x25, 0xb2, 0xf1, 0x26, 0x07, 0x62, 0x89, 0x4b, 0x04, - 0xa0, 0x70, 0x69, 0x00, 0xfe, 0x91, 0x01, 0x24, 0xaf, 0x05, 0xb6, 0x9c, 0x96, 0xe4, 0x89, 0x7e, - 0x05, 0x66, 0x7a, 0xea, 0x5a, 0x61, 0x24, 0x1b, 0x4a, 0x78, 0xa3, 0x08, 0xf1, 0x68, 0x07, 0x8a, - 0xf2, 0x64, 0xc5, 0xd9, 0xd2, 0x50, 0xc4, 0xc5, 0xdd, 0x10, 0x71, 0x3e, 0xa8, 0x55, 0x13, 0x6a, - 0x22, 0xcc, 0xad, 0x53, 0x9f, 0xe0, 0x58, 0x02, 0x5a, 0x03, 0xb0, 0x7c, 0x47, 0xdf, 0x21, 0x15, - 0xe3, 0x1d, 0x44, 0x7c, 0x1b, 0xc4, 0x1a, 0x15, 0x7a, 0x0b, 0xa6, 0xb9, 0xa7, 0xd4, 0x82, 0xe1, - 0x73, 0xe9, 0xce, 0x27, 0xf7, 0x75, 0xb3, 0xc0, 0x9b, 0x16, 0x7f, 0xc2, 0x42, 0x02, 0xba, 0x0b, - 0x79, 0x91, 0x16, 0x32, 0x2a, 0x13, 0x0e, 0x9a, 0xe2, 0xd6, 0xa1, 0xa6, 0xe4, 0xf3, 0xe8, 0x09, - 0x2b, 0x89, 0xe6, 0xbb, 0x50, 0xdc, 0x71, 0xda, 0xd4, 0xe3, 0xea, 0xb8, 0x83, 0x59, 0xe2, 0x96, - 0x15, 0x39, 0x38, 0x0c, 0x7e, 0x88, 0xe7, 0x51, 0x77, 0x2d, 0xd7, 0x93, 0x77, 0xa9, 0x5c, 0x1c, - 0xf5, 0x77, 0x38, 0x10, 0x4b, 0xdc, 0xf5, 0x2b, 0xbc, 0x51, 0xff, 0xf4, 0x51, 0x6d, 0xea, 0xe1, - 0xa3, 0xda, 0xd4, 0xfb, 0x8f, 0x54, 0xd3, 0xfe, 0x5b, 0x09, 0x60, 0xf7, 0xe0, 0x7b, 0xa4, 0x2d, - 0x8b, 0xc1, 0xe5, 0x1b, 0x20, 0x3e, 0x7c, 0xa9, 0xc5, 0xa3, 0xd8, 0x96, 0x64, 0x86, 0x86, 0x2f, - 0x0d, 0x87, 0x13, 0x94, 0xa8, 0x01, 0xc5, 0x68, 0x2b, 0xa4, 0xc2, 0xb6, 0x18, 0xa6, 0x41, 0xb4, - 0x3a, 0xc2, 0x31, 0x4d, 0xa2, 0x32, 0x4d, 0x5f, 0x5a, 0x99, 0x9a, 0x90, 0xed, 0x3b, 0xb6, 0x88, - 0x4a, 0xb1, 0xf9, 0x85, 0xb0, 0x33, 0xdc, 0xde, 0x6e, 0x9d, 0x0f, 0x6a, 0x2f, 0x8e, 0x5b, 0xa9, - 0x06, 0xa7, 0x3e, 0x61, 0xf5, 0xdb, 0xdb, 0x2d, 0xcc, 0x99, 0x2f, 0x3a, 0xbd, 0xf9, 0x09, 0x4f, - 0xef, 0x1a, 0x80, 0xfa, 0x6a, 0xce, 0x2d, 0x8f, 0x61, 0x94, 0x9d, 0x37, 0x22, 0x0c, 0xd6, 0xa8, - 0x10, 0x83, 0xc5, 0x36, 0xbf, 0xdc, 0xf3, 0x64, 0x77, 0x7a, 0x84, 0x05, 0x56, 0x4f, 0xee, 0x88, - 0x26, 0x4b, 0xd5, 0x17, 0x94, 0x9a, 0xc5, 0x8d, 0x61, 0x61, 0x78, 0x54, 0x3e, 0xf2, 0x60, 0xd1, - 0x56, 0xd7, 0xd4, 0x58, 0x69, 0x71, 0x62, 0xa5, 0xcf, 0x73, 0x85, 0xad, 0x61, 0x41, 0x78, 0x54, - 0x36, 0xfa, 0x0e, 0x54, 0x43, 0xe0, 0xe8, 0xae, 0x40, 0x6c, 0xad, 0xb2, 0xcd, 0xa5, 0xb3, 0x41, - 0xad, 0xda, 0x1a, 0x4b, 0x85, 0x9f, 0x22, 0x01, 0xd9, 0x90, 0xef, 0xca, 0xb1, 0xab, 0x24, 0x5a, - 0xe5, 0x57, 0xd2, 0x7d, 0x45, 0x9c, 0xfd, 0x75, 0x7d, 0xdc, 0x8a, 0xee, 0xc2, 0x6a, 0xd2, 0x52, - 0xb2, 0xd1, 0x03, 0x28, 0x59, 0xae, 0xeb, 0x05, 0x96, 0xdc, 0x5e, 0xcc, 0x0a, 0x55, 0xeb, 0x13, - 0xab, 0x5a, 0x8f, 0x65, 0x0c, 0x8d, 0x77, 0x1a, 0x06, 0xeb, 0xaa, 0xd0, 0x7d, 0x98, 0xf7, 0xee, - 0xbb, 0x84, 0x62, 0x72, 0x48, 0x28, 0x71, 0xdb, 0x84, 0x55, 0xca, 0x42, 0xfb, 0x17, 0x53, 0x6a, - 0x4f, 0x30, 0xc7, 0x29, 0x9d, 0x84, 0x33, 0x3c, 0xac, 0x05, 0xd5, 0x01, 0x0e, 0x1d, 0x57, 0x0d, - 0xe9, 0x95, 0xb9, 0x78, 0xcd, 0xb9, 0x15, 0x41, 0xb1, 0x46, 0x81, 0x5e, 0x87, 0x52, 0xbb, 0xdb, - 0x67, 0x01, 0x91, 0xfb, 0xd4, 0x79, 0x71, 0x82, 0xa2, 0xef, 0xdb, 0x88, 0x51, 0x58, 0xa7, 0x43, - 0x47, 0x30, 0xeb, 0x68, 0xb7, 0x81, 0xca, 0x82, 0xc8, 0xc5, 0xb5, 0x89, 0xaf, 0x00, 0xac, 0xb9, + 0xa0, 0x70, 0x69, 0x00, 0xfe, 0x91, 0x01, 0x24, 0x9f, 0x05, 0xb6, 0x9c, 0x96, 0xe4, 0x8d, 0x7e, + 0x09, 0x66, 0x7a, 0xea, 0x59, 0x61, 0x24, 0x1b, 0x4a, 0xf8, 0xa2, 0x08, 0xf1, 0x68, 0x07, 0x8a, + 0xf2, 0x66, 0xc5, 0xd9, 0xd2, 0x50, 0xc4, 0xc5, 0xdd, 0x10, 0x71, 0x3e, 0xa8, 0x55, 0x13, 0x6a, + 0x22, 0xcc, 0x3b, 0xa7, 0x3e, 0xc1, 0xb1, 0x04, 0xb4, 0x06, 0x60, 0xf9, 0x8e, 0xbe, 0x43, 0x2a, + 0xc6, 0x3b, 0x88, 0xf8, 0x35, 0x88, 0x35, 0x2a, 0xf4, 0x26, 0x4c, 0x73, 0x4f, 0xa9, 0x05, 0xc3, + 0x17, 0xd2, 0xdd, 0x4f, 0xee, 0xeb, 0x66, 0x81, 0x37, 0x2d, 0xfe, 0x0b, 0x0b, 0x09, 0xe8, 0x0e, + 0xe4, 0x45, 0x5a, 0xc8, 0xa8, 0x4c, 0x38, 0x68, 0x8a, 0x57, 0x87, 0x9a, 0x92, 0xcf, 0xa3, 0x5f, + 0x58, 0x49, 0x34, 0xdf, 0x85, 0xe2, 0x8e, 0xd3, 0xa6, 0x1e, 0x57, 0xc7, 0x1d, 0xcc, 0x12, 0xaf, + 0xac, 0xc8, 0xc1, 0x61, 0xf0, 0x43, 0x3c, 0x8f, 0xba, 0x6b, 0xb9, 0x9e, 0x7c, 0x4b, 0xe5, 0xe2, + 0xa8, 0xbf, 0xcd, 0x81, 0x58, 0xe2, 0xae, 0x5f, 0xe1, 0x8d, 0xfa, 0x67, 0x0f, 0x6b, 0x53, 0x0f, + 0x1e, 0xd6, 0xa6, 0x3e, 0x78, 0xa8, 0x9a, 0xf6, 0xdf, 0x4a, 0x00, 0xbb, 0x07, 0xdf, 0x27, 0x6d, + 0x59, 0x0c, 0x2e, 0xdf, 0x00, 0xf1, 0xe1, 0x4b, 0x2d, 0x1e, 0xc5, 0xb6, 0x24, 0x33, 0x34, 0x7c, + 0x69, 0x38, 0x9c, 0xa0, 0x44, 0x0d, 0x28, 0x46, 0x5b, 0x21, 0x15, 0xb6, 0xc5, 0x30, 0x0d, 0xa2, + 0xd5, 0x11, 0x8e, 0x69, 0x12, 0x95, 0x69, 0xfa, 0xd2, 0xca, 0xd4, 0x84, 0x6c, 0xdf, 0xb1, 0x45, + 0x54, 0x8a, 0xcd, 0x2f, 0x85, 0x9d, 0xe1, 0xd6, 0x76, 0xeb, 0x7c, 0x50, 0x7b, 0x7e, 0xdc, 0x4a, + 0x35, 0x38, 0xf5, 0x09, 0xab, 0xdf, 0xda, 0x6e, 0x61, 0xce, 0x7c, 0xd1, 0xed, 0xcd, 0x4f, 0x78, + 0x7b, 0xd7, 0x00, 0xd4, 0xa9, 0x39, 0xb7, 0xbc, 0x86, 0x51, 0x76, 0xde, 0x88, 0x30, 0x58, 0xa3, + 0x42, 0x0c, 0x16, 0xdb, 0xfc, 0x71, 0xcf, 0x93, 0xdd, 0xe9, 0x11, 0x16, 0x58, 0x3d, 0xb9, 0x23, + 0x9a, 0x2c, 0x55, 0x9f, 0x53, 0x6a, 0x16, 0x37, 0x86, 0x85, 0xe1, 0x51, 0xf9, 0xc8, 0x83, 0x45, + 0x5b, 0x3d, 0x53, 0x63, 0xa5, 0xc5, 0x89, 0x95, 0x3e, 0xcb, 0x15, 0xb6, 0x86, 0x05, 0xe1, 0x51, + 0xd9, 0xe8, 0xbb, 0x50, 0x0d, 0x81, 0xa3, 0xbb, 0x02, 0xb1, 0xb5, 0xca, 0x36, 0x97, 0xce, 0x06, + 0xb5, 0x6a, 0x6b, 0x2c, 0x15, 0x7e, 0x82, 0x04, 0x64, 0x43, 0xbe, 0x2b, 0xc7, 0xae, 0x92, 0x68, + 0x95, 0x5f, 0x4b, 0x77, 0x8a, 0x38, 0xfb, 0xeb, 0xfa, 0xb8, 0x15, 0xbd, 0x85, 0xd5, 0xa4, 0xa5, + 0x64, 0xa3, 0xfb, 0x50, 0xb2, 0x5c, 0xd7, 0x0b, 0x2c, 0xb9, 0xbd, 0x98, 0x15, 0xaa, 0xd6, 0x27, + 0x56, 0xb5, 0x1e, 0xcb, 0x18, 0x1a, 0xef, 0x34, 0x0c, 0xd6, 0x55, 0xa1, 0x7b, 0x30, 0xef, 0xdd, + 0x73, 0x09, 0xc5, 0xe4, 0x90, 0x50, 0xe2, 0xb6, 0x09, 0xab, 0x94, 0x85, 0xf6, 0x2f, 0xa7, 0xd4, + 0x9e, 0x60, 0x8e, 0x53, 0x3a, 0x09, 0x67, 0x78, 0x58, 0x0b, 0xaa, 0x03, 0x1c, 0x3a, 0xae, 0x1a, + 0xd2, 0x2b, 0x73, 0xf1, 0x9a, 0x73, 0x2b, 0x82, 0x62, 0x8d, 0x02, 0xbd, 0x0a, 0xa5, 0x76, 0xb7, + 0xcf, 0x02, 0x22, 0xf7, 0xa9, 0xf3, 0xe2, 0x06, 0x45, 0xe7, 0xdb, 0x88, 0x51, 0x58, 0xa7, 0x43, + 0x47, 0x30, 0xeb, 0x68, 0xaf, 0x81, 0xca, 0x82, 0xc8, 0xc5, 0xb5, 0x89, 0x9f, 0x00, 0xac, 0xb9, 0xc0, 0x2b, 0x91, 0x0e, 0xc1, 0x09, 0xc9, 0xa8, 0x0f, 0xe5, 0x9e, 0xde, 0x6a, 0x2a, 0x8b, 0xc2, - 0x8f, 0xd7, 0xd2, 0xa9, 0x1a, 0x6d, 0x86, 0xf1, 0x00, 0x91, 0xc0, 0xe1, 0xa4, 0x96, 0xea, 0x97, - 0xa0, 0xf4, 0x1f, 0xce, 0xc4, 0x7c, 0xa6, 0x1e, 0xce, 0x98, 0x89, 0x66, 0xea, 0x0f, 0x32, 0x30, - 0x97, 0x8c, 0x73, 0x74, 0xf7, 0x34, 0xc6, 0xae, 0xe5, 0xc3, 0x66, 0x90, 0x1d, 0xdb, 0x0c, 0x54, - 0xcd, 0x9d, 0x7e, 0x96, 0x9a, 0x9b, 0x6c, 0xe7, 0xb9, 0x54, 0xed, 0xbc, 0x0e, 0xc0, 0xe7, 0x13, + 0x8f, 0xd7, 0xd2, 0xa9, 0x1a, 0x6d, 0x86, 0xf1, 0x00, 0x91, 0xc0, 0xe1, 0xa4, 0x96, 0xea, 0x57, + 0xa0, 0xf4, 0x1f, 0xce, 0xc4, 0x7c, 0xa6, 0x1e, 0xce, 0x98, 0x89, 0x66, 0xea, 0x0f, 0x33, 0x30, + 0x97, 0x8c, 0x73, 0xf4, 0xf6, 0x34, 0xc6, 0xae, 0xe5, 0xc3, 0x66, 0x90, 0x1d, 0xdb, 0x0c, 0x54, + 0xcd, 0x9d, 0x7e, 0x9a, 0x9a, 0x9b, 0x6c, 0xe7, 0xb9, 0x54, 0xed, 0xbc, 0x0e, 0xc0, 0xe7, 0x13, 0xea, 0x75, 0xbb, 0x84, 0x8a, 0x12, 0x5d, 0x50, 0x8b, 0xf7, 0x08, 0x8a, 0x35, 0x0a, 0xb4, 0x05, 0xe8, 0xa0, 0xeb, 0xb5, 0x8f, 0x85, 0x0b, 0xc2, 0xf2, 0x22, 0x8a, 0x73, 0x41, 0x2e, 0x2f, 0x9b, - 0x23, 0x58, 0x7c, 0x01, 0x87, 0x39, 0x03, 0xb9, 0x3d, 0x3e, 0xe6, 0x99, 0xbf, 0x30, 0x60, 0x56, - 0x3c, 0x4d, 0xb2, 0x8e, 0xad, 0x41, 0xee, 0xd0, 0x0b, 0x57, 0x2e, 0x05, 0xf9, 0xcf, 0xc5, 0x16, - 0x07, 0x60, 0x09, 0x7f, 0x86, 0x7d, 0xed, 0x2e, 0x24, 0xf7, 0xa0, 0xe8, 0x4d, 0x19, 0x19, 0x23, - 0x5a, 0x54, 0x4e, 0x16, 0x15, 0xf3, 0x2a, 0x14, 0xb1, 0xe7, 0x05, 0x7b, 0x56, 0x70, 0xc4, 0xb8, - 0xe1, 0x3e, 0x7f, 0x50, 0xdf, 0x26, 0x0c, 0x17, 0x18, 0x2c, 0xe1, 0xe6, 0xcf, 0x0d, 0x78, 0x61, - 0xec, 0x8a, 0x9b, 0x47, 0xb8, 0x1d, 0xbd, 0x29, 0x93, 0xa2, 0x08, 0xc7, 0x74, 0x58, 0xa3, 0xe2, - 0x93, 0x7a, 0x62, 0x2f, 0x3e, 0x3c, 0xa9, 0x27, 0xb4, 0xe1, 0x24, 0xad, 0xf9, 0xcf, 0x0c, 0xe4, - 0xe5, 0xb5, 0xfd, 0xbf, 0x7c, 0x39, 0x7b, 0x19, 0xf2, 0x4c, 0xe8, 0x51, 0xe6, 0x45, 0x4d, 0x43, - 0x6a, 0xc7, 0x0a, 0x2b, 0x86, 0x65, 0xc2, 0x98, 0xd5, 0x09, 0x0f, 0x53, 0x3c, 0x2c, 0x4b, 0x30, - 0x0e, 0xf1, 0xe8, 0x0d, 0xc8, 0x53, 0x62, 0xb1, 0xe8, 0xde, 0xb0, 0x14, 0x8a, 0xc4, 0x02, 0x7a, - 0x3e, 0xa8, 0xcd, 0x2a, 0xe1, 0xe2, 0x1d, 0x2b, 0x6a, 0x74, 0x17, 0x66, 0x6c, 0x12, 0x58, 0x4e, - 0x37, 0x1c, 0x4c, 0x5f, 0x9b, 0x64, 0xbd, 0xd1, 0x92, 0xac, 0xcd, 0x12, 0xb7, 0x49, 0xbd, 0xe0, - 0x50, 0x20, 0x2f, 0x04, 0x6d, 0xcf, 0x96, 0xff, 0x8c, 0xe5, 0xe2, 0x42, 0xb0, 0xe1, 0xd9, 0x04, - 0x0b, 0x8c, 0xf9, 0xd0, 0x80, 0x92, 0x94, 0xb4, 0x61, 0xf5, 0x19, 0x41, 0xab, 0xd1, 0x57, 0xc8, - 0x70, 0x87, 0xa3, 0xc9, 0x34, 0x1f, 0xe6, 0xcf, 0x07, 0xb5, 0xa2, 0x20, 0x13, 0x93, 0x7d, 0xf8, - 0x01, 0x9a, 0x8f, 0x32, 0x97, 0xf8, 0xe8, 0x25, 0xc8, 0x89, 0xec, 0x57, 0xce, 0x8c, 0xe6, 0x5d, - 0x71, 0x40, 0xb0, 0xc4, 0x99, 0x1f, 0x67, 0xa0, 0x9c, 0xf8, 0xb8, 0x14, 0xc3, 0x6d, 0xb4, 0x4a, - 0xcb, 0xa4, 0x58, 0xcf, 0x8e, 0xff, 0x3f, 0xf3, 0x5b, 0x90, 0x6f, 0xf3, 0xef, 0x0b, 0xff, 0x50, - 0x5e, 0x9d, 0x24, 0x14, 0xc2, 0x33, 0x71, 0x26, 0x89, 0x57, 0x86, 0x95, 0x40, 0x74, 0x03, 0x16, - 0x29, 0x09, 0xe8, 0xe9, 0xfa, 0x61, 0x40, 0xa8, 0x7e, 0x3f, 0xcc, 0xc5, 0xe3, 0x1f, 0x1e, 0x26, - 0xc0, 0xa3, 0x3c, 0x61, 0xe9, 0xce, 0x3f, 0x43, 0xe9, 0x36, 0xbb, 0x30, 0xfd, 0x3f, 0xbc, 0xaa, - 0x7c, 0x1b, 0x8a, 0xf1, 0x30, 0xf9, 0x09, 0xab, 0x34, 0xbf, 0x0b, 0x05, 0x9e, 0x8d, 0xe1, 0x25, - 0xe8, 0x92, 0xce, 0x98, 0xec, 0x59, 0x99, 0x34, 0x3d, 0xcb, 0xec, 0x41, 0xf9, 0xb6, 0x6f, 0x3f, - 0xe3, 0x3f, 0x78, 0x99, 0xd4, 0x1d, 0x61, 0x0d, 0xe4, 0xbf, 0xe2, 0xbc, 0x78, 0xcb, 0x05, 0x92, - 0x56, 0xbc, 0xf5, 0x6d, 0x90, 0xb6, 0xc1, 0xfd, 0xb1, 0x01, 0x20, 0xb6, 0x19, 0x9b, 0x27, 0xc4, - 0x0d, 0xb8, 0x1f, 0x78, 0xc0, 0x87, 0xfd, 0x20, 0x4e, 0xad, 0xc0, 0xa0, 0xdb, 0x90, 0xf7, 0xc4, - 0x4c, 0xab, 0x56, 0xaa, 0x13, 0x6e, 0xa7, 0xa2, 0x24, 0x97, 0x83, 0x31, 0x56, 0xc2, 0x9a, 0x2b, - 0x8f, 0x9f, 0x2c, 0x4d, 0x7d, 0xf8, 0x64, 0x69, 0xea, 0xa3, 0x27, 0x4b, 0x53, 0xef, 0x9d, 0x2d, - 0x19, 0x8f, 0xcf, 0x96, 0x8c, 0x0f, 0xcf, 0x96, 0x8c, 0x8f, 0xce, 0x96, 0x8c, 0x8f, 0xcf, 0x96, - 0x8c, 0x87, 0x7f, 0x5d, 0x9a, 0xba, 0x9b, 0x39, 0x59, 0xfd, 0x77, 0x00, 0x00, 0x00, 0xff, 0xff, - 0x1a, 0xd0, 0x3c, 0x48, 0x01, 0x24, 0x00, 0x00, + 0x23, 0x58, 0x7c, 0x01, 0x87, 0x39, 0x03, 0xb9, 0x3d, 0x3e, 0xe6, 0x99, 0xbf, 0x34, 0x60, 0x56, + 0xfc, 0x9a, 0x64, 0x1d, 0x5b, 0x83, 0xdc, 0xa1, 0x17, 0xae, 0x5c, 0x0a, 0xf2, 0x3f, 0x17, 0x5b, + 0x1c, 0x80, 0x25, 0xfc, 0x29, 0xf6, 0xb5, 0xef, 0x1b, 0x90, 0x5c, 0x84, 0xa2, 0x37, 0x64, 0x68, + 0x8c, 0x68, 0x53, 0x39, 0x61, 0x58, 0x5e, 0x1f, 0x37, 0xe8, 0x3f, 0x93, 0x6a, 0xeb, 0x75, 0x15, + 0x8a, 0xd8, 0xf3, 0x82, 0x3d, 0x2b, 0x38, 0x62, 0xfc, 0xe0, 0x3e, 0xff, 0xa1, 0x7c, 0x23, 0x0e, + 0x2e, 0x30, 0x58, 0xc2, 0xcd, 0x5f, 0x18, 0xf0, 0xdc, 0xd8, 0x15, 0x39, 0xcf, 0x90, 0x76, 0xf4, + 0xa5, 0x4e, 0x14, 0x65, 0x48, 0x4c, 0x87, 0x35, 0x2a, 0x3e, 0xe9, 0x27, 0xf6, 0xea, 0xc3, 0x93, + 0x7e, 0x42, 0x1b, 0x4e, 0xd2, 0x9a, 0xff, 0xcc, 0x40, 0x5e, 0x3e, 0xfb, 0xff, 0xcb, 0x8f, 0xbb, + 0x17, 0x21, 0xcf, 0x84, 0x1e, 0x65, 0x5e, 0xd4, 0x74, 0xa4, 0x76, 0xac, 0xb0, 0x62, 0xd8, 0x26, + 0x8c, 0x59, 0x9d, 0xf0, 0x32, 0xc6, 0xc3, 0xb6, 0x04, 0xe3, 0x10, 0x8f, 0x5e, 0x83, 0x3c, 0x25, + 0x16, 0x8b, 0xde, 0x1d, 0x4b, 0xa1, 0x48, 0x2c, 0xa0, 0xe7, 0x83, 0xda, 0xac, 0x12, 0x2e, 0xbe, + 0xb1, 0xa2, 0x46, 0x77, 0x60, 0xc6, 0x26, 0x81, 0xe5, 0x74, 0xc3, 0xc1, 0xf6, 0x95, 0x49, 0xd6, + 0x23, 0x2d, 0xc9, 0xda, 0x2c, 0x71, 0x9b, 0xd4, 0x07, 0x0e, 0x05, 0xf2, 0x42, 0xd2, 0xf6, 0x6c, + 0xf9, 0x9f, 0xb5, 0x5c, 0x5c, 0x48, 0x36, 0x3c, 0x9b, 0x60, 0x81, 0x31, 0x1f, 0x18, 0x50, 0x92, + 0x92, 0x36, 0xac, 0x3e, 0x23, 0x68, 0x35, 0x3a, 0x85, 0x0c, 0x77, 0x38, 0xda, 0x4c, 0xf3, 0xc7, + 0xc0, 0xf9, 0xa0, 0x56, 0x14, 0x64, 0xe2, 0x65, 0x10, 0x1e, 0x40, 0xf3, 0x51, 0xe6, 0x12, 0x1f, + 0xbd, 0x00, 0x39, 0x71, 0x7b, 0x94, 0x33, 0xa3, 0x79, 0x59, 0x5c, 0x30, 0x2c, 0x71, 0xe6, 0x27, + 0x19, 0x28, 0x27, 0x0e, 0x97, 0x62, 0x38, 0x8e, 0x56, 0x71, 0x99, 0x14, 0xeb, 0xdd, 0xf1, 0xff, + 0x0f, 0xfd, 0x36, 0xe4, 0xdb, 0xfc, 0x7c, 0xe1, 0x3f, 0xa4, 0x57, 0x27, 0x09, 0x85, 0xf0, 0x4c, + 0x9c, 0x49, 0xe2, 0x93, 0x61, 0x25, 0x10, 0xdd, 0x80, 0x45, 0x4a, 0x02, 0x7a, 0xba, 0x7e, 0x18, + 0x10, 0xaa, 0xbf, 0x2f, 0x73, 0xf1, 0xf8, 0x88, 0x87, 0x09, 0xf0, 0x28, 0x4f, 0x58, 0xfa, 0xf3, + 0x4f, 0x51, 0xfa, 0xcd, 0x2e, 0x4c, 0xff, 0x0f, 0x9f, 0x3a, 0xdf, 0x81, 0x62, 0x3c, 0x8c, 0x7e, + 0xca, 0x2a, 0xcd, 0xef, 0x41, 0x81, 0x67, 0x63, 0xf8, 0x88, 0xba, 0xa4, 0xb3, 0x26, 0x7b, 0x5e, + 0x26, 0x4d, 0xcf, 0x33, 0x7b, 0x50, 0xbe, 0xe5, 0xdb, 0x4f, 0xf9, 0x1f, 0xc0, 0x4c, 0xea, 0x8e, + 0xb2, 0x06, 0xf2, 0xbf, 0xea, 0xbc, 0x78, 0xcb, 0x05, 0x94, 0x56, 0xbc, 0xf5, 0x6d, 0x92, 0xb6, + 0x01, 0xfe, 0x89, 0x01, 0x20, 0xb6, 0x21, 0x9b, 0x27, 0xc4, 0x0d, 0xb8, 0x1f, 0x78, 0xc0, 0x87, + 0xfd, 0x20, 0x6e, 0xad, 0xc0, 0xa0, 0x5b, 0x90, 0xf7, 0xc4, 0x4c, 0xac, 0x56, 0xb2, 0x13, 0x6e, + 0xb7, 0xa2, 0x24, 0x97, 0x83, 0x35, 0x56, 0xc2, 0x9a, 0x2b, 0x8f, 0x1e, 0x2f, 0x4d, 0x7d, 0xf4, + 0x78, 0x69, 0xea, 0xe3, 0xc7, 0x4b, 0x53, 0xef, 0x9d, 0x2d, 0x19, 0x8f, 0xce, 0x96, 0x8c, 0x8f, + 0xce, 0x96, 0x8c, 0x8f, 0xcf, 0x96, 0x8c, 0x4f, 0xce, 0x96, 0x8c, 0x07, 0x7f, 0x5d, 0x9a, 0xba, + 0x93, 0x39, 0x59, 0xfd, 0x77, 0x00, 0x00, 0x00, 0xff, 0xff, 0xc3, 0x66, 0x55, 0x2c, 0x41, 0x24, + 0x00, 0x00, } diff --git a/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/generated.proto b/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/generated.proto index d07903ca77..36bda1fe59 100644 --- a/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/generated.proto +++ b/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/generated.proto @@ -754,6 +754,10 @@ message Preconditions { // Specifies the target UID. // +optional optional string uid = 1; + + // Specifies the target ResourceVersion + // +optional + optional string resourceVersion = 2; } // RootPaths lists the paths available at root. diff --git a/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/helpers.go b/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/helpers.go index b41d549a25..b4dc78b3ea 100644 --- a/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/helpers.go +++ b/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/helpers.go @@ -228,6 +228,12 @@ func NewUIDPreconditions(uid string) *Preconditions { return &Preconditions{UID: &u} } +// NewRVDeletionPrecondition returns a DeleteOptions with a ResourceVersion precondition set. +func NewRVDeletionPrecondition(rv string) *DeleteOptions { + p := Preconditions{ResourceVersion: &rv} + return &DeleteOptions{Preconditions: &p} +} + // HasObjectMetaSystemFieldValues returns true if fields that are managed by the system on ObjectMeta have values. func HasObjectMetaSystemFieldValues(meta Object) bool { return !meta.GetCreationTimestamp().Time.IsZero() || diff --git a/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/types.go b/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/types.go index 23cbc2c6df..fd6395256a 100644 --- a/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/types.go +++ b/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/types.go @@ -575,6 +575,9 @@ type Preconditions struct { // Specifies the target UID. // +optional UID *types.UID `json:"uid,omitempty" protobuf:"bytes,1,opt,name=uid,casttype=k8s.io/apimachinery/pkg/types.UID"` + // Specifies the target ResourceVersion + // +optional + ResourceVersion *string `json:"resourceVersion,omitempty" protobuf:"bytes,2,opt,name=resourceVersion"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/types_swagger_doc_generated.go b/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/types_swagger_doc_generated.go index 34b8b4f4ee..3b1a09e57f 100644 --- a/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/types_swagger_doc_generated.go +++ b/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/types_swagger_doc_generated.go @@ -294,8 +294,9 @@ func (PatchOptions) SwaggerDoc() map[string]string { } var map_Preconditions = map[string]string{ - "": "Preconditions must be fulfilled before an operation (update, delete, etc.) is carried out.", - "uid": "Specifies the target UID.", + "": "Preconditions must be fulfilled before an operation (update, delete, etc.) is carried out.", + "uid": "Specifies the target UID.", + "resourceVersion": "Specifies the target ResourceVersion", } func (Preconditions) SwaggerDoc() map[string]string { diff --git a/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/zz_generated.deepcopy.go b/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/zz_generated.deepcopy.go index 5b09ac54c4..68498b8d20 100644 --- a/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/zz_generated.deepcopy.go +++ b/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/zz_generated.deepcopy.go @@ -831,6 +831,11 @@ func (in *Preconditions) DeepCopyInto(out *Preconditions) { *out = new(types.UID) **out = **in } + if in.ResourceVersion != nil { + in, out := &in.ResourceVersion, &out.ResourceVersion + *out = new(string) + **out = **in + } return } diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/BUILD b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/BUILD index 1c2892a40e..10379edd55 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/BUILD @@ -7,14 +7,17 @@ go_library( importpath = "k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager", visibility = ["//visibility:public"], deps = [ + "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/meta:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal:go_default_library", "//vendor/k8s.io/kube-openapi/pkg/util/proto:go_default_library", "//vendor/sigs.k8s.io/structured-merge-diff/fieldpath:go_default_library", "//vendor/sigs.k8s.io/structured-merge-diff/merge:go_default_library", + "//vendor/sigs.k8s.io/yaml:go_default_library", ], ) @@ -41,6 +44,7 @@ go_test( embed = [":go_default_library"], deps = [ "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/meta:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/fieldmanager.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/fieldmanager.go index 7879dacb47..212bcadfe2 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/fieldmanager.go +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/fieldmanager.go @@ -20,14 +20,17 @@ import ( "fmt" "time" + "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal" openapiproto "k8s.io/kube-openapi/pkg/util/proto" "sigs.k8s.io/structured-merge-diff/fieldpath" "sigs.k8s.io/structured-merge-diff/merge" + "sigs.k8s.io/yaml" ) // FieldManager updates the managed fields and merge applied @@ -149,9 +152,20 @@ func (f *FieldManager) Apply(liveObj runtime.Object, patch []byte, fieldManager if err != nil { return nil, fmt.Errorf("failed to decode managed fields: %v", err) } - // We can assume that patchObj is already on the proper version: - // it shouldn't have to be converted so that it's not defaulted. - // TODO (jennybuckley): Explicitly checkt that patchObj is in the proper version. + // Check that the patch object has the same version as the live object + patchObj := &unstructured.Unstructured{Object: map[string]interface{}{}} + + if err := yaml.Unmarshal(patch, &patchObj.Object); err != nil { + return nil, fmt.Errorf("error decoding YAML: %v", err) + } + if patchObj.GetAPIVersion() != f.groupVersion.String() { + return nil, + errors.NewBadRequest( + fmt.Sprintf("Incorrect version specified in apply patch. "+ + "Specified patch version: %s, expected: %s", + patchObj.GetAPIVersion(), f.groupVersion.String())) + } + liveObjVersioned, err := f.toVersioned(liveObj) if err != nil { return nil, fmt.Errorf("failed to convert live object to proper version: %v", err) diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/fieldmanager_test.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/fieldmanager_test.go index f512c98b76..776dfd0a1e 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/fieldmanager_test.go +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/fieldmanager_test.go @@ -18,9 +18,11 @@ package fieldmanager_test import ( "errors" + "net/http" "testing" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" @@ -72,7 +74,7 @@ func TestApplyStripsFields(t *testing.T) { obj := &corev1.Pod{} newObj, err := f.Apply(obj, []byte(`{ - "apiVersion": "v1", + "apiVersion": "apps/v1", "kind": "Deployment", "metadata": { "name": "b", @@ -85,7 +87,7 @@ func TestApplyStripsFields(t *testing.T) { "managedFields": [{ "manager": "apply", "operation": "Apply", - "apiVersion": "v1", + "apiVersion": "apps/v1", "fields": { "f:metadata": { "f:labels": { @@ -111,13 +113,46 @@ func TestApplyStripsFields(t *testing.T) { } } +func TestVersionCheck(t *testing.T) { + f := NewTestFieldManager(t) + + obj := &corev1.Pod{} + + // patch has 'apiVersion: apps/v1' and live version is apps/v1 -> no errors + _, err := f.Apply(obj, []byte(`{ + "apiVersion": "apps/v1", + "kind": "Deployment", + }`), "fieldmanager_test", false) + if err != nil { + t.Fatalf("failed to apply object: %v", err) + } + + // patch has 'apiVersion: apps/v2' but live version is apps/v1 -> error + _, err = f.Apply(obj, []byte(`{ + "apiVersion": "apps/v2", + "kind": "Deployment", + }`), "fieldmanager_test", false) + if err == nil { + t.Fatalf("expected an error from mismatched patch and live versions") + } + switch typ := err.(type) { + default: + t.Fatalf("expected error to be of type %T was %T", apierrors.StatusError{}, typ) + case apierrors.APIStatus: + if typ.Status().Code != http.StatusBadRequest { + t.Fatalf("expected status code to be %d but was %d", + http.StatusBadRequest, typ.Status().Code) + } + } +} + func TestApplyDoesNotStripLabels(t *testing.T) { f := NewTestFieldManager(t) obj := &corev1.Pod{} newObj, err := f.Apply(obj, []byte(`{ - "apiVersion": "v1", + "apiVersion": "apps/v1", "kind": "Pod", "metadata": { "labels": { diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/typeconverter.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/typeconverter.go index 80ac84128f..e80e5634bc 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/typeconverter.go +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/typeconverter.go @@ -105,7 +105,12 @@ func (c *typeConverter) YAMLToTyped(from []byte) (typed.TypedValue, error) { return nil, fmt.Errorf("error decoding YAML: %v", err) } - return c.ObjectToTyped(unstructured) + gvk := unstructured.GetObjectKind().GroupVersionKind() + t := c.parser.Type(gvk) + if t == nil { + return nil, fmt.Errorf("no corresponding type for %v", gvk) + } + return t.FromYAML(typed.YAMLObject(string(from))) } func (c *typeConverter) TypedToObject(value typed.TypedValue) (runtime.Object, error) { diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/typeconverter_test.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/typeconverter_test.go index c36d686545..8e41f07960 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/typeconverter_test.go +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/typeconverter_test.go @@ -68,7 +68,6 @@ metadata: labels: app: nginx spec: - replicas: 3 selector: matchLabels: app: nginx @@ -91,7 +90,6 @@ metadata: labels: app: nginx spec: - replicas: 3 selector: matchLabels: app: nginx diff --git a/staging/src/k8s.io/apiserver/pkg/registry/generic/registry/store.go b/staging/src/k8s.io/apiserver/pkg/registry/generic/registry/store.go index c84a3276c4..be15e2bb3b 100644 --- a/staging/src/k8s.io/apiserver/pkg/registry/generic/registry/store.go +++ b/staging/src/k8s.io/apiserver/pkg/registry/generic/registry/store.go @@ -448,6 +448,7 @@ func (e *Store) Update(ctx context.Context, name string, objInfo rest.UpdatedObj storagePreconditions := &storage.Preconditions{} if preconditions := objInfo.Preconditions(); preconditions != nil { storagePreconditions.UID = preconditions.UID + storagePreconditions.ResourceVersion = preconditions.ResourceVersion } out := e.NewFunc() @@ -879,6 +880,7 @@ func (e *Store) Delete(ctx context.Context, name string, options *metav1.DeleteO var preconditions storage.Preconditions if options.Preconditions != nil { preconditions.UID = options.Preconditions.UID + preconditions.ResourceVersion = options.Preconditions.ResourceVersion } graceful, pendingGraceful, err := rest.BeforeDelete(e.DeleteStrategy, ctx, obj, options) if err != nil { diff --git a/staging/src/k8s.io/apiserver/pkg/registry/rest/delete.go b/staging/src/k8s.io/apiserver/pkg/registry/rest/delete.go index 7c39f6be10..16c9e1b40d 100644 --- a/staging/src/k8s.io/apiserver/pkg/registry/rest/delete.go +++ b/staging/src/k8s.io/apiserver/pkg/registry/rest/delete.go @@ -77,8 +77,13 @@ func BeforeDelete(strategy RESTDeleteStrategy, ctx context.Context, obj runtime. return false, false, errors.NewInvalid(schema.GroupKind{Group: metav1.GroupName, Kind: "DeleteOptions"}, "", errs) } // Checking the Preconditions here to fail early. They'll be enforced later on when we actually do the deletion, too. - if options.Preconditions != nil && options.Preconditions.UID != nil && *options.Preconditions.UID != objectMeta.GetUID() { - return false, false, errors.NewConflict(schema.GroupResource{Group: gvk.Group, Resource: gvk.Kind}, objectMeta.GetName(), fmt.Errorf("the UID in the precondition (%s) does not match the UID in record (%s). The object might have been deleted and then recreated", *options.Preconditions.UID, objectMeta.GetUID())) + if options.Preconditions != nil { + if options.Preconditions.UID != nil && *options.Preconditions.UID != objectMeta.GetUID() { + return false, false, errors.NewConflict(schema.GroupResource{Group: gvk.Group, Resource: gvk.Kind}, objectMeta.GetName(), fmt.Errorf("the UID in the precondition (%s) does not match the UID in record (%s). The object might have been deleted and then recreated", *options.Preconditions.UID, objectMeta.GetUID())) + } + if options.Preconditions.ResourceVersion != nil && *options.Preconditions.ResourceVersion != objectMeta.GetResourceVersion() { + return false, false, errors.NewConflict(schema.GroupResource{Group: gvk.Group, Resource: gvk.Kind}, objectMeta.GetName(), fmt.Errorf("the ResourceVersion in the precondition (%s) does not match the ResourceVersion in record (%s). The object might have been modified", *options.Preconditions.ResourceVersion, objectMeta.GetResourceVersion())) + } } gracefulStrategy, ok := strategy.(RESTGracefulDeleteStrategy) if !ok { diff --git a/staging/src/k8s.io/apiserver/pkg/registry/rest/resttest/resttest.go b/staging/src/k8s.io/apiserver/pkg/registry/rest/resttest/resttest.go index 8e6b555426..f29f4325b6 100644 --- a/staging/src/k8s.io/apiserver/pkg/registry/rest/resttest/resttest.go +++ b/staging/src/k8s.io/apiserver/pkg/registry/rest/resttest/resttest.go @@ -909,6 +909,45 @@ func (t *Tester) testDeleteWithUID(obj runtime.Object, createFn CreateFunc, getF } } +// This test the fast-fail path. We test that the precondition gets verified +// again before deleting the object in tests of pkg/storage/etcd. +func (t *Tester) testDeleteWithResourceVersion(obj runtime.Object, createFn CreateFunc, getFn GetFunc, isNotFoundFn IsErrorFunc, opts metav1.DeleteOptions) { + ctx := t.TestContext() + + foo := obj.DeepCopyObject() + t.setObjectMeta(foo, t.namer(1)) + objectMeta := t.getObjectMetaOrFail(foo) + objectMeta.SetResourceVersion("RV0000") + if err := createFn(ctx, foo); err != nil { + t.Errorf("unexpected error: %v", err) + } + opts.Preconditions = metav1.NewRVDeletionPrecondition("RV1111").Preconditions + obj, wasDeleted, err := t.storage.(rest.GracefulDeleter).Delete(ctx, objectMeta.GetName(), &opts) + if err == nil || !errors.IsConflict(err) { + t.Errorf("unexpected error: %v", err) + } + if wasDeleted { + t.Errorf("unexpected, object %s should not have been deleted immediately", objectMeta.GetName()) + } + obj, _, err = t.storage.(rest.GracefulDeleter).Delete(ctx, objectMeta.GetName(), metav1.NewRVDeletionPrecondition("RV0000")) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + + if !t.returnDeletedObject { + if status, ok := obj.(*metav1.Status); !ok { + t.Errorf("expected status of delete, got %v", status) + } else if status.Status != metav1.StatusSuccess { + t.Errorf("expected success, got: %v", status.Status) + } + } + + _, err = getFn(ctx, foo) + if err == nil || !isNotFoundFn(err) { + t.Errorf("unexpected error: %v", err) + } +} + // ============================================================================= // Graceful Deletion tests. diff --git a/staging/src/k8s.io/apiserver/pkg/storage/interfaces.go b/staging/src/k8s.io/apiserver/pkg/storage/interfaces.go index 56dd6dbdc9..cff425f3ab 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/interfaces.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/interfaces.go @@ -98,6 +98,9 @@ type Preconditions struct { // Specifies the target UID. // +optional UID *types.UID `json:"uid,omitempty"` + // Specifies the target ResourceVersion + // +optional + ResourceVersion *string `json:"resourceVersion,omitempty"` } // NewUIDPreconditions returns a Preconditions with UID set. @@ -125,8 +128,14 @@ func (p *Preconditions) Check(key string, obj runtime.Object) error { objMeta.GetUID()) return NewInvalidObjError(key, err) } + if p.ResourceVersion != nil && *p.ResourceVersion != objMeta.GetResourceVersion() { + err := fmt.Sprintf( + "Precondition failed: ResourceVersion in precondition: %v, ResourceVersion in object meta: %v", + *p.ResourceVersion, + objMeta.GetResourceVersion()) + return NewInvalidObjError(key, err) + } return nil - } // Interface offers a common interface for object marshaling/unmarshaling operations and diff --git a/test/e2e/framework/ingress/ingress_utils.go b/test/e2e/framework/ingress/ingress_utils.go index 6292aa9b24..5fa377d304 100644 --- a/test/e2e/framework/ingress/ingress_utils.go +++ b/test/e2e/framework/ingress/ingress_utils.go @@ -667,12 +667,26 @@ func (j *TestJig) pollIngressWithCert(ing *extensions.Ingress, address string, k } // WaitForIngress waits for the Ingress to get an address. +// WaitForIngress returns when it gets the first 200 response func (j *TestJig) WaitForIngress(waitForNodePort bool) { if err := j.WaitForGivenIngressWithTimeout(j.Ingress, waitForNodePort, framework.LoadBalancerPollTimeout); err != nil { framework.Failf("error in waiting for ingress to get an address: %s", err) } } +// WaitForIngressToStable waits for the LB return 100 consecutive 200 responses. +func (j *TestJig) WaitForIngressToStable() { + if err := wait.Poll(10*time.Second, framework.LoadBalancerCreateTimeoutDefault, func() (bool, error) { + _, err := j.GetDistinctResponseFromIngress() + if err != nil { + return false, nil + } + return true, nil + }); err != nil { + framework.Failf("error in waiting for ingress to stablize: %v", err) + } +} + // WaitForGivenIngressWithTimeout waits till the ingress acquires an IP, // then waits for its hosts/urls to respond to a protocol check (either // http or https). If waitForNodePort is true, the NodePort of the Service diff --git a/test/e2e/framework/networking_utils.go b/test/e2e/framework/networking_utils.go index 7d43570aa9..acaf74553f 100644 --- a/test/e2e/framework/networking_utils.go +++ b/test/e2e/framework/networking_utils.go @@ -17,7 +17,6 @@ limitations under the License. package framework import ( - "bytes" "encoding/json" "fmt" "io/ioutil" @@ -708,13 +707,131 @@ func CheckReachabilityFromPod(expectToBeReachable bool, timeout time.Duration, n ExpectNoError(err) } -// Does an HTTP GET, but does not reuse TCP connections -// This masks problems where the iptables rule has changed, but we don't see it -// This is intended for relatively quick requests (status checks), so we set a short (5 seconds) timeout -func httpGetNoConnectionPool(url string) (*http.Response, error) { - return httpGetNoConnectionPoolTimeout(url, 5*time.Second) +type HTTPPokeParams struct { + Timeout time.Duration + ExpectCode int // default = 200 + BodyContains string + RetriableCodes []int } +type HTTPPokeResult struct { + Status HTTPPokeStatus + Code int // HTTP code: 0 if the connection was not made + Error error // if there was any error + Body []byte // if code != 0 +} + +type HTTPPokeStatus string + +const ( + HTTPSuccess HTTPPokeStatus = "Success" + HTTPError HTTPPokeStatus = "UnknownError" + // Any time we add new errors, we should audit all callers of this. + HTTPTimeout HTTPPokeStatus = "TimedOut" + HTTPRefused HTTPPokeStatus = "ConnectionRefused" + HTTPRetryCode HTTPPokeStatus = "RetryCode" + HTTPWrongCode HTTPPokeStatus = "WrongCode" + HTTPBadResponse HTTPPokeStatus = "BadResponse" +) + +// PokeHTTP tries to connect to a host on a port for a given URL path. Callers +// can specify additional success parameters, if desired. +// +// The result status will be characterized as precisely as possible, given the +// known users of this. +// +// The result code will be zero in case of any failure to connect, or non-zero +// if the HTTP transaction completed (even if the other test params make this a +// failure). +// +// The result error will be populated for any status other than Success. +// +// The result body will be populated if the HTTP transaction was completed, even +// if the other test params make this a failure). +func PokeHTTP(host string, port int, path string, params *HTTPPokeParams) HTTPPokeResult { + hostPort := net.JoinHostPort(host, strconv.Itoa(port)) + url := fmt.Sprintf("http://%s%s", hostPort, path) + + ret := HTTPPokeResult{} + + // Sanity check inputs, because it has happened. These are the only things + // that should hard fail the test - they are basically ASSERT()s. + if host == "" { + Failf("Got empty host for HTTP poke (%s)", url) + return ret + } + if port == 0 { + Failf("Got port==0 for HTTP poke (%s)", url) + return ret + } + + // Set default params. + if params == nil { + params = &HTTPPokeParams{} + } + if params.ExpectCode == 0 { + params.ExpectCode = http.StatusOK + } + + Logf("Poking %q", url) + + resp, err := httpGetNoConnectionPoolTimeout(url, params.Timeout) + if err != nil { + ret.Error = err + neterr, ok := err.(net.Error) + if ok && neterr.Timeout() { + ret.Status = HTTPTimeout + } else if strings.Contains(err.Error(), "connection refused") { + ret.Status = HTTPRefused + } else { + ret.Status = HTTPError + } + Logf("Poke(%q): %v", url, err) + return ret + } + + ret.Code = resp.StatusCode + + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + ret.Status = HTTPError + ret.Error = fmt.Errorf("error reading HTTP body: %v", err) + Logf("Poke(%q): %v", url, ret.Error) + return ret + } + ret.Body = make([]byte, len(body)) + copy(ret.Body, body) + + if resp.StatusCode != params.ExpectCode { + for _, code := range params.RetriableCodes { + if resp.StatusCode == code { + ret.Error = fmt.Errorf("retriable status code: %d", resp.StatusCode) + ret.Status = HTTPRetryCode + Logf("Poke(%q): %v", url, ret.Error) + return ret + } + } + ret.Status = HTTPWrongCode + ret.Error = fmt.Errorf("bad status code: %d", resp.StatusCode) + Logf("Poke(%q): %v", url, ret.Error) + return ret + } + + if params.BodyContains != "" && !strings.Contains(string(body), params.BodyContains) { + ret.Status = HTTPBadResponse + ret.Error = fmt.Errorf("response does not contain expected substring: %q", string(body)) + Logf("Poke(%q): %v", url, ret.Error) + return ret + } + + ret.Status = HTTPSuccess + Logf("Poke(%q): success", url) + return ret +} + +// Does an HTTP GET, but does not reuse TCP connections +// This masks problems where the iptables rule has changed, but we don't see it func httpGetNoConnectionPoolTimeout(url string, timeout time.Duration) (*http.Response, error) { tr := utilnet.SetTransportDefaults(&http.Transport{ DisableKeepAlives: true, @@ -727,178 +844,126 @@ func httpGetNoConnectionPoolTimeout(url string, timeout time.Duration) (*http.Re return client.Get(url) } -func TestReachableHTTP(ip string, port int, request string, expect string) (bool, error) { - return TestReachableHTTPWithContent(ip, port, request, expect, nil) +type UDPPokeParams struct { + Timeout time.Duration + Response string } -func TestReachableHTTPWithRetriableErrorCodes(ip string, port int, request string, expect string, retriableErrCodes []int) (bool, error) { - return TestReachableHTTPWithContentTimeoutWithRetriableErrorCodes(ip, port, request, expect, nil, retriableErrCodes, time.Second*5) +type UDPPokeResult struct { + Status UDPPokeStatus + Error error // if there was any error + Response []byte // if code != 0 } -func TestReachableHTTPWithContent(ip string, port int, request string, expect string, content *bytes.Buffer) (bool, error) { - return TestReachableHTTPWithContentTimeout(ip, port, request, expect, content, 5*time.Second) -} +type UDPPokeStatus string -func TestReachableHTTPWithContentTimeout(ip string, port int, request string, expect string, content *bytes.Buffer, timeout time.Duration) (bool, error) { - return TestReachableHTTPWithContentTimeoutWithRetriableErrorCodes(ip, port, request, expect, content, []int{}, timeout) -} +const ( + UDPSuccess UDPPokeStatus = "Success" + UDPError UDPPokeStatus = "UnknownError" + // Any time we add new errors, we should audit all callers of this. + UDPTimeout UDPPokeStatus = "TimedOut" + UDPRefused UDPPokeStatus = "ConnectionRefused" + UDPBadResponse UDPPokeStatus = "BadResponse" +) -func TestReachableHTTPWithContentTimeoutWithRetriableErrorCodes(ip string, port int, request string, expect string, content *bytes.Buffer, retriableErrCodes []int, timeout time.Duration) (bool, error) { +// PokeUDP tries to connect to a host on a port and send the given request. Callers +// can specify additional success parameters, if desired. +// +// The result status will be characterized as precisely as possible, given the +// known users of this. +// +// The result error will be populated for any status other than Success. +// +// The result response will be populated if the UDP transaction was completed, even +// if the other test params make this a failure). +func PokeUDP(host string, port int, request string, params *UDPPokeParams) UDPPokeResult { + hostPort := net.JoinHostPort(host, strconv.Itoa(port)) + url := fmt.Sprintf("udp://%s", hostPort) - ipPort := net.JoinHostPort(ip, strconv.Itoa(port)) - url := fmt.Sprintf("http://%s%s", ipPort, request) - if ip == "" { - Failf("Got empty IP for reachability check (%s)", url) - return false, nil + ret := UDPPokeResult{} + + // Sanity check inputs, because it has happened. These are the only things + // that should hard fail the test - they are basically ASSERT()s. + if host == "" { + Failf("Got empty host for UDP poke (%s)", url) + return ret } if port == 0 { - Failf("Got port==0 for reachability check (%s)", url) - return false, nil + Failf("Got port==0 for UDP poke (%s)", url) + return ret } - Logf("Testing HTTP reachability of %v", url) + // Set default params. + if params == nil { + params = &UDPPokeParams{} + } - resp, err := httpGetNoConnectionPoolTimeout(url, timeout) + Logf("Poking %v", url) + + con, err := net.Dial("udp", hostPort) if err != nil { - Logf("Got error testing for reachability of %s: %v", url, err) - return false, nil + ret.Status = UDPError + ret.Error = err + Logf("Poke(%q): %v", url, err) + return ret } - defer resp.Body.Close() - body, err := ioutil.ReadAll(resp.Body) + + _, err = con.Write([]byte(fmt.Sprintf("%s\n", request))) if err != nil { - Logf("Got error reading response from %s: %v", url, err) - return false, nil - } - if resp.StatusCode != 200 { - for _, code := range retriableErrCodes { - if resp.StatusCode == code { - Logf("Got non-success status %q when trying to access %s, but the error code is retriable", resp.Status, url) - return false, nil - } + ret.Error = err + neterr, ok := err.(net.Error) + if ok && neterr.Timeout() { + ret.Status = UDPTimeout + } else if strings.Contains(err.Error(), "connection refused") { + ret.Status = UDPRefused + } else { + ret.Status = UDPError } - return false, fmt.Errorf("received non-success return status %q trying to access %s; got body: %s", - resp.Status, url, string(body)) - } - if !strings.Contains(string(body), expect) { - return false, fmt.Errorf("received response body without expected substring %q: %s", expect, string(body)) - } - if content != nil { - content.Write(body) - } - return true, nil -} - -func TestNotReachableHTTP(ip string, port int) (bool, error) { - return TestNotReachableHTTPTimeout(ip, port, 5*time.Second) -} - -func TestNotReachableHTTPTimeout(ip string, port int, timeout time.Duration) (bool, error) { - ipPort := net.JoinHostPort(ip, strconv.Itoa(port)) - url := fmt.Sprintf("http://%s", ipPort) - if ip == "" { - Failf("Got empty IP for non-reachability check (%s)", url) - return false, nil - } - if port == 0 { - Failf("Got port==0 for non-reachability check (%s)", url) - return false, nil + Logf("Poke(%q): %v", url, err) + return ret } - Logf("Testing HTTP non-reachability of %v", url) + if params.Timeout != 0 { + err = con.SetDeadline(time.Now().Add(params.Timeout)) + if err != nil { + ret.Status = UDPError + ret.Error = err + Logf("Poke(%q): %v", url, err) + return ret + } + } - resp, err := httpGetNoConnectionPoolTimeout(url, timeout) + bufsize := len(params.Response) + 1 + if bufsize == 0 { + bufsize = 4096 + } + var buf []byte = make([]byte, bufsize) + n, err := con.Read(buf) if err != nil { - Logf("Confirmed that %s is not reachable", url) - return true, nil + ret.Error = err + neterr, ok := err.(net.Error) + if ok && neterr.Timeout() { + ret.Status = UDPTimeout + } else if strings.Contains(err.Error(), "connection refused") { + ret.Status = UDPRefused + } else { + ret.Status = UDPError + } + Logf("Poke(%q): %v", url, err) + return ret } - resp.Body.Close() - return false, nil -} + ret.Response = buf[0:n] -func TestReachableUDP(ip string, port int, request string, expect string) (bool, error) { - ipPort := net.JoinHostPort(ip, strconv.Itoa(port)) - uri := fmt.Sprintf("udp://%s", ipPort) - if ip == "" { - Failf("Got empty IP for reachability check (%s)", uri) - return false, nil - } - if port == 0 { - Failf("Got port==0 for reachability check (%s)", uri) - return false, nil + if params.Response != "" && string(ret.Response) != params.Response { + ret.Status = UDPBadResponse + ret.Error = fmt.Errorf("response does not match expected string: %q", string(ret.Response)) + Logf("Poke(%q): %v", url, ret.Error) + return ret } - Logf("Testing UDP reachability of %v", uri) - - con, err := net.Dial("udp", ipPort) - if err != nil { - return false, fmt.Errorf("Failed to dial %s: %v", ipPort, err) - } - - _, err = con.Write([]byte(fmt.Sprintf("%s\n", request))) - if err != nil { - return false, fmt.Errorf("Failed to send request: %v", err) - } - - var buf []byte = make([]byte, len(expect)+1) - - err = con.SetDeadline(time.Now().Add(3 * time.Second)) - if err != nil { - return false, fmt.Errorf("Failed to set deadline: %v", err) - } - - _, err = con.Read(buf) - if err != nil { - return false, nil - } - - if !strings.Contains(string(buf), expect) { - return false, fmt.Errorf("Failed to retrieve %q, got %q", expect, string(buf)) - } - - Logf("Successfully reached %v", uri) - return true, nil -} - -func TestNotReachableUDP(ip string, port int, request string) (bool, error) { - ipPort := net.JoinHostPort(ip, strconv.Itoa(port)) - uri := fmt.Sprintf("udp://%s", ipPort) - if ip == "" { - Failf("Got empty IP for reachability check (%s)", uri) - return false, nil - } - if port == 0 { - Failf("Got port==0 for reachability check (%s)", uri) - return false, nil - } - - Logf("Testing UDP non-reachability of %v", uri) - - con, err := net.Dial("udp", ipPort) - if err != nil { - Logf("Confirmed that %s is not reachable", uri) - return true, nil - } - - _, err = con.Write([]byte(fmt.Sprintf("%s\n", request))) - if err != nil { - Logf("Confirmed that %s is not reachable", uri) - return true, nil - } - - var buf []byte = make([]byte, 1) - - err = con.SetDeadline(time.Now().Add(3 * time.Second)) - if err != nil { - return false, fmt.Errorf("Failed to set deadline: %v", err) - } - - _, err = con.Read(buf) - if err != nil { - Logf("Confirmed that %s is not reachable", uri) - return true, nil - } - - return false, nil + ret.Status = UDPSuccess + Logf("Poke(%q): success", url) + return ret } func TestHitNodesFromOutside(externalIP string, httpPort int32, timeout time.Duration, expectedHosts sets.String) error { @@ -911,13 +976,12 @@ func TestHitNodesFromOutsideWithCount(externalIP string, httpPort int32, timeout hittedHosts := sets.NewString() count := 0 condition := func() (bool, error) { - var respBody bytes.Buffer - reached, err := TestReachableHTTPWithContentTimeout(externalIP, int(httpPort), "/hostname", "", &respBody, - 1*time.Second) - if err != nil || !reached { + result := PokeHTTP(externalIP, int(httpPort), "/hostname", &HTTPPokeParams{Timeout: 1 * time.Second}) + if result.Status != HTTPSuccess { return false, nil } - hittedHost := strings.TrimSpace(respBody.String()) + + hittedHost := strings.TrimSpace(string(result.Body)) if !expectedHosts.Has(hittedHost) { Logf("Error hitting unexpected host: %v, reset counter: %v", hittedHost, count) count = 0 diff --git a/test/e2e/framework/service_util.go b/test/e2e/framework/service_util.go index 7ef9d711cd..b98ddba72e 100644 --- a/test/e2e/framework/service_util.go +++ b/test/e2e/framework/service_util.go @@ -623,6 +623,7 @@ func (j *ServiceTestJig) waitForConditionOrFail(namespace, name string, timeout // name as the jig and runs the "netexec" container. func (j *ServiceTestJig) newRCTemplate(namespace string) *v1.ReplicationController { var replicas int32 = 1 + var grace int64 = 3 // so we don't race with kube-proxy when scaling up/down rc := &v1.ReplicationController{ ObjectMeta: metav1.ObjectMeta{ @@ -654,7 +655,7 @@ func (j *ServiceTestJig) newRCTemplate(namespace string) *v1.ReplicationControll }, }, }, - TerminationGracePeriodSeconds: new(int64), + TerminationGracePeriodSeconds: &grace, }, }, }, @@ -737,6 +738,28 @@ func (j *ServiceTestJig) RunOrFail(namespace string, tweak func(rc *v1.Replicati return result } +func (j *ServiceTestJig) Scale(namespace string, replicas int) { + rc := j.Name + scale, err := j.Client.CoreV1().ReplicationControllers(namespace).GetScale(rc, metav1.GetOptions{}) + if err != nil { + Failf("Failed to get scale for RC %q: %v", rc, err) + } + + scale.Spec.Replicas = int32(replicas) + _, err = j.Client.CoreV1().ReplicationControllers(namespace).UpdateScale(rc, scale) + if err != nil { + Failf("Failed to scale RC %q: %v", rc, err) + } + pods, err := j.waitForPodsCreated(namespace, replicas) + if err != nil { + Failf("Failed waiting for pods: %v", err) + } + if err := j.waitForPodsReady(namespace, pods); err != nil { + Failf("Failed waiting for pods to be running: %v", err) + } + return +} + func (j *ServiceTestJig) waitForPdbReady(namespace string) error { timeout := 2 * time.Minute for start := time.Now(); time.Since(start) < timeout; time.Sleep(2 * time.Second) { @@ -875,9 +898,19 @@ func (j *ServiceTestJig) TestReachableHTTP(host string, port int, timeout time.D } func (j *ServiceTestJig) TestReachableHTTPWithRetriableErrorCodes(host string, port int, retriableErrCodes []int, timeout time.Duration) { - if err := wait.PollImmediate(Poll, timeout, func() (bool, error) { - return TestReachableHTTPWithRetriableErrorCodes(host, port, "/echo?msg=hello", "hello", retriableErrCodes) - }); err != nil { + pollfn := func() (bool, error) { + result := PokeHTTP(host, port, "/echo?msg=hello", + &HTTPPokeParams{ + BodyContains: "hello", + RetriableCodes: retriableErrCodes, + }) + if result.Status == HTTPSuccess { + return true, nil + } + return false, nil // caller can retry + } + + if err := wait.PollImmediate(Poll, timeout, pollfn); err != nil { if err == wait.ErrWaitTimeout { Failf("Could not reach HTTP service through %v:%v after %v", host, port, timeout) } else { @@ -887,36 +920,87 @@ func (j *ServiceTestJig) TestReachableHTTPWithRetriableErrorCodes(host string, p } func (j *ServiceTestJig) TestNotReachableHTTP(host string, port int, timeout time.Duration) { - if err := wait.PollImmediate(Poll, timeout, func() (bool, error) { return TestNotReachableHTTP(host, port) }); err != nil { - Failf("Could still reach HTTP service through %v:%v after %v: %v", host, port, timeout, err) + pollfn := func() (bool, error) { + result := PokeHTTP(host, port, "/", nil) + if result.Code == 0 { + return true, nil + } + return false, nil // caller can retry + } + + if err := wait.PollImmediate(Poll, timeout, pollfn); err != nil { + Failf("HTTP service %v:%v reachable after %v: %v", host, port, timeout, err) + } +} + +func (j *ServiceTestJig) TestRejectedHTTP(host string, port int, timeout time.Duration) { + pollfn := func() (bool, error) { + result := PokeHTTP(host, port, "/", nil) + if result.Status == HTTPRefused { + return true, nil + } + return false, nil // caller can retry + } + + if err := wait.PollImmediate(Poll, timeout, pollfn); err != nil { + Failf("HTTP service %v:%v not rejected: %v", host, port, err) } } func (j *ServiceTestJig) TestReachableUDP(host string, port int, timeout time.Duration) { - if err := wait.PollImmediate(Poll, timeout, func() (bool, error) { return TestReachableUDP(host, port, "echo hello", "hello") }); err != nil { + pollfn := func() (bool, error) { + result := PokeUDP(host, port, "echo hello", &UDPPokeParams{ + Timeout: 3 * time.Second, + Response: "hello", + }) + if result.Status == UDPSuccess { + return true, nil + } + return false, nil // caller can retry + } + + if err := wait.PollImmediate(Poll, timeout, pollfn); err != nil { Failf("Could not reach UDP service through %v:%v after %v: %v", host, port, timeout, err) } } func (j *ServiceTestJig) TestNotReachableUDP(host string, port int, timeout time.Duration) { - if err := wait.PollImmediate(Poll, timeout, func() (bool, error) { return TestNotReachableUDP(host, port, "echo hello") }); err != nil { - Failf("Could still reach UDP service through %v:%v after %v: %v", host, port, timeout, err) + pollfn := func() (bool, error) { + result := PokeUDP(host, port, "echo hello", &UDPPokeParams{Timeout: 3 * time.Second}) + if result.Status != UDPSuccess && result.Status != UDPError { + return true, nil + } + return false, nil // caller can retry + } + if err := wait.PollImmediate(Poll, timeout, pollfn); err != nil { + Failf("UDP service %v:%v reachable after %v: %v", host, port, timeout, err) + } +} + +func (j *ServiceTestJig) TestRejectedUDP(host string, port int, timeout time.Duration) { + pollfn := func() (bool, error) { + result := PokeUDP(host, port, "echo hello", &UDPPokeParams{Timeout: 3 * time.Second}) + if result.Status == UDPRefused { + return true, nil + } + return false, nil // caller can retry + } + if err := wait.PollImmediate(Poll, timeout, pollfn); err != nil { + Failf("UDP service %v:%v not rejected: %v", host, port, err) } } func (j *ServiceTestJig) GetHTTPContent(host string, port int, timeout time.Duration, url string) bytes.Buffer { var body bytes.Buffer - var err error if pollErr := wait.PollImmediate(Poll, timeout, func() (bool, error) { - var result bool - result, err = TestReachableHTTPWithContent(host, port, url, "", &body) - if err != nil { - Logf("Error hitting %v:%v%v, retrying: %v", host, port, url, err) - return false, nil + result := PokeHTTP(host, port, url, nil) + if result.Status == HTTPSuccess { + body.Write(result.Body) + return true, nil } - return result, nil + return false, nil }); pollErr != nil { - Failf("Could not reach HTTP service through %v:%v%v after %v: %v", host, port, url, timeout, err) + Failf("Could not reach HTTP service through %v:%v%v after %v: %v", host, port, url, timeout, pollErr) } return body } @@ -929,7 +1013,7 @@ func testHTTPHealthCheckNodePort(ip string, port int, request string) (bool, err return false, fmt.Errorf("Invalid input ip or port") } Logf("Testing HTTP health check on %v", url) - resp, err := httpGetNoConnectionPool(url) + resp, err := httpGetNoConnectionPoolTimeout(url, 5*time.Second) if err != nil { Logf("Got error testing for reachability of %s: %v", url, err) return false, err diff --git a/test/e2e/network/firewall.go b/test/e2e/network/firewall.go index db43181ab7..cfa3ffcaef 100644 --- a/test/e2e/network/firewall.go +++ b/test/e2e/network/firewall.go @@ -188,11 +188,11 @@ var _ = SIGDescribe("Firewall rule", func() { }) func assertNotReachableHTTPTimeout(ip string, port int, timeout time.Duration) { - unreachable, err := framework.TestNotReachableHTTPTimeout(ip, port, timeout) - if err != nil { - framework.Failf("Unexpected error checking for reachability of %s:%d: %v", ip, port, err) + result := framework.PokeHTTP(ip, port, "/", &framework.HTTPPokeParams{Timeout: timeout}) + if result.Status == framework.HTTPError { + framework.Failf("Unexpected error checking for reachability of %s:%d: %v", ip, port, result.Error) } - if !unreachable { + if result.Code != 0 { framework.Failf("Was unexpectedly able to reach %s:%d", ip, port) } } diff --git a/test/e2e/network/ingress.go b/test/e2e/network/ingress.go index 1fc1180a08..2b4470f0bb 100644 --- a/test/e2e/network/ingress.go +++ b/test/e2e/network/ingress.go @@ -529,9 +529,10 @@ var _ = SIGDescribe("Loadbalancing: L7", func() { _, err = f.ClientSet.CoreV1().Services(ns).Update(&svc) Expect(err).NotTo(HaveOccurred()) } - wait.Poll(5*time.Second, framework.LoadBalancerPollTimeout, func() (bool, error) { - return gceController.BackendServiceUsingIG(jig.GetServicePorts(true)) + err = wait.Poll(5*time.Second, framework.LoadBalancerPollTimeout, func() (bool, error) { + return gceController.BackendServiceUsingIG(jig.GetServicePorts(false)) }) + Expect(err).NotTo(HaveOccurred(), "Expect backend service to target IG, but failed to observe") jig.WaitForIngress(true) By("Switch backend service to use NEG") @@ -542,9 +543,10 @@ var _ = SIGDescribe("Loadbalancing: L7", func() { _, err = f.ClientSet.CoreV1().Services(ns).Update(&svc) Expect(err).NotTo(HaveOccurred()) } - wait.Poll(5*time.Second, framework.LoadBalancerPollTimeout, func() (bool, error) { + err = wait.Poll(5*time.Second, framework.LoadBalancerPollTimeout, func() (bool, error) { return gceController.BackendServiceUsingNEG(jig.GetServicePorts(false)) }) + Expect(err).NotTo(HaveOccurred(), "Expect backend service to target NEG, but failed to observe") jig.WaitForIngress(true) }) @@ -556,7 +558,7 @@ var _ = SIGDescribe("Loadbalancing: L7", func() { svcPorts := jig.GetServicePorts(false) usingNEG, err := gceController.BackendServiceUsingNEG(svcPorts) Expect(err).NotTo(HaveOccurred()) - Expect(usingNEG).To(BeTrue()) + Expect(usingNEG).To(BeTrue(), "Expect backend service to be using NEG. But not.") // ClusterIP ServicePorts have no NodePort for _, sp := range svcPorts { @@ -574,18 +576,21 @@ var _ = SIGDescribe("Loadbalancing: L7", func() { _, err = f.ClientSet.AppsV1().Deployments(ns).UpdateScale(name, scale) Expect(err).NotTo(HaveOccurred()) } - wait.Poll(10*time.Second, negUpdateTimeout, func() (bool, error) { + err = wait.Poll(10*time.Second, negUpdateTimeout, func() (bool, error) { res, err := jig.GetDistinctResponseFromIngress() if err != nil { return false, nil } + framework.Logf("Expecting %d backends, got %d", num, res.Len()) return res.Len() == num, nil }) + Expect(err).NotTo(HaveOccurred()) } By("Create a basic HTTP ingress using NEG") jig.CreateIngress(filepath.Join(ingress.IngressManifestPath, "neg"), ns, map[string]string{}, map[string]string{}) jig.WaitForIngress(true) + jig.WaitForIngressToStable() usingNEG, err := gceController.BackendServiceUsingNEG(jig.GetServicePorts(false)) Expect(err).NotTo(HaveOccurred()) Expect(usingNEG).To(BeTrue()) @@ -611,6 +616,7 @@ var _ = SIGDescribe("Loadbalancing: L7", func() { By("Create a basic HTTP ingress using NEG") jig.CreateIngress(filepath.Join(ingress.IngressManifestPath, "neg"), ns, map[string]string{}, map[string]string{}) jig.WaitForIngress(true) + jig.WaitForIngressToStable() usingNEG, err := gceController.BackendServiceUsingNEG(jig.GetServicePorts(false)) Expect(err).NotTo(HaveOccurred()) Expect(usingNEG).To(BeTrue()) @@ -621,13 +627,15 @@ var _ = SIGDescribe("Loadbalancing: L7", func() { scale.Spec.Replicas = int32(replicas) _, err = f.ClientSet.AppsV1().Deployments(ns).UpdateScale(name, scale) Expect(err).NotTo(HaveOccurred()) - wait.Poll(10*time.Second, framework.LoadBalancerPollTimeout, func() (bool, error) { + + err = wait.Poll(10*time.Second, framework.LoadBalancerPollTimeout, func() (bool, error) { res, err := jig.GetDistinctResponseFromIngress() if err != nil { return false, nil } return res.Len() == replicas, nil }) + Expect(err).NotTo(HaveOccurred()) By("Trigger rolling update and observe service disruption") deploy, err := f.ClientSet.AppsV1().Deployments(ns).Get(name, metav1.GetOptions{}) @@ -637,7 +645,7 @@ var _ = SIGDescribe("Loadbalancing: L7", func() { deploy.Spec.Template.Spec.TerminationGracePeriodSeconds = &gracePeriod _, err = f.ClientSet.AppsV1().Deployments(ns).Update(deploy) Expect(err).NotTo(HaveOccurred()) - wait.Poll(10*time.Second, framework.LoadBalancerPollTimeout, func() (bool, error) { + err = wait.Poll(10*time.Second, framework.LoadBalancerPollTimeout, func() (bool, error) { res, err := jig.GetDistinctResponseFromIngress() Expect(err).NotTo(HaveOccurred()) deploy, err := f.ClientSet.AppsV1().Deployments(ns).Get(name, metav1.GetOptions{}) @@ -655,6 +663,7 @@ var _ = SIGDescribe("Loadbalancing: L7", func() { return false, nil } }) + Expect(err).NotTo(HaveOccurred()) }) It("should sync endpoints for both Ingress-referenced NEG and standalone NEG", func() { @@ -669,7 +678,7 @@ var _ = SIGDescribe("Loadbalancing: L7", func() { _, err = f.ClientSet.AppsV1().Deployments(ns).UpdateScale(name, scale) Expect(err).NotTo(HaveOccurred()) } - wait.Poll(10*time.Second, negUpdateTimeout, func() (bool, error) { + err = wait.Poll(10*time.Second, negUpdateTimeout, func() (bool, error) { svc, err := f.ClientSet.CoreV1().Services(ns).Get(name, metav1.GetOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -716,6 +725,7 @@ var _ = SIGDescribe("Loadbalancing: L7", func() { return true, nil }) + Expect(err).NotTo(HaveOccurred()) } By("Create a basic HTTP ingress using NEG") @@ -1108,7 +1118,7 @@ func detectHttpVersionAndSchemeTest(f *framework.Framework, jig *ingress.TestJig } func detectNegAnnotation(f *framework.Framework, jig *ingress.TestJig, gceController *gce.GCEIngressController, ns, name string, negs int) { - wait.Poll(5*time.Second, framework.LoadBalancerPollTimeout, func() (bool, error) { + if err := wait.Poll(5*time.Second, negUpdateTimeout, func() (bool, error) { svc, err := f.ClientSet.CoreV1().Services(ns).Get(name, metav1.GetOptions{}) if err != nil { return false, nil @@ -1150,5 +1160,7 @@ func detectNegAnnotation(f *framework.Framework, jig *ingress.TestJig, gceContro } return gceController.BackendServiceUsingNEG(jig.GetServicePorts(false)) - }) + }); err != nil { + Expect(err).NotTo(HaveOccurred()) + } } diff --git a/test/e2e/network/service.go b/test/e2e/network/service.go index 3df7628386..55f743992d 100644 --- a/test/e2e/network/service.go +++ b/test/e2e/network/service.go @@ -791,11 +791,47 @@ var _ = SIGDescribe("Services", func() { jig.TestReachableUDP(nodeIP, udpNodePort, framework.KubeProxyLagTimeout) By("hitting the TCP service's LoadBalancer") - jig.TestReachableHTTP(tcpIngressIP, svcPort, loadBalancerCreateTimeout) // this may actually recreate the LB + jig.TestReachableHTTP(tcpIngressIP, svcPort, loadBalancerCreateTimeout) if loadBalancerSupportsUDP { By("hitting the UDP service's LoadBalancer") - jig.TestReachableUDP(udpIngressIP, svcPort, loadBalancerCreateTimeout) // this may actually recreate the LB) + jig.TestReachableUDP(udpIngressIP, svcPort, loadBalancerCreateTimeout) + } + + By("Scaling the pods to 0") + jig.Scale(ns1, 0) + jig.Scale(ns2, 0) + + By("looking for ICMP REJECT on the TCP service's NodePort") + jig.TestRejectedHTTP(nodeIP, tcpNodePort, framework.KubeProxyLagTimeout) + + By("looking for ICMP REJECT on the UDP service's NodePort") + jig.TestRejectedUDP(nodeIP, udpNodePort, framework.KubeProxyLagTimeout) + + By("looking for ICMP REJECT on the TCP service's LoadBalancer") + jig.TestRejectedHTTP(tcpIngressIP, svcPort, loadBalancerCreateTimeout) + + if loadBalancerSupportsUDP { + By("looking for ICMP REJECT on the UDP service's LoadBalancer") + jig.TestRejectedUDP(udpIngressIP, svcPort, loadBalancerCreateTimeout) + } + + By("Scaling the pods to 1") + jig.Scale(ns1, 1) + jig.Scale(ns2, 1) + + By("hitting the TCP service's NodePort") + jig.TestReachableHTTP(nodeIP, tcpNodePort, framework.KubeProxyLagTimeout) + + By("hitting the UDP service's NodePort") + jig.TestReachableUDP(nodeIP, udpNodePort, framework.KubeProxyLagTimeout) + + By("hitting the TCP service's LoadBalancer") + jig.TestReachableHTTP(tcpIngressIP, svcPort, loadBalancerCreateTimeout) + + if loadBalancerSupportsUDP { + By("hitting the UDP service's LoadBalancer") + jig.TestReachableUDP(udpIngressIP, svcPort, loadBalancerCreateTimeout) } // Change the services back to ClusterIP. @@ -2063,14 +2099,18 @@ var _ = SIGDescribe("ESIPP [Slow] [DisabledForLargeClusters]", func() { for nodeName, nodeIPs := range endpointNodeMap { By(fmt.Sprintf("checking kube-proxy health check fails on node with endpoint (%s), public IP %s", nodeName, nodeIPs[0])) var body bytes.Buffer - var result bool - var err error - if pollErr := wait.PollImmediate(framework.Poll, framework.ServiceTestTimeout, func() (bool, error) { - result, err = framework.TestReachableHTTPWithContent(nodeIPs[0], healthCheckNodePort, "/healthz", "", &body) - return !result, nil - }); pollErr != nil { - framework.Failf("Kube-proxy still exposing health check on node %v:%v, after ESIPP was turned off. Last err %v, last body %v", - nodeName, healthCheckNodePort, err, body.String()) + pollfn := func() (bool, error) { + result := framework.PokeHTTP(nodeIPs[0], healthCheckNodePort, "/healthz", nil) + if result.Code == 0 { + return true, nil + } + body.Reset() + body.Write(result.Body) + return false, nil + } + if pollErr := wait.PollImmediate(framework.Poll, framework.ServiceTestTimeout, pollfn); pollErr != nil { + framework.Failf("Kube-proxy still exposing health check on node %v:%v, after ESIPP was turned off. body %s", + nodeName, healthCheckNodePort, body.String()) } } diff --git a/test/e2e/storage/csi_mock_volume.go b/test/e2e/storage/csi_mock_volume.go index 7a41fa3020..9bc15116ca 100644 --- a/test/e2e/storage/csi_mock_volume.go +++ b/test/e2e/storage/csi_mock_volume.go @@ -379,7 +379,7 @@ var _ = utils.SIGDescribe("CSI mock volume", func() { _, _, pod3 := createPod() Expect(pod3).NotTo(BeNil(), "while creating third pod") err = waitForMaxVolumeCondition(pod3, m.cs) - Expect(err).NotTo(HaveOccurred(), "while waiting for max volume condition") + Expect(err).NotTo(HaveOccurred(), "while waiting for max volume condition on pod : %+v", pod3) }) }) diff --git a/test/e2e/storage/drivers/csi.go b/test/e2e/storage/drivers/csi.go index e579913cfd..07023a6fd9 100644 --- a/test/e2e/storage/drivers/csi.go +++ b/test/e2e/storage/drivers/csi.go @@ -367,7 +367,7 @@ func (g *gcePDCSIDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) { if pattern.FsType == "xfs" { framework.SkipUnlessNodeOSDistroIs("ubuntu", "custom") } - if pattern.FeatureTag == "sig-windows" { + if pattern.FeatureTag == "[sig-windows]" { framework.Skipf("Skipping tests for windows since CSI does not support it yet") } } @@ -473,7 +473,7 @@ func (g *gcePDExternalCSIDriver) SkipUnsupportedTest(pattern testpatterns.TestPa if pattern.FsType == "xfs" { framework.SkipUnlessNodeOSDistroIs("ubuntu", "custom") } - if pattern.FeatureTag == "sig-windows" { + if pattern.FeatureTag == "[sig-windows]" { framework.Skipf("Skipping tests for windows since CSI does not support it yet") } } diff --git a/test/e2e/storage/drivers/in_tree.go b/test/e2e/storage/drivers/in_tree.go index 886a00a470..76465ea1fb 100644 --- a/test/e2e/storage/drivers/in_tree.go +++ b/test/e2e/storage/drivers/in_tree.go @@ -1106,21 +1106,18 @@ var _ testsuites.DynamicPVTestDriver = &gcePdDriver{} // InitGcePdDriver returns gcePdDriver that implements TestDriver interface func InitGcePdDriver() testsuites.TestDriver { - var supportedTypes sets.String - var capFsGroup bool - if framework.NodeOSDistroIs("windows") { - supportedTypes = sets.NewString("ntfs") - capFsGroup = false - } else { - supportedTypes = sets.NewString( - "", // Default fsType - "ext2", - "ext3", - "ext4", - "xfs", - ) - capFsGroup = true - } + // In current test structure, it first initialize the driver and then set up + // the new framework, so we cannot get the correct OS here. So here set to + // support all fs types including both linux and windows. We have code to check Node OS later + // during test. + supportedTypes := sets.NewString( + "", // Default fsType + "ext2", + "ext3", + "ext4", + "xfs", + "ntfs", + ) return &gcePdDriver{ driverInfo: testsuites.DriverInfo{ Name: "gcepd", @@ -1129,7 +1126,7 @@ func InitGcePdDriver() testsuites.TestDriver { SupportedMountOption: sets.NewString("debug", "nouid32"), Capabilities: map[testsuites.Capability]bool{ testsuites.CapPersistence: true, - testsuites.CapFsGroup: capFsGroup, + testsuites.CapFsGroup: true, testsuites.CapBlock: true, testsuites.CapExec: true, }, @@ -1143,7 +1140,7 @@ func (g *gcePdDriver) GetDriverInfo() *testsuites.DriverInfo { func (g *gcePdDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) { framework.SkipUnlessProviderIs("gce", "gke") - if pattern.FeatureTag == "sig-windows" { + if pattern.FeatureTag == "[sig-windows]" { framework.SkipUnlessNodeOSDistroIs("windows") } } diff --git a/test/e2e/storage/testpatterns/testpattern.go b/test/e2e/storage/testpatterns/testpattern.go index 410ab85f4f..415ce45d56 100644 --- a/test/e2e/storage/testpatterns/testpattern.go +++ b/test/e2e/storage/testpatterns/testpattern.go @@ -152,21 +152,21 @@ var ( Name: "Inline-volume (ntfs)", VolType: InlineVolume, FsType: "ntfs", - FeatureTag: "sig-windows", + FeatureTag: "[sig-windows]", } // NtfsPreprovisionedPV is TestPattern for "Pre-provisioned PV (ntfs)" NtfsPreprovisionedPV = TestPattern{ Name: "Pre-provisioned PV (ntfs)", VolType: PreprovisionedPV, FsType: "ntfs", - FeatureTag: "sig-windows", + FeatureTag: "[sig-windows]", } // NtfsDynamicPV is TestPattern for "Dynamic PV (xfs)" NtfsDynamicPV = TestPattern{ Name: "Dynamic PV (ntfs)", VolType: DynamicPV, FsType: "ntfs", - FeatureTag: "sig-windows", + FeatureTag: "[sig-windows]", } // Definitions for Filesystem volume mode diff --git a/test/e2e/storage/testsuites/volume_io.go b/test/e2e/storage/testsuites/volume_io.go index 1ec29ba5a0..7f6227a258 100644 --- a/test/e2e/storage/testsuites/volume_io.go +++ b/test/e2e/storage/testsuites/volume_io.go @@ -125,7 +125,7 @@ func (t *volumeIOTestSuite) defineTests(driver TestDriver, pattern testpatterns. fileSizes := createFileSizes(dInfo.MaxFileSize) testFile := fmt.Sprintf("%s_io_test_%s", dInfo.Name, f.Namespace.Name) var fsGroup *int64 - if dInfo.Capabilities[CapFsGroup] { + if !framework.NodeOSDistroIs("windows") && dInfo.Capabilities[CapFsGroup] { fsGroupVal := int64(1234) fsGroup = &fsGroupVal } diff --git a/test/e2e/storage/testsuites/volumes.go b/test/e2e/storage/testsuites/volumes.go index 957b4e5cd1..5ffadae835 100644 --- a/test/e2e/storage/testsuites/volumes.go +++ b/test/e2e/storage/testsuites/volumes.go @@ -152,7 +152,7 @@ func (t *volumesTestSuite) defineTests(driver TestDriver, pattern testpatterns.T } config := convertTestConfig(l.config) var fsGroup *int64 - if dInfo.Capabilities[CapFsGroup] { + if framework.NodeOSDistroIs("windows") && dInfo.Capabilities[CapFsGroup] { fsGroupVal := int64(1234) fsGroup = &fsGroupVal } @@ -185,7 +185,12 @@ func testScriptInPod( ) suffix := generateSuffixForPodName(volumeType) fileName := fmt.Sprintf("test-%s", suffix) - content := fmt.Sprintf("ls %s", volPath) + var content string + if framework.NodeOSDistroIs("windows") { + content = fmt.Sprintf("ls -n %s", volPath) + } else { + content = fmt.Sprintf("ls %s", volPath) + } command := framework.GenerateWriteandExecuteScriptFileCmd(content, fileName, volPath) pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ diff --git a/test/e2e_node/device_plugin.go b/test/e2e_node/device_plugin.go index 138100a815..f0c7bdbe19 100644 --- a/test/e2e_node/device_plugin.go +++ b/test/e2e_node/device_plugin.go @@ -45,17 +45,12 @@ const ( ) // Serial because the test restarts Kubelet -var _ = framework.KubeDescribe("Device Plugin [Feature:DevicePlugin][NodeFeature:DevicePlugin][Serial]", func() { - f := framework.NewDefaultFramework("device-plugin-errors") - testDevicePlugin(f, false, pluginapi.DevicePluginPath) -}) - var _ = framework.KubeDescribe("Device Plugin [Feature:DevicePluginProbe][NodeFeature:DevicePluginProbe][Serial]", func() { f := framework.NewDefaultFramework("device-plugin-errors") - testDevicePlugin(f, true, "/var/lib/kubelet/plugins_registry") + testDevicePlugin(f, "/var/lib/kubelet/plugins_registry") }) -func testDevicePlugin(f *framework.Framework, enablePluginWatcher bool, pluginSockDir string) { +func testDevicePlugin(f *framework.Framework, pluginSockDir string) { pluginSockDir = filepath.Join(pluginSockDir) + "/" Context("DevicePlugin", func() { By("Enabling support for Kubelet Plugins Watcher") @@ -63,7 +58,6 @@ func testDevicePlugin(f *framework.Framework, enablePluginWatcher bool, pluginSo if initialConfig.FeatureGates == nil { initialConfig.FeatureGates = map[string]bool{} } - initialConfig.FeatureGates[string(features.KubeletPluginsWatcher)] = enablePluginWatcher initialConfig.FeatureGates[string(features.KubeletPodResources)] = true }) It("Verifies the Kubelet device plugin functionality.", func() { diff --git a/test/integration/apiserver/apply/apply_test.go b/test/integration/apiserver/apply/apply_test.go index d0787190fa..e0858940c8 100644 --- a/test/integration/apiserver/apply/apply_test.go +++ b/test/integration/apiserver/apply/apply_test.go @@ -119,6 +119,19 @@ func TestApplyAlsoCreates(t *testing.T) { if err != nil { t.Fatalf("Failed to retrieve object: %v", err) } + + // Test that we can re apply with a different field manager and don't get conflicts + _, err = client.CoreV1().RESTClient().Patch(types.ApplyPatchType). + Namespace("default"). + Resource(tc.resource). + Name(tc.name). + Param("fieldManager", "apply_test_2"). + Body([]byte(tc.body)). + Do(). + Get() + if err != nil { + t.Fatalf("Failed to re-apply object using Apply patch: %v", err) + } } } diff --git a/test/integration/framework/BUILD b/test/integration/framework/BUILD index e6b14df36e..9d16fe337a 100644 --- a/test/integration/framework/BUILD +++ b/test/integration/framework/BUILD @@ -4,6 +4,7 @@ load( "@io_bazel_rules_go//go:def.bzl", "go_library", ) +load("//build:platforms.bzl", "go_platform_constraint") go_library( name = "go_default_library", @@ -15,9 +16,23 @@ go_library( "test_server.go", "util.go", ], - data = [ - "@com_coreos_etcd//:etcd", - ], + data = select({ + go_platform_constraint( + arch = "arm64", + os = "linux", + ): [ + "@com_coreos_etcd_arm64//:etcd", + ], + go_platform_constraint( + arch = "ppc64le", + os = "linux", + ): [ + "@com_coreos_etcd_ppc64le//:etcd", + ], + "//conditions:default": [ + "@com_coreos_etcd_amd64//:etcd", + ], + }), importpath = "k8s.io/kubernetes/test/integration/framework", deps = [ "//cmd/kube-apiserver/app:go_default_library", diff --git a/test/integration/framework/etcd.go b/test/integration/framework/etcd.go index 97aa267f4b..f66469a370 100644 --- a/test/integration/framework/etcd.go +++ b/test/integration/framework/etcd.go @@ -24,6 +24,7 @@ import ( "os" "os/exec" "path/filepath" + "runtime" "strings" "k8s.io/klog" @@ -43,7 +44,8 @@ You can use 'hack/install-etcd.sh' to install a copy in third_party/. // getEtcdPath returns a path to an etcd executable. func getEtcdPath() (string, error) { - bazelPath := filepath.Join(os.Getenv("RUNFILES_DIR"), "com_coreos_etcd/etcd") + bazelPath := filepath.Join(os.Getenv("RUNFILES_DIR"), fmt.Sprintf("com_coreos_etcd_%s", runtime.GOARCH), "etcd") + p, err := exec.LookPath(bazelPath) if err == nil { return p, nil