mirror of https://github.com/k3s-io/k3s
Merge pull request #51483 from yujuhong/e2e-net-tiers
Automatic merge from submit-queue e2e: Add tests for network tiers in GCE This test depends on #51301, which adds the new feature. Only the `e2e: Add tests for network tiers in GCE` commit is new. #51301 should pass this new test.pull/6/head
Kubernetes Submit Queue
GitHub
commit
022919d1a4
|
|
@ -83,6 +83,7 @@ go_library(
|
|||
go_test(
|
||||
name = "go_default_test",
|
||||
srcs = [
|
||||
"gce_annotations_test.go",
|
||||
"gce_disks_test.go",
|
||||
"gce_healthchecks_test.go",
|
||||
"gce_loadbalancer_external_test.go",
|
||||
|
|
@ -100,6 +101,7 @@ go_test(
|
|||
"//vendor/google.golang.org/api/compute/v1:go_default_library",
|
||||
"//vendor/google.golang.org/api/googleapi:go_default_library",
|
||||
"//vendor/k8s.io/api/core/v1:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
|
||||
],
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -17,11 +17,16 @@ limitations under the License.
|
|||
package gce
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/golang/glog"
|
||||
|
||||
"k8s.io/api/core/v1"
|
||||
)
|
||||
|
||||
type LoadBalancerType string
|
||||
type NetworkTier string
|
||||
|
||||
const (
|
||||
// ServiceAnnotationLoadBalancerType is annotated on a service with type LoadBalancer
|
||||
|
|
@ -39,6 +44,17 @@ const (
|
|||
ServiceAnnotationILBBackendShare = "alpha.cloud.google.com/load-balancer-backend-share"
|
||||
// This annotation did not correctly specify "alpha", so both annotations will be checked.
|
||||
deprecatedServiceAnnotationILBBackendShare = "cloud.google.com/load-balancer-backend-share"
|
||||
|
||||
// NetworkTierAnnotationKey is annotated on a Service object to indicate which
|
||||
// network tier a GCP LB should use. The valid values are "Standard" and
|
||||
// "Premium" (default).
|
||||
NetworkTierAnnotationKey = "cloud.google.com/network-tier"
|
||||
NetworkTierAnnotationStandard = "Standard"
|
||||
NetworkTierAnnotationPremium = "Premium"
|
||||
|
||||
NetworkTierStandard NetworkTier = NetworkTierAnnotationStandard
|
||||
NetworkTierPremium NetworkTier = NetworkTierAnnotationPremium
|
||||
NetworkTierDefault NetworkTier = NetworkTierPremium
|
||||
)
|
||||
|
||||
// GetLoadBalancerAnnotationType returns the type of GCP load balancer which should be assembled.
|
||||
|
|
@ -77,3 +93,42 @@ func GetLoadBalancerAnnotationBackendShare(service *v1.Service) bool {
|
|||
|
||||
return false
|
||||
}
|
||||
|
||||
// GetServiceNetworkTier returns the network tier of GCP load balancer
|
||||
// which should be assembled, and an error if the specified tier is not
|
||||
// supported.
|
||||
func GetServiceNetworkTier(service *v1.Service) (NetworkTier, error) {
|
||||
l, ok := service.Annotations[NetworkTierAnnotationKey]
|
||||
if !ok {
|
||||
return NetworkTierDefault, nil
|
||||
}
|
||||
|
||||
v := NetworkTier(l)
|
||||
switch v {
|
||||
case NetworkTierStandard:
|
||||
fallthrough
|
||||
case NetworkTierPremium:
|
||||
return v, nil
|
||||
default:
|
||||
return NetworkTierDefault, fmt.Errorf("unsupported network tier: %q", v)
|
||||
}
|
||||
}
|
||||
|
||||
// ToGCEValue converts NetworkTier to a string that we can populate the
|
||||
// NetworkTier field of GCE objects.
|
||||
func (n NetworkTier) ToGCEValue() string {
|
||||
return strings.ToUpper(string(n))
|
||||
}
|
||||
|
||||
// NetworkTierGCEValueToType converts the value of the NetworkTier field of a
|
||||
// GCE object to the NetworkTier type.
|
||||
func NetworkTierGCEValueToType(s string) NetworkTier {
|
||||
switch s {
|
||||
case "STANDARD":
|
||||
return NetworkTierStandard
|
||||
case "PREMIUM":
|
||||
return NetworkTierPremium
|
||||
default:
|
||||
return NetworkTier(s)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,70 @@
|
|||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package gce
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestServiceNetworkTierAnnotationKey(t *testing.T) {
|
||||
createTestService := func() *v1.Service {
|
||||
return &v1.Service{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
UID: "randome-uid",
|
||||
Name: "test-svc",
|
||||
Namespace: "test-ns",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
for testName, testCase := range map[string]struct {
|
||||
annotations map[string]string
|
||||
expectedTier NetworkTier
|
||||
expectErr bool
|
||||
}{
|
||||
"Use the default when the annotation does not exist": {
|
||||
annotations: nil,
|
||||
expectedTier: NetworkTierDefault,
|
||||
},
|
||||
"Standard tier": {
|
||||
annotations: map[string]string{NetworkTierAnnotationKey: "Standard"},
|
||||
expectedTier: NetworkTierStandard,
|
||||
},
|
||||
"Premium tier": {
|
||||
annotations: map[string]string{NetworkTierAnnotationKey: "Premium"},
|
||||
expectedTier: NetworkTierPremium,
|
||||
},
|
||||
"Report an error on invalid network tier value": {
|
||||
annotations: map[string]string{NetworkTierAnnotationKey: "Unknown-tier"},
|
||||
expectedTier: NetworkTierPremium,
|
||||
expectErr: true,
|
||||
},
|
||||
} {
|
||||
t.Run(testName, func(t *testing.T) {
|
||||
svc := createTestService()
|
||||
svc.Annotations = testCase.annotations
|
||||
actualTier, err := GetServiceNetworkTier(svc)
|
||||
assert.Equal(t, testCase.expectedTier, actualTier)
|
||||
assert.Equal(t, testCase.expectErr, err != nil)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -99,6 +99,14 @@ func (gce *GCECloud) ListRegionForwardingRules(region string) (*compute.Forwardi
|
|||
return v, mc.Observe(err)
|
||||
}
|
||||
|
||||
// ListRegionForwardingRules lists all RegionalForwardingRules in the project & region.
|
||||
func (gce *GCECloud) ListAlphaRegionForwardingRules(region string) (*computealpha.ForwardingRuleList, error) {
|
||||
mc := newForwardingRuleMetricContextWithVersion("list", region, computeAlphaVersion)
|
||||
// TODO: use PageToken to list all not just the first 500
|
||||
v, err := gce.serviceAlpha.ForwardingRules.List(gce.projectID, region).Do()
|
||||
return v, mc.Observe(err)
|
||||
}
|
||||
|
||||
// CreateRegionForwardingRule creates and returns a
|
||||
// RegionalForwardingRule that points to the given BackendService
|
||||
func (gce *GCECloud) CreateRegionForwardingRule(rule *compute.ForwardingRule, region string) error {
|
||||
|
|
|
|||
|
|
@ -483,6 +483,31 @@ func (j *ServiceTestJig) UpdateServiceOrFail(namespace, name string, update func
|
|||
return svc
|
||||
}
|
||||
|
||||
func (j *ServiceTestJig) WaitForNewIngressIPOrFail(namespace, name, existingIP string, timeout time.Duration) *v1.Service {
|
||||
var service *v1.Service
|
||||
Logf("Waiting up to %v for service %q to get a new ingress IP", timeout, name)
|
||||
pollFunc := func() (bool, error) {
|
||||
svc, err := j.Client.Core().Services(namespace).Get(name, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if len(svc.Status.LoadBalancer.Ingress) == 0 {
|
||||
return false, nil
|
||||
}
|
||||
ip := svc.Status.LoadBalancer.Ingress[0].IP
|
||||
if ip == "" || ip == existingIP {
|
||||
return false, nil
|
||||
}
|
||||
// Got a new IP.
|
||||
service = svc
|
||||
return true, nil
|
||||
}
|
||||
if err := wait.PollImmediate(Poll, timeout, pollFunc); err != nil {
|
||||
Failf("Timeout waiting for service %q to have a new ingress IP", name)
|
||||
}
|
||||
return service
|
||||
}
|
||||
|
||||
func (j *ServiceTestJig) ChangeServiceNodePortOrFail(namespace, name string, initial int) *v1.Service {
|
||||
var err error
|
||||
var service *v1.Service
|
||||
|
|
@ -1421,3 +1446,10 @@ func EnableAndDisableInternalLB() (enable func(svc *v1.Service), disable func(sv
|
|||
|
||||
return
|
||||
}
|
||||
|
||||
func GetServiceLoadBalancerCreationTimeout(cs clientset.Interface) time.Duration {
|
||||
if nodes := GetReadySchedulableNodesOrDie(cs); len(nodes.Items) > LargeClusterMinNodesNumber {
|
||||
return LoadBalancerCreateTimeoutLarge
|
||||
}
|
||||
return LoadBalancerCreateTimeoutDefault
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ go_library(
|
|||
"ingress.go",
|
||||
"kube_proxy.go",
|
||||
"network_policy.go",
|
||||
"network_tiers.go",
|
||||
"networking.go",
|
||||
"networking_perf.go",
|
||||
"no_snat.go",
|
||||
|
|
@ -42,6 +43,7 @@ go_library(
|
|||
"//test/utils:go_default_library",
|
||||
"//vendor/github.com/onsi/ginkgo:go_default_library",
|
||||
"//vendor/github.com/onsi/gomega:go_default_library",
|
||||
"//vendor/google.golang.org/api/compute/v0.alpha:go_default_library",
|
||||
"//vendor/k8s.io/api/core/v1:go_default_library",
|
||||
"//vendor/k8s.io/api/rbac/v1beta1:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,165 @@
|
|||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package network
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
computealpha "google.golang.org/api/compute/v0.alpha"
|
||||
|
||||
"k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/util/wait"
|
||||
clientset "k8s.io/client-go/kubernetes"
|
||||
"k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
|
||||
"k8s.io/kubernetes/pkg/cloudprovider"
|
||||
gcecloud "k8s.io/kubernetes/pkg/cloudprovider/providers/gce"
|
||||
"k8s.io/kubernetes/test/e2e/framework"
|
||||
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
var _ = SIGDescribe("Services [Feature:GCEAlphaFeature][Slow]", func() {
|
||||
f := framework.NewDefaultFramework("services")
|
||||
|
||||
var cs clientset.Interface
|
||||
var internalClientset internalclientset.Interface
|
||||
serviceLBNames := []string{}
|
||||
|
||||
BeforeEach(func() {
|
||||
// This test suite requires the GCE environment.
|
||||
framework.SkipUnlessProviderIs("gce")
|
||||
cs = f.ClientSet
|
||||
internalClientset = f.InternalClientset
|
||||
})
|
||||
|
||||
AfterEach(func() {
|
||||
if CurrentGinkgoTestDescription().Failed {
|
||||
framework.DescribeSvc(f.Namespace.Name)
|
||||
}
|
||||
for _, lb := range serviceLBNames {
|
||||
framework.Logf("cleaning gce resource for %s", lb)
|
||||
framework.CleanupServiceGCEResources(cs, lb, framework.TestContext.CloudConfig.Zone)
|
||||
}
|
||||
//reset serviceLBNames
|
||||
serviceLBNames = []string{}
|
||||
})
|
||||
It("should be able to create and tear down a standard-tier load balancer [Slow]", func() {
|
||||
lagTimeout := framework.LoadBalancerLagTimeoutDefault
|
||||
createTimeout := framework.GetServiceLoadBalancerCreationTimeout(cs)
|
||||
|
||||
svcName := "net-tiers-svc"
|
||||
ns := f.Namespace.Name
|
||||
jig := framework.NewServiceTestJig(cs, svcName)
|
||||
|
||||
By("creating a pod to be part of the service " + svcName)
|
||||
jig.RunOrFail(ns, nil)
|
||||
|
||||
By("creating a Service of type LoadBalancer using the standard network tier")
|
||||
svc := jig.CreateTCPServiceOrFail(ns, func(svc *v1.Service) {
|
||||
svc.Spec.Type = v1.ServiceTypeLoadBalancer
|
||||
setNetworkTier(svc, gcecloud.NetworkTierAnnotationStandard)
|
||||
})
|
||||
// Record the LB name for test cleanup.
|
||||
serviceLBNames = append(serviceLBNames, cloudprovider.GetLoadBalancerName(svc))
|
||||
|
||||
svc = jig.WaitForLoadBalancerOrFail(ns, svcName, createTimeout)
|
||||
lbIngress := &svc.Status.LoadBalancer.Ingress[0]
|
||||
svcPort := int(svc.Spec.Ports[0].Port)
|
||||
ingressIP := framework.GetIngressPoint(lbIngress)
|
||||
|
||||
By("running sanity and reachability checks")
|
||||
jig.SanityCheckService(svc, v1.ServiceTypeLoadBalancer)
|
||||
jig.TestReachableHTTP(ingressIP, svcPort, lagTimeout)
|
||||
// Check the network tier of the forwarding rule.
|
||||
netTier, err := getLBNetworkTierByIP(ingressIP)
|
||||
Expect(err).NotTo(HaveOccurred(), "failed to get the network tier of the load balancer")
|
||||
Expect(netTier).To(Equal(gcecloud.NetworkTierStandard))
|
||||
|
||||
By("updating the Service to use the premium (default) tier")
|
||||
existingIP := ingressIP
|
||||
svc = jig.UpdateServiceOrFail(ns, svcName, func(svc *v1.Service) {
|
||||
clearNetworkTier(svc)
|
||||
})
|
||||
// Wait until the ingress IP changes. Each tier has its own pool of
|
||||
// IPs, so changing tiers implies changing IPs.
|
||||
svc = jig.WaitForNewIngressIPOrFail(ns, svcName, existingIP, createTimeout)
|
||||
lbIngress = &svc.Status.LoadBalancer.Ingress[0]
|
||||
ingressIP = framework.GetIngressPoint(lbIngress)
|
||||
|
||||
By("running sanity and reachability checks")
|
||||
jig.SanityCheckService(svc, v1.ServiceTypeLoadBalancer)
|
||||
jig.TestReachableHTTP(ingressIP, svcPort, lagTimeout)
|
||||
// Check the network tier of the forwarding rule.
|
||||
netTier, err = getLBNetworkTierByIP(ingressIP)
|
||||
Expect(err).NotTo(HaveOccurred(), "failed to get the network tier of the load balancer")
|
||||
Expect(netTier).To(Equal(gcecloud.NetworkTierPremium))
|
||||
|
||||
// TODO: Add tests for user-requested IPs.
|
||||
})
|
||||
})
|
||||
|
||||
func getLBNetworkTierByIP(ip string) (gcecloud.NetworkTier, error) {
|
||||
var rule *computealpha.ForwardingRule
|
||||
// Retry a few times to tolerate flakes.
|
||||
err := wait.PollImmediate(5*time.Second, 15*time.Second, func() (bool, error) {
|
||||
obj, err := getGCEForwardingRuleByIP(ip)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
rule = obj
|
||||
return true, nil
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return gcecloud.NetworkTierGCEValueToType(rule.NetworkTier), nil
|
||||
}
|
||||
|
||||
func getGCEForwardingRuleByIP(ip string) (*computealpha.ForwardingRule, error) {
|
||||
cloud, err := framework.GetGCECloud()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ruleList, err := cloud.ListAlphaRegionForwardingRules(cloud.Region())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, rule := range ruleList.Items {
|
||||
if rule.IPAddress == ip {
|
||||
return rule, nil
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("forwarding rule with ip %q not found", ip)
|
||||
}
|
||||
|
||||
func setNetworkTier(svc *v1.Service, tier string) {
|
||||
key := gcecloud.NetworkTierAnnotationKey
|
||||
if svc.ObjectMeta.Annotations == nil {
|
||||
svc.ObjectMeta.Annotations = map[string]string{}
|
||||
}
|
||||
svc.ObjectMeta.Annotations[key] = tier
|
||||
}
|
||||
|
||||
func clearNetworkTier(svc *v1.Service) {
|
||||
key := gcecloud.NetworkTierAnnotationKey
|
||||
if svc.ObjectMeta.Annotations == nil {
|
||||
return
|
||||
}
|
||||
delete(svc.ObjectMeta.Annotations, key)
|
||||
}
|
||||
Loading…
Reference in New Issue