Improve the upgrade test for ingress.

pull/6/head
Rohit Ramkumar 2018-01-09 21:07:01 -08:00
parent ddea2dd56f
commit 2941c4bcbc
10 changed files with 307 additions and 2 deletions

View File

@ -372,6 +372,19 @@ func CleanupGCEIngressController(gceController *GCEIngressController) {
}
}
func (cont *GCEIngressController) ListGlobalForwardingRules() []*compute.ForwardingRule {
gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud)
fwdList := []*compute.ForwardingRule{}
l, err := gceCloud.ListGlobalForwardingRules()
Expect(err).NotTo(HaveOccurred())
for _, fwd := range l {
if cont.isOwned(fwd.Name) {
fwdList = append(fwdList, fwd)
}
}
return fwdList
}
func (cont *GCEIngressController) deleteForwardingRule(del bool) string {
msg := ""
fwList := []compute.ForwardingRule{}
@ -394,6 +407,13 @@ func (cont *GCEIngressController) deleteForwardingRule(del bool) string {
return msg
}
func (cont *GCEIngressController) GetGlobalAddress(ipName string) *compute.Address {
gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud)
ip, err := gceCloud.GetGlobalAddress(ipName)
Expect(err).NotTo(HaveOccurred())
return ip
}
func (cont *GCEIngressController) deleteAddresses(del bool) string {
msg := ""
ipList := []compute.Address{}
@ -414,6 +434,32 @@ func (cont *GCEIngressController) deleteAddresses(del bool) string {
return msg
}
func (cont *GCEIngressController) ListTargetHttpProxies() []*compute.TargetHttpProxy {
gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud)
tpList := []*compute.TargetHttpProxy{}
l, err := gceCloud.ListTargetHttpProxies()
Expect(err).NotTo(HaveOccurred())
for _, tp := range l {
if cont.isOwned(tp.Name) {
tpList = append(tpList, tp)
}
}
return tpList
}
func (cont *GCEIngressController) ListTargetHttpsProxies() []*compute.TargetHttpsProxy {
gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud)
tpsList := []*compute.TargetHttpsProxy{}
l, err := gceCloud.ListTargetHttpsProxies()
Expect(err).NotTo(HaveOccurred())
for _, tps := range l {
if cont.isOwned(tps.Name) {
tpsList = append(tpsList, tps)
}
}
return tpsList
}
func (cont *GCEIngressController) deleteTargetProxy(del bool) string {
msg := ""
tpList := []compute.TargetHttpProxy{}
@ -449,6 +495,19 @@ func (cont *GCEIngressController) deleteTargetProxy(del bool) string {
return msg
}
func (cont *GCEIngressController) ListUrlMaps() []*compute.UrlMap {
gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud)
umList := []*compute.UrlMap{}
l, err := gceCloud.ListUrlMaps()
Expect(err).NotTo(HaveOccurred())
for _, um := range l {
if cont.isOwned(um.Name) {
umList = append(umList, um)
}
}
return umList
}
func (cont *GCEIngressController) deleteURLMap(del bool) (msg string) {
gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud)
umList, err := gceCloud.ListUrlMaps()
@ -478,6 +537,19 @@ func (cont *GCEIngressController) deleteURLMap(del bool) (msg string) {
return msg
}
func (cont *GCEIngressController) ListGlobalBackendServices() []*compute.BackendService {
gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud)
beList := []*compute.BackendService{}
l, err := gceCloud.ListGlobalBackendServices()
Expect(err).NotTo(HaveOccurred())
for _, be := range l {
if cont.isOwned(be.Name) {
beList = append(beList, be)
}
}
return beList
}
func (cont *GCEIngressController) deleteBackendService(del bool) (msg string) {
gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud)
beList, err := gceCloud.ListGlobalBackendServices()
@ -537,6 +609,19 @@ func (cont *GCEIngressController) deleteHTTPHealthCheck(del bool) (msg string) {
return msg
}
func (cont *GCEIngressController) ListSslCertificates() []*compute.SslCertificate {
gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud)
sslList := []*compute.SslCertificate{}
l, err := gceCloud.ListSslCertificates()
Expect(err).NotTo(HaveOccurred())
for _, ssl := range l {
if cont.isOwned(ssl.Name) {
sslList = append(sslList, ssl)
}
}
return sslList
}
func (cont *GCEIngressController) deleteSSLCertificate(del bool) (msg string) {
gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud)
sslList, err := gceCloud.ListSslCertificates()
@ -565,6 +650,19 @@ func (cont *GCEIngressController) deleteSSLCertificate(del bool) (msg string) {
return msg
}
func (cont *GCEIngressController) ListInstanceGroups() []*compute.InstanceGroup {
gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud)
igList := []*compute.InstanceGroup{}
l, err := gceCloud.ListInstanceGroups(cont.Cloud.Zone)
Expect(err).NotTo(HaveOccurred())
for _, ig := range l {
if cont.isOwned(ig.Name) {
igList = append(igList, ig)
}
}
return igList
}
func (cont *GCEIngressController) deleteInstanceGroup(del bool) (msg string) {
gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud)
// TODO: E2E cloudprovider has only 1 zone, but the cluster can have many.
@ -658,6 +756,12 @@ func (cont *GCEIngressController) canDelete(resourceName, creationTimestamp stri
return canDeleteWithTimestamp(resourceName, creationTimestamp)
}
// isOwned returns true if the resourceName ends in a suffix matching this
// controller UID.
func (cont *GCEIngressController) isOwned(resourceName string) bool {
return cont.canDelete(resourceName, "", false)
}
// canDeleteNEG returns true if either the name contains this controller's UID,
// or the creationTimestamp exceeds the maxAge and del is set to true.
func (cont *GCEIngressController) canDeleteNEG(resourceName, creationTimestamp string, delOldResources bool) bool {

View File

@ -40,6 +40,15 @@ func EtcdUpgrade(target_storage, target_version string) error {
}
}
func IngressUpgrade() error {
switch TestContext.Provider {
case "gce":
return ingressUpgradeGCE()
default:
return fmt.Errorf("IngressUpgrade() is not implemented for provider %s", TestContext.Provider)
}
}
func MasterUpgrade(v string) error {
switch TestContext.Provider {
case "gce":
@ -64,6 +73,15 @@ func etcdUpgradeGCE(target_storage, target_version string) error {
return err
}
func ingressUpgradeGCE() error {
// Flip glbc image from latest release image to HEAD to simulate an upgrade.
// Kubelet should restart glbc automatically.
sshResult, err := NodeExec(GetMasterHost(), "sudo sed -i -re 's/(image:)(.*)/\\1 gcr.io\\/e2e-ingress-gce\\/ingress-gce-e2e-glbc-amd64:latest/' /etc/kubernetes/manifests/glbc.manifest")
// TODO(rramkumar): Ensure glbc pod is in "Running" state before proceeding.
LogSSHResult(sshResult)
return err
}
// TODO(mrhohn): Remove this function when kube-proxy is run as a DaemonSet by default.
func MasterUpgradeGCEWithKubeProxyDaemonSet(v string, enableKubeProxyDaemonSet bool) error {
return masterUpgradeGCE(v, enableKubeProxyDaemonSet)

View File

@ -72,6 +72,11 @@ var kubeProxyDowngradeTests = []upgrades.Test{
&upgrades.IngressUpgradeTest{},
}
// Upgrade ingress with custom image.
var ingressUpgradeTests = []upgrades.Test{
&upgrades.IngressUpgradeTest{},
}
var _ = SIGDescribe("Upgrade [Feature:Upgrade]", func() {
f := framework.NewDefaultFramework("cluster-upgrade")
@ -201,6 +206,31 @@ var _ = SIGDescribe("etcd Upgrade [Feature:EtcdUpgrade]", func() {
})
})
var _ = SIGDescribe("ingress Upgrade [Feature:IngressUpgrade]", func() {
f := framework.NewDefaultFramework("ingress-upgrade")
// Create the frameworks here because we can only create them
// in a "Describe".
testFrameworks := createUpgradeFrameworks(ingressUpgradeTests)
Describe("ingress upgrade", func() {
It("should maintain a functioning ingress", func() {
upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), "")
framework.ExpectNoError(err)
testSuite := &junit.TestSuite{Name: "ingress upgrade"}
ingressTest := &junit.TestCase{Name: "[sig-networking] ingress-upgrade", Classname: "upgrade_tests"}
testSuite.TestCases = append(testSuite.TestCases, ingressTest)
upgradeFunc := func() {
start := time.Now()
defer finalizeUpgradeTest(start, ingressTest)
framework.ExpectNoError(framework.IngressUpgrade())
}
runUpgradeSuite(f, ingressUpgradeTests, testFrameworks, testSuite, upgCtx, upgrades.IngressUpgrade, upgradeFunc)
})
})
})
var _ = Describe("[sig-apps] stateful Upgrade [Feature:StatefulUpgrade]", func() {
f := framework.NewDefaultFramework("stateful-upgrade")

View File

@ -0,0 +1,11 @@
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: static-ip
# This annotation is added by the test upon allocating a staticip.
# annotations:
# kubernetes.io/ingress.global-static-ip-name: "staticip"
spec:
backend:
serviceName: echoheaders-https
servicePort: 80

View File

@ -0,0 +1,16 @@
apiVersion: v1
kind: ReplicationController
metadata:
name: echoheaders-https
spec:
replicas: 2
template:
metadata:
labels:
app: echoheaders-https
spec:
containers:
- name: echoheaders-https
image: gcr.io/google_containers/echoserver:1.6
ports:
- containerPort: 8080

View File

@ -0,0 +1,15 @@
apiVersion: v1
kind: Service
metadata:
name: echoheaders-https
labels:
app: echoheaders-https
spec:
type: NodePort
ports:
- port: 80
targetPort: 8080
protocol: TCP
name: http
selector:
app: echoheaders-https

View File

@ -13,4 +13,3 @@ spec:
backend:
serviceName: echoheaders-https
servicePort: 80

View File

@ -29,9 +29,11 @@ go_library(
"//test/e2e/common:go_default_library",
"//test/e2e/framework:go_default_library",
"//test/utils/image:go_default_library",
"//vendor/github.com/davecgh/go-spew/spew:go_default_library",
"//vendor/github.com/onsi/ginkgo:go_default_library",
"//vendor/github.com/onsi/gomega:go_default_library",
"//vendor/github.com/onsi/gomega/gstruct:go_default_library",
"//vendor/google.golang.org/api/compute/v1:go_default_library",
"//vendor/k8s.io/api/autoscaling/v1:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/api/extensions/v1beta1:go_default_library",

View File

@ -17,12 +17,17 @@ limitations under the License.
package upgrades
import (
"encoding/json"
"fmt"
"net/http"
"path/filepath"
"reflect"
"github.com/davecgh/go-spew/spew"
. "github.com/onsi/ginkgo"
compute "google.golang.org/api/compute/v1"
extensions "k8s.io/api/extensions/v1beta1"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/kubernetes/test/e2e/framework"
)
@ -30,16 +35,34 @@ import (
// IngressUpgradeTest adapts the Ingress e2e for upgrade testing
type IngressUpgradeTest struct {
gceController *framework.GCEIngressController
// holds GCP resources pre-upgrade
resourceStore *GCPResourceStore
jig *framework.IngressTestJig
httpClient *http.Client
ip string
ipName string
}
// GCPResourceStore keeps track of the GCP resources spun up by an ingress.
// Note: Fields are exported so that we can utilize reflection.
type GCPResourceStore struct {
Fw *compute.Firewall
FwdList []*compute.ForwardingRule
UmList []*compute.UrlMap
TpList []*compute.TargetHttpProxy
TpsList []*compute.TargetHttpsProxy
SslList []*compute.SslCertificate
BeList []*compute.BackendService
Ip *compute.Address
IgList []*compute.InstanceGroup
}
func (IngressUpgradeTest) Name() string { return "ingress-upgrade" }
// Setup creates a GLBC, allocates an ip, and an ingress resource,
// then waits for a successful connectivity check to the ip.
// Also keeps track of all load balancer resources for cross-checking
// during an IngressUpgrade.
func (t *IngressUpgradeTest) Setup(f *framework.Framework) {
framework.SkipUnlessProviderIs("gce", "gke")
@ -66,13 +89,18 @@ func (t *IngressUpgradeTest) Setup(f *framework.Framework) {
// Create a working basic Ingress
By(fmt.Sprintf("allocated static ip %v: %v through the GCE cloud provider", t.ipName, t.ip))
jig.CreateIngress(filepath.Join(framework.IngressManifestPath, "static-ip"), ns.Name, map[string]string{
jig.CreateIngress(filepath.Join(framework.IngressManifestPath, "static-ip-2"), ns.Name, map[string]string{
"kubernetes.io/ingress.global-static-ip-name": t.ipName,
"kubernetes.io/ingress.allow-http": "false",
}, map[string]string{})
jig.AddHTTPS("tls-secret", "ingress.test.com")
By("waiting for Ingress to come up with ip: " + t.ip)
framework.ExpectNoError(framework.PollURL(fmt.Sprintf("https://%v/", t.ip), "", framework.LoadBalancerPollTimeout, jig.PollInterval, t.httpClient, false))
By("keeping track of GCP resources created by Ingress")
t.resourceStore = &GCPResourceStore{}
t.populateGCPResourceStore(t.resourceStore)
}
// Test waits for the upgrade to complete, and then verifies
@ -85,6 +113,8 @@ func (t *IngressUpgradeTest) Test(f *framework.Framework, done <-chan struct{},
// while it's down will leak cloud resources, because the ingress
// controller doesn't checkpoint to disk.
t.verify(f, done, true)
case IngressUpgrade:
t.verify(f, done, true)
default:
// Currently ingress gets disrupted across node upgrade, because endpoints
// get killed and we don't have any guarantees that 2 nodes don't overlap
@ -99,6 +129,7 @@ func (t *IngressUpgradeTest) Teardown(f *framework.Framework) {
if CurrentGinkgoTestDescription().Failed {
framework.DescribeIng(t.gceController.Ns)
}
if t.jig.Ingress != nil {
By("Deleting ingress")
t.jig.TryDeleteIngress()
@ -122,4 +153,80 @@ func (t *IngressUpgradeTest) verify(f *framework.Framework, done <-chan struct{}
}
By("hitting the Ingress IP " + t.ip)
framework.ExpectNoError(framework.PollURL(fmt.Sprintf("https://%v/", t.ip), "", framework.LoadBalancerPollTimeout, t.jig.PollInterval, t.httpClient, false))
// We want to manually trigger a sync because then we can easily verify
// a correct sync completed after update.
By("updating ingress spec to manually trigger a sync")
t.jig.Update(func(ing *extensions.Ingress) {
ing.Spec.TLS[0].Hosts = append(ing.Spec.TLS[0].Hosts, "ingress.test.com")
ing.Spec.Rules = append(
ing.Spec.Rules,
extensions.IngressRule{
Host: "ingress.test.com",
IngressRuleValue: extensions.IngressRuleValue{
HTTP: &extensions.HTTPIngressRuleValue{
Paths: []extensions.HTTPIngressPath{
{
Path: "/test",
// Note: Dependant on using "static-ip-2" manifest.
Backend: *(ing.Spec.Backend),
},
},
},
},
})
})
// WaitForIngress() tests that all paths are pinged, which is how we know
// everything is synced with the cloud.
t.jig.WaitForIngress(false)
By("comparing GCP resources post-upgrade")
postUpgradeResourceStore := &GCPResourceStore{}
t.populateGCPResourceStore(postUpgradeResourceStore)
framework.ExpectNoError(compareGCPResourceStores(t.resourceStore, postUpgradeResourceStore, func(v1 reflect.Value, v2 reflect.Value) error {
i1 := v1.Interface()
i2 := v2.Interface()
// Skip verifying the UrlMap since we did that via WaitForIngress()
if !reflect.DeepEqual(i1, i2) && (v1.Type() != reflect.TypeOf([]*compute.UrlMap{})) {
return spew.Errorf("resources after ingress upgrade were different:\n Pre-Upgrade: %#v\n Post-Upgrade: %#v", i1, i2)
}
return nil
}))
}
func (t *IngressUpgradeTest) populateGCPResourceStore(resourceStore *GCPResourceStore) {
cont := t.gceController
resourceStore.Fw = cont.GetFirewallRule()
resourceStore.FwdList = cont.ListGlobalForwardingRules()
resourceStore.UmList = cont.ListUrlMaps()
resourceStore.TpList = cont.ListTargetHttpProxies()
resourceStore.TpsList = cont.ListTargetHttpsProxies()
resourceStore.SslList = cont.ListSslCertificates()
resourceStore.BeList = cont.ListGlobalBackendServices()
resourceStore.Ip = cont.GetGlobalAddress(t.ipName)
resourceStore.IgList = cont.ListInstanceGroups()
}
func compareGCPResourceStores(rs1 *GCPResourceStore, rs2 *GCPResourceStore, compare func(v1 reflect.Value, v2 reflect.Value) error) error {
// Before we do a comparison, remove the ServerResponse field from the
// Compute API structs. This is needed because two objects could be the same
// but their ServerResponse will be different if they were populated through
// separate API calls.
rs1Json, _ := json.Marshal(rs1)
rs2Json, _ := json.Marshal(rs2)
rs1New := &GCPResourceStore{}
rs2New := &GCPResourceStore{}
json.Unmarshal(rs1Json, rs1New)
json.Unmarshal(rs2Json, rs2New)
// Iterate through struct fields and perform equality checks on the fields.
// We do this rather than performing a deep equal on the struct itself because
// it is easier to log which field, if any, is not the same.
rs1V := reflect.ValueOf(*rs1New)
rs2V := reflect.ValueOf(*rs2New)
for i := 0; i < rs1V.NumField(); i++ {
if err := compare(rs1V.Field(i), rs2V.Field(i)); err != nil {
return err
}
}
return nil
}

View File

@ -40,6 +40,9 @@ const (
// EtcdUpgrade indicates that only etcd is being upgraded (or migrated
// between storage versions).
EtcdUpgrade
// IngressUpgrade indicates that only ingress is being upgraded.
IngressUpgrade
)
// Test is an interface for upgrade tests.