mirror of https://github.com/k3s-io/k3s
Improve the upgrade test for ingress.
parent
ddea2dd56f
commit
2941c4bcbc
|
@ -372,6 +372,19 @@ func CleanupGCEIngressController(gceController *GCEIngressController) {
|
|||
}
|
||||
}
|
||||
|
||||
func (cont *GCEIngressController) ListGlobalForwardingRules() []*compute.ForwardingRule {
|
||||
gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud)
|
||||
fwdList := []*compute.ForwardingRule{}
|
||||
l, err := gceCloud.ListGlobalForwardingRules()
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
for _, fwd := range l {
|
||||
if cont.isOwned(fwd.Name) {
|
||||
fwdList = append(fwdList, fwd)
|
||||
}
|
||||
}
|
||||
return fwdList
|
||||
}
|
||||
|
||||
func (cont *GCEIngressController) deleteForwardingRule(del bool) string {
|
||||
msg := ""
|
||||
fwList := []compute.ForwardingRule{}
|
||||
|
@ -394,6 +407,13 @@ func (cont *GCEIngressController) deleteForwardingRule(del bool) string {
|
|||
return msg
|
||||
}
|
||||
|
||||
func (cont *GCEIngressController) GetGlobalAddress(ipName string) *compute.Address {
|
||||
gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud)
|
||||
ip, err := gceCloud.GetGlobalAddress(ipName)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
return ip
|
||||
}
|
||||
|
||||
func (cont *GCEIngressController) deleteAddresses(del bool) string {
|
||||
msg := ""
|
||||
ipList := []compute.Address{}
|
||||
|
@ -414,6 +434,32 @@ func (cont *GCEIngressController) deleteAddresses(del bool) string {
|
|||
return msg
|
||||
}
|
||||
|
||||
func (cont *GCEIngressController) ListTargetHttpProxies() []*compute.TargetHttpProxy {
|
||||
gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud)
|
||||
tpList := []*compute.TargetHttpProxy{}
|
||||
l, err := gceCloud.ListTargetHttpProxies()
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
for _, tp := range l {
|
||||
if cont.isOwned(tp.Name) {
|
||||
tpList = append(tpList, tp)
|
||||
}
|
||||
}
|
||||
return tpList
|
||||
}
|
||||
|
||||
func (cont *GCEIngressController) ListTargetHttpsProxies() []*compute.TargetHttpsProxy {
|
||||
gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud)
|
||||
tpsList := []*compute.TargetHttpsProxy{}
|
||||
l, err := gceCloud.ListTargetHttpsProxies()
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
for _, tps := range l {
|
||||
if cont.isOwned(tps.Name) {
|
||||
tpsList = append(tpsList, tps)
|
||||
}
|
||||
}
|
||||
return tpsList
|
||||
}
|
||||
|
||||
func (cont *GCEIngressController) deleteTargetProxy(del bool) string {
|
||||
msg := ""
|
||||
tpList := []compute.TargetHttpProxy{}
|
||||
|
@ -449,6 +495,19 @@ func (cont *GCEIngressController) deleteTargetProxy(del bool) string {
|
|||
return msg
|
||||
}
|
||||
|
||||
func (cont *GCEIngressController) ListUrlMaps() []*compute.UrlMap {
|
||||
gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud)
|
||||
umList := []*compute.UrlMap{}
|
||||
l, err := gceCloud.ListUrlMaps()
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
for _, um := range l {
|
||||
if cont.isOwned(um.Name) {
|
||||
umList = append(umList, um)
|
||||
}
|
||||
}
|
||||
return umList
|
||||
}
|
||||
|
||||
func (cont *GCEIngressController) deleteURLMap(del bool) (msg string) {
|
||||
gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud)
|
||||
umList, err := gceCloud.ListUrlMaps()
|
||||
|
@ -478,6 +537,19 @@ func (cont *GCEIngressController) deleteURLMap(del bool) (msg string) {
|
|||
return msg
|
||||
}
|
||||
|
||||
func (cont *GCEIngressController) ListGlobalBackendServices() []*compute.BackendService {
|
||||
gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud)
|
||||
beList := []*compute.BackendService{}
|
||||
l, err := gceCloud.ListGlobalBackendServices()
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
for _, be := range l {
|
||||
if cont.isOwned(be.Name) {
|
||||
beList = append(beList, be)
|
||||
}
|
||||
}
|
||||
return beList
|
||||
}
|
||||
|
||||
func (cont *GCEIngressController) deleteBackendService(del bool) (msg string) {
|
||||
gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud)
|
||||
beList, err := gceCloud.ListGlobalBackendServices()
|
||||
|
@ -537,6 +609,19 @@ func (cont *GCEIngressController) deleteHTTPHealthCheck(del bool) (msg string) {
|
|||
return msg
|
||||
}
|
||||
|
||||
func (cont *GCEIngressController) ListSslCertificates() []*compute.SslCertificate {
|
||||
gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud)
|
||||
sslList := []*compute.SslCertificate{}
|
||||
l, err := gceCloud.ListSslCertificates()
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
for _, ssl := range l {
|
||||
if cont.isOwned(ssl.Name) {
|
||||
sslList = append(sslList, ssl)
|
||||
}
|
||||
}
|
||||
return sslList
|
||||
}
|
||||
|
||||
func (cont *GCEIngressController) deleteSSLCertificate(del bool) (msg string) {
|
||||
gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud)
|
||||
sslList, err := gceCloud.ListSslCertificates()
|
||||
|
@ -565,6 +650,19 @@ func (cont *GCEIngressController) deleteSSLCertificate(del bool) (msg string) {
|
|||
return msg
|
||||
}
|
||||
|
||||
func (cont *GCEIngressController) ListInstanceGroups() []*compute.InstanceGroup {
|
||||
gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud)
|
||||
igList := []*compute.InstanceGroup{}
|
||||
l, err := gceCloud.ListInstanceGroups(cont.Cloud.Zone)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
for _, ig := range l {
|
||||
if cont.isOwned(ig.Name) {
|
||||
igList = append(igList, ig)
|
||||
}
|
||||
}
|
||||
return igList
|
||||
}
|
||||
|
||||
func (cont *GCEIngressController) deleteInstanceGroup(del bool) (msg string) {
|
||||
gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud)
|
||||
// TODO: E2E cloudprovider has only 1 zone, but the cluster can have many.
|
||||
|
@ -658,6 +756,12 @@ func (cont *GCEIngressController) canDelete(resourceName, creationTimestamp stri
|
|||
return canDeleteWithTimestamp(resourceName, creationTimestamp)
|
||||
}
|
||||
|
||||
// isOwned returns true if the resourceName ends in a suffix matching this
|
||||
// controller UID.
|
||||
func (cont *GCEIngressController) isOwned(resourceName string) bool {
|
||||
return cont.canDelete(resourceName, "", false)
|
||||
}
|
||||
|
||||
// canDeleteNEG returns true if either the name contains this controller's UID,
|
||||
// or the creationTimestamp exceeds the maxAge and del is set to true.
|
||||
func (cont *GCEIngressController) canDeleteNEG(resourceName, creationTimestamp string, delOldResources bool) bool {
|
||||
|
|
|
@ -40,6 +40,15 @@ func EtcdUpgrade(target_storage, target_version string) error {
|
|||
}
|
||||
}
|
||||
|
||||
func IngressUpgrade() error {
|
||||
switch TestContext.Provider {
|
||||
case "gce":
|
||||
return ingressUpgradeGCE()
|
||||
default:
|
||||
return fmt.Errorf("IngressUpgrade() is not implemented for provider %s", TestContext.Provider)
|
||||
}
|
||||
}
|
||||
|
||||
func MasterUpgrade(v string) error {
|
||||
switch TestContext.Provider {
|
||||
case "gce":
|
||||
|
@ -64,6 +73,15 @@ func etcdUpgradeGCE(target_storage, target_version string) error {
|
|||
return err
|
||||
}
|
||||
|
||||
func ingressUpgradeGCE() error {
|
||||
// Flip glbc image from latest release image to HEAD to simulate an upgrade.
|
||||
// Kubelet should restart glbc automatically.
|
||||
sshResult, err := NodeExec(GetMasterHost(), "sudo sed -i -re 's/(image:)(.*)/\\1 gcr.io\\/e2e-ingress-gce\\/ingress-gce-e2e-glbc-amd64:latest/' /etc/kubernetes/manifests/glbc.manifest")
|
||||
// TODO(rramkumar): Ensure glbc pod is in "Running" state before proceeding.
|
||||
LogSSHResult(sshResult)
|
||||
return err
|
||||
}
|
||||
|
||||
// TODO(mrhohn): Remove this function when kube-proxy is run as a DaemonSet by default.
|
||||
func MasterUpgradeGCEWithKubeProxyDaemonSet(v string, enableKubeProxyDaemonSet bool) error {
|
||||
return masterUpgradeGCE(v, enableKubeProxyDaemonSet)
|
||||
|
|
|
@ -72,6 +72,11 @@ var kubeProxyDowngradeTests = []upgrades.Test{
|
|||
&upgrades.IngressUpgradeTest{},
|
||||
}
|
||||
|
||||
// Upgrade ingress with custom image.
|
||||
var ingressUpgradeTests = []upgrades.Test{
|
||||
&upgrades.IngressUpgradeTest{},
|
||||
}
|
||||
|
||||
var _ = SIGDescribe("Upgrade [Feature:Upgrade]", func() {
|
||||
f := framework.NewDefaultFramework("cluster-upgrade")
|
||||
|
||||
|
@ -201,6 +206,31 @@ var _ = SIGDescribe("etcd Upgrade [Feature:EtcdUpgrade]", func() {
|
|||
})
|
||||
})
|
||||
|
||||
var _ = SIGDescribe("ingress Upgrade [Feature:IngressUpgrade]", func() {
|
||||
f := framework.NewDefaultFramework("ingress-upgrade")
|
||||
|
||||
// Create the frameworks here because we can only create them
|
||||
// in a "Describe".
|
||||
testFrameworks := createUpgradeFrameworks(ingressUpgradeTests)
|
||||
Describe("ingress upgrade", func() {
|
||||
It("should maintain a functioning ingress", func() {
|
||||
upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), "")
|
||||
framework.ExpectNoError(err)
|
||||
|
||||
testSuite := &junit.TestSuite{Name: "ingress upgrade"}
|
||||
ingressTest := &junit.TestCase{Name: "[sig-networking] ingress-upgrade", Classname: "upgrade_tests"}
|
||||
testSuite.TestCases = append(testSuite.TestCases, ingressTest)
|
||||
|
||||
upgradeFunc := func() {
|
||||
start := time.Now()
|
||||
defer finalizeUpgradeTest(start, ingressTest)
|
||||
framework.ExpectNoError(framework.IngressUpgrade())
|
||||
}
|
||||
runUpgradeSuite(f, ingressUpgradeTests, testFrameworks, testSuite, upgCtx, upgrades.IngressUpgrade, upgradeFunc)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
var _ = Describe("[sig-apps] stateful Upgrade [Feature:StatefulUpgrade]", func() {
|
||||
f := framework.NewDefaultFramework("stateful-upgrade")
|
||||
|
||||
|
|
|
@ -0,0 +1,11 @@
|
|||
apiVersion: extensions/v1beta1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: static-ip
|
||||
# This annotation is added by the test upon allocating a staticip.
|
||||
# annotations:
|
||||
# kubernetes.io/ingress.global-static-ip-name: "staticip"
|
||||
spec:
|
||||
backend:
|
||||
serviceName: echoheaders-https
|
||||
servicePort: 80
|
|
@ -0,0 +1,16 @@
|
|||
apiVersion: v1
|
||||
kind: ReplicationController
|
||||
metadata:
|
||||
name: echoheaders-https
|
||||
spec:
|
||||
replicas: 2
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: echoheaders-https
|
||||
spec:
|
||||
containers:
|
||||
- name: echoheaders-https
|
||||
image: gcr.io/google_containers/echoserver:1.6
|
||||
ports:
|
||||
- containerPort: 8080
|
|
@ -0,0 +1,15 @@
|
|||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: echoheaders-https
|
||||
labels:
|
||||
app: echoheaders-https
|
||||
spec:
|
||||
type: NodePort
|
||||
ports:
|
||||
- port: 80
|
||||
targetPort: 8080
|
||||
protocol: TCP
|
||||
name: http
|
||||
selector:
|
||||
app: echoheaders-https
|
|
@ -13,4 +13,3 @@ spec:
|
|||
backend:
|
||||
serviceName: echoheaders-https
|
||||
servicePort: 80
|
||||
|
||||
|
|
|
@ -29,9 +29,11 @@ go_library(
|
|||
"//test/e2e/common:go_default_library",
|
||||
"//test/e2e/framework:go_default_library",
|
||||
"//test/utils/image:go_default_library",
|
||||
"//vendor/github.com/davecgh/go-spew/spew:go_default_library",
|
||||
"//vendor/github.com/onsi/ginkgo:go_default_library",
|
||||
"//vendor/github.com/onsi/gomega:go_default_library",
|
||||
"//vendor/github.com/onsi/gomega/gstruct:go_default_library",
|
||||
"//vendor/google.golang.org/api/compute/v1:go_default_library",
|
||||
"//vendor/k8s.io/api/autoscaling/v1:go_default_library",
|
||||
"//vendor/k8s.io/api/core/v1:go_default_library",
|
||||
"//vendor/k8s.io/api/extensions/v1beta1:go_default_library",
|
||||
|
|
|
@ -17,12 +17,17 @@ limitations under the License.
|
|||
package upgrades
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
. "github.com/onsi/ginkgo"
|
||||
|
||||
compute "google.golang.org/api/compute/v1"
|
||||
extensions "k8s.io/api/extensions/v1beta1"
|
||||
"k8s.io/apimachinery/pkg/util/wait"
|
||||
"k8s.io/kubernetes/test/e2e/framework"
|
||||
)
|
||||
|
@ -30,16 +35,34 @@ import (
|
|||
// IngressUpgradeTest adapts the Ingress e2e for upgrade testing
|
||||
type IngressUpgradeTest struct {
|
||||
gceController *framework.GCEIngressController
|
||||
// holds GCP resources pre-upgrade
|
||||
resourceStore *GCPResourceStore
|
||||
jig *framework.IngressTestJig
|
||||
httpClient *http.Client
|
||||
ip string
|
||||
ipName string
|
||||
}
|
||||
|
||||
// GCPResourceStore keeps track of the GCP resources spun up by an ingress.
|
||||
// Note: Fields are exported so that we can utilize reflection.
|
||||
type GCPResourceStore struct {
|
||||
Fw *compute.Firewall
|
||||
FwdList []*compute.ForwardingRule
|
||||
UmList []*compute.UrlMap
|
||||
TpList []*compute.TargetHttpProxy
|
||||
TpsList []*compute.TargetHttpsProxy
|
||||
SslList []*compute.SslCertificate
|
||||
BeList []*compute.BackendService
|
||||
Ip *compute.Address
|
||||
IgList []*compute.InstanceGroup
|
||||
}
|
||||
|
||||
func (IngressUpgradeTest) Name() string { return "ingress-upgrade" }
|
||||
|
||||
// Setup creates a GLBC, allocates an ip, and an ingress resource,
|
||||
// then waits for a successful connectivity check to the ip.
|
||||
// Also keeps track of all load balancer resources for cross-checking
|
||||
// during an IngressUpgrade.
|
||||
func (t *IngressUpgradeTest) Setup(f *framework.Framework) {
|
||||
framework.SkipUnlessProviderIs("gce", "gke")
|
||||
|
||||
|
@ -66,13 +89,18 @@ func (t *IngressUpgradeTest) Setup(f *framework.Framework) {
|
|||
|
||||
// Create a working basic Ingress
|
||||
By(fmt.Sprintf("allocated static ip %v: %v through the GCE cloud provider", t.ipName, t.ip))
|
||||
jig.CreateIngress(filepath.Join(framework.IngressManifestPath, "static-ip"), ns.Name, map[string]string{
|
||||
jig.CreateIngress(filepath.Join(framework.IngressManifestPath, "static-ip-2"), ns.Name, map[string]string{
|
||||
"kubernetes.io/ingress.global-static-ip-name": t.ipName,
|
||||
"kubernetes.io/ingress.allow-http": "false",
|
||||
}, map[string]string{})
|
||||
jig.AddHTTPS("tls-secret", "ingress.test.com")
|
||||
|
||||
By("waiting for Ingress to come up with ip: " + t.ip)
|
||||
framework.ExpectNoError(framework.PollURL(fmt.Sprintf("https://%v/", t.ip), "", framework.LoadBalancerPollTimeout, jig.PollInterval, t.httpClient, false))
|
||||
|
||||
By("keeping track of GCP resources created by Ingress")
|
||||
t.resourceStore = &GCPResourceStore{}
|
||||
t.populateGCPResourceStore(t.resourceStore)
|
||||
}
|
||||
|
||||
// Test waits for the upgrade to complete, and then verifies
|
||||
|
@ -85,6 +113,8 @@ func (t *IngressUpgradeTest) Test(f *framework.Framework, done <-chan struct{},
|
|||
// while it's down will leak cloud resources, because the ingress
|
||||
// controller doesn't checkpoint to disk.
|
||||
t.verify(f, done, true)
|
||||
case IngressUpgrade:
|
||||
t.verify(f, done, true)
|
||||
default:
|
||||
// Currently ingress gets disrupted across node upgrade, because endpoints
|
||||
// get killed and we don't have any guarantees that 2 nodes don't overlap
|
||||
|
@ -99,6 +129,7 @@ func (t *IngressUpgradeTest) Teardown(f *framework.Framework) {
|
|||
if CurrentGinkgoTestDescription().Failed {
|
||||
framework.DescribeIng(t.gceController.Ns)
|
||||
}
|
||||
|
||||
if t.jig.Ingress != nil {
|
||||
By("Deleting ingress")
|
||||
t.jig.TryDeleteIngress()
|
||||
|
@ -122,4 +153,80 @@ func (t *IngressUpgradeTest) verify(f *framework.Framework, done <-chan struct{}
|
|||
}
|
||||
By("hitting the Ingress IP " + t.ip)
|
||||
framework.ExpectNoError(framework.PollURL(fmt.Sprintf("https://%v/", t.ip), "", framework.LoadBalancerPollTimeout, t.jig.PollInterval, t.httpClient, false))
|
||||
|
||||
// We want to manually trigger a sync because then we can easily verify
|
||||
// a correct sync completed after update.
|
||||
By("updating ingress spec to manually trigger a sync")
|
||||
t.jig.Update(func(ing *extensions.Ingress) {
|
||||
ing.Spec.TLS[0].Hosts = append(ing.Spec.TLS[0].Hosts, "ingress.test.com")
|
||||
ing.Spec.Rules = append(
|
||||
ing.Spec.Rules,
|
||||
extensions.IngressRule{
|
||||
Host: "ingress.test.com",
|
||||
IngressRuleValue: extensions.IngressRuleValue{
|
||||
HTTP: &extensions.HTTPIngressRuleValue{
|
||||
Paths: []extensions.HTTPIngressPath{
|
||||
{
|
||||
Path: "/test",
|
||||
// Note: Dependant on using "static-ip-2" manifest.
|
||||
Backend: *(ing.Spec.Backend),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
})
|
||||
// WaitForIngress() tests that all paths are pinged, which is how we know
|
||||
// everything is synced with the cloud.
|
||||
t.jig.WaitForIngress(false)
|
||||
By("comparing GCP resources post-upgrade")
|
||||
postUpgradeResourceStore := &GCPResourceStore{}
|
||||
t.populateGCPResourceStore(postUpgradeResourceStore)
|
||||
framework.ExpectNoError(compareGCPResourceStores(t.resourceStore, postUpgradeResourceStore, func(v1 reflect.Value, v2 reflect.Value) error {
|
||||
i1 := v1.Interface()
|
||||
i2 := v2.Interface()
|
||||
// Skip verifying the UrlMap since we did that via WaitForIngress()
|
||||
if !reflect.DeepEqual(i1, i2) && (v1.Type() != reflect.TypeOf([]*compute.UrlMap{})) {
|
||||
return spew.Errorf("resources after ingress upgrade were different:\n Pre-Upgrade: %#v\n Post-Upgrade: %#v", i1, i2)
|
||||
}
|
||||
return nil
|
||||
}))
|
||||
}
|
||||
|
||||
func (t *IngressUpgradeTest) populateGCPResourceStore(resourceStore *GCPResourceStore) {
|
||||
cont := t.gceController
|
||||
resourceStore.Fw = cont.GetFirewallRule()
|
||||
resourceStore.FwdList = cont.ListGlobalForwardingRules()
|
||||
resourceStore.UmList = cont.ListUrlMaps()
|
||||
resourceStore.TpList = cont.ListTargetHttpProxies()
|
||||
resourceStore.TpsList = cont.ListTargetHttpsProxies()
|
||||
resourceStore.SslList = cont.ListSslCertificates()
|
||||
resourceStore.BeList = cont.ListGlobalBackendServices()
|
||||
resourceStore.Ip = cont.GetGlobalAddress(t.ipName)
|
||||
resourceStore.IgList = cont.ListInstanceGroups()
|
||||
}
|
||||
|
||||
func compareGCPResourceStores(rs1 *GCPResourceStore, rs2 *GCPResourceStore, compare func(v1 reflect.Value, v2 reflect.Value) error) error {
|
||||
// Before we do a comparison, remove the ServerResponse field from the
|
||||
// Compute API structs. This is needed because two objects could be the same
|
||||
// but their ServerResponse will be different if they were populated through
|
||||
// separate API calls.
|
||||
rs1Json, _ := json.Marshal(rs1)
|
||||
rs2Json, _ := json.Marshal(rs2)
|
||||
rs1New := &GCPResourceStore{}
|
||||
rs2New := &GCPResourceStore{}
|
||||
json.Unmarshal(rs1Json, rs1New)
|
||||
json.Unmarshal(rs2Json, rs2New)
|
||||
|
||||
// Iterate through struct fields and perform equality checks on the fields.
|
||||
// We do this rather than performing a deep equal on the struct itself because
|
||||
// it is easier to log which field, if any, is not the same.
|
||||
rs1V := reflect.ValueOf(*rs1New)
|
||||
rs2V := reflect.ValueOf(*rs2New)
|
||||
for i := 0; i < rs1V.NumField(); i++ {
|
||||
if err := compare(rs1V.Field(i), rs2V.Field(i)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
|
@ -40,6 +40,9 @@ const (
|
|||
// EtcdUpgrade indicates that only etcd is being upgraded (or migrated
|
||||
// between storage versions).
|
||||
EtcdUpgrade
|
||||
|
||||
// IngressUpgrade indicates that only ingress is being upgraded.
|
||||
IngressUpgrade
|
||||
)
|
||||
|
||||
// Test is an interface for upgrade tests.
|
||||
|
|
Loading…
Reference in New Issue