mirror of https://github.com/k3s-io/k3s
199 lines
9.1 KiB
Go
199 lines
9.1 KiB
Go
/*
|
|
Copyright 2016 The Kubernetes Authors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package network
|
|
|
|
import (
|
|
"fmt"
|
|
"time"
|
|
|
|
"k8s.io/api/core/v1"
|
|
"k8s.io/apimachinery/pkg/util/sets"
|
|
clientset "k8s.io/client-go/kubernetes"
|
|
cloudprovider "k8s.io/cloud-provider"
|
|
gcecloud "k8s.io/kubernetes/pkg/cloudprovider/providers/gce"
|
|
"k8s.io/kubernetes/pkg/master/ports"
|
|
"k8s.io/kubernetes/test/e2e/framework"
|
|
"k8s.io/kubernetes/test/e2e/framework/providers/gce"
|
|
|
|
. "github.com/onsi/ginkgo"
|
|
. "github.com/onsi/gomega"
|
|
)
|
|
|
|
var _ = SIGDescribe("Firewall rule", func() {
|
|
var firewall_test_name = "firewall-test"
|
|
f := framework.NewDefaultFramework(firewall_test_name)
|
|
|
|
var cs clientset.Interface
|
|
var cloudConfig framework.CloudConfig
|
|
var gceCloud *gcecloud.Cloud
|
|
|
|
BeforeEach(func() {
|
|
framework.SkipUnlessProviderIs("gce")
|
|
|
|
var err error
|
|
cs = f.ClientSet
|
|
cloudConfig = framework.TestContext.CloudConfig
|
|
gceCloud, err = gce.GetGCECloud()
|
|
Expect(err).NotTo(HaveOccurred())
|
|
})
|
|
|
|
// This test takes around 6 minutes to run
|
|
It("[Slow] [Serial] should create valid firewall rules for LoadBalancer type service", func() {
|
|
ns := f.Namespace.Name
|
|
// This source ranges is just used to examine we have exact same things on LB firewall rules
|
|
firewallTestSourceRanges := []string{"0.0.0.0/1", "128.0.0.0/1"}
|
|
serviceName := "firewall-test-loadbalancer"
|
|
|
|
By("Getting cluster ID")
|
|
clusterID, err := gce.GetClusterID(cs)
|
|
Expect(err).NotTo(HaveOccurred())
|
|
framework.Logf("Got cluster ID: %v", clusterID)
|
|
|
|
jig := framework.NewServiceTestJig(cs, serviceName)
|
|
nodeList := jig.GetNodes(framework.MaxNodesForEndpointsTests)
|
|
Expect(nodeList).NotTo(BeNil())
|
|
nodesNames := jig.GetNodesNames(framework.MaxNodesForEndpointsTests)
|
|
if len(nodesNames) <= 0 {
|
|
framework.Failf("Expect at least 1 node, got: %v", nodesNames)
|
|
}
|
|
nodesSet := sets.NewString(nodesNames...)
|
|
|
|
By("Creating a LoadBalancer type service with ExternalTrafficPolicy=Global")
|
|
svc := jig.CreateLoadBalancerService(ns, serviceName, framework.LoadBalancerCreateTimeoutDefault, func(svc *v1.Service) {
|
|
svc.Spec.Ports = []v1.ServicePort{{Protocol: v1.ProtocolTCP, Port: gce.FirewallTestHttpPort}}
|
|
svc.Spec.LoadBalancerSourceRanges = firewallTestSourceRanges
|
|
})
|
|
defer func() {
|
|
jig.UpdateServiceOrFail(svc.Namespace, svc.Name, func(svc *v1.Service) {
|
|
svc.Spec.Type = v1.ServiceTypeNodePort
|
|
svc.Spec.LoadBalancerSourceRanges = nil
|
|
})
|
|
Expect(cs.CoreV1().Services(svc.Namespace).Delete(svc.Name, nil)).NotTo(HaveOccurred())
|
|
By("Waiting for the local traffic health check firewall rule to be deleted")
|
|
localHCFwName := gce.MakeHealthCheckFirewallNameForLBService(clusterID, cloudprovider.DefaultLoadBalancerName(svc), false)
|
|
_, err := gce.WaitForFirewallRule(gceCloud, localHCFwName, false, framework.LoadBalancerCleanupTimeout)
|
|
Expect(err).NotTo(HaveOccurred())
|
|
}()
|
|
svcExternalIP := svc.Status.LoadBalancer.Ingress[0].IP
|
|
|
|
By("Checking if service's firewall rule is correct")
|
|
lbFw := gce.ConstructFirewallForLBService(svc, cloudConfig.NodeTag)
|
|
fw, err := gceCloud.GetFirewall(lbFw.Name)
|
|
Expect(err).NotTo(HaveOccurred())
|
|
Expect(gce.VerifyFirewallRule(fw, lbFw, cloudConfig.Network, false)).NotTo(HaveOccurred())
|
|
|
|
By("Checking if service's nodes health check firewall rule is correct")
|
|
nodesHCFw := gce.ConstructHealthCheckFirewallForLBService(clusterID, svc, cloudConfig.NodeTag, true)
|
|
fw, err = gceCloud.GetFirewall(nodesHCFw.Name)
|
|
Expect(err).NotTo(HaveOccurred())
|
|
Expect(gce.VerifyFirewallRule(fw, nodesHCFw, cloudConfig.Network, false)).NotTo(HaveOccurred())
|
|
|
|
// OnlyLocal service is needed to examine which exact nodes the requests are being forwarded to by the Load Balancer on GCE
|
|
By("Updating LoadBalancer service to ExternalTrafficPolicy=Local")
|
|
svc = jig.UpdateServiceOrFail(svc.Namespace, svc.Name, func(svc *v1.Service) {
|
|
svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyTypeLocal
|
|
})
|
|
|
|
By("Waiting for the nodes health check firewall rule to be deleted")
|
|
_, err = gce.WaitForFirewallRule(gceCloud, nodesHCFw.Name, false, framework.LoadBalancerCleanupTimeout)
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
By("Waiting for the correct local traffic health check firewall rule to be created")
|
|
localHCFw := gce.ConstructHealthCheckFirewallForLBService(clusterID, svc, cloudConfig.NodeTag, false)
|
|
fw, err = gce.WaitForFirewallRule(gceCloud, localHCFw.Name, true, framework.LoadBalancerCreateTimeoutDefault)
|
|
Expect(err).NotTo(HaveOccurred())
|
|
Expect(gce.VerifyFirewallRule(fw, localHCFw, cloudConfig.Network, false)).NotTo(HaveOccurred())
|
|
|
|
By(fmt.Sprintf("Creating netexec pods on at most %v nodes", framework.MaxNodesForEndpointsTests))
|
|
for i, nodeName := range nodesNames {
|
|
podName := fmt.Sprintf("netexec%v", i)
|
|
jig.LaunchNetexecPodOnNode(f, nodeName, podName, gce.FirewallTestHttpPort, gce.FirewallTestUdpPort, true)
|
|
defer func() {
|
|
framework.Logf("Cleaning up the netexec pod: %v", podName)
|
|
Expect(cs.CoreV1().Pods(ns).Delete(podName, nil)).NotTo(HaveOccurred())
|
|
}()
|
|
}
|
|
|
|
// Send requests from outside of the cluster because internal traffic is whitelisted
|
|
By("Accessing the external service ip from outside, all non-master nodes should be reached")
|
|
Expect(framework.TestHitNodesFromOutside(svcExternalIP, gce.FirewallTestHttpPort, framework.LoadBalancerCreateTimeoutDefault, nodesSet)).NotTo(HaveOccurred())
|
|
|
|
// Check if there are overlapping tags on the firewall that extend beyond just the vms in our cluster
|
|
// by removing the tag on one vm and make sure it doesn't get any traffic. This is an imperfect
|
|
// simulation, we really want to check that traffic doesn't reach a vm outside the GKE cluster, but
|
|
// that's much harder to do in the current e2e framework.
|
|
By(fmt.Sprintf("Removing tags from one of the nodes: %v", nodesNames[0]))
|
|
nodesSet.Delete(nodesNames[0])
|
|
// Instance could run in a different zone in multi-zone test. Figure out which zone
|
|
// it is in before proceeding.
|
|
zone := cloudConfig.Zone
|
|
if zoneInLabel, ok := nodeList.Items[0].Labels[v1.LabelZoneFailureDomain]; ok {
|
|
zone = zoneInLabel
|
|
}
|
|
removedTags := gce.SetInstanceTags(cloudConfig, nodesNames[0], zone, []string{})
|
|
defer func() {
|
|
By("Adding tags back to the node and wait till the traffic is recovered")
|
|
nodesSet.Insert(nodesNames[0])
|
|
gce.SetInstanceTags(cloudConfig, nodesNames[0], zone, removedTags)
|
|
// Make sure traffic is recovered before exit
|
|
Expect(framework.TestHitNodesFromOutside(svcExternalIP, gce.FirewallTestHttpPort, framework.LoadBalancerCreateTimeoutDefault, nodesSet)).NotTo(HaveOccurred())
|
|
}()
|
|
|
|
By("Accessing serivce through the external ip and examine got no response from the node without tags")
|
|
Expect(framework.TestHitNodesFromOutsideWithCount(svcExternalIP, gce.FirewallTestHttpPort, framework.LoadBalancerCreateTimeoutDefault, nodesSet, 15)).NotTo(HaveOccurred())
|
|
})
|
|
|
|
It("should have correct firewall rules for e2e cluster", func() {
|
|
nodes := framework.GetReadySchedulableNodesOrDie(cs)
|
|
if len(nodes.Items) <= 0 {
|
|
framework.Failf("Expect at least 1 node, got: %v", len(nodes.Items))
|
|
}
|
|
|
|
By("Checking if e2e firewall rules are correct")
|
|
for _, expFw := range gce.GetE2eFirewalls(cloudConfig.MasterName, cloudConfig.MasterTag, cloudConfig.NodeTag, cloudConfig.Network, cloudConfig.ClusterIPRange) {
|
|
fw, err := gceCloud.GetFirewall(expFw.Name)
|
|
Expect(err).NotTo(HaveOccurred())
|
|
Expect(gce.VerifyFirewallRule(fw, expFw, cloudConfig.Network, false)).NotTo(HaveOccurred())
|
|
}
|
|
|
|
By("Checking well known ports on master and nodes are not exposed externally")
|
|
nodeAddrs := framework.NodeAddresses(nodes, v1.NodeExternalIP)
|
|
if len(nodeAddrs) == 0 {
|
|
framework.Failf("did not find any node addresses")
|
|
}
|
|
|
|
masterAddresses := framework.GetAllMasterAddresses(cs)
|
|
for _, masterAddress := range masterAddresses {
|
|
assertNotReachableHTTPTimeout(masterAddress, ports.InsecureKubeControllerManagerPort, gce.FirewallTestTcpTimeout)
|
|
assertNotReachableHTTPTimeout(masterAddress, ports.InsecureSchedulerPort, gce.FirewallTestTcpTimeout)
|
|
}
|
|
assertNotReachableHTTPTimeout(nodeAddrs[0], ports.KubeletPort, gce.FirewallTestTcpTimeout)
|
|
assertNotReachableHTTPTimeout(nodeAddrs[0], ports.KubeletReadOnlyPort, gce.FirewallTestTcpTimeout)
|
|
assertNotReachableHTTPTimeout(nodeAddrs[0], ports.ProxyStatusPort, gce.FirewallTestTcpTimeout)
|
|
})
|
|
})
|
|
|
|
func assertNotReachableHTTPTimeout(ip string, port int, timeout time.Duration) {
|
|
result := framework.PokeHTTP(ip, port, "/", &framework.HTTPPokeParams{Timeout: timeout})
|
|
if result.Status == framework.HTTPError {
|
|
framework.Failf("Unexpected error checking for reachability of %s:%d: %v", ip, port, result.Error)
|
|
}
|
|
if result.Code != 0 {
|
|
framework.Failf("Was unexpectedly able to reach %s:%d", ip, port)
|
|
}
|
|
}
|