k3s/pkg/cloudprovider/providers/gce/gce_loadbalancer_utils_test.go

/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// This file contains shared functions and variables to set up for tests for
// ExternalLoadBalancer and InternalLoadBalancers. It currently cannot live in a
// separate package from GCE because then it would cause a circular import.

package gce

import (
	"context"
	"fmt"
	"strings"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	compute "google.golang.org/api/compute/v1"

	"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud"
	"k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/tools/record"
	servicehelpers "k8s.io/cloud-provider/service/helpers"
)

// TODO(yankaiz): Create shared error types for both test/non-test codes.
const (
	eventReasonManualChange = "LoadBalancerManualChange"
	eventMsgFirewallChange  = "Firewall change required by network admin"
	errPrefixGetTargetPool  = "error getting load balancer's target pool:"
	errStrLbNoHosts         = "Cannot EnsureLoadBalancer() with no hosts"
	wrongTier               = "SupremeLuxury"
	errStrUnsupportedTier   = "unsupported network tier: \"" + wrongTier + "\""
)

func fakeLoadbalancerService(lbType string) *v1.Service {
	return &v1.Service{
		ObjectMeta: metav1.ObjectMeta{
			Name:        "",
			Annotations: map[string]string{ServiceAnnotationLoadBalancerType: lbType},
		},
		Spec: v1.ServiceSpec{
			SessionAffinity: v1.ServiceAffinityClientIP,
			Type:            v1.ServiceTypeLoadBalancer,
			Ports:           []v1.ServicePort{{Protocol: v1.ProtocolTCP, Port: int32(123)}},
		},
	}
}

var (
	FilewallChangeMsg = fmt.Sprintf("%s %s %s", v1.EventTypeNormal, eventReasonManualChange, eventMsgFirewallChange)
)

func createAndInsertNodes(gce *Cloud, nodeNames []string, zoneName string) ([]*v1.Node, error) {
	nodes := []*v1.Node{}

	for _, name := range nodeNames {
		// Inserting the same node name twice causes an error - here we check if
		// the instance exists already before insertion.
		// TestUpdateExternalLoadBalancer inserts a new node, and relies on an older
		// node to already have been inserted.
		instance, _ := gce.getInstanceByName(name)

		if instance == nil {
			err := gce.InsertInstance(
				gce.ProjectID(),
				zoneName,
				&compute.Instance{
					Name: name,
					Tags: &compute.Tags{
						Items: []string{name},
					},
				},
			)
			if err != nil {
				return nodes, err
			}
		}

		nodes = append(
			nodes,
			&v1.Node{
				ObjectMeta: metav1.ObjectMeta{
					Name: name,
					Labels: map[string]string{
						v1.LabelHostname:          name,
						v1.LabelZoneFailureDomain: zoneName,
					},
				},
				Status: v1.NodeStatus{
					NodeInfo: v1.NodeSystemInfo{
						KubeProxyVersion: "v1.7.2",
					},
				},
			},
		)

	}

	return nodes, nil
}

func assertExternalLbResources(t *testing.T, gce *Cloud, apiService *v1.Service, vals TestClusterValues, nodeNames []string) {
	lbName := gce.GetLoadBalancerName(context.TODO(), "", apiService)
	hcName := MakeNodesHealthCheckName(vals.ClusterID)

	// Check that Firewalls are created for the LoadBalancer and the HealthCheck
	fwNames := []string{
		MakeFirewallName(lbName), // Firewalls for external LBs are prefixed with k8s-fw-
		MakeHealthCheckFirewallName(vals.ClusterID, hcName, true),
	}

	for _, fwName := range fwNames {
		firewall, err := gce.GetFirewall(fwName)
		require.NoError(t, err)
		assert.Equal(t, nodeNames, firewall.TargetTags)
		assert.NotEmpty(t, firewall.SourceRanges)
	}

	// Check that TargetPool is Created
	pool, err := gce.GetTargetPool(lbName, gce.region)
	require.NoError(t, err)
	assert.Equal(t, lbName, pool.Name)
	assert.NotEmpty(t, pool.HealthChecks)
	assert.Equal(t, 1, len(pool.Instances))

	// Check that HealthCheck is created
	healthcheck, err := gce.GetHTTPHealthCheck(hcName)
	require.NoError(t, err)
	assert.Equal(t, hcName, healthcheck.Name)

	// Check that ForwardingRule is created
	fwdRule, err := gce.GetRegionForwardingRule(lbName, gce.region)
	require.NoError(t, err)
	assert.Equal(t, lbName, fwdRule.Name)
	assert.Equal(t, "TCP", fwdRule.IPProtocol)
	assert.Equal(t, "123-123", fwdRule.PortRange)
}

func assertExternalLbResourcesDeleted(t *testing.T, gce *Cloud, apiService *v1.Service, vals TestClusterValues, firewallsDeleted bool) {
	lbName := gce.GetLoadBalancerName(context.TODO(), "", apiService)
	hcName := MakeNodesHealthCheckName(vals.ClusterID)

	if firewallsDeleted {
		// Check that Firewalls are deleted for the LoadBalancer and the HealthCheck
		fwNames := []string{
			MakeFirewallName(lbName),
			MakeHealthCheckFirewallName(vals.ClusterID, hcName, true),
		}

		for _, fwName := range fwNames {
			firewall, err := gce.GetFirewall(fwName)
			require.Error(t, err)
			assert.Nil(t, firewall)
		}

		// Check forwarding rule is deleted
		fwdRule, err := gce.GetRegionForwardingRule(lbName, gce.region)
		require.Error(t, err)
		assert.Nil(t, fwdRule)
	}

	// Check that TargetPool is deleted
	pool, err := gce.GetTargetPool(lbName, gce.region)
	require.Error(t, err)
	assert.Nil(t, pool)

	// Check that HealthCheck is deleted
	healthcheck, err := gce.GetHTTPHealthCheck(hcName)
	require.Error(t, err)
	assert.Nil(t, healthcheck)

}

func assertInternalLbResources(t *testing.T, gce *Cloud, apiService *v1.Service, vals TestClusterValues, nodeNames []string) {
	lbName := gce.GetLoadBalancerName(context.TODO(), "", apiService)

	// Check that Instance Group is created
	igName := makeInstanceGroupName(vals.ClusterID)
	ig, err := gce.GetInstanceGroup(igName, vals.ZoneName)
	assert.NoError(t, err)
	assert.Equal(t, igName, ig.Name)

	// Check that Firewalls are created for the LoadBalancer and the HealthCheck
	fwNames := []string{
		lbName, // Firewalls for internal LBs are named the same name as the loadbalancer.
		makeHealthCheckFirewallName(lbName, vals.ClusterID, true),
	}

	for _, fwName := range fwNames {
		firewall, err := gce.GetFirewall(fwName)
		require.NoError(t, err)
		assert.Equal(t, nodeNames, firewall.TargetTags)
		assert.NotEmpty(t, firewall.SourceRanges)
	}

	// Check that HealthCheck is created
	sharedHealthCheck := !servicehelpers.RequestsOnlyLocalTraffic(apiService)
	hcName := makeHealthCheckName(lbName, vals.ClusterID, sharedHealthCheck)
	healthcheck, err := gce.GetHealthCheck(hcName)
	require.NoError(t, err)
	assert.Equal(t, hcName, healthcheck.Name)

	// Check that BackendService exists
	sharedBackend := shareBackendService(apiService)
	backendServiceName := makeBackendServiceName(lbName, vals.ClusterID, sharedBackend, cloud.SchemeInternal, "TCP", apiService.Spec.SessionAffinity)
	backendServiceLink := gce.getBackendServiceLink(backendServiceName)

	bs, err := gce.GetRegionBackendService(backendServiceName, gce.region)
	require.NoError(t, err)
	assert.Equal(t, "TCP", bs.Protocol)
	assert.Equal(
		t,
		[]string{healthcheck.SelfLink},
		bs.HealthChecks,
	)

	// Check that ForwardingRule is created
	fwdRule, err := gce.GetRegionForwardingRule(lbName, gce.region)
	require.NoError(t, err)
	assert.Equal(t, lbName, fwdRule.Name)
	assert.Equal(t, "TCP", fwdRule.IPProtocol)
	assert.Equal(t, backendServiceLink, fwdRule.BackendService)
	// if no Subnetwork specified, defaults to the GCE NetworkURL
	assert.Equal(t, gce.NetworkURL(), fwdRule.Subnetwork)
}

func assertInternalLbResourcesDeleted(t *testing.T, gce *Cloud, apiService *v1.Service, vals TestClusterValues, firewallsDeleted bool) {
	lbName := gce.GetLoadBalancerName(context.TODO(), "", apiService)
	sharedHealthCheck := !servicehelpers.RequestsOnlyLocalTraffic(apiService)
	hcName := makeHealthCheckName(lbName, vals.ClusterID, sharedHealthCheck)

	// ensureExternalLoadBalancer and ensureInternalLoadBalancer both create
	// Firewalls with the same name.
	if firewallsDeleted {
		// Check that Firewalls are deleted for the LoadBalancer and the HealthCheck
		fwNames := []string{
			MakeFirewallName(lbName),
			MakeHealthCheckFirewallName(vals.ClusterID, hcName, true),
		}

		for _, fwName := range fwNames {
			firewall, err := gce.GetFirewall(fwName)
			require.Error(t, err)
			assert.Nil(t, firewall)
		}

		// Check forwarding rule is deleted
		fwdRule, err := gce.GetRegionForwardingRule(lbName, gce.region)
		require.Error(t, err)
		assert.Nil(t, fwdRule)
	}

	// Check that Instance Group is deleted
	igName := makeInstanceGroupName(vals.ClusterID)
	ig, err := gce.GetInstanceGroup(igName, vals.ZoneName)
	assert.Error(t, err)
	assert.Nil(t, ig)

	// Check that HealthCheck is deleted
	healthcheck, err := gce.GetHealthCheck(hcName)
	require.Error(t, err)
	assert.Nil(t, healthcheck)
}

func checkEvent(t *testing.T, recorder *record.FakeRecorder, expected string, shouldMatch bool) bool {
	select {
	case received := <-recorder.Events:
		if strings.HasPrefix(received, expected) != shouldMatch {
			t.Errorf(received)
			if shouldMatch {
				t.Errorf("Should receive message \"%v\" but got \"%v\".", expected, received)
			} else {
				t.Errorf("Unexpected event \"%v\".", received)
			}
		}
		return false
	case <-time.After(2 * time.Second):
		if shouldMatch {
			t.Errorf("Should receive message \"%v\" but got timed out.", expected)
		}
		return true
	}
}