Merge pull request #54507 from micahhausler/aws-elb-security-policy

Automatic merge from submit-queue (batch tested with PRs 54134, 54507). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Added service annotation for AWS ELB SSL policy

**What this PR does / why we need it**:

This work adds a new supported service annotation for AWS clusters, `service.beta.kubernetes.io/aws-load-balancer-ssl-negotiation-policy`, which lets users specify which [predefined AWS SSL policy](http://docs.aws.amazon.com/elasticloadbalancing/latest/classic/elb-security-policy-table.html) they would like to use.

**Which issue this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close that issue when PR gets merged)*: fixes #

Fixes #43744

**Special notes for your reviewer**:

While this PR doesn't allow users to define their own cipher policy in an annotation, a user could (out of band) create their own policy on an ELB with the naming convention `k8s-SSLNegotiationPolicy-<my-policy-name>` and specify it with the above annotation.

This is my second k8s PR, and I don't have experience with an e2e test, would that be required for this change? I did run this in a kubeadm cluster and it worked like a charm. I was able to choose different predefined policies, and revert to the default policy when I removed the annotation.

**Release note**:

```release-note
Added service annotation for AWS ELB SSL policy
```

pkg/cloudprovider/providers/aws/aws.go

@@ -129,6 +129,11 @@ const ServiceAnnotationLoadBalancerCertificate = "service.beta.kubernetes.io/aws
 // listeners. Defaults to '*' (all).
 const ServiceAnnotationLoadBalancerSSLPorts = "service.beta.kubernetes.io/aws-load-balancer-ssl-ports"
 
+// ServiceAnnotationLoadBalancerSSLNegotiationPolicy is the annotation used on
+// the service to specify a SSL negotiation settings for the HTTPS/SSL listeners
+// of your load balancer. Defaults to AWS's default
+const ServiceAnnotationLoadBalancerSSLNegotiationPolicy = "service.beta.kubernetes.io/aws-load-balancer-ssl-negotiation-policy"
+
 // ServiceAnnotationLoadBalancerBEProtocol is the annotation used on the service
 // to specify the protocol spoken by the backend (pod) behind a listener.
 // If `http` (default) or `https`, an HTTPS listener that terminates the
@@ -259,6 +264,8 @@ type ELB interface {
 	DeregisterInstancesFromLoadBalancer(*elb.DeregisterInstancesFromLoadBalancerInput) (*elb.DeregisterInstancesFromLoadBalancerOutput, error)
 	CreateLoadBalancerPolicy(*elb.CreateLoadBalancerPolicyInput) (*elb.CreateLoadBalancerPolicyOutput, error)
 	SetLoadBalancerPoliciesForBackendServer(*elb.SetLoadBalancerPoliciesForBackendServerInput) (*elb.SetLoadBalancerPoliciesForBackendServerOutput, error)
+	SetLoadBalancerPoliciesOfListener(input *elb.SetLoadBalancerPoliciesOfListenerInput) (*elb.SetLoadBalancerPoliciesOfListenerOutput, error)
+	DescribeLoadBalancerPolicies(input *elb.DescribeLoadBalancerPoliciesInput) (*elb.DescribeLoadBalancerPoliciesOutput, error)
 
 	DetachLoadBalancerFromSubnets(*elb.DetachLoadBalancerFromSubnetsInput) (*elb.DetachLoadBalancerFromSubnetsOutput, error)
 	AttachLoadBalancerToSubnets(*elb.AttachLoadBalancerToSubnetsInput) (*elb.AttachLoadBalancerToSubnetsOutput, error)
@@ -3101,6 +3108,20 @@ func (c *Cloud) EnsureLoadBalancer(clusterName string, apiService *v1.Service, n
 		return nil, err
 	}
 
+	if sslPolicyName, ok := annotations[ServiceAnnotationLoadBalancerSSLNegotiationPolicy]; ok {
+		err := c.ensureSSLNegotiationPolicy(loadBalancer, sslPolicyName)
+		if err != nil {
+			return nil, err
+		}
+
+		for _, port := range c.getLoadBalancerTLSPorts(loadBalancer) {
+			err := c.setSSLNegotiationPolicy(loadBalancerName, sslPolicyName, port)
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+
 	if path, healthCheckNodePort := service.GetServiceHealthCheckPathPort(apiService); path != "" {
 		glog.V(4).Infof("service %v (%v) needs health checks on :%d%s)", apiService.Name, loadBalancerName, healthCheckNodePort, path)
 		err = c.ensureLoadBalancerHealthCheck(loadBalancer, "HTTP", healthCheckNodePort, path)
@@ -3492,6 +3513,19 @@ func (c *Cloud) UpdateLoadBalancer(clusterName string, service *v1.Service, node
 		return fmt.Errorf("Load balancer not found")
 	}
 
+	if sslPolicyName, ok := service.Annotations[ServiceAnnotationLoadBalancerSSLNegotiationPolicy]; ok {
+		err := c.ensureSSLNegotiationPolicy(lb, sslPolicyName)
+		if err != nil {
+			return err
+		}
+		for _, port := range c.getLoadBalancerTLSPorts(lb) {
+			err := c.setSSLNegotiationPolicy(loadBalancerName, sslPolicyName, port)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
 	err = c.ensureLoadBalancerInstances(aws.StringValue(lb.LoadBalancerName), lb.Instances, instances)
 	if err != nil {
 		return nil

pkg/cloudprovider/providers/aws/aws_fakes.go

@@ -352,6 +352,14 @@ func (elb *FakeELB) SetLoadBalancerPoliciesForBackendServer(*elb.SetLoadBalancer
 	panic("Not implemented")
 }
 
+func (elb *FakeELB) SetLoadBalancerPoliciesOfListener(input *elb.SetLoadBalancerPoliciesOfListenerInput) (*elb.SetLoadBalancerPoliciesOfListenerOutput, error) {
+	panic("Not implemented")
+}
+
+func (elb *FakeELB) DescribeLoadBalancerPolicies(input *elb.DescribeLoadBalancerPoliciesInput) (*elb.DescribeLoadBalancerPoliciesOutput, error) {
+	panic("Not implemented")
+}
+
 func (elb *FakeELB) DescribeLoadBalancerAttributes(*elb.DescribeLoadBalancerAttributesInput) (*elb.DescribeLoadBalancerAttributesOutput, error) {
 	panic("Not implemented")
 }

pkg/cloudprovider/providers/aws/aws_loadbalancer.go

@@ -23,6 +23,7 @@ import (
 	"strings"
 
 	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/aws/aws-sdk-go/service/elb"
 	"github.com/golang/glog"
@@ -33,6 +34,8 @@ import (
 
 const ProxyProtocolPolicyName = "k8s-proxyprotocol-enabled"
 
+const SSLNegotiationPolicyNameFormat = "k8s-SSLNegotiationPolicy-%s"
+
 // getLoadBalancerAdditionalTags converts the comma separated list of key-value
 // pairs in the ServiceAnnotationLoadBalancerAdditionalTags annotation and returns
 // it as a map.
@@ -471,6 +474,78 @@ func (c *Cloud) ensureLoadBalancerInstances(loadBalancerName string, lbInstances
 	return nil
 }
 
+func (c *Cloud) getLoadBalancerTLSPorts(loadBalancer *elb.LoadBalancerDescription) []int64 {
+	ports := []int64{}
+
+	for _, listenerDescription := range loadBalancer.ListenerDescriptions {
+		protocol := aws.StringValue(listenerDescription.Listener.Protocol)
+		if protocol == "SSL" || protocol == "HTTPS" {
+			ports = append(ports, aws.Int64Value(listenerDescription.Listener.LoadBalancerPort))
+		}
+	}
+	return ports
+}
+
+func (c *Cloud) ensureSSLNegotiationPolicy(loadBalancer *elb.LoadBalancerDescription, policyName string) error {
+	glog.V(2).Info("Describing load balancer policies on load balancer")
+	result, err := c.elb.DescribeLoadBalancerPolicies(&elb.DescribeLoadBalancerPoliciesInput{
+		LoadBalancerName: loadBalancer.LoadBalancerName,
+		PolicyNames: []*string{
+			aws.String(fmt.Sprintf(SSLNegotiationPolicyNameFormat, policyName)),
+		},
+	})
+	if err != nil {
+		if aerr, ok := err.(awserr.Error); ok {
+			switch aerr.Code() {
+			case "PolicyNotFound":
+				// TODO change from string to `elb.ErrCodePolicyNotFoundException` once the AWS SDK is updated
+			default:
+				return fmt.Errorf("error describing security policies on load balancer: %q", err)
+			}
+		}
+	}
+
+	if len(result.PolicyDescriptions) > 0 {
+		return nil
+	}
+
+	glog.V(2).Infof("Creating SSL negotiation policy '%s' on load balancer", fmt.Sprintf(SSLNegotiationPolicyNameFormat, policyName))
+	// there is an upper limit of 98 policies on an ELB, we're pretty safe from
+	// running into it
+	_, err = c.elb.CreateLoadBalancerPolicy(&elb.CreateLoadBalancerPolicyInput{
+		LoadBalancerName: loadBalancer.LoadBalancerName,
+		PolicyName:       aws.String(fmt.Sprintf(SSLNegotiationPolicyNameFormat, policyName)),
+		PolicyTypeName:   aws.String("SSLNegotiationPolicyType"),
+		PolicyAttributes: []*elb.PolicyAttribute{
+			{
+				AttributeName:  aws.String("Reference-Security-Policy"),
+				AttributeValue: aws.String(policyName),
+			},
+		},
+	})
+	if err != nil {
+		return fmt.Errorf("error creating security policy on load balancer: %q", err)
+	}
+	return nil
+}
+
+func (c *Cloud) setSSLNegotiationPolicy(loadBalancerName, sslPolicyName string, port int64) error {
+	policyName := fmt.Sprintf(SSLNegotiationPolicyNameFormat, sslPolicyName)
+	request := &elb.SetLoadBalancerPoliciesOfListenerInput{
+		LoadBalancerName: aws.String(loadBalancerName),
+		LoadBalancerPort: aws.Int64(port),
+		PolicyNames: []*string{
+			aws.String(policyName),
+		},
+	}
+	glog.V(2).Infof("Setting SSL negotiation policy '%s' on load balancer", policyName)
+	_, err := c.elb.SetLoadBalancerPoliciesOfListener(request)
+	if err != nil {
+		return fmt.Errorf("error setting SSL negotiation policy '%s' on load balancer: %q", policyName, err)
+	}
+	return nil
+}
+
 func (c *Cloud) createProxyProtocolPolicy(loadBalancerName string) error {
 	request := &elb.CreateLoadBalancerPolicyInput{
 		LoadBalancerName: aws.String(loadBalancerName),