github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Add external services v2 support.

pull/6/head
Brendan Burns 2014-11-11 20:08:33 -08:00
parent aa711af39e
commit 2aa52d043b
14 changed files with 104 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
docs/services.md
View File

@ -127,16 +127,10 @@ being aware of which `pods` they are accessing.
![Services detailed diagram](services_detail.png)
## External Services
For some parts of your application (e.g. your frontend) you want to expose a service on an external (publically visible) IP address. To achieve this, you can set the ```createExternalLoadBalancer``` flag on the service. This sets up a cloud provider specific load balancer (assuming that it is supported by your cloud provider) and also sets up IPTables rules on each host that map packets from the specified External IP address to the service proxy in the same manner as internal service IP addresses.
## Shortcomings
Part of the `service` specification is a `createExternalLoadBalancer` flag,
which tells the master to make an external load balancer that points to the
service. In order to do this today, the service proxy must answer on a known
(i.e. not random) port. In this case, the service port is promoted to the
proxy port. This means that it is still possible for users to collide with
each other's services or with other pods. We expect most `services` will not
set this flag, mitigating the exposure.
We expect that using iptables for portals will work at small scale, but will
not scale to large clusters with thousands of services. See [the original
design proposal for

2
pkg/api/types.go
View File

@ -576,6 +576,8 @@ type ServiceSpec struct {
// CreateExternalLoadBalancer indicates whether a load balancer should be created for this service.
CreateExternalLoadBalancer bool `json:"createExternalLoadBalancer,omitempty" yaml:"createExternalLoadBalancer,omitempty"`
// PublicIPs are used by external load balancers.
PublicIPs []string `json:"publicIPs,omitempty" yaml:"publicIPs,omitempty"`
// ContainerPort is the name of the port on the container to direct traffic to.
// Optional, if unspecified use the first port on the container.

2
pkg/api/v1beta1/conversion.go
View File

@ -370,6 +370,7 @@ func init() {
return err
}
out.CreateExternalLoadBalancer = in.Spec.CreateExternalLoadBalancer
out.PublicIPs = in.Spec.PublicIPs
out.ContainerPort = in.Spec.ContainerPort
out.PortalIP = in.Spec.PortalIP
out.ProxyPort = in.Spec.ProxyPort
@ -392,6 +393,7 @@ func init() {
return err
}
out.Spec.CreateExternalLoadBalancer = in.CreateExternalLoadBalancer
out.Spec.PublicIPs = in.PublicIPs
out.Spec.ContainerPort = in.ContainerPort
out.Spec.PortalIP = in.PortalIP
out.Spec.ProxyPort = in.ProxyPort

2
pkg/api/v1beta1/types.go
View File

@ -467,6 +467,8 @@ type Service struct {
// This service will route traffic to pods having labels matching this selector.
Selector map[string]string `json:"selector,omitempty" yaml:"selector,omitempty"`
CreateExternalLoadBalancer bool `json:"createExternalLoadBalancer,omitempty" yaml:"createExternalLoadBalancer,omitempty"`
// PublicIPs are used by external load balancers.
PublicIPs []string `json:"publicIPs,omitempty" yaml:"publicIPs,omitempty"`
// ContainerPort is the name of the port on the container to direct traffic to.
// Optional, if unspecified use the first port on the container.

2
pkg/api/v1beta2/conversion.go
View File

@ -299,6 +299,7 @@ func init() {
return err
}
out.CreateExternalLoadBalancer = in.Spec.CreateExternalLoadBalancer
out.PublicIPs = in.Spec.PublicIPs
out.ContainerPort = in.Spec.ContainerPort
out.PortalIP = in.Spec.PortalIP
out.ProxyPort = in.Spec.ProxyPort
@ -322,6 +323,7 @@ func init() {
return err
}
out.Spec.CreateExternalLoadBalancer = in.CreateExternalLoadBalancer
out.Spec.PublicIPs = in.PublicIPs
out.Spec.ContainerPort = in.ContainerPort
out.Spec.PortalIP = in.PortalIP
out.Spec.ProxyPort = in.ProxyPort

2
pkg/api/v1beta2/types.go
View File

@ -432,6 +432,8 @@ type Service struct {
// This service will route traffic to pods having labels matching this selector.
Selector map[string]string `json:"selector,omitempty" yaml:"selector,omitempty"`
CreateExternalLoadBalancer bool `json:"createExternalLoadBalancer,omitempty" yaml:"createExternalLoadBalancer,omitempty"`
// PublicIPs are used by external load balancers.
PublicIPs []string `json:"publicIPs,omitempty" yaml:"publicIPs,omitempty"`
// ContainerPort is the name of the port on the container to direct traffic to.
// Optional, if unspecified use the first port on the container.

2
pkg/api/v1beta3/types.go
View File

@ -614,6 +614,8 @@ type ServiceSpec struct {
// CreateExternalLoadBalancer indicates whether a load balancer should be created for this service.
CreateExternalLoadBalancer bool `json:"createExternalLoadBalancer,omitempty" yaml:"createExternalLoadBalancer,omitempty"`
// PublicIPs are used by external load balancers.
PublicIPs []string `json:"publicIPs,omitempty" yaml:"publicIPs,omitempty"`
// ContainerPort is the name of the port on the container to direct traffic to.
// Optional, if unspecified use the first port on the container.

4
pkg/cloudprovider/cloud.go
View File

@ -47,8 +47,8 @@ type TCPLoadBalancer interface {
// TCPLoadBalancerExists returns whether the specified load balancer exists.
// TODO: Break this up into different interfaces (LB, etc) when we have more than one type of service
TCPLoadBalancerExists(name, region string) (bool, error)
// CreateTCPLoadBalancer creates a new tcp load balancer.
CreateTCPLoadBalancer(name, region string, port int, hosts []string) error
// CreateTCPLoadBalancer creates a new tcp load balancer. Returns the IP address of the balancer
CreateTCPLoadBalancer(name, region string, externalIP net.IP, port int, hosts []string) (net.IP, error)
// UpdateTCPLoadBalancer updates hosts under the specified load balancer.
UpdateTCPLoadBalancer(name, region string, hosts []string) error
// DeleteTCPLoadBalancer deletes a specified load balancer.

5
pkg/cloudprovider/fake/fake.go
View File

@ -34,6 +34,7 @@ type FakeCloud struct {
NodeResources *api.NodeResources
ClusterList []string
MasterName string
ExternalIP net.IP
cloudprovider.Zone
}
@ -83,9 +84,9 @@ func (f *FakeCloud) TCPLoadBalancerExists(name, region string) (bool, error) {
// CreateTCPLoadBalancer is a test-spy implementation of TCPLoadBalancer.CreateTCPLoadBalancer.
// It adds an entry "create" into the internal method call record.
func (f *FakeCloud) CreateTCPLoadBalancer(name, region string, port int, hosts []string) error {
func (f *FakeCloud) CreateTCPLoadBalancer(name, region string, externalIP net.IP, port int, hosts []string) (net.IP, error) {
f.addCall("create")
return f.Err
return f.ExternalIP, f.Err
}
// UpdateTCPLoadBalancer is a test-spy implementation of TCPLoadBalancer.UpdateTCPLoadBalancer.

22
pkg/cloudprovider/gce/gce.go
View File

@ -192,10 +192,10 @@ func (gce *GCECloud) TCPLoadBalancerExists(name, region string) (bool, error) {
}
// CreateTCPLoadBalancer is an implementation of TCPLoadBalancer.CreateTCPLoadBalancer.
func (gce *GCECloud) CreateTCPLoadBalancer(name, region string, port int, hosts []string) error {
func (gce *GCECloud) CreateTCPLoadBalancer(name, region string, externalIP net.IP, port int, hosts []string) (net.IP, error) {
pool, err := gce.makeTargetPool(name, region, hosts)
if err != nil {
return err
return nil, err
}
req := &compute.ForwardingRule{
Name: name,
@ -203,8 +203,22 @@ func (gce *GCECloud) CreateTCPLoadBalancer(name, region string, port int, hosts
PortRange: strconv.Itoa(port),
Target: pool,
}
_, err = gce.service.ForwardingRules.Insert(gce.projectID, region, req).Do()
return err
if len(externalIP) > 0 {
req.IPAddress = externalIP.String()
}
op, err := gce.service.ForwardingRules.Insert(gce.projectID, region, req).Do()
if err != nil {
return nil, err
}
err = gce.waitForRegionOp(op, region)
if err != nil {
return nil, err
}
fwd, err := gce.service.ForwardingRules.Get(gce.projectID, region, name).Do()
if err != nil {
return nil, err
}
return net.ParseIP(fwd.IPAddress), nil
}
// UpdateTCPLoadBalancer is an implementation of TCPLoadBalancer.UpdateTCPLoadBalancer.

51
pkg/proxy/proxier.go
View File

@ -40,6 +40,7 @@ type serviceInfo struct {
timeout time.Duration
mu sync.Mutex // protects active
active bool
publicIP []string
}
func (si *serviceInfo) isActive() bool {
@ -443,7 +444,7 @@ func (proxier *Proxier) OnUpdate(services []api.Service) {
if exists && info.isActive() && info.portalPort == service.Spec.Port && info.portalIP.Equal(serviceIP) {
continue
}
if exists && (info.portalPort != service.Spec.Port || !info.portalIP.Equal(serviceIP)) {
if exists && (info.portalPort != service.Spec.Port || !info.portalIP.Equal(serviceIP) || service.Spec.CreateExternalLoadBalancer != (len(info.publicIP) > 0)) {
glog.V(4).Infof("Something changed for service %q: stopping it", service.Name)
err := proxier.closePortal(service.Name, info)
if err != nil {
@ -462,6 +463,9 @@ func (proxier *Proxier) OnUpdate(services []api.Service) {
}
info.portalIP = serviceIP
info.portalPort = service.Spec.Port
if service.Spec.CreateExternalLoadBalancer {
info.publicIP = service.Spec.PublicIPs
}
err = proxier.openPortal(service.Name, info)
if err != nil {
glog.Errorf("Failed to open portal for %q: %s", service.Name, err)
@ -494,6 +498,25 @@ func (proxier *Proxier) openPortal(service string, info *serviceInfo) error {
if !existed {
glog.Infof("Opened iptables portal for service %q on %s:%d", service, info.portalIP, info.portalPort)
}
if len(info.publicIP) > 0 {
return proxier.openExternalPortal(service, info)
}
return nil
}
func (proxier *Proxier) openExternalPortal(service string, info *serviceInfo) error {
for _, publicIP := range info.publicIP {
proxier.iptables.EnsureRule(iptables.TableNAT, iptables.ChainPostrouting, iptablesRoutingArgs(publicIP)...)
args := iptablesPortalArgs(net.ParseIP(publicIP), info.portalPort, info.protocol, proxier.listenAddress, info.proxyPort, service)
existed, err := proxier.iptables.EnsureRule(iptables.TableNAT, iptablesProxyChain, args...)
if err != nil {
glog.Errorf("Failed to install iptables %s rule for service %q", iptablesProxyChain, service)
return err
}
if !existed {
glog.Infof("Opened iptables external portal for service %q on %s:%d", service, publicIP, info.proxyPort)
}
}
return nil
}
@ -503,10 +526,26 @@ func (proxier *Proxier) closePortal(service string, info *serviceInfo) error {
glog.Errorf("Failed to delete iptables %s rule for service %q", iptablesProxyChain, service)
return err
}
if len(info.publicIP) > 0 {
return proxier.closeExternalPortal(service, info)
}
glog.Infof("Closed iptables portal for service %q", service)
return nil
}
func (proxier *Proxier) closeExternalPortal(service string, info *serviceInfo) error {
for _, publicIP := range info.publicIP {
proxier.iptables.DeleteRule(iptables.TableNAT, iptables.ChainPostrouting, iptablesRoutingArgs(publicIP)...)
args := iptablesPortalArgs(net.ParseIP(publicIP), info.portalPort, info.protocol, proxier.listenAddress, info.proxyPort, service)
if err := proxier.iptables.DeleteRule(iptables.TableNAT, iptablesProxyChain, args...); err != nil {
glog.Errorf("Failed to delete external iptables %s rule for service %q", iptablesProxyChain, service)
return err
}
}
glog.Infof("Closed external iptables portal for service %q", service)
return nil
}
var iptablesProxyChain iptables.Chain = "KUBE-PROXY"
// Ensure that the iptables infrastructure we use is set up. This can safely be called periodically.
@ -538,6 +577,16 @@ var localhostIPv4 = net.ParseIP("127.0.0.1")
var zeroIPv6 = net.ParseIP("::0")
var localhostIPv6 = net.ParseIP("::1")
// Build an iptables args to route in a specific external ip
func iptablesRoutingArgs(destIP string) []string {
return []string{
"!",
"-d", destIP + "/32",
"-o", "eth0",
"-j", "MASQUERADE",
}
}
// Build a slice of iptables args for a portal rule.
func iptablesPortalArgs(destIP net.IP, destPort int, protocol api.Protocol, proxyIP net.IP, proxyPort int, service string) []string {
args := []string{

16
pkg/registry/service/rest.go
View File

@ -133,13 +133,21 @@ func (rs *REST) Create(ctx api.Context, obj runtime.Object) (<-chan apiserver.RE
if err != nil {
return nil, err
}
err = balancer.CreateTCPLoadBalancer(service.Name, zone.Region, service.Spec.Port, hostsFromMinionList(hosts))
var ip net.IP
if len(service.Spec.PublicIPs) > 0 {
for _, publicIP := range service.Spec.PublicIPs {
ip, err = balancer.CreateTCPLoadBalancer(service.Name, zone.Region, net.ParseIP(publicIP), service.Spec.Port, hostsFromMinionList(hosts))
if err != nil {
break
}
}
} else {
ip, err = balancer.CreateTCPLoadBalancer(service.Name, zone.Region, nil, service.Spec.Port, hostsFromMinionList(hosts))
}
if err != nil {
return nil, err
}
// External load-balancers require a known port for the service proxy.
// TODO: If we end up brokering HostPorts between Pods and Services, this can be any port.
service.Spec.ProxyPort = service.Spec.Port
service.Spec.PublicIPs = []string{ip.String()}
}
err := rs.registry.CreateService(ctx, service)
if err != nil {

2
pkg/registry/service/rest_test.go
View File

@ -638,7 +638,7 @@ func TestServiceRegistryIPExternalLoadBalancer(t *testing.T) {
if created_service.Spec.PortalIP != "1.2.3.1" {
t.Errorf("Unexpected PortalIP: %s", created_service.Spec.PortalIP)
}
if created_service.Spec.ProxyPort != 6502 {
if created_service.Spec.ProxyPort != 0 {
t.Errorf("Unexpected ProxyPort: %d", created_service.Spec.ProxyPort)
}
}

5
pkg/util/iptables/iptables.go
View File

@ -54,8 +54,9 @@ const (
type Chain string
const (
ChainPrerouting Chain = "PREROUTING"
ChainOutput Chain = "OUTPUT"
ChainPostrouting Chain = "POSTROUTING"
ChainPrerouting Chain = "PREROUTING"
ChainOutput Chain = "OUTPUT"
)
// runner implements Interface in terms of exec("iptables").