mirror of https://github.com/k3s-io/k3s
add tcp or udp proxy for service addresses
parent
fcb234e580
commit
48647fb9b5
|
@ -249,15 +249,15 @@ func NewProxyServerDefault(config *options.ProxyServerConfig) (*ProxyServer, err
|
||||||
userspace.CleanupLeftovers(iptInterface)
|
userspace.CleanupLeftovers(iptInterface)
|
||||||
} else {
|
} else {
|
||||||
glog.V(0).Info("Using userspace Proxier.")
|
glog.V(0).Info("Using userspace Proxier.")
|
||||||
// This is a proxy.LoadBalancer which NewProxier needs but has methods we don't need for
|
|
||||||
// our config.EndpointsConfigHandler.
|
|
||||||
loadBalancer := userspace.NewLoadBalancerRR()
|
|
||||||
// set EndpointsConfigHandler to our loadBalancer
|
|
||||||
endpointsHandler = loadBalancer
|
|
||||||
|
|
||||||
var proxierUserspace proxy.ProxyProvider
|
var proxierUserspace proxy.ProxyProvider
|
||||||
|
|
||||||
if runtime.GOOS == "windows" {
|
if runtime.GOOS == "windows" {
|
||||||
|
// This is a proxy.LoadBalancer which NewProxier needs but has methods we don't need for
|
||||||
|
// our config.EndpointsConfigHandler.
|
||||||
|
loadBalancer := winuserspace.NewLoadBalancerRR()
|
||||||
|
// set EndpointsConfigHandler to our loadBalancer
|
||||||
|
endpointsHandler = loadBalancer
|
||||||
proxierUserspace, err = winuserspace.NewProxier(
|
proxierUserspace, err = winuserspace.NewProxier(
|
||||||
loadBalancer,
|
loadBalancer,
|
||||||
net.ParseIP(config.BindAddress),
|
net.ParseIP(config.BindAddress),
|
||||||
|
@ -268,6 +268,11 @@ func NewProxyServerDefault(config *options.ProxyServerConfig) (*ProxyServer, err
|
||||||
config.UDPIdleTimeout.Duration,
|
config.UDPIdleTimeout.Duration,
|
||||||
)
|
)
|
||||||
} else {
|
} else {
|
||||||
|
// This is a proxy.LoadBalancer which NewProxier needs but has methods we don't need for
|
||||||
|
// our config.EndpointsConfigHandler.
|
||||||
|
loadBalancer := userspace.NewLoadBalancerRR()
|
||||||
|
// set EndpointsConfigHandler to our loadBalancer
|
||||||
|
endpointsHandler = loadBalancer
|
||||||
proxierUserspace, err = userspace.NewProxier(
|
proxierUserspace, err = userspace.NewProxier(
|
||||||
loadBalancer,
|
loadBalancer,
|
||||||
net.ParseIP(config.BindAddress),
|
net.ParseIP(config.BindAddress),
|
||||||
|
|
|
@ -47,3 +47,15 @@ type ServicePortName struct {
|
||||||
func (spn ServicePortName) String() string {
|
func (spn ServicePortName) String() string {
|
||||||
return fmt.Sprintf("%s:%s", spn.NamespacedName.String(), spn.Port)
|
return fmt.Sprintf("%s:%s", spn.NamespacedName.String(), spn.Port)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ServicePortPortalName carries a namespace + name + portname + portalip. This is the unique
|
||||||
|
// identfier for a load-balanced service.
|
||||||
|
type ServicePortPortalName struct {
|
||||||
|
types.NamespacedName
|
||||||
|
Port string
|
||||||
|
PortalIPName string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (spn ServicePortPortalName) String() string {
|
||||||
|
return fmt.Sprintf("%s:%s:%s", spn.NamespacedName.String(), spn.Port, spn.PortalIPName)
|
||||||
|
}
|
||||||
|
|
|
@ -19,7 +19,6 @@ package winuserspace
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"net"
|
"net"
|
||||||
"strconv"
|
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
|
@ -28,7 +27,7 @@ import (
|
||||||
"github.com/golang/glog"
|
"github.com/golang/glog"
|
||||||
|
|
||||||
"k8s.io/apimachinery/pkg/types"
|
"k8s.io/apimachinery/pkg/types"
|
||||||
utilerrors "k8s.io/apimachinery/pkg/util/errors"
|
//utilerrors "k8s.io/apimachinery/pkg/util/errors"
|
||||||
utilnet "k8s.io/apimachinery/pkg/util/net"
|
utilnet "k8s.io/apimachinery/pkg/util/net"
|
||||||
"k8s.io/apimachinery/pkg/util/runtime"
|
"k8s.io/apimachinery/pkg/util/runtime"
|
||||||
"k8s.io/kubernetes/pkg/api"
|
"k8s.io/kubernetes/pkg/api"
|
||||||
|
@ -37,25 +36,25 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
type portal struct {
|
type portal struct {
|
||||||
ip net.IP
|
ip string
|
||||||
port int
|
port int
|
||||||
isExternal bool
|
isExternal bool
|
||||||
}
|
}
|
||||||
|
|
||||||
type serviceInfo struct {
|
type serviceInfo struct {
|
||||||
isAliveAtomic int32 // Only access this with atomic ops
|
isAliveAtomic int32 // Only access this with atomic ops
|
||||||
portal portal
|
portal portal
|
||||||
protocol api.Protocol
|
protocol api.Protocol
|
||||||
proxyPort int
|
proxyPort int
|
||||||
socket proxySocket
|
socket proxySocket
|
||||||
timeout time.Duration
|
timeout time.Duration
|
||||||
activeClients *clientCache
|
activeClients *clientCache
|
||||||
nodePort int
|
//nodePort int
|
||||||
loadBalancerStatus api.LoadBalancerStatus
|
//loadBalancerStatus api.LoadBalancerStatus
|
||||||
sessionAffinityType api.ServiceAffinity
|
sessionAffinityType api.ServiceAffinity
|
||||||
stickyMaxAgeMinutes int
|
stickyMaxAgeMinutes int
|
||||||
// Deprecated, but required for back-compat (including e2e)
|
// Deprecated, but required for back-compat (including e2e)
|
||||||
externalIPs []string
|
//externalIPs []string
|
||||||
}
|
}
|
||||||
|
|
||||||
func (info *serviceInfo) setAlive(b bool) {
|
func (info *serviceInfo) setAlive(b bool) {
|
||||||
|
@ -85,16 +84,16 @@ func logTimeout(err error) bool {
|
||||||
type Proxier struct {
|
type Proxier struct {
|
||||||
loadBalancer LoadBalancer
|
loadBalancer LoadBalancer
|
||||||
mu sync.Mutex // protects serviceMap
|
mu sync.Mutex // protects serviceMap
|
||||||
serviceMap map[proxy.ServicePortName]*serviceInfo
|
serviceMap map[proxy.ServicePortPortalName]*serviceInfo
|
||||||
syncPeriod time.Duration
|
syncPeriod time.Duration
|
||||||
udpIdleTimeout time.Duration
|
udpIdleTimeout time.Duration
|
||||||
portMapMutex sync.Mutex
|
portMapMutex sync.Mutex
|
||||||
portMap map[portMapKey]*portMapValue
|
portMap map[portMapKey]*portMapValue
|
||||||
numProxyLoops int32 // use atomic ops to access this; mostly for testing
|
numProxyLoops int32 // use atomic ops to access this; mostly for testing
|
||||||
listenIP net.IP
|
//listenIP net.IP
|
||||||
netsh netsh.Interface
|
netsh netsh.Interface
|
||||||
hostIP net.IP
|
hostIP net.IP
|
||||||
proxyPorts PortAllocator
|
//proxyPorts PortAllocator
|
||||||
}
|
}
|
||||||
|
|
||||||
// assert Proxier is a ProxyProvider
|
// assert Proxier is a ProxyProvider
|
||||||
|
@ -114,7 +113,7 @@ func (k *portMapKey) String() string {
|
||||||
|
|
||||||
// A value for the portMap
|
// A value for the portMap
|
||||||
type portMapValue struct {
|
type portMapValue struct {
|
||||||
owner proxy.ServicePortName
|
owner proxy.ServicePortPortalName
|
||||||
socket interface {
|
socket interface {
|
||||||
Close() error
|
Close() error
|
||||||
}
|
}
|
||||||
|
@ -153,31 +152,23 @@ func NewProxier(loadBalancer LoadBalancer, listenIP net.IP, netsh netsh.Interfac
|
||||||
return nil, fmt.Errorf("failed to select a host interface: %v", err)
|
return nil, fmt.Errorf("failed to select a host interface: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
proxyPorts := newPortAllocator(pr)
|
|
||||||
|
|
||||||
glog.V(2).Infof("Setting proxy IP to %v and initializing iptables", hostIP)
|
glog.V(2).Infof("Setting proxy IP to %v and initializing iptables", hostIP)
|
||||||
return createProxier(loadBalancer, listenIP, netsh, hostIP, proxyPorts, syncPeriod, udpIdleTimeout)
|
return createProxier(loadBalancer, listenIP, netsh, hostIP, syncPeriod, udpIdleTimeout)
|
||||||
}
|
}
|
||||||
|
|
||||||
func createProxier(loadBalancer LoadBalancer, listenIP net.IP, netsh netsh.Interface, hostIP net.IP, proxyPorts PortAllocator, syncPeriod, udpIdleTimeout time.Duration) (*Proxier, error) {
|
func createProxier(loadBalancer LoadBalancer, listenIP net.IP, netsh netsh.Interface, hostIP net.IP, syncPeriod, udpIdleTimeout time.Duration) (*Proxier, error) {
|
||||||
// convenient to pass nil for tests..
|
|
||||||
if proxyPorts == nil {
|
|
||||||
proxyPorts = newPortAllocator(utilnet.PortRange{})
|
|
||||||
}
|
|
||||||
return &Proxier{
|
return &Proxier{
|
||||||
loadBalancer: loadBalancer,
|
loadBalancer: loadBalancer,
|
||||||
serviceMap: make(map[proxy.ServicePortName]*serviceInfo),
|
serviceMap: make(map[proxy.ServicePortPortalName]*serviceInfo),
|
||||||
portMap: make(map[portMapKey]*portMapValue),
|
portMap: make(map[portMapKey]*portMapValue),
|
||||||
syncPeriod: syncPeriod,
|
syncPeriod: syncPeriod,
|
||||||
udpIdleTimeout: udpIdleTimeout,
|
udpIdleTimeout: udpIdleTimeout,
|
||||||
listenIP: listenIP,
|
|
||||||
netsh: netsh,
|
netsh: netsh,
|
||||||
hostIP: hostIP,
|
hostIP: hostIP,
|
||||||
proxyPorts: proxyPorts,
|
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Sync is called to immediately synchronize the proxier state to iptables
|
// Sync is called to immediately synchronize the proxier state
|
||||||
func (proxier *Proxier) Sync() {
|
func (proxier *Proxier) Sync() {
|
||||||
proxier.ensurePortals()
|
proxier.ensurePortals()
|
||||||
proxier.cleanupStaleStickySessions()
|
proxier.cleanupStaleStickySessions()
|
||||||
|
@ -196,7 +187,7 @@ func (proxier *Proxier) SyncLoop() {
|
||||||
|
|
||||||
// Ensure that portals exist for all services.
|
// Ensure that portals exist for all services.
|
||||||
func (proxier *Proxier) ensurePortals() {
|
func (proxier *Proxier) ensurePortals() {
|
||||||
proxier.mu.Lock()
|
/*proxier.mu.Lock()
|
||||||
defer proxier.mu.Unlock()
|
defer proxier.mu.Unlock()
|
||||||
// NB: This does not remove rules that should not be present.
|
// NB: This does not remove rules that should not be present.
|
||||||
for name, info := range proxier.serviceMap {
|
for name, info := range proxier.serviceMap {
|
||||||
|
@ -204,52 +195,56 @@ func (proxier *Proxier) ensurePortals() {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
glog.Errorf("Failed to ensure portal for %q: %v", name, err)
|
glog.Errorf("Failed to ensure portal for %q: %v", name, err)
|
||||||
}
|
}
|
||||||
}
|
}*/
|
||||||
}
|
}
|
||||||
|
|
||||||
// cleanupStaleStickySessions cleans up any stale sticky session records in the hash map.
|
// cleanupStaleStickySessions cleans up any stale sticky session records in the hash map.
|
||||||
func (proxier *Proxier) cleanupStaleStickySessions() {
|
func (proxier *Proxier) cleanupStaleStickySessions() {
|
||||||
proxier.mu.Lock()
|
proxier.mu.Lock()
|
||||||
defer proxier.mu.Unlock()
|
defer proxier.mu.Unlock()
|
||||||
for name := range proxier.serviceMap {
|
spnMap := make(map[proxy.ServicePortName]bool)
|
||||||
|
for k := range proxier.serviceMap {
|
||||||
|
spn := proxy.ServicePortName{NamespacedName: types.NamespacedName{Namespace: k.Namespace, Name: k.Name}, Port: k.Port}
|
||||||
|
spnMap[spn] = true
|
||||||
|
}
|
||||||
|
for name := range spnMap {
|
||||||
proxier.loadBalancer.CleanupStaleStickySessions(name)
|
proxier.loadBalancer.CleanupStaleStickySessions(name)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// This assumes proxier.mu is not locked.
|
// This assumes proxier.mu is not locked.
|
||||||
func (proxier *Proxier) stopProxy(service proxy.ServicePortName, info *serviceInfo) error {
|
func (proxier *Proxier) stopProxy(service proxy.ServicePortPortalName, info *serviceInfo) error {
|
||||||
proxier.mu.Lock()
|
proxier.mu.Lock()
|
||||||
defer proxier.mu.Unlock()
|
defer proxier.mu.Unlock()
|
||||||
return proxier.stopProxyInternal(service, info)
|
return proxier.stopProxyInternal(service, info)
|
||||||
}
|
}
|
||||||
|
|
||||||
// This assumes proxier.mu is locked.
|
// This assumes proxier.mu is locked.
|
||||||
func (proxier *Proxier) stopProxyInternal(service proxy.ServicePortName, info *serviceInfo) error {
|
func (proxier *Proxier) stopProxyInternal(service proxy.ServicePortPortalName, info *serviceInfo) error {
|
||||||
delete(proxier.serviceMap, service)
|
delete(proxier.serviceMap, service)
|
||||||
info.setAlive(false)
|
info.setAlive(false)
|
||||||
err := info.socket.Close()
|
err := info.socket.Close()
|
||||||
port := info.socket.ListenPort()
|
|
||||||
proxier.proxyPorts.Release(port)
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
func (proxier *Proxier) getServiceInfo(service proxy.ServicePortName) (*serviceInfo, bool) {
|
func (proxier *Proxier) getServiceInfo(service proxy.ServicePortPortalName) (*serviceInfo, bool) {
|
||||||
proxier.mu.Lock()
|
proxier.mu.Lock()
|
||||||
defer proxier.mu.Unlock()
|
defer proxier.mu.Unlock()
|
||||||
info, ok := proxier.serviceMap[service]
|
info, ok := proxier.serviceMap[service]
|
||||||
return info, ok
|
return info, ok
|
||||||
}
|
}
|
||||||
|
|
||||||
func (proxier *Proxier) setServiceInfo(service proxy.ServicePortName, info *serviceInfo) {
|
func (proxier *Proxier) setServiceInfo(service proxy.ServicePortPortalName, info *serviceInfo) {
|
||||||
proxier.mu.Lock()
|
proxier.mu.Lock()
|
||||||
defer proxier.mu.Unlock()
|
defer proxier.mu.Unlock()
|
||||||
proxier.serviceMap[service] = info
|
proxier.serviceMap[service] = info
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
// addServiceOnPort starts listening for a new service, returning the serviceInfo.
|
// addServiceOnPort starts listening for a new service, returning the serviceInfo.
|
||||||
// Pass proxyPort=0 to allocate a random port. The timeout only applies to UDP
|
// Pass proxyPort=0 to allocate a random port. The timeout only applies to UDP
|
||||||
// connections, for now.
|
// connections, for now.
|
||||||
func (proxier *Proxier) addServiceOnPort(service proxy.ServicePortName, protocol api.Protocol, proxyPort int, timeout time.Duration) (*serviceInfo, error) {
|
func (proxier *Proxier) addServiceOnPort(service proxy.ServicePortPortalName, protocol api.Protocol, proxyPort int, timeout time.Duration) (*serviceInfo, error) {
|
||||||
sock, err := newProxySocket(protocol, proxier.listenIP, proxyPort)
|
sock, err := newProxySocket(protocol, proxier.listenIP, proxyPort)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
@ -286,13 +281,14 @@ func (proxier *Proxier) addServiceOnPort(service proxy.ServicePortName, protocol
|
||||||
|
|
||||||
return si, nil
|
return si, nil
|
||||||
}
|
}
|
||||||
|
*/
|
||||||
|
|
||||||
// OnServiceUpdate manages the active set of service proxies.
|
// OnServiceUpdate manages the active set of service proxies.
|
||||||
// Active service proxies are reinitialized if found in the update set or
|
// Active service proxies are reinitialized if found in the update set or
|
||||||
// shutdown if missing from the update set.
|
// shutdown if missing from the update set.
|
||||||
func (proxier *Proxier) OnServiceUpdate(services []api.Service) {
|
func (proxier *Proxier) OnServiceUpdate(services []api.Service) {
|
||||||
glog.V(4).Infof("Received update notice: %+v", services)
|
glog.V(4).Infof("Received update notice: %+v", services)
|
||||||
activeServices := make(map[proxy.ServicePortName]bool) // use a map as a set
|
activeServices := make(map[proxy.ServicePortPortalName]bool) // use a map as a set
|
||||||
for i := range services {
|
for i := range services {
|
||||||
service := &services[i]
|
service := &services[i]
|
||||||
|
|
||||||
|
@ -304,84 +300,143 @@ func (proxier *Proxier) OnServiceUpdate(services []api.Service) {
|
||||||
|
|
||||||
for i := range service.Spec.Ports {
|
for i := range service.Spec.Ports {
|
||||||
servicePort := &service.Spec.Ports[i]
|
servicePort := &service.Spec.Ports[i]
|
||||||
serviceName := proxy.ServicePortName{NamespacedName: types.NamespacedName{Namespace: service.Namespace, Name: service.Name}, Port: servicePort.Name}
|
// create a slice of source IPs
|
||||||
activeServices[serviceName] = true
|
var listenIPs []string
|
||||||
serviceIP := net.ParseIP(service.Spec.ClusterIP)
|
listenIPs = append(listenIPs, service.Spec.ClusterIP)
|
||||||
info, exists := proxier.getServiceInfo(serviceName)
|
|
||||||
// TODO: check health of the socket? What if ProxyLoop exited?
|
/*for _, ip := range service.Spec.ExternalIPs {
|
||||||
if exists && sameConfig(info, service, servicePort) {
|
listenIPs = append(listenIPs, ip)
|
||||||
// Nothing changed.
|
|
||||||
continue
|
|
||||||
}
|
}
|
||||||
if exists {
|
|
||||||
glog.V(4).Infof("Something changed for service %q: stopping it", serviceName)
|
for _, ip := range service.Status.LoadBalancer.Ingress {
|
||||||
err := proxier.closePortal(serviceName, info)
|
listenIPs = append(listenIPs, ip)
|
||||||
if err != nil {
|
}
|
||||||
glog.Errorf("Failed to close portal for %q: %v", serviceName, err)
|
|
||||||
|
if int(service.Spec.Ports[i]) != 0 {
|
||||||
|
listenIPs = append(listenIPs, "")
|
||||||
|
}*/
|
||||||
|
|
||||||
|
for _, listenIP := range listenIPs {
|
||||||
|
serviceName := proxy.ServicePortPortalName{NamespacedName: types.NamespacedName{Namespace: service.Namespace, Name: service.Name}, Port: servicePort.Name, PortalIPName: listenIP}
|
||||||
|
activeServices[serviceName] = true
|
||||||
|
serviceIP := net.ParseIP(listenIP)
|
||||||
|
info, exists := proxier.getServiceInfo(serviceName)
|
||||||
|
if exists && sameConfig(info, service, servicePort) {
|
||||||
|
// Nothing changed.
|
||||||
|
continue
|
||||||
}
|
}
|
||||||
err = proxier.stopProxy(serviceName, info)
|
if exists {
|
||||||
if err != nil {
|
glog.V(4).Infof("Something changed for service %q: stopping it", serviceName)
|
||||||
glog.Errorf("Failed to stop service %q: %v", serviceName, err)
|
|
||||||
|
// turn off the proxy
|
||||||
|
err := proxier.stopProxy(serviceName, info)
|
||||||
|
if err != nil {
|
||||||
|
glog.Errorf("Failed to stop service %q: %v", serviceName, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// close the PortalProxy if it is not a node port
|
||||||
|
if serviceIP != nil {
|
||||||
|
args := proxier.netshIpv4AddressDeleteArgs(serviceIP)
|
||||||
|
if err := proxier.netsh.DeleteIPAddress(args); err != nil {
|
||||||
|
glog.Errorf("Failed to delete IP address for service %q, error %s", serviceName, err.Error())
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// TODO(ajh) release the node port
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
proxyPort, err := proxier.proxyPorts.AllocateNext()
|
glog.V(1).Infof("Adding new service %q at %s:%d/%s", serviceName, serviceIP, servicePort.Port, servicePort.Protocol)
|
||||||
if err != nil {
|
|
||||||
glog.Errorf("failed to allocate proxy port for service %q: %v", serviceName, err)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
glog.V(1).Infof("Adding new service %q at %s:%d/%s", serviceName, serviceIP, servicePort.Port, servicePort.Protocol)
|
// add the IP address if it is not a node port
|
||||||
info, err = proxier.addServiceOnPort(serviceName, servicePort.Protocol, proxyPort, proxier.udpIdleTimeout)
|
if serviceIP != nil {
|
||||||
if err != nil {
|
args := proxier.netshIpv4AddressAddArgs(serviceIP)
|
||||||
glog.Errorf("Failed to start proxy for %q: %v", serviceName, err)
|
existed, err := proxier.netsh.EnsureIPAddress(args, serviceIP)
|
||||||
continue
|
if err != nil {
|
||||||
}
|
glog.Errorf("Failed to add ip address for service %q, args:%v", serviceName, args)
|
||||||
info.portal.ip = serviceIP
|
continue
|
||||||
info.portal.port = int(servicePort.Port)
|
}
|
||||||
info.externalIPs = service.Spec.ExternalIPs
|
if !existed {
|
||||||
// Deep-copy in case the service instance changes
|
glog.V(3).Infof("Added ip address to fowarder interface for service %q on %s %s:%d", serviceName, servicePort.Protocol, serviceIP, int(servicePort.Port))
|
||||||
info.loadBalancerStatus = *api.LoadBalancerStatusDeepCopy(&service.Status.LoadBalancer)
|
}
|
||||||
info.nodePort = int(servicePort.NodePort)
|
} else {
|
||||||
info.sessionAffinityType = service.Spec.SessionAffinity
|
// TODO(ajh) handle the node port
|
||||||
glog.V(4).Infof("info: %#v", info)
|
}
|
||||||
|
|
||||||
err = proxier.openPortal(serviceName, info)
|
// add the listener, proxy
|
||||||
if err != nil {
|
sock, err := newProxySocket(servicePort.Protocol, serviceIP, int(servicePort.Port))
|
||||||
glog.Errorf("Failed to open portal for %q: %v", serviceName, err)
|
if err != nil {
|
||||||
|
glog.Errorf("failed to create a new proxy socket for service %q: %v", serviceName, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
si := &serviceInfo{
|
||||||
|
isAliveAtomic: 1,
|
||||||
|
portal: portal{ip: listenIP, port: int(servicePort.Port), isExternal: false},
|
||||||
|
protocol: servicePort.Protocol,
|
||||||
|
proxyPort: int(servicePort.Port),
|
||||||
|
socket: sock,
|
||||||
|
timeout: proxier.udpIdleTimeout,
|
||||||
|
activeClients: newClientCache(),
|
||||||
|
sessionAffinityType: service.Spec.SessionAffinity, // default
|
||||||
|
stickyMaxAgeMinutes: 180, // TODO: parameterize this in the API.
|
||||||
|
}
|
||||||
|
glog.V(4).Infof("info: %#v", si)
|
||||||
|
proxier.setServiceInfo(serviceName, si)
|
||||||
|
|
||||||
|
glog.V(2).Infof("Proxying for service %q on %s port %d", serviceName, servicePort.Protocol, int(servicePort.Port))
|
||||||
|
go func(service proxy.ServicePortPortalName, proxier *Proxier) {
|
||||||
|
defer runtime.HandleCrash()
|
||||||
|
atomic.AddInt32(&proxier.numProxyLoops, 1)
|
||||||
|
sock.ProxyLoop(service, si, proxier)
|
||||||
|
atomic.AddInt32(&proxier.numProxyLoops, -1)
|
||||||
|
}(serviceName, proxier)
|
||||||
|
}
|
||||||
|
if len(listenIPs) > 0 {
|
||||||
|
// only one loadbalancer per service endpoint
|
||||||
|
servicePortName := proxy.ServicePortName{NamespacedName: types.NamespacedName{Namespace: service.Namespace, Name: service.Name}, Port: servicePort.Name}
|
||||||
|
proxier.loadBalancer.NewService(servicePortName, service.Spec.SessionAffinity, 180)
|
||||||
}
|
}
|
||||||
proxier.loadBalancer.NewService(serviceName, info.sessionAffinityType, info.stickyMaxAgeMinutes)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
proxier.mu.Lock()
|
proxier.mu.Lock()
|
||||||
defer proxier.mu.Unlock()
|
defer proxier.mu.Unlock()
|
||||||
|
|
||||||
|
spnMap := make(map[proxy.ServicePortName]bool)
|
||||||
|
|
||||||
for name, info := range proxier.serviceMap {
|
for name, info := range proxier.serviceMap {
|
||||||
|
spn := proxy.ServicePortName{NamespacedName: types.NamespacedName{Namespace: name.Namespace, Name: name.Name}, Port: name.Port}
|
||||||
|
spnMap[spn] = spnMap[spn] || activeServices[name]
|
||||||
|
|
||||||
if !activeServices[name] {
|
if !activeServices[name] {
|
||||||
|
serviceIP := net.ParseIP(info.portal.ip)
|
||||||
glog.V(1).Infof("Stopping service %q", name)
|
glog.V(1).Infof("Stopping service %q", name)
|
||||||
err := proxier.closePortal(name, info)
|
|
||||||
if err != nil {
|
// turn off the proxy
|
||||||
glog.Errorf("Failed to close portal for %q: %v", name, err)
|
err := proxier.stopProxy(name, info)
|
||||||
}
|
|
||||||
err = proxier.stopProxyInternal(name, info)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
glog.Errorf("Failed to stop service %q: %v", name, err)
|
glog.Errorf("Failed to stop service %q: %v", name, err)
|
||||||
}
|
}
|
||||||
proxier.loadBalancer.DeleteService(name)
|
|
||||||
|
// close the PortalProxy if it is not a node port
|
||||||
|
if serviceIP != nil {
|
||||||
|
args := proxier.netshIpv4AddressDeleteArgs(serviceIP)
|
||||||
|
if err := proxier.netsh.DeleteIPAddress(args); err != nil {
|
||||||
|
glog.Errorf("Failed to stop service %q: %v", name, err)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// TODO(ajh) release the node port
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// only delete spn if all listen ips show inactive
|
||||||
|
for k := range spnMap {
|
||||||
|
if !spnMap[k] {
|
||||||
|
proxier.loadBalancer.DeleteService(k)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func sameConfig(info *serviceInfo, service *api.Service, port *api.ServicePort) bool {
|
func sameConfig(info *serviceInfo, service *api.Service, port *api.ServicePort) bool {
|
||||||
if info.protocol != port.Protocol || info.portal.port != int(port.Port) || info.nodePort != int(port.NodePort) {
|
if info.protocol != port.Protocol || info.portal.port != int(port.Port) {
|
||||||
return false
|
|
||||||
}
|
|
||||||
if !info.portal.ip.Equal(net.ParseIP(service.Spec.ClusterIP)) {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
if !ipsEqual(info.externalIPs, service.Spec.ExternalIPs) {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
if !api.LoadBalancerStatusEqual(&info.loadBalancerStatus, &service.Status.LoadBalancer) {
|
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
if info.sessionAffinityType != service.Spec.SessionAffinity {
|
if info.sessionAffinityType != service.Spec.SessionAffinity {
|
||||||
|
@ -402,6 +457,7 @@ func ipsEqual(lhs, rhs []string) bool {
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
func (proxier *Proxier) openPortal(service proxy.ServicePortName, info *serviceInfo) error {
|
func (proxier *Proxier) openPortal(service proxy.ServicePortName, info *serviceInfo) error {
|
||||||
err := proxier.openOnePortal(info.portal, info.protocol, proxier.listenIP, info.proxyPort, service)
|
err := proxier.openOnePortal(info.portal, info.protocol, proxier.listenIP, info.proxyPort, service)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -429,7 +485,9 @@ func (proxier *Proxier) openPortal(service proxy.ServicePortName, info *serviceI
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
*/
|
||||||
|
|
||||||
|
/*
|
||||||
func (proxier *Proxier) openOnePortal(portal portal, protocol api.Protocol, proxyIP net.IP, proxyPort int, name proxy.ServicePortName) error {
|
func (proxier *Proxier) openOnePortal(portal portal, protocol api.Protocol, proxyIP net.IP, proxyPort int, name proxy.ServicePortName) error {
|
||||||
if protocol == api.ProtocolUDP {
|
if protocol == api.ProtocolUDP {
|
||||||
glog.Warningf("Not adding rule for %q on %s:%d as UDP protocol is not supported by netsh portproxy", name, portal.ip, portal.port)
|
glog.Warningf("Not adding rule for %q on %s:%d as UDP protocol is not supported by netsh portproxy", name, portal.ip, portal.port)
|
||||||
|
@ -461,7 +519,9 @@ func (proxier *Proxier) openOnePortal(portal portal, protocol api.Protocol, prox
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
*/
|
||||||
|
|
||||||
|
/*
|
||||||
// claimNodePort marks a port as being owned by a particular service, or returns error if already claimed.
|
// claimNodePort marks a port as being owned by a particular service, or returns error if already claimed.
|
||||||
// Idempotent: reclaiming with the same owner is not an error
|
// Idempotent: reclaiming with the same owner is not an error
|
||||||
func (proxier *Proxier) claimNodePort(ip net.IP, port int, protocol api.Protocol, owner proxy.ServicePortName) error {
|
func (proxier *Proxier) claimNodePort(ip net.IP, port int, protocol api.Protocol, owner proxy.ServicePortName) error {
|
||||||
|
@ -494,7 +554,9 @@ func (proxier *Proxier) claimNodePort(ip net.IP, port int, protocol api.Protocol
|
||||||
}
|
}
|
||||||
return fmt.Errorf("Port conflict detected on port %s. %v vs %v", key.String(), owner, existing)
|
return fmt.Errorf("Port conflict detected on port %s. %v vs %v", key.String(), owner, existing)
|
||||||
}
|
}
|
||||||
|
*/
|
||||||
|
|
||||||
|
/*
|
||||||
// releaseNodePort releases a claim on a port. Returns an error if the owner does not match the claim.
|
// releaseNodePort releases a claim on a port. Returns an error if the owner does not match the claim.
|
||||||
// Tolerates release on an unclaimed port, to simplify .
|
// Tolerates release on an unclaimed port, to simplify .
|
||||||
func (proxier *Proxier) releaseNodePort(ip net.IP, port int, protocol api.Protocol, owner proxy.ServicePortName) error {
|
func (proxier *Proxier) releaseNodePort(ip net.IP, port int, protocol api.Protocol, owner proxy.ServicePortName) error {
|
||||||
|
@ -515,7 +577,9 @@ func (proxier *Proxier) releaseNodePort(ip net.IP, port int, protocol api.Protoc
|
||||||
existing.socket.Close()
|
existing.socket.Close()
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
*/
|
||||||
|
|
||||||
|
/*
|
||||||
func (proxier *Proxier) openNodePort(nodePort int, protocol api.Protocol, proxyIP net.IP, proxyPort int, name proxy.ServicePortName) error {
|
func (proxier *Proxier) openNodePort(nodePort int, protocol api.Protocol, proxyIP net.IP, proxyPort int, name proxy.ServicePortName) error {
|
||||||
if protocol == api.ProtocolUDP {
|
if protocol == api.ProtocolUDP {
|
||||||
glog.Warningf("Not adding node port rule for %q on port %d as UDP protocol is not supported by netsh portproxy", name, nodePort)
|
glog.Warningf("Not adding node port rule for %q on port %d as UDP protocol is not supported by netsh portproxy", name, nodePort)
|
||||||
|
@ -539,8 +603,9 @@ func (proxier *Proxier) openNodePort(nodePort int, protocol api.Protocol, proxyI
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}*/
|
||||||
|
|
||||||
|
/*
|
||||||
func (proxier *Proxier) closePortal(service proxy.ServicePortName, info *serviceInfo) error {
|
func (proxier *Proxier) closePortal(service proxy.ServicePortName, info *serviceInfo) error {
|
||||||
// Collect errors and report them all at the end.
|
// Collect errors and report them all at the end.
|
||||||
el := proxier.closeOnePortal(info.portal, info.protocol, proxier.listenIP, info.proxyPort, service)
|
el := proxier.closeOnePortal(info.portal, info.protocol, proxier.listenIP, info.proxyPort, service)
|
||||||
|
@ -561,8 +626,9 @@ func (proxier *Proxier) closePortal(service proxy.ServicePortName, info *service
|
||||||
glog.Errorf("Some errors closing iptables portals for service %q", service)
|
glog.Errorf("Some errors closing iptables portals for service %q", service)
|
||||||
}
|
}
|
||||||
return utilerrors.NewAggregate(el)
|
return utilerrors.NewAggregate(el)
|
||||||
}
|
}*/
|
||||||
|
|
||||||
|
/*
|
||||||
func (proxier *Proxier) closeOnePortal(portal portal, protocol api.Protocol, proxyIP net.IP, proxyPort int, name proxy.ServicePortName) []error {
|
func (proxier *Proxier) closeOnePortal(portal portal, protocol api.Protocol, proxyIP net.IP, proxyPort int, name proxy.ServicePortName) []error {
|
||||||
el := []error{}
|
el := []error{}
|
||||||
|
|
||||||
|
@ -587,8 +653,9 @@ func (proxier *Proxier) closeOnePortal(portal portal, protocol api.Protocol, pro
|
||||||
}
|
}
|
||||||
|
|
||||||
return el
|
return el
|
||||||
}
|
}*/
|
||||||
|
|
||||||
|
/*
|
||||||
func (proxier *Proxier) closeNodePort(nodePort int, protocol api.Protocol, proxyIP net.IP, proxyPort int, name proxy.ServicePortName) []error {
|
func (proxier *Proxier) closeNodePort(nodePort int, protocol api.Protocol, proxyIP net.IP, proxyPort int, name proxy.ServicePortName) []error {
|
||||||
el := []error{}
|
el := []error{}
|
||||||
|
|
||||||
|
@ -603,7 +670,7 @@ func (proxier *Proxier) closeNodePort(nodePort int, protocol api.Protocol, proxy
|
||||||
}
|
}
|
||||||
|
|
||||||
return el
|
return el
|
||||||
}
|
}*/
|
||||||
|
|
||||||
func isLocalIP(ip net.IP) (bool, error) {
|
func isLocalIP(ip net.IP) (bool, error) {
|
||||||
addrs, err := net.InterfaceAddrs()
|
addrs, err := net.InterfaceAddrs()
|
||||||
|
@ -633,6 +700,7 @@ func isClosedError(err error) bool {
|
||||||
return strings.HasSuffix(err.Error(), "use of closed network connection")
|
return strings.HasSuffix(err.Error(), "use of closed network connection")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
func (proxier *Proxier) netshPortProxyAddArgs(destIP net.IP, destPort int, proxyIP net.IP, proxyPort int, service proxy.ServicePortName) []string {
|
func (proxier *Proxier) netshPortProxyAddArgs(destIP net.IP, destPort int, proxyIP net.IP, proxyPort int, service proxy.ServicePortName) []string {
|
||||||
args := []string{
|
args := []string{
|
||||||
"interface", "portproxy", "set", "v4tov4",
|
"interface", "portproxy", "set", "v4tov4",
|
||||||
|
@ -646,6 +714,7 @@ func (proxier *Proxier) netshPortProxyAddArgs(destIP net.IP, destPort int, proxy
|
||||||
|
|
||||||
return args
|
return args
|
||||||
}
|
}
|
||||||
|
*/
|
||||||
|
|
||||||
func (proxier *Proxier) netshIpv4AddressAddArgs(destIP net.IP) []string {
|
func (proxier *Proxier) netshIpv4AddressAddArgs(destIP net.IP) []string {
|
||||||
intName := proxier.netsh.GetInterfaceToAddIP()
|
intName := proxier.netsh.GetInterfaceToAddIP()
|
||||||
|
@ -658,6 +727,7 @@ func (proxier *Proxier) netshIpv4AddressAddArgs(destIP net.IP) []string {
|
||||||
return args
|
return args
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
func (proxier *Proxier) netshPortProxyDeleteArgs(destIP net.IP, destPort int, proxyIP net.IP, proxyPort int, service proxy.ServicePortName) []string {
|
func (proxier *Proxier) netshPortProxyDeleteArgs(destIP net.IP, destPort int, proxyIP net.IP, proxyPort int, service proxy.ServicePortName) []string {
|
||||||
args := []string{
|
args := []string{
|
||||||
"interface", "portproxy", "delete", "v4tov4",
|
"interface", "portproxy", "delete", "v4tov4",
|
||||||
|
@ -669,6 +739,7 @@ func (proxier *Proxier) netshPortProxyDeleteArgs(destIP net.IP, destPort int, pr
|
||||||
|
|
||||||
return args
|
return args
|
||||||
}
|
}
|
||||||
|
*/
|
||||||
|
|
||||||
func (proxier *Proxier) netshIpv4AddressDeleteArgs(destIP net.IP) []string {
|
func (proxier *Proxier) netshIpv4AddressDeleteArgs(destIP net.IP) []string {
|
||||||
intName := proxier.netsh.GetInterfaceToAddIP()
|
intName := proxier.netsh.GetInterfaceToAddIP()
|
||||||
|
|
|
@ -26,6 +26,7 @@ import (
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/golang/glog"
|
"github.com/golang/glog"
|
||||||
|
"k8s.io/apimachinery/pkg/types"
|
||||||
"k8s.io/apimachinery/pkg/util/runtime"
|
"k8s.io/apimachinery/pkg/util/runtime"
|
||||||
"k8s.io/kubernetes/pkg/api"
|
"k8s.io/kubernetes/pkg/api"
|
||||||
"k8s.io/kubernetes/pkg/proxy"
|
"k8s.io/kubernetes/pkg/proxy"
|
||||||
|
@ -40,7 +41,7 @@ type proxySocket interface {
|
||||||
// while sessions are active.
|
// while sessions are active.
|
||||||
Close() error
|
Close() error
|
||||||
// ProxyLoop proxies incoming connections for the specified service to the service endpoints.
|
// ProxyLoop proxies incoming connections for the specified service to the service endpoints.
|
||||||
ProxyLoop(service proxy.ServicePortName, info *serviceInfo, proxier *Proxier)
|
ProxyLoop(service proxy.ServicePortPortalName, info *serviceInfo, proxier *Proxier)
|
||||||
// ListenPort returns the host port that the proxySocket is listening on
|
// ListenPort returns the host port that the proxySocket is listening on
|
||||||
ListenPort() int
|
ListenPort() int
|
||||||
}
|
}
|
||||||
|
@ -86,10 +87,11 @@ func (tcp *tcpProxySocket) ListenPort() int {
|
||||||
return tcp.port
|
return tcp.port
|
||||||
}
|
}
|
||||||
|
|
||||||
func tryConnect(service proxy.ServicePortName, srcAddr net.Addr, protocol string, proxier *Proxier) (out net.Conn, err error) {
|
func tryConnect(service proxy.ServicePortPortalName, srcAddr net.Addr, protocol string, proxier *Proxier) (out net.Conn, err error) {
|
||||||
sessionAffinityReset := false
|
sessionAffinityReset := false
|
||||||
for _, dialTimeout := range endpointDialTimeout {
|
for _, dialTimeout := range endpointDialTimeout {
|
||||||
endpoint, err := proxier.loadBalancer.NextEndpoint(service, srcAddr, sessionAffinityReset)
|
servicePortName := proxy.ServicePortName{NamespacedName: types.NamespacedName{Namespace: service.Namespace, Name: service.Name}, Port: service.Port}
|
||||||
|
endpoint, err := proxier.loadBalancer.NextEndpoint(servicePortName, srcAddr, sessionAffinityReset)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
glog.Errorf("Couldn't find an endpoint for %s: %v", service, err)
|
glog.Errorf("Couldn't find an endpoint for %s: %v", service, err)
|
||||||
return nil, err
|
return nil, err
|
||||||
|
@ -111,7 +113,7 @@ func tryConnect(service proxy.ServicePortName, srcAddr net.Addr, protocol string
|
||||||
return nil, fmt.Errorf("failed to connect to an endpoint.")
|
return nil, fmt.Errorf("failed to connect to an endpoint.")
|
||||||
}
|
}
|
||||||
|
|
||||||
func (tcp *tcpProxySocket) ProxyLoop(service proxy.ServicePortName, myInfo *serviceInfo, proxier *Proxier) {
|
func (tcp *tcpProxySocket) ProxyLoop(service proxy.ServicePortPortalName, myInfo *serviceInfo, proxier *Proxier) {
|
||||||
for {
|
for {
|
||||||
if !myInfo.isAlive() {
|
if !myInfo.isAlive() {
|
||||||
// The service port was closed or replaced.
|
// The service port was closed or replaced.
|
||||||
|
@ -197,7 +199,7 @@ func newClientCache() *clientCache {
|
||||||
return &clientCache{clients: map[string]net.Conn{}}
|
return &clientCache{clients: map[string]net.Conn{}}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (udp *udpProxySocket) ProxyLoop(service proxy.ServicePortName, myInfo *serviceInfo, proxier *Proxier) {
|
func (udp *udpProxySocket) ProxyLoop(service proxy.ServicePortPortalName, myInfo *serviceInfo, proxier *Proxier) {
|
||||||
var buffer [4096]byte // 4KiB should be enough for most whole-packets
|
var buffer [4096]byte // 4KiB should be enough for most whole-packets
|
||||||
for {
|
for {
|
||||||
if !myInfo.isAlive() {
|
if !myInfo.isAlive() {
|
||||||
|
@ -241,7 +243,7 @@ func (udp *udpProxySocket) ProxyLoop(service proxy.ServicePortName, myInfo *serv
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (udp *udpProxySocket) getBackendConn(activeClients *clientCache, cliAddr net.Addr, proxier *Proxier, service proxy.ServicePortName, timeout time.Duration) (net.Conn, error) {
|
func (udp *udpProxySocket) getBackendConn(activeClients *clientCache, cliAddr net.Addr, proxier *Proxier, service proxy.ServicePortPortalName, timeout time.Duration) (net.Conn, error) {
|
||||||
activeClients.mu.Lock()
|
activeClients.mu.Lock()
|
||||||
defer activeClients.mu.Unlock()
|
defer activeClients.mu.Unlock()
|
||||||
|
|
||||||
|
|
|
@ -167,12 +167,12 @@ func (runner *runner) DeleteIPAddress(args []string) error {
|
||||||
|
|
||||||
// GetInterfaceToAddIP returns the interface name where Service IP needs to be added
|
// GetInterfaceToAddIP returns the interface name where Service IP needs to be added
|
||||||
// IP Address needs to be added for netsh portproxy to redirect traffic
|
// IP Address needs to be added for netsh portproxy to redirect traffic
|
||||||
// Reads Environment variable INTERFACE_TO_ADD_SERVICE_IP, if it is not defined then "vEthernet (HNS Internal NIC)" is returned
|
// Reads Environment variable INTERFACE_TO_ADD_SERVICE_IP, if it is not defined then "vEthernet (forwarder)" is returned
|
||||||
func (runner *runner) GetInterfaceToAddIP() string {
|
func (runner *runner) GetInterfaceToAddIP() string {
|
||||||
if iface := os.Getenv("INTERFACE_TO_ADD_SERVICE_IP"); len(iface) > 0 {
|
if iface := os.Getenv("INTERFACE_TO_ADD_SERVICE_IP"); len(iface) > 0 {
|
||||||
return iface
|
return iface
|
||||||
}
|
}
|
||||||
return "vEthernet (HNS Internal NIC)"
|
return "vEthernet (forwarder)"
|
||||||
}
|
}
|
||||||
|
|
||||||
// Restore is part of Interface.
|
// Restore is part of Interface.
|
||||||
|
|
Loading…
Reference in New Issue