mirror of https://github.com/k3s-io/k3s
273 lines
9.1 KiB
Go
273 lines
9.1 KiB
Go
// Copyright 2018 flannel authors
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package hostgw
|
|
|
|
import (
|
|
"fmt"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/Microsoft/hcsshim"
|
|
"github.com/flannel-io/flannel/backend"
|
|
"github.com/flannel-io/flannel/pkg/ip"
|
|
"github.com/flannel-io/flannel/pkg/routing"
|
|
"github.com/flannel-io/flannel/subnet"
|
|
"github.com/pkg/errors"
|
|
"golang.org/x/net/context"
|
|
"k8s.io/apimachinery/pkg/util/json"
|
|
"k8s.io/apimachinery/pkg/util/wait"
|
|
log "k8s.io/klog"
|
|
)
|
|
|
|
func init() {
|
|
backend.Register("host-gw", New)
|
|
}
|
|
|
|
type HostgwBackend struct {
|
|
sm subnet.Manager
|
|
extIface *backend.ExternalInterface
|
|
}
|
|
|
|
func New(sm subnet.Manager, extIface *backend.ExternalInterface) (backend.Backend, error) {
|
|
if !extIface.ExtAddr.Equal(extIface.IfaceAddr) {
|
|
return nil, fmt.Errorf("your PublicIP differs from interface IP, meaning that probably you're on a NAT, which is not supported by host-gw backend")
|
|
}
|
|
|
|
be := &HostgwBackend{
|
|
sm: sm,
|
|
extIface: extIface,
|
|
}
|
|
|
|
return be, nil
|
|
}
|
|
|
|
func (be *HostgwBackend) RegisterNetwork(ctx context.Context, wg *sync.WaitGroup, config *subnet.Config) (backend.Network, error) {
|
|
// 1. Parse configuration
|
|
cfg := struct {
|
|
Name string
|
|
DNSServerList string
|
|
}{}
|
|
if len(config.Backend) > 0 {
|
|
if err := json.Unmarshal(config.Backend, &cfg); err != nil {
|
|
return nil, errors.Wrap(err, "error decoding windows host-gw backend config")
|
|
}
|
|
}
|
|
if len(cfg.Name) == 0 {
|
|
cfg.Name = "cbr0"
|
|
}
|
|
log.Infof("HOST-GW config: %+v", cfg)
|
|
|
|
n := &backend.RouteNetwork{
|
|
SimpleNetwork: backend.SimpleNetwork{
|
|
ExtIface: be.extIface,
|
|
},
|
|
SM: be.sm,
|
|
BackendType: "host-gw",
|
|
Mtu: be.extIface.Iface.MTU,
|
|
LinkIndex: be.extIface.Iface.Index,
|
|
}
|
|
n.GetRoute = func(lease *subnet.Lease) *routing.Route {
|
|
return &routing.Route{
|
|
DestinationSubnet: lease.Subnet.ToIPNet(),
|
|
GatewayAddress: lease.Attrs.PublicIP.ToIP(),
|
|
InterfaceIndex: n.LinkIndex,
|
|
}
|
|
}
|
|
|
|
// 2. Acquire the lease form subnet manager
|
|
attrs := subnet.LeaseAttrs{
|
|
PublicIP: ip.FromIP(be.extIface.ExtAddr),
|
|
BackendType: "host-gw",
|
|
}
|
|
|
|
l, err := be.sm.AcquireLease(ctx, &attrs)
|
|
switch err {
|
|
case nil:
|
|
n.SubnetLease = l
|
|
|
|
case context.Canceled, context.DeadlineExceeded:
|
|
return nil, err
|
|
|
|
default:
|
|
return nil, errors.Wrap(err, "failed to acquire lease")
|
|
}
|
|
|
|
// 3. Check if the network exists and has the expected settings
|
|
createNewNetwork := true
|
|
expectedSubnet := n.SubnetLease.Subnet
|
|
expectedAddressPrefix := expectedSubnet.String()
|
|
expectedGatewayAddress := (expectedSubnet.IP + 1).String()
|
|
expectedPodGatewayAddress := expectedSubnet.IP + 2
|
|
networkName := cfg.Name
|
|
var waitErr, lastErr error
|
|
|
|
existingNetwork, err := hcsshim.GetHNSNetworkByName(networkName)
|
|
if err == nil {
|
|
for _, subnet := range existingNetwork.Subnets {
|
|
if subnet.AddressPrefix == expectedAddressPrefix && subnet.GatewayAddress == expectedGatewayAddress {
|
|
createNewNetwork = false
|
|
log.Infof("Found existing HNSNetwork %s", networkName)
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
// 4. Create a new HNSNetwork
|
|
expectedNetwork := existingNetwork
|
|
if createNewNetwork {
|
|
if existingNetwork != nil {
|
|
if _, err := existingNetwork.Delete(); err != nil {
|
|
return nil, errors.Wrapf(err, "failed to delete existing HNSNetwork %s", networkName)
|
|
}
|
|
log.Infof("Deleted stale HNSNetwork %s", networkName)
|
|
}
|
|
|
|
expectedNetwork = &hcsshim.HNSNetwork{
|
|
Name: networkName,
|
|
Type: "L2Bridge",
|
|
DNSServerList: cfg.DNSServerList,
|
|
Subnets: []hcsshim.Subnet{
|
|
{
|
|
AddressPrefix: expectedAddressPrefix,
|
|
GatewayAddress: expectedGatewayAddress,
|
|
},
|
|
},
|
|
}
|
|
jsonRequest, err := json.Marshal(expectedNetwork)
|
|
if err != nil {
|
|
return nil, errors.Wrapf(err, "failed to marshal %+v", expectedNetwork)
|
|
}
|
|
|
|
log.Infof("Attempting to create HNSNetwork %s", string(jsonRequest))
|
|
newNetwork, err := hcsshim.HNSNetworkRequest("POST", "", string(jsonRequest))
|
|
if err != nil {
|
|
return nil, errors.Wrapf(err, "failed to create HNSNetwork %s", networkName)
|
|
}
|
|
|
|
// Wait for the network to populate Management IP
|
|
log.Infof("Waiting to get ManagementIP from HNSNetwork %s", networkName)
|
|
var newNetworkID = newNetwork.Id
|
|
waitErr = wait.Poll(500*time.Millisecond, 30*time.Second, func() (done bool, err error) {
|
|
newNetwork, lastErr = hcsshim.HNSNetworkRequest("GET", newNetworkID, "")
|
|
return newNetwork != nil && len(newNetwork.ManagementIP) != 0, nil
|
|
})
|
|
if waitErr == wait.ErrWaitTimeout {
|
|
// Do not swallow the root cause
|
|
if lastErr != nil {
|
|
waitErr = lastErr
|
|
}
|
|
return nil, errors.Wrapf(waitErr, "timeout, failed to get management IP from HNSNetwork %s", networkName)
|
|
}
|
|
|
|
// Wait for the interface with the management IP
|
|
log.Infof("Waiting to get net interface for HNSNetwork %s (%s)", networkName, newNetwork.ManagementIP)
|
|
managementIP, err := ip.ParseIP4(newNetwork.ManagementIP)
|
|
if err != nil {
|
|
return nil, errors.Wrapf(err, "Failed to parse management ip (%s)", newNetwork.ManagementIP)
|
|
}
|
|
|
|
waitErr = wait.Poll(500*time.Millisecond, 5*time.Second, func() (done bool, err error) {
|
|
_, lastErr = ip.GetInterfaceByIP(managementIP.ToIP())
|
|
return lastErr == nil, nil
|
|
})
|
|
if waitErr == wait.ErrWaitTimeout {
|
|
return nil, errors.Wrapf(lastErr, "timeout, failed to get net interface for HNSNetwork %s (%s)", networkName, newNetwork.ManagementIP)
|
|
}
|
|
|
|
log.Infof("Created HNSNetwork %s", networkName)
|
|
expectedNetwork = newNetwork
|
|
}
|
|
|
|
// 5. Ensure a 1.2 endpoint on this network in the host compartment
|
|
createNewBridgeEndpoint := true
|
|
bridgeEndpointName := networkName + "_ep"
|
|
existingBridgeEndpoint, err := hcsshim.GetHNSEndpointByName(bridgeEndpointName)
|
|
if err == nil && existingBridgeEndpoint.IPAddress.String() == expectedPodGatewayAddress.String() {
|
|
log.Infof("Found existing bridge HNSEndpoint %s", bridgeEndpointName)
|
|
createNewBridgeEndpoint = false
|
|
}
|
|
|
|
// 6. Create a bridge HNSEndpoint
|
|
expectedBridgeEndpoint := existingBridgeEndpoint
|
|
if createNewBridgeEndpoint {
|
|
if existingBridgeEndpoint != nil {
|
|
if _, err = existingBridgeEndpoint.Delete(); err != nil {
|
|
return nil, errors.Wrapf(err, "failed to delete existing bridge HNSEndpoint %s", bridgeEndpointName)
|
|
}
|
|
log.Infof("Deleted stale bridge HNSEndpoint %s", bridgeEndpointName)
|
|
}
|
|
|
|
expectedBridgeEndpoint = &hcsshim.HNSEndpoint{
|
|
Name: bridgeEndpointName,
|
|
IPAddress: expectedPodGatewayAddress.ToIP(),
|
|
VirtualNetwork: expectedNetwork.Id,
|
|
}
|
|
|
|
log.Infof("Attempting to create bridge HNSEndpoint %+v", expectedBridgeEndpoint)
|
|
if expectedBridgeEndpoint, err = expectedBridgeEndpoint.Create(); err != nil {
|
|
return nil, errors.Wrapf(err, "failed to create bridge HNSEndpoint %s", bridgeEndpointName)
|
|
}
|
|
|
|
log.Infof("Created bridge HNSEndpoint %s", bridgeEndpointName)
|
|
}
|
|
|
|
// Wait for the bridgeEndpoint to attach to the host
|
|
log.Infof("Waiting to attach bridge endpoint %s to host", bridgeEndpointName)
|
|
waitErr = wait.Poll(500*time.Millisecond, 5*time.Second, func() (done bool, err error) {
|
|
lastErr = expectedBridgeEndpoint.HostAttach(1)
|
|
if lastErr == nil {
|
|
return true, nil
|
|
}
|
|
// See https://github.com/flannel-io/flannel/issues/1391 and
|
|
// hcsshim lacks some validations to detect the error, so we judge it by error message.
|
|
if strings.Contains(lastErr.Error(), "This endpoint is already attached to the switch.") {
|
|
return true, nil
|
|
}
|
|
return false, nil
|
|
})
|
|
if waitErr == wait.ErrWaitTimeout {
|
|
return nil, errors.Wrapf(lastErr, "failed to hot attach bridge HNSEndpoint %s to host compartment", bridgeEndpointName)
|
|
}
|
|
log.Infof("Attached bridge endpoint %s to host successfully", bridgeEndpointName)
|
|
|
|
// 7. Enable forwarding on the host interface and endpoint
|
|
for _, interfaceIpAddress := range []string{expectedNetwork.ManagementIP, expectedBridgeEndpoint.IPAddress.String()} {
|
|
ipv4, err := ip.ParseIP4(interfaceIpAddress)
|
|
if err != nil {
|
|
return nil, errors.Wrapf(err, "Failed to parse expected net interface ip (%s)", interfaceIpAddress)
|
|
}
|
|
|
|
netInterface, err := ip.GetInterfaceByIP(ipv4.ToIP())
|
|
if err != nil {
|
|
return nil, errors.Wrapf(err, "failed to find interface for IP Address %s", interfaceIpAddress)
|
|
}
|
|
log.Infof("Found %+v interface with IP %s", netInterface, interfaceIpAddress)
|
|
|
|
// When a new hns network is created, the interface is modified, esp the name, index
|
|
if expectedNetwork.ManagementIP == ipv4.String() {
|
|
n.LinkIndex = netInterface.Index
|
|
n.Name = netInterface.Name
|
|
}
|
|
|
|
if err := ip.EnableForwardingForInterface(netInterface); err != nil {
|
|
return nil, errors.Wrapf(err, "failed to enable forwarding on %s index %d", netInterface.Name, netInterface.Index)
|
|
}
|
|
log.Infof("Enabled forwarding on %s index %d", netInterface.Name, netInterface.Index)
|
|
}
|
|
|
|
return n, nil
|
|
}
|