From c23f12765e04af778846ff019f2be8a2cf196d9a Mon Sep 17 00:00:00 2001 From: Segator Date: Tue, 26 Nov 2019 21:30:12 +0100 Subject: [PATCH] hostgw flannel support --- pkg/agent/flannel/flannel.go | 1 + pkg/agent/flannel/setup.go | 2 +- pkg/cli/cmds/server.go | 2 +- .../coreos/flannel/backend/hostgw/hostgw.go | 89 ++++++ .../flannel/backend/hostgw/hostgw_windows.go | 253 ++++++++++++++++++ 5 files changed, 345 insertions(+), 2 deletions(-) create mode 100644 vendor/github.com/coreos/flannel/backend/hostgw/hostgw.go create mode 100644 vendor/github.com/coreos/flannel/backend/hostgw/hostgw_windows.go diff --git a/pkg/agent/flannel/flannel.go b/pkg/agent/flannel/flannel.go index 8526772813..6577498f4a 100644 --- a/pkg/agent/flannel/flannel.go +++ b/pkg/agent/flannel/flannel.go @@ -30,6 +30,7 @@ import ( // Backends need to be imported for their init() to get executed and them to register _ "github.com/coreos/flannel/backend/extension" + _ "github.com/coreos/flannel/backend/hostgw" _ "github.com/coreos/flannel/backend/ipsec" _ "github.com/coreos/flannel/backend/vxlan" ) diff --git a/pkg/agent/flannel/setup.go b/pkg/agent/flannel/setup.go index caf629799f..809503265c 100644 --- a/pkg/agent/flannel/setup.go +++ b/pkg/agent/flannel/setup.go @@ -166,4 +166,4 @@ func setupStrongSwan(nodeConfig *config.Node) error { // make new strongswan link return os.Symlink(dataDir, nodeConfig.AgentConfig.StrongSwanDir) -} \ No newline at end of file +} diff --git a/pkg/cli/cmds/server.go b/pkg/cli/cmds/server.go index 138d130297..a00ee84930 100644 --- a/pkg/cli/cmds/server.go +++ b/pkg/cli/cmds/server.go @@ -114,7 +114,7 @@ func NewServerCommand(action func(*cli.Context) error) cli.Command { }, cli.StringFlag{ Name: "flannel-backend", - Usage: fmt.Sprintf("(networking) One of 'none', 'vxlan', 'ipsec', or 'wireguard'"), + Usage: fmt.Sprintf("(networking) One of 'none', 'vxlan', 'ipsec', 'hostgw', or 'wireguard'"), Destination: &ServerConfig.FlannelBackend, Value: "vxlan", }, diff --git a/vendor/github.com/coreos/flannel/backend/hostgw/hostgw.go b/vendor/github.com/coreos/flannel/backend/hostgw/hostgw.go new file mode 100644 index 0000000000..ebcbfc8eb4 --- /dev/null +++ b/vendor/github.com/coreos/flannel/backend/hostgw/hostgw.go @@ -0,0 +1,89 @@ +// +build !windows + +// Copyright 2015 flannel authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +build !windows + +package hostgw + +import ( + "fmt" + + "sync" + + "github.com/coreos/flannel/backend" + "github.com/coreos/flannel/pkg/ip" + "github.com/coreos/flannel/subnet" + "github.com/vishvananda/netlink" + "golang.org/x/net/context" +) + +func init() { + backend.Register("host-gw", New) +} + +type HostgwBackend struct { + sm subnet.Manager + extIface *backend.ExternalInterface +} + +func New(sm subnet.Manager, extIface *backend.ExternalInterface) (backend.Backend, error) { + if !extIface.ExtAddr.Equal(extIface.IfaceAddr) { + return nil, fmt.Errorf("your PublicIP differs from interface IP, meaning that probably you're on a NAT, which is not supported by host-gw backend") + } + + be := &HostgwBackend{ + sm: sm, + extIface: extIface, + } + return be, nil +} + +func (be *HostgwBackend) RegisterNetwork(ctx context.Context, wg sync.WaitGroup, config *subnet.Config) (backend.Network, error) { + n := &backend.RouteNetwork{ + SimpleNetwork: backend.SimpleNetwork{ + ExtIface: be.extIface, + }, + SM: be.sm, + BackendType: "host-gw", + Mtu: be.extIface.Iface.MTU, + LinkIndex: be.extIface.Iface.Index, + } + n.GetRoute = func(lease *subnet.Lease) *netlink.Route { + return &netlink.Route{ + Dst: lease.Subnet.ToIPNet(), + Gw: lease.Attrs.PublicIP.ToIP(), + LinkIndex: n.LinkIndex, + } + } + + attrs := subnet.LeaseAttrs{ + PublicIP: ip.FromIP(be.extIface.ExtAddr), + BackendType: "host-gw", + } + + l, err := be.sm.AcquireLease(ctx, &attrs) + switch err { + case nil: + n.SubnetLease = l + + case context.Canceled, context.DeadlineExceeded: + return nil, err + + default: + return nil, fmt.Errorf("failed to acquire lease: %v", err) + } + + return n, nil +} diff --git a/vendor/github.com/coreos/flannel/backend/hostgw/hostgw_windows.go b/vendor/github.com/coreos/flannel/backend/hostgw/hostgw_windows.go new file mode 100644 index 0000000000..9741f189f7 --- /dev/null +++ b/vendor/github.com/coreos/flannel/backend/hostgw/hostgw_windows.go @@ -0,0 +1,253 @@ +// Copyright 2018 flannel authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hostgw + +import ( + "fmt" + "strconv" + "sync" + "time" + + "github.com/Microsoft/hcsshim" + "github.com/coreos/flannel/backend" + "github.com/coreos/flannel/pkg/ip" + "github.com/coreos/flannel/subnet" + log "github.com/golang/glog" + "github.com/juju/errors" + "github.com/rakelkar/gonetsh/netroute" + "github.com/rakelkar/gonetsh/netsh" + "golang.org/x/net/context" + "k8s.io/apimachinery/pkg/util/json" + "k8s.io/apimachinery/pkg/util/wait" + utilexec "k8s.io/utils/exec" +) + +func init() { + backend.Register("host-gw", New) +} + +type HostgwBackend struct { + sm subnet.Manager + extIface *backend.ExternalInterface +} + +func New(sm subnet.Manager, extIface *backend.ExternalInterface) (backend.Backend, error) { + if !extIface.ExtAddr.Equal(extIface.IfaceAddr) { + return nil, fmt.Errorf("your PublicIP differs from interface IP, meaning that probably you're on a NAT, which is not supported by host-gw backend") + } + + be := &HostgwBackend{ + sm: sm, + extIface: extIface, + } + + return be, nil +} + +func (be *HostgwBackend) RegisterNetwork(ctx context.Context, wg sync.WaitGroup, config *subnet.Config) (backend.Network, error) { + // 1. Parse configuration + cfg := struct { + Name string + DNSServerList string + }{} + if len(config.Backend) > 0 { + if err := json.Unmarshal(config.Backend, &cfg); err != nil { + return nil, errors.Annotate(err, "error decoding windows host-gw backend config") + } + } + if len(cfg.Name) == 0 { + cfg.Name = "cbr0" + } + log.Infof("HOST-GW config: %+v", cfg) + + n := &backend.RouteNetwork{ + SimpleNetwork: backend.SimpleNetwork{ + ExtIface: be.extIface, + }, + SM: be.sm, + BackendType: "host-gw", + Mtu: be.extIface.Iface.MTU, + LinkIndex: be.extIface.Iface.Index, + } + n.GetRoute = func(lease *subnet.Lease) *netroute.Route { + return &netroute.Route{ + DestinationSubnet: lease.Subnet.ToIPNet(), + GatewayAddress: lease.Attrs.PublicIP.ToIP(), + LinkIndex: n.LinkIndex, + } + } + + // 2. Acquire the lease form subnet manager + attrs := subnet.LeaseAttrs{ + PublicIP: ip.FromIP(be.extIface.ExtAddr), + BackendType: "host-gw", + } + + l, err := be.sm.AcquireLease(ctx, &attrs) + switch err { + case nil: + n.SubnetLease = l + + case context.Canceled, context.DeadlineExceeded: + return nil, err + + default: + return nil, errors.Annotate(err, "failed to acquire lease") + } + + // 3. Check if the network exists and has the expected settings + netshHelper := netsh.New(utilexec.New()) + createNewNetwork := true + expectedSubnet := n.SubnetLease.Subnet + expectedAddressPrefix := expectedSubnet.String() + expectedGatewayAddress := (expectedSubnet.IP + 1).String() + expectedPodGatewayAddress := expectedSubnet.IP + 2 + networkName := cfg.Name + var waitErr, lastErr error + + existingNetwork, err := hcsshim.GetHNSNetworkByName(networkName) + if err == nil { + for _, subnet := range existingNetwork.Subnets { + if subnet.AddressPrefix == expectedAddressPrefix && subnet.GatewayAddress == expectedGatewayAddress { + createNewNetwork = false + log.Infof("Found existing HNSNetwork %s", networkName) + break + } + } + } + + // 4. Create a new HNSNetwork + expectedNetwork := existingNetwork + if createNewNetwork { + if existingNetwork != nil { + if _, err := existingNetwork.Delete(); err != nil { + return nil, errors.Annotatef(err, "failed to delete existing HNSNetwork %s", networkName) + } + log.Infof("Deleted stale HNSNetwork %s", networkName) + } + + expectedNetwork = &hcsshim.HNSNetwork{ + Name: networkName, + Type: "L2Bridge", + DNSServerList: cfg.DNSServerList, + Subnets: []hcsshim.Subnet{ + { + AddressPrefix: expectedAddressPrefix, + GatewayAddress: expectedGatewayAddress, + }, + }, + } + jsonRequest, err := json.Marshal(expectedNetwork) + if err != nil { + return nil, errors.Annotatef(err, "failed to marshal %+v", expectedNetwork) + } + + log.Infof("Attempting to create HNSNetwork %s", string(jsonRequest)) + newNetwork, err := hcsshim.HNSNetworkRequest("POST", "", string(jsonRequest)) + if err != nil { + return nil, errors.Annotatef(err, "failed to create HNSNetwork %s", networkName) + } + + // Wait for the network to populate Management IP + log.Infof("Waiting to get ManagementIP from HNSNetwork %s", networkName) + waitErr = wait.Poll(500*time.Millisecond, 30*time.Second, func() (done bool, err error) { + newNetwork, lastErr = hcsshim.HNSNetworkRequest("GET", newNetwork.Id, "") + return newNetwork != nil && len(newNetwork.ManagementIP) != 0, nil + }) + if waitErr == wait.ErrWaitTimeout { + return nil, errors.Annotatef(waitErr, "timeout, failed to get management IP from HNSNetwork %s", networkName) + } + + // Wait for the interface with the management IP + log.Infof("Waiting to get net interface for HNSNetwork %s (%s)", networkName, newNetwork.ManagementIP) + waitErr = wait.Poll(500*time.Millisecond, 5*time.Second, func() (done bool, err error) { + _, lastErr = netshHelper.GetInterfaceByIP(newNetwork.ManagementIP) + return lastErr == nil, nil + }) + if waitErr == wait.ErrWaitTimeout { + return nil, errors.Annotatef(lastErr, "timeout, failed to get net interface for HNSNetwork %s (%s)", networkName, newNetwork.ManagementIP) + } + + log.Infof("Created HNSNetwork %s", networkName) + expectedNetwork = newNetwork + } + + // 5. Ensure a 1.2 endpoint on this network in the host compartment + createNewBridgeEndpoint := true + bridgeEndpointName := networkName + "_ep" + existingBridgeEndpoint, err := hcsshim.GetHNSEndpointByName(bridgeEndpointName) + if err == nil && existingBridgeEndpoint.IPAddress.String() == expectedPodGatewayAddress.String() { + log.Infof("Found existing bridge HNSEndpoint %s", bridgeEndpointName) + createNewBridgeEndpoint = false + } + + // 6. Create a bridge HNSEndpoint + expectedBridgeEndpoint := existingBridgeEndpoint + if createNewBridgeEndpoint { + if existingBridgeEndpoint != nil { + if _, err = existingBridgeEndpoint.Delete(); err != nil { + return nil, errors.Annotatef(err, "failed to delete existing bridge HNSEndpoint %s", bridgeEndpointName) + } + log.Infof("Deleted stale bridge HNSEndpoint %s", bridgeEndpointName) + } + + expectedBridgeEndpoint = &hcsshim.HNSEndpoint{ + Name: bridgeEndpointName, + IPAddress: expectedPodGatewayAddress.ToIP(), + VirtualNetwork: expectedNetwork.Id, + } + + log.Infof("Attempting to create bridge HNSEndpoint %+v", expectedBridgeEndpoint) + if expectedBridgeEndpoint, err = expectedBridgeEndpoint.Create(); err != nil { + return nil, errors.Annotatef(err, "failed to create bridge HNSEndpoint %s", bridgeEndpointName) + } + + log.Infof("Created bridge HNSEndpoint %s", bridgeEndpointName) + } + + // Wait for the bridgeEndpoint to attach to the host + log.Infof("Waiting to attach bridge endpoint %s to host", bridgeEndpointName) + waitErr = wait.Poll(500*time.Millisecond, 5*time.Second, func() (done bool, err error) { + lastErr = expectedBridgeEndpoint.HostAttach(1) + return lastErr == nil, nil + }) + if waitErr == wait.ErrWaitTimeout { + return nil, errors.Annotatef(lastErr, "failed to hot attach bridge HNSEndpoint %s to host compartment", bridgeEndpointName) + } + log.Infof("Attached bridge endpoint %s to host successfully", bridgeEndpointName) + + // 7. Enable forwarding on the host interface and endpoint + for _, interfaceIpAddress := range []string{expectedNetwork.ManagementIP, expectedBridgeEndpoint.IPAddress.String()} { + netInterface, err := netshHelper.GetInterfaceByIP(interfaceIpAddress) + if err != nil { + return nil, errors.Annotatef(err, "failed to find interface for IP Address %s", interfaceIpAddress) + } + log.Infof("Found %+v interface with IP %s", netInterface, interfaceIpAddress) + + // When a new hns network is created, the interface is modified, esp the name, index + if expectedNetwork.ManagementIP == netInterface.IpAddress { + n.LinkIndex = netInterface.Idx + n.Name = netInterface.Name + } + + interfaceIdx := strconv.Itoa(netInterface.Idx) + if err := netshHelper.EnableForwarding(interfaceIdx); err != nil { + return nil, errors.Annotatef(err, "failed to enable forwarding on %s index %s", netInterface.Name, interfaceIdx) + } + log.Infof("Enabled forwarding on %s index %s", netInterface.Name, interfaceIdx) + } + + return n, nil +}