Merge pull request #74825 from ksubrmnn/preserve_dip

Windows support for preserving the destination IP as the VIP when loadbalancing with DSR

go.mod

@@ -14,7 +14,7 @@ require (
 	github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab
 	github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd
 	github.com/Microsoft/go-winio v0.4.11
-	github.com/Microsoft/hcsshim v0.0.0-20190110205307-69ac8d3f7fc1
+	github.com/Microsoft/hcsshim v0.8.6
 	github.com/PuerkitoBio/purell v1.1.0
 	github.com/Rican7/retry v0.1.0 // indirect
 	github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e
@@ -207,7 +207,7 @@ replace (
 	github.com/JeffAshton/win_pdh => github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab
 	github.com/MakeNowJust/heredoc => github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd
 	github.com/Microsoft/go-winio => github.com/Microsoft/go-winio v0.4.11
-	github.com/Microsoft/hcsshim => github.com/Microsoft/hcsshim v0.0.0-20190110205307-69ac8d3f7fc1
+	github.com/Microsoft/hcsshim => github.com/Microsoft/hcsshim v0.8.6
 	github.com/NYTimes/gziphandler => github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46
 	github.com/PuerkitoBio/purell => github.com/PuerkitoBio/purell v1.1.0
 	github.com/PuerkitoBio/urlesc => github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578

go.sum

@@ -19,8 +19,8 @@ github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd h1:sjQovDkwrZp
 github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd/go.mod h1:64YHyfSL2R96J44Nlwm39UHepQbyR5q10x7iYa1ks2E=
 github.com/Microsoft/go-winio v0.4.11 h1:zoIOcVf0xPN1tnMVbTtEdI+P8OofVk3NObnwOQ6nK2Q=
 github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
-github.com/Microsoft/hcsshim v0.0.0-20190110205307-69ac8d3f7fc1 h1:QOfYwlBe/tzoRD0V12RMxgNooJcv4aQX3S6M/N5wOzg=
+github.com/Microsoft/hcsshim v0.8.6 h1:ZfF0+zZeYdzMIVMZHKtDKJvLHj76XCuVae/jNkjj0IA=
-github.com/Microsoft/hcsshim v0.0.0-20190110205307-69ac8d3f7fc1/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg=
+github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg=
 github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46 h1:lsxEuwrXEAokXB9qhlbKWPpo3KMLZQ5WB5WLQRW1uq0=
 github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
 github.com/PuerkitoBio/purell v1.1.0 h1:rmGxhojJlM0tuKtfdvliR84CFHljx9ag64t2xmVkjK4=

pkg/proxy/winkernel/hnsV1.go

@@ -33,7 +33,7 @@ type HostNetworkService interface {
 	getEndpointByIpAddress(ip string, networkName string) (*endpointsInfo, error)
 	createEndpoint(ep *endpointsInfo, networkName string) (*endpointsInfo, error)
 	deleteEndpoint(hnsID string) error
-	getLoadBalancer(endpoints []endpointsInfo, isILB bool, isDSR bool, sourceVip string, vip string, protocol uint16, internalPort uint16, externalPort uint16) (*loadBalancerInfo, error)
+	getLoadBalancer(endpoints []endpointsInfo, flags loadBalancerFlags, sourceVip string, vip string, protocol uint16, internalPort uint16, externalPort uint16) (*loadBalancerInfo, error)
 	deleteLoadBalancer(hnsID string) error
 }
 
@@ -148,13 +148,13 @@ func (hns hnsV1) deleteEndpoint(hnsID string) error {
 	return err
 }
 
-func (hns hnsV1) getLoadBalancer(endpoints []endpointsInfo, isILB bool, isDSR bool, sourceVip string, vip string, protocol uint16, internalPort uint16, externalPort uint16) (*loadBalancerInfo, error) {
+func (hns hnsV1) getLoadBalancer(endpoints []endpointsInfo, flags loadBalancerFlags, sourceVip string, vip string, protocol uint16, internalPort uint16, externalPort uint16) (*loadBalancerInfo, error) {
 	plists, err := hcsshim.HNSListPolicyListRequest()
 	if err != nil {
 		return nil, err
 	}
 
-	if isDSR {
+	if flags.isDSR {
 		klog.V(3).Info("DSR is not supported in V1. Using non DSR instead")
 	}
 
@@ -167,7 +167,7 @@ func (hns hnsV1) getLoadBalancer(endpoints []endpointsInfo, isILB bool, isDSR bo
 		if err = json.Unmarshal(plist.Policies[0], &elbPolicy); err != nil {
 			continue
 		}
-		if elbPolicy.Protocol == protocol && elbPolicy.InternalPort == internalPort && elbPolicy.ExternalPort == externalPort && elbPolicy.ILB == isILB {
+		if elbPolicy.Protocol == protocol && elbPolicy.InternalPort == internalPort && elbPolicy.ExternalPort == externalPort && elbPolicy.ILB == flags.isILB {
 			if len(vip) > 0 {
 				if len(elbPolicy.VIPs) == 0 || elbPolicy.VIPs[0] != vip {
 					continue
@@ -190,7 +190,7 @@ func (hns hnsV1) getLoadBalancer(endpoints []endpointsInfo, isILB bool, isDSR bo
 	}
 	lb, err := hcsshim.AddLoadBalancer(
 		hnsEndpoints,
-		isILB,
+		flags.isILB,
 		sourceVip,
 		vip,
 		protocol,

pkg/proxy/winkernel/hnsV2.go

@@ -169,7 +169,7 @@ func (hns hnsV2) deleteEndpoint(hnsID string) error {
 	}
 	return err
 }
-func (hns hnsV2) getLoadBalancer(endpoints []endpointsInfo, isILB bool, isDSR bool, sourceVip string, vip string, protocol uint16, internalPort uint16, externalPort uint16) (*loadBalancerInfo, error) {
+func (hns hnsV2) getLoadBalancer(endpoints []endpointsInfo, flags loadBalancerFlags, sourceVip string, vip string, protocol uint16, internalPort uint16, externalPort uint16) (*loadBalancerInfo, error) {
 	plists, err := hcn.ListLoadBalancers()
 	if err != nil {
 		return nil, err
@@ -181,7 +181,7 @@ func (hns hnsV2) getLoadBalancer(endpoints []endpointsInfo, isILB bool, isDSR bo
 		}
 		// Validate if input meets any of the policy lists
 		lbPortMapping := plist.PortMappings[0]
-		if lbPortMapping.Protocol == uint32(protocol) && lbPortMapping.InternalPort == internalPort && lbPortMapping.ExternalPort == externalPort && (lbPortMapping.Flags&1 != 0) == isILB {
+		if lbPortMapping.Protocol == uint32(protocol) && lbPortMapping.InternalPort == internalPort && lbPortMapping.ExternalPort == externalPort && (lbPortMapping.Flags&1 != 0) == flags.isILB {
 			if len(vip) > 0 {
 				if len(plist.FrontendVIPs) == 0 || plist.FrontendVIPs[0] != vip {
 					continue
@@ -207,10 +207,30 @@ func (hns hnsV2) getLoadBalancer(endpoints []endpointsInfo, isILB bool, isDSR bo
 	if len(vip) > 0 {
 		vips = append(vips, vip)
 	}
+
+	lbPortMappingFlags := hcn.LoadBalancerPortMappingFlagsNone
+	if flags.isILB {
+		lbPortMappingFlags |= hcn.LoadBalancerPortMappingFlagsILB
+	}
+	if flags.useMUX {
+		lbPortMappingFlags |= hcn.LoadBalancerPortMappingFlagsUseMux
+	}
+	if flags.preserveDIP {
+		lbPortMappingFlags |= hcn.LoadBalancerPortMappingFlagsPreserveDIP
+	}
+	if flags.localRoutedVIP {
+		lbPortMappingFlags |= hcn.LoadBalancerPortMappingFlagsLocalRoutedVIP
+	}
+
+	lbFlags := hcn.LoadBalancerFlagsNone
+	if flags.isDSR {
+		lbFlags |= hcn.LoadBalancerFlagsDSR
+	}
+
 	lb, err := hcn.AddLoadBalancer(
 		hnsEndpoints,
-		isILB,
+		lbFlags,
-		isDSR,
+		lbPortMappingFlags,
 		sourceVip,
 		vips,
 		protocol,

pkg/proxy/winkernel/hns_test.go

@@ -354,8 +354,8 @@ func testGetLoadBalancerExisting(t *testing.T, hns HostNetworkService) {
 	Endpoints := []hcn.HostComputeEndpoint{*Endpoint}
 	LoadBalancer, err := hcn.AddLoadBalancer(
 		Endpoints,
-		false,
+		hcn.LoadBalancerFlagsNone,
-		false,
+		hcn.LoadBalancerPortMappingFlagsNone,
 		sourceVip,
 		[]string{serviceVip},
 		protocol,
@@ -371,7 +371,7 @@ func testGetLoadBalancerExisting(t *testing.T, hns HostNetworkService) {
 		hnsID: Endpoint.Id,
 	}
 	endpoints := []endpointsInfo{*endpoint}
-	lb, err := hns.getLoadBalancer(endpoints, false, false, sourceVip, serviceVip, protocol, internalPort, externalPort)
+	lb, err := hns.getLoadBalancer(endpoints, loadBalancerFlags{}, sourceVip, serviceVip, protocol, internalPort, externalPort)
 	if err != nil {
 		t.Error(err)
 	}
@@ -419,7 +419,7 @@ func testGetLoadBalancerNew(t *testing.T, hns HostNetworkService) {
 		hnsID: Endpoint.Id,
 	}
 	endpoints := []endpointsInfo{*endpoint}
-	lb, err := hns.getLoadBalancer(endpoints, false, false, sourceVip, serviceVip, protocol, internalPort, externalPort)
+	lb, err := hns.getLoadBalancer(endpoints, loadBalancerFlags{}, sourceVip, serviceVip, protocol, internalPort, externalPort)
 	if err != nil {
 		t.Error(err)
 	}
@@ -469,8 +469,8 @@ func testDeleteLoadBalancer(t *testing.T, hns HostNetworkService) {
 	Endpoints := []hcn.HostComputeEndpoint{*Endpoint}
 	LoadBalancer, err := hcn.AddLoadBalancer(
 		Endpoints,
-		false,
+		hcn.LoadBalancerFlagsNone,
-		false,
+		hcn.LoadBalancerPortMappingFlagsNone,
 		sourceVip,
 		[]string{serviceVip},
 		protocol,

pkg/proxy/winkernel/proxier.go

@@ -91,6 +91,14 @@ type loadBalancerInfo struct {
 	hnsID string
 }
 
+type loadBalancerFlags struct {
+	isILB          bool
+	isDSR          bool
+	localRoutedVIP bool
+	useMUX         bool
+	preserveDIP    bool
+}
+
 // internal struct for string service information
 type serviceInfo struct {
 	clusterIP                net.IP
@@ -111,6 +119,7 @@ type serviceInfo struct {
 	policyApplied            bool
 	remoteEndpoint           *endpointsInfo
 	hns                      HostNetworkService
+	preserveDIP              bool
 }
 
 type hnsNetworkInfo struct {
@@ -204,6 +213,14 @@ func newServiceInfo(svcPortName proxy.ServicePortName, port *v1.ServicePort, ser
 	if service.Spec.SessionAffinity == v1.ServiceAffinityClientIP && service.Spec.SessionAffinityConfig != nil {
 		stickyMaxAgeSeconds = int(*service.Spec.SessionAffinityConfig.ClientIP.TimeoutSeconds)
 	}
+
+	klog.Infof("Service %q preserve-destination: %v", svcPortName.NamespacedName.String(), service.Annotations["preserve-destination"])
+
+	preserveDIP := service.Annotations["preserve-destination"] == "true"
+	err := hcn.DSRSupported()
+	if err != nil {
+		preserveDIP = false
+	}
 	info := &serviceInfo{
 		clusterIP: net.ParseIP(service.Spec.ClusterIP),
 		port:      int(port.Port),
@@ -219,6 +236,7 @@ func newServiceInfo(svcPortName proxy.ServicePortName, port *v1.ServicePort, ser
 		loadBalancerSourceRanges: make([]string, len(service.Spec.LoadBalancerSourceRanges)),
 		onlyNodeLocalEndpoints:   onlyNodeLocalEndpoints,
 		hns:                      hns,
+		preserveDIP:              preserveDIP,
 	}
 
 	copy(info.loadBalancerSourceRanges, service.Spec.LoadBalancerSourceRanges)
@@ -513,7 +531,7 @@ func NewProxier(
 	var hns HostNetworkService
 	hns = hnsV1{}
 	supportedFeatures := hcn.GetSupportedFeatures()
-	if supportedFeatures.RemoteSubnet {
+	if supportedFeatures.Api.V2 {
 		hns = hnsV2{}
 	}
 
@@ -999,6 +1017,7 @@ func (proxier *Proxier) syncProxyRules() {
 		}
 
 		var hnsEndpoints []endpointsInfo
+		var hnsLocalEndpoints []endpointsInfo
 		klog.V(4).Infof("====Applying Policy for %s====", svcName)
 		// Create Remote endpoints for every endpoint, corresponding to the service
 		containsPublicIP := false
@@ -1087,6 +1106,9 @@ func (proxier *Proxier) syncProxyRules() {
 			// Save the hnsId for reference
 			LogJson(newHnsEndpoint, "Hns Endpoint resource", 1)
 			hnsEndpoints = append(hnsEndpoints, *newHnsEndpoint)
+			if newHnsEndpoint.isLocal {
+				hnsLocalEndpoints = append(hnsLocalEndpoints, *newHnsEndpoint)
+			}
 			ep.hnsID = newHnsEndpoint.hnsID
 			ep.refCount++
 			Log(ep, "Endpoint resource found", 3)
@@ -1112,8 +1134,7 @@ func (proxier *Proxier) syncProxyRules() {
 		}
 		hnsLoadBalancer, err := hns.getLoadBalancer(
 			hnsEndpoints,
-			false,
+			loadBalancerFlags{isDSR: proxier.isDSR},
-			proxier.isDSR,
 			sourceVip,
 			svcInfo.clusterIP.String(),
 			Enum(svcInfo.protocol),
@@ -1132,8 +1153,7 @@ func (proxier *Proxier) syncProxyRules() {
 		if svcInfo.nodePort > 0 {
 			hnsLoadBalancer, err := hns.getLoadBalancer(
 				hnsEndpoints,
-				false,
+				loadBalancerFlags{localRoutedVIP: true},
-				false,
 				sourceVip,
 				"",
 				Enum(svcInfo.protocol),
@@ -1154,8 +1174,7 @@ func (proxier *Proxier) syncProxyRules() {
 			// Try loading existing policies, if already available
 			hnsLoadBalancer, err = hns.getLoadBalancer(
 				hnsEndpoints,
-				false,
+				loadBalancerFlags{},
-				false,
 				sourceVip,
 				externalIP.ip,
 				Enum(svcInfo.protocol),
@@ -1172,10 +1191,13 @@ func (proxier *Proxier) syncProxyRules() {
 		// Create a Load Balancer Policy for each loadbalancer ingress
 		for _, lbIngressIP := range svcInfo.loadBalancerIngressIPs {
 			// Try loading existing policies, if already available
+			lbIngressEndpoints := hnsEndpoints
+			if svcInfo.preserveDIP {
+				lbIngressEndpoints = hnsLocalEndpoints
+			}
 			hnsLoadBalancer, err := hns.getLoadBalancer(
-				hnsEndpoints,
+				lbIngressEndpoints,
-				false,
+				loadBalancerFlags{isDSR: svcInfo.preserveDIP || proxier.isDSR, useMUX: svcInfo.preserveDIP, preserveDIP: svcInfo.preserveDIP},
-				false,
 				sourceVip,
 				lbIngressIP.ip,
 				Enum(svcInfo.protocol),

pkg/proxy/winkernel/proxier_test.go

@@ -110,7 +110,7 @@ func (hns fakeHNS) createEndpoint(ep *endpointsInfo, networkName string) (*endpo
 func (hns fakeHNS) deleteEndpoint(hnsID string) error {
 	return nil
 }
-func (hns fakeHNS) getLoadBalancer(endpoints []endpointsInfo, isILB bool, isDSR bool, sourceVip string, vip string, protocol uint16, internalPort uint16, externalPort uint16) (*loadBalancerInfo, error) {
+func (hns fakeHNS) getLoadBalancer(endpoints []endpointsInfo, flags loadBalancerFlags, sourceVip string, vip string, protocol uint16, internalPort uint16, externalPort uint16) (*loadBalancerInfo, error) {
 	return &loadBalancerInfo{
 		hnsID: guid,
 	}, nil

vendor/github.com/Microsoft/hcsshim/appveyor.yml

@@ -6,13 +6,14 @@ clone_folder: c:\gopath\src\github.com\Microsoft\hcsshim
 
 environment:
   GOPATH: c:\gopath
-  PATH: C:\mingw-w64\x86_64-7.2.0-posix-seh-rt_v5-rev1\mingw64\bin;%GOPATH%\bin;%PATH%
+  PATH: C:\mingw-w64\x86_64-7.2.0-posix-seh-rt_v5-rev1\mingw64\bin;%GOPATH%\bin;C:\gometalinter-2.0.12-windows-amd64;%PATH%
+
+stack: go 1.11
 
 build_script:
-  - go get -u github.com/alecthomas/gometalinter
+  - appveyor DownloadFile https://github.com/alecthomas/gometalinter/releases/download/v2.0.12/gometalinter-2.0.12-windows-amd64.zip
-  - gometalinter.exe --install
+  - 7z x gometalinter-2.0.12-windows-amd64.zip -y -oC:\ > NUL
   - gometalinter.exe --config .gometalinter.json ./...
-  - go get -v -d -t -tags "functional integration admin" ./...
   - go build ./cmd/wclayer
   - go build ./cmd/runhcs
   - go build ./cmd/tar2ext4

vendor/github.com/Microsoft/hcsshim/hcn/hcn.go

@@ -143,6 +143,15 @@ func RemoteSubnetSupported() error {
 	return platformDoesNotSupportError("Remote Subnet")
 }
 
+// HostRouteSupported returns an error if the HCN version does not support Host Route policies.
+func HostRouteSupported() error {
+	supported := GetSupportedFeatures()
+	if supported.HostRoute {
+		return nil
+	}
+	return platformDoesNotSupportError("Host Route")
+}
+
 // DSRSupported returns an error if the HCN version does not support Direct Server Return.
 func DSRSupported() error {
 	supported := GetSupportedFeatures()

vendor/github.com/Microsoft/hcsshim/hcn/hcnglobals.go

@@ -27,6 +27,8 @@ var (
 	V2ApiSupport = Version{Major: 9, Minor: 1}
 	// Remote Subnet allows for Remote Subnet policies on Overlay networks
 	RemoteSubnetVersion = Version{Major: 9, Minor: 2}
+	// A Host Route policy allows for local container to local host communication Overlay networks
+	HostRouteVersion = Version{Major: 9, Minor: 2}
 	// HNS 10.2 allows for Direct Server Return for loadbalancing
 	DSRVersion = Version{Major: 10, Minor: 2}
 )

vendor/github.com/Microsoft/hcsshim/hcn/hcnloadbalancer.go

@@ -10,10 +10,10 @@ import (
 
 // LoadBalancerPortMapping is associated with HostComputeLoadBalancer
 type LoadBalancerPortMapping struct {
-	Protocol     uint32 `json:",omitempty"` // EX: TCP = 6, UDP = 17
+	Protocol     uint32                       `json:",omitempty"` // EX: TCP = 6, UDP = 17
-	InternalPort uint16 `json:",omitempty"`
+	InternalPort uint16                       `json:",omitempty"`
-	ExternalPort uint16 `json:",omitempty"`
+	ExternalPort uint16                       `json:",omitempty"`
-	Flags        uint32 `json:",omitempty"` // 0: None, 1: EnableILB, 2: LocalRoutedVip
+	Flags        LoadBalancerPortMappingFlags `json:",omitempty"`
 }
 
 // HostComputeLoadBalancer represents software load balancer.
@@ -24,9 +24,35 @@ type HostComputeLoadBalancer struct {
 	FrontendVIPs         []string                  `json:",omitempty"`
 	PortMappings         []LoadBalancerPortMapping `json:",omitempty"`
 	SchemaVersion        SchemaVersion             `json:",omitempty"`
-	Flags                uint32                    `json:",omitempty"` // 0: None, 1: EnableDirectServerReturn
+	Flags                LoadBalancerFlags         `json:",omitempty"` // 0: None, 1: EnableDirectServerReturn
 }
 
+//LoadBalancerFlags modify settings for a loadbalancer.
+type LoadBalancerFlags uint32
+
+var (
+	// LoadBalancerFlagsNone is the default.
+	LoadBalancerFlagsNone LoadBalancerFlags = 0
+	// LoadBalancerFlagsDSR enables Direct Server Return (DSR)
+	LoadBalancerFlagsDSR LoadBalancerFlags = 1
+)
+
+// LoadBalancerPortMappingFlags are special settings on a loadbalancer.
+type LoadBalancerPortMappingFlags uint32
+
+var (
+	// LoadBalancerPortMappingFlagsNone is the default.
+	LoadBalancerPortMappingFlagsNone LoadBalancerPortMappingFlags
+	// LoadBalancerPortMappingFlagsILB enables internal loadbalancing.
+	LoadBalancerPortMappingFlagsILB LoadBalancerPortMappingFlags = 1
+	// LoadBalancerPortMappingFlagsLocalRoutedVIP enables VIP access from the host.
+	LoadBalancerPortMappingFlagsLocalRoutedVIP LoadBalancerPortMappingFlags = 2
+	// LoadBalancerPortMappingFlagsUseMux enables DSR for NodePort access of VIP.
+	LoadBalancerPortMappingFlagsUseMux LoadBalancerPortMappingFlags = 4
+	// LoadBalancerPortMappingFlagsPreserveDIP delivers packets with destination IP as the VIP.
+	LoadBalancerPortMappingFlagsPreserveDIP LoadBalancerPortMappingFlags = 8
+)
+
 func getLoadBalancer(loadBalancerGuid guid.GUID, query string) (*HostComputeLoadBalancer, error) {
 	// Open loadBalancer.
 	var (
@@ -280,20 +306,8 @@ func (loadBalancer *HostComputeLoadBalancer) RemoveEndpoint(endpoint *HostComput
 }
 
 // AddLoadBalancer for the specified endpoints
-func AddLoadBalancer(endpoints []HostComputeEndpoint, isILB bool, isDSR bool, sourceVIP string, frontendVIPs []string, protocol uint16, internalPort uint16, externalPort uint16) (*HostComputeLoadBalancer, error) {
+func AddLoadBalancer(endpoints []HostComputeEndpoint, flags LoadBalancerFlags, portMappingFlags LoadBalancerPortMappingFlags, sourceVIP string, frontendVIPs []string, protocol uint16, internalPort uint16, externalPort uint16) (*HostComputeLoadBalancer, error) {
-	logrus.Debugf("hcn::HostComputeLoadBalancer::AddLoadBalancer endpointId=%v, isILB=%v, sourceVIP=%s, frontendVIPs=%v, protocol=%v, internalPort=%v, externalPort=%v", endpoints, isILB, sourceVIP, frontendVIPs, protocol, internalPort, externalPort)
+	logrus.Debugf("hcn::HostComputeLoadBalancer::AddLoadBalancer endpointId=%v, LoadBalancerFlags=%v, LoadBalancerPortMappingFlags=%v, sourceVIP=%s, frontendVIPs=%v, protocol=%v, internalPort=%v, externalPort=%v", endpoints, flags, portMappingFlags, sourceVIP, frontendVIPs, protocol, internalPort, externalPort)
-
-	var portMappingFlags uint32
-	portMappingFlags = 0
-	if isILB {
-		portMappingFlags = 1
-	}
-
-	var lbFlags uint32
-	lbFlags = 0
-	if isDSR {
-		lbFlags = 1 // EnableDirectServerReturn
-	}
 
 	loadBalancer := &HostComputeLoadBalancer{
 		SourceVIP: sourceVIP,
@@ -310,7 +324,7 @@ func AddLoadBalancer(endpoints []HostComputeEndpoint, isILB bool, isDSR bool, so
 			Major: 2,
 			Minor: 0,
 		},
-		Flags: lbFlags,
+		Flags: flags,
 	}
 
 	for _, endpoint := range endpoints {

vendor/github.com/Microsoft/hcsshim/hcn/hcnnetwork.go

@@ -62,6 +62,15 @@ const (
 	Overlay     NetworkType = "Overlay"
 )
 
+// NetworkFlags are various network flags.
+type NetworkFlags uint32
+
+// NetworkFlags const
+const (
+	None                NetworkFlags = 0
+	EnableNonPersistent NetworkFlags = 8
+)
+
 // HostComputeNetwork represents a network
 type HostComputeNetwork struct {
 	Id            string          `json:"ID,omitempty"`
@@ -71,7 +80,7 @@ type HostComputeNetwork struct {
 	MacPool       MacPool         `json:",omitempty"`
 	Dns           Dns             `json:",omitempty"`
 	Ipams         []Ipam          `json:",omitempty"`
-	Flags         uint32          `json:",omitempty"` // 0: None
+	Flags         NetworkFlags    `json:",omitempty"` // 0: None
 	SchemaVersion SchemaVersion   `json:",omitempty"`
 }
 

vendor/github.com/Microsoft/hcsshim/hcn/hcnpolicy.go

@@ -40,6 +40,7 @@ const (
 	InterfaceConstraint NetworkPolicyType = "InterfaceConstraint"
 	ProviderAddress     NetworkPolicyType = "ProviderAddress"
 	RemoteSubnetRoute   NetworkPolicyType = "RemoteSubnetRoute"
+	HostRoute           NetworkPolicyType = "HostRoute"
 )
 
 // NetworkPolicy is a collection of Policy settings for a Network.

vendor/github.com/Microsoft/hcsshim/hcn/hcnsupport.go

@@ -9,6 +9,7 @@ type SupportedFeatures struct {
 	Acl          AclFeatures `json:"ACL"`
 	Api          ApiSupport  `json:"API"`
 	RemoteSubnet bool        `json:"RemoteSubnet"`
+	HostRoute    bool        `json:"HostRoute"`
 	DSR          bool        `json:"DSR"`
 }
 
@@ -50,6 +51,7 @@ func GetSupportedFeatures() SupportedFeatures {
 	}
 
 	features.RemoteSubnet = isFeatureSupported(globals.Version, RemoteSubnetVersion)
+	features.HostRoute = isFeatureSupported(globals.Version, HostRouteVersion)
 	features.DSR = isFeatureSupported(globals.Version, DSRVersion)
 
 	return features

vendor/github.com/Microsoft/hcsshim/internal/hcs/log.go

@@ -7,9 +7,14 @@ func logOperationBegin(ctx logrus.Fields, msg string) {
 }
 
 func logOperationEnd(ctx logrus.Fields, msg string, err error) {
+	// Copy the log and fields first.
+	log := logrus.WithFields(ctx)
 	if err == nil {
-		logrus.WithFields(ctx).Debug(msg)
+		log.Debug(msg)
 	} else {
-		logrus.WithFields(ctx).WithError(err).Error(msg)
+		// Edit only the copied field data to avoid race conditions on the
+		// write.
+		log.Data[logrus.ErrorKey] = err
+		log.Error(msg)
 	}
 }

vendor/github.com/Microsoft/hcsshim/internal/hcs/process.go

@@ -31,9 +31,8 @@ func newProcess(process hcsProcess, processID int, computeSystem *System) *Proce
 		processID: processID,
 		system:    computeSystem,
 		logctx: logrus.Fields{
-			logfields.HCSOperation: "",
+			logfields.ContainerID: computeSystem.ID(),
-			logfields.ContainerID:  computeSystem.ID(),
+			logfields.ProcessID:   processID,
-			logfields.ProcessID:    processID,
 		},
 	}
 }
@@ -88,13 +87,12 @@ func (process *Process) SystemID() string {
 }
 
 func (process *Process) logOperationBegin(operation string) {
-	process.logctx[logfields.HCSOperation] = operation
 	logOperationBegin(
 		process.logctx,
-		"hcsshim::Process - Begin Operation")
+		operation+" - Begin Operation")
 }
 
-func (process *Process) logOperationEnd(err error) {
+func (process *Process) logOperationEnd(operation string, err error) {
 	var result string
 	if err == nil {
 		result = "Success"
@@ -104,9 +102,8 @@ func (process *Process) logOperationEnd(err error) {
 
 	logOperationEnd(
 		process.logctx,
-		"hcsshim::Process - End Operation - "+result,
+		operation+" - End Operation - "+result,
 		err)
-	process.logctx[logfields.HCSOperation] = ""
 }
 
 // Signal signals the process with `options`.
@@ -116,7 +113,7 @@ func (process *Process) Signal(options guestrequest.SignalProcessOptions) (err e
 
 	operation := "hcsshim::Process::Signal"
 	process.logOperationBegin(operation)
-	defer func() { process.logOperationEnd(err) }()
+	defer func() { process.logOperationEnd(operation, err) }()
 
 	if process.handle == 0 {
 		return makeProcessError(process, operation, ErrAlreadyClosed, nil)
@@ -148,7 +145,7 @@ func (process *Process) Kill() (err error) {
 
 	operation := "hcsshim::Process::Kill"
 	process.logOperationBegin(operation)
-	defer func() { process.logOperationEnd(err) }()
+	defer func() { process.logOperationEnd(operation, err) }()
 
 	if process.handle == 0 {
 		return makeProcessError(process, operation, ErrAlreadyClosed, nil)
@@ -170,7 +167,7 @@ func (process *Process) Kill() (err error) {
 func (process *Process) Wait() (err error) {
 	operation := "hcsshim::Process::Wait"
 	process.logOperationBegin(operation)
-	defer func() { process.logOperationEnd(err) }()
+	defer func() { process.logOperationEnd(operation, err) }()
 
 	err = waitForNotification(process.callbackNumber, hcsNotificationProcessExited, nil)
 	if err != nil {
@@ -185,7 +182,7 @@ func (process *Process) Wait() (err error) {
 func (process *Process) WaitTimeout(timeout time.Duration) (err error) {
 	operation := "hcssshim::Process::WaitTimeout"
 	process.logOperationBegin(operation)
-	defer func() { process.logOperationEnd(err) }()
+	defer func() { process.logOperationEnd(operation, err) }()
 
 	err = waitForNotification(process.callbackNumber, hcsNotificationProcessExited, &timeout)
 	if err != nil {
@@ -202,7 +199,7 @@ func (process *Process) ResizeConsole(width, height uint16) (err error) {
 
 	operation := "hcsshim::Process::ResizeConsole"
 	process.logOperationBegin(operation)
-	defer func() { process.logOperationEnd(err) }()
+	defer func() { process.logOperationEnd(operation, err) }()
 
 	if process.handle == 0 {
 		return makeProcessError(process, operation, ErrAlreadyClosed, nil)
@@ -239,7 +236,7 @@ func (process *Process) Properties() (_ *ProcessStatus, err error) {
 
 	operation := "hcsshim::Process::Properties"
 	process.logOperationBegin(operation)
-	defer func() { process.logOperationEnd(err) }()
+	defer func() { process.logOperationEnd(operation, err) }()
 
 	if process.handle == 0 {
 		return nil, makeProcessError(process, operation, ErrAlreadyClosed, nil)
@@ -275,7 +272,7 @@ func (process *Process) Properties() (_ *ProcessStatus, err error) {
 func (process *Process) ExitCode() (_ int, err error) {
 	operation := "hcsshim::Process::ExitCode"
 	process.logOperationBegin(operation)
-	defer func() { process.logOperationEnd(err) }()
+	defer func() { process.logOperationEnd(operation, err) }()
 
 	properties, err := process.Properties()
 	if err != nil {
@@ -302,7 +299,7 @@ func (process *Process) Stdio() (_ io.WriteCloser, _ io.ReadCloser, _ io.ReadClo
 
 	operation := "hcsshim::Process::Stdio"
 	process.logOperationBegin(operation)
-	defer func() { process.logOperationEnd(err) }()
+	defer func() { process.logOperationEnd(operation, err) }()
 
 	if process.handle == 0 {
 		return nil, nil, nil, makeProcessError(process, operation, ErrAlreadyClosed, nil)
@@ -346,7 +343,7 @@ func (process *Process) CloseStdin() (err error) {
 
 	operation := "hcsshim::Process::CloseStdin"
 	process.logOperationBegin(operation)
-	defer func() { process.logOperationEnd(err) }()
+	defer func() { process.logOperationEnd(operation, err) }()
 
 	if process.handle == 0 {
 		return makeProcessError(process, operation, ErrAlreadyClosed, nil)
@@ -384,7 +381,7 @@ func (process *Process) Close() (err error) {
 
 	operation := "hcsshim::Process::Close"
 	process.logOperationBegin(operation)
-	defer func() { process.logOperationEnd(err) }()
+	defer func() { process.logOperationEnd(operation, err) }()
 
 	// Don't double free this
 	if process.handle == 0 {

vendor/github.com/Microsoft/hcsshim/internal/hcs/system.go

@@ -49,20 +49,18 @@ func newSystem(id string) *System {
 	return &System{
 		id: id,
 		logctx: logrus.Fields{
-			logfields.HCSOperation: "",
+			logfields.ContainerID: id,
-			logfields.ContainerID:  id,
 		},
 	}
 }
 
 func (computeSystem *System) logOperationBegin(operation string) {
-	computeSystem.logctx[logfields.HCSOperation] = operation
 	logOperationBegin(
 		computeSystem.logctx,
-		"hcsshim::ComputeSystem - Begin Operation")
+		operation+" - Begin Operation")
 }
 
-func (computeSystem *System) logOperationEnd(err error) {
+func (computeSystem *System) logOperationEnd(operation string, err error) {
 	var result string
 	if err == nil {
 		result = "Success"
@@ -72,9 +70,8 @@ func (computeSystem *System) logOperationEnd(err error) {
 
 	logOperationEnd(
 		computeSystem.logctx,
-		"hcsshim::ComputeSystem - End Operation - "+result,
+		operation+" - End Operation - "+result,
 		err)
-	computeSystem.logctx[logfields.HCSOperation] = ""
 }
 
 // CreateComputeSystem creates a new compute system with the given configuration but does not start it.
@@ -83,7 +80,7 @@ func CreateComputeSystem(id string, hcsDocumentInterface interface{}) (_ *System
 
 	computeSystem := newSystem(id)
 	computeSystem.logOperationBegin(operation)
-	defer func() { computeSystem.logOperationEnd(err) }()
+	defer func() { computeSystem.logOperationEnd(operation, err) }()
 
 	hcsDocumentB, err := json.Marshal(hcsDocumentInterface)
 	if err != nil {
@@ -133,7 +130,13 @@ func OpenComputeSystem(id string) (_ *System, err error) {
 
 	computeSystem := newSystem(id)
 	computeSystem.logOperationBegin(operation)
-	defer func() { computeSystem.logOperationEnd(err) }()
+	defer func() {
+		if IsNotExist(err) {
+			computeSystem.logOperationEnd(operation, nil)
+		} else {
+			computeSystem.logOperationEnd(operation, err)
+		}
+	}()
 
 	var (
 		handle  hcsSystem
@@ -157,12 +160,10 @@ func OpenComputeSystem(id string) (_ *System, err error) {
 // GetComputeSystems gets a list of the compute systems on the system that match the query
 func GetComputeSystems(q schema1.ComputeSystemQuery) (_ []schema1.ContainerProperties, err error) {
 	operation := "hcsshim::GetComputeSystems"
-	fields := logrus.Fields{
+	fields := logrus.Fields{}
-		logfields.HCSOperation: operation,
-	}
 	logOperationBegin(
 		fields,
-		"hcsshim::ComputeSystem - Begin Operation")
+		operation+" - Begin Operation")
 
 	defer func() {
 		var result string
@@ -174,7 +175,7 @@ func GetComputeSystems(q schema1.ComputeSystemQuery) (_ []schema1.ContainerPrope
 
 		logOperationEnd(
 			fields,
-			"hcsshim::ComputeSystem - End Operation - "+result,
+			operation+" - End Operation - "+result,
 			err)
 	}()
 
@@ -221,7 +222,7 @@ func (computeSystem *System) Start() (err error) {
 
 	operation := "hcsshim::ComputeSystem::Start"
 	computeSystem.logOperationBegin(operation)
-	defer func() { computeSystem.logOperationEnd(err) }()
+	defer func() { computeSystem.logOperationEnd(operation, err) }()
 
 	if computeSystem.handle == 0 {
 		return makeSystemError(computeSystem, "Start", "", ErrAlreadyClosed, nil)
@@ -278,7 +279,13 @@ func (computeSystem *System) Shutdown() (err error) {
 
 	operation := "hcsshim::ComputeSystem::Shutdown"
 	computeSystem.logOperationBegin(operation)
-	defer func() { computeSystem.logOperationEnd(err) }()
+	defer func() {
+		if IsAlreadyStopped(err) {
+			computeSystem.logOperationEnd(operation, nil)
+		} else {
+			computeSystem.logOperationEnd(operation, err)
+		}
+	}()
 
 	if computeSystem.handle == 0 {
 		return makeSystemError(computeSystem, "Shutdown", "", ErrAlreadyClosed, nil)
@@ -304,7 +311,13 @@ func (computeSystem *System) Terminate() (err error) {
 
 	operation := "hcsshim::ComputeSystem::Terminate"
 	computeSystem.logOperationBegin(operation)
-	defer func() { computeSystem.logOperationEnd(err) }()
+	defer func() {
+		if IsPending(err) {
+			computeSystem.logOperationEnd(operation, nil)
+		} else {
+			computeSystem.logOperationEnd(operation, err)
+		}
+	}()
 
 	if computeSystem.handle == 0 {
 		return makeSystemError(computeSystem, "Terminate", "", ErrAlreadyClosed, nil)
@@ -326,7 +339,7 @@ func (computeSystem *System) Terminate() (err error) {
 func (computeSystem *System) Wait() (err error) {
 	operation := "hcsshim::ComputeSystem::Wait"
 	computeSystem.logOperationBegin(operation)
-	defer func() { computeSystem.logOperationEnd(err) }()
+	defer func() { computeSystem.logOperationEnd(operation, err) }()
 
 	err = waitForNotification(computeSystem.callbackNumber, hcsNotificationSystemExited, nil)
 	if err != nil {
@@ -341,10 +354,10 @@ func (computeSystem *System) Wait() (err error) {
 func (computeSystem *System) WaitExpectedError(expected error) (err error) {
 	operation := "hcsshim::ComputeSystem::WaitExpectedError"
 	computeSystem.logOperationBegin(operation)
-	defer func() { computeSystem.logOperationEnd(err) }()
+	defer func() { computeSystem.logOperationEnd(operation, err) }()
 
 	err = waitForNotification(computeSystem.callbackNumber, hcsNotificationSystemExited, nil)
-	if err != nil && err != expected {
+	if err != nil && getInnerError(err) != expected {
 		return makeSystemError(computeSystem, "WaitExpectedError", "", err, nil)
 	}
 
@@ -356,7 +369,7 @@ func (computeSystem *System) WaitExpectedError(expected error) (err error) {
 func (computeSystem *System) WaitTimeout(timeout time.Duration) (err error) {
 	operation := "hcsshim::ComputeSystem::WaitTimeout"
 	computeSystem.logOperationBegin(operation)
-	defer func() { computeSystem.logOperationEnd(err) }()
+	defer func() { computeSystem.logOperationEnd(operation, err) }()
 
 	err = waitForNotification(computeSystem.callbackNumber, hcsNotificationSystemExited, &timeout)
 	if err != nil {
@@ -372,7 +385,7 @@ func (computeSystem *System) Properties(types ...schema1.PropertyType) (_ *schem
 
 	operation := "hcsshim::ComputeSystem::Properties"
 	computeSystem.logOperationBegin(operation)
-	defer func() { computeSystem.logOperationEnd(err) }()
+	defer func() { computeSystem.logOperationEnd(operation, err) }()
 
 	queryj, err := json.Marshal(schema1.PropertyQuery{types})
 	if err != nil {
@@ -411,7 +424,7 @@ func (computeSystem *System) Pause() (err error) {
 
 	operation := "hcsshim::ComputeSystem::Pause"
 	computeSystem.logOperationBegin(operation)
-	defer func() { computeSystem.logOperationEnd(err) }()
+	defer func() { computeSystem.logOperationEnd(operation, err) }()
 
 	if computeSystem.handle == 0 {
 		return makeSystemError(computeSystem, "Pause", "", ErrAlreadyClosed, nil)
@@ -436,7 +449,7 @@ func (computeSystem *System) Resume() (err error) {
 
 	operation := "hcsshim::ComputeSystem::Resume"
 	computeSystem.logOperationBegin(operation)
-	defer func() { computeSystem.logOperationEnd(err) }()
+	defer func() { computeSystem.logOperationEnd(operation, err) }()
 
 	if computeSystem.handle == 0 {
 		return makeSystemError(computeSystem, "Resume", "", ErrAlreadyClosed, nil)
@@ -461,7 +474,7 @@ func (computeSystem *System) CreateProcess(c interface{}) (_ *Process, err error
 
 	operation := "hcsshim::ComputeSystem::CreateProcess"
 	computeSystem.logOperationBegin(operation)
-	defer func() { computeSystem.logOperationEnd(err) }()
+	defer func() { computeSystem.logOperationEnd(operation, err) }()
 
 	var (
 		processInfo   hcsProcessInformation
@@ -521,7 +534,7 @@ func (computeSystem *System) OpenProcess(pid int) (_ *Process, err error) {
 
 	operation := "hcsshim::ComputeSystem::OpenProcess"
 	computeSystem.logOperationBegin(operation)
-	defer func() { computeSystem.logOperationEnd(err) }()
+	defer func() { computeSystem.logOperationEnd(operation, err) }()
 
 	var (
 		processHandle hcsProcess
@@ -555,7 +568,7 @@ func (computeSystem *System) Close() (err error) {
 
 	operation := "hcsshim::ComputeSystem::Close"
 	computeSystem.logOperationBegin(operation)
-	defer func() { computeSystem.logOperationEnd(err) }()
+	defer func() { computeSystem.logOperationEnd(operation, err) }()
 
 	// Don't double free this
 	if computeSystem.handle == 0 {
@@ -642,7 +655,7 @@ func (computeSystem *System) Modify(config interface{}) (err error) {
 
 	operation := "hcsshim::ComputeSystem::Modify"
 	computeSystem.logOperationBegin(operation)
-	defer func() { computeSystem.logOperationEnd(err) }()
+	defer func() { computeSystem.logOperationEnd(operation, err) }()
 
 	if computeSystem.handle == 0 {
 		return makeSystemError(computeSystem, "Modify", "", ErrAlreadyClosed, nil)

vendor/github.com/Microsoft/hcsshim/internal/logfields/fields.go

@@ -26,11 +26,6 @@ const (
 	Uint32       = "uint32"
 	Uint64       = "uint64"
 
-	// HCS
-
-	HCSOperation       = "hcs-op"
-	HCSOperationResult = "hcs-op-result"
-
 	// runhcs
 
 	VMShimOperation = "vmshim-op"

vendor/github.com/Microsoft/hcsshim/internal/safefile/safeopen.go

@@ -87,7 +87,7 @@ func OpenRoot(path string) (*os.File, error) {
 
 func ntRelativePath(path string) ([]uint16, error) {
 	path = filepath.Clean(path)
-	if strings.Contains(":", path) {
+	if strings.Contains(path, ":") {
 		// Since alternate data streams must follow the file they
 		// are attached to, finding one here (out of order) is invalid.
 		return nil, errors.New("path contains invalid character `:`")

vendor/github.com/Microsoft/hcsshim/internal/schema2/plan9_share.go

@@ -20,6 +20,13 @@ type Plan9Share struct {
 
 	Port int32 `json:"Port,omitempty"`
 
+	// Flags are marked private. Until they are exported correctly
+	//
+	// ReadOnly      0x00000001
+	// LinuxMetadata 0x00000004
+	// CaseSensitive 0x00000008
+	Flags int32 `json:"Flags,omitempty"`
+
 	ReadOnly bool `json:"ReadOnly,omitempty"`
 
 	UseShareRootIdentity bool `json:"UseShareRootIdentity,omitempty"`

vendor/github.com/Microsoft/hcsshim/vendor.conf

@@ -0,0 +1,21 @@
+github.com/blang/semver v3.1.0
+github.com/containerd/console c12b1e7919c14469339a5d38f2f8ed9b64a9de23
+github.com/containerd/go-runc 5a6d9f37cfa36b15efba46dc7ea349fa9b7143c3
+github.com/hashicorp/errwrap 7554cd9344cec97297fa6649b055a8c98c2a1e55
+github.com/hashicorp/go-multierror ed905158d87462226a13fe39ddf685ea65f1c11f
+github.com/konsorten/go-windows-terminal-sequences v1.0.1
+github.com/linuxkit/virtsock 8e79449dea0735c1c056d814934dd035734cc97c
+github.com/Microsoft/go-winio 16cfc975803886a5e47c4257a24c8d8c52e178b2
+github.com/Microsoft/opengcs v0.3.9
+github.com/opencontainers/runtime-spec eba862dc2470385a233c7507392675cbeadf7353
+github.com/opencontainers/runtime-tools 1d69bd0f9c39677d0630e50664fbc3154ae61b88
+github.com/pkg/errors v0.8.1
+github.com/sirupsen/logrus v1.3.0
+github.com/syndtr/gocapability db04d3cc01c8b54962a58ec7e491717d06cfcc16
+github.com/urfave/cli 7bc6a0acffa589f415f88aca16cc1de5ffd66f9c
+github.com/xeipuuv/gojsonpointer 4e3ac2762d5f479393488629ee9370b50873b3a6
+github.com/xeipuuv/gojsonreference bd5ef7bd5415a7ac448318e64f11a24cd21e594b
+github.com/xeipuuv/gojsonschema 1d523034197ff1f222f6429836dd36a2457a1874
+golang.org/x/crypto ff983b9c42bc9fbf91556e191cc8efb585c16908
+golang.org/x/sync 37e7f081c4d4c64e13b10787722085407fe5d15f
+golang.org/x/sys e5ecc2a6747ce8d4af18ed98b3de5ae30eb3a5bb

vendor/modules.txt

@@ -32,7 +32,7 @@ github.com/JeffAshton/win_pdh
 github.com/MakeNowJust/heredoc
 # github.com/Microsoft/go-winio v0.4.11 => github.com/Microsoft/go-winio v0.4.11
 github.com/Microsoft/go-winio
-# github.com/Microsoft/hcsshim v0.0.0-20190110205307-69ac8d3f7fc1 => github.com/Microsoft/hcsshim v0.0.0-20190110205307-69ac8d3f7fc1
+# github.com/Microsoft/hcsshim v0.8.6 => github.com/Microsoft/hcsshim v0.8.6
 github.com/Microsoft/hcsshim
 github.com/Microsoft/hcsshim/hcn
 github.com/Microsoft/hcsshim/internal/cni