Merge pull request #29666 from vefimova/fix_29270

Automatic merge from submit-queue

Fixed merging of host's and dns' search lines

Fixed forming of pod's Search line in resolv.conf:
 - exclude duplicates while merging of host's and dns' search lines to form pod's one
 - truncate pod's search line if it exceeds resolver limits: is > 255 chars and containes > 6 searches
 - monitoring the resolv.conf file which is used by kubelet (set thru --resolv-conf="") and logging and eventing if search line in it consists of more than 3 entries (or 6 if Cluster Domain is set) or its lenght is > 255 chars
 - logging and eventing when a pod's search line is > 255 chars or containes > 6 searches during forming

Fixes #29270

**Release note**:

```release-note
Fixed forming resolver search line for pods: exclude duplicates, obey libc limitations, logging and eventing appropriately.
```

pkg/kubelet/kubelet.go

@@ -1243,6 +1243,11 @@ func (kl *Kubelet) Run(updates <-chan kubetypes.PodUpdate) {
 	// handled by pod workers).
 	go wait.Until(kl.podKiller, 1*time.Second, wait.NeverStop)
 
+	// Start gorouting responsible for checking limits in resolv.conf
+	if kl.resolverConfig != "" {
+		go wait.Until(func() { kl.checkLimitsForResolvConf() }, 30*time.Second, wait.NeverStop)
+	}
+
 	// Start component sync loops.
 	kl.statusManager.Start()
 	kl.probeManager.Start()
@@ -1299,6 +1304,8 @@ func (kl *Kubelet) GetClusterDNS(pod *v1.Pod) ([]string, []string, error) {
 		if kl.resolverConfig == "" {
 			hostDNS = []string{"127.0.0.1"}
 			hostSearch = []string{"."}
+		} else {
+			hostSearch = kl.formDNSSearchForDNSDefault(hostSearch, pod)
 		}
 		return hostDNS, hostSearch, nil
 	}
@@ -1307,15 +1314,7 @@ func (kl *Kubelet) GetClusterDNS(pod *v1.Pod) ([]string, []string, error) {
 	// the pod. The cluster DNS server itself will forward queries to other nameservers that is configured to use,
 	// in case the cluster DNS server cannot resolve the DNS query itself
 	dns := []string{kl.clusterDNS.String()}
-
+	dnsSearch := kl.formDNSSearch(hostSearch, pod)
-	var dnsSearch []string
-	if kl.clusterDomain != "" {
-		nsSvcDomain := fmt.Sprintf("%s.svc.%s", pod.Namespace, kl.clusterDomain)
-		svcDomain := fmt.Sprintf("svc.%s", kl.clusterDomain)
-		dnsSearch = append([]string{nsSvcDomain, svcDomain, kl.clusterDomain}, hostSearch...)
-	} else {
-		dnsSearch = hostSearch
-	}
 	return dns, dnsSearch, nil
 }
 

pkg/kubelet/kubelet_network.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"os"
 	"strings"
 
 	"github.com/golang/glog"
@@ -89,6 +90,120 @@ func (kl *Kubelet) providerRequiresNetworkingConfiguration() bool {
 	return supported
 }
 
+func omitDuplicates(kl *Kubelet, pod *v1.Pod, combinedSearch []string) []string {
+	uniqueDomains := map[string]bool{}
+
+	for _, dnsDomain := range combinedSearch {
+		if _, exists := uniqueDomains[dnsDomain]; !exists {
+			combinedSearch[len(uniqueDomains)] = dnsDomain
+			uniqueDomains[dnsDomain] = true
+		} else {
+			log := fmt.Sprintf("Found and omitted duplicated dns domain in host search line: '%s' during merging with cluster dns domains", dnsDomain)
+			kl.recorder.Event(pod, v1.EventTypeWarning, "DNSSearchForming", log)
+			glog.Error(log)
+		}
+	}
+	return combinedSearch[:len(uniqueDomains)]
+}
+
+func formDNSSearchFitsLimits(kl *Kubelet, pod *v1.Pod, composedSearch []string) []string {
+	// resolver file Search line current limitations
+	resolvSearchLineDNSDomainsLimit := 6
+	resolvSearchLineLenLimit := 255
+	limitsExceeded := false
+
+	if len(composedSearch) > resolvSearchLineDNSDomainsLimit {
+		composedSearch = composedSearch[:resolvSearchLineDNSDomainsLimit]
+		limitsExceeded = true
+	}
+
+	if resolvSearchhLineStrLen := len(strings.Join(composedSearch, " ")); resolvSearchhLineStrLen > resolvSearchLineLenLimit {
+		cutDomainsNum := 0
+		cutDoaminsLen := 0
+		for i := len(composedSearch) - 1; i >= 0; i-- {
+			cutDoaminsLen += len(composedSearch[i]) + 1
+			cutDomainsNum++
+
+			if (resolvSearchhLineStrLen - cutDoaminsLen) <= resolvSearchLineLenLimit {
+				break
+			}
+		}
+
+		composedSearch = composedSearch[:(len(composedSearch) - cutDomainsNum)]
+		limitsExceeded = true
+	}
+
+	if limitsExceeded {
+		log := fmt.Sprintf("Search Line limits were exceeded, some dns names have been omitted, the applied search line is: %s", strings.Join(composedSearch, " "))
+		kl.recorder.Event(pod, v1.EventTypeWarning, "DNSSearchForming", log)
+		glog.Error(log)
+	}
+	return composedSearch
+}
+
+func (kl *Kubelet) formDNSSearchForDNSDefault(hostSearch []string, pod *v1.Pod) []string {
+	return formDNSSearchFitsLimits(kl, pod, hostSearch)
+}
+
+func (kl *Kubelet) formDNSSearch(hostSearch []string, pod *v1.Pod) []string {
+	if kl.clusterDomain == "" {
+		formDNSSearchFitsLimits(kl, pod, hostSearch)
+		return hostSearch
+	}
+
+	nsSvcDomain := fmt.Sprintf("%s.svc.%s", pod.Namespace, kl.clusterDomain)
+	svcDomain := fmt.Sprintf("svc.%s", kl.clusterDomain)
+	dnsSearch := []string{nsSvcDomain, svcDomain, kl.clusterDomain}
+
+	combinedSearch := append(dnsSearch, hostSearch...)
+
+	combinedSearch = omitDuplicates(kl, pod, combinedSearch)
+	return formDNSSearchFitsLimits(kl, pod, combinedSearch)
+}
+
+func (kl *Kubelet) checkLimitsForResolvConf() {
+	// resolver file Search line current limitations
+	resolvSearchLineDNSDomainsLimit := 6
+	resolvSearchLineLenLimit := 255
+
+	f, err := os.Open(kl.resolverConfig)
+	if err != nil {
+		kl.recorder.Event(kl.nodeRef, v1.EventTypeWarning, "checkLimitsForResolvConf", err.Error())
+		glog.Error("checkLimitsForResolvConf: " + err.Error())
+		return
+	}
+	defer f.Close()
+
+	_, hostSearch, err := kl.parseResolvConf(f)
+	if err != nil {
+		kl.recorder.Event(kl.nodeRef, v1.EventTypeWarning, "checkLimitsForResolvConf", err.Error())
+		glog.Error("checkLimitsForResolvConf: " + err.Error())
+		return
+	}
+
+	domainCntLimit := resolvSearchLineDNSDomainsLimit
+
+	if kl.clusterDomain != "" {
+		domainCntLimit -= 3
+	}
+
+	if len(hostSearch) > domainCntLimit {
+		log := fmt.Sprintf("Resolv.conf file '%s' contains search line consisting of more than %d domains!", kl.resolverConfig, domainCntLimit)
+		kl.recorder.Event(kl.nodeRef, v1.EventTypeWarning, "checkLimitsForResolvConf", log)
+		glog.Error("checkLimitsForResolvConf: " + log)
+		return
+	}
+
+	if len(strings.Join(hostSearch, " ")) > resolvSearchLineLenLimit {
+		log := fmt.Sprintf("Resolv.conf file '%s' contains search line which length is more than allowed %d chars!", kl.resolverConfig, resolvSearchLineLenLimit)
+		kl.recorder.Event(kl.nodeRef, v1.EventTypeWarning, "checkLimitsForResolvConf", log)
+		glog.Error("checkLimitsForResolvConf: " + log)
+		return
+	}
+
+	return
+}
+
 // parseResolveConf reads a resolv.conf file from the given reader, and parses
 // it into nameservers and searches, possibly returning an error.
 // TODO: move to utility package

pkg/kubelet/kubelet_network_test.go

@@ -17,12 +17,14 @@ limitations under the License.
 package kubelet
 
 import (
+	"fmt"
 	"net"
 	"reflect"
 	"strings"
 	"testing"
 
 	"k8s.io/kubernetes/pkg/api/v1"
+	"k8s.io/kubernetes/pkg/client/record"
 	"k8s.io/kubernetes/pkg/util/bandwidth"
 )
 
@@ -112,6 +114,84 @@ func TestParseResolvConf(t *testing.T) {
 	}
 }
 
+func TestComposeDNSSearch(t *testing.T) {
+	testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */)
+	kubelet := testKubelet.kubelet
+
+	recorder := record.NewFakeRecorder(20)
+	kubelet.recorder = recorder
+
+	pod := podWithUidNameNs("", "test_pod", "testNS")
+	kubelet.clusterDomain = "TEST"
+
+	testCases := []struct {
+		dnsNames     []string
+		hostNames    []string
+		resultSearch []string
+		events       []string
+	}{
+		{
+			[]string{"testNS.svc.TEST", "svc.TEST", "TEST"},
+			[]string{},
+			[]string{"testNS.svc.TEST", "svc.TEST", "TEST"},
+			[]string{},
+		},
+
+		{
+			[]string{"testNS.svc.TEST", "svc.TEST", "TEST"},
+			[]string{"AAA", "svc.TEST", "BBB", "TEST"},
+			[]string{"testNS.svc.TEST", "svc.TEST", "TEST", "AAA", "BBB"},
+			[]string{
+				"Found and omitted duplicated dns domain in host search line: 'svc.TEST' during merging with cluster dns domains",
+				"Found and omitted duplicated dns domain in host search line: 'TEST' during merging with cluster dns domains",
+			},
+		},
+
+		{
+			[]string{"testNS.svc.TEST", "svc.TEST", "TEST"},
+			[]string{"AAA", strings.Repeat("B", 256), "BBB"},
+			[]string{"testNS.svc.TEST", "svc.TEST", "TEST", "AAA"},
+			[]string{"Search Line limits were exceeded, some dns names have been omitted, the applied search line is: testNS.svc.TEST svc.TEST TEST AAA"},
+		},
+
+		{
+			[]string{"testNS.svc.TEST", "svc.TEST", "TEST"},
+			[]string{"AAA", "TEST", "BBB", "TEST", "CCC", "DDD"},
+			[]string{"testNS.svc.TEST", "svc.TEST", "TEST", "AAA", "BBB", "CCC"},
+			[]string{
+				"Found and omitted duplicated dns domain in host search line: 'TEST' during merging with cluster dns domains",
+				"Found and omitted duplicated dns domain in host search line: 'TEST' during merging with cluster dns domains",
+				"Search Line limits were exceeded, some dns names have been omitted, the applied search line is: testNS.svc.TEST svc.TEST TEST AAA BBB CCC",
+			},
+		},
+	}
+
+	fetchEvent := func(recorder *record.FakeRecorder) string {
+		select {
+		case event := <-recorder.Events:
+			return event
+		default:
+			return "No more events!"
+		}
+	}
+
+	for i, tc := range testCases {
+		dnsSearch := kubelet.formDNSSearch(tc.hostNames, pod)
+
+		if !reflect.DeepEqual(dnsSearch, tc.resultSearch) {
+			t.Errorf("[%d] expected search line %#v, got %#v", i, tc.resultSearch, dnsSearch)
+		}
+
+		for _, expectedEvent := range tc.events {
+			expected := fmt.Sprintf("%s %s %s", v1.EventTypeWarning, "DNSSearchForming", expectedEvent)
+			event := fetchEvent(recorder)
+			if event != expected {
+				t.Errorf("[%d] expected event '%s', got '%s", i, expected, event)
+			}
+		}
+	}
+}
+
 func TestCleanupBandwidthLimits(t *testing.T) {
 	testPod := func(name, ingress string) *v1.Pod {
 		pod := podWithUidNameNs("", name, "")

pkg/kubelet/kubelet_pods_test.go

@@ -211,7 +211,11 @@ func TestGenerateRunContainerOptions_DNSConfigurationParams(t *testing.T) {
 	} else if options[0].DNS[0] != clusterNS {
 		t.Errorf("expected nameserver %s, got %v", clusterNS, options[0].DNS[0])
 	}
-	if len(options[0].DNSSearch) != len(options[1].DNSSearch)+3 {
+	expLength := len(options[1].DNSSearch) + 3
+	if expLength > 6 {
+		expLength = 6
+	}
+	if len(options[0].DNSSearch) != expLength {
 		t.Errorf("expected prepend of cluster domain, got %+v", options[0].DNSSearch)
 	} else if options[0].DNSSearch[0] != ".svc."+kubelet.clusterDomain {
 		t.Errorf("expected domain %s, got %s", ".svc."+kubelet.clusterDomain, options[0].DNSSearch)