Merge pull request #54773 from phsiao/dns_options_handling

Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Support copying "options" in resolv.conf into pod sandbox when dnsPolicy is Default

**What this PR does / why we need it**:

This PR adds support for copying "options" from host's /etc/resolv.conf (or --resolv-conf) into pod's resolv.conf when dnsPolicy is Default.  Being able to customize options is important because it is common to leverage options to fine-tune the behavior of DNS client.

**Which issue this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close that issue when PR gets merged)*: fixes #42542

**Special notes for your reviewer**:

I originally wanted to also tackle the issue of copying options for when dnsPolicy is ClusterFirst, but with ability to "merge" with default options (ndots:5 more specifically) when it makes sense.  I decided to leave it off for now because the "merging" may need more discussions.   Happy to add that to this PR or create another PR for that if it makes sense and is clear what should be done.   I think even when dnsPolicy is ClusterFirst it is important to allow customization.

**Release note**:

```kubelet: add support for copying "options" from /etc/resolv.conf (or --resolv-conf if it is used) into pod's /etc/resolv.conf when dnsPolicy is Default.```

pkg/kubelet/container/helpers.go

@@ -47,7 +47,7 @@ type HandlerRunner interface {
 // able to get necessary informations like the RunContainerOptions, DNS settings, Host IP.
 type RuntimeHelper interface {
 	GenerateRunContainerOptions(pod *v1.Pod, container *v1.Container, podIP string) (contOpts *RunContainerOptions, err error)
-	GetClusterDNS(pod *v1.Pod) (dnsServers []string, dnsSearches []string, useClusterFirstPolicy bool, err error)
+	GetClusterDNS(pod *v1.Pod) (dnsServers []string, dnsSearches []string, dnsOptions []string, useClusterFirstPolicy bool, err error)
 	// GetPodCgroupParent returns the CgroupName identifer, and its literal cgroupfs form on the host
 	// of a pod.
 	GetPodCgroupParent(pod *v1.Pod) string

pkg/kubelet/container/testing/fake_runtime_helper.go

@@ -44,8 +44,8 @@ func (f *FakeRuntimeHelper) GetPodCgroupParent(pod *v1.Pod) string {
 	return ""
 }
 
-func (f *FakeRuntimeHelper) GetClusterDNS(pod *v1.Pod) ([]string, []string, bool, error) {
+func (f *FakeRuntimeHelper) GetClusterDNS(pod *v1.Pod) ([]string, []string, []string, bool, error) {
-	return f.DNSServers, f.DNSSearches, false, f.Err
+	return f.DNSServers, f.DNSSearches, nil, false, f.Err
 }
 
 // This is not used by docker runtime.

pkg/kubelet/kubelet.go

@@ -1401,21 +1401,21 @@ func (kl *Kubelet) GetKubeClient() clientset.Interface {
 	return kl.kubeClient
 }
 
-// GetClusterDNS returns a list of the DNS servers and a list of the DNS search
+// GetClusterDNS returns a list of the DNS servers, a list of the DNS search
-// domains of the cluster.
+// domains of the cluster, and a list of resolv.conf options.
-func (kl *Kubelet) GetClusterDNS(pod *v1.Pod) ([]string, []string, bool, error) {
+func (kl *Kubelet) GetClusterDNS(pod *v1.Pod) ([]string, []string, []string, bool, error) {
-	var hostDNS, hostSearch []string
+	var hostDNS, hostSearch, hostOptions []string
 	// Get host DNS settings
 	if kl.resolverConfig != "" {
 		f, err := os.Open(kl.resolverConfig)
 		if err != nil {
-			return nil, nil, false, err
+			return nil, nil, nil, false, err
 		}
 		defer f.Close()
 
-		hostDNS, hostSearch, err = kl.parseResolvConf(f)
+		hostDNS, hostSearch, hostOptions, err = kl.parseResolvConf(f)
 		if err != nil {
-			return nil, nil, false, err
+			return nil, nil, nil, false, err
 		}
 	}
 	useClusterFirstPolicy := ((pod.Spec.DNSPolicy == v1.DNSClusterFirst && !kubecontainer.IsHostNetworkPod(pod)) || pod.Spec.DNSPolicy == v1.DNSClusterFirstWithHostNet)
@@ -1444,7 +1444,7 @@ func (kl *Kubelet) GetClusterDNS(pod *v1.Pod) ([]string, []string, bool, error)
 		} else {
 			hostSearch = kl.formDNSSearchForDNSDefault(hostSearch, pod)
 		}
-		return hostDNS, hostSearch, useClusterFirstPolicy, nil
+		return hostDNS, hostSearch, hostOptions, useClusterFirstPolicy, nil
 	}
 
 	// for a pod with DNSClusterFirst policy, the cluster DNS server is the only nameserver configured for
@@ -1456,7 +1456,7 @@ func (kl *Kubelet) GetClusterDNS(pod *v1.Pod) ([]string, []string, bool, error)
 	}
 	dnsSearch := kl.formDNSSearch(hostSearch, pod)
 
-	return dns, dnsSearch, useClusterFirstPolicy, nil
+	return dns, dnsSearch, hostOptions, useClusterFirstPolicy, nil
 }
 
 // syncPod is the transaction script for the sync of a single pod.
@@ -2237,7 +2237,7 @@ func (kl *Kubelet) setupDNSinContainerizedMounter(mounterPath string) {
 		if err != nil {
 			glog.Error("Could not open resolverConf file")
 		} else {
-			_, hostSearch, err := kl.parseResolvConf(f)
+			_, hostSearch, _, err := kl.parseResolvConf(f)
 			if err != nil {
 				glog.Errorf("Error for parsing the reslov.conf file: %v", err)
 			} else {

pkg/kubelet/kubelet_network.go

@@ -169,7 +169,7 @@ func (kl *Kubelet) checkLimitsForResolvConf() {
 	}
 	defer f.Close()
 
-	_, hostSearch, err := kl.parseResolvConf(f)
+	_, hostSearch, _, err := kl.parseResolvConf(f)
 	if err != nil {
 		kl.recorder.Event(kl.nodeRef, v1.EventTypeWarning, "checkLimitsForResolvConf", err.Error())
 		glog.Error("checkLimitsForResolvConf: " + err.Error())
@@ -200,12 +200,12 @@ func (kl *Kubelet) checkLimitsForResolvConf() {
 }
 
 // parseResolveConf reads a resolv.conf file from the given reader, and parses
-// it into nameservers and searches, possibly returning an error.
+// it into nameservers, searches and options, possibly returning an error.
 // TODO: move to utility package
-func (kl *Kubelet) parseResolvConf(reader io.Reader) (nameservers []string, searches []string, err error) {
+func (kl *Kubelet) parseResolvConf(reader io.Reader) (nameservers []string, searches []string, options []string, err error) {
 	file, err := ioutil.ReadAll(reader)
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, nil, err
 	}
 
 	// Lines of the form "nameserver 1.2.3.4" accumulate.
@@ -214,6 +214,10 @@ func (kl *Kubelet) parseResolvConf(reader io.Reader) (nameservers []string, sear
 	// Lines of the form "search example.com" overrule - last one wins.
 	searches = []string{}
 
+	// Lines of the form "option ndots:5 attempts:2" overrule - last one wins.
+	// Each option is recorded as an element in the array.
+	options = []string{}
+
 	lines := strings.Split(string(file), "\n")
 	for l := range lines {
 		trimmed := strings.TrimSpace(lines[l])
@@ -230,13 +234,16 @@ func (kl *Kubelet) parseResolvConf(reader io.Reader) (nameservers []string, sear
 		if fields[0] == "search" {
 			searches = fields[1:]
 		}
+		if fields[0] == "options" {
+			options = fields[1:]
+		}
 	}
 
 	// There used to be code here to scrub DNS for each cloud, but doesn't
 	// make sense anymore since cloudproviders are being factored out.
 	// contact @thockin or @wlan0 for more information
 
-	return nameservers, searches, nil
+	return nameservers, searches, options, nil
 }
 
 // syncNetworkStatus updates the network state

pkg/kubelet/kubelet_network_test.go

@@ -74,39 +74,44 @@ func TestParseResolvConf(t *testing.T) {
 		data        string
 		nameservers []string
 		searches    []string
+		options     []string
 	}{
-		{"", []string{}, []string{}},
+		{"", []string{}, []string{}, []string{}},
-		{" ", []string{}, []string{}},
+		{" ", []string{}, []string{}, []string{}},
-		{"\n", []string{}, []string{}},
+		{"\n", []string{}, []string{}, []string{}},
-		{"\t\n\t", []string{}, []string{}},
+		{"\t\n\t", []string{}, []string{}, []string{}},
-		{"#comment\n", []string{}, []string{}},
+		{"#comment\n", []string{}, []string{}, []string{}},
-		{" #comment\n", []string{}, []string{}},
+		{" #comment\n", []string{}, []string{}, []string{}},
-		{"#comment\n#comment", []string{}, []string{}},
+		{"#comment\n#comment", []string{}, []string{}, []string{}},
-		{"#comment\nnameserver", []string{}, []string{}},
+		{"#comment\nnameserver", []string{}, []string{}, []string{}},
-		{"#comment\nnameserver\nsearch", []string{}, []string{}},
+		{"#comment\nnameserver\nsearch", []string{}, []string{}, []string{}},
-		{"nameserver 1.2.3.4", []string{"1.2.3.4"}, []string{}},
+		{"nameserver 1.2.3.4", []string{"1.2.3.4"}, []string{}, []string{}},
-		{" nameserver 1.2.3.4", []string{"1.2.3.4"}, []string{}},
+		{" nameserver 1.2.3.4", []string{"1.2.3.4"}, []string{}, []string{}},
-		{"\tnameserver 1.2.3.4", []string{"1.2.3.4"}, []string{}},
+		{"\tnameserver 1.2.3.4", []string{"1.2.3.4"}, []string{}, []string{}},
-		{"nameserver\t1.2.3.4", []string{"1.2.3.4"}, []string{}},
+		{"nameserver\t1.2.3.4", []string{"1.2.3.4"}, []string{}, []string{}},
-		{"nameserver \t 1.2.3.4", []string{"1.2.3.4"}, []string{}},
+		{"nameserver \t 1.2.3.4", []string{"1.2.3.4"}, []string{}, []string{}},
-		{"nameserver 1.2.3.4\nnameserver 5.6.7.8", []string{"1.2.3.4", "5.6.7.8"}, []string{}},
+		{"nameserver 1.2.3.4\nnameserver 5.6.7.8", []string{"1.2.3.4", "5.6.7.8"}, []string{}, []string{}},
-		{"nameserver 1.2.3.4 #comment", []string{"1.2.3.4"}, []string{}},
+		{"nameserver 1.2.3.4 #comment", []string{"1.2.3.4"}, []string{}, []string{}},
-		{"search foo", []string{}, []string{"foo"}},
+		{"search foo", []string{}, []string{"foo"}, []string{}},
-		{"search foo bar", []string{}, []string{"foo", "bar"}},
+		{"search foo bar", []string{}, []string{"foo", "bar"}, []string{}},
-		{"search foo bar bat\n", []string{}, []string{"foo", "bar", "bat"}},
+		{"search foo bar bat\n", []string{}, []string{"foo", "bar", "bat"}, []string{}},
-		{"search foo\nsearch bar", []string{}, []string{"bar"}},
+		{"search foo\nsearch bar", []string{}, []string{"bar"}, []string{}},
-		{"nameserver 1.2.3.4\nsearch foo bar", []string{"1.2.3.4"}, []string{"foo", "bar"}},
+		{"nameserver 1.2.3.4\nsearch foo bar", []string{"1.2.3.4"}, []string{"foo", "bar"}, []string{}},
-		{"nameserver 1.2.3.4\nsearch foo\nnameserver 5.6.7.8\nsearch bar", []string{"1.2.3.4", "5.6.7.8"}, []string{"bar"}},
+		{"nameserver 1.2.3.4\nsearch foo\nnameserver 5.6.7.8\nsearch bar", []string{"1.2.3.4", "5.6.7.8"}, []string{"bar"}, []string{}},
-		{"#comment\nnameserver 1.2.3.4\n#comment\nsearch foo\ncomment", []string{"1.2.3.4"}, []string{"foo"}},
+		{"#comment\nnameserver 1.2.3.4\n#comment\nsearch foo\ncomment", []string{"1.2.3.4"}, []string{"foo"}, []string{}},
+		{"options ndots:5 attempts:2", []string{}, []string{}, []string{"ndots:5", "attempts:2"}},
+		{"options ndots:1\noptions ndots:5 attempts:3", []string{}, []string{}, []string{"ndots:5", "attempts:3"}},
+		{"nameserver 1.2.3.4\nsearch foo\nnameserver 5.6.7.8\nsearch bar\noptions ndots:5 attempts:4", []string{"1.2.3.4", "5.6.7.8"}, []string{"bar"}, []string{"ndots:5", "attempts:4"}},
 	}
 	testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */)
 	defer testKubelet.Cleanup()
 	kubelet := testKubelet.kubelet
 	for i, tc := range testCases {
-		ns, srch, err := kubelet.parseResolvConf(strings.NewReader(tc.data))
+		ns, srch, opts, err := kubelet.parseResolvConf(strings.NewReader(tc.data))
 		require.NoError(t, err)
 		assert.EqualValues(t, tc.nameservers, ns, "test case [%d]: name servers", i)
 		assert.EqualValues(t, tc.searches, srch, "test case [%d] searches", i)
+		assert.EqualValues(t, tc.options, opts, "test case [%d] options", i)
 	}
 }
 

pkg/kubelet/kubelet_test.go

@@ -2194,7 +2194,7 @@ func TestGetClusterDNS(t *testing.T) {
 	}, 4)
 	for i, pod := range pods {
 		var err error
-		options[i].DNS, options[i].DNSSearch, _, err = kubelet.GetClusterDNS(pod)
+		options[i].DNS, options[i].DNSSearch, _, _, err = kubelet.GetClusterDNS(pod)
 		if err != nil {
 			t.Fatalf("failed to generate container options: %v", err)
 		}
@@ -2227,7 +2227,7 @@ func TestGetClusterDNS(t *testing.T) {
 	kubelet.resolverConfig = "/etc/resolv.conf"
 	for i, pod := range pods {
 		var err error
-		options[i].DNS, options[i].DNSSearch, _, err = kubelet.GetClusterDNS(pod)
+		options[i].DNS, options[i].DNSSearch, _, _, err = kubelet.GetClusterDNS(pod)
 		if err != nil {
 			t.Fatalf("failed to generate container options: %v", err)
 		}

pkg/kubelet/kuberuntime/kuberuntime_sandbox.go

@@ -74,14 +74,16 @@ func (m *kubeGenericRuntimeManager) generatePodSandboxConfig(pod *v1.Pod, attemp
 		Annotations: newPodAnnotations(pod),
 	}
 
-	dnsServers, dnsSearches, useClusterFirstPolicy, err := m.runtimeHelper.GetClusterDNS(pod)
+	dnsServers, dnsSearches, dnsOptions, useClusterFirstPolicy, err := m.runtimeHelper.GetClusterDNS(pod)
 	if err != nil {
 		return nil, err
 	}
 	podSandboxConfig.DnsConfig = &runtimeapi.DNSConfig{
 		Servers:  dnsServers,
 		Searches: dnsSearches,
+		Options:  dnsOptions,
 	}
+
 	if useClusterFirstPolicy {
 		podSandboxConfig.DnsConfig.Options = defaultDNSOptions
 	}

pkg/kubelet/rkt/rkt.go

@@ -1041,7 +1041,7 @@ func (r *Runtime) generateRunCommand(pod *v1.Pod, uuid, networkNamespaceID strin
 		}
 	} else {
 		// Setup DNS.
-		dnsServers, dnsSearches, _, err := r.runtimeHelper.GetClusterDNS(pod)
+		dnsServers, dnsSearches, _, _, err := r.runtimeHelper.GetClusterDNS(pod)
 		if err != nil {
 			return "", err
 		}