github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

refactor nsenter to new pkg/util

pull/6/head
Di Xu 2017-09-01 09:53:16 +08:00
parent 4f2d7b93da
commit 46b0b3491f
4 changed files with 203 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
pkg/util/mount/mount_linux.go
View File

@ -350,9 +350,7 @@ func parseProcMounts(content []byte) ([]MountPoint, error) {
}
func (mounter *Mounter) MakeRShared(path string) error {
mountCmd := defaultMountCommand
mountArgs := []string{}
return doMakeRShared(path, procMountInfoPath, mountCmd, mountArgs)
return doMakeRShared(path, procMountInfoPath)
}
// formatAndMount uses unix utils to format and mount the given disk
@ -523,7 +521,7 @@ func parseMountInfo(filename string) ([]mountInfo, error) {
// path is shared and bind-mounts it as rshared if needed. mountCmd and
// mountArgs are expected to contain mount-like command, doMakeRShared will add
// '--bind <path> <path>' and '--make-rshared <path>' to mountArgs.
func doMakeRShared(path string, mountInfoFilename string, mountCmd string, mountArgs []string) error {
func doMakeRShared(path string, mountInfoFilename string) error {
shared, err := isShared(path, mountInfoFilename)
if err != nil {
return err

136
pkg/util/mount/nsenter_mount.go
View File

@ -25,78 +25,30 @@ import (
"strings"
"github.com/golang/glog"
"k8s.io/utils/exec"
"k8s.io/kubernetes/pkg/util/nsenter"
)
// NsenterMounter is part of experimental support for running the kubelet
// in a container. Currently, all docker containers receive their own mount
// namespaces. NsenterMounter works by executing nsenter to run commands in
const (
// hostProcMountsPath is the default mount path for rootfs
hostProcMountsPath = "/rootfs/proc/1/mounts"
// hostProcMountinfoPath is the default mount info path for rootfs
hostProcMountinfoPath = "/rootfs/proc/1/mountinfo"
)
// Currently, all docker containers receive their own mount namespaces.
// NsenterMounter works by executing nsenter to run commands in
// the host's mount namespace.
//
// NsenterMounter requires:
//
// 1. Docker >= 1.6 due to the dependency on the slave propagation mode
// of the bind-mount of the kubelet root directory in the container.
// Docker 1.5 used a private propagation mode for bind-mounts, so mounts
// performed in the host's mount namespace do not propagate out to the
// bind-mount in this docker version.
// 2. The host's root filesystem must be available at /rootfs
// 3. The nsenter binary must be on the Kubelet process' PATH in the container's
// filesystem.
// 4. The Kubelet process must have CAP_SYS_ADMIN (required by nsenter); at
// the present, this effectively means that the kubelet is running in a
// privileged container.
// 5. The volume path used by the Kubelet must be the same inside and outside
// the container and be writable by the container (to initialize volume)
// contents. TODO: remove this requirement.
// 6. The host image must have mount, findmnt, and umount binaries in /bin,
// /usr/sbin, or /usr/bin
// 7. The host image should have systemd-run in /bin, /usr/sbin, or /usr/bin
// For more information about mount propagation modes, see:
// https://www.kernel.org/doc/Documentation/filesystems/sharedsubtree.txt
type NsenterMounter struct {
// a map of commands to their paths on the host filesystem
paths map[string]string
ne *nsenter.Nsenter
}
func NewNsenterMounter() *NsenterMounter {
m := &NsenterMounter{
paths: map[string]string{
"mount": "",
"findmnt": "",
"umount": "",
"systemd-run": "",
},
}
// search for the mount command in other locations besides /usr/bin
for binary := range m.paths {
// default to root
m.paths[binary] = filepath.Join("/", binary)
for _, path := range []string{"/bin", "/usr/sbin", "/usr/bin"} {
binPath := filepath.Join(path, binary)
if _, err := os.Stat(filepath.Join(hostRootFsPath, binPath)); err != nil {
continue
}
m.paths[binary] = binPath
break
}
// TODO: error, so that the kubelet can stop if the mounts don't exist
// (don't forget that systemd-run is optional)
}
return m
return &NsenterMounter{ne: nsenter.NewNsenter()}
}
// NsenterMounter implements mount.Interface
var _ = Interface(&NsenterMounter{})
const (
hostRootFsPath = "/rootfs"
hostProcMountsPath = "/rootfs/proc/1/mounts"
hostProcMountinfoPath = "/rootfs/proc/1/mountinfo"
hostMountNamespacePath = "/rootfs/proc/1/ns/mnt"
nsenterPath = "nsenter"
)
// Mount runs mount(8) in the host's root mount namespace. Aside from this
// aspect, Mount has the same semantics as the mounter returned by mount.New()
func (n *NsenterMounter) Mount(source string, target string, fstype string, options []string) error {
@ -116,26 +68,22 @@ func (n *NsenterMounter) Mount(source string, target string, fstype string, opti
// doNsenterMount nsenters the host's mount namespace and performs the
// requested mount.
func (n *NsenterMounter) doNsenterMount(source, target, fstype string, options []string) error {
glog.V(5).Infof("nsenter Mounting %s %s %s %v", source, target, fstype, options)
args := n.makeNsenterArgs(source, target, fstype, options)
glog.V(5).Infof("Mount command: %v %v", nsenterPath, args)
exec := exec.New()
outputBytes, err := exec.Command(nsenterPath, args...).CombinedOutput()
glog.V(5).Infof("nsenter mount %s %s %s %v", source, target, fstype, options)
cmd, args := n.makeNsenterArgs(source, target, fstype, options)
outputBytes, err := n.ne.Exec(cmd, args).CombinedOutput()
if len(outputBytes) != 0 {
glog.V(5).Infof("Output of mounting %s to %s: %v", source, target, string(outputBytes))
}
return err
}
// makeNsenterArgs makes a list of argument to nsenter in order to do the
// requested mount.
func (n *NsenterMounter) makeNsenterArgs(source, target, fstype string, options []string) []string {
mountCmd := n.absHostPath("mount")
func (n *NsenterMounter) makeNsenterArgs(source, target, fstype string, options []string) (string, []string) {
mountCmd := n.ne.AbsHostPath("mount")
mountArgs := makeMountArgs(source, target, fstype, options)
if systemdRunPath, hasSystemd := n.paths["systemd-run"]; hasSystemd {
if systemdRunPath, hasSystemd := n.ne.SupportsSystemd(); hasSystemd {
// Complete command line:
// nsenter --mount=/rootfs/proc/1/ns/mnt -- /bin/systemd-run --description=... --scope -- /bin/mount -t <type> <what> <where>
// Expected flow is:
@ -165,34 +113,20 @@ func (n *NsenterMounter) makeNsenterArgs(source, target, fstype string, options
// No code here, mountCmd and mountArgs use /bin/mount
}
nsenterArgs := []string{
"--mount=" + hostMountNamespacePath,
"--",
mountCmd,
}
nsenterArgs = append(nsenterArgs, mountArgs...)
return nsenterArgs
return mountCmd, mountArgs
}
// Unmount runs umount(8) in the host's mount namespace.
func (n *NsenterMounter) Unmount(target string) error {
args := []string{
"--mount=" + hostMountNamespacePath,
"--",
n.absHostPath("umount"),
target,
}
args := []string{target}
// No need to execute systemd-run here, it's enough that unmount is executed
// in the host's mount namespace. It will finish appropriate fuse daemon(s)
// running in any scope.
glog.V(5).Infof("Unmount command: %v %v", nsenterPath, args)
exec := exec.New()
outputBytes, err := exec.Command(nsenterPath, args...).CombinedOutput()
glog.V(5).Infof("nsenter unmount args: %v", args)
outputBytes, err := n.ne.Exec("umount", args).CombinedOutput()
if len(outputBytes) != 0 {
glog.V(5).Infof("Output of unmounting %s: %v", target, string(outputBytes))
}
return err
}
@ -207,7 +141,7 @@ func (m *NsenterMounter) IsNotMountPoint(dir string) (bool, error) {
func (*NsenterMounter) IsMountPointMatch(mp MountPoint, dir string) bool {
deletedDir := fmt.Sprintf("%s\\040(deleted)", dir)
return ((mp.Path == dir) || (mp.Path == deletedDir))
return (mp.Path == dir) || (mp.Path == deletedDir)
}
// IsLikelyNotMountPoint determines whether a path is a mountpoint by calling findmnt
@ -227,11 +161,9 @@ func (n *NsenterMounter) IsLikelyNotMountPoint(file string) (bool, error) {
// the first of multiple possible mountpoints using --first-only.
// Also add fstype output to make sure that the output of target file will give the full path
// TODO: Need more refactoring for this function. Track the solution with issue #26996
args := []string{"--mount=" + hostMountNamespacePath, "--", n.absHostPath("findmnt"), "-o", "target,fstype", "--noheadings", "--first-only", "--target", file}
glog.V(5).Infof("findmnt command: %v %v", nsenterPath, args)
exec := exec.New()
out, err := exec.Command(nsenterPath, args...).CombinedOutput()
args := []string{"-o", "target,fstype", "--noheadings", "--first-only", "--target", file}
glog.V(5).Infof("nsenter findmnt args: %v", args)
out, err := n.ne.Exec("findmnt", args).CombinedOutput()
if err != nil {
glog.V(2).Infof("Failed findmnt command for path %s: %v", file, err)
// Different operating systems behave differently for paths which are not mount points.
@ -285,20 +217,6 @@ func (n *NsenterMounter) GetDeviceNameFromMount(mountPath, pluginDir string) (st
return getDeviceNameFromMount(n, mountPath, pluginDir)
}
func (n *NsenterMounter) absHostPath(command string) string {
path, ok := n.paths[command]
if !ok {
return command
}
return path
}
func (n *NsenterMounter) MakeRShared(path string) error {
nsenterCmd := nsenterPath
nsenterArgs := []string{
"--mount=" + hostMountNamespacePath,
"--",
n.absHostPath("mount"),
}
return doMakeRShared(path, hostProcMountinfoPath, nsenterCmd, nsenterArgs)
return doMakeRShared(path, hostProcMountinfoPath)
}

124
pkg/util/nsenter/nsenter.go Normal file
View File

@ -0,0 +1,124 @@
// +build linux
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package nsenter
import (
"fmt"
"os"
"path/filepath"
"k8s.io/utils/exec"
"github.com/golang/glog"
)
const (
hostRootFsPath = "/rootfs"
// hostProcMountNsPath is the default mount namespace for rootfs
hostProcMountNsPath = "/rootfs/proc/1/ns/mnt"
// nsenterPath is the default nsenter command
nsenterPath = "nsenter"
)
// Nsenter is part of experimental support for running the kubelet
// in a container.
//
// Nsenter requires:
//
// 1. Docker >= 1.6 due to the dependency on the slave propagation mode
// of the bind-mount of the kubelet root directory in the container.
// Docker 1.5 used a private propagation mode for bind-mounts, so mounts
// performed in the host's mount namespace do not propagate out to the
// bind-mount in this docker version.
// 2. The host's root filesystem must be available at /rootfs
// 3. The nsenter binary must be on the Kubelet process' PATH in the container's
// filesystem.
// 4. The Kubelet process must have CAP_SYS_ADMIN (required by nsenter); at
// the present, this effectively means that the kubelet is running in a
// privileged container.
// 5. The volume path used by the Kubelet must be the same inside and outside
// the container and be writable by the container (to initialize volume)
// contents. TODO: remove this requirement.
// 6. The host image must have "mount", "findmnt", "umount", "stat", "touch",
// "mkdir", "ls", "sh" and "chmod" binaries in /bin, /usr/sbin, or /usr/bin
// 7. The host image should have systemd-run in /bin, /usr/sbin, or /usr/bin
// For more information about mount propagation modes, see:
// https://www.kernel.org/doc/Documentation/filesystems/sharedsubtree.txt
type Nsenter struct {
// a map of commands to their paths on the host filesystem
paths map[string]string
}
// NewNsenter constructs a new instance of Nsenter
func NewNsenter() *Nsenter {
ne := &Nsenter{
paths: map[string]string{
"mount": "",
"findmnt": "",
"umount": "",
"systemd-run": "",
"stat": "",
"touch": "",
"mkdir": "",
"ls": "",
"sh": "",
"chmod": "",
},
}
// search for the required commands in other locations besides /usr/bin
for binary := range ne.paths {
// default to root
ne.paths[binary] = filepath.Join("/", binary)
for _, path := range []string{"/bin", "/usr/sbin", "/usr/bin"} {
binPath := filepath.Join(path, binary)
if _, err := os.Stat(filepath.Join(hostRootFsPath, binPath)); err != nil {
continue
}
ne.paths[binary] = binPath
break
}
// TODO: error, so that the kubelet can stop if the paths don't exist
// (don't forget that systemd-run is optional)
}
return ne
}
// Exec executes nsenter commands in hostProcMountNsPath mount namespace
func (ne *Nsenter) Exec(cmd string, args []string) exec.Cmd {
fullArgs := append([]string{fmt.Sprintf("--mount=%s", hostProcMountNsPath), "--"},
append([]string{ne.AbsHostPath(cmd)}, args...)...)
glog.V(5).Infof("Running nsenter command: %v %v", nsenterPath, fullArgs)
exec := exec.New()
return exec.Command(nsenterPath, fullArgs...)
}
// AbsHostPath returns the absolute runnable path for a specified command
func (ne *Nsenter) AbsHostPath(command string) string {
path, ok := ne.paths[command]
if !ok {
return command
}
return path
}
// SupportsSystemd checks whether command systemd-run exists
func (ne *Nsenter) SupportsSystemd() (string, bool) {
systemdRunPath, hasSystemd := ne.paths["systemd-run"]
return systemdRunPath, hasSystemd
}

50
pkg/util/nsenter/nsenter_unsupported.go Normal file
View File

@ -0,0 +1,50 @@
// +build !linux
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package nsenter
import (
"k8s.io/utils/exec"
)
// Nsenter is part of experimental support for running the kubelet
// in a container.
type Nsenter struct {
// a map of commands to their paths on the host filesystem
Paths map[string]string
}
// NewNsenter constructs a new instance of Nsenter
func NewNsenter() *Nsenter {
return &Nsenter{}
}
// Exec executes nsenter commands in hostProcMountNsPath mount namespace
func (ne *Nsenter) Exec(args ...string) exec.Cmd {
return nil
}
// AbsHostPath returns the absolute runnable path for a specified command
func (ne *Nsenter) AbsHostPath(command string) string {
return ""
}
// SupportsSystemd checks whether command systemd-run exists
func (ne *Nsenter) SupportsSystemd() (string, bool) {
return "", false
}