github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #45199 from yujuhong/clean-up-dockertools 

Automatic merge from submit-queue

Clean up code in dockertools

Move functions and sub packages to dockershim.
Part of #43234.
pull/6/head
Kubernetes Submit Queue 2017-05-02 20:55:59 -07:00 committed by GitHub
parent fc51574bb4 c35c00f3f1
commit 2a87baba6c
19 changed files with 299 additions and 392 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
cmd/kubelet/app/server.go
View File

@ -939,18 +939,6 @@ func RunDockershim(c *componentconfig.KubeletConfiguration, dockershimRootDir st
dockerClient := dockertools.ConnectToDockerOrDie(c.DockerEndpoint, c.RuntimeRequestTimeout.Duration,
c.ImagePullProgressDeadline.Duration)
// Initialize docker exec handler.
var dockerExecHandler dockertools.ExecHandler
switch c.DockerExecHandlerName {
case "native":
dockerExecHandler = &dockertools.NativeExecHandler{}
case "nsenter":
dockerExecHandler = &dockertools.NsenterExecHandler{}
default:
glog.Warningf("Unknown Docker exec handler %q; defaulting to native", c.DockerExecHandlerName)
dockerExecHandler = &dockertools.NativeExecHandler{}
}
// Initialize network plugin settings.
binDir := c.CNIBinDir
if binDir == "" {
@ -976,7 +964,7 @@ func RunDockershim(c *componentconfig.KubeletConfiguration, dockershimRootDir st
}
ds, err := dockershim.NewDockerService(dockerClient, c.SeccompProfileRoot, c.PodInfraContainerImage,
streamingConfig, &pluginSettings, c.RuntimeCgroups, c.CgroupDriver, dockerExecHandler, dockershimRootDir,
streamingConfig, &pluginSettings, c.RuntimeCgroups, c.CgroupDriver, c.DockerExecHandlerName, dockershimRootDir,
!c.DockerEnableSharedPID)
if err != nil {
return err

10
pkg/kubelet/dockershim/BUILD
View File

@ -21,6 +21,7 @@ go_library(
"docker_sandbox.go",
"docker_service.go",
"docker_streaming.go",
"exec.go",
"helpers.go",
"naming.go",
"security_context.go",
@ -37,8 +38,8 @@ go_library(
"//pkg/kubelet/container:go_default_library",
"//pkg/kubelet/dockershim/cm:go_default_library",
"//pkg/kubelet/dockershim/errors:go_default_library",
"//pkg/kubelet/dockershim/securitycontext:go_default_library",
"//pkg/kubelet/dockertools:go_default_library",
"//pkg/kubelet/dockertools/securitycontext:go_default_library",
"//pkg/kubelet/leaky:go_default_library",
"//pkg/kubelet/network:go_default_library",
"//pkg/kubelet/network/cni:go_default_library",
@ -49,7 +50,10 @@ go_library(
"//pkg/kubelet/types:go_default_library",
"//pkg/kubelet/util/cache:go_default_library",
"//pkg/kubelet/util/ioutils:go_default_library",
"//pkg/security/apparmor:go_default_library",
"//pkg/util/exec:go_default_library",
"//pkg/util/hash:go_default_library",
"//pkg/util/term:go_default_library",
"//vendor/github.com/blang/semver:go_default_library",
"//vendor/github.com/docker/engine-api/types:go_default_library",
"//vendor/github.com/docker/engine-api/types/container:go_default_library",
@ -57,6 +61,7 @@ go_library(
"//vendor/github.com/docker/engine-api/types/strslice:go_default_library",
"//vendor/github.com/docker/go-connections/nat:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
@ -86,9 +91,9 @@ go_test(
"//pkg/kubelet/container:go_default_library",
"//pkg/kubelet/container/testing:go_default_library",
"//pkg/kubelet/dockershim/errors:go_default_library",
"//pkg/kubelet/dockershim/securitycontext:go_default_library",
"//pkg/kubelet/dockershim/testing:go_default_library",
"//pkg/kubelet/dockertools:go_default_library",
"//pkg/kubelet/dockertools/securitycontext:go_default_library",
"//pkg/kubelet/network:go_default_library",
"//pkg/kubelet/network/testing:go_default_library",
"//pkg/kubelet/types:go_default_library",
@ -119,6 +124,7 @@ filegroup(
"//pkg/kubelet/dockershim/cm:all-srcs",
"//pkg/kubelet/dockershim/errors:all-srcs",
"//pkg/kubelet/dockershim/remote:all-srcs",
"//pkg/kubelet/dockershim/securitycontext:all-srcs",
"//pkg/kubelet/dockershim/testing:all-srcs",
],
tags = ["automanaged"],

22
pkg/kubelet/dockershim/docker_image.go
View File

@ -17,6 +17,8 @@ limitations under the License.
package dockershim
import (
"fmt"
dockertypes "github.com/docker/engine-api/types"
runtimeapi "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime"
"k8s.io/kubernetes/pkg/kubelet/dockertools"
@ -80,7 +82,7 @@ func (ds *dockerService) PullImage(image *runtimeapi.ImageSpec, auth *runtimeapi
return "", err
}
return dockertools.GetImageRef(ds.client, image.Image)
return getImageRef(ds.client, image.Image)
}
// RemoveImage removes the image.
@ -101,3 +103,21 @@ func (ds *dockerService) RemoveImage(image *runtimeapi.ImageSpec) error {
_, err = ds.client.RemoveImage(image.Image, dockertypes.ImageRemoveOptions{PruneChildren: true})
return err
}
// getImageRef returns the image digest if exists, or else returns the image ID.
func getImageRef(client dockertools.DockerInterface, image string) (string, error) {
img, err := client.InspectImageByRef(image)
if err != nil {
return "", err
}
if img == nil {
return "", fmt.Errorf("unable to inspect image %s", image)
}
// Returns the digest if it exist.
if len(img.RepoDigests) > 0 {
return img.RepoDigests[0], nil
}
return img.ID, nil
}

42
pkg/kubelet/dockershim/docker_service.go
View File

@ -20,12 +20,14 @@ import (
"fmt"
"io"
"net/http"
"strconv"
"time"
"github.com/blang/semver"
dockertypes "github.com/docker/engine-api/types"
"github.com/golang/glog"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/kubernetes/pkg/api/v1"
"k8s.io/kubernetes/pkg/apis/componentconfig"
internalapi "k8s.io/kubernetes/pkg/kubelet/api"
@ -147,12 +149,23 @@ var internalLabelKeys []string = []string{containerTypeLabelKey, containerLogPat
// NOTE: Anything passed to DockerService should be eventually handled in another way when we switch to running the shim as a different process.
func NewDockerService(client dockertools.DockerInterface, seccompProfileRoot string, podSandboxImage string, streamingConfig *streaming.Config,
pluginSettings *NetworkPluginSettings, cgroupsName string, kubeCgroupDriver string, execHandler dockertools.ExecHandler, dockershimRootDir string, disableSharedPID bool) (DockerService, error) {
pluginSettings *NetworkPluginSettings, cgroupsName string, kubeCgroupDriver string, execHandlerName, dockershimRootDir string, disableSharedPID bool) (DockerService, error) {
c := dockertools.NewInstrumentedDockerInterface(client)
checkpointHandler, err := NewPersistentCheckpointHandler(dockershimRootDir)
if err != nil {
return nil, err
}
var execHandler ExecHandler
switch execHandlerName {
case "native":
execHandler = &NativeExecHandler{}
case "nsenter":
execHandler = &NsenterExecHandler{}
default:
glog.Warningf("Unknown Docker exec handler %q; defaulting to native", execHandlerName)
execHandler = &NativeExecHandler{}
}
ds := &dockerService{
seccompProfileRoot: seccompProfileRoot,
client: c,
@ -478,7 +491,32 @@ func (d *dockerLegacyService) GetContainerLogs(pod *v1.Pod, containerID kubecont
if err != nil {
return err
}
return dockertools.GetContainerLogs(d.client, pod, containerID, logOptions, stdout, stderr, container.Config.Tty)
var since int64
if logOptions.SinceSeconds != nil {
t := metav1.Now().Add(-time.Duration(*logOptions.SinceSeconds) * time.Second)
since = t.Unix()
}
if logOptions.SinceTime != nil {
since = logOptions.SinceTime.Unix()
}
opts := dockertypes.ContainerLogsOptions{
ShowStdout: true,
ShowStderr: true,
Since: strconv.FormatInt(since, 10),
Timestamps: logOptions.Timestamps,
Follow: logOptions.Follow,
}
if logOptions.TailLines != nil {
opts.Tail = strconv.FormatInt(*logOptions.TailLines, 10)
}
sopts := dockertools.StreamOptions{
OutputStream: stdout,
ErrorStream: stderr,
RawTerminal: container.Config.Tty,
}
return d.client.Logs(containerID.ID, opts, sopts)
}
// criSupportedLogDrivers are log drivers supported by native CRI integration.

90
pkg/kubelet/dockershim/docker_streaming.go
View File

@ -21,11 +21,15 @@ import (
"fmt"
"io"
"math"
"os/exec"
"strings"
"time"
dockertypes "github.com/docker/engine-api/types"
"github.com/golang/glog"
"k8s.io/kubernetes/pkg/client/unversioned/remotecommand"
runtimeapi "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime"
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
"k8s.io/kubernetes/pkg/kubelet/dockertools"
"k8s.io/kubernetes/pkg/kubelet/server/streaming"
"k8s.io/kubernetes/pkg/kubelet/util/ioutils"
@ -33,7 +37,7 @@ import (
type streamingRuntime struct {
client dockertools.DockerInterface
execHandler dockertools.ExecHandler
execHandler ExecHandler
}
var _ streaming.Runtime = &streamingRuntime{}
@ -57,14 +61,14 @@ func (r *streamingRuntime) Attach(containerID string, in io.Reader, out, errw io
return err
}
return dockertools.AttachContainer(r.client, containerID, in, out, errw, tty, resize)
return attachContainer(r.client, containerID, in, out, errw, tty, resize)
}
func (r *streamingRuntime) PortForward(podSandboxID string, port int32, stream io.ReadWriteCloser) error {
if port < 0 || port > math.MaxUint16 {
return fmt.Errorf("invalid port %d", port)
}
return dockertools.PortForward(r.client, podSandboxID, port, stream)
return portForward(r.client, podSandboxID, port, stream)
}
// ExecSync executes a command in the container, and returns the stdout output.
@ -128,3 +132,83 @@ func checkContainerStatus(client dockertools.DockerInterface, containerID string
}
return container, nil
}
func attachContainer(client dockertools.DockerInterface, containerID string, stdin io.Reader, stdout, stderr io.WriteCloser, tty bool, resize <-chan remotecommand.TerminalSize) error {
// Have to start this before the call to client.AttachToContainer because client.AttachToContainer is a blocking
// call :-( Otherwise, resize events don't get processed and the terminal never resizes.
kubecontainer.HandleResizing(resize, func(size remotecommand.TerminalSize) {
client.ResizeContainerTTY(containerID, int(size.Height), int(size.Width))
})
// TODO(random-liu): Do we really use the *Logs* field here?
opts := dockertypes.ContainerAttachOptions{
Stream: true,
Stdin: stdin != nil,
Stdout: stdout != nil,
Stderr: stderr != nil,
}
sopts := dockertools.StreamOptions{
InputStream: stdin,
OutputStream: stdout,
ErrorStream: stderr,
RawTerminal: tty,
}
return client.AttachToContainer(containerID, opts, sopts)
}
func portForward(client dockertools.DockerInterface, podInfraContainerID string, port int32, stream io.ReadWriteCloser) error {
container, err := client.InspectContainer(podInfraContainerID)
if err != nil {
return err
}
if !container.State.Running {
return fmt.Errorf("container not running (%s)", container.ID)
}
containerPid := container.State.Pid
socatPath, lookupErr := exec.LookPath("socat")
if lookupErr != nil {
return fmt.Errorf("unable to do port forwarding: socat not found.")
}
args := []string{"-t", fmt.Sprintf("%d", containerPid), "-n", socatPath, "-", fmt.Sprintf("TCP4:localhost:%d", port)}
nsenterPath, lookupErr := exec.LookPath("nsenter")
if lookupErr != nil {
return fmt.Errorf("unable to do port forwarding: nsenter not found.")
}
commandString := fmt.Sprintf("%s %s", nsenterPath, strings.Join(args, " "))
glog.V(4).Infof("executing port forwarding command: %s", commandString)
command := exec.Command(nsenterPath, args...)
command.Stdout = stream
stderr := new(bytes.Buffer)
command.Stderr = stderr
// If we use Stdin, command.Run() won't return until the goroutine that's copying
// from stream finishes. Unfortunately, if you have a client like telnet connected
// via port forwarding, as long as the user's telnet client is connected to the user's
// local listener that port forwarding sets up, the telnet session never exits. This
// means that even if socat has finished running, command.Run() won't ever return
// (because the client still has the connection and stream open).
//
// The work around is to use StdinPipe(), as Wait() (called by Run()) closes the pipe
// when the command (socat) exits.
inPipe, err := command.StdinPipe()
if err != nil {
return fmt.Errorf("unable to do port forwarding: error creating stdin pipe: %v", err)
}
go func() {
io.Copy(inPipe, stream)
inPipe.Close()
}()
if err := command.Run(); err != nil {
return fmt.Errorf("%v: %s", err, stderr.String())
}
return nil
}

32
pkg/kubelet/dockertools/exec.go → pkg/kubelet/dockershim/exec.go
View File

@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package dockertools
package dockershim
import (
"fmt"
@ -25,22 +25,44 @@ import (
dockertypes "github.com/docker/engine-api/types"
"github.com/golang/glog"
"k8s.io/kubernetes/pkg/client/unversioned/remotecommand"
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
"k8s.io/kubernetes/pkg/kubelet/dockertools"
utilexec "k8s.io/kubernetes/pkg/util/exec"
"k8s.io/kubernetes/pkg/util/term"
)
// ExecHandler knows how to execute a command in a running Docker container.
type ExecHandler interface {
ExecInContainer(client DockerInterface, container *dockertypes.ContainerJSON, cmd []string, stdin io.Reader, stdout, stderr io.WriteCloser, tty bool, resize <-chan remotecommand.TerminalSize, timeout time.Duration) error
ExecInContainer(client dockertools.DockerInterface, container *dockertypes.ContainerJSON, cmd []string, stdin io.Reader, stdout, stderr io.WriteCloser, tty bool, resize <-chan remotecommand.TerminalSize, timeout time.Duration) error
}
// NsenterExecHandler executes commands in Docker containers using nsenter.
type NsenterExecHandler struct{}
type dockerExitError struct {
Inspect *dockertypes.ContainerExecInspect
}
func (d *dockerExitError) String() string {
return d.Error()
}
func (d *dockerExitError) Error() string {
return fmt.Sprintf("Error executing in Docker Container: %d", d.Inspect.ExitCode)
}
func (d *dockerExitError) Exited() bool {
return !d.Inspect.Running
}
func (d *dockerExitError) ExitStatus() int {
return d.Inspect.ExitCode
}
// TODO should we support nsenter in a container, running with elevated privs and --pid=host?
func (*NsenterExecHandler) ExecInContainer(client DockerInterface, container *dockertypes.ContainerJSON, cmd []string, stdin io.Reader, stdout, stderr io.WriteCloser, tty bool, resize <-chan remotecommand.TerminalSize, timeout time.Duration) error {
func (*NsenterExecHandler) ExecInContainer(client dockertools.DockerInterface, container *dockertypes.ContainerJSON, cmd []string, stdin io.Reader, stdout, stderr io.WriteCloser, tty bool, resize <-chan remotecommand.TerminalSize, timeout time.Duration) error {
nsenter, err := exec.LookPath("nsenter")
if err != nil {
return fmt.Errorf("exec unavailable - unable to locate nsenter")
@ -111,7 +133,7 @@ func (*NsenterExecHandler) ExecInContainer(client DockerInterface, container *do
// NativeExecHandler executes commands in Docker containers using Docker's exec API.
type NativeExecHandler struct{}
func (*NativeExecHandler) ExecInContainer(client DockerInterface, container *dockertypes.ContainerJSON, cmd []string, stdin io.Reader, stdout, stderr io.WriteCloser, tty bool, resize <-chan remotecommand.TerminalSize, timeout time.Duration) error {
func (*NativeExecHandler) ExecInContainer(client dockertools.DockerInterface, container *dockertypes.ContainerJSON, cmd []string, stdin io.Reader, stdout, stderr io.WriteCloser, tty bool, resize <-chan remotecommand.TerminalSize, timeout time.Duration) error {
createOpts := dockertypes.ExecConfig{
Cmd: cmd,
AttachStdin: stdin != nil,
@ -131,7 +153,7 @@ func (*NativeExecHandler) ExecInContainer(client DockerInterface, container *doc
})
startOpts := dockertypes.ExecStartCheck{Detach: false, Tty: tty}
streamOpts := StreamOptions{
streamOpts := dockertools.StreamOptions{
InputStream: stdin,
OutputStream: stdout,
ErrorStream: stderr,

119
pkg/kubelet/dockershim/helpers.go
View File

@ -17,7 +17,12 @@ limitations under the License.
package dockershim
import (
"bytes"
"crypto/md5"
"encoding/json"
"fmt"
"io/ioutil"
"path/filepath"
"regexp"
"strconv"
"strings"
@ -28,14 +33,21 @@ import (
dockernat "github.com/docker/go-connections/nat"
"github.com/golang/glog"
"k8s.io/kubernetes/pkg/api/v1"
v1helper "k8s.io/kubernetes/pkg/api/v1/helper"
runtimeapi "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime"
"k8s.io/kubernetes/pkg/kubelet/dockertools"
"k8s.io/kubernetes/pkg/kubelet/types"
"k8s.io/kubernetes/pkg/security/apparmor"
)
const (
annotationPrefix = "annotation."
// Docker changed the API for specifying options in v1.11
securityOptSeparatorChangeVersion = "1.23.0" // Corresponds to docker 1.11.x
securityOptSeparatorOld = ':'
securityOptSeparatorNew = '='
)
var (
@ -43,7 +55,9 @@ var (
// Docker changes the security option separator from ':' to '=' in the 1.23
// API version.
optsSeparatorChangeVersion = semver.MustParse(dockertools.SecurityOptSeparatorChangeVersion)
optsSeparatorChangeVersion = semver.MustParse(securityOptSeparatorChangeVersion)
defaultSeccompOpt = []dockerOpt{{"seccomp", "unconfined", ""}}
)
// generateEnvList converts KeyValue list to a list of strings, in the form of
@ -181,17 +195,57 @@ func makePortsAndBindings(pm []*runtimeapi.PortMapping) (map[dockernat.Port]stru
return exposedPorts, portBindings
}
func getSeccompDockerOpts(annotations map[string]string, ctrName, profileRoot string) ([]dockerOpt, error) {
profile, profileOK := annotations[v1.SeccompContainerAnnotationKeyPrefix+ctrName]
if !profileOK {
// try the pod profile
profile, profileOK = annotations[v1.SeccompPodAnnotationKey]
if !profileOK {
// return early the default
return defaultSeccompOpt, nil
}
}
if profile == "unconfined" {
// return early the default
return defaultSeccompOpt, nil
}
if profile == "docker/default" {
// return nil so docker will load the default seccomp profile
return nil, nil
}
if !strings.HasPrefix(profile, "localhost/") {
return nil, fmt.Errorf("unknown seccomp profile option: %s", profile)
}
name := strings.TrimPrefix(profile, "localhost/") // by pod annotation validation, name is a valid subpath
fname := filepath.Join(profileRoot, filepath.FromSlash(name))
file, err := ioutil.ReadFile(fname)
if err != nil {
return nil, fmt.Errorf("cannot load seccomp profile %q: %v", name, err)
}
b := bytes.NewBuffer(nil)
if err := json.Compact(b, file); err != nil {
return nil, err
}
// Rather than the full profile, just put the filename & md5sum in the event log.
msg := fmt.Sprintf("%s(md5:%x)", name, md5.Sum(file))
return []dockerOpt{{"seccomp", b.String(), msg}}, nil
}
// getSeccompSecurityOpts gets container seccomp options from container and sandbox
// config, currently from sandbox annotations.
// It is an experimental feature and may be promoted to official runtime api in the future.
func getSeccompSecurityOpts(containerName string, sandboxConfig *runtimeapi.PodSandboxConfig, seccompProfileRoot string, separator rune) ([]string, error) {
seccompOpts, err := dockertools.GetSeccompOpts(sandboxConfig.GetAnnotations(), containerName, seccompProfileRoot)
seccompOpts, err := getSeccompDockerOpts(sandboxConfig.GetAnnotations(), containerName, seccompProfileRoot)
if err != nil {
return nil, err
}
fmtOpts := dockertools.FmtDockerOpts(seccompOpts, separator)
return fmtOpts, nil
return fmtDockerOpts(seccompOpts, separator), nil
}
// getApparmorSecurityOpts gets apparmor options from container config.
@ -200,12 +254,12 @@ func getApparmorSecurityOpts(sc *runtimeapi.LinuxContainerSecurityContext, separ
return nil, nil
}
appArmorOpts, err := dockertools.GetAppArmorOpts(sc.ApparmorProfile)
appArmorOpts, err := getAppArmorOpts(sc.ApparmorProfile)
if err != nil {
return nil, err
}
fmtOpts := dockertools.FmtDockerOpts(appArmorOpts, separator)
fmtOpts := fmtDockerOpts(appArmorOpts, separator)
return fmtOpts, nil
}
@ -258,10 +312,23 @@ func (f *dockerFilter) AddLabel(key, value string) {
f.Add("label", fmt.Sprintf("%s=%s", key, value))
}
// parseUserFromImageUser splits the user out of an user:group string.
func parseUserFromImageUser(id string) string {
if id == "" {
return id
}
// split instances where the id may contain user:group
if strings.Contains(id, ":") {
return strings.Split(id, ":")[0]
}
// no group, just return the id
return id
}
// getUserFromImageUser gets uid or user name of the image user.
// If user is numeric, it will be treated as uid; or else, it is treated as user name.
func getUserFromImageUser(imageUser string) (*int64, string) {
user := dockertools.GetUserFromImageUser(imageUser)
user := parseUserFromImageUser(imageUser)
// return both nil if user is not specified in the image.
if user == "" {
return nil, ""
@ -321,9 +388,9 @@ func getSecurityOptSeparator(v *semver.Version) rune {
case -1:
// Current version is less than the API change version; use the old
// separator.
return dockertools.SecurityOptSeparatorOld
return securityOptSeparatorOld
default:
return dockertools.SecurityOptSeparatorNew
return securityOptSeparatorNew
}
}
@ -342,3 +409,35 @@ func ensureSandboxImageExists(client dockertools.DockerInterface, image string)
}
return nil
}
func getAppArmorOpts(profile string) ([]dockerOpt, error) {
if profile == "" || profile == apparmor.ProfileRuntimeDefault {
// The docker applies the default profile by default.
return nil, nil
}
// Assume validation has already happened.
profileName := strings.TrimPrefix(profile, apparmor.ProfileNamePrefix)
return []dockerOpt{{"apparmor", profileName, ""}}, nil
}
// fmtDockerOpts formats the docker security options using the given separator.
func fmtDockerOpts(opts []dockerOpt, sep rune) []string {
fmtOpts := make([]string, len(opts))
for i, opt := range opts {
fmtOpts[i] = fmt.Sprintf("%s%c%s", opt.key, sep, opt.value)
}
return fmtOpts
}
type dockerOpt struct {
// The key-value pair passed to docker.
key, value string
// The alternative value to use in log/event messages.
msg string
}
// Expose key/value from dockertools
func (d dockerOpt) GetKV() (string, string) {
return d.key, d.value
}

2
pkg/kubelet/dockershim/security_context.go
View File

@ -26,7 +26,7 @@ import (
"k8s.io/kubernetes/pkg/api/v1"
runtimeapi "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime"
"k8s.io/kubernetes/pkg/kubelet/dockertools/securitycontext"
"k8s.io/kubernetes/pkg/kubelet/dockershim/securitycontext"
knetwork "k8s.io/kubernetes/pkg/kubelet/network"
)

2
pkg/kubelet/dockershim/security_context_test.go
View File

@ -26,7 +26,7 @@ import (
"github.com/stretchr/testify/assert"
runtimeapi "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime"
"k8s.io/kubernetes/pkg/kubelet/dockertools/securitycontext"
"k8s.io/kubernetes/pkg/kubelet/dockershim/securitycontext"
)
func TestModifyContainerConfig(t *testing.T) {

0
pkg/kubelet/dockertools/securitycontext/BUILD → pkg/kubelet/dockershim/securitycontext/BUILD
View File

2
pkg/kubelet/dockertools/securitycontext/doc.go → pkg/kubelet/dockershim/securitycontext/doc.go
View File

@ -15,4 +15,4 @@ limitations under the License.
*/
// Package securitycontext contains security context api implementations
package securitycontext // import "k8s.io/kubernetes/pkg/kubelet/dockertools/securitycontext"
package securitycontext // import "k8s.io/kubernetes/pkg/kubelet/dockershim/securitycontext"

0
pkg/kubelet/dockertools/securitycontext/fake.go → pkg/kubelet/dockershim/securitycontext/fake.go
View File

0
pkg/kubelet/dockertools/securitycontext/provider.go → pkg/kubelet/dockershim/securitycontext/provider.go
View File

0
pkg/kubelet/dockertools/securitycontext/provider_test.go → pkg/kubelet/dockershim/securitycontext/provider_test.go
View File

0
pkg/kubelet/dockertools/securitycontext/types.go → pkg/kubelet/dockershim/securitycontext/types.go
View File

0
pkg/kubelet/dockertools/securitycontext/util.go → pkg/kubelet/dockershim/securitycontext/util.go
View File

11
pkg/kubelet/dockertools/BUILD
View File

@ -14,7 +14,6 @@ go_library(
"docker.go",
"docker_manager.go",
"docker_manager_linux.go",
"exec.go",
"fake_docker_client.go",
"instrumented_docker.go",
"kube_docker_client.go",
@ -22,15 +21,11 @@ go_library(
tags = ["automanaged"],
deps = [
"//pkg/api/v1:go_default_library",
"//pkg/client/unversioned/remotecommand:go_default_library",
"//pkg/credentialprovider:go_default_library",
"//pkg/kubelet/container:go_default_library",
"//pkg/kubelet/images:go_default_library",
"//pkg/kubelet/leaky:go_default_library",
"//pkg/kubelet/metrics:go_default_library",
"//pkg/security/apparmor:go_default_library",
"//pkg/util/exec:go_default_library",
"//pkg/util/term:go_default_library",
"//vendor/github.com/docker/distribution/digest:go_default_library",
"//vendor/github.com/docker/distribution/reference:go_default_library",
"//vendor/github.com/docker/docker/pkg/jsonmessage:go_default_library",
@ -40,7 +35,6 @@ go_library(
"//vendor/github.com/docker/engine-api/types/container:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/golang.org/x/net/context:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library",
"//vendor/k8s.io/client-go/util/clock:go_default_library",
@ -82,9 +76,6 @@ filegroup(
filegroup(
name = "all-srcs",
srcs = [
":package-srcs",
"//pkg/kubelet/dockertools/securitycontext:all-srcs",
],
srcs = [":package-srcs"],
tags = ["automanaged"],
)

332
pkg/kubelet/dockertools/docker_manager.go
View File

@ -16,38 +16,8 @@ limitations under the License.
package dockertools
import (
"bytes"
"crypto/md5"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"time"
dockertypes "github.com/docker/engine-api/types"
"github.com/golang/glog"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/kubernetes/pkg/api/v1"
"k8s.io/kubernetes/pkg/client/unversioned/remotecommand"
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
"k8s.io/kubernetes/pkg/security/apparmor"
)
const (
DockerType = "docker"
dockerDefaultLoggingDriver = "json-file"
// Docker changed the API for specifying options in v1.11
SecurityOptSeparatorChangeVersion = "1.23.0" // Corresponds to docker 1.11.x
SecurityOptSeparatorOld = ':'
SecurityOptSeparatorNew = '='
DockerType = "docker"
// https://docs.docker.com/engine/reference/api/docker_remote_api/
// docker version should be at least 1.10.x
@ -56,304 +26,4 @@ const (
statusRunningPrefix = "Up"
statusExitedPrefix = "Exited"
statusCreatedPrefix = "Created"
ndotsDNSOption = "options ndots:5\n"
)
var (
defaultSeccompOpt = []dockerOpt{{"seccomp", "unconfined", ""}}
)
// GetImageRef returns the image digest if exists, or else returns the image ID.
// It is exported for reusing in dockershim.
func GetImageRef(client DockerInterface, image string) (string, error) {
img, err := client.InspectImageByRef(image)
if err != nil {
return "", err
}
if img == nil {
return "", fmt.Errorf("unable to inspect image %s", image)
}
// Returns the digest if it exist.
if len(img.RepoDigests) > 0 {
return img.RepoDigests[0], nil
}
return img.ID, nil
}
// Temporarily export this function to share with dockershim.
// TODO: clean this up.
func GetContainerLogs(client DockerInterface, pod *v1.Pod, containerID kubecontainer.ContainerID, logOptions *v1.PodLogOptions, stdout, stderr io.Writer, rawTerm bool) error {
var since int64
if logOptions.SinceSeconds != nil {
t := metav1.Now().Add(-time.Duration(*logOptions.SinceSeconds) * time.Second)
since = t.Unix()
}
if logOptions.SinceTime != nil {
since = logOptions.SinceTime.Unix()
}
opts := dockertypes.ContainerLogsOptions{
ShowStdout: true,
ShowStderr: true,
Since: strconv.FormatInt(since, 10),
Timestamps: logOptions.Timestamps,
Follow: logOptions.Follow,
}
if logOptions.TailLines != nil {
opts.Tail = strconv.FormatInt(*logOptions.TailLines, 10)
}
sopts := StreamOptions{
OutputStream: stdout,
ErrorStream: stderr,
RawTerminal: rawTerm,
}
return client.Logs(containerID.ID, opts, sopts)
}
// Temporarily export this function to share with dockershim.
// TODO: clean this up.
func AttachContainer(client DockerInterface, containerID string, stdin io.Reader, stdout, stderr io.WriteCloser, tty bool, resize <-chan remotecommand.TerminalSize) error {
// Have to start this before the call to client.AttachToContainer because client.AttachToContainer is a blocking
// call :-( Otherwise, resize events don't get processed and the terminal never resizes.
kubecontainer.HandleResizing(resize, func(size remotecommand.TerminalSize) {
client.ResizeContainerTTY(containerID, int(size.Height), int(size.Width))
})
// TODO(random-liu): Do we really use the *Logs* field here?
opts := dockertypes.ContainerAttachOptions{
Stream: true,
Stdin: stdin != nil,
Stdout: stdout != nil,
Stderr: stderr != nil,
}
sopts := StreamOptions{
InputStream: stdin,
OutputStream: stdout,
ErrorStream: stderr,
RawTerminal: tty,
}
return client.AttachToContainer(containerID, opts, sopts)
}
// Temporarily export this function to share with dockershim.
func PortForward(client DockerInterface, podInfraContainerID string, port int32, stream io.ReadWriteCloser) error {
container, err := client.InspectContainer(podInfraContainerID)
if err != nil {
return err
}
if !container.State.Running {
return fmt.Errorf("container not running (%s)", container.ID)
}
containerPid := container.State.Pid
socatPath, lookupErr := exec.LookPath("socat")
if lookupErr != nil {
return fmt.Errorf("unable to do port forwarding: socat not found.")
}
args := []string{"-t", fmt.Sprintf("%d", containerPid), "-n", socatPath, "-", fmt.Sprintf("TCP4:localhost:%d", port)}
nsenterPath, lookupErr := exec.LookPath("nsenter")
if lookupErr != nil {
return fmt.Errorf("unable to do port forwarding: nsenter not found.")
}
commandString := fmt.Sprintf("%s %s", nsenterPath, strings.Join(args, " "))
glog.V(4).Infof("executing port forwarding command: %s", commandString)
command := exec.Command(nsenterPath, args...)
command.Stdout = stream
stderr := new(bytes.Buffer)
command.Stderr = stderr
// If we use Stdin, command.Run() won't return until the goroutine that's copying
// from stream finishes. Unfortunately, if you have a client like telnet connected
// via port forwarding, as long as the user's telnet client is connected to the user's
// local listener that port forwarding sets up, the telnet session never exits. This
// means that even if socat has finished running, command.Run() won't ever return
// (because the client still has the connection and stream open).
//
// The work around is to use StdinPipe(), as Wait() (called by Run()) closes the pipe
// when the command (socat) exits.
inPipe, err := command.StdinPipe()
if err != nil {
return fmt.Errorf("unable to do port forwarding: error creating stdin pipe: %v", err)
}
go func() {
io.Copy(inPipe, stream)
inPipe.Close()
}()
if err := command.Run(); err != nil {
return fmt.Errorf("%v: %s", err, stderr.String())
}
return nil
}
// Temporarily export this function to share with dockershim.
// TODO: clean this up.
func GetAppArmorOpts(profile string) ([]dockerOpt, error) {
if profile == "" || profile == apparmor.ProfileRuntimeDefault {
// The docker applies the default profile by default.
return nil, nil
}
// Assume validation has already happened.
profileName := strings.TrimPrefix(profile, apparmor.ProfileNamePrefix)
return []dockerOpt{{"apparmor", profileName, ""}}, nil
}
// Temporarily export this function to share with dockershim.
// TODO: clean this up.
func GetSeccompOpts(annotations map[string]string, ctrName, profileRoot string) ([]dockerOpt, error) {
profile, profileOK := annotations[v1.SeccompContainerAnnotationKeyPrefix+ctrName]
if !profileOK {
// try the pod profile
profile, profileOK = annotations[v1.SeccompPodAnnotationKey]
if !profileOK {
// return early the default
return defaultSeccompOpt, nil
}
}
if profile == "unconfined" {
// return early the default
return defaultSeccompOpt, nil
}
if profile == "docker/default" {
// return nil so docker will load the default seccomp profile
return nil, nil
}
if !strings.HasPrefix(profile, "localhost/") {
return nil, fmt.Errorf("unknown seccomp profile option: %s", profile)
}
name := strings.TrimPrefix(profile, "localhost/") // by pod annotation validation, name is a valid subpath
fname := filepath.Join(profileRoot, filepath.FromSlash(name))
file, err := ioutil.ReadFile(fname)
if err != nil {
return nil, fmt.Errorf("cannot load seccomp profile %q: %v", name, err)
}
b := bytes.NewBuffer(nil)
if err := json.Compact(b, file); err != nil {
return nil, err
}
// Rather than the full profile, just put the filename & md5sum in the event log.
msg := fmt.Sprintf("%s(md5:%x)", name, md5.Sum(file))
return []dockerOpt{{"seccomp", b.String(), msg}}, nil
}
// FmtDockerOpts formats the docker security options using the given separator.
func FmtDockerOpts(opts []dockerOpt, sep rune) []string {
fmtOpts := make([]string, len(opts))
for i, opt := range opts {
fmtOpts[i] = fmt.Sprintf("%s%c%s", opt.key, sep, opt.value)
}
return fmtOpts
}
type dockerOpt struct {
// The key-value pair passed to docker.
key, value string
// The alternative value to use in log/event messages.
msg string
}
// Expose key/value from dockertools
func (d dockerOpt) GetKV() (string, string) {
return d.key, d.value
}
// GetUserFromImageUser splits the user out of an user:group string.
func GetUserFromImageUser(id string) string {
if id == "" {
return id
}
// split instances where the id may contain user:group
if strings.Contains(id, ":") {
return strings.Split(id, ":")[0]
}
// no group, just return the id
return id
}
type dockerExitError struct {
Inspect *dockertypes.ContainerExecInspect
}
func (d *dockerExitError) String() string {
return d.Error()
}
func (d *dockerExitError) Error() string {
return fmt.Sprintf("Error executing in Docker Container: %d", d.Inspect.ExitCode)
}
func (d *dockerExitError) Exited() bool {
return !d.Inspect.Running
}
func (d *dockerExitError) ExitStatus() int {
return d.Inspect.ExitCode
}
// RewriteResolvFile rewrites resolv.conf file generated by docker.
// Exported for reusing in dockershim.
func RewriteResolvFile(resolvFilePath string, dns []string, dnsSearch []string, useClusterFirstPolicy bool) error {
if len(resolvFilePath) == 0 {
glog.Errorf("ResolvConfPath is empty.")
return nil
}
if _, err := os.Stat(resolvFilePath); os.IsNotExist(err) {
return fmt.Errorf("ResolvConfPath %q does not exist", resolvFilePath)
}
var resolvFileContent []string
for _, srv := range dns {
resolvFileContent = append(resolvFileContent, "nameserver "+srv)
}
if len(dnsSearch) > 0 {
resolvFileContent = append(resolvFileContent, "search "+strings.Join(dnsSearch, " "))
}
if len(resolvFileContent) > 0 {
if useClusterFirstPolicy {
resolvFileContent = append(resolvFileContent, ndotsDNSOption)
}
resolvFileContentStr := strings.Join(resolvFileContent, "\n")
resolvFileContentStr += "\n"
glog.V(4).Infof("Will attempt to re-write config file %s with: \n%s", resolvFilePath, resolvFileContent)
if err := rewriteFile(resolvFilePath, resolvFileContentStr); err != nil {
glog.Errorf("resolv.conf could not be updated: %v", err)
return err
}
}
return nil
}
func rewriteFile(filePath, stringToWrite string) error {
f, err := os.OpenFile(filePath, os.O_TRUNC|os.O_WRONLY, 0644)
if err != nil {
return err
}
defer f.Close()
_, err = f.WriteString(stringToWrite)
return err
}

13
pkg/kubelet/kubelet.go
View File

@ -364,17 +364,6 @@ func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Kub
KernelMemcgNotification: kubeCfg.ExperimentalKernelMemcgNotification,
}
var dockerExecHandler dockertools.ExecHandler
switch kubeCfg.DockerExecHandlerName {
case "native":
dockerExecHandler = &dockertools.NativeExecHandler{}
case "nsenter":
dockerExecHandler = &dockertools.NsenterExecHandler{}
default:
glog.Warningf("Unknown Docker exec handler %q; defaulting to native", kubeCfg.DockerExecHandlerName)
dockerExecHandler = &dockertools.NativeExecHandler{}
}
serviceIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
if kubeDeps.KubeClient != nil {
serviceLW := cache.NewListWatchFromClient(kubeDeps.KubeClient.Core().RESTClient(), "services", metav1.NamespaceAll, fields.Everything())
@ -556,7 +545,7 @@ func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Kub
// Create and start the CRI shim running as a grpc server.
streamingConfig := getStreamingConfig(kubeCfg, kubeDeps)
ds, err := dockershim.NewDockerService(klet.dockerClient, kubeCfg.SeccompProfileRoot, kubeCfg.PodInfraContainerImage,
streamingConfig, &pluginSettings, kubeCfg.RuntimeCgroups, kubeCfg.CgroupDriver, dockerExecHandler, dockershimRootDir,
streamingConfig, &pluginSettings, kubeCfg.RuntimeCgroups, kubeCfg.CgroupDriver, kubeCfg.DockerExecHandlerName, dockershimRootDir,
!kubeCfg.DockerEnableSharedPID)
if err != nil {
return nil, err