/* Copyright 2014 Google Inc. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package dockertools import ( "bufio" "bytes" "errors" "fmt" "hash/adler32" "io" "io/ioutil" "math/rand" "os" "os/exec" "strconv" "strings" "github.com/GoogleCloudPlatform/kubernetes/pkg/api" "github.com/GoogleCloudPlatform/kubernetes/pkg/credentialprovider" "github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/leaky" "github.com/GoogleCloudPlatform/kubernetes/pkg/types" "github.com/GoogleCloudPlatform/kubernetes/pkg/util" docker "github.com/fsouza/go-dockerclient" "github.com/golang/glog" ) const ( PodInfraContainerName = leaky.PodInfraContainerName DockerPrefix = "docker://" ) // DockerInterface is an abstract interface for testability. It abstracts the interface of docker.Client. type DockerInterface interface { ListContainers(options docker.ListContainersOptions) ([]docker.APIContainers, error) InspectContainer(id string) (*docker.Container, error) CreateContainer(docker.CreateContainerOptions) (*docker.Container, error) StartContainer(id string, hostConfig *docker.HostConfig) error StopContainer(id string, timeout uint) error RemoveContainer(opts docker.RemoveContainerOptions) error InspectImage(image string) (*docker.Image, error) ListImages(opts docker.ListImagesOptions) ([]docker.APIImages, error) PullImage(opts docker.PullImageOptions, auth docker.AuthConfiguration) error RemoveImage(image string) error Logs(opts docker.LogsOptions) error Version() (*docker.Env, error) CreateExec(docker.CreateExecOptions) (*docker.Exec, error) StartExec(string, docker.StartExecOptions) error } // DockerID is an ID of docker container. It is a type to make it clear when we're working with docker container Ids type DockerID string // DockerPuller is an abstract interface for testability. It abstracts image pull operations. type DockerPuller interface { Pull(image string) error IsImagePresent(image string) (bool, error) } // dockerPuller is the default implementation of DockerPuller. type dockerPuller struct { client DockerInterface keyring credentialprovider.DockerKeyring } type throttledDockerPuller struct { puller dockerPuller limiter util.RateLimiter } // NewDockerPuller creates a new instance of the default implementation of DockerPuller. func NewDockerPuller(client DockerInterface, qps float32, burst int) DockerPuller { dp := dockerPuller{ client: client, keyring: credentialprovider.NewDockerKeyring(), } if qps == 0.0 { return dp } return &throttledDockerPuller{ puller: dp, limiter: util.NewTokenBucketRateLimiter(qps, burst), } } type dockerContainerCommandRunner struct { client DockerInterface } // The first version of docker that supports exec natively is 1.3.0 == API 1.15 var dockerAPIVersionWithExec = []uint{1, 15} // Returns the major and minor version numbers of docker server. func (d *dockerContainerCommandRunner) GetDockerServerVersion() ([]uint, error) { env, err := d.client.Version() if err != nil { return nil, fmt.Errorf("failed to get docker server version - %v", err) } version := []uint{} for _, entry := range *env { if strings.Contains(strings.ToLower(entry), "apiversion") || strings.Contains(strings.ToLower(entry), "api version") { elems := strings.Split(strings.Split(entry, "=")[1], ".") for _, elem := range elems { val, err := strconv.ParseUint(elem, 10, 32) if err != nil { return nil, fmt.Errorf("failed to parse docker server version %q: %v", entry, err) } version = append(version, uint(val)) } return version, nil } } return nil, fmt.Errorf("docker server version missing from server version output - %+v", env) } func (d *dockerContainerCommandRunner) nativeExecSupportExists() (bool, error) { version, err := d.GetDockerServerVersion() if err != nil { return false, err } if len(dockerAPIVersionWithExec) != len(version) { return false, fmt.Errorf("unexpected docker version format. Expecting %v format, got %v", dockerAPIVersionWithExec, version) } for idx, val := range dockerAPIVersionWithExec { if version[idx] < val { return false, nil } } return true, nil } func (d *dockerContainerCommandRunner) getRunInContainerCommand(containerID string, cmd []string) (*exec.Cmd, error) { args := append([]string{"exec"}, cmd...) command := exec.Command("/usr/sbin/nsinit", args...) command.Dir = fmt.Sprintf("/var/lib/docker/execdriver/native/%s", containerID) return command, nil } func (d *dockerContainerCommandRunner) runInContainerUsingNsinit(containerID string, cmd []string) ([]byte, error) { c, err := d.getRunInContainerCommand(containerID, cmd) if err != nil { return nil, err } return c.CombinedOutput() } // RunInContainer uses nsinit to run the command inside the container identified by containerID func (d *dockerContainerCommandRunner) RunInContainer(containerID string, cmd []string) ([]byte, error) { // If native exec support does not exist in the local docker daemon use nsinit. useNativeExec, err := d.nativeExecSupportExists() if err != nil { return nil, err } if !useNativeExec { return d.runInContainerUsingNsinit(containerID, cmd) } createOpts := docker.CreateExecOptions{ Container: containerID, Cmd: cmd, AttachStdin: false, AttachStdout: true, AttachStderr: true, Tty: false, } execObj, err := d.client.CreateExec(createOpts) if err != nil { return nil, fmt.Errorf("failed to run in container - Exec setup failed - %v", err) } var buf bytes.Buffer wrBuf := bufio.NewWriter(&buf) startOpts := docker.StartExecOptions{ Detach: false, Tty: false, OutputStream: wrBuf, ErrorStream: wrBuf, RawTerminal: false, } errChan := make(chan error, 1) go func() { errChan <- d.client.StartExec(execObj.ID, startOpts) }() wrBuf.Flush() return buf.Bytes(), <-errChan } // ExecInContainer uses nsenter to run the command inside the container identified by containerID. // // TODO: // - match cgroups of container // - should we support `docker exec`? // - should we support nsenter in a container, running with elevated privs and --pid=host? func (d *dockerContainerCommandRunner) ExecInContainer(containerId string, cmd []string, stdin io.Reader, stdout, stderr io.WriteCloser, tty bool) error { container, err := d.client.InspectContainer(containerId) if err != nil { return err } if !container.State.Running { return fmt.Errorf("container not running (%s)", container) } containerPid := container.State.Pid // TODO what if the container doesn't have `env`??? args := []string{"-t", fmt.Sprintf("%d", containerPid), "-m", "-i", "-u", "-n", "-p", "--", "env", "-i"} args = append(args, fmt.Sprintf("HOSTNAME=%s", container.Config.Hostname)) args = append(args, container.Config.Env...) args = append(args, cmd...) command := exec.Command("nsenter", args...) // TODO use exec.LookPath if tty { p, err := StartPty(command) if err != nil { return err } defer p.Close() // make sure to close the stdout stream defer stdout.Close() if stdin != nil { go io.Copy(p, stdin) } if stdout != nil { go io.Copy(stdout, p) } return command.Wait() } else { cp := func(dst io.WriteCloser, src io.Reader, closeDst bool) { defer func() { if closeDst { dst.Close() } }() io.Copy(dst, src) } if stdin != nil { inPipe, err := command.StdinPipe() if err != nil { return err } go func() { cp(inPipe, stdin, false) inPipe.Close() }() } if stdout != nil { outPipe, err := command.StdoutPipe() if err != nil { return err } go cp(stdout, outPipe, true) } if stderr != nil { errPipe, err := command.StderrPipe() if err != nil { return err } go cp(stderr, errPipe, true) } return command.Run() } } // PortForward executes socat in the pod's network namespace and copies // data between stream (representing the user's local connection on their // computer) and the specified port in the container. // // TODO: // - match cgroups of container // - should we support nsenter + socat on the host? (current impl) // - should we support nsenter + socat in a container, running with elevated privs and --pid=host? func (d *dockerContainerCommandRunner) PortForward(podInfraContainerID string, port uint16, stream io.ReadWriteCloser) error { container, err := d.client.InspectContainer(podInfraContainerID) if err != nil { return err } if !container.State.Running { return fmt.Errorf("container not running (%s)", container) } containerPid := container.State.Pid // TODO use exec.LookPath for socat / what if the host doesn't have it??? args := []string{"-t", fmt.Sprintf("%d", containerPid), "-n", "socat", "-", fmt.Sprintf("TCP4:localhost:%d", port)} // TODO use exec.LookPath command := exec.Command("nsenter", args...) in, err := command.StdinPipe() if err != nil { return err } out, err := command.StdoutPipe() if err != nil { return err } go io.Copy(in, stream) go io.Copy(stream, out) return command.Run() } // NewDockerContainerCommandRunner creates a ContainerCommandRunner which uses nsinit to run a command // inside a container. func NewDockerContainerCommandRunner(client DockerInterface) ContainerCommandRunner { return &dockerContainerCommandRunner{client: client} } func (p dockerPuller) Pull(image string) error { image, tag := parseImageName(image) // If no tag was specified, use the default "latest". if len(tag) == 0 { tag = "latest" } opts := docker.PullImageOptions{ Repository: image, Tag: tag, } creds, ok := p.keyring.Lookup(image) if !ok { glog.V(1).Infof("Pulling image %s without credentials", image) } err := p.client.PullImage(opts, creds) // If there was no error, or we had credentials, just return the error. if err == nil || ok { return err } // Image spec: [/]/[: 1 { var err error hash, err = strconv.ParseUint(nameParts[1], 16, 32) if err != nil { glog.Warningf("invalid container hash: %s", nameParts[1]) } } // Pod fullname. podFullName = parts[2] + "_" + parts[3] // Pod UID. podUID = types.UID(parts[4]) return } func GetRunningContainers(client DockerInterface, ids []string) ([]*docker.Container, error) { result := []*docker.Container{} if client == nil { return nil, fmt.Errorf("unexpected nil docker client.") } for ix := range ids { status, err := client.InspectContainer(ids[ix]) if err != nil { return nil, err } if status != nil && status.State.Running { result = append(result, status) } } return result, nil } // Parses image name including a tag and returns image name and tag. // TODO: Future Docker versions can parse the tag on daemon side, see // https://github.com/dotcloud/docker/issues/6876 // So this can be deprecated at some point. func parseImageName(image string) (string, string) { tag := "" parts := strings.SplitN(image, "/", 2) repo := "" if len(parts) == 2 { repo = parts[0] image = parts[1] } parts = strings.SplitN(image, ":", 2) if len(parts) == 2 { image = parts[0] tag = parts[1] } if repo != "" { image = fmt.Sprintf("%s/%s", repo, image) } return image, tag } // Get a docker endpoint, either from the string passed in, or $DOCKER_HOST environment variables func getDockerEndpoint(dockerEndpoint string) string { var endpoint string if len(dockerEndpoint) > 0 { endpoint = dockerEndpoint } else if len(os.Getenv("DOCKER_HOST")) > 0 { endpoint = os.Getenv("DOCKER_HOST") } else { endpoint = "unix:///var/run/docker.sock" } glog.Infof("Connecting to docker on %s", endpoint) return endpoint } func ConnectToDockerOrDie(dockerEndpoint string) DockerInterface { if dockerEndpoint == "fake://" { return &FakeDockerClient{ VersionInfo: []string{"apiVersion=1.16"}, } } client, err := docker.NewClient(getDockerEndpoint(dockerEndpoint)) if err != nil { glog.Fatal("Couldn't connect to docker.") } return client } type ContainerCommandRunner interface { RunInContainer(containerID string, cmd []string) ([]byte, error) GetDockerServerVersion() ([]uint, error) ExecInContainer(containerID string, cmd []string, in io.Reader, out, err io.WriteCloser, tty bool) error PortForward(podInfraContainerID string, port uint16, stream io.ReadWriteCloser) error }