diff --git a/trash.lock b/trash.lock index ed803f8a43..fc8a10c595 100755 --- a/trash.lock +++ b/trash.lock @@ -38,7 +38,7 @@ import: - package: github.com/containerd/continuity version: bd77b46c8352f74eb12c85bdc01f4b90f69d66b4 - package: github.com/containerd/cri - version: v1.2-k3s.2 + version: v1.2.7-k3s1 repo: https://github.com/rancher/cri.git - package: github.com/containerd/fifo version: 3d5202aec260678c48179c56f40e6f38a095738c @@ -53,8 +53,8 @@ import: - package: github.com/containernetworking/cni version: v0.6.0 - package: github.com/containernetworking/plugins - version: 9810b7d5137b171c4e07ce59bb18be9feccec557 - repo: https://github.com/ibuildthecloud/plugins.git + version: v0.7.5-k3s1 + repo: https://github.com/rancher/plugins.git - package: github.com/coreos/etcd version: v3.3.10 - package: github.com/coreos/flannel @@ -65,7 +65,7 @@ import: - package: github.com/coreos/go-semver version: v0.3.0 - package: github.com/coreos/go-systemd - version: 48702e0da86bd25e76cfef347e2adeb434a0d0a6 + version: v14 - package: github.com/coreos/pkg version: v4 - package: github.com/cyphar/filepath-securejoin @@ -107,7 +107,7 @@ import: - package: github.com/go-sql-driver/mysql version: v1.4.1 - package: github.com/godbus/dbus - version: c7fdd8b5cd55e87b4e1f4e372cdb1db61dd6c66f + version: v3 - package: github.com/gogo/googleapis version: 08a7655d27152912db7aaf4f983275eaf8d128ef - package: github.com/gogo/protobuf @@ -134,7 +134,7 @@ import: - package: github.com/gregjones/httpcache version: 787624de3eb7bd915c329cba748687a3b22666a6 - package: github.com/grpc-ecosystem/go-grpc-prometheus - version: 6b7015e65d366bf3f19b2b2a000a831940f0f7e0 + version: v1.1 - package: github.com/hashicorp/errwrap version: 7554cd9344cec97297fa6649b055a8c98c2a1e55 - package: github.com/hashicorp/go-multierror @@ -209,7 +209,7 @@ import: - package: github.com/opencontainers/runtime-tools version: v0.6.0 - package: github.com/opencontainers/selinux - version: v1.2.1 + version: v1.2.2 - package: github.com/pborman/uuid version: ca53cad383cad2479bbba7f7a1a05797ec1386e4 - package: github.com/peterbourgon/diskv diff --git a/vendor.conf b/vendor.conf index 1c03c21434..3a56c406d8 100644 --- a/vendor.conf +++ b/vendor.conf @@ -30,13 +30,14 @@ golang.org/x/crypto a49355c7e3f8fe157a85be2f77e6e269a0f89602 gopkg.in/freddierice/go-losetup.v1 fc9adea44124401d8bfef3a97eaf61b5d44cc2c6 github.com/urfave/cli 8e01ec4cd3e2d84ab2fe90d8210528ffbb06d8ff golang.org/x/sync 450f422ab23cf9881c94e2db30cac0eb1b7cf80c +github.com/docker/docker c12f09bf99b54f274a5ae241dd154fa74020cbab # flannel github.com/golang/glog 23def4e6c14b4da8ac2ed8007337bc5eb5007998 github.com/coreos/go-iptables 47f22b0dd3355c0ba570ba12b0b8a36bf214c04b # cni -github.com/containernetworking/plugins 9810b7d5137b171c4e07ce59bb18be9feccec557 https://github.com/ibuildthecloud/plugins.git +github.com/containernetworking/plugins v0.7.5-k3s1 https://github.com/rancher/plugins.git github.com/j-keck/arping 2cf9dc699c5640a7e2c81403a44127bf28033600 github.com/alexflint/go-filemutex 72bdc8eae2aef913234599b837f5dda445ca9bd9 @@ -52,11 +53,11 @@ github.com/containerd/typeurl a93fcdb778cd272c6e9b3028b2f42d813e785d40 github.com/containerd/fifo 3d5202aec260678c48179c56f40e6f38a095738c github.com/containerd/btrfs 2e1aa0ddf94f91fa282b6ed87c23bf0d64911244 github.com/containerd/continuity bd77b46c8352f74eb12c85bdc01f4b90f69d66b4 -github.com/coreos/go-systemd 48702e0da86bd25e76cfef347e2adeb434a0d0a6 +github.com/coreos/go-systemd v14 github.com/docker/go-metrics 4ea375f7759c82740c893fc030bc37088d2ec098 github.com/docker/go-events 9461782956ad83b30282bf90e31fa6a70c255ba9 #github.com/docker/go-units v0.3.3 -github.com/godbus/dbus c7fdd8b5cd55e87b4e1f4e372cdb1db61dd6c66f +github.com/godbus/dbus v3 #github.com/prometheus/client_golang f4fb1b73fb099f396a7f0036bf86aa8def4ed823 #github.com/prometheus/client_model 99fa1f4be8e564e8a6b613da7fa6f46c9edafc6c #github.com/prometheus/common 89604d197083d4781071d3c65855d24ecfb0a563 @@ -69,16 +70,16 @@ github.com/golang/protobuf v1.1.0 #github.com/opencontainers/runtime-spec eba862dc2470385a233c7507392675cbeadf7353 # v1.0.1-45-geba862d github.com/opencontainers/runc v1.0.0-rc8 github.com/sirupsen/logrus v1.0.3 -github.com/urfave/cli 7bc6a0acffa589f415f88aca16cc1de5ffd66f9c +#github.com/urfave/cli 7bc6a0acffa589f415f88aca16cc1de5ffd66f9c golang.org/x/net b3756b4b77d7b13260a0a2ec658753cf48922eac google.golang.org/grpc v1.12.0 github.com/pkg/errors v0.8.0 github.com/opencontainers/go-digest c9281466c8b2f606084ac71339773efd177436e7 golang.org/x/sys 41f3e6584952bb034a481797859f6ab34b6803bd https://github.com/golang/sys github.com/opencontainers/image-spec v1.0.1 -golang.org/x/sync 450f422ab23cf9881c94e2db30cac0eb1b7cf80c +#golang.org/x/sync 450f422ab23cf9881c94e2db30cac0eb1b7cf80c github.com/BurntSushi/toml a368813c5e648fee92e5f6c30e3944ff9d5e8895 -github.com/grpc-ecosystem/go-grpc-prometheus 6b7015e65d366bf3f19b2b2a000a831940f0f7e0 +github.com/grpc-ecosystem/go-grpc-prometheus v1.1 github.com/Microsoft/go-winio v0.4.12 github.com/Microsoft/hcsshim v0.8.6 google.golang.org/genproto d80a6e20e776b0b17a324d0ba1ab50a39c8e8944 @@ -93,14 +94,14 @@ go.etcd.io/bbolt v1.3.1-etcd.8 github.com/kubernetes-sigs/cri-tools v1.14.0-k3s1 https://github.com/rancher/cri-tools.git # cri dependencies -github.com/containerd/cri v1.2-k3s.2 https://github.com/rancher/cri.git +github.com/containerd/cri v1.2.7-k3s1 https://github.com/rancher/cri.git github.com/containerd/go-cni 40bcf8ec8acd7372be1d77031d585d5d8e561c90 github.com/blang/semver v3.1.0 github.com/containernetworking/cni v0.6.0 #github.com/containernetworking/plugins v0.7.5 github.com/davecgh/go-spew v1.1.0 github.com/docker/distribution 0d3efadf0154c2b8a4e7b6621fff9809655cc580 -github.com/docker/docker c12f09bf99b54f274a5ae241dd154fa74020cbab +#github.com/docker/docker 86f080cff0914e9694068ed78d503701667c4c00 github.com/docker/spdystream 449fdfce4d962303d702fec724ef0ad181c92528 github.com/emicklei/go-restful v2.2.1 github.com/ghodss/yaml v1.0.0 @@ -112,13 +113,13 @@ github.com/json-iterator/go 1.1.5 github.com/modern-go/reflect2 1.0.1 github.com/modern-go/concurrent 1.0.3 github.com/opencontainers/runtime-tools v0.6.0 -github.com/opencontainers/selinux v1.2.1 +github.com/opencontainers/selinux v1.2.2 github.com/seccomp/libseccomp-golang 32f571b70023028bd57d9288c20efbcb237f3ce0 github.com/tchap/go-patricia v2.2.6 github.com/xeipuuv/gojsonpointer 4e3ac2762d5f479393488629ee9370b50873b3a6 github.com/xeipuuv/gojsonreference bd5ef7bd5415a7ac448318e64f11a24cd21e594b github.com/xeipuuv/gojsonschema 1d523034197ff1f222f6429836dd36a2457a1874 -golang.org/x/crypto 49796115aa4b964c318aad4f3084fdb41e9aa067 +#golang.org/x/crypto 49796115aa4b964c318aad4f3084fdb41e9aa067 golang.org/x/oauth2 a6bd8cefa1811bd24b86f8902872e4e8225f74c4 golang.org/x/time f51c12702a4d776e4c1fa9b0fabab841babae631 gopkg.in/inf.v0 3887ee99ecf07df5b447e9b00d9c0b2adaa9f3e4 diff --git a/vendor/github.com/containerd/cri/pkg/server/container_execsync.go b/vendor/github.com/containerd/cri/pkg/server/container_execsync.go index fd54120aed..ba64cd021e 100644 --- a/vendor/github.com/containerd/cri/pkg/server/container_execsync.go +++ b/vendor/github.com/containerd/cri/pkg/server/container_execsync.go @@ -131,7 +131,7 @@ func (c *criService) execInContainer(ctx context.Context, id string, opts execOp defer func() { deferCtx, deferCancel := ctrdutil.DeferContext() defer deferCancel() - if _, err := process.Delete(deferCtx); err != nil { + if _, err := process.Delete(deferCtx, containerd.WithProcessKill); err != nil { logrus.WithError(err).Errorf("Failed to delete exec process %q for container %q", execID, id) } }() diff --git a/vendor/github.com/containerd/cri/pkg/server/container_remove.go b/vendor/github.com/containerd/cri/pkg/server/container_remove.go index fe37a8ac72..9b70cac003 100644 --- a/vendor/github.com/containerd/cri/pkg/server/container_remove.go +++ b/vendor/github.com/containerd/cri/pkg/server/container_remove.go @@ -105,6 +105,9 @@ func setContainerRemoving(container containerstore.Container) error { if status.State() == runtime.ContainerState_CONTAINER_UNKNOWN { return status, errors.New("container state is unknown, to stop first") } + if status.Starting { + return status, errors.New("container is in starting state, can't be removed") + } if status.Removing { return status, errors.New("container is already in removing state") } diff --git a/vendor/github.com/containerd/cri/pkg/server/container_start.go b/vendor/github.com/containerd/cri/pkg/server/container_start.go index 59fa5ed696..cd75f2a1b1 100644 --- a/vendor/github.com/containerd/cri/pkg/server/container_start.go +++ b/vendor/github.com/containerd/cri/pkg/server/container_start.go @@ -38,64 +38,48 @@ import ( // StartContainer starts the container. func (c *criService) StartContainer(ctx context.Context, r *runtime.StartContainerRequest) (retRes *runtime.StartContainerResponse, retErr error) { - container, err := c.containerStore.Get(r.GetContainerId()) + cntr, err := c.containerStore.Get(r.GetContainerId()) if err != nil { return nil, errors.Wrapf(err, "an error occurred when try to find container %q", r.GetContainerId()) } - var startErr error - // update container status in one transaction to avoid race with event monitor. - if err := container.Status.UpdateSync(func(status containerstore.Status) (containerstore.Status, error) { - // Always apply status change no matter startContainer fails or not. Because startContainer - // may change container state no matter it fails or succeeds. - startErr = c.startContainer(ctx, container, &status) - return status, nil - }); startErr != nil { - return nil, startErr - } else if err != nil { - return nil, errors.Wrapf(err, "failed to update container %q metadata", container.ID) - } - return &runtime.StartContainerResponse{}, nil -} - -// startContainer actually starts the container. The function needs to be run in one transaction. Any updates -// to the status passed in will be applied no matter the function returns error or not. -func (c *criService) startContainer(ctx context.Context, - cntr containerstore.Container, - status *containerstore.Status) (retErr error) { id := cntr.ID meta := cntr.Metadata container := cntr.Container config := meta.Config - // Return error if container is not in created state. - if status.State() != runtime.ContainerState_CONTAINER_CREATED { - return errors.Errorf("container %q is in %s state", id, criContainerStateToString(status.State())) + // Set starting state to prevent other start/remove operations against this container + // while it's being started. + if err := setContainerStarting(cntr); err != nil { + return nil, errors.Wrapf(err, "failed to set starting state for container %q", id) } - // Do not start the container when there is a removal in progress. - if status.Removing { - return errors.Errorf("container %q is in removing state", id) - } - defer func() { if retErr != nil { // Set container to exited if fail to start. - status.Pid = 0 - status.FinishedAt = time.Now().UnixNano() - status.ExitCode = errorStartExitCode - status.Reason = errorStartReason - status.Message = retErr.Error() + if err := cntr.Status.UpdateSync(func(status containerstore.Status) (containerstore.Status, error) { + status.Pid = 0 + status.FinishedAt = time.Now().UnixNano() + status.ExitCode = errorStartExitCode + status.Reason = errorStartReason + status.Message = retErr.Error() + return status, nil + }); err != nil { + logrus.WithError(err).Errorf("failed to set start failure state for container %q", id) + } + } + if err := resetContainerStarting(cntr); err != nil { + logrus.WithError(err).Errorf("failed to reset starting state for container %q", id) } }() // Get sandbox config from sandbox store. sandbox, err := c.sandboxStore.Get(meta.SandboxID) if err != nil { - return errors.Wrapf(err, "sandbox %q not found", meta.SandboxID) + return nil, errors.Wrapf(err, "sandbox %q not found", meta.SandboxID) } sandboxID := meta.SandboxID if sandbox.Status.Get().State != sandboxstore.StateReady { - return errors.Errorf("sandbox container %q is not running", sandboxID) + return nil, errors.Errorf("sandbox container %q is not running", sandboxID) } ioCreation := func(id string) (_ containerdio.IO, err error) { @@ -110,7 +94,7 @@ func (c *criService) startContainer(ctx context.Context, ctrInfo, err := container.Info(ctx) if err != nil { - return errors.Wrap(err, "failed to get container info") + return nil, errors.Wrap(err, "failed to get container info") } var taskOpts []containerd.NewTaskOpts @@ -120,7 +104,7 @@ func (c *criService) startContainer(ctx context.Context, } task, err := container.NewTask(ctx, ioCreation, taskOpts...) if err != nil { - return errors.Wrap(err, "failed to create containerd task") + return nil, errors.Wrap(err, "failed to create containerd task") } defer func() { if retErr != nil { @@ -133,15 +117,61 @@ func (c *criService) startContainer(ctx context.Context, } }() + // wait is a long running background request, no timeout needed. + exitCh, err := task.Wait(ctrdutil.NamespacedContext()) + if err != nil { + return nil, errors.Wrap(err, "failed to wait for containerd task") + } + // Start containerd task. if err := task.Start(ctx); err != nil { - return errors.Wrapf(err, "failed to start containerd task %q", id) + return nil, errors.Wrapf(err, "failed to start containerd task %q", id) } // Update container start timestamp. - status.Pid = task.Pid() - status.StartedAt = time.Now().UnixNano() - return nil + if err := cntr.Status.UpdateSync(func(status containerstore.Status) (containerstore.Status, error) { + status.Pid = task.Pid() + status.StartedAt = time.Now().UnixNano() + return status, nil + }); err != nil { + return nil, errors.Wrapf(err, "failed to update container %q state", id) + } + + // start the monitor after updating container state, this ensures that + // event monitor receives the TaskExit event and update container state + // after this. + c.eventMonitor.startExitMonitor(context.Background(), id, task.Pid(), exitCh) + + return &runtime.StartContainerResponse{}, nil +} + +// setContainerStarting sets the container into starting state. In starting state, the +// container will not be removed or started again. +func setContainerStarting(container containerstore.Container) error { + return container.Status.Update(func(status containerstore.Status) (containerstore.Status, error) { + // Return error if container is not in created state. + if status.State() != runtime.ContainerState_CONTAINER_CREATED { + return status, errors.Errorf("container is in %s state", criContainerStateToString(status.State())) + } + // Do not start the container when there is a removal in progress. + if status.Removing { + return status, errors.New("container is in removing state, can't be started") + } + if status.Starting { + return status, errors.New("container is already in starting state") + } + status.Starting = true + return status, nil + }) +} + +// resetContainerStarting resets the container starting state on start failure. So +// that we could remove the container later. +func resetContainerStarting(container containerstore.Container) error { + return container.Status.Update(func(status containerstore.Status) (containerstore.Status, error) { + status.Starting = false + return status, nil + }) } // createContainerLoggers creates container loggers and return write closer for stdout and stderr. diff --git a/vendor/github.com/containerd/cri/pkg/server/container_stop.go b/vendor/github.com/containerd/cri/pkg/server/container_stop.go index 7b8f16e6c5..8286b7d60c 100644 --- a/vendor/github.com/containerd/cri/pkg/server/container_stop.go +++ b/vendor/github.com/containerd/cri/pkg/server/container_stop.go @@ -19,7 +19,6 @@ package server import ( "time" - "github.com/containerd/containerd" eventtypes "github.com/containerd/containerd/api/events" "github.com/containerd/containerd/errdefs" "github.com/docker/docker/pkg/signal" @@ -29,6 +28,7 @@ import ( "golang.org/x/sys/unix" runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" + ctrdutil "github.com/containerd/cri/pkg/containerd/util" "github.com/containerd/cri/pkg/store" containerstore "github.com/containerd/cri/pkg/store/container" ) @@ -75,36 +75,34 @@ func (c *criService) stopContainer(ctx context.Context, container containerstore return errors.Wrapf(err, "failed to get task for container %q", id) } // Don't return for unknown state, some cleanup needs to be done. - if state != runtime.ContainerState_CONTAINER_UNKNOWN { - return nil + if state == runtime.ContainerState_CONTAINER_UNKNOWN { + return cleanupUnknownContainer(ctx, id, container) } - // Task is an interface, explicitly set it to nil just in case. - task = nil + return nil } // Handle unknown state. if state == runtime.ContainerState_CONTAINER_UNKNOWN { - status, err := getTaskStatus(ctx, task) + // Start an exit handler for containers in unknown state. + waitCtx, waitCancel := context.WithCancel(ctrdutil.NamespacedContext()) + defer waitCancel() + exitCh, err := task.Wait(waitCtx) if err != nil { - return errors.Wrapf(err, "failed to get task status for %q", id) - } - switch status.Status { - case containerd.Running, containerd.Created: - // The task is still running, continue stopping the task. - case containerd.Stopped: - // The task has exited. If the task exited after containerd - // started, the event monitor will receive its exit event; if it - // exited before containerd started, the event monitor will never - // receive its exit event. - // However, we can't tell that because the task state was not - // successfully loaded during containerd start (container is - // in UNKNOWN state). - // So always do cleanup here, just in case that we've missed the - // exit event. - return cleanupUnknownContainer(ctx, id, status, container) - default: - return errors.Wrapf(err, "unsupported task status %q", status.Status) + if !errdefs.IsNotFound(err) { + return errors.Wrapf(err, "failed to wait for task for %q", id) + } + return cleanupUnknownContainer(ctx, id, container) } + + exitCtx, exitCancel := context.WithCancel(context.Background()) + stopCh := c.eventMonitor.startExitMonitor(exitCtx, id, task.Pid(), exitCh) + defer func() { + exitCancel() + // This ensures that exit monitor is stopped before + // `Wait` is cancelled, so no exit event is generated + // because of the `Wait` cancellation. + <-stopCh + }() } // We only need to kill the task. The event handler will Delete the @@ -177,19 +175,13 @@ func (c *criService) waitContainerStop(ctx context.Context, container containers } // cleanupUnknownContainer cleanup stopped container in unknown state. -func cleanupUnknownContainer(ctx context.Context, id string, status containerd.Status, - cntr containerstore.Container) error { +func cleanupUnknownContainer(ctx context.Context, id string, cntr containerstore.Container) error { // Reuse handleContainerExit to do the cleanup. - // NOTE(random-liu): If the task did exit after containerd started, both - // the event monitor and the cleanup function would update the container - // state. The final container state will be whatever being updated first. - // There is no way to completely avoid this race condition, and for best - // effort unknown state container cleanup, this seems acceptable. return handleContainerExit(ctx, &eventtypes.TaskExit{ ContainerID: id, ID: id, Pid: 0, - ExitStatus: status.ExitStatus, - ExitedAt: status.ExitTime, + ExitStatus: unknownExitCode, + ExitedAt: time.Now(), }, cntr) } diff --git a/vendor/github.com/containerd/cri/pkg/server/events.go b/vendor/github.com/containerd/cri/pkg/server/events.go index f94355d703..9f54454880 100644 --- a/vendor/github.com/containerd/cri/pkg/server/events.go +++ b/vendor/github.com/containerd/cri/pkg/server/events.go @@ -50,13 +50,17 @@ const ( // Add a timeout for each event handling, events that timeout will be requeued and // handled again in the future. handleEventTimeout = 10 * time.Second + + exitChannelSize = 1024 ) // eventMonitor monitors containerd event and updates internal state correspondingly. // TODO(random-liu): Handle event for each container in a separate goroutine. type eventMonitor struct { - c *criService - ch <-chan *events.Envelope + c *criService + ch <-chan *events.Envelope + // exitCh receives container/sandbox exit events from exit monitors. + exitCh chan *eventtypes.TaskExit errCh <-chan error ctx context.Context cancel context.CancelFunc @@ -89,6 +93,7 @@ func newEventMonitor(c *criService) *eventMonitor { c: c, ctx: ctx, cancel: cancel, + exitCh: make(chan *eventtypes.TaskExit, exitChannelSize), backOff: newBackOff(), } } @@ -98,13 +103,38 @@ func (em *eventMonitor) subscribe(subscriber events.Subscriber) { // note: filters are any match, if you want any match but not in namespace foo // then you have to manually filter namespace foo filters := []string{ - `topic=="/tasks/exit"`, `topic=="/tasks/oom"`, `topic~="/images/"`, } em.ch, em.errCh = subscriber.Subscribe(em.ctx, filters...) } +// startExitMonitor starts an exit monitor for a given container/sandbox. +func (em *eventMonitor) startExitMonitor(ctx context.Context, id string, pid uint32, exitCh <-chan containerd.ExitStatus) <-chan struct{} { + stopCh := make(chan struct{}) + go func() { + defer close(stopCh) + select { + case exitRes := <-exitCh: + exitStatus, exitedAt, err := exitRes.Result() + if err != nil { + logrus.WithError(err).Errorf("Failed to get task exit status for %q", id) + exitStatus = unknownExitCode + exitedAt = time.Now() + } + em.exitCh <- &eventtypes.TaskExit{ + ContainerID: id, + ID: id, + Pid: pid, + ExitStatus: exitStatus, + ExitedAt: exitedAt, + } + case <-ctx.Done(): + } + }() + return stopCh +} + func convertEvent(e *gogotypes.Any) (string, interface{}, error) { id := "" evt, err := typeurl.UnmarshalAny(e) @@ -113,8 +143,6 @@ func convertEvent(e *gogotypes.Any) (string, interface{}, error) { } switch e := evt.(type) { - case *eventtypes.TaskExit: - id = e.ContainerID case *eventtypes.TaskOOM: id = e.ContainerID case *eventtypes.ImageCreate: @@ -142,6 +170,18 @@ func (em *eventMonitor) start() <-chan error { defer close(errCh) for { select { + case e := <-em.exitCh: + logrus.Debugf("Received exit event %+v", e) + id := e.ID + if em.backOff.isInBackOff(id) { + logrus.Infof("Events for %q is in backoff, enqueue event %+v", id, e) + em.backOff.enBackOff(id, e) + break + } + if err := em.handleEvent(e); err != nil { + logrus.WithError(err).Errorf("Failed to handle exit event %+v for %s", e, id) + em.backOff.enBackOff(id, e) + } case e := <-em.ch: logrus.Debugf("Received containerd event timestamp - %v, namespace - %q, topic - %q", e.Timestamp, e.Namespace, e.Topic) if e.Namespace != constants.K8sContainerdNamespace { @@ -217,8 +257,7 @@ func (em *eventMonitor) handleEvent(any interface{}) error { } else if err != store.ErrNotExist { return errors.Wrap(err, "can't find container for TaskExit event") } - // Use GetAll to include sandbox in init state. - sb, err := em.c.sandboxStore.GetAll(e.ID) + sb, err := em.c.sandboxStore.Get(e.ID) if err == nil { if err := handleSandboxExit(ctx, e, sb); err != nil { return errors.Wrap(err, "failed to handle sandbox TaskExit event") @@ -326,15 +365,7 @@ func handleSandboxExit(ctx context.Context, e *eventtypes.TaskExit, sb sandboxst } } err = sb.Status.Update(func(status sandboxstore.Status) (sandboxstore.Status, error) { - // NOTE(random-liu): We SHOULD NOT change INIT state here. - // If sandbox state is INIT when event monitor receives an TaskExit event, - // it means that sandbox start has failed. In that case, `RunPodSandbox` will - // cleanup everything immediately. - // Once sandbox state goes out of INIT, it becomes visable to the user, which - // is not what we want. - if status.State != sandboxstore.StateInit { - status.State = sandboxstore.StateNotReady - } + status.State = sandboxstore.StateNotReady status.Pid = 0 return status, nil }) diff --git a/vendor/github.com/containerd/cri/pkg/server/helpers.go b/vendor/github.com/containerd/cri/pkg/server/helpers.go index 5283a3de20..51c1a9241d 100644 --- a/vendor/github.com/containerd/cri/pkg/server/helpers.go +++ b/vendor/github.com/containerd/cri/pkg/server/helpers.go @@ -25,12 +25,9 @@ import ( "regexp" "strconv" "strings" - "time" "github.com/BurntSushi/toml" - "github.com/containerd/containerd" "github.com/containerd/containerd/containers" - "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/runtime/linux/runctypes" runcoptions "github.com/containerd/containerd/runtime/v2/runc/options" "github.com/containerd/typeurl" @@ -580,31 +577,6 @@ func unknownSandboxStatus() sandboxstore.Status { } } -// unknownExitStatus generates containerd.Status for container exited with unknown exit code. -func unknownExitStatus() containerd.Status { - return containerd.Status{ - Status: containerd.Stopped, - ExitStatus: unknownExitCode, - ExitTime: time.Now(), - } -} - -// getTaskStatus returns status for a given task. It returns unknown exit status if -// the task is nil or not found. -func getTaskStatus(ctx context.Context, task containerd.Task) (containerd.Status, error) { - if task == nil { - return unknownExitStatus(), nil - } - status, err := task.Status(ctx) - if err != nil { - if !errdefs.IsNotFound(err) { - return containerd.Status{}, err - } - return unknownExitStatus(), nil - } - return status, nil -} - func getCurrentOOMScoreAdj() (int, error) { b, err := ioutil.ReadFile("/proc/self/oom_score_adj") if err != nil { diff --git a/vendor/github.com/containerd/cri/pkg/server/image_pull.go b/vendor/github.com/containerd/cri/pkg/server/image_pull.go index 3461b493af..74dca4877c 100644 --- a/vendor/github.com/containerd/cri/pkg/server/image_pull.go +++ b/vendor/github.com/containerd/cri/pkg/server/image_pull.go @@ -102,6 +102,7 @@ func (c *criService) PullImage(ctx context.Context, r *runtime.PullImageRequest) containerd.WithResolver(resolver), containerd.WithPullSnapshotter(c.config.ContainerdConfig.Snapshotter), containerd.WithPullUnpack, + containerd.WithPullLabel(imageLabelKey, imageLabelValue), ) if err != nil { return nil, errors.Wrapf(err, "failed to pull and unpack image %q", ref) diff --git a/vendor/github.com/containerd/cri/pkg/server/restart.go b/vendor/github.com/containerd/cri/pkg/server/restart.go index 7e53883d74..e4be81dc5f 100644 --- a/vendor/github.com/containerd/cri/pkg/server/restart.go +++ b/vendor/github.com/containerd/cri/pkg/server/restart.go @@ -34,6 +34,7 @@ import ( "golang.org/x/net/context" runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" + ctrdutil "github.com/containerd/cri/pkg/containerd/util" "github.com/containerd/cri/pkg/netns" cio "github.com/containerd/cri/pkg/server/io" containerstore "github.com/containerd/cri/pkg/store/container" @@ -57,7 +58,7 @@ func (c *criService) recover(ctx context.Context) error { return errors.Wrap(err, "failed to list sandbox containers") } for _, sandbox := range sandboxes { - sb, err := loadSandbox(ctx, sandbox) + sb, err := c.loadSandbox(ctx, sandbox) if err != nil { logrus.WithError(err).Errorf("Failed to load sandbox %q", sandbox.ID()) continue @@ -275,6 +276,22 @@ func (c *criService) loadContainer(ctx context.Context, cntr containerd.Containe status.StartedAt = time.Now().UnixNano() status.Pid = t.Pid() } + // Wait for the task for exit monitor. + // wait is a long running background request, no timeout needed. + exitCh, err := t.Wait(ctrdutil.NamespacedContext()) + if err != nil { + if !errdefs.IsNotFound(err) { + return errors.Wrap(err, "failed to wait for task") + } + // Container was in running state, but its task has been deleted, + // set unknown exited state. + status.FinishedAt = time.Now().UnixNano() + status.ExitCode = unknownExitCode + status.Reason = unknownExitReason + } else { + // Start exit monitor. + c.eventMonitor.startExitMonitor(context.Background(), id, status.Pid, exitCh) + } case containerd.Stopped: // Task is stopped. Updata status and delete the task. if _, err := t.Delete(ctx, containerd.WithProcessKill); err != nil && !errdefs.IsNotFound(err) { @@ -304,7 +321,7 @@ func (c *criService) loadContainer(ctx context.Context, cntr containerd.Containe } // loadSandbox loads sandbox from containerd. -func loadSandbox(ctx context.Context, cntr containerd.Container) (sandboxstore.Sandbox, error) { +func (c *criService) loadSandbox(ctx context.Context, cntr containerd.Container) (sandboxstore.Sandbox, error) { ctx, cancel := context.WithTimeout(ctx, loadContainerTimeout) defer cancel() var sandbox sandboxstore.Sandbox @@ -358,9 +375,20 @@ func loadSandbox(ctx context.Context, cntr containerd.Container) (sandboxstore.S status.State = sandboxstore.StateNotReady } else { if taskStatus.Status == containerd.Running { - // Task is running, set sandbox state as READY. - status.State = sandboxstore.StateReady - status.Pid = t.Pid() + // Wait for the task for sandbox monitor. + // wait is a long running background request, no timeout needed. + exitCh, err := t.Wait(ctrdutil.NamespacedContext()) + if err != nil { + if !errdefs.IsNotFound(err) { + return status, errors.Wrap(err, "failed to wait for task") + } + status.State = sandboxstore.StateNotReady + } else { + // Task is running, set sandbox state as READY. + status.State = sandboxstore.StateReady + status.Pid = t.Pid() + c.eventMonitor.startExitMonitor(context.Background(), meta.ID, status.Pid, exitCh) + } } else { // Task is not running. Delete the task and set sandbox state as NOTREADY. if _, err := t.Delete(ctx, containerd.WithProcessKill); err != nil && !errdefs.IsNotFound(err) { diff --git a/vendor/github.com/containerd/cri/pkg/server/sandbox_run.go b/vendor/github.com/containerd/cri/pkg/server/sandbox_run.go index 79041928c4..0657241763 100644 --- a/vendor/github.com/containerd/cri/pkg/server/sandbox_run.go +++ b/vendor/github.com/containerd/cri/pkg/server/sandbox_run.go @@ -82,7 +82,7 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox RuntimeHandler: r.GetRuntimeHandler(), }, sandboxstore.Status{ - State: sandboxstore.StateInit, + State: sandboxstore.StateUnknown, }, ) @@ -247,88 +247,65 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox if err != nil { return nil, errors.Wrap(err, "failed to get sandbox container info") } + + // Create sandbox task in containerd. + log.Tracef("Create sandbox container (id=%q, name=%q).", + id, name) + + var taskOpts []containerd.NewTaskOpts + // TODO(random-liu): Remove this after shim v1 is deprecated. + if c.config.NoPivot && ociRuntime.Type == linuxRuntime { + taskOpts = append(taskOpts, containerd.WithNoPivotRoot) + } + // We don't need stdio for sandbox container. + task, err := container.NewTask(ctx, containerdio.NullIO, taskOpts...) + if err != nil { + return nil, errors.Wrap(err, "failed to create containerd task") + } + defer func() { + if retErr != nil { + deferCtx, deferCancel := ctrdutil.DeferContext() + defer deferCancel() + // Cleanup the sandbox container if an error is returned. + if _, err := task.Delete(deferCtx, containerd.WithProcessKill); err != nil && !errdefs.IsNotFound(err) { + logrus.WithError(err).Errorf("Failed to delete sandbox container %q", id) + } + } + }() + + // wait is a long running background request, no timeout needed. + exitCh, err := task.Wait(ctrdutil.NamespacedContext()) + if err != nil { + return nil, errors.Wrap(err, "failed to wait for sandbox container task") + } + + if err := task.Start(ctx); err != nil { + return nil, errors.Wrapf(err, "failed to start sandbox container task %q", id) + } + if err := sandbox.Status.Update(func(status sandboxstore.Status) (sandboxstore.Status, error) { + // Set the pod sandbox as ready after successfully start sandbox container. + status.Pid = task.Pid() + status.State = sandboxstore.StateReady status.CreatedAt = info.CreatedAt return status, nil }); err != nil { - return nil, errors.Wrap(err, "failed to update sandbox created timestamp") + return nil, errors.Wrap(err, "failed to update sandbox status") } // Add sandbox into sandbox store in INIT state. sandbox.Container = container + if err := c.sandboxStore.Add(sandbox); err != nil { return nil, errors.Wrapf(err, "failed to add sandbox %+v into store", sandbox) } - defer func() { - // Delete sandbox from sandbox store if there is an error. - if retErr != nil { - c.sandboxStore.Delete(id) - } - }() - // NOTE(random-liu): Sandbox state only stay in INIT state after this point - // and before the end of this function. - // * If `Update` succeeds, sandbox state will become READY in one transaction. - // * If `Update` fails, sandbox will be removed from the store in the defer above. - // * If containerd stops at any point before `Update` finishes, because sandbox - // state is not checkpointed, it will be recovered from corresponding containerd task - // status during restart: - // * If the task is running, sandbox state will be READY, - // * Or else, sandbox state will be NOTREADY. + + // start the monitor after adding sandbox into the store, this ensures + // that sandbox is in the store, when event monitor receives the TaskExit event. // - // In any case, sandbox will leave INIT state, so it's safe to ignore sandbox - // in INIT state in other functions. - - // Start sandbox container in one transaction to avoid race condition with - // event monitor. - if err := sandbox.Status.Update(func(status sandboxstore.Status) (_ sandboxstore.Status, retErr error) { - // NOTE(random-liu): We should not change the sandbox state to NOTREADY - // if `Update` fails. - // - // If `Update` fails, the sandbox will be cleaned up by all the defers - // above. We should not let user see this sandbox, or else they will - // see the sandbox disappear after the defer clean up, which may confuse - // them. - // - // Given so, we should keep the sandbox in INIT state if `Update` fails, - // and ignore sandbox in INIT state in all the inspection functions. - - // Create sandbox task in containerd. - log.Tracef("Create sandbox container (id=%q, name=%q).", - id, name) - - var taskOpts []containerd.NewTaskOpts - // TODO(random-liu): Remove this after shim v1 is deprecated. - if c.config.NoPivot && ociRuntime.Type == linuxRuntime { - taskOpts = append(taskOpts, containerd.WithNoPivotRoot) - } - // We don't need stdio for sandbox container. - task, err := container.NewTask(ctx, containerdio.NullIO, taskOpts...) - if err != nil { - return status, errors.Wrap(err, "failed to create containerd task") - } - defer func() { - if retErr != nil { - deferCtx, deferCancel := ctrdutil.DeferContext() - defer deferCancel() - // Cleanup the sandbox container if an error is returned. - // It's possible that task is deleted by event monitor. - if _, err := task.Delete(deferCtx, containerd.WithProcessKill); err != nil && !errdefs.IsNotFound(err) { - logrus.WithError(err).Errorf("Failed to delete sandbox container %q", id) - } - } - }() - - if err := task.Start(ctx); err != nil { - return status, errors.Wrapf(err, "failed to start sandbox container task %q", id) - } - - // Set the pod sandbox as ready after successfully start sandbox container. - status.Pid = task.Pid() - status.State = sandboxstore.StateReady - return status, nil - }); err != nil { - return nil, errors.Wrap(err, "failed to start sandbox container") - } + // TaskOOM from containerd may come before sandbox is added to store, + // but we don't care about sandbox TaskOOM right now, so it is fine. + c.eventMonitor.startExitMonitor(context.Background(), id, task.Pid(), exitCh) return &runtime.RunPodSandboxResponse{PodSandboxId: id}, nil } diff --git a/vendor/github.com/containerd/cri/pkg/server/sandbox_stop.go b/vendor/github.com/containerd/cri/pkg/server/sandbox_stop.go index 7c522e309c..e1bb09e4c3 100644 --- a/vendor/github.com/containerd/cri/pkg/server/sandbox_stop.go +++ b/vendor/github.com/containerd/cri/pkg/server/sandbox_stop.go @@ -19,7 +19,6 @@ package server import ( "time" - "github.com/containerd/containerd" eventtypes "github.com/containerd/containerd/api/events" "github.com/containerd/containerd/errdefs" cni "github.com/containerd/go-cni" @@ -29,6 +28,7 @@ import ( "golang.org/x/sys/unix" runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" + ctrdutil "github.com/containerd/cri/pkg/containerd/util" sandboxstore "github.com/containerd/cri/pkg/store/sandbox" ) @@ -97,6 +97,7 @@ func (c *criService) StopPodSandbox(ctx context.Context, r *runtime.StopPodSandb // `task.Delete` is not called here because it will be called when // the event monitor handles the `TaskExit` event. func (c *criService) stopSandboxContainer(ctx context.Context, sandbox sandboxstore.Sandbox) error { + id := sandbox.ID container := sandbox.Container state := sandbox.Status.Get().State task, err := container.Task(ctx, nil) @@ -105,29 +106,35 @@ func (c *criService) stopSandboxContainer(ctx context.Context, sandbox sandboxst return errors.Wrap(err, "failed to get sandbox container") } // Don't return for unknown state, some cleanup needs to be done. - if state != sandboxstore.StateUnknown { - return nil + if state == sandboxstore.StateUnknown { + return cleanupUnknownSandbox(ctx, id, sandbox) } - // Task is an interface, explicitly set it to nil just in case. - task = nil + return nil } // Handle unknown state. // The cleanup logic is the same with container unknown state. if state == sandboxstore.StateUnknown { - status, err := getTaskStatus(ctx, task) + // Start an exit handler for containers in unknown state. + waitCtx, waitCancel := context.WithCancel(ctrdutil.NamespacedContext()) + defer waitCancel() + exitCh, err := task.Wait(waitCtx) if err != nil { - return errors.Wrapf(err, "failed to get task status for %q", sandbox.ID) - } - switch status.Status { - case containerd.Running, containerd.Created: - // The task is still running, continue stopping the task. - case containerd.Stopped: - // The task has exited, explicitly cleanup. - return cleanupUnknownSandbox(ctx, sandbox.ID, status, sandbox) - default: - return errors.Wrapf(err, "unsupported task status %q", status.Status) + if !errdefs.IsNotFound(err) { + return errors.Wrap(err, "failed to wait for task") + } + return cleanupUnknownSandbox(ctx, id, sandbox) } + + exitCtx, exitCancel := context.WithCancel(context.Background()) + stopCh := c.eventMonitor.startExitMonitor(exitCtx, id, task.Pid(), exitCh) + defer func() { + exitCancel() + // This ensures that exit monitor is stopped before + // `Wait` is cancelled, so no exit event is generated + // because of the `Wait` cancellation. + <-stopCh + }() } // Kill the sandbox container. @@ -166,14 +173,13 @@ func (c *criService) teardownPod(id string, path string, config *runtime.PodSand } // cleanupUnknownSandbox cleanup stopped sandbox in unknown state. -func cleanupUnknownSandbox(ctx context.Context, id string, status containerd.Status, - sandbox sandboxstore.Sandbox) error { +func cleanupUnknownSandbox(ctx context.Context, id string, sandbox sandboxstore.Sandbox) error { // Reuse handleSandboxExit to do the cleanup. return handleSandboxExit(ctx, &eventtypes.TaskExit{ ContainerID: id, ID: id, Pid: 0, - ExitStatus: status.ExitStatus, - ExitedAt: status.ExitTime, + ExitStatus: unknownExitCode, + ExitedAt: time.Now(), }, sandbox) } diff --git a/vendor/github.com/containerd/cri/pkg/store/container/status.go b/vendor/github.com/containerd/cri/pkg/store/container/status.go index b73a2ef66d..d074b659c3 100644 --- a/vendor/github.com/containerd/cri/pkg/store/container/status.go +++ b/vendor/github.com/containerd/cri/pkg/store/container/status.go @@ -88,6 +88,10 @@ type Status struct { // Human-readable message indicating details about why container is in its // current state. Message string + // Starting indicates that the container is in starting state. + // This field doesn't need to be checkpointed. + // TODO(now): Add unit test. + Starting bool `json:"-"` // Removing indicates that the container is in removing state. // This field doesn't need to be checkpointed. Removing bool `json:"-"` diff --git a/vendor/github.com/containerd/cri/pkg/store/sandbox/sandbox.go b/vendor/github.com/containerd/cri/pkg/store/sandbox/sandbox.go index 69189069f9..2fc01c5b35 100644 --- a/vendor/github.com/containerd/cri/pkg/store/sandbox/sandbox.go +++ b/vendor/github.com/containerd/cri/pkg/store/sandbox/sandbox.go @@ -86,22 +86,9 @@ func (s *Store) Add(sb Sandbox) error { return nil } -// Get returns the sandbox with specified id. Returns store.ErrNotExist -// if the sandbox doesn't exist. +// Get returns the sandbox with specified id. +// Returns store.ErrNotExist if the sandbox doesn't exist. func (s *Store) Get(id string) (Sandbox, error) { - sb, err := s.GetAll(id) - if err != nil { - return sb, err - } - if sb.Status.Get().State == StateInit { - return Sandbox{}, store.ErrNotExist - } - return sb, nil -} - -// GetAll returns the sandbox with specified id, including sandbox in unknown -// state. Returns store.ErrNotExist if the sandbox doesn't exist. -func (s *Store) GetAll(id string) (Sandbox, error) { s.lock.RLock() defer s.lock.RUnlock() id, err := s.idIndex.Get(id) @@ -123,9 +110,6 @@ func (s *Store) List() []Sandbox { defer s.lock.RUnlock() var sandboxes []Sandbox for _, sb := range s.sandboxes { - if sb.Status.Get().State == StateInit { - continue - } sandboxes = append(sandboxes, sb) } return sandboxes diff --git a/vendor/github.com/containerd/cri/pkg/store/sandbox/status.go b/vendor/github.com/containerd/cri/pkg/store/sandbox/status.go index 201b19ba47..c5480bb5da 100644 --- a/vendor/github.com/containerd/cri/pkg/store/sandbox/status.go +++ b/vendor/github.com/containerd/cri/pkg/store/sandbox/status.go @@ -26,11 +26,11 @@ import ( // | | // | Create(Run) | Load // | | -// Start +----v----+ | -// (failed) | | | -// +-------------+ INIT | +-----------+ -// | | | | | -// | +----+----+ | | +// Start | | +// (failed) | | +// +------------------+ +-----------+ +// | | | | +// | | | | // | | | | // | | Start(Run) | | // | | | | @@ -53,23 +53,17 @@ import ( // +-------------> DELETED // State is the sandbox state we use in containerd/cri. -// It includes init and unknown, which are internal states not defined in CRI. +// It includes unknown, which is internal states not defined in CRI. // The state mapping from internal states to CRI states: // * ready -> ready // * not ready -> not ready -// * init -> not exist // * unknown -> not ready type State uint32 const ( - // StateInit is init state of sandbox. Sandbox - // is in init state before its corresponding sandbox container - // is created. Sandbox in init state should be ignored by most - // functions, unless the caller needs to update sandbox state. - StateInit State = iota // StateReady is ready state, it means sandbox container // is running. - StateReady + StateReady = iota // StateNotReady is notready state, it ONLY means sandbox // container is not running. // StopPodSandbox should still be called for NOTREADY sandbox to diff --git a/vendor/github.com/containerd/cri/vendor.conf b/vendor/github.com/containerd/cri/vendor.conf index ba4ab0f2fe..33ad681ada 100644 --- a/vendor/github.com/containerd/cri/vendor.conf +++ b/vendor/github.com/containerd/cri/vendor.conf @@ -1,14 +1,14 @@ github.com/beorn7/perks 4c0e84591b9aa9e6dcfdf3e020114cd81f89d5f9 github.com/blang/semver v3.1.0 github.com/BurntSushi/toml a368813c5e648fee92e5f6c30e3944ff9d5e8895 -github.com/containerd/cgroups dbea6f2bd41658b84b00417ceefa416b979cbf10 +github.com/containerd/cgroups 4994991857f9b0ae8dc439551e8bebdbb4bf66c1 github.com/containerd/console c12b1e7919c14469339a5d38f2f8ed9b64a9de23 -github.com/containerd/containerd v1.2.5 +github.com/containerd/containerd v1.2.7 github.com/containerd/continuity bd77b46c8352f74eb12c85bdc01f4b90f69d66b4 github.com/containerd/fifo 3d5202aec260678c48179c56f40e6f38a095738c github.com/containerd/go-cni 40bcf8ec8acd7372be1d77031d585d5d8e561c90 github.com/containerd/go-runc 5a6d9f37cfa36b15efba46dc7ea349fa9b7143c3 -github.com/containerd/ttrpc 2a805f71863501300ae1976d29f0454ae003e85a +github.com/containerd/ttrpc f82148331ad2181edea8f3f649a1f7add6c3f9c2 github.com/containerd/typeurl a93fcdb778cd272c6e9b3028b2f42d813e785d40 github.com/containernetworking/cni v0.6.0 github.com/containernetworking/plugins v0.7.5 @@ -39,10 +39,10 @@ github.com/modern-go/concurrent 1.0.3 github.com/modern-go/reflect2 1.0.1 github.com/opencontainers/go-digest c9281466c8b2f606084ac71339773efd177436e7 github.com/opencontainers/image-spec v1.0.1 -github.com/opencontainers/runc 2b18fe1d885ee5083ef9f0838fee39b62d653e30 +github.com/opencontainers/runc v1.0.0-rc8 github.com/opencontainers/runtime-spec eba862dc2470385a233c7507392675cbeadf7353 github.com/opencontainers/runtime-tools v0.6.0 -github.com/opencontainers/selinux v1.2.1 +github.com/opencontainers/selinux v1.2.2 github.com/pkg/errors v0.8.0 github.com/pmezard/go-difflib v1.0.0 github.com/prometheus/client_golang f4fb1b73fb099f396a7f0036bf86aa8def4ed823 diff --git a/vendor/github.com/containernetworking/plugins/plugins/meta/portmap/chain.go b/vendor/github.com/containernetworking/plugins/plugins/meta/portmap/chain.go index 04c0478652..eb5531703f 100644 --- a/vendor/github.com/containernetworking/plugins/plugins/meta/portmap/chain.go +++ b/vendor/github.com/containernetworking/plugins/plugins/meta/portmap/chain.go @@ -29,6 +29,8 @@ type chain struct { entryRules [][]string // the rules that "point" to this chain rules [][]string // the rules this chain contains + + prependEntry bool // whether or not the entry rules should be prepended } // setup idempotently creates the chain. It will not error if the chain exists. @@ -45,19 +47,19 @@ func (c *chain) setup(ipt *iptables.IPTables) error { } // Add the rules to the chain - for i := len(c.rules) - 1; i >= 0; i-- { - if err := prependUnique(ipt, c.table, c.name, c.rules[i]); err != nil { + for _, rule := range c.rules { + if err := insertUnique(ipt, c.table, c.name, false, rule); err != nil { return err } } // Add the entry rules to the entry chains for _, entryChain := range c.entryChains { - for i := len(c.entryRules) - 1; i >= 0; i-- { + for _, rule := range c.entryRules { r := []string{} - r = append(r, c.entryRules[i]...) + r = append(r, rule...) r = append(r, "-j", c.name) - if err := prependUnique(ipt, c.table, entryChain, r); err != nil { + if err := insertUnique(ipt, c.table, entryChain, c.prependEntry, r); err != nil { return err } } @@ -105,8 +107,9 @@ func (c *chain) teardown(ipt *iptables.IPTables) error { return nil } -// prependUnique will prepend a rule to a chain, if it does not already exist -func prependUnique(ipt *iptables.IPTables, table, chain string, rule []string) error { +// insertUnique will add a rule to a chain if it does not already exist. +// By default the rule is appended, unless prepend is true. +func insertUnique(ipt *iptables.IPTables, table, chain string, prepend bool, rule []string) error { exists, err := ipt.Exists(table, chain, rule...) if err != nil { return err @@ -115,7 +118,11 @@ func prependUnique(ipt *iptables.IPTables, table, chain string, rule []string) e return nil } - return ipt.Insert(table, chain, 1, rule...) + if prepend { + return ipt.Insert(table, chain, 1, rule...) + } else { + return ipt.Append(table, chain, rule...) + } } func chainExists(ipt *iptables.IPTables, tableName, chainName string) (bool, error) { diff --git a/vendor/github.com/containernetworking/plugins/plugins/meta/portmap/portmap.go b/vendor/github.com/containernetworking/plugins/plugins/meta/portmap/portmap.go index 4a1672006f..6e7e33be1e 100644 --- a/vendor/github.com/containernetworking/plugins/plugins/meta/portmap/portmap.go +++ b/vendor/github.com/containernetworking/plugins/plugins/meta/portmap/portmap.go @@ -255,6 +255,10 @@ func genMarkMasqChain(markBit int) chain { table: "nat", name: MarkMasqChainName, entryChains: []string{"POSTROUTING"}, + // Only this entry chain needs to be prepended, because otherwise it is + // stomped on by the masquerading rules created by the CNI ptp and bridge + // plugins. + prependEntry: true, entryRules: [][]string{{ "-m", "comment", "--comment", "CNI portfwd requiring masquerade", diff --git a/vendor/github.com/opencontainers/selinux/VERSION b/vendor/github.com/opencontainers/selinux/VERSION index b6bb93f7c7..23aa839063 100644 --- a/vendor/github.com/opencontainers/selinux/VERSION +++ b/vendor/github.com/opencontainers/selinux/VERSION @@ -1 +1 @@ -1.3.0-dev +1.2.2 diff --git a/vendor/github.com/opencontainers/selinux/go-selinux/selinux_linux.go b/vendor/github.com/opencontainers/selinux/go-selinux/selinux_linux.go index 51fa8de68a..d7786c33c1 100644 --- a/vendor/github.com/opencontainers/selinux/go-selinux/selinux_linux.go +++ b/vendor/github.com/opencontainers/selinux/go-selinux/selinux_linux.go @@ -406,7 +406,14 @@ func SocketLabel() (string, error) { // SetKeyLabel takes a process label and tells the kernel to assign the // label to the next kernel keyring that gets created func SetKeyLabel(label string) error { - return writeCon("/proc/self/attr/keycreate", label) + err := writeCon("/proc/self/attr/keycreate", label) + if os.IsNotExist(err) { + return nil + } + if label == "" && os.IsPermission(err) && !GetEnabled() { + return nil + } + return err } // KeyLabel retrieves the current kernel keyring label setting