From 33b0702edc59bdb5a3f5d3ecb0e629b5154d544e Mon Sep 17 00:00:00 2001 From: Manjunath A Kumatagi Date: Thu, 6 Jul 2017 21:31:35 +0530 Subject: [PATCH] Multiarch support for pets images --- test/images/BUILD | 1 + test/images/pets/peer-finder/BASEIMAGE | 4 + test/images/pets/peer-finder/BUILD | 35 ++++++ .../Makefile => peer-finder/Dockerfile} | 15 +-- .../pets/{zookeeper => peer-finder}/Makefile | 20 ++-- test/images/pets/peer-finder/README.md | 39 ++++++ test/images/pets/peer-finder/VERSION | 1 + test/images/pets/peer-finder/peer-finder.go | 111 ++++++++++++++++++ test/images/pets/redis-installer/BASEIMAGE | 4 + .../{redis => redis-installer}/Dockerfile | 10 +- test/images/pets/redis-installer/Makefile | 25 ++++ .../pets/{redis => redis-installer}/README.md | 0 test/images/pets/redis-installer/VERSION | 1 + .../{redis => redis-installer}/install.sh | 0 .../{redis => redis-installer}/on-start.sh | 0 .../images/pets/zookeeper-installer/BASEIMAGE | 4 + .../Dockerfile | 10 +- test/images/pets/zookeeper-installer/Makefile | 25 ++++ .../README.md | 0 test/images/pets/zookeeper-installer/VERSION | 1 + .../install.sh | 0 .../on-start.sh | 0 22 files changed, 278 insertions(+), 28 deletions(-) create mode 100644 test/images/pets/peer-finder/BASEIMAGE create mode 100644 test/images/pets/peer-finder/BUILD rename test/images/pets/{redis/Makefile => peer-finder/Dockerfile} (72%) rename test/images/pets/{zookeeper => peer-finder}/Makefile (71%) create mode 100644 test/images/pets/peer-finder/README.md create mode 100644 test/images/pets/peer-finder/VERSION create mode 100644 test/images/pets/peer-finder/peer-finder.go create mode 100644 test/images/pets/redis-installer/BASEIMAGE rename test/images/pets/{redis => redis-installer}/Dockerfile (86%) create mode 100644 test/images/pets/redis-installer/Makefile rename test/images/pets/{redis => redis-installer}/README.md (100%) create mode 100644 test/images/pets/redis-installer/VERSION rename test/images/pets/{redis => redis-installer}/install.sh (100%) rename test/images/pets/{redis => redis-installer}/on-start.sh (100%) create mode 100644 test/images/pets/zookeeper-installer/BASEIMAGE rename test/images/pets/{zookeeper => zookeeper-installer}/Dockerfile (84%) create mode 100644 test/images/pets/zookeeper-installer/Makefile rename test/images/pets/{zookeeper => zookeeper-installer}/README.md (100%) create mode 100644 test/images/pets/zookeeper-installer/VERSION rename test/images/pets/{zookeeper => zookeeper-installer}/install.sh (100%) rename test/images/pets/{zookeeper => zookeeper-installer}/on-start.sh (100%) diff --git a/test/images/BUILD b/test/images/BUILD index 40a9c7db39..c39d264524 100644 --- a/test/images/BUILD +++ b/test/images/BUILD @@ -32,6 +32,7 @@ filegroup( "//test/images/no-snat-test:all-srcs", "//test/images/no-snat-test-proxy:all-srcs", "//test/images/nonewprivs:all-srcs", + "//test/images/pets/peer-finder:all-srcs", "//test/images/port-forward-tester:all-srcs", "//test/images/porter:all-srcs", "//test/images/resource-consumer:all-srcs", diff --git a/test/images/pets/peer-finder/BASEIMAGE b/test/images/pets/peer-finder/BASEIMAGE new file mode 100644 index 0000000000..40fc100e7a --- /dev/null +++ b/test/images/pets/peer-finder/BASEIMAGE @@ -0,0 +1,4 @@ +amd64=gcr.io/google-containers/debian-base-amd64:0.2 +arm=gcr.io/google-containers/debian-base-arm:0.2 +arm64=gcr.io/google-containers/debian-base-arm64:0.2 +ppc64le=gcr.io/google-containers/debian-base-ppc64le:0.2 diff --git a/test/images/pets/peer-finder/BUILD b/test/images/pets/peer-finder/BUILD new file mode 100644 index 0000000000..c62754211b --- /dev/null +++ b/test/images/pets/peer-finder/BUILD @@ -0,0 +1,35 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_binary", + "go_library", +) + +go_binary( + name = "peer-finder", + library = ":go_default_library", + tags = ["automanaged"], +) + +go_library( + name = "go_default_library", + srcs = ["peer-finder.go"], + tags = ["automanaged"], + deps = ["//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library"], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], +) diff --git a/test/images/pets/redis/Makefile b/test/images/pets/peer-finder/Dockerfile similarity index 72% rename from test/images/pets/redis/Makefile rename to test/images/pets/peer-finder/Dockerfile index edf42c18b0..b233e2cf21 100644 --- a/test/images/pets/redis/Makefile +++ b/test/images/pets/peer-finder/Dockerfile @@ -12,16 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -all: push +FROM BASEIMAGE -TAG = e2e -PREFIX = gcr.io/google_containers/redis-install-3.2.0 +CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/ -container: - docker build --pull -t $(PREFIX):$(TAG) . +RUN clean-install wget bash dnsutils -push: container - gcloud docker -- push $(PREFIX):$(TAG) +COPY peer-finder / -clean: - docker rmi $(PREFIX):$(TAG) +EXPOSE 9376 +ENTRYPOINT ["/peer-finder"] diff --git a/test/images/pets/zookeeper/Makefile b/test/images/pets/peer-finder/Makefile similarity index 71% rename from test/images/pets/zookeeper/Makefile rename to test/images/pets/peer-finder/Makefile index 410cad57a1..b76395dcd5 100644 --- a/test/images/pets/zookeeper/Makefile +++ b/test/images/pets/peer-finder/Makefile @@ -12,16 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -all: push +SRCS = peer-finder +ARCH ?= amd64 +TARGET ?= $(CURDIR) +GOLANG_VERSION ?= latest +SRC_DIR = pets/peer-finder +export -TAG = e2e -PREFIX = gcr.io/google_containers/zookeeper-install-3.5.0-alpha +bin: + ../../image-util.sh bin $(SRCS) -container: - docker build --pull -t $(PREFIX):$(TAG) . - -push: container - gcloud docker -- push $(PREFIX):$(TAG) - -clean: - docker rmi $(PREFIX):$(TAG) +.PHONY: bin diff --git a/test/images/pets/peer-finder/README.md b/test/images/pets/peer-finder/README.md new file mode 100644 index 0000000000..f523a57979 --- /dev/null +++ b/test/images/pets/peer-finder/README.md @@ -0,0 +1,39 @@ +# Peer finder + +This is a simple peer finder daemon that is useful with StatefulSet and related use cases. + +All it does is watch DNS for changes in the set of endpoints that are part of the governing service +of the PetSet. It periodically looks up the SRV record of the DNS entry that corresponds to a Kubernetes +Service which enumerates the set of peers for this the specified service. + +Be sure to use the `service.alpha.kubernetes.io/tolerate-unready-endpoints` on the governing service +of the StatefulSet so that all peers are listed in endpoints before any peers are started. + +There are several ways to bundle it with your main application. + +1. In an [init container](http://kubernetes.io/docs/user-guide/pods/init-container/), + to help your pod determine its peers when it it first started (determine the desired set of + peers from the governing service of the StatefulSet. For this use case, the `--on-start` option + can be used, but the `--on-change` option should not be used since the init container will no + longer be running after the pod is started. An example of an `--on-start` script would be to + edit a configuration file for the main app to insert the list of peers. This file needs to be + on a Volume shared between the init container and the main container. +2. In a sidecar (e.g. a second container in the same pod as the main app), in which case the `--on-change` + option can be used, but `--on-start` may not be useful without a way to guarantee the ordering + of the sidecar relative to the main app container. An example of an on-change script would be to + send an administrative command to the main container over the localhost network. (Note that signalling + is not practical since pods currently do not share a PID namespace). +3. As pid 1 of the main container, in which case both `--on-change` and `--on-start` may be used. + In this mode, the ordering of the peer-finder relative to the main app is ensured by having the peer + finder start the main app. An example script would be to modify a configuration file and send SIGHUP + to the main process. +4. Both 1 and 2. + +Options 1 and 2 and 4 may be preferable since they do not require changes to the main container image. +Option 3 is useful is signalling is necessary. + +The peer-finder tool is intended to help legacy applications run in containers on Kubernetes. +If possible, it may be preferable to modify an applications to poll DNS itself to determine its peer set. + +Not all StatefulSets are able to be scaled. For unscalable StatefulSets, only the on-start message is needed, and +so option 1 is a good choice. diff --git a/test/images/pets/peer-finder/VERSION b/test/images/pets/peer-finder/VERSION new file mode 100644 index 0000000000..d3827e75a5 --- /dev/null +++ b/test/images/pets/peer-finder/VERSION @@ -0,0 +1 @@ +1.0 diff --git a/test/images/pets/peer-finder/peer-finder.go b/test/images/pets/peer-finder/peer-finder.go new file mode 100644 index 0000000000..22dbfdca2b --- /dev/null +++ b/test/images/pets/peer-finder/peer-finder.go @@ -0,0 +1,111 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// A small utility program to lookup hostnames of endpoints in a service. +package main + +import ( + "flag" + "fmt" + "log" + "net" + "os" + "os/exec" + "sort" + "strings" + "time" + + "k8s.io/apimachinery/pkg/util/sets" +) + +const ( + pollPeriod = 1 * time.Second +) + +var ( + onChange = flag.String("on-change", "", "Script to run on change, must accept a new line separated list of peers via stdin.") + onStart = flag.String("on-start", "", "Script to run on start, must accept a new line separated list of peers via stdin.") + svc = flag.String("service", "", "Governing service responsible for the DNS records of the domain this pod is in.") + namespace = flag.String("ns", "", "The namespace this pod is running in. If unspecified, the POD_NAMESPACE env var is used.") + domain = flag.String("domain", "cluster.local", "The Cluster Domain which is used by the Cluster.") +) + +func lookup(svcName string) (sets.String, error) { + endpoints := sets.NewString() + _, srvRecords, err := net.LookupSRV("", "", svcName) + if err != nil { + return endpoints, err + } + for _, srvRecord := range srvRecords { + // The SRV records ends in a "." for the root domain + ep := fmt.Sprintf("%v", srvRecord.Target[:len(srvRecord.Target)-1]) + endpoints.Insert(ep) + } + return endpoints, nil +} + +func shellOut(sendStdin, script string) { + log.Printf("execing: %v with stdin: %v", script, sendStdin) + // TODO: Switch to sending stdin from go + out, err := exec.Command("bash", "-c", fmt.Sprintf("echo -e '%v' | %v", sendStdin, script)).CombinedOutput() + if err != nil { + log.Fatalf("Failed to execute %v: %v, err: %v", script, string(out), err) + } + log.Print(string(out)) +} + +func main() { + flag.Parse() + + ns := *namespace + if ns == "" { + ns = os.Getenv("POD_NAMESPACE") + } + if *svc == "" || ns == "" || (*onChange == "" && *onStart == "") { + log.Fatalf("Incomplete args, require -on-change and/or -on-start, -service and -ns or an env var for POD_NAMESPACE.") + } + + hostname, err := os.Hostname() + if err != nil { + log.Fatalf("Failed to get hostname: %s", err) + } + + svcLocalSuffix := strings.Join([]string{"svc", *domain}, ".") + myName := strings.Join([]string{hostname, *svc, ns, svcLocalSuffix}, ".") + script := *onStart + if script == "" { + script = *onChange + log.Printf("No on-start supplied, on-change %v will be applied on start.", script) + } + for newPeers, peers := sets.NewString(), sets.NewString(); script != ""; time.Sleep(pollPeriod) { + newPeers, err = lookup(*svc) + if err != nil { + log.Printf("%v", err) + continue + } + if newPeers.Equal(peers) || !newPeers.Has(myName) { + continue + } + peerList := newPeers.List() + sort.Strings(peerList) + log.Printf("Peer list updated\nwas %v\nnow %v", peers.List(), newPeers.List()) + shellOut(strings.Join(peerList, "\n"), script) + peers = newPeers + script = *onChange + } + // TODO: Exit if there's no on-change? + log.Printf("Peer finder exiting") +} diff --git a/test/images/pets/redis-installer/BASEIMAGE b/test/images/pets/redis-installer/BASEIMAGE new file mode 100644 index 0000000000..40fc100e7a --- /dev/null +++ b/test/images/pets/redis-installer/BASEIMAGE @@ -0,0 +1,4 @@ +amd64=gcr.io/google-containers/debian-base-amd64:0.2 +arm=gcr.io/google-containers/debian-base-arm:0.2 +arm64=gcr.io/google-containers/debian-base-arm64:0.2 +ppc64le=gcr.io/google-containers/debian-base-ppc64le:0.2 diff --git a/test/images/pets/redis/Dockerfile b/test/images/pets/redis-installer/Dockerfile similarity index 86% rename from test/images/pets/redis/Dockerfile rename to test/images/pets/redis-installer/Dockerfile index b29227fb08..0038c6bab1 100644 --- a/test/images/pets/redis/Dockerfile +++ b/test/images/pets/redis-installer/Dockerfile @@ -14,10 +14,12 @@ # TODO: get rid of bash dependency and switch to plain busybox. # The tar in busybox also doesn't seem to understand compression. -FROM debian:jessie +FROM BASEIMAGE + +CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/ # TODO: just use standard redis when there is one for 3.2.0. -RUN apt-get update && apt-get install -y wget make gcc +RUN clean-install wget make gcc libc-dev # See README.md RUN wget -qO /redis-3.2.0.tar.gz http://download.redis.io/releases/redis-3.2.0.tar.gz && \ @@ -31,8 +33,8 @@ RUN cd /tmp/redis-3.2.0 && make distclean && mkdir -p /redis && \ rm -rf /tmp/redis-3.2.0 ADD on-start.sh / -# See contrib/pets/peer-finder for details -RUN wget -qO /peer-finder https://storage.googleapis.com/kubernetes-release/pets/peer-finder + +COPY peer-finder / ADD install.sh / RUN chmod -c 755 /install.sh /on-start.sh /peer-finder Entrypoint ["/install.sh"] diff --git a/test/images/pets/redis-installer/Makefile b/test/images/pets/redis-installer/Makefile new file mode 100644 index 0000000000..b76395dcd5 --- /dev/null +++ b/test/images/pets/redis-installer/Makefile @@ -0,0 +1,25 @@ +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +SRCS = peer-finder +ARCH ?= amd64 +TARGET ?= $(CURDIR) +GOLANG_VERSION ?= latest +SRC_DIR = pets/peer-finder +export + +bin: + ../../image-util.sh bin $(SRCS) + +.PHONY: bin diff --git a/test/images/pets/redis/README.md b/test/images/pets/redis-installer/README.md similarity index 100% rename from test/images/pets/redis/README.md rename to test/images/pets/redis-installer/README.md diff --git a/test/images/pets/redis-installer/VERSION b/test/images/pets/redis-installer/VERSION new file mode 100644 index 0000000000..d3827e75a5 --- /dev/null +++ b/test/images/pets/redis-installer/VERSION @@ -0,0 +1 @@ +1.0 diff --git a/test/images/pets/redis/install.sh b/test/images/pets/redis-installer/install.sh similarity index 100% rename from test/images/pets/redis/install.sh rename to test/images/pets/redis-installer/install.sh diff --git a/test/images/pets/redis/on-start.sh b/test/images/pets/redis-installer/on-start.sh similarity index 100% rename from test/images/pets/redis/on-start.sh rename to test/images/pets/redis-installer/on-start.sh diff --git a/test/images/pets/zookeeper-installer/BASEIMAGE b/test/images/pets/zookeeper-installer/BASEIMAGE new file mode 100644 index 0000000000..40fc100e7a --- /dev/null +++ b/test/images/pets/zookeeper-installer/BASEIMAGE @@ -0,0 +1,4 @@ +amd64=gcr.io/google-containers/debian-base-amd64:0.2 +arm=gcr.io/google-containers/debian-base-arm:0.2 +arm64=gcr.io/google-containers/debian-base-arm64:0.2 +ppc64le=gcr.io/google-containers/debian-base-ppc64le:0.2 diff --git a/test/images/pets/zookeeper/Dockerfile b/test/images/pets/zookeeper-installer/Dockerfile similarity index 84% rename from test/images/pets/zookeeper/Dockerfile rename to test/images/pets/zookeeper-installer/Dockerfile index 780615933f..0843b858af 100644 --- a/test/images/pets/zookeeper/Dockerfile +++ b/test/images/pets/zookeeper-installer/Dockerfile @@ -14,13 +14,15 @@ # TODO: get rid of bash dependency and switch to plain busybox. # The tar in busybox also doesn't seem to understand compression. -FROM debian:jessie +FROM BASEIMAGE -RUN apt-get update && apt-get install -y wget netcat +CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/ + +RUN clean-install wget netcat ADD on-start.sh / -# See contrib/pets/peer-finder for details -RUN wget -qO /peer-finder https://storage.googleapis.com/kubernetes-release/pets/peer-finder + +COPY peer-finder / # See README.md RUN wget -q -O /zookeeper-3.5.0-alpha.tar.gz http://apache.mirrors.pair.com/zookeeper/zookeeper-3.5.0-alpha/zookeeper-3.5.0-alpha.tar.gz && \ diff --git a/test/images/pets/zookeeper-installer/Makefile b/test/images/pets/zookeeper-installer/Makefile new file mode 100644 index 0000000000..b76395dcd5 --- /dev/null +++ b/test/images/pets/zookeeper-installer/Makefile @@ -0,0 +1,25 @@ +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +SRCS = peer-finder +ARCH ?= amd64 +TARGET ?= $(CURDIR) +GOLANG_VERSION ?= latest +SRC_DIR = pets/peer-finder +export + +bin: + ../../image-util.sh bin $(SRCS) + +.PHONY: bin diff --git a/test/images/pets/zookeeper/README.md b/test/images/pets/zookeeper-installer/README.md similarity index 100% rename from test/images/pets/zookeeper/README.md rename to test/images/pets/zookeeper-installer/README.md diff --git a/test/images/pets/zookeeper-installer/VERSION b/test/images/pets/zookeeper-installer/VERSION new file mode 100644 index 0000000000..d3827e75a5 --- /dev/null +++ b/test/images/pets/zookeeper-installer/VERSION @@ -0,0 +1 @@ +1.0 diff --git a/test/images/pets/zookeeper/install.sh b/test/images/pets/zookeeper-installer/install.sh similarity index 100% rename from test/images/pets/zookeeper/install.sh rename to test/images/pets/zookeeper-installer/install.sh diff --git a/test/images/pets/zookeeper/on-start.sh b/test/images/pets/zookeeper-installer/on-start.sh similarity index 100% rename from test/images/pets/zookeeper/on-start.sh rename to test/images/pets/zookeeper-installer/on-start.sh