From 7ab1fc46001ccd0158c5b5cc0f2cfbd6250bf991 Mon Sep 17 00:00:00 2001 From: Mik Vyatskov Date: Tue, 11 Oct 2016 15:06:33 +0200 Subject: [PATCH] Add logs generator --- hack/.linted_packages | 1 + hack/verify-flags/known-flags.txt | 2 + test/images/logs-generator/Dockerfile | 23 ++++++ test/images/logs-generator/Makefile | 32 ++++++++ test/images/logs-generator/README.md | 57 ++++++++++++++ test/images/logs-generator/logs_generator.go | 82 ++++++++++++++++++++ 6 files changed, 197 insertions(+) create mode 100644 test/images/logs-generator/Dockerfile create mode 100644 test/images/logs-generator/Makefile create mode 100644 test/images/logs-generator/README.md create mode 100644 test/images/logs-generator/logs_generator.go diff --git a/hack/.linted_packages b/hack/.linted_packages index 19d5c94df2..a168357293 100644 --- a/hack/.linted_packages +++ b/hack/.linted_packages @@ -223,6 +223,7 @@ test/images/clusterapi-tester test/images/entrypoint-tester test/images/fakegitserver test/images/goproxy +test/images/logs-generator test/images/mount-tester test/images/n-way-http test/images/port-forward-tester diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index a4f1457368..6e6499072b 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -570,3 +570,5 @@ www-prefix zone-name garbage-collector-enabled viper-config +log-lines-total +run-duration diff --git a/test/images/logs-generator/Dockerfile b/test/images/logs-generator/Dockerfile new file mode 100644 index 0000000000..17990a9f65 --- /dev/null +++ b/test/images/logs-generator/Dockerfile @@ -0,0 +1,23 @@ +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM gcr.io/google_containers/ubuntu-slim:0.4 + +MAINTAINER Mik Vyatskov + +COPY logs-generator / + +CMD /logs-generator --logtostderr \ + --log-lines-total=${LOGS_GENERATOR_LINES_TOTAL} \ + --run-duration=${LOGS_GENERATOR_DURATION} \ No newline at end of file diff --git a/test/images/logs-generator/Makefile b/test/images/logs-generator/Makefile new file mode 100644 index 0000000000..a170e57062 --- /dev/null +++ b/test/images/logs-generator/Makefile @@ -0,0 +1,32 @@ +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +TAG = v0.1.0 +PREFIX = gcr.io/google_containers + +all: build + +build: binary container + +binary: + go build -a --ldflags '-w' -o logs-generator . + +container: + docker build -t $(PREFIX)/logs-generator:$(TAG) . + +push: + gcloud docker push $(PREFIX)/logs-generator:$(TAG) + +clean: + rm -f logs-generator \ No newline at end of file diff --git a/test/images/logs-generator/README.md b/test/images/logs-generator/README.md new file mode 100644 index 0000000000..9130ce2ceb --- /dev/null +++ b/test/images/logs-generator/README.md @@ -0,0 +1,57 @@ +# Logs Generator + +## Overview + +Logs generator is a tool to create predictable load on the logs delivery system. +Is generates random lines with predictable format and predictable average length. +Each line can be later uniquely identified to ensure logs delivery. + +## Usage + +Tool is parametrized with the total number of number that should be generated and the duration of +the generation process. For example, if you want to create a throughput of 100 lines per second +for a minute, you set total number of lines to 6000 and duration to 1 minute. + +Parameters are passed through environment variables. There are no defaults, you should always +set up container parameters. Total number of line is parametrized through env variable +`LOGS_GENERATOR_LINES_TOTAL` and duration in go format is parametrized through env variable +`LOGS_GENERATOR_DURATION`. + +Inside the container all log lines are written to the stdout. + +Each line is on average 100 bytes long and follows this pattern: + +``` +2000-12-31T12:59:59Z /api/v1/namespaces//endpoints/ +``` + +Where `` refers to the number from 0 to `total_lines - 1`, which is unique for each +line in a given run of the container. + +## Image + +Image is located in the public repository of Google Container Registry under the name + +``` +gcr.io/google_containers/logs-generator:v0.1.0 +``` + +## Examples + +``` +docker run -i \ + -e "LOGS_GENERATOR_LINES_TOTAL=10" \ + -e "LOGS_GENERATOR_DURATION=1s" \ + gcr.io/google_containers/logs-generator:v0.1.0 +``` + +``` +kubectl run logs-generator \ + --generator=run-pod/v1 \ + --image=gcr.io/google_containers/logs-generator:v0.1.0 \ + --restart=Never \ + --env "LOGS_GENERATOR_LINES_TOTAL=1000" \ + --env "LOGS_GENERATOR_DURATION=1m" +``` + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/test/images/logs-generator/README.md?pixel)]() \ No newline at end of file diff --git a/test/images/logs-generator/logs_generator.go b/test/images/logs-generator/logs_generator.go new file mode 100644 index 0000000000..4da580e0c5 --- /dev/null +++ b/test/images/logs-generator/logs_generator.go @@ -0,0 +1,82 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "flag" + "fmt" + "time" + + "github.com/golang/glog" + "k8s.io/kubernetes/pkg/util/rand" +) + +var ( + httpMethods = []string{ + "GET", + "POST", + "PUT", + } + namespaces = []string{ + "kube-system", + "default", + "ns", + } +) + +var ( + linesTotal = flag.Int("log-lines-total", 0, "Total lines that should be generated by the end of the run") + duration = flag.Duration("run-duration", 0, "Total duration of the run") +) + +func main() { + flag.Parse() + + if *linesTotal <= 0 { + glog.Fatalf("Invalid total number of lines: %d", *linesTotal) + } + + if *duration <= 0 { + glog.Fatalf("Invalid duration: %v", *duration) + } + + generateLogs(*linesTotal, *duration) +} + +// Outputs linesTotal lines of logs to stdout uniformly for duration +func generateLogs(linesTotal int, duration time.Duration) { + delay := duration / time.Duration(linesTotal) + rand.Seed(time.Now().UnixNano()) + + tick := time.Tick(delay) + for id := 0; id < linesTotal; id++ { + glog.Info(generateLogLine(id)) + <-tick + } +} + +// Generates apiserver-like line with average length of 100 symbols +func generateLogLine(id int) string { + method := httpMethods[rand.Intn(len(httpMethods))] + namespace := namespaces[rand.Intn(len(namespaces))] + + podName := rand.String(rand.IntnRange(3, 5)) + url := fmt.Sprintf("/api/v1/namespaces/%s/pods/%s", namespace, podName) + status := rand.IntnRange(200, 600) + + return fmt.Sprintf("%d %s %s %d", id, method, url, status) +}