github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #62151 from thockin/build-cleanup-with-go-1-10 

Automatic merge from submit-queue (batch tested with PRs 62495, 63003, 62829, 62151, 62002). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Build cleanup with go 1.10

Simplify our build/test scripts now that go 1.10 is in play.  It has caching that seems to actually work.

**Release note**:
```release-note
NONE
```
pull/8/head
Kubernetes Submit Queue 2018-04-23 22:45:24 -07:00 committed by GitHub
parent 1aa67bee2d 59ef5e2379
commit fce3ad5198
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 189 additions and 842 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

271
build/root/Makefile.generated_files
View File

@ -35,7 +35,7 @@ SHELL := /bin/bash
# This rule collects all the generated file sets into a single rule. Other
# rules should depend on this to ensure generated files are rebuilt.
.PHONY: generated_files
generated_files: gen_deepcopy gen_defaulter gen_conversion gen_openapi
generated_files: gen_deepcopy gen_defaulter gen_conversion gen_openapi gen_bindata
.PHONY: verify_generated_files
verify_generated_files: verify_gen_deepcopy \
@ -486,110 +486,6 @@ $(DEFAULTER_GEN):
hack/make-rules/build.sh ./vendor/k8s.io/code-generator/cmd/defaulter-gen
touch $@
#
# Open-api generation
#
# Any package that wants open-api functions generated must include a
# comment-tag in column 0 of one file of the form:
# // +k8s:openapi-gen=true
#
# The result file, in each pkg, of open-api generation.
OPENAPI_BASENAME := $(GENERATED_FILE_PREFIX)openapi
OPENAPI_FILENAME := $(OPENAPI_BASENAME).go
OPENAPI_OUTPUT_PKG := pkg/generated/openapi
# The tool used to generate open apis.
OPENAPI_GEN := $(BIN_DIR)/openapi-gen
# Find all the directories that request open-api generation.
ifeq ($(DBG_MAKEFILE),1)
$(warning ***** finding all +k8s:openapi-gen tags)
endif
OPENAPI_DIRS := $(shell \
grep --color=never -l '+k8s:openapi-gen=' $(ALL_K8S_TAG_FILES) \
| xargs -n1 dirname \
| LC_ALL=C sort -u \
)
OPENAPI_OUTFILE := $(OPENAPI_OUTPUT_PKG)/$(OPENAPI_FILENAME)
# This rule is the user-friendly entrypoint for openapi generation.
.PHONY: gen_openapi
gen_openapi: $(OPENAPI_OUTFILE) $(OPENAPI_GEN)
# For each dir in OPENAPI_DIRS, this establishes a dependency between the
# output file and the input files that should trigger a rebuild.
#
# Note that this is a deps-only statement, not a full rule (see below). This
# has to be done in a distinct step because wildcards don't work in static
# pattern rules.
#
# The '$(eval)' is needed because this has a different RHS for each LHS, and
# would otherwise produce results that make can't parse.
#
# We depend on the $(GOFILES_META).stamp to detect when the set of input files
# has changed. This allows us to detect deleted input files.
$(foreach dir, $(OPENAPI_DIRS), $(eval \
$(OPENAPI_OUTFILE): $(META_DIR)/$(dir)/$(GOFILES_META).stamp \
$(gofiles__$(dir)) \
))
# How to regenerate open-api code. This emits a single file for all results.
$(OPENAPI_OUTFILE): $(OPENAPI_GEN) $(OPENAPI_GEN)
function run_gen_openapi() { \
./hack/run-in-gopath.sh $(OPENAPI_GEN) \
--v $(KUBE_VERBOSE) \
--logtostderr \
-i $$(echo $(addprefix $(PRJ_SRC_PATH)/, $(OPENAPI_DIRS)) | sed 's/ /,/g') \
-p $(PRJ_SRC_PATH)/$(OPENAPI_OUTPUT_PKG) \
-O $(OPENAPI_BASENAME) \
"$$@"; \
}; \
run_gen_openapi
# This calculates the dependencies for the generator tool, so we only rebuild
# it when needed. It is PHONY so that it always runs, but it only updates the
# file if the contents have actually changed. We 'sinclude' this later.
.PHONY: $(META_DIR)/$(OPENAPI_GEN).mk
$(META_DIR)/$(OPENAPI_GEN).mk:
mkdir -p $(@D); \
(echo -n "$(OPENAPI_GEN): "; \
./hack/run-in-gopath.sh go list \
-f '{{.ImportPath}}{{"\n"}}{{range .Deps}}{{.}}{{"\n"}}{{end}}' \
./vendor/k8s.io/code-generator/cmd/openapi-gen \
| grep --color=never "^$(PRJ_SRC_PATH)/" \
| xargs ./hack/run-in-gopath.sh go list \
-f '{{$$d := .Dir}}{{$$d}}{{"\n"}}{{range .GoFiles}}{{$$d}}/{{.}}{{"\n"}}{{end}}' \
| paste -sd' ' - \
| sed 's/ / \\=,/g' \
| tr '=,' '\n\t' \
| sed "s|$$(pwd -P)/||"; \
) > $@.tmp; \
if ! cmp -s $@.tmp $@; then \
if [[ "$(DBG_CODEGEN)" == 1 ]]; then \
echo "DBG: $(OPENAPI_GEN).mk changed"; \
fi; \
cat $@.tmp > $@; \
rm -f $@.tmp; \
fi
# Include dependency info for the generator tool. This will cause the rule of
# the same name to be considered and if it is updated, make will restart.
sinclude $(META_DIR)/$(OPENAPI_GEN).mk
# How to build the generator tool. The deps for this are defined in
# the $(OPENAPI_GEN).mk, above.
#
# A word on the need to touch: This rule might trigger if, for example, a
# non-Go file was added or deleted from a directory on which this depends.
# This target needs to be reconsidered, but Go realizes it doesn't actually
# have to be rebuilt. In that case, make will forever see the dependency as
# newer than the binary, and try to rebuild it over and over. So we touch it,
# and make is happy.
$(OPENAPI_GEN):
hack/make-rules/build.sh ./vendor/k8s.io/code-generator/cmd/openapi-gen
touch $@
#
# Conversion generation
#
@ -805,3 +701,168 @@ sinclude $(META_DIR)/$(CONVERSION_GEN).mk
$(CONVERSION_GEN):
hack/make-rules/build.sh ./vendor/k8s.io/code-generator/cmd/conversion-gen
touch $@
#
# Open-api generation
#
# Any package that wants open-api functions generated must include a
# comment-tag in column 0 of one file of the form:
# // +k8s:openapi-gen=true
#
# The result file, in each pkg, of open-api generation.
OPENAPI_BASENAME := $(GENERATED_FILE_PREFIX)openapi
OPENAPI_FILENAME := $(OPENAPI_BASENAME).go
OPENAPI_OUTPUT_PKG := pkg/generated/openapi
# The tool used to generate open apis.
OPENAPI_GEN := $(BIN_DIR)/openapi-gen
# Find all the directories that request open-api generation.
ifeq ($(DBG_MAKEFILE),1)
$(warning ***** finding all +k8s:openapi-gen tags)
endif
OPENAPI_DIRS := $(shell \
grep --color=never -l '+k8s:openapi-gen=' $(ALL_K8S_TAG_FILES) \
| xargs -n1 dirname \
| LC_ALL=C sort -u \
)
OPENAPI_OUTFILE := $(OPENAPI_OUTPUT_PKG)/$(OPENAPI_FILENAME)
# This rule is the user-friendly entrypoint for openapi generation.
.PHONY: gen_openapi
gen_openapi: $(OPENAPI_OUTFILE) $(OPENAPI_GEN)
# For each dir in OPENAPI_DIRS, this establishes a dependency between the
# output file and the input files that should trigger a rebuild.
#
# Note that this is a deps-only statement, not a full rule (see below). This
# has to be done in a distinct step because wildcards don't work in static
# pattern rules.
#
# The '$(eval)' is needed because this has a different RHS for each LHS, and
# would otherwise produce results that make can't parse.
#
# We depend on the $(GOFILES_META).stamp to detect when the set of input files
# has changed. This allows us to detect deleted input files.
$(foreach dir, $(OPENAPI_DIRS), $(eval \
$(OPENAPI_OUTFILE): $(META_DIR)/$(dir)/$(GOFILES_META).stamp \
$(gofiles__$(dir)) \
))
# How to regenerate open-api code. This emits a single file for all results.
$(OPENAPI_OUTFILE): $(OPENAPI_GEN) $(OPENAPI_GEN)
function run_gen_openapi() { \
./hack/run-in-gopath.sh $(OPENAPI_GEN) \
--v $(KUBE_VERBOSE) \
--logtostderr \
-i $$(echo $(addprefix $(PRJ_SRC_PATH)/, $(OPENAPI_DIRS)) | sed 's/ /,/g') \
-p $(PRJ_SRC_PATH)/$(OPENAPI_OUTPUT_PKG) \
-O $(OPENAPI_BASENAME) \
"$$@"; \
}; \
run_gen_openapi
# This calculates the dependencies for the generator tool, so we only rebuild
# it when needed. It is PHONY so that it always runs, but it only updates the
# file if the contents have actually changed. We 'sinclude' this later.
.PHONY: $(META_DIR)/$(OPENAPI_GEN).mk
$(META_DIR)/$(OPENAPI_GEN).mk:
mkdir -p $(@D); \
(echo -n "$(OPENAPI_GEN): "; \
./hack/run-in-gopath.sh go list \
-f '{{.ImportPath}}{{"\n"}}{{range .Deps}}{{.}}{{"\n"}}{{end}}' \
./vendor/k8s.io/code-generator/cmd/openapi-gen \
| grep --color=never "^$(PRJ_SRC_PATH)/" \
| xargs ./hack/run-in-gopath.sh go list \
-f '{{$$d := .Dir}}{{$$d}}{{"\n"}}{{range .GoFiles}}{{$$d}}/{{.}}{{"\n"}}{{end}}' \
| paste -sd' ' - \
| sed 's/ / \\=,/g' \
| tr '=,' '\n\t' \
| sed "s|$$(pwd -P)/||"; \
) > $@.tmp; \
if ! cmp -s $@.tmp $@; then \
if [[ "$(DBG_CODEGEN)" == 1 ]]; then \
echo "DBG: $(OPENAPI_GEN).mk changed"; \
fi; \
cat $@.tmp > $@; \
rm -f $@.tmp; \
fi
# Include dependency info for the generator tool. This will cause the rule of
# the same name to be considered and if it is updated, make will restart.
sinclude $(META_DIR)/$(OPENAPI_GEN).mk
# How to build the generator tool. The deps for this are defined in
# the $(OPENAPI_GEN).mk, above.
#
# A word on the need to touch: This rule might trigger if, for example, a
# non-Go file was added or deleted from a directory on which this depends.
# This target needs to be reconsidered, but Go realizes it doesn't actually
# have to be rebuilt. In that case, make will forever see the dependency as
# newer than the binary, and try to rebuild it over and over. So we touch it,
# and make is happy.
$(OPENAPI_GEN):
hack/make-rules/build.sh ./vendor/k8s.io/code-generator/cmd/openapi-gen
touch $@
#
# bindata generation
#
# The tool used to generate bindata files.
BINDATA_GEN := $(BIN_DIR)/go-bindata
# A wrapper script that generates all bindata files. It is fast enough that we
# don't care.
BINDATA_SCRIPT := hack/generate-bindata.sh
# This rule is the user-friendly entrypoint for bindata generation.
.PHONY: gen_bindata
gen_bindata: $(BINDATA_GEN) FORCE
./hack/run-in-gopath.sh $(BINDATA_SCRIPT)
FORCE:
# This calculates the dependencies for the generator tool, so we only rebuild
# it when needed. It is PHONY so that it always runs, but it only updates the
# file if the contents have actually changed. We 'sinclude' this later.
.PHONY: $(META_DIR)/$(BINDATA_GEN).mk
$(META_DIR)/$(BINDATA_GEN).mk:
mkdir -p $(@D); \
(echo -n "$(BINDATA_GEN): "; \
./hack/run-in-gopath.sh go list \
-f '{{.ImportPath}}{{"\n"}}{{range .Deps}}{{.}}{{"\n"}}{{end}}' \
./vendor/github.com/jteeuwen/go-bindata/go-bindata \
| grep --color=never "^$(PRJ_SRC_PATH)/" \
| xargs ./hack/run-in-gopath.sh go list \
-f '{{$$d := .Dir}}{{$$d}}{{"\n"}}{{range .GoFiles}}{{$$d}}/{{.}}{{"\n"}}{{end}}' \
| paste -sd' ' - \
| sed 's/ / \\=,/g' \
| tr '=,' '\n\t' \
| sed "s|$$(pwd -P)/||"; \
) > $@.tmp; \
if ! cmp -s $@.tmp $@; then \
if [[ "$(DBG_CODEGEN)" == 1 ]]; then \
echo "DBG: $(BINDATA_GEN).mk changed"; \
fi; \
cat $@.tmp > $@; \
rm -f $@.tmp; \
fi
# Include dependency info for the generator tool. This will cause the rule of
# the same name to be considered and if it is updated, make will restart.
sinclude $(META_DIR)/$(BINDATA_GEN).mk
# How to build the generator tool. The deps for this are defined in
# the $(BINDATA_GEN).mk, above.
#
# A word on the need to touch: This rule might trigger if, for example, a
# non-Go file was added or deleted from a directory on which this depends.
# This target needs to be reconsidered, but Go realizes it doesn't actually
# have to be rebuilt. In that case, make will forever see the dependency as
# newer than the binary, and try to rebuild it over and over. So we touch it,
# and make is happy.
$(BINDATA_GEN):
hack/make-rules/build.sh ./vendor/github.com/jteeuwen/go-bindata/go-bindata
touch $@

1
hack/BUILD
View File

@ -19,7 +19,6 @@ filegroup(
srcs = [
":package-srcs",
"//hack/boilerplate:all-srcs",
"//hack/cmd/teststale:all-srcs",
"//hack/e2e-internal:all-srcs",
"//hack/lib:all-srcs",
"//hack/make-rules:all-srcs",

39
hack/cmd/teststale/BUILD
View File

@ -1,39 +0,0 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_binary",
"go_library",
"go_test",
)
go_binary(
name = "teststale",
embed = [":go_default_library"],
)
go_test(
name = "go_default_test",
srcs = ["teststale_test.go"],
embed = [":go_default_library"],
)
go_library(
name = "go_default_library",
srcs = ["teststale.go"],
importpath = "k8s.io/kubernetes/hack/cmd/teststale",
deps = ["//vendor/github.com/golang/glog:go_default_library"],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
)

209
hack/cmd/teststale/teststale.go
View File

@ -1,209 +0,0 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// teststale checks the staleness of a test binary. go test -c builds a test
// binary but it does no staleness check. In other words, every time one runs
// go test -c, it compiles the test packages and links the binary even when
// nothing has changed. This program helps to mitigate that problem by allowing
// to check the staleness of a given test package and its binary.
package main
import (
"encoding/json"
"flag"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"time"
"github.com/golang/glog"
)
const usageHelp = "" +
`This program checks the staleness of a given test package and its test
binary so that one can make a decision about re-building the test binary.
Usage:
teststale -binary=/path/to/test/binary -package=package
Example:
teststale -binary="$HOME/gosrc/bin/e2e.test" -package="k8s.io/kubernetes/test/e2e"
`
var (
binary = flag.String("binary", "", "filesystem path to the test binary file. Example: \"$HOME/gosrc/bin/e2e.test\"")
pkgPath = flag.String("package", "", "import path of the test package in the format used while importing packages. Example: \"k8s.io/kubernetes/test/e2e\"")
)
func usage() {
fmt.Fprintln(os.Stderr, usageHelp)
fmt.Fprintln(os.Stderr, "Flags:")
flag.PrintDefaults()
os.Exit(2)
}
// golist is an interface emulating the `go list` command to get package information.
// TODO: Evaluate using `go/build` package instead. It doesn't provide staleness
// information, but we can probably run `go list` and `go/build.Import()` concurrently
// in goroutines and merge the results. Evaluate if that's faster.
type golist interface {
pkgInfo(pkgPaths []string) ([]pkg, error)
}
// execmd implements the `golist` interface.
type execcmd struct {
cmd string
args []string
env []string
}
func (e *execcmd) pkgInfo(pkgPaths []string) ([]pkg, error) {
args := append(e.args, pkgPaths...)
cmd := exec.Command(e.cmd, args...)
cmd.Env = e.env
stdout, err := cmd.StdoutPipe()
if err != nil {
return nil, fmt.Errorf("failed to obtain the metadata output stream: %v", err)
}
dec := json.NewDecoder(stdout)
// Start executing the command
if err := cmd.Start(); err != nil {
return nil, fmt.Errorf("command did not start: %v", err)
}
var pkgs []pkg
for {
var p pkg
if err := dec.Decode(&p); err == io.EOF {
break
} else if err != nil {
return nil, fmt.Errorf("failed to unmarshal metadata for package %s: %v", p.ImportPath, err)
}
pkgs = append(pkgs, p)
}
if err := cmd.Wait(); err != nil {
return nil, fmt.Errorf("command did not complete: %v", err)
}
return pkgs, nil
}
type pkg struct {
Dir string
ImportPath string
Target string
Stale bool
TestGoFiles []string
TestImports []string
XTestGoFiles []string
XTestImports []string
}
func (p *pkg) isNewerThan(cmd golist, buildTime time.Time) bool {
// If the package itself is stale, then we have to rebuild the whole thing anyway.
if p.Stale {
return true
}
// Test for file staleness
for _, f := range p.TestGoFiles {
if isNewerThan(filepath.Join(p.Dir, f), buildTime) {
glog.V(4).Infof("test Go file %s is stale", f)
return true
}
}
for _, f := range p.XTestGoFiles {
if isNewerThan(filepath.Join(p.Dir, f), buildTime) {
glog.V(4).Infof("external test Go file %s is stale", f)
return true
}
}
imps := []string{}
imps = append(imps, p.TestImports...)
imps = append(imps, p.XTestImports...)
// This calls `go list` the second time. This is required because the first
// call to `go list` checks the staleness of the package in question by
// looking the non-test dependencies, but it doesn't look at the test
// dependencies. However, it returns the list of test dependencies. This
// second call to `go list` checks the staleness of all the test
// dependencies.
pkgs, err := cmd.pkgInfo(imps)
if err != nil || len(pkgs) < 1 {
glog.V(4).Infof("failed to obtain metadata for packages %s: %v", imps, err)
return true
}
for _, p := range pkgs {
if p.Stale {
glog.V(4).Infof("import %q is stale", p.ImportPath)
return true
}
}
return false
}
func isNewerThan(filename string, buildTime time.Time) bool {
stat, err := os.Stat(filename)
if err != nil {
return true
}
return stat.ModTime().After(buildTime)
}
// isTestStale checks if the test binary is stale and needs to rebuilt.
// Some of the ideas here are inspired by how Go does staleness checks.
func isTestStale(cmd golist, binPath, pkgPath string) bool {
bStat, err := os.Stat(binPath)
if err != nil {
glog.V(4).Infof("Couldn't obtain the modified time of the binary %s: %v", binPath, err)
return true
}
buildTime := bStat.ModTime()
pkgs, err := cmd.pkgInfo([]string{pkgPath})
if err != nil || len(pkgs) < 1 {
glog.V(4).Infof("Couldn't retrieve test package information for package %s: %v", pkgPath, err)
return false
}
return pkgs[0].isNewerThan(cmd, buildTime)
}
func main() {
flag.Usage = usage
flag.Parse()
cmd := &execcmd{
cmd: "go",
args: []string{
"list",
"-json",
},
env: os.Environ(),
}
if !isTestStale(cmd, *binary, *pkgPath) {
os.Exit(1)
}
}

325
hack/cmd/teststale/teststale_test.go
View File

@ -1,325 +0,0 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main
import (
"fmt"
"io/ioutil"
"math/rand"
"os"
"path"
"path/filepath"
"testing"
"time"
)
const (
// seed for rand.Source to generate data for files
seed int64 = 42
// 1K binary file
binLen = 1024
// Directory of the test package relative to $GOPATH
testImportDir = "example.com/proj/pkg"
)
var (
pastHour = time.Now().Add(-1 * time.Hour)
// The test package we are testing against
testPkg = path.Join(testImportDir, "test")
)
// fakegolist implements the `golist` interface providing fake package information for testing.
type fakegolist struct {
dir string
importMap map[string]pkg
testFiles []string
binfile string
}
func newFakegolist() (*fakegolist, error) {
dir, err := ioutil.TempDir("", "teststale")
if err != nil {
// test can't proceed without a temp directory.
return nil, fmt.Errorf("failed to create a temp directory for testing: %v", err)
}
// Set the temp directory as the $GOPATH
if err := os.Setenv("GOPATH", dir); err != nil {
// can't proceed without pointing the $GOPATH to the temp directory.
return nil, fmt.Errorf("failed to set \"$GOPATH\" pointing to %q: %v", dir, err)
}
// Setup $GOPATH directory layout.
// Yeah! I am bored of repeatedly writing "if err != nil {}"!
if os.MkdirAll(filepath.Join(dir, "bin"), 0750) != nil ||
os.MkdirAll(filepath.Join(dir, "pkg", "linux_amd64"), 0750) != nil ||
os.MkdirAll(filepath.Join(dir, "src"), 0750) != nil {
return nil, fmt.Errorf("failed to setup the $GOPATH directory structure")
}
// Create a temp file to represent the test binary.
binfile, err := ioutil.TempFile("", "testbin")
if err != nil {
return nil, fmt.Errorf("failed to create the temp file to represent the test binary: %v", err)
}
// Could have used crypto/rand instead, but it doesn't matter.
rr := rand.New(rand.NewSource(42))
bin := make([]byte, binLen)
if _, err = rr.Read(bin); err != nil {
return nil, fmt.Errorf("couldn't read from the random source: %v", err)
}
if _, err := binfile.Write(bin); err != nil {
return nil, fmt.Errorf("couldn't write to the binary file %q: %v", binfile.Name(), err)
}
if err := binfile.Close(); err != nil {
// It is arguable whether this should be fatal.
return nil, fmt.Errorf("failed to close the binary file %q: %v", binfile.Name(), err)
}
if err := os.Chtimes(binfile.Name(), time.Now(), time.Now()); err != nil {
return nil, fmt.Errorf("failed to modify the mtime of the binary file %q: %v", binfile.Name(), err)
}
// Create test source files directory.
testdir := filepath.Join(dir, "src", testPkg)
if err := os.MkdirAll(testdir, 0750); err != nil {
return nil, fmt.Errorf("failed to create test source directory %q: %v", testdir, err)
}
fgl := &fakegolist{
dir: dir,
importMap: map[string]pkg{
"example.com/proj/pkg/test": {
Dir: path.Join(dir, "src", testPkg),
ImportPath: testPkg,
Target: path.Join(dir, "pkg", "linux_amd64", testImportDir, "test.a"),
Stale: false,
TestGoFiles: []string{
"foo_test.go",
"bar_test.go",
},
TestImports: []string{
"example.com/proj/pkg/p1",
"example.com/proj/pkg/p1/c11",
"example.com/proj/pkg/p2",
"example.com/proj/cmd/p3/c12/c23",
"strings",
"testing",
},
XTestGoFiles: []string{
"xfoo_test.go",
"xbar_test.go",
"xbaz_test.go",
},
XTestImports: []string{
"example.com/proj/pkg/test",
"example.com/proj/pkg/p1",
"example.com/proj/cmd/p3/c12/c23",
"os",
"testing",
},
},
"example.com/proj/pkg/p1": {Stale: false},
"example.com/proj/pkg/p1/c11": {Stale: false},
"example.com/proj/pkg/p2": {Stale: false},
"example.com/proj/cmd/p3/c12/c23": {Stale: false},
"strings": {Stale: false},
"testing": {Stale: false},
"os": {Stale: false},
},
testFiles: []string{
"foo_test.go",
"bar_test.go",
"xfoo_test.go",
"xbar_test.go",
"xbaz_test.go",
},
binfile: binfile.Name(),
}
// Create test source files.
for _, fn := range fgl.testFiles {
fp := filepath.Join(testdir, fn)
if _, err := os.Create(fp); err != nil {
return nil, fmt.Errorf("failed to create the test file %q: %v", fp, err)
}
if err := os.Chtimes(fp, time.Now(), pastHour); err != nil {
return nil, fmt.Errorf("failed to modify the mtime of the test file %q: %v", binfile.Name(), err)
}
}
return fgl, nil
}
func (fgl *fakegolist) pkgInfo(pkgPaths []string) ([]pkg, error) {
var pkgs []pkg
for _, path := range pkgPaths {
p, ok := fgl.importMap[path]
if !ok {
return nil, fmt.Errorf("package %q not found", path)
}
pkgs = append(pkgs, p)
}
return pkgs, nil
}
func (fgl *fakegolist) chMtime(filename string, mtime time.Time) error {
for _, fn := range fgl.testFiles {
if fn == filename {
fp := filepath.Join(fgl.dir, "src", testPkg, fn)
if err := os.Chtimes(fp, time.Now(), mtime); err != nil {
return fmt.Errorf("failed to modify the mtime of %q: %v", filename, err)
}
return nil
}
}
return fmt.Errorf("file %q not found", filename)
}
func (fgl *fakegolist) chStale(pkg string, stale bool) error {
if p, ok := fgl.importMap[pkg]; ok {
p.Stale = stale
fgl.importMap[pkg] = p
return nil
}
return fmt.Errorf("package %q not found", pkg)
}
func (fgl *fakegolist) cleanup() {
os.RemoveAll(fgl.dir)
os.Remove(fgl.binfile)
}
func TestIsTestStale(t *testing.T) {
cases := []struct {
fileMtime map[string]time.Time
pkgStaleness map[string]bool
result bool
}{
// Basic test: binary is fresh, all modifications were before the binary was built.
{
result: false,
},
// A local test file is new, hence binary must be stale.
{
fileMtime: map[string]time.Time{
"foo_test.go": time.Now().Add(1 * time.Hour),
},
result: true,
},
// Test package is new, so binary must be stale.
{
pkgStaleness: map[string]bool{
"example.com/proj/pkg/test": true,
},
result: true,
},
// Test package dependencies are new, so binary must be stale.
{
pkgStaleness: map[string]bool{
"example.com/proj/cmd/p3/c12/c23": true,
"strings": true,
},
result: true,
},
// External test files are new, hence binary must be stale.
{
fileMtime: map[string]time.Time{
"xfoo_test.go": time.Now().Add(1 * time.Hour),
"xbar_test.go": time.Now().Add(2 * time.Hour),
},
result: true,
},
// External test dependency is new, so binary must be stale.
{
pkgStaleness: map[string]bool{
"os": true,
},
result: true,
},
// Multiple source files and dependencies are new, so binary must be stale.
{
fileMtime: map[string]time.Time{
"foo_test.go": time.Now().Add(1 * time.Hour),
"xfoo_test.go": time.Now().Add(2 * time.Hour),
"xbar_test.go": time.Now().Add(3 * time.Hour),
},
pkgStaleness: map[string]bool{
"example.com/proj/pkg/p1": true,
"example.com/proj/pkg/p1/c11": true,
"example.com/proj/pkg/p2": true,
"example.com/proj/cmd/p3/c12/c23": true,
"strings": true,
"os": true,
},
result: true,
},
// Everything is new, so binary must be stale.
{
fileMtime: map[string]time.Time{
"foo_test.go": time.Now().Add(3 * time.Hour),
"bar_test.go": time.Now().Add(1 * time.Hour),
"xfoo_test.go": time.Now().Add(2 * time.Hour),
"xbar_test.go": time.Now().Add(1 * time.Hour),
"xbaz_test.go": time.Now().Add(2 * time.Hour),
},
pkgStaleness: map[string]bool{
"example.com/proj/pkg/p1": true,
"example.com/proj/pkg/p1/c11": true,
"example.com/proj/pkg/p2": true,
"example.com/proj/cmd/p3/c12/c23": true,
"example.com/proj/pkg/test": true,
"strings": true,
"testing": true,
"os": true,
},
result: true,
},
}
for _, tc := range cases {
fgl, err := newFakegolist()
if err != nil {
t.Fatalf("failed to setup the test: %v", err)
}
defer fgl.cleanup()
for fn, mtime := range tc.fileMtime {
if err := fgl.chMtime(fn, mtime); err != nil {
t.Fatalf("failed to change the mtime of %q: %v", fn, err)
}
}
for pkg, stale := range tc.pkgStaleness {
if err := fgl.chStale(pkg, stale); err != nil {
t.Fatalf("failed to change the staleness of %q: %v", pkg, err)
}
}
if tc.result != isTestStale(fgl, fgl.binfile, testPkg) {
if tc.result {
t.Errorf("Expected test package %q to be stale", testPkg)
} else {
t.Errorf("Expected test package %q to be not stale", testPkg)
}
}
}
}

8
hack/generate-bindata.sh
View File

@ -18,10 +18,8 @@ set -o errexit
set -o pipefail
set -o nounset
if [[ -z "${KUBE_ROOT:-}" ]]; then
KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
fi
export KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
source "${KUBE_ROOT}/hack/lib/init.sh"
source "${KUBE_ROOT}/hack/lib/logging.sh"
if [[ ! -d "${KUBE_ROOT}/examples" ]]; then
@ -31,7 +29,7 @@ fi
# kube::golang::build_kube_toolchain installs the vendored go-bindata in
# $GOPATH/bin, so make sure that's explicitly part of our $PATH.
export PATH="${GOPATH}/bin:${PATH}"
export PATH="${KUBE_OUTPUT_BINPATH}:${PATH}"
if ! which go-bindata &>/dev/null ; then
echo "Cannot find go-bindata."

153
hack/lib/golang.sh
View File

@ -213,11 +213,6 @@ readonly KUBE_STATIC_LIBRARIES=(
kubectl
)
# Add any files with those //generate annotations in the array below.
readonly KUBE_BINDATAS=(
test/e2e/generated/gobindata_util.go
)
kube::golang::is_statically_linked_library() {
local e
for e in "${KUBE_STATIC_LIBRARIES[@]}"; do [[ "$1" == *"/$e" ]] && return 0; done;
@ -355,6 +350,7 @@ kube::golang::setup_env() {
kube::golang::create_gopath_tree
export GOPATH="${KUBE_GOPATH}"
export GOCACHE="${KUBE_GOPATH}/cache"
# Append KUBE_EXTRA_GOPATH to the GOPATH if it is defined.
if [[ -n ${KUBE_EXTRA_GOPATH:-} ]]; then
@ -416,52 +412,9 @@ kube::golang::place_bins() {
done
}
kube::golang::fallback_if_stdlib_not_installable() {
local go_root_dir=$(go env GOROOT);
local go_host_os=$(go env GOHOSTOS);
local go_host_arch=$(go env GOHOSTARCH);
local cgo_pkg_dir=${go_root_dir}/pkg/${go_host_os}_${go_host_arch}_cgo;
if [ -e ${cgo_pkg_dir} ]; then
return 0;
fi
if [ -w ${go_root_dir}/pkg ]; then
return 0;
fi
kube::log::status "+++ Warning: stdlib pkg with cgo flag not found.";
kube::log::status "+++ Warning: stdlib pkg cannot be rebuilt since ${go_root_dir}/pkg is not writable by `whoami`";
kube::log::status "+++ Warning: Make ${go_root_dir}/pkg writable for `whoami` for a one-time stdlib install, Or"
kube::log::status "+++ Warning: Rebuild stdlib using the command 'CGO_ENABLED=0 go install -a -installsuffix cgo std'";
kube::log::status "+++ Falling back to go build, which is slower";
use_go_build=true
}
# Builds the toolchain necessary for building kube. This needs to be
# built only on the host platform.
# TODO: Find this a proper home.
# Ideally, not a shell script because testing shell scripts is painful.
kube::golang::build_kube_toolchain() {
local targets=(
hack/cmd/teststale
vendor/github.com/jteeuwen/go-bindata/go-bindata
)
local binaries
binaries=($(kube::golang::binaries_from_targets "${targets[@]}"))
kube::log::status "Building the toolchain targets:" "${binaries[@]}"
go install "${goflags[@]:+${goflags[@]}}" \
-gcflags "${gogcflags}" \
-ldflags "${goldflags}" \
"${binaries[@]:+${binaries[@]}}"
}
# Try and replicate the native binary placement of go install without
# calling go install.
kube::golang::output_filename_for_binary() {
kube::golang::outfile_for_binary() {
local binary=$1
local platform=$2
local output_path="${KUBE_GOPATH}/bin"
@ -477,7 +430,6 @@ kube::golang::output_filename_for_binary() {
kube::golang::build_binaries_for_platform() {
local platform=$1
local use_go_build=${2-}
local -a statics=()
local -a nonstatics=()
@ -496,79 +448,25 @@ kube::golang::build_binaries_for_platform() {
done
if [[ "${#statics[@]}" != 0 ]]; then
kube::golang::fallback_if_stdlib_not_installable;
CGO_ENABLED=0 go install -installsuffix static "${goflags[@]:+${goflags[@]}}" \
-gcflags "${gogcflags}" \
-ldflags "${goldflags}" \
"${statics[@]:+${statics[@]}}"
fi
if [[ -n ${use_go_build:-} ]]; then
kube::log::progress " "
for binary in "${statics[@]:+${statics[@]}}"; do
local outfile=$(kube::golang::output_filename_for_binary "${binary}" "${platform}")
CGO_ENABLED=0 go build -o "${outfile}" \
"${goflags[@]:+${goflags[@]}}" \
-gcflags "${gogcflags}" \
-ldflags "${goldflags}" \
"${binary}"
kube::log::progress "*"
done
for binary in "${nonstatics[@]:+${nonstatics[@]}}"; do
local outfile=$(kube::golang::output_filename_for_binary "${binary}" "${platform}")
go build -o "${outfile}" \
"${goflags[@]:+${goflags[@]}}" \
-gcflags "${gogcflags}" \
-ldflags "${goldflags}" \
"${binary}"
kube::log::progress "*"
done
kube::log::progress "\n"
else
# Use go install.
if [[ "${#nonstatics[@]}" != 0 ]]; then
go install "${goflags[@]:+${goflags[@]}}" \
-gcflags "${gogcflags}" \
-ldflags "${goldflags}" \
"${nonstatics[@]:+${nonstatics[@]}}"
fi
if [[ "${#statics[@]}" != 0 ]]; then
CGO_ENABLED=0 go install -installsuffix cgo "${goflags[@]:+${goflags[@]}}" \
-gcflags "${gogcflags}" \
-ldflags "${goldflags}" \
"${statics[@]:+${statics[@]}}"
fi
if [[ "${#nonstatics[@]}" != 0 ]]; then
go install "${goflags[@]:+${goflags[@]}}" \
-gcflags "${gogcflags}" \
-ldflags "${goldflags}" \
"${nonstatics[@]:+${nonstatics[@]}}"
fi
for test in "${tests[@]:+${tests[@]}}"; do
local outfile=$(kube::golang::output_filename_for_binary "${test}" \
"${platform}")
local outfile=$(kube::golang::outfile_for_binary "${test}" "${platform}")
local testpkg="$(dirname ${test})"
# Staleness check always happens on the host machine, so we don't
# have to locate the `teststale` binaries for the other platforms.
# Since we place the host binaries in `$KUBE_GOPATH/bin`, we can
# assume that the binary exists there, if it exists at all.
# Otherwise, something has gone wrong with building the `teststale`
# binary and we should safely proceed building the test binaries
# assuming that they are stale. There is no good reason to error
# out.
if test -x "${KUBE_GOPATH}/bin/teststale" && ! "${KUBE_GOPATH}/bin/teststale" -binary "${outfile}" -package "${testpkg}"
then
continue
fi
# `go test -c` below directly builds the binary. It builds the packages,
# but it never installs them. `go test -i` only installs the dependencies
# of the test, but not the test package itself. So neither `go test -c`
# nor `go test -i` installs, for example, test/e2e.a. And without that,
# doing a staleness check on k8s.io/kubernetes/test/e2e package always
# returns true (always stale). And that's why we need to install the
# test package.
go install "${goflags[@]:+${goflags[@]}}" \
-gcflags "${gogcflags}" \
-ldflags "${goldflags}" \
"${testpkg}"
mkdir -p "$(dirname ${outfile})"
go test -i -c \
go test -c \
"${goflags[@]:+${goflags[@]}}" \
-gcflags "${gogcflags}" \
-ldflags "${goldflags}" \
@ -629,14 +527,11 @@ kube::golang::build_binaries() {
goldflags="${GOLDFLAGS:-} $(kube::version::ldflags)"
gogcflags="${GOGCFLAGS:-}"
local use_go_build
local -a targets=()
local arg
for arg; do
if [[ "${arg}" == "--use_go_build" ]]; then
use_go_build=true
elif [[ "${arg}" == -* ]]; then
if [[ "${arg}" == -* ]]; then
# Assume arguments starting with a dash are flags to pass to go.
goflags+=("${arg}")
else
@ -671,26 +566,14 @@ kube::golang::build_binaries() {
fi
fi
# First build the toolchain before building any other targets
kube::golang::build_kube_toolchain
kube::log::status "Generating bindata:" "${KUBE_BINDATAS[@]}"
for bindata in "${KUBE_BINDATAS[@]}"; do
# Only try to generate bindata if the file exists, since in some cases
# one-off builds of individual directories may exclude some files.
if [[ -f "${KUBE_ROOT}/${bindata}" ]]; then
go generate "${goflags[@]:+${goflags[@]}}" "${KUBE_ROOT}/${bindata}"
fi
done
if [[ "${parallel}" == "true" ]]; then
kube::log::status "Building go targets for {${platforms[*]}} in parallel (output will appear in a burst when complete):" "${targets[@]}"
local platform
for platform in "${platforms[@]}"; do (
kube::golang::set_platform_envs "${platform}"
kube::log::status "${platform}: go build started"
kube::golang::build_binaries_for_platform ${platform} ${use_go_build:-}
kube::log::status "${platform}: go build finished"
kube::log::status "${platform}: build started"
kube::golang::build_binaries_for_platform ${platform}
kube::log::status "${platform}: build finished"
) &> "/tmp//${platform//\//_}.build" &
done
@ -709,7 +592,7 @@ kube::golang::build_binaries() {
kube::log::status "Building go targets for ${platform}:" "${targets[@]}"
(
kube::golang::set_platform_envs "${platform}"
kube::golang::build_binaries_for_platform ${platform} ${use_go_build:-}
kube::golang::build_binaries_for_platform ${platform}
)
done
fi

18
hack/make-rules/test.sh
View File

@ -278,12 +278,6 @@ runTests() {
# command, which is much faster.
if [[ ! ${KUBE_COVER} =~ ^[yY]$ ]]; then
kube::log::status "Running tests without code coverage"
# `go test` does not install the things it builds. `go test -i` installs
# the build artifacts but doesn't run the tests. The two together provide
# a large speedup for tests that do not need to be rebuilt.
go test -i "${goflags[@]:+${goflags[@]}}" \
${KUBE_RACE} ${KUBE_TIMEOUT} "${@}" \
"${testargs[@]:+${testargs[@]}}"
go test "${goflags[@]:+${goflags[@]}}" \
${KUBE_RACE} ${KUBE_TIMEOUT} "${@}" \
"${testargs[@]:+${testargs[@]}}" \
@ -319,21 +313,11 @@ runTests() {
for path in $(echo $cover_ignore_dirs | sed 's/|/ /g'); do
echo -e "skipped\tk8s.io/kubernetes/$path"
done
#
# `go test` does not install the things it builds. `go test -i` installs
# the build artifacts but doesn't run the tests. The two together provide
# a large speedup for tests that do not need to be rebuilt.
printf "%s\n" "${@}" \
| grep -Ev $cover_ignore_dirs \
| xargs -I{} -n 1 -P ${KUBE_COVERPROCS} \
bash -c "set -o pipefail; _pkg=\"\$0\"; _pkg_out=\${_pkg//\//_}; \
go test -i ${goflags[@]:+${goflags[@]}} \
${KUBE_RACE} \
${KUBE_TIMEOUT} \
-cover -covermode=\"${KUBE_COVERMODE}\" \
-coverprofile=\"${cover_report_dir}/\${_pkg}/${cover_profile}\" \
\"\${_pkg}\" \
${testargs[@]:+${testargs[@]}}
go test ${goflags[@]:+${goflags[@]}} \
${KUBE_RACE} \
${KUBE_TIMEOUT} \

4
hack/verify-typecheck.sh
View File

@ -25,9 +25,7 @@ kube::golang::verify_go_version
cd "${KUBE_ROOT}"
if [[ ! -f test/e2e/generated/bindata.go ]]; then
make --no-print-directory -C "${KUBE_ROOT}" verify_generated_files
fi
make --no-print-directory -C "${KUBE_ROOT}" generated_files
ret=0
go run test/typecheck/main.go "$@" || ret=$?

2
test/e2e/generated/gobindata_util.go
View File

@ -16,8 +16,6 @@ limitations under the License.
package generated
//go:generate ../../../hack/generate-bindata.sh
import "github.com/golang/glog"
/*

1
test/test_owners.csv
View File

@ -544,7 +544,6 @@ k8s.io/kubernetes/cmd/libs/go2idl/go-to-protobuf/protobuf,smarterclayton,0,
k8s.io/kubernetes/cmd/libs/go2idl/openapi-gen/generators,davidopp,1,
k8s.io/kubernetes/examples,Random-Liu,0,
k8s.io/kubernetes/hack,thockin,1,
k8s.io/kubernetes/hack/cmd/teststale,thockin,1,
k8s.io/kubernetes/pkg/api,Q-Lee,1,
k8s.io/kubernetes/pkg/api/endpoints,cjcullen,1,
k8s.io/kubernetes/pkg/api/events,jlowdermilk,1,

1 name owner auto-assigned sig
544 k8s.io/kubernetes/cmd/libs/go2idl/openapi-gen/generators davidopp 1
545 k8s.io/kubernetes/examples Random-Liu 0
546 k8s.io/kubernetes/hack thockin 1
k8s.io/kubernetes/hack/cmd/teststale thockin 1
547 k8s.io/kubernetes/pkg/api Q-Lee 1
548 k8s.io/kubernetes/pkg/api/endpoints cjcullen 1
549 k8s.io/kubernetes/pkg/api/events jlowdermilk 1