Add a helper tool to emit Makefile snippets

This loads the whole go universe one time, so should be faster that
repeated calls to `go list`

hack/make-rules/BUILD

@@ -106,7 +106,10 @@ filegroup(
 
 filegroup(
     name = "all-srcs",
-    srcs = [":package-srcs"],
+    srcs = [
+        ":package-srcs",
+        "//hack/make-rules/helpers/go2make:all-srcs",
+    ],
     tags = ["automanaged"],
     visibility = ["//visibility:public"],
 )

hack/make-rules/helpers/go2make/BUILD

@@ -0,0 +1,41 @@
+# gazelle:exclude testdata
+
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "go2make.go",
+        "pkgwalk.go",
+    ],
+    importpath = "k8s.io/kubernetes/hack/make-rules/helpers/go2make",
+    visibility = ["//visibility:private"],
+    deps = ["//vendor/github.com/spf13/pflag:go_default_library"],
+)
+
+go_binary(
+    name = "go2make",
+    embed = [":go_default_library"],
+    visibility = ["//visibility:public"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["pkgwalk_test.go"],
+    data = glob(["testdata/**"]),
+    embed = [":go_default_library"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

hack/make-rules/helpers/go2make/go2make.go

@@ -0,0 +1,215 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"bytes"
+	goflag "flag"
+	"fmt"
+	"go/build"
+	"io"
+	"os"
+	"sort"
+	"strings"
+
+	"github.com/spf13/pflag"
+)
+
+var flPrune = pflag.StringSlice("prune", nil, "sub-packages to prune (recursive, may be specified multiple times)")
+var flDebug = pflag.BoolP("debug", "d", false, "enable debugging output")
+var flHelp = pflag.BoolP("help", "h", false, "print help and exit")
+
+func main() {
+	pflag.CommandLine.AddGoFlagSet(goflag.CommandLine)
+	pflag.Usage = func() { help(os.Stderr) }
+	pflag.Parse()
+
+	debug("PWD", getwd())
+
+	build.Default.BuildTags = []string{"ignore_autogenerated"}
+	build.Default.UseAllFiles = false
+
+	if *flHelp {
+		help(os.Stdout)
+		os.Exit(0)
+	}
+	if len(pflag.Args()) == 0 {
+		help(os.Stderr)
+		os.Exit(1)
+	}
+	for _, in := range pflag.Args() {
+		if strings.HasSuffix(in, "/...") {
+			// Recurse.
+			debug("starting", in)
+			pkgName := strings.TrimSuffix(in, "/...")
+			if err := WalkPkg(pkgName, visitPkg); err != nil {
+				fmt.Fprintln(os.Stderr, err)
+				os.Exit(1)
+			}
+		} else {
+			// Import one package.
+			if err := saveImport(in); err != nil {
+				fmt.Fprintln(os.Stderr, err)
+				os.Exit(2)
+			}
+		}
+	}
+}
+
+func help(out io.Writer) {
+	fmt.Fprintf(out, "Usage: %s [FLAG...] <PKG...>\n", os.Args[0])
+	fmt.Fprintf(out, "\n")
+	fmt.Fprintf(out, "go2make calculates all of the dependencies of a set of Go packages and prints\n")
+	fmt.Fprintf(out, "them as variable definitions suitable for use as a Makefile.\n")
+	fmt.Fprintf(out, "\n")
+	fmt.Fprintf(out, "Package specifications may be simple (e.g. 'example.com/txt/color') or\n")
+	fmt.Fprintf(out, "recursive (e.g. 'example.com/txt/...')\n")
+	fmt.Fprintf(out, " Example:\n")
+	fmt.Fprintf(out, "  $ %s ./example.com/pretty\n", os.Args[0])
+	fmt.Fprintf(out, "  example.com/txt/split := \\\n")
+	fmt.Fprintf(out, "    /go/src/example.com/txt/split/ \\\n")
+	fmt.Fprintf(out, "    /go/src/example.com/txt/split/split.go \\\n")
+	fmt.Fprintf(out, "  ./example.com/pretty := \\\n")
+	fmt.Fprintf(out, "    /go/src/example.com/pretty/ \\\n")
+	fmt.Fprintf(out, "    /go/src/example.com/pretty/print.go \\\n")
+	fmt.Fprintf(out, "    /go/src/example.com/txt/split/ \\\n")
+	fmt.Fprintf(out, "    /go/src/example.com/txt/split/split.go\n")
+	fmt.Fprintf(out, "\n")
+	fmt.Fprintf(out, " Flags:\n")
+
+	pflag.PrintDefaults()
+}
+
+func debug(items ...interface{}) {
+	if *flDebug {
+		x := []interface{}{"DBG:"}
+		x = append(x, items...)
+		fmt.Println(x...)
+	}
+}
+
+func visitPkg(importPath, absPath string) error {
+	debug("visit", importPath)
+	return saveImport(importPath)
+}
+
+func prune(pkgName string) bool {
+	for _, pr := range *flPrune {
+		if pr == pkgName {
+			return true
+		}
+	}
+	return false
+}
+
+// cache keeps track of which packages we have already loaded.
+var cache = map[string]*build.Package{}
+
+func saveImport(pkgName string) error {
+	if cache[pkgName] != nil {
+		return nil
+	}
+	if prune(pkgName) {
+		debug("prune", pkgName)
+		return ErrSkipPkg
+	}
+	pkg, err := loadPackage(pkgName)
+	if err != nil {
+		return err
+	}
+	debug("save", pkgName)
+	cache[pkgName] = pkg
+
+	debug("recurse", pkgName)
+	defer func() { debug("done   ", pkgName) }()
+	if !pkg.Goroot && (len(pkg.GoFiles)+len(pkg.Imports) > 0) {
+		// Process deps of this package before the package itself.
+		for _, impName := range pkg.Imports {
+			if impName == "C" {
+				continue
+			}
+			debug("depends on", impName)
+			saveImport(impName)
+		}
+
+		// Emit a variable for each package.
+		var buf bytes.Buffer
+		buf.WriteString(pkgName)
+		buf.WriteString(" := ")
+
+		// Packages depend on their own directories, their own files, and
+		// transitive list of all deps' directories and files.
+		all := map[string]struct{}{}
+		all[pkg.Dir+"/"] = struct{}{}
+		filesForPkg(pkg, all)
+		for _, imp := range pkg.Imports {
+			pkg := cache[imp]
+			if pkg == nil || pkg.Goroot {
+				continue
+			}
+			all[pkg.Dir+"/"] = struct{}{}
+			filesForPkg(pkg, all)
+		}
+		// Sort and de-dup them.
+		files := flatten(all)
+		for _, f := range files {
+			buf.WriteString(" \\\n  ")
+			buf.WriteString(f)
+		}
+
+		fmt.Println(buf.String())
+	}
+	return nil
+}
+
+func filesForPkg(pkg *build.Package, all map[string]struct{}) {
+	for _, file := range pkg.GoFiles {
+		if pkg.Dir != "." {
+			file = pkg.Dir + "/" + file
+		}
+		all[file] = struct{}{}
+	}
+}
+
+func flatten(all map[string]struct{}) []string {
+	list := make([]string, 0, len(all))
+	for k := range all {
+		list = append(list, k)
+	}
+	sort.Strings(list)
+	return list
+}
+
+func loadPackage(pkgName string) (*build.Package, error) {
+	debug("load", pkgName)
+	pkg, err := build.Import(pkgName, getwd(), 0)
+	if err != nil {
+		// We can ignore NoGoError.  Anything else is real.
+		if _, ok := err.(*build.NoGoError); !ok {
+			return nil, err
+		}
+	}
+	return pkg, nil
+}
+
+func getwd() string {
+	pwd, err := os.Getwd()
+	if err != nil {
+		panic(fmt.Sprintf("can't get working directory: %v", err))
+	}
+	return pwd
+}

hack/make-rules/helpers/go2make/pkgwalk.go

@@ -0,0 +1,117 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"go/build"
+	"os"
+	"path"
+	"sort"
+)
+
+// VisitFunc is a function called by WalkPkg to examine a single package.
+type VisitFunc func(importPath string, absPath string) error
+
+// ErrSkipPkg can be returned by a VisitFunc to indicate that the package in
+// question should not be walked any further.
+var ErrSkipPkg = fmt.Errorf("package skipped")
+
+// WalkPkg recursively visits all packages under pkgName.  This is similar
+// to filepath.Walk, except that it follows symlinks.  A package is always
+// visited before the children of that package.  If visit returns ErrSkipPkg,
+// pkgName will not be walked.
+func WalkPkg(pkgName string, visit VisitFunc) error {
+	// Visit the package itself.
+	pkg, err := findPackage(pkgName)
+	if err != nil {
+		return err
+	}
+	if err := visit(pkg.ImportPath, pkg.Dir); err == ErrSkipPkg {
+		return nil
+	} else if err != nil {
+		return err
+	}
+
+	// Read all of the child dirents and find sub-packages.
+	infos, err := readDirInfos(pkg.Dir)
+	if err != nil {
+		return err
+	}
+	for _, info := range infos {
+		if !info.IsDir() {
+			continue
+		}
+		name := info.Name()
+		if name[0] == '_' || (len(name) > 1 && name[0] == '.') || name == "testdata" {
+			continue
+		}
+		// Don't use path.Join() because it drops leading `./` via path.Clean().
+		err := WalkPkg(pkgName+"/"+name, visit)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// findPackage finds a Go package.
+func findPackage(pkgName string) (*build.Package, error) {
+	debug("find", pkgName)
+	pkg, err := build.Import(pkgName, getwd(), build.FindOnly)
+	if err != nil {
+		return nil, err
+	}
+	return pkg, nil
+}
+
+// readDirInfos returns a list of os.FileInfo structures for the dirents under
+// dirPath.  The result list is sorted by name.  This is very similar to
+// ioutil.ReadDir, except that it follows symlinks.
+func readDirInfos(dirPath string) ([]os.FileInfo, error) {
+	names, err := readDirNames(dirPath)
+	if err != nil {
+		return nil, err
+	}
+	sort.Strings(names)
+
+	infos := make([]os.FileInfo, 0, len(names))
+	for _, n := range names {
+		info, err := os.Stat(path.Join(dirPath, n))
+		if err != nil {
+			return nil, err
+		}
+		infos = append(infos, info)
+	}
+	return infos, nil
+}
+
+// readDirNames returns a list of all dirents in dirPath.  The result list is
+// not sorted or filtered.
+func readDirNames(dirPath string) ([]string, error) {
+	d, err := os.Open(dirPath)
+	if err != nil {
+		return nil, err
+	}
+	defer d.Close()
+
+	names, err := d.Readdirnames(-1)
+	if err != nil {
+		return nil, err
+	}
+	return names, nil
+}

hack/make-rules/helpers/go2make/pkgwalk_test.go

@@ -0,0 +1,222 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"path"
+	"reflect"
+	"sort"
+	"testing"
+)
+
+func Test_WalkPkg(t *testing.T) {
+	testCases := []struct {
+		pkg      string
+		fail     bool
+		expected []string
+	}{
+		{
+			pkg:  "./testdata/nonexistent-dir",
+			fail: true,
+		},
+		{
+			pkg:      "./testdata/dir-without-gofiles",
+			expected: []string{"./testdata/dir-without-gofiles"},
+		},
+		{
+			pkg:      "./testdata/dir-with-gofiles",
+			expected: []string{"./testdata/dir-with-gofiles", "./testdata/dir-with-gofiles/subdir"},
+		},
+	}
+
+	for i, tc := range testCases {
+		visited := []string{}
+		err := WalkPkg(tc.pkg, func(imp, abs string) error {
+			if _, base := path.Split(imp); base == "skipme" {
+				return ErrSkipPkg
+			}
+			visited = append(visited, imp)
+			return nil
+		})
+		if err != nil && tc.fail {
+			continue
+		}
+		if err != nil {
+			t.Errorf("[%d] unexpected error: %v", i, err)
+			continue
+		}
+		if tc.fail {
+			t.Errorf("[%d] expected error", i)
+			continue
+		}
+		if !reflect.DeepEqual(visited, tc.expected) {
+			t.Errorf("[%d] unexpected results: %v", i, visited)
+		}
+	}
+}
+func Test_findPackage(t *testing.T) {
+	testCases := []struct {
+		pkg  string
+		fail bool
+	}{
+		{
+			pkg:  "./testdata/nonexistent-dir",
+			fail: true,
+		},
+		{
+			pkg: "./testdata/dir-without-gofiles",
+		},
+		{
+			pkg: "./testdata/dir-with-gofiles",
+		},
+	}
+
+	for i, tc := range testCases {
+		_, err := findPackage(tc.pkg)
+		if err != nil && tc.fail {
+			continue
+		}
+		if err != nil {
+			t.Errorf("[%d] unexpected error: %v", i, err)
+			continue
+		}
+		if tc.fail {
+			t.Errorf("[%d] expected error", i)
+			continue
+		}
+	}
+}
+
+func Test_readDirInfos(t *testing.T) {
+	testCases := []struct {
+		dir      string
+		fail     bool
+		expected map[string]bool
+	}{
+		{
+			dir:  "./testdata/nonexistent-dir",
+			fail: true,
+		},
+		{
+			dir:      "./testdata/dir-without-gofiles",
+			expected: map[string]bool{"README": true},
+		},
+		{
+			dir: "./testdata/dir-with-gofiles",
+			expected: map[string]bool{
+				"README":      true,
+				"foo.go":      true,
+				"bar.go":      true,
+				"subdir":      true,
+				"testdata":    true,
+				"_underscore": true,
+				".dot":        true,
+				"skipme":      true,
+			},
+		},
+	}
+
+	for i, tc := range testCases {
+		infos, err := readDirInfos(tc.dir)
+		if err != nil && tc.fail {
+			continue
+		}
+		if err != nil {
+			t.Errorf("[%d] unexpected error: %v", i, err)
+			continue
+		}
+		if tc.fail {
+			t.Errorf("[%d] expected error", i)
+			continue
+		}
+		result := make([]string, len(infos))
+		sorted := make([]string, len(infos))
+		for i, inf := range infos {
+			result[i] = inf.Name()
+			sorted[i] = inf.Name()
+		}
+		sort.Strings(sorted)
+		if !reflect.DeepEqual(result, sorted) {
+			t.Errorf("[%d] result was not sorted: %v", i, result)
+		}
+		for _, r := range result {
+			if !tc.expected[r] {
+				t.Errorf("[%d] got unexpected result: %s", i, r)
+			} else {
+				delete(tc.expected, r)
+			}
+		}
+		for r := range tc.expected {
+			t.Errorf("[%d] missing expected result: %s", i, r)
+		}
+	}
+}
+
+func Test_readDirNames(t *testing.T) {
+	testCases := []struct {
+		dir      string
+		fail     bool
+		expected map[string]bool
+	}{
+		{
+			dir:  "./testdata/nonexistent-dir",
+			fail: true,
+		},
+		{
+			dir:      "./testdata/dir-without-gofiles",
+			expected: map[string]bool{"README": true},
+		},
+		{
+			dir: "./testdata/dir-with-gofiles",
+			expected: map[string]bool{
+				"README":      true,
+				"foo.go":      true,
+				"bar.go":      true,
+				"subdir":      true,
+				"testdata":    true,
+				"_underscore": true,
+				".dot":        true,
+				"skipme":      true,
+			},
+		},
+	}
+
+	for i, tc := range testCases {
+		result, err := readDirNames(tc.dir)
+		if err != nil && tc.fail {
+			continue
+		}
+		if err != nil {
+			t.Errorf("[%d] unexpected error: %v", i, err)
+			continue
+		}
+		if tc.fail {
+			t.Errorf("[%d] expected error", i)
+			continue
+		}
+		for _, r := range result {
+			if !tc.expected[r] {
+				t.Errorf("[%d] got unexpected result: %s", i, r)
+			} else {
+				delete(tc.expected, r)
+			}
+		}
+		for r := range tc.expected {
+			t.Errorf("[%d] missing expected result: %s", i, r)
+		}
+	}
+}

hack/make-rules/helpers/go2make/testdata/dir-with-gofiles/.dot/README

@@ -0,0 +1 @@
+README

hack/make-rules/helpers/go2make/testdata/dir-with-gofiles/.dot/subsubdir/README

@@ -0,0 +1 @@
+README

hack/make-rules/helpers/go2make/testdata/dir-with-gofiles/README

@@ -0,0 +1 @@
+README

hack/make-rules/helpers/go2make/testdata/dir-with-gofiles/_underscore/README

@@ -0,0 +1 @@
+README

hack/make-rules/helpers/go2make/testdata/dir-with-gofiles/_underscore/subsubdir/README

@@ -0,0 +1 @@
+README

hack/make-rules/helpers/go2make/testdata/dir-with-gofiles/bar.go

@@ -0,0 +1,20 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gofiles
+
+func bar() {
+}

hack/make-rules/helpers/go2make/testdata/dir-with-gofiles/foo.go

@@ -0,0 +1,20 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gofiles
+
+func foo() {
+}

hack/make-rules/helpers/go2make/testdata/dir-with-gofiles/skipme/README

@@ -0,0 +1 @@
+README

hack/make-rules/helpers/go2make/testdata/dir-with-gofiles/skipme/subsubdir/README

@@ -0,0 +1 @@
+README

hack/make-rules/helpers/go2make/testdata/dir-with-gofiles/subdir/README

@@ -0,0 +1 @@
+README

hack/make-rules/helpers/go2make/testdata/dir-with-gofiles/testdata/README

@@ -0,0 +1 @@
+README

hack/make-rules/helpers/go2make/testdata/dir-with-gofiles/testdata/subsubdir/README

@@ -0,0 +1 @@
+README

hack/make-rules/helpers/go2make/testdata/dir-without-gofiles/README

@@ -0,0 +1 @@
+README