diff --git a/hack/make-rules/verify.sh b/hack/make-rules/verify.sh index 0ba36b3d53..7ed9d3723e 100755 --- a/hack/make-rules/verify.sh +++ b/hack/make-rules/verify.sh @@ -30,6 +30,7 @@ EXCLUDED_PATTERNS=( "verify-linkcheck.sh" # runs in separate Jenkins job once per day due to high network usage "verify-test-owners.sh" # TODO(rmmh): figure out how to avoid endless conflicts "verify-*-dockerized.sh" # Don't run any scripts that intended to be run dockerized + "verify-typecheck.sh" # runs in separate typecheck job ) # Only run whitelisted fast checks in quick mode. diff --git a/hack/verify-typecheck.sh b/hack/verify-typecheck.sh new file mode 100755 index 0000000000..8380e54d27 --- /dev/null +++ b/hack/verify-typecheck.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# Copyright 2018 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. +source "${KUBE_ROOT}/hack/lib/init.sh" + +kube::golang::verify_go_version + +cd "${KUBE_ROOT}" + +ret=0 +go run test/typecheck/main.go "$@" || ret=$? +if [[ $ret -ne 0 ]]; then + echo "!!! Type Check has failed. This may cause cross platform build failures." >&2 + echo "!!! Please see https://git.k8s.io/kubernetes/test/typecheck for more information." >&2 + exit 1 +fi diff --git a/test/BUILD b/test/BUILD index aa84e47065..d8087c512e 100644 --- a/test/BUILD +++ b/test/BUILD @@ -21,6 +21,7 @@ filegroup( "//test/list:all-srcs", "//test/soak/cauldron:all-srcs", "//test/soak/serve_hostnames:all-srcs", + "//test/typecheck:all-srcs", "//test/utils:all-srcs", ], tags = ["automanaged"], diff --git a/test/typecheck/BUILD b/test/typecheck/BUILD new file mode 100644 index 0000000000..4b303e5a05 --- /dev/null +++ b/test/typecheck/BUILD @@ -0,0 +1,51 @@ +package(default_visibility = ["//visibility:public"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_binary", + "go_library", + "go_test", +) + +go_binary( + name = "list", + embed = [":go_default_library"], +) + +go_library( + name = "go_default_library", + srcs = ["main.go"], + importpath = "k8s.io/kubernetes/test/typecheck", + deps = [ + "//test/typecheck/srcimporter:go_default_library", + "//third_party/forked/golang/go/types:go_default_library", + "//vendor/golang.org/x/crypto/ssh/terminal:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [ + ":package-srcs", + "//test/typecheck/srcimporter:all-srcs", + ], + tags = ["automanaged"], +) + +go_binary( + name = "typecheck", + embed = [":go_default_library"], +) + +go_test( + name = "go_default_test", + srcs = ["main_test.go"], + embed = [":go_default_library"], +) diff --git a/test/typecheck/README b/test/typecheck/README new file mode 100644 index 0000000000..986bbd785b --- /dev/null +++ b/test/typecheck/README @@ -0,0 +1,23 @@ +Typecheck does cross-platform typechecking of source code for all Go build +platforms. + +The primary benefit is speed: a full Kubernetes cross-build takes 20 minutes +and >40GB of RAM, while this takes under 2 minutes and <8GB of RAM. + +It uses Go's built-in parsing and typechecking libraries (go/parser and +go/types), which unfortunately are not what the go compiler uses. Occasional +mismatches will occur, but overall they correspond closely. + +Failures can be ignored if they don't block the build: + +Things go/types errors on that go build doesn't: + True errors (according to the spec): + These should be fixed whenever possible. Ignore if a fix isn't possible + or is in progress (e.g., vendored code). + - unused variables in closures + False errors: + These should be ignored and reported upstream if applicable. + - type checking mismatches between staging and generated types +Things go build fails on that we don't: + Please send examples of this to rmmh@ and extend this list. + - CGo errors, including syntax and linker errors. diff --git a/test/typecheck/main.go b/test/typecheck/main.go new file mode 100644 index 0000000000..e9e885cb31 --- /dev/null +++ b/test/typecheck/main.go @@ -0,0 +1,347 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// do a fast type check of kubernetes code, for all platforms. +package main + +import ( + "flag" + "fmt" + "go/ast" + "go/build" + "go/parser" + "go/token" + "log" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "sync/atomic" + "time" + + "golang.org/x/crypto/ssh/terminal" + // TODO(rmmh): remove this when golang/go#23712 is fixed, and the + // fix is the current minimum Go version to build Kubernetes. + "k8s.io/kubernetes/test/typecheck/srcimporter" + "k8s.io/kubernetes/third_party/forked/golang/go/types" +) + +var ( + verbose = flag.Bool("verbose", false, "print more information") + cross = flag.Bool("cross", true, "build for all platforms") + platforms = flag.String("platform", "", "comma-separated list of platforms to typecheck") + timings = flag.Bool("time", false, "output times taken for each phase") + defuses = flag.Bool("defuse", false, "output defs/uses") + serial = flag.Bool("serial", false, "don't type check platforms in parallel") + + isTerminal = terminal.IsTerminal(int(os.Stdout.Fd())) + logPrefix = "" + + // When processed in order, windows and darwin are early to make + // interesting OS-based errors happen earlier. + crossPlatforms = []string{ + "linux/amd64", "windows/386", + "darwin/amd64", "linux/arm", + "linux/386", "windows/amd64", + "linux/arm64", "linux/ppc64le", + "linux/s390x", "darwin/386", + } +) + +type analyzer struct { + fset *token.FileSet // positions are relative to fset + conf types.Config + ctx build.Context + failed bool + platform string + donePaths map[string]interface{} +} + +func newAnalyzer(platform string) *analyzer { + ctx := build.Default + platSplit := strings.Split(platform, "/") + ctx.GOOS, ctx.GOARCH = platSplit[0], platSplit[1] + ctx.CgoEnabled = true + + a := &analyzer{ + platform: platform, + fset: token.NewFileSet(), + ctx: ctx, + donePaths: make(map[string]interface{}), + } + a.conf = types.Config{ + FakeImportC: true, + Error: a.handleError, + Sizes: types.SizesFor("gc", a.ctx.GOARCH), + } + + a.conf.Importer = srcimporter.New( + &a.ctx, a.fset, make(map[string]*types.Package)) + + if *verbose { + fmt.Printf("context: %#v\n", ctx) + } + + return a +} + +func (a *analyzer) handleError(err error) { + if e, ok := err.(types.Error); ok { + // useful for some ignores: + // path := e.Fset.Position(e.Pos).String() + ignore := false + // TODO(rmmh): read ignores from a file, so this code can + // be Kubernetes-agnostic. Unused ignores should be treated as + // errors, to ensure coverage isn't overly broad. + if strings.Contains(e.Msg, "GetOpenAPIDefinitions") { + // TODO(rmmh): figure out why this happens. + // cmd/kube-apiserver/app/server.go:392:70 + // test/integration/framework/master_utils.go:131:84 + ignore = true + } + if ignore { + if *verbose { + fmt.Println("ignoring error:", err) + } + return + } + } + // TODO(rmmh): dedup errors across platforms? + fmt.Fprintf(os.Stderr, "%sERROR(%s) %s\n", logPrefix, a.platform, err) + a.failed = true +} + +// collect extracts test metadata from a file. +func (a *analyzer) collect(dir string) { + if _, ok := a.donePaths[dir]; ok { + return + } + a.donePaths[dir] = nil + + // Create the AST by parsing src. + fs, err := parser.ParseDir(a.fset, dir, nil, parser.AllErrors) + + if err != nil { + fmt.Println(logPrefix+"ERROR(syntax)", err) + a.failed = true + return + } + + if len(fs) > 1 && *verbose { + fmt.Println("multiple packages in dir:", dir) + } + + for _, p := range fs { + // returns first error, but a.handleError deals with it + files := a.filterFiles(p.Files) + if *verbose { + fmt.Printf("path: %s package: %s files: ", dir, p.Name) + for _, f := range files { + fname := filepath.Base(a.fset.File(f.Pos()).Name()) + fmt.Printf("%s ", fname) + } + fmt.Printf("\n") + } + a.typeCheck(dir, files) + } +} + +// filterFiles restricts a list of files to only those that should be built by +// the current platform. This includes both build suffixes (_windows.go) and build +// tags ("// +build !linux" at the beginning). +func (a *analyzer) filterFiles(fs map[string]*ast.File) []*ast.File { + files := []*ast.File{} + for _, f := range fs { + fpath := a.fset.File(f.Pos()).Name() + dir, name := filepath.Split(fpath) + matches, err := a.ctx.MatchFile(dir, name) + if err != nil { + fmt.Fprintf(os.Stderr, "%sERROR reading %s: %s\n", logPrefix, fpath, err) + a.failed = true + continue + } + if matches { + files = append(files, f) + } + } + return files +} + +func (a *analyzer) typeCheck(dir string, files []*ast.File) error { + info := types.Info{ + Defs: make(map[*ast.Ident]types.Object), + Uses: make(map[*ast.Ident]types.Object), + } + + // NOTE: this type check does a *recursive* import, but srcimporter + // doesn't do a full type check (ignores function bodies)-- this has + // some additional overhead. + // + // This means that we need to ensure that typeCheck runs on all + // code we will be compiling. + // + // TODO(rmmh): Customize our forked srcimporter to do this better. + pkg, err := a.conf.Check(dir, a.fset, files, &info) + if err != nil { + return err // type error + } + + // A significant fraction of vendored code only compiles on Linux, + // but it's only imported by code that has build-guards for Linux. + // Track vendored code to type-check it in a second pass. + for _, imp := range pkg.Imports() { + if strings.HasPrefix(imp.Path(), "k8s.io/kubernetes/vendor/") { + vendorPath := imp.Path()[len("k8s.io/kubernetes/"):] + if *verbose { + fmt.Println("recursively checking vendor path:", vendorPath) + } + a.collect(vendorPath) + } + } + + if *defuses { + for id, obj := range info.Defs { + fmt.Printf("%s: %q defines %v\n", + a.fset.Position(id.Pos()), id.Name, obj) + } + for id, obj := range info.Uses { + fmt.Printf("%s: %q uses %v\n", + a.fset.Position(id.Pos()), id.Name, obj) + } + } + + return nil +} + +type collector struct { + dirs []string +} + +// handlePath walks the filesystem recursively, collecting directories, +// ignoring some unneeded directories (hidden/vendored) that are handled +// specially later. +func (c *collector) handlePath(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + // Ignore hidden directories (.git, .cache, etc) + if len(path) > 1 && path[0] == '.' || + // Staging code is symlinked from vendor/k8s.io, and uses import + // paths as if it were inside of vendor/. It fails typechecking + // inside of staging/, but works when typechecked as part of vendor/. + path == "staging" || + // OS-specific vendor code tends to be imported by OS-specific + // packages. We recursively typecheck imported vendored packages for + // each OS, but don't typecheck everything for every OS. + path == "vendor" || + path == "_output" || + // This is a weird one. /testdata/ is *mostly* ignored by Go, + // and this translates to kubernetes/vendor not working. + // edit/record.go doesn't compile without gopkg.in/yaml.v2 + // in $GOSRC/$GOROOT (both typecheck and the shell script). + path == "pkg/kubectl/cmd/testdata/edit" { + return filepath.SkipDir + } + c.dirs = append(c.dirs, path) + } + return nil +} + +func main() { + flag.Parse() + args := flag.Args() + + if *verbose { + *serial = true // to avoid confusing interleaved logs + } + + if len(args) == 0 { + args = append(args, ".") + } + + c := collector{} + for _, arg := range args { + err := filepath.Walk(arg, c.handlePath) + if err != nil { + log.Fatalf("Error walking: %v", err) + } + } + sort.Strings(c.dirs) + + ps := crossPlatforms[:] + if *platforms != "" { + ps = strings.Split(*platforms, ",") + } else if !*cross { + ps = ps[:1] + } + + fmt.Println("type-checking: ", strings.Join(ps, ", ")) + + var wg sync.WaitGroup + var processedDirs int64 + var currentWork int64 // (dir_index << 8) | platform_index + statuses := make([]int, len(ps)) + for i, p := range ps { + wg.Add(1) + fn := func(i int, p string) { + start := time.Now() + a := newAnalyzer(p) + for n, dir := range c.dirs { + a.collect(dir) + atomic.AddInt64(&processedDirs, 1) + atomic.StoreInt64(¤tWork, int64(n<<8|i)) + } + if a.failed { + statuses[i] = 1 + } + if *timings { + fmt.Printf("%s took %.1fs\n", p, time.Since(start).Seconds()) + } + wg.Done() + } + if *serial { + fn(i, p) + } else { + go fn(i, p) + } + } + if isTerminal { + logPrefix = "\r" // clear status bar when printing + // Display a status bar so devs can estimate completion times. + go func() { + total := len(ps) * len(c.dirs) + for proc := 0; proc < total; proc = int(atomic.LoadInt64(&processedDirs)) { + work := atomic.LoadInt64(¤tWork) + dir := c.dirs[work>>8] + platform := ps[work&0xFF] + if len(dir) > 80 { + dir = dir[:80] + } + fmt.Printf("\r%d/%d \033[2m%-13s\033[0m %-80s", proc, total, platform, dir) + time.Sleep(50 * time.Millisecond) + } + }() + } + wg.Wait() + fmt.Println() + for _, status := range statuses { + if status != 0 { + os.Exit(status) + } + } +} diff --git a/test/typecheck/main_test.go b/test/typecheck/main_test.go new file mode 100644 index 0000000000..803963f2a8 --- /dev/null +++ b/test/typecheck/main_test.go @@ -0,0 +1,157 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "errors" + "fmt" + "go/ast" + "go/parser" + "io/ioutil" + "os" + "path/filepath" + "strings" + "testing" +) + +var packageCases = []struct { + code string + errs map[string]string +}{ + // Empty: no problems! + {"", map[string]string{"linux/amd64": ""}}, + // Slightly less empty: no problems! + {"func getRandomNumber() int { return 4; }", map[string]string{"darwin/386": ""}}, + // Fixed in #59243 + {`import "golang.org/x/sys/unix" + func f(err error) { + if err != unix.ENXIO { + panic("woops") + } + }`, map[string]string{"linux/amd64": "", "windows/amd64": "test.go:4:13: ENXIO not declared by package unix"}}, + // Fixed in #51984 + {`import "golang.org/x/sys/unix" + const linuxHugetlbfsMagic = 0x958458f6 + func IsHugeTlbfs() bool { + buf := unix.Statfs_t{} + unix.Statfs("/tmp/", &buf) + return buf.Type == linuxHugetlbfsMagic + }`, map[string]string{ + "linux/amd64": "", + "linux/386": "test.go:7:22: linuxHugetlbfsMagic (untyped int constant 2508478710) overflows int32", + }}, + // Fixed in #51873 + {`var a = map[string]interface{}{"num1": 9223372036854775807}`, + map[string]string{"linux/arm": "test.go:2:40: 9223372036854775807 (untyped int constant) overflows int"}}, +} + +var testFiles = map[string]string{ + "golang.org/x/sys/unix/empty.go": `package unix`, + "golang.org/x/sys/unix/errno_linux.go": `// +build linux + package unix + + type Errno string + func (e Errno) Error() string { return string(e) } + + var ENXIO = Errno("3")`, + "golang.org/x/sys/unix/ztypes_linux_amd64.go": `// +build amd64,linux + package unix + type Statfs_t struct { + Type int64 + } + func Statfs(path string, statfs *Statfs_t) {} + `, + "golang.org/x/sys/unix/ztypes_linux_386.go": `// +build i386,linux + package unix + type Statfs_t struct { + Type int32 + } + func Statfs(path string, statfs *Statfs_t) {} + `, +} + +func TestHandlePackage(t *testing.T) { + // When running in Bazel, we don't have access to Go source code. Fake it instead! + tmpDir, err := ioutil.TempDir("", "test_typecheck") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpDir) + + for path, data := range testFiles { + path := filepath.Join(tmpDir, "src", path) + err := os.MkdirAll(filepath.Dir(path), 0755) + if err != nil { + t.Fatal(err) + } + err = ioutil.WriteFile(path, []byte(data), 0644) + if err != nil { + t.Fatal(err) + } + fmt.Println(path) + } + + for _, test := range packageCases { + for platform, expectedErr := range test.errs { + a := newAnalyzer(platform) + // Make Imports happen relative to our faked up GOROOT. + a.ctx.GOROOT = tmpDir + a.ctx.GOPATH = "" + + errs := []string{} + a.conf.Error = func(err error) { + errs = append(errs, err.Error()) + } + + code := "package test\n" + test.code + parsed, err := parser.ParseFile(a.fset, "test.go", strings.NewReader(code), parser.AllErrors) + if err != nil { + t.Fatal(err) + } + a.typeCheck(tmpDir, []*ast.File{parsed}) + + if expectedErr == "" { + if len(errs) > 0 { + t.Errorf("code:\n%s\ngot %v\nwant %v", + code, errs, expectedErr) + } + } else { + if len(errs) != 1 { + t.Errorf("code:\n%s\ngot %v\nwant %v", + code, errs, expectedErr) + } else { + if errs[0] != expectedErr { + t.Errorf("code:\n%s\ngot %v\nwant %v", + code, errs[0], expectedErr) + } + } + } + } + } +} + +func TestHandlePath(t *testing.T) { + c := collector{} + e := errors.New("ex") + i, _ := os.Stat(".") // i.IsDir() == true + if c.handlePath("foo", nil, e) != e { + t.Error("handlePath not returning errors") + } + if c.handlePath("vendor", i, nil) != filepath.SkipDir { + t.Error("should skip vendor") + } +} diff --git a/test/typecheck/srcimporter/BUILD b/test/typecheck/srcimporter/BUILD new file mode 100644 index 0000000000..b28cfb53aa --- /dev/null +++ b/test/typecheck/srcimporter/BUILD @@ -0,0 +1,23 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["srcimporter.go"], + importpath = "k8s.io/kubernetes/test/typecheck/srcimporter", + visibility = ["//visibility:public"], + deps = ["//third_party/forked/golang/go/types:go_default_library"], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/test/typecheck/srcimporter/srcimporter.go b/test/typecheck/srcimporter/srcimporter.go new file mode 100644 index 0000000000..c9fd84af2f --- /dev/null +++ b/test/typecheck/srcimporter/srcimporter.go @@ -0,0 +1,244 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Forked from go's go/internal/srcimporter + +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package srcimporter implements importing directly +// from source files rather than installed packages. +package srcimporter + +import ( + "fmt" + "go/ast" + "go/build" + "go/parser" + "go/token" + "path/filepath" + "sync" + + "k8s.io/kubernetes/third_party/forked/golang/go/types" +) + +// An Importer provides the context for importing packages from source code. +type Importer struct { + ctxt *build.Context + fset *token.FileSet + sizes types.Sizes + packages map[string]*types.Package +} + +// New returns a new Importer for the given context, file set, and map +// of packages. The context is used to resolve import paths to package paths, +// and identifying the files belonging to the package. If the context provides +// non-nil file system functions, they are used instead of the regular package +// os functions. The file set is used to track position information of package +// files; and imported packages are added to the packages map. +func New(ctxt *build.Context, fset *token.FileSet, packages map[string]*types.Package) *Importer { + return &Importer{ + ctxt: ctxt, + fset: fset, + sizes: types.SizesFor(ctxt.Compiler, ctxt.GOARCH), // uses go/types default if GOARCH not found + packages: packages, + } +} + +// Importing is a sentinel taking the place in Importer.packages +// for a package that is in the process of being imported. +var importing types.Package + +// Import (path) is a shortcut for ImportFrom(path, "", 0). +func (p *Importer) Import(path string) (*types.Package, error) { + return p.ImportFrom(path, "", 0) +} + +// ImportFrom imports the package with the given import path resolved from the given srcDir, +// adds the new package to the set of packages maintained by the importer, and returns the +// package. Package path resolution and file system operations are controlled by the context +// maintained with the importer. The import mode must be zero but is otherwise ignored. +// Packages that are not comprised entirely of pure Go files may fail to import because the +// type checker may not be able to determine all exported entities (e.g. due to cgo dependencies). +func (p *Importer) ImportFrom(path, srcDir string, mode types.ImportMode) (*types.Package, error) { + if mode != 0 { + panic("non-zero import mode") + } + + // determine package path (do vendor resolution) + var bp *build.Package + var err error + switch { + default: + if abs, err := p.absPath(srcDir); err == nil { // see issue #14282 + srcDir = abs + } + bp, err = p.ctxt.Import(path, srcDir, build.FindOnly) + + case build.IsLocalImport(path): + // "./x" -> "srcDir/x" + bp, err = p.ctxt.ImportDir(filepath.Join(srcDir, path), build.FindOnly) + + case p.isAbsPath(path): + return nil, fmt.Errorf("invalid absolute import path %q", path) + } + if err != nil { + return nil, err // err may be *build.NoGoError - return as is + } + + // package unsafe is known to the type checker + if bp.ImportPath == "unsafe" { + return types.Unsafe, nil + } + + // no need to re-import if the package was imported completely before + pkg := p.packages[bp.ImportPath] + if pkg != nil { + if pkg == &importing { + return nil, fmt.Errorf("import cycle through package %q", bp.ImportPath) + } + if !pkg.Complete() { + // Package exists but is not complete - we cannot handle this + // at the moment since the source importer replaces the package + // wholesale rather than augmenting it (see #19337 for details). + // Return incomplete package with error (see #16088). + return pkg, fmt.Errorf("reimported partially imported package %q", bp.ImportPath) + } + return pkg, nil + } + + p.packages[bp.ImportPath] = &importing + defer func() { + // clean up in case of error + // TODO(gri) Eventually we may want to leave a (possibly empty) + // package in the map in all cases (and use that package to + // identify cycles). See also issue 16088. + if p.packages[bp.ImportPath] == &importing { + p.packages[bp.ImportPath] = nil + } + }() + + // collect package files + bp, err = p.ctxt.ImportDir(bp.Dir, 0) + if err != nil { + return nil, err // err may be *build.NoGoError - return as is + } + var filenames []string + filenames = append(filenames, bp.GoFiles...) + filenames = append(filenames, bp.CgoFiles...) + + files, err := p.parseFiles(bp.Dir, filenames) + if err != nil { + return nil, err + } + + // type-check package files + var firstHardErr error + conf := types.Config{ + IgnoreFuncBodies: true, + FakeImportC: true, + // continue type-checking after the first error + Error: func(err error) { + if firstHardErr == nil && !err.(types.Error).Soft { + firstHardErr = err + } + }, + Importer: p, + Sizes: p.sizes, + } + pkg, err = conf.Check(bp.ImportPath, p.fset, files, nil) + if err != nil { + // If there was a hard error it is possibly unsafe + // to use the package as it may not be fully populated. + // Do not return it (see also #20837, #20855). + if firstHardErr != nil { + pkg = nil + err = firstHardErr // give preference to first hard error over any soft error + } + return pkg, fmt.Errorf("type-checking package %q failed (%v)", bp.ImportPath, err) + } + if firstHardErr != nil { + // this can only happen if we have a bug in go/types + panic("package is not safe yet no error was returned") + } + + p.packages[bp.ImportPath] = pkg + return pkg, nil +} + +func (p *Importer) parseFiles(dir string, filenames []string) ([]*ast.File, error) { + open := p.ctxt.OpenFile // possibly nil + + files := make([]*ast.File, len(filenames)) + errors := make([]error, len(filenames)) + + var wg sync.WaitGroup + wg.Add(len(filenames)) + for i, filename := range filenames { + go func(i int, filepath string) { + defer wg.Done() + if open != nil { + src, err := open(filepath) + if err != nil { + errors[i] = fmt.Errorf("opening package file %s failed (%v)", filepath, err) + return + } + files[i], errors[i] = parser.ParseFile(p.fset, filepath, src, 0) + src.Close() // ignore Close error - parsing may have succeeded which is all we need + } else { + // Special-case when ctxt doesn't provide a custom OpenFile and use the + // parser's file reading mechanism directly. This appears to be quite a + // bit faster than opening the file and providing an io.ReaderCloser in + // both cases. + // TODO(gri) investigate performance difference (issue #19281) + files[i], errors[i] = parser.ParseFile(p.fset, filepath, nil, 0) + } + }(i, p.joinPath(dir, filename)) + } + wg.Wait() + + // if there are errors, return the first one for deterministic results + for _, err := range errors { + if err != nil { + return nil, err + } + } + + return files, nil +} + +// context-controlled file system operations + +func (p *Importer) absPath(path string) (string, error) { + // TODO(gri) This should be using p.ctxt.AbsPath which doesn't + // exist but probably should. See also issue #14282. + return filepath.Abs(path) +} + +func (p *Importer) isAbsPath(path string) bool { + if f := p.ctxt.IsAbsPath; f != nil { + return f(path) + } + return filepath.IsAbs(path) +} + +func (p *Importer) joinPath(elem ...string) string { + if f := p.ctxt.JoinPath; f != nil { + return f(elem...) + } + return filepath.Join(elem...) +}