github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Add test/typecheck, a fast typecheck for all build platforms. 

Most of the time spent compiling is spent optimizing and linking
binary code. Most errors occur at the syntax or semantic (type) layers.
Go's compiler is importable as a normal package, so we can do fast
syntax and type checking for the 10 platforms we build on.

This currently takes ~6 minutes of CPU time (parallelized).

This makes presubmit cross builds superfluous, since it should catch
most cross-build breaks (generally Unix and 64-bit assumptions).

Example output:

$ time go run test/typecheck/main.go
type-checking:  linux/amd64, windows/386, darwin/amd64, linux/arm, linux/386, windows/amd64, linux/arm64, linux/ppc64le, linux/s390x, darwin/386
ERROR(windows/amd64) pkg/proxy/ipvs/proxier.go:1708:27: ENXIO not declared by package unix
ERROR(windows/386) pkg/proxy/ipvs/proxier.go:1708:27: ENXIO not declared by package unix

real	0m45.083s
user	6m15.504s
sys	1m14.000s
pull/6/head
Ryan Hitchman 2018-02-02 14:27:52 -08:00
parent 4d2e43f53f
commit dd40e612dd
9 changed files with 881 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
hack/make-rules/verify.sh
View File

@ -30,6 +30,7 @@ EXCLUDED_PATTERNS=(
"verify-linkcheck.sh" # runs in separate Jenkins job once per day due to high network usage "verify-linkcheck.sh" # runs in separate Jenkins job once per day due to high network usage
"verify-test-owners.sh" # TODO(rmmh): figure out how to avoid endless conflicts "verify-test-owners.sh" # TODO(rmmh): figure out how to avoid endless conflicts
"verify-*-dockerized.sh" # Don't run any scripts that intended to be run dockerized "verify-*-dockerized.sh" # Don't run any scripts that intended to be run dockerized
"verify-typecheck.sh" # runs in separate typecheck job
) )
# Only run whitelisted fast checks in quick mode. # Only run whitelisted fast checks in quick mode.

34
hack/verify-typecheck.sh Executable file
View File

@ -0,0 +1,34 @@
#!/bin/bash
# Copyright 2018 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -o errexit
set -o nounset
set -o pipefail
KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
source "${KUBE_ROOT}/hack/lib/init.sh"
kube::golang::verify_go_version
cd "${KUBE_ROOT}"
ret=0
go run test/typecheck/main.go "$@" || ret=$?
if [[ $ret -ne 0 ]]; then
echo "!!! Type Check has failed. This may cause cross platform build failures." >&2
echo "!!! Please see https://git.k8s.io/kubernetes/test/typecheck for more information." >&2
exit 1
fi

1
test/BUILD
View File

@ -21,6 +21,7 @@ filegroup(
"//test/list:all-srcs", "//test/list:all-srcs",
"//test/soak/cauldron:all-srcs", "//test/soak/cauldron:all-srcs",
"//test/soak/serve_hostnames:all-srcs", "//test/soak/serve_hostnames:all-srcs",
"//test/typecheck:all-srcs",
"//test/utils:all-srcs", "//test/utils:all-srcs",
], ],
tags = ["automanaged"], tags = ["automanaged"],

51
test/typecheck/BUILD Normal file
View File

@ -0,0 +1,51 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_binary",
"go_library",
"go_test",
)
go_binary(
name = "list",
embed = [":go_default_library"],
)
go_library(
name = "go_default_library",
srcs = ["main.go"],
importpath = "k8s.io/kubernetes/test/typecheck",
deps = [
"//test/typecheck/srcimporter:go_default_library",
"//third_party/forked/golang/go/types:go_default_library",
"//vendor/golang.org/x/crypto/ssh/terminal:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [
":package-srcs",
"//test/typecheck/srcimporter:all-srcs",
],
tags = ["automanaged"],
)
go_binary(
name = "typecheck",
embed = [":go_default_library"],
)
go_test(
name = "go_default_test",
srcs = ["main_test.go"],
embed = [":go_default_library"],
)

23
test/typecheck/README Normal file
View File

@ -0,0 +1,23 @@
Typecheck does cross-platform typechecking of source code for all Go build
platforms.
The primary benefit is speed: a full Kubernetes cross-build takes 20 minutes
and >40GB of RAM, while this takes under 2 minutes and <8GB of RAM.
It uses Go's built-in parsing and typechecking libraries (go/parser and
go/types), which unfortunately are not what the go compiler uses. Occasional
mismatches will occur, but overall they correspond closely.
Failures can be ignored if they don't block the build:
Things go/types errors on that go build doesn't:
True errors (according to the spec):
These should be fixed whenever possible. Ignore if a fix isn't possible
or is in progress (e.g., vendored code).
- unused variables in closures
False errors:
These should be ignored and reported upstream if applicable.
- type checking mismatches between staging and generated types
Things go build fails on that we don't:
Please send examples of this to rmmh@ and extend this list.
- CGo errors, including syntax and linker errors.

347
test/typecheck/main.go Normal file
View File

@ -0,0 +1,347 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// do a fast type check of kubernetes code, for all platforms.
package main
import (
"flag"
"fmt"
"go/ast"
"go/build"
"go/parser"
"go/token"
"log"
"os"
"path/filepath"
"sort"
"strings"
"sync"
"sync/atomic"
"time"
"golang.org/x/crypto/ssh/terminal"
// TODO(rmmh): remove this when golang/go#23712 is fixed, and the
// fix is the current minimum Go version to build Kubernetes.
"k8s.io/kubernetes/test/typecheck/srcimporter"
"k8s.io/kubernetes/third_party/forked/golang/go/types"
)
var (
verbose = flag.Bool("verbose", false, "print more information")
cross = flag.Bool("cross", true, "build for all platforms")
platforms = flag.String("platform", "", "comma-separated list of platforms to typecheck")
timings = flag.Bool("time", false, "output times taken for each phase")
defuses = flag.Bool("defuse", false, "output defs/uses")
serial = flag.Bool("serial", false, "don't type check platforms in parallel")
isTerminal = terminal.IsTerminal(int(os.Stdout.Fd()))
logPrefix = ""
// When processed in order, windows and darwin are early to make
// interesting OS-based errors happen earlier.
crossPlatforms = []string{
"linux/amd64", "windows/386",
"darwin/amd64", "linux/arm",
"linux/386", "windows/amd64",
"linux/arm64", "linux/ppc64le",
"linux/s390x", "darwin/386",
}
)
type analyzer struct {
fset *token.FileSet // positions are relative to fset
conf types.Config
ctx build.Context
failed bool
platform string
donePaths map[string]interface{}
}
func newAnalyzer(platform string) *analyzer {
ctx := build.Default
platSplit := strings.Split(platform, "/")
ctx.GOOS, ctx.GOARCH = platSplit[0], platSplit[1]
ctx.CgoEnabled = true
a := &analyzer{
platform: platform,
fset: token.NewFileSet(),
ctx: ctx,
donePaths: make(map[string]interface{}),
}
a.conf = types.Config{
FakeImportC: true,
Error: a.handleError,
Sizes: types.SizesFor("gc", a.ctx.GOARCH),
}
a.conf.Importer = srcimporter.New(
&a.ctx, a.fset, make(map[string]*types.Package))
if *verbose {
fmt.Printf("context: %#v\n", ctx)
}
return a
}
func (a *analyzer) handleError(err error) {
if e, ok := err.(types.Error); ok {
// useful for some ignores:
// path := e.Fset.Position(e.Pos).String()
ignore := false
// TODO(rmmh): read ignores from a file, so this code can
// be Kubernetes-agnostic. Unused ignores should be treated as
// errors, to ensure coverage isn't overly broad.
if strings.Contains(e.Msg, "GetOpenAPIDefinitions") {
// TODO(rmmh): figure out why this happens.
// cmd/kube-apiserver/app/server.go:392:70
// test/integration/framework/master_utils.go:131:84
ignore = true
}
if ignore {
if *verbose {
fmt.Println("ignoring error:", err)
}
return
}
}
// TODO(rmmh): dedup errors across platforms?
fmt.Fprintf(os.Stderr, "%sERROR(%s) %s\n", logPrefix, a.platform, err)
a.failed = true
}
// collect extracts test metadata from a file.
func (a *analyzer) collect(dir string) {
if _, ok := a.donePaths[dir]; ok {
return
}
a.donePaths[dir] = nil
// Create the AST by parsing src.
fs, err := parser.ParseDir(a.fset, dir, nil, parser.AllErrors)
if err != nil {
fmt.Println(logPrefix+"ERROR(syntax)", err)
a.failed = true
return
}
if len(fs) > 1 && *verbose {
fmt.Println("multiple packages in dir:", dir)
}
for _, p := range fs {
// returns first error, but a.handleError deals with it
files := a.filterFiles(p.Files)
if *verbose {
fmt.Printf("path: %s package: %s files: ", dir, p.Name)
for _, f := range files {
fname := filepath.Base(a.fset.File(f.Pos()).Name())
fmt.Printf("%s ", fname)
}
fmt.Printf("\n")
}
a.typeCheck(dir, files)
}
}
// filterFiles restricts a list of files to only those that should be built by
// the current platform. This includes both build suffixes (_windows.go) and build
// tags ("// +build !linux" at the beginning).
func (a *analyzer) filterFiles(fs map[string]*ast.File) []*ast.File {
files := []*ast.File{}
for _, f := range fs {
fpath := a.fset.File(f.Pos()).Name()
dir, name := filepath.Split(fpath)
matches, err := a.ctx.MatchFile(dir, name)
if err != nil {
fmt.Fprintf(os.Stderr, "%sERROR reading %s: %s\n", logPrefix, fpath, err)
a.failed = true
continue
}
if matches {
files = append(files, f)
}
}
return files
}
func (a *analyzer) typeCheck(dir string, files []*ast.File) error {
info := types.Info{
Defs: make(map[*ast.Ident]types.Object),
Uses: make(map[*ast.Ident]types.Object),
}
// NOTE: this type check does a *recursive* import, but srcimporter
// doesn't do a full type check (ignores function bodies)-- this has
// some additional overhead.
//
// This means that we need to ensure that typeCheck runs on all
// code we will be compiling.
//
// TODO(rmmh): Customize our forked srcimporter to do this better.
pkg, err := a.conf.Check(dir, a.fset, files, &info)
if err != nil {
return err // type error
}
// A significant fraction of vendored code only compiles on Linux,
// but it's only imported by code that has build-guards for Linux.
// Track vendored code to type-check it in a second pass.
for _, imp := range pkg.Imports() {
if strings.HasPrefix(imp.Path(), "k8s.io/kubernetes/vendor/") {
vendorPath := imp.Path()[len("k8s.io/kubernetes/"):]
if *verbose {
fmt.Println("recursively checking vendor path:", vendorPath)
}
a.collect(vendorPath)
}
}
if *defuses {
for id, obj := range info.Defs {
fmt.Printf("%s: %q defines %v\n",
a.fset.Position(id.Pos()), id.Name, obj)
}
for id, obj := range info.Uses {
fmt.Printf("%s: %q uses %v\n",
a.fset.Position(id.Pos()), id.Name, obj)
}
}
return nil
}
type collector struct {
dirs []string
}
// handlePath walks the filesystem recursively, collecting directories,
// ignoring some unneeded directories (hidden/vendored) that are handled
// specially later.
func (c *collector) handlePath(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if info.IsDir() {
// Ignore hidden directories (.git, .cache, etc)
if len(path) > 1 && path[0] == '.' ||
// Staging code is symlinked from vendor/k8s.io, and uses import
// paths as if it were inside of vendor/. It fails typechecking
// inside of staging/, but works when typechecked as part of vendor/.
path == "staging" ||
// OS-specific vendor code tends to be imported by OS-specific
// packages. We recursively typecheck imported vendored packages for
// each OS, but don't typecheck everything for every OS.
path == "vendor" ||
path == "_output" ||
// This is a weird one. /testdata/ is *mostly* ignored by Go,
// and this translates to kubernetes/vendor not working.
// edit/record.go doesn't compile without gopkg.in/yaml.v2
// in $GOSRC/$GOROOT (both typecheck and the shell script).
path == "pkg/kubectl/cmd/testdata/edit" {
return filepath.SkipDir
}
c.dirs = append(c.dirs, path)
}
return nil
}
func main() {
flag.Parse()
args := flag.Args()
if *verbose {
*serial = true // to avoid confusing interleaved logs
}
if len(args) == 0 {
args = append(args, ".")
}
c := collector{}
for _, arg := range args {
err := filepath.Walk(arg, c.handlePath)
if err != nil {
log.Fatalf("Error walking: %v", err)
}
}
sort.Strings(c.dirs)
ps := crossPlatforms[:]
if *platforms != "" {
ps = strings.Split(*platforms, ",")
} else if !*cross {
ps = ps[:1]
}
fmt.Println("type-checking: ", strings.Join(ps, ", "))
var wg sync.WaitGroup
var processedDirs int64
var currentWork int64 // (dir_index << 8) | platform_index
statuses := make([]int, len(ps))
for i, p := range ps {
wg.Add(1)
fn := func(i int, p string) {
start := time.Now()
a := newAnalyzer(p)
for n, dir := range c.dirs {
a.collect(dir)
atomic.AddInt64(&processedDirs, 1)
atomic.StoreInt64(&currentWork, int64(n<<8|i))
}
if a.failed {
statuses[i] = 1
}
if *timings {
fmt.Printf("%s took %.1fs\n", p, time.Since(start).Seconds())
}
wg.Done()
}
if *serial {
fn(i, p)
} else {
go fn(i, p)
}
}
if isTerminal {
logPrefix = "\r" // clear status bar when printing
// Display a status bar so devs can estimate completion times.
go func() {
total := len(ps) * len(c.dirs)
for proc := 0; proc < total; proc = int(atomic.LoadInt64(&processedDirs)) {
work := atomic.LoadInt64(&currentWork)
dir := c.dirs[work>>8]
platform := ps[work&0xFF]
if len(dir) > 80 {
dir = dir[:80]
}
fmt.Printf("\r%d/%d \033[2m%-13s\033[0m %-80s", proc, total, platform, dir)
time.Sleep(50 * time.Millisecond)
}
}()
}
wg.Wait()
fmt.Println()
for _, status := range statuses {
if status != 0 {
os.Exit(status)
}
}
}

157
test/typecheck/main_test.go Normal file
View File

@ -0,0 +1,157 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main
import (
"errors"
"fmt"
"go/ast"
"go/parser"
"io/ioutil"
"os"
"path/filepath"
"strings"
"testing"
)
var packageCases = []struct {
code string
errs map[string]string
}{
// Empty: no problems!
{"", map[string]string{"linux/amd64": ""}},
// Slightly less empty: no problems!
{"func getRandomNumber() int { return 4; }", map[string]string{"darwin/386": ""}},
// Fixed in #59243
{`import "golang.org/x/sys/unix"
func f(err error) {
if err != unix.ENXIO {
panic("woops")
}
}`, map[string]string{"linux/amd64": "", "windows/amd64": "test.go:4:13: ENXIO not declared by package unix"}},
// Fixed in #51984
{`import "golang.org/x/sys/unix"
const linuxHugetlbfsMagic = 0x958458f6
func IsHugeTlbfs() bool {
buf := unix.Statfs_t{}
unix.Statfs("/tmp/", &buf)
return buf.Type == linuxHugetlbfsMagic
}`, map[string]string{
"linux/amd64": "",
"linux/386": "test.go:7:22: linuxHugetlbfsMagic (untyped int constant 2508478710) overflows int32",
}},
// Fixed in #51873
{`var a = map[string]interface{}{"num1": 9223372036854775807}`,
map[string]string{"linux/arm": "test.go:2:40: 9223372036854775807 (untyped int constant) overflows int"}},
}
var testFiles = map[string]string{
"golang.org/x/sys/unix/empty.go": `package unix`,
"golang.org/x/sys/unix/errno_linux.go": `// +build linux
package unix
type Errno string
func (e Errno) Error() string { return string(e) }
var ENXIO = Errno("3")`,
"golang.org/x/sys/unix/ztypes_linux_amd64.go": `// +build amd64,linux
package unix
type Statfs_t struct {
Type int64
}
func Statfs(path string, statfs *Statfs_t) {}
`,
"golang.org/x/sys/unix/ztypes_linux_386.go": `// +build i386,linux
package unix
type Statfs_t struct {
Type int32
}
func Statfs(path string, statfs *Statfs_t) {}
`,
}
func TestHandlePackage(t *testing.T) {
// When running in Bazel, we don't have access to Go source code. Fake it instead!
tmpDir, err := ioutil.TempDir("", "test_typecheck")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpDir)
for path, data := range testFiles {
path := filepath.Join(tmpDir, "src", path)
err := os.MkdirAll(filepath.Dir(path), 0755)
if err != nil {
t.Fatal(err)
}
err = ioutil.WriteFile(path, []byte(data), 0644)
if err != nil {
t.Fatal(err)
}
fmt.Println(path)
}
for _, test := range packageCases {
for platform, expectedErr := range test.errs {
a := newAnalyzer(platform)
// Make Imports happen relative to our faked up GOROOT.
a.ctx.GOROOT = tmpDir
a.ctx.GOPATH = ""
errs := []string{}
a.conf.Error = func(err error) {
errs = append(errs, err.Error())
}
code := "package test\n" + test.code
parsed, err := parser.ParseFile(a.fset, "test.go", strings.NewReader(code), parser.AllErrors)
if err != nil {
t.Fatal(err)
}
a.typeCheck(tmpDir, []*ast.File{parsed})
if expectedErr == "" {
if len(errs) > 0 {
t.Errorf("code:\n%s\ngot %v\nwant %v",
code, errs, expectedErr)
}
} else {
if len(errs) != 1 {
t.Errorf("code:\n%s\ngot %v\nwant %v",
code, errs, expectedErr)
} else {
if errs[0] != expectedErr {
t.Errorf("code:\n%s\ngot %v\nwant %v",
code, errs[0], expectedErr)
}
}
}
}
}
}
func TestHandlePath(t *testing.T) {
c := collector{}
e := errors.New("ex")
i, _ := os.Stat(".") // i.IsDir() == true
if c.handlePath("foo", nil, e) != e {
t.Error("handlePath not returning errors")
}
if c.handlePath("vendor", i, nil) != filepath.SkipDir {
t.Error("should skip vendor")
}
}

23
test/typecheck/srcimporter/BUILD Normal file
View File

@ -0,0 +1,23 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library")
go_library(
name = "go_default_library",
srcs = ["srcimporter.go"],
importpath = "k8s.io/kubernetes/test/typecheck/srcimporter",
visibility = ["//visibility:public"],
deps = ["//third_party/forked/golang/go/types:go_default_library"],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

244
test/typecheck/srcimporter/srcimporter.go Normal file
View File

@ -0,0 +1,244 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Forked from go's go/internal/srcimporter
// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package srcimporter implements importing directly
// from source files rather than installed packages.
package srcimporter
import (
"fmt"
"go/ast"
"go/build"
"go/parser"
"go/token"
"path/filepath"
"sync"
"k8s.io/kubernetes/third_party/forked/golang/go/types"
)
// An Importer provides the context for importing packages from source code.
type Importer struct {
ctxt *build.Context
fset *token.FileSet
sizes types.Sizes
packages map[string]*types.Package
}
// New returns a new Importer for the given context, file set, and map
// of packages. The context is used to resolve import paths to package paths,
// and identifying the files belonging to the package. If the context provides
// non-nil file system functions, they are used instead of the regular package
// os functions. The file set is used to track position information of package
// files; and imported packages are added to the packages map.
func New(ctxt *build.Context, fset *token.FileSet, packages map[string]*types.Package) *Importer {
return &Importer{
ctxt: ctxt,
fset: fset,
sizes: types.SizesFor(ctxt.Compiler, ctxt.GOARCH), // uses go/types default if GOARCH not found
packages: packages,
}
}
// Importing is a sentinel taking the place in Importer.packages
// for a package that is in the process of being imported.
var importing types.Package
// Import (path) is a shortcut for ImportFrom(path, "", 0).
func (p *Importer) Import(path string) (*types.Package, error) {
return p.ImportFrom(path, "", 0)
}
// ImportFrom imports the package with the given import path resolved from the given srcDir,
// adds the new package to the set of packages maintained by the importer, and returns the
// package. Package path resolution and file system operations are controlled by the context
// maintained with the importer. The import mode must be zero but is otherwise ignored.
// Packages that are not comprised entirely of pure Go files may fail to import because the
// type checker may not be able to determine all exported entities (e.g. due to cgo dependencies).
func (p *Importer) ImportFrom(path, srcDir string, mode types.ImportMode) (*types.Package, error) {
if mode != 0 {
panic("non-zero import mode")
}
// determine package path (do vendor resolution)
var bp *build.Package
var err error
switch {
default:
if abs, err := p.absPath(srcDir); err == nil { // see issue #14282
srcDir = abs
}
bp, err = p.ctxt.Import(path, srcDir, build.FindOnly)
case build.IsLocalImport(path):
// "./x" -> "srcDir/x"
bp, err = p.ctxt.ImportDir(filepath.Join(srcDir, path), build.FindOnly)
case p.isAbsPath(path):
return nil, fmt.Errorf("invalid absolute import path %q", path)
}
if err != nil {
return nil, err // err may be *build.NoGoError - return as is
}
// package unsafe is known to the type checker
if bp.ImportPath == "unsafe" {
return types.Unsafe, nil
}
// no need to re-import if the package was imported completely before
pkg := p.packages[bp.ImportPath]
if pkg != nil {
if pkg == &importing {
return nil, fmt.Errorf("import cycle through package %q", bp.ImportPath)
}
if !pkg.Complete() {
// Package exists but is not complete - we cannot handle this
// at the moment since the source importer replaces the package
// wholesale rather than augmenting it (see #19337 for details).
// Return incomplete package with error (see #16088).
return pkg, fmt.Errorf("reimported partially imported package %q", bp.ImportPath)
}
return pkg, nil
}
p.packages[bp.ImportPath] = &importing
defer func() {
// clean up in case of error
// TODO(gri) Eventually we may want to leave a (possibly empty)
// package in the map in all cases (and use that package to
// identify cycles). See also issue 16088.
if p.packages[bp.ImportPath] == &importing {
p.packages[bp.ImportPath] = nil
}
}()
// collect package files
bp, err = p.ctxt.ImportDir(bp.Dir, 0)
if err != nil {
return nil, err // err may be *build.NoGoError - return as is
}
var filenames []string
filenames = append(filenames, bp.GoFiles...)
filenames = append(filenames, bp.CgoFiles...)
files, err := p.parseFiles(bp.Dir, filenames)
if err != nil {
return nil, err
}
// type-check package files
var firstHardErr error
conf := types.Config{
IgnoreFuncBodies: true,
FakeImportC: true,
// continue type-checking after the first error
Error: func(err error) {
if firstHardErr == nil && !err.(types.Error).Soft {
firstHardErr = err
}
},
Importer: p,
Sizes: p.sizes,
}
pkg, err = conf.Check(bp.ImportPath, p.fset, files, nil)
if err != nil {
// If there was a hard error it is possibly unsafe
// to use the package as it may not be fully populated.
// Do not return it (see also #20837, #20855).
if firstHardErr != nil {
pkg = nil
err = firstHardErr // give preference to first hard error over any soft error
}
return pkg, fmt.Errorf("type-checking package %q failed (%v)", bp.ImportPath, err)
}
if firstHardErr != nil {
// this can only happen if we have a bug in go/types
panic("package is not safe yet no error was returned")
}
p.packages[bp.ImportPath] = pkg
return pkg, nil
}
func (p *Importer) parseFiles(dir string, filenames []string) ([]*ast.File, error) {
open := p.ctxt.OpenFile // possibly nil
files := make([]*ast.File, len(filenames))
errors := make([]error, len(filenames))
var wg sync.WaitGroup
wg.Add(len(filenames))
for i, filename := range filenames {
go func(i int, filepath string) {
defer wg.Done()
if open != nil {
src, err := open(filepath)
if err != nil {
errors[i] = fmt.Errorf("opening package file %s failed (%v)", filepath, err)
return
}
files[i], errors[i] = parser.ParseFile(p.fset, filepath, src, 0)
src.Close() // ignore Close error - parsing may have succeeded which is all we need
} else {
// Special-case when ctxt doesn't provide a custom OpenFile and use the
// parser's file reading mechanism directly. This appears to be quite a
// bit faster than opening the file and providing an io.ReaderCloser in
// both cases.
// TODO(gri) investigate performance difference (issue #19281)
files[i], errors[i] = parser.ParseFile(p.fset, filepath, nil, 0)
}
}(i, p.joinPath(dir, filename))
}
wg.Wait()
// if there are errors, return the first one for deterministic results
for _, err := range errors {
if err != nil {
return nil, err
}
}
return files, nil
}
// context-controlled file system operations
func (p *Importer) absPath(path string) (string, error) {
// TODO(gri) This should be using p.ctxt.AbsPath which doesn't
// exist but probably should. See also issue #14282.
return filepath.Abs(path)
}
func (p *Importer) isAbsPath(path string) bool {
if f := p.ctxt.IsAbsPath; f != nil {
return f(path)
}
return filepath.IsAbs(path)
}
func (p *Importer) joinPath(elem ...string) string {
if f := p.ctxt.JoinPath; f != nil {
return f(elem...)
}
return filepath.Join(elem...)
}