diff --git a/hack/BUILD b/hack/BUILD index 0eaeae8c5c..9b04cb6c32 100644 --- a/hack/BUILD +++ b/hack/BUILD @@ -19,6 +19,7 @@ filegroup( srcs = [ ":package-srcs", "//hack/boilerplate:all-srcs", + "//hack/conformance:all-srcs", "//hack/e2e-internal:all-srcs", "//hack/lib:all-srcs", "//hack/make-rules:all-srcs", diff --git a/hack/conformance/BUILD b/hack/conformance/BUILD new file mode 100644 index 0000000000..b0ce3f2e08 --- /dev/null +++ b/hack/conformance/BUILD @@ -0,0 +1,32 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") + +go_library( + name = "go_default_library", + srcs = ["check_conformance_test_requirements.go"], + importpath = "k8s.io/kubernetes/hack/conformance", + visibility = ["//visibility:private"], + deps = [ + "//vendor/github.com/pkg/errors:go_default_library", + "//vendor/github.com/spf13/cobra:go_default_library", + ], +) + +go_binary( + name = "conformance", + embed = [":go_default_library"], + visibility = ["//visibility:public"], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/hack/conformance/check_conformance_test_requirements.go b/hack/conformance/check_conformance_test_requirements.go new file mode 100755 index 0000000000..a0519483b9 --- /dev/null +++ b/hack/conformance/check_conformance_test_requirements.go @@ -0,0 +1,157 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// This tool is for checking conformance e2e tests follow the requirements +// which is https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/conformance-tests.md#conformance-test-requirements +package main + +import ( + "bufio" + "bytes" + "fmt" + "io/ioutil" + "os" + "regexp" + + "github.com/pkg/errors" + "github.com/spf13/cobra" +) + +const ( + //e.g. framework.ConformanceIt("should provide secure master service ", func() { + patternStartConformance = "framework.ConformanceIt\\(.*, func\\(\\) {$" + patternEndConformance = "}\\)$" + patternSkipProviderIs = "Skip.*ProviderIs\\(" +) + +// This function checks the requirement: it works for all providers (e.g., no SkipIfProviderIs/SkipUnlessProviderIs calls) +func checkAllProviders(e2eFile string) error { + checkFailed := false + inConformanceCode := false + + regStartConformance := regexp.MustCompile(patternStartConformance) + regEndConformance := regexp.MustCompile(patternEndConformance) + regSkipProviderIs := regexp.MustCompile(patternSkipProviderIs) + + fileInput, err := ioutil.ReadFile(e2eFile) + if err != nil { + return errors.Wrapf(err, "Failed to read file %s", e2eFile) + } + scanner := bufio.NewScanner(bytes.NewReader(fileInput)) + scanner.Split(bufio.ScanLines) + + for scanner.Scan() { + line := scanner.Text() + if regStartConformance.MatchString(line) { + if inConformanceCode { + return errors.Errorf("Missed the end of previous conformance test. There might be a bug in this script.") + } + inConformanceCode = true + } + if inConformanceCode { + if regSkipProviderIs.MatchString(line) { + // To list all invalid places in a single operation of this tool, here doesn't return error and continues checking. + fmt.Fprintf(os.Stderr, "%v: Conformance test should not call SkipIfProviderIs()/SkipUnlessProviderIs()\n", e2eFile) + checkFailed = true + } + if regEndConformance.MatchString(line) { + inConformanceCode = false + } + } + } + if inConformanceCode { + return errors.Errorf("Missed the end of previous conformance test. There might be a bug in this script.") + } + if checkFailed { + return errors.Errorf("We need to fix the above errors.") + } + return nil +} + +func processFile(e2ePath string) error { + regGoFile := regexp.MustCompile(".*.go") + + files, err := ioutil.ReadDir(e2ePath) + if err != nil { + return errors.Wrapf(err, "Failed to read dir %s", e2ePath) + } + for _, file := range files { + if file.IsDir() { + continue + } + if !regGoFile.MatchString(file.Name()) { + continue + } + e2eFile := fmt.Sprintf("%s/%s", e2ePath, file.Name()) + err = checkAllProviders(e2eFile) + if err != nil { + return err + } + } + return nil +} + +func processDir(e2ePath string) error { + err := processFile(e2ePath) + if err != nil { + return err + } + + // Search sub directories if exist + files, err := ioutil.ReadDir(e2ePath) + if err != nil { + return errors.Wrapf(err, "Failed to read dir %s", e2ePath) + } + for _, file := range files { + if !file.IsDir() { + continue + } + err = processDir(fmt.Sprintf("%s/%s", e2ePath, file.Name())) + if err != nil { + return err + } + } + return nil +} + +func newCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "check_conformance_test_requirements [e2e-test-path]", + Short: "Check conformance test code follows the requirements", + Run: func(cmd *cobra.Command, args []string) { + if len(args) != 1 { + cmd.Help() + os.Exit(1) + } + e2eRootPath := args[0] + err := processDir(e2eRootPath) + if err != nil { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + os.Exit(1) + } + os.Exit(0) + }, + } + return cmd +} + +func main() { + command := newCommand() + if err := command.Execute(); err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } +}