diff --git a/test/BUILD b/test/BUILD index 831bcb91f1..aa84e47065 100644 --- a/test/BUILD +++ b/test/BUILD @@ -11,6 +11,7 @@ filegroup( name = "all-srcs", srcs = [ ":package-srcs", + "//test/conformance:all-srcs", "//test/e2e:all-srcs", "//test/e2e_node:all-srcs", "//test/fixtures:all-srcs", diff --git a/test/conformance/BUILD b/test/conformance/BUILD new file mode 100644 index 0000000000..27bbed8156 --- /dev/null +++ b/test/conformance/BUILD @@ -0,0 +1,42 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") + +go_library( + name = "go_default_library", + srcs = ["walk.go"], + visibility = ["//visibility:private"], +) + +go_binary( + name = "conformance", + library = ":go_default_library", + visibility = ["//visibility:public"], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) + +genrule( + name = "list_conformance_tests", + srcs = ["//test/e2e:all-srcs"], + outs = ["conformance.txt"], + cmd = "./$(location :conformance) $(locations //test/e2e:all-srcs) > $@", + message = "Listing all conformance tests.", + tools = [":conformance"], +) + +sh_test( + name = "conformance_test", + srcs = ["conformance_test.sh"], + data = ["testdata/conformance.txt", ":list_conformance_tests"], +) diff --git a/test/conformance/OWNERS b/test/conformance/OWNERS new file mode 100644 index 0000000000..155eb146fb --- /dev/null +++ b/test/conformance/OWNERS @@ -0,0 +1,7 @@ +# This is the owner of the test code. The test data itself is owned by sig-architecture. +reviewers: + - mml + - cheftako +approvers: + - mml + - cheftako diff --git a/test/conformance/conformance_test.sh b/test/conformance/conformance_test.sh new file mode 100755 index 0000000000..b0e5eb1cd8 --- /dev/null +++ b/test/conformance/conformance_test.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# echo ${TEST_SRCDIR} +# pwd +# env | grep --color=always test/conformance +# find ${TEST_SRCDIR} -ls | grep --color=always test/conformance + +set -o errexit + +diff -u test/conformance/testdata/conformance.txt test/conformance/conformance.txt +echo PASS diff --git a/test/conformance/testdata/OWNERS b/test/conformance/testdata/OWNERS new file mode 100644 index 0000000000..ff12a8f545 --- /dev/null +++ b/test/conformance/testdata/OWNERS @@ -0,0 +1,9 @@ +# To be owned by sig-architecture. +# TODO(mml): Exclude parent owners once +# https://github.com/kubernetes/test-infra/issues/5197 is implemented. +reviewers: + - bgrant0607 + - smarterclayton +approvers: + - bgrant0607 + - smarterclayton diff --git a/test/conformance/testdata/conformance.txt b/test/conformance/testdata/conformance.txt new file mode 100755 index 0000000000..55811df566 --- /dev/null +++ b/test/conformance/testdata/conformance.txt @@ -0,0 +1,150 @@ +test/e2e/apimachinery/custom_resource_definition.go: "creating/deleting custom resource definition objects works " +test/e2e/apps/rc.go: "should serve a basic image on each replica with a public image " +test/e2e/apps/replica_set.go: "should serve a basic image on each replica with a public image " +test/e2e/auth/service_accounts.go: "should mount an API token into pods " +test/e2e/auth/service_accounts.go: "should allow opting out of API token automount " +test/e2e/common/configmap.go: "should be consumable via environment variable " +test/e2e/common/configmap.go: "should be consumable via the environment " +test/e2e/common/configmap_volume.go: "should be consumable from pods in volume " +test/e2e/common/configmap_volume.go: "should be consumable from pods in volume with defaultMode set " +test/e2e/common/configmap_volume.go: "should be consumable from pods in volume as non-root " +test/e2e/common/configmap_volume.go: "should be consumable from pods in volume with mappings " +test/e2e/common/configmap_volume.go: "should be consumable from pods in volume with mappings and Item mode set" +test/e2e/common/configmap_volume.go: "should be consumable from pods in volume with mappings as non-root " +test/e2e/common/configmap_volume.go: "updates should be reflected in volume " +test/e2e/common/configmap_volume.go: "optional updates should be reflected in volume " +test/e2e/common/configmap_volume.go: "should be consumable in multiple volumes in the same pod " +test/e2e/common/container_probe.go: "with readiness probe should not be ready before initial delay and never restart " +test/e2e/common/container_probe.go: "with readiness probe that fails should never be ready and never restart " +test/e2e/common/container_probe.go: "should be restarted with a exec \"cat /tmp/health\" liveness probe" +test/e2e/common/container_probe.go: "should *not* be restarted with a exec \"cat /tmp/health\" liveness probe" +test/e2e/common/container_probe.go: "should be restarted with a /healthz http liveness probe " +test/e2e/common/container_probe.go: "should have monotonically increasing restart count [Slow]" +test/e2e/common/container_probe.go: "should *not* be restarted with a /healthz http liveness probe " +test/e2e/common/container_probe.go: "should be restarted with a docker exec liveness probe with timeout " +test/e2e/common/docker_containers.go: "should use the image defaults if command and args are blank " +test/e2e/common/docker_containers.go: "should be able to override the image's default arguments (docker cmd) " +test/e2e/common/docker_containers.go: "should be able to override the image's default commmand (docker entrypoint) " +test/e2e/common/docker_containers.go: "should be able to override the image's default command and arguments " +test/e2e/common/downward_api.go: "should provide pod name and namespace as env vars " +test/e2e/common/downward_api.go: "should provide pod IP as an env var " +test/e2e/common/downward_api.go: "should provide host IP as an env var " +test/e2e/common/downward_api.go: "should provide container's limits.cpu/memory and requests.cpu/memory as env vars " +test/e2e/common/downward_api.go: "should provide default limits.cpu/memory from node allocatable " +test/e2e/common/downward_api.go: "should provide pod UID as env vars " +test/e2e/common/downwardapi_volume.go: "should provide podname only " +test/e2e/common/downwardapi_volume.go: "should set DefaultMode on files " +test/e2e/common/downwardapi_volume.go: "should set mode on item file " +test/e2e/common/downwardapi_volume.go: "should update labels on modification " +test/e2e/common/downwardapi_volume.go: "should update annotations on modification " +test/e2e/common/downwardapi_volume.go: "should provide container's cpu limit " +test/e2e/common/downwardapi_volume.go: "should provide container's memory limit " +test/e2e/common/downwardapi_volume.go: "should provide container's cpu request " +test/e2e/common/downwardapi_volume.go: "should provide container's memory request " +test/e2e/common/downwardapi_volume.go: "should provide node allocatable (cpu) as default cpu limit if the limit is not set " +test/e2e/common/downwardapi_volume.go: "should provide node allocatable (memory) as default memory limit if the limit is not set " +test/e2e/common/empty_dir.go: "volume on tmpfs should have the correct mode [sig-storage]" +test/e2e/common/empty_dir.go: "should support (root,0644,tmpfs) [sig-storage]" +test/e2e/common/empty_dir.go: "should support (root,0666,tmpfs) [sig-storage]" +test/e2e/common/empty_dir.go: "should support (root,0777,tmpfs) [sig-storage]" +test/e2e/common/empty_dir.go: "should support (non-root,0644,tmpfs) [sig-storage]" +test/e2e/common/empty_dir.go: "should support (non-root,0666,tmpfs) [sig-storage]" +test/e2e/common/empty_dir.go: "should support (non-root,0777,tmpfs) [sig-storage]" +test/e2e/common/empty_dir.go: "volume on default medium should have the correct mode [sig-storage]" +test/e2e/common/empty_dir.go: "should support (root,0644,default) [sig-storage]" +test/e2e/common/empty_dir.go: "should support (root,0666,default) [sig-storage]" +test/e2e/common/empty_dir.go: "should support (root,0777,default) [sig-storage]" +test/e2e/common/empty_dir.go: "should support (non-root,0644,default) [sig-storage]" +test/e2e/common/empty_dir.go: "should support (non-root,0666,default) [sig-storage]" +test/e2e/common/empty_dir.go: "should support (non-root,0777,default) [sig-storage]" +test/e2e/common/expansion.go: "should allow composing env vars into new env vars " +test/e2e/common/expansion.go: "should allow substituting values in a container's command " +test/e2e/common/expansion.go: "should allow substituting values in a container's args " +test/e2e/common/host_path.go: "should give a volume the correct mode [sig-storage]" +test/e2e/common/kubelet_etc_hosts.go: "should test kubelet managed /etc/hosts file " +test/e2e/common/networking.go: "should function for intra-pod communication: http " +test/e2e/common/networking.go: "should function for intra-pod communication: udp " +test/e2e/common/networking.go: "should function for node-pod communication: http " +test/e2e/common/networking.go: "should function for node-pod communication: udp " +test/e2e/common/pods.go: "should get a host IP " +test/e2e/common/pods.go: "should be submitted and removed " +test/e2e/common/pods.go: "should be updated " +test/e2e/common/pods.go: "should allow activeDeadlineSeconds to be updated " +test/e2e/common/pods.go: "should contain environment variables for services " +test/e2e/common/projected.go: "should be consumable from pods in volume [sig-storage]" +test/e2e/common/projected.go: "should be consumable from pods in volume with defaultMode set [sig-storage]" +test/e2e/common/projected.go: "should be consumable from pods in volume as non-root with defaultMode and fsGroup set [sig-storage]" +test/e2e/common/projected.go: "should be consumable from pods in volume with mappings [sig-storage]" +test/e2e/common/projected.go: "should be consumable from pods in volume with mappings and Item Mode set [sig-storage]" +test/e2e/common/projected.go: "should be consumable in multiple volumes in a pod [sig-storage]" +test/e2e/common/projected.go: "optional updates should be reflected in volume [sig-storage]" +test/e2e/common/projected.go: "should be consumable from pods in volume [sig-storage]" +test/e2e/common/projected.go: "should be consumable from pods in volume with defaultMode set [sig-storage]" +test/e2e/common/projected.go: "should be consumable from pods in volume as non-root [sig-storage]" +test/e2e/common/projected.go: "should be consumable from pods in volume with mappings [sig-storage]" +test/e2e/common/projected.go: "should be consumable from pods in volume with mappings and Item mode set [sig-storage]" +test/e2e/common/projected.go: "should be consumable from pods in volume with mappings as non-root [sig-storage]" +test/e2e/common/projected.go: "updates should be reflected in volume [sig-storage]" +test/e2e/common/projected.go: "optional updates should be reflected in volume [sig-storage]" +test/e2e/common/projected.go: "should be consumable in multiple volumes in the same pod [sig-storage]" +test/e2e/common/projected.go: "should provide podname only [sig-storage]" +test/e2e/common/projected.go: "should set DefaultMode on files [sig-storage]" +test/e2e/common/projected.go: "should set mode on item file [sig-storage]" +test/e2e/common/projected.go: "should update labels on modification [sig-storage]" +test/e2e/common/projected.go: "should update annotations on modification [sig-storage]" +test/e2e/common/projected.go: "should provide container's cpu limit [sig-storage]" +test/e2e/common/projected.go: "should provide container's memory limit [sig-storage]" +test/e2e/common/projected.go: "should provide container's cpu request [sig-storage]" +test/e2e/common/projected.go: "should provide container's memory request [sig-storage]" +test/e2e/common/projected.go: "should provide node allocatable (cpu) as default cpu limit if the limit is not set [sig-storage]" +test/e2e/common/projected.go: "should provide node allocatable (memory) as default memory limit if the limit is not set [sig-storage]" +test/e2e/common/projected.go: "should project all components that make up the projection API [sig-storage] [Projection]" +test/e2e/common/secrets.go: "should be consumable from pods in env vars " +test/e2e/common/secrets.go: "should be consumable via the environment " +test/e2e/common/secrets_volume.go: "should be consumable from pods in volume " +test/e2e/common/secrets_volume.go: "should be consumable from pods in volume with defaultMode set " +test/e2e/common/secrets_volume.go: "should be consumable from pods in volume as non-root with defaultMode and fsGroup set " +test/e2e/common/secrets_volume.go: "should be consumable from pods in volume with mappings " +test/e2e/common/secrets_volume.go: "should be consumable from pods in volume with mappings and Item Mode set " +test/e2e/common/secrets_volume.go: "should be consumable in multiple volumes in a pod " +test/e2e/common/secrets_volume.go: "optional updates should be reflected in volume " +test/e2e/events.go: "should be sent by kubelets and the scheduler about pods scheduling and running " +test/e2e/kubectl/kubectl.go: "should create and stop a replication controller " +test/e2e/kubectl/kubectl.go: "should scale a replication controller " +test/e2e/kubectl/kubectl.go: "should do a rolling update of a replication controller " +test/e2e/kubectl/kubectl.go: "should create and stop a working application " +test/e2e/kubectl/kubectl.go: "should check if v1 is in available api versions " +test/e2e/kubectl/kubectl.go: "should check if Kubernetes master services is included in cluster-info " +test/e2e/kubectl/kubectl.go: "should check if kubectl describe prints relevant information for rc and pods " +test/e2e/kubectl/kubectl.go: "should create services for rc " +test/e2e/kubectl/kubectl.go: "should update the label on a resource " +test/e2e/kubectl/kubectl.go: "should be able to retrieve and filter logs " +test/e2e/kubectl/kubectl.go: "should add annotations for pods in rc " +test/e2e/kubectl/kubectl.go: "should check is all data is printed " +test/e2e/kubectl/kubectl.go: "should create an rc or deployment from an image " +test/e2e/kubectl/kubectl.go: "should create an rc from an image " +test/e2e/kubectl/kubectl.go: "should support rolling-update to same image " +test/e2e/kubectl/kubectl.go: "should create a deployment from an image " +test/e2e/kubectl/kubectl.go: "should create a job from an image when restart is OnFailure " +test/e2e/kubectl/kubectl.go: "should create a pod from an image when restart is Never " +test/e2e/kubectl/kubectl.go: "should update a single-container pod's image " +test/e2e/kubectl/kubectl.go: "should create a job from an image, then delete the job " +test/e2e/kubectl/kubectl.go: "should support proxy with --port 0 " +test/e2e/kubectl/kubectl.go: "should support --unix-socket=/path " +test/e2e/network/dns.go: "should provide DNS for the cluster " +test/e2e/network/dns.go: "should provide DNS for services " +test/e2e/network/proxy.go: "should proxy logs on node with explicit kubelet port " +test/e2e/network/proxy.go: "should proxy logs on node " +test/e2e/network/proxy.go: "should proxy logs on node with explicit kubelet port using proxy subresource " +test/e2e/network/proxy.go: "should proxy logs on node using proxy subresource " +test/e2e/network/proxy.go: "should proxy through a service and a pod " +test/e2e/network/service.go: "should provide secure master service " +test/e2e/network/service.go: "should serve a basic endpoint from pods " +test/e2e/network/service.go: "should serve multiport endpoints from pods " +test/e2e/network/service_latency.go: "should not be very high " +test/e2e/pods.go: "should be submitted and removed [Flaky]" +test/e2e/pods.go: "should be submitted and removed " +test/e2e/pre_stop.go: "should call prestop when killing a pod " +test/e2e/scheduling/predicates.go: "validates resource limits of pods that are allowed to run " +test/e2e/scheduling/predicates.go: "validates that NodeSelector is respected if not matching " +test/e2e/scheduling/predicates.go: "validates that NodeSelector is respected if matching " diff --git a/test/conformance/walk.go b/test/conformance/walk.go new file mode 100644 index 0000000000..9637620f18 --- /dev/null +++ b/test/conformance/walk.go @@ -0,0 +1,166 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package main provides a tool that scans kubernetes e2e test source code +// looking for conformance test declarations, which it emits on stdout. It +// also looks for legacy, manually added "[Conformance]" tags and reports an +// error if it finds any. +// +// This approach is not air tight, but it will serve our purpose as a +// pre-submit check. +package main + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "os" + "path/filepath" + "strings" +) + +type visitor struct { + FileSet *token.FileSet +} + +func newVisitor() *visitor { + return &visitor{ + FileSet: token.NewFileSet(), + } +} + +func (v *visitor) isConformanceCall(call *ast.CallExpr) bool { + switch fun := call.Fun.(type) { + case *ast.SelectorExpr: + if fun.Sel != nil { + return fun.Sel.Name == "ConformanceIt" + } + } + return false +} + +func (v *visitor) isLegacyItCall(call *ast.CallExpr) bool { + switch fun := call.Fun.(type) { + case *ast.Ident: + if fun.Name != "It" { + return false + } + if len(call.Args) < 1 { + v.failf(call, "Not enough arguments to It()") + } + default: + return false + } + + switch arg := call.Args[0].(type) { + case *ast.BasicLit: + if arg.Kind != token.STRING { + v.failf(arg, "Unexpected non-string argument to It()") + } + if strings.Contains(arg.Value, "[Conformance]") { + return true + } + default: + // non-literal argument to It()... we just ignore these even though they could be a way to "sneak in" a conformance test + } + + return false +} + +func (v *visitor) failf(expr ast.Expr, format string, a ...interface{}) { + msg := fmt.Sprintf(format, a...) + fmt.Fprintf(os.Stderr, "ERROR at %v: %s\n", v.FileSet.Position(expr.Pos()), msg) + os.Exit(65) +} + +func (v *visitor) emit(arg ast.Expr) { + switch at := arg.(type) { + case *ast.BasicLit: + if at.Kind != token.STRING { + v.failf(at, "framework.ConformanceIt() called with non-string argument") + return + } + fmt.Printf("%s: %s\n", v.FileSet.Position(at.Pos()).Filename, at.Value) + default: + v.failf(at, "framework.ConformanceIt() called with non-literal argument") + fmt.Fprintf(os.Stderr, "ERROR: non-literal argument %v at %v\n", arg, v.FileSet.Position(arg.Pos())) + } +} + +// Visit visits each node looking for either calls to framework.ConformanceIt, +// which it will emit in its list of conformance tests, or legacy calls to +// It() with a manually embedded [Conformance] tag, which it will complain +// about. +func (v *visitor) Visit(node ast.Node) (w ast.Visitor) { + switch t := node.(type) { + case *ast.CallExpr: + if v.isConformanceCall(t) { + v.emit(t.Args[0]) + } else if v.isLegacyItCall(t) { + v.failf(t, "Using It() with manual [Conformance] tag is no longer allowed. Use framework.ConformanceIt() instead.") + return nil + } + } + return v +} + +func scandir(dir string) { + v := newVisitor() + pkg, err := parser.ParseDir(v.FileSet, dir, nil, 0) + if err != nil { + panic(err) + } + + for _, p := range pkg { + ast.Walk(v, p) + } +} + +func scanfile(path string) { + v := newVisitor() + file, err := parser.ParseFile(v.FileSet, path, nil, 0) + if err != nil { + panic(err) + } + + ast.Walk(v, file) +} + +func main() { + args := os.Args[1:] + if len(args) < 1 { + fmt.Fprintf(os.Stderr, "USAGE: %s [...]\n", os.Args[0]) + os.Exit(64) + } + + for _, arg := range args { + filepath.Walk(arg, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + scandir(path) + } else { + // TODO(mml): Remove this once we have all-go-srcs build rules. See https://github.com/kubernetes/repo-infra/pull/45 + if strings.HasSuffix(path, ".go") { + scanfile(path) + } + } + return nil + }) + } +}