github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Add conformance test regression test. 

This test creates a golden list of existing conformance tests.  Efforts
to add or remove conformance tests will require you to rebuild the
golden list, and changes to the golden list will be reviewed by
sig-architecture.
pull/6/head
Matt Liggett 2017-10-10 14:34:48 -07:00
parent a5967cbaf1
commit fad431118a
7 changed files with 400 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
test/BUILD
View File

@ -11,6 +11,7 @@ filegroup(
name = "all-srcs",
srcs = [
":package-srcs",
"//test/conformance:all-srcs",
"//test/e2e:all-srcs",
"//test/e2e_node:all-srcs",
"//test/fixtures:all-srcs",

42
test/conformance/BUILD Normal file
View File

@ -0,0 +1,42 @@
load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
go_library(
name = "go_default_library",
srcs = ["walk.go"],
visibility = ["//visibility:private"],
)
go_binary(
name = "conformance",
library = ":go_default_library",
visibility = ["//visibility:public"],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)
genrule(
name = "list_conformance_tests",
srcs = ["//test/e2e:all-srcs"],
outs = ["conformance.txt"],
cmd = "./$(location :conformance) $(locations //test/e2e:all-srcs) > $@",
message = "Listing all conformance tests.",
tools = [":conformance"],
)
sh_test(
name = "conformance_test",
srcs = ["conformance_test.sh"],
data = ["testdata/conformance.txt", ":list_conformance_tests"],
)

7
test/conformance/OWNERS Normal file
View File

@ -0,0 +1,7 @@
# This is the owner of the test code. The test data itself is owned by sig-architecture.
reviewers:
- mml
- cheftako
approvers:
- mml
- cheftako

25
test/conformance/conformance_test.sh Executable file
View File

@ -0,0 +1,25 @@
#!/bin/bash
# Copyright 2017 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# echo ${TEST_SRCDIR}
# pwd
# env | grep --color=always test/conformance
# find ${TEST_SRCDIR} -ls | grep --color=always test/conformance
set -o errexit
diff -u test/conformance/testdata/conformance.txt test/conformance/conformance.txt
echo PASS

9
test/conformance/testdata/OWNERS vendored Normal file
View File

@ -0,0 +1,9 @@
# To be owned by sig-architecture.
# TODO(mml): Exclude parent owners once
# https://github.com/kubernetes/test-infra/issues/5197 is implemented.
reviewers:
- bgrant0607
- smarterclayton
approvers:
- bgrant0607
- smarterclayton

150
test/conformance/testdata/conformance.txt vendored Executable file
View File

@ -0,0 +1,150 @@
test/e2e/apimachinery/custom_resource_definition.go: "creating/deleting custom resource definition objects works "
test/e2e/apps/rc.go: "should serve a basic image on each replica with a public image "
test/e2e/apps/replica_set.go: "should serve a basic image on each replica with a public image "
test/e2e/auth/service_accounts.go: "should mount an API token into pods "
test/e2e/auth/service_accounts.go: "should allow opting out of API token automount "
test/e2e/common/configmap.go: "should be consumable via environment variable "
test/e2e/common/configmap.go: "should be consumable via the environment "
test/e2e/common/configmap_volume.go: "should be consumable from pods in volume "
test/e2e/common/configmap_volume.go: "should be consumable from pods in volume with defaultMode set "
test/e2e/common/configmap_volume.go: "should be consumable from pods in volume as non-root "
test/e2e/common/configmap_volume.go: "should be consumable from pods in volume with mappings "
test/e2e/common/configmap_volume.go: "should be consumable from pods in volume with mappings and Item mode set"
test/e2e/common/configmap_volume.go: "should be consumable from pods in volume with mappings as non-root "
test/e2e/common/configmap_volume.go: "updates should be reflected in volume "
test/e2e/common/configmap_volume.go: "optional updates should be reflected in volume "
test/e2e/common/configmap_volume.go: "should be consumable in multiple volumes in the same pod "
test/e2e/common/container_probe.go: "with readiness probe should not be ready before initial delay and never restart "
test/e2e/common/container_probe.go: "with readiness probe that fails should never be ready and never restart "
test/e2e/common/container_probe.go: "should be restarted with a exec \"cat /tmp/health\" liveness probe"
test/e2e/common/container_probe.go: "should *not* be restarted with a exec \"cat /tmp/health\" liveness probe"
test/e2e/common/container_probe.go: "should be restarted with a /healthz http liveness probe "
test/e2e/common/container_probe.go: "should have monotonically increasing restart count [Slow]"
test/e2e/common/container_probe.go: "should *not* be restarted with a /healthz http liveness probe "
test/e2e/common/container_probe.go: "should be restarted with a docker exec liveness probe with timeout "
test/e2e/common/docker_containers.go: "should use the image defaults if command and args are blank "
test/e2e/common/docker_containers.go: "should be able to override the image's default arguments (docker cmd) "
test/e2e/common/docker_containers.go: "should be able to override the image's default commmand (docker entrypoint) "
test/e2e/common/docker_containers.go: "should be able to override the image's default command and arguments "
test/e2e/common/downward_api.go: "should provide pod name and namespace as env vars "
test/e2e/common/downward_api.go: "should provide pod IP as an env var "
test/e2e/common/downward_api.go: "should provide host IP as an env var "
test/e2e/common/downward_api.go: "should provide container's limits.cpu/memory and requests.cpu/memory as env vars "
test/e2e/common/downward_api.go: "should provide default limits.cpu/memory from node allocatable "
test/e2e/common/downward_api.go: "should provide pod UID as env vars "
test/e2e/common/downwardapi_volume.go: "should provide podname only "
test/e2e/common/downwardapi_volume.go: "should set DefaultMode on files "
test/e2e/common/downwardapi_volume.go: "should set mode on item file "
test/e2e/common/downwardapi_volume.go: "should update labels on modification "
test/e2e/common/downwardapi_volume.go: "should update annotations on modification "
test/e2e/common/downwardapi_volume.go: "should provide container's cpu limit "
test/e2e/common/downwardapi_volume.go: "should provide container's memory limit "
test/e2e/common/downwardapi_volume.go: "should provide container's cpu request "
test/e2e/common/downwardapi_volume.go: "should provide container's memory request "
test/e2e/common/downwardapi_volume.go: "should provide node allocatable (cpu) as default cpu limit if the limit is not set "
test/e2e/common/downwardapi_volume.go: "should provide node allocatable (memory) as default memory limit if the limit is not set "
test/e2e/common/empty_dir.go: "volume on tmpfs should have the correct mode [sig-storage]"
test/e2e/common/empty_dir.go: "should support (root,0644,tmpfs) [sig-storage]"
test/e2e/common/empty_dir.go: "should support (root,0666,tmpfs) [sig-storage]"
test/e2e/common/empty_dir.go: "should support (root,0777,tmpfs) [sig-storage]"
test/e2e/common/empty_dir.go: "should support (non-root,0644,tmpfs) [sig-storage]"
test/e2e/common/empty_dir.go: "should support (non-root,0666,tmpfs) [sig-storage]"
test/e2e/common/empty_dir.go: "should support (non-root,0777,tmpfs) [sig-storage]"
test/e2e/common/empty_dir.go: "volume on default medium should have the correct mode [sig-storage]"
test/e2e/common/empty_dir.go: "should support (root,0644,default) [sig-storage]"
test/e2e/common/empty_dir.go: "should support (root,0666,default) [sig-storage]"
test/e2e/common/empty_dir.go: "should support (root,0777,default) [sig-storage]"
test/e2e/common/empty_dir.go: "should support (non-root,0644,default) [sig-storage]"
test/e2e/common/empty_dir.go: "should support (non-root,0666,default) [sig-storage]"
test/e2e/common/empty_dir.go: "should support (non-root,0777,default) [sig-storage]"
test/e2e/common/expansion.go: "should allow composing env vars into new env vars "
test/e2e/common/expansion.go: "should allow substituting values in a container's command "
test/e2e/common/expansion.go: "should allow substituting values in a container's args "
test/e2e/common/host_path.go: "should give a volume the correct mode [sig-storage]"
test/e2e/common/kubelet_etc_hosts.go: "should test kubelet managed /etc/hosts file "
test/e2e/common/networking.go: "should function for intra-pod communication: http "
test/e2e/common/networking.go: "should function for intra-pod communication: udp "
test/e2e/common/networking.go: "should function for node-pod communication: http "
test/e2e/common/networking.go: "should function for node-pod communication: udp "
test/e2e/common/pods.go: "should get a host IP "
test/e2e/common/pods.go: "should be submitted and removed "
test/e2e/common/pods.go: "should be updated "
test/e2e/common/pods.go: "should allow activeDeadlineSeconds to be updated "
test/e2e/common/pods.go: "should contain environment variables for services "
test/e2e/common/projected.go: "should be consumable from pods in volume [sig-storage]"
test/e2e/common/projected.go: "should be consumable from pods in volume with defaultMode set [sig-storage]"
test/e2e/common/projected.go: "should be consumable from pods in volume as non-root with defaultMode and fsGroup set [sig-storage]"
test/e2e/common/projected.go: "should be consumable from pods in volume with mappings [sig-storage]"
test/e2e/common/projected.go: "should be consumable from pods in volume with mappings and Item Mode set [sig-storage]"
test/e2e/common/projected.go: "should be consumable in multiple volumes in a pod [sig-storage]"
test/e2e/common/projected.go: "optional updates should be reflected in volume [sig-storage]"
test/e2e/common/projected.go: "should be consumable from pods in volume [sig-storage]"
test/e2e/common/projected.go: "should be consumable from pods in volume with defaultMode set [sig-storage]"
test/e2e/common/projected.go: "should be consumable from pods in volume as non-root [sig-storage]"
test/e2e/common/projected.go: "should be consumable from pods in volume with mappings [sig-storage]"
test/e2e/common/projected.go: "should be consumable from pods in volume with mappings and Item mode set [sig-storage]"
test/e2e/common/projected.go: "should be consumable from pods in volume with mappings as non-root [sig-storage]"
test/e2e/common/projected.go: "updates should be reflected in volume [sig-storage]"
test/e2e/common/projected.go: "optional updates should be reflected in volume [sig-storage]"
test/e2e/common/projected.go: "should be consumable in multiple volumes in the same pod [sig-storage]"
test/e2e/common/projected.go: "should provide podname only [sig-storage]"
test/e2e/common/projected.go: "should set DefaultMode on files [sig-storage]"
test/e2e/common/projected.go: "should set mode on item file [sig-storage]"
test/e2e/common/projected.go: "should update labels on modification [sig-storage]"
test/e2e/common/projected.go: "should update annotations on modification [sig-storage]"
test/e2e/common/projected.go: "should provide container's cpu limit [sig-storage]"
test/e2e/common/projected.go: "should provide container's memory limit [sig-storage]"
test/e2e/common/projected.go: "should provide container's cpu request [sig-storage]"
test/e2e/common/projected.go: "should provide container's memory request [sig-storage]"
test/e2e/common/projected.go: "should provide node allocatable (cpu) as default cpu limit if the limit is not set [sig-storage]"
test/e2e/common/projected.go: "should provide node allocatable (memory) as default memory limit if the limit is not set [sig-storage]"
test/e2e/common/projected.go: "should project all components that make up the projection API [sig-storage] [Projection]"
test/e2e/common/secrets.go: "should be consumable from pods in env vars "
test/e2e/common/secrets.go: "should be consumable via the environment "
test/e2e/common/secrets_volume.go: "should be consumable from pods in volume "
test/e2e/common/secrets_volume.go: "should be consumable from pods in volume with defaultMode set "
test/e2e/common/secrets_volume.go: "should be consumable from pods in volume as non-root with defaultMode and fsGroup set "
test/e2e/common/secrets_volume.go: "should be consumable from pods in volume with mappings "
test/e2e/common/secrets_volume.go: "should be consumable from pods in volume with mappings and Item Mode set "
test/e2e/common/secrets_volume.go: "should be consumable in multiple volumes in a pod "
test/e2e/common/secrets_volume.go: "optional updates should be reflected in volume "
test/e2e/events.go: "should be sent by kubelets and the scheduler about pods scheduling and running "
test/e2e/kubectl/kubectl.go: "should create and stop a replication controller "
test/e2e/kubectl/kubectl.go: "should scale a replication controller "
test/e2e/kubectl/kubectl.go: "should do a rolling update of a replication controller "
test/e2e/kubectl/kubectl.go: "should create and stop a working application "
test/e2e/kubectl/kubectl.go: "should check if v1 is in available api versions "
test/e2e/kubectl/kubectl.go: "should check if Kubernetes master services is included in cluster-info "
test/e2e/kubectl/kubectl.go: "should check if kubectl describe prints relevant information for rc and pods "
test/e2e/kubectl/kubectl.go: "should create services for rc "
test/e2e/kubectl/kubectl.go: "should update the label on a resource "
test/e2e/kubectl/kubectl.go: "should be able to retrieve and filter logs "
test/e2e/kubectl/kubectl.go: "should add annotations for pods in rc "
test/e2e/kubectl/kubectl.go: "should check is all data is printed "
test/e2e/kubectl/kubectl.go: "should create an rc or deployment from an image "
test/e2e/kubectl/kubectl.go: "should create an rc from an image "
test/e2e/kubectl/kubectl.go: "should support rolling-update to same image "
test/e2e/kubectl/kubectl.go: "should create a deployment from an image "
test/e2e/kubectl/kubectl.go: "should create a job from an image when restart is OnFailure "
test/e2e/kubectl/kubectl.go: "should create a pod from an image when restart is Never "
test/e2e/kubectl/kubectl.go: "should update a single-container pod's image "
test/e2e/kubectl/kubectl.go: "should create a job from an image, then delete the job "
test/e2e/kubectl/kubectl.go: "should support proxy with --port 0 "
test/e2e/kubectl/kubectl.go: "should support --unix-socket=/path "
test/e2e/network/dns.go: "should provide DNS for the cluster "
test/e2e/network/dns.go: "should provide DNS for services "
test/e2e/network/proxy.go: "should proxy logs on node with explicit kubelet port "
test/e2e/network/proxy.go: "should proxy logs on node "
test/e2e/network/proxy.go: "should proxy logs on node with explicit kubelet port using proxy subresource "
test/e2e/network/proxy.go: "should proxy logs on node using proxy subresource "
test/e2e/network/proxy.go: "should proxy through a service and a pod "
test/e2e/network/service.go: "should provide secure master service "
test/e2e/network/service.go: "should serve a basic endpoint from pods "
test/e2e/network/service.go: "should serve multiport endpoints from pods "
test/e2e/network/service_latency.go: "should not be very high "
test/e2e/pods.go: "should be submitted and removed [Flaky]"
test/e2e/pods.go: "should be submitted and removed "
test/e2e/pre_stop.go: "should call prestop when killing a pod "
test/e2e/scheduling/predicates.go: "validates resource limits of pods that are allowed to run "
test/e2e/scheduling/predicates.go: "validates that NodeSelector is respected if not matching "
test/e2e/scheduling/predicates.go: "validates that NodeSelector is respected if matching "

166
test/conformance/walk.go Normal file
View File

@ -0,0 +1,166 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package main provides a tool that scans kubernetes e2e test source code
// looking for conformance test declarations, which it emits on stdout. It
// also looks for legacy, manually added "[Conformance]" tags and reports an
// error if it finds any.
//
// This approach is not air tight, but it will serve our purpose as a
// pre-submit check.
package main
import (
"fmt"
"go/ast"
"go/parser"
"go/token"
"os"
"path/filepath"
"strings"
)
type visitor struct {
FileSet *token.FileSet
}
func newVisitor() *visitor {
return &visitor{
FileSet: token.NewFileSet(),
}
}
func (v *visitor) isConformanceCall(call *ast.CallExpr) bool {
switch fun := call.Fun.(type) {
case *ast.SelectorExpr:
if fun.Sel != nil {
return fun.Sel.Name == "ConformanceIt"
}
}
return false
}
func (v *visitor) isLegacyItCall(call *ast.CallExpr) bool {
switch fun := call.Fun.(type) {
case *ast.Ident:
if fun.Name != "It" {
return false
}
if len(call.Args) < 1 {
v.failf(call, "Not enough arguments to It()")
}
default:
return false
}
switch arg := call.Args[0].(type) {
case *ast.BasicLit:
if arg.Kind != token.STRING {
v.failf(arg, "Unexpected non-string argument to It()")
}
if strings.Contains(arg.Value, "[Conformance]") {
return true
}
default:
// non-literal argument to It()... we just ignore these even though they could be a way to "sneak in" a conformance test
}
return false
}
func (v *visitor) failf(expr ast.Expr, format string, a ...interface{}) {
msg := fmt.Sprintf(format, a...)
fmt.Fprintf(os.Stderr, "ERROR at %v: %s\n", v.FileSet.Position(expr.Pos()), msg)
os.Exit(65)
}
func (v *visitor) emit(arg ast.Expr) {
switch at := arg.(type) {
case *ast.BasicLit:
if at.Kind != token.STRING {
v.failf(at, "framework.ConformanceIt() called with non-string argument")
return
}
fmt.Printf("%s: %s\n", v.FileSet.Position(at.Pos()).Filename, at.Value)
default:
v.failf(at, "framework.ConformanceIt() called with non-literal argument")
fmt.Fprintf(os.Stderr, "ERROR: non-literal argument %v at %v\n", arg, v.FileSet.Position(arg.Pos()))
}
}
// Visit visits each node looking for either calls to framework.ConformanceIt,
// which it will emit in its list of conformance tests, or legacy calls to
// It() with a manually embedded [Conformance] tag, which it will complain
// about.
func (v *visitor) Visit(node ast.Node) (w ast.Visitor) {
switch t := node.(type) {
case *ast.CallExpr:
if v.isConformanceCall(t) {
v.emit(t.Args[0])
} else if v.isLegacyItCall(t) {
v.failf(t, "Using It() with manual [Conformance] tag is no longer allowed. Use framework.ConformanceIt() instead.")
return nil
}
}
return v
}
func scandir(dir string) {
v := newVisitor()
pkg, err := parser.ParseDir(v.FileSet, dir, nil, 0)
if err != nil {
panic(err)
}
for _, p := range pkg {
ast.Walk(v, p)
}
}
func scanfile(path string) {
v := newVisitor()
file, err := parser.ParseFile(v.FileSet, path, nil, 0)
if err != nil {
panic(err)
}
ast.Walk(v, file)
}
func main() {
args := os.Args[1:]
if len(args) < 1 {
fmt.Fprintf(os.Stderr, "USAGE: %s <DIR or FILE> [...]\n", os.Args[0])
os.Exit(64)
}
for _, arg := range args {
filepath.Walk(arg, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if info.IsDir() {
scandir(path)
} else {
// TODO(mml): Remove this once we have all-go-srcs build rules. See https://github.com/kubernetes/repo-infra/pull/45
if strings.HasSuffix(path, ".go") {
scanfile(path)
}
}
return nil
})
}
}