mirror of https://github.com/k3s-io/k3s
Initial revision of conformance node environment conformance tests
Phillip Wittrock
parent
6b358e43dd
commit
0e2ade336a
|
|
@ -0,0 +1,258 @@
|
||||||
|
/*
|
||||||
|
Copyright 2015 The Kubernetes Authors All rights reserved.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
// Build the binary with `go build conformance.go`, then run the conformance binary on a node candidate. If compiled
|
||||||
|
// on a non-linux machine, must be cross compiled for the host.
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"io/ioutil"
|
||||||
|
"net"
|
||||||
|
"os/exec"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"errors"
|
||||||
|
"k8s.io/kubernetes/pkg/kubelet/cadvisor"
|
||||||
|
"os"
|
||||||
|
)
|
||||||
|
|
||||||
|
const success = "\033[0;32mSUCESS\033[0m"
|
||||||
|
const failed = "\033[0;31mFAILED\033[0m"
|
||||||
|
const notConfigured = "\033[0;34mNOT CONFIGURED\033[0m"
|
||||||
|
const skipped = "\033[0;34mSKIPPED\033[0m"
|
||||||
|
|
||||||
|
var checkFlag = flag.String(
|
||||||
|
"check", "all", "what to check for conformance. One or more of all,container-runtime,daemons,dns,firewall,kernel")
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
// Set this to false to undo util/logs.go settings it to true. Prevents cadvisor log spam.
|
||||||
|
// Remove this once util/logs.go stops setting the flag to true.
|
||||||
|
flag.Set("logtostderr", "false")
|
||||||
|
flag.Parse()
|
||||||
|
}
|
||||||
|
|
||||||
|
// TODO: Should we write an e2e test for this?
|
||||||
|
func main() {
|
||||||
|
o := strings.Split(*checkFlag, ",")
|
||||||
|
errs := check(o...)
|
||||||
|
if len(errs) > 0 {
|
||||||
|
os.Exit(1)
|
||||||
|
} else {
|
||||||
|
os.Exit(0)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// check returns errors found while checking the provided components. Will prevent errors to stdout.
|
||||||
|
func check(options ...string) []error {
|
||||||
|
errs := []error{}
|
||||||
|
for _, c := range options {
|
||||||
|
switch c {
|
||||||
|
case "all":
|
||||||
|
errs = appendNotNil(errs, kernel())
|
||||||
|
errs = appendNotNil(errs, containerRuntime())
|
||||||
|
errs = appendNotNil(errs, daemons())
|
||||||
|
errs = appendNotNil(errs, firewall())
|
||||||
|
errs = appendNotNil(errs, dns())
|
||||||
|
case "containerruntime":
|
||||||
|
errs = appendNotNil(errs, containerRuntime())
|
||||||
|
case "daemons":
|
||||||
|
errs = appendNotNil(errs, daemons())
|
||||||
|
case "dns":
|
||||||
|
errs = appendNotNil(errs, dns())
|
||||||
|
case "firewall":
|
||||||
|
errs = appendNotNil(errs, firewall())
|
||||||
|
case "kernel":
|
||||||
|
errs = appendNotNil(errs, kernel())
|
||||||
|
default:
|
||||||
|
fmt.Printf("Unrecognized option %s", c)
|
||||||
|
errs = append(errs, errors.New(fmt.Sprintf("Unrecognized option %s", c)))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return errs
|
||||||
|
}
|
||||||
|
|
||||||
|
const dockerVersionRegex = `1\.[7-9]\.[0-9]+`
|
||||||
|
|
||||||
|
// containerRuntime checks that a suitable container runtime is installed and recognized by cadvisor: docker 1.7-1.9
|
||||||
|
func containerRuntime() error {
|
||||||
|
dockerRegex, err := regexp.Compile(dockerVersionRegex)
|
||||||
|
if err != nil {
|
||||||
|
// This should never happen and can only be fixed by changing the code
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Setup cadvisor to check the container environment
|
||||||
|
c, err := cadvisor.New(0 /*don't start the http server*/)
|
||||||
|
if err != nil {
|
||||||
|
return printError("Container Runtime Check: %s Could not start cadvisor %v", failed, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
vi, err := c.VersionInfo()
|
||||||
|
if err != nil {
|
||||||
|
return printError("Container Runtime Check: %s Could not get VersionInfo %v", failed, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
d := vi.DockerVersion
|
||||||
|
if !dockerRegex.Match([]byte(d)) {
|
||||||
|
return printError(
|
||||||
|
"Container Runtime Check: %s Docker version %s does not matching %s. You may need to run as root or the "+
|
||||||
|
"user the kubelet will run under.", failed, d, dockerVersionRegex)
|
||||||
|
}
|
||||||
|
|
||||||
|
return printSuccess("Container Runtime Check: %s", success)
|
||||||
|
}
|
||||||
|
|
||||||
|
const kubeletClusterDnsRegexStr = `\/kubelet.*--cluster-dns=(\S+) `
|
||||||
|
const kubeletClusterDomainRegexStr = `\/kubelet.*--cluster-domain=(\S+)`
|
||||||
|
|
||||||
|
// dns checks that cluster dns has been properly configured and can resolve the kubernetes.default service
|
||||||
|
func dns() error {
|
||||||
|
dnsRegex, err := regexp.Compile(kubeletClusterDnsRegexStr)
|
||||||
|
if err != nil {
|
||||||
|
// This should never happen and can only be fixed by changing the code
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
domainRegex, err := regexp.Compile(kubeletClusterDomainRegexStr)
|
||||||
|
if err != nil {
|
||||||
|
// This should never happen and can only be fixed by changing the code
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
h, err := net.LookupHost("kubernetes.default")
|
||||||
|
if err == nil {
|
||||||
|
return printSuccess("Dns Check (Optional): %s", success)
|
||||||
|
}
|
||||||
|
if len(h) > 0 {
|
||||||
|
return printSuccess("Dns Check (Optional): %s", success)
|
||||||
|
}
|
||||||
|
|
||||||
|
kubecmd, err := exec.Command("ps", "aux").CombinedOutput()
|
||||||
|
|
||||||
|
// look for the dns flag and parse the value
|
||||||
|
dns := dnsRegex.FindStringSubmatch(string(kubecmd))
|
||||||
|
if len(dns) < 2 {
|
||||||
|
return printSuccess(
|
||||||
|
"Dns Check (Optional): %s No hosts resolve to kubernetes.default. kubelet will need to set "+
|
||||||
|
"--cluster-dns and --cluster-domain when run", notConfigured)
|
||||||
|
}
|
||||||
|
|
||||||
|
// look for the domain flag and parse the value
|
||||||
|
domain := domainRegex.FindStringSubmatch(string(kubecmd))
|
||||||
|
if len(domain) < 2 {
|
||||||
|
return printSuccess(
|
||||||
|
"Dns Check (Optional): %s No hosts resolve to kubernetes.default. kubelet will need to set "+
|
||||||
|
"--cluster-dns and --cluster-domain when run", notConfigured)
|
||||||
|
}
|
||||||
|
|
||||||
|
// do a lookup with the flags the kubelet is running with
|
||||||
|
nsArgs := []string{"-q=a", fmt.Sprintf("kubernetes.default.%s", domain[1]), dns[1]}
|
||||||
|
if err = exec.Command("nslookup", nsArgs...).Run(); err != nil {
|
||||||
|
// Mark this as failed since there was a clear intention to set it up, but it is done so improperly
|
||||||
|
return printError(
|
||||||
|
"Dns Check (Optional): %s No hosts resolve to kubernetes.default kubelet found, but cannot resolve "+
|
||||||
|
"kubernetes.default using nslookup %s error: %v", failed, strings.Join(nsArgs, " "), err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Can resolve kubernetes.default using the kubelete dns and domain values
|
||||||
|
return printSuccess("Dns Check (Optional): %s", success)
|
||||||
|
}
|
||||||
|
|
||||||
|
const cmdlineCGroupMemory = `cgroup_enable=memory`
|
||||||
|
|
||||||
|
// kernel checks that the kernel has been configured correctly to support the required cgroup features
|
||||||
|
func kernel() error {
|
||||||
|
cmdline, err := ioutil.ReadFile("/proc/cmdline")
|
||||||
|
if err != nil {
|
||||||
|
return printError("Kernel Command Line Check %s: Could not check /proc/cmdline", failed)
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(cmdline), cmdlineCGroupMemory) {
|
||||||
|
return printError("Kernel Command Line Check %s: cgroup_enable=memory not enabled in /proc/cmdline", failed)
|
||||||
|
}
|
||||||
|
return printSuccess("Kernel Command Line %s", success)
|
||||||
|
}
|
||||||
|
|
||||||
|
const iptablesInputRegexStr = `Chain INPUT \(policy DROP\)`
|
||||||
|
const iptablesForwardRegexStr = `Chain FORWARD \(policy DROP\)`
|
||||||
|
|
||||||
|
// firewall checks that iptables does not have common firewall rules setup that would disrupt traffic
|
||||||
|
func firewall() error {
|
||||||
|
out, err := exec.Command("iptables", "-L", "INPUT").CombinedOutput()
|
||||||
|
if err != nil {
|
||||||
|
return printSuccess("Firewall Iptables Check %s: Could not run iptables", skipped)
|
||||||
|
}
|
||||||
|
inputRegex, err := regexp.Compile(iptablesInputRegexStr)
|
||||||
|
if err != nil {
|
||||||
|
// This should never happen and can only be fixed by changing the code
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
if inputRegex.Match(out) {
|
||||||
|
return printError("Firewall Iptables Check %s: Found INPUT rule matching %s", failed, iptablesInputRegexStr)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check GCE forward rules
|
||||||
|
out, err = exec.Command("iptables", "-L", "FORWARD").CombinedOutput()
|
||||||
|
if err != nil {
|
||||||
|
return printSuccess("Firewall Iptables Check %s: Could not run iptables", skipped)
|
||||||
|
}
|
||||||
|
forwardRegex, err := regexp.Compile(iptablesForwardRegexStr)
|
||||||
|
if err != nil {
|
||||||
|
// This should never happen and can only be fixed by changing the code
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
if forwardRegex.Match(out) {
|
||||||
|
return printError("Firewall Iptables Check %s: Found FORWARD rule matching %s", failed, iptablesInputRegexStr)
|
||||||
|
}
|
||||||
|
|
||||||
|
return printSuccess("Firewall Iptables Check %s", success)
|
||||||
|
}
|
||||||
|
|
||||||
|
// daemons checks that the required node programs are running: kubelet, kube-proxy, and docker
|
||||||
|
func daemons() error {
|
||||||
|
if exec.Command("pgrep", "-f", "kubelet").Run() != nil {
|
||||||
|
return printError("Daemon Check %s: kubelet process not found", failed)
|
||||||
|
}
|
||||||
|
|
||||||
|
if exec.Command("pgrep", "-f", "kube-proxy").Run() != nil {
|
||||||
|
return printError("Daemon Check %s: kube-proxy process not found", failed)
|
||||||
|
}
|
||||||
|
|
||||||
|
return printSuccess("Daemon Check %s", success)
|
||||||
|
}
|
||||||
|
|
||||||
|
// printError provides its arguments to print a format string to the console (newline terminated) and returns an
|
||||||
|
// error with the same string
|
||||||
|
func printError(s string, args ...interface{}) error {
|
||||||
|
es := fmt.Sprintf(s, args...)
|
||||||
|
fmt.Println(es)
|
||||||
|
return errors.New(es)
|
||||||
|
}
|
||||||
|
|
||||||
|
// printSuccess provides its arguments to print a format string to the console (newline terminated) and returns nil
|
||||||
|
func printSuccess(s string, args ...interface{}) error {
|
||||||
|
fmt.Println(fmt.Sprintf(s, args...))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// appendNotNil appends err to errs iff err is not nil
|
||||||
|
func appendNotNil(errs []error, err error) []error {
|
||||||
|
if err != nil {
|
||||||
|
return append(errs, err)
|
||||||
|
}
|
||||||
|
return errs
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue