diff --git a/cmd/k3s/main.go b/cmd/k3s/main.go index 20055c7e63..016ec4bac7 100644 --- a/cmd/k3s/main.go +++ b/cmd/k3s/main.go @@ -29,6 +29,7 @@ func main() { cmds.NewKubectlCommand(externalCLIAction("kubectl")), cmds.NewCRICTL(externalCLIAction("crictl")), cmds.NewCtrCommand(externalCLIAction("ctr")), + cmds.NewCheckConfigCommand(externalCLIAction("check-config")), } err := app.Run(os.Args) diff --git a/contrib/util/check-config.sh b/contrib/util/check-config.sh new file mode 100755 index 0000000000..fade3575b2 --- /dev/null +++ b/contrib/util/check-config.sh @@ -0,0 +1,490 @@ +#!/bin/sh +set -e + +EXITCODE=0 + +# rancher/k3s modified from +# https://github.com/moby/moby/blob/c831882/contrib/check-config.sh + +# bits of this were adapted from lxc-checkconfig for moby +# see also https://github.com/lxc/lxc/blob/lxc-1.0.2/src/lxc/lxc-checkconfig.in + +uname=$(uname -r) + +possibleConfigs=" + /proc/config.gz + /boot/config-${uname} + /boot/config-${uname##*-} + /usr/src/linux-${uname}/.config + /usr/src/linux/.config +" + +if [ $# -gt 0 ]; then + CONFIG="$1" +fi + +if ! command -v zgrep >/dev/null 2>&1; then + zgrep() { + zcat "$2" | grep "$1" + } +fi + +configFormat=gz + +dogrep() { + if [ "$configFormat" = "gz" ]; then + zgrep "$1" "$2" + else + grep "$1" "$2" + fi +} + +kernelVersion="$(uname -r)" +kernelMajor="${kernelVersion%%.*}" +kernelMinor="${kernelVersion#$kernelMajor.}" +kernelMinor="${kernelMinor%%.*}" + +is_set() { + dogrep "CONFIG_$1=[y|m]" "$CONFIG" > /dev/null +} +is_set_in_kernel() { + dogrep "CONFIG_$1=y" "$CONFIG" > /dev/null +} +is_set_as_module() { + dogrep "CONFIG_$1=m" "$CONFIG" > /dev/null +} + +color() { + codes= + if [ "$1" = 'bold' ]; then + codes=1 + shift + fi + if [ "$#" -gt 0 ]; then + code= + case "$1" in + # see https://en.wikipedia.org/wiki/ANSI_escape_code#Colors + black) code=30 ;; + red) code=31 ;; + green) code=32 ;; + yellow) code=33 ;; + blue) code=34 ;; + magenta) code=35 ;; + cyan) code=36 ;; + white) code=37 ;; + esac + if [ "$code" ]; then + [ "$codes" ] && codes="${codes};" + codes="${codes}${code}" + fi + fi + printf '\033['"$codes"'m' +} +wrap_color() { + text="$1" + shift + color "$@" + echo -n "$text" + color reset + echo +} + +wrap_good() { + echo "$(wrap_color "$1" white): $(wrap_color "$2" green)" +} +wrap_bad() { + echo "$(wrap_color "$1" bold): $(wrap_color "$2" bold red)" +} +wrap_warn() { + echo "$(wrap_color "$1" bold): $(wrap_color "$2" bold yellow)" +} +warning() { + wrap_color >&2 "$*" yellow +} + +check_flag() { + if is_set_in_kernel "$1"; then + wrap_good "CONFIG_$1" 'enabled' + elif is_set_as_module "$1"; then + wrap_good "CONFIG_$1" 'enabled (as module)' + else + if [ "$IS_ERROR" = 1 ]; then + wrap_bad "CONFIG_$1" 'missing' + EXITCODE=1 + else + wrap_warn "CONFIG_$1" 'missing' + fi + fi +} + +check_flags() { + for flag in "$@"; do + echo -n "- "; check_flag "$flag" + done +} + +check_command() { + if command -v "$1" >/dev/null 2>&1; then + wrap_good "$1 command" 'available' + else + wrap_bad "$1 command" 'missing' + EXITCODE=1 + fi +} + +check_device() { + if [ -c "$1" ]; then + wrap_good "$1" 'present' + else + wrap_bad "$1" 'missing' + EXITCODE=1 + fi +} + +check_distro_userns() { + [ -s /etc/os-release ] || return 0 + source /etc/os-release 2>/dev/null || true + if ( echo "${ID}" | grep -q -E '^(centos|rhel)$' ) && \ + ( echo "${VERSION_ID}" | grep -q -E '^7' ); then + # this is a CentOS7 or RHEL7 system + if ! grep -q "user_namespace.enable=1" /proc/cmdline; then + # no user namespace support enabled + wrap_bad " (RHEL7/CentOS7" "User namespaces disabled; add 'user_namespace.enable=1' to boot command line)" + EXITCODE=1 + fi + fi +} + +# --- + +echo + +{ + BINDIR=$(dirname "$0") + cd $BINDIR + echo "Verifying binaries in $BINDIR:" + + if [ -s .sha256sums ]; then + sumsTemp=$(mktemp) + if sha256sum -c .sha256sums >$sumsTemp 2>&1; then + wrap_good '- sha256sum' 'good' + else + wrap_bad '- sha256sum' 'does not match' + cat $sumsTemp | sed -e 's/^/ ... /' + EXITCODE=1 + fi + rm -f $sumsTemp + else + wrap_warn '- sha256sum' 'sha256sums unavailable' + fi + + linkFail=0 + if [ -s .links ]; then + while read file link; do + if [ "$(readlink $file)" != "$link" ]; then + wrap_bad '- links' "$file should link to $link" + linkFail=1 + fi + done <.links + if [ $linkFail -eq 0 ]; then + wrap_good '- links' 'good' + else + EXITCODE=1 + fi + else + wrap_warn '- links' 'link list unavailable' + fi + + cd - >/dev/null +} + +echo + +{ + version_ge() { + [ "$1" = "$2" ] || [ "$(printf '%s\n' "$@" | sort -V | head -n 1)" != "$1" ] + } + + echo "System:" + iptablesInfo=$(iptables --version) + iptablesVersion=$(echo $iptablesInfo | awk '{ print $2 }') + if version_ge $iptablesVersion v1.8.0; then + iptablesMode=$(echo $iptablesInfo | awk '{ print $3 }') + if [ "$iptablesMode" != "(legacy)" ]; then + wrap_bad "- $iptablesInfo" 'should be older than v1.8.0 or in legacy mode' + EXITCODE=1 + else + wrap_good "- $iptablesInfo" 'ok' + fi + else + wrap_good "- $iptablesInfo" 'older than v1.8' + fi + + totalSwap=$(free | grep -i '^swap:' | awk '{ print $2 }') + if [ "$totalSwap" != "0" ]; then + wrap_warn '- swap' 'should be disabled' + else + wrap_good '- swap' 'disabled' + fi + + if ip route | grep -v cni0 | grep -q -E '^10\.(42|43)\.'; then + wrap_warn '- routes' 'default CIDRs 10.42.0.0/16 or 10.43.0.0/16 already routed' + else + wrap_good '- routes' 'ok' + fi +} + +echo + +{ + check_limit_over() + { + if [ "$(cat "$1")" -le "$2" ]; then + wrap_bad "- $1" "$(cat "$1")" + wrap_color " This should be set to at least $2, for example set: sysctl -w kernel/keys/root_maxkeys=1000000" bold black + EXITCODE=1 + else + wrap_good "- $1" "$(cat "$1")" + fi + } + + echo 'Limits:' + check_limit_over /proc/sys/kernel/keys/root_maxkeys 10000 +} + +echo + +# --- + +SUDO= +[ $(id -u) -ne 0 ] && SUDO=sudo +lsmod | grep -q configs || $SUDO modprobe configs || true + +if [ -z "$CONFIG" ]; then + for tryConfig in ${possibleConfigs}; do + if [ -e "$tryConfig" ]; then + CONFIG="$tryConfig" + break + fi + done +fi +if [ ! -e "$CONFIG" ]; then + case "$CONFIG" in + -*) + warning "error: argument $CONFIG" + ;; + *) + warning "error: cannot find kernel config $CONFIG" + ;; + esac + warning " try running this script again, specifying the kernel config:" + warning " set CONFIG=/path/to/kernel/.config or add argument /path/to/kernel/.config" + exit 1 +fi + +wrap_color "info: reading kernel config from $CONFIG ..." cyan +zcat $CONFIG >/dev/null 2>&1 || configFormat= + +echo + +echo 'Generally Necessary:' + +echo -n '- ' +cgroupSubsystemDir="$(awk '/[, ](cpu|cpuacct|cpuset|devices|freezer|memory)[, ]/ && $3 == "cgroup" { print $2 }' /proc/mounts | head -n1)" +cgroupDir="$(dirname "$cgroupSubsystemDir")" +if [ -d "$cgroupDir/cpu" ] || [ -d "$cgroupDir/cpuacct" ] || [ -d "$cgroupDir/cpuset" ] || [ -d "$cgroupDir/devices" ] || [ -d "$cgroupDir/freezer" ] || [ -d "$cgroupDir/memory" ]; then + echo "$(wrap_good 'cgroup hierarchy' 'properly mounted') [$cgroupDir]" +else + if [ "$cgroupSubsystemDir" ]; then + echo "$(wrap_bad 'cgroup hierarchy' 'single mountpoint!') [$cgroupSubsystemDir]" + else + wrap_bad 'cgroup hierarchy' 'nonexistent??' + fi + EXITCODE=1 + echo " $(wrap_color '(see https://github.com/tianon/cgroupfs-mount)' yellow)" +fi + +if [ "$(cat /sys/module/apparmor/parameters/enabled 2>/dev/null)" = 'Y' ]; then + echo -n '- ' + if command -v apparmor_parser &> /dev/null; then + wrap_good 'apparmor' 'enabled and tools installed' + else + wrap_bad 'apparmor' 'enabled, but apparmor_parser missing' + echo -n ' ' + if command -v apt-get &> /dev/null; then + wrap_color '(use "apt-get install apparmor" to fix this)' + elif command -v yum &> /dev/null; then + wrap_color '(your best bet is "yum install apparmor-parser")' + else + wrap_color '(look for an "apparmor" package for your distribution)' + fi + EXITCODE=1 + fi +fi + +flags=" + NAMESPACES NET_NS PID_NS IPC_NS UTS_NS + CGROUPS CGROUP_CPUACCT CGROUP_DEVICE CGROUP_FREEZER CGROUP_SCHED CPUSETS MEMCG + KEYS + VETH BRIDGE BRIDGE_NETFILTER + NF_NAT_IPV4 IP_NF_FILTER IP_NF_TARGET_MASQUERADE + NETFILTER_XT_MATCH_ADDRTYPE NETFILTER_XT_MATCH_CONNTRACK NETFILTER_XT_MATCH_IPVS + IP_NF_NAT NF_NAT NF_NAT_NEEDED + POSIX_MQUEUE +" +IS_ERROR=1 check_flags $flags +if [ "$kernelMajor" -lt 4 ] || ( [ "$kernelMajor" -eq 4 ] && [ "$kernelMinor" -lt 8 ] ); then + check_flags DEVPTS_MULTIPLE_INSTANCES +fi + +echo + +echo 'Optional Features:' +{ + check_flags USER_NS + check_distro_userns +} +{ + check_flags SECCOMP +} +{ + check_flags CGROUP_PIDS +} +# { +# CODE=${EXITCODE} +# check_flags MEMCG_SWAP MEMCG_SWAP_ENABLED +# if [ -e /sys/fs/cgroup/memory/memory.memsw.limit_in_bytes ]; then +# echo " $(wrap_color '(cgroup swap accounting is currently enabled)' bold black)" +# EXITCODE=${CODE} +# elif is_set MEMCG_SWAP && ! is_set MEMCG_SWAP_ENABLED; then +# echo " $(wrap_color '(cgroup swap accounting is currently not enabled, you can enable it by setting boot option "swapaccount=1")' bold black)" +# fi +# } +# { +# if is_set LEGACY_VSYSCALL_NATIVE; then +# echo -n "- "; wrap_bad "CONFIG_LEGACY_VSYSCALL_NATIVE" 'enabled' +# echo " $(wrap_color '(dangerous, provides an ASLR-bypassing target with usable ROP gadgets.)' bold black)" +# elif is_set LEGACY_VSYSCALL_EMULATE; then +# echo -n "- "; wrap_good "CONFIG_LEGACY_VSYSCALL_EMULATE" 'enabled' +# elif is_set LEGACY_VSYSCALL_NONE; then +# echo -n "- "; wrap_bad "CONFIG_LEGACY_VSYSCALL_NONE" 'enabled' +# echo " $(wrap_color '(containers using eglibc <= 2.13 will not work. Switch to' bold black)" +# echo " $(wrap_color ' "CONFIG_VSYSCALL_[NATIVE|EMULATE]" or use "vsyscall=[native|emulate]"' bold black)" +# echo " $(wrap_color ' on kernel command line. Note that this will disable ASLR for the,' bold black)" +# echo " $(wrap_color ' VDSO which may assist in exploiting security vulnerabilities.)' bold black)" +# # else Older kernels (prior to 3dc33bd30f3e, released in v4.40-rc1) do +# # not have these LEGACY_VSYSCALL options and are effectively +# # LEGACY_VSYSCALL_EMULATE. Even older kernels are presumably +# # effectively LEGACY_VSYSCALL_NATIVE. +# fi +# } + +if [ "$kernelMajor" -lt 4 ] || ( [ "$kernelMajor" -eq 4 ] && [ "$kernelMinor" -le 5 ] ); then + check_flags MEMCG_KMEM +fi + +if [ "$kernelMajor" -lt 3 ] || ( [ "$kernelMajor" -eq 3 ] && [ "$kernelMinor" -le 18 ] ); then + check_flags RESOURCE_COUNTERS +fi + +if [ "$kernelMajor" -lt 3 ] || ( [ "$kernelMajor" -eq 3 ] && [ "$kernelMinor" -le 13 ] ); then + netprio=NETPRIO_CGROUP +else + netprio=CGROUP_NET_PRIO +fi + +# IOSCHED_CFQ CFQ_GROUP_IOSCHED +flags=" + BLK_CGROUP BLK_DEV_THROTTLING + CGROUP_PERF + CGROUP_HUGETLB + NET_CLS_CGROUP $netprio + CFS_BANDWIDTH FAIR_GROUP_SCHED RT_GROUP_SCHED + IP_NF_TARGET_REDIRECT + IP_VS + IP_VS_NFCT + IP_VS_PROTO_TCP + IP_VS_PROTO_UDP + IP_VS_RR +" +check_flags $flags + +# if ! is_set EXT4_USE_FOR_EXT2; then +# check_flags EXT3_FS EXT3_FS_XATTR EXT3_FS_POSIX_ACL EXT3_FS_SECURITY +# if ! is_set EXT3_FS || ! is_set EXT3_FS_XATTR || ! is_set EXT3_FS_POSIX_ACL || ! is_set EXT3_FS_SECURITY; then +# echo " $(wrap_color '(enable these ext3 configs if you are using ext3 as backing filesystem)' bold black)" +# fi +# fi + +check_flags EXT4_FS EXT4_FS_POSIX_ACL EXT4_FS_SECURITY +if ! is_set EXT4_FS || ! is_set EXT4_FS_POSIX_ACL || ! is_set EXT4_FS_SECURITY; then + if is_set EXT4_USE_FOR_EXT2; then + echo " $(wrap_color 'enable these ext4 configs if you are using ext3 or ext4 as backing filesystem' bold black)" + else + echo " $(wrap_color 'enable these ext4 configs if you are using ext4 as backing filesystem' bold black)" + fi +fi + +echo '- Network Drivers:' +echo " - \"$(wrap_color 'overlay' blue)\":" +check_flags VXLAN | sed 's/^/ /' # BRIDGE_VLAN_FILTERING +echo ' Optional (for encrypted networks):' +check_flags CRYPTO CRYPTO_AEAD CRYPTO_GCM CRYPTO_SEQIV CRYPTO_GHASH \ + XFRM XFRM_USER XFRM_ALGO INET_ESP INET_XFRM_MODE_TRANSPORT | sed 's/^/ /' +# echo " - \"$(wrap_color 'ipvlan' blue)\":" +# check_flags IPVLAN | sed 's/^/ /' +# echo " - \"$(wrap_color 'macvlan' blue)\":" +# check_flags MACVLAN DUMMY | sed 's/^/ /' +# echo " - \"$(wrap_color 'ftp,tftp client in container' blue)\":" +# check_flags NF_NAT_FTP NF_CONNTRACK_FTP NF_NAT_TFTP NF_CONNTRACK_TFTP | sed 's/^/ /' + +# only fail if no storage drivers available +CODE=${EXITCODE} +EXITCODE=0 +STORAGE=1 + +echo '- Storage Drivers:' +# echo " - \"$(wrap_color 'aufs' blue)\":" +# check_flags AUFS_FS | sed 's/^/ /' +# if ! is_set AUFS_FS && grep -q aufs /proc/filesystems; then +# echo " $(wrap_color '(note that some kernels include AUFS patches but not the AUFS_FS flag)' bold black)" +# fi +# [ "$EXITCODE" = 0 ] && STORAGE=0 +# EXITCODE=0 + +# echo " - \"$(wrap_color 'btrfs' blue)\":" +# check_flags BTRFS_FS | sed 's/^/ /' +# check_flags BTRFS_FS_POSIX_ACL | sed 's/^/ /' +# [ "$EXITCODE" = 0 ] && STORAGE=0 +# EXITCODE=0 + +# echo " - \"$(wrap_color 'devicemapper' blue)\":" +# check_flags BLK_DEV_DM DM_THIN_PROVISIONING | sed 's/^/ /' +# [ "$EXITCODE" = 0 ] && STORAGE=0 +# EXITCODE=0 + +echo " - \"$(wrap_color 'overlay' blue)\":" +check_flags OVERLAY_FS | sed 's/^/ /' +[ "$EXITCODE" = 0 ] && STORAGE=0 +EXITCODE=0 + +# echo " - \"$(wrap_color 'zfs' blue)\":" +# echo -n " - "; check_device /dev/zfs +# echo -n " - "; check_command zfs +# echo -n " - "; check_command zpool +# [ "$EXITCODE" = 0 ] && STORAGE=0 +# EXITCODE=0 + +EXITCODE=$CODE +[ "$STORAGE" = 1 ] && EXITCODE=1 + +# --- + +echo +if [ $EXITCODE -eq 0 ]; then + wrap_good 'STATUS' 'pass' +else + wrap_bad 'STATUS' 'fail' +fi + +exit $EXITCODE diff --git a/pkg/cli/cmds/check-config.go b/pkg/cli/cmds/check-config.go new file mode 100644 index 0000000000..ead97e47a9 --- /dev/null +++ b/pkg/cli/cmds/check-config.go @@ -0,0 +1,15 @@ +package cmds + +import ( + "github.com/urfave/cli" +) + +func NewCheckConfigCommand(action func(*cli.Context) error) cli.Command { + return cli.Command{ + Name: "check-config", + Usage: "Run config check", + SkipFlagParsing: true, + SkipArgReorder: true, + Action: action, + } +} diff --git a/scripts/package-cli b/scripts/package-cli index 8d473e82e4..0aac10bb24 100755 --- a/scripts/package-cli +++ b/scripts/package-cli @@ -17,11 +17,24 @@ for i in bridge flannel host-local loopback portmap; do ln -s cni ./bin/$i done +cp contrib/util/check-config.sh bin/check-config rm -rf build/data mkdir -p build/data build/out mkdir -p dist/artifacts +( + set +x + cd bin + find . -not -path '*/\.*' -type f -exec sha256sum {} \; | sed -e 's| \./| |' | sort -k2 >.sha256sums + ( + for f in $(find . -type l); do + echo $f $(readlink $f) + done + ) | sed -e 's|^\./||' | sort >.links + set -x +) + tar cvzf ./build/out/data.tar.gz --exclude ./bin/hyperkube ./bin ./etc HASH=$(sha256sum ./build/out/data.tar.gz | awk '{print $1}') diff --git a/scripts/sonobuoy b/scripts/sonobuoy index 8d7285ad00..182739f973 100755 --- a/scripts/sonobuoy +++ b/scripts/sonobuoy @@ -55,6 +55,7 @@ docker run -d --name ${K3S_SERVER} --privileged \ K3S_IP=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' ${K3S_SERVER}) echo "Started ${K3S_SERVER} @ ${K3S_IP}:${K3S_PORT}" +docker exec ${K3S_SERVER} check-config || true timeout --foreground 1m bash -c wait-for-kubeconfig verify-valid-versions ${K3S_SERVER}