diff --git a/tests/e2e/amd64_resource_files/hardened_psp.yaml b/tests/e2e/amd64_resource_files/hardened_psp.yaml new file mode 100644 index 0000000000..2da8a613bb --- /dev/null +++ b/tests/e2e/amd64_resource_files/hardened_psp.yaml @@ -0,0 +1,217 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: default-network-dns-policy + namespace: kube-system +spec: + ingress: + - ports: + - port: 53 + protocol: TCP + - port: 53 + protocol: UDP + podSelector: + matchLabels: + k8s-app: kube-dns + policyTypes: + - Ingress +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: cis1.5-compliant-psp +spec: + privileged: false + allowPrivilegeEscalation: false + requiredDropCapabilities: + - ALL + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: psp:restricted + labels: + addonmanager.kubernetes.io/mode: EnsureExists +rules: +- apiGroups: ['extensions'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - cis1.5-compliant-psp +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: default:restricted + labels: + addonmanager.kubernetes.io/mode: EnsureExists +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:restricted +subjects: +- kind: Group + name: system:authenticated + apiGroup: rbac.authorization.k8s.io +--- +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: intra-namespace + namespace: kube-system +spec: + podSelector: {} + ingress: + - from: + - namespaceSelector: + matchLabels: + name: kube-system +--- +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: intra-namespace + namespace: default +spec: + podSelector: {} + ingress: + - from: + - namespaceSelector: + matchLabels: + name: default +--- +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: intra-namespace + namespace: kube-public +spec: + podSelector: {} + ingress: + - from: + - namespaceSelector: + matchLabels: + name: kube-public +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: system-unrestricted-psp +spec: + allowPrivilegeEscalation: true + allowedCapabilities: + - '*' + fsGroup: + rule: RunAsAny + hostIPC: true + hostNetwork: true + hostPID: true + hostPorts: + - max: 65535 + min: 0 + privileged: true + runAsUser: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + volumes: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system-unrestricted-node-psp-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system-unrestricted-psp-role +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:nodes +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system-unrestricted-psp-role +rules: +- apiGroups: + - policy + resourceNames: + - system-unrestricted-psp + resources: + - podsecuritypolicies + verbs: + - use +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: system-unrestricted-svc-acct-psp-rolebinding + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system-unrestricted-psp-role +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:serviceaccounts +--- +# Reference https://rancher.com/docs/k3s/latest/en/security/hardening_guide/#networkpolicies +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-all-svclbtraefik-ingress + namespace: kube-system +spec: + podSelector: + matchLabels: + app: svclb-traefik + ingress: + - {} + policyTypes: + - Ingress +--- +# Reference https://rancher.com/docs/k3s/latest/en/security/hardening_guide/#networkpolicies +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-all-metrics-server + namespace: kube-system +spec: + podSelector: + matchLabels: + k8s-app: metrics-server + ingress: + - {} + policyTypes: + - Ingress +--- diff --git a/tests/e2e/scripts/harden.sh b/tests/e2e/scripts/harden.sh new file mode 100644 index 0000000000..3b65139ab2 --- /dev/null +++ b/tests/e2e/scripts/harden.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +echo "vm.panic_on_oom=0 +vm.overcommit_memory=1 +kernel.panic=10 +kernel.panic_on_oops=1 +kernel.keys.root_maxbytes=25000000 +" >> /etc/sysctl.d/90-kubelet.conf +sysctl -p /etc/sysctl.d/90-kubelet.conf \ No newline at end of file diff --git a/tests/e2e/splitserver/Vagrantfile b/tests/e2e/splitserver/Vagrantfile index 24f6a03285..7fe76db28a 100644 --- a/tests/e2e/splitserver/Vagrantfile +++ b/tests/e2e/splitserver/Vagrantfile @@ -40,7 +40,7 @@ def provision(vm, role, role_num, node_num) k3s.args = "server" k3s.config = <<~YAML cluster-init: true - note-external-ip: #{NETWORK_PREFIX}.100 + node-external-ip: #{NETWORK_PREFIX}.100 flannel-iface: eth1 disable-apiserver: true disable-controller-manager: true diff --git a/tests/e2e/vagrantdefaults.rb b/tests/e2e/vagrantdefaults.rb index 895fbd7852..2dda456015 100644 --- a/tests/e2e/vagrantdefaults.rb +++ b/tests/e2e/vagrantdefaults.rb @@ -1,13 +1,11 @@ def defaultOSConfigure(vm) - if vm.box.include?("ubuntu2004") - vm.provision "shell", inline: "systemd-resolve --set-dns=8.8.8.8 --interface=eth0" - vm.provision "shell", inline: "apt install -y jq" - end - if vm.box.include?("Leap") - vm.provision "shell", inline: "zypper install -y jq" - end - if vm.box.include?("microos") - vm.provision "shell", inline: "transactional-update pkg install -y jq" + if vm.box.include?("generic/ubuntu") + vm.provision "Set DNS", type: "shell", inline: "systemd-resolve --set-dns=8.8.8.8 --interface=eth0" + vm.provision "Install jq", type: "shell", inline: "apt install -y jq" + elsif vm.box.include?("Leap") + vm.provision "Install jq", type: "shell", inline: "zypper install -y jq" + elsif vm.box.include?("microos") + vm.provision "Install jq", type: "shell", inline: "transactional-update pkg install -y jq" vm.provision 'reload', run: 'once' end end \ No newline at end of file diff --git a/tests/e2e/validatecluster/Vagrantfile b/tests/e2e/validatecluster/Vagrantfile index d39a1aaa0e..3623550e96 100644 --- a/tests/e2e/validatecluster/Vagrantfile +++ b/tests/e2e/validatecluster/Vagrantfile @@ -5,19 +5,21 @@ NODE_BOXES = (ENV['E2E_NODE_BOXES'] || ['generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004']) GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master") RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "") -EXTERNAL_DB = (ENV['E2E_EXTERNAL_DB'] || "mysql") +EXTERNAL_DB = (ENV['E2E_EXTERNAL_DB'] || "etcd") +HARDENED = (ENV['E2E_HARDENED'] || "") NODE_CPUS = (ENV['E2E_NODE_CPUS'] || 2).to_i NODE_MEMORY = (ENV['E2E_NODE_MEMORY'] || 1024).to_i # Virtualbox >= 6.1.28 require `/etc/vbox/network.conf` for expanded private networks NETWORK_PREFIX = "10.10.10" install_type = "" -db_type = "" +hardened_arg = "" def provision(vm, role, role_num, node_num) vm.box = NODE_BOXES[node_num] vm.hostname = role # An expanded netmask is required to allow VM<-->VM communication, virtualbox defaults to /32 - vm.network "private_network", ip: "#{NETWORK_PREFIX}.#{100+node_num}", netmask: "255.255.255.0" + node_ip = "#{NETWORK_PREFIX}.#{100+node_num}" + vm.network "private_network", ip: node_ip, netmask: "24" vagrant_defaults = '../vagrantdefaults.rb' load vagrant_defaults if File.exists?(vagrant_defaults) @@ -29,51 +31,60 @@ def provision(vm, role, role_num, node_num) else # Grabs the last 5 commit SHA's from the given branch, then purges any commits that do not have a passing CI build # MicroOS requires it not be in a /tmp/ or other root system folder - vm.provision "shell", path: "../scripts/latest_commit.sh", args: [GITHUB_BRANCH, "/home/vagrant/k3s_commits"] + vm.provision "Get latest commit", type: "shell", path: "../scripts/latest_commit.sh", args: [GITHUB_BRANCH, "/home/vagrant/k3s_commits"] install_type = "INSTALL_K3S_COMMIT=$(head\ -n\ 1\ /home/vagrant/k3s_commits)" end vm.provision "shell", inline: "ping -c 2 k3s.io" + db_type = getDBType(role, role_num, vm) + + if !HARDENED.empty? + vm.provision "Set kernel parameters", type: "shell", path: "../scripts/harden.sh" + hardened_arg = "protect-kernel-defaults: true\nkube-apiserver-arg: \"enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount\"" + end + if role.include?("server") && role_num == 0 - if EXTERNAL_DB == "mysql" - dockerInstall(vm) - vm.provision "shell", inline: "docker run -d -p 3306:3306 --name #{EXTERNAL_DB} -e MYSQL_ROOT_PASSWORD=e2e mysql:5.7" - vm.provision "shell", inline: "echo \"Wait for mysql to startup\"; sleep 10" - db_type = "--datastore-endpoint='mysql://root:e2e@tcp(#{NETWORK_PREFIX}.100:3306)/k3s'" - elsif EXTERNAL_DB == "postgres" - dockerInstall(vm) - vm.provision "shell", inline: "docker run -d -p 5432:5432 --name #{EXTERNAL_DB} -e POSTGRES_PASSWORD=e2e postgres:14-alpine" - vm.provision "shell", inline: "echo \"Wait for postgres to startup\"; sleep 10" - db_type = "--datastore-endpoint='postgres://postgres:e2e@#{NETWORK_PREFIX}.100:5432/k3s?sslmode=disable'" - elsif EXTERNAL_DB == "" || EXTERNAL_DB == "etcd" - db_type = "--cluster-init" - else - puts "Unknown EXTERNAL_DB: " + EXTERNAL_DB - abort - end - - vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s| - k3s.args = "server #{db_type} --node-external-ip=#{NETWORK_PREFIX}.100 --flannel-iface=eth1" - k3s.env = %W[K3S_KUBECONFIG_MODE=0644 K3S_TOKEN=vagrant #{install_type}] + vm.provision 'k3s-primary-server', type: 'k3s', run: 'once' do |k3s| + k3s.args = "server " + k3s.config = <<~YAML + token: vagrant + node-external-ip: #{NETWORK_PREFIX}.100 + flannel-iface: eth1 + #{db_type} + #{hardened_arg} + YAML + k3s.env = %W[K3S_KUBECONFIG_MODE=0644 #{install_type}] k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 end + elsif role.include?("server") && role_num != 0 - if EXTERNAL_DB == "mysql" - db_type = "--datastore-endpoint='mysql://root:e2e@tcp(#{NETWORK_PREFIX}.100:3306)/k3s'" - elsif EXTERNAL_DB == "postgres" - db_type = "--datastore-endpoint='postgres://postgres:e2e@#{NETWORK_PREFIX}.100:5432/k3s?sslmode=disable'" - end - - vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s| - k3s.args = "server #{db_type} --server https://#{NETWORK_PREFIX}.100:6443 --flannel-iface=eth1" + vm.provision 'k3s-secondary-server', type: 'k3s', run: 'once' do |k3s| + k3s.args = "server" + k3s.config = <<~YAML + server: "https://#{NETWORK_PREFIX}.100:6443" + token: vagrant + node-external-ip: #{node_ip} + flannel-iface: eth1 + #{db_type} + #{hardened_arg} + YAML k3s.env = %W[K3S_KUBECONFIG_MODE=0644 K3S_TOKEN=vagrant #{install_type}] k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 end end + if role.include?("agent") - vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s| - k3s.args = %W[agent #{db_type} --server https://#{NETWORK_PREFIX}.100:6443 --flannel-iface=eth1] - k3s.env = %W[K3S_KUBECONFIG_MODE=0644 K3S_TOKEN=vagrant #{install_type}] + vm.provision 'k3s-agent', type: 'k3s', run: 'once' do |k3s| + k3s.args = "agent" + k3s.config = <<~YAML + server: "https://#{NETWORK_PREFIX}.100:6443" + token: vagrant + node-external-ip: #{node_ip} + flannel-iface: eth1 + #{db_type} + #{hardened_arg} + YAML + k3s.env = %W[K3S_KUBECONFIG_MODE=0644 #{install_type}] k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 end end @@ -85,6 +96,39 @@ def provision(vm, role, role_num, node_num) end end +def getDBType(role, role_num, vm) + + if EXTERNAL_DB == "mysql" + if role.include?("server") && role_num == 0 + dockerInstall(vm) + vm.provision "Start mysql", inline: "docker run -d -p 3306:3306 --name #{EXTERNAL_DB} -e MYSQL_ROOT_PASSWORD=e2e mysql:5.7" + vm.provision "shell", inline: "echo \"Wait for mysql to startup\"; sleep 10" + return "datastore-endpoint: 'mysql://root:e2e@tcp(#{NETWORK_PREFIX}.100:3306)/k3s'" + elsif role.include?("server") && role_num != 0 + return "datastore-endpoint: 'mysql://root:e2e@tcp(#{NETWORK_PREFIX}.100:3306)/k3s'" + end + + elsif EXTERNAL_DB == "postgres" + if role.include?("server") && role_num == 0 + dockerInstall(vm) + vm.provision "Start postgres", inline: "docker run -d -p 5432:5432 --name #{EXTERNAL_DB} -e POSTGRES_PASSWORD=e2e postgres:14-alpine" + vm.provision "shell", inline: "echo \"Wait for postgres to startup\"; sleep 10" + return "datastore-endpoint: 'postgres://postgres:e2e@#{NETWORK_PREFIX}.100:5432/k3s?sslmode=disable'" + elsif role.include?("server") && role_num != 0 + return "datastore-endpoint: 'postgres://postgres:e2e@#{NETWORK_PREFIX}.100:5432/k3s?sslmode=disable'" + end + + elsif ( EXTERNAL_DB == "" || EXTERNAL_DB == "etcd" ) + if role.include?("server") && role_num == 0 + return "cluster-init: true" + end + else + puts "Unknown EXTERNAL_DB: " + EXTERNAL_DB + abort + end + return "" +end + def dockerInstall(vm) vm.provider "libvirt" do |v| v.memory = NODE_MEMORY + 1024 @@ -92,7 +136,7 @@ def dockerInstall(vm) vm.provider "virtualbox" do |v| v.memory = NODE_MEMORY + 1024 end - if vm.box.include?("ubuntu2004") + if vm.box.include?("ubuntu") vm.provision "shell", inline: "apt install -y docker.io" end if vm.box.include?("Leap") @@ -105,6 +149,7 @@ def dockerInstall(vm) end end + Vagrant.configure("2") do |config| config.vagrant.plugins = ["vagrant-k3s", "vagrant-reload"] # Default provider is libvirt, virtualbox is only provided as a backup