diff --git a/cluster/common.sh b/cluster/common.sh index 6379219cb7..04050449aa 100755 --- a/cluster/common.sh +++ b/cluster/common.sh @@ -321,6 +321,9 @@ function find-tar() { # KUBE_MANIFESTS_TAR function find-release-tars() { SERVER_BINARY_TAR=$(find-tar kubernetes-server-linux-amd64.tar.gz) + if [[ "${NUM_WINDOWS_NODES}" -gt "0" && "${USE_RELEASE_NODE_BINARIES:-false}" == "false" ]]; then + NODE_BINARY_TAR=$(find-tar kubernetes-node-windows-amd64.tar.gz) + fi # This tarball is used by GCI, Ubuntu Trusty, and Container Linux. KUBE_MANIFESTS_TAR= diff --git a/cluster/gce/config-common.sh b/cluster/gce/config-common.sh index a6b18002c6..910983d197 100644 --- a/cluster/gce/config-common.sh +++ b/cluster/gce/config-common.sh @@ -14,26 +14,36 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Returns the total number of Linux and Windows nodes in the cluster. +# # Vars assumed: # NUM_NODES +# NUM_WINDOWS_NODES +function get-num-nodes { + echo "$((${NUM_NODES} + ${NUM_WINDOWS_NODES}))" +} + +# Vars assumed: +# NUM_NODES +# NUM_WINDOWS_NODES function get-master-size { local suggested_master_size=1 - if [[ "${NUM_NODES}" -gt "5" ]]; then + if [[ "$(get-num-nodes)" -gt "5" ]]; then suggested_master_size=2 fi - if [[ "${NUM_NODES}" -gt "10" ]]; then + if [[ "$(get-num-nodes)" -gt "10" ]]; then suggested_master_size=4 fi - if [[ "${NUM_NODES}" -gt "100" ]]; then + if [[ "$(get-num-nodes)" -gt "100" ]]; then suggested_master_size=8 fi - if [[ "${NUM_NODES}" -gt "250" ]]; then + if [[ "$(get-num-nodes)" -gt "250" ]]; then suggested_master_size=16 fi - if [[ "${NUM_NODES}" -gt "500" ]]; then + if [[ "$(get-num-nodes)" -gt "500" ]]; then suggested_master_size=32 fi - if [[ "${NUM_NODES}" -gt "3000" ]]; then + if [[ "$(get-num-nodes)" -gt "3000" ]]; then suggested_master_size=64 fi echo "${suggested_master_size}" @@ -41,12 +51,13 @@ function get-master-size { # Vars assumed: # NUM_NODES +# NUM_WINDOWS_NODES function get-master-root-disk-size() { local suggested_master_root_disk_size="20GB" - if [[ "${NUM_NODES}" -gt "500" ]]; then + if [[ "$(get-num-nodes)" -gt "500" ]]; then suggested_master_root_disk_size="100GB" fi - if [[ "${NUM_NODES}" -gt "3000" ]]; then + if [[ "$(get-num-nodes)" -gt "3000" ]]; then suggested_master_root_disk_size="500GB" fi echo "${suggested_master_root_disk_size}" @@ -54,12 +65,13 @@ function get-master-root-disk-size() { # Vars assumed: # NUM_NODES +# NUM_WINDOWS_NODES function get-master-disk-size() { local suggested_master_disk_size="20GB" - if [[ "${NUM_NODES}" -gt "500" ]]; then + if [[ "$(get-num-nodes)" -gt "500" ]]; then suggested_master_disk_size="100GB" fi - if [[ "${NUM_NODES}" -gt "3000" ]]; then + if [[ "$(get-num-nodes)" -gt "3000" ]]; then suggested_master_disk_size="200GB" fi echo "${suggested_master_disk_size}" @@ -72,13 +84,13 @@ function get-node-ip-range { return fi local suggested_range="10.40.0.0/22" - if [[ "${NUM_NODES}" -gt 1000 ]]; then + if [[ "$(get-num-nodes)" -gt 1000 ]]; then suggested_range="10.40.0.0/21" fi - if [[ "${NUM_NODES}" -gt 2000 ]]; then + if [[ "$(get-num-nodes)" -gt 2000 ]]; then suggested_range="10.40.0.0/20" fi - if [[ "${NUM_NODES}" -gt 4000 ]]; then + if [[ "$(get-num-nodes)" -gt 4000 ]]; then suggested_range="10.40.0.0/19" fi echo "${suggested_range}" @@ -86,13 +98,13 @@ function get-node-ip-range { function get-cluster-ip-range { local suggested_range="10.64.0.0/14" - if [[ "${NUM_NODES}" -gt 1000 ]]; then + if [[ "$(get-num-nodes)" -gt 1000 ]]; then suggested_range="10.64.0.0/13" fi - if [[ "${NUM_NODES}" -gt 2000 ]]; then + if [[ "$(get-num-nodes)" -gt 2000 ]]; then suggested_range="10.64.0.0/12" fi - if [[ "${NUM_NODES}" -gt 4000 ]]; then + if [[ "$(get-num-nodes)" -gt 4000 ]]; then suggested_range="10.64.0.0/11" fi echo "${suggested_range}" @@ -114,3 +126,26 @@ function get-alias-range-size() { # NOTE: Avoid giving nodes empty scopes, because kubelet needs a service account # in order to initialize properly. NODE_SCOPES="${NODE_SCOPES:-monitoring,logging-write,storage-ro}" + +# Root directory for Kubernetes files on Windows nodes. +WINDOWS_K8S_DIR="C:\etc\kubernetes" +# Directory where Kubernetes binaries will be installed on Windows nodes. +WINDOWS_NODE_DIR="${WINDOWS_K8S_DIR}\node\bin" +# Directory where Kubernetes log files will be stored on Windows nodes. +WINDOWS_LOGS_DIR="${WINDOWS_K8S_DIR}\logs" +# Directory where CNI binaries will be stored on Windows nodes. +WINDOWS_CNI_DIR="${WINDOWS_K8S_DIR}\cni" +# Directory where CNI config files will be stored on Windows nodes. +WINDOWS_CNI_CONFIG_DIR="${WINDOWS_K8S_DIR}\cni\config" +# Pod manifests directory for Windows nodes on Windows nodes. +WINDOWS_MANIFESTS_DIR="${WINDOWS_K8S_DIR}\manifests" +# Directory where cert/key files will be stores on Windows nodes. +WINDOWS_PKI_DIR="${WINDOWS_K8S_DIR}\pki" +# Path for kubelet config file on Windows nodes. +WINDOWS_KUBELET_CONFIG_FILE="${WINDOWS_K8S_DIR}\kubelet-config.yaml" +# Path for kubeconfig file on Windows nodes. +WINDOWS_KUBECONFIG_FILE="${WINDOWS_K8S_DIR}\kubelet.kubeconfig" +# Path for bootstrap kubeconfig file on Windows nodes. +WINDOWS_BOOTSTRAP_KUBECONFIG_FILE="${WINDOWS_K8S_DIR}\kubelet.bootstrap-kubeconfig" +# Path for kube-proxy kubeconfig file on Windows nodes. +WINDOWS_KUBEPROXY_KUBECONFIG_FILE="${WINDOWS_K8S_DIR}\kubeproxy.kubeconfig" diff --git a/cluster/gce/config-default.sh b/cluster/gce/config-default.sh index 37e81cd53e..4a3f171d08 100755 --- a/cluster/gce/config-default.sh +++ b/cluster/gce/config-default.sh @@ -29,6 +29,7 @@ RELEASE_REGION_FALLBACK=${RELEASE_REGION_FALLBACK:-false} REGIONAL_KUBE_ADDONS=${REGIONAL_KUBE_ADDONS:-true} NODE_SIZE=${NODE_SIZE:-n1-standard-2} NUM_NODES=${NUM_NODES:-3} +NUM_WINDOWS_NODES=${NUM_WINDOWS_NODES:-0} MASTER_SIZE=${MASTER_SIZE:-n1-standard-$(get-master-size)} MASTER_MIN_CPU_ARCHITECTURE=${MASTER_MIN_CPU_ARCHITECTURE:-} # To allow choosing better architectures. MASTER_DISK_TYPE=pd-ssd @@ -44,6 +45,7 @@ NODE_LOCAL_SSDS=${NODE_LOCAL_SSDS:-0} # fluentd is not running as a manifest pod with appropriate label. # TODO(piosz): remove this in 1.8 NODE_LABELS="${KUBE_NODE_LABELS:-beta.kubernetes.io/fluentd-ds-ready=true}" +WINDOWS_NODE_LABELS="${WINDOWS_NODE_LABELS:-}" # An extension to local SSDs allowing users to specify block/fs and SCSI/NVMe devices # Format of this variable will be "#,scsi/nvme,block/fs" you can specify multiple @@ -63,6 +65,7 @@ MIG_WAIT_UNTIL_STABLE_TIMEOUT=${MIG_WAIT_UNTIL_STABLE_TIMEOUT:-1800} MASTER_OS_DISTRIBUTION=${KUBE_MASTER_OS_DISTRIBUTION:-${KUBE_OS_DISTRIBUTION:-gci}} NODE_OS_DISTRIBUTION=${KUBE_NODE_OS_DISTRIBUTION:-${KUBE_OS_DISTRIBUTION:-gci}} +WINDOWS_NODE_OS_DISTRIBUTION=${WINDOWS_NODE_OS_DISTRIBUTION:-win1803} if [[ "${MASTER_OS_DISTRIBUTION}" == "cos" ]]; then MASTER_OS_DISTRIBUTION="gci" @@ -173,15 +176,19 @@ HEAPSTER_MACHINE_TYPE="${HEAPSTER_MACHINE_TYPE:-}" # NON_MASTER_NODE_LABELS are labels will only be applied on non-master nodes. NON_MASTER_NODE_LABELS="${KUBE_NON_MASTER_NODE_LABELS:-}" +WINDOWS_NON_MASTER_NODE_LABELS="${WINDOWS_NON_MASTER_NODE_LABELS:-}" if [[ "${PREEMPTIBLE_MASTER}" == "true" ]]; then NODE_LABELS="${NODE_LABELS},cloud.google.com/gke-preemptible=true" + WINDOWS_NODE_LABELS="${WINDOWS_NODE_LABELS},cloud.google.com/gke-preemptible=true" elif [[ "${PREEMPTIBLE_NODE}" == "true" ]]; then NON_MASTER_NODE_LABELS="${NON_MASTER_NODE_LABELS},cloud.google.com/gke-preemptible=true" + WINDOWS_NON_MASTER_NODE_LABELS="${WINDOWS_NON_MASTER_NODE_LABELS},cloud.google.com/gke-preemptible=true" fi # To avoid running Calico on a node that is not configured appropriately, # label each Node so that the DaemonSet can run the Pods only on ready Nodes. +# Windows nodes do not support Calico. if [[ ${NETWORK_POLICY_PROVIDER:-} == "calico" ]]; then NON_MASTER_NODE_LABELS="${NON_MASTER_NODE_LABELS:+${NON_MASTER_NODE_LABELS},}projectcalico.org/ds-ready=true" fi @@ -194,6 +201,7 @@ CUSTOM_TYPHA_DEPLOYMENT_YAML="${KUBE_CUSTOM_TYPHA_DEPLOYMENT_YAML:-}" # To avoid running netd on a node that is not configured appropriately, # label each Node so that the DaemonSet can run the Pods only on ready Nodes. +# Windows nodes do not support netd. if [[ ${ENABLE_NETD:-} == "true" ]]; then NON_MASTER_NODE_LABELS="${NON_MASTER_NODE_LABELS:+${NON_MASTER_NODE_LABELS},}cloud.google.com/gke-netd-ready=true" fi @@ -467,3 +475,7 @@ ENABLE_NODE_TERMINATION_HANDLER="${ENABLE_NODE_TERMINATION_HANDLER:-false}" if [[ "${NODE_TERMINATION_HANDLER_IMAGE:-}" ]]; then PROVIDER_VARS="${PROVIDER_VARS:-} NODE_TERMINATION_HANDLER_IMAGE" fi + +# Taint Windows nodes by default to prevent Linux workloads from being +# scheduled onto them. +WINDOWS_NODE_TAINTS="${WINDOWS_NODE_TAINTS:-node.kubernetes.io/os=windows:NoSchedule}" diff --git a/cluster/gce/config-test.sh b/cluster/gce/config-test.sh index aca76802e7..0edb195e63 100755 --- a/cluster/gce/config-test.sh +++ b/cluster/gce/config-test.sh @@ -29,6 +29,7 @@ RELEASE_REGION_FALLBACK=${RELEASE_REGION_FALLBACK:-false} REGIONAL_KUBE_ADDONS=${REGIONAL_KUBE_ADDONS:-true} NODE_SIZE=${NODE_SIZE:-n1-standard-2} NUM_NODES=${NUM_NODES:-3} +NUM_WINDOWS_NODES=${NUM_WINDOWS_NODES:-0} MASTER_SIZE=${MASTER_SIZE:-n1-standard-$(get-master-size)} MASTER_MIN_CPU_ARCHITECTURE=${MASTER_MIN_CPU_ARCHITECTURE:-} # To allow choosing better architectures. MASTER_DISK_TYPE=pd-ssd @@ -44,6 +45,7 @@ NODE_LOCAL_SSDS=${NODE_LOCAL_SSDS:-0} # fluentd is not running as a manifest pod with appropriate label. # TODO(piosz): remove this in 1.8 NODE_LABELS="${KUBE_NODE_LABELS:-beta.kubernetes.io/fluentd-ds-ready=true}" +WINDOWS_NODE_LABELS="${WINDOWS_NODE_LABELS:-}" # An extension to local SSDs allowing users to specify block/fs and SCSI/NVMe devices # Format of this variable will be "#,scsi/nvme,block/fs" you can specify multiple @@ -66,6 +68,8 @@ MIG_WAIT_UNTIL_STABLE_TIMEOUT=${MIG_WAIT_UNTIL_STABLE_TIMEOUT:-1800} MASTER_OS_DISTRIBUTION=${KUBE_MASTER_OS_DISTRIBUTION:-${KUBE_OS_DISTRIBUTION:-gci}} NODE_OS_DISTRIBUTION=${KUBE_NODE_OS_DISTRIBUTION:-${KUBE_OS_DISTRIBUTION:-gci}} +WINDOWS_NODE_OS_DISTRIBUTION=${WINDOWS_NODE_OS_DISTRIBUTION:-win1803} + if [[ "${MASTER_OS_DISTRIBUTION}" == "cos" ]]; then MASTER_OS_DISTRIBUTION="gci" fi @@ -81,7 +85,7 @@ fi # To avoid failing large tests due to some flakes in starting nodes, allow # for a small percentage of nodes to not start during cluster startup. -ALLOWED_NOTREADY_NODES="${ALLOWED_NOTREADY_NODES:-$((NUM_NODES / 100))}" +ALLOWED_NOTREADY_NODES="${ALLOWED_NOTREADY_NODES:-$(($(get-num-nodes) / 100))}" # By default a cluster will be started with the master and nodes # on Container-optimized OS (cos, previously known as gci). If @@ -215,11 +219,14 @@ KUBEPROXY_TEST_ARGS="${KUBEPROXY_TEST_ARGS:-} ${TEST_CLUSTER_API_CONTENT_TYPE}" # NON_MASTER_NODE_LABELS are labels will only be applied on non-master nodes. NON_MASTER_NODE_LABELS="${KUBE_NON_MASTER_NODE_LABELS:-}" +WINDOWS_NON_MASTER_NODE_LABELS="${WINDOWS_NON_MASTER_NODE_LABELS:-}" if [[ "${PREEMPTIBLE_MASTER}" == "true" ]]; then NODE_LABELS="${NODE_LABELS},cloud.google.com/gke-preemptible=true" + WINDOWS_NODE_LABELS="${WINDOWS_NODE_LABELS},cloud.google.com/gke-preemptible=true" elif [[ "${PREEMPTIBLE_NODE}" == "true" ]]; then NON_MASTER_NODE_LABELS="${NON_MASTER_NODE_LABELS},cloud.google.com/gke-preemptible=true" + WINDOWS_NON_MASTER_NODE_LABELS="${WINDOWS_NON_MASTER_NODE_LABELS},cloud.google.com/gke-preemptible=true" fi # Optional: Enable netd. @@ -230,6 +237,7 @@ CUSTOM_TYPHA_DEPLOYMENT_YAML="${KUBE_CUSTOM_TYPHA_DEPLOYMENT_YAML:-}" # To avoid running netd on a node that is not configured appropriately, # label each Node so that the DaemonSet can run the Pods only on ready Nodes. +# Windows nodes do not support netd. if [[ ${ENABLE_NETD:-} == "true" ]]; then NON_MASTER_NODE_LABELS="${NON_MASTER_NODE_LABELS:+${NON_MASTER_NODE_LABELS},}cloud.google.com/gke-netd-ready=true" fi @@ -238,6 +246,7 @@ ENABLE_NODELOCAL_DNS="${KUBE_ENABLE_NODELOCAL_DNS:-false}" # To avoid running Calico on a node that is not configured appropriately, # label each Node so that the DaemonSet can run the Pods only on ready Nodes. +# Windows nodes do not support Calico. if [[ ${NETWORK_POLICY_PROVIDER:-} == "calico" ]]; then NON_MASTER_NODE_LABELS="${NON_MASTER_NODE_LABELS:+${NON_MASTER_NODE_LABELS},}projectcalico.org/ds-ready=true" fi @@ -486,3 +495,7 @@ ENABLE_NODE_TERMINATION_HANDLER="${ENABLE_NODE_TERMINATION_HANDLER:-false}" if [[ "${NODE_TERMINATION_HANDLER_IMAGE:-}" ]]; then PROVIDER_VARS="${PROVIDER_VARS:-} NODE_TERMINATION_HANDLER_IMAGE" fi + +# Taint Windows nodes by default to prevent Linux workloads from being +# scheduled onto them. +WINDOWS_NODE_TAINTS="${WINDOWS_NODE_TAINTS:-node.kubernetes.io/os=windows:NoSchedule}" diff --git a/cluster/gce/gci/node-helper.sh b/cluster/gce/gci/node-helper.sh index fd4ca7378a..88c675d67c 100755 --- a/cluster/gce/gci/node-helper.sh +++ b/cluster/gce/gci/node-helper.sh @@ -17,7 +17,7 @@ # A library of helper functions and constant for GCI distro source "${KUBE_ROOT}/cluster/gce/gci/helper.sh" -function get-node-instance-metadata { +function get-node-instance-metadata-from-file { local metadata="" metadata+="kube-env=${KUBE_TEMP}/node-kube-env.yaml," metadata+="kubelet-config=${KUBE_TEMP}/node-kubelet-config.yaml," @@ -34,8 +34,8 @@ function get-node-instance-metadata { } # $1: template name (required). -function create-node-instance-template { +function create-linux-node-instance-template { local template_name="$1" ensure-gci-metadata-files - create-node-template "$template_name" "${scope_flags[*]}" "$(get-node-instance-metadata)" + create-node-template "${template_name}" "${scope_flags[*]}" "$(get-node-instance-metadata-from-file)" "" "linux" } diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index de359618c7..265fbe059e 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -32,6 +32,8 @@ else exit 1 fi +source "${KUBE_ROOT}/cluster/gce/${WINDOWS_NODE_OS_DISTRIBUTION}/node-helper.sh" + if [[ "${MASTER_OS_DISTRIBUTION}" == "trusty" || "${MASTER_OS_DISTRIBUTION}" == "gci" || "${MASTER_OS_DISTRIBUTION}" == "ubuntu" ]]; then source "${KUBE_ROOT}/cluster/gce/${MASTER_OS_DISTRIBUTION}/master-helper.sh" else @@ -57,23 +59,50 @@ fi # Sets node image based on the specified os distro. Currently this function only # supports gci and debian. -function set-node-image() { - if [[ "${NODE_OS_DISTRIBUTION}" == "gci" ]]; then - DEFAULT_GCI_PROJECT=google-containers - if [[ "${GCI_VERSION}" == "cos"* ]]; then - DEFAULT_GCI_PROJECT=cos-cloud - fi - - # If the node image is not set, we use the latest GCI image. - # Otherwise, we respect whatever is set by the user. - NODE_IMAGE=${KUBE_GCE_NODE_IMAGE:-${GCI_VERSION}} - NODE_IMAGE_PROJECT=${KUBE_GCE_NODE_PROJECT:-${DEFAULT_GCI_PROJECT}} +# +# Requires: +# NODE_OS_DISTRIBUTION +# Sets: +# DEFAULT_GCI_PROJECT +# NODE_IMAGE +# NODE_IMAGE_PROJECT +function set-linux-node-image() { + if [[ "${NODE_OS_DISTRIBUTION}" == "gci" ]]; then + DEFAULT_GCI_PROJECT=google-containers + if [[ "${GCI_VERSION}" == "cos"* ]]; then + DEFAULT_GCI_PROJECT=cos-cloud fi + + # If the node image is not set, we use the latest GCI image. + # Otherwise, we respect whatever is set by the user. + NODE_IMAGE=${KUBE_GCE_NODE_IMAGE:-${GCI_VERSION}} + NODE_IMAGE_PROJECT=${KUBE_GCE_NODE_PROJECT:-${DEFAULT_GCI_PROJECT}} + fi } -set-node-image +# Requires: +# WINDOWS_NODE_OS_DISTRIBUTION +# Sets: +# WINDOWS_NODE_IMAGE_FAMILY +# WINDOWS_NODE_IMAGE_PROJECT +function set-windows-node-image() { + WINDOWS_NODE_IMAGE_PROJECT="windows-cloud" + if [[ "${WINDOWS_NODE_OS_DISTRIBUTION}" == "win1803" ]]; then + WINDOWS_NODE_IMAGE_FAMILY="windows-1803-core-for-containers" + elif [[ "${WINDOWS_NODE_OS_DISTRIBUTION}" == "win2019" ]]; then + WINDOWS_NODE_IMAGE_FAMILY="windows-2019-core-for-containers" + elif [[ "${WINDOWS_NODE_OS_DISTRIBUTION}" == "win1809" ]]; then + WINDOWS_NODE_IMAGE_FAMILY="windows-1809-core-for-containers" + else + echo "Unknown WINDOWS_NODE_OS_DISTRIBUTION ${WINDOWS_NODE_OS_DISTRIBUTION}" >&2 + exit 1 + fi +} -# Verfiy cluster autoscaler configuration. +set-linux-node-image +set-windows-node-image + +# Verify cluster autoscaler configuration. if [[ "${ENABLE_CLUSTER_AUTOSCALER}" == "true" ]]; then if [[ -z $AUTOSCALER_MIN_NODES ]]; then echo "AUTOSCALER_MIN_NODES not set." @@ -342,7 +371,7 @@ function upload-tars() { fi } -# Detect minions created in the minion group +# Detect Linux and Windows nodes created in the instance group. # # Assumed vars: # NODE_INSTANCE_PREFIX @@ -535,23 +564,29 @@ function write-master-env { KUBERNETES_MASTER_NAME="${MASTER_NAME}" fi - construct-kubelet-flags true - build-kube-env true "${KUBE_TEMP}/master-kube-env.yaml" - build-kubelet-config true "${KUBE_TEMP}/master-kubelet-config.yaml" + construct-linux-kubelet-flags true + build-linux-kube-env true "${KUBE_TEMP}/master-kube-env.yaml" + build-kubelet-config true "linux" "${KUBE_TEMP}/master-kubelet-config.yaml" build-kube-master-certs "${KUBE_TEMP}/kube-master-certs.yaml" } -function write-node-env { +function write-linux-node-env { if [[ -z "${KUBERNETES_MASTER_NAME:-}" ]]; then KUBERNETES_MASTER_NAME="${MASTER_NAME}" fi - construct-kubelet-flags false - build-kube-env false "${KUBE_TEMP}/node-kube-env.yaml" - build-kubelet-config false "${KUBE_TEMP}/node-kubelet-config.yaml" + construct-linux-kubelet-flags false + build-linux-kube-env false "${KUBE_TEMP}/node-kube-env.yaml" + build-kubelet-config false "linux" "${KUBE_TEMP}/node-kubelet-config.yaml" } -function build-node-labels { +function write-windows-node-env { + construct-windows-kubelet-flags + build-windows-kube-env "${KUBE_TEMP}/windows-node-kube-env.yaml" + build-kubelet-config false "windows" "${KUBE_TEMP}/windows-node-kubelet-config.yaml" +} + +function build-linux-node-labels { local master=$1 local node_labels="" if [[ "${KUBE_PROXY_DAEMONSET:-}" == "true" && "${master}" != "true" ]]; then @@ -568,6 +603,17 @@ function build-node-labels { echo $node_labels } +function build-windows-node-labels { + local node_labels="" + if [[ -n "${WINDOWS_NODE_LABELS:-}" ]]; then + node_labels="${node_labels:+${node_labels},}${WINDOWS_NODE_LABELS}" + fi + if [[ -n "${WINDOWS_NON_MASTER_NODE_LABELS:-}" ]]; then + node_labels="${node_labels:+${node_labels},}${WINDOWS_NON_MASTER_NODE_LABELS}" + fi + echo $node_labels +} + # yaml-map-string-stringarray converts the encoded structure to yaml format, and echoes the result # under the provided name. If the encoded structure is empty, echoes nothing. # 1: name to be output in yaml @@ -645,12 +691,26 @@ function yaml-map-string-string { fi } -# $1: if 'true', we're rendering flags for a master, else a node -function construct-kubelet-flags { - local master=$1 +# Returns kubelet flags used on both Linux and Windows nodes. +function construct-common-kubelet-flags { local flags="${KUBELET_TEST_LOG_LEVEL:-"--v=2"} ${KUBELET_TEST_ARGS:-}" - flags+=" --allow-privileged=true" flags+=" --cloud-provider=gce" + # TODO(mtaufen): ROTATE_CERTIFICATES seems unused; delete it? + if [[ -n "${ROTATE_CERTIFICATES:-}" ]]; then + flags+=" --rotate-certificates=true" + fi + if [[ -n "${MAX_PODS_PER_NODE:-}" ]]; then + flags+=" --max-pods=${MAX_PODS_PER_NODE}" + fi + echo $flags +} + +# Sets KUBELET_ARGS with the kubelet flags for Linux nodes. +# $1: if 'true', we're rendering flags for a master, else a node +function construct-linux-kubelet-flags { + local master="$1" + local flags="$(construct-common-kubelet-flags)" + flags+=" --allow-privileged=true" # Keep in sync with CONTAINERIZED_MOUNTER_HOME in configure-helper.sh flags+=" --experimental-mounter-path=/home/kubernetes/containerized_mounter/mounter" flags+=" --experimental-check-node-capabilities-before-mount=true" @@ -695,60 +755,175 @@ function construct-kubelet-flags { flags+=" --non-masquerade-cidr=${NON_MASQUERADE_CIDR}" fi flags+=" --volume-plugin-dir=${VOLUME_PLUGIN_DIR}" - local node_labels=$(build-node-labels ${master}) + local node_labels="$(build-linux-node-labels ${master})" if [[ -n "${node_labels:-}" ]]; then flags+=" --node-labels=${node_labels}" fi if [[ -n "${NODE_TAINTS:-}" ]]; then flags+=" --register-with-taints=${NODE_TAINTS}" fi - # TODO(mtaufen): ROTATE_CERTIFICATES seems unused; delete it? - if [[ -n "${ROTATE_CERTIFICATES:-}" ]]; then - flags+=" --rotate-certificates=true" - fi if [[ -n "${CONTAINER_RUNTIME:-}" ]]; then flags+=" --container-runtime=${CONTAINER_RUNTIME}" fi if [[ -n "${CONTAINER_RUNTIME_ENDPOINT:-}" ]]; then flags+=" --container-runtime-endpoint=${CONTAINER_RUNTIME_ENDPOINT}" fi - if [[ -n "${MAX_PODS_PER_NODE:-}" ]]; then - flags+=" --max-pods=${MAX_PODS_PER_NODE}" + + KUBELET_ARGS="${flags}" +} + +# Sets KUBELET_ARGS with the kubelet flags for Windows nodes. +function construct-windows-kubelet-flags { + local flags="$(construct-common-kubelet-flags)" + + # Note: NODE_KUBELET_TEST_ARGS is empty in typical kube-up runs. + flags+=" ${NODE_KUBELET_TEST_ARGS:-}" + + local node_labels="$(build-windows-node-labels)" + if [[ -n "${node_labels:-}" ]]; then + flags+=" --node-labels=${node_labels}" fi + # Concatenate common and windows-only node taints and apply them. + local node_taints="${NODE_TAINTS:-}" + if [[ -n "${node_taints}" && -n "${WINDOWS_NODE_TAINTS:-}" ]]; then + node_taints+=":${WINDOWS_NODE_TAINTS}" + else + node_taints="${WINDOWS_NODE_TAINTS:-}" + fi + if [[ -n "${node_taints}" ]]; then + flags+=" --register-with-taints=${node_taints}" + fi + + # Many of these flags were adapted from + # https://github.com/Microsoft/SDN/blob/master/Kubernetes/windows/start-kubelet.ps1. + flags+=" --config=${WINDOWS_KUBELET_CONFIG_FILE}" + + # Path to a kubeconfig file that will be used to get client certificate for + # kubelet. If the file specified by --kubeconfig does not exist, the bootstrap + # kubeconfig is used to request a client certificate from the API server. On + # success, a kubeconfig file referencing the generated client certificate and + # key is written to the path specified by --kubeconfig. The client certificate + # and key file will be stored in the directory pointed by --cert-dir. + # + # See also: + # https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet-tls-bootstrapping/ + flags+=" --bootstrap-kubeconfig=${WINDOWS_BOOTSTRAP_KUBECONFIG_FILE}" + flags+=" --kubeconfig=${WINDOWS_KUBECONFIG_FILE}" + + # The directory where the TLS certs are located. + flags+=" --cert-dir=${WINDOWS_PKI_DIR}" + + flags+=" --network-plugin=cni" + flags+=" --cni-bin-dir=${WINDOWS_CNI_DIR}" + flags+=" --cni-conf-dir=${WINDOWS_CNI_CONFIG_DIR}" + flags+=" --pod-manifest-path=${WINDOWS_MANIFESTS_DIR}" + + # Windows images are large and we don't have gcr mirrors yet. Allow longer + # pull progress deadline. + flags+=" --image-pull-progress-deadline=5m" + flags+=" --enable-debugging-handlers=true" + + # Configure kubelet to run as a windows service. + flags+=" --windows-service=true" + + # TODO(mtaufen): Configure logging for kubelet running as a service. I haven't + # been able to figure out how to direct stdout/stderr into log files when + # configuring it to run via sc.exe, so we just manually override logging + # config here. + flags+=" --log-file=${WINDOWS_LOGS_DIR}\kubelet.log" + # klog sets this to true internally, so need to override to false so we + # actually log to the file + flags+=" --logtostderr=false" + + # Configure flags with explicit empty string values. We can't escape + # double-quotes, because they still break sc.exe after expansion in the + # binPath parameter, and single-quotes get parsed as characters instead of + # string delimiters. + flags+=" --resolv-conf=" + + # Both --cgroups-per-qos and --enforce-node-allocatable should be disabled on + # windows; the latter requires the former to be enabled to work. + flags+=" --cgroups-per-qos=false --enforce-node-allocatable=" + + # Turn off kernel memory cgroup notification. + flags+=" --experimental-kernel-memcg-notification=false" + KUBELET_ARGS="${flags}" } # $1: if 'true', we're rendering config for a master, else a node function build-kubelet-config { - local master=$1 - local file=$2 + local master="$1" + local os="$2" + local file="$3" rm -f "${file}" { - declare quoted_dns_server_ip - declare quoted_dns_domain - quoted_dns_server_ip=$(yaml-quote "${DNS_SERVER_IP}") - if [[ "${ENABLE_NODELOCAL_DNS:-}" == "true" ]]; then - quoted_dns_server_ip=$(yaml-quote "${LOCAL_DNS_IP}") + print-common-kubelet-config + if [[ "${master}" == "true" ]]; then + print-master-kubelet-config + else + print-common-node-kubelet-config + if [[ "${os}" == "linux" ]]; then + print-linux-node-kubelet-config + elif [[ "${os}" == "windows" ]]; then + print-windows-node-kubelet-config + else + echo "Unknown OS ${os}" >&2 + exit 1 + fi fi - quoted_dns_domain=$(yaml-quote "${DNS_DOMAIN}") - cat < "${file}" +} + +# cat the Kubelet config yaml in common between masters, linux nodes, and +# windows nodes +function print-common-kubelet-config { + declare quoted_dns_server_ip + declare quoted_dns_domain + quoted_dns_server_ip=$(yaml-quote "${DNS_SERVER_IP}") + if [[ "${ENABLE_NODELOCAL_DNS:-}" == "true" ]]; then + quoted_dns_server_ip=$(yaml-quote "${LOCAL_DNS_IP}") + fi + quoted_dns_domain=$(yaml-quote "${DNS_DOMAIN}") + cat <>$file <>$file </dev/null 2>&1; then sha1sum "$1" | awk '{ print $1 }' @@ -1521,6 +1730,7 @@ for c in required: if missing: for c in missing: print ("missing required gcloud component \"{0}\"".format(c)) + print ("Try running `gcloud components install {0}`".format(c)) exit(1) ' """${version}""" fi @@ -1670,19 +1880,23 @@ function validate-node-local-ssds-ext(){ # Robustly try to create an instance template. # $1: The name of the instance template. # $2: The scopes flag. -# $3: String of comma-separated metadata entries (must all be from a file). +# $3: String of comma-separated metadata-from-file entries. +# $4: String of comma-separated metadata (key=value) entries. +# $5: the node OS ("linux" or "windows"). function create-node-template() { detect-project detect-subnetworks local template_name="$1" + local metadata_values="$4" + local os="$5" # First, ensure the template doesn't exist. # TODO(zmerlynn): To make this really robust, we need to parse the output and # add retries. Just relying on a non-zero exit code doesn't # distinguish an ephemeral failed call from a "not-exists". - if gcloud compute instance-templates describe "$template_name" --project "${PROJECT}" &>/dev/null; then + if gcloud compute instance-templates describe "${template_name}" --project "${PROJECT}" &>/dev/null; then echo "Instance template ${1} already exists; deleting." >&2 - if ! gcloud compute instance-templates delete "$template_name" --project "${PROJECT}" --quiet &>/dev/null; then + if ! gcloud compute instance-templates delete "${template_name}" --project "${PROJECT}" --quiet &>/dev/null; then echo -e "${color_yellow}Failed to delete existing instance template${color_norm}" >&2 exit 2 fi @@ -1737,17 +1951,28 @@ function create-node-template() { "${ENABLE_IP_ALIASES:-}" \ "${IP_ALIAS_SIZE:-}") + local node_image_flags="" + if [[ "${os}" == 'linux' ]]; then + node_image_flags="--image-project ${NODE_IMAGE_PROJECT} --image ${NODE_IMAGE}" + elif [[ "${os}" == 'windows' ]]; then + node_image_flags="--image-project ${WINDOWS_NODE_IMAGE_PROJECT} --image-family ${WINDOWS_NODE_IMAGE_FAMILY}" + else + echo "Unknown OS ${os}" >&2 + exit 1 + fi + + local metadata_flag="${metadata_values:+--metadata ${metadata_values}}" + local attempt=1 while true; do echo "Attempt ${attempt} to create ${1}" >&2 if ! ${gcloud} compute instance-templates create \ - "$template_name" \ + "${template_name}" \ --project "${PROJECT}" \ --machine-type "${NODE_SIZE}" \ --boot-disk-type "${NODE_DISK_TYPE}" \ --boot-disk-size "${NODE_DISK_SIZE}" \ - --image-project="${NODE_IMAGE_PROJECT}" \ - --image "${NODE_IMAGE}" \ + ${node_image_flags} \ --service-account "${NODE_SERVICE_ACCOUNT}" \ --tags "${NODE_TAG}" \ ${accelerator_args} \ @@ -1756,19 +1981,20 @@ function create-node-template() { ${network} \ ${preemptible_minions} \ $2 \ - --metadata-from-file $3 >&2; then + --metadata-from-file $3 \ + ${metadata_flag} >&2; then if (( attempt > 5 )); then - echo -e "${color_red}Failed to create instance template $template_name ${color_norm}" >&2 + echo -e "${color_red}Failed to create instance template ${template_name} ${color_norm}" >&2 exit 2 fi - echo -e "${color_yellow}Attempt ${attempt} failed to create instance template $template_name. Retrying.${color_norm}" >&2 + echo -e "${color_yellow}Attempt ${attempt} failed to create instance template ${template_name}. Retrying.${color_norm}" >&2 attempt=$(($attempt+1)) sleep $(($attempt * 5)) # In case the previous attempt failed with something like a # Backend Error and left the entry laying around, delete it # before we try again. - gcloud compute instance-templates delete "$template_name" --project "${PROJECT}" &>/dev/null || true + gcloud compute instance-templates delete "${template_name}" --project "${PROJECT}" &>/dev/null || true else break fi @@ -1799,7 +2025,9 @@ function kube-up() { parse-master-env create-subnetworks detect-subnetworks - create-nodes + # Windows nodes take longer to boot and setup so create them first. + create-windows-nodes + create-linux-nodes elif [[ ${KUBE_REPLICATE_EXISTING_MASTER:-} == "true" ]]; then if [[ "${MASTER_OS_DISTRIBUTION}" != "gci" && "${MASTER_OS_DISTRIBUTION}" != "ubuntu" ]]; then echo "Master replication supported only for gci and ubuntu" @@ -1822,7 +2050,9 @@ function kube-up() { create-master create-nodes-firewall create-nodes-template - create-nodes + # Windows nodes take longer to boot and setup so create them first. + create-windows-nodes + create-linux-nodes check-cluster fi } @@ -1897,6 +2127,17 @@ function create-network() { --source-ranges "0.0.0.0/0" \ --allow "tcp:22" & fi + + # Open up TCP 3389 to allow RDP connections. + if [[ ${NUM_WINDOWS_NODES} -gt 0 ]]; then + if ! gcloud compute firewall-rules describe --project "${NETWORK_PROJECT}" "${NETWORK}-default-rdp" &>/dev/null; then + gcloud compute firewall-rules create "${NETWORK}-default-rdp" \ + --project "${NETWORK_PROJECT}" \ + --network "${NETWORK}" \ + --source-ranges "0.0.0.0/0" \ + --allow "tcp:3389" & + fi + fi } function expand-default-subnetwork() { @@ -2187,7 +2428,7 @@ function create-master() { create-etcd-certs ${MASTER_NAME} create-etcd-apiserver-certs "etcd-${MASTER_NAME}" ${MASTER_NAME} - if [[ "${NUM_NODES}" -ge "50" ]]; then + if [[ "$(get-num-nodes)" -ge "50" ]]; then # We block on master creation for large clusters to avoid doing too much # unnecessary work in case master start-up fails (like creation of nodes). create-master-instance "${MASTER_RESERVED_IP}" @@ -2377,17 +2618,25 @@ function create-nodes-template() { local scope_flags=$(get-scope-flags) - write-node-env + write-linux-node-env + write-windows-node-env - local template_name="${NODE_INSTANCE_PREFIX}-template" - create-node-instance-template $template_name + # NOTE: these template names and their format must match + # create-[linux,windows]-nodes() as well as get-template()! + # TODO(pjh): find a better way to manage these (get-template() is annoying). + local linux_template_name="${NODE_INSTANCE_PREFIX}-template" + local windows_template_name="${NODE_INSTANCE_PREFIX}-template-windows" + create-linux-node-instance-template $linux_template_name + create-windows-node-instance-template $windows_template_name "${scope_flags[*]}" } # Assumes: # - MAX_INSTANCES_PER_MIG # - NUM_NODES +# - NUM_WINDOWS_NODES # exports: # - NUM_MIGS +# - NUM_WINDOWS_MIGS function set_num_migs() { local defaulted_max_instances_per_mig=${MAX_INSTANCES_PER_MIG:-1000} @@ -2396,6 +2645,7 @@ function set_num_migs() { defaulted_max_instances_per_mig=1000 fi export NUM_MIGS=$(((${NUM_NODES} + ${defaulted_max_instances_per_mig} - 1) / ${defaulted_max_instances_per_mig})) + export NUM_WINDOWS_MIGS=$(((${NUM_WINDOWS_NODES} + ${defaulted_max_instances_per_mig} - 1) / ${defaulted_max_instances_per_mig})) } # Assumes: @@ -2404,7 +2654,7 @@ function set_num_migs() { # - NUM_NODES # - PROJECT # - ZONE -function create-nodes() { +function create-linux-nodes() { local template_name="${NODE_INSTANCE_PREFIX}-template" if [[ -z "${HEAPSTER_MACHINE_TYPE:-}" ]]; then @@ -2434,7 +2684,7 @@ function create-nodes() { --zone "${ZONE}" \ --base-instance-name "${group_name}" \ --size "${this_mig_size}" \ - --template "$template_name" || true; + --template "${template_name}" || true; gcloud compute instance-groups managed wait-until-stable \ "${group_name}" \ --zone "${ZONE}" \ @@ -2444,6 +2694,44 @@ function create-nodes() { wait } +# Assumes: +# - NUM_WINDOWS_MIGS +# - NODE_INSTANCE_PREFIX +# - NUM_WINDOWS_NODES +# - PROJECT +# - ZONE +function create-windows-nodes() { + local template_name="${NODE_INSTANCE_PREFIX}-template-windows" + + local -r nodes="${NUM_WINDOWS_NODES}" + local instances_left=${nodes} + + for ((i=1; i<=${NUM_WINDOWS_MIGS}; i++)); do + local group_name="${NODE_INSTANCE_PREFIX}-windows-group-$i" + if [[ $i == ${NUM_WINDOWS_MIGS} ]]; then + # TODO: We don't add a suffix for the last group to keep backward compatibility when there's only one MIG. + # We should change it at some point, but note #18545 when changing this. + group_name="${NODE_INSTANCE_PREFIX}-windows-group" + fi + # Spread the remaining number of nodes evenly + this_mig_size=$((${instances_left} / (${NUM_WINDOWS_MIGS}-${i}+1))) + instances_left=$((instances_left-${this_mig_size})) + + gcloud compute instance-groups managed \ + create "${group_name}" \ + --project "${PROJECT}" \ + --zone "${ZONE}" \ + --base-instance-name "${group_name}" \ + --size "${this_mig_size}" \ + --template "${template_name}" || true; + gcloud compute instance-groups managed wait-until-stable \ + "${group_name}" \ + --zone "${ZONE}" \ + --project "${PROJECT}" \ + --timeout "${MIG_WAIT_UNTIL_STABLE_TIMEOUT}" || true; + done +} + # Assumes: # - NODE_INSTANCE_PREFIX # - PROJECT @@ -2486,7 +2774,7 @@ function create-heapster-node() { --tags "${NODE_TAG}" \ ${network} \ $(get-scope-flags) \ - --metadata-from-file "$(get-node-instance-metadata)" + --metadata-from-file "$(get-linux-node-instance-metadata-from-file)" } # Assumes: @@ -2513,6 +2801,11 @@ function create-cluster-autoscaler-mig-config() { echo "AUTOSCALER_MAX_NODES must be greater or equal ${NUM_MIGS}" exit 2 fi + if [[ ${NUM_WINDOWS_MIGS} -gt 0 ]]; then + # TODO(pjh): implement Windows support in this function. + echo "Not implemented yet: autoscaler config for Windows MIGs" + exit 2 + fi # The code assumes that the migs were created with create-nodes # function which tries to evenly spread nodes across the migs. diff --git a/cluster/gce/win1803/OWNERS b/cluster/gce/win1803/OWNERS new file mode 100644 index 0000000000..04490edb64 --- /dev/null +++ b/cluster/gce/win1803/OWNERS @@ -0,0 +1,2 @@ +approvers: +- yujuhong diff --git a/cluster/gce/win1803/README-GCE-Windows-kube-up.md b/cluster/gce/win1803/README-GCE-Windows-kube-up.md new file mode 100644 index 0000000000..a01703a0ae --- /dev/null +++ b/cluster/gce/win1803/README-GCE-Windows-kube-up.md @@ -0,0 +1,187 @@ +# Starting a Windows Kubernetes cluster on GCE using kube-up + +## Bring up the cluster + +Prerequisites: a Google Cloud Platform project. + +### 0. Prepare your environment + +Clone this repository under your `$GOPATH/src` directory on a Linux machine. +Then, optionally clean/prepare your environment using these commands: + +``` +# Remove files that interfere with get-kube / kube-up: +rm -rf ./kubernetes/; rm -f kubernetes.tar.gz; rm -f ~/.kube/config + +# Set the default gcloud project for this shell. This is optional but convenient +# if you're working with multiple projects and don't want to repeatedly switch +# between gcloud config configurations. +export CLOUDSDK_CORE_PROJECT= +``` + +### 1. Build Kubernetes + +The most straightforward approach to build those binaries is to run `make +release`. However, that builds binaries for all supported platforms, and can be +slow. You can speed up the process by following the instructions below to only +build the necessary binaries. +``` +# Fetch the PR: https://github.com/pjh/kubernetes/pull/43 +git remote add pjh https://github.com/pjh/kubernetes +git fetch pjh pull/43/head + +# Get the commit hash and cherry-pick the commit to your current branch +BUILD_WIN_COMMIT=$(git ls-remote pjh | grep refs/pull/43/head | cut -f 1) +git cherry-pick $BUILD_WIN_COMMIT + +# Build binaries for both Linux and Windows +make quick-release +``` + +### 2 Create a Kubernetes cluster + +You can create a regular Kubernetes cluster or an end-to-end test cluster. +Please make sure you set the environment variables properly following the +instructions in the previous section. + +First, set the following environment variables which are required for +controlling the number of Linux and Windows nodes in the cluster and for +enabling IP aliases (which are required for Windows pod routing): + +``` +export NUM_NODES=2 # number of Linux nodes +export NUM_WINDOWS_NODES=2 +export KUBE_GCE_ENABLE_IP_ALIASES=true +``` + +If you wish to use `netd` as the CNI plugin for Linux nodes, set these +variables: + +``` +export KUBE_ENABLE_NETD=true +export KUBE_CUSTOM_NETD_YAML=$(curl -s \ + https://raw.githubusercontent.com/GoogleCloudPlatform/netd/master/netd.yaml \ + | sed -e 's/^/ /') +``` + +Now bring up a cluster using one of the following two methods: + +#### 2.a Create a regular Kubernetes cluster + +``` +# Invoke kube-up.sh with these environment variables: +# PROJECT: text name of your GCP project. +# KUBERNETES_SKIP_CONFIRM: skips any kube-up prompts. +PROJECT=${CLOUDSDK_CORE_PROJECT} KUBERNETES_SKIP_CONFIRM=y ./cluster/kube-up.sh +``` + +To teardown the cluster run: + +``` +PROJECT=${CLOUDSDK_CORE_PROJECT} KUBERNETES_SKIP_CONFIRM=y ./cluster/kube-down.sh +``` + +#### 2.b Create a Kubernetes end-to-end (E2E) test cluster + +``` +PROJECT=${CLOUDSDK_CORE_PROJECT} go run ./hack/e2e.go -- --up +``` +This command, by default, tears down the existing E2E cluster and create a new +one. + +No matter what type of cluster you chose to create, the result should be a +Kubernetes cluster with one Linux master node, `NUM_NODES` Linux worker nodes +and `NUM_WINDOWS_NODES` Windows worker nodes. + +## Validating the cluster + +Invoke this script to run a smoke test that verifies that the cluster has been +brought up correctly: + +``` +cluster/gce/win1803/smoke-test.sh +``` + +## Running tests against the cluster + +These steps are based on +[kubernetes-sigs/windows-testing](https://github.com/kubernetes-sigs/windows-testing). + +* TODO(pjh): use patched `cluster/local/util.sh` from + https://github.com/pjh/kubernetes/blob/windows-up/cluster/local/util.sh. + +* If necessary run `alias kubectl=client/bin/kubectl` . + +* Set the following environment variables (these values should make sense if + you built your cluster using the kube-up steps above): + + ``` + export KUBE_HOME=$(pwd) + export KUBECONFIG=~/.kube/config + export KUBE_MASTER=local + export KUBE_MASTER_NAME=kubernetes-master + export KUBE_MASTER_IP=$(kubectl get node ${KUBE_MASTER_NAME} -o jsonpath='{.status.addresses[?(@.type=="ExternalIP")].address}') + export KUBE_MASTER_URL=https://${KUBE_MASTER_IP} + export KUBE_MASTER_PORT=443 + ``` + +* Download the list of Windows e2e tests: + + ``` + curl https://raw.githubusercontent.com/e2e-win/e2e-win-prow-deployment/master/repo-list.txt -o ${KUBE_HOME}/repo-list.yaml + export KUBE_TEST_REPO_LIST=${KUBE_HOME}/repo-list.yaml + ``` + +* Download and configure the list of tests to exclude: + + ``` + curl https://raw.githubusercontent.com/e2e-win/e2e-win-prow-deployment/master/exclude_conformance_test.txt -o ${KUBE_HOME}/exclude_conformance_test.txt + export EXCLUDED_TESTS=$(cat exclude_conformance_test.txt | + tr -d '\r' | # remove Windows carriage returns + tr -s '\n' '|' | # coalesce newlines into | + tr -s ' ' '.' | # coalesce spaces into . + sed -e 's/[]\[()]/\\&/g' | # escape brackets and parentheses + sed -e 's/.$//g') # remove final | added by tr + ``` + +* Taint the Linux nodes so that test pods will not land on them: + + ``` + export LINUX_NODES=$(kubectl get nodes -l beta.kubernetes.io/os=linux,kubernetes.io/hostname!=${KUBE_MASTER_NAME} -o name) + export LINUX_NODE_COUNT=$(echo ${LINUX_NODES} | wc -w) + for node in $LINUX_NODES; do + kubectl taint node $node node-under-test=false:NoSchedule + done + ``` + +* Build necessary test binaries: + + ``` + make WHAT=test/e2e/e2e.test + ``` + +* Run the tests with flags that point at the "local" (already-running) cluster + and that permit the `NoSchedule` Linux nodes: + + ``` + export KUBETEST_ARGS="--ginkgo.noColor=true "\ + "--report-dir=${KUBE_HOME}/e2e-reports "\ + "--allowed-not-ready-nodes=${LINUX_NODE_COUNT} "\ + "--ginkgo.dryRun=false "\ + "--ginkgo.focus=\[Conformance\] "\ + "--ginkgo.skip=${EXCLUDED_TESTS}" + + go run ${KUBE_HOME}/hack/e2e.go -- --verbose-commands \ + --ginkgo-parallel=4 \ + --check-version-skew=false --test --provider=local \ + --test_args="${KUBETEST_ARGS}" &> ${KUBE_HOME}/conformance.out + ``` + + TODO: copy log files from Windows nodes using some command like: + + ``` + scp -r -o PreferredAuthentications=keyboard-interactive,password \ + -o PubkeyAuthentication=no \ + user@kubernetes-minion-windows-group-mk0p:C:\\etc\\kubernetes\\logs \ + kubetest-logs/ + ``` diff --git a/cluster/gce/win1803/common.psm1 b/cluster/gce/win1803/common.psm1 new file mode 100644 index 0000000000..46bf53eeaf --- /dev/null +++ b/cluster/gce/win1803/common.psm1 @@ -0,0 +1,90 @@ +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +<# +.SYNOPSIS + Library containing common variables and code used by other PowerShell modules + and scripts for configuring Windows nodes. +#> + +# REDO_STEPS affects the behavior of a node that is rebooted after initial +# bringup. When true, on a reboot the scripts will redo steps that were +# determined to have already been completed once (e.g. to overwrite +# already-existing config files). When false the scripts will perform the +# minimum required steps to re-join this node to the cluster. +$REDO_STEPS = $false +Export-ModuleMember -Variable REDO_STEPS + +# Writes $Message to the console. Terminates the script if $Fatal is set. +function Log-Output { + param ( + [parameter(Mandatory=$true)] [string]$Message, + [switch]$Fatal + ) + Write-Host "${Message}" + if (${Fatal}) { + Exit 1 + } +} + +# Checks if a file should be written or overwritten by testing if it already +# exists and checking the value of the global $REDO_STEPS variable. Emits an +# informative message if the file already exists. +# +# Returns $true if the file does not exist, or if it does but the global +# $REDO_STEPS variable is set to $true. Returns $false if the file exists and +# the caller should not overwrite it. +function ShouldWrite-File { + param ( + [parameter(Mandatory=$true)] [string]$Filename + ) + if (Test-Path $Filename) { + if ($REDO_STEPS) { + Log-Output "Warning: $Filename already exists, will overwrite it" + return $true + } + Log-Output "Skip: $Filename already exists, not overwriting it" + return $false + } + return $true +} + +# Returns the GCE instance metadata value for $Key. If the key is not present +# in the instance metadata returns $Default if set, otherwise returns $null. +function Get-InstanceMetadataValue { + param ( + [parameter(Mandatory=$true)] [string]$Key, + [parameter(Mandatory=$false)] [string]$Default + ) + + $url = ("http://metadata.google.internal/computeMetadata/v1/instance/" + + "attributes/$Key") + try { + $client = New-Object Net.WebClient + $client.Headers.Add('Metadata-Flavor', 'Google') + return ($client.DownloadString($url)).Trim() + } + catch [System.Net.WebException] { + if ($Default) { + return $Default + } + else { + Log-Output "Failed to retrieve value for $Key." + return $null + } + } +} + +# Export all public functions: +Export-ModuleMember -Function *-* diff --git a/cluster/gce/win1803/configure.ps1 b/cluster/gce/win1803/configure.ps1 new file mode 100644 index 0000000000..1a0addc6fb --- /dev/null +++ b/cluster/gce/win1803/configure.ps1 @@ -0,0 +1,119 @@ +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +<# +.SYNOPSIS + Top-level script that runs on Windows nodes to join them to the K8s cluster. +#> + +$ErrorActionPreference = 'Stop' + +# Turn on tracing to debug +# Set-PSDebug -Trace 1 + +# Update TLS setting to enable Github downloads and disable progress bar to +# increase download speed. +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 +$ProgressPreference = 'SilentlyContinue' + +# Returns the GCE instance metadata value for $Key. If the key is not present +# in the instance metadata returns $Default if set, otherwise returns $null. +function Get-InstanceMetadataValue { + param ( + [parameter(Mandatory=$true)] [string]$Key, + [parameter(Mandatory=$false)] [string]$Default + ) + + $url = ("http://metadata.google.internal/computeMetadata/v1/instance/" + + "attributes/$Key") + try { + $client = New-Object Net.WebClient + $client.Headers.Add('Metadata-Flavor', 'Google') + return ($client.DownloadString($url)).Trim() + } + catch [System.Net.WebException] { + if ($Default) { + return $Default + } + else { + Write-Host "Failed to retrieve value for $Key." + return $null + } + } +} + +# Fetches the value of $MetadataKey, saves it to C:\$Filename and imports it as +# a PowerShell module. +# +# Note: this function depends on common.psm1. +function FetchAndImport-ModuleFromMetadata { + param ( + [parameter(Mandatory=$true)] [string]$MetadataKey, + [parameter(Mandatory=$true)] [string]$Filename + ) + + $module = Get-InstanceMetadataValue $MetadataKey + if (Test-Path C:\$Filename) { + if (-not $REDO_STEPS) { + Log-Output "Skip: C:\$Filename already exists, not overwriting" + Import-Module -Force C:\$Filename + return + } + Log-Output "Warning: C:\$Filename already exists, will overwrite it." + } + New-Item -ItemType file -Force C:\$Filename | Out-Null + Set-Content C:\$Filename $module + Import-Module -Force C:\$Filename +} + +try { + # Don't use FetchAndImport-ModuleFromMetadata for common.psm1 - the common + # module includes variables and functions that any other function may depend + # on. + $module = Get-InstanceMetadataValue 'common-psm1' + New-Item -ItemType file -Force C:\common.psm1 | Out-Null + Set-Content C:\common.psm1 $module + Import-Module -Force C:\common.psm1 + + # TODO(pjh): update the function to set $Filename automatically from the key, + # then put these calls into a loop over a list of XYZ-psm1 keys. + FetchAndImport-ModuleFromMetadata 'k8s-node-setup-psm1' 'k8s-node-setup.psm1' + + Set-PrerequisiteOptions + $kube_env = Fetch-KubeEnv + Set-EnvironmentVars + Create-Directories + Download-HelperScripts + + Create-PauseImage + DownloadAndInstall-KubernetesBinaries + Create-NodePki + Create-KubeletKubeconfig + Create-KubeproxyKubeconfig + Set-PodCidr + Configure-HostNetworkingService + Configure-CniNetworking + Configure-Kubelet + + Start-WorkerServices + Log-Output 'Waiting 15 seconds for node to join cluster.' + Start-Sleep 15 + Verify-WorkerServices +} +catch { + Write-Host 'Exception caught in script:' + Write-Host $_.InvocationInfo.PositionMessage + Write-Host "Kubernetes Windows node setup failed: $($_.Exception.Message)" + exit 1 +} diff --git a/cluster/gce/win1803/k8s-node-setup.psm1 b/cluster/gce/win1803/k8s-node-setup.psm1 new file mode 100644 index 0000000000..561c075779 --- /dev/null +++ b/cluster/gce/win1803/k8s-node-setup.psm1 @@ -0,0 +1,1028 @@ +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +<# +.SYNOPSIS + Library for configuring Windows nodes and joining them to the cluster. + +.NOTES + This module depends on common.psm1. + + Some portions copied / adapted from + https://github.com/Microsoft/SDN/blob/master/Kubernetes/windows/start-kubelet.ps1. + +.EXAMPLE + Suggested usage for dev/test: + [Net.ServicePointManager]::SecurityProtocol = ` + [Net.SecurityProtocolType]::Tls12 + Invoke-WebRequest ` + https://github.com/kubernetes/kubernetes/raw/windows-up/cluster/gce/win1803/k8s-node-setup.psm1 ` + -OutFile C:\k8s-node-setup.psm1 + Invoke-WebRequest ` + https://github.com/kubernetes/kubernetes/raw/windows-up/cluster/gce/win1803/configure.ps1 ` + -OutFile C:\configure.ps1 + Import-Module -Force C:\k8s-node-setup.psm1 # -Force to override existing + # Execute functions manually or run configure.ps1. +#> + +# TODO: update scripts for these style guidelines: +# - Remove {} around variable references unless actually needed for clarity. +# - Always use single-quoted strings unless actually interpolating variables +# or using escape characters. +# - Use "approved verbs": +# https://docs.microsoft.com/en-us/powershell/developer/cmdlet/approved-verbs-for-windows-powershell-commands +# - Document functions using proper syntax: +# https://technet.microsoft.com/en-us/library/hh847834(v=wps.620).aspx + +$INFRA_CONTAINER = "kubeletwin/pause" +$GCE_METADATA_SERVER = "169.254.169.254" +# The "management" interface is used by the kubelet and by Windows pods to talk +# to the rest of the Kubernetes cluster *without NAT*. This interface does not +# exist until an initial HNS network has been created on the Windows node - see +# Add_InitialHnsNetwork(). +$MGMT_ADAPTER_NAME = "vEthernet (Ethernet*" + +Import-Module -Force C:\common.psm1 + +# Writes a TODO with $Message to the console. +function Log_Todo { + param ( + [parameter(Mandatory=$true)] [string]$Message + ) + Log-Output "TODO: ${Message}" +} + +# Writes a not-implemented warning with $Message to the console and exits the +# script. +function Log_NotImplemented { + param ( + [parameter(Mandatory=$true)] [string]$Message + ) + Log-Output "Not implemented yet: ${Message}" -Fatal +} + +# Fails and exits if the route to the GCE metadata server is not present, +# otherwise does nothing and emits nothing. +function Verify_GceMetadataServerRouteIsPresent { + Try { + Get-NetRoute ` + -ErrorAction "Stop" ` + -AddressFamily IPv4 ` + -DestinationPrefix ${GCE_METADATA_SERVER}/32 | Out-Null + } Catch [Microsoft.PowerShell.Cmdletization.Cim.CimJobException] { + Log-Output -Fatal ` + ("GCE metadata server route is not present as expected.`n" + + "$(Get-NetRoute -AddressFamily IPv4 | Out-String)") + } +} + +# Checks if the route to the GCE metadata server is present. Returns when the +# route is NOT present or after a timeout has expired. +function WaitFor_GceMetadataServerRouteToBeRemoved { + $elapsed = 0 + $timeout = 60 + Log-Output ("Waiting up to ${timeout} seconds for GCE metadata server " + + "route to be removed") + while (${elapsed} -lt ${timeout}) { + Try { + Get-NetRoute ` + -ErrorAction "Stop" ` + -AddressFamily IPv4 ` + -DestinationPrefix ${GCE_METADATA_SERVER}/32 | Out-Null + } Catch [Microsoft.PowerShell.Cmdletization.Cim.CimJobException] { + break + } + $sleeptime = 2 + Start-Sleep ${sleeptime} + ${elapsed} += ${sleeptime} + } +} + +# Adds a route to the GCE metadata server to every network interface. +function Add_GceMetadataServerRoute { + # Before setting up HNS the 1803 VM has a "vEthernet (nat)" interface and a + # "Ethernet" interface, and the route to the metadata server exists on the + # Ethernet interface. After adding the HNS network a "vEthernet (Ethernet)" + # interface is added, and it seems to subsume the routes of the "Ethernet" + # interface (trying to add routes on the Ethernet interface at this point just + # results in "New-NetRoute : Element not found" errors). I don't know what's + # up with that, but since it's hard to know what's the right thing to do here + # we just try to add the route on all of the network adapters. + Get-NetAdapter | ForEach-Object { + $adapter_index = $_.InterfaceIndex + New-NetRoute ` + -ErrorAction Ignore ` + -DestinationPrefix "${GCE_METADATA_SERVER}/32" ` + -InterfaceIndex ${adapter_index} | Out-Null + } +} + +# Fetches the kube-env from the instance metadata. +# +# Returns: a PowerShell Hashtable object containing the key-value pairs from +# kube-env. +function Fetch-KubeEnv { + # Testing / debugging: + # First: + # ${kube_env} = Get-InstanceMetadataValue 'kube-env' + # or: + # ${kube_env} = [IO.File]::ReadAllText(".\kubeEnv.txt") + # ${kube_env_table} = ConvertFrom-Yaml ${kube_env} + # ${kube_env_table} + # ${kube_env_table}.GetType() + + # The type of kube_env is a powershell String. + $kube_env = Get-InstanceMetadataValue 'kube-env' + $kube_env_table = ConvertFrom-Yaml ${kube_env} + return ${kube_env_table} +} + +# Sets the environment variable $Key to $Value at the Machine scope (will +# be present in the environment for all new shells after a reboot). +function Set_MachineEnvironmentVar { + param ( + [parameter(Mandatory=$true)] [string]$Key, + [parameter(Mandatory=$true)] [string]$Value + ) + [Environment]::SetEnvironmentVariable($Key, $Value, "Machine") +} + +# Sets the environment variable $Key to $Value in the current shell. +function Set_CurrentShellEnvironmentVar { + param ( + [parameter(Mandatory=$true)] [string]$Key, + [parameter(Mandatory=$true)] [string]$Value + ) + $expression = '$env:' + $Key + ' = "' + $Value + '"' + Invoke-Expression ${expression} +} + +# Sets environment variables used by Kubernetes binaries and by other functions +# in this module. Depends on numerous ${kube_env} keys. +function Set-EnvironmentVars { + # Turning the kube-env values into environment variables is not required but + # it makes debugging this script easier, and it also makes the syntax a lot + # easier (${env:K8S_DIR} can be expanded within a string but + # ${kube_env}['K8S_DIR'] cannot be afaik). + $env_vars = @{ + "K8S_DIR" = ${kube_env}['K8S_DIR'] + "NODE_DIR" = ${kube_env}['NODE_DIR'] + "CNI_DIR" = ${kube_env}['CNI_DIR'] + "CNI_CONFIG_DIR" = ${kube_env}['CNI_CONFIG_DIR'] + "PKI_DIR" = ${kube_env}['PKI_DIR'] + "KUBELET_CONFIG" = ${kube_env}['KUBELET_CONFIG_FILE'] + "BOOTSTRAP_KUBECONFIG" = ${kube_env}['BOOTSTRAP_KUBECONFIG_FILE'] + "KUBEPROXY_KUBECONFIG" = ${kube_env}['KUBEPROXY_KUBECONFIG_FILE'] + + "Path" = ${env:Path} + ";" + ${kube_env}['NODE_DIR'] + "KUBE_NETWORK" = "l2bridge".ToLower() + "CA_CERT_BUNDLE_PATH" = ${kube_env}['PKI_DIR'] + '\ca-certificates.crt' + "KUBELET_CERT_PATH" = ${kube_env}['PKI_DIR'] + '\kubelet.crt' + "KUBELET_KEY_PATH" = ${kube_env}['PKI_DIR'] + '\kubelet.key' + + # TODO(pjh): these are only in flags, can be removed from env once flags are + # moved to util.sh: + "LOGS_DIR" = ${kube_env}['LOGS_DIR'] + "MANIFESTS_DIR" = ${kube_env}['MANIFESTS_DIR'] + "KUBECONFIG" = ${kube_env}['KUBECONFIG_FILE'] + } + + # Set the environment variables in two ways: permanently on the machine (only + # takes effect after a reboot), and in the current shell. + $env_vars.GetEnumerator() | ForEach-Object{ + $message = "Setting environment variable: " + $_.key + " = " + $_.value + Log-Output ${message} + Set_MachineEnvironmentVar $_.key $_.value + Set_CurrentShellEnvironmentVar $_.key $_.value + } +} + +# Configures various settings and prerequisites needed for the rest of the +# functions in this module and the Kubernetes binaries to operate properly. +function Set-PrerequisiteOptions { + # The Windows firewall interferes with Kubernetes networking; GCE's firewall + # should be sufficient. + Log-Output "Disabling Windows Firewall" + Set-NetFirewallProfile -Profile Domain, Public, Private -Enabled False + + # Windows updates cause the node to reboot at arbitrary times. + Log-Output "Disabling Windows Update service" + sc.exe config wuauserv start=disabled + sc.exe stop wuauserv + + # Windows Defender periodically consumes 100% of the CPU. + # TODO(pjh): this (all of a sudden, ugh) started failing with "The term + # 'Set-MpPreference' is not recognized...". Investigate and fix or remove. + #Log-Output "Disabling Windows Defender service" + #Set-MpPreference -DisableRealtimeMonitoring $true + #Uninstall-WindowsFeature -Name 'Windows-Defender' + + # Use TLS 1.2: needed for Invoke-WebRequest downloads from github.com. + [Net.ServicePointManager]::SecurityProtocol = ` + [Net.SecurityProtocolType]::Tls12 + + # https://github.com/cloudbase/powershell-yaml + Log-Output "Installing powershell-yaml module from external repo" + Install-Module -Name powershell-yaml -Force +} + +# Creates directories where other functions in this module will read and write +# data. +function Create-Directories { + Log-Output "Creating ${env:K8S_DIR} and its subdirectories." + ForEach ($dir in ("${env:K8S_DIR}", "${env:NODE_DIR}", "${env:LOGS_DIR}", + "${env:CNI_DIR}", "${env:CNI_CONFIG_DIR}", "${env:MANIFESTS_DIR}", + "${env:PKI_DIR}")) { + mkdir -Force $dir + } +} + +# Downloads some external helper scripts needed by other functions in this +# module. +function Download-HelperScripts { + if (-not (ShouldWrite-File ${env:K8S_DIR}\hns.psm1)) { + return + } + Invoke-WebRequest ` + https://github.com/Microsoft/SDN/raw/master/Kubernetes/windows/hns.psm1 ` + -OutFile ${env:K8S_DIR}\hns.psm1 +} + +# Takes the Windows version string from the cluster bash scripts (e.g. +# 'win1803') and returns the correct label to use for containers on this +# version of Windows. Returns $null if $WinVersion is unknown. +function Get_ContainerVersionLabel { + param ( + [parameter(Mandatory=$true)] [string]$WinVersion + ) + # -match does regular expression matching. + if ($WinVersion -match '1803') { + return '1803' + } + elseif ($WinVersion -match '1809') { + return '1809' + } + elseif ($WinVersion -match '2019') { + return 'ltsc2019' + } + Throw ("Unknown Windows version $WinVersion, don't know its container " + + "version label") +} + +# Builds the pause image with name $INFRA_CONTAINER. +function Create-PauseImage { + $version_label = Get_ContainerVersionLabel ` + $(Get-InstanceMetadataValue 'win-version') + $pause_dir = "${env:K8S_DIR}\pauseimage" + $dockerfile = "$pause_dir\Dockerfile" + mkdir -Force $pause_dir + if (ShouldWrite-File $dockerfile) { + New-Item -Force -ItemType file $dockerfile | Out-Null + Set-Content ` + $dockerfile ` + ("FROM microsoft/nanoserver:${version_label}`n`n" + + "CMD cmd /c ping -t localhost > nul") + } + + if (($(docker images -a) -like "*${INFRA_CONTAINER}*") -and + (-not $REDO_STEPS)) { + Log-Output "Skip: ${INFRA_CONTAINER} already built" + return + } + docker build -t ${INFRA_CONTAINER} $pause_dir +} + +# Downloads the Kubernetes binaries from kube-env's NODE_BINARY_TAR_URL and +# puts them in a subdirectory of $env:K8S_DIR. +# +# Required ${kube_env} keys: +# NODE_BINARY_TAR_URL +function DownloadAndInstall-KubernetesBinaries { + # Assume that presence of kubelet.exe indicates that the kubernetes binaries + # were already previously downloaded to this node. + if (-not (ShouldWrite-File ${env:NODE_DIR}\kubelet.exe)) { + return + } + + $tmp_dir = 'C:\k8s_tmp' + New-Item -Force -ItemType 'directory' $tmp_dir | Out-Null + + $uri = ${kube_env}['NODE_BINARY_TAR_URL'] + $filename = Split-Path -leaf $uri + + # Disable progress bar to increase download speed. + $ProgressPreference = 'SilentlyContinue' + Invoke-WebRequest $uri -OutFile ${tmp_dir}\${filename} + + # TODO(yujuhong): Verify hash of the tarball. + + # Change the directory to the parent directory of ${env:K8S_DIR} and untar. + # This (over-)writes ${dest_dir}/kubernetes/node/bin/*.exe files. + $dest_dir = (Get-Item ${env:K8S_DIR}).Parent.Fullname + tar xzf ${tmp_dir}\${filename} -C ${dest_dir} + + # Clean up the temporary directory + Remove-Item -Force -Recurse $tmp_dir +} + +# TODO(pjh): this is copied from +# https://github.com/Microsoft/SDN/blob/master/Kubernetes/windows/start-kubelet.ps1#L98. +# See if there's a way to fetch or construct the "management subnet" so that +# this is not needed. +function ConvertTo_DecimalIP +{ + param( + [parameter(Mandatory = $true, Position = 0)] + [Net.IPAddress] $IPAddress + ) + + $i = 3; $decimal_ip = 0; + $IPAddress.GetAddressBytes() | % { + $decimal_ip += $_ * [Math]::Pow(256, $i); $i-- + } + return [UInt32]$decimal_ip +} + +# TODO(pjh): this is copied from +# https://github.com/Microsoft/SDN/blob/master/Kubernetes/windows/start-kubelet.ps1#L98. +# See if there's a way to fetch or construct the "management subnet" so that +# this is not needed. +function ConvertTo_DottedDecimalIP +{ + param( + [parameter(Mandatory = $true, Position = 0)] + [Uint32] $IPAddress + ) + + $dotted_ip = $(for ($i = 3; $i -gt -1; $i--) { + $remainder = $IPAddress % [Math]::Pow(256, $i) + ($IPAddress - $remainder) / [Math]::Pow(256, $i) + $IPAddress = $remainder + }) + return [String]::Join(".", $dotted_ip) +} + +# TODO(pjh): this is copied from +# https://github.com/Microsoft/SDN/blob/master/Kubernetes/windows/start-kubelet.ps1#L98. +# See if there's a way to fetch or construct the "management subnet" so that +# this is not needed. +function ConvertTo_MaskLength +{ + param( + [parameter(Mandatory = $True, Position = 0)] + [Net.IPAddress] $SubnetMask + ) + + $bits = "$($SubnetMask.GetAddressBytes() | % { + [Convert]::ToString($_, 2) + } )" -replace "[\s0]" + return $bits.Length +} + +# Returns the "management" subnet on which the Windows pods+kubelet will +# communicate with the rest of the Kubernetes cluster without NAT. In GCE this +# is the subnet that VM internal IPs are allocated from. +# +# This function will fail if Add_InitialHnsNetwork() has not been called first. +function Get_MgmtSubnet { + $net_adapter = Get_MgmtNetAdapter + + # TODO(pjh): applying the primary interface's subnet mask to its IP address + # *should* give us the GCE network subnet that VM IP addresses are being + # allocated from... however it might be more accurate or straightforward to + # just fetch the IP address range for the VPC subnet that the kube-up script + # creates (kubernetes-subnet-default). + $addr = (Get-NetIPAddress ` + -InterfaceAlias ${net_adapter}.ifAlias ` + -AddressFamily IPv4).IPAddress + $mask = (Get-WmiObject Win32_NetworkAdapterConfiguration | + Where-Object InterfaceIndex -eq $(${net_adapter}.ifIndex)).IPSubnet[0] + $mgmt_subnet = ` + (ConvertTo_DecimalIP ${addr}) -band (ConvertTo_DecimalIP ${mask}) + $mgmt_subnet = ConvertTo_DottedDecimalIP ${mgmt_subnet} + return "${mgmt_subnet}/$(ConvertTo_MaskLength $mask)" +} + +# Returns a network adapter object for the "management" interface via which the +# Windows pods+kubelet will communicate with the rest of the Kubernetes cluster. +# +# This function will fail if Add_InitialHnsNetwork() has not been called first. +function Get_MgmtNetAdapter { + $net_adapter = Get-NetAdapter | Where-Object Name -like ${MGMT_ADAPTER_NAME} + if (-not ${net_adapter}) { + Throw ("Failed to find a suitable network adapter, check your network " + + "settings.") + } + + return $net_adapter +} + +# Decodes the base64 $Data string and writes it as binary to $File. Does +# nothing if $File already exists and $REDO_STEPS is not set. +function Write_PkiData { + param ( + [parameter(Mandatory=$true)] [string] $Data, + [parameter(Mandatory=$true)] [string] $File + ) + + if (-not (ShouldWrite-File $File)) { + return + } + + # This command writes out a PEM certificate file, analogous to "base64 + # --decode" on Linux. See https://stackoverflow.com/a/51914136/1230197. + [IO.File]::WriteAllBytes($File, [Convert]::FromBase64String($Data)) + Log_Todo ("need to set permissions correctly on ${File}; not sure what the " + + "Windows equivalent of 'umask 077' is") + # Linux: owned by root, rw by user only. + # -rw------- 1 root root 1.2K Oct 12 00:56 ca-certificates.crt + # -rw------- 1 root root 1.3K Oct 12 00:56 kubelet.crt + # -rw------- 1 root root 1.7K Oct 12 00:56 kubelet.key + # Windows: + # https://docs.microsoft.com/en-us/dotnet/api/system.io.fileattributes + # https://docs.microsoft.com/en-us/dotnet/api/system.io.fileattributes +} + +# Creates the node PKI files in $env:PKI_DIR. +# +# Required ${kube_env} keys: +# CA_CERT +# KUBELET_CERT +# KUBELET_KEY +function Create-NodePki { + Log-Output "Creating node pki files" + + $CA_CERT_BUNDLE = ${kube_env}['CA_CERT'] + $KUBELET_CERT = ${kube_env}['KUBELET_CERT'] + $KUBELET_KEY = ${kube_env}['KUBELET_KEY'] + + Write_PkiData "${CA_CERT_BUNDLE}" ${env:CA_CERT_BUNDLE_PATH} + Write_PkiData "${KUBELET_CERT}" ${env:KUBELET_CERT_PATH} + Write_PkiData "${KUBELET_KEY}" ${env:KUBELET_KEY_PATH} + Get-ChildItem ${env:PKI_DIR} +} + +# Creates the kubelet kubeconfig at $env:BOOTSTRAP_KUBECONFIG. +# +# Create-NodePki() must be called first. +# +# Required ${kube_env} keys: +# KUBERNETES_MASTER_NAME: the apiserver IP address. +function Create-KubeletKubeconfig { + # The API server IP address comes from KUBERNETES_MASTER_NAME in kube-env, I + # think. cluster/gce/gci/configure-helper.sh?l=2801 + $apiserverAddress = ${kube_env}['KUBERNETES_MASTER_NAME'] + + # TODO(pjh): set these using kube-env values. + $createBootstrapConfig = $true + $fetchBootstrapConfig = $false + + if (${createBootstrapConfig}) { + if (-not (ShouldWrite-File ${env:BOOTSTRAP_KUBECONFIG})) { + return + } + New-Item -Force -ItemType file ${env:BOOTSTRAP_KUBECONFIG} | Out-Null + # TODO(mtaufen): is user "kubelet" correct? Other examples use e.g. + # "system:node:$(hostname)". + Set-Content ${env:BOOTSTRAP_KUBECONFIG} ` +'apiVersion: v1 +kind: Config +users: +- name: kubelet + user: + client-certificate: KUBELET_CERT_PATH + client-key: KUBELET_KEY_PATH +clusters: +- name: local + cluster: + server: https://APISERVER_ADDRESS + certificate-authority: CA_CERT_BUNDLE_PATH +contexts: +- context: + cluster: local + user: kubelet + name: service-account-context +current-context: service-account-context'.` + replace('KUBELET_CERT_PATH', ${env:KUBELET_CERT_PATH}).` + replace('KUBELET_KEY_PATH', ${env:KUBELET_KEY_PATH}).` + replace('APISERVER_ADDRESS', ${apiserverAddress}).` + replace('CA_CERT_BUNDLE_PATH', ${env:CA_CERT_BUNDLE_PATH}) + Log-Output ("kubelet bootstrap kubeconfig:`n" + + "$(Get-Content -Raw ${env:BOOTSTRAP_KUBECONFIG})") + } + elseif (${fetchBootstrapConfig}) { + Log_NotImplemented ` + "fetching kubelet bootstrap-kubeconfig file from metadata" + # get-metadata-value "instance/attributes/bootstrap-kubeconfig" > + # /var/lib/kubelet/bootstrap-kubeconfig + Log-Output ("kubelet bootstrap kubeconfig:`n" + + "$(Get-Content -Raw ${env:BOOTSTRAP_KUBECONFIG})") + } + else { + Log_NotImplemented "fetching kubelet kubeconfig file from metadata" + } +} + +# Creates the kube-proxy user kubeconfig file at $env:KUBEPROXY_KUBECONFIG. +# +# Create-NodePki() must be called first. +# +# Required ${kube_env} keys: +# CA_CERT +# KUBE_PROXY_TOKEN +function Create-KubeproxyKubeconfig { + if (-not (ShouldWrite-File ${env:KUBEPROXY_KUBECONFIG})) { + return + } + + New-Item -Force -ItemType file ${env:KUBEPROXY_KUBECONFIG} | Out-Null + + # In configure-helper.sh kubelet kubeconfig uses certificate-authority while + # kubeproxy kubeconfig uses certificate-authority-data, ugh. Does it matter? + # Use just one or the other for consistency? + Set-Content ${env:KUBEPROXY_KUBECONFIG} ` +'apiVersion: v1 +kind: Config +users: +- name: kube-proxy + user: + token: KUBEPROXY_TOKEN +clusters: +- name: local + cluster: + certificate-authority-data: CA_CERT +contexts: +- context: + cluster: local + user: kube-proxy + name: service-account-context +current-context: service-account-context'.` + replace('KUBEPROXY_TOKEN', ${kube_env}['KUBE_PROXY_TOKEN']).` + replace('CA_CERT', ${kube_env}['CA_CERT']) + + Log-Output ("kubeproxy kubeconfig:`n" + + "$(Get-Content -Raw ${env:KUBEPROXY_KUBECONFIG})") +} + +# Returns the IP alias range configured for this GCE instance. +function Get_IpAliasRange { + $url = ("http://${GCE_METADATA_SERVER}/computeMetadata/v1/instance/" + + "network-interfaces/0/ip-aliases/0") + $client = New-Object Net.WebClient + $client.Headers.Add('Metadata-Flavor', 'Google') + return ($client.DownloadString($url)).Trim() +} + +# Retrieves the pod CIDR and sets it in $env:POD_CIDR. +function Set-PodCidr { + while($true) { + $pod_cidr = Get_IpAliasRange + if (-not $?) { + Log-Output ${pod_cIDR} + Log-Output "Retrying Get_IpAliasRange..." + Start-Sleep -sec 1 + continue + } + break + } + + Log-Output "fetched pod CIDR (same as IP alias range): ${pod_cidr}" + Set_MachineEnvironmentVar "POD_CIDR" ${pod_cidr} + Set_CurrentShellEnvironmentVar "POD_CIDR" ${pod_cidr} +} + +# Adds an initial HNS network on the Windows node which forces the creation of +# a virtual switch and the "management" interface that will be used to +# communicate with the rest of the Kubernetes cluster without NAT. +# +# Note that adding the initial HNS network may cause connectivity to the GCE +# metadata server to be lost due to a Windows bug. +# Configure-HostNetworkingService() restores connectivity, look there for +# details. +# +# Download-HelperScripts() must have been called first. +function Add_InitialHnsNetwork { + $INITIAL_HNS_NETWORK = 'External' + + # This comes from + # https://github.com/Microsoft/SDN/blob/master/Kubernetes/flannel/l2bridge/start.ps1#L74 + # (or + # https://github.com/Microsoft/SDN/blob/master/Kubernetes/windows/start-kubelet.ps1#L206). + # + # daschott noted on Slack: "L2bridge networks require an external vSwitch. + # The first network ("External") with hardcoded values in the script is just + # a placeholder to create an external vSwitch. This is purely for convenience + # to be able to remove/modify the actual HNS network ("cbr0") or rejoin the + # nodes without a network blip. Creating a vSwitch takes time, causes network + # blips, and it makes it more likely to hit the issue where flanneld is + # stuck, so we want to do this as rarely as possible." + $hns_network = Get-HnsNetwork | Where-Object Name -eq $INITIAL_HNS_NETWORK + if ($hns_network) { + if ($REDO_STEPS) { + Log-Output ("Warning: initial '$INITIAL_HNS_NETWORK' HNS network " + + "already exists, removing it and recreating it") + $hns_network | Remove-HnsNetwork + $hns_network = $null + } + else { + Log-Output ("Skip: initial '$INITIAL_HNS_NETWORK' HNS network " + + "already exists, not recreating it") + return + } + } + Log-Output ("Creating initial HNS network to force creation of " + + "${MGMT_ADAPTER_NAME} interface") + # Note: RDP connection will hiccup when running this command. + New-HNSNetwork ` + -Type "L2Bridge" ` + -AddressPrefix "192.168.255.0/30" ` + -Gateway "192.168.255.1" ` + -Name $INITIAL_HNS_NETWORK ` + -Verbose +} + +# Configures HNS on the Windows node to enable Kubernetes networking: +# - Creates the "management" interface associated with an initial HNS network. +# - Creates the HNS network $env:KUBE_NETWORK for pod networking. +# - Creates an HNS endpoint for pod networking. +# - Adds necessary routes on the management interface. +# - Verifies that the GCE metadata server connection remains intact. +# +# Prerequisites: +# $env:POD_CIDR is set (by Set-PodCidr). +# Download-HelperScripts() has been called. +function Configure-HostNetworkingService { + Import-Module -Force ${env:K8S_DIR}\hns.psm1 + + Add_InitialHnsNetwork + + # For Windows nodes the pod gateway IP address is the .1 address in the pod + # CIDR for the host, but from inside containers it's the .2 address. + $pod_gateway = ` + ${env:POD_CIDR}.substring(0, ${env:POD_CIDR}.lastIndexOf('.')) + '.1' + $pod_endpoint_gateway = ` + ${env:POD_CIDR}.substring(0, ${env:POD_CIDR}.lastIndexOf('.')) + '.2' + Log-Output ("Setting up Windows node HNS networking: " + + "podCidr = ${env:POD_CIDR}, podGateway = ${pod_gateway}, " + + "podEndpointGateway = ${pod_endpoint_gateway}") + + $hns_network = Get-HnsNetwork | Where-Object Name -eq ${env:KUBE_NETWORK} + if ($hns_network) { + if ($REDO_STEPS) { + Log-Output ("Warning: ${env:KUBE_NETWORK} HNS network already exists, " + + "removing it and recreating it") + $hns_network | Remove-HnsNetwork + $hns_network = $null + } + else { + Log-Output "Skip: ${env:KUBE_NETWORK} HNS network already exists" + } + } + $created_hns_network = $false + if (-not $hns_network) { + # Note: RDP connection will hiccup when running this command. + $hns_network = New-HNSNetwork ` + -Type "L2Bridge" ` + -AddressPrefix ${env:POD_CIDR} ` + -Gateway ${pod_gateway} ` + -Name ${env:KUBE_NETWORK} ` + -Verbose + $created_hns_network = $true + } + + $endpoint_name = "cbr0" + $vnic_name = "vEthernet (${endpoint_name})" + + $hns_endpoint = Get-HnsEndpoint | Where-Object Name -eq $endpoint_name + # Note: we don't expect to ever enter this block currently - while the HNS + # network does seem to persist across reboots, the HNS endpoints do not. + if ($hns_endpoint) { + if ($REDO_STEPS) { + Log-Output ("Warning: HNS endpoint $endpoint_name already exists, " + + "removing it and recreating it") + $hns_endpoint | Remove-HnsEndpoint + $hns_endpoint = $null + } + else { + Log-Output "Skip: HNS endpoint $endpoint_name already exists" + } + } + if (-not $hns_endpoint) { + $hns_endpoint = New-HnsEndpoint ` + -NetworkId ${hns_network}.Id ` + -Name ${endpoint_name} ` + -IPAddress ${pod_endpoint_gateway} ` + -Gateway "0.0.0.0" ` + -Verbose + # TODO(pjh): find out: why is this always CompartmentId 1? + Attach-HnsHostEndpoint ` + -EndpointID ${hns_endpoint}.Id ` + -CompartmentID 1 ` + -Verbose + netsh interface ipv4 set interface "${vnic_name}" forwarding=enabled + } + + Get-HNSPolicyList | Remove-HnsPolicyList + + # Add a route from the management NIC to the pod CIDR. + # + # When a packet from a Kubernetes service backend arrives on the destination + # Windows node, the reverse SNAT will be applied and the source address of + # the packet gets replaced from the pod IP to the service VIP. The packet + # will then leave the VM and return back through hairpinning. + # + # When IP alias is enabled, IP forwarding is disabled for anti-spoofing; + # the packet with the service VIP will get blocked and be lost. With this + # route, the packet will be routed to the pod subnetwork, and not leave the + # VM. + $mgmt_net_adapter = Get_MgmtNetAdapter + New-NetRoute ` + -ErrorAction Ignore ` + -InterfaceAlias ${mgmt_net_adapter}.ifAlias ` + -DestinationPrefix ${env:POD_CIDR} ` + -NextHop "0.0.0.0" ` + -Verbose + + if ($created_hns_network) { + # There is an HNS bug where the route to the GCE metadata server will be + # removed when the HNS network is created: + # https://github.com/Microsoft/hcsshim/issues/299#issuecomment-425491610. + # The behavior here is very unpredictable: the route may only be removed + # after some delay, or it may appear to be removed then you'll add it back + # but then it will be removed once again. So, we first wait a long + # unfortunate amount of time to ensure that things have quiesced, then we + # wait until we're sure the route is really gone before re-adding it again. + Log-Output "Waiting 45 seconds for host network state to quiesce" + Start-Sleep 45 + WaitFor_GceMetadataServerRouteToBeRemoved + Log-Output "Re-adding the GCE metadata server route" + Add_GceMetadataServerRoute + } + Verify_GceMetadataServerRouteIsPresent + + Log-Output "Host network setup complete" +} + +# Downloads the Windows CNI binaries and writes a CNI config file under +# $env:CNI_CONFIG_DIR. +# +# Prerequisites: +# $env:POD_CIDR is set (by Set-PodCidr). +# The "management" interface exists (Configure-HostNetworkingService). +# The HNS network for pod networking has been configured +# (Configure-HostNetworkingService). +# +# Required ${kube_env} keys: +# DNS_SERVER_IP +# DNS_DOMAIN +# CLUSTER_IP_RANGE +# SERVICE_CLUSTER_IP_RANGE +function Configure-CniNetworking { + if ((ShouldWrite-File ${env:CNI_DIR}\win-bridge.exe) -or + (ShouldWrite-File ${env:CNI_DIR}\host-local.exe)) { + Invoke-WebRequest ` + https://github.com/yujuhong/gce-k8s-windows-testing/raw/master/windows-cni-plugins.zip ` + -OutFile ${env:CNI_DIR}\windows-cni-plugins.zip + rm ${env:CNI_DIR}\*.exe + Expand-Archive ${env:CNI_DIR}\windows-cni-plugins.zip ${env:CNI_DIR} + mv ${env:CNI_DIR}\bin\*.exe ${env:CNI_DIR}\ + rmdir ${env:CNI_DIR}\bin + } + if (-not ((Test-Path ${env:CNI_DIR}\win-bridge.exe) -and ` + (Test-Path ${env:CNI_DIR}\host-local.exe))) { + Log-Output ` + "win-bridge.exe and host-local.exe not found in ${env:CNI_DIR}" ` + -Fatal + } + + $l2bridge_conf = "${env:CNI_CONFIG_DIR}\l2bridge.conf" + if (-not (ShouldWrite-File ${l2bridge_conf})) { + return + } + + $mgmt_ip = (Get_MgmtNetAdapter | + Get-NetIPAddress -AddressFamily IPv4).IPAddress + $mgmt_subnet = Get_MgmtSubnet + Log-Output ("using mgmt IP ${mgmt_ip} and mgmt subnet ${mgmt_subnet} for " + + "CNI config") + + # Explanation of the CNI config values: + # CLUSTER_CIDR: the cluster CIDR from which pod CIDRs are allocated. + # POD_CIDR: the pod CIDR assigned to this node. + # MGMT_SUBNET: the subnet on which the Windows pods + kubelet will + # communicate with the rest of the cluster without NAT (i.e. the subnet + # that VM internal IPs are allocated from). + # MGMT_IP: the IP address assigned to the node's primary network interface + # (i.e. the internal IP of the GCE VM). + # SERVICE_CIDR: the CIDR used for kubernetes services. + # DNS_SERVER_IP: the cluster's DNS server IP address. + # DNS_DOMAIN: the cluster's DNS domain, e.g. "cluster.local". + New-Item -Force -ItemType file ${l2bridge_conf} | Out-Null + Set-Content ${l2bridge_conf} ` +'{ + "cniVersion": "0.2.0", + "name": "l2bridge", + "type": "win-bridge", + "capabilities": { + "portMappings": true + }, + "ipam": { + "type": "host-local", + "subnet": "POD_CIDR" + }, + "dns": { + "Nameservers": [ + "DNS_SERVER_IP" + ], + "Search": [ + "DNS_DOMAIN" + ] + }, + "Policies": [ + { + "Name": "EndpointPolicy", + "Value": { + "Type": "OutBoundNAT", + "ExceptionList": [ + "CLUSTER_CIDR", + "SERVICE_CIDR", + "MGMT_SUBNET" + ] + } + }, + { + "Name": "EndpointPolicy", + "Value": { + "Type": "ROUTE", + "DestinationPrefix": "SERVICE_CIDR", + "NeedEncap": true + } + }, + { + "Name": "EndpointPolicy", + "Value": { + "Type": "ROUTE", + "DestinationPrefix": "MGMT_IP/32", + "NeedEncap": true + } + } + ] +}'.replace('POD_CIDR', ${env:POD_CIDR}).` + replace('DNS_SERVER_IP', ${kube_env}['DNS_SERVER_IP']).` + replace('DNS_DOMAIN', ${kube_env}['DNS_DOMAIN']).` + replace('MGMT_IP', ${mgmt_ip}).` + replace('CLUSTER_CIDR', ${kube_env}['CLUSTER_IP_RANGE']).` + replace('SERVICE_CIDR', ${kube_env}['SERVICE_CLUSTER_IP_RANGE']).` + replace('MGMT_SUBNET', ${mgmt_subnet}) + + Log-Output "CNI config:`n$(Get-Content -Raw ${l2bridge_conf})" +} + +# Fetches the kubelet config from the instance metadata and puts it at +# $env:KUBELET_CONFIG. +function Configure-Kubelet { + if (-not (ShouldWrite-File ${env:KUBELET_CONFIG})) { + return + } + + # The Kubelet config is built by build-kubelet-config() in + # cluster/gce/util.sh, and stored in the metadata server under the + # 'kubelet-config' key. + $kubelet_config = Get-InstanceMetadataValue 'kubelet-config' + Set-Content ${env:KUBELET_CONFIG} $kubelet_config + Log-Output "Kubelet config:`n$(Get-Content -Raw ${env:KUBELET_CONFIG})" +} + +# Sets up the kubelet and kube-proxy arguments and starts them as native +# Windows services. +# +# Required ${kube_env} keys: +# KUBELET_ARGS +# KUBERNETES_MASTER_NAME +# CLUSTER_IP_RANGE +function Start-WorkerServices { + $kubelet_args_str = ${kube_env}['KUBELET_ARGS'] + $kubelet_args = $kubelet_args_str.Split(" ") + Log-Output "kubelet_args from metadata: ${kubelet_args}" + + $additional_arg_list = @(` + "--pod-infra-container-image=${INFRA_CONTAINER}" + ) + $kubelet_args = ${kubelet_args} + ${additional_arg_list} + + # kubeproxy is started on Linux nodes using + # kube-manifests/kubernetes/gci-trusty/kube-proxy.manifest, which is + # generated by start-kube-proxy in configure-helper.sh and contains e.g.: + # kube-proxy --master=https://35.239.84.171 + # --kubeconfig=/var/lib/kube-proxy/kubeconfig --cluster-cidr=10.64.0.0/14 + # --resource-container="" --oom-score-adj=-998 --v=2 + # --feature-gates=ExperimentalCriticalPodAnnotation=true + # --iptables-sync-period=1m --iptables-min-sync-period=10s + # --ipvs-sync-period=1m --ipvs-min-sync-period=10s + # And also with various volumeMounts and "securityContext: privileged: true". + $apiserver_address = ${kube_env}['KUBERNETES_MASTER_NAME'] + $kubeproxy_args = @(` + "--v=4", + "--master=https://${apiserver_address}", + "--kubeconfig=${env:KUBEPROXY_KUBECONFIG}", + "--proxy-mode=kernelspace", + "--hostname-override=$(hostname)", + "--cluster-cidr=$(${kube_env}['CLUSTER_IP_RANGE'])", + + # Configure kube-proxy to run as a windows service. + "--windows-service=true", + + # TODO(mtaufen): Configure logging for kube-proxy running as a service. + # I haven't been able to figure out how to direct stdout/stderr into log + # files when configuring it to run via sc.exe, so we just manually + # override logging config here. + "--log-file=${env:LOGS_DIR}\kube-proxy.log", + # klog sets this to true intenrally, so need to override to false + # so we actually log to the file + "--logtostderr=false", + + # Configure flags with explicit empty string values. We can't escape + # double-quotes, because they still break sc.exe after expansion in the + # binPath parameter, and single-quotes get parsed as characters instead + # of string delimiters. + "--resource-container=" + ) + + # TODO(pjh): kubelet is emitting these messages: + # I1023 23:44:11.761915 2468 kubelet.go:274] Adding pod path: + # C:\etc\kubernetes + # I1023 23:44:11.775601 2468 file.go:68] Watching path + # "C:\\etc\\kubernetes" + # ... + # E1023 23:44:31.794327 2468 file.go:182] Can't process manifest file + # "C:\\etc\\kubernetes\\hns.psm1": C:\etc\kubernetes\hns.psm1: couldn't parse + # as pod(yaml: line 10: did not find expected ), please check + # config file. + # + # Figure out how to change the directory that the kubelet monitors for new + # pod manifests. + + # We configure the service to restart on failure, after 10s wait. We reset + # the restart count to 0 each time, so we re-use our restart/10000 action on + # each failure. Note it currently restarts even when explicitly stopped, you + # have to delete the service entry to *really* kill it (e.g. `sc.exe delete + # kubelet`). See issue #72900. + if (Get-Process | Where-Object Name -eq "kubelet") { + Log-Output -Fatal ` + "A kubelet process is already running, don't know what to do" + } + Log-Output "Creating kubelet service" + sc.exe create kubelet binPath= "${env:NODE_DIR}\kubelet.exe ${kubelet_args}" start= demand + sc.exe failure kubelet reset= 0 actions= restart/10000 + Log-Output "Starting kubelet service" + sc.exe start kubelet + + Log-Output "Waiting 10 seconds for kubelet to stabilize" + Start-Sleep 10 + + if (Get-Process | Where-Object Name -eq "kube-proxy") { + Log-Output -Fatal ` + "A kube-proxy process is already running, don't know what to do" + } + Log-Output "Creating kube-proxy service" + sc.exe create kube-proxy binPath= "${env:NODE_DIR}\kube-proxy.exe ${kubeproxy_args}" start= demand + sc.exe failure kube-proxy reset= 0 actions= restart/10000 + Log-Output "Starting kube-proxy service" + sc.exe start kube-proxy + + # F1020 23:08:52.000083 9136 server.go:361] unable to load in-cluster + # configuration, KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT must be + # defined + # TODO(pjh): still getting errors like these in kube-proxy log: + # E1023 04:03:58.143449 4840 reflector.go:205] k8s.io/kubernetes/pkg/client/informers/informers_generated/internalversion/factory.go:129: Failed to list *core.Endpoints: Get https://35.239.84.171/api/v1/endpoints?limit=500&resourceVersion=0: dial tcp 35.239.84.171:443: connectex: A connection attempt failed because the connected party did not properly respond after a period of time, or established connection failed because connected host has failed to respond. + # E1023 04:03:58.150266 4840 reflector.go:205] k8s.io/kubernetes/pkg/client/informers/informers_generated/internalversion/factory.go:129: Failed to list *core.Service: Get https://35.239.84.171/api/v1/services?limit=500&resourceVersion=0: dial tcp 35.239.84.171:443: connectex: A connection attempt failed because the connected party did not properly respond after a period of time, or established connection failed because connected host has failed to respond. + + Log_Todo ("verify that jobs are still running; print more details about " + + "the background jobs.") + Log-Output "$(Get-Service kube* | Out-String)" + Verify_GceMetadataServerRouteIsPresent + Log-Output "Kubernetes components started successfully" +} + +# Runs 'kubectl get nodes'. +# TODO(pjh): run more verification commands. +function Verify-WorkerServices { + Log-Output ("kubectl get nodes:`n" + + "$(& ${env:NODE_DIR}\kubectl.exe get nodes | Out-String)") + Verify_GceMetadataServerRouteIsPresent + Log_Todo "run more verification commands." +} + +# Export all public functions: +Export-ModuleMember -Function *-* diff --git a/cluster/gce/win1803/node-helper.sh b/cluster/gce/win1803/node-helper.sh new file mode 100755 index 0000000000..4ea1b749bf --- /dev/null +++ b/cluster/gce/win1803/node-helper.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash + +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# A library of helper functions and constants for Windows nodes. + +function get-windows-node-instance-metadata-from-file { + local metadata="" + metadata+="cluster-name=${KUBE_TEMP}/cluster-name.txt," + metadata+="kube-env=${KUBE_TEMP}/windows-node-kube-env.yaml," + metadata+="kubelet-config=${KUBE_TEMP}/windows-node-kubelet-config.yaml," + # To get startup script output run "gcloud compute instances + # get-serial-port-output " from the location where you're running + # kube-up. + metadata+="windows-startup-script-ps1=${KUBE_ROOT}/cluster/gce/${WINDOWS_NODE_OS_DISTRIBUTION}/configure.ps1," + metadata+="common-psm1=${KUBE_ROOT}/cluster/gce/${WINDOWS_NODE_OS_DISTRIBUTION}/common.psm1," + metadata+="k8s-node-setup-psm1=${KUBE_ROOT}/cluster/gce/${WINDOWS_NODE_OS_DISTRIBUTION}/k8s-node-setup.psm1," + metadata+="user-profile-psm1=${KUBE_ROOT}/cluster/gce/${WINDOWS_NODE_OS_DISTRIBUTION}/user-profile.psm1," + metadata+="${NODE_EXTRA_METADATA}" + echo "${metadata}" +} + +function get-windows-node-instance-metadata { + local metadata="" + metadata+="k8s-version=${KUBE_VERSION:-v1.13.2}," + metadata+="serial-port-enable=1," + # This enables logging the serial port output. + # https://cloud.google.com/compute/docs/instances/viewing-serial-port-output + metadata+="serial-port-logging-enable=true," + metadata+="win-version=${WINDOWS_NODE_OS_DISTRIBUTION}" + echo "${metadata}" +} + +# $1: template name (required). +# $2: scopes flag. +function create-windows-node-instance-template { + local template_name="$1" + local scopes_flag="$2" + create-node-template "${template_name}" "${scopes_flag}" "$(get-windows-node-instance-metadata-from-file)" "$(get-windows-node-instance-metadata)" "windows" +} diff --git a/cluster/gce/win1803/smoke-test.sh b/cluster/gce/win1803/smoke-test.sh new file mode 100755 index 0000000000..bd128ed925 --- /dev/null +++ b/cluster/gce/win1803/smoke-test.sh @@ -0,0 +1,672 @@ +#!/bin/bash + +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# A small smoke test to run against a just-deployed kube-up cluster with Windows +# nodes. Performs checks such as: +# 1) Verifying that all Windows nodes have status Ready. +# 2) Verifying that no system pods are attempting to run on Windows nodes. +# 3) Verifying pairwise connectivity between most of the following: Linux +# pods, Windows pods, K8s services, and the Internet. +# 4) Verifying that basic DNS resolution works in Windows pods. +# +# This script assumes that it is run from the root of the kubernetes repository +# and that kubectl is present at client/bin/kubectl. +# +# TODOs: +# - Implement the node-to-pod checks. +# - Capture stdout for each command to a file and only print it when the test +# fails. +# - Move copy-pasted code into reusable functions. +# - Continue running all checks after one fails. +# - Test service connectivity by running a test pod with an http server and +# exposing it as a service (rather than curl-ing from existing system +# services that don't serve http requests). +# - Add test retries for transient errors, such as: +# "error: unable to upgrade connection: Authorization error +# (user=kube-apiserver, verb=create, resource=nodes, subresource=proxy)" + +# Override this to use a different kubectl binary. +kubectl=kubectl +linux_deployment_timeout=60 +windows_deployment_timeout=240 +output_file=/tmp/k8s-smoke-test.out + +function check_windows_nodes_are_ready { + # kubectl filtering is the worst. + statuses=$(${kubectl} get nodes -l beta.kubernetes.io/os=windows \ + -o jsonpath='{.items[*].status.conditions[?(@.type=="Ready")].status}') + for status in $statuses; do + if [[ $status == "False" ]]; then + echo "ERROR: some Windows node has status != Ready" + echo "kubectl get nodes -l beta.kubernetes.io/os=windows" + ${kubectl} get nodes -l beta.kubernetes.io/os=windows + exit 1 + fi + done + echo "Verified that all Windows nodes have status Ready" +} + +function check_no_system_pods_on_windows_nodes { + windows_system_pods=$(${kubectl} get pods --namespace kube-system \ + -o wide | grep -E "Pending|windows" | wc -w) + if [[ $windows_system_pods -ne 0 ]]; then + echo "ERROR: there are kube-system pods trying to run on Windows nodes" + echo "kubectl get pods --namespace kube-system -o wide" + ${kubectl} get pods --namespace kube-system -o wide + exit 1 + fi + echo "Verified that all system pods are running on Linux nodes" +} + +linux_webserver_deployment=linux-nginx +linux_webserver_pod_label=nginx + +function deploy_linux_webserver_pod { + echo "Writing example deployment to $linux_webserver_deployment.yaml" + cat < $linux_webserver_deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: $linux_webserver_deployment + labels: + app: $linux_webserver_pod_label +spec: + replicas: 1 + selector: + matchLabels: + app: $linux_webserver_pod_label + template: + metadata: + labels: + app: $linux_webserver_pod_label + spec: + containers: + - name: nginx + image: nginx:1.7.9 + nodeSelector: + beta.kubernetes.io/os: linux +EOF + + if ! ${kubectl} create -f $linux_webserver_deployment.yaml; then + echo "kubectl create -f $linux_webserver_deployment.yaml failed" + exit 1 + fi + + timeout=$linux_deployment_timeout + while [[ $timeout -gt 0 ]]; do + echo "Waiting for Linux $linux_webserver_pod_label pods to become Ready" + statuses=$(${kubectl} get pods -l app=$linux_webserver_pod_label \ + -o jsonpath='{.items[*].status.conditions[?(@.type=="Ready")].status}' \ + | grep "False" | wc -w) + if [[ $statuses -eq 0 ]]; then + break + else + sleep 10 + (( timeout=timeout-10 )) + fi + done + + if [[ $timeout -gt 0 ]]; then + echo "All $linux_webserver_pod_label pods became Ready" + else + echo "ERROR: Not all $linux_webserver_pod_label pods became Ready" + echo "kubectl get pods -l app=$linux_webserver_pod_label" + ${kubectl} get pods -l app=$linux_webserver_pod_label + cleanup_deployments + exit 1 + fi +} + +# Returns the name of an arbitrary Linux webserver pod. +function get_linux_webserver_pod_name { + $kubectl get pods -l app=$linux_webserver_pod_label \ + -o jsonpath='{.items[0].metadata.name}' +} + +# Returns the IP address of an arbitrary Linux webserver pod. +function get_linux_webserver_pod_ip { + $kubectl get pods -l app=$linux_webserver_pod_label \ + -o jsonpath='{.items[0].status.podIP}' +} + +function undeploy_linux_webserver_pod { + ${kubectl} delete deployment $linux_webserver_deployment +} + +linux_command_deployment=linux-ubuntu +linux_command_pod_label=ubuntu + +function deploy_linux_command_pod { + echo "Writing example deployment to $linux_command_deployment.yaml" + cat < $linux_command_deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: $linux_command_deployment + labels: + app: $linux_command_pod_label +spec: + replicas: 1 + selector: + matchLabels: + app: $linux_command_pod_label + template: + metadata: + labels: + app: $linux_command_pod_label + spec: + containers: + - name: ubuntu + image: ubuntu + command: ["sleep", "123456"] + nodeSelector: + beta.kubernetes.io/os: linux +EOF + + if ! ${kubectl} create -f $linux_command_deployment.yaml; then + echo "kubectl create -f $linux_command_deployment.yaml failed" + exit 1 + fi + + timeout=$linux_deployment_timeout + while [[ $timeout -gt 0 ]]; do + echo "Waiting for Linux $linux_command_pod_label pods to become Ready" + statuses=$(${kubectl} get pods -l app=$linux_command_pod_label \ + -o jsonpath='{.items[*].status.conditions[?(@.type=="Ready")].status}' \ + | grep "False" | wc -w) + if [[ $statuses -eq 0 ]]; then + break + else + sleep 10 + (( timeout=timeout-10 )) + fi + done + + if [[ $timeout -gt 0 ]]; then + echo "All $linux_command_pod_label pods became Ready" + else + echo "ERROR: Not all $linux_command_pod_label pods became Ready" + echo "kubectl get pods -l app=$linux_command_pod_label" + ${kubectl} get pods -l app=$linux_command_pod_label + cleanup_deployments + exit 1 + fi +} + +# Returns the name of an arbitrary Linux command pod. +function get_linux_command_pod_name { + $kubectl get pods -l app=$linux_command_pod_label \ + -o jsonpath='{.items[0].metadata.name}' +} + +# Returns the IP address of an arbitrary Linux command pod. +function get_linux_command_pod_ip { + $kubectl get pods -l app=$linux_command_pod_label \ + -o jsonpath='{.items[0].status.podIP}' +} + +# Installs test executables (ping, curl) in the Linux command pod. +# NOTE: this assumes that there is only one Linux "command pod". +# TODO(pjh): fix this. +function prepare_linux_command_pod { + local linux_command_pod + linux_command_pod="$(get_linux_command_pod_name)" + + echo "Installing test utilities in Linux command pod, may take a minute" + $kubectl exec "$linux_command_pod" -- apt-get update > /dev/null + $kubectl exec "$linux_command_pod" -- \ + apt-get install -y iputils-ping curl > /dev/null +} + +function undeploy_linux_command_pod { + ${kubectl} delete deployment $linux_command_deployment +} + +windows_webserver_deployment=windows-nettest +windows_webserver_pod_label=nettest + +function deploy_windows_webserver_pod { + echo "Writing example deployment to $windows_webserver_deployment.yaml" + cat < $windows_webserver_deployment.yaml +# You can run a pod with the e2eteam/nettest:1.0 image (which should listen on +# :8080) and create another pod on a different node (linux would be +# easier) to curl the http server: +# curl http://:8080/read +apiVersion: apps/v1 +kind: Deployment +metadata: + name: $windows_webserver_deployment + labels: + app: $windows_webserver_pod_label +spec: + replicas: 1 + selector: + matchLabels: + app: $windows_webserver_pod_label + template: + metadata: + labels: + app: $windows_webserver_pod_label + spec: + containers: + - name: nettest + image: e2eteam/nettest:1.0 + nodeSelector: + beta.kubernetes.io/os: windows + tolerations: + - effect: NoSchedule + key: node.kubernetes.io/os + operator: Equal + value: windows +EOF + + if ! ${kubectl} create -f $windows_webserver_deployment.yaml; then + echo "kubectl create -f $windows_webserver_deployment.yaml failed" + exit 1 + fi + + timeout=$windows_deployment_timeout + while [[ $timeout -gt 0 ]]; do + echo "Waiting for Windows $windows_webserver_pod_label pods to become Ready" + statuses=$(${kubectl} get pods -l app=$windows_webserver_pod_label \ + -o jsonpath='{.items[*].status.conditions[?(@.type=="Ready")].status}' \ + | grep "False" | wc -w) + if [[ $statuses -eq 0 ]]; then + break + else + sleep 10 + (( timeout=timeout-10 )) + fi + done + + if [[ $timeout -gt 0 ]]; then + echo "All $windows_webserver_pod_label pods became Ready" + else + echo "ERROR: Not all $windows_webserver_pod_label pods became Ready" + echo "kubectl get pods -l app=$windows_webserver_pod_label" + ${kubectl} get pods -l app=$windows_webserver_pod_label + cleanup_deployments + exit 1 + fi +} + +function get_windows_webserver_pod_name { + $kubectl get pods -l app=$windows_webserver_pod_label \ + -o jsonpath='{.items[0].metadata.name}' +} + +function get_windows_webserver_pod_ip { + $kubectl get pods -l app=$windows_webserver_pod_label \ + -o jsonpath='{.items[0].status.podIP}' +} + +function undeploy_windows_webserver_pod { + ${kubectl} delete deployment $windows_webserver_deployment +} + +windows_command_deployment=windows-powershell +windows_command_pod_label=powershell + +function deploy_windows_command_pod { + echo "Writing example deployment to $windows_command_deployment.yaml" + cat < $windows_command_deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: $windows_command_deployment + labels: + app: $windows_command_pod_label +spec: + replicas: 1 + selector: + matchLabels: + app: $windows_command_pod_label + template: + metadata: + labels: + app: $windows_command_pod_label + spec: + containers: + - name: nettest + image: e2eteam/nettest:1.0 + nodeSelector: + beta.kubernetes.io/os: windows + tolerations: + - effect: NoSchedule + key: node.kubernetes.io/os + operator: Equal + value: windows +EOF + + if ! ${kubectl} create -f $windows_command_deployment.yaml; then + echo "kubectl create -f $windows_command_deployment.yaml failed" + exit 1 + fi + + timeout=$windows_deployment_timeout + while [[ $timeout -gt 0 ]]; do + echo "Waiting for Windows $windows_command_pod_label pods to become Ready" + statuses=$(${kubectl} get pods -l app=$windows_command_pod_label \ + -o jsonpath='{.items[*].status.conditions[?(@.type=="Ready")].status}' \ + | grep "False" | wc -w) + if [[ $statuses -eq 0 ]]; then + break + else + sleep 10 + (( timeout=timeout-10 )) + fi + done + + if [[ $timeout -gt 0 ]]; then + echo "All $windows_command_pod_label pods became Ready" + else + echo "ERROR: Not all $windows_command_pod_label pods became Ready" + echo "kubectl get pods -l app=$windows_command_pod_label" + ${kubectl} get pods -l app=$windows_command_pod_label + cleanup_deployments + exit 1 + fi +} + +function get_windows_command_pod_name { + $kubectl get pods -l app=$windows_command_pod_label \ + -o jsonpath='{.items[0].metadata.name}' +} + +function get_windows_command_pod_ip { + $kubectl get pods -l app=$windows_command_pod_label \ + -o jsonpath='{.items[0].status.podIP}' +} + +function undeploy_windows_command_pod { + ${kubectl} delete deployment $windows_command_deployment +} + +function test_linux_node_to_linux_pod { + echo "TODO: ${FUNCNAME[0]}" +} + +function test_linux_node_to_windows_pod { + echo "TODO: ${FUNCNAME[0]}" +} + +function test_linux_pod_to_linux_pod { + echo "TEST: ${FUNCNAME[0]}" + local linux_command_pod + linux_command_pod="$(get_linux_command_pod_name)" + local linux_webserver_pod_ip + linux_webserver_pod_ip="$(get_linux_webserver_pod_ip)" + + if ! $kubectl exec "$linux_command_pod" -- curl -m 20 \ + "http://$linux_webserver_pod_ip" &> $output_file; then + cleanup_deployments + echo "Failing output: $(cat $output_file)" + echo "FAILED: ${FUNCNAME[0]}" + exit 1 + fi +} + +# TODO(pjh): this test flakily fails on brand-new clusters, not sure why. +# % Total % Received % Xferd Average Speed Time Time Time Current +# Dload Upload Total Spent Left Speed +# 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 +# curl: (6) Could not resolve host: +# command terminated with exit code 6 +function test_linux_pod_to_windows_pod { + echo "TEST: ${FUNCNAME[0]}" + local linux_command_pod + linux_command_pod="$(get_linux_command_pod_name)" + local windows_webserver_pod_ip + windows_webserver_pod_ip="$(get_windows_webserver_pod_ip)" + + if ! $kubectl exec "$linux_command_pod" -- curl -m 20 \ + "http://$windows_webserver_pod_ip:8080/read" &> $output_file; then + cleanup_deployments + echo "Failing output: $(cat $output_file)" + echo "FAILED: ${FUNCNAME[0]}" + echo "This test seems to be flaky. TODO(pjh): investigate." + exit 1 + fi +} + +function test_linux_pod_to_internet { + echo "TEST: ${FUNCNAME[0]}" + local linux_command_pod + linux_command_pod="$(get_linux_command_pod_name)" + local internet_ip="8.8.8.8" # Google DNS + + # This is expected to return 404 (not found). + if ! $kubectl exec "$linux_command_pod" -- curl -m 20 \ + "http://$internet_ip" > $output_file; then + cleanup_deployments + echo "Failing output: $(cat $output_file)" + echo "FAILED: ${FUNCNAME[0]}" + exit 1 + fi +} + +function test_linux_pod_to_k8s_service { + echo "TEST: ${FUNCNAME[0]}" + local linux_command_pod + linux_command_pod="$(get_linux_command_pod_name)" + local service="heapster" + local service_ip + service_ip=$($kubectl get service --namespace kube-system $service \ + -o jsonpath='{.spec.clusterIP}') + local service_port + service_port=$($kubectl get service --namespace kube-system $service \ + -o jsonpath='{.spec.ports[?(@.protocol=="TCP")].port}') + echo "curl-ing $service address from Linux pod: $service_ip:$service_port" + + # curl-ing the heapster service results in an expected 404 response code. The + # curl command does not set a failure return code in this case. + if ! $kubectl exec "$linux_command_pod" -- \ + curl -m 20 "http://$service_ip:$service_port" &> $output_file; then + cleanup_deployments + echo "Failing output: $(cat $output_file)" + echo "FAILED: ${FUNCNAME[0]}" + exit 1 + fi +} + +function test_windows_node_to_linux_pod { + echo "TODO: ${FUNCNAME[0]}" +} + +function test_windows_node_to_windows_pod { + echo "TODO: ${FUNCNAME[0]}" +} + +# TODO(pjh): this test failed for me once with +# error: unable to upgrade connection: container not found ("nettest") +# Maybe the container crashed for some reason? Investigate if it happens more. +# +# TODO(pjh): another one-time failure: +# error: unable to upgrade connection: Authorization error +# (user=kube-apiserver, verb=create, resource=nodes, subresource=proxy) +function test_windows_pod_to_linux_pod { + echo "TEST: ${FUNCNAME[0]}" + local windows_command_pod + windows_command_pod="$(get_windows_command_pod_name)" + local linux_webserver_pod_ip + linux_webserver_pod_ip="$(get_linux_webserver_pod_ip)" + + if ! $kubectl exec "$windows_command_pod" -- powershell.exe \ + "curl -UseBasicParsing http://$linux_webserver_pod_ip" > \ + $output_file; then + cleanup_deployments + echo "Failing output: $(cat $output_file)" + echo "FAILED: ${FUNCNAME[0]}" + exit 1 + fi +} + +function test_windows_pod_to_windows_pod { + echo "TEST: ${FUNCNAME[0]}" + local windows_command_pod + windows_command_pod="$(get_windows_command_pod_name)" + local windows_webserver_pod_ip + windows_webserver_pod_ip="$(get_windows_webserver_pod_ip)" + + if ! $kubectl exec "$windows_command_pod" -- powershell.exe \ + "curl -UseBasicParsing http://$windows_webserver_pod_ip:8080/read" \ + > $output_file; then + cleanup_deployments + echo "Failing output: $(cat $output_file)" + echo "FAILED: ${FUNCNAME[0]}" + exit 1 + fi +} + +function test_windows_pod_to_internet { + echo "TEST: ${FUNCNAME[0]}" + local windows_command_pod + windows_command_pod="$(get_windows_command_pod_name)" + local internet_ip="8.8.8.8" + + # This snippet tests Internet connectivity without depending on DNS by + # attempting to curl Google's well-known DNS IP, 8.8.8.8. On success we expect + # to get back a 404 status code; on failure the response object will have a + # status code of 0 or some other HTTP code. + if ! $kubectl exec "$windows_command_pod" -- powershell.exe \ + "\$response = try { \` + (curl -UseBasicParsing http://$internet_ip \` + -ErrorAction Stop).BaseResponse \` + } catch [System.Net.WebException] { \` + \$_.Exception.Response \` + }; \` + \$statusCodeInt = [int]\$response.StatusCode; \` + if (\$statusCodeInt -eq 404) { \` + exit 0 \` + } else { \` + Write-Host \"curl $internet_ip got unexpected status code \$statusCodeInt\" + exit 1 \` + }" > $output_file; then + cleanup_deployments + echo "Failing output: $(cat $output_file)" + echo "FAILED: ${FUNCNAME[0]}" + exit 1 + fi +} + +function test_windows_pod_to_k8s_service { + echo "TEST: ${FUNCNAME[0]}" + local windows_command_pod + windows_command_pod="$(get_windows_command_pod_name)" + local service="heapster" + local service_ip + service_ip=$($kubectl get service --namespace kube-system $service \ + -o jsonpath='{.spec.clusterIP}') + local service_port + service_port=$($kubectl get service --namespace kube-system $service \ + -o jsonpath='{.spec.ports[?(@.protocol=="TCP")].port}') + local service_address="$service_ip:$service_port" + + echo "curl-ing $service address from Windows pod: $service_address" + # Performing a web request to the heapster service results in an expected 404 + # response; this code snippet filters out the expected 404 from other status + # codes that indicate failure. + if ! $kubectl exec "$windows_command_pod" -- powershell.exe \ + "\$response = try { \` + (curl -UseBasicParsing http://$service_address \` + -ErrorAction Stop).BaseResponse \` + } catch [System.Net.WebException] { \` + \$_.Exception.Response \` + }; \` + \$statusCodeInt = [int]\$response.StatusCode; \` + if (\$statusCodeInt -eq 404) { \` + exit 0 \` + } else { \` + Write-Host \"curl $service_address got unexpected status code \$statusCodeInt\" + exit 1 \` + }" > $output_file; then + cleanup_deployments + echo "Failing output: $(cat $output_file)" + echo "FAILED: ${FUNCNAME[0]}" + exit 1 + fi +} + +function test_kube_dns_in_windows_pod { + echo "TEST: ${FUNCNAME[0]}" + local windows_command_pod + windows_command_pod="$(get_windows_command_pod_name)" + local service="kube-dns" + local service_ip + service_ip=$($kubectl get service --namespace kube-system $service \ + -o jsonpath='{.spec.clusterIP}') + + if ! $kubectl exec "$windows_command_pod" -- powershell.exe \ + "Resolve-DnsName www.bing.com -server $service_ip" > $output_file; then + cleanup_deployments + echo "Failing output: $(cat $output_file)" + echo "FAILED: ${FUNCNAME[0]}" + exit 1 + fi +} + +function test_dns_just_works_in_windows_pod { + echo "TEST: ${FUNCNAME[0]}" + local windows_command_pod + windows_command_pod="$(get_windows_command_pod_name)" + + if ! $kubectl exec "$windows_command_pod" -- powershell.exe \ + "curl -UseBasicParsing http://www.bing.com" > $output_file; then + cleanup_deployments + echo "Failing output: $(cat $output_file)" + echo "FAILED: ${FUNCNAME[0]}" + exit 1 + fi +} + +function cleanup_deployments { + undeploy_linux_webserver_pod + undeploy_linux_command_pod + undeploy_windows_webserver_pod + undeploy_windows_command_pod +} + +check_windows_nodes_are_ready +check_no_system_pods_on_windows_nodes + +deploy_linux_webserver_pod +deploy_linux_command_pod +deploy_windows_webserver_pod +deploy_windows_command_pod +prepare_linux_command_pod +echo "" + +test_linux_node_to_linux_pod +test_linux_node_to_windows_pod +test_linux_pod_to_linux_pod +test_linux_pod_to_windows_pod +test_linux_pod_to_k8s_service + +# Note: test_windows_node_to_k8s_service is not supported at this time. +# https://docs.microsoft.com/en-us/virtualization/windowscontainers/kubernetes/common-problems#my-windows-node-cannot-access-my-services-using-the-service-ip +test_windows_node_to_linux_pod +test_windows_node_to_windows_pod +test_windows_pod_to_linux_pod +test_windows_pod_to_windows_pod +test_windows_pod_to_internet +test_windows_pod_to_k8s_service +test_kube_dns_in_windows_pod +test_dns_just_works_in_windows_pod +echo "" + +cleanup_deployments +echo "All tests passed!" +exit 0 diff --git a/cluster/gce/win1803/user-profile.psm1 b/cluster/gce/win1803/user-profile.psm1 new file mode 100644 index 0000000000..62abd57811 --- /dev/null +++ b/cluster/gce/win1803/user-profile.psm1 @@ -0,0 +1,337 @@ + +<# +.Synopsis + Rough PS functions to create new user profiles +.DESCRIPTION + Call the Create-NewProfile function directly to create a new profile +.EXAMPLE + Create-NewProfile -Username 'testUser1' -Password 'testUser1' +.NOTES + Created by: Josh Rickard (@MS_dministrator) and Thom Schumacher (@driberif) + Forked by: @crshnbrn66, then @pjh (2018-11-08). See + https://gist.github.com/pjh/9753cd14400f4e3d4567f4553ba75f1d/revisions + Date: 24MAR2017 + Location: https://gist.github.com/crshnbrn66/7e81bf20408c05ddb2b4fdf4498477d8 + + Contact: https://github.com/MSAdministrator + MSAdministrator.com + https://github.com/crshnbrn66 + powershellposse.com +#> + + +#Function to create the new local user first +function New-LocalUser +{ + [CmdletBinding()] + [Alias()] + [OutputType([int])] + Param + ( + # Param1 help description + [Parameter(Mandatory=$true, + ValueFromPipelineByPropertyName=$true, + Position=0)] + $userName, + # Param2 help description + [string] + $password + ) + + $system = [ADSI]"WinNT://$env:COMPUTERNAME"; + $user = $system.Create("user",$userName); + $user.SetPassword($password); + $user.SetInfo(); + + $flag=$user.UserFlags.value -bor 0x10000; + $user.put("userflags",$flag); + $user.SetInfo(); + + $group = [ADSI]("WinNT://$env:COMPUTERNAME/Users"); + $group.PSBase.Invoke("Add", $user.PSBase.Path); +} + +#function to register a native method +function Register-NativeMethod +{ + [CmdletBinding()] + [Alias()] + [OutputType([int])] + Param + ( + # Param1 help description + [Parameter(Mandatory=$true, + ValueFromPipelineByPropertyName=$true, + Position=0)] + [string]$dll, + + # Param2 help description + [Parameter(Mandatory=$true, + ValueFromPipelineByPropertyName=$true, + Position=1)] + [string] + $methodSignature + ) + + $script:nativeMethods += [PSCustomObject]@{ Dll = $dll; Signature = $methodSignature; } +} +function Get-Win32LastError +{ + [CmdletBinding()] + [Alias()] + [OutputType([int])] + Param($typeName = 'LastError') + if (-not ([System.Management.Automation.PSTypeName]$typeName).Type) + { + $lasterrorCode = $script:lasterror | ForEach-Object{ + '[DllImport("kernel32.dll", SetLastError = true)] + public static extern uint GetLastError();' + } + Add-Type @" + using System; + using System.Text; + using System.Runtime.InteropServices; + public static class $typeName { + $lasterrorCode + } +"@ + } +} +#function to add native method +function Add-NativeMethods +{ + [CmdletBinding()] + [Alias()] + [OutputType([int])] + Param($typeName = 'NativeMethods') + + $nativeMethodsCode = $script:nativeMethods | ForEach-Object { " + [DllImport(`"$($_.Dll)`")] + public static extern $($_.Signature); + " } + + Add-Type @" + using System; + using System.Text; + using System.Runtime.InteropServices; + public static class $typeName { + $nativeMethodsCode + } +"@ +} + +#Main function to create the new user profile +function Create-NewProfile { + + [CmdletBinding()] + [Alias()] + [OutputType([int])] + Param + ( + # Param1 help description + [Parameter(Mandatory=$true, + ValueFromPipelineByPropertyName=$true, + Position=0)] + [string]$UserName, + + # Param2 help description + [Parameter(Mandatory=$true, + ValueFromPipelineByPropertyName=$true, + Position=1)] + [string] + $Password + ) + + Write-Verbose "Creating local user $Username"; + + try + { + New-LocalUser -username $UserName -password $Password; + } + catch + { + Write-Error $_.Exception.Message; + break; + } + $methodName = 'UserEnvCP' + $script:nativeMethods = @(); + + if (-not ([System.Management.Automation.PSTypeName]$MethodName).Type) + { + Register-NativeMethod "userenv.dll" "int CreateProfile([MarshalAs(UnmanagedType.LPWStr)] string pszUserSid,` + [MarshalAs(UnmanagedType.LPWStr)] string pszUserName,` + [Out][MarshalAs(UnmanagedType.LPWStr)] StringBuilder pszProfilePath, uint cchProfilePath)"; + + Add-NativeMethods -typeName $MethodName; + } + + $localUser = New-Object System.Security.Principal.NTAccount("$UserName"); + $userSID = $localUser.Translate([System.Security.Principal.SecurityIdentifier]); + $sb = new-object System.Text.StringBuilder(260); + $pathLen = $sb.Capacity; + + Write-Verbose "Creating user profile for $Username"; + + try + { + [UserEnvCP]::CreateProfile($userSID.Value, $Username, $sb, $pathLen) | Out-Null; + } + catch + { + Write-Error $_.Exception.Message; + break; + } +} + +function New-ProfileFromSID { + + [CmdletBinding()] + [Alias()] + [OutputType([int])] + Param + ( + # Param1 help description + [Parameter(Mandatory=$true, + ValueFromPipelineByPropertyName=$true, + Position=0)] + [string]$UserName, + [string]$domain = 'PHCORP' + ) + $methodname = 'UserEnvCP2' + $script:nativeMethods = @(); + + if (-not ([System.Management.Automation.PSTypeName]$methodname).Type) + { + Register-NativeMethod "userenv.dll" "int CreateProfile([MarshalAs(UnmanagedType.LPWStr)] string pszUserSid,` + [MarshalAs(UnmanagedType.LPWStr)] string pszUserName,` + [Out][MarshalAs(UnmanagedType.LPWStr)] StringBuilder pszProfilePath, uint cchProfilePath)"; + + Add-NativeMethods -typeName $methodname; + } + + $sb = new-object System.Text.StringBuilder(260); + $pathLen = $sb.Capacity; + + Write-Verbose "Creating user profile for $Username"; + #$SID= ((get-aduser -id $UserName -ErrorAction Stop).sid.value) + if($domain) + { + $objUser = New-Object System.Security.Principal.NTAccount($domain, $UserName) + $strSID = $objUser.Translate([System.Security.Principal.SecurityIdentifier]) + $SID = $strSID.Value + } + else + { + $objUser = New-Object System.Security.Principal.NTAccount($UserName) + $strSID = $objUser.Translate([System.Security.Principal.SecurityIdentifier]) + $SID = $strSID.Value + } + Write-Verbose "$UserName SID: $SID" + try + { + $result = [UserEnvCP2]::CreateProfile($SID, $Username, $sb, $pathLen) + if($result -eq '-2147024713') + { + $status = "$userName already exists" + write-verbose "$username Creation Result: $result" + } + elseif($result -eq '-2147024809') + { + $staus = "$username Not Found" + write-verbose "$username creation result: $result" + } + elseif($result -eq 0) + { + $status = "$username Profile has been created" + write-verbose "$username Creation Result: $result" + } + else + { + $status = "$UserName unknown return result: $result" + } + } + catch + { + Write-Error $_.Exception.Message; + break; + } + $status +} +Function Remove-Profile { + + [CmdletBinding()] + [Alias()] + [OutputType([int])] + Param + ( + # Param1 help description + [Parameter(Mandatory=$true, + ValueFromPipelineByPropertyName=$true, + Position=0)] + [string]$UserName, + [string]$ProfilePath, + [string]$domain = 'PHCORP' + ) + $methodname = 'userenvDP' + $script:nativeMethods = @(); + + if (-not ([System.Management.Automation.PSTypeName]"$methodname.profile").Type) + { + add-type @" +using System.Runtime.InteropServices; + +namespace $typename +{ + public static class UserEnv + { + [DllImport("userenv.dll", CharSet = CharSet.Unicode, ExactSpelling = false, SetLastError = true)] + public static extern bool DeleteProfile(string sidString, string profilePath, string computerName); + + [DllImport("kernel32.dll")] + public static extern uint GetLastError(); + } + + public static class Profile + { + public static uint Delete(string sidString) + { //Profile path and computer name are optional + if (!UserEnv.DeleteProfile(sidString, null, null)) + { + return UserEnv.GetLastError(); + } + + return 0; + } + } +} +"@ + } + + #$SID= ((get-aduser -id $UserName -ErrorAction Stop).sid.value) + if($domain) + { + $objUser = New-Object System.Security.Principal.NTAccount($domain, $UserName) + $strSID = $objUser.Translate([System.Security.Principal.SecurityIdentifier]) + $SID = $strSID.Value + } + else + { + $objUser = New-Object System.Security.Principal.NTAccount($UserName) + $strSID = $objUser.Translate([System.Security.Principal.SecurityIdentifier]) + $SID = $strSID.Value + } + Write-Verbose "$UserName SID: $SID" + try + { + #http://stackoverflow.com/questions/31949002/c-sharp-delete-user-profile + $result = [userenvDP.Profile]::Delete($SID) + } + catch + { + Write-Error $_.Exception.Message; + break; + } + $LastError +} + +Export-ModuleMember Create-NewProfile diff --git a/cluster/validate-cluster.sh b/cluster/validate-cluster.sh index 857c1c948a..3f7cd3d29c 100755 --- a/cluster/validate-cluster.sh +++ b/cluster/validate-cluster.sh @@ -50,9 +50,8 @@ function kubectl_retry() { ALLOWED_NOTREADY_NODES="${ALLOWED_NOTREADY_NODES:-0}" CLUSTER_READY_ADDITIONAL_TIME_SECONDS="${CLUSTER_READY_ADDITIONAL_TIME_SECONDS:-30}" -EXPECTED_NUM_NODES="${NUM_NODES}" - if [[ "${KUBERNETES_PROVIDER:-}" == "gce" ]]; then + EXPECTED_NUM_NODES="$(get-num-nodes)" echo "Validating gce cluster, MULTIZONE=${MULTIZONE:-}" # In multizone mode we need to add instances for all nodes in the region. if [[ "${MULTIZONE:-}" == "true" ]]; then @@ -60,6 +59,8 @@ if [[ "${KUBERNETES_PROVIDER:-}" == "gce" ]]; then --filter="name ~ '${NODE_INSTANCE_PREFIX}.*' AND zone:($(gcloud -q compute zones list --project="${PROJECT}" --filter=region=${REGION} --format=csv[no-heading]\(name\) | tr "\n" "," | sed "s/,$//"))" | wc -l) echo "Computing number of nodes, NODE_INSTANCE_PREFIX=${NODE_INSTANCE_PREFIX}, REGION=${REGION}, EXPECTED_NUM_NODES=${EXPECTED_NUM_NODES}" fi +else + EXPECTED_NUM_NODES="${NUM_NODES}" fi if [[ "${REGISTER_MASTER_KUBELET:-}" == "true" ]]; then