diff --git a/cluster/gce/config-default.sh b/cluster/gce/config-default.sh index 4e4a08d907..0762e57b3b 100755 --- a/cluster/gce/config-default.sh +++ b/cluster/gce/config-default.sh @@ -112,8 +112,8 @@ ADMISSION_CONTROL=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota,Pe # Optional: if set to true kube-up will automatically check for existing resources and clean them up. KUBE_UP_AUTOMATIC_CLEANUP=${KUBE_UP_AUTOMATIC_CLEANUP:-false} -# OpenContrail networking plugin specific settings -NETWORK_PROVIDER="${NETWORK_PROVIDER:-none}" # opencontrail, flannel +# Networking plugin specific settings. +NETWORK_PROVIDER="${NETWORK_PROVIDER:-none}" # opencontrail, flannel, kubenet OPENCONTRAIL_TAG="${OPENCONTRAIL_TAG:-R2.20}" OPENCONTRAIL_KUBERNETES_TAG="${OPENCONTRAIL_KUBERNETES_TAG:-master}" OPENCONTRAIL_PUBLIC_SUBNET="${OPENCONTRAIL_PUBLIC_SUBNET:-10.1.0.0/16}" diff --git a/cluster/saltbase/salt/cni/init.sls b/cluster/saltbase/salt/cni/init.sls new file mode 100644 index 0000000000..544f33f522 --- /dev/null +++ b/cluster/saltbase/salt/cni/init.sls @@ -0,0 +1,20 @@ +/opt/cni: + file.directory: + - user: root + - group: root + - mode: 755 + - makedirs: True + +# These are all available CNI network plugins. +cni-tar: + archive: + - extracted + - user: root + - name: /opt/cni + - makedirs: True + - source: https://storage.googleapis.com/kubernetes-release/network-plugins/cni-09214926.tar.gz + - tar_options: v + - source_hash: md5=58f8631f912dd88be6a0920775db7274 + - archive_format: tar + - if_missing: /opt/cni/bin + diff --git a/cluster/saltbase/salt/docker/docker-defaults b/cluster/saltbase/salt/docker/docker-defaults index 9ea5a12141..5b97c16d6a 100644 --- a/cluster/saltbase/salt/docker/docker-defaults +++ b/cluster/saltbase/salt/docker/docker-defaults @@ -3,5 +3,9 @@ {% if pillar.get('e2e_storage_test_environment', '').lower() == 'true' -%} {% set e2e_opts = '-s devicemapper' -%} {% endif -%} -DOCKER_OPTS="{{grains_opts}} {{e2e_opts}} --bridge=cbr0 --iptables=false --ip-masq=false --log-level=warn" +{% set bridge_opts = "--bridge=cbr0" %} +{% if pillar.get('network_provider', '').lower() == 'kubenet' %} + {% set bridge_opts = "" %} +{% endif -%} +DOCKER_OPTS="{{grains_opts}} {{e2e_opts}} {{bridge_opts}} --iptables=false --ip-masq=false --log-level=warn" DOCKER_NOFILE=1000000 diff --git a/cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest b/cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest index e19b4d12a6..39444e9100 100644 --- a/cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest +++ b/cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest @@ -14,6 +14,8 @@ # This is expected to be a short-term compromise. {% if pillar.get('network_provider', '').lower() == 'flannel' %} {% set allocate_node_cidrs = "--allocate-node-cidrs=false" -%} +{% elif pillar.get('network_provider', '').lower() == 'kubenet' %} + {% set allocate_node_cidrs = "--allocate-node-cidrs=true" -%} {% elif pillar['allocate_node_cidrs'] is defined -%} {% set allocate_node_cidrs = "--allocate-node-cidrs=" + pillar['allocate_node_cidrs'] -%} {% endif -%} diff --git a/cluster/saltbase/salt/kubelet/default b/cluster/saltbase/salt/kubelet/default index 396d71df65..4b051edc43 100644 --- a/cluster/saltbase/salt/kubelet/default +++ b/cluster/saltbase/salt/kubelet/default @@ -26,6 +26,7 @@ {% set debugging_handlers = "--enable-debugging-handlers=true" -%} +{% set reconcile_cidr_args = "" -%} {% if grains['roles'][0] == 'kubernetes-master' -%} {% if grains.cloud in ['aws', 'gce', 'vagrant', 'vsphere'] -%} @@ -33,7 +34,8 @@ # running on the master. {% if grains.kubelet_api_servers is defined -%} {% set api_servers_with_port = "--api-servers=https://" + grains.kubelet_api_servers -%} - {% set master_kubelet_args = master_kubelet_args + "--register-schedulable=false --reconcile-cidr=false" -%} + {% set master_kubelet_args = master_kubelet_args + "--register-schedulable=false" -%} + {% set reconcile_cidr_args = "--reconcile-cidr=false" -%} {% else -%} {% set api_servers_with_port = "" -%} {% endif -%} @@ -116,8 +118,13 @@ {% endif -%} {% set pod_cidr = "" %} -{% if grains['roles'][0] == 'kubernetes-master' and grains.get('cbr-cidr') %} - {% set pod_cidr = "--pod-cidr=" + grains['cbr-cidr'] %} +{% if grains['roles'][0] == 'kubernetes-master' %} + {% if grains.get('cbr-cidr') %} + {% set pod_cidr = "--pod-cidr=" + grains['cbr-cidr'] %} + {% elif api_servers_with_port == '' and pillar.get('network_provider', '').lower() == 'kubenet' %} + # Kubelet standalone mode needs a PodCIDR since there is no controller-manager + {% set pod_cidr = "--pod-cidr=10.76.0.0/16" %} + {% endif -%} {% endif %} {% set cpu_cfs_quota = "" %} @@ -133,6 +140,11 @@ {% set network_plugin = "" -%} {% if pillar.get('network_provider', '').lower() == 'opencontrail' %} {% set network_plugin = "--network-plugin=opencontrail" %} +{% elif pillar.get('network_provider', '').lower() == 'kubenet' %} + {% set network_plugin = "--network-plugin=kubenet" -%} + {% if reconcile_cidr_args == '' -%} + {% set reconcile_cidr_args = "--reconcile-cidr=true" -%} + {% endif -%} {% endif -%} {% set kubelet_port = "" -%} @@ -146,4 +158,4 @@ {% endif -%} # test_args has to be kept at the end, so they'll overwrite any prior configuration -DAEMON_ARGS="{{daemon_args}} {{api_servers_with_port}} {{debugging_handlers}} {{hostname_override}} {{cloud_provider}} {{config}} {{manifest_url}} --allow-privileged={{pillar['allow_privileged']}} {{log_level}} {{cluster_dns}} {{cluster_domain}} {{docker_root}} {{kubelet_root}} {{configure_cbr0}} {{non_masquerade_cidr}} {{cgroup_root}} {{system_container}} {{pod_cidr}} {{ master_kubelet_args }} {{cpu_cfs_quota}} {{network_plugin}} {{kubelet_port}} {{experimental_flannel_overlay}} {{test_args}}" +DAEMON_ARGS="{{daemon_args}} {{api_servers_with_port}} {{debugging_handlers}} {{hostname_override}} {{cloud_provider}} {{config}} {{manifest_url}} --allow-privileged={{pillar['allow_privileged']}} {{log_level}} {{cluster_dns}} {{cluster_domain}} {{docker_root}} {{kubelet_root}} {{configure_cbr0}} {{non_masquerade_cidr}} {{cgroup_root}} {{system_container}} {{pod_cidr}} {{ master_kubelet_args }} {{cpu_cfs_quota}} {{network_plugin}} {{kubelet_port}} {{experimental_flannel_overlay}} {{ reconcile_cidr_args }} {{test_args}}" diff --git a/cluster/saltbase/salt/top.sls b/cluster/saltbase/salt/top.sls index f3d32f779b..e96c6390ce 100644 --- a/cluster/saltbase/salt/top.sls +++ b/cluster/saltbase/salt/top.sls @@ -15,6 +15,8 @@ base: - docker {% if pillar.get('network_provider', '').lower() == 'flannel' %} - flannel +{% elif pillar.get('network_provider', '').lower() == 'kubenet' %} + - cni {% endif %} - helpers - cadvisor @@ -46,6 +48,8 @@ base: {% if pillar.get('network_provider', '').lower() == 'flannel' %} - flannel-server - flannel +{% elif pillar.get('network_provider', '').lower() == 'kubenet' %} + - cni {% endif %} - kube-apiserver - kube-controller-manager diff --git a/cluster/vagrant/config-default.sh b/cluster/vagrant/config-default.sh index fd420c368b..3172a8ace4 100755 --- a/cluster/vagrant/config-default.sh +++ b/cluster/vagrant/config-default.sh @@ -97,8 +97,12 @@ octets=($(echo "$SERVICE_CLUSTER_IP_RANGE" | sed -e 's|/.*||' -e 's/\./ /g')) service_ip=$(echo "${octets[*]}" | sed 's/ /./g') MASTER_EXTRA_SANS="IP:${service_ip},DNS:kubernetes,DNS:kubernetes.default,DNS:kubernetes.default.svc,DNS:kubernetes.default.svc.${DNS_DOMAIN},DNS:${MASTER_NAME}" +NETWORK_PROVIDER="${NETWORK_PROVIDER:-none}" # opencontrail, kubenet, etc +if [ "${NETWORK_PROVIDER}" == "kubenet" ]; then + CLUSTER_IP_RANGE="${CONTAINER_SUBNET}" +fi + # OpenContrail networking plugin specific settings -NETWORK_PROVIDER="${NETWORK_PROVIDER:-none}" # opencontrail OPENCONTRAIL_TAG="${OPENCONTRAIL_TAG:-R2.20}" OPENCONTRAIL_KUBERNETES_TAG="${OPENCONTRAIL_KUBERNETES_TAG:-master}" OPENCONTRAIL_PUBLIC_SUBNET="${OPENCONTRAIL_PUBLIC_SUBNET:-10.1.0.0/16}" diff --git a/cluster/vagrant/provision-master.sh b/cluster/vagrant/provision-master.sh index da6c611d43..46e1e4db9c 100755 --- a/cluster/vagrant/provision-master.sh +++ b/cluster/vagrant/provision-master.sh @@ -57,7 +57,9 @@ echo "127.0.0.1 localhost" >> /etc/hosts # enables cmds like 'kubectl get pods' echo "$MASTER_IP $MASTER_NAME" >> /etc/hosts # Configure the master network -provision-network-master +if [ "${NETWORK_PROVIDER}" != "kubenet" ]; then + provision-network-master +fi write-salt-config kubernetes-master diff --git a/cluster/vagrant/provision-node.sh b/cluster/vagrant/provision-node.sh index 2fb64dd313..544224a67e 100755 --- a/cluster/vagrant/provision-node.sh +++ b/cluster/vagrant/provision-node.sh @@ -53,7 +53,9 @@ for (( i=0; i<${#NODE_NAMES[@]}; i++)); do done # Configure network -provision-network-node +if [ "${NETWORK_PROVIDER}" != "kubenet" ]; then + provision-network-node +fi write-salt-config kubernetes-pool diff --git a/cluster/vagrant/provision-utils.sh b/cluster/vagrant/provision-utils.sh index 286d86221d..f095d71cfc 100755 --- a/cluster/vagrant/provision-utils.sh +++ b/cluster/vagrant/provision-utils.sh @@ -38,6 +38,7 @@ instance_prefix: '$(echo "$INSTANCE_PREFIX" | sed -e "s/'/''/g")' admission_control: '$(echo "$ADMISSION_CONTROL" | sed -e "s/'/''/g")' enable_cpu_cfs_quota: '$(echo "$ENABLE_CPU_CFS_QUOTA" | sed -e "s/'/''/g")' network_provider: '$(echo "$NETWORK_PROVIDER" | sed -e "s/'/''/g")' +cluster_cidr: '$(echo "$CLUSTER_IP_RANGE" | sed -e "s/'/''/g")' opencontrail_tag: '$(echo "$OPENCONTRAIL_TAG" | sed -e "s/'/''/g")' opencontrail_kubernetes_tag: '$(echo "$OPENCONTRAIL_KUBERNETES_TAG" | sed -e "s/'/''/g")' opencontrail_public_subnet: '$(echo "$OPENCONTRAIL_PUBLIC_SUBNET" | sed -e "s/'/''/g")' diff --git a/cluster/vagrant/util.sh b/cluster/vagrant/util.sh index 875fa2a2cd..0f90b1e2e0 100755 --- a/cluster/vagrant/util.sh +++ b/cluster/vagrant/util.sh @@ -153,6 +153,7 @@ function echo-kube-env() { echo "NODE_NAMES=(${NODE_NAMES[@]})" echo "NODE_IPS=(${NODE_IPS[@]})" echo "CONTAINER_SUBNET='${CONTAINER_SUBNET}'" + echo "CLUSTER_IP_RANGE='${CLUSTER_IP_RANGE}'" echo "MASTER_CONTAINER_SUBNET='${MASTER_CONTAINER_SUBNET}'" echo "NODE_CONTAINER_NETMASKS='${NODE_CONTAINER_NETMASKS[@]}'" echo "NODE_CONTAINER_SUBNETS=(${NODE_CONTAINER_SUBNETS[@]})" diff --git a/cmd/kubelet/app/plugins.go b/cmd/kubelet/app/plugins.go index 9dc450a584..31b50b4510 100644 --- a/cmd/kubelet/app/plugins.go +++ b/cmd/kubelet/app/plugins.go @@ -25,6 +25,7 @@ import ( "k8s.io/kubernetes/pkg/kubelet/network" "k8s.io/kubernetes/pkg/kubelet/network/cni" "k8s.io/kubernetes/pkg/kubelet/network/exec" + "k8s.io/kubernetes/pkg/kubelet/network/kubenet" // Volume plugins "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/aws_ebs" @@ -88,6 +89,7 @@ func ProbeNetworkPlugins(pluginDir string) []network.NetworkPlugin { // for each existing plugin, add to the list allPlugins = append(allPlugins, exec.ProbeNetworkPlugins(pluginDir)...) allPlugins = append(allPlugins, cni.ProbeNetworkPlugins(pluginDir)...) + allPlugins = append(allPlugins, kubenet.NewPlugin()) return allPlugins } diff --git a/docs/admin/network-plugins.md b/docs/admin/network-plugins.md new file mode 100644 index 0000000000..41249f83b3 --- /dev/null +++ b/docs/admin/network-plugins.md @@ -0,0 +1,66 @@ + + + + +WARNING +WARNING +WARNING +WARNING +WARNING + +

PLEASE NOTE: This document applies to the HEAD of the source tree

+ +If you are using a released version of Kubernetes, you should +refer to the docs that go with that version. + +Documentation for other releases can be found at +[releases.k8s.io](http://releases.k8s.io). + +-- + + + + + +# Network Plugins + +__Disclaimer__: Network plugins are in alpha. Its contents will change rapidly. + +Network plugins in Kubernetes come in a few flavors: +* Plain vanilla exec plugins - deprecated in favor of CNI plugins. +* CNI plugins: adhere to the appc/CNI specification, designed for interoperability. +* Kubenet plugin: implements basic `cbr0` using the `bridge` and `host-local` CNI plugins + +## Installation + +The kubelet has a single default network plugin, and a default network common to the entire cluster. It probes for plugins when it starts up, remembers what it found, and executes the selected plugin at appropriate times in the pod lifecycle (this is only true for docker, as rkt manages its own CNI plugins). There are two Kubelet command line parameters to keep in mind when using plugins: +* `network-plugin-dir`: Kubelet probes this directory for plugins on startup +* `network-plugin`: The network plugin to use from `network-plugin-dir`. It must match the name reported by a plugin probed from the plugin directory. For CNI plugins, this is simply "cni". + +### Exec + +Place plugins in `network-plugin-dir/plugin-name/plugin-name`, i.e if you have a bridge plugin and `network-plugin-dir` is `/usr/lib/kubernetes`, you'd place the bridge plugin executable at `/usr/lib/kubernetes/bridge/bridge`. See [this comment](../../pkg/kubelet/network/exec/exec.go) for more details. + +### CNI + +The CNI plugin is selected by passing Kubelet the `--network-plugin=cni` command-line option. Kubelet reads the first CNI configuration file from `--network-plugin-dir` and uses the CNI configuration from that file to set up each pod's network. The CNI configuration file must match the [CNI specification](https://github.com/appc/cni/blob/master/SPEC.md), and any required CNI plugins referenced by the configuration must be present in `/opt/cni/bin`. + +### kubenet + +The Linux-only kubenet plugin provides functionality similar to the `--configure-cbr0` kubelet command-line option. It creates a Linux bridge named `cbr0` and creates a veth pair for each pod with the host end of each pair connected to `cbr0`. The pod end of the pair is assigned an IP address allocated from a range assigned to the node through either configuration or by the controller-manager. `cbr0` is assigned an MTU matching the smallest MTU of an enabled normal interface on the host. The kubenet plugin is currently mutually exclusive with, and will eventually replace, the --configure-cbr0 option. It is also currently incompatible with the flannel experimental overlay. + +The plugin requires a few things: +* The standard CNI `bridge` and `host-local` plugins to be placed in `/opt/cni/bin`. +* Kubelet must be run with the `--network-plugin=kubenet` argument to enable the plugin +* Kubelet must also be run with the `--reconcile-cidr` argument to ensure the IP subnet assigned to the node by configuration or the controller-manager is propagated to the plugin +* The node must be assigned an IP subnet through either the `--pod-cidr` kubelet command-line option or the `--allocate-node-cidrs=true --cluster-cidr=` controller-manager command-line options. + + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/docs/admin/network-plugins.md?pixel)]() + diff --git a/hack/local-up-cluster.sh b/hack/local-up-cluster.sh index f1cba1f2d5..876a9a7b0d 100755 --- a/hack/local-up-cluster.sh +++ b/hack/local-up-cluster.sh @@ -24,6 +24,7 @@ DOCKERIZE_KUBELET=${DOCKERIZE_KUBELET:-""} ALLOW_PRIVILEGED=${ALLOW_PRIVILEGED:-""} ALLOW_SECURITY_CONTEXT=${ALLOW_SECURITY_CONTEXT:-""} RUNTIME_CONFIG=${RUNTIME_CONFIG:-""} +NET_PLUGIN=${NET_PLUGIN:-""} KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. cd "${KUBE_ROOT}" @@ -255,12 +256,18 @@ function start_apiserver { } function start_controller_manager { + node_cidr_args="" + if [[ "${NET_PLUGIN}" == "kubenet" ]]; then + node_cidr_args="--allocate-node-cidrs=true --cluster-cidr=10.1.0.0/16 " + fi + CTLRMGR_LOG=/tmp/kube-controller-manager.log sudo -E "${GO_OUT}/kube-controller-manager" \ --v=${LOG_LEVEL} \ --service-account-private-key-file="${SERVICE_ACCOUNT_KEY}" \ --root-ca-file="${ROOT_CA_FILE}" \ --enable-hostpath-provisioner="${ENABLE_HOSTPATH_PROVISIONER}" \ + ${node_cidr_args} \ --master="${API_HOST}:${API_PORT}" >"${CTLRMGR_LOG}" 2>&1 & CTLRMGR_PID=$! } @@ -283,6 +290,16 @@ function start_kubelet { fi fi + net_plugin_args="" + if [[ -n "${NET_PLUGIN}" ]]; then + net_plugin_args="--network-plugin=${NET_PLUGIN}" + fi + + kubenet_plugin_args="" + if [[ "${NET_PLUGIN}" == "kubenet" ]]; then + kubenet_plugin_args="--reconcile-cidr=true " + fi + sudo -E "${GO_OUT}/kubelet" ${priv_arg}\ --v=${LOG_LEVEL} \ --chaos-chance="${CHAOS_CHANCE}" \ @@ -294,6 +311,8 @@ function start_kubelet { --api-servers="${API_HOST}:${API_PORT}" \ --cpu-cfs-quota=${CPU_CFS_QUOTA} \ --cluster-dns="127.0.0.1" \ + ${net_plugin_args} \ + ${kubenet_plugin_args} \ --port="$KUBELET_PORT" >"${KUBELET_LOG}" 2>&1 & KUBELET_PID=$! else diff --git a/pkg/kubelet/dockertools/manager.go b/pkg/kubelet/dockertools/manager.go index 21495a49d3..7442323283 100644 --- a/pkg/kubelet/dockertools/manager.go +++ b/pkg/kubelet/dockertools/manager.go @@ -1654,7 +1654,7 @@ func (dm *DockerManager) createPodInfraContainer(pod *api.Pod) (kubecontainer.Do netNamespace := "" var ports []api.ContainerPort - if dm.networkPlugin.Name() == "cni" { + if dm.networkPlugin.Name() == "cni" || dm.networkPlugin.Name() == "kubenet" { netNamespace = "none" } diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 0bc00a9cc1..be40407569 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -269,6 +269,13 @@ func NewMainKubelet( oomWatcher := NewOOMWatcher(cadvisorInterface, recorder) + // TODO: remove when internal cbr0 implementation gets removed in favor + // of the kubenet network plugin + if networkPluginName == "kubenet" { + configureCBR0 = false + flannelExperimentalOverlay = false + } + klet := &Kubelet{ hostname: hostname, nodeName: nodeName, @@ -638,6 +645,7 @@ type Kubelet struct { resolverConfig string // Optionally shape the bandwidth of a pod + // TODO: remove when kubenet plugin is ready shaper bandwidth.BandwidthShaper // True if container cpu limits should be enforced via cgroup CFS quota @@ -2595,6 +2603,8 @@ func (kl *Kubelet) updateRuntimeUp() { kl.runtimeState.setRuntimeSync(kl.clock.Now()) } +// TODO: remove when kubenet plugin is ready +// NOTE!!! if you make changes here, also make them to kubenet func (kl *Kubelet) reconcileCBR0(podCIDR string) error { if podCIDR == "" { glog.V(5).Info("PodCIDR not set. Will not configure cbr0.") diff --git a/pkg/kubelet/network/kubenet/kubenet_linux.go b/pkg/kubelet/network/kubenet/kubenet_linux.go new file mode 100644 index 0000000000..4eebe27e39 --- /dev/null +++ b/pkg/kubelet/network/kubenet/kubenet_linux.go @@ -0,0 +1,287 @@ +// +build linux + +/* +Copyright 2014 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubenet + +import ( + "fmt" + "net" + "strings" + "syscall" + + "github.com/vishvananda/netlink" + + "github.com/appc/cni/libcni" + "github.com/golang/glog" + kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" + "k8s.io/kubernetes/pkg/kubelet/dockertools" + "k8s.io/kubernetes/pkg/kubelet/network" + "k8s.io/kubernetes/pkg/util/bandwidth" +) + +const ( + KubenetPluginName = "kubenet" + BridgeName = "cbr0" + DefaultCNIDir = "/opt/cni/bin" + DefaultInterfaceName = "eth0" +) + +type kubenetNetworkPlugin struct { + host network.Host + netConfig *libcni.NetworkConfig + cniConfig *libcni.CNIConfig + shaper bandwidth.BandwidthShaper + + podCIDRs map[kubecontainer.DockerID]string + MTU int +} + +func NewPlugin() network.NetworkPlugin { + return &kubenetNetworkPlugin{ + podCIDRs: make(map[kubecontainer.DockerID]string), + MTU: 1460, + } +} + +func (plugin *kubenetNetworkPlugin) Init(host network.Host) error { + plugin.host = host + plugin.cniConfig = &libcni.CNIConfig{ + Path: []string{DefaultCNIDir}, + } + + if link, err := findMinMTU(); err == nil { + plugin.MTU = link.MTU + glog.V(5).Infof("Using interface %s MTU %d as bridge MTU", link.Name, link.MTU) + } else { + glog.Warningf("Failed to find default bridge MTU: %v", err) + } + + return nil +} + +func findMinMTU() (*net.Interface, error) { + intfs, err := net.Interfaces() + if err != nil { + return nil, err + } + + mtu := 999999 + defIntfIndex := -1 + for i, intf := range intfs { + if ((intf.Flags & net.FlagUp) != 0) && (intf.Flags&(net.FlagLoopback|net.FlagPointToPoint) == 0) { + if intf.MTU < mtu { + mtu = intf.MTU + defIntfIndex = i + } + } + } + + if mtu >= 999999 || mtu < 576 || defIntfIndex < 0 { + return nil, fmt.Errorf("no suitable interface", BridgeName) + } + + return &intfs[defIntfIndex], nil +} + +const NET_CONFIG_TEMPLATE = `{ + "cniVersion": "0.1.0", + "name": "kubenet", + "type": "bridge", + "bridge": "%s", + "mtu": %d, + "addIf": "%s", + "isGateway": true, + "ipMasq": true, + "ipam": { + "type": "host-local", + "subnet": "%s", + "gateway": "%s", + "routes": [ + { "dst": "0.0.0.0/0" } + ] + } +}` + +func (plugin *kubenetNetworkPlugin) Event(name string, details map[string]interface{}) { + if name != network.NET_PLUGIN_EVENT_POD_CIDR_CHANGE { + return + } + + podCIDR, ok := details[network.NET_PLUGIN_EVENT_POD_CIDR_CHANGE_DETAIL_CIDR].(string) + if !ok { + glog.Warningf("%s event didn't contain pod CIDR", network.NET_PLUGIN_EVENT_POD_CIDR_CHANGE) + return + } + + if plugin.netConfig != nil { + glog.V(5).Infof("Ignoring subsequent pod CIDR update to %s", podCIDR) + return + } + + glog.V(5).Infof("PodCIDR is set to %q", podCIDR) + _, cidr, err := net.ParseCIDR(podCIDR) + if err == nil { + // Set bridge address to first address in IPNet + cidr.IP.To4()[3] += 1 + + json := fmt.Sprintf(NET_CONFIG_TEMPLATE, BridgeName, plugin.MTU, DefaultInterfaceName, podCIDR, cidr.IP.String()) + plugin.netConfig, err = libcni.ConfFromBytes([]byte(json)) + if err == nil { + glog.V(5).Infof("CNI network config:\n%s", json) + + // Ensure cbr0 has no conflicting addresses; CNI's 'bridge' + // plugin will bail out if the bridge has an unexpected one + plugin.clearBridgeAddressesExcept(cidr.IP.String()) + } + } + + if err != nil { + glog.Warningf("Failed to generate CNI network config: %v", err) + } +} + +func (plugin *kubenetNetworkPlugin) clearBridgeAddressesExcept(keep string) { + bridge, err := netlink.LinkByName(BridgeName) + if err != nil { + return + } + + addrs, err := netlink.AddrList(bridge, syscall.AF_INET) + if err != nil { + return + } + + for _, addr := range addrs { + if addr.IPNet.String() != keep { + glog.V(5).Infof("Removing old address %s from %s", addr.IPNet.String(), BridgeName) + netlink.AddrDel(bridge, &addr) + } + } +} + +func (plugin *kubenetNetworkPlugin) Name() string { + return KubenetPluginName +} + +func (plugin *kubenetNetworkPlugin) SetUpPod(namespace string, name string, id kubecontainer.DockerID) error { + // Can't set up pods if we don't have a PodCIDR yet + if plugin.netConfig == nil { + return fmt.Errorf("Kubenet needs a PodCIDR to set up pods") + } + + runtime, ok := plugin.host.GetRuntime().(*dockertools.DockerManager) + if !ok { + return fmt.Errorf("Kubenet execution called on non-docker runtime") + } + netnsPath, err := runtime.GetNetNS(id.ContainerID()) + if err != nil { + return err + } + + rt := buildCNIRuntimeConf(name, namespace, id.ContainerID(), netnsPath) + if err != nil { + return fmt.Errorf("Error building CNI config: %v", err) + } + + res, err := plugin.cniConfig.AddNetwork(plugin.netConfig, rt) + if err != nil { + return fmt.Errorf("Error adding container to network: %v", err) + } + if res.IP4 == nil { + return fmt.Errorf("CNI plugin reported no IPv4 address for container %v.", id) + } + + plugin.podCIDRs[id] = res.IP4.IP.String() + + // The first SetUpPod call creates the bridge; ensure shaping is enabled + if plugin.shaper == nil { + plugin.shaper = bandwidth.NewTCShaper(BridgeName) + if plugin.shaper == nil { + return fmt.Errorf("Failed to create bandwidth shaper!") + } + plugin.shaper.ReconcileInterface() + } + + // TODO: get ingress/egress from Pod.Spec and add pod CIDR to shaper + + return nil +} + +func (plugin *kubenetNetworkPlugin) TearDownPod(namespace string, name string, id kubecontainer.DockerID) error { + if plugin.netConfig == nil { + return fmt.Errorf("Kubenet needs a PodCIDR to tear down pods") + } + + runtime, ok := plugin.host.GetRuntime().(*dockertools.DockerManager) + if !ok { + return fmt.Errorf("Kubenet execution called on non-docker runtime") + } + netnsPath, err := runtime.GetNetNS(id.ContainerID()) + if err != nil { + return err + } + + rt := buildCNIRuntimeConf(name, namespace, id.ContainerID(), netnsPath) + if err != nil { + return fmt.Errorf("Error building CNI config: %v", err) + } + + // no cached CIDR is Ok during teardown + if cidr, ok := plugin.podCIDRs[id]; ok { + glog.V(5).Infof("Removing pod CIDR %s from shaper", cidr) + // shaper wants /32 + if addr, _, err := net.ParseCIDR(cidr); err != nil { + if err = plugin.shaper.Reset(fmt.Sprintf("%s/32", addr.String())); err != nil { + glog.Warningf("Failed to remove pod CIDR %s from shaper: %v", cidr, err) + } + } + } + delete(plugin.podCIDRs, id) + + if err := plugin.cniConfig.DelNetwork(plugin.netConfig, rt); err != nil { + return fmt.Errorf("Error removing container from network: %v", err) + } + + return nil +} + +// TODO: Use the addToNetwork function to obtain the IP of the Pod. That will assume idempotent ADD call to the plugin. +// Also fix the runtime's call to Status function to be done only in the case that the IP is lost, no need to do periodic calls +func (plugin *kubenetNetworkPlugin) Status(namespace string, name string, id kubecontainer.DockerID) (*network.PodNetworkStatus, error) { + cidr, ok := plugin.podCIDRs[id] + if !ok { + return nil, fmt.Errorf("No IP address found for pod %v", id) + } + + ip, _, err := net.ParseCIDR(strings.Trim(cidr, "\n")) + if err != nil { + return nil, err + } + return &network.PodNetworkStatus{IP: ip}, nil +} + +func buildCNIRuntimeConf(podName string, podNs string, podInfraContainerID kubecontainer.ContainerID, podNetnsPath string) *libcni.RuntimeConf { + glog.V(4).Infof("Kubenet: using netns path %v", podNetnsPath) + glog.V(4).Infof("Kubenet: using podns path %v", podNs) + + return &libcni.RuntimeConf{ + ContainerID: podInfraContainerID.ID, + NetNS: podNetnsPath, + IfName: DefaultInterfaceName, + } +} diff --git a/pkg/kubelet/network/kubenet/kubenet_unsupported.go b/pkg/kubelet/network/kubenet/kubenet_unsupported.go new file mode 100644 index 0000000000..751db29137 --- /dev/null +++ b/pkg/kubelet/network/kubenet/kubenet_unsupported.go @@ -0,0 +1,55 @@ +// +build !linux + +/* +Copyright 2014 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubenet + +import ( + "fmt" + + kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" + "k8s.io/kubernetes/pkg/kubelet/network" +) + +type kubenetNetworkPlugin struct { +} + +func NewPlugin() network.NetworkPlugin { + return &kubenetNetworkPlugin{} +} + +func (plugin *kubenetNetworkPlugin) Init(host network.Host) error { + return fmt.Errorf("Kubenet is not supported in this build") +} +func (plugin *kubenetNetworkPlugin) Event(name string, details map[string]interface{}) { +} + +func (plugin *kubenetNetworkPlugin) Name() string { + return "kubenet" +} + +func (plugin *kubenetNetworkPlugin) SetUpPod(namespace string, name string, id kubecontainer.DockerID) error { + return fmt.Errorf("Kubenet is not supported in this build") +} + +func (plugin *kubenetNetworkPlugin) TearDownPod(namespace string, name string, id kubecontainer.DockerID) error { + return fmt.Errorf("Kubenet is not supported in this build") +} + +func (plugin *kubenetNetworkPlugin) Status(namespace string, name string, id kubecontainer.DockerID) (*network.PodNetworkStatus, error) { + return nil, fmt.Errorf("Kubenet is not supported in this build") +}