diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 8e4a67f956..cda9db2508 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -23,6 +23,21 @@ "ImportPath": "github.com/abbot/go-http-auth", "Rev": "c0ef4539dfab4d21c8ef20ba2924f9fc6f186d35" }, + { + "ImportPath": "github.com/appc/cni/libcni", + "Comment": "v0.1.0-27-g2a58bd9", + "Rev": "2a58bd9379ca33579f0cf631945b717aa4fa373d" + }, + { + "ImportPath": "github.com/appc/cni/pkg/invoke", + "Comment": "v0.1.0-27-g2a58bd9", + "Rev": "2a58bd9379ca33579f0cf631945b717aa4fa373d" + }, + { + "ImportPath": "github.com/appc/cni/pkg/types", + "Comment": "v0.1.0-27-g2a58bd9", + "Rev": "2a58bd9379ca33579f0cf631945b717aa4fa373d" + }, { "ImportPath": "github.com/appc/spec/schema", "Comment": "v0.6.1-30-gc928a0c", diff --git a/Godeps/_workspace/src/github.com/appc/cni/libcni/api.go b/Godeps/_workspace/src/github.com/appc/cni/libcni/api.go new file mode 100644 index 0000000000..4ad714a03e --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/cni/libcni/api.go @@ -0,0 +1,65 @@ +// Copyright 2015 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package libcni + +import ( + "strings" + + "github.com/appc/cni/pkg/invoke" + "github.com/appc/cni/pkg/types" +) + +type RuntimeConf struct { + ContainerID string + NetNS string + IfName string + Args [][2]string +} + +type NetworkConfig struct { + Network *types.NetConf + Bytes []byte +} + +type CNI interface { + AddNetwork(net *NetworkConfig, rt *RuntimeConf) (*types.Result, error) + DelNetwork(net *NetworkConfig, rt *RuntimeConf) error +} + +type CNIConfig struct { + Path []string +} + +func (c *CNIConfig) AddNetwork(net *NetworkConfig, rt *RuntimeConf) (*types.Result, error) { + pluginPath := invoke.FindInPath(net.Network.Type, c.Path) + return invoke.ExecPluginWithResult(pluginPath, net.Bytes, c.args("ADD", rt)) +} + +func (c *CNIConfig) DelNetwork(net *NetworkConfig, rt *RuntimeConf) error { + pluginPath := invoke.FindInPath(net.Network.Type, c.Path) + return invoke.ExecPluginWithoutResult(pluginPath, net.Bytes, c.args("DEL", rt)) +} + +// ===== +func (c *CNIConfig) args(action string, rt *RuntimeConf) *invoke.Args { + return &invoke.Args{ + Command: action, + ContainerID: rt.ContainerID, + NetNS: rt.NetNS, + PluginArgs: rt.Args, + IfName: rt.IfName, + Path: strings.Join(c.Path, ":"), + } +} diff --git a/Godeps/_workspace/src/github.com/appc/cni/libcni/conf.go b/Godeps/_workspace/src/github.com/appc/cni/libcni/conf.go new file mode 100644 index 0000000000..47babeb447 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/cni/libcni/conf.go @@ -0,0 +1,85 @@ +// Copyright 2015 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package libcni + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "sort" +) + +func ConfFromBytes(bytes []byte) (*NetworkConfig, error) { + conf := &NetworkConfig{Bytes: bytes} + if err := json.Unmarshal(bytes, &conf.Network); err != nil { + return nil, fmt.Errorf("error parsing configuration: %s", err) + } + return conf, nil +} + +func ConfFromFile(filename string) (*NetworkConfig, error) { + bytes, err := ioutil.ReadFile(filename) + if err != nil { + return nil, fmt.Errorf("error reading %s: %s", filename, err) + } + return ConfFromBytes(bytes) +} + +func ConfFiles(dir string) ([]string, error) { + // In part, adapted from rkt/networking/podenv.go#listFiles + files, err := ioutil.ReadDir(dir) + switch { + case err == nil: // break + case os.IsNotExist(err): + return nil, nil + default: + return nil, err + } + + confFiles := []string{} + for _, f := range files { + if f.IsDir() { + continue + } + if filepath.Ext(f.Name()) == ".conf" { + confFiles = append(confFiles, filepath.Join(dir, f.Name())) + } + } + return confFiles, nil +} + +func LoadConf(dir, name string) (*NetworkConfig, error) { + files, err := ConfFiles(dir) + switch { + case err != nil: + return nil, err + case len(files) == 0: + return nil, fmt.Errorf("no net configurations found") + } + sort.Strings(files) + + for _, confFile := range files { + conf, err := ConfFromFile(confFile) + if err != nil { + return nil, err + } + if conf.Network.Name == name { + return conf, nil + } + } + return nil, fmt.Errorf(`no net configuration with name "%s" in %s`, name, dir) +} diff --git a/Godeps/_workspace/src/github.com/appc/cni/pkg/invoke/args.go b/Godeps/_workspace/src/github.com/appc/cni/pkg/invoke/args.go new file mode 100644 index 0000000000..6f0a813ad3 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/cni/pkg/invoke/args.go @@ -0,0 +1,76 @@ +// Copyright 2015 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package invoke + +import ( + "os" + "strings" +) + +type CNIArgs interface { + // For use with os/exec; i.e., return nil to inherit the + // environment from this process + AsEnv() []string +} + +type inherited struct{} + +var inheritArgsFromEnv inherited + +func (_ *inherited) AsEnv() []string { + return nil +} + +func ArgsFromEnv() CNIArgs { + return &inheritArgsFromEnv +} + +type Args struct { + Command string + ContainerID string + NetNS string + PluginArgs [][2]string + PluginArgsStr string + IfName string + Path string +} + +func (args *Args) AsEnv() []string { + env := os.Environ() + pluginArgsStr := args.PluginArgsStr + if pluginArgsStr == "" { + pluginArgsStr = stringify(args.PluginArgs) + } + + env = append(env, + "CNI_COMMAND="+args.Command, + "CNI_CONTAINERID="+args.ContainerID, + "CNI_NETNS="+args.NetNS, + "CNI_ARGS="+pluginArgsStr, + "CNI_IFNAME="+args.IfName, + "CNI_PATH="+args.Path) + return env +} + +// taken from rkt/networking/net_plugin.go +func stringify(pluginArgs [][2]string) string { + entries := make([]string, len(pluginArgs)) + + for i, kv := range pluginArgs { + entries[i] = strings.Join(kv[:], "=") + } + + return strings.Join(entries, ";") +} diff --git a/Godeps/_workspace/src/github.com/appc/cni/pkg/invoke/exec.go b/Godeps/_workspace/src/github.com/appc/cni/pkg/invoke/exec.go new file mode 100644 index 0000000000..cf0cff47e9 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/cni/pkg/invoke/exec.go @@ -0,0 +1,80 @@ +// Copyright 2015 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package invoke + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + + "github.com/appc/cni/pkg/types" +) + +func pluginErr(err error, output []byte) error { + if _, ok := err.(*exec.ExitError); ok { + emsg := types.Error{} + if perr := json.Unmarshal(output, &emsg); perr != nil { + return fmt.Errorf("netplugin failed but error parsing its diagnostic message %q: %v", string(output), perr) + } + details := "" + if emsg.Details != "" { + details = fmt.Sprintf("; %v", emsg.Details) + } + return fmt.Errorf("%v%v", emsg.Msg, details) + } + + return err +} + +func ExecPluginWithResult(pluginPath string, netconf []byte, args CNIArgs) (*types.Result, error) { + stdoutBytes, err := execPlugin(pluginPath, netconf, args) + if err != nil { + return nil, err + } + + res := &types.Result{} + err = json.Unmarshal(stdoutBytes, res) + return res, err +} + +func ExecPluginWithoutResult(pluginPath string, netconf []byte, args CNIArgs) error { + _, err := execPlugin(pluginPath, netconf, args) + return err +} + +func execPlugin(pluginPath string, netconf []byte, args CNIArgs) ([]byte, error) { + if pluginPath == "" { + return nil, fmt.Errorf("could not find %q plugin", filepath.Base(pluginPath)) + } + + stdout := &bytes.Buffer{} + + c := exec.Cmd{ + Env: args.AsEnv(), + Path: pluginPath, + Args: []string{pluginPath}, + Stdin: bytes.NewBuffer(netconf), + Stdout: stdout, + Stderr: os.Stderr, + } + if err := c.Run(); err != nil { + return nil, pluginErr(err, stdout.Bytes()) + } + + return stdout.Bytes(), nil +} diff --git a/Godeps/_workspace/src/github.com/appc/cni/pkg/invoke/find.go b/Godeps/_workspace/src/github.com/appc/cni/pkg/invoke/find.go new file mode 100644 index 0000000000..dfad12bc64 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/cni/pkg/invoke/find.go @@ -0,0 +1,37 @@ +// Copyright 2015 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package invoke + +import ( + "os" + "path/filepath" + "strings" +) + +func FindInPath(plugin string, path []string) string { + for _, p := range path { + fullname := filepath.Join(p, plugin) + if fi, err := os.Stat(fullname); err == nil && fi.Mode().IsRegular() { + return fullname + } + } + return "" +} + +// Find returns the full path of the plugin by searching in CNI_PATH +func Find(plugin string) string { + paths := strings.Split(os.Getenv("CNI_PATH"), ":") + return FindInPath(plugin, paths) +} diff --git a/Godeps/_workspace/src/github.com/appc/cni/pkg/types/args.go b/Godeps/_workspace/src/github.com/appc/cni/pkg/types/args.go new file mode 100644 index 0000000000..68162435a8 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/cni/pkg/types/args.go @@ -0,0 +1,50 @@ +// Copyright 2015 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package types + +import ( + "encoding" + "fmt" + "reflect" + "strings" +) + +func LoadArgs(args string, container interface{}) error { + if args == "" { + return nil + } + + containerValue := reflect.ValueOf(container) + + pairs := strings.Split(args, ";") + for _, pair := range pairs { + kv := strings.Split(pair, "=") + if len(kv) != 2 { + return fmt.Errorf("ARGS: invalid pair %q", pair) + } + keyString := kv[0] + valueString := kv[1] + keyField := containerValue.Elem().FieldByName(keyString) + if !keyField.IsValid() { + return fmt.Errorf("ARGS: invalid key %q", keyString) + } + u := keyField.Addr().Interface().(encoding.TextUnmarshaler) + err := u.UnmarshalText([]byte(valueString)) + if err != nil { + return fmt.Errorf("ARGS: error parsing value of pair %q: %v)", pair, err) + } + } + return nil +} diff --git a/Godeps/_workspace/src/github.com/appc/cni/pkg/types/types.go b/Godeps/_workspace/src/github.com/appc/cni/pkg/types/types.go new file mode 100644 index 0000000000..21ba32d61f --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/cni/pkg/types/types.go @@ -0,0 +1,166 @@ +// Copyright 2015 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package types + +import ( + "encoding/json" + "net" + "os" +) + +// like net.IPNet but adds JSON marshalling and unmarshalling +type IPNet net.IPNet + +// ParseCIDR takes a string like "10.2.3.1/24" and +// return IPNet with "10.2.3.1" and /24 mask +func ParseCIDR(s string) (*net.IPNet, error) { + ip, ipn, err := net.ParseCIDR(s) + if err != nil { + return nil, err + } + + ipn.IP = ip + return ipn, nil +} + +func (n IPNet) MarshalJSON() ([]byte, error) { + return json.Marshal((*net.IPNet)(&n).String()) +} + +func (n *IPNet) UnmarshalJSON(data []byte) error { + var s string + if err := json.Unmarshal(data, &s); err != nil { + return err + } + + tmp, err := ParseCIDR(s) + if err != nil { + return err + } + + *n = IPNet(*tmp) + return nil +} + +// NetConf describes a network. +type NetConf struct { + Name string `json:"name,omitempty"` + Type string `json:"type,omitempty"` + IPAM struct { + Type string `json:"type,omitempty"` + } `json:"ipam,omitempty"` +} + +// Result is what gets returned from the plugin (via stdout) to the caller +type Result struct { + IP4 *IPConfig `json:"ip4,omitempty"` + IP6 *IPConfig `json:"ip6,omitempty"` +} + +func (r *Result) Print() error { + return prettyPrint(r) +} + +// IPConfig contains values necessary to configure an interface +type IPConfig struct { + IP net.IPNet + Gateway net.IP + Routes []Route +} + +type Route struct { + Dst net.IPNet + GW net.IP +} + +type Error struct { + Code uint `json:"code"` + Msg string `json:"msg"` + Details string `json:"details,omitempty"` +} + +func (e *Error) Error() string { + return e.Msg +} + +func (e *Error) Print() error { + return prettyPrint(e) +} + +// net.IPNet is not JSON (un)marshallable so this duality is needed +// for our custom IPNet type + +// JSON (un)marshallable types +type ipConfig struct { + IP IPNet `json:"ip"` + Gateway net.IP `json:"gateway,omitempty"` + Routes []Route `json:"routes,omitempty"` +} + +type route struct { + Dst IPNet `json:"dst"` + GW net.IP `json:"gw,omitempty"` +} + +func (c *IPConfig) MarshalJSON() ([]byte, error) { + ipc := ipConfig{ + IP: IPNet(c.IP), + Gateway: c.Gateway, + Routes: c.Routes, + } + + return json.Marshal(ipc) +} + +func (c *IPConfig) UnmarshalJSON(data []byte) error { + ipc := ipConfig{} + if err := json.Unmarshal(data, &ipc); err != nil { + return err + } + + c.IP = net.IPNet(ipc.IP) + c.Gateway = ipc.Gateway + c.Routes = ipc.Routes + return nil +} + +func (r *Route) UnmarshalJSON(data []byte) error { + rt := route{} + if err := json.Unmarshal(data, &rt); err != nil { + return err + } + + r.Dst = net.IPNet(rt.Dst) + r.GW = rt.GW + return nil +} + +func (r *Route) MarshalJSON() ([]byte, error) { + rt := route{ + Dst: IPNet(r.Dst), + GW: r.GW, + } + + return json.Marshal(rt) +} + +func prettyPrint(obj interface{}) error { + data, err := json.MarshalIndent(obj, "", " ") + if err != nil { + return err + } + _, err = os.Stdout.Write(data) + return err +} diff --git a/cmd/kubelet/app/plugins.go b/cmd/kubelet/app/plugins.go index 9a5b5e0417..d2be67656a 100644 --- a/cmd/kubelet/app/plugins.go +++ b/cmd/kubelet/app/plugins.go @@ -22,6 +22,7 @@ import ( _ "k8s.io/kubernetes/pkg/credentialprovider/gcp" // Network plugins "k8s.io/kubernetes/pkg/kubelet/network" + "k8s.io/kubernetes/pkg/kubelet/network/cni" "k8s.io/kubernetes/pkg/kubelet/network/exec" // Volume plugins "k8s.io/kubernetes/pkg/volume" @@ -78,6 +79,7 @@ func ProbeNetworkPlugins(pluginDir string) []network.NetworkPlugin { // for each existing plugin, add to the list allPlugins = append(allPlugins, exec.ProbeNetworkPlugins(pluginDir)...) + allPlugins = append(allPlugins, cni.ProbeNetworkPlugins(pluginDir)...) return allPlugins } diff --git a/pkg/kubelet/dockertools/manager.go b/pkg/kubelet/dockertools/manager.go index 4407b29971..8a76aea12a 100644 --- a/pkg/kubelet/dockertools/manager.go +++ b/pkg/kubelet/dockertools/manager.go @@ -74,6 +74,8 @@ const ( kubernetesPodLabel = "io.kubernetes.pod.data" kubernetesTerminationGracePeriodLabel = "io.kubernetes.pod.terminationGracePeriod" kubernetesContainerLabel = "io.kubernetes.container.name" + + DockerNetnsFmt = "/proc/%v/ns/net" ) // DockerManager implements the Runtime interface. @@ -1181,6 +1183,32 @@ func (dm *DockerManager) PortForward(pod *kubecontainer.Pod, port uint16, stream return command.Run() } +// Get the IP address of a container's interface using nsenter +func (dm *DockerManager) GetContainerIP(containerID, interfaceName string) (string, error) { + _, lookupErr := exec.LookPath("nsenter") + if lookupErr != nil { + return "", fmt.Errorf("Unable to obtain IP address of container: missing nsenter.") + } + container, err := dm.client.InspectContainer(containerID) + if err != nil { + return "", err + } + + if !container.State.Running { + return "", fmt.Errorf("container not running (%s)", container.ID) + } + + containerPid := container.State.Pid + extractIPCmd := fmt.Sprintf("ip -4 addr show %s | grep inet | awk -F\" \" '{print $2}'", interfaceName) + args := []string{"-t", fmt.Sprintf("%d", containerPid), "-n", "--", "bash", "-c", extractIPCmd} + command := exec.Command("nsenter", args...) + out, err := command.CombinedOutput() + if err != nil { + return "", err + } + return string(out), nil +} + // Kills all containers in the specified pod func (dm *DockerManager) KillPod(pod *api.Pod, runningPod kubecontainer.Pod) error { // Send the kills in parallel since they may take a long time. Len + 1 since there @@ -1526,6 +1554,10 @@ func (dm *DockerManager) createPodInfraContainer(pod *api.Pod) (kubeletTypes.Doc netNamespace := "" var ports []api.ContainerPort + if dm.networkPlugin.Name() == "cni" { + netNamespace = "none" + } + if pod.Spec.HostNetwork { netNamespace = "host" } else { @@ -1935,3 +1967,14 @@ func getIPCMode(pod *api.Pod, ipcMode string) string { } return ipcMode } + +// GetNetNs returns the network namespace path for the given container +func (dm *DockerManager) GetNetNs(containerID string) (string, error) { + inspectResult, err := dm.client.InspectContainer(string(containerID)) + if err != nil { + glog.Errorf("Error inspecting container: '%v'", err) + return "", err + } + netnsPath := fmt.Sprintf(DockerNetnsFmt, inspectResult.State.Pid) + return netnsPath, nil +} diff --git a/pkg/kubelet/network/cni/cni.go b/pkg/kubelet/network/cni/cni.go new file mode 100644 index 0000000000..ff92a59ff4 --- /dev/null +++ b/pkg/kubelet/network/cni/cni.go @@ -0,0 +1,198 @@ +/* +Copyright 2014 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cni + +import ( + "fmt" + "github.com/appc/cni/libcni" + cniTypes "github.com/appc/cni/pkg/types" + "github.com/golang/glog" + "k8s.io/kubernetes/pkg/kubelet/dockertools" + "k8s.io/kubernetes/pkg/kubelet/network" + kubeletTypes "k8s.io/kubernetes/pkg/kubelet/types" + "net" + "sort" + "strings" +) + +const ( + CNIPluginName = "cni" + DefaultNetDir = "/etc/cni/net.d" + DefaultCNIDir = "/opt/cni/bin" + DefaultInterfaceName = "eth0" + VendorCNIDirTemplate = "/opt/%s/bin" +) + +type cniNetworkPlugin struct { + defaultNetwork *cniNetwork + host network.Host +} + +type cniNetwork struct { + name string + NetworkConfig *libcni.NetworkConfig + CNIConfig *libcni.CNIConfig +} + +func ProbeNetworkPlugins(pluginDir string) []network.NetworkPlugin { + configList := make([]network.NetworkPlugin, 0) + network, err := getDefaultCNINetwork(pluginDir) + if err != nil { + return configList + } + return append(configList, &cniNetworkPlugin{defaultNetwork: network}) +} + +func getDefaultCNINetwork(pluginDir string) (*cniNetwork, error) { + if pluginDir == "" { + pluginDir = DefaultNetDir + } + files, err := libcni.ConfFiles(pluginDir) + switch { + case err != nil: + return nil, err + case len(files) == 0: + return nil, fmt.Errorf("No networks found in %s", pluginDir) + } + + sort.Strings(files) + for _, confFile := range files { + conf, err := libcni.ConfFromFile(confFile) + if err != nil { + glog.Warningf("Error loading CNI config file %s: %v", confFile, err) + continue + } + // Search for vendor-specific plugins as well as default plugins in the CNI codebase. + vendorCNIDir := fmt.Sprintf(VendorCNIDirTemplate, conf.Network.Type) + cninet := &libcni.CNIConfig{ + Path: []string{DefaultCNIDir, vendorCNIDir}, + } + network := &cniNetwork{name: conf.Network.Name, NetworkConfig: conf, CNIConfig: cninet} + return network, nil + } + return nil, fmt.Errorf("No valid networks found in %s", pluginDir) +} + +func (plugin *cniNetworkPlugin) Init(host network.Host) error { + plugin.host = host + return nil +} + +func (plugin *cniNetworkPlugin) Name() string { + return CNIPluginName +} + +func (plugin *cniNetworkPlugin) SetUpPod(namespace string, name string, id kubeletTypes.DockerID) error { + runtime, ok := plugin.host.GetRuntime().(*dockertools.DockerManager) + if !ok { + return fmt.Errorf("CNI execution called on non-docker runtime") + } + netns, err := runtime.GetNetNs(string(id)) + if err != nil { + return err + } + + _, err = plugin.defaultNetwork.addToNetwork(name, namespace, string(id), netns) + if err != nil { + glog.Errorf("Error while adding to cni network: %s", err) + return err + } + + return err +} + +func (plugin *cniNetworkPlugin) TearDownPod(namespace string, name string, id kubeletTypes.DockerID) error { + runtime, ok := plugin.host.GetRuntime().(*dockertools.DockerManager) + if !ok { + return fmt.Errorf("CNI execution called on non-docker runtime") + } + netns, err := runtime.GetNetNs(string(id)) + if err != nil { + return err + } + + return plugin.defaultNetwork.deleteFromNetwork(name, namespace, string(id), netns) +} + +func (plugin *cniNetworkPlugin) Status(namespace string, name string, id kubeletTypes.DockerID) (*network.PodNetworkStatus, error) { + runtime, ok := plugin.host.GetRuntime().(*dockertools.DockerManager) + if !ok { + return nil, fmt.Errorf("CNI execution called on non-docker runtime") + } + ipStr, err := runtime.GetContainerIP(string(id), DefaultInterfaceName) + if err != nil { + return nil, err + } + ip, _, err := net.ParseCIDR(strings.Trim(ipStr, "\n")) + if err != nil { + return nil, err + } + return &network.PodNetworkStatus{IP: ip}, nil +} + +func (network *cniNetwork) addToNetwork(podName string, podNamespace string, podInfraContainerID string, podNetnsPath string) (*cniTypes.Result, error) { + rt, err := buildCNIRuntimeConf(podName, podNamespace, podInfraContainerID, podNetnsPath) + if err != nil { + glog.Errorf("Error adding network: %v", err) + return nil, err + } + + netconf, cninet := network.NetworkConfig, network.CNIConfig + glog.V(4).Infof("About to run with conf.Network.Type=%v, c.Path=%v", netconf.Network.Type, cninet.Path) + res, err := cninet.AddNetwork(netconf, rt) + if err != nil { + glog.Errorf("Error adding network: %v", err) + return nil, err + } + + return res, nil +} + +func (network *cniNetwork) deleteFromNetwork(podName string, podNamespace string, podInfraContainerID string, podNetnsPath string) error { + rt, err := buildCNIRuntimeConf(podName, podNamespace, podInfraContainerID, podNetnsPath) + if err != nil { + glog.Errorf("Error deleting network: %v", err) + return err + } + + netconf, cninet := network.NetworkConfig, network.CNIConfig + glog.V(4).Infof("About to run with conf.Network.Type=%v, c.Path=%v", netconf.Network.Type, cninet.Path) + err = cninet.DelNetwork(netconf, rt) + if err != nil { + glog.Errorf("Error deleting network: %v", err) + return err + } + return nil +} + +func buildCNIRuntimeConf(podName string, podNs string, podInfraContainerID string, podNetnsPath string) (*libcni.RuntimeConf, error) { + glog.V(4).Infof("Got netns path %v", podNetnsPath) + glog.V(4).Infof("Using netns path %v", podNs) + + rt := &libcni.RuntimeConf{ + ContainerID: podInfraContainerID, + NetNS: podNetnsPath, + IfName: DefaultInterfaceName, + Args: [][2]string{ + {"K8S_POD_NAMESPACE", podNs}, + {"K8S_POD_NAME", podName}, + {"K8S_POD_INFRA_CONTAINER_ID", podInfraContainerID}, + }, + } + + return rt, nil +} diff --git a/pkg/kubelet/network/cni/cni_test.go b/pkg/kubelet/network/cni/cni_test.go new file mode 100644 index 0000000000..23e17cf0a9 --- /dev/null +++ b/pkg/kubelet/network/cni/cni_test.go @@ -0,0 +1,191 @@ +// +build linux + +/* +Copyright 2014 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cni + +import ( + "bytes" + "fmt" + "io/ioutil" + "math/rand" + "os" + "path" + "testing" + "text/template" + + docker "github.com/fsouza/go-dockerclient" + cadvisorApi "github.com/google/cadvisor/info/v1" + + "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/client/record" + client "k8s.io/kubernetes/pkg/client/unversioned" + kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" + "k8s.io/kubernetes/pkg/kubelet/dockertools" + "k8s.io/kubernetes/pkg/kubelet/network" + "k8s.io/kubernetes/pkg/util/sets" +) + +// The temp dir where test plugins will be stored. +const testNetworkConfigPath = "/tmp/fake/plugins/net/cni" + +func installPluginUnderTest(t *testing.T, vendorName string, plugName string) { + pluginDir := path.Join(testNetworkConfigPath, plugName) + err := os.MkdirAll(pluginDir, 0777) + if err != nil { + t.Fatalf("Failed to create plugin config dir: %v", err) + } + pluginConfig := path.Join(pluginDir, plugName+".conf") + f, err := os.Create(pluginConfig) + if err != nil { + t.Fatalf("Failed to install plugin") + } + networkConfig := fmt.Sprintf("{ \"name\": \"%s\", \"type\": \"%s\" }", plugName, vendorName) + + _, err = f.WriteString(networkConfig) + if err != nil { + t.Fatalf("Failed to write network config file (%v)", err) + } + f.Close() + + vendorCNIDir := fmt.Sprintf(VendorCNIDirTemplate, vendorName) + err = os.MkdirAll(vendorCNIDir, 0777) + if err != nil { + t.Fatalf("Failed to create plugin dir: %v", err) + } + pluginExec := path.Join(vendorCNIDir, vendorName) + f, err = os.Create(pluginExec) + + const execScriptTempl = `#!/bin/bash +export $(echo ${CNI_ARGS} | sed 's/;/ /g') &> /dev/null +echo -n "$CNI_COMMAND $CNI_NETNS $K8S_POD_NAMESPACE $K8S_POD_NAME $K8S_POD_INFRA_CONTAINER_ID" >& {{.OutputFile}} +echo -n "{ \"ip4\": { \"ip\": \"10.1.0.23/24\" } }" +` + execTemplateData := &map[string]interface{}{ + "OutputFile": path.Join(pluginDir, plugName+".out"), + } + + tObj := template.Must(template.New("test").Parse(execScriptTempl)) + buf := &bytes.Buffer{} + if err := tObj.Execute(buf, *execTemplateData); err != nil { + t.Fatalf("Error in executing script template - %v", err) + } + execScript := buf.String() + _, err = f.WriteString(execScript) + if err != nil { + t.Fatalf("Failed to write plugin exec - %v", err) + } + + err = f.Chmod(0777) + if err != nil { + t.Fatalf("Failed to set exec perms on plugin") + } + + f.Close() +} + +func tearDownPlugin(plugName string, vendorName string) { + err := os.RemoveAll(testNetworkConfigPath) + if err != nil { + fmt.Printf("Error in cleaning up test: %v", err) + } + vendorCNIDir := fmt.Sprintf(VendorCNIDirTemplate, vendorName) + err = os.RemoveAll(vendorCNIDir) + if err != nil { + fmt.Printf("Error in cleaning up test: %v", err) + } +} + +type fakeNetworkHost struct { + kubeClient client.Interface +} + +func NewFakeHost(kubeClient client.Interface) *fakeNetworkHost { + host := &fakeNetworkHost{kubeClient: kubeClient} + return host +} + +func (fnh *fakeNetworkHost) GetPodByName(name, namespace string) (*api.Pod, bool) { + return nil, false +} + +func (fnh *fakeNetworkHost) GetKubeClient() client.Interface { + return nil +} + +func (nh *fakeNetworkHost) GetRuntime() kubecontainer.Runtime { + dm, fakeDockerClient := newTestDockerManager() + fakeDockerClient.Container = &docker.Container{ + ID: "foobar", + State: docker.State{Pid: 12345}, + } + return dm +} + +func newTestDockerManager() (*dockertools.DockerManager, *dockertools.FakeDockerClient) { + fakeDocker := &dockertools.FakeDockerClient{VersionInfo: docker.Env{"Version=1.1.3", "ApiVersion=1.15"}, Errors: make(map[string]error), RemovedImages: sets.String{}} + fakeRecorder := &record.FakeRecorder{} + readinessManager := kubecontainer.NewReadinessManager() + containerRefManager := kubecontainer.NewRefManager() + networkPlugin, _ := network.InitNetworkPlugin([]network.NetworkPlugin{}, "", network.NewFakeHost(nil)) + dockerManager := dockertools.NewFakeDockerManager( + fakeDocker, + fakeRecorder, + readinessManager, + containerRefManager, + &cadvisorApi.MachineInfo{}, + dockertools.PodInfraContainerImage, + 0, 0, "", + kubecontainer.FakeOS{}, + networkPlugin, + nil, + nil) + + return dockerManager, fakeDocker +} + +func TestCNIPlugin(t *testing.T) { + // install some random plugin + pluginName := fmt.Sprintf("test%d", rand.Intn(1000)) + vendorName := fmt.Sprintf("test_vendor%d", rand.Intn(1000)) + defer tearDownPlugin(pluginName, vendorName) + installPluginUnderTest(t, vendorName, pluginName) + + plug, err := network.InitNetworkPlugin(ProbeNetworkPlugins(path.Join(testNetworkConfigPath, pluginName)), "cni", NewFakeHost(nil)) + if err != nil { + t.Fatalf("Failed to select the desired plugin: %v", err) + } + + err = plug.SetUpPod("podNamespace", "podName", "dockerid2345") + if err != nil { + t.Errorf("Expected nil: %v", err) + } + output, err := ioutil.ReadFile(path.Join(testNetworkConfigPath, pluginName, pluginName+".out")) + expectedOutput := "ADD /proc/12345/ns/net podNamespace podName dockerid2345" + if string(output) != expectedOutput { + t.Errorf("Mismatch in expected output for setup hook. Expected '%s', got '%s'", expectedOutput, string(output)) + } + err = plug.TearDownPod("podNamespace", "podName", "dockerid4545454") + if err != nil { + t.Errorf("Expected nil: %v", err) + } + output, err = ioutil.ReadFile(path.Join(testNetworkConfigPath, pluginName, pluginName+".out")) + expectedOutput = "DEL /proc/12345/ns/net podNamespace podName dockerid4545454" + if string(output) != expectedOutput { + t.Errorf("Mismatch in expected output for setup hook. Expected '%s', got '%s'", expectedOutput, string(output)) + } +} diff --git a/pkg/kubelet/network/plugins.go b/pkg/kubelet/network/plugins.go index 5576f80a09..25c2224785 100644 --- a/pkg/kubelet/network/plugins.go +++ b/pkg/kubelet/network/plugins.go @@ -25,6 +25,7 @@ import ( "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/unversioned" client "k8s.io/kubernetes/pkg/client/unversioned" + kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" kubeletTypes "k8s.io/kubernetes/pkg/kubelet/types" "k8s.io/kubernetes/pkg/util/errors" "k8s.io/kubernetes/pkg/util/validation" @@ -73,6 +74,9 @@ type Host interface { // GetKubeClient returns a client interface GetKubeClient() client.Interface + + // GetContainerRuntime returns the container runtime that implements the containers (e.g. docker/rkt) + GetRuntime() kubecontainer.Runtime } // InitNetworkPlugin inits the plugin that matches networkPluginName. Plugins must have unique names. diff --git a/pkg/kubelet/network/testing.go b/pkg/kubelet/network/testing.go index 969fc7f6e7..6b82534e38 100644 --- a/pkg/kubelet/network/testing.go +++ b/pkg/kubelet/network/testing.go @@ -22,6 +22,7 @@ package network import ( "k8s.io/kubernetes/pkg/api" client "k8s.io/kubernetes/pkg/client/unversioned" + kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" ) type fakeNetworkHost struct { @@ -40,3 +41,7 @@ func (fnh *fakeNetworkHost) GetPodByName(name, namespace string) (*api.Pod, bool func (fnh *fakeNetworkHost) GetKubeClient() client.Interface { return nil } + +func (nh *fakeNetworkHost) GetRuntime() kubecontainer.Runtime { + return &kubecontainer.FakeRuntime{} +} diff --git a/pkg/kubelet/networks.go b/pkg/kubelet/networks.go index a88a9f125c..5ec7a00fe7 100644 --- a/pkg/kubelet/networks.go +++ b/pkg/kubelet/networks.go @@ -19,6 +19,7 @@ package kubelet import ( "k8s.io/kubernetes/pkg/api" client "k8s.io/kubernetes/pkg/client/unversioned" + kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" ) // This just exports required functions from kubelet proper, for use by network @@ -34,3 +35,7 @@ func (nh *networkHost) GetPodByName(name, namespace string) (*api.Pod, bool) { func (nh *networkHost) GetKubeClient() client.Interface { return nh.kubelet.kubeClient } + +func (nh *networkHost) GetRuntime() kubecontainer.Runtime { + return nh.kubelet.GetRuntime() +}