From 3745e0f88caf728d901cf6365ae790ed4dec0135 Mon Sep 17 00:00:00 2001 From: Angus Lees Date: Tue, 30 Aug 2016 12:26:25 +1000 Subject: [PATCH] openstack: Support config-drive for local metadata Config-drive is an alternate no-network method for publishing local instance metadata on OpenStack. This change implements support for fetching data from config-drive, and tries it before querying the network metadata service (since config-drive will fail quickly if not available). Note config-drive involves mounting the filesystem with label "config-2", so anyone using config-drive and running kubelet in a container will need to ensure /dev/disk/by-label/config-2 is available inside the container (read-only). --- .../providers/openstack/metadata.go | 156 ++++++++++++++++++ .../providers/openstack/metadata_test.go | 86 ++++++++++ .../providers/openstack/openstack.go | 56 +------ 3 files changed, 245 insertions(+), 53 deletions(-) create mode 100644 pkg/cloudprovider/providers/openstack/metadata.go create mode 100644 pkg/cloudprovider/providers/openstack/metadata_test.go diff --git a/pkg/cloudprovider/providers/openstack/metadata.go b/pkg/cloudprovider/providers/openstack/metadata.go new file mode 100644 index 0000000000..97948f45a7 --- /dev/null +++ b/pkg/cloudprovider/providers/openstack/metadata.go @@ -0,0 +1,156 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package openstack + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "net/http" + "os" + "path/filepath" + "strings" + + "github.com/golang/glog" + "k8s.io/kubernetes/pkg/util/exec" + "k8s.io/kubernetes/pkg/util/mount" +) + +// metadataUrl is URL to OpenStack metadata server. It's hardcoded IPv4 +// link-local address as documented in "OpenStack Cloud Administrator Guide", +// chapter Compute - Networking with nova-network. +// http://docs.openstack.org/admin-guide-cloud/compute-networking-nova.html#metadata-service +const metadataUrl = "http://169.254.169.254/openstack/2012-08-10/meta_data.json" + +// Config drive is defined as an iso9660 or vfat (deprecated) drive +// with the "config-2" label. +// http://docs.openstack.org/user-guide/cli-config-drive.html +const configDriveLabel = "config-2" +const configDrivePath = "openstack/2012-08-10/meta_data.json" + +var ErrBadMetadata = errors.New("Invalid OpenStack metadata, got empty uuid") + +// Assumes the "2012-08-10" meta_data.json format. +// See http://docs.openstack.org/user-guide/cli_config_drive.html +type Metadata struct { + Uuid string `json:"uuid"` + Name string `json:"name"` + AvailabilityZone string `json:"availability_zone"` + // .. and other fields we don't care about. Expand as necessary. +} + +// parseMetadataUUID reads JSON from OpenStack metadata server and parses +// instance ID out of it. +func parseMetadata(r io.Reader) (*Metadata, error) { + var metadata Metadata + json := json.NewDecoder(r) + if err := json.Decode(&metadata); err != nil { + return nil, err + } + + if metadata.Uuid == "" { + return nil, ErrBadMetadata + } + + return &metadata, nil +} + +func getMetadataFromConfigDrive() (*Metadata, error) { + // Try to read instance UUID from config drive. + dev := "/dev/disk/by-label/" + configDriveLabel + if _, err := os.Stat(dev); os.IsNotExist(err) { + out, err := exec.New().Command( + "blkid", "-l", + "-t", "LABEL="+configDriveLabel, + "-o", "device", + ).CombinedOutput() + if err != nil { + glog.V(2).Infof("Unable to run blkid: %v", err) + return nil, err + } + dev = strings.TrimSpace(string(out)) + } + + mntdir, err := ioutil.TempDir("", "configdrive") + if err != nil { + return nil, err + } + defer os.Remove(mntdir) + + glog.V(4).Infof("Attempting to mount configdrive %s on %s", dev, mntdir) + + mounter := mount.New() + err = mounter.Mount(dev, mntdir, "iso9660", []string{"ro"}) + if err != nil { + err = mounter.Mount(dev, mntdir, "vfat", []string{"ro"}) + } + if err != nil { + glog.Errorf("Error mounting configdrive %s: %v", dev, err) + return nil, err + } + defer mounter.Unmount(mntdir) + + glog.V(4).Infof("Configdrive mounted on %s", mntdir) + + f, err := os.Open( + filepath.Join(mntdir, configDrivePath)) + if err != nil { + glog.Errorf("Error reading %s on config drive: %v", configDrivePath, err) + return nil, err + } + defer f.Close() + + return parseMetadata(f) +} + +func getMetadataFromMetadataService() (*Metadata, error) { + // Try to get JSON from metdata server. + glog.V(4).Infof("Attempting to fetch metadata from %s", metadataUrl) + resp, err := http.Get(metadataUrl) + if err != nil { + glog.V(3).Infof("Cannot read %s: %v", metadataUrl, err) + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + err = fmt.Errorf("Unexpected status code when reading metadata from %s: %s", metadataUrl, resp.Status) + glog.V(3).Infof("%v", err) + return nil, err + } + + return parseMetadata(resp.Body) +} + +// Metadata is fixed for the current host, so cache the value process-wide +var metadataCache *Metadata + +func getMetadata() (*Metadata, error) { + if metadataCache == nil { + md, err := getMetadataFromConfigDrive() + if err != nil { + md, err = getMetadataFromMetadataService() + } + if err != nil { + return nil, err + } + metadataCache = md + } + return metadataCache, nil +} diff --git a/pkg/cloudprovider/providers/openstack/metadata_test.go b/pkg/cloudprovider/providers/openstack/metadata_test.go new file mode 100644 index 0000000000..feeb04b0f9 --- /dev/null +++ b/pkg/cloudprovider/providers/openstack/metadata_test.go @@ -0,0 +1,86 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package openstack + +import ( + "strings" + "testing" +) + +var FakeMetadata = Metadata{ + Uuid: "83679162-1378-4288-a2d4-70e13ec132aa", + Name: "test", + AvailabilityZone: "nova", +} + +func SetMetadataFixture(value *Metadata) { + metadataCache = value +} + +func ClearMetadata() { + metadataCache = nil +} + +func TestParseMetadata(t *testing.T) { + _, err := parseMetadata(strings.NewReader("bogus")) + if err == nil { + t.Errorf("Should fail when bad data is provided: %s", err) + } + + data := strings.NewReader(` +{ + "availability_zone": "nova", + "files": [ + { + "content_path": "/content/0000", + "path": "/etc/network/interfaces" + }, + { + "content_path": "/content/0001", + "path": "known_hosts" + } + ], + "hostname": "test.novalocal", + "launch_index": 0, + "name": "test", + "meta": { + "role": "webservers", + "essential": "false" + }, + "public_keys": { + "mykey": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQDBqUfVvCSez0/Wfpd8dLLgZXV9GtXQ7hnMN+Z0OWQUyebVEHey1CXuin0uY1cAJMhUq8j98SiW+cU0sU4J3x5l2+xi1bodDm1BtFWVeLIOQINpfV1n8fKjHB+ynPpe1F6tMDvrFGUlJs44t30BrujMXBe8Rq44cCk6wqyjATA3rQ== Generated by Nova\n" + }, + "uuid": "83679162-1378-4288-a2d4-70e13ec132aa" +} +`) + md, err := parseMetadata(data) + if err != nil { + t.Fatalf("Should succeed when provided with valid data: %s", err) + } + + if md.Name != "test" { + t.Errorf("incorrect name: %s", md.Name) + } + + if md.Uuid != "83679162-1378-4288-a2d4-70e13ec132aa" { + t.Errorf("incorrect uuid: %s", md.Uuid) + } + + if md.AvailabilityZone != "nova" { + t.Errorf("incorrect az: %s", md.AvailabilityZone) + } +} diff --git a/pkg/cloudprovider/providers/openstack/openstack.go b/pkg/cloudprovider/providers/openstack/openstack.go index 1ae83f18b2..a422df1aec 100644 --- a/pkg/cloudprovider/providers/openstack/openstack.go +++ b/pkg/cloudprovider/providers/openstack/openstack.go @@ -17,7 +17,6 @@ limitations under the License. package openstack import ( - "encoding/json" "errors" "fmt" "io" @@ -42,12 +41,6 @@ import ( const ProviderName = "openstack" -// metadataUrl is URL to OpenStack metadata server. It's hardcoded IPv4 -// link-local address as documented in "OpenStack Cloud Administrator Guide", -// chapter Compute - Networking with nova-network. -// http://docs.openstack.org/admin-guide-cloud/compute-networking-nova.html#metadata-service -const metadataUrl = "http://169.254.169.254/openstack/2012-08-10/meta_data.json" - var ErrNotFound = errors.New("Failed to find object") var ErrMultipleResults = errors.New("Multiple results where only one expected") var ErrNoAddressFound = errors.New("No address found for host") @@ -152,27 +145,6 @@ func readConfig(config io.Reader) (Config, error) { return cfg, err } -// parseMetadataUUID reads JSON from OpenStack metadata server and parses -// instance ID out of it. -func parseMetadataUUID(jsonData []byte) (string, error) { - // We should receive an object with { 'uuid': '' } and couple of other - // properties (which we ignore). - - obj := struct{ UUID string }{} - err := json.Unmarshal(jsonData, &obj) - if err != nil { - return "", err - } - - uuid := obj.UUID - if uuid == "" { - err = fmt.Errorf("cannot parse OpenStack metadata, got empty uuid") - return "", err - } - - return uuid, nil -} - func readInstanceID() (string, error) { // Try to find instance ID on the local filesystem (created by cloud-init) const instanceIDFile = "/var/lib/cloud/data/instance-id" @@ -184,37 +156,15 @@ func readInstanceID() (string, error) { if instanceID != "" { return instanceID, nil } - // Fall through with empty instanceID and try metadata server. + // Fall through to metadata server lookup } - glog.V(5).Infof("Cannot read %s: '%v', trying metadata server", instanceIDFile, err) - // Try to get JSON from metdata server. - resp, err := http.Get(metadataUrl) + md, err := getMetadata() if err != nil { - glog.V(3).Infof("Cannot read %s: %v", metadataUrl, err) return "", err } - if resp.StatusCode != 200 { - err = fmt.Errorf("got unexpected status code when reading metadata from %s: %s", metadataUrl, resp.Status) - glog.V(3).Infof("%v", err) - return "", err - } - - defer resp.Body.Close() - bodyBytes, err := ioutil.ReadAll(resp.Body) - if err != nil { - glog.V(3).Infof("Cannot get HTTP response body from %s: %v", metadataUrl, err) - return "", err - } - instanceID, err := parseMetadataUUID(bodyBytes) - if err != nil { - glog.V(3).Infof("Cannot parse instance ID from metadata from %s: %v", metadataUrl, err) - return "", err - } - - glog.V(3).Infof("Got instance id from %s: %s", metadataUrl, instanceID) - return instanceID, nil + return md.Uuid, nil } func newOpenStack(cfg Config) (*OpenStack, error) {