From b22ccc6780c843b29dbd0da6e44f20c56870b09b Mon Sep 17 00:00:00 2001 From: Miao Luo Date: Wed, 2 Nov 2016 22:31:47 -0700 Subject: [PATCH] Support persistent volume on Photon Controller platform 1. Enable Photon Controller as cloud provider 2. Support Photon persistent disk as volume source/persistent volume source --- api/swagger-spec/apps_v1alpha1.json | 21 + api/swagger-spec/batch_v1.json | 21 + api/swagger-spec/extensions_v1beta1.json | 21 + api/swagger-spec/v1.json | 25 + cmd/kube-controller-manager/app/BUILD | 2 + cmd/kube-controller-manager/app/plugins.go | 7 + cmd/kubelet/app/BUILD | 1 + cmd/kubelet/app/plugins.go | 2 + pkg/api/types.go | 14 + pkg/api/v1/types.go | 15 + pkg/api/validation/validation.go | 24 + pkg/apis/extensions/types.go | 1 + pkg/cloudprovider/providers/BUILD | 1 + pkg/cloudprovider/providers/photon/BUILD | 37 ++ pkg/cloudprovider/providers/photon/OWNERS | 4 + pkg/cloudprovider/providers/photon/photon.go | 573 ++++++++++++++++++ .../providers/photon/photon_test.go | 216 +++++++ pkg/cloudprovider/providers/providers.go | 1 + pkg/kubectl/describe.go | 12 + pkg/security/podsecuritypolicy/util/util.go | 5 +- pkg/volume/photon_pd/BUILD | 55 ++ pkg/volume/photon_pd/OWNERS | 4 + pkg/volume/photon_pd/attacher.go | 295 +++++++++ pkg/volume/photon_pd/attacher_test.go | 328 ++++++++++ pkg/volume/photon_pd/photon_pd.go | 387 ++++++++++++ pkg/volume/photon_pd/photon_pd_test.go | 239 ++++++++ pkg/volume/photon_pd/photon_util.go | 145 +++++ test/test_owners.csv | 2 + 28 files changed, 2457 insertions(+), 1 deletion(-) create mode 100644 pkg/cloudprovider/providers/photon/BUILD create mode 100644 pkg/cloudprovider/providers/photon/OWNERS create mode 100644 pkg/cloudprovider/providers/photon/photon.go create mode 100644 pkg/cloudprovider/providers/photon/photon_test.go create mode 100644 pkg/volume/photon_pd/BUILD create mode 100644 pkg/volume/photon_pd/OWNERS create mode 100644 pkg/volume/photon_pd/attacher.go create mode 100644 pkg/volume/photon_pd/attacher_test.go create mode 100644 pkg/volume/photon_pd/photon_pd.go create mode 100644 pkg/volume/photon_pd/photon_pd_test.go create mode 100644 pkg/volume/photon_pd/photon_util.go diff --git a/api/swagger-spec/apps_v1alpha1.json b/api/swagger-spec/apps_v1alpha1.json index c7a58ac05a..28b05ff8d4 100644 --- a/api/swagger-spec/apps_v1alpha1.json +++ b/api/swagger-spec/apps_v1alpha1.json @@ -1452,6 +1452,10 @@ "azureDisk": { "$ref": "v1.AzureDiskVolumeSource", "description": "AzureDisk represents an Azure Data Disk mount on the host and bind mount to the pod." + }, + "photonPersistentDisk": { + "$ref": "v1.PhotonPersistentDiskVolumeSource", + "description": "PhotonPersistentDisk represents a PhotonController persistent disk attached and mounted on kubelets host machine" } } }, @@ -2085,6 +2089,23 @@ "id": "v1.AzureDataDiskCachingMode", "properties": {} }, + "v1.PhotonPersistentDiskVolumeSource": { + "id": "v1.PhotonPersistentDiskVolumeSource", + "description": "Represents a Photon Controller persistent disk resource.", + "required": [ + "pdID" + ], + "properties": { + "pdID": { + "type": "string", + "description": "ID that identifies Photon Controller persistent disk" + }, + "fsType": { + "type": "string", + "description": "Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified." + } + } + }, "v1.Container": { "id": "v1.Container", "description": "A single application container that you want to run within a pod.", diff --git a/api/swagger-spec/batch_v1.json b/api/swagger-spec/batch_v1.json index 69f97e7c66..46b6765c57 100644 --- a/api/swagger-spec/batch_v1.json +++ b/api/swagger-spec/batch_v1.json @@ -1457,6 +1457,10 @@ "azureDisk": { "$ref": "v1.AzureDiskVolumeSource", "description": "AzureDisk represents an Azure Data Disk mount on the host and bind mount to the pod." + }, + "photonPersistentDisk": { + "$ref": "v1.PhotonPersistentDiskVolumeSource", + "description": "PhotonPersistentDisk represents a PhotonController persistent disk attached and mounted on kubelets host machine" } } }, @@ -2090,6 +2094,23 @@ "id": "v1.AzureDataDiskCachingMode", "properties": {} }, + "v1.PhotonPersistentDiskVolumeSource": { + "id": "v1.PhotonPersistentDiskVolumeSource", + "description": "Represents a Photon Controller persistent disk resource.", + "required": [ + "pdID" + ], + "properties": { + "pdID": { + "type": "string", + "description": "ID that identifies Photon Controller persistent disk" + }, + "fsType": { + "type": "string", + "description": "Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified." + } + } + }, "v1.Container": { "id": "v1.Container", "description": "A single application container that you want to run within a pod.", diff --git a/api/swagger-spec/extensions_v1beta1.json b/api/swagger-spec/extensions_v1beta1.json index 97f6b5e754..10ed09470b 100644 --- a/api/swagger-spec/extensions_v1beta1.json +++ b/api/swagger-spec/extensions_v1beta1.json @@ -8224,6 +8224,10 @@ "azureDisk": { "$ref": "v1.AzureDiskVolumeSource", "description": "AzureDisk represents an Azure Data Disk mount on the host and bind mount to the pod." + }, + "photonPersistentDisk": { + "$ref": "v1.PhotonPersistentDiskVolumeSource", + "description": "PhotonPersistentDisk represents a PhotonController persistent disk attached and mounted on kubelets host machine" } } }, @@ -8857,6 +8861,23 @@ "id": "v1.AzureDataDiskCachingMode", "properties": {} }, + "v1.PhotonPersistentDiskVolumeSource": { + "id": "v1.PhotonPersistentDiskVolumeSource", + "description": "Represents a Photon Controller persistent disk resource.", + "required": [ + "pdID" + ], + "properties": { + "pdID": { + "type": "string", + "description": "ID that identifies Photon Controller persistent disk" + }, + "fsType": { + "type": "string", + "description": "Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified." + } + } + }, "v1.Container": { "id": "v1.Container", "description": "A single application container that you want to run within a pod.", diff --git a/api/swagger-spec/v1.json b/api/swagger-spec/v1.json index 825e48cf59..7b8f1cda86 100644 --- a/api/swagger-spec/v1.json +++ b/api/swagger-spec/v1.json @@ -17654,6 +17654,10 @@ "$ref": "v1.AzureDiskVolumeSource", "description": "AzureDisk represents an Azure Data Disk mount on the host and bind mount to the pod." }, + "photonPersistentDisk": { + "$ref": "v1.PhotonPersistentDiskVolumeSource", + "description": "PhotonPersistentDisk represents a PhotonController persistent disk attached and mounted on kubelets host machine" + }, "accessModes": { "type": "array", "items": { @@ -18104,6 +18108,23 @@ "id": "v1.AzureDataDiskCachingMode", "properties": {} }, + "v1.PhotonPersistentDiskVolumeSource": { + "id": "v1.PhotonPersistentDiskVolumeSource", + "description": "Represents a Photon Controller persistent disk resource.", + "required": [ + "pdID" + ], + "properties": { + "pdID": { + "type": "string", + "description": "ID that identifies Photon Controller persistent disk" + }, + "fsType": { + "type": "string", + "description": "Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified." + } + } + }, "v1.PersistentVolumeStatus": { "id": "v1.PersistentVolumeStatus", "description": "PersistentVolumeStatus is the current status of a persistent volume.", @@ -18362,6 +18383,10 @@ "azureDisk": { "$ref": "v1.AzureDiskVolumeSource", "description": "AzureDisk represents an Azure Data Disk mount on the host and bind mount to the pod." + }, + "photonPersistentDisk": { + "$ref": "v1.PhotonPersistentDiskVolumeSource", + "description": "PhotonPersistentDisk represents a PhotonController persistent disk attached and mounted on kubelets host machine" } } }, diff --git a/cmd/kube-controller-manager/app/BUILD b/cmd/kube-controller-manager/app/BUILD index 924f74aaa0..e92a609333 100644 --- a/cmd/kube-controller-manager/app/BUILD +++ b/cmd/kube-controller-manager/app/BUILD @@ -38,6 +38,7 @@ go_library( "//pkg/cloudprovider/providers/azure:go_default_library", "//pkg/cloudprovider/providers/gce:go_default_library", "//pkg/cloudprovider/providers/openstack:go_default_library", + "//pkg/cloudprovider/providers/photon:go_default_library", "//pkg/cloudprovider/providers/vsphere:go_default_library", "//pkg/controller:go_default_library", "//pkg/controller/certificates:go_default_library", @@ -83,6 +84,7 @@ go_library( "//pkg/volume/glusterfs:go_default_library", "//pkg/volume/host_path:go_default_library", "//pkg/volume/nfs:go_default_library", + "//pkg/volume/photon_pd:go_default_library", "//pkg/volume/quobyte:go_default_library", "//pkg/volume/rbd:go_default_library", "//pkg/volume/vsphere_volume:go_default_library", diff --git a/cmd/kube-controller-manager/app/plugins.go b/cmd/kube-controller-manager/app/plugins.go index 0ba397e8e1..6b3a833f8d 100644 --- a/cmd/kube-controller-manager/app/plugins.go +++ b/cmd/kube-controller-manager/app/plugins.go @@ -34,6 +34,7 @@ import ( "k8s.io/kubernetes/pkg/cloudprovider/providers/azure" "k8s.io/kubernetes/pkg/cloudprovider/providers/gce" "k8s.io/kubernetes/pkg/cloudprovider/providers/openstack" + "k8s.io/kubernetes/pkg/cloudprovider/providers/photon" "k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere" utilconfig "k8s.io/kubernetes/pkg/util/config" "k8s.io/kubernetes/pkg/util/io" @@ -47,6 +48,7 @@ import ( "k8s.io/kubernetes/pkg/volume/glusterfs" "k8s.io/kubernetes/pkg/volume/host_path" "k8s.io/kubernetes/pkg/volume/nfs" + "k8s.io/kubernetes/pkg/volume/photon_pd" "k8s.io/kubernetes/pkg/volume/quobyte" "k8s.io/kubernetes/pkg/volume/rbd" "k8s.io/kubernetes/pkg/volume/vsphere_volume" @@ -67,6 +69,7 @@ func ProbeAttachableVolumePlugins(config componentconfig.VolumeConfiguration) [] allPlugins = append(allPlugins, flexvolume.ProbeVolumePlugins(config.FlexVolumePluginDir)...) allPlugins = append(allPlugins, vsphere_volume.ProbeVolumePlugins()...) allPlugins = append(allPlugins, azure_dd.ProbeVolumePlugins()...) + allPlugins = append(allPlugins, photon_pd.ProbeVolumePlugins()...) return allPlugins } @@ -124,6 +127,8 @@ func ProbeControllerVolumePlugins(cloud cloudprovider.Interface, config componen allPlugins = append(allPlugins, vsphere_volume.ProbeVolumePlugins()...) case azure.CloudProviderName == cloud.ProviderName(): allPlugins = append(allPlugins, azure_dd.ProbeVolumePlugins()...) + case photon.ProviderName == cloud.ProviderName(): + allPlugins = append(allPlugins, photon_pd.ProbeVolumePlugins()...) } } @@ -154,6 +159,8 @@ func NewAlphaVolumeProvisioner(cloud cloudprovider.Interface, config componentco return getProvisionablePluginFromVolumePlugins(vsphere_volume.ProbeVolumePlugins()) case cloud != nil && azure.CloudProviderName == cloud.ProviderName(): return getProvisionablePluginFromVolumePlugins(azure_dd.ProbeVolumePlugins()) + case cloud != nil && photon.ProviderName == cloud.ProviderName(): + return getProvisionablePluginFromVolumePlugins(photon_pd.ProbeVolumePlugins()) } return nil, nil } diff --git a/cmd/kubelet/app/BUILD b/cmd/kubelet/app/BUILD index 650a3fd4c5..731f63ef85 100644 --- a/cmd/kubelet/app/BUILD +++ b/cmd/kubelet/app/BUILD @@ -93,6 +93,7 @@ go_library( "//pkg/volume/host_path:go_default_library", "//pkg/volume/iscsi:go_default_library", "//pkg/volume/nfs:go_default_library", + "//pkg/volume/photon_pd:go_default_library", "//pkg/volume/quobyte:go_default_library", "//pkg/volume/rbd:go_default_library", "//pkg/volume/secret:go_default_library", diff --git a/cmd/kubelet/app/plugins.go b/cmd/kubelet/app/plugins.go index 833198118a..d125852622 100644 --- a/cmd/kubelet/app/plugins.go +++ b/cmd/kubelet/app/plugins.go @@ -45,6 +45,7 @@ import ( "k8s.io/kubernetes/pkg/volume/host_path" "k8s.io/kubernetes/pkg/volume/iscsi" "k8s.io/kubernetes/pkg/volume/nfs" + "k8s.io/kubernetes/pkg/volume/photon_pd" "k8s.io/kubernetes/pkg/volume/quobyte" "k8s.io/kubernetes/pkg/volume/rbd" "k8s.io/kubernetes/pkg/volume/secret" @@ -86,6 +87,7 @@ func ProbeVolumePlugins(pluginDir string) []volume.VolumePlugin { allPlugins = append(allPlugins, configmap.ProbeVolumePlugins()...) allPlugins = append(allPlugins, vsphere_volume.ProbeVolumePlugins()...) allPlugins = append(allPlugins, azure_dd.ProbeVolumePlugins()...) + allPlugins = append(allPlugins, photon_pd.ProbeVolumePlugins()...) return allPlugins } diff --git a/pkg/api/types.go b/pkg/api/types.go index cb864b79f2..6599fd6b3d 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -289,6 +289,8 @@ type VolumeSource struct { // AzureDisk represents an Azure Data Disk mount on the host and bind mount to the pod. // +optional AzureDisk *AzureDiskVolumeSource `json:"azureDisk,omitempty"` + // PhotonPersistentDisk represents a Photon Controller persistent disk attached and mounted on kubelets host machine + PhotonPersistentDisk *PhotonPersistentDiskVolumeSource `json:"photonPersistentDisk,omitempty"` } // Similar to VolumeSource but meant for the administrator who creates PVs. @@ -349,6 +351,8 @@ type PersistentVolumeSource struct { // AzureDisk represents an Azure Data Disk mount on the host and bind mount to the pod. // +optional AzureDisk *AzureDiskVolumeSource `json:"azureDisk,omitempty"` + // PhotonPersistentDisk represents a Photon Controller persistent disk attached and mounted on kubelets host machine + PhotonPersistentDisk *PhotonPersistentDiskVolumeSource `json:"photonPersistentDisk,omitempty"` } type PersistentVolumeClaimVolumeSource struct { @@ -936,6 +940,16 @@ type VsphereVirtualDiskVolumeSource struct { FSType string `json:"fsType,omitempty"` } +// Represents a Photon Controller persistent disk resource. +type PhotonPersistentDiskVolumeSource struct { + // ID that identifies Photon Controller persistent disk + PdID string `json:"pdID"` + // Filesystem type to mount. + // Must be a filesystem type supported by the host operating system. + // Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + FSType string `json:"fsType,omitempty"` +} + type AzureDataDiskCachingMode string const ( diff --git a/pkg/api/v1/types.go b/pkg/api/v1/types.go index cfa692572b..4d729bd9e8 100644 --- a/pkg/api/v1/types.go +++ b/pkg/api/v1/types.go @@ -322,6 +322,8 @@ type VolumeSource struct { // AzureDisk represents an Azure Data Disk mount on the host and bind mount to the pod. // +optional AzureDisk *AzureDiskVolumeSource `json:"azureDisk,omitempty" protobuf:"bytes,22,opt,name=azureDisk"` + // PhotonPersistentDisk represents a PhotonController persistent disk attached and mounted on kubelets host machine + PhotonPersistentDisk *PhotonPersistentDiskVolumeSource `json:"photonPersistentDisk,omitempty" protobuf:"bytes,23,opt,name=photonPersistentDisk"` } // PersistentVolumeClaimVolumeSource references the user's PVC in the same namespace. @@ -405,6 +407,8 @@ type PersistentVolumeSource struct { // AzureDisk represents an Azure Data Disk mount on the host and bind mount to the pod. // +optional AzureDisk *AzureDiskVolumeSource `json:"azureDisk,omitempty" protobuf:"bytes,16,opt,name=azureDisk"` + // PhotonPersistentDisk represents a PhotonController persistent disk attached and mounted on kubelets host machine + PhotonPersistentDisk *PhotonPersistentDiskVolumeSource `json:"photonPersistentDisk,omitempty" protobuf:"bytes,17,opt,name=photonPersistentDisk"` } // +genclient=true @@ -1023,6 +1027,17 @@ type VsphereVirtualDiskVolumeSource struct { // +optional FSType string `json:"fsType,omitempty" protobuf:"bytes,2,opt,name=fsType"` } + +// Represents a Photon Controller persistent disk resource. +type PhotonPersistentDiskVolumeSource struct { + // ID that identifies Photon Controller persistent disk + PdID string `json:"pdID" protobuf:"bytes,1,opt,name=pdID"` + // Filesystem type to mount. + // Must be a filesystem type supported by the host operating system. + // Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + FSType string `json:"fsType,omitempty" protobuf:"bytes,2,opt,name=fsType"` +} + type AzureDataDiskCachingMode string const ( diff --git a/pkg/api/validation/validation.go b/pkg/api/validation/validation.go index 412b597060..3def988307 100644 --- a/pkg/api/validation/validation.go +++ b/pkg/api/validation/validation.go @@ -665,6 +665,14 @@ func validateVolumeSource(source *api.VolumeSource, fldPath *field.Path) field.E allErrs = append(allErrs, validateVsphereVolumeSource(source.VsphereVolume, fldPath.Child("vsphereVolume"))...) } } + if source.PhotonPersistentDisk != nil { + if numVolumes > 0 { + allErrs = append(allErrs, field.Forbidden(fldPath.Child("photonPersistentDisk"), "may not specify more than 1 volume type")) + } else { + numVolumes++ + allErrs = append(allErrs, validatePhotonPersistentDiskVolumeSource(source.PhotonPersistentDisk, fldPath.Child("photonPersistentDisk"))...) + } + } if source.AzureDisk != nil { numVolumes++ allErrs = append(allErrs, validateAzureDisk(source.AzureDisk, fldPath.Child("azureDisk"))...) @@ -1008,6 +1016,14 @@ func validateVsphereVolumeSource(cd *api.VsphereVirtualDiskVolumeSource, fldPath return allErrs } +func validatePhotonPersistentDiskVolumeSource(cd *api.PhotonPersistentDiskVolumeSource, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + if len(cd.PdID) == 0 { + allErrs = append(allErrs, field.Required(fldPath.Child("pdID"), "")) + } + return allErrs +} + // ValidatePersistentVolumeName checks that a name is appropriate for a // PersistentVolumeName object. var ValidatePersistentVolumeName = NameIsDNSSubdomain @@ -1159,6 +1175,14 @@ func ValidatePersistentVolume(pv *api.PersistentVolume) field.ErrorList { allErrs = append(allErrs, validateVsphereVolumeSource(pv.Spec.VsphereVolume, specPath.Child("vsphereVolume"))...) } } + if pv.Spec.PhotonPersistentDisk != nil { + if numVolumes > 0 { + allErrs = append(allErrs, field.Forbidden(specPath.Child("photonPersistentDisk"), "may not specify more than 1 volume type")) + } else { + numVolumes++ + allErrs = append(allErrs, validatePhotonPersistentDiskVolumeSource(pv.Spec.PhotonPersistentDisk, specPath.Child("photonPersistentDisk"))...) + } + } if pv.Spec.AzureDisk != nil { numVolumes++ allErrs = append(allErrs, validateAzureDisk(pv.Spec.AzureDisk, specPath.Child("azureDisk"))...) diff --git a/pkg/apis/extensions/types.go b/pkg/apis/extensions/types.go index f80ac23c45..ded03596dc 100644 --- a/pkg/apis/extensions/types.go +++ b/pkg/apis/extensions/types.go @@ -898,6 +898,7 @@ var ( VsphereVolume FSType = "vsphereVolume" Quobyte FSType = "quobyte" AzureDisk FSType = "azureDisk" + PhotonPersistentDisk FSType = "photonPersistentDisk" All FSType = "*" ) diff --git a/pkg/cloudprovider/providers/BUILD b/pkg/cloudprovider/providers/BUILD index 8617ca2fed..e49fa11427 100644 --- a/pkg/cloudprovider/providers/BUILD +++ b/pkg/cloudprovider/providers/BUILD @@ -22,6 +22,7 @@ go_library( "//pkg/cloudprovider/providers/mesos:go_default_library", "//pkg/cloudprovider/providers/openstack:go_default_library", "//pkg/cloudprovider/providers/ovirt:go_default_library", + "//pkg/cloudprovider/providers/photon:go_default_library", "//pkg/cloudprovider/providers/rackspace:go_default_library", "//pkg/cloudprovider/providers/vsphere:go_default_library", ], diff --git a/pkg/cloudprovider/providers/photon/BUILD b/pkg/cloudprovider/providers/photon/BUILD new file mode 100644 index 0000000000..f0fcbc78f9 --- /dev/null +++ b/pkg/cloudprovider/providers/photon/BUILD @@ -0,0 +1,37 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_binary", + "go_library", + "go_test", + "cgo_library", +) + +go_library( + name = "go_default_library", + srcs = ["photon.go"], + tags = ["automanaged"], + deps = [ + "//pkg/api:go_default_library", + "//pkg/cloudprovider:go_default_library", + "//pkg/types:go_default_library", + "//vendor:github.com/golang/glog", + "//vendor:github.com/vmware/photon-controller-go-sdk/photon", + "//vendor:gopkg.in/gcfg.v1", + ], +) + +go_test( + name = "go_default_test", + srcs = ["photon_test.go"], + library = "go_default_library", + tags = ["automanaged"], + deps = [ + "//pkg/cloudprovider:go_default_library", + "//pkg/types:go_default_library", + "//pkg/util/rand:go_default_library", + ], +) diff --git a/pkg/cloudprovider/providers/photon/OWNERS b/pkg/cloudprovider/providers/photon/OWNERS new file mode 100644 index 0000000000..b8170a5d6d --- /dev/null +++ b/pkg/cloudprovider/providers/photon/OWNERS @@ -0,0 +1,4 @@ +maintainers: +- luomiao +- kerneltime +- abrarshivani diff --git a/pkg/cloudprovider/providers/photon/photon.go b/pkg/cloudprovider/providers/photon/photon.go new file mode 100644 index 0000000000..3bf5e75578 --- /dev/null +++ b/pkg/cloudprovider/providers/photon/photon.go @@ -0,0 +1,573 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// This version of Photon cloud provider supports the disk interface +// for Photon persistent disk volume plugin. LoadBalancer, Routes, and +// Zones are currently not supported. +// The use of Photon cloud provider requires to start kubelet, kube-apiserver, +// and kube-controller-manager with config flag: '--cloud-provider=photon +// --cloud-config=[path_to_config_file]'. When running multi-node kubernetes +// using docker, the config file should be located inside /etc/kubernetes. +package photon + +import ( + "bytes" + "errors" + "fmt" + "github.com/golang/glog" + "github.com/vmware/photon-controller-go-sdk/photon" + "gopkg.in/gcfg.v1" + "io" + "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/cloudprovider" + k8stypes "k8s.io/kubernetes/pkg/types" + "log" + "os/exec" + "strings" +) + +const ( + ProviderName = "photon" + DiskSpecKind = "persistent-disk" +) + +// Global variable pointing to photon client +var photonClient *photon.Client +var logger *log.Logger = nil + +// overrideIP indicates if the hostname is overriden by IP address, such as when +// running multi-node kubernetes using docker. In this case the user should set +// overrideIP = true in cloud config file. Default value is false. +var overrideIP bool = false + +// Photon is an implementation of the cloud provider interface for Photon Controller. +type PCCloud struct { + cfg *PCConfig + // InstanceID of the server where this PCCloud object is instantiated. + localInstanceID string + // local $HOSTNAME + localHostname string + // hostname from K8S, could be overridden + localK8sHostname string + // Photon project ID. We assume that there is only one Photon Controller project + // in the environment per current Photon Controller deployment methodology. + projID string + cloudprovider.Zone +} + +type PCConfig struct { + Global struct { + // the Photon Controller endpoint IP address + CloudTarget string `gcfg:"target"` + // when the Photon Controller authentication is enabled, set to true; + // otherwise, set to false. + IgnoreCertificate bool `gcfg:"ignoreCertificate"` + // Photon Controller tenant name + Tenant string `gcfg:"tenant"` + // Photon Controller project name + Project string `gcfg:"project"` + // when kubelet is started with '--hostname-override=${IP_ADDRESS}', set to true; + // otherwise, set to false. + OverrideIP bool `gcfg:"overrideIP"` + } +} + +// Disks is interface for manipulation with PhotonController Persistent Disks. +type Disks interface { + // AttachDisk attaches given disk to given node. Current node + // is used when nodeName is empty string. + AttachDisk(pdID string, nodeName k8stypes.NodeName) error + + // DetachDisk detaches given disk to given node. Current node + // is used when nodeName is empty string. + DetachDisk(pdID string, nodeName k8stypes.NodeName) error + + // DiskIsAttached checks if a disk is attached to the given node. + DiskIsAttached(pdID string, nodeName k8stypes.NodeName) (bool, error) + + // DisksAreAttached is a batch function to check if a list of disks are attached + // to the node with the specified NodeName. + DisksAreAttached(pdIDs []string, nodeName k8stypes.NodeName) (map[string]bool, error) + + // CreateDisk creates a new PD with given properties. + CreateDisk(volumeOptions *VolumeOptions) (pdID string, err error) + + // DeleteDisk deletes PD. + DeleteDisk(pdID string) error +} + +// VolumeOptions specifies capacity, tags, name and flavorID for a volume. +type VolumeOptions struct { + CapacityGB int + Tags map[string]string + Name string + Flavor string +} + +func logError(msg string, err error) error { + s := "Photon Cloud Provider: " + msg + ". Error [" + err.Error() + "]" + glog.Errorf(s) + return fmt.Errorf(s) +} + +func readConfig(config io.Reader) (PCConfig, error) { + if config == nil { + err := fmt.Errorf("cloud provider config file is missing. Please restart kubelet with --cloud-provider=photon --cloud-config=[path_to_config_file]") + return PCConfig{}, err + } + + var cfg PCConfig + err := gcfg.ReadInto(&cfg, config) + return cfg, err +} + +func init() { + cloudprovider.RegisterCloudProvider(ProviderName, func(config io.Reader) (cloudprovider.Interface, error) { + cfg, err := readConfig(config) + if err != nil { + return nil, logError("failed to read in cloud provider config file", err) + } + return newPCCloud(cfg) + }) +} + +// Retrieve the Photon VM ID from the Photon Controller endpoint based on the node name +func getVMIDbyNodename(project string, nodeName string) (string, error) { + vmList, err := photonClient.Projects.GetVMs(project, nil) + if err != nil { + return "", fmt.Errorf("Failed to GetVMs from project %s with nodeName %s, error: [%v]", project, nodeName, err) + } + + for _, vm := range vmList.Items { + if vm.Name == nodeName { + if vm.State == "STARTED" { + return vm.ID, nil + } + } + } + + return "", fmt.Errorf("No matching started VM is found with name %s", nodeName) +} + +// Retrieve the Photon VM ID from the Photon Controller endpoint based on the IP address +func getVMIDbyIP(project string, IPAddress string) (string, error) { + vmList, err := photonClient.Projects.GetVMs(project, nil) + if err != nil { + return "", fmt.Errorf("Failed to GetVMs for project %s, error [%v]", project, err) + } + + for _, vm := range vmList.Items { + task, err := photonClient.VMs.GetNetworks(vm.ID) + if err != nil { + glog.Warningf("Photon Cloud Provider: GetNetworks failed for vm.ID %s, error [%v]", vm.ID, err) + } else { + task, err = photonClient.Tasks.Wait(task.ID) + if err != nil { + glog.Warning("Photon Cloud Provider: Wait task for GetNetworks failed for vm.ID %s, error [%v]", vm.ID, err) + } else { + networkConnections := task.ResourceProperties.(map[string]interface{}) + networks := networkConnections["networkConnections"].([]interface{}) + for _, nt := range networks { + network := nt.(map[string]interface{}) + if val, ok := network["ipAddress"]; ok && val != nil { + ipAddr := val.(string) + if ipAddr == IPAddress { + return vm.ID, nil + } + } + } + } + } + } + + return "", fmt.Errorf("No matching VM is found with IP %s", IPAddress) +} + +// Retrieve the the Photon project ID from the Photon Controller endpoint based on the project name +func getProjIDbyName(tenantName, projName string) (string, error) { + tenants, err := photonClient.Tenants.GetAll() + if err != nil { + return "", fmt.Errorf("GetAll tenants failed with error [%v]", err) + } + + for _, tenant := range tenants.Items { + if tenant.Name == tenantName { + projects, err := photonClient.Tenants.GetProjects(tenant.ID, nil) + if err != nil { + return "", fmt.Errorf("Failed to GetProjects for tenant %s, error [%v]", tenantName, err) + } + + for _, project := range projects.Items { + if project.Name == projName { + return project.ID, nil + } + } + } + } + + return "", fmt.Errorf("No matching tenant/project name is found with %s/%s", tenantName, projName) +} + +func newPCCloud(cfg PCConfig) (*PCCloud, error) { + if len(cfg.Global.CloudTarget) == 0 { + return nil, fmt.Errorf("Photon Controller endpoint was not specified.") + } + + //TODO: add handling of certification enabled situation + options := &photon.ClientOptions{ + IgnoreCertificate: cfg.Global.IgnoreCertificate, + } + + photonClient = photon.NewClient(cfg.Global.CloudTarget, options, logger) + status, err := photonClient.Status.Get() + if err != nil { + return nil, logError("new client creation failed", err) + } + glog.V(2).Info("Photon Cloud Provider: Status of the new photon controller client: %v", status) + + // Get Photon Controller project ID for future use + projID, err := getProjIDbyName(cfg.Global.Tenant, cfg.Global.Project) + if err != nil { + return nil, logError("getProjIDbyName failed when creating new Photon Controller client", err) + } + + // Get local hostname for localInstanceID + cmd := exec.Command("bash", "-c", `echo $HOSTNAME`) + var out bytes.Buffer + cmd.Stdout = &out + err = cmd.Run() + if err != nil { + return nil, logError("get local hostname bash command failed", err) + } + if out.Len() == 0 { + return nil, logError("unable to retrieve hostname for Instance ID", nil) + } + hostname := strings.TrimRight(out.String(), "\n") + vmID, err := getVMIDbyNodename(projID, hostname) + if err != nil { + return nil, logError("getVMIDbyNodename failed when creating new Photon Controller client", err) + } + + pc := PCCloud{ + cfg: &cfg, + localInstanceID: vmID, + localHostname: hostname, + localK8sHostname: "", + projID: projID, + } + + overrideIP = cfg.Global.OverrideIP + + return &pc, nil +} + +// Instances returns an implementation of Instances for Photon Controller. +func (pc *PCCloud) Instances() (cloudprovider.Instances, bool) { + return pc, true +} + +// List is an implementation of Instances.List. +func (pc *PCCloud) List(filter string) ([]k8stypes.NodeName, error) { + return nil, nil +} + +// NodeAddresses is an implementation of Instances.NodeAddresses. +func (pc *PCCloud) NodeAddresses(nodeName k8stypes.NodeName) ([]api.NodeAddress, error) { + addrs := []api.NodeAddress{} + name := string(nodeName) + + var vmID string + var err error + if name == pc.localK8sHostname { + vmID = pc.localInstanceID + } else { + vmID, err = getInstanceID(name, pc.projID) + if err != nil { + return addrs, logError("getInstanceID failed for NodeAddresses", err) + } + } + + // Retrieve the Photon VM's IP addresses from the Photon Controller endpoint based on the VM ID + vmList, err := photonClient.Projects.GetVMs(pc.projID, nil) + if err != nil { + return addrs, fmt.Errorf("Photon Cloud Provider: Failed to GetVMs for project %s, error [%v]", pc.projID, err) + } + + for _, vm := range vmList.Items { + if vm.ID == vmID { + task, err := photonClient.VMs.GetNetworks(vm.ID) + if err != nil { + return addrs, logError("GetNetworks failed for node "+name+" with vm.ID "+vm.ID, err) + } else { + task, err = photonClient.Tasks.Wait(task.ID) + if err != nil { + return addrs, logError("Wait task for GetNetworks failed for node"+name+" with vm.ID "+vm.ID, err) + } else { + networkConnections := task.ResourceProperties.(map[string]interface{}) + networks := networkConnections["networkConnections"].([]interface{}) + for _, nt := range networks { + network := nt.(map[string]interface{}) + if val, ok := network["ipAddress"]; ok && val != nil { + ipAddr := val.(string) + if ipAddr != "-" { + api.AddToNodeAddresses(&addrs, + api.NodeAddress{ + // TODO: figure out the type of the IP + Type: api.NodeInternalIP, + Address: ipAddr, + }, + ) + } + } + } + return addrs, nil + } + } + } + } + + return addrs, logError("Failed to find the node "+name+" from Photon Controller endpoint", nil) +} + +func (pc *PCCloud) AddSSHKeyToAllInstances(user string, keyData []byte) error { + return errors.New("unimplemented") +} + +func (pc *PCCloud) CurrentNodeName(hostname string) (k8stypes.NodeName, error) { + pc.localK8sHostname = hostname + return k8stypes.NodeName(hostname), nil +} + +func getInstanceID(name string, projID string) (string, error) { + var vmID string + var err error + + if overrideIP == true { + vmID, err = getVMIDbyIP(projID, name) + } else { + vmID, err = getVMIDbyNodename(projID, name) + } + if err != nil { + return "", err + } + + if vmID == "" { + err = cloudprovider.InstanceNotFound + } + + return vmID, err +} + +// ExternalID returns the cloud provider ID of the specified instance (deprecated). +func (pc *PCCloud) ExternalID(nodeName k8stypes.NodeName) (string, error) { + name := string(nodeName) + if name == pc.localK8sHostname { + return pc.localInstanceID, nil + } else { + ID, err := getInstanceID(name, pc.projID) + if err != nil { + return ID, logError("getInstanceID failed for ExternalID", err) + } else { + return ID, nil + } + } +} + +// InstanceID returns the cloud provider ID of the specified instance. +func (pc *PCCloud) InstanceID(nodeName k8stypes.NodeName) (string, error) { + name := string(nodeName) + if name == pc.localK8sHostname { + return pc.localInstanceID, nil + } else { + ID, err := getInstanceID(name, pc.projID) + if err != nil { + return ID, logError("getInstanceID failed for InstanceID", err) + } else { + return ID, nil + } + } +} + +func (pc *PCCloud) InstanceType(nodeName k8stypes.NodeName) (string, error) { + return "", nil +} + +func (pc *PCCloud) Clusters() (cloudprovider.Clusters, bool) { + return nil, true +} + +// ProviderName returns the cloud provider ID. +func (pc *PCCloud) ProviderName() string { + return ProviderName +} + +// LoadBalancer returns an implementation of LoadBalancer for Photon Controller. +func (pc *PCCloud) LoadBalancer() (cloudprovider.LoadBalancer, bool) { + return nil, false +} + +// Zones returns an implementation of Zones for Photon Controller. +func (pc *PCCloud) Zones() (cloudprovider.Zones, bool) { + glog.V(4).Info("Claiming to support Zones") + return pc, true +} + +func (pc *PCCloud) GetZone() (cloudprovider.Zone, error) { + return pc.Zone, nil +} + +// Routes returns a false since the interface is not supported for photon controller. +func (pc *PCCloud) Routes() (cloudprovider.Routes, bool) { + return nil, false +} + +// ScrubDNS filters DNS settings for pods. +func (pc *PCCloud) ScrubDNS(nameservers, searches []string) (nsOut, srchOut []string) { + return nameservers, searches +} + +// Attaches given virtual disk volume to the compute running kubelet. +func (pc *PCCloud) AttachDisk(pdID string, nodeName k8stypes.NodeName) error { + operation := &photon.VmDiskOperation{ + DiskID: pdID, + } + + vmID, err := pc.InstanceID(nodeName) + if err != nil { + return logError("pc.InstanceID failed for AttachDisk", err) + } + + task, err := photonClient.VMs.AttachDisk(vmID, operation) + if err != nil { + return logError("Failed to attach disk with pdID "+pdID, err) + } + + _, err = photonClient.Tasks.Wait(task.ID) + if err != nil { + return logError("Failed to wait for task to attach disk with pdID "+pdID, err) + } + + return nil +} + +// Detaches given virtual disk volume from the compute running kubelet. +func (pc *PCCloud) DetachDisk(pdID string, nodeName k8stypes.NodeName) error { + operation := &photon.VmDiskOperation{ + DiskID: pdID, + } + + vmID, err := pc.InstanceID(nodeName) + if err != nil { + return logError("pc.InstanceID failed for DetachDisk", err) + } + + task, err := photonClient.VMs.DetachDisk(vmID, operation) + if err != nil { + return logError("Failed to detach disk with pdID "+pdID, err) + } + + _, err = photonClient.Tasks.Wait(task.ID) + if err != nil { + return logError("Failed to wait for task to detach disk with pdID "+pdID, err) + } + + return nil +} + +// DiskIsAttached returns if disk is attached to the VM using controllers supported by the plugin. +func (pc *PCCloud) DiskIsAttached(pdID string, nodeName k8stypes.NodeName) (bool, error) { + disk, err := photonClient.Disks.Get(pdID) + if err != nil { + return false, logError("Failed to Get disk with pdID "+pdID, err) + } + + vmID, err := pc.InstanceID(nodeName) + if err != nil { + return false, logError("pc.InstanceID failed for DiskIsAttached", err) + } + + for _, vm := range disk.VMs { + if strings.Compare(vm, vmID) == 0 { + return true, nil + } + } + + return false, nil +} + +// DisksAreAttached returns if disks are attached to the VM using controllers supported by the plugin. +func (pc *PCCloud) DisksAreAttached(pdIDs []string, nodeName k8stypes.NodeName) (map[string]bool, error) { + attached := make(map[string]bool) + for _, pdID := range pdIDs { + attached[pdID] = false + } + + vmID, err := pc.InstanceID(nodeName) + if err != nil { + return attached, logError("pc.InstanceID failed for DiskIsAttached", err) + } + + for _, pdID := range pdIDs { + disk, err := photonClient.Disks.Get(pdID) + if err != nil { + glog.Warningf("Photon Cloud Provider: failed to get VMs for persistent disk %s, err [%v]", pdID, err) + } else { + for _, vm := range disk.VMs { + if strings.Compare(vm, vmID) == 0 { + attached[pdID] = true + } + } + } + } + + return attached, nil +} + +// Create a volume of given size (in GB). +func (pc *PCCloud) CreateDisk(volumeOptions *VolumeOptions) (pdID string, err error) { + diskSpec := photon.DiskCreateSpec{} + diskSpec.Name = volumeOptions.Name + diskSpec.Flavor = volumeOptions.Flavor + diskSpec.CapacityGB = volumeOptions.CapacityGB + diskSpec.Kind = DiskSpecKind + + task, err := photonClient.Projects.CreateDisk(pc.projID, &diskSpec) + if err != nil { + return "", logError("Failed to CreateDisk", err) + } + + waitTask, err := photonClient.Tasks.Wait(task.ID) + if err != nil { + return "", logError("Failed to wait for task to CreateDisk", err) + } + + return waitTask.Entity.ID, nil +} + +// Deletes a volume given volume name. +func (pc *PCCloud) DeleteDisk(pdID string) error { + task, err := photonClient.Disks.Delete(pdID) + if err != nil { + return logError("Failed to DeleteDisk", err) + } + + _, err = photonClient.Tasks.Wait(task.ID) + if err != nil { + return logError("Failed to wait for task to DeleteDisk", err) + } + + return nil +} diff --git a/pkg/cloudprovider/providers/photon/photon_test.go b/pkg/cloudprovider/providers/photon/photon_test.go new file mode 100644 index 0000000000..7106dab2b1 --- /dev/null +++ b/pkg/cloudprovider/providers/photon/photon_test.go @@ -0,0 +1,216 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package photon + +import ( + "log" + "os" + "strconv" + "strings" + "testing" + + "k8s.io/kubernetes/pkg/cloudprovider" + "k8s.io/kubernetes/pkg/types" + "k8s.io/kubernetes/pkg/util/rand" +) + +func configFromEnv() (TestVM string, TestFlavor string, cfg PCConfig, ok bool) { + var IgnoreCertificate bool + var OverrideIP bool + var err error + cfg.Global.CloudTarget = os.Getenv("PHOTON_TARGET") + cfg.Global.Tenant = os.Getenv("PHOTON_TENANT") + cfg.Global.Project = os.Getenv("PHOTON_PROJECT") + if os.Getenv("PHOTON_IGNORE_CERTIFICATE") != "" { + IgnoreCertificate, err = strconv.ParseBool(os.Getenv("PHOTON_IGNORE_CERTIFICATE")) + } else { + IgnoreCertificate = false + } + if err != nil { + log.Fatal(err) + } + cfg.Global.IgnoreCertificate = IgnoreCertificate + if os.Getenv("PHOTON_OVERRIDE_IP") != "" { + OverrideIP, err = strconv.ParseBool(os.Getenv("PHOTON_OVERRIDE_IP")) + } else { + OverrideIP = false + } + if err != nil { + log.Fatal(err) + } + cfg.Global.OverrideIP = OverrideIP + + TestVM = os.Getenv("PHOTON_TEST_VM") + if os.Getenv("PHOTON_TEST_FLAVOR") != "" { + TestFlavor = os.Getenv("PHOTON_TEST_FLAVOR") + } else { + TestFlavor = "" + } + if err != nil { + log.Fatal(err) + } + + ok = (cfg.Global.CloudTarget != "" && + cfg.Global.Tenant != "" && + cfg.Global.Project != "" && + TestVM != "") + + return +} + +func TestReadConfig(t *testing.T) { + _, err := readConfig(nil) + if err == nil { + t.Errorf("Should fail when no config is provided: %s", err) + } + + cfg, err := readConfig(strings.NewReader(` +[Global] +target = 0.0.0.0 +ignoreCertificate = true +tenant = tenant +project = project +overrideIP = false +`)) + if err != nil { + t.Fatalf("Should succeed when a valid config is provided: %s", err) + } + + if cfg.Global.CloudTarget != "0.0.0.0" { + t.Errorf("incorrect photon target ip: %s", cfg.Global.CloudTarget) + } + + if cfg.Global.Tenant != "tenant" { + t.Errorf("incorrect tenant: %s", cfg.Global.Tenant) + } + + if cfg.Global.Project != "project" { + t.Errorf("incorrect project: %s", cfg.Global.Project) + } +} + +func TestNewPCCloud(t *testing.T) { + _, _, cfg, ok := configFromEnv() + if !ok { + t.Skipf("No config found in environment") + } + + _, err := newPCCloud(cfg) + if err != nil { + t.Fatalf("Failed to create new Photon client: %s", err) + } +} + +func TestInstances(t *testing.T) { + testVM, _, cfg, ok := configFromEnv() + if !ok { + t.Skipf("No config found in environment") + } + NodeName := types.NodeName(testVM) + + pc, err := newPCCloud(cfg) + if err != nil { + t.Fatalf("Failed to create new Photon client: %s", err) + } + + i, ok := pc.Instances() + if !ok { + t.Fatalf("Instances() returned false") + } + + externalId, err := i.ExternalID(NodeName) + if err != nil { + t.Fatalf("Instances.ExternalID(%s) failed: %s", testVM, err) + } + t.Logf("Found ExternalID(%s) = %s\n", testVM, externalId) + + nonExistingVM := types.NodeName(rand.String(15)) + externalId, err = i.ExternalID(nonExistingVM) + if err == cloudprovider.InstanceNotFound { + t.Logf("VM %s was not found as expected\n", nonExistingVM) + } else if err == nil { + t.Fatalf("Instances.ExternalID did not fail as expected, VM %s was found", nonExistingVM) + } else { + t.Fatalf("Instances.ExternalID did not fail as expected, err: %v", err) + } + + instanceId, err := i.InstanceID(NodeName) + if err != nil { + t.Fatalf("Instances.InstanceID(%s) failed: %s", testVM, err) + } + t.Logf("Found InstanceID(%s) = %s\n", testVM, instanceId) + + instanceId, err = i.InstanceID(nonExistingVM) + if err == cloudprovider.InstanceNotFound { + t.Logf("VM %s was not found as expected\n", nonExistingVM) + } else if err == nil { + t.Fatalf("Instances.InstanceID did not fail as expected, VM %s was found", nonExistingVM) + } else { + t.Fatalf("Instances.InstanceID did not fail as expected, err: %v", err) + } + + addrs, err := i.NodeAddresses(NodeName) + if err != nil { + t.Fatalf("Instances.NodeAddresses(%s) failed: %s", testVM, err) + } + t.Logf("Found NodeAddresses(%s) = %s\n", testVM, addrs) +} + +func TestVolumes(t *testing.T) { + testVM, testFlavor, cfg, ok := configFromEnv() + if !ok { + t.Skipf("No config found in environment") + } + + pc, err := newPCCloud(cfg) + if err != nil { + t.Fatalf("Failed to create new Photon client: %s", err) + } + + NodeName := types.NodeName(testVM) + + volumeOptions := &VolumeOptions{ + CapacityGB: 2, + Tags: nil, + Name: "kubernetes-test-volume-" + rand.String(10), + Flavor: testFlavor} + + pdID, err := pc.CreateDisk(volumeOptions) + if err != nil { + t.Fatalf("Cannot create a Photon persistent disk: %v", err) + } + + err = pc.AttachDisk(pdID, NodeName) + if err != nil { + t.Fatalf("Cannot attach persistent disk(%s) to VM(%s): %v", pdID, testVM, err) + } + + _, err = pc.DiskIsAttached(pdID, NodeName) + if err != nil { + t.Fatalf("Cannot attach persistent disk(%s) to VM(%s): %v", pdID, testVM, err) + } + + err = pc.DetachDisk(pdID, NodeName) + if err != nil { + t.Fatalf("Cannot detach persisten disk(%s) from VM(%s): %v", pdID, testVM, err) + } + + err = pc.DeleteDisk(pdID) + if err != nil { + t.Fatalf("Cannot delete persisten disk(%s): %v", pdID, err) + } +} diff --git a/pkg/cloudprovider/providers/providers.go b/pkg/cloudprovider/providers/providers.go index 5fe9f7331c..4bc1572c90 100644 --- a/pkg/cloudprovider/providers/providers.go +++ b/pkg/cloudprovider/providers/providers.go @@ -25,6 +25,7 @@ import ( _ "k8s.io/kubernetes/pkg/cloudprovider/providers/mesos" _ "k8s.io/kubernetes/pkg/cloudprovider/providers/openstack" _ "k8s.io/kubernetes/pkg/cloudprovider/providers/ovirt" + _ "k8s.io/kubernetes/pkg/cloudprovider/providers/photon" _ "k8s.io/kubernetes/pkg/cloudprovider/providers/rackspace" _ "k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere" ) diff --git a/pkg/kubectl/describe.go b/pkg/kubectl/describe.go index 98281ad122..b8ae59a7a3 100644 --- a/pkg/kubectl/describe.go +++ b/pkg/kubectl/describe.go @@ -572,6 +572,8 @@ func describeVolumes(volumes []api.Volume, out io.Writer, space string) { printVsphereVolumeSource(volume.VolumeSource.VsphereVolume, out) case volume.VolumeSource.Cinder != nil: printCinderVolumeSource(volume.VolumeSource.Cinder, out) + case volume.VolumeSource.PhotonPersistentDisk != nil: + printPhotonPersistentDiskVolumeSource(volume.VolumeSource.PhotonPersistentDisk, out) default: fmt.Fprintf(out, " \n") } @@ -706,6 +708,14 @@ func printVsphereVolumeSource(vsphere *api.VsphereVirtualDiskVolumeSource, out i " FSType:\t%v\n", vsphere.VolumePath, vsphere.FSType) } + +func printPhotonPersistentDiskVolumeSource(photon *api.PhotonPersistentDiskVolumeSource, out io.Writer) { + fmt.Fprintf(out, " Type:\tPhotonPersistentDisk (a Persistent Disk resource in photon platform)\n"+ + " PdID:\t%v\n"+ + " FSType:\t%v\n", + photon.PdID, photon.FSType) +} + func printCinderVolumeSource(cinder *api.CinderVolumeSource, out io.Writer) { fmt.Fprintf(out, " Type:\tCinder (a Persistent Disk resource in OpenStack)\n"+ " VolumeID:\t%v\n"+ @@ -772,6 +782,8 @@ func (d *PersistentVolumeDescriber) Describe(namespace, name string, describerSe printCinderVolumeSource(pv.Spec.Cinder, out) case pv.Spec.AzureDisk != nil: printAzureDiskVolumeSource(pv.Spec.AzureDisk, out) + case pv.Spec.PhotonPersistentDisk != nil: + printPhotonPersistentDiskVolumeSource(pv.Spec.PhotonPersistentDisk, out) } if events != nil { diff --git a/pkg/security/podsecuritypolicy/util/util.go b/pkg/security/podsecuritypolicy/util/util.go index 0b423b948c..427df33204 100644 --- a/pkg/security/podsecuritypolicy/util/util.go +++ b/pkg/security/podsecuritypolicy/util/util.go @@ -60,7 +60,8 @@ func GetAllFSTypesAsSet() sets.String { string(extensions.ConfigMap), string(extensions.VsphereVolume), string(extensions.Quobyte), - string(extensions.AzureDisk)) + string(extensions.AzureDisk), + string(extensions.PhotonPersistentDisk)) return fstypes } @@ -111,6 +112,8 @@ func GetVolumeFSType(v api.Volume) (extensions.FSType, error) { return extensions.Quobyte, nil case v.AzureDisk != nil: return extensions.AzureDisk, nil + case v.PhotonPersistentDisk != nil: + return extensions.PhotonPersistentDisk, nil } return "", fmt.Errorf("unknown volume type for volume: %#v", v) diff --git a/pkg/volume/photon_pd/BUILD b/pkg/volume/photon_pd/BUILD new file mode 100644 index 0000000000..d24b466797 --- /dev/null +++ b/pkg/volume/photon_pd/BUILD @@ -0,0 +1,55 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_binary", + "go_library", + "go_test", + "cgo_library", +) + +go_library( + name = "go_default_library", + srcs = [ + "attacher.go", + "photon_pd.go", + "photon_util.go", + ], + tags = ["automanaged"], + deps = [ + "//pkg/api:go_default_library", + "//pkg/api/resource:go_default_library", + "//pkg/cloudprovider:go_default_library", + "//pkg/cloudprovider/providers/photon:go_default_library", + "//pkg/types:go_default_library", + "//pkg/util/exec:go_default_library", + "//pkg/util/keymutex:go_default_library", + "//pkg/util/mount:go_default_library", + "//pkg/util/strings:go_default_library", + "//pkg/volume:go_default_library", + "//pkg/volume/util:go_default_library", + "//vendor:github.com/golang/glog", + ], +) + +go_test( + name = "go_default_test", + srcs = [ + "attacher_test.go", + "photon_pd_test.go", + ], + library = "go_default_library", + tags = ["automanaged"], + deps = [ + "//pkg/api:go_default_library", + "//pkg/cloudprovider/providers/photon:go_default_library", + "//pkg/types:go_default_library", + "//pkg/util/mount:go_default_library", + "//pkg/util/testing:go_default_library", + "//pkg/volume:go_default_library", + "//pkg/volume/testing:go_default_library", + "//vendor:github.com/golang/glog", + ], +) diff --git a/pkg/volume/photon_pd/OWNERS b/pkg/volume/photon_pd/OWNERS new file mode 100644 index 0000000000..b8170a5d6d --- /dev/null +++ b/pkg/volume/photon_pd/OWNERS @@ -0,0 +1,4 @@ +maintainers: +- luomiao +- kerneltime +- abrarshivani diff --git a/pkg/volume/photon_pd/attacher.go b/pkg/volume/photon_pd/attacher.go new file mode 100644 index 0000000000..02a75d0240 --- /dev/null +++ b/pkg/volume/photon_pd/attacher.go @@ -0,0 +1,295 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package photon_pd + +import ( + "fmt" + "os" + "path" + "path/filepath" + "strings" + "time" + + "github.com/golang/glog" + "k8s.io/kubernetes/pkg/cloudprovider/providers/photon" + "k8s.io/kubernetes/pkg/types" + "k8s.io/kubernetes/pkg/util/exec" + "k8s.io/kubernetes/pkg/util/keymutex" + "k8s.io/kubernetes/pkg/util/mount" + "k8s.io/kubernetes/pkg/volume" + volumeutil "k8s.io/kubernetes/pkg/volume/util" +) + +type photonPersistentDiskAttacher struct { + host volume.VolumeHost + photonDisks photon.Disks +} + +var _ volume.Attacher = &photonPersistentDiskAttacher{} +var _ volume.AttachableVolumePlugin = &photonPersistentDiskPlugin{} + +// Singleton key mutex for keeping attach operations for the same host atomic +var attachdetachMutex = keymutex.NewKeyMutex() + +func (plugin *photonPersistentDiskPlugin) NewAttacher() (volume.Attacher, error) { + photonCloud, err := getCloudProvider(plugin.host.GetCloudProvider()) + if err != nil { + glog.Errorf("Photon Controller attacher: NewAttacher failed to get cloud provider") + return nil, err + } + + return &photonPersistentDiskAttacher{ + host: plugin.host, + photonDisks: photonCloud, + }, nil +} + +// Attaches the volume specified by the given spec to the given host. +// On success, returns the device path where the device was attached on the +// node. +// Callers are responsible for retryinging on failure. +// Callers are responsible for thread safety between concurrent attach and +// detach operations. +func (attacher *photonPersistentDiskAttacher) Attach(spec *volume.Spec, nodeName types.NodeName) (string, error) { + hostName := string(nodeName) + volumeSource, _, err := getVolumeSource(spec) + if err != nil { + glog.Errorf("Photon Controller attacher: Attach failed to get volume source") + return "", err + } + + glog.V(4).Infof("Photon Controller: Attach disk called for host %s", hostName) + + // Keeps concurrent attach operations to same host atomic + attachdetachMutex.LockKey(hostName) + defer attachdetachMutex.UnlockKey(hostName) + + // TODO: if disk is already attached? + err = attacher.photonDisks.AttachDisk(volumeSource.PdID, nodeName) + if err != nil { + glog.Errorf("Error attaching volume %q: %+v", volumeSource.PdID, err) + return "", err + } + + PdidWithNoHypens := strings.Replace(volumeSource.PdID, "-", "", -1) + return path.Join(diskByIDPath, diskPhotonPrefix+PdidWithNoHypens), nil +} + +func (attacher *photonPersistentDiskAttacher) VolumesAreAttached(specs []*volume.Spec, nodeName types.NodeName) (map[*volume.Spec]bool, error) { + volumesAttachedCheck := make(map[*volume.Spec]bool) + volumeSpecMap := make(map[string]*volume.Spec) + pdIDList := []string{} + for _, spec := range specs { + volumeSource, _, err := getVolumeSource(spec) + if err != nil { + glog.Errorf("Error getting volume (%q) source : %v", spec.Name(), err) + continue + } + + pdIDList = append(pdIDList, volumeSource.PdID) + volumesAttachedCheck[spec] = true + volumeSpecMap[volumeSource.PdID] = spec + } + attachedResult, err := attacher.photonDisks.DisksAreAttached(pdIDList, nodeName) + if err != nil { + glog.Errorf( + "Error checking if volumes (%v) are attached to current node (%q). err=%v", + pdIDList, nodeName, err) + return volumesAttachedCheck, err + } + + for pdID, attached := range attachedResult { + if !attached { + spec := volumeSpecMap[pdID] + volumesAttachedCheck[spec] = false + glog.V(2).Infof("VolumesAreAttached: check volume %q (specName: %q) is no longer attached", pdID, spec.Name()) + } + } + return volumesAttachedCheck, nil +} + +func (attacher *photonPersistentDiskAttacher) WaitForAttach(spec *volume.Spec, devicePath string, timeout time.Duration) (string, error) { + volumeSource, _, err := getVolumeSource(spec) + if err != nil { + glog.Errorf("Photon Controller attacher: WaitForAttach failed to get volume source") + return "", err + } + + if devicePath == "" { + return "", fmt.Errorf("WaitForAttach failed for PD %s: devicePath is empty.", volumeSource.PdID) + } + + // scan scsi path to discover the new disk + scsiHostScan() + + ticker := time.NewTicker(checkSleepDuration) + defer ticker.Stop() + + timer := time.NewTimer(timeout) + defer timer.Stop() + + for { + select { + case <-ticker.C: + glog.V(4).Infof("Checking PD %s is attached", volumeSource.PdID) + checkPath, err := verifyDevicePath(devicePath) + if err != nil { + // Log error, if any, and continue checking periodically. See issue #11321 + glog.Warningf("Photon Controller attacher: WaitForAttach with devicePath %s Checking PD %s Error verify path", devicePath, volumeSource.PdID) + } else if checkPath != "" { + // A device path has successfully been created for the VMDK + glog.V(4).Infof("Successfully found attached PD %s.", volumeSource.PdID) + // map path with spec.Name() + volName := spec.Name() + realPath, _ := filepath.EvalSymlinks(devicePath) + deviceName := path.Base(realPath) + volNameToDeviceName[volName] = deviceName + return devicePath, nil + } + case <-timer.C: + return "", fmt.Errorf("Could not find attached PD %s. Timeout waiting for mount paths to be created.", volumeSource.PdID) + } + } +} + +// GetDeviceMountPath returns a path where the device should +// point which should be bind mounted for individual volumes. +func (attacher *photonPersistentDiskAttacher) GetDeviceMountPath(spec *volume.Spec) (string, error) { + volumeSource, _, err := getVolumeSource(spec) + if err != nil { + glog.Errorf("Photon Controller attacher: GetDeviceMountPath failed to get volume source") + return "", err + } + + return makeGlobalPDPath(attacher.host, volumeSource.PdID), nil +} + +// GetMountDeviceRefs finds all other references to the device referenced +// by deviceMountPath; returns a list of paths. +func (plugin *photonPersistentDiskPlugin) GetDeviceMountRefs(deviceMountPath string) ([]string, error) { + mounter := plugin.host.GetMounter() + return mount.GetMountRefs(mounter, deviceMountPath) +} + +// MountDevice mounts device to global mount point. +func (attacher *photonPersistentDiskAttacher) MountDevice(spec *volume.Spec, devicePath string, deviceMountPath string) error { + mounter := attacher.host.GetMounter() + notMnt, err := mounter.IsLikelyNotMountPoint(deviceMountPath) + if err != nil { + if os.IsNotExist(err) { + if err := os.MkdirAll(deviceMountPath, 0750); err != nil { + glog.Errorf("Failed to create directory at %#v. err: %s", deviceMountPath, err) + return err + } + notMnt = true + } else { + return err + } + } + + volumeSource, _, err := getVolumeSource(spec) + if err != nil { + glog.Errorf("Photon Controller attacher: MountDevice failed to get volume source. err: %s", err) + return err + } + + options := []string{} + + if notMnt { + diskMounter := &mount.SafeFormatAndMount{Interface: mounter, Runner: exec.New()} + err = diskMounter.FormatAndMount(devicePath, deviceMountPath, volumeSource.FSType, options) + if err != nil { + os.Remove(deviceMountPath) + return err + } + glog.V(4).Infof("formatting spec %v devicePath %v deviceMountPath %v fs %v with options %+v", spec.Name(), devicePath, deviceMountPath, volumeSource.FSType, options) + } + return nil +} + +type photonPersistentDiskDetacher struct { + mounter mount.Interface + photonDisks photon.Disks +} + +var _ volume.Detacher = &photonPersistentDiskDetacher{} + +func (plugin *photonPersistentDiskPlugin) NewDetacher() (volume.Detacher, error) { + photonCloud, err := getCloudProvider(plugin.host.GetCloudProvider()) + if err != nil { + glog.Errorf("Photon Controller attacher: NewDetacher failed to get cloud provider. err: %s", err) + return nil, err + } + + return &photonPersistentDiskDetacher{ + mounter: plugin.host.GetMounter(), + photonDisks: photonCloud, + }, nil +} + +// Detach the given device from the given host. +func (detacher *photonPersistentDiskDetacher) Detach(deviceMountPath string, nodeName types.NodeName) error { + + hostName := string(nodeName) + pdID := deviceMountPath + attached, err := detacher.photonDisks.DiskIsAttached(pdID, nodeName) + if err != nil { + // Log error and continue with detach + glog.Errorf( + "Error checking if persistent disk (%q) is already attached to current node (%q). Will continue and try detach anyway. err=%v", + pdID, hostName, err) + } + + if err == nil && !attached { + // Volume is already detached from node. + glog.V(4).Infof("detach operation was successful. persistent disk %q is already detached from node %q.", pdID, hostName) + return nil + } + + attachdetachMutex.LockKey(hostName) + defer attachdetachMutex.UnlockKey(hostName) + if err := detacher.photonDisks.DetachDisk(pdID, nodeName); err != nil { + glog.Errorf("Error detaching volume %q: %v", pdID, err) + return err + } + return nil +} + +func (detacher *photonPersistentDiskDetacher) WaitForDetach(devicePath string, timeout time.Duration) error { + ticker := time.NewTicker(checkSleepDuration) + defer ticker.Stop() + timer := time.NewTimer(timeout) + defer timer.Stop() + + for { + select { + case <-ticker.C: + glog.V(4).Infof("Checking device %q is detached.", devicePath) + if pathExists, err := volumeutil.PathExists(devicePath); err != nil { + return fmt.Errorf("Error checking if device path exists: %v", err) + } else if !pathExists { + return nil + } + case <-timer.C: + return fmt.Errorf("Timeout reached; Device %v is still attached", devicePath) + } + } +} + +func (detacher *photonPersistentDiskDetacher) UnmountDevice(deviceMountPath string) error { + return volumeutil.UnmountPath(deviceMountPath, detacher.mounter) +} diff --git a/pkg/volume/photon_pd/attacher_test.go b/pkg/volume/photon_pd/attacher_test.go new file mode 100644 index 0000000000..357bb174c9 --- /dev/null +++ b/pkg/volume/photon_pd/attacher_test.go @@ -0,0 +1,328 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package photon_pd + +import ( + "errors" + "testing" + + "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/cloudprovider/providers/photon" + "k8s.io/kubernetes/pkg/volume" + volumetest "k8s.io/kubernetes/pkg/volume/testing" + + "github.com/golang/glog" + "k8s.io/kubernetes/pkg/types" +) + +func TestGetDeviceName_Volume(t *testing.T) { + plugin := newPlugin() + name := "my-photon-volume" + spec := createVolSpec(name, false) + + deviceName, err := plugin.GetVolumeName(spec) + if err != nil { + t.Errorf("GetDeviceName error: %v", err) + } + if deviceName != name { + t.Errorf("GetDeviceName error: expected %s, got %s", name, deviceName) + } +} + +func TestGetDeviceName_PersistentVolume(t *testing.T) { + plugin := newPlugin() + name := "my-photon-pv" + spec := createPVSpec(name, true) + + deviceName, err := plugin.GetVolumeName(spec) + if err != nil { + t.Errorf("GetDeviceName error: %v", err) + } + if deviceName != name { + t.Errorf("GetDeviceName error: expected %s, got %s", name, deviceName) + } +} + +// One testcase for TestAttachDetach table test below +type testcase struct { + name string + // For fake Photon cloud provider: + attach attachCall + detach detachCall + diskIsAttached diskIsAttachedCall + t *testing.T + + // Actual test to run + test func(test *testcase) (string, error) + // Expected return of the test + expectedDevice string + expectedError error +} + +func TestAttachDetach(t *testing.T) { + diskName := "000-000-000" + nodeName := types.NodeName("instance") + readOnly := false + spec := createVolSpec(diskName, readOnly) + attachError := errors.New("Fake attach error") + detachError := errors.New("Fake detach error") + diskCheckError := errors.New("Fake DiskIsAttached error") + tests := []testcase{ + // Successful Attach call + { + name: "Attach_Positive", + attach: attachCall{diskName, nodeName, nil}, + test: func(testcase *testcase) (string, error) { + attacher := newAttacher(testcase) + return attacher.Attach(spec, nodeName) + }, + expectedDevice: "/dev/disk/by-id/wwn-0x000000000", + }, + + // Attach call fails + { + name: "Attach_Negative", + attach: attachCall{diskName, nodeName, attachError}, + test: func(testcase *testcase) (string, error) { + attacher := newAttacher(testcase) + return attacher.Attach(spec, nodeName) + }, + expectedError: attachError, + }, + + // Detach succeeds + { + name: "Detach_Positive", + diskIsAttached: diskIsAttachedCall{diskName, nodeName, true, nil}, + detach: detachCall{diskName, nodeName, nil}, + test: func(testcase *testcase) (string, error) { + detacher := newDetacher(testcase) + return "", detacher.Detach(diskName, nodeName) + }, + }, + + // Disk is already detached + { + name: "Detach_Positive_AlreadyDetached", + diskIsAttached: diskIsAttachedCall{diskName, nodeName, false, nil}, + test: func(testcase *testcase) (string, error) { + detacher := newDetacher(testcase) + return "", detacher.Detach(diskName, nodeName) + }, + }, + + // Detach succeeds when DiskIsAttached fails + { + name: "Detach_Positive_CheckFails", + diskIsAttached: diskIsAttachedCall{diskName, nodeName, false, diskCheckError}, + detach: detachCall{diskName, nodeName, nil}, + test: func(testcase *testcase) (string, error) { + detacher := newDetacher(testcase) + return "", detacher.Detach(diskName, nodeName) + }, + }, + + // Detach fails + { + name: "Detach_Negative", + diskIsAttached: diskIsAttachedCall{diskName, nodeName, false, diskCheckError}, + detach: detachCall{diskName, nodeName, detachError}, + test: func(testcase *testcase) (string, error) { + detacher := newDetacher(testcase) + return "", detacher.Detach(diskName, nodeName) + }, + expectedError: detachError, + }, + } + + for _, testcase := range tests { + testcase.t = t + device, err := testcase.test(&testcase) + if err != testcase.expectedError { + t.Errorf("%s failed: expected err=%q, got %q", testcase.name, testcase.expectedError.Error(), err.Error()) + } + if device != testcase.expectedDevice { + t.Errorf("%s failed: expected device=%q, got %q", testcase.name, testcase.expectedDevice, device) + } + t.Logf("Test %q succeeded", testcase.name) + } +} + +// newPlugin creates a new gcePersistentDiskPlugin with fake cloud, NewAttacher +// and NewDetacher won't work. +func newPlugin() *photonPersistentDiskPlugin { + host := volumetest.NewFakeVolumeHost("/tmp", nil, nil) + plugins := ProbeVolumePlugins() + plugin := plugins[0] + plugin.Init(host) + return plugin.(*photonPersistentDiskPlugin) +} + +func newAttacher(testcase *testcase) *photonPersistentDiskAttacher { + return &photonPersistentDiskAttacher{ + host: nil, + photonDisks: testcase, + } +} + +func newDetacher(testcase *testcase) *photonPersistentDiskDetacher { + return &photonPersistentDiskDetacher{ + photonDisks: testcase, + } +} + +func createVolSpec(name string, readOnly bool) *volume.Spec { + return &volume.Spec{ + Volume: &api.Volume{ + VolumeSource: api.VolumeSource{ + PhotonPersistentDisk: &api.PhotonPersistentDiskVolumeSource{ + PdID: name, + }, + }, + }, + } +} + +func createPVSpec(name string, readOnly bool) *volume.Spec { + return &volume.Spec{ + PersistentVolume: &api.PersistentVolume{ + Spec: api.PersistentVolumeSpec{ + PersistentVolumeSource: api.PersistentVolumeSource{ + PhotonPersistentDisk: &api.PhotonPersistentDiskVolumeSource{ + PdID: name, + }, + }, + }, + }, + } +} + +// Fake PhotonPD implementation + +type attachCall struct { + diskName string + nodeName types.NodeName + ret error +} + +type detachCall struct { + diskName string + nodeName types.NodeName + ret error +} + +type diskIsAttachedCall struct { + diskName string + nodeName types.NodeName + isAttached bool + ret error +} + +func (testcase *testcase) AttachDisk(diskName string, nodeName types.NodeName) error { + expected := &testcase.attach + + if expected.diskName == "" && expected.nodeName == "" { + // testcase.attach looks uninitialized, test did not expect to call + // AttachDisk + testcase.t.Errorf("Unexpected AttachDisk call!") + return errors.New("Unexpected AttachDisk call!") + } + + if expected.diskName != diskName { + testcase.t.Errorf("Unexpected AttachDisk call: expected diskName %s, got %s", expected.diskName, diskName) + return errors.New("Unexpected AttachDisk call: wrong diskName") + } + + if expected.nodeName != nodeName { + testcase.t.Errorf("Unexpected AttachDisk call: expected nodeName %s, got %s", expected.nodeName, nodeName) + return errors.New("Unexpected AttachDisk call: wrong nodeName") + } + + glog.V(4).Infof("AttachDisk call: %s, %s, returning %v", diskName, nodeName, expected.ret) + + return expected.ret +} + +func (testcase *testcase) DetachDisk(diskName string, nodeName types.NodeName) error { + expected := &testcase.detach + + if expected.diskName == "" && expected.nodeName == "" { + // testcase.detach looks uninitialized, test did not expect to call + // DetachDisk + testcase.t.Errorf("Unexpected DetachDisk call!") + return errors.New("Unexpected DetachDisk call!") + } + + if expected.diskName != diskName { + testcase.t.Errorf("Unexpected DetachDisk call: expected diskName %s, got %s", expected.diskName, diskName) + return errors.New("Unexpected DetachDisk call: wrong diskName") + } + + if expected.nodeName != nodeName { + testcase.t.Errorf("Unexpected DetachDisk call: expected nodeName %s, got %s", expected.nodeName, nodeName) + return errors.New("Unexpected DetachDisk call: wrong nodeName") + } + + glog.V(4).Infof("DetachDisk call: %s, %s, returning %v", diskName, nodeName, expected.ret) + + return expected.ret +} + +func (testcase *testcase) DiskIsAttached(diskName string, nodeName types.NodeName) (bool, error) { + expected := &testcase.diskIsAttached + + if expected.diskName == "" && expected.nodeName == "" { + // testcase.diskIsAttached looks uninitialized, test did not expect to + // call DiskIsAttached + testcase.t.Errorf("Unexpected DiskIsAttached call!") + return false, errors.New("Unexpected DiskIsAttached call!") + } + + if expected.diskName != diskName { + testcase.t.Errorf("Unexpected DiskIsAttached call: expected diskName %s, got %s", expected.diskName, diskName) + return false, errors.New("Unexpected DiskIsAttached call: wrong diskName") + } + + if expected.nodeName != nodeName { + testcase.t.Errorf("Unexpected DiskIsAttached call: expected nodeName %s, got %s", expected.nodeName, nodeName) + return false, errors.New("Unexpected DiskIsAttached call: wrong nodeName") + } + + glog.V(4).Infof("DiskIsAttached call: %s, %s, returning %v, %v", diskName, nodeName, expected.isAttached, expected.ret) + + return expected.isAttached, expected.ret +} + +func (testcase *testcase) DisksAreAttached(diskNames []string, nodeName types.NodeName) (map[string]bool, error) { + return nil, errors.New("Not implemented") +} + +func (testcase *testcase) CreateDisk(volumeOptions *photon.VolumeOptions) (volumeName string, err error) { + return "", errors.New("Not implemented") +} + +func (testcase *testcase) DeleteDisk(volumeName string) error { + return errors.New("Not implemented") +} + +func (testcase *testcase) GetVolumeLabels(volumeName string) (map[string]string, error) { + return map[string]string{}, errors.New("Not implemented") +} + +func (testcase *testcase) GetDiskPath(volumeName string) (string, error) { + return "", errors.New("Not implemented") +} diff --git a/pkg/volume/photon_pd/photon_pd.go b/pkg/volume/photon_pd/photon_pd.go new file mode 100644 index 0000000000..b293b8b7a6 --- /dev/null +++ b/pkg/volume/photon_pd/photon_pd.go @@ -0,0 +1,387 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package photon_pd + +import ( + "fmt" + "os" + "path" + + "github.com/golang/glog" + "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/api/resource" + "k8s.io/kubernetes/pkg/types" + "k8s.io/kubernetes/pkg/util/exec" + "k8s.io/kubernetes/pkg/util/mount" + utilstrings "k8s.io/kubernetes/pkg/util/strings" + "k8s.io/kubernetes/pkg/volume" +) + +// This is the primary entrypoint for volume plugins. +func ProbeVolumePlugins() []volume.VolumePlugin { + return []volume.VolumePlugin{&photonPersistentDiskPlugin{}} +} + +type photonPersistentDiskPlugin struct { + host volume.VolumeHost +} + +var _ volume.VolumePlugin = &photonPersistentDiskPlugin{} +var _ volume.PersistentVolumePlugin = &photonPersistentDiskPlugin{} +var _ volume.DeletableVolumePlugin = &photonPersistentDiskPlugin{} +var _ volume.ProvisionableVolumePlugin = &photonPersistentDiskPlugin{} + +const ( + photonPersistentDiskPluginName = "kubernetes.io/photon-pd" +) + +func (plugin *photonPersistentDiskPlugin) Init(host volume.VolumeHost) error { + plugin.host = host + return nil +} + +func (plugin *photonPersistentDiskPlugin) GetPluginName() string { + return photonPersistentDiskPluginName +} + +func (plugin *photonPersistentDiskPlugin) GetVolumeName(spec *volume.Spec) (string, error) { + volumeSource, _, err := getVolumeSource(spec) + if err != nil { + glog.Errorf("Photon volume plugin: GetVolumeName failed to get volume source") + return "", err + } + + return volumeSource.PdID, nil +} + +func (plugin *photonPersistentDiskPlugin) CanSupport(spec *volume.Spec) bool { + return (spec.PersistentVolume != nil && spec.PersistentVolume.Spec.PhotonPersistentDisk != nil) || + (spec.Volume != nil && spec.Volume.PhotonPersistentDisk != nil) +} + +func (plugin *photonPersistentDiskPlugin) RequiresRemount() bool { + return false +} + +func (plugin *photonPersistentDiskPlugin) NewMounter(spec *volume.Spec, pod *api.Pod, _ volume.VolumeOptions) (volume.Mounter, error) { + return plugin.newMounterInternal(spec, pod.UID, &PhotonDiskUtil{}, plugin.host.GetMounter()) +} + +func (plugin *photonPersistentDiskPlugin) NewUnmounter(volName string, podUID types.UID) (volume.Unmounter, error) { + return plugin.newUnmounterInternal(volName, podUID, &PhotonDiskUtil{}, plugin.host.GetMounter()) +} + +func (plugin *photonPersistentDiskPlugin) newMounterInternal(spec *volume.Spec, podUID types.UID, manager pdManager, mounter mount.Interface) (volume.Mounter, error) { + vvol, _, err := getVolumeSource(spec) + if err != nil { + glog.Errorf("Photon volume plugin: newMounterInternal failed to get volume source") + return nil, err + } + + pdID := vvol.PdID + fsType := vvol.FSType + + return &photonPersistentDiskMounter{ + photonPersistentDisk: &photonPersistentDisk{ + podUID: podUID, + volName: spec.Name(), + pdID: pdID, + manager: manager, + mounter: mounter, + plugin: plugin, + }, + fsType: fsType, + diskMounter: &mount.SafeFormatAndMount{Interface: mounter, Runner: exec.New()}}, nil +} + +func (plugin *photonPersistentDiskPlugin) newUnmounterInternal(volName string, podUID types.UID, manager pdManager, mounter mount.Interface) (volume.Unmounter, error) { + return &photonPersistentDiskUnmounter{ + &photonPersistentDisk{ + podUID: podUID, + volName: volName, + manager: manager, + mounter: mounter, + plugin: plugin, + }}, nil +} + +func (plugin *photonPersistentDiskPlugin) ConstructVolumeSpec(volumeName, mountPath string) (*volume.Spec, error) { + photonPersistentDisk := &api.Volume{ + Name: volumeName, + VolumeSource: api.VolumeSource{ + PhotonPersistentDisk: &api.PhotonPersistentDiskVolumeSource{ + PdID: volumeName, + }, + }, + } + return volume.NewSpecFromVolume(photonPersistentDisk), nil +} + +// Abstract interface to disk operations. +type pdManager interface { + // Creates a volume + CreateVolume(provisioner *photonPersistentDiskProvisioner) (pdID string, volumeSizeGB int, err error) + // Deletes a volume + DeleteVolume(deleter *photonPersistentDiskDeleter) error +} + +// photonPersistentDisk volumes are disk resources are attached to the kubelet's host machine and exposed to the pod. +type photonPersistentDisk struct { + volName string + podUID types.UID + // Unique identifier of the volume, used to find the disk resource in the provider. + pdID string + // Filesystem type, optional. + fsType string + // Utility interface that provides API calls to the provider to attach/detach disks. + manager pdManager + // Mounter interface that provides system calls to mount the global path to the pod local path. + mounter mount.Interface + plugin *photonPersistentDiskPlugin + volume.MetricsNil +} + +var _ volume.Mounter = &photonPersistentDiskMounter{} + +type photonPersistentDiskMounter struct { + *photonPersistentDisk + fsType string + diskMounter *mount.SafeFormatAndMount +} + +func (b *photonPersistentDiskMounter) GetAttributes() volume.Attributes { + return volume.Attributes{ + SupportsSELinux: true, + } +} + +// SetUp attaches the disk and bind mounts to the volume path. +func (b *photonPersistentDiskMounter) SetUp(fsGroup *int64) error { + return b.SetUpAt(b.GetPath(), fsGroup) +} + +// SetUp attaches the disk and bind mounts to the volume path. +func (b *photonPersistentDiskMounter) SetUpAt(dir string, fsGroup *int64) error { + glog.V(4).Infof("Photon Persistent Disk setup %s to %s", b.pdID, dir) + + // TODO: handle failed mounts here. + notmnt, err := b.mounter.IsLikelyNotMountPoint(dir) + if err != nil && !os.IsNotExist(err) { + glog.Errorf("cannot validate mount point: %s %v", dir, err) + return err + } + if !notmnt { + return nil + } + + if err := os.MkdirAll(dir, 0750); err != nil { + glog.Errorf("mkdir failed on disk %s (%v)", dir, err) + return err + } + + options := []string{"bind"} + + // Perform a bind mount to the full path to allow duplicate mounts of the same PD. + globalPDPath := makeGlobalPDPath(b.plugin.host, b.pdID) + glog.V(4).Infof("attempting to mount %s", dir) + + err = b.mounter.Mount(globalPDPath, dir, "", options) + if err != nil { + notmnt, mntErr := b.mounter.IsLikelyNotMountPoint(dir) + if mntErr != nil { + glog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr) + return err + } + if !notmnt { + if mntErr = b.mounter.Unmount(dir); mntErr != nil { + glog.Errorf("Failed to unmount: %v", mntErr) + return err + } + notmnt, mntErr := b.mounter.IsLikelyNotMountPoint(dir) + if mntErr != nil { + glog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr) + return err + } + if !notmnt { + glog.Errorf("%s is still mounted, despite call to unmount(). Will try again next sync loop.", b.GetPath()) + return err + } + } + os.Remove(dir) + glog.Errorf("Mount of disk %s failed: %v", dir, err) + return err + } + + return nil +} + +var _ volume.Unmounter = &photonPersistentDiskUnmounter{} + +type photonPersistentDiskUnmounter struct { + *photonPersistentDisk +} + +// Unmounts the bind mount, and detaches the disk only if the PD +// resource was the last reference to that disk on the kubelet. +func (c *photonPersistentDiskUnmounter) TearDown() error { + err := c.TearDownAt(c.GetPath()) + if err != nil { + return err + } + + removeFromScsiSubsystem(c.volName) + return nil +} + +// Unmounts the bind mount, and detaches the disk only if the PD +// resource was the last reference to that disk on the kubelet. +func (c *photonPersistentDiskUnmounter) TearDownAt(dir string) error { + glog.V(4).Infof("Photon Controller Volume TearDown of %s", dir) + notmnt, err := c.mounter.IsLikelyNotMountPoint(dir) + if err != nil { + return err + } + if notmnt { + return os.Remove(dir) + } + + if err := c.mounter.Unmount(dir); err != nil { + glog.Errorf("Unmount failed: %v", err) + return err + } + + notmnt, mntErr := c.mounter.IsLikelyNotMountPoint(dir) + if mntErr != nil { + glog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr) + return err + } + if notmnt { + return os.Remove(dir) + } + return fmt.Errorf("Failed to unmount volume dir") +} + +func makeGlobalPDPath(host volume.VolumeHost, devName string) string { + return path.Join(host.GetPluginDir(photonPersistentDiskPluginName), "mounts", devName) +} + +func (ppd *photonPersistentDisk) GetPath() string { + name := photonPersistentDiskPluginName + return ppd.plugin.host.GetPodVolumeDir(ppd.podUID, utilstrings.EscapeQualifiedNameForDisk(name), ppd.volName) +} + +// TODO: supporting more access mode for PhotonController persistent disk +func (plugin *photonPersistentDiskPlugin) GetAccessModes() []api.PersistentVolumeAccessMode { + return []api.PersistentVolumeAccessMode{ + api.ReadWriteOnce, + } +} + +type photonPersistentDiskDeleter struct { + *photonPersistentDisk +} + +var _ volume.Deleter = &photonPersistentDiskDeleter{} + +func (plugin *photonPersistentDiskPlugin) NewDeleter(spec *volume.Spec) (volume.Deleter, error) { + return plugin.newDeleterInternal(spec, &PhotonDiskUtil{}) +} + +func (plugin *photonPersistentDiskPlugin) newDeleterInternal(spec *volume.Spec, manager pdManager) (volume.Deleter, error) { + if spec.PersistentVolume != nil && spec.PersistentVolume.Spec.PhotonPersistentDisk == nil { + return nil, fmt.Errorf("spec.PersistentVolumeSource.PhotonPersistentDisk is nil") + } + return &photonPersistentDiskDeleter{ + &photonPersistentDisk{ + volName: spec.Name(), + pdID: spec.PersistentVolume.Spec.PhotonPersistentDisk.PdID, + manager: manager, + plugin: plugin, + }}, nil +} + +func (r *photonPersistentDiskDeleter) Delete() error { + return r.manager.DeleteVolume(r) +} + +type photonPersistentDiskProvisioner struct { + *photonPersistentDisk + options volume.VolumeOptions +} + +var _ volume.Provisioner = &photonPersistentDiskProvisioner{} + +func (plugin *photonPersistentDiskPlugin) NewProvisioner(options volume.VolumeOptions) (volume.Provisioner, error) { + return plugin.newProvisionerInternal(options, &PhotonDiskUtil{}) +} + +func (plugin *photonPersistentDiskPlugin) newProvisionerInternal(options volume.VolumeOptions, manager pdManager) (volume.Provisioner, error) { + return &photonPersistentDiskProvisioner{ + photonPersistentDisk: &photonPersistentDisk{ + manager: manager, + plugin: plugin, + }, + options: options, + }, nil +} + +func (p *photonPersistentDiskProvisioner) Provision() (*api.PersistentVolume, error) { + pdID, sizeGB, err := p.manager.CreateVolume(p) + if err != nil { + return nil, err + } + + pv := &api.PersistentVolume{ + ObjectMeta: api.ObjectMeta{ + Name: p.options.PVName, + Labels: map[string]string{}, + Annotations: map[string]string{ + "kubernetes.io/createdby": "photon-volume-dynamic-provisioner", + }, + }, + Spec: api.PersistentVolumeSpec{ + PersistentVolumeReclaimPolicy: p.options.PersistentVolumeReclaimPolicy, + AccessModes: p.options.PVC.Spec.AccessModes, + Capacity: api.ResourceList{ + api.ResourceName(api.ResourceStorage): resource.MustParse(fmt.Sprintf("%dGi", sizeGB)), + }, + PersistentVolumeSource: api.PersistentVolumeSource{ + PhotonPersistentDisk: &api.PhotonPersistentDiskVolumeSource{ + PdID: pdID, + FSType: "ext4", + }, + }, + }, + } + if len(p.options.PVC.Spec.AccessModes) == 0 { + pv.Spec.AccessModes = p.plugin.GetAccessModes() + } + + return pv, nil +} + +func getVolumeSource( + spec *volume.Spec) (*api.PhotonPersistentDiskVolumeSource, bool, error) { + if spec.Volume != nil && spec.Volume.PhotonPersistentDisk != nil { + return spec.Volume.PhotonPersistentDisk, spec.ReadOnly, nil + } else if spec.PersistentVolume != nil && + spec.PersistentVolume.Spec.PhotonPersistentDisk != nil { + return spec.PersistentVolume.Spec.PhotonPersistentDisk, spec.ReadOnly, nil + } + + return nil, false, fmt.Errorf("Spec does not reference a Photon Controller persistent disk type") +} diff --git a/pkg/volume/photon_pd/photon_pd_test.go b/pkg/volume/photon_pd/photon_pd_test.go new file mode 100644 index 0000000000..e6e1be5e3d --- /dev/null +++ b/pkg/volume/photon_pd/photon_pd_test.go @@ -0,0 +1,239 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package photon_pd + +import ( + "fmt" + "os" + "path" + "testing" + + "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/types" + "k8s.io/kubernetes/pkg/util/mount" + utiltesting "k8s.io/kubernetes/pkg/util/testing" + "k8s.io/kubernetes/pkg/volume" + volumetest "k8s.io/kubernetes/pkg/volume/testing" +) + +func TestCanSupport(t *testing.T) { + tmpDir, err := utiltesting.MkTmpdir("photonpdTest") + if err != nil { + t.Fatalf("can't make a temp dir: %v", err) + } + defer os.RemoveAll(tmpDir) + plugMgr := volume.VolumePluginMgr{} + plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) + + plug, err := plugMgr.FindPluginByName("kubernetes.io/photon-pd") + if err != nil { + t.Errorf("Can't find the plugin by name") + } + if plug.GetPluginName() != "kubernetes.io/photon-pd" { + t.Errorf("Wrong name: %s", plug.GetPluginName()) + } + if !plug.CanSupport(&volume.Spec{Volume: &api.Volume{VolumeSource: api.VolumeSource{PhotonPersistentDisk: &api.PhotonPersistentDiskVolumeSource{}}}}) { + t.Errorf("Expected true") + } + if !plug.CanSupport(&volume.Spec{PersistentVolume: &api.PersistentVolume{Spec: api.PersistentVolumeSpec{PersistentVolumeSource: api.PersistentVolumeSource{PhotonPersistentDisk: &api.PhotonPersistentDiskVolumeSource{}}}}}) { + t.Errorf("Expected true") + } +} + +func TestGetAccessModes(t *testing.T) { + tmpDir, err := utiltesting.MkTmpdir("photonpdTest") + if err != nil { + t.Fatalf("can't make a temp dir: %v", err) + } + defer os.RemoveAll(tmpDir) + plugMgr := volume.VolumePluginMgr{} + plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) + + plug, err := plugMgr.FindPersistentPluginByName("kubernetes.io/photon-pd") + if err != nil { + t.Errorf("Can't find the plugin by name") + } + + if !contains(plug.GetAccessModes(), api.ReadWriteOnce) { + t.Errorf("Expected to support AccessModeTypes: %s", api.ReadWriteOnce) + } + if contains(plug.GetAccessModes(), api.ReadOnlyMany) { + t.Errorf("Expected not to support AccessModeTypes: %s", api.ReadOnlyMany) + } +} + +func contains(modes []api.PersistentVolumeAccessMode, mode api.PersistentVolumeAccessMode) bool { + for _, m := range modes { + if m == mode { + return true + } + } + return false +} + +type fakePDManager struct { +} + +func (fake *fakePDManager) CreateVolume(c *photonPersistentDiskProvisioner) (pdID string, volumeSizeGB int, err error) { + return "test-photon-pd-id", 10, nil +} + +func (fake *fakePDManager) DeleteVolume(cd *photonPersistentDiskDeleter) error { + if cd.pdID != "test-photon-pd-id" { + return fmt.Errorf("Deleter got unexpected volume name: %s", cd.pdID) + } + return nil +} + +func TestPlugin(t *testing.T) { + tmpDir, err := utiltesting.MkTmpdir("photonpdTest") + if err != nil { + t.Fatalf("can't make a temp dir: %v", err) + } + defer os.RemoveAll(tmpDir) + plugMgr := volume.VolumePluginMgr{} + plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) + + plug, err := plugMgr.FindPluginByName("kubernetes.io/photon-pd") + if err != nil { + t.Errorf("Can't find the plugin by name") + } + spec := &api.Volume{ + Name: "vol1", + VolumeSource: api.VolumeSource{ + PhotonPersistentDisk: &api.PhotonPersistentDiskVolumeSource{ + PdID: "pdid", + FSType: "ext4", + }, + }, + } + fakeManager := &fakePDManager{} + fakeMounter := &mount.FakeMounter{} + mounter, err := plug.(*photonPersistentDiskPlugin).newMounterInternal(volume.NewSpecFromVolume(spec), types.UID("poduid"), fakeManager, fakeMounter) + if err != nil { + t.Errorf("Failed to make a new Mounter: %v", err) + } + if mounter == nil { + t.Errorf("Got a nil Mounter") + } + + volPath := path.Join(tmpDir, "pods/poduid/volumes/kubernetes.io~photon-pd/vol1") + path := mounter.GetPath() + if path != volPath { + t.Errorf("Got unexpected path: %s", path) + } + + if err := mounter.SetUp(nil); err != nil { + t.Errorf("Expected success, got: %v", err) + } + if _, err := os.Stat(path); err != nil { + if os.IsNotExist(err) { + t.Errorf("SetUp() failed, volume path not created: %s", path) + } else { + t.Errorf("SetUp() failed: %v", err) + } + } + if _, err := os.Stat(path); err != nil { + if os.IsNotExist(err) { + t.Errorf("SetUp() failed, volume path not created: %s", path) + } else { + t.Errorf("SetUp() failed: %v", err) + } + } + + fakeManager = &fakePDManager{} + unmounter, err := plug.(*photonPersistentDiskPlugin).newUnmounterInternal("vol1", types.UID("poduid"), fakeManager, fakeMounter) + if err != nil { + t.Errorf("Failed to make a new Unmounter: %v", err) + } + if unmounter == nil { + t.Errorf("Got a nil Unmounter") + } + + if err := unmounter.TearDown(); err != nil { + t.Errorf("Expected success, got: %v", err) + } + if _, err := os.Stat(path); err == nil { + t.Errorf("TearDown() failed, volume path still exists: %s", path) + } else if !os.IsNotExist(err) { + t.Errorf("SetUp() failed: %v", err) + } + + // Test Provisioner + options := volume.VolumeOptions{ + PVC: volumetest.CreateTestPVC("10Gi", []api.PersistentVolumeAccessMode{api.ReadWriteOnce}), + PersistentVolumeReclaimPolicy: api.PersistentVolumeReclaimDelete, + } + provisioner, err := plug.(*photonPersistentDiskPlugin).newProvisionerInternal(options, &fakePDManager{}) + persistentSpec, err := provisioner.Provision() + if err != nil { + t.Errorf("Provision() failed: %v", err) + } + + if persistentSpec.Spec.PersistentVolumeSource.PhotonPersistentDisk.PdID != "test-photon-pd-id" { + t.Errorf("Provision() returned unexpected persistent disk ID: %s", persistentSpec.Spec.PersistentVolumeSource.PhotonPersistentDisk.PdID) + } + cap := persistentSpec.Spec.Capacity[api.ResourceStorage] + size := cap.Value() + if size != 10*1024*1024*1024 { + t.Errorf("Provision() returned unexpected volume size: %v", size) + } + + // Test Deleter + volSpec := &volume.Spec{ + PersistentVolume: persistentSpec, + } + deleter, err := plug.(*photonPersistentDiskPlugin).newDeleterInternal(volSpec, &fakePDManager{}) + err = deleter.Delete() + if err != nil { + t.Errorf("Deleter() failed: %v", err) + } +} + +func TestMounterAndUnmounterTypeAssert(t *testing.T) { + tmpDir, err := utiltesting.MkTmpdir("photonpdTest") + if err != nil { + t.Fatalf("can't make a temp dir: %v", err) + } + defer os.RemoveAll(tmpDir) + plugMgr := volume.VolumePluginMgr{} + plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) + + plug, err := plugMgr.FindPluginByName("kubernetes.io/photon-pd") + if err != nil { + t.Errorf("Can't find the plugin by name") + } + spec := &api.Volume{ + Name: "vol1", + VolumeSource: api.VolumeSource{ + PhotonPersistentDisk: &api.PhotonPersistentDiskVolumeSource{ + PdID: "pdid", + FSType: "ext4", + }, + }, + } + + mounter, err := plug.(*photonPersistentDiskPlugin).newMounterInternal(volume.NewSpecFromVolume(spec), types.UID("poduid"), &fakePDManager{}, &mount.FakeMounter{}) + if _, ok := mounter.(volume.Unmounter); ok { + t.Errorf("Volume Mounter can be type-assert to Unmounter") + } + + unmounter, err := plug.(*photonPersistentDiskPlugin).newUnmounterInternal("vol1", types.UID("poduid"), &fakePDManager{}, &mount.FakeMounter{}) + if _, ok := unmounter.(volume.Mounter); ok { + t.Errorf("Volume Unmounter can be type-assert to Mounter") + } +} diff --git a/pkg/volume/photon_pd/photon_util.go b/pkg/volume/photon_pd/photon_util.go new file mode 100644 index 0000000000..b63535f900 --- /dev/null +++ b/pkg/volume/photon_pd/photon_util.go @@ -0,0 +1,145 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package photon_pd + +import ( + "errors" + "fmt" + "io/ioutil" + "strings" + "time" + + "github.com/golang/glog" + "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/cloudprovider" + "k8s.io/kubernetes/pkg/cloudprovider/providers/photon" + "k8s.io/kubernetes/pkg/volume" + volumeutil "k8s.io/kubernetes/pkg/volume/util" +) + +const ( + maxRetries = 10 + checkSleepDuration = time.Second + diskByIDPath = "/dev/disk/by-id/" + diskPhotonPrefix = "wwn-0x" +) + +var ErrProbeVolume = errors.New("Error scanning attached volumes") +var volNameToDeviceName = make(map[string]string) + +type PhotonDiskUtil struct{} + +func logError(msg string, err error) error { + s := "Photon Controller utility: " + msg + ". Error [" + err.Error() + "]" + glog.Errorf(s) + return fmt.Errorf(s) +} + +func removeFromScsiSubsystem(volName string) { + // TODO: if using pvscsi controller, this won't be needed + deviceName := volNameToDeviceName[volName] + fileName := "/sys/block/" + deviceName + "/device/delete" + data := []byte("1") + ioutil.WriteFile(fileName, data, 0666) +} + +func scsiHostScan() { + // TODO: if using pvscsi controller, this won't be needed + scsi_path := "/sys/class/scsi_host/" + if dirs, err := ioutil.ReadDir(scsi_path); err == nil { + for _, f := range dirs { + name := scsi_path + f.Name() + "/scan" + data := []byte("- - -") + ioutil.WriteFile(name, data, 0666) + glog.Errorf("scsiHostScan scan for %s", name) + } + } +} + +func verifyDevicePath(path string) (string, error) { + if pathExists, err := volumeutil.PathExists(path); err != nil { + return "", fmt.Errorf("Error checking if path exists: %v", err) + } else if pathExists { + return path, nil + } + + glog.V(4).Infof("verifyDevicePath: path not exists yet") + return "", nil +} + +// CreateVolume creates a PhotonController persistent disk. +func (util *PhotonDiskUtil) CreateVolume(p *photonPersistentDiskProvisioner) (pdID string, capacityGB int, err error) { + cloud, err := getCloudProvider(p.plugin.host.GetCloudProvider()) + if err != nil { + return "", 0, logError("CreateVolume failed to get cloud provider", err) + } + + capacity := p.options.PVC.Spec.Resources.Requests[api.ResourceName(api.ResourceStorage)] + volSizeBytes := capacity.Value() + // PhotonController works with GB, convert to GB with rounding up + volSizeGB := int(volume.RoundUpSize(volSizeBytes, 1024*1024*1024)) + name := volume.GenerateVolumeName(p.options.ClusterName, p.options.PVName, 255) + volumeOptions := &photon.VolumeOptions{ + CapacityGB: volSizeGB, + Tags: *p.options.CloudTags, + Name: name, + } + + for parameter, value := range p.options.Parameters { + switch strings.ToLower(parameter) { + case "flavor": + volumeOptions.Flavor = value + default: + return "", 0, logError("invalid option "+parameter+" for volume plugin "+p.plugin.GetPluginName(), err) + } + } + + pdID, err = cloud.CreateDisk(volumeOptions) + if err != nil { + return "", 0, logError("failed to CreateDisk", err) + } + + glog.V(4).Infof("Successfully created Photon Controller persistent disk %s", name) + return pdID, volSizeGB, nil +} + +// DeleteVolume deletes a vSphere volume. +func (util *PhotonDiskUtil) DeleteVolume(pd *photonPersistentDiskDeleter) error { + cloud, err := getCloudProvider(pd.plugin.host.GetCloudProvider()) + if err != nil { + return logError("DeleteVolume failed to get cloud provider", err) + } + + if err = cloud.DeleteDisk(pd.pdID); err != nil { + return logError("failed to DeleteDisk for pdID "+pd.pdID, err) + } + + glog.V(4).Infof("Successfully deleted PhotonController persistent disk %s", pd.pdID) + return nil +} + +func getCloudProvider(cloud cloudprovider.Interface) (*photon.PCCloud, error) { + if cloud == nil { + return nil, logError("Cloud provider not initialized properly", nil) + } + + pcc := cloud.(*photon.PCCloud) + if pcc == nil { + return nil, logError("Invalid cloud provider: expected Photon Controller", nil) + } + return pcc, nil +} diff --git a/test/test_owners.csv b/test/test_owners.csv index be056d9e98..381db478fc 100644 --- a/test/test_owners.csv +++ b/test/test_owners.csv @@ -573,6 +573,7 @@ k8s.io/kubernetes/pkg/cloudprovider/providers/gce,yifan-gu,1 k8s.io/kubernetes/pkg/cloudprovider/providers/mesos,mml,1 k8s.io/kubernetes/pkg/cloudprovider/providers/openstack,Q-Lee,1 k8s.io/kubernetes/pkg/cloudprovider/providers/ovirt,girishkalele,1 +k8s.io/kubernetes/pkg/cloudprovider/providers/photon,luomiao,0 k8s.io/kubernetes/pkg/cloudprovider/providers/rackspace,caesarxuchao,1 k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere,apelisse,1 k8s.io/kubernetes/pkg/controller,mikedanese,1 @@ -845,6 +846,7 @@ k8s.io/kubernetes/pkg/volume/glusterfs,timstclair,1 k8s.io/kubernetes/pkg/volume/host_path,jbeda,1 k8s.io/kubernetes/pkg/volume/iscsi,cjcullen,1 k8s.io/kubernetes/pkg/volume/nfs,justinsb,1 +k8s.io/kubernetes/pkg/volume/photon_pd,luomiao,0 k8s.io/kubernetes/pkg/volume/quobyte,yujuhong,1 k8s.io/kubernetes/pkg/volume/rbd,piosz,1 k8s.io/kubernetes/pkg/volume/secret,rmmh,1