github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #41558 from kubernetes/revert-40000-storage-ga-remove-alpha 

Revert "Remove alpha provisioning"
pull/6/head
Shyam JVS 2017-02-16 15:54:05 +01:00 committed by GitHub
parent 8ecc256e88 2ed7acfbcc
commit 7d4383c6e1
15 changed files with 175 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/kube-controller-manager/app/BUILD
View File

@ -69,6 +69,7 @@ go_library(
"//pkg/controller/ttl:go_default_library",
"//pkg/controller/volume/attachdetach:go_default_library",
"//pkg/controller/volume/persistentvolume:go_default_library",
"//pkg/features:go_default_library",
"//pkg/quota/install:go_default_library",
"//pkg/serviceaccount:go_default_library",
"//pkg/util/configz:go_default_library",
@ -97,6 +98,7 @@ go_library(
"//vendor:k8s.io/apimachinery/pkg/util/sets",
"//vendor:k8s.io/apimachinery/pkg/util/wait",
"//vendor:k8s.io/apiserver/pkg/server/healthz",
"//vendor:k8s.io/apiserver/pkg/util/feature",
"//vendor:k8s.io/client-go/discovery",
"//vendor:k8s.io/client-go/dynamic",
"//vendor:k8s.io/client-go/kubernetes/typed/core/v1",

5
cmd/kube-controller-manager/app/controllermanager.go
View File

@ -472,9 +472,14 @@ func StartControllers(controllers map[string]InitFunc, s *options.CMServer, root
glog.Infof("Will not configure cloud provider routes for allocate-node-cidrs: %v, configure-cloud-routes: %v.", s.AllocateNodeCIDRs, s.ConfigureCloudRoutes)
}
alphaProvisioner, err := NewAlphaVolumeProvisioner(cloud, s.VolumeConfiguration)
if err != nil {
return fmt.Errorf("an backward-compatible provisioner could not be created: %v, but one was expected. Provisioning will not work. This functionality is considered an early Alpha version.", err)
}
params := persistentvolumecontroller.ControllerParameters{
KubeClient: clientBuilder.ClientOrDie("persistent-volume-binder"),
SyncPeriod: s.PVClaimBinderSyncPeriod.Duration,
AlphaProvisioner: alphaProvisioner,
VolumePlugins: ProbeControllerVolumePlugins(cloud, s.VolumeConfiguration),
Cloud: cloud,
ClusterName: s.ClusterName,

32
cmd/kube-controller-manager/app/plugins.go
View File

@ -29,6 +29,7 @@ import (
// Volume plugins
"github.com/golang/glog"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/kubernetes/pkg/cloudprovider"
"k8s.io/kubernetes/pkg/cloudprovider/providers/aws"
"k8s.io/kubernetes/pkg/cloudprovider/providers/azure"
@ -36,6 +37,7 @@ import (
"k8s.io/kubernetes/pkg/cloudprovider/providers/openstack"
"k8s.io/kubernetes/pkg/cloudprovider/providers/photon"
"k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/util/io"
"k8s.io/kubernetes/pkg/volume"
"k8s.io/kubernetes/pkg/volume/aws_ebs"
@ -134,6 +136,36 @@ func ProbeControllerVolumePlugins(cloud cloudprovider.Interface, config componen
return allPlugins
}
// NewAlphaVolumeProvisioner returns a volume provisioner to use when running in
// a cloud or development environment. The alpha implementation of provisioning
// allows 1 implied provisioner per cloud and is here only for compatibility
// with Kubernetes 1.3
// TODO: remove in Kubernetes 1.5
func NewAlphaVolumeProvisioner(cloud cloudprovider.Interface, config componentconfig.VolumeConfiguration) (volume.ProvisionableVolumePlugin, error) {
switch {
case !utilfeature.DefaultFeatureGate.Enabled(features.DynamicVolumeProvisioning):
return nil, nil
case cloud == nil && config.EnableHostPathProvisioning:
return getProvisionablePluginFromVolumePlugins(host_path.ProbeVolumePlugins(
volume.VolumeConfig{
ProvisioningEnabled: true,
}))
case cloud != nil && aws.ProviderName == cloud.ProviderName():
return getProvisionablePluginFromVolumePlugins(aws_ebs.ProbeVolumePlugins())
case cloud != nil && gce.ProviderName == cloud.ProviderName():
return getProvisionablePluginFromVolumePlugins(gce_pd.ProbeVolumePlugins())
case cloud != nil && openstack.ProviderName == cloud.ProviderName():
return getProvisionablePluginFromVolumePlugins(cinder.ProbeVolumePlugins())
case cloud != nil && vsphere.ProviderName == cloud.ProviderName():
return getProvisionablePluginFromVolumePlugins(vsphere_volume.ProbeVolumePlugins())
case cloud != nil && azure.CloudProviderName == cloud.ProviderName():
return getProvisionablePluginFromVolumePlugins(azure_dd.ProbeVolumePlugins())
case cloud != nil && photon.ProviderName == cloud.ProviderName():
return getProvisionablePluginFromVolumePlugins(photon_pd.ProbeVolumePlugins())
}
return nil, nil
}
func getProvisionablePluginFromVolumePlugins(plugins []volume.VolumePlugin) (volume.ProvisionableVolumePlugin, error) {
for _, plugin := range plugins {
if provisonablePlugin, ok := plugin.(volume.ProvisionableVolumePlugin); ok {

2
examples/cockroachdb/cockroachdb-statefulset.yaml
View File

@ -183,6 +183,8 @@ spec:
volumeClaimTemplates:
- metadata:
name: datadir
annotations:
volume.alpha.kubernetes.io/storage-class: anything
spec:
accessModes:
- "ReadWriteOnce"

2
examples/storage/cassandra/README.md
View File

@ -254,6 +254,8 @@ spec:
volumeClaimTemplates:
- metadata:
name: cassandra-data
annotations:
volume.alpha.kubernetes.io/storage-class: anything
spec:
accessModes: [ "ReadWriteOnce" ]
resources:

2
examples/storage/cassandra/cassandra-statefulset.yaml
View File

@ -77,6 +77,8 @@ spec:
volumeClaimTemplates:
- metadata:
name: cassandra-data
annotations:
volume.alpha.kubernetes.io/storage-class: anything
spec:
accessModes: [ "ReadWriteOnce" ]
resources:

2
examples/volumes/nfs/provisioner/nfs-server-gce-pv.yaml
View File

@ -4,6 +4,8 @@ metadata:
name: nfs-pv-provisioning-demo
labels:
demo: nfs-pv-provisioning
annotations:
volume.alpha.kubernetes.io/storage-class: any
spec:
accessModes: [ "ReadWriteOnce" ]
resources:

14
pkg/apis/storage/util/helpers.go
View File

@ -27,6 +27,10 @@ import (
const IsDefaultStorageClassAnnotation = "storageclass.beta.kubernetes.io/is-default-class"
const BetaIsDefaultStorageClassAnnotation = "storageclass.beta.kubernetes.io/is-default-class"
// AlphaStorageClassAnnotation represents the previous alpha storage class
// annotation. it's no longer used and held here for posterity.
const AlphaStorageClassAnnotation = "volume.alpha.kubernetes.io/storage-class"
// BetaStorageClassAnnotation represents the beta/previous StorageClass annotation.
// It's currently still used and will be held for backwards compatibility
const BetaStorageClassAnnotation = "volume.beta.kubernetes.io/storage-class"
@ -71,7 +75,7 @@ func GetClaimStorageClass(claim *api.PersistentVolumeClaim) string {
// GetStorageClassAnnotation returns the StorageClass value
// if the annotation is set, empty string if not
// TODO: remove Beta when no longer used or needed
// TODO: remove Alpha and Beta when no longer used or needed
func GetStorageClassAnnotation(obj metav1.ObjectMeta) string {
if class, ok := obj.Annotations[StorageClassAnnotation]; ok {
return class
@ -79,13 +83,16 @@ func GetStorageClassAnnotation(obj metav1.ObjectMeta) string {
if class, ok := obj.Annotations[BetaStorageClassAnnotation]; ok {
return class
}
if class, ok := obj.Annotations[AlphaStorageClassAnnotation]; ok {
return class
}
return ""
}
// HasStorageClassAnnotation returns a boolean
// if the annotation is set
// TODO: remove Beta when no longer used or needed
// TODO: remove Alpha and Beta when no longer used or needed
func HasStorageClassAnnotation(obj metav1.ObjectMeta) bool {
if _, found := obj.Annotations[StorageClassAnnotation]; found {
return found
@ -93,6 +100,9 @@ func HasStorageClassAnnotation(obj metav1.ObjectMeta) bool {
if _, found := obj.Annotations[BetaStorageClassAnnotation]; found {
return found
}
if _, found := obj.Annotations[AlphaStorageClassAnnotation]; found {
return found
}
return false

17
pkg/apis/storage/v1beta1/util/helpers.go
View File

@ -27,6 +27,10 @@ import (
const IsDefaultStorageClassAnnotation = "storageclass.beta.kubernetes.io/is-default-class"
const BetaIsDefaultStorageClassAnnotation = "storageclass.beta.kubernetes.io/is-default-class"
// AlphaStorageClassAnnotation represents the previous alpha storage class
// annotation. it's no longer used and held here for posterity.
const AlphaStorageClassAnnotation = "volume.alpha.kubernetes.io/storage-class"
// BetaStorageClassAnnotation represents the beta/previous StorageClass annotation.
// It's currently still used and will be held for backwards compatibility
const BetaStorageClassAnnotation = "volume.beta.kubernetes.io/storage-class"
@ -71,7 +75,7 @@ func GetClaimStorageClass(claim *v1.PersistentVolumeClaim) string {
// GetStorageClassAnnotation returns the StorageClass value
// if the annotation is set, empty string if not
// TODO: remove Beta when no longer used or needed
// TODO: remove Alpha and Beta when no longer used or needed
func GetStorageClassAnnotation(obj metav1.ObjectMeta) string {
if class, ok := obj.Annotations[StorageClassAnnotation]; ok {
return class
@ -79,12 +83,16 @@ func GetStorageClassAnnotation(obj metav1.ObjectMeta) string {
if class, ok := obj.Annotations[BetaStorageClassAnnotation]; ok {
return class
}
if class, ok := obj.Annotations[AlphaStorageClassAnnotation]; ok {
return class
}
return ""
}
// HasStorageClassAnnotation returns a boolean
// if the annotation is set
// TODO: remove Beta when no longer used or needed
// TODO: remove Alpha and Beta when no longer used or needed
func HasStorageClassAnnotation(obj metav1.ObjectMeta) bool {
if _, found := obj.Annotations[StorageClassAnnotation]; found {
return found
@ -92,7 +100,12 @@ func HasStorageClassAnnotation(obj metav1.ObjectMeta) bool {
if _, found := obj.Annotations[BetaStorageClassAnnotation]; found {
return found
}
if _, found := obj.Annotations[AlphaStorageClassAnnotation]; found {
return found
}
return false
}
// IsDefaultAnnotationText returns a pretty Yes/No String if

3
pkg/controller/volume/persistentvolume/framework_test.go
View File

@ -834,6 +834,9 @@ func wrapTestWithPluginCalls(expectedRecycleCalls, expectedDeleteCalls []error,
provisionCalls: expectedProvisionCalls,
}
ctrl.volumePluginMgr.InitPlugins([]vol.VolumePlugin{plugin}, ctrl)
if expectedProvisionCalls != nil {
ctrl.alphaProvisioner = plugin
}
return toWrap(ctrl, reactor, test)
}
}

8
pkg/controller/volume/persistentvolume/index.go
View File

@ -130,6 +130,14 @@ func (pvIndex *persistentVolumeOrderedIndex) findByClaim(claim *v1.PersistentVol
return volume, nil
}
// In Alpha dynamic provisioning, we do now want not match claims
// with existing PVs, findByClaim must find only PVs that are
// pre-bound to the claim (by dynamic provisioning). TODO: remove in
// 1.5
if metav1.HasAnnotation(claim.ObjectMeta, storageutil.AlphaStorageClassAnnotation) {
continue
}
// filter out:
// - volumes bound to another claim
// - volumes whose labels don't match the claim's selector, if specified

34
pkg/controller/volume/persistentvolume/provision_test.go
View File

@ -95,6 +95,10 @@ var provision2Success = provisionCall{
expectedParameters: class2Parameters,
}
var provisionAlphaSuccess = provisionCall{
ret: nil,
}
// Test single call to syncVolume, expecting provisioning to happen.
// 1. Fill in the controller with initial data
// 2. Call the syncVolume *once*.
@ -356,6 +360,36 @@ func TestProvisionSync(t *testing.T) {
runSyncTests(t, tests, storageClasses)
}
func TestAlphaProvisionSync(t *testing.T) {
tests := []controllerTest{
{
// Provision a volume with alpha annotation
"14-1 - successful alpha provisioning",
novolumes,
newVolumeArray("pvc-uid14-1", "1Gi", "uid14-1", "claim14-1", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, annBoundByController, annDynamicallyProvisioned),
newClaimArray("claim14-1", "uid14-1", "1Gi", "", v1.ClaimPending, storageutil.AlphaStorageClassAnnotation),
// Binding will be completed in the next syncClaim
newClaimArray("claim14-1", "uid14-1", "1Gi", "", v1.ClaimPending, storageutil.AlphaStorageClassAnnotation, annStorageProvisioner),
noevents, noerrors, wrapTestWithProvisionCalls([]provisionCall{provisionAlphaSuccess}, testSyncClaim),
},
{
// Provision success - there is already a volume available, still
// we provision a new one when requested.
"14-2 - no alpha provisioning when there is a volume available",
newVolumeArray("volume14-2", "1Gi", "", "", v1.VolumePending, v1.PersistentVolumeReclaimRetain),
[]*v1.PersistentVolume{
newVolume("volume14-2", "1Gi", "", "", v1.VolumePending, v1.PersistentVolumeReclaimRetain),
newVolume("pvc-uid14-2", "1Gi", "uid14-2", "claim14-2", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, annBoundByController, annDynamicallyProvisioned),
},
newClaimArray("claim14-2", "uid14-2", "1Gi", "", v1.ClaimPending, storageutil.AlphaStorageClassAnnotation),
// Binding will be completed in the next syncClaim
newClaimArray("claim14-2", "uid14-2", "1Gi", "", v1.ClaimPending, storageutil.AlphaStorageClassAnnotation, annStorageProvisioner),
noevents, noerrors, wrapTestWithProvisionCalls([]provisionCall{provisionAlphaSuccess}, testSyncClaim),
},
}
runSyncTests(t, tests, []*storage.StorageClass{})
}
// Test multiple calls to syncClaim/syncVolume and periodic sync of all
// volume/claims. The test follows this pattern:
// 0. Load the controller with initial data.

49
pkg/controller/volume/persistentvolume/pv_controller.go
View File

@ -203,6 +203,10 @@ type PersistentVolumeController struct {
createProvisionedPVRetryCount int
createProvisionedPVInterval time.Duration
// Provisioner for annAlphaClass.
// TODO: remove in 1.5
alphaProvisioner vol.ProvisionableVolumePlugin
}
// syncClaim is the main controller method to decide what to do with a claim.
@ -239,7 +243,7 @@ func (ctrl *PersistentVolumeController) syncUnboundClaim(claim *v1.PersistentVol
glog.V(4).Infof("synchronizing unbound PersistentVolumeClaim[%s]: no volume found", claimToClaimKey(claim))
// No PV could be found
// OBSERVATION: pvc is "Pending", will retry
if storageutil.GetClaimStorageClass(claim) != "" {
if storageutil.GetClaimStorageClass(claim) != "" || metav1.HasAnnotation(claim.ObjectMeta, storageutil.AlphaStorageClassAnnotation) {
if err = ctrl.provisionClaim(claim); err != nil {
return err
}
@ -1336,7 +1340,12 @@ func (ctrl *PersistentVolumeController) provisionClaimOperation(claimObj interfa
// Add annBoundByController (used in deleting the volume)
metav1.SetMetaDataAnnotation(&volume.ObjectMeta, annBoundByController, "yes")
metav1.SetMetaDataAnnotation(&volume.ObjectMeta, annDynamicallyProvisioned, plugin.GetPluginName())
metav1.SetMetaDataAnnotation(&volume.ObjectMeta, storageutil.StorageClassAnnotation, claimClass)
// For Alpha provisioning behavior, do not add storage.BetaStorageClassAnnotations for volumes created
// by storage.AlphaStorageClassAnnotation
// TODO: remove this check in 1.5, storage.StorageClassAnnotation will be always non-empty there.
if claimClass != "" {
metav1.SetMetaDataAnnotation(&volume.ObjectMeta, storageutil.StorageClassAnnotation, claimClass)
}
// Try to create the PV object several times
for i := 0; i < ctrl.createProvisionedPVRetryCount; i++ {
@ -1438,6 +1447,20 @@ func (ctrl *PersistentVolumeController) newRecyclerEventRecorder(volume *v1.Pers
// It returns either the provisioning plugin or nil when an external
// provisioner is requested.
func (ctrl *PersistentVolumeController) findProvisionablePlugin(claim *v1.PersistentVolumeClaim) (vol.ProvisionableVolumePlugin, *storage.StorageClass, error) {
// TODO: remove this alpha behavior in 1.5
alpha := metav1.HasAnnotation(claim.ObjectMeta, storageutil.AlphaStorageClassAnnotation)
beta := metav1.HasAnnotation(claim.ObjectMeta, storageutil.BetaStorageClassAnnotation)
if alpha && beta {
// Both Alpha and Beta annotations are set. Do beta.
alpha = false
msg := fmt.Sprintf("both %q and %q annotations are present, using %q", storageutil.AlphaStorageClassAnnotation, storageutil.BetaStorageClassAnnotation, storageutil.BetaStorageClassAnnotation)
ctrl.eventRecorder.Event(claim, v1.EventTypeNormal, "ProvisioningIgnoreAlpha", msg)
}
if alpha {
// Fall back to fixed list of provisioner plugins
return ctrl.findAlphaProvisionablePlugin()
}
// provisionClaim() which leads here is never called with claimClass=="", we
// can save some checks.
claimClass := storageutil.GetClaimStorageClass(claim)
@ -1465,6 +1488,28 @@ func (ctrl *PersistentVolumeController) findProvisionablePlugin(claim *v1.Persis
return plugin, class, nil
}
// findAlphaProvisionablePlugin returns a volume plugin compatible with
// Kubernetes 1.3.
// TODO: remove in Kubernetes 1.5
func (ctrl *PersistentVolumeController) findAlphaProvisionablePlugin() (vol.ProvisionableVolumePlugin, *storage.StorageClass, error) {
if ctrl.alphaProvisioner == nil {
return nil, nil, fmt.Errorf("cannot find volume plugin for alpha provisioning")
}
// Return a dummy StorageClass instance with no parameters
storageClass := &storage.StorageClass{
TypeMeta: metav1.TypeMeta{
Kind: "StorageClass",
},
ObjectMeta: metav1.ObjectMeta{
Name: "",
},
Provisioner: ctrl.alphaProvisioner.GetPluginName(),
}
glog.V(4).Infof("using alpha provisioner %s", ctrl.alphaProvisioner.GetPluginName())
return ctrl.alphaProvisioner, storageClass, nil
}
// findDeletablePlugin finds a deleter plugin for a given volume. It returns
// either the deleter plugin or nil when an external deleter is requested.
func (ctrl *PersistentVolumeController) findDeletablePlugin(volume *v1.PersistentVolume) (vol.DeletableVolumePlugin, error) {

7
pkg/controller/volume/persistentvolume/pv_controller_base.go
View File

@ -53,6 +53,7 @@ import (
type ControllerParameters struct {
KubeClient clientset.Interface
SyncPeriod time.Duration
AlphaProvisioner vol.ProvisionableVolumePlugin
VolumePlugins []vol.VolumePlugin
Cloud cloudprovider.Interface
ClusterName string
@ -81,11 +82,17 @@ func NewController(p ControllerParameters) *PersistentVolumeController {
clusterName: p.ClusterName,
createProvisionedPVRetryCount: createProvisionedPVRetryCount,
createProvisionedPVInterval: createProvisionedPVInterval,
alphaProvisioner: p.AlphaProvisioner,
claimQueue: workqueue.NewNamed("claims"),
volumeQueue: workqueue.NewNamed("volumes"),
}
controller.volumePluginMgr.InitPlugins(p.VolumePlugins, controller)
if controller.alphaProvisioner != nil {
if err := controller.alphaProvisioner.Init(controller); err != nil {
glog.Errorf("PersistentVolumeController: error initializing alpha provisioner plugin: %v", err)
}
}
volumeSource := p.VolumeSource
if volumeSource == nil {

18
test/e2e/volume_provisioning.go
View File

@ -190,26 +190,12 @@ var _ = framework.KubeDescribe("Dynamic provisioning", func() {
It("should create and delete alpha persistent volumes [Slow] [Volume]", func() {
framework.SkipUnlessProviderIs("openstack", "gce", "aws", "gke", "vsphere")
// Check there is a default storage class
classes, err := c.Storage().StorageClasses().List(metav1.ListOptions{})
Expect(err).NotTo(HaveOccurred())
found := false
for _, class := range classes.Items {
if storageutil.IsDefaultAnnotation(class.ObjectMeta) {
found = true
break
}
}
if !found {
Skip("No default class found")
}
By("creating a claim with an alpha dynamic provisioning annotation")
claim := newClaim(ns, "", true)
defer func() {
c.Core().PersistentVolumeClaims(ns).Delete(claim.Name, nil)
}()
claim, err = c.Core().PersistentVolumeClaims(ns).Create(claim)
claim, err := c.Core().PersistentVolumeClaims(ns).Create(claim)
Expect(err).NotTo(HaveOccurred())
if framework.ProviderIs("vsphere") {
@ -277,7 +263,7 @@ func newClaim(ns, suffix string, alpha bool) *v1.PersistentVolumeClaim {
if alpha {
claim.Annotations = map[string]string{
"volume.alpha.kubernetes.io/storage-class": "anything",
storageutil.AlphaStorageClassAnnotation: "",
}
} else {
claim.Annotations = map[string]string{