github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

API changes for persistent local volumes. 

Includes:
- A new volume type, LocalVolumeSource.  This only supports
file-based local volumes for now.
- New alpha annotation in PV: NodeAffinity
- Validation + tests for specifying LocalVolumeSource and PV
NodeAffinity
- Alpha feature gate
pull/6/head
Michelle Au 2017-04-17 22:24:24 -07:00
parent f0e5a999e9
commit d848be195f
10 changed files with 558 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
pkg/api/helper/helpers.go
View File

@ -594,3 +594,29 @@ func PersistentVolumeClaimHasClass(claim *api.PersistentVolumeClaim) bool {
return false
}
// GetStorageNodeAffinityFromAnnotation gets the json serialized data from PersistentVolume.Annotations
// and converts it to the NodeAffinity type in api.
// TODO: update when storage node affinity graduates to beta
func GetStorageNodeAffinityFromAnnotation(annotations map[string]string) (*api.NodeAffinity, error) {
if len(annotations) > 0 && annotations[api.AlphaStorageNodeAffinityAnnotation] != "" {
var affinity api.NodeAffinity
err := json.Unmarshal([]byte(annotations[api.AlphaStorageNodeAffinityAnnotation]), &affinity)
if err != nil {
return nil, err
}
return &affinity, nil
}
return nil, nil
}
// Converts NodeAffinity type to Alpha annotation for use in PersistentVolumes
// TODO: update when storage node affinity graduates to beta
func StorageNodeAffinityToAlphaAnnotation(annotations map[string]string, affinity *api.NodeAffinity) error {
json, err := json.Marshal(*affinity)
if err != nil {
return err
}
annotations[api.AlphaStorageNodeAffinityAnnotation] = string(json)
return nil
}

87
pkg/api/helper/helpers_test.go
View File

@ -266,3 +266,90 @@ func TestSysctlsFromPodAnnotation(t *testing.T) {
}
}
}
// TODO: remove when alpha support for topology constraints is removed
func TestGetNodeAffinityFromAnnotations(t *testing.T) {
testCases := []struct {
annotations map[string]string
expectErr bool
}{
{
annotations: nil,
expectErr: false,
},
{
annotations: map[string]string{},
expectErr: false,
},
{
annotations: map[string]string{
api.AlphaStorageNodeAffinityAnnotation: `{
"requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": [
{ "matchExpressions": [
{ "key": "test-key1",
"operator": "In",
"values": ["test-value1", "test-value2"]
},
{ "key": "test-key2",
"operator": "In",
"values": ["test-value1", "test-value2"]
}
]}
]}
}`,
},
expectErr: false,
},
{
annotations: map[string]string{
api.AlphaStorageNodeAffinityAnnotation: `[{
"requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": [
{ "matchExpressions": [
{ "key": "test-key1",
"operator": "In",
"values": ["test-value1", "test-value2"]
},
{ "key": "test-key2",
"operator": "In",
"values": ["test-value1", "test-value2"]
}
]}
]}
}]`,
},
expectErr: true,
},
{
annotations: map[string]string{
api.AlphaStorageNodeAffinityAnnotation: `{
"requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms":
"matchExpressions": [
{ "key": "test-key1",
"operator": "In",
"values": ["test-value1", "test-value2"]
},
{ "key": "test-key2",
"operator": "In",
"values": ["test-value1", "test-value2"]
}
]}
}
}`,
},
expectErr: true,
},
}
for i, tc := range testCases {
_, err := GetStorageNodeAffinityFromAnnotation(tc.annotations)
if err == nil && tc.expectErr {
t.Errorf("[%v]expected error but got none.", i)
}
if err != nil && !tc.expectErr {
t.Errorf("[%v]did not expect error but got: %v", i, err)
}
}
}

15
pkg/api/types.go
View File

@ -381,6 +381,9 @@ type PersistentVolumeSource struct {
// ScaleIO represents a ScaleIO persistent volume attached and mounted on Kubernetes nodes.
// +optional
ScaleIO *ScaleIOVolumeSource
// Local represents directly-attached storage with node affinity
// +optional
Local *LocalVolumeSource
}
type PersistentVolumeClaimVolumeSource struct {
@ -399,6 +402,10 @@ const (
// MountOptionAnnotation defines mount option annotation used in PVs
MountOptionAnnotation = "volume.beta.kubernetes.io/mount-options"
// AlphaStorageNodeAffinityAnnotation defines node affinity policies for a PersistentVolume.
// Value is a string of the json representation of type NodeAffinity
AlphaStorageNodeAffinityAnnotation = "volume.alpha.kubernetes.io/node-affinity"
)
// +genclient=true
@ -1223,6 +1230,14 @@ type KeyToPath struct {
Mode *int32
}
// Local represents directly-attached storage with node affinity
type LocalVolumeSource struct {
// The full path to the volume on the node
// For alpha, this path must be a directory
// Once block as a source is supported, then this path can point to a block device
Path string
}
// ContainerPort represents a network port in a single container
type ContainerPort struct {
// Optional: If specified, this must be an IANA_SVC_NAME Each named port

26
pkg/api/v1/helper/helpers.go
View File

@ -498,3 +498,29 @@ func PersistentVolumeClaimHasClass(claim *v1.PersistentVolumeClaim) bool {
return false
}
// GetStorageNodeAffinityFromAnnotation gets the json serialized data from PersistentVolume.Annotations
// and converts it to the NodeAffinity type in api.
// TODO: update when storage node affinity graduates to beta
func GetStorageNodeAffinityFromAnnotation(annotations map[string]string) (*v1.NodeAffinity, error) {
if len(annotations) > 0 && annotations[v1.AlphaStorageNodeAffinityAnnotation] != "" {
var affinity v1.NodeAffinity
err := json.Unmarshal([]byte(annotations[v1.AlphaStorageNodeAffinityAnnotation]), &affinity)
if err != nil {
return nil, err
}
return &affinity, nil
}
return nil, nil
}
// Converts NodeAffinity type to Alpha annotation for use in PersistentVolumes
// TODO: update when storage node affinity graduates to beta
func StorageNodeAffinityToAlphaAnnotation(annotations map[string]string, affinity *v1.NodeAffinity) error {
json, err := json.Marshal(*affinity)
if err != nil {
return err
}
annotations[v1.AlphaStorageNodeAffinityAnnotation] = string(json)
return nil
}

87
pkg/api/v1/helper/helpers_test.go
View File

@ -499,3 +499,90 @@ func TestGetAffinityFromPodAnnotations(t *testing.T) {
}
}
}
// TODO: remove when alpha support for topology constraints is removed
func TestGetNodeAffinityFromAnnotations(t *testing.T) {
testCases := []struct {
annotations map[string]string
expectErr bool
}{
{
annotations: nil,
expectErr: false,
},
{
annotations: map[string]string{},
expectErr: false,
},
{
annotations: map[string]string{
v1.AlphaStorageNodeAffinityAnnotation: `{
"requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": [
{ "matchExpressions": [
{ "key": "test-key1",
"operator": "In",
"values": ["test-value1", "test-value2"]
},
{ "key": "test-key2",
"operator": "In",
"values": ["test-value1", "test-value2"]
}
]}
]}
}`,
},
expectErr: false,
},
{
annotations: map[string]string{
v1.AlphaStorageNodeAffinityAnnotation: `[{
"requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": [
{ "matchExpressions": [
{ "key": "test-key1",
"operator": "In",
"values": ["test-value1", "test-value2"]
},
{ "key": "test-key2",
"operator": "In",
"values": ["test-value1", "test-value2"]
}
]}
]}
}]`,
},
expectErr: true,
},
{
annotations: map[string]string{
v1.AlphaStorageNodeAffinityAnnotation: `{
"requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms":
"matchExpressions": [
{ "key": "test-key1",
"operator": "In",
"values": ["test-value1", "test-value2"]
},
{ "key": "test-key2",
"operator": "In",
"values": ["test-value1", "test-value2"]
}
]}
}
}`,
},
expectErr: true,
},
}
for i, tc := range testCases {
_, err := GetStorageNodeAffinityFromAnnotation(tc.annotations)
if err == nil && tc.expectErr {
t.Errorf("[%v]expected error but got none.", i)
}
if err != nil && !tc.expectErr {
t.Errorf("[%v]did not expect error but got: %v", i, err)
}
}
}

15
pkg/api/v1/types.go
View File

@ -439,6 +439,9 @@ type PersistentVolumeSource struct {
// ScaleIO represents a ScaleIO persistent volume attached and mounted on Kubernetes nodes.
// +optional
ScaleIO *ScaleIOVolumeSource `json:"scaleIO,omitempty" protobuf:"bytes,19,opt,name=scaleIO"`
// Local represents directly-attached storage with node affinity
// +optional
Local *LocalVolumeSource `json:"local,omitempty" protobuf:"bytes,20,opt,name=local"`
}
const (
@ -448,6 +451,10 @@ const (
// MountOptionAnnotation defines mount option annotation used in PVs
MountOptionAnnotation = "volume.beta.kubernetes.io/mount-options"
// AlphaStorageNodeAffinityAnnotation defines node affinity policies for a PersistentVolume.
// Value is a string of the json representation of type NodeAffinity
AlphaStorageNodeAffinityAnnotation = "volume.alpha.kubernetes.io/node-affinity"
)
// +genclient=true
@ -1310,6 +1317,14 @@ type KeyToPath struct {
Mode *int32 `json:"mode,omitempty" protobuf:"varint,3,opt,name=mode"`
}
// Local represents directly-attached storage with node affinity
type LocalVolumeSource struct {
// The full path to the volume on the node
// For alpha, this path must be a directory
// Once block as a source is supported, then this path can point to a block device
Path string `json:"path" protobuf:"bytes,1,opt,name=path"`
}
// ContainerPort represents a network port in a single container.
type ContainerPort struct {
// If specified, this must be an IANA_SVC_NAME and unique within the pod. Each

59
pkg/api/validation/validation.go
View File

@ -1101,6 +1101,14 @@ func validateScaleIOVolumeSource(sio *api.ScaleIOVolumeSource, fldPath *field.Pa
return allErrs
}
func validateLocalVolumeSource(ls *api.LocalVolumeSource, fldPath *field.Path) field.ErrorList {
allErrs := field.ErrorList{}
if ls.Path == "" {
allErrs = append(allErrs, field.Required(fldPath.Child("path"), ""))
}
return allErrs
}
// ValidatePersistentVolumeName checks that a name is appropriate for a
// PersistentVolumeName object.
var ValidatePersistentVolumeName = NameIsDNSSubdomain
@ -1110,7 +1118,8 @@ var supportedAccessModes = sets.NewString(string(api.ReadWriteOnce), string(api.
var supportedReclaimPolicy = sets.NewString(string(api.PersistentVolumeReclaimDelete), string(api.PersistentVolumeReclaimRecycle), string(api.PersistentVolumeReclaimRetain))
func ValidatePersistentVolume(pv *api.PersistentVolume) field.ErrorList {
allErrs := ValidateObjectMeta(&pv.ObjectMeta, false, ValidatePersistentVolumeName, field.NewPath("metadata"))
metaPath := field.NewPath("metadata")
allErrs := ValidateObjectMeta(&pv.ObjectMeta, false, ValidatePersistentVolumeName, metaPath)
specPath := field.NewPath("spec")
if len(pv.Spec.AccessModes) == 0 {
@ -1139,6 +1148,9 @@ func ValidatePersistentVolume(pv *api.PersistentVolume) field.ErrorList {
}
}
nodeAffinitySpecified, errs := validateStorageNodeAffinityAnnotation(pv.ObjectMeta.Annotations, metaPath.Child("annotations"))
allErrs = append(allErrs, errs...)
numVolumes := 0
if pv.Spec.HostPath != nil {
if numVolumes > 0 {
@ -1290,6 +1302,22 @@ func ValidatePersistentVolume(pv *api.PersistentVolume) field.ErrorList {
allErrs = append(allErrs, validateScaleIOVolumeSource(pv.Spec.ScaleIO, specPath.Child("scaleIO"))...)
}
}
if pv.Spec.Local != nil {
if numVolumes > 0 {
allErrs = append(allErrs, field.Forbidden(specPath.Child("local"), "may not specify more than 1 volume type"))
} else {
numVolumes++
if !utilfeature.DefaultFeatureGate.Enabled(features.PersistentLocalVolumes) {
allErrs = append(allErrs, field.Forbidden(specPath.Child("local"), "Local volumes are disabled by feature-gate"))
}
allErrs = append(allErrs, validateLocalVolumeSource(pv.Spec.Local, specPath.Child("local"))...)
// NodeAffinity is required
if !nodeAffinitySpecified {
allErrs = append(allErrs, field.Required(metaPath.Child("annotations"), "Local volume requires node affinity"))
}
}
}
if numVolumes == 0 {
allErrs = append(allErrs, field.Required(specPath, "must specify a volume type"))
@ -4044,3 +4072,32 @@ func sysctlIntersection(a []api.Sysctl, b []api.Sysctl) []string {
}
return result
}
// validateStorageNodeAffinityAnnotation tests that the serialized TopologyConstraints in PersistentVolume.Annotations has valid data
func validateStorageNodeAffinityAnnotation(annotations map[string]string, fldPath *field.Path) (bool, field.ErrorList) {
allErrs := field.ErrorList{}
na, err := helper.GetStorageNodeAffinityFromAnnotation(annotations)
if err != nil {
allErrs = append(allErrs, field.Invalid(fldPath, api.AlphaStorageNodeAffinityAnnotation, err.Error()))
return false, allErrs
}
if na == nil {
return false, allErrs
}
if !utilfeature.DefaultFeatureGate.Enabled(features.PersistentLocalVolumes) {
allErrs = append(allErrs, field.Forbidden(fldPath, "Storage node affinity is disabled by feature-gate"))
}
policySpecified := false
if na.RequiredDuringSchedulingIgnoredDuringExecution != nil {
allErrs = append(allErrs, ValidateNodeSelector(na.RequiredDuringSchedulingIgnoredDuringExecution, fldPath.Child("requiredDuringSchedulingIgnoredDuringExecution"))...)
policySpecified = true
}
if len(na.PreferredDuringSchedulingIgnoredDuringExecution) > 0 {
allErrs = append(allErrs, field.Forbidden(fldPath.Child("preferredDuringSchedulingIgnoredDuringExection"), "Storage node affinity does not support preferredDuringSchedulingIgnoredDuringExecution"))
}
return policySpecified, allErrs
}

229
pkg/api/validation/validation_test.go
View File

@ -58,6 +58,24 @@ func testVolume(name string, namespace string, spec api.PersistentVolumeSpec) *a
}
}
func testVolumeWithNodeAffinity(t *testing.T, name string, namespace string, affinity *api.NodeAffinity, spec api.PersistentVolumeSpec) *api.PersistentVolume {
objMeta := metav1.ObjectMeta{Name: name}
if namespace != "" {
objMeta.Namespace = namespace
}
objMeta.Annotations = map[string]string{}
err := helper.StorageNodeAffinityToAlphaAnnotation(objMeta.Annotations, affinity)
if err != nil {
t.Fatalf("Failed to get node affinity annotation: %v", err)
}
return &api.PersistentVolume{
ObjectMeta: objMeta,
Spec: spec,
}
}
func TestValidatePersistentVolumes(t *testing.T) {
scenarios := map[string]struct {
isExpectedFailure bool
@ -213,6 +231,42 @@ func TestValidatePersistentVolumes(t *testing.T) {
StorageClassName: "-invalid-",
}),
},
// LocalVolume alpha feature disabled
// TODO: remove when no longer alpha
"alpha disabled valid local volume": {
isExpectedFailure: true,
volume: testVolumeWithNodeAffinity(
t,
"valid-local-volume",
"",
&api.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &api.NodeSelector{
NodeSelectorTerms: []api.NodeSelectorTerm{
{
MatchExpressions: []api.NodeSelectorRequirement{
{
Key: "test-label-key",
Operator: api.NodeSelectorOpIn,
Values: []string{"test-label-value"},
},
},
},
},
},
},
api.PersistentVolumeSpec{
Capacity: api.ResourceList{
api.ResourceName(api.ResourceStorage): resource.MustParse("10G"),
},
AccessModes: []api.PersistentVolumeAccessMode{api.ReadWriteOnce},
PersistentVolumeSource: api.PersistentVolumeSource{
Local: &api.LocalVolumeSource{
Path: "/foo",
},
},
StorageClassName: "test-storage-class",
}),
},
}
for name, scenario := range scenarios {
@ -227,6 +281,181 @@ func TestValidatePersistentVolumes(t *testing.T) {
}
func TestValidateLocalVolumes(t *testing.T) {
scenarios := map[string]struct {
isExpectedFailure bool
volume *api.PersistentVolume
}{
"valid local volume": {
isExpectedFailure: false,
volume: testVolumeWithNodeAffinity(
t,
"valid-local-volume",
"",
&api.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &api.NodeSelector{
NodeSelectorTerms: []api.NodeSelectorTerm{
{
MatchExpressions: []api.NodeSelectorRequirement{
{
Key: "test-label-key",
Operator: api.NodeSelectorOpIn,
Values: []string{"test-label-value"},
},
},
},
},
},
},
api.PersistentVolumeSpec{
Capacity: api.ResourceList{
api.ResourceName(api.ResourceStorage): resource.MustParse("10G"),
},
AccessModes: []api.PersistentVolumeAccessMode{api.ReadWriteOnce},
PersistentVolumeSource: api.PersistentVolumeSource{
Local: &api.LocalVolumeSource{
Path: "/foo",
},
},
StorageClassName: "test-storage-class",
}),
},
"invalid local volume nil annotations": {
isExpectedFailure: true,
volume: testVolume(
"invalid-local-volume-nil-annotations",
"",
api.PersistentVolumeSpec{
Capacity: api.ResourceList{
api.ResourceName(api.ResourceStorage): resource.MustParse("10G"),
},
AccessModes: []api.PersistentVolumeAccessMode{api.ReadWriteOnce},
PersistentVolumeSource: api.PersistentVolumeSource{
Local: &api.LocalVolumeSource{
Path: "/foo",
},
},
StorageClassName: "test-storage-class",
}),
},
"invalid local volume empty affinity": {
isExpectedFailure: true,
volume: testVolumeWithNodeAffinity(
t,
"invalid-local-volume-empty-affinity",
"",
&api.NodeAffinity{},
api.PersistentVolumeSpec{
Capacity: api.ResourceList{
api.ResourceName(api.ResourceStorage): resource.MustParse("10G"),
},
AccessModes: []api.PersistentVolumeAccessMode{api.ReadWriteOnce},
PersistentVolumeSource: api.PersistentVolumeSource{
Local: &api.LocalVolumeSource{
Path: "/foo",
},
},
StorageClassName: "test-storage-class",
}),
},
"invalid local volume preferred affinity": {
isExpectedFailure: true,
volume: testVolumeWithNodeAffinity(
t,
"invalid-local-volume-preferred-affinity",
"",
&api.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &api.NodeSelector{
NodeSelectorTerms: []api.NodeSelectorTerm{
{
MatchExpressions: []api.NodeSelectorRequirement{
{
Key: "test-label-key",
Operator: api.NodeSelectorOpIn,
Values: []string{"test-label-value"},
},
},
},
},
},
PreferredDuringSchedulingIgnoredDuringExecution: []api.PreferredSchedulingTerm{
{
Weight: 10,
Preference: api.NodeSelectorTerm{
MatchExpressions: []api.NodeSelectorRequirement{
{
Key: "test-label-key",
Operator: api.NodeSelectorOpIn,
Values: []string{"test-label-value"},
},
},
},
},
},
},
api.PersistentVolumeSpec{
Capacity: api.ResourceList{
api.ResourceName(api.ResourceStorage): resource.MustParse("10G"),
},
AccessModes: []api.PersistentVolumeAccessMode{api.ReadWriteOnce},
PersistentVolumeSource: api.PersistentVolumeSource{
Local: &api.LocalVolumeSource{
Path: "/foo",
},
},
StorageClassName: "test-storage-class",
}),
},
"invalid local volume empty path": {
isExpectedFailure: true,
volume: testVolumeWithNodeAffinity(
t,
"invalid-local-volume-empty-path",
"",
&api.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &api.NodeSelector{
NodeSelectorTerms: []api.NodeSelectorTerm{
{
MatchExpressions: []api.NodeSelectorRequirement{
{
Key: "test-label-key",
Operator: api.NodeSelectorOpIn,
Values: []string{"test-label-value"},
},
},
},
},
},
},
api.PersistentVolumeSpec{
Capacity: api.ResourceList{
api.ResourceName(api.ResourceStorage): resource.MustParse("10G"),
},
AccessModes: []api.PersistentVolumeAccessMode{api.ReadWriteOnce},
PersistentVolumeSource: api.PersistentVolumeSource{
Local: &api.LocalVolumeSource{},
},
StorageClassName: "test-storage-class",
}),
},
}
err := utilfeature.DefaultFeatureGate.Set("PersistentLocalVolumes=true")
if err != nil {
t.Errorf("Failed to enable feature gate for LocalPersistentVolumes: %v", err)
return
}
for name, scenario := range scenarios {
errs := ValidatePersistentVolume(scenario.volume)
if len(errs) == 0 && scenario.isExpectedFailure {
t.Errorf("Unexpected success for scenario: %s", name)
}
if len(errs) > 0 && !scenario.isExpectedFailure {
t.Errorf("Unexpected failure for scenario: %s - %+v", name, errs)
}
}
}
func testVolumeClaim(name string, namespace string, spec api.PersistentVolumeClaimSpec) *api.PersistentVolumeClaim {
return &api.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace},

7
pkg/features/kube_features.go
View File

@ -88,6 +88,12 @@ const (
// Changes the logic behind evicting Pods from not ready Nodes
// to take advantage of NoExecute Taints and Tolerations.
TaintBasedEvictions utilfeature.Feature = "TaintBasedEvictions"
// owner: @msau
// alpha: v1.7
//
// A new volume type that supports local disks on a node.
PersistentLocalVolumes utilfeature.Feature = "PersistentLocalVolumes"
)
func init() {
@ -107,6 +113,7 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS
AffinityInAnnotations: {Default: false, PreRelease: utilfeature.Alpha},
Accelerators: {Default: false, PreRelease: utilfeature.Alpha},
TaintBasedEvictions: {Default: false, PreRelease: utilfeature.Alpha},
PersistentLocalVolumes: {Default: false, PreRelease: utilfeature.Alpha},
// inherited features from generic apiserver, relisted here to get a conflict if it is changed
// unintentionally on either side:

8
pkg/printers/internalversion/describe.go
View File

@ -912,6 +912,12 @@ func printScaleIOVolumeSource(sio *api.ScaleIOVolumeSource, w PrefixWriter) {
sio.Gateway, sio.System, sio.ProtectionDomain, sio.StoragePool, sio.StorageMode, sio.VolumeName, sio.FSType, sio.ReadOnly)
}
func printLocalVolumeSource(ls *api.LocalVolumeSource, w PrefixWriter) {
w.Write(LEVEL_2, "Type:\tLocalVolume (a persistent volume backed by local storage on a node)\n"+
" Path:\t%v\n",
ls.Path)
}
type PersistentVolumeDescriber struct {
clientset.Interface
}
@ -981,6 +987,8 @@ func describePersistentVolume(pv *api.PersistentVolume, events *api.EventList) (
printPortworxVolumeSource(pv.Spec.PortworxVolume, w)
case pv.Spec.ScaleIO != nil:
printScaleIOVolumeSource(pv.Spec.ScaleIO, w)
case pv.Spec.Local != nil:
printLocalVolumeSource(pv.Spec.Local, w)
}
if events != nil {