mirror of https://github.com/k3s-io/k3s
Introduce storage v1alpha1 and VolumeAttachment
Introduce the v1alpha1 version to the Kubernetes storage API. And add a new VolumeAttachment object to that version. This object will initially be used only by the new CSI Volume Plugin. Eventually existing volume plugins can be refactored to use it too.pull/6/head
Saad Ali
parent
81fa823a6c
commit
d96c105d71
|
|
@ -223,6 +223,7 @@ var apiVersionPriorities = map[schema.GroupVersion]priority{
|
|||
{Group: "settings.k8s.io", Version: "v1alpha1"}: {group: 16900, version: 9},
|
||||
{Group: "storage.k8s.io", Version: "v1"}: {group: 16800, version: 15},
|
||||
{Group: "storage.k8s.io", Version: "v1beta1"}: {group: 16800, version: 9},
|
||||
{Group: "storage.k8s.io", Version: "v1alpha1"}: {group: 16800, version: 1},
|
||||
{Group: "apiextensions.k8s.io", Version: "v1beta1"}: {group: 16700, version: 9},
|
||||
{Group: "admissionregistration.k8s.io", Version: "v1alpha1"}: {group: 16700, version: 9},
|
||||
{Group: "scheduling.k8s.io", Version: "v1alpha1"}: {group: 16600, version: 9},
|
||||
|
|
|
|||
|
|
@ -64,6 +64,7 @@ import (
|
|||
"k8s.io/kubernetes/pkg/apis/batch"
|
||||
"k8s.io/kubernetes/pkg/apis/extensions"
|
||||
"k8s.io/kubernetes/pkg/apis/networking"
|
||||
"k8s.io/kubernetes/pkg/apis/storage"
|
||||
"k8s.io/kubernetes/pkg/capabilities"
|
||||
"k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
|
||||
informers "k8s.io/kubernetes/pkg/client/informers/informers_generated/internalversion"
|
||||
|
|
@ -556,7 +557,10 @@ func BuildStorageFactory(s *options.ServerRunOptions) (*serverstorage.DefaultSto
|
|||
s.Etcd.StorageConfig, s.Etcd.DefaultStorageMediaType, legacyscheme.Codecs,
|
||||
serverstorage.NewDefaultResourceEncodingConfig(legacyscheme.Registry), storageGroupsToEncodingVersion,
|
||||
// FIXME (soltysh): this GroupVersionResource override should be configurable
|
||||
[]schema.GroupVersionResource{batch.Resource("cronjobs").WithVersion("v1beta1")},
|
||||
[]schema.GroupVersionResource{
|
||||
batch.Resource("cronjobs").WithVersion("v1beta1"),
|
||||
storage.Resource("volumeattachments").WithVersion("v1alpha1"),
|
||||
},
|
||||
master.DefaultAPIResourceConfigSource(), s.APIEnablement.RuntimeConfig)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error in initializing storage factory: %s", err)
|
||||
|
|
|
|||
|
|
@ -73,6 +73,7 @@ pkg/apis/storage
|
|||
pkg/apis/storage/util
|
||||
pkg/apis/storage/v1
|
||||
pkg/apis/storage/v1/util
|
||||
pkg/apis/storage/v1alpha1
|
||||
pkg/apis/storage/v1beta1
|
||||
pkg/apis/storage/v1beta1/util
|
||||
pkg/auth/authorizer/abac
|
||||
|
|
@ -458,6 +459,7 @@ staging/src/k8s.io/api/rbac/v1beta1
|
|||
staging/src/k8s.io/api/scheduling/v1alpha1
|
||||
staging/src/k8s.io/api/settings/v1alpha1
|
||||
staging/src/k8s.io/api/storage/v1
|
||||
staging/src/k8s.io/api/storage/v1alpha1
|
||||
staging/src/k8s.io/api/storage/v1beta1
|
||||
staging/src/k8s.io/apiextensions-apiserver/examples/client-go/pkg/apis/cr
|
||||
staging/src/k8s.io/apiextensions-apiserver/examples/client-go/pkg/apis/cr/v1
|
||||
|
|
@ -666,6 +668,8 @@ staging/src/k8s.io/client-go/kubernetes/typed/settings/v1alpha1
|
|||
staging/src/k8s.io/client-go/kubernetes/typed/settings/v1alpha1/fake
|
||||
staging/src/k8s.io/client-go/kubernetes/typed/storage/v1
|
||||
staging/src/k8s.io/client-go/kubernetes/typed/storage/v1/fake
|
||||
staging/src/k8s.io/client-go/kubernetes/typed/storage/v1alpha1
|
||||
staging/src/k8s.io/client-go/kubernetes/typed/storage/v1alpha1/fake
|
||||
staging/src/k8s.io/client-go/kubernetes/typed/storage/v1beta1
|
||||
staging/src/k8s.io/client-go/kubernetes/typed/storage/v1beta1/fake
|
||||
staging/src/k8s.io/client-go/plugin/pkg/auth/authenticator/token/oidc/testing
|
||||
|
|
|
|||
|
|
@ -76,6 +76,7 @@ rbac.authorization.k8s.io/v1beta1 \
|
|||
rbac.authorization.k8s.io/v1alpha1 \
|
||||
scheduling.k8s.io/v1alpha1 \
|
||||
settings.k8s.io/v1alpha1 \
|
||||
storage.k8s.io/v1alpha1 \
|
||||
storage.k8s.io/v1beta1 \
|
||||
storage.k8s.io/v1 \
|
||||
}"
|
||||
|
|
|
|||
|
|
@ -67,6 +67,7 @@ PACKAGES=(
|
|||
k8s.io/api/imagepolicy/v1alpha1
|
||||
k8s.io/api/scheduling/v1alpha1
|
||||
k8s.io/api/settings/v1alpha1
|
||||
k8s.io/api/storage/v1alpha1
|
||||
k8s.io/api/storage/v1beta1
|
||||
k8s.io/api/storage/v1
|
||||
k8s.io/api/admissionregistration/v1alpha1
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ go_library(
|
|||
"//pkg/api/legacyscheme:go_default_library",
|
||||
"//pkg/apis/storage:go_default_library",
|
||||
"//pkg/apis/storage/v1:go_default_library",
|
||||
"//pkg/apis/storage/v1alpha1:go_default_library",
|
||||
"//pkg/apis/storage/v1beta1:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/apimachinery/announced:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library",
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ import (
|
|||
"k8s.io/kubernetes/pkg/api/legacyscheme"
|
||||
"k8s.io/kubernetes/pkg/apis/storage"
|
||||
"k8s.io/kubernetes/pkg/apis/storage/v1"
|
||||
"k8s.io/kubernetes/pkg/apis/storage/v1alpha1"
|
||||
"k8s.io/kubernetes/pkg/apis/storage/v1beta1"
|
||||
)
|
||||
|
||||
|
|
@ -37,14 +38,18 @@ func init() {
|
|||
func Install(groupFactoryRegistry announced.APIGroupFactoryRegistry, registry *registered.APIRegistrationManager, scheme *runtime.Scheme) {
|
||||
if err := announced.NewGroupMetaFactory(
|
||||
&announced.GroupMetaFactoryArgs{
|
||||
GroupName: storage.GroupName,
|
||||
VersionPreferenceOrder: []string{v1.SchemeGroupVersion.Version, v1beta1.SchemeGroupVersion.Version},
|
||||
RootScopedKinds: sets.NewString("StorageClass"),
|
||||
GroupName: storage.GroupName,
|
||||
VersionPreferenceOrder: []string{v1.SchemeGroupVersion.Version, v1beta1.SchemeGroupVersion.Version, v1alpha1.SchemeGroupVersion.Version},
|
||||
RootScopedKinds: sets.NewString(
|
||||
"StorageClass",
|
||||
"VolumeAttachment",
|
||||
),
|
||||
AddInternalObjectsToScheme: storage.AddToScheme,
|
||||
},
|
||||
announced.VersionToSchemeFunc{
|
||||
v1.SchemeGroupVersion.Version: v1.AddToScheme,
|
||||
v1beta1.SchemeGroupVersion.Version: v1beta1.AddToScheme,
|
||||
v1.SchemeGroupVersion.Version: v1.AddToScheme,
|
||||
v1beta1.SchemeGroupVersion.Version: v1beta1.AddToScheme,
|
||||
v1alpha1.SchemeGroupVersion.Version: v1alpha1.AddToScheme,
|
||||
},
|
||||
).Announce(groupFactoryRegistry).RegisterAndEnable(registry, scheme); err != nil {
|
||||
panic(err)
|
||||
|
|
|
|||
|
|
@ -46,6 +46,8 @@ func addKnownTypes(scheme *runtime.Scheme) error {
|
|||
scheme.AddKnownTypes(SchemeGroupVersion,
|
||||
&StorageClass{},
|
||||
&StorageClassList{},
|
||||
&VolumeAttachment{},
|
||||
&VolumeAttachmentList{},
|
||||
)
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -80,3 +80,110 @@ type StorageClassList struct {
|
|||
// Items is the list of StorageClasses
|
||||
Items []StorageClass
|
||||
}
|
||||
|
||||
// +genclient
|
||||
// +genclient:nonNamespaced
|
||||
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
|
||||
|
||||
// Captures the intent to attach or detach the specified volume to/from
|
||||
// the specified node.
|
||||
//
|
||||
// VolumeAttachment objects are non-namespaced.
|
||||
type VolumeAttachment struct {
|
||||
metav1.TypeMeta
|
||||
|
||||
// Standard object metadata.
|
||||
// More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata
|
||||
// +optional
|
||||
metav1.ObjectMeta
|
||||
|
||||
// Specification of the desired attach/detach volume behavior.
|
||||
// Populated by the Kubernetes system.
|
||||
Spec VolumeAttachmentSpec
|
||||
|
||||
// Status of the VolumeAttachment request.
|
||||
// Populated by the entity completing the attach or detach
|
||||
// operation, i.e. the external-attacher.
|
||||
// +optional
|
||||
Status VolumeAttachmentStatus
|
||||
}
|
||||
|
||||
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
|
||||
|
||||
// VolumeAttachmentList is a collection of VolumeAttachment objects.
|
||||
type VolumeAttachmentList struct {
|
||||
metav1.TypeMeta
|
||||
// Standard list metadata
|
||||
// More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata
|
||||
// +optional
|
||||
metav1.ListMeta
|
||||
|
||||
// Items is the list of VolumeAttachments
|
||||
Items []VolumeAttachment
|
||||
}
|
||||
|
||||
// The specification of a VolumeAttachment request.
|
||||
type VolumeAttachmentSpec struct {
|
||||
// Attacher indicates the name of the volume driver that MUST handle this
|
||||
// request. This is the name returned by GetPluginName().
|
||||
Attacher string
|
||||
|
||||
// Source represents the volume that should be attached.
|
||||
Source VolumeAttachmentSource
|
||||
|
||||
// The node that the volume should be attached to.
|
||||
NodeName string
|
||||
}
|
||||
|
||||
// VolumeAttachmentSource represents a volume that should be attached.
|
||||
// Right now only PersistenVolumes can be attached via external attacher,
|
||||
// in future we may allow also inline volumes in pods.
|
||||
// Exactly one member can be set.
|
||||
type VolumeAttachmentSource struct {
|
||||
// Name of the persistent volume to attach.
|
||||
// +optional
|
||||
PersistentVolumeName *string
|
||||
|
||||
// Placeholder for *VolumeSource to accommodate inline volumes in pods.
|
||||
}
|
||||
|
||||
// The status of a VolumeAttachment request.
|
||||
type VolumeAttachmentStatus struct {
|
||||
// Indicates the volume is successfully attached.
|
||||
// This field must only be set by the entity completing the attach
|
||||
// operation, i.e. the external-attacher.
|
||||
Attached bool
|
||||
|
||||
// Upon successful attach, this field is populated with any
|
||||
// information returned by the attach operation that must be passed
|
||||
// into subsequent WaitForAttach or Mount calls.
|
||||
// This field must only be set by the entity completing the attach
|
||||
// operation, i.e. the external-attacher.
|
||||
// +optional
|
||||
AttachmentMetadata map[string]string
|
||||
|
||||
// The last error encountered during attach operation, if any.
|
||||
// This field must only be set by the entity completing the attach
|
||||
// operation, i.e. the external-attacher.
|
||||
// +optional
|
||||
AttachError *VolumeError
|
||||
|
||||
// The last error encountered during detach operation, if any.
|
||||
// This field must only be set by the entity completing the detach
|
||||
// operation, i.e. the external-attacher.
|
||||
// +optional
|
||||
DetachError *VolumeError
|
||||
}
|
||||
|
||||
// Captures an error encountered during a volume operation.
|
||||
type VolumeError struct {
|
||||
// Time the error was encountered.
|
||||
// +optional
|
||||
Time metav1.Time
|
||||
|
||||
// String detailing the error encountered during Attach or Detach operation.
|
||||
// This string maybe logged, so it should not contain sensitive
|
||||
// information.
|
||||
// +optional
|
||||
Message string
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,22 @@
|
|||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// +k8s:conversion-gen=k8s.io/kubernetes/pkg/apis/storage
|
||||
// +k8s:conversion-gen-external-types=../../../../vendor/k8s.io/api/storage/v1alpha1
|
||||
// +groupName=storage.k8s.io
|
||||
// +k8s:defaulter-gen=TypeMeta
|
||||
// +k8s:defaulter-gen-input=../../../../vendor/k8s.io/api/storage/v1alpha1
|
||||
package v1alpha1 // import "k8s.io/kubernetes/pkg/apis/storage/v1alpha1"
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package v1alpha1
|
||||
|
||||
import (
|
||||
storagev1alpha1 "k8s.io/api/storage/v1alpha1"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
)
|
||||
|
||||
// GroupName is the group name use in this package
|
||||
const GroupName = "storage.k8s.io"
|
||||
|
||||
// SchemeGroupVersion is group version used to register these objects
|
||||
var SchemeGroupVersion = schema.GroupVersion{Group: GroupName, Version: "v1alpha1"}
|
||||
|
||||
// Resource takes an unqualified resource and returns a Group qualified GroupResource
|
||||
func Resource(resource string) schema.GroupResource {
|
||||
return SchemeGroupVersion.WithResource(resource).GroupResource()
|
||||
}
|
||||
|
||||
var (
|
||||
localSchemeBuilder = &storagev1alpha1.SchemeBuilder
|
||||
AddToScheme = localSchemeBuilder.AddToScheme
|
||||
)
|
||||
|
|
@ -20,6 +20,7 @@ import (
|
|||
"reflect"
|
||||
"strings"
|
||||
|
||||
apiequality "k8s.io/apimachinery/pkg/api/equality"
|
||||
"k8s.io/apimachinery/pkg/util/sets"
|
||||
"k8s.io/apimachinery/pkg/util/validation"
|
||||
"k8s.io/apimachinery/pkg/util/validation/field"
|
||||
|
|
@ -30,6 +31,14 @@ import (
|
|||
"k8s.io/kubernetes/pkg/features"
|
||||
)
|
||||
|
||||
const (
|
||||
maxProvisionerParameterSize = 256 * (1 << 10) // 256 kB
|
||||
maxProvisionerParameterLen = 512
|
||||
|
||||
maxAttachedVolumeMetadataSize = 256 * (1 << 10) // 256 kB
|
||||
maxVolumeErrorMessageSize = 1024
|
||||
)
|
||||
|
||||
// ValidateStorageClass validates a StorageClass.
|
||||
func ValidateStorageClass(storageClass *storage.StorageClass) field.ErrorList {
|
||||
allErrs := apivalidation.ValidateObjectMeta(&storageClass.ObjectMeta, false, apivalidation.ValidateClassName, field.NewPath("metadata"))
|
||||
|
|
@ -72,9 +81,6 @@ func validateProvisioner(provisioner string, fldPath *field.Path) field.ErrorLis
|
|||
return allErrs
|
||||
}
|
||||
|
||||
const maxProvisionerParameterSize = 256 * (1 << 10) // 256 kB
|
||||
const maxProvisionerParameterLen = 512
|
||||
|
||||
// validateParameters tests that keys are qualified names and that provisionerParameter are < 256kB.
|
||||
func validateParameters(params map[string]string, fldPath *field.Path) field.ErrorList {
|
||||
var totalSize int64
|
||||
|
|
@ -121,3 +127,94 @@ func validateAllowVolumeExpansion(allowExpand *bool, fldPath *field.Path) field.
|
|||
}
|
||||
return allErrs
|
||||
}
|
||||
|
||||
// ValidateVolumeAttachment validates a VolumeAttachment.
|
||||
func ValidateVolumeAttachment(volumeAttachment *storage.VolumeAttachment) field.ErrorList {
|
||||
allErrs := apivalidation.ValidateObjectMeta(&volumeAttachment.ObjectMeta, false, apivalidation.ValidateClassName, field.NewPath("metadata"))
|
||||
allErrs = append(allErrs, validateVolumeAttachmentSpec(&volumeAttachment.Spec, field.NewPath("spec"))...)
|
||||
allErrs = append(allErrs, validateVolumeAttachmentStatus(&volumeAttachment.Status, field.NewPath("status"))...)
|
||||
return allErrs
|
||||
}
|
||||
|
||||
// ValidateVolumeAttachmentSpec tests that the specified VolumeAttachmentSpec
|
||||
// has valid data.
|
||||
func validateVolumeAttachmentSpec(
|
||||
spec *storage.VolumeAttachmentSpec, fldPath *field.Path) field.ErrorList {
|
||||
allErrs := field.ErrorList{}
|
||||
allErrs = append(allErrs, validateAttacher(spec.Attacher, fldPath.Child("attacher"))...)
|
||||
allErrs = append(allErrs, validateVolumeAttachmentSource(&spec.Source, fldPath.Child("source"))...)
|
||||
allErrs = append(allErrs, validateNodeName(spec.NodeName, fldPath.Child("nodeName"))...)
|
||||
return allErrs
|
||||
}
|
||||
|
||||
// validateAttacher tests if attacher is a valid qualified name.
|
||||
func validateAttacher(attacher string, fldPath *field.Path) field.ErrorList {
|
||||
allErrs := field.ErrorList{}
|
||||
if len(attacher) == 0 {
|
||||
allErrs = append(allErrs, field.Required(fldPath, attacher))
|
||||
}
|
||||
return allErrs
|
||||
}
|
||||
|
||||
// validateSource tests if the source is valid for VolumeAttachment.
|
||||
func validateVolumeAttachmentSource(source *storage.VolumeAttachmentSource, fldPath *field.Path) field.ErrorList {
|
||||
allErrs := field.ErrorList{}
|
||||
if source.PersistentVolumeName == nil || len(*source.PersistentVolumeName) == 0 {
|
||||
allErrs = append(allErrs, field.Required(fldPath, ""))
|
||||
}
|
||||
return allErrs
|
||||
}
|
||||
|
||||
// validateNodeName tests if the nodeName is valid for VolumeAttachment.
|
||||
func validateNodeName(nodeName string, fldPath *field.Path) field.ErrorList {
|
||||
allErrs := field.ErrorList{}
|
||||
for _, msg := range apivalidation.ValidateNodeName(nodeName, false /* prefix */) {
|
||||
allErrs = append(allErrs, field.Invalid(fldPath, nodeName, msg))
|
||||
}
|
||||
return allErrs
|
||||
}
|
||||
|
||||
// validaVolumeAttachmentStatus tests if volumeAttachmentStatus is valid.
|
||||
func validateVolumeAttachmentStatus(status *storage.VolumeAttachmentStatus, fldPath *field.Path) field.ErrorList {
|
||||
allErrs := field.ErrorList{}
|
||||
allErrs = append(allErrs, validateAttachmentMetadata(status.AttachmentMetadata, fldPath.Child("attachmentMetadata"))...)
|
||||
allErrs = append(allErrs, validateVolumeError(status.AttachError, fldPath.Child("attachError"))...)
|
||||
allErrs = append(allErrs, validateVolumeError(status.DetachError, fldPath.Child("detachError"))...)
|
||||
return allErrs
|
||||
}
|
||||
|
||||
func validateAttachmentMetadata(metadata map[string]string, fldPath *field.Path) field.ErrorList {
|
||||
allErrs := field.ErrorList{}
|
||||
|
||||
var size int64
|
||||
for k, v := range metadata {
|
||||
size += (int64)(len(k)) + (int64)(len(v))
|
||||
}
|
||||
if size > maxAttachedVolumeMetadataSize {
|
||||
allErrs = append(allErrs, field.TooLong(fldPath, metadata, maxAttachedVolumeMetadataSize))
|
||||
}
|
||||
return allErrs
|
||||
}
|
||||
|
||||
func validateVolumeError(e *storage.VolumeError, fldPath *field.Path) field.ErrorList {
|
||||
allErrs := field.ErrorList{}
|
||||
|
||||
if e == nil {
|
||||
return allErrs
|
||||
}
|
||||
if len(e.Message) > maxVolumeErrorMessageSize {
|
||||
allErrs = append(allErrs, field.TooLong(fldPath.Child("message"), e.Message, maxAttachedVolumeMetadataSize))
|
||||
}
|
||||
return allErrs
|
||||
}
|
||||
|
||||
// ValidateVolumeAttachmentUpdate validates a VolumeAttachment.
|
||||
func ValidateVolumeAttachmentUpdate(new, old *storage.VolumeAttachment) field.ErrorList {
|
||||
allErrs := ValidateVolumeAttachment(new)
|
||||
|
||||
// Spec is read-only
|
||||
if !apiequality.Semantic.DeepEqual(old.Spec, new.Spec) {
|
||||
allErrs = append(allErrs, field.Invalid(field.NewPath("spec"), new.Spec, "field is immutable"))
|
||||
}
|
||||
return allErrs
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ package validation
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
|
|
@ -157,3 +158,281 @@ func TestAlphaExpandPersistentVolumesFeatureValidation(t *testing.T) {
|
|||
}
|
||||
|
||||
}
|
||||
|
||||
func TestVolumeAttachmentValidation(t *testing.T) {
|
||||
volumeName := "pv-name"
|
||||
empty := ""
|
||||
successCases := []storage.VolumeAttachment{
|
||||
{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "foo"},
|
||||
Spec: storage.VolumeAttachmentSpec{
|
||||
Attacher: "myattacher",
|
||||
Source: storage.VolumeAttachmentSource{
|
||||
PersistentVolumeName: &volumeName,
|
||||
},
|
||||
NodeName: "mynode",
|
||||
},
|
||||
},
|
||||
{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "foo-with-status"},
|
||||
Spec: storage.VolumeAttachmentSpec{
|
||||
Attacher: "myattacher",
|
||||
Source: storage.VolumeAttachmentSource{
|
||||
PersistentVolumeName: &volumeName,
|
||||
},
|
||||
NodeName: "mynode",
|
||||
},
|
||||
Status: storage.VolumeAttachmentStatus{
|
||||
Attached: true,
|
||||
AttachmentMetadata: map[string]string{
|
||||
"foo": "bar",
|
||||
},
|
||||
AttachError: &storage.VolumeError{
|
||||
Time: metav1.Time{},
|
||||
Message: "hello world",
|
||||
},
|
||||
DetachError: &storage.VolumeError{
|
||||
Time: metav1.Time{},
|
||||
Message: "hello world",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, volumeAttachment := range successCases {
|
||||
if errs := ValidateVolumeAttachment(&volumeAttachment); len(errs) != 0 {
|
||||
t.Errorf("expected success: %v", errs)
|
||||
}
|
||||
}
|
||||
errorCases := []storage.VolumeAttachment{
|
||||
{
|
||||
// Empty attacher name
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "foo"},
|
||||
Spec: storage.VolumeAttachmentSpec{
|
||||
Attacher: "",
|
||||
NodeName: "mynode",
|
||||
},
|
||||
},
|
||||
{
|
||||
// Invalid attacher name
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "foo"},
|
||||
Spec: storage.VolumeAttachmentSpec{
|
||||
Attacher: "invalid!@#$%^&*()",
|
||||
NodeName: "mynode",
|
||||
},
|
||||
},
|
||||
{
|
||||
// Empty node name
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "foo"},
|
||||
Spec: storage.VolumeAttachmentSpec{
|
||||
Attacher: "myattacher",
|
||||
NodeName: "",
|
||||
},
|
||||
},
|
||||
{
|
||||
// No volume name
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "foo"},
|
||||
Spec: storage.VolumeAttachmentSpec{
|
||||
Attacher: "myattacher",
|
||||
NodeName: "node",
|
||||
Source: storage.VolumeAttachmentSource{
|
||||
PersistentVolumeName: nil,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
// Empty volume name
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "foo"},
|
||||
Spec: storage.VolumeAttachmentSpec{
|
||||
Attacher: "myattacher",
|
||||
NodeName: "node",
|
||||
Source: storage.VolumeAttachmentSource{
|
||||
PersistentVolumeName: &empty,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
// Too long error message
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "foo"},
|
||||
Spec: storage.VolumeAttachmentSpec{
|
||||
Attacher: "myattacher",
|
||||
NodeName: "node",
|
||||
Source: storage.VolumeAttachmentSource{
|
||||
PersistentVolumeName: &volumeName,
|
||||
},
|
||||
},
|
||||
Status: storage.VolumeAttachmentStatus{
|
||||
Attached: true,
|
||||
AttachmentMetadata: map[string]string{
|
||||
"foo": "bar",
|
||||
},
|
||||
AttachError: &storage.VolumeError{
|
||||
Time: metav1.Time{},
|
||||
Message: "hello world",
|
||||
},
|
||||
DetachError: &storage.VolumeError{
|
||||
Time: metav1.Time{},
|
||||
Message: strings.Repeat("a", maxVolumeErrorMessageSize+1),
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
// Too long metadata
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "foo"},
|
||||
Spec: storage.VolumeAttachmentSpec{
|
||||
Attacher: "myattacher",
|
||||
NodeName: "node",
|
||||
Source: storage.VolumeAttachmentSource{
|
||||
PersistentVolumeName: &volumeName,
|
||||
},
|
||||
},
|
||||
Status: storage.VolumeAttachmentStatus{
|
||||
Attached: true,
|
||||
AttachmentMetadata: map[string]string{
|
||||
"foo": strings.Repeat("a", maxAttachedVolumeMetadataSize),
|
||||
},
|
||||
AttachError: &storage.VolumeError{
|
||||
Time: metav1.Time{},
|
||||
Message: "hello world",
|
||||
},
|
||||
DetachError: &storage.VolumeError{
|
||||
Time: metav1.Time{},
|
||||
Message: "hello world",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, volumeAttachment := range errorCases {
|
||||
if errs := ValidateVolumeAttachment(&volumeAttachment); len(errs) == 0 {
|
||||
t.Errorf("Expected failure for test: %v", volumeAttachment)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestVolumeAttachmentUpdateValidation(t *testing.T) {
|
||||
volumeName := "foo"
|
||||
newVolumeName := "bar"
|
||||
|
||||
old := storage.VolumeAttachment{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "foo"},
|
||||
Spec: storage.VolumeAttachmentSpec{
|
||||
Attacher: "myattacher",
|
||||
Source: storage.VolumeAttachmentSource{
|
||||
PersistentVolumeName: &volumeName,
|
||||
},
|
||||
NodeName: "mynode",
|
||||
},
|
||||
}
|
||||
successCases := []storage.VolumeAttachment{
|
||||
{
|
||||
// no change
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "foo"},
|
||||
Spec: storage.VolumeAttachmentSpec{
|
||||
Attacher: "myattacher",
|
||||
Source: storage.VolumeAttachmentSource{
|
||||
PersistentVolumeName: &volumeName,
|
||||
},
|
||||
NodeName: "mynode",
|
||||
},
|
||||
},
|
||||
{
|
||||
// modify status
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "foo"},
|
||||
Spec: storage.VolumeAttachmentSpec{
|
||||
Attacher: "myattacher",
|
||||
Source: storage.VolumeAttachmentSource{
|
||||
PersistentVolumeName: &volumeName,
|
||||
},
|
||||
NodeName: "mynode",
|
||||
},
|
||||
Status: storage.VolumeAttachmentStatus{
|
||||
Attached: true,
|
||||
AttachmentMetadata: map[string]string{
|
||||
"foo": "bar",
|
||||
},
|
||||
AttachError: &storage.VolumeError{
|
||||
Time: metav1.Time{},
|
||||
Message: "hello world",
|
||||
},
|
||||
DetachError: &storage.VolumeError{
|
||||
Time: metav1.Time{},
|
||||
Message: "hello world",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, volumeAttachment := range successCases {
|
||||
if errs := ValidateVolumeAttachmentUpdate(&volumeAttachment, &old); len(errs) != 0 {
|
||||
t.Errorf("expected success: %v", errs)
|
||||
}
|
||||
}
|
||||
|
||||
errorCases := []storage.VolumeAttachment{
|
||||
{
|
||||
// change attacher
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "foo"},
|
||||
Spec: storage.VolumeAttachmentSpec{
|
||||
Attacher: "another-attacher",
|
||||
Source: storage.VolumeAttachmentSource{
|
||||
PersistentVolumeName: &volumeName,
|
||||
},
|
||||
NodeName: "mynode",
|
||||
},
|
||||
},
|
||||
{
|
||||
// change volume
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "foo"},
|
||||
Spec: storage.VolumeAttachmentSpec{
|
||||
Attacher: "myattacher",
|
||||
Source: storage.VolumeAttachmentSource{
|
||||
PersistentVolumeName: &newVolumeName,
|
||||
},
|
||||
NodeName: "mynode",
|
||||
},
|
||||
},
|
||||
{
|
||||
// change node
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "foo"},
|
||||
Spec: storage.VolumeAttachmentSpec{
|
||||
Attacher: "myattacher",
|
||||
Source: storage.VolumeAttachmentSource{
|
||||
PersistentVolumeName: &volumeName,
|
||||
},
|
||||
NodeName: "anothernode",
|
||||
},
|
||||
},
|
||||
{
|
||||
// add invalid status
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "foo"},
|
||||
Spec: storage.VolumeAttachmentSpec{
|
||||
Attacher: "myattacher",
|
||||
Source: storage.VolumeAttachmentSource{
|
||||
PersistentVolumeName: &volumeName,
|
||||
},
|
||||
NodeName: "mynode",
|
||||
},
|
||||
Status: storage.VolumeAttachmentStatus{
|
||||
Attached: true,
|
||||
AttachmentMetadata: map[string]string{
|
||||
"foo": "bar",
|
||||
},
|
||||
AttachError: &storage.VolumeError{
|
||||
Time: metav1.Time{},
|
||||
Message: strings.Repeat("a", maxAttachedVolumeMetadataSize),
|
||||
},
|
||||
DetachError: &storage.VolumeError{
|
||||
Time: metav1.Time{},
|
||||
Message: "hello world",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, volumeAttachment := range errorCases {
|
||||
if errs := ValidateVolumeAttachmentUpdate(&volumeAttachment, &old); len(errs) == 0 {
|
||||
t.Errorf("Expected failure for test: %v", volumeAttachment)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -82,6 +82,7 @@ filegroup(
|
|||
"//pkg/registry/settings/rest:all-srcs",
|
||||
"//pkg/registry/storage/rest:all-srcs",
|
||||
"//pkg/registry/storage/storageclass:all-srcs",
|
||||
"//pkg/registry/storage/volumeattachment:all-srcs",
|
||||
],
|
||||
tags = ["automanaged"],
|
||||
)
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ go_library(
|
|||
"//pkg/api/legacyscheme:go_default_library",
|
||||
"//pkg/apis/storage:go_default_library",
|
||||
"//pkg/registry/storage/storageclass/storage:go_default_library",
|
||||
"//pkg/registry/storage/volumeattachment/storage:go_default_library",
|
||||
"//vendor/k8s.io/api/storage/v1:go_default_library",
|
||||
"//vendor/k8s.io/api/storage/v1beta1:go_default_library",
|
||||
"//vendor/k8s.io/apiserver/pkg/registry/generic:go_default_library",
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ package rest
|
|||
|
||||
import (
|
||||
storageapiv1 "k8s.io/api/storage/v1"
|
||||
storageapiv1alpha1 "k8s.io/api/storage/v1alpha1"
|
||||
storageapiv1beta1 "k8s.io/api/storage/v1beta1"
|
||||
"k8s.io/apiserver/pkg/registry/generic"
|
||||
"k8s.io/apiserver/pkg/registry/rest"
|
||||
|
|
@ -26,6 +27,7 @@ import (
|
|||
"k8s.io/kubernetes/pkg/api/legacyscheme"
|
||||
storageapi "k8s.io/kubernetes/pkg/apis/storage"
|
||||
storageclassstore "k8s.io/kubernetes/pkg/registry/storage/storageclass/storage"
|
||||
volumeattachmentstore "k8s.io/kubernetes/pkg/registry/storage/volumeattachment/storage"
|
||||
)
|
||||
|
||||
type RESTStorageProvider struct {
|
||||
|
|
@ -36,6 +38,10 @@ func (p RESTStorageProvider) NewRESTStorage(apiResourceConfigSource serverstorag
|
|||
// If you add a version here, be sure to add an entry in `k8s.io/kubernetes/cmd/kube-apiserver/app/aggregator.go with specific priorities.
|
||||
// TODO refactor the plumbing to provide the information in the APIGroupInfo
|
||||
|
||||
if apiResourceConfigSource.AnyResourcesForVersionEnabled(storageapiv1alpha1.SchemeGroupVersion) {
|
||||
apiGroupInfo.VersionedResourcesStorageMap[storageapiv1alpha1.SchemeGroupVersion.Version] = p.v1alpha1Storage(apiResourceConfigSource, restOptionsGetter)
|
||||
apiGroupInfo.GroupMeta.GroupVersion = storageapiv1alpha1.SchemeGroupVersion
|
||||
}
|
||||
if apiResourceConfigSource.AnyResourcesForVersionEnabled(storageapiv1beta1.SchemeGroupVersion) {
|
||||
apiGroupInfo.VersionedResourcesStorageMap[storageapiv1beta1.SchemeGroupVersion.Version] = p.v1beta1Storage(apiResourceConfigSource, restOptionsGetter)
|
||||
apiGroupInfo.GroupMeta.GroupVersion = storageapiv1beta1.SchemeGroupVersion
|
||||
|
|
@ -48,6 +54,19 @@ func (p RESTStorageProvider) NewRESTStorage(apiResourceConfigSource serverstorag
|
|||
return apiGroupInfo, true
|
||||
}
|
||||
|
||||
func (p RESTStorageProvider) v1alpha1Storage(apiResourceConfigSource serverstorage.APIResourceConfigSource, restOptionsGetter generic.RESTOptionsGetter) map[string]rest.Storage {
|
||||
version := storageapiv1alpha1.SchemeGroupVersion
|
||||
|
||||
storage := map[string]rest.Storage{}
|
||||
|
||||
if apiResourceConfigSource.ResourceEnabled(version.WithResource("volumeattachments")) {
|
||||
volumeAttachmentStorage := volumeattachmentstore.NewREST(restOptionsGetter)
|
||||
storage["volumeattachments"] = volumeAttachmentStorage
|
||||
}
|
||||
|
||||
return storage
|
||||
}
|
||||
|
||||
func (p RESTStorageProvider) v1beta1Storage(apiResourceConfigSource serverstorage.APIResourceConfigSource, restOptionsGetter generic.RESTOptionsGetter) map[string]rest.Storage {
|
||||
version := storageapiv1beta1.SchemeGroupVersion
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,49 @@
|
|||
load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
|
||||
|
||||
go_library(
|
||||
name = "go_default_library",
|
||||
srcs = [
|
||||
"doc.go",
|
||||
"strategy.go",
|
||||
],
|
||||
importpath = "k8s.io/kubernetes/pkg/registry/storage/volumeattachment",
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//pkg/api/legacyscheme:go_default_library",
|
||||
"//pkg/apis/storage:go_default_library",
|
||||
"//pkg/apis/storage/validation:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/util/validation/field:go_default_library",
|
||||
"//vendor/k8s.io/apiserver/pkg/endpoints/request:go_default_library",
|
||||
"//vendor/k8s.io/apiserver/pkg/storage/names:go_default_library",
|
||||
],
|
||||
)
|
||||
|
||||
go_test(
|
||||
name = "go_default_test",
|
||||
srcs = ["strategy_test.go"],
|
||||
importpath = "k8s.io/kubernetes/pkg/registry/storage/volumeattachment",
|
||||
library = ":go_default_library",
|
||||
deps = [
|
||||
"//pkg/apis/storage:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
|
||||
"//vendor/k8s.io/apiserver/pkg/endpoints/request:go_default_library",
|
||||
],
|
||||
)
|
||||
|
||||
filegroup(
|
||||
name = "package-srcs",
|
||||
srcs = glob(["**"]),
|
||||
tags = ["automanaged"],
|
||||
visibility = ["//visibility:private"],
|
||||
)
|
||||
|
||||
filegroup(
|
||||
name = "all-srcs",
|
||||
srcs = [
|
||||
":package-srcs",
|
||||
"//pkg/registry/storage/volumeattachment/storage:all-srcs",
|
||||
],
|
||||
tags = ["automanaged"],
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Package volumeattachment provides Registry interface and its REST
|
||||
// implementation for storing volumeattachment api objects.
|
||||
package volumeattachment
|
||||
|
|
@ -0,0 +1,48 @@
|
|||
load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
|
||||
|
||||
go_library(
|
||||
name = "go_default_library",
|
||||
srcs = ["storage.go"],
|
||||
importpath = "k8s.io/kubernetes/pkg/registry/storage/volumeattachment/storage",
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//pkg/apis/storage:go_default_library",
|
||||
"//pkg/registry/storage/volumeattachment:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
|
||||
"//vendor/k8s.io/apiserver/pkg/registry/generic:go_default_library",
|
||||
"//vendor/k8s.io/apiserver/pkg/registry/generic/registry:go_default_library",
|
||||
],
|
||||
)
|
||||
|
||||
go_test(
|
||||
name = "go_default_test",
|
||||
srcs = ["storage_test.go"],
|
||||
importpath = "k8s.io/kubernetes/pkg/registry/storage/volumeattachment/storage",
|
||||
library = ":go_default_library",
|
||||
deps = [
|
||||
"//pkg/api/testapi:go_default_library",
|
||||
"//pkg/apis/storage:go_default_library",
|
||||
"//pkg/registry/registrytest:go_default_library",
|
||||
"//vendor/k8s.io/api/storage/v1alpha1:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/fields:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
|
||||
"//vendor/k8s.io/apiserver/pkg/registry/generic:go_default_library",
|
||||
"//vendor/k8s.io/apiserver/pkg/storage/etcd/testing:go_default_library",
|
||||
],
|
||||
)
|
||||
|
||||
filegroup(
|
||||
name = "package-srcs",
|
||||
srcs = glob(["**"]),
|
||||
tags = ["automanaged"],
|
||||
visibility = ["//visibility:private"],
|
||||
)
|
||||
|
||||
filegroup(
|
||||
name = "all-srcs",
|
||||
srcs = [":package-srcs"],
|
||||
tags = ["automanaged"],
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apiserver/pkg/registry/generic"
|
||||
genericregistry "k8s.io/apiserver/pkg/registry/generic/registry"
|
||||
storageapi "k8s.io/kubernetes/pkg/apis/storage"
|
||||
"k8s.io/kubernetes/pkg/registry/storage/volumeattachment"
|
||||
)
|
||||
|
||||
// REST object that will work against persistent volumes.
|
||||
type REST struct {
|
||||
*genericregistry.Store
|
||||
}
|
||||
|
||||
// NewREST returns a RESTStorage object that will work against persistent volumes.
|
||||
func NewREST(optsGetter generic.RESTOptionsGetter) *REST {
|
||||
store := &genericregistry.Store{
|
||||
NewFunc: func() runtime.Object { return &storageapi.VolumeAttachment{} },
|
||||
NewListFunc: func() runtime.Object { return &storageapi.VolumeAttachmentList{} },
|
||||
DefaultQualifiedResource: storageapi.Resource("volumeattachments"),
|
||||
|
||||
CreateStrategy: volumeattachment.Strategy,
|
||||
UpdateStrategy: volumeattachment.Strategy,
|
||||
DeleteStrategy: volumeattachment.Strategy,
|
||||
ReturnDeletedObject: true,
|
||||
}
|
||||
options := &generic.StoreOptions{RESTOptions: optsGetter}
|
||||
if err := store.CompleteWithOptions(options); err != nil {
|
||||
panic(err) // TODO: Propagate error up
|
||||
}
|
||||
|
||||
return &REST{store}
|
||||
}
|
||||
|
|
@ -0,0 +1,190 @@
|
|||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
storageapiv1alpha1 "k8s.io/api/storage/v1alpha1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/fields"
|
||||
"k8s.io/apimachinery/pkg/labels"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apiserver/pkg/registry/generic"
|
||||
etcdtesting "k8s.io/apiserver/pkg/storage/etcd/testing"
|
||||
"k8s.io/kubernetes/pkg/api/testapi"
|
||||
storageapi "k8s.io/kubernetes/pkg/apis/storage"
|
||||
"k8s.io/kubernetes/pkg/registry/registrytest"
|
||||
)
|
||||
|
||||
func newStorage(t *testing.T) (*REST, *etcdtesting.EtcdTestServer) {
|
||||
etcdStorage, server := registrytest.NewEtcdStorage(t, storageapi.GroupName)
|
||||
restOptions := generic.RESTOptions{
|
||||
StorageConfig: etcdStorage,
|
||||
Decorator: generic.UndecoratedStorage,
|
||||
DeleteCollectionWorkers: 1,
|
||||
ResourcePrefix: "volumeattachments",
|
||||
}
|
||||
volumeAttachmentStorage := NewREST(restOptions)
|
||||
return volumeAttachmentStorage, server
|
||||
}
|
||||
|
||||
func validNewVolumeAttachment(name string) *storageapi.VolumeAttachment {
|
||||
pvName := "foo"
|
||||
return &storageapi.VolumeAttachment{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: name,
|
||||
},
|
||||
Spec: storageapi.VolumeAttachmentSpec{
|
||||
Attacher: "valid-attacher",
|
||||
Source: storageapi.VolumeAttachmentSource{
|
||||
PersistentVolumeName: &pvName,
|
||||
},
|
||||
NodeName: "valid-node",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func validChangedVolumeAttachment() *storageapi.VolumeAttachment {
|
||||
return validNewVolumeAttachment("foo")
|
||||
}
|
||||
|
||||
func TestCreate(t *testing.T) {
|
||||
if *testapi.Storage.GroupVersion() != storageapiv1alpha1.SchemeGroupVersion {
|
||||
// skip the test for all versions exception v1alpha1
|
||||
return
|
||||
}
|
||||
|
||||
storage, server := newStorage(t)
|
||||
defer server.Terminate(t)
|
||||
defer storage.Store.DestroyFunc()
|
||||
test := registrytest.New(t, storage.Store).ClusterScope()
|
||||
volumeAttachment := validNewVolumeAttachment("foo")
|
||||
volumeAttachment.ObjectMeta = metav1.ObjectMeta{GenerateName: "foo"}
|
||||
pvName := "foo"
|
||||
test.TestCreate(
|
||||
// valid
|
||||
volumeAttachment,
|
||||
// invalid
|
||||
&storageapi.VolumeAttachment{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "*BadName!"},
|
||||
Spec: storageapi.VolumeAttachmentSpec{
|
||||
Attacher: "invalid-attacher-!@#$%^&*()",
|
||||
Source: storageapi.VolumeAttachmentSource{
|
||||
PersistentVolumeName: &pvName,
|
||||
},
|
||||
NodeName: "invalid-node-!@#$%^&*()",
|
||||
},
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
func TestUpdate(t *testing.T) {
|
||||
if *testapi.Storage.GroupVersion() != storageapiv1alpha1.SchemeGroupVersion {
|
||||
// skip the test for all versions except v1alpha1
|
||||
return
|
||||
}
|
||||
|
||||
storage, server := newStorage(t)
|
||||
defer server.Terminate(t)
|
||||
defer storage.Store.DestroyFunc()
|
||||
test := registrytest.New(t, storage.Store).ClusterScope()
|
||||
test.TestUpdate(
|
||||
// valid
|
||||
validNewVolumeAttachment("foo"),
|
||||
// updateFunc
|
||||
func(obj runtime.Object) runtime.Object {
|
||||
object := obj.(*storageapi.VolumeAttachment)
|
||||
object.Status.Attached = true
|
||||
return object
|
||||
},
|
||||
//invalid update
|
||||
func(obj runtime.Object) runtime.Object {
|
||||
object := obj.(*storageapi.VolumeAttachment)
|
||||
object.Spec.Attacher = "invalid-attacher-!@#$%^&*()"
|
||||
return object
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
func TestDelete(t *testing.T) {
|
||||
if *testapi.Storage.GroupVersion() != storageapiv1alpha1.SchemeGroupVersion {
|
||||
// skip the test for all versions except v1alpha1
|
||||
return
|
||||
}
|
||||
|
||||
storage, server := newStorage(t)
|
||||
defer server.Terminate(t)
|
||||
defer storage.Store.DestroyFunc()
|
||||
test := registrytest.New(t, storage.Store).ClusterScope().ReturnDeletedObject()
|
||||
test.TestDelete(validNewVolumeAttachment("foo"))
|
||||
}
|
||||
|
||||
func TestGet(t *testing.T) {
|
||||
if *testapi.Storage.GroupVersion() != storageapiv1alpha1.SchemeGroupVersion {
|
||||
// skip the test for all versions except v1alpha1
|
||||
return
|
||||
}
|
||||
|
||||
storage, server := newStorage(t)
|
||||
defer server.Terminate(t)
|
||||
defer storage.Store.DestroyFunc()
|
||||
test := registrytest.New(t, storage.Store).ClusterScope()
|
||||
test.TestGet(validNewVolumeAttachment("foo"))
|
||||
}
|
||||
|
||||
func TestList(t *testing.T) {
|
||||
if *testapi.Storage.GroupVersion() != storageapiv1alpha1.SchemeGroupVersion {
|
||||
// skip the test for all versions except v1alpha1
|
||||
return
|
||||
}
|
||||
|
||||
storage, server := newStorage(t)
|
||||
defer server.Terminate(t)
|
||||
defer storage.Store.DestroyFunc()
|
||||
test := registrytest.New(t, storage.Store).ClusterScope()
|
||||
test.TestList(validNewVolumeAttachment("foo"))
|
||||
}
|
||||
|
||||
func TestWatch(t *testing.T) {
|
||||
if *testapi.Storage.GroupVersion() != storageapiv1alpha1.SchemeGroupVersion {
|
||||
// skip the test for all versions except v1alpha1
|
||||
return
|
||||
}
|
||||
|
||||
storage, server := newStorage(t)
|
||||
defer server.Terminate(t)
|
||||
defer storage.Store.DestroyFunc()
|
||||
test := registrytest.New(t, storage.Store).ClusterScope()
|
||||
test.TestWatch(
|
||||
validNewVolumeAttachment("foo"),
|
||||
// matching labels
|
||||
[]labels.Set{},
|
||||
// not matching labels
|
||||
[]labels.Set{
|
||||
{"foo": "bar"},
|
||||
},
|
||||
// matching fields
|
||||
[]fields.Set{
|
||||
{"metadata.name": "foo"},
|
||||
},
|
||||
// not matching fields
|
||||
[]fields.Set{
|
||||
{"metadata.name": "bar"},
|
||||
},
|
||||
)
|
||||
}
|
||||
|
|
@ -0,0 +1,73 @@
|
|||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package volumeattachment
|
||||
|
||||
import (
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/util/validation/field"
|
||||
genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
|
||||
"k8s.io/apiserver/pkg/storage/names"
|
||||
"k8s.io/kubernetes/pkg/api/legacyscheme"
|
||||
"k8s.io/kubernetes/pkg/apis/storage"
|
||||
"k8s.io/kubernetes/pkg/apis/storage/validation"
|
||||
)
|
||||
|
||||
// volumeAttachmentStrategy implements behavior for VolumeAttachment objects
|
||||
type volumeAttachmentStrategy struct {
|
||||
runtime.ObjectTyper
|
||||
names.NameGenerator
|
||||
}
|
||||
|
||||
// Strategy is the default logic that applies when creating and updating
|
||||
// VolumeAttachment objects via the REST API.
|
||||
var Strategy = volumeAttachmentStrategy{legacyscheme.Scheme, names.SimpleNameGenerator}
|
||||
|
||||
func (volumeAttachmentStrategy) NamespaceScoped() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// ResetBeforeCreate clears the Status field which is not allowed to be set by end users on creation.
|
||||
func (volumeAttachmentStrategy) PrepareForCreate(ctx genericapirequest.Context, obj runtime.Object) {
|
||||
}
|
||||
|
||||
func (volumeAttachmentStrategy) Validate(ctx genericapirequest.Context, obj runtime.Object) field.ErrorList {
|
||||
volumeAttachment := obj.(*storage.VolumeAttachment)
|
||||
return validation.ValidateVolumeAttachment(volumeAttachment)
|
||||
}
|
||||
|
||||
// Canonicalize normalizes the object after validation.
|
||||
func (volumeAttachmentStrategy) Canonicalize(obj runtime.Object) {
|
||||
}
|
||||
|
||||
func (volumeAttachmentStrategy) AllowCreateOnUpdate() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// PrepareForUpdate sets the Status fields which is not allowed to be set by an end user updating a PV
|
||||
func (volumeAttachmentStrategy) PrepareForUpdate(ctx genericapirequest.Context, obj, old runtime.Object) {
|
||||
}
|
||||
|
||||
func (volumeAttachmentStrategy) ValidateUpdate(ctx genericapirequest.Context, obj, old runtime.Object) field.ErrorList {
|
||||
newVolumeAttachmentObj := obj.(*storage.VolumeAttachment)
|
||||
oldVolumeAttachmentObj := old.(*storage.VolumeAttachment)
|
||||
errorList := validation.ValidateVolumeAttachment(newVolumeAttachmentObj)
|
||||
return append(errorList, validation.ValidateVolumeAttachmentUpdate(newVolumeAttachmentObj, oldVolumeAttachmentObj)...)
|
||||
}
|
||||
|
||||
func (volumeAttachmentStrategy) AllowUnconditionalUpdate() bool {
|
||||
return false
|
||||
}
|
||||
|
|
@ -0,0 +1,77 @@
|
|||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package volumeattachment
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
|
||||
"k8s.io/kubernetes/pkg/apis/storage"
|
||||
)
|
||||
|
||||
func TestVolumeAttachmentStrategy(t *testing.T) {
|
||||
ctx := genericapirequest.NewDefaultContext()
|
||||
if Strategy.NamespaceScoped() {
|
||||
t.Errorf("VolumeAttachment must not be namespace scoped")
|
||||
}
|
||||
if Strategy.AllowCreateOnUpdate() {
|
||||
t.Errorf("VolumeAttachment should not allow create on update")
|
||||
}
|
||||
|
||||
pvName := "name"
|
||||
volumeAttachment := &storage.VolumeAttachment{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "valid-attachment",
|
||||
},
|
||||
Spec: storage.VolumeAttachmentSpec{
|
||||
Attacher: "valid-attacher",
|
||||
Source: storage.VolumeAttachmentSource{
|
||||
PersistentVolumeName: &pvName,
|
||||
},
|
||||
NodeName: "valid-node",
|
||||
},
|
||||
}
|
||||
|
||||
Strategy.PrepareForCreate(ctx, volumeAttachment)
|
||||
|
||||
errs := Strategy.Validate(ctx, volumeAttachment)
|
||||
if len(errs) != 0 {
|
||||
t.Errorf("unexpected error validating %v", errs)
|
||||
}
|
||||
|
||||
newVolumeAttachment := &storage.VolumeAttachment{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "valid-attachment-2",
|
||||
},
|
||||
Spec: storage.VolumeAttachmentSpec{
|
||||
Attacher: "valid-attacher-2",
|
||||
Source: storage.VolumeAttachmentSource{
|
||||
PersistentVolumeName: &pvName,
|
||||
},
|
||||
NodeName: "valid-node-2",
|
||||
},
|
||||
}
|
||||
|
||||
Strategy.PrepareForUpdate(ctx, newVolumeAttachment, volumeAttachment)
|
||||
|
||||
errs = Strategy.ValidateUpdate(ctx, newVolumeAttachment, volumeAttachment)
|
||||
if len(errs) == 0 {
|
||||
t.Errorf("Expected a validation error")
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// +k8s:deepcopy-gen=package,register
|
||||
// +groupName=storage.k8s.io
|
||||
// +k8s:openapi-gen=true
|
||||
package v1alpha1 // import "k8s.io/api/storage/v1alpha1"
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package v1alpha1
|
||||
|
||||
import (
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
)
|
||||
|
||||
// GroupName is the group name use in this package
|
||||
const GroupName = "storage.k8s.io"
|
||||
|
||||
// SchemeGroupVersion is group version used to register these objects
|
||||
var SchemeGroupVersion = schema.GroupVersion{Group: GroupName, Version: "v1alpha1"}
|
||||
|
||||
// Resource takes an unqualified resource and returns a Group qualified GroupResource
|
||||
func Resource(resource string) schema.GroupResource {
|
||||
return SchemeGroupVersion.WithResource(resource).GroupResource()
|
||||
}
|
||||
|
||||
var (
|
||||
SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes)
|
||||
AddToScheme = SchemeBuilder.AddToScheme
|
||||
)
|
||||
|
||||
// Adds the list of known types to the given scheme.
|
||||
func addKnownTypes(scheme *runtime.Scheme) error {
|
||||
scheme.AddKnownTypes(SchemeGroupVersion,
|
||||
&VolumeAttachment{},
|
||||
&VolumeAttachmentList{},
|
||||
)
|
||||
|
||||
metav1.AddToGroupVersion(scheme, SchemeGroupVersion)
|
||||
return nil
|
||||
}
|
||||
|
|
@ -0,0 +1,126 @@
|
|||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package v1alpha1
|
||||
|
||||
import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
|
||||
// +genclient
|
||||
// +genclient:nonNamespaced
|
||||
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
|
||||
|
||||
// VolumeAttachment captures the intent to attach or detach the specified volume
|
||||
// to/from the specified node.
|
||||
//
|
||||
// VolumeAttachment objects are non-namespaced.
|
||||
type VolumeAttachment struct {
|
||||
metav1.TypeMeta `json:",inline"`
|
||||
|
||||
// Standard object metadata.
|
||||
// More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata
|
||||
// +optional
|
||||
metav1.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`
|
||||
|
||||
// Specification of the desired attach/detach volume behavior.
|
||||
// Populated by the Kubernetes system.
|
||||
Spec VolumeAttachmentSpec `json:"spec" protobuf:"bytes,2,opt,name=spec"`
|
||||
|
||||
// Status of the VolumeAttachment request.
|
||||
// Populated by the entity completing the attach or detach
|
||||
// operation, i.e. the external-attacher.
|
||||
// +optional
|
||||
Status VolumeAttachmentStatus `json:"status,omitempty" protobuf:"bytes,3,opt,name=status"`
|
||||
}
|
||||
|
||||
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
|
||||
|
||||
// VolumeAttachmentList is a collection of VolumeAttachment objects.
|
||||
type VolumeAttachmentList struct {
|
||||
metav1.TypeMeta `json:",inline"`
|
||||
// Standard list metadata
|
||||
// More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata
|
||||
// +optional
|
||||
metav1.ListMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`
|
||||
|
||||
// Items is the list of VolumeAttachments
|
||||
Items []VolumeAttachment `json:"items" protobuf:"bytes,2,rep,name=items"`
|
||||
}
|
||||
|
||||
// VolumeAttachmentSpec is the specification of a VolumeAttachment request.
|
||||
type VolumeAttachmentSpec struct {
|
||||
// Attacher indicates the name of the volume driver that MUST handle this
|
||||
// request. This is the name returned by GetPluginName().
|
||||
Attacher string `json:"attacher" protobuf:"bytes,1,opt,name=attacher"`
|
||||
|
||||
// Source represents the volume that should be attached.
|
||||
Source VolumeAttachmentSource `json:"source" protobuf:"bytes,2,opt,name=source"`
|
||||
|
||||
// The node that the volume should be attached to.
|
||||
NodeName string `json:"nodeName" protobuf:"bytes,3,opt,name=nodeName"`
|
||||
}
|
||||
|
||||
// VolumeAttachmentSource represents a volume that should be attached.
|
||||
// Right now only PersistenVolumes can be attached via external attacher,
|
||||
// in future we may allow also inline volumes in pods.
|
||||
// Exactly one member can be set.
|
||||
type VolumeAttachmentSource struct {
|
||||
// Name of the persistent volume to attach.
|
||||
// +optional
|
||||
PersistentVolumeName *string `json:"persistentVolumeName,omitempty" protobuf:"bytes,1,opt,name=persistentVolumeName"`
|
||||
|
||||
// Placeholder for *VolumeSource to accommodate inline volumes in pods.
|
||||
}
|
||||
|
||||
// VolumeAttachmentStatus is the status of a VolumeAttachment request.
|
||||
type VolumeAttachmentStatus struct {
|
||||
// Indicates the volume is successfully attached.
|
||||
// This field must only be set by the entity completing the attach
|
||||
// operation, i.e. the external-attacher.
|
||||
Attached bool `json:"attached" protobuf:"varint,1,opt,name=attached"`
|
||||
|
||||
// Upon successful attach, this field is populated with any
|
||||
// information returned by the attach operation that must be passed
|
||||
// into subsequent WaitForAttach or Mount calls.
|
||||
// This field must only be set by the entity completing the attach
|
||||
// operation, i.e. the external-attacher.
|
||||
// +optional
|
||||
AttachmentMetadata map[string]string `json:"attachmentMetadata,omitempty" protobuf:"bytes,2,rep,name=attachmentMetadata"`
|
||||
|
||||
// The last error encountered during attach operation, if any.
|
||||
// This field must only be set by the entity completing the attach
|
||||
// operation, i.e. the external-attacher.
|
||||
// +optional
|
||||
AttachError *VolumeError `json:"attachError,omitempty" protobuf:"bytes,3,opt,name=attachError,casttype=VolumeError"`
|
||||
|
||||
// The last error encountered during detach operation, if any.
|
||||
// This field must only be set by the entity completing the detach
|
||||
// operation, i.e. the external-attacher.
|
||||
// +optional
|
||||
DetachError *VolumeError `json:"detachError,omitempty" protobuf:"bytes,4,opt,name=detachError,casttype=VolumeError"`
|
||||
}
|
||||
|
||||
// VolumeError captures an error encountered during a volume operation.
|
||||
type VolumeError struct {
|
||||
// Time the error was encountered.
|
||||
// +optional
|
||||
Time metav1.Time `json:"time,omitempty" protobuf:"bytes,1,opt,name=time"`
|
||||
|
||||
// String detailing the error encountered during Attach or Detach operation.
|
||||
// This string maybe logged, so it should not contain sensitive
|
||||
// information.
|
||||
// +optional
|
||||
Message string `json:"message,omitempty" protobuf:"bytes,2,opt,name=message"`
|
||||
}
|
||||
|
|
@ -289,6 +289,13 @@ var etcdStorageData = map[schema.GroupVersionResource]struct {
|
|||
},
|
||||
// --
|
||||
|
||||
// k8s.io/kubernetes/pkg/apis/storage/v1alpha1
|
||||
gvr("storage.k8s.io", "v1alpha1", "volumeattachments"): {
|
||||
stub: `{"metadata": {"name": "va1"}, "spec": {"attacher": "gce", "nodeName": "localhost", "source": {"persistentVolumeName": "pv1"}}}`,
|
||||
expectedEtcdPath: "/registry/volumeattachments/va1",
|
||||
},
|
||||
// --
|
||||
|
||||
// k8s.io/kubernetes/pkg/apis/storage/v1beta1
|
||||
gvr("storage.k8s.io", "v1beta1", "storageclasses"): {
|
||||
stub: `{"metadata": {"name": "sc1"}, "provisioner": "aws"}`,
|
||||
|
|
|
|||
Loading…
Reference in New Issue