diff --git a/pkg/util/resizefs/BUILD b/pkg/util/resizefs/BUILD index 6301c603a9..00fedd0c1d 100644 --- a/pkg/util/resizefs/BUILD +++ b/pkg/util/resizefs/BUILD @@ -56,7 +56,6 @@ go_library( "@io_bazel_rules_go//go/platform:linux": [ "//pkg/util/mount:go_default_library", "//vendor/github.com/golang/glog:go_default_library", - "//vendor/k8s.io/utils/exec:go_default_library", ], "@io_bazel_rules_go//go/platform:nacl": [ "//pkg/util/mount:go_default_library", diff --git a/pkg/util/resizefs/resizefs_linux.go b/pkg/util/resizefs/resizefs_linux.go index 6a4d82d6c0..518eba2bb6 100644 --- a/pkg/util/resizefs/resizefs_linux.go +++ b/pkg/util/resizefs/resizefs_linux.go @@ -23,14 +23,6 @@ import ( "github.com/golang/glog" "k8s.io/kubernetes/pkg/util/mount" - utilexec "k8s.io/utils/exec" -) - -const ( - // 'fsck' found errors and corrected them - fsckErrorsCorrected = 1 - // 'fsck' found errors but exited without correcting them - fsckErrorsUncorrected = 4 ) // ResizeFs Provides support for resizing file systems @@ -44,11 +36,11 @@ func NewResizeFs(mounter *mount.SafeFormatAndMount) *ResizeFs { } // Resize perform resize of file system -func (resizefs *ResizeFs) Resize(devicePath string) (bool, error) { +func (resizefs *ResizeFs) Resize(devicePath string, deviceMountPath string) (bool, error) { format, err := resizefs.mounter.GetDiskFormat(devicePath) if err != nil { - formatErr := fmt.Errorf("error checking format for device %s: %v", devicePath, err) + formatErr := fmt.Errorf("ResizeFS.Resize - error checking format for device %s: %v", devicePath, err) return false, formatErr } @@ -58,63 +50,14 @@ func (resizefs *ResizeFs) Resize(devicePath string) (bool, error) { return false, nil } - deviceOpened, err := resizefs.mounter.DeviceOpened(devicePath) - - if err != nil { - deviceOpenErr := fmt.Errorf("error verifying if device %s is open: %v", devicePath, err) - return false, deviceOpenErr - } - - if deviceOpened { - deviceAlreadyOpenErr := fmt.Errorf("the device %s is already in use", devicePath) - return false, deviceAlreadyOpenErr - } - + glog.V(3).Infof("ResizeFS.Resize - Expanding mounted volume %s", devicePath) switch format { case "ext3", "ext4": - fsckErr := resizefs.extFsck(devicePath, format) - if fsckErr != nil { - return false, fsckErr - } return resizefs.extResize(devicePath) case "xfs": - fsckErr := resizefs.fsckDevice(devicePath) - if fsckErr != nil { - return false, fsckErr - } - return resizefs.xfsResize(devicePath) + return resizefs.xfsResize(deviceMountPath) } - return false, fmt.Errorf("resize of format %s is not supported for device %s", format, devicePath) -} - -func (resizefs *ResizeFs) fsckDevice(devicePath string) error { - glog.V(4).Infof("Checking for issues with fsck on device: %s", devicePath) - args := []string{"-a", devicePath} - out, err := resizefs.mounter.Exec.Run("fsck", args...) - if err != nil { - ee, isExitError := err.(utilexec.ExitError) - switch { - case err == utilexec.ErrExecutableNotFound: - glog.Warningf("'fsck' not found on system; continuing resizing without running 'fsck'.") - case isExitError && ee.ExitStatus() == fsckErrorsCorrected: - glog.V(2).Infof("Device %s has errors which were corrected by fsck: %s", devicePath, string(out)) - case isExitError && ee.ExitStatus() == fsckErrorsUncorrected: - return fmt.Errorf("'fsck' found errors on device %s but could not correct them: %s", devicePath, string(out)) - case isExitError && ee.ExitStatus() > fsckErrorsUncorrected: - glog.Infof("`fsck` error %s", string(out)) - } - } - return nil -} - -func (resizefs *ResizeFs) extFsck(devicePath string, fsType string) error { - glog.V(4).Infof("Checking for issues with fsck.%s on device: %s", fsType, devicePath) - args := []string{"-f", "-y", devicePath} - out, err := resizefs.mounter.Run("fsck."+fsType, args...) - if err != nil { - return fmt.Errorf("running fsck.%s failed on %s with error: %v\n Output: %s", fsType, devicePath, err, string(out)) - } - return nil + return false, fmt.Errorf("ResizeFS.Resize - resize of format %s is not supported for device %s mounted at %s", format, devicePath, deviceMountPath) } func (resizefs *ResizeFs) extResize(devicePath string) (bool, error) { @@ -129,15 +72,15 @@ func (resizefs *ResizeFs) extResize(devicePath string) (bool, error) { } -func (resizefs *ResizeFs) xfsResize(devicePath string) (bool, error) { - args := []string{"-d", devicePath} +func (resizefs *ResizeFs) xfsResize(deviceMountPath string) (bool, error) { + args := []string{"-d", deviceMountPath} output, err := resizefs.mounter.Exec.Run("xfs_growfs", args...) if err == nil { - glog.V(2).Infof("Device %s resized successfully", devicePath) + glog.V(2).Infof("Device %s resized successfully", deviceMountPath) return true, nil } - resizeError := fmt.Errorf("resize of device %s failed: %v. xfs_growfs output: %s", devicePath, err, string(output)) + resizeError := fmt.Errorf("resize of device %s failed: %v. xfs_growfs output: %s", deviceMountPath, err, string(output)) return false, resizeError } diff --git a/pkg/util/resizefs/resizefs_unsupported.go b/pkg/util/resizefs/resizefs_unsupported.go index 9241d7d5b2..dd4dd017e8 100644 --- a/pkg/util/resizefs/resizefs_unsupported.go +++ b/pkg/util/resizefs/resizefs_unsupported.go @@ -35,6 +35,6 @@ func NewResizeFs(mounter *mount.SafeFormatAndMount) *ResizeFs { } // Resize perform resize of file system -func (resizefs *ResizeFs) Resize(devicePath string) (bool, error) { +func (resizefs *ResizeFs) Resize(devicePath string, deviceMountPath string) (bool, error) { return false, fmt.Errorf("Resize is not supported for this build") } diff --git a/pkg/volume/util/operationexecutor/operation_generator.go b/pkg/volume/util/operationexecutor/operation_generator.go index 357f1a387e..ad175a5868 100644 --- a/pkg/volume/util/operationexecutor/operation_generator.go +++ b/pkg/volume/util/operationexecutor/operation_generator.go @@ -495,14 +495,6 @@ func (og *operationGenerator) GenerateMountVolumeFunc( glog.Infof(volumeToMount.GenerateMsgDetailed("MountVolume.WaitForAttach succeeded", fmt.Sprintf("DevicePath %q", devicePath))) - // resizeFileSystem will resize the file system if user has requested a resize of - // underlying persistent volume and is allowed to do so. - resizeSimpleError, resizeDetailedError := og.resizeFileSystem(volumeToMount, devicePath, volumePlugin.GetPluginName()) - - if resizeSimpleError != nil || resizeDetailedError != nil { - return resizeSimpleError, resizeDetailedError - } - deviceMountPath, err := volumeAttacher.GetDeviceMountPath(volumeToMount.VolumeSpec) if err != nil { @@ -529,6 +521,15 @@ func (og *operationGenerator) GenerateMountVolumeFunc( // On failure, return error. Caller will log and retry. return volumeToMount.GenerateError("MountVolume.MarkDeviceAsMounted failed", markDeviceMountedErr) } + + // resizeFileSystem will resize the file system if user has requested a resize of + // underlying persistent volume and is allowed to do so. + resizeSimpleError, resizeDetailedError := og.resizeFileSystem(volumeToMount, devicePath, deviceMountPath, volumePlugin.GetPluginName()) + + if resizeSimpleError != nil || resizeDetailedError != nil { + return resizeSimpleError, resizeDetailedError + } + } if og.checkNodeCapabilitiesBeforeMount { @@ -586,7 +587,7 @@ func (og *operationGenerator) GenerateMountVolumeFunc( }, nil } -func (og *operationGenerator) resizeFileSystem(volumeToMount VolumeToMount, devicePath string, pluginName string) (simpleErr, detailedErr error) { +func (og *operationGenerator) resizeFileSystem(volumeToMount VolumeToMount, devicePath, deviceMountPath, pluginName string) (simpleErr, detailedErr error) { if !utilfeature.DefaultFeatureGate.Enabled(features.ExpandPersistentVolumes) { glog.V(6).Infof("Resizing is not enabled for this volume %s", volumeToMount.VolumeName) return nil, nil @@ -626,7 +627,7 @@ func (og *operationGenerator) resizeFileSystem(volumeToMount VolumeToMount, devi } resizer := resizefs.NewResizeFs(diskFormatter) - resizeStatus, resizeErr := resizer.Resize(devicePath) + resizeStatus, resizeErr := resizer.Resize(devicePath, deviceMountPath) if resizeErr != nil { return volumeToMount.GenerateError("MountVolume.resizeFileSystem failed", resizeErr) diff --git a/test/e2e/framework/deployment_util.go b/test/e2e/framework/deployment_util.go index d5544e1998..b7b52bf3bf 100644 --- a/test/e2e/framework/deployment_util.go +++ b/test/e2e/framework/deployment_util.go @@ -216,8 +216,8 @@ func CheckDeploymentRevisionAndImage(c clientset.Interface, ns, deploymentName, return testutils.CheckDeploymentRevisionAndImage(c, ns, deploymentName, revision, image) } -func CreateDeployment(client clientset.Interface, replicas int32, podLabels map[string]string, namespace string, pvclaims []*v1.PersistentVolumeClaim, command string) (*extensions.Deployment, error) { - deploymentSpec := MakeDeployment(replicas, podLabels, namespace, pvclaims, false, command) +func CreateDeployment(client clientset.Interface, replicas int32, podLabels map[string]string, nodeSelector map[string]string, namespace string, pvclaims []*v1.PersistentVolumeClaim, command string) (*extensions.Deployment, error) { + deploymentSpec := MakeDeployment(replicas, podLabels, nodeSelector, namespace, pvclaims, false, command) deployment, err := client.ExtensionsV1beta1().Deployments(namespace).Create(deploymentSpec) if err != nil { return nil, fmt.Errorf("deployment %q Create API error: %v", deploymentSpec.Name, err) @@ -232,7 +232,7 @@ func CreateDeployment(client clientset.Interface, replicas int32, podLabels map[ // MakeDeployment creates a deployment definition based on the namespace. The deployment references the PVC's // name. A slice of BASH commands can be supplied as args to be run by the pod -func MakeDeployment(replicas int32, podLabels map[string]string, namespace string, pvclaims []*v1.PersistentVolumeClaim, isPrivileged bool, command string) *extensions.Deployment { +func MakeDeployment(replicas int32, podLabels map[string]string, nodeSelector map[string]string, namespace string, pvclaims []*v1.PersistentVolumeClaim, isPrivileged bool, command string) *extensions.Deployment { if len(command) == 0 { command = "while true; do sleep 1; done" } @@ -276,6 +276,9 @@ func MakeDeployment(replicas int32, podLabels map[string]string, namespace strin } deploymentSpec.Spec.Template.Spec.Containers[0].VolumeMounts = volumeMounts deploymentSpec.Spec.Template.Spec.Volumes = volumes + if nodeSelector != nil { + deploymentSpec.Spec.Template.Spec.NodeSelector = nodeSelector + } return deploymentSpec } diff --git a/test/e2e/storage/BUILD b/test/e2e/storage/BUILD index d31a9b3a86..02c97070e9 100644 --- a/test/e2e/storage/BUILD +++ b/test/e2e/storage/BUILD @@ -12,6 +12,7 @@ go_library( "csi_volumes.go", "empty_dir_wrapper.go", "flexvolume.go", + "mounted_volume_resize.go", "pd.go", "persistent_volumes.go", "persistent_volumes-disruptive.go", @@ -29,6 +30,7 @@ go_library( "//pkg/api/testapi:go_default_library", "//pkg/apis/core/v1/helper:go_default_library", "//pkg/apis/storage/v1/util:go_default_library", + "//pkg/client/conditions:go_default_library", "//pkg/kubelet/apis:go_default_library", "//pkg/kubelet/metrics:go_default_library", "//pkg/util/slice:go_default_library", diff --git a/test/e2e/storage/mounted_volume_resize.go b/test/e2e/storage/mounted_volume_resize.go new file mode 100644 index 0000000000..895e561cfd --- /dev/null +++ b/test/e2e/storage/mounted_volume_resize.go @@ -0,0 +1,172 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package storage + +import ( + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "k8s.io/api/core/v1" + extensions "k8s.io/api/extensions/v1beta1" + storage "k8s.io/api/storage/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + utilerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/apimachinery/pkg/util/wait" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/pkg/client/conditions" + "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/storage/utils" +) + +var _ = utils.SIGDescribe("Mounted volume expand [Feature:ExpandPersistentVolumes] [Slow]", func() { + var ( + c clientset.Interface + ns string + err error + pvc *v1.PersistentVolumeClaim + resizableSc *storage.StorageClass + nodeName string + isNodeLabeled bool + nodeKeyValueLabel map[string]string + nodeLabelValue string + nodeKey string + ) + + f := framework.NewDefaultFramework("mounted-volume-expand") + BeforeEach(func() { + framework.SkipUnlessProviderIs("aws", "gce") + c = f.ClientSet + ns = f.Namespace.Name + framework.ExpectNoError(framework.WaitForAllNodesSchedulable(c, framework.TestContext.NodeSchedulableTimeout)) + + nodeList := framework.GetReadySchedulableNodesOrDie(f.ClientSet) + if len(nodeList.Items) != 0 { + nodeName = nodeList.Items[0].Name + } else { + framework.Failf("Unable to find ready and schedulable Node") + } + + nodeKey = "mounted_volume_expand" + + if !isNodeLabeled { + nodeLabelValue = ns + nodeKeyValueLabel = make(map[string]string) + nodeKeyValueLabel[nodeKey] = nodeLabelValue + framework.AddOrUpdateLabelOnNode(c, nodeName, nodeKey, nodeLabelValue) + isNodeLabeled = true + } + + test := storageClassTest{ + name: "default", + claimSize: "2Gi", + } + resizableSc, err = createResizableStorageClass(test, ns, "resizing", c) + Expect(err).NotTo(HaveOccurred(), "Error creating resizable storage class") + Expect(*resizableSc.AllowVolumeExpansion).To(BeTrue()) + + pvc = newClaim(test, ns, "default") + pvc.Spec.StorageClassName = &resizableSc.Name + pvc, err = c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Create(pvc) + Expect(err).NotTo(HaveOccurred(), "Error creating pvc") + }) + + framework.AddCleanupAction(func() { + if len(nodeLabelValue) > 0 { + framework.RemoveLabelOffNode(c, nodeName, nodeKey) + } + }) + + AfterEach(func() { + framework.Logf("AfterEach: Cleaning up resources for mounted volume resize") + + if c != nil { + if errs := framework.PVPVCCleanup(c, ns, nil, pvc); len(errs) > 0 { + framework.Failf("AfterEach: Failed to delete PVC and/or PV. Errors: %v", utilerrors.NewAggregate(errs)) + } + pvc, nodeName, isNodeLabeled, nodeLabelValue = nil, "", false, "" + nodeKeyValueLabel = make(map[string]string) + } + }) + + It("Should verify mounted devices can be resized", func() { + By("Waiting for PVC to be in bound phase") + pvcClaims := []*v1.PersistentVolumeClaim{pvc} + pvs, err := framework.WaitForPVClaimBoundPhase(c, pvcClaims, framework.ClaimProvisionTimeout) + Expect(err).NotTo(HaveOccurred(), "Failed waiting for PVC to be bound %v", err) + Expect(len(pvs)).To(Equal(1)) + + By("Creating a deployment with the provisioned volume") + deployment, err := framework.CreateDeployment(c, int32(1), map[string]string{"test": "app"}, nodeKeyValueLabel, ns, pvcClaims, "") + defer c.ExtensionsV1beta1().Deployments(ns).Delete(deployment.Name, &metav1.DeleteOptions{}) + + By("Expanding current pvc") + newSize := resource.MustParse("6Gi") + pvc, err = expandPVCSize(pvc, newSize, c) + Expect(err).NotTo(HaveOccurred(), "While updating pvc for more size") + Expect(pvc).NotTo(BeNil()) + + pvcSize := pvc.Spec.Resources.Requests[v1.ResourceStorage] + if pvcSize.Cmp(newSize) != 0 { + framework.Failf("error updating pvc size %q", pvc.Name) + } + + By("Waiting for cloudprovider resize to finish") + err = waitForControllerVolumeResize(pvc, c) + Expect(err).NotTo(HaveOccurred(), "While waiting for pvc resize to finish") + + By("Getting a pod from deployment") + podList, err := framework.GetPodsForDeployment(c, deployment) + Expect(podList.Items).NotTo(BeEmpty()) + pod := podList.Items[0] + + By("Deleting the pod from deployment") + err = framework.DeletePodWithWait(f, c, &pod) + Expect(err).NotTo(HaveOccurred(), "while deleting pod for resizing") + + By("Waiting for deployment to create new pod") + pod, err = waitForDeploymentToRecreatePod(c, deployment) + Expect(err).NotTo(HaveOccurred(), "While waiting for pod to be recreated") + + By("Waiting for file system resize to finish") + pvc, err = waitForFSResize(pvc, c) + Expect(err).NotTo(HaveOccurred(), "while waiting for fs resize to finish") + + pvcConditions := pvc.Status.Conditions + Expect(len(pvcConditions)).To(Equal(0), "pvc should not have conditions") + }) +}) + +func waitForDeploymentToRecreatePod(client clientset.Interface, deployment *extensions.Deployment) (v1.Pod, error) { + var runningPod v1.Pod + waitErr := wait.PollImmediate(10*time.Second, 5*time.Minute, func() (bool, error) { + podList, err := framework.GetPodsForDeployment(client, deployment) + for _, pod := range podList.Items { + switch pod.Status.Phase { + case v1.PodRunning: + runningPod = pod + return true, nil + case v1.PodFailed, v1.PodSucceeded: + return false, conditions.ErrPodCompleted + } + return false, nil + } + return false, err + }) + return runningPod, waitErr +} diff --git a/test/e2e/storage/vsphere/vsphere_volume_node_poweroff.go b/test/e2e/storage/vsphere/vsphere_volume_node_poweroff.go index 23e7bd0824..42140257fb 100644 --- a/test/e2e/storage/vsphere/vsphere_volume_node_poweroff.go +++ b/test/e2e/storage/vsphere/vsphere_volume_node_poweroff.go @@ -97,7 +97,7 @@ var _ = utils.SIGDescribe("Node Poweroff [Feature:vsphere] [Slow] [Disruptive]", volumePath := pvs[0].Spec.VsphereVolume.VolumePath By("Creating a Deployment") - deployment, err := framework.CreateDeployment(client, int32(1), map[string]string{"test": "app"}, namespace, pvclaims, "") + deployment, err := framework.CreateDeployment(client, int32(1), map[string]string{"test": "app"}, nil, namespace, pvclaims, "") defer client.ExtensionsV1beta1().Deployments(namespace).Delete(deployment.Name, &metav1.DeleteOptions{}) By("Get pod from the deployement")