Implement file system resizing support on kubelet start

pkg/kubelet/events/event.go

@@ -52,6 +52,8 @@ const (
 	FailedDetachVolume                   = "FailedDetachVolume"
 	FailedMountVolume                    = "FailedMount"
 	VolumeResizeFailed                   = "VolumeResizeFailed"
+	FileSystemResizeFailed               = "FileSystemResizeFailed"
+	FileSystemResizeSuccess              = "FileSystemResizeSuccessful"
 	FailedUnMountVolume                  = "FailedUnMount"
 	FailedMapVolume                      = "FailedMapVolume"
 	FailedUnmapDevice                    = "FailedUnmapDevice"

pkg/util/BUILD

@@ -47,6 +47,7 @@ filegroup(
         "//pkg/util/procfs:all-srcs",
         "//pkg/util/reflector/prometheus:all-srcs",
         "//pkg/util/removeall:all-srcs",
+        "//pkg/util/resizefs:all-srcs",
         "//pkg/util/resourcecontainer:all-srcs",
         "//pkg/util/rlimit:all-srcs",
         "//pkg/util/selinux:all-srcs",

pkg/util/mount/mount_linux.go

@@ -498,7 +498,7 @@ func (mounter *SafeFormatAndMount) formatAndMount(source string, target string,
 	if mountErr != nil {
 		// Mount failed. This indicates either that the disk is unformatted or
 		// it contains an unexpected filesystem.
-		existingFormat, err := mounter.getDiskFormat(source)
+		existingFormat, err := mounter.GetDiskFormat(source)
 		if err != nil {
 			return err
 		}
@@ -536,8 +536,8 @@ func (mounter *SafeFormatAndMount) formatAndMount(source string, target string,
 	return mountErr
 }
 
-// getDiskFormat uses 'lsblk' to see if the given disk is unformated
-func (mounter *SafeFormatAndMount) getDiskFormat(disk string) (string, error) {
+// GetDiskFormat uses 'lsblk' to see if the given disk is unformated
+func (mounter *SafeFormatAndMount) GetDiskFormat(disk string) (string, error) {
 	args := []string{"-n", "-o", "FSTYPE", disk}
 	glog.V(4).Infof("Attempting to determine if disk %q is formatted using lsblk with args: (%v)", disk, args)
 	dataOut, err := mounter.Exec.Run("lsblk", args...)

pkg/util/resizefs/BUILD

+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "resizefs_unsupported.go",
+    ] + select({
+        "@io_bazel_rules_go//go/platform:linux_amd64": [
+            "resizefs_linux.go",
+        ],
+        "//conditions:default": [],
+    }),
+    importpath = "k8s.io/kubernetes/pkg/util/resizefs",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/util/mount:go_default_library",
+    ] + select({
+        "@io_bazel_rules_go//go/platform:linux_amd64": [
+            "//vendor/github.com/golang/glog:go_default_library",
+            "//vendor/k8s.io/utils/exec:go_default_library",
+        ],
+        "//conditions:default": [],
+    }),
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

pkg/util/resizefs/resizefs_linux.go

+// +build linux
+
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package resizefs
+
+import (
+	"fmt"
+
+	"github.com/golang/glog"
+	"k8s.io/kubernetes/pkg/util/mount"
+	utilexec "k8s.io/utils/exec"
+)
+
+const (
+	// 'fsck' found errors and corrected them
+	fsckErrorsCorrected = 1
+	// 'fsck' found errors but exited without correcting them
+	fsckErrorsUncorrected = 4
+)
+
+// ResizeFs Provides support for resizing file systems
+type ResizeFs struct {
+	mounter *mount.SafeFormatAndMount
+}
+
+// NewResizeFs returns new instance of resizer
+func NewResizeFs(mounter *mount.SafeFormatAndMount) *ResizeFs {
+	return &ResizeFs{mounter: mounter}
+}
+
+// Resize perform resize of file system
+func (resizefs *ResizeFs) Resize(devicePath string) (bool, error) {
+	format, err := resizefs.mounter.GetDiskFormat(devicePath)
+
+	if err != nil {
+		formatErr := fmt.Errorf("error checking format for device %s: %v", devicePath, err)
+		return false, formatErr
+	}
+
+	// If disk has no format, there is no need to resize the disk because mkfs.*
+	// by default will use whole disk anyways.
+	if format == "" {
+		return false, nil
+	}
+
+	deviceOpened, err := resizefs.mounter.DeviceOpened(devicePath)
+
+	if err != nil {
+		deviceOpenErr := fmt.Errorf("error verifying if device %s is open: %v", devicePath, err)
+		return false, deviceOpenErr
+	}
+
+	if deviceOpened {
+		deviceAlreadyOpenErr := fmt.Errorf("the device %s is already in use", devicePath)
+		return false, deviceAlreadyOpenErr
+	}
+
+	switch format {
+	case "ext3", "ext4":
+		fsckErr := resizefs.extFsck(devicePath, format)
+		if fsckErr != nil {
+			return false, fsckErr
+		}
+		return resizefs.extResize(devicePath)
+	case "xfs":
+		fsckErr := resizefs.fsckDevice(devicePath)
+		if fsckErr != nil {
+			return false, fsckErr
+		}
+		return resizefs.xfsResize(devicePath)
+	}
+	return false, fmt.Errorf("resize of format %s is not supported for device %s", format, devicePath)
+}
+
+func (resizefs *ResizeFs) fsckDevice(devicePath string) error {
+	glog.V(4).Infof("Checking for issues with fsck on device: %s", devicePath)
+	args := []string{"-a", devicePath}
+	out, err := resizefs.mounter.Exec.Run("fsck", args...)
+	if err != nil {
+		ee, isExitError := err.(utilexec.ExitError)
+		switch {
+		case err == utilexec.ErrExecutableNotFound:
+			glog.Warningf("'fsck' not found on system; continuing resizing without running 'fsck'.")
+		case isExitError && ee.ExitStatus() == fsckErrorsCorrected:
+			glog.V(2).Infof("Device %s has errors which were corrected by fsck: %s", devicePath, string(out))
+		case isExitError && ee.ExitStatus() == fsckErrorsUncorrected:
+			return fmt.Errorf("'fsck' found errors on device %s but could not correct them: %s", devicePath, string(out))
+		case isExitError && ee.ExitStatus() > fsckErrorsUncorrected:
+			glog.Infof("`fsck` error %s", string(out))
+		}
+	}
+	return nil
+}
+
+func (resizefs *ResizeFs) extFsck(devicePath string, fsType string) error {
+	glog.V(4).Infof("Checking for issues with fsck.%s on device: %s", fsType, devicePath)
+	args := []string{"-f", "-y", devicePath}
+	out, err := resizefs.mounter.Run("fsck."+fsType, args...)
+	if err != nil {
+		return fmt.Errorf("running fsck.%s failed on %s with error: %v\n Output: %s", fsType, devicePath, err, string(out))
+	}
+	return nil
+}
+
+func (resizefs *ResizeFs) extResize(devicePath string) (bool, error) {
+	output, err := resizefs.mounter.Exec.Run("resize2fs", devicePath)
+	if err == nil {
+		glog.V(2).Infof("Device %s resized successfully", devicePath)
+		return true, nil
+	}
+
+	resizeError := fmt.Errorf("resize of device %s failed: %v. resize2fs output: %s", devicePath, err, string(output))
+	return false, resizeError
+
+}
+
+func (resizefs *ResizeFs) xfsResize(devicePath string) (bool, error) {
+	args := []string{"-d", devicePath}
+	output, err := resizefs.mounter.Exec.Run("xfs_growfs", args...)
+
+	if err == nil {
+		glog.V(2).Infof("Device %s resized successfully", devicePath)
+		return true, nil
+	}
+
+	resizeError := fmt.Errorf("resize of device %s failed: %v. xfs_growfs output: %s", devicePath, err, string(output))
+	return false, resizeError
+}

pkg/util/resizefs/resizefs_unsupported.go

+// +build !linux
+
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package resizefs
+
+import (
+	"fmt"
+
+	"k8s.io/kubernetes/pkg/util/mount"
+)
+
+// ResizeFs Provides support for resizing file systems
+type ResizeFs struct {
+	mounter *mount.SafeFormatAndMount
+}
+
+// NewResizeFs returns new instance of resizer
+func NewResizeFs(mounter *mount.SafeFormatAndMount) *ResizeFs {
+	return &ResizeFs{mounter: mounter}
+}
+
+// Resize perform resize of file system
+func (resizefs *ResizeFs) Resize(devicePath string) (bool, error) {
+	return false, fmt.Errorf("Resize is not supported for this build")
+}

pkg/volume/util/operationexecutor/BUILD

@@ -18,6 +18,7 @@ go_library(
         "//pkg/features:go_default_library",
         "//pkg/kubelet/events:go_default_library",
         "//pkg/util/mount:go_default_library",
+        "//pkg/util/resizefs:go_default_library",
         "//pkg/volume:go_default_library",
         "//pkg/volume/util:go_default_library",
         "//pkg/volume/util/nestedpendingoperations:go_default_library",
@@ -28,6 +29,7 @@ go_library(
         "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/strategicpatch:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
         "//vendor/k8s.io/client-go/kubernetes:go_default_library",
         "//vendor/k8s.io/client-go/tools/record:go_default_library",

pkg/volume/util/operationexecutor/operation_executor_test.go

@@ -80,7 +80,6 @@ func TestOperationExecutor_MountVolume_ConcurrentMountForAttachablePlugins(t *te
 	volumesToMount := make([]VolumeToMount, numVolumesToAttach)
 	pdName := "pd-volume"
 	volumeName := v1.UniqueVolumeName(pdName)
-
 	// Act
 	for i := range volumesToMount {
 		podName := "pod-" + strconv.Itoa((i + 1))

pkg/volume/util/operationexecutor/operation_generator.go

@@ -17,6 +17,7 @@ limitations under the License.
 package operationexecutor
 
 import (
+	"encoding/json"
 	"fmt"
 	"strings"
 	"time"
@@ -26,6 +27,7 @@ import (
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/strategicpatch"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/record"
@@ -33,6 +35,7 @@ import (
 	"k8s.io/kubernetes/pkg/features"
 	kevents "k8s.io/kubernetes/pkg/kubelet/events"
 	"k8s.io/kubernetes/pkg/util/mount"
+	"k8s.io/kubernetes/pkg/util/resizefs"
 	"k8s.io/kubernetes/pkg/volume"
 	"k8s.io/kubernetes/pkg/volume/util"
 	"k8s.io/kubernetes/pkg/volume/util/volumehelper"
@@ -451,6 +454,13 @@ func (og *operationGenerator) GenerateMountVolumeFunc(
 
 			glog.Infof(volumeToMount.GenerateMsgDetailed("MountVolume.WaitForAttach succeeded", fmt.Sprintf("DevicePath %q", devicePath)))
 
+			mounter := og.volumePluginMgr.Host.GetMounter(volumePlugin.GetPluginName())
+			resizeError := og.resizeFileSystem(volumeToMount, devicePath, mounter)
+
+			if resizeError != nil {
+				return volumeToMount.GenerateErrorDetailed("MountVolume.Resize failed", resizeError)
+			}
+
 			deviceMountPath, err :=
 				volumeAttacher.GetDeviceMountPath(volumeToMount.VolumeSpec)
 			if err != nil {
@@ -528,6 +538,65 @@ func (og *operationGenerator) GenerateMountVolumeFunc(
 	}, volumePlugin.GetPluginName(), nil
 }
 
+func (og *operationGenerator) resizeFileSystem(volumeToMount VolumeToMount, devicePath string, mounter mount.Interface) error {
+	if !utilfeature.DefaultFeatureGate.Enabled(features.ExpandPersistentVolumes) {
+		glog.V(6).Infof("Resizing is not enabled for this volume %s", volumeToMount.VolumeName)
+		return nil
+	}
+
+	// Get expander, if possible
+	expandableVolumePlugin, _ :=
+		og.volumePluginMgr.FindExpandablePluginBySpec(volumeToMount.VolumeSpec)
+
+	if expandableVolumePlugin != nil &&
+		expandableVolumePlugin.RequiresFSResize() &&
+		volumeToMount.VolumeSpec.PersistentVolume != nil {
+		pv := volumeToMount.VolumeSpec.PersistentVolume
+		pvc, err := og.kubeClient.CoreV1().PersistentVolumeClaims(pv.Spec.ClaimRef.Namespace).Get(pv.Spec.ClaimRef.Name, metav1.GetOptions{})
+		if err != nil {
+			// Return error rather than leave the file system un-resized, caller will log and retry
+			return volumeToMount.GenerateErrorDetailed("MountVolume get PVC failed", err)
+		}
+
+		pvcStatusCap := pvc.Status.Capacity[v1.ResourceStorage]
+		pvSpecCap := pv.Spec.Capacity[v1.ResourceStorage]
+		if pvcStatusCap.Cmp(pvSpecCap) < 0 {
+			// File system resize was requested, proceed
+			glog.V(4).Infof(volumeToMount.GenerateMsgDetailed("MountVolume.resizeFileSystem entering", fmt.Sprintf("DevicePath %q", volumeToMount.DevicePath)))
+
+			diskFormatter := &mount.SafeFormatAndMount{
+				Interface: mounter,
+				Exec:      og.volumePluginMgr.Host.GetExec(expandableVolumePlugin.GetPluginName()),
+			}
+
+			resizer := resizefs.NewResizeFs(diskFormatter)
+			resizeStatus, resizeErr := resizer.Resize(devicePath)
+
+			if resizeErr != nil {
+				resizeDetailedError := volumeToMount.GenerateErrorDetailed("MountVolume.resizeFileSystem failed", resizeErr)
+				glog.Error(resizeDetailedError)
+				og.recorder.Eventf(volumeToMount.Pod, v1.EventTypeWarning, kevents.FileSystemResizeFailed, resizeDetailedError.Error())
+				return resizeDetailedError
+			}
+
+			if resizeStatus {
+				simpleMsg, detailedMsg := volumeToMount.GenerateMsg("MountVolume.resizeFileSystem succeeded", "")
+				og.recorder.Eventf(volumeToMount.Pod, v1.EventTypeNormal, kevents.FileSystemResizeSuccess, simpleMsg)
+				glog.Infof(detailedMsg)
+			}
+
+			// File system resize succeeded, now update the PVC's Capacity to match the PV's
+			err = updatePVCStatusCapacity(pvc.Name, pvc, pv.Spec.Capacity, og.kubeClient)
+			if err != nil {
+				// On retry, resizeFileSystem will be called again but do nothing
+				return volumeToMount.GenerateErrorDetailed("MountVolume update PVC status failed", err)
+			}
+			return nil
+		}
+	}
+	return nil
+}
+
 func (og *operationGenerator) GenerateUnmountVolumeFunc(
 	volumeToUnmount MountedVolume,
 	actualStateOfWorld ActualStateOfWorldMounterUpdater) (func() error, string, error) {
@@ -1104,6 +1173,7 @@ func (og *operationGenerator) GenerateExpandVolumeFunc(
 				og.recorder.Eventf(pvcWithResizeRequest.PVC, v1.EventTypeWarning, kevents.VolumeResizeFailed, expandErr.Error())
 				return expandErr
 			}
+			glog.Infof("ExpandVolume succeeded for volume %s", pvcWithResizeRequest.QualifiedName())
 			newSize = updatedSize
 			// k8s doesn't have transactions, we can't guarantee that after updating PV - updating PVC will be
 			// successful, that is why all PVCs for which pvc.Spec.Size > pvc.Status.Size must be reprocessed
@@ -1115,6 +1185,7 @@ func (og *operationGenerator) GenerateExpandVolumeFunc(
 				og.recorder.Eventf(pvcWithResizeRequest.PVC, v1.EventTypeWarning, kevents.VolumeResizeFailed, updateErr.Error())
 				return updateErr
 			}
+			glog.Infof("ExpandVolume.UpdatePV succeeded for volume %s", pvcWithResizeRequest.QualifiedName())
 		}
 
 		// No Cloudprovider resize needed, lets mark resizing as done
@@ -1190,3 +1261,30 @@ func isDeviceOpened(deviceToDetach AttachedVolume, mounter mount.Interface) (boo
 	}
 	return deviceOpened, nil
 }
+
+func updatePVCStatusCapacity(pvcName string, pvc *v1.PersistentVolumeClaim, capacity v1.ResourceList, client clientset.Interface) error {
+	pvcCopy := pvc.DeepCopy()
+
+	oldData, err := json.Marshal(pvcCopy)
+	if err != nil {
+		return fmt.Errorf("Failed to marshal oldData for pvc %q with %v", pvcName, err)
+	}
+
+	pvcCopy.Status.Capacity = capacity
+	newData, err := json.Marshal(pvcCopy)
+
+	if err != nil {
+		return fmt.Errorf("Failed to marshal newData for pvc %q with %v", pvcName, err)
+	}
+	patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, pvcCopy)
+
+	if err != nil {
+		return fmt.Errorf("Failed to CreateTwoWayMergePatch for pvc %q with %v ", pvcName, err)
+	}
+	_, err = client.CoreV1().PersistentVolumeClaims(pvc.Namespace).
+		Patch(pvcName, types.StrategicMergePatchType, patchBytes, "status")
+	if err != nil {
+		return fmt.Errorf("Failed to patch PVC %q with %v", pvcName, err)
+	}
+	return nil
+}