Merge pull request #74393 from codenrhoden/refactor-subpath

cmd/kubelet/app/BUILD

@@ -108,6 +108,7 @@ go_library(
         "//pkg/volume/scaleio:go_default_library",
         "//pkg/volume/secret:go_default_library",
         "//pkg/volume/storageos:go_default_library",
+        "//pkg/volume/util/subpath:go_default_library",
         "//pkg/volume/vsphere_volume:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",

cmd/kubelet/app/server.go

@@ -95,6 +95,7 @@ import (
 	"k8s.io/kubernetes/pkg/util/rlimit"
 	"k8s.io/kubernetes/pkg/version"
 	"k8s.io/kubernetes/pkg/version/verflag"
+	"k8s.io/kubernetes/pkg/volume/util/subpath"
 	nodeapiclientset "k8s.io/node-api/pkg/client/clientset/versioned"
 	"k8s.io/utils/exec"
 	"k8s.io/utils/nsenter"
@@ -364,6 +365,7 @@ func UnsecuredDependencies(s *options.KubeletServer) (*kubelet.Dependencies, err
 	}
 
 	mounter := mount.New(s.ExperimentalMounterPath)
+	subpather := subpath.New(mounter)
 	var pluginRunner = exec.New()
 	if s.Containerized {
 		klog.V(2).Info("Running kubelet in containerized mode")
@@ -372,6 +374,8 @@ func UnsecuredDependencies(s *options.KubeletServer) (*kubelet.Dependencies, err
 			return nil, err
 		}
 		mounter = mount.NewNsenterMounter(s.RootDirectory, ne)
+		// NSenter only valid on Linux
+		subpather = subpath.NewNSEnter(mounter, ne, s.RootDirectory)
 		// an exec interface which can use nsenter for flex plugin calls
 		pluginRunner, err = nsenter.NewNsenter(nsenter.DefaultHostRootFsPath, exec.New())
 		if err != nil {
@@ -399,6 +403,7 @@ func UnsecuredDependencies(s *options.KubeletServer) (*kubelet.Dependencies, err
 		CSIClient:           nil,
 		EventClient:         nil,
 		Mounter:             mounter,
+		Subpather:           subpather,
 		OOMAdjuster:         oom.NewOOMAdjuster(),
 		OSInterface:         kubecontainer.RealOS{},
 		VolumePlugins:       ProbeVolumePlugins(),

pkg/controller/.import-restrictions

@@ -167,6 +167,7 @@
       "AllowedPrefixes": [
         "k8s.io/kubernetes/pkg/api/legacyscheme",
         "k8s.io/kubernetes/pkg/api/v1/endpoints",
+        "k8s.io/kubernetes/pkg/api/v1/node",
         "k8s.io/kubernetes/pkg/api/v1/pod",
         "k8s.io/kubernetes/pkg/apis/apps/v1",
         "k8s.io/kubernetes/pkg/apis/autoscaling",
@@ -241,6 +242,7 @@
         "k8s.io/kubernetes/pkg/volume/util",
         "k8s.io/kubernetes/pkg/volume/util/operationexecutor",
         "k8s.io/kubernetes/pkg/volume/util/recyclerclient",
+        "k8s.io/kubernetes/pkg/volume/util/subpath",
         "k8s.io/kubernetes/pkg/volume/util/types",
         "k8s.io/kubernetes/pkg/volume/util/volumepathhandler",
         "k8s.io/kubernetes/pkg/api/service",

pkg/controller/volume/attachdetach/BUILD

@@ -22,6 +22,7 @@ go_library(
         "//pkg/volume:go_default_library",
         "//pkg/volume/util:go_default_library",
         "//pkg/volume/util/operationexecutor:go_default_library",
+        "//pkg/volume/util/subpath:go_default_library",
         "//pkg/volume/util/volumepathhandler:go_default_library",
         "//staging/src/k8s.io/api/authentication/v1:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",

pkg/controller/volume/attachdetach/attach_detach_controller.go

@@ -52,6 +52,7 @@ import (
 	"k8s.io/kubernetes/pkg/volume"
 	volumeutil "k8s.io/kubernetes/pkg/volume/util"
 	"k8s.io/kubernetes/pkg/volume/util/operationexecutor"
+	"k8s.io/kubernetes/pkg/volume/util/subpath"
 	"k8s.io/kubernetes/pkg/volume/util/volumepathhandler"
 )
 
@@ -768,3 +769,8 @@ func (adc *attachDetachController) GetEventRecorder() record.EventRecorder {
 func (adc *attachDetachController) GetCSIClient() csiclient.Interface {
 	return adc.csiClient
 }
+
+func (adc *attachDetachController) GetSubpather() subpath.Interface {
+	// Subpaths not needed in attachdetach controller
+	return nil
+}

pkg/controller/volume/expand/BUILD

@@ -19,6 +19,7 @@ go_library(
         "//pkg/volume:go_default_library",
         "//pkg/volume/util:go_default_library",
         "//pkg/volume/util/operationexecutor:go_default_library",
+        "//pkg/volume/util/subpath:go_default_library",
         "//pkg/volume/util/volumepathhandler:go_default_library",
         "//staging/src/k8s.io/api/authentication/v1:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",

pkg/controller/volume/expand/expand_controller.go

@@ -46,6 +46,7 @@ import (
 	"k8s.io/kubernetes/pkg/volume"
 	"k8s.io/kubernetes/pkg/volume/util"
 	"k8s.io/kubernetes/pkg/volume/util/operationexecutor"
+	"k8s.io/kubernetes/pkg/volume/util/subpath"
 	"k8s.io/kubernetes/pkg/volume/util/volumepathhandler"
 )
 
@@ -339,3 +340,8 @@ func (expc *expandController) GetCSIClient() csiclientset.Interface {
 	// No volume plugin in expand controller needs csi.storage.k8s.io
 	return nil
 }
+
+func (expc *expandController) GetSubpather() subpath.Interface {
+	// not needed for expand controller
+	return nil
+}

pkg/controller/volume/persistentvolume/BUILD

@@ -33,6 +33,7 @@ go_library(
         "//pkg/volume:go_default_library",
         "//pkg/volume/util:go_default_library",
         "//pkg/volume/util/recyclerclient:go_default_library",
+        "//pkg/volume/util/subpath:go_default_library",
         "//staging/src/k8s.io/api/authentication/v1:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
         "//staging/src/k8s.io/api/storage/v1:go_default_library",

pkg/controller/volume/persistentvolume/volume_host.go

@@ -30,6 +30,7 @@ import (
 	"k8s.io/klog"
 	"k8s.io/kubernetes/pkg/util/mount"
 	vol "k8s.io/kubernetes/pkg/volume"
+	"k8s.io/kubernetes/pkg/volume/util/subpath"
 )
 
 // VolumeHost interface implementation for PersistentVolumeController.
@@ -136,3 +137,8 @@ func (ctrl *PersistentVolumeController) GetCSIClient() csiclientset.Interface {
 	// No volume plugin needs csi.storage.k8s.io client in PV controller.
 	return nil
 }
+
+func (ctrl *PersistentVolumeController) GetSubpather() subpath.Interface {
+	// No volume plugin needs Subpaths in PV controller.
+	return nil
+}

pkg/kubelet/BUILD

@@ -112,6 +112,7 @@ go_library(
         "//pkg/volume:go_default_library",
         "//pkg/volume/csi:go_default_library",
         "//pkg/volume/util:go_default_library",
+        "//pkg/volume/util/subpath:go_default_library",
         "//pkg/volume/util/types:go_default_library",
         "//pkg/volume/util/volumepathhandler:go_default_library",
         "//pkg/volume/validation:go_default_library",
@@ -220,6 +221,7 @@ go_test(
         "//pkg/volume/host_path:go_default_library",
         "//pkg/volume/testing:go_default_library",
         "//pkg/volume/util:go_default_library",
+        "//pkg/volume/util/subpath:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/equality:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",

pkg/kubelet/kubelet.go

@@ -113,6 +113,7 @@ import (
 	"k8s.io/kubernetes/pkg/util/oom"
 	"k8s.io/kubernetes/pkg/volume"
 	"k8s.io/kubernetes/pkg/volume/csi"
+	"k8s.io/kubernetes/pkg/volume/util/subpath"
 	nodeapiclientset "k8s.io/node-api/pkg/client/clientset/versioned"
 	utilexec "k8s.io/utils/exec"
 	"k8s.io/utils/integer"
@@ -255,6 +256,7 @@ type Dependencies struct {
 	OSInterface             kubecontainer.OSInterface
 	PodConfig               *config.PodConfig
 	Recorder                record.EventRecorder
+	Subpather               subpath.Interface
 	VolumePlugins           []volume.VolumePlugin
 	DynamicPluginProber     volume.DynamicPluginProber
 	TLSOptions              *server.TLSOptions
@@ -519,6 +521,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
 		cgroupsPerQOS:                           kubeCfg.CgroupsPerQOS,
 		cgroupRoot:                              kubeCfg.CgroupRoot,
 		mounter:                                 kubeDeps.Mounter,
+		subpather:                               kubeDeps.Subpather,
 		maxPods:                                 int(kubeCfg.MaxPods),
 		podsPerCore:                             int(kubeCfg.PodsPerCore),
 		syncLoopMonitor:                         atomic.Value{},
@@ -1099,6 +1102,9 @@ type Kubelet struct {
 	// Mounter to use for volumes.
 	mounter mount.Interface
 
+	// subpather to execute subpath actions
+	subpather subpath.Interface
+
 	// Manager of non-Runtime containers.
 	containerManager cm.ContainerManager
 

pkg/kubelet/kubelet_pods.go

@@ -61,6 +61,7 @@ import (
 	"k8s.io/kubernetes/pkg/kubelet/util/format"
 	mountutil "k8s.io/kubernetes/pkg/util/mount"
 	volumeutil "k8s.io/kubernetes/pkg/volume/util"
+	"k8s.io/kubernetes/pkg/volume/util/subpath"
 	"k8s.io/kubernetes/pkg/volume/util/volumepathhandler"
 	volumevalidation "k8s.io/kubernetes/pkg/volume/validation"
 	"k8s.io/kubernetes/third_party/forked/golang/expansion"
@@ -127,7 +128,7 @@ func (kl *Kubelet) makeBlockVolumes(pod *v1.Pod, container *v1.Container, podVol
 }
 
 // makeMounts determines the mount points for the given container.
-func makeMounts(pod *v1.Pod, podDir string, container *v1.Container, hostName, hostDomain, podIP string, podVolumes kubecontainer.VolumeMap, mounter mountutil.Interface, expandEnvs []kubecontainer.EnvVar) ([]kubecontainer.Mount, func(), error) {
+func makeMounts(pod *v1.Pod, podDir string, container *v1.Container, hostName, hostDomain, podIP string, podVolumes kubecontainer.VolumeMap, mounter mountutil.Interface, subpather subpath.Interface, expandEnvs []kubecontainer.EnvVar) ([]kubecontainer.Mount, func(), error) {
 	// Kubernetes only mounts on /etc/hosts if:
 	// - container is not an infrastructure (pause) container
 	// - container is not already mounting on /etc/hosts
@@ -206,13 +207,13 @@ func makeMounts(pod *v1.Pod, podDir string, container *v1.Container, hostName, h
 				if err != nil {
 					return nil, cleanupAction, err
 				}
-				if err := mounter.SafeMakeDir(subPath, volumePath, perm); err != nil {
+				if err := subpather.SafeMakeDir(subPath, volumePath, perm); err != nil {
 					// Don't pass detailed error back to the user because it could give information about host filesystem
 					klog.Errorf("failed to create subPath directory for volumeMount %q of container %q: %v", mount.Name, container.Name, err)
 					return nil, cleanupAction, fmt.Errorf("failed to create subPath directory for volumeMount %q of container %q", mount.Name, container.Name)
 				}
 			}
-			hostPath, cleanupAction, err = mounter.PrepareSafeSubpath(mountutil.Subpath{
+			hostPath, cleanupAction, err = subpather.PrepareSafeSubpath(subpath.Subpath{
 				VolumeMountIndex: i,
 				Path:             hostPath,
 				VolumeName:       vol.InnerVolumeSpecName,
@@ -464,7 +465,7 @@ func (kl *Kubelet) GenerateRunContainerOptions(pod *v1.Pod, container *v1.Contai
 	}
 	opts.Envs = append(opts.Envs, envs...)
 
-	mounts, cleanupAction, err := makeMounts(pod, kl.getPodDir(pod.UID), container, hostname, hostDomainName, podIP, volumes, kl.mounter, opts.Envs)
+	mounts, cleanupAction, err := makeMounts(pod, kl.getPodDir(pod.UID), container, hostname, hostDomainName, podIP, volumes, kl.mounter, kl.subpather, opts.Envs)
 	if err != nil {
 		return nil, cleanupAction, err
 	}

pkg/kubelet/kubelet_pods_linux_test.go

@@ -29,6 +29,7 @@ import (
 	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
 	"k8s.io/kubernetes/pkg/util/mount"
 	volumetest "k8s.io/kubernetes/pkg/volume/testing"
+	"k8s.io/kubernetes/pkg/volume/util/subpath"
 )
 
 func TestMakeMounts(t *testing.T) {
@@ -241,13 +242,14 @@ func TestMakeMounts(t *testing.T) {
 	for name, tc := range testCases {
 		t.Run(name, func(t *testing.T) {
 			fm := &mount.FakeMounter{}
+			fsp := &subpath.FakeSubpath{}
 			pod := v1.Pod{
 				Spec: v1.PodSpec{
 					HostNetwork: true,
 				},
 			}
 
-			mounts, _, err := makeMounts(&pod, "/pod", &tc.container, "fakepodname", "", "", tc.podVolumes, fm, nil)
+			mounts, _, err := makeMounts(&pod, "/pod", &tc.container, "fakepodname", "", "", tc.podVolumes, fm, fsp, nil)
 
 			// validate only the error if we expect an error
 			if tc.expectErr {

pkg/kubelet/kubelet_pods_test.go

@@ -49,10 +49,12 @@ import (
 	"k8s.io/kubernetes/pkg/kubelet/server/portforward"
 	"k8s.io/kubernetes/pkg/kubelet/server/remotecommand"
 	"k8s.io/kubernetes/pkg/util/mount"
+	"k8s.io/kubernetes/pkg/volume/util/subpath"
 )
 
 func TestDisabledSubpath(t *testing.T) {
 	fm := &mount.FakeMounter{}
+	fsp := &subpath.FakeSubpath{}
 	pod := v1.Pod{
 		Spec: v1.PodSpec{
 			HostNetwork: true,
@@ -95,7 +97,7 @@ func TestDisabledSubpath(t *testing.T) {
 
 	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.VolumeSubpath, false)()
 	for name, test := range cases {
-		_, _, err := makeMounts(&pod, "/pod", &test.container, "fakepodname", "", "", podVolumes, fm, nil)
+		_, _, err := makeMounts(&pod, "/pod", &test.container, "fakepodname", "", "", podVolumes, fm, fsp, nil)
 		if err != nil && !test.expectError {
 			t.Errorf("test %v failed: %v", name, err)
 		}

pkg/kubelet/kubelet_pods_windows_test.go

@@ -23,6 +23,7 @@ import (
 	"k8s.io/api/core/v1"
 	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
 	"k8s.io/kubernetes/pkg/util/mount"
+	"k8s.io/kubernetes/pkg/volume/util/subpath"
 )
 
 func TestMakeMountsWindows(t *testing.T) {
@@ -82,7 +83,8 @@ func TestMakeMountsWindows(t *testing.T) {
 	}
 
 	fm := &mount.FakeMounter{}
-	mounts, _, _ := makeMounts(&pod, "/pod", &container, "fakepodname", "", "", podVolumes, fm, nil)
+	fsp := &subpath.FakeSubpath{}
+	mounts, _, _ := makeMounts(&pod, "/pod", &container, "fakepodname", "", "", podVolumes, fm, fsp, nil)
 
 	expectedMounts := []kubecontainer.Mount{
 		{

pkg/kubelet/kubelet_test.go

@@ -75,6 +75,7 @@ import (
 	_ "k8s.io/kubernetes/pkg/volume/host_path"
 	volumetest "k8s.io/kubernetes/pkg/volume/testing"
 	"k8s.io/kubernetes/pkg/volume/util"
+	"k8s.io/kubernetes/pkg/volume/util/subpath"
 )
 
 func init() {
@@ -176,6 +177,7 @@ func newTestKubeletWithImageList(
 	kubelet.heartbeatClient = fakeKubeClient
 	kubelet.os = &containertest.FakeOS{}
 	kubelet.mounter = &mount.FakeMounter{}
+	kubelet.subpather = &subpath.FakeSubpath{}
 
 	kubelet.hostname = testKubeletHostname
 	kubelet.nodeName = types.NodeName(testKubeletHostname)

pkg/kubelet/volume_host.go

@@ -40,6 +40,7 @@ import (
 	"k8s.io/kubernetes/pkg/util/mount"
 	"k8s.io/kubernetes/pkg/volume"
 	"k8s.io/kubernetes/pkg/volume/util"
+	"k8s.io/kubernetes/pkg/volume/util/subpath"
 )
 
 // NewInitializedVolumePluginMgr returns a new instance of
@@ -126,6 +127,10 @@ func (kvh *kubeletVolumeHost) GetCSIClient() csiclientset.Interface {
 	return kvh.kubelet.csiClient
 }
 
+func (kvh *kubeletVolumeHost) GetSubpather() subpath.Interface {
+	return kvh.kubelet.subpather
+}
+
 func (kvh *kubeletVolumeHost) NewWrapperMounter(
 	volName string,
 	spec volume.Spec,

pkg/kubemark/BUILD

@@ -36,6 +36,7 @@ go_library(
         "//pkg/volume/emptydir:go_default_library",
         "//pkg/volume/projected:go_default_library",
         "//pkg/volume/secret:go_default_library",
+        "//pkg/volume/util/subpath:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/fields:go_default_library",

pkg/kubemark/hollow_kubelet.go

@@ -35,6 +35,7 @@ import (
 	"k8s.io/kubernetes/pkg/volume/emptydir"
 	"k8s.io/kubernetes/pkg/volume/projected"
 	"k8s.io/kubernetes/pkg/volume/secret"
+	"k8s.io/kubernetes/pkg/volume/util/subpath"
 	"k8s.io/kubernetes/test/utils"
 
 	"k8s.io/klog"
@@ -78,6 +79,7 @@ func NewHollowKubelet(
 		TLSOptions:         nil,
 		OOMAdjuster:        oom.NewFakeOOMAdjuster(),
 		Mounter:            mount.New("" /* default mount path */),
+		Subpather:          &subpath.FakeSubpath{},
 	}
 
 	return &HollowKubelet{

pkg/util/mount/BUILD

@@ -80,8 +80,6 @@ go_test(
         "//vendor/k8s.io/utils/exec/testing:go_default_library",
     ] + select({
         "@io_bazel_rules_go//go/platform:linux": [
-            "//vendor/golang.org/x/sys/unix:go_default_library",
-            "//vendor/k8s.io/klog:go_default_library",
             "//vendor/k8s.io/utils/exec:go_default_library",
             "//vendor/k8s.io/utils/nsenter:go_default_library",
         ],

pkg/util/mount/exec_mount.go

@@ -144,18 +144,6 @@ func (m *execMounter) EvalHostSymlinks(pathname string) (string, error) {
 	return m.wrappedMounter.EvalHostSymlinks(pathname)
 }
 
-func (m *execMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
-	return m.wrappedMounter.PrepareSafeSubpath(subPath)
-}
-
-func (m *execMounter) CleanSubPaths(podDir string, volumeName string) error {
-	return m.wrappedMounter.CleanSubPaths(podDir, volumeName)
-}
-
-func (m *execMounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
-	return m.wrappedMounter.SafeMakeDir(pathname, base, perm)
-}
-
 func (m *execMounter) GetMountRefs(pathname string) ([]string, error) {
 	return m.wrappedMounter.GetMountRefs(pathname)
 }

pkg/util/mount/exec_mount_unsupported.go

@@ -91,18 +91,6 @@ func (m *execMounter) EvalHostSymlinks(pathname string) (string, error) {
 	return "", errors.New("not implemented")
 }
 
-func (mounter *execMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
-	return subPath.Path, nil, nil
-}
-
-func (mounter *execMounter) CleanSubPaths(podDir string, volumeName string) error {
-	return nil
-}
-
-func (mounter *execMounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
-	return nil
-}
-
 func (mounter *execMounter) GetMountRefs(pathname string) ([]string, error) {
 	return nil, errors.New("not implemented")
 }

pkg/util/mount/fake.go

@@ -220,17 +220,6 @@ func (f *FakeMounter) EvalHostSymlinks(pathname string) (string, error) {
 	return pathname, nil
 }
 
-func (f *FakeMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
-	return subPath.Path, nil, nil
-}
-
-func (f *FakeMounter) CleanSubPaths(podDir string, volumeName string) error {
-	return nil
-}
-func (mounter *FakeMounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
-	return nil
-}
-
 func (f *FakeMounter) GetMountRefs(pathname string) ([]string, error) {
 	realpath, err := filepath.EvalSymlinks(pathname)
 	if err != nil {

pkg/util/mount/mount.go

@@ -84,35 +84,12 @@ type Interface interface {
 	// MakeDir creates a new directory.
 	// Will operate in the host mount namespace if kubelet is running in a container
 	MakeDir(pathname string) error
-	// SafeMakeDir creates subdir within given base. It makes sure that the
-	// created directory does not escape given base directory mis-using
-	// symlinks. Note that the function makes sure that it creates the directory
-	// somewhere under the base, nothing else. E.g. if the directory already
-	// exists, it may exist outside of the base due to symlinks.
-	// This method should be used if the directory to create is inside volume
-	// that's under user control. User must not be able to use symlinks to
-	// escape the volume to create directories somewhere else.
-	SafeMakeDir(subdir string, base string, perm os.FileMode) error
 	// Will operate in the host mount namespace if kubelet is running in a container.
 	// Error is returned on any other error than "file not found".
 	ExistsPath(pathname string) (bool, error)
 	// EvalHostSymlinks returns the path name after evaluating symlinks.
 	// Will operate in the host mount namespace if kubelet is running in a container.
 	EvalHostSymlinks(pathname string) (string, error)
-	// CleanSubPaths removes any bind-mounts created by PrepareSafeSubpath in given
-	// pod volume directory.
-	CleanSubPaths(podDir string, volumeName string) error
-	// PrepareSafeSubpath does everything that's necessary to prepare a subPath
-	// that's 1) inside given volumePath and 2) immutable after this call.
-	//
-	// newHostPath - location of prepared subPath. It should be used instead of
-	// hostName when running the container.
-	// cleanupAction - action to run when the container is running or it failed to start.
-	//
-	// CleanupAction must be called immediately after the container with given
-	// subpath starts. On the other hand, Interface.CleanSubPaths must be called
-	// when the pod finishes.
-	PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error)
 	// GetMountRefs finds all mount references to the path, returns a
 	// list of paths. Path could be a mountpoint path, device or a normal
 	// directory (for bind mount).
@@ -355,15 +332,15 @@ func PathWithinBase(fullPath, basePath string) bool {
 	if err != nil {
 		return false
 	}
-	if startsWithBackstep(rel) {
+	if StartsWithBackstep(rel) {
 		// Needed to escape the base path
 		return false
 	}
 	return true
 }
 
-// startsWithBackstep checks if the given path starts with a backstep segment
-func startsWithBackstep(rel string) bool {
+// StartsWithBackstep checks if the given path starts with a backstep segment
+func StartsWithBackstep(rel string) bool {
 	// normalize to / and check for ../
 	return rel == ".." || strings.HasPrefix(filepath.ToSlash(rel), "../")
 }

pkg/util/mount/mount_linux.go

@@ -21,7 +21,6 @@ package mount
 import (
 	"errors"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"os/exec"
 	"path"
@@ -53,14 +52,6 @@ const (
 	fsckErrorsCorrected = 1
 	// 'fsck' found errors but exited without correcting them
 	fsckErrorsUncorrected = 4
-
-	// place for subpath mounts
-	// TODO: pass in directory using kubelet_getters instead
-	containerSubPathDirectoryName = "volume-subpaths"
-	// syscall.Openat flags used to traverse directories not following symlinks
-	nofollowFlags = unix.O_RDONLY | unix.O_NOFOLLOW
-	// flags for getting file descriptor without following the symlink
-	openFDFlags = unix.O_NOFOLLOW | unix.O_PATH
 )
 
 // Mounter provides the default implementation of mount.Interface
@@ -726,282 +717,6 @@ func getSELinuxSupport(path string, mountInfoFilename string) (bool, error) {
 	return false, nil
 }
 
-func (mounter *Mounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
-	newHostPath, err = doBindSubPath(mounter, subPath)
-
-	// There is no action when the container starts. Bind-mount will be cleaned
-	// when container stops by CleanSubPaths.
-	cleanupAction = nil
-	return newHostPath, cleanupAction, err
-}
-
-// This implementation is shared between Linux and NsEnterMounter
-func safeOpenSubPath(mounter Interface, subpath Subpath) (int, error) {
-	if !PathWithinBase(subpath.Path, subpath.VolumePath) {
-		return -1, fmt.Errorf("subpath %q not within volume path %q", subpath.Path, subpath.VolumePath)
-	}
-	fd, err := doSafeOpen(subpath.Path, subpath.VolumePath)
-	if err != nil {
-		return -1, fmt.Errorf("error opening subpath %v: %v", subpath.Path, err)
-	}
-	return fd, nil
-}
-
-// prepareSubpathTarget creates target for bind-mount of subpath. It returns
-// "true" when the target already exists and something is mounted there.
-// Given Subpath must have all paths with already resolved symlinks and with
-// paths relevant to kubelet (when it runs in a container).
-// This function is called also by NsEnterMounter. It works because
-// /var/lib/kubelet is mounted from the host into the container with Kubelet as
-// /var/lib/kubelet too.
-func prepareSubpathTarget(mounter Interface, subpath Subpath) (bool, string, error) {
-	// Early check for already bind-mounted subpath.
-	bindPathTarget := getSubpathBindTarget(subpath)
-	notMount, err := mounter.IsNotMountPoint(bindPathTarget)
-	if err != nil {
-		if !os.IsNotExist(err) {
-			return false, "", fmt.Errorf("error checking path %s for mount: %s", bindPathTarget, err)
-		}
-		// Ignore ErrorNotExist: the file/directory will be created below if it does not exist yet.
-		notMount = true
-	}
-	if !notMount {
-		// It's already mounted
-		klog.V(5).Infof("Skipping bind-mounting subpath %s: already mounted", bindPathTarget)
-		return true, bindPathTarget, nil
-	}
-
-	// bindPathTarget is in /var/lib/kubelet and thus reachable without any
-	// translation even to containerized kubelet.
-	bindParent := filepath.Dir(bindPathTarget)
-	err = os.MkdirAll(bindParent, 0750)
-	if err != nil && !os.IsExist(err) {
-		return false, "", fmt.Errorf("error creating directory %s: %s", bindParent, err)
-	}
-
-	t, err := os.Lstat(subpath.Path)
-	if err != nil {
-		return false, "", fmt.Errorf("lstat %s failed: %s", subpath.Path, err)
-	}
-
-	if t.Mode()&os.ModeDir > 0 {
-		if err = os.Mkdir(bindPathTarget, 0750); err != nil && !os.IsExist(err) {
-			return false, "", fmt.Errorf("error creating directory %s: %s", bindPathTarget, err)
-		}
-	} else {
-		// "/bin/touch <bindPathTarget>".
-		// A file is enough for all possible targets (symlink, device, pipe,
-		// socket, ...), bind-mounting them into a file correctly changes type
-		// of the target file.
-		if err = ioutil.WriteFile(bindPathTarget, []byte{}, 0640); err != nil {
-			return false, "", fmt.Errorf("error creating file %s: %s", bindPathTarget, err)
-		}
-	}
-	return false, bindPathTarget, nil
-}
-
-func getSubpathBindTarget(subpath Subpath) string {
-	// containerName is DNS label, i.e. safe as a directory name.
-	return filepath.Join(subpath.PodDir, containerSubPathDirectoryName, subpath.VolumeName, subpath.ContainerName, strconv.Itoa(subpath.VolumeMountIndex))
-}
-
-func doBindSubPath(mounter Interface, subpath Subpath) (hostPath string, err error) {
-	// Linux, kubelet runs on the host:
-	// - safely open the subpath
-	// - bind-mount /proc/<pid of kubelet>/fd/<fd> to subpath target
-	// User can't change /proc/<pid of kubelet>/fd/<fd> to point to a bad place.
-
-	// Evaluate all symlinks here once for all subsequent functions.
-	newVolumePath, err := filepath.EvalSymlinks(subpath.VolumePath)
-	if err != nil {
-		return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.VolumePath, err)
-	}
-	newPath, err := filepath.EvalSymlinks(subpath.Path)
-	if err != nil {
-		return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.Path, err)
-	}
-	klog.V(5).Infof("doBindSubPath %q (%q) for volumepath %q", subpath.Path, newPath, subpath.VolumePath)
-	subpath.VolumePath = newVolumePath
-	subpath.Path = newPath
-
-	fd, err := safeOpenSubPath(mounter, subpath)
-	if err != nil {
-		return "", err
-	}
-	defer syscall.Close(fd)
-
-	alreadyMounted, bindPathTarget, err := prepareSubpathTarget(mounter, subpath)
-	if err != nil {
-		return "", err
-	}
-	if alreadyMounted {
-		return bindPathTarget, nil
-	}
-
-	success := false
-	defer func() {
-		// Cleanup subpath on error
-		if !success {
-			klog.V(4).Infof("doBindSubPath() failed for %q, cleaning up subpath", bindPathTarget)
-			if cleanErr := cleanSubPath(mounter, subpath); cleanErr != nil {
-				klog.Errorf("Failed to clean subpath %q: %v", bindPathTarget, cleanErr)
-			}
-		}
-	}()
-
-	kubeletPid := os.Getpid()
-	mountSource := fmt.Sprintf("/proc/%d/fd/%v", kubeletPid, fd)
-
-	// Do the bind mount
-	options := []string{"bind"}
-	klog.V(5).Infof("bind mounting %q at %q", mountSource, bindPathTarget)
-	if err = mounter.Mount(mountSource, bindPathTarget, "" /*fstype*/, options); err != nil {
-		return "", fmt.Errorf("error mounting %s: %s", subpath.Path, err)
-	}
-	success = true
-
-	klog.V(3).Infof("Bound SubPath %s into %s", subpath.Path, bindPathTarget)
-	return bindPathTarget, nil
-}
-
-func (mounter *Mounter) CleanSubPaths(podDir string, volumeName string) error {
-	return doCleanSubPaths(mounter, podDir, volumeName)
-}
-
-// This implementation is shared between Linux and NsEnterMounter
-func doCleanSubPaths(mounter Interface, podDir string, volumeName string) error {
-	// scan /var/lib/kubelet/pods/<uid>/volume-subpaths/<volume>/*
-	subPathDir := filepath.Join(podDir, containerSubPathDirectoryName, volumeName)
-	klog.V(4).Infof("Cleaning up subpath mounts for %s", subPathDir)
-
-	containerDirs, err := ioutil.ReadDir(subPathDir)
-	if err != nil {
-		if os.IsNotExist(err) {
-			return nil
-		}
-		return fmt.Errorf("error reading %s: %s", subPathDir, err)
-	}
-
-	for _, containerDir := range containerDirs {
-		if !containerDir.IsDir() {
-			klog.V(4).Infof("Container file is not a directory: %s", containerDir.Name())
-			continue
-		}
-		klog.V(4).Infof("Cleaning up subpath mounts for container %s", containerDir.Name())
-
-		// scan /var/lib/kubelet/pods/<uid>/volume-subpaths/<volume>/<container name>/*
-		fullContainerDirPath := filepath.Join(subPathDir, containerDir.Name())
-		err = filepath.Walk(fullContainerDirPath, func(path string, info os.FileInfo, err error) error {
-			if path == fullContainerDirPath {
-				// Skip top level directory
-				return nil
-			}
-
-			// pass through errors and let doCleanSubPath handle them
-			if err = doCleanSubPath(mounter, fullContainerDirPath, filepath.Base(path)); err != nil {
-				return err
-			}
-			return nil
-		})
-		if err != nil {
-			return fmt.Errorf("error processing %s: %s", fullContainerDirPath, err)
-		}
-
-		// Whole container has been processed, remove its directory.
-		if err := os.Remove(fullContainerDirPath); err != nil {
-			return fmt.Errorf("error deleting %s: %s", fullContainerDirPath, err)
-		}
-		klog.V(5).Infof("Removed %s", fullContainerDirPath)
-	}
-	// Whole pod volume subpaths have been cleaned up, remove its subpath directory.
-	if err := os.Remove(subPathDir); err != nil {
-		return fmt.Errorf("error deleting %s: %s", subPathDir, err)
-	}
-	klog.V(5).Infof("Removed %s", subPathDir)
-
-	// Remove entire subpath directory if it's the last one
-	podSubPathDir := filepath.Join(podDir, containerSubPathDirectoryName)
-	if err := os.Remove(podSubPathDir); err != nil && !os.IsExist(err) {
-		return fmt.Errorf("error deleting %s: %s", podSubPathDir, err)
-	}
-	klog.V(5).Infof("Removed %s", podSubPathDir)
-	return nil
-}
-
-// doCleanSubPath tears down the single subpath bind mount
-func doCleanSubPath(mounter Interface, fullContainerDirPath, subPathIndex string) error {
-	// process /var/lib/kubelet/pods/<uid>/volume-subpaths/<volume>/<container name>/<subPathName>
-	klog.V(4).Infof("Cleaning up subpath mounts for subpath %v", subPathIndex)
-	fullSubPath := filepath.Join(fullContainerDirPath, subPathIndex)
-
-	if err := CleanupMountPoint(fullSubPath, mounter, true); err != nil {
-		return fmt.Errorf("error cleaning subpath mount %s: %s", fullSubPath, err)
-	}
-
-	klog.V(4).Infof("Successfully cleaned subpath directory %s", fullSubPath)
-	return nil
-}
-
-// cleanSubPath will teardown the subpath bind mount and any remove any directories if empty
-func cleanSubPath(mounter Interface, subpath Subpath) error {
-	containerDir := filepath.Join(subpath.PodDir, containerSubPathDirectoryName, subpath.VolumeName, subpath.ContainerName)
-
-	// Clean subdir bindmount
-	if err := doCleanSubPath(mounter, containerDir, strconv.Itoa(subpath.VolumeMountIndex)); err != nil && !os.IsNotExist(err) {
-		return err
-	}
-
-	// Recusively remove directories if empty
-	if err := removeEmptyDirs(subpath.PodDir, containerDir); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-// removeEmptyDirs works backwards from endDir to baseDir and removes each directory
-// if it is empty.  It stops once it encounters a directory that has content
-func removeEmptyDirs(baseDir, endDir string) error {
-	if !PathWithinBase(endDir, baseDir) {
-		return fmt.Errorf("endDir %q is not within baseDir %q", endDir, baseDir)
-	}
-
-	for curDir := endDir; curDir != baseDir; curDir = filepath.Dir(curDir) {
-		s, err := os.Stat(curDir)
-		if err != nil {
-			if os.IsNotExist(err) {
-				klog.V(5).Infof("curDir %q doesn't exist, skipping", curDir)
-				continue
-			}
-			return fmt.Errorf("error stat %q: %v", curDir, err)
-		}
-		if !s.IsDir() {
-			return fmt.Errorf("path %q not a directory", curDir)
-		}
-
-		err = os.Remove(curDir)
-		if os.IsExist(err) {
-			klog.V(5).Infof("Directory %q not empty, not removing", curDir)
-			break
-		} else if err != nil {
-			return fmt.Errorf("error removing directory %q: %v", curDir, err)
-		}
-		klog.V(5).Infof("Removed directory %q", curDir)
-	}
-	return nil
-}
-
-func (mounter *Mounter) SafeMakeDir(subdir string, base string, perm os.FileMode) error {
-	realBase, err := filepath.EvalSymlinks(base)
-	if err != nil {
-		return fmt.Errorf("error resolving symlinks in %s: %s", base, err)
-	}
-
-	realFullPath := filepath.Join(realBase, subdir)
-
-	return doSafeMakeDir(realFullPath, realBase, perm)
-}
-
 func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) {
 	if _, err := os.Stat(pathname); os.IsNotExist(err) {
 		return []string{}, nil
@@ -1049,237 +764,6 @@ func getMode(pathname string) (os.FileMode, error) {
 	return info.Mode(), nil
 }
 
-// This implementation is shared between Linux and NsEnterMounter. Both pathname
-// and base must be either already resolved symlinks or thet will be resolved in
-// kubelet's mount namespace (in case it runs containerized).
-func doSafeMakeDir(pathname string, base string, perm os.FileMode) error {
-	klog.V(4).Infof("Creating directory %q within base %q", pathname, base)
-
-	if !PathWithinBase(pathname, base) {
-		return fmt.Errorf("path %s is outside of allowed base %s", pathname, base)
-	}
-
-	// Quick check if the directory already exists
-	s, err := os.Stat(pathname)
-	if err == nil {
-		// Path exists
-		if s.IsDir() {
-			// The directory already exists. It can be outside of the parent,
-			// but there is no race-proof check.
-			klog.V(4).Infof("Directory %s already exists", pathname)
-			return nil
-		}
-		return &os.PathError{Op: "mkdir", Path: pathname, Err: syscall.ENOTDIR}
-	}
-
-	// Find all existing directories
-	existingPath, toCreate, err := findExistingPrefix(base, pathname)
-	if err != nil {
-		return fmt.Errorf("error opening directory %s: %s", pathname, err)
-	}
-	// Ensure the existing directory is inside allowed base
-	fullExistingPath, err := filepath.EvalSymlinks(existingPath)
-	if err != nil {
-		return fmt.Errorf("error opening directory %s: %s", existingPath, err)
-	}
-	if !PathWithinBase(fullExistingPath, base) {
-		return fmt.Errorf("path %s is outside of allowed base %s", fullExistingPath, err)
-	}
-
-	klog.V(4).Infof("%q already exists, %q to create", fullExistingPath, filepath.Join(toCreate...))
-	parentFD, err := doSafeOpen(fullExistingPath, base)
-	if err != nil {
-		return fmt.Errorf("cannot open directory %s: %s", existingPath, err)
-	}
-	childFD := -1
-	defer func() {
-		if parentFD != -1 {
-			if err = syscall.Close(parentFD); err != nil {
-				klog.V(4).Infof("Closing FD %v failed for safemkdir(%v): %v", parentFD, pathname, err)
-			}
-		}
-		if childFD != -1 {
-			if err = syscall.Close(childFD); err != nil {
-				klog.V(4).Infof("Closing FD %v failed for safemkdir(%v): %v", childFD, pathname, err)
-			}
-		}
-	}()
-
-	currentPath := fullExistingPath
-	// create the directories one by one, making sure nobody can change
-	// created directory into symlink.
-	for _, dir := range toCreate {
-		currentPath = filepath.Join(currentPath, dir)
-		klog.V(4).Infof("Creating %s", dir)
-		err = syscall.Mkdirat(parentFD, currentPath, uint32(perm))
-		if err != nil {
-			return fmt.Errorf("cannot create directory %s: %s", currentPath, err)
-		}
-		// Dive into the created directory
-		childFD, err := syscall.Openat(parentFD, dir, nofollowFlags, 0)
-		if err != nil {
-			return fmt.Errorf("cannot open %s: %s", currentPath, err)
-		}
-		// We can be sure that childFD is safe to use. It could be changed
-		// by user after Mkdirat() and before Openat(), however:
-		// - it could not be changed to symlink - we use nofollowFlags
-		// - it could be changed to a file (or device, pipe, socket, ...)
-		//   but either subsequent Mkdirat() fails or we mount this file
-		//   to user's container. Security is no violated in both cases
-		//   and user either gets error or the file that it can already access.
-
-		if err = syscall.Close(parentFD); err != nil {
-			klog.V(4).Infof("Closing FD %v failed for safemkdir(%v): %v", parentFD, pathname, err)
-		}
-		parentFD = childFD
-		childFD = -1
-	}
-
-	// Everything was created. mkdirat(..., perm) above was affected by current
-	// umask and we must apply the right permissions to the last directory
-	// (that's the one that will be available to the container as subpath)
-	// so user can read/write it. This is the behavior of previous code.
-	// TODO: chmod all created directories, not just the last one.
-	// parentFD is the last created directory.
-
-	// Translate perm (os.FileMode) to uint32 that fchmod() expects
-	kernelPerm := uint32(perm & os.ModePerm)
-	if perm&os.ModeSetgid > 0 {
-		kernelPerm |= syscall.S_ISGID
-	}
-	if perm&os.ModeSetuid > 0 {
-		kernelPerm |= syscall.S_ISUID
-	}
-	if perm&os.ModeSticky > 0 {
-		kernelPerm |= syscall.S_ISVTX
-	}
-	if err = syscall.Fchmod(parentFD, kernelPerm); err != nil {
-		return fmt.Errorf("chmod %q failed: %s", currentPath, err)
-	}
-	return nil
-}
-
-// findExistingPrefix finds prefix of pathname that exists. In addition, it
-// returns list of remaining directories that don't exist yet.
-func findExistingPrefix(base, pathname string) (string, []string, error) {
-	rel, err := filepath.Rel(base, pathname)
-	if err != nil {
-		return base, nil, err
-	}
-	dirs := strings.Split(rel, string(filepath.Separator))
-
-	// Do OpenAt in a loop to find the first non-existing dir. Resolve symlinks.
-	// This should be faster than looping through all dirs and calling os.Stat()
-	// on each of them, as the symlinks are resolved only once with OpenAt().
-	currentPath := base
-	fd, err := syscall.Open(currentPath, syscall.O_RDONLY, 0)
-	if err != nil {
-		return pathname, nil, fmt.Errorf("error opening %s: %s", currentPath, err)
-	}
-	defer func() {
-		if err = syscall.Close(fd); err != nil {
-			klog.V(4).Infof("Closing FD %v failed for findExistingPrefix(%v): %v", fd, pathname, err)
-		}
-	}()
-	for i, dir := range dirs {
-		// Using O_PATH here will prevent hangs in case user replaces directory with
-		// fifo
-		childFD, err := syscall.Openat(fd, dir, unix.O_PATH, 0)
-		if err != nil {
-			if os.IsNotExist(err) {
-				return currentPath, dirs[i:], nil
-			}
-			return base, nil, err
-		}
-		if err = syscall.Close(fd); err != nil {
-			klog.V(4).Infof("Closing FD %v failed for findExistingPrefix(%v): %v", fd, pathname, err)
-		}
-		fd = childFD
-		currentPath = filepath.Join(currentPath, dir)
-	}
-	return pathname, []string{}, nil
-}
-
-// This implementation is shared between Linux and NsEnterMounter
-// Open path and return its fd.
-// Symlinks are disallowed (pathname must already resolve symlinks),
-// and the path must be within the base directory.
-func doSafeOpen(pathname string, base string) (int, error) {
-	pathname = filepath.Clean(pathname)
-	base = filepath.Clean(base)
-
-	// Calculate segments to follow
-	subpath, err := filepath.Rel(base, pathname)
-	if err != nil {
-		return -1, err
-	}
-	segments := strings.Split(subpath, string(filepath.Separator))
-
-	// Assumption: base is the only directory that we have under control.
-	// Base dir is not allowed to be a symlink.
-	parentFD, err := syscall.Open(base, nofollowFlags, 0)
-	if err != nil {
-		return -1, fmt.Errorf("cannot open directory %s: %s", base, err)
-	}
-	defer func() {
-		if parentFD != -1 {
-			if err = syscall.Close(parentFD); err != nil {
-				klog.V(4).Infof("Closing FD %v failed for safeopen(%v): %v", parentFD, pathname, err)
-			}
-		}
-	}()
-
-	childFD := -1
-	defer func() {
-		if childFD != -1 {
-			if err = syscall.Close(childFD); err != nil {
-				klog.V(4).Infof("Closing FD %v failed for safeopen(%v): %v", childFD, pathname, err)
-			}
-		}
-	}()
-
-	currentPath := base
-
-	// Follow the segments one by one using openat() to make
-	// sure the user cannot change already existing directories into symlinks.
-	for _, seg := range segments {
-		currentPath = filepath.Join(currentPath, seg)
-		if !PathWithinBase(currentPath, base) {
-			return -1, fmt.Errorf("path %s is outside of allowed base %s", currentPath, base)
-		}
-
-		klog.V(5).Infof("Opening path %s", currentPath)
-		childFD, err = syscall.Openat(parentFD, seg, openFDFlags, 0)
-		if err != nil {
-			return -1, fmt.Errorf("cannot open %s: %s", currentPath, err)
-		}
-
-		var deviceStat unix.Stat_t
-		err := unix.Fstat(childFD, &deviceStat)
-		if err != nil {
-			return -1, fmt.Errorf("Error running fstat on %s with %v", currentPath, err)
-		}
-		fileFmt := deviceStat.Mode & syscall.S_IFMT
-		if fileFmt == syscall.S_IFLNK {
-			return -1, fmt.Errorf("Unexpected symlink found %s", currentPath)
-		}
-
-		// Close parentFD
-		if err = syscall.Close(parentFD); err != nil {
-			return -1, fmt.Errorf("closing fd for %q failed: %v", filepath.Dir(currentPath), err)
-		}
-		// Set child to new parent
-		parentFD = childFD
-		childFD = -1
-	}
-
-	// We made it to the end, return this fd, don't close it
-	finalFD := parentFD
-	parentFD = -1
-
-	return finalFD, nil
-}
-
 // searchMountPoints finds all mount references to the source, returns a list of
 // mountpoints.
 // This function assumes source cannot be device.

pkg/util/mount/mount_linux_test.go

@@ -25,14 +25,10 @@ import (
 	"os"
 	"path/filepath"
 	"reflect"
-	"strconv"
 	"strings"
-	"syscall"
 	"testing"
 
 	"k8s.io/utils/exec"
-
-	"k8s.io/klog"
 )
 
 func TestReadProcMountsFrom(t *testing.T) {
@@ -424,822 +420,6 @@ func TestPathWithinBase(t *testing.T) {
 	}
 }
 
-func TestSafeMakeDir(t *testing.T) {
-	defaultPerm := os.FileMode(0750) + os.ModeDir
-	tests := []struct {
-		name string
-		// Function that prepares directory structure for the test under given
-		// base.
-		prepare     func(base string) error
-		path        string
-		checkPath   string
-		perm        os.FileMode
-		expectError bool
-	}{
-		{
-			"directory-does-not-exist",
-			func(base string) error {
-				return nil
-			},
-			"test/directory",
-			"test/directory",
-			defaultPerm,
-			false,
-		},
-		{
-			"directory-with-sgid",
-			func(base string) error {
-				return nil
-			},
-			"test/directory",
-			"test/directory",
-			os.FileMode(0777) + os.ModeDir + os.ModeSetgid,
-			false,
-		},
-		{
-			"directory-with-suid",
-			func(base string) error {
-				return nil
-			},
-			"test/directory",
-			"test/directory",
-			os.FileMode(0777) + os.ModeDir + os.ModeSetuid,
-			false,
-		},
-		{
-			"directory-with-sticky-bit",
-			func(base string) error {
-				return nil
-			},
-			"test/directory",
-			"test/directory",
-			os.FileMode(0777) + os.ModeDir + os.ModeSticky,
-			false,
-		},
-		{
-			"directory-exists",
-			func(base string) error {
-				return os.MkdirAll(filepath.Join(base, "test/directory"), 0750)
-			},
-			"test/directory",
-			"test/directory",
-			defaultPerm,
-			false,
-		},
-		{
-			"create-base",
-			func(base string) error {
-				return nil
-			},
-			"",
-			"",
-			defaultPerm,
-			false,
-		},
-		{
-			"escape-base-using-dots",
-			func(base string) error {
-				return nil
-			},
-			"..",
-			"",
-			defaultPerm,
-			true,
-		},
-		{
-			"escape-base-using-dots-2",
-			func(base string) error {
-				return nil
-			},
-			"test/../../..",
-			"",
-			defaultPerm,
-			true,
-		},
-		{
-			"follow-symlinks",
-			func(base string) error {
-				if err := os.MkdirAll(filepath.Join(base, "destination"), defaultPerm); err != nil {
-					return err
-				}
-				return os.Symlink("destination", filepath.Join(base, "test"))
-			},
-			"test/directory",
-			"destination/directory",
-			defaultPerm,
-			false,
-		},
-		{
-			"follow-symlink-loop",
-			func(base string) error {
-				return os.Symlink("test", filepath.Join(base, "test"))
-			},
-			"test/directory",
-			"",
-			defaultPerm,
-			true,
-		},
-		{
-			"follow-symlink-multiple follow",
-			func(base string) error {
-				/* test1/dir points to test2 and test2/dir points to test1 */
-				if err := os.MkdirAll(filepath.Join(base, "test1"), defaultPerm); err != nil {
-					return err
-				}
-				if err := os.MkdirAll(filepath.Join(base, "test2"), defaultPerm); err != nil {
-					return err
-				}
-				if err := os.Symlink(filepath.Join(base, "test2"), filepath.Join(base, "test1/dir")); err != nil {
-					return err
-				}
-				if err := os.Symlink(filepath.Join(base, "test1"), filepath.Join(base, "test2/dir")); err != nil {
-					return err
-				}
-				return nil
-			},
-			"test1/dir/dir/dir/dir/dir/dir/dir/foo",
-			"test2/foo",
-			defaultPerm,
-			false,
-		},
-		{
-			"danglink-symlink",
-			func(base string) error {
-				return os.Symlink("non-existing", filepath.Join(base, "test"))
-			},
-			"test/directory",
-			"",
-			defaultPerm,
-			true,
-		},
-		{
-			"non-directory",
-			func(base string) error {
-				return ioutil.WriteFile(filepath.Join(base, "test"), []byte{}, defaultPerm)
-			},
-			"test/directory",
-			"",
-			defaultPerm,
-			true,
-		},
-		{
-			"non-directory-final",
-			func(base string) error {
-				return ioutil.WriteFile(filepath.Join(base, "test"), []byte{}, defaultPerm)
-			},
-			"test",
-			"",
-			defaultPerm,
-			true,
-		},
-		{
-			"escape-with-relative-symlink",
-			func(base string) error {
-				if err := os.MkdirAll(filepath.Join(base, "dir"), defaultPerm); err != nil {
-					return err
-				}
-				if err := os.MkdirAll(filepath.Join(base, "exists"), defaultPerm); err != nil {
-					return err
-				}
-				return os.Symlink("../exists", filepath.Join(base, "dir/test"))
-			},
-			"dir/test",
-			"",
-			defaultPerm,
-			false,
-		},
-		{
-			"escape-with-relative-symlink-not-exists",
-			func(base string) error {
-				if err := os.MkdirAll(filepath.Join(base, "dir"), defaultPerm); err != nil {
-					return err
-				}
-				return os.Symlink("../not-exists", filepath.Join(base, "dir/test"))
-			},
-			"dir/test",
-			"",
-			defaultPerm,
-			true,
-		},
-		{
-			"escape-with-symlink",
-			func(base string) error {
-				return os.Symlink("/", filepath.Join(base, "test"))
-			},
-			"test/directory",
-			"",
-			defaultPerm,
-			true,
-		},
-	}
-
-	for i := range tests {
-		test := tests[i]
-		t.Run(test.name, func(t *testing.T) {
-			base, err := ioutil.TempDir("", "safe-make-dir-"+test.name+"-")
-			if err != nil {
-				t.Fatalf(err.Error())
-			}
-			defer os.RemoveAll(base)
-			test.prepare(base)
-			pathToCreate := filepath.Join(base, test.path)
-			err = doSafeMakeDir(pathToCreate, base, test.perm)
-			if err != nil && !test.expectError {
-				t.Fatal(err)
-			}
-			if err != nil {
-				t.Logf("got error: %s", err)
-			}
-			if err == nil && test.expectError {
-				t.Fatalf("expected error, got none")
-			}
-
-			if test.checkPath != "" {
-				st, err := os.Stat(filepath.Join(base, test.checkPath))
-				if err != nil {
-					t.Fatalf("cannot read path %s", test.checkPath)
-				}
-				actualMode := st.Mode()
-				if actualMode != test.perm {
-					if actualMode^test.perm == os.ModeSetgid && test.perm&os.ModeSetgid == 0 {
-						// when TMPDIR is a kubernetes emptydir, the sticky gid bit is set due to fsgroup
-						t.Logf("masking bit from %o", actualMode)
-					} else {
-						t.Errorf("expected permissions %o, got %o (%b)", test.perm, actualMode, test.perm^actualMode)
-					}
-				}
-			}
-		})
-	}
-}
-
-func TestRemoveEmptyDirs(t *testing.T) {
-	defaultPerm := os.FileMode(0750)
-	tests := []struct {
-		name string
-		// Function that prepares directory structure for the test under given
-		// base.
-		prepare func(base string) error
-		// Function that validates directory structure after the test
-		validate    func(base string) error
-		baseDir     string
-		endDir      string
-		expectError bool
-	}{
-		{
-			name: "all-empty",
-			prepare: func(base string) error {
-				return os.MkdirAll(filepath.Join(base, "a/b/c"), defaultPerm)
-			},
-			validate: func(base string) error {
-				return validateDirEmpty(filepath.Join(base, "a"))
-			},
-			baseDir:     "a",
-			endDir:      "a/b/c",
-			expectError: false,
-		},
-		{
-			name: "dir-not-empty",
-			prepare: func(base string) error {
-				if err := os.MkdirAll(filepath.Join(base, "a/b/c"), defaultPerm); err != nil {
-					return err
-				}
-				return os.Mkdir(filepath.Join(base, "a/b/d"), defaultPerm)
-			},
-			validate: func(base string) error {
-				if err := validateDirNotExists(filepath.Join(base, "a/b/c")); err != nil {
-					return err
-				}
-				return validateDirExists(filepath.Join(base, "a/b"))
-			},
-			baseDir:     "a",
-			endDir:      "a/b/c",
-			expectError: false,
-		},
-		{
-			name: "path-not-within-base",
-			prepare: func(base string) error {
-				return os.MkdirAll(filepath.Join(base, "a/b/c"), defaultPerm)
-			},
-			validate: func(base string) error {
-				return validateDirExists(filepath.Join(base, "a"))
-			},
-			baseDir:     "a",
-			endDir:      "b/c",
-			expectError: true,
-		},
-		{
-			name: "path-already-deleted",
-			prepare: func(base string) error {
-				return nil
-			},
-			validate: func(base string) error {
-				return nil
-			},
-			baseDir:     "a",
-			endDir:      "a/b/c",
-			expectError: false,
-		},
-		{
-			name: "path-not-dir",
-			prepare: func(base string) error {
-				if err := os.MkdirAll(filepath.Join(base, "a/b"), defaultPerm); err != nil {
-					return err
-				}
-				return ioutil.WriteFile(filepath.Join(base, "a/b", "c"), []byte{}, defaultPerm)
-			},
-			validate: func(base string) error {
-				if err := validateDirExists(filepath.Join(base, "a/b")); err != nil {
-					return err
-				}
-				return validateFileExists(filepath.Join(base, "a/b/c"))
-			},
-			baseDir:     "a",
-			endDir:      "a/b/c",
-			expectError: true,
-		},
-	}
-
-	for _, test := range tests {
-		klog.V(4).Infof("test %q", test.name)
-		base, err := ioutil.TempDir("", "remove-empty-dirs-"+test.name+"-")
-		if err != nil {
-			t.Fatalf(err.Error())
-		}
-		if err = test.prepare(base); err != nil {
-			os.RemoveAll(base)
-			t.Fatalf("failed to prepare test %q: %v", test.name, err.Error())
-		}
-
-		err = removeEmptyDirs(filepath.Join(base, test.baseDir), filepath.Join(base, test.endDir))
-		if err != nil && !test.expectError {
-			t.Errorf("test %q failed: %v", test.name, err)
-		}
-		if err == nil && test.expectError {
-			t.Errorf("test %q failed: expected error, got success", test.name)
-		}
-
-		if err = test.validate(base); err != nil {
-			t.Errorf("test %q failed validation: %v", test.name, err)
-		}
-
-		os.RemoveAll(base)
-	}
-}
-
-func TestCleanSubPaths(t *testing.T) {
-	defaultPerm := os.FileMode(0750)
-	testVol := "vol1"
-
-	tests := []struct {
-		name string
-		// Function that prepares directory structure for the test under given
-		// base.
-		prepare func(base string) ([]MountPoint, error)
-		// Function that validates directory structure after the test
-		validate    func(base string) error
-		expectError bool
-	}{
-		{
-			name: "not-exists",
-			prepare: func(base string) ([]MountPoint, error) {
-				return nil, nil
-			},
-			validate: func(base string) error {
-				return nil
-			},
-			expectError: false,
-		},
-		{
-			name: "subpath-not-mount",
-			prepare: func(base string) ([]MountPoint, error) {
-				return nil, os.MkdirAll(filepath.Join(base, containerSubPathDirectoryName, testVol, "container1", "0"), defaultPerm)
-			},
-			validate: func(base string) error {
-				return validateDirNotExists(filepath.Join(base, containerSubPathDirectoryName))
-			},
-			expectError: false,
-		},
-		{
-			name: "subpath-file",
-			prepare: func(base string) ([]MountPoint, error) {
-				path := filepath.Join(base, containerSubPathDirectoryName, testVol, "container1")
-				if err := os.MkdirAll(path, defaultPerm); err != nil {
-					return nil, err
-				}
-				return nil, ioutil.WriteFile(filepath.Join(path, "0"), []byte{}, defaultPerm)
-			},
-			validate: func(base string) error {
-				return validateDirNotExists(filepath.Join(base, containerSubPathDirectoryName))
-			},
-			expectError: false,
-		},
-		{
-			name: "subpath-container-not-dir",
-			prepare: func(base string) ([]MountPoint, error) {
-				path := filepath.Join(base, containerSubPathDirectoryName, testVol)
-				if err := os.MkdirAll(path, defaultPerm); err != nil {
-					return nil, err
-				}
-				return nil, ioutil.WriteFile(filepath.Join(path, "container1"), []byte{}, defaultPerm)
-			},
-			validate: func(base string) error {
-				return validateDirExists(filepath.Join(base, containerSubPathDirectoryName, testVol))
-			},
-			expectError: true,
-		},
-		{
-			name: "subpath-multiple-container-not-dir",
-			prepare: func(base string) ([]MountPoint, error) {
-				path := filepath.Join(base, containerSubPathDirectoryName, testVol)
-				if err := os.MkdirAll(filepath.Join(path, "container1"), defaultPerm); err != nil {
-					return nil, err
-				}
-				return nil, ioutil.WriteFile(filepath.Join(path, "container2"), []byte{}, defaultPerm)
-			},
-			validate: func(base string) error {
-				path := filepath.Join(base, containerSubPathDirectoryName, testVol)
-				if err := validateDirNotExists(filepath.Join(path, "container1")); err != nil {
-					return err
-				}
-				return validateFileExists(filepath.Join(path, "container2"))
-			},
-			expectError: true,
-		},
-		{
-			name: "subpath-mount",
-			prepare: func(base string) ([]MountPoint, error) {
-				path := filepath.Join(base, containerSubPathDirectoryName, testVol, "container1", "0")
-				if err := os.MkdirAll(path, defaultPerm); err != nil {
-					return nil, err
-				}
-				mounts := []MountPoint{{Device: "/dev/sdb", Path: path}}
-				return mounts, nil
-			},
-			validate: func(base string) error {
-				return validateDirNotExists(filepath.Join(base, containerSubPathDirectoryName))
-			},
-		},
-		{
-			name: "subpath-mount-multiple",
-			prepare: func(base string) ([]MountPoint, error) {
-				path := filepath.Join(base, containerSubPathDirectoryName, testVol, "container1", "0")
-				path2 := filepath.Join(base, containerSubPathDirectoryName, testVol, "container1", "1")
-				path3 := filepath.Join(base, containerSubPathDirectoryName, testVol, "container2", "1")
-				if err := os.MkdirAll(path, defaultPerm); err != nil {
-					return nil, err
-				}
-				if err := os.MkdirAll(path2, defaultPerm); err != nil {
-					return nil, err
-				}
-				if err := os.MkdirAll(path3, defaultPerm); err != nil {
-					return nil, err
-				}
-				mounts := []MountPoint{
-					{Device: "/dev/sdb", Path: path},
-					{Device: "/dev/sdb", Path: path3},
-				}
-				return mounts, nil
-			},
-			validate: func(base string) error {
-				return validateDirNotExists(filepath.Join(base, containerSubPathDirectoryName))
-			},
-		},
-		{
-			name: "subpath-mount-multiple-vols",
-			prepare: func(base string) ([]MountPoint, error) {
-				path := filepath.Join(base, containerSubPathDirectoryName, testVol, "container1", "0")
-				path2 := filepath.Join(base, containerSubPathDirectoryName, "vol2", "container1", "1")
-				if err := os.MkdirAll(path, defaultPerm); err != nil {
-					return nil, err
-				}
-				if err := os.MkdirAll(path2, defaultPerm); err != nil {
-					return nil, err
-				}
-				mounts := []MountPoint{
-					{Device: "/dev/sdb", Path: path},
-				}
-				return mounts, nil
-			},
-			validate: func(base string) error {
-				baseSubdir := filepath.Join(base, containerSubPathDirectoryName)
-				if err := validateDirNotExists(filepath.Join(baseSubdir, testVol)); err != nil {
-					return err
-				}
-				return validateDirExists(baseSubdir)
-			},
-		},
-	}
-
-	for _, test := range tests {
-		klog.V(4).Infof("test %q", test.name)
-		base, err := ioutil.TempDir("", "clean-subpaths-"+test.name+"-")
-		if err != nil {
-			t.Fatalf(err.Error())
-		}
-		mounts, err := test.prepare(base)
-		if err != nil {
-			os.RemoveAll(base)
-			t.Fatalf("failed to prepare test %q: %v", test.name, err.Error())
-		}
-
-		fm := &FakeMounter{MountPoints: mounts}
-
-		err = doCleanSubPaths(fm, base, testVol)
-		if err != nil && !test.expectError {
-			t.Errorf("test %q failed: %v", test.name, err)
-		}
-		if err == nil && test.expectError {
-			t.Errorf("test %q failed: expected error, got success", test.name)
-		}
-		if err = test.validate(base); err != nil {
-			t.Errorf("test %q failed validation: %v", test.name, err)
-		}
-
-		os.RemoveAll(base)
-	}
-}
-
-var (
-	testVol       = "vol1"
-	testPod       = "pod0"
-	testContainer = "container0"
-	testSubpath   = 1
-)
-
-func setupFakeMounter(testMounts []string) *FakeMounter {
-	mounts := []MountPoint{}
-	for _, mountPoint := range testMounts {
-		mounts = append(mounts, MountPoint{Device: "/foo", Path: mountPoint})
-	}
-	return &FakeMounter{MountPoints: mounts}
-}
-
-func getTestPaths(base string) (string, string) {
-	return filepath.Join(base, testVol),
-		filepath.Join(base, testPod, containerSubPathDirectoryName, testVol, testContainer, strconv.Itoa(testSubpath))
-}
-
-func TestBindSubPath(t *testing.T) {
-	defaultPerm := os.FileMode(0750)
-
-	tests := []struct {
-		name string
-		// Function that prepares directory structure for the test under given
-		// base.
-		prepare     func(base string) ([]string, string, string, error)
-		expectError bool
-	}{
-		{
-			name: "subpath-dir",
-			prepare: func(base string) ([]string, string, string, error) {
-				volpath, _ := getTestPaths(base)
-				subpath := filepath.Join(volpath, "dir0")
-				return nil, volpath, subpath, os.MkdirAll(subpath, defaultPerm)
-			},
-			expectError: false,
-		},
-		{
-			name: "subpath-dir-symlink",
-			prepare: func(base string) ([]string, string, string, error) {
-				volpath, _ := getTestPaths(base)
-				subpath := filepath.Join(volpath, "dir0")
-				if err := os.MkdirAll(subpath, defaultPerm); err != nil {
-					return nil, "", "", err
-				}
-				subpathLink := filepath.Join(volpath, "dirLink")
-				return nil, volpath, subpath, os.Symlink(subpath, subpathLink)
-			},
-			expectError: false,
-		},
-		{
-			name: "subpath-file",
-			prepare: func(base string) ([]string, string, string, error) {
-				volpath, _ := getTestPaths(base)
-				subpath := filepath.Join(volpath, "file0")
-				if err := os.MkdirAll(volpath, defaultPerm); err != nil {
-					return nil, "", "", err
-				}
-				return nil, volpath, subpath, ioutil.WriteFile(subpath, []byte{}, defaultPerm)
-			},
-			expectError: false,
-		},
-		{
-			name: "subpath-not-exists",
-			prepare: func(base string) ([]string, string, string, error) {
-				volpath, _ := getTestPaths(base)
-				subpath := filepath.Join(volpath, "file0")
-				return nil, volpath, subpath, nil
-			},
-			expectError: true,
-		},
-		{
-			name: "subpath-outside",
-			prepare: func(base string) ([]string, string, string, error) {
-				volpath, _ := getTestPaths(base)
-				subpath := filepath.Join(volpath, "dir0")
-				if err := os.MkdirAll(volpath, defaultPerm); err != nil {
-					return nil, "", "", err
-				}
-				return nil, volpath, subpath, os.Symlink(base, subpath)
-			},
-			expectError: true,
-		},
-		{
-			name: "subpath-symlink-child-outside",
-			prepare: func(base string) ([]string, string, string, error) {
-				volpath, _ := getTestPaths(base)
-				subpathDir := filepath.Join(volpath, "dir0")
-				subpath := filepath.Join(subpathDir, "child0")
-				if err := os.MkdirAll(subpathDir, defaultPerm); err != nil {
-					return nil, "", "", err
-				}
-				return nil, volpath, subpath, os.Symlink(base, subpath)
-			},
-			expectError: true,
-		},
-		{
-			name: "subpath-child-outside-exists",
-			prepare: func(base string) ([]string, string, string, error) {
-				volpath, _ := getTestPaths(base)
-				subpathDir := filepath.Join(volpath, "dir0")
-				child := filepath.Join(base, "child0")
-				subpath := filepath.Join(subpathDir, "child0")
-				if err := os.MkdirAll(volpath, defaultPerm); err != nil {
-					return nil, "", "", err
-				}
-				// touch file outside
-				if err := ioutil.WriteFile(child, []byte{}, defaultPerm); err != nil {
-					return nil, "", "", err
-				}
-
-				// create symlink for subpath dir
-				return nil, volpath, subpath, os.Symlink(base, subpathDir)
-			},
-			expectError: true,
-		},
-		{
-			name: "subpath-child-outside-not-exists",
-			prepare: func(base string) ([]string, string, string, error) {
-				volpath, _ := getTestPaths(base)
-				subpathDir := filepath.Join(volpath, "dir0")
-				subpath := filepath.Join(subpathDir, "child0")
-				if err := os.MkdirAll(volpath, defaultPerm); err != nil {
-					return nil, "", "", err
-				}
-				// create symlink for subpath dir
-				return nil, volpath, subpath, os.Symlink(base, subpathDir)
-			},
-			expectError: true,
-		},
-		{
-			name: "subpath-child-outside-exists-middle-dir-symlink",
-			prepare: func(base string) ([]string, string, string, error) {
-				volpath, _ := getTestPaths(base)
-				subpathDir := filepath.Join(volpath, "dir0")
-				symlinkDir := filepath.Join(subpathDir, "linkDir0")
-				child := filepath.Join(base, "child0")
-				subpath := filepath.Join(symlinkDir, "child0")
-				if err := os.MkdirAll(subpathDir, defaultPerm); err != nil {
-					return nil, "", "", err
-				}
-				// touch file outside
-				if err := ioutil.WriteFile(child, []byte{}, defaultPerm); err != nil {
-					return nil, "", "", err
-				}
-
-				// create symlink for middle dir
-				return nil, volpath, subpath, os.Symlink(base, symlinkDir)
-			},
-			expectError: true,
-		},
-		{
-			name: "subpath-backstepping",
-			prepare: func(base string) ([]string, string, string, error) {
-				volpath, _ := getTestPaths(base)
-				subpath := filepath.Join(volpath, "dir0")
-				symlinkBase := filepath.Join(volpath, "..")
-				if err := os.MkdirAll(volpath, defaultPerm); err != nil {
-					return nil, "", "", err
-				}
-
-				// create symlink for subpath
-				return nil, volpath, subpath, os.Symlink(symlinkBase, subpath)
-			},
-			expectError: true,
-		},
-		{
-			name: "subpath-mountdir-already-exists",
-			prepare: func(base string) ([]string, string, string, error) {
-				volpath, subpathMount := getTestPaths(base)
-				if err := os.MkdirAll(subpathMount, defaultPerm); err != nil {
-					return nil, "", "", err
-				}
-
-				subpath := filepath.Join(volpath, "dir0")
-				return nil, volpath, subpath, os.MkdirAll(subpath, defaultPerm)
-			},
-			expectError: false,
-		},
-		{
-			name: "subpath-mount-already-exists",
-			prepare: func(base string) ([]string, string, string, error) {
-				volpath, subpathMount := getTestPaths(base)
-				mounts := []string{subpathMount}
-				if err := os.MkdirAll(subpathMount, defaultPerm); err != nil {
-					return nil, "", "", err
-				}
-
-				subpath := filepath.Join(volpath, "dir0")
-				return mounts, volpath, subpath, os.MkdirAll(subpath, defaultPerm)
-			},
-			expectError: false,
-		},
-		{
-			name: "mount-unix-socket",
-			prepare: func(base string) ([]string, string, string, error) {
-				volpath, subpathMount := getTestPaths(base)
-				mounts := []string{subpathMount}
-				if err := os.MkdirAll(volpath, defaultPerm); err != nil {
-					return nil, "", "", err
-				}
-
-				socketFile, socketCreateError := createSocketFile(volpath)
-
-				return mounts, volpath, socketFile, socketCreateError
-			},
-			expectError: false,
-		},
-		{
-			name: "subpath-mounting-fifo",
-			prepare: func(base string) ([]string, string, string, error) {
-				volpath, subpathMount := getTestPaths(base)
-				mounts := []string{subpathMount}
-				if err := os.MkdirAll(volpath, defaultPerm); err != nil {
-					return nil, "", "", err
-				}
-
-				testFifo := filepath.Join(volpath, "mount_test.fifo")
-				err := syscall.Mkfifo(testFifo, 0)
-				return mounts, volpath, testFifo, err
-			},
-			expectError: false,
-		},
-	}
-
-	for _, test := range tests {
-		klog.V(4).Infof("test %q", test.name)
-		base, err := ioutil.TempDir("", "bind-subpath-"+test.name+"-")
-		if err != nil {
-			t.Fatalf(err.Error())
-		}
-
-		mounts, volPath, subPath, err := test.prepare(base)
-		if err != nil {
-			os.RemoveAll(base)
-			t.Fatalf("failed to prepare test %q: %v", test.name, err.Error())
-		}
-
-		fm := setupFakeMounter(mounts)
-
-		subpath := Subpath{
-			VolumeMountIndex: testSubpath,
-			Path:             subPath,
-			VolumeName:       testVol,
-			VolumePath:       volPath,
-			PodDir:           filepath.Join(base, "pod0"),
-			ContainerName:    testContainer,
-		}
-
-		_, subpathMount := getTestPaths(base)
-		bindPathTarget, err := doBindSubPath(fm, subpath)
-		if test.expectError {
-			if err == nil {
-				t.Errorf("test %q failed: expected error, got success", test.name)
-			}
-			if bindPathTarget != "" {
-				t.Errorf("test %q failed: expected empty bindPathTarget, got %v", test.name, bindPathTarget)
-			}
-			if err = validateDirNotExists(subpathMount); err != nil {
-				t.Errorf("test %q failed: %v", test.name, err)
-			}
-		}
-		if !test.expectError {
-			if err != nil {
-				t.Errorf("test %q failed: %v", test.name, err)
-			}
-			if bindPathTarget != subpathMount {
-				t.Errorf("test %q failed: expected bindPathTarget %v, got %v", test.name, subpathMount, bindPathTarget)
-			}
-			if err = validateFileExists(subpathMount); err != nil {
-				t.Errorf("test %q failed: %v", test.name, err)
-			}
-		}
-
-		os.RemoveAll(base)
-	}
-}
-
 func TestParseMountInfo(t *testing.T) {
 	info :=
 		`62 0 253:0 / / rw,relatime shared:1 - ext4 /dev/mapper/ssd-root rw,seclabel,data=ordered
@@ -1461,189 +641,6 @@ func TestGetSELinuxSupport(t *testing.T) {
 	}
 }
 
-func TestSafeOpen(t *testing.T) {
-	defaultPerm := os.FileMode(0750)
-
-	tests := []struct {
-		name string
-		// Function that prepares directory structure for the test under given
-		// base.
-		prepare     func(base string) error
-		path        string
-		expectError bool
-	}{
-		{
-			"directory-does-not-exist",
-			func(base string) error {
-				return nil
-			},
-			"test/directory",
-			true,
-		},
-		{
-			"directory-exists",
-			func(base string) error {
-				return os.MkdirAll(filepath.Join(base, "test/directory"), 0750)
-			},
-			"test/directory",
-			false,
-		},
-		{
-			"escape-base-using-dots",
-			func(base string) error {
-				return nil
-			},
-			"..",
-			true,
-		},
-		{
-			"escape-base-using-dots-2",
-			func(base string) error {
-				return os.MkdirAll(filepath.Join(base, "test"), 0750)
-			},
-			"test/../../..",
-			true,
-		},
-		{
-			"symlink",
-			func(base string) error {
-				if err := os.MkdirAll(filepath.Join(base, "destination"), defaultPerm); err != nil {
-					return err
-				}
-				return os.Symlink("destination", filepath.Join(base, "test"))
-			},
-			"test",
-			true,
-		},
-		{
-			"symlink-nested",
-			func(base string) error {
-				if err := os.MkdirAll(filepath.Join(base, "dir1/dir2"), defaultPerm); err != nil {
-					return err
-				}
-				return os.Symlink("dir1", filepath.Join(base, "dir1/dir2/test"))
-			},
-			"test",
-			true,
-		},
-		{
-			"symlink-loop",
-			func(base string) error {
-				return os.Symlink("test", filepath.Join(base, "test"))
-			},
-			"test",
-			true,
-		},
-		{
-			"symlink-not-exists",
-			func(base string) error {
-				return os.Symlink("non-existing", filepath.Join(base, "test"))
-			},
-			"test",
-			true,
-		},
-		{
-			"non-directory",
-			func(base string) error {
-				return ioutil.WriteFile(filepath.Join(base, "test"), []byte{}, defaultPerm)
-			},
-			"test/directory",
-			true,
-		},
-		{
-			"non-directory-final",
-			func(base string) error {
-				return ioutil.WriteFile(filepath.Join(base, "test"), []byte{}, defaultPerm)
-			},
-			"test",
-			false,
-		},
-		{
-			"escape-with-relative-symlink",
-			func(base string) error {
-				if err := os.MkdirAll(filepath.Join(base, "dir"), defaultPerm); err != nil {
-					return err
-				}
-				if err := os.MkdirAll(filepath.Join(base, "exists"), defaultPerm); err != nil {
-					return err
-				}
-				return os.Symlink("../exists", filepath.Join(base, "dir/test"))
-			},
-			"dir/test",
-			true,
-		},
-		{
-			"escape-with-relative-symlink-not-exists",
-			func(base string) error {
-				if err := os.MkdirAll(filepath.Join(base, "dir"), defaultPerm); err != nil {
-					return err
-				}
-				return os.Symlink("../not-exists", filepath.Join(base, "dir/test"))
-			},
-			"dir/test",
-			true,
-		},
-		{
-			"escape-with-symlink",
-			func(base string) error {
-				return os.Symlink("/", filepath.Join(base, "test"))
-			},
-			"test",
-			true,
-		},
-		{
-			"mount-unix-socket",
-			func(base string) error {
-				socketFile, socketError := createSocketFile(base)
-
-				if socketError != nil {
-					return fmt.Errorf("Error preparing socket file %s with %v", socketFile, socketError)
-				}
-				return nil
-			},
-			"mt.sock",
-			false,
-		},
-		{
-			"mounting-unix-socket-in-middle",
-			func(base string) error {
-				testSocketFile, socketError := createSocketFile(base)
-
-				if socketError != nil {
-					return fmt.Errorf("Error preparing socket file %s with %v", testSocketFile, socketError)
-				}
-				return nil
-			},
-			"mt.sock/bar",
-			true,
-		},
-	}
-
-	for _, test := range tests {
-		klog.V(4).Infof("test %q", test.name)
-		base, err := ioutil.TempDir("", "safe-open-"+test.name+"-")
-		if err != nil {
-			t.Fatalf(err.Error())
-		}
-
-		test.prepare(base)
-		pathToCreate := filepath.Join(base, test.path)
-		fd, err := doSafeOpen(pathToCreate, base)
-		if err != nil && !test.expectError {
-			t.Errorf("test %q: %s", test.name, err)
-		}
-		if err != nil {
-			klog.Infof("got error: %s", err)
-		}
-		if err == nil && test.expectError {
-			t.Errorf("test %q: expected error, got none", test.name)
-		}
-
-		syscall.Close(fd)
-		os.RemoveAll(base)
-	}
-}
-
 func createSocketFile(socketDir string) (string, error) {
 	testSocketFile := filepath.Join(socketDir, "mt.sock")
 
@@ -1663,139 +660,6 @@ func createSocketFile(socketDir string) (string, error) {
 	return testSocketFile, socketCreateError
 }
 
-func TestFindExistingPrefix(t *testing.T) {
-	defaultPerm := os.FileMode(0750)
-	tests := []struct {
-		name string
-		// Function that prepares directory structure for the test under given
-		// base.
-		prepare      func(base string) error
-		path         string
-		expectedPath string
-		expectedDirs []string
-		expectError  bool
-	}{
-		{
-			"directory-does-not-exist",
-			func(base string) error {
-				return nil
-			},
-			"directory",
-			"",
-			[]string{"directory"},
-			false,
-		},
-		{
-			"directory-exists",
-			func(base string) error {
-				return os.MkdirAll(filepath.Join(base, "test/directory"), 0750)
-			},
-			"test/directory",
-			"test/directory",
-			[]string{},
-			false,
-		},
-		{
-			"follow-symlinks",
-			func(base string) error {
-				if err := os.MkdirAll(filepath.Join(base, "destination/directory"), defaultPerm); err != nil {
-					return err
-				}
-				return os.Symlink("destination", filepath.Join(base, "test"))
-			},
-			"test/directory",
-			"test/directory",
-			[]string{},
-			false,
-		},
-		{
-			"follow-symlink-loop",
-			func(base string) error {
-				return os.Symlink("test", filepath.Join(base, "test"))
-			},
-			"test/directory",
-			"",
-			nil,
-			true,
-		},
-		{
-			"follow-symlink-multiple follow",
-			func(base string) error {
-				/* test1/dir points to test2 and test2/dir points to test1 */
-				if err := os.MkdirAll(filepath.Join(base, "test1"), defaultPerm); err != nil {
-					return err
-				}
-				if err := os.MkdirAll(filepath.Join(base, "test2"), defaultPerm); err != nil {
-					return err
-				}
-				if err := os.Symlink(filepath.Join(base, "test2"), filepath.Join(base, "test1/dir")); err != nil {
-					return err
-				}
-				if err := os.Symlink(filepath.Join(base, "test1"), filepath.Join(base, "test2/dir")); err != nil {
-					return err
-				}
-				return nil
-			},
-			"test1/dir/dir/foo/bar",
-			"test1/dir/dir",
-			[]string{"foo", "bar"},
-			false,
-		},
-		{
-			"danglink-symlink",
-			func(base string) error {
-				return os.Symlink("non-existing", filepath.Join(base, "test"))
-			},
-			// OS returns IsNotExist error both for dangling symlink and for
-			// non-existing directory.
-			"test/directory",
-			"",
-			[]string{"test", "directory"},
-			false,
-		},
-		{
-			"with-fifo-in-middle",
-			func(base string) error {
-				testFifo := filepath.Join(base, "mount_test.fifo")
-				return syscall.Mkfifo(testFifo, 0)
-			},
-			"mount_test.fifo/directory",
-			"",
-			nil,
-			true,
-		},
-	}
-
-	for _, test := range tests {
-		klog.V(4).Infof("test %q", test.name)
-		base, err := ioutil.TempDir("", "find-prefix-"+test.name+"-")
-		if err != nil {
-			t.Fatalf(err.Error())
-		}
-		test.prepare(base)
-		path := filepath.Join(base, test.path)
-		existingPath, dirs, err := findExistingPrefix(base, path)
-		if err != nil && !test.expectError {
-			t.Errorf("test %q: %s", test.name, err)
-		}
-		if err != nil {
-			klog.Infof("got error: %s", err)
-		}
-		if err == nil && test.expectError {
-			t.Errorf("test %q: expected error, got none", test.name)
-		}
-
-		fullExpectedPath := filepath.Join(base, test.expectedPath)
-		if existingPath != fullExpectedPath {
-			t.Errorf("test %q: expected path %q, got %q", test.name, fullExpectedPath, existingPath)
-		}
-		if !reflect.DeepEqual(dirs, test.expectedDirs) {
-			t.Errorf("test %q: expected dirs %v, got %v", test.name, test.expectedDirs, dirs)
-		}
-		os.RemoveAll(base)
-	}
-}
-
 func TestGetFileType(t *testing.T) {
 	mounter := Mounter{"fake/path", false}
 

pkg/util/mount/mount_unsupported.go

@@ -110,18 +110,6 @@ func (mounter *Mounter) EvalHostSymlinks(pathname string) (string, error) {
 	return "", unsupportedErr
 }
 
-func (mounter *Mounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
-	return subPath.Path, nil, unsupportedErr
-}
-
-func (mounter *Mounter) CleanSubPaths(podDir string, volumeName string) error {
-	return unsupportedErr
-}
-
-func (mounter *Mounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
-	return unsupportedErr
-}
-
 func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) {
 	return nil, errors.New("not implemented")
 }

pkg/util/mount/mount_windows.go

@@ -26,7 +26,6 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
-	"syscall"
 
 	"k8s.io/klog"
 
@@ -279,123 +278,6 @@ func (mounter *Mounter) EvalHostSymlinks(pathname string) (string, error) {
 	return filepath.EvalSymlinks(pathname)
 }
 
-// check whether hostPath is within volume path
-// this func will lock all intermediate subpath directories, need to close handle outside of this func after container started
-func lockAndCheckSubPath(volumePath, hostPath string) ([]uintptr, error) {
-	if len(volumePath) == 0 || len(hostPath) == 0 {
-		return []uintptr{}, nil
-	}
-
-	finalSubPath, err := filepath.EvalSymlinks(hostPath)
-	if err != nil {
-		return []uintptr{}, fmt.Errorf("cannot read link %s: %s", hostPath, err)
-	}
-	finalVolumePath, err := filepath.EvalSymlinks(volumePath)
-	if err != nil {
-		return []uintptr{}, fmt.Errorf("cannot read link %s: %s", volumePath, err)
-	}
-
-	return lockAndCheckSubPathWithoutSymlink(finalVolumePath, finalSubPath)
-}
-
-// lock all intermediate subPath directories and check they are all within volumePath
-// volumePath & subPath should not contain any symlink, otherwise it will return error
-func lockAndCheckSubPathWithoutSymlink(volumePath, subPath string) ([]uintptr, error) {
-	if len(volumePath) == 0 || len(subPath) == 0 {
-		return []uintptr{}, nil
-	}
-
-	// get relative path to volumePath
-	relSubPath, err := filepath.Rel(volumePath, subPath)
-	if err != nil {
-		return []uintptr{}, fmt.Errorf("Rel(%s, %s) error: %v", volumePath, subPath, err)
-	}
-	if startsWithBackstep(relSubPath) {
-		return []uintptr{}, fmt.Errorf("SubPath %q not within volume path %q", subPath, volumePath)
-	}
-
-	if relSubPath == "." {
-		// volumePath and subPath are equal
-		return []uintptr{}, nil
-	}
-
-	fileHandles := []uintptr{}
-	var errorResult error
-
-	currentFullPath := volumePath
-	dirs := strings.Split(relSubPath, string(os.PathSeparator))
-	for _, dir := range dirs {
-		// lock intermediate subPath directory first
-		currentFullPath = filepath.Join(currentFullPath, dir)
-		handle, err := lockPath(currentFullPath)
-		if err != nil {
-			errorResult = fmt.Errorf("cannot lock path %s: %s", currentFullPath, err)
-			break
-		}
-		fileHandles = append(fileHandles, handle)
-
-		// make sure intermediate subPath directory does not contain symlink any more
-		stat, err := os.Lstat(currentFullPath)
-		if err != nil {
-			errorResult = fmt.Errorf("Lstat(%q) error: %v", currentFullPath, err)
-			break
-		}
-		if stat.Mode()&os.ModeSymlink != 0 {
-			errorResult = fmt.Errorf("subpath %q is an unexpected symlink after EvalSymlinks", currentFullPath)
-			break
-		}
-
-		if !PathWithinBase(currentFullPath, volumePath) {
-			errorResult = fmt.Errorf("SubPath %q not within volume path %q", currentFullPath, volumePath)
-			break
-		}
-	}
-
-	return fileHandles, errorResult
-}
-
-// unlockPath unlock directories
-func unlockPath(fileHandles []uintptr) {
-	if fileHandles != nil {
-		for _, handle := range fileHandles {
-			syscall.CloseHandle(syscall.Handle(handle))
-		}
-	}
-}
-
-// lockPath locks a directory or symlink, return handle, exec "syscall.CloseHandle(handle)" to unlock the path
-func lockPath(path string) (uintptr, error) {
-	if len(path) == 0 {
-		return uintptr(syscall.InvalidHandle), syscall.ERROR_FILE_NOT_FOUND
-	}
-	pathp, err := syscall.UTF16PtrFromString(path)
-	if err != nil {
-		return uintptr(syscall.InvalidHandle), err
-	}
-	access := uint32(syscall.GENERIC_READ)
-	sharemode := uint32(syscall.FILE_SHARE_READ)
-	createmode := uint32(syscall.OPEN_EXISTING)
-	flags := uint32(syscall.FILE_FLAG_BACKUP_SEMANTICS | syscall.FILE_FLAG_OPEN_REPARSE_POINT)
-	fd, err := syscall.CreateFile(pathp, access, sharemode, nil, createmode, flags, 0)
-	return uintptr(fd), err
-}
-
-// Lock all directories in subPath and check they're not symlinks.
-func (mounter *Mounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
-	handles, err := lockAndCheckSubPath(subPath.VolumePath, subPath.Path)
-
-	// Unlock the directories when the container starts
-	cleanupAction = func() {
-		unlockPath(handles)
-	}
-	return subPath.Path, cleanupAction, err
-}
-
-// No bind-mounts for subpaths are necessary on Windows
-func (mounter *Mounter) CleanSubPaths(podDir string, volumeName string) error {
-	return nil
-}
-
 func (mounter *SafeFormatAndMount) formatAndMount(source string, target string, fstype string, options []string) error {
 	// Try to mount the disk
 	klog.V(4).Infof("Attempting to formatAndMount disk: %s %s %s", fstype, source, target)
@@ -522,126 +404,3 @@ func (mounter *Mounter) GetMode(pathname string) (os.FileMode, error) {
 	}
 	return info.Mode(), nil
 }
-
-// SafeMakeDir makes sure that the created directory does not escape given base directory mis-using symlinks.
-func (mounter *Mounter) SafeMakeDir(subdir string, base string, perm os.FileMode) error {
-	realBase, err := filepath.EvalSymlinks(base)
-	if err != nil {
-		return fmt.Errorf("error resolving symlinks in %s: %s", base, err)
-	}
-
-	realFullPath := filepath.Join(realBase, subdir)
-	return doSafeMakeDir(realFullPath, realBase, perm)
-}
-
-func doSafeMakeDir(pathname string, base string, perm os.FileMode) error {
-	klog.V(4).Infof("Creating directory %q within base %q", pathname, base)
-
-	if !PathWithinBase(pathname, base) {
-		return fmt.Errorf("path %s is outside of allowed base %s", pathname, base)
-	}
-
-	// Quick check if the directory already exists
-	s, err := os.Stat(pathname)
-	if err == nil {
-		// Path exists
-		if s.IsDir() {
-			// The directory already exists. It can be outside of the parent,
-			// but there is no race-proof check.
-			klog.V(4).Infof("Directory %s already exists", pathname)
-			return nil
-		}
-		return &os.PathError{Op: "mkdir", Path: pathname, Err: syscall.ENOTDIR}
-	}
-
-	// Find all existing directories
-	existingPath, toCreate, err := findExistingPrefix(base, pathname)
-	if err != nil {
-		return fmt.Errorf("error opening directory %s: %s", pathname, err)
-	}
-	if len(toCreate) == 0 {
-		return nil
-	}
-
-	// Ensure the existing directory is inside allowed base
-	fullExistingPath, err := filepath.EvalSymlinks(existingPath)
-	if err != nil {
-		return fmt.Errorf("error opening existing directory %s: %s", existingPath, err)
-	}
-	fullBasePath, err := filepath.EvalSymlinks(base)
-	if err != nil {
-		return fmt.Errorf("cannot read link %s: %s", base, err)
-	}
-	if !PathWithinBase(fullExistingPath, fullBasePath) {
-		return fmt.Errorf("path %s is outside of allowed base %s", fullExistingPath, err)
-	}
-
-	// lock all intermediate directories from fullBasePath to fullExistingPath (top to bottom)
-	fileHandles, err := lockAndCheckSubPathWithoutSymlink(fullBasePath, fullExistingPath)
-	defer unlockPath(fileHandles)
-	if err != nil {
-		return err
-	}
-
-	klog.V(4).Infof("%q already exists, %q to create", fullExistingPath, filepath.Join(toCreate...))
-	currentPath := fullExistingPath
-	// create the directories one by one, making sure nobody can change
-	// created directory into symlink by lock that directory immediately
-	for _, dir := range toCreate {
-		currentPath = filepath.Join(currentPath, dir)
-		klog.V(4).Infof("Creating %s", dir)
-		if err := os.Mkdir(currentPath, perm); err != nil {
-			return fmt.Errorf("cannot create directory %s: %s", currentPath, err)
-		}
-		handle, err := lockPath(currentPath)
-		if err != nil {
-			return fmt.Errorf("cannot lock path %s: %s", currentPath, err)
-		}
-		defer syscall.CloseHandle(syscall.Handle(handle))
-		// make sure newly created directory does not contain symlink after lock
-		stat, err := os.Lstat(currentPath)
-		if err != nil {
-			return fmt.Errorf("Lstat(%q) error: %v", currentPath, err)
-		}
-		if stat.Mode()&os.ModeSymlink != 0 {
-			return fmt.Errorf("subpath %q is an unexpected symlink after Mkdir", currentPath)
-		}
-	}
-
-	return nil
-}
-
-// findExistingPrefix finds prefix of pathname that exists. In addition, it
-// returns list of remaining directories that don't exist yet.
-func findExistingPrefix(base, pathname string) (string, []string, error) {
-	rel, err := filepath.Rel(base, pathname)
-	if err != nil {
-		return base, nil, err
-	}
-
-	if startsWithBackstep(rel) {
-		return base, nil, fmt.Errorf("pathname(%s) is not within base(%s)", pathname, base)
-	}
-
-	if rel == "." {
-		// base and pathname are equal
-		return pathname, []string{}, nil
-	}
-
-	dirs := strings.Split(rel, string(filepath.Separator))
-
-	parent := base
-	currentPath := base
-	for i, dir := range dirs {
-		parent = currentPath
-		currentPath = filepath.Join(parent, dir)
-		if _, err := os.Lstat(currentPath); err != nil {
-			if os.IsNotExist(err) {
-				return parent, dirs[i:], nil
-			}
-			return base, nil, err
-		}
-	}
-
-	return pathname, []string{}, nil
-}

pkg/util/mount/mount_windows_test.go

@@ -134,409 +134,6 @@ func TestGetMountRefs(t *testing.T) {
 	}
 }
 
-func TestDoSafeMakeDir(t *testing.T) {
-	base, err := ioutil.TempDir("", "TestDoSafeMakeDir")
-	if err != nil {
-		t.Fatalf(err.Error())
-	}
-
-	defer os.RemoveAll(base)
-
-	testingVolumePath := filepath.Join(base, "testingVolumePath")
-	os.MkdirAll(testingVolumePath, 0755)
-	defer os.RemoveAll(testingVolumePath)
-
-	tests := []struct {
-		volumePath    string
-		subPath       string
-		expectError   bool
-		symlinkTarget string
-	}{
-		{
-			volumePath:    testingVolumePath,
-			subPath:       ``,
-			expectError:   true,
-			symlinkTarget: "",
-		},
-		{
-			volumePath:    testingVolumePath,
-			subPath:       filepath.Join(testingVolumePath, `x`),
-			expectError:   false,
-			symlinkTarget: "",
-		},
-		{
-			volumePath:    testingVolumePath,
-			subPath:       filepath.Join(testingVolumePath, `a\b\c\d`),
-			expectError:   false,
-			symlinkTarget: "",
-		},
-		{
-			volumePath:    testingVolumePath,
-			subPath:       filepath.Join(testingVolumePath, `symlink`),
-			expectError:   false,
-			symlinkTarget: base,
-		},
-		{
-			volumePath:    testingVolumePath,
-			subPath:       filepath.Join(testingVolumePath, `symlink\c\d`),
-			expectError:   true,
-			symlinkTarget: "",
-		},
-		{
-			volumePath:    testingVolumePath,
-			subPath:       filepath.Join(testingVolumePath, `symlink\y926`),
-			expectError:   true,
-			symlinkTarget: "",
-		},
-		{
-			volumePath:    testingVolumePath,
-			subPath:       filepath.Join(testingVolumePath, `a\b\symlink`),
-			expectError:   false,
-			symlinkTarget: base,
-		},
-		{
-			volumePath:    testingVolumePath,
-			subPath:       filepath.Join(testingVolumePath, `a\x\symlink`),
-			expectError:   false,
-			symlinkTarget: filepath.Join(testingVolumePath, `a`),
-		},
-	}
-
-	for _, test := range tests {
-		if len(test.volumePath) > 0 && len(test.subPath) > 0 && len(test.symlinkTarget) > 0 {
-			// make all parent sub directories
-			if parent := filepath.Dir(test.subPath); parent != "." {
-				os.MkdirAll(parent, 0755)
-			}
-
-			// make last element as symlink
-			linkPath := test.subPath
-			if _, err := os.Stat(linkPath); err != nil && os.IsNotExist(err) {
-				if err := makeLink(linkPath, test.symlinkTarget); err != nil {
-					t.Fatalf("unexpected error: %v", fmt.Errorf("mklink link(%q) target(%q) error: %q", linkPath, test.symlinkTarget, err))
-				}
-			}
-		}
-
-		err := doSafeMakeDir(test.subPath, test.volumePath, os.FileMode(0755))
-		if test.expectError {
-			assert.NotNil(t, err, "Expect error during doSafeMakeDir(%s, %s)", test.subPath, test.volumePath)
-			continue
-		}
-		assert.Nil(t, err, "Expect no error during doSafeMakeDir(%s, %s)", test.subPath, test.volumePath)
-		if _, err := os.Stat(test.subPath); os.IsNotExist(err) {
-			t.Errorf("subPath should exists after doSafeMakeDir(%s, %s)", test.subPath, test.volumePath)
-		}
-	}
-}
-
-func TestLockAndCheckSubPath(t *testing.T) {
-	base, err := ioutil.TempDir("", "TestLockAndCheckSubPath")
-	if err != nil {
-		t.Fatalf(err.Error())
-	}
-
-	defer os.RemoveAll(base)
-
-	testingVolumePath := filepath.Join(base, "testingVolumePath")
-
-	tests := []struct {
-		volumePath          string
-		subPath             string
-		expectedHandleCount int
-		expectError         bool
-		symlinkTarget       string
-	}{
-		{
-			volumePath:          `c:\`,
-			subPath:             ``,
-			expectedHandleCount: 0,
-			expectError:         false,
-			symlinkTarget:       "",
-		},
-		{
-			volumePath:          ``,
-			subPath:             `a`,
-			expectedHandleCount: 0,
-			expectError:         false,
-			symlinkTarget:       "",
-		},
-		{
-			volumePath:          testingVolumePath,
-			subPath:             filepath.Join(testingVolumePath, `a`),
-			expectedHandleCount: 1,
-			expectError:         false,
-			symlinkTarget:       "",
-		},
-		{
-			volumePath:          testingVolumePath,
-			subPath:             filepath.Join(testingVolumePath, `a\b\c\d`),
-			expectedHandleCount: 4,
-			expectError:         false,
-			symlinkTarget:       "",
-		},
-		{
-			volumePath:          testingVolumePath,
-			subPath:             filepath.Join(testingVolumePath, `symlink`),
-			expectedHandleCount: 0,
-			expectError:         true,
-			symlinkTarget:       base,
-		},
-		{
-			volumePath:          testingVolumePath,
-			subPath:             filepath.Join(testingVolumePath, `a\b\c\symlink`),
-			expectedHandleCount: 0,
-			expectError:         true,
-			symlinkTarget:       base,
-		},
-		{
-			volumePath:          testingVolumePath,
-			subPath:             filepath.Join(testingVolumePath, `a\b\c\d\symlink`),
-			expectedHandleCount: 2,
-			expectError:         false,
-			symlinkTarget:       filepath.Join(testingVolumePath, `a\b`),
-		},
-	}
-
-	for _, test := range tests {
-		if len(test.volumePath) > 0 && len(test.subPath) > 0 {
-			os.MkdirAll(test.volumePath, 0755)
-			if len(test.symlinkTarget) == 0 {
-				// make all intermediate sub directories
-				os.MkdirAll(test.subPath, 0755)
-			} else {
-				// make all parent sub directories
-				if parent := filepath.Dir(test.subPath); parent != "." {
-					os.MkdirAll(parent, 0755)
-				}
-
-				// make last element as symlink
-				linkPath := test.subPath
-				if _, err := os.Stat(linkPath); err != nil && os.IsNotExist(err) {
-					if err := makeLink(linkPath, test.symlinkTarget); err != nil {
-						t.Fatalf("unexpected error: %v", fmt.Errorf("mklink link(%q) target(%q) error: %q", linkPath, test.symlinkTarget, err))
-					}
-				}
-			}
-		}
-
-		fileHandles, err := lockAndCheckSubPath(test.volumePath, test.subPath)
-		unlockPath(fileHandles)
-		assert.Equal(t, test.expectedHandleCount, len(fileHandles))
-		if test.expectError {
-			assert.NotNil(t, err, "Expect error during LockAndCheckSubPath(%s, %s)", test.volumePath, test.subPath)
-			continue
-		}
-		assert.Nil(t, err, "Expect no error during LockAndCheckSubPath(%s, %s)", test.volumePath, test.subPath)
-	}
-
-	// remove dir will happen after closing all file handles
-	assert.Nil(t, os.RemoveAll(testingVolumePath), "Expect no error during remove dir %s", testingVolumePath)
-}
-
-func TestLockAndCheckSubPathWithoutSymlink(t *testing.T) {
-	base, err := ioutil.TempDir("", "TestLockAndCheckSubPathWithoutSymlink")
-	if err != nil {
-		t.Fatalf(err.Error())
-	}
-
-	defer os.RemoveAll(base)
-
-	testingVolumePath := filepath.Join(base, "testingVolumePath")
-
-	tests := []struct {
-		volumePath          string
-		subPath             string
-		expectedHandleCount int
-		expectError         bool
-		symlinkTarget       string
-	}{
-		{
-			volumePath:          `c:\`,
-			subPath:             ``,
-			expectedHandleCount: 0,
-			expectError:         false,
-			symlinkTarget:       "",
-		},
-		{
-			volumePath:          ``,
-			subPath:             `a`,
-			expectedHandleCount: 0,
-			expectError:         false,
-			symlinkTarget:       "",
-		},
-		{
-			volumePath:          testingVolumePath,
-			subPath:             filepath.Join(testingVolumePath, `a`),
-			expectedHandleCount: 1,
-			expectError:         false,
-			symlinkTarget:       "",
-		},
-		{
-			volumePath:          testingVolumePath,
-			subPath:             filepath.Join(testingVolumePath, `a\b\c\d`),
-			expectedHandleCount: 4,
-			expectError:         false,
-			symlinkTarget:       "",
-		},
-		{
-			volumePath:          testingVolumePath,
-			subPath:             filepath.Join(testingVolumePath, `symlink`),
-			expectedHandleCount: 1,
-			expectError:         true,
-			symlinkTarget:       base,
-		},
-		{
-			volumePath:          testingVolumePath,
-			subPath:             filepath.Join(testingVolumePath, `a\b\c\symlink`),
-			expectedHandleCount: 4,
-			expectError:         true,
-			symlinkTarget:       base,
-		},
-		{
-			volumePath:          testingVolumePath,
-			subPath:             filepath.Join(testingVolumePath, `a\b\c\d\symlink`),
-			expectedHandleCount: 5,
-			expectError:         true,
-			symlinkTarget:       filepath.Join(testingVolumePath, `a\b`),
-		},
-	}
-
-	for _, test := range tests {
-		if len(test.volumePath) > 0 && len(test.subPath) > 0 {
-			os.MkdirAll(test.volumePath, 0755)
-			if len(test.symlinkTarget) == 0 {
-				// make all intermediate sub directories
-				os.MkdirAll(test.subPath, 0755)
-			} else {
-				// make all parent sub directories
-				if parent := filepath.Dir(test.subPath); parent != "." {
-					os.MkdirAll(parent, 0755)
-				}
-
-				// make last element as symlink
-				linkPath := test.subPath
-				if _, err := os.Stat(linkPath); err != nil && os.IsNotExist(err) {
-					if err := makeLink(linkPath, test.symlinkTarget); err != nil {
-						t.Fatalf("unexpected error: %v", fmt.Errorf("mklink link(%q) target(%q) error: %q", linkPath, test.symlinkTarget, err))
-					}
-				}
-			}
-		}
-
-		fileHandles, err := lockAndCheckSubPathWithoutSymlink(test.volumePath, test.subPath)
-		unlockPath(fileHandles)
-		assert.Equal(t, test.expectedHandleCount, len(fileHandles))
-		if test.expectError {
-			assert.NotNil(t, err, "Expect error during LockAndCheckSubPath(%s, %s)", test.volumePath, test.subPath)
-			continue
-		}
-		assert.Nil(t, err, "Expect no error during LockAndCheckSubPath(%s, %s)", test.volumePath, test.subPath)
-	}
-
-	// remove dir will happen after closing all file handles
-	assert.Nil(t, os.RemoveAll(testingVolumePath), "Expect no error during remove dir %s", testingVolumePath)
-}
-
-func TestFindExistingPrefix(t *testing.T) {
-	base, err := ioutil.TempDir("", "TestFindExistingPrefix")
-	if err != nil {
-		t.Fatalf(err.Error())
-	}
-
-	defer os.RemoveAll(base)
-
-	testingVolumePath := filepath.Join(base, "testingVolumePath")
-
-	tests := []struct {
-		base                    string
-		pathname                string
-		expectError             bool
-		expectedExistingPath    string
-		expectedToCreateDirs    []string
-		createSubPathBeforeTest bool
-	}{
-		{
-			base:                    `c:\tmp\a`,
-			pathname:                `c:\tmp\b`,
-			expectError:             true,
-			expectedExistingPath:    "",
-			expectedToCreateDirs:    []string{},
-			createSubPathBeforeTest: false,
-		},
-		{
-			base:                    ``,
-			pathname:                `c:\tmp\b`,
-			expectError:             true,
-			expectedExistingPath:    "",
-			expectedToCreateDirs:    []string{},
-			createSubPathBeforeTest: false,
-		},
-		{
-			base:                    `c:\tmp\a`,
-			pathname:                `d:\tmp\b`,
-			expectError:             true,
-			expectedExistingPath:    "",
-			expectedToCreateDirs:    []string{},
-			createSubPathBeforeTest: false,
-		},
-		{
-			base:                    testingVolumePath,
-			pathname:                testingVolumePath,
-			expectError:             false,
-			expectedExistingPath:    testingVolumePath,
-			expectedToCreateDirs:    []string{},
-			createSubPathBeforeTest: false,
-		},
-		{
-			base:                    testingVolumePath,
-			pathname:                filepath.Join(testingVolumePath, `a\b`),
-			expectError:             false,
-			expectedExistingPath:    filepath.Join(testingVolumePath, `a\b`),
-			expectedToCreateDirs:    []string{},
-			createSubPathBeforeTest: true,
-		},
-		{
-			base:                    testingVolumePath,
-			pathname:                filepath.Join(testingVolumePath, `a\b\c\`),
-			expectError:             false,
-			expectedExistingPath:    filepath.Join(testingVolumePath, `a\b`),
-			expectedToCreateDirs:    []string{`c`},
-			createSubPathBeforeTest: false,
-		},
-		{
-			base:                    testingVolumePath,
-			pathname:                filepath.Join(testingVolumePath, `a\b\c\d`),
-			expectError:             false,
-			expectedExistingPath:    filepath.Join(testingVolumePath, `a\b`),
-			expectedToCreateDirs:    []string{`c`, `d`},
-			createSubPathBeforeTest: false,
-		},
-	}
-
-	for _, test := range tests {
-		if test.createSubPathBeforeTest {
-			os.MkdirAll(test.pathname, 0755)
-		}
-
-		existingPath, toCreate, err := findExistingPrefix(test.base, test.pathname)
-		if test.expectError {
-			assert.NotNil(t, err, "Expect error during findExistingPrefix(%s, %s)", test.base, test.pathname)
-			continue
-		}
-		assert.Nil(t, err, "Expect no error during findExistingPrefix(%s, %s)", test.base, test.pathname)
-
-		assert.Equal(t, test.expectedExistingPath, existingPath, "Expect result not equal with findExistingPrefix(%s, %s) return: %q, expected: %q",
-			test.base, test.pathname, existingPath, test.expectedExistingPath)
-
-		assert.Equal(t, test.expectedToCreateDirs, toCreate, "Expect result not equal with findExistingPrefix(%s, %s) return: %q, expected: %q",
-			test.base, test.pathname, toCreate, test.expectedToCreateDirs)
-
-	}
-	// remove dir will happen after closing all file handles
-	assert.Nil(t, os.RemoveAll(testingVolumePath), "Expect no error during remove dir %s", testingVolumePath)
-}
-
 func TestPathWithinBase(t *testing.T) {
 	tests := []struct {
 		fullPath       string

pkg/util/mount/nsenter_mount.go

@@ -23,9 +23,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
-	"syscall"
 
-	"golang.org/x/sys/unix"
 	"k8s.io/klog"
 	"k8s.io/utils/nsenter"
 	utilpath "k8s.io/utils/path"
@@ -298,51 +296,6 @@ func (mounter *NsenterMounter) EvalHostSymlinks(pathname string) (string, error)
 	return mounter.ne.EvalSymlinks(pathname, true)
 }
 
-func (mounter *NsenterMounter) CleanSubPaths(podDir string, volumeName string) error {
-	return doCleanSubPaths(mounter, podDir, volumeName)
-}
-
-func (mounter *NsenterMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
-	// Bind-mount the subpath to avoid using symlinks in subpaths.
-	newHostPath, err = doNsEnterBindSubPath(mounter, subPath)
-
-	// There is no action when the container starts. Bind-mount will be cleaned
-	// when container stops by CleanSubPaths.
-	cleanupAction = nil
-	return newHostPath, cleanupAction, err
-}
-
-func (mounter *NsenterMounter) SafeMakeDir(subdir string, base string, perm os.FileMode) error {
-	fullSubdirPath := filepath.Join(base, subdir)
-	evaluatedSubdirPath, err := mounter.ne.EvalSymlinks(fullSubdirPath, false /* mustExist */)
-	if err != nil {
-		return fmt.Errorf("error resolving symlinks in %s: %s", fullSubdirPath, err)
-	}
-	evaluatedSubdirPath = filepath.Clean(evaluatedSubdirPath)
-
-	evaluatedBase, err := mounter.ne.EvalSymlinks(base, true /* mustExist */)
-	if err != nil {
-		return fmt.Errorf("error resolving symlinks in %s: %s", base, err)
-	}
-	evaluatedBase = filepath.Clean(evaluatedBase)
-
-	rootDir := filepath.Clean(mounter.rootDir)
-	if PathWithinBase(evaluatedBase, rootDir) {
-		// Base is in /var/lib/kubelet. This directory is shared between the
-		// container with kubelet and the host. We don't need to add '/rootfs'.
-		// This is useful when /rootfs is mounted as read-only - we can still
-		// create subpaths for paths in /var/lib/kubelet.
-		return doSafeMakeDir(evaluatedSubdirPath, evaluatedBase, perm)
-	}
-
-	// Base is somewhere on the host's filesystem. Add /rootfs and try to make
-	// the directory there.
-	// This requires /rootfs to be writable.
-	kubeletSubdirPath := mounter.ne.KubeletPath(evaluatedSubdirPath)
-	kubeletBase := mounter.ne.KubeletPath(evaluatedBase)
-	return doSafeMakeDir(kubeletSubdirPath, kubeletBase, perm)
-}
-
 func (mounter *NsenterMounter) GetMountRefs(pathname string) ([]string, error) {
 	exists, err := mounter.ExistsPath(pathname)
 	if err != nil {
@@ -358,95 +311,6 @@ func (mounter *NsenterMounter) GetMountRefs(pathname string) ([]string, error) {
 	return searchMountPoints(hostpath, hostProcMountinfoPath)
 }
 
-func doNsEnterBindSubPath(mounter *NsenterMounter, subpath Subpath) (hostPath string, err error) {
-	// Linux, kubelet runs in a container:
-	// - safely open the subpath
-	// - bind-mount the subpath to target (this can be unsafe)
-	// - check that we mounted the right thing by comparing device ID and inode
-	//   of the subpath (via safely opened fd) and the target (that's under our
-	//   control)
-
-	// Evaluate all symlinks here once for all subsequent functions.
-	evaluatedHostVolumePath, err := mounter.ne.EvalSymlinks(subpath.VolumePath, true /*mustExist*/)
-	if err != nil {
-		return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.VolumePath, err)
-	}
-	evaluatedHostSubpath, err := mounter.ne.EvalSymlinks(subpath.Path, true /*mustExist*/)
-	if err != nil {
-		return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.Path, err)
-	}
-	klog.V(5).Infof("doBindSubPath %q (%q) for volumepath %q", subpath.Path, evaluatedHostSubpath, subpath.VolumePath)
-	subpath.VolumePath = mounter.ne.KubeletPath(evaluatedHostVolumePath)
-	subpath.Path = mounter.ne.KubeletPath(evaluatedHostSubpath)
-
-	// Check the subpath is correct and open it
-	fd, err := safeOpenSubPath(mounter, subpath)
-	if err != nil {
-		return "", err
-	}
-	defer syscall.Close(fd)
-
-	alreadyMounted, bindPathTarget, err := prepareSubpathTarget(mounter, subpath)
-	if err != nil {
-		return "", err
-	}
-	if alreadyMounted {
-		return bindPathTarget, nil
-	}
-
-	success := false
-	defer func() {
-		// Cleanup subpath on error
-		if !success {
-			klog.V(4).Infof("doNsEnterBindSubPath() failed for %q, cleaning up subpath", bindPathTarget)
-			if cleanErr := cleanSubPath(mounter, subpath); cleanErr != nil {
-				klog.Errorf("Failed to clean subpath %q: %v", bindPathTarget, cleanErr)
-			}
-		}
-	}()
-
-	// Leap of faith: optimistically expect that nobody has modified previously
-	// expanded evalSubPath with evil symlinks and bind-mount it.
-	// Mount is done on the host! don't use kubelet path!
-	klog.V(5).Infof("bind mounting %q at %q", evaluatedHostSubpath, bindPathTarget)
-	if err = mounter.Mount(evaluatedHostSubpath, bindPathTarget, "" /*fstype*/, []string{"bind"}); err != nil {
-		return "", fmt.Errorf("error mounting %s: %s", evaluatedHostSubpath, err)
-	}
-
-	// Check that the bind-mount target is the same inode and device as the
-	// source that we keept open, i.e. we mounted the right thing.
-	err = checkDeviceInode(fd, bindPathTarget)
-	if err != nil {
-		return "", fmt.Errorf("error checking bind mount for subpath %s: %s", subpath.VolumePath, err)
-	}
-
-	success = true
-	klog.V(3).Infof("Bound SubPath %s into %s", subpath.Path, bindPathTarget)
-	return bindPathTarget, nil
-}
-
-// checkDeviceInode checks that opened file and path represent the same file.
-func checkDeviceInode(fd int, path string) error {
-	var srcStat, dstStat unix.Stat_t
-	err := unix.Fstat(fd, &srcStat)
-	if err != nil {
-		return fmt.Errorf("error running fstat on subpath FD: %v", err)
-	}
-
-	err = unix.Stat(path, &dstStat)
-	if err != nil {
-		return fmt.Errorf("error running fstat on %s: %v", path, err)
-	}
-
-	if srcStat.Dev != dstStat.Dev {
-		return fmt.Errorf("different device number")
-	}
-	if srcStat.Ino != dstStat.Ino {
-		return fmt.Errorf("different inode")
-	}
-	return nil
-}
-
 func (mounter *NsenterMounter) GetFSGroup(pathname string) (int64, error) {
 	hostPath, err := mounter.ne.EvalSymlinks(pathname, true /* mustExist */)
 	if err != nil {

pkg/util/mount/nsenter_mount_test.go

@@ -23,10 +23,8 @@ import (
 	"os"
 	"os/user"
 	"path/filepath"
-	"strings"
 	"testing"
 
-	"golang.org/x/sys/unix"
 	"k8s.io/utils/nsenter"
 )
 
@@ -76,80 +74,6 @@ func TestParseFindMnt(t *testing.T) {
 	}
 }
 
-func TestCheckDeviceInode(t *testing.T) {
-	testDir, err := ioutil.TempDir("", "nsenter-mounter-device-")
-	if err != nil {
-		t.Fatalf("Cannot create temporary directory: %s", err)
-	}
-	defer os.RemoveAll(testDir)
-
-	tests := []struct {
-		name        string
-		srcPath     string
-		dstPath     string
-		expectError string
-	}{
-		{
-			name:        "the same file",
-			srcPath:     filepath.Join(testDir, "1"),
-			dstPath:     filepath.Join(testDir, "1"),
-			expectError: "",
-		},
-		{
-			name:        "different file on the same FS",
-			srcPath:     filepath.Join(testDir, "2.1"),
-			dstPath:     filepath.Join(testDir, "2.2"),
-			expectError: "different inode",
-		},
-		{
-			name:    "different file on different device",
-			srcPath: filepath.Join(testDir, "3"),
-			// /proc is always on a different "device" than /tmp (or $TEMP)
-			dstPath:     "/proc/self/status",
-			expectError: "different device",
-		},
-	}
-
-	for _, test := range tests {
-		if err := ioutil.WriteFile(test.srcPath, []byte{}, 0644); err != nil {
-			t.Errorf("Test %q: cannot create srcPath %s: %s", test.name, test.srcPath, err)
-			continue
-		}
-
-		// Don't create dst if it exists
-		if _, err := os.Stat(test.dstPath); os.IsNotExist(err) {
-			if err := ioutil.WriteFile(test.dstPath, []byte{}, 0644); err != nil {
-				t.Errorf("Test %q: cannot create dstPath %s: %s", test.name, test.dstPath, err)
-				continue
-			}
-		} else if err != nil {
-			t.Errorf("Test %q: cannot check existence of dstPath %s: %s", test.name, test.dstPath, err)
-			continue
-		}
-
-		fd, err := unix.Open(test.srcPath, unix.O_CREAT, 0644)
-		if err != nil {
-			t.Errorf("Test %q: cannot open srcPath %s: %s", test.name, test.srcPath, err)
-			continue
-		}
-
-		err = checkDeviceInode(fd, test.dstPath)
-
-		if test.expectError == "" && err != nil {
-			t.Errorf("Test %q: expected no error, got %s", test.name, err)
-		}
-		if test.expectError != "" {
-			if err == nil {
-				t.Errorf("Test %q: expected error, got none", test.name)
-			} else {
-				if !strings.Contains(err.Error(), test.expectError) {
-					t.Errorf("Test %q: expected error %q, got %q", test.name, test.expectError, err)
-				}
-			}
-		}
-	}
-}
-
 func newFakeNsenterMounter(tmpdir string, t *testing.T) (mounter *NsenterMounter, rootfsPath string, varlibPath string, err error) {
 	rootfsPath = filepath.Join(tmpdir, "rootfs")
 	if err := os.Mkdir(rootfsPath, 0755); err != nil {
@@ -504,221 +428,3 @@ func writeRootfsFile(rootfs, path string, mode os.FileMode) (string, error) {
 	}
 	return fullPath, nil
 }
-
-func TestNsenterSafeMakeDir(t *testing.T) {
-	tests := []struct {
-		name        string
-		prepare     func(base, rootfs, varlib string) (expectedDir string, err error)
-		subdir      string
-		expectError bool
-		// If true, "base" directory for SafeMakeDir will be /var/lib/kubelet
-		baseIsVarLib bool
-	}{
-		{
-			name: "simple directory",
-			// evaluated in base
-			subdir: "some/subdirectory/structure",
-			prepare: func(base, rootfs, varlib string) (expectedDir string, err error) {
-				// expected to be created in /roots/
-				expectedDir = filepath.Join(rootfs, base, "some/subdirectory/structure")
-				return expectedDir, nil
-			},
-		},
-		{
-			name: "simple existing directory",
-			// evaluated in base
-			subdir: "some/subdirectory/structure",
-			prepare: func(base, rootfs, varlib string) (expectedDir string, err error) {
-				// On the host: directory exists
-				hostPath := filepath.Join(base, "some/subdirectory/structure")
-				if err := os.MkdirAll(hostPath, 0755); err != nil {
-					return "", err
-				}
-				// In rootfs: directory exists
-				kubeletPath := filepath.Join(rootfs, hostPath)
-				if err := os.MkdirAll(kubeletPath, 0755); err != nil {
-					return "", err
-				}
-				// expected to be created in /roots/
-				expectedDir = kubeletPath
-				return expectedDir, nil
-			},
-		},
-		{
-			name: "absolute symlink into safe place",
-			// evaluated in base
-			subdir: "some/subdirectory/structure",
-			prepare: func(base, rootfs, varlib string) (expectedDir string, err error) {
-				// On the host: /base/other/subdirectory exists, /base/some is link to /base/other
-				hostPath := filepath.Join(base, "other/subdirectory")
-				if err := os.MkdirAll(hostPath, 0755); err != nil {
-					return "", err
-				}
-				somePath := filepath.Join(base, "some")
-				otherPath := filepath.Join(base, "other")
-				if err := os.Symlink(otherPath, somePath); err != nil {
-					return "", err
-				}
-
-				// In rootfs: /base/other/subdirectory exists
-				kubeletPath := filepath.Join(rootfs, hostPath)
-				if err := os.MkdirAll(kubeletPath, 0755); err != nil {
-					return "", err
-				}
-				// expected 'structure' to be created
-				expectedDir = filepath.Join(rootfs, hostPath, "structure")
-				return expectedDir, nil
-			},
-		},
-		{
-			name: "relative symlink into safe place",
-			// evaluated in base
-			subdir: "some/subdirectory/structure",
-			prepare: func(base, rootfs, varlib string) (expectedDir string, err error) {
-				// On the host: /base/other/subdirectory exists, /base/some is link to other
-				hostPath := filepath.Join(base, "other/subdirectory")
-				if err := os.MkdirAll(hostPath, 0755); err != nil {
-					return "", err
-				}
-				somePath := filepath.Join(base, "some")
-				if err := os.Symlink("other", somePath); err != nil {
-					return "", err
-				}
-
-				// In rootfs: /base/other/subdirectory exists
-				kubeletPath := filepath.Join(rootfs, hostPath)
-				if err := os.MkdirAll(kubeletPath, 0755); err != nil {
-					return "", err
-				}
-				// expected 'structure' to be created
-				expectedDir = filepath.Join(rootfs, hostPath, "structure")
-				return expectedDir, nil
-			},
-		},
-		{
-			name: "symlink into unsafe place",
-			// evaluated in base
-			subdir: "some/subdirectory/structure",
-			prepare: func(base, rootfs, varlib string) (expectedDir string, err error) {
-				// On the host: /base/some is link to /bin/other
-				somePath := filepath.Join(base, "some")
-				if err := os.Symlink("/bin", somePath); err != nil {
-					return "", err
-				}
-				return "", nil
-			},
-			expectError: true,
-		},
-		{
-			name: "simple directory in /var/lib/kubelet",
-			// evaluated in varlib
-			subdir:       "some/subdirectory/structure",
-			baseIsVarLib: true,
-			prepare: func(base, rootfs, varlib string) (expectedDir string, err error) {
-				// expected to be created in /base/var/lib/kubelet, not in /rootfs!
-				expectedDir = filepath.Join(varlib, "some/subdirectory/structure")
-				return expectedDir, nil
-			},
-		},
-		{
-			name: "safe symlink in /var/lib/kubelet",
-			// evaluated in varlib
-			subdir:       "some/subdirectory/structure",
-			baseIsVarLib: true,
-			prepare: func(base, rootfs, varlib string) (expectedDir string, err error) {
-				// On the host: /varlib/kubelet/other/subdirectory exists, /varlib/some is link to other
-				hostPath := filepath.Join(varlib, "other/subdirectory")
-				if err := os.MkdirAll(hostPath, 0755); err != nil {
-					return "", err
-				}
-				somePath := filepath.Join(varlib, "some")
-				if err := os.Symlink("other", somePath); err != nil {
-					return "", err
-				}
-
-				// expected to be created in /base/var/lib/kubelet, not in /rootfs!
-				expectedDir = filepath.Join(varlib, "other/subdirectory/structure")
-				return expectedDir, nil
-			},
-		},
-		{
-			name: "unsafe symlink in /var/lib/kubelet",
-			// evaluated in varlib
-			subdir:       "some/subdirectory/structure",
-			baseIsVarLib: true,
-			prepare: func(base, rootfs, varlib string) (expectedDir string, err error) {
-				// On the host: /varlib/some is link to /bin
-				somePath := filepath.Join(varlib, "some")
-				if err := os.Symlink("/bin", somePath); err != nil {
-					return "", err
-				}
-
-				return "", nil
-			},
-			expectError: true,
-		},
-	}
-	for _, test := range tests {
-		tmpdir, err := ioutil.TempDir("", "nsenter-get-mode-")
-		if err != nil {
-			t.Error(err)
-			continue
-		}
-		defer os.RemoveAll(tmpdir)
-
-		mounter, rootfs, varlib, err := newFakeNsenterMounter(tmpdir, t)
-		if err != nil {
-			t.Error(err)
-			continue
-		}
-		// Prepare base directory for the test
-		testBase := filepath.Join(tmpdir, "base")
-		if err := os.Mkdir(testBase, 0755); err != nil {
-			t.Error(err)
-			continue
-		}
-		// Prepare base directory also in /rootfs
-		rootfsBase := filepath.Join(rootfs, testBase)
-		if err := os.MkdirAll(rootfsBase, 0755); err != nil {
-			t.Error(err)
-			continue
-		}
-
-		expectedDir := ""
-		if test.prepare != nil {
-			expectedDir, err = test.prepare(testBase, rootfs, varlib)
-			if err != nil {
-				t.Error(err)
-				continue
-			}
-		}
-
-		if test.baseIsVarLib {
-			// use /var/lib/kubelet as the test base so we can test creating
-			// subdirs there directly in /var/lib/kubenet and not in
-			// /rootfs/var/lib/kubelet
-			testBase = varlib
-		}
-
-		err = mounter.SafeMakeDir(test.subdir, testBase, 0755)
-		if err != nil && !test.expectError {
-			t.Errorf("Test %q: unexpected error: %s", test.name, err)
-		}
-		if test.expectError {
-			if err == nil {
-				t.Errorf("Test %q: expected error, got none", test.name)
-			} else {
-				if !strings.Contains(err.Error(), "is outside of allowed base") {
-					t.Errorf("Test %q: expected error to contain \"is outside of allowed base\", got this one instead: %s", test.name, err)
-				}
-			}
-		}
-
-		if expectedDir != "" {
-			_, err := os.Stat(expectedDir)
-			if err != nil {
-				t.Errorf("Test %q: expected %q to exist, got error: %s", test.name, expectedDir, err)
-			}
-		}
-	}
-}

pkg/util/mount/nsenter_mount_unsupported.go

@@ -93,18 +93,6 @@ func (*NsenterMounter) EvalHostSymlinks(pathname string) (string, error) {
 	return "", errors.New("not implemented")
 }
 
-func (*NsenterMounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
-	return nil
-}
-
-func (*NsenterMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
-	return subPath.Path, nil, nil
-}
-
-func (*NsenterMounter) CleanSubPaths(podDir string, volumeName string) error {
-	return nil
-}
-
 func (*NsenterMounter) GetMountRefs(pathname string) ([]string, error) {
 	return nil, errors.New("not implemented")
 }

pkg/volume/BUILD

@@ -21,6 +21,7 @@ go_library(
         "//pkg/util/mount:go_default_library",
         "//pkg/volume/util/fs:go_default_library",
         "//pkg/volume/util/recyclerclient:go_default_library",
+        "//pkg/volume/util/subpath:go_default_library",
         "//staging/src/k8s.io/api/authentication/v1:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",

pkg/volume/plugins.go

@@ -36,6 +36,7 @@ import (
 	"k8s.io/klog"
 	"k8s.io/kubernetes/pkg/util/mount"
 	"k8s.io/kubernetes/pkg/volume/util/recyclerclient"
+	"k8s.io/kubernetes/pkg/volume/util/subpath"
 )
 
 type ProbeOperation uint32
@@ -367,6 +368,9 @@ type VolumeHost interface {
 
 	// Returns the event recorder of kubelet.
 	GetEventRecorder() record.EventRecorder
+
+	// Returns an interface that should be used to execute subpath operations
+	GetSubpather() subpath.Interface
 }
 
 // VolumePluginMgr tracks registered plugins.

pkg/volume/testing/BUILD

@@ -17,6 +17,7 @@ go_library(
         "//pkg/volume:go_default_library",
         "//pkg/volume/util:go_default_library",
         "//pkg/volume/util/recyclerclient:go_default_library",
+        "//pkg/volume/util/subpath:go_default_library",
         "//pkg/volume/util/volumepathhandler:go_default_library",
         "//staging/src/k8s.io/api/authentication/v1:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",

pkg/volume/testing/testing.go

@@ -42,6 +42,7 @@ import (
 	. "k8s.io/kubernetes/pkg/volume"
 	"k8s.io/kubernetes/pkg/volume/util"
 	"k8s.io/kubernetes/pkg/volume/util/recyclerclient"
+	"k8s.io/kubernetes/pkg/volume/util/subpath"
 	"k8s.io/kubernetes/pkg/volume/util/volumepathhandler"
 	utilstrings "k8s.io/utils/strings"
 )
@@ -71,6 +72,7 @@ type fakeVolumeHost struct {
 	exec       mount.Exec
 	nodeLabels map[string]string
 	nodeName   string
+	subpather  subpath.Interface
 }
 
 func NewFakeVolumeHost(rootDir string, kubeClient clientset.Interface, plugins []VolumePlugin) *fakeVolumeHost {
@@ -101,6 +103,7 @@ func newFakeVolumeHost(rootDir string, kubeClient clientset.Interface, plugins [
 	}
 	host.exec = mount.NewFakeExec(nil)
 	host.pluginMgr.InitPlugins(plugins, nil /* prober */, host)
+	host.subpather = &subpath.FakeSubpath{}
 	return host
 }
 
@@ -149,6 +152,10 @@ func (f *fakeVolumeHost) GetMounter(pluginName string) mount.Interface {
 	return f.mounter
 }
 
+func (f *fakeVolumeHost) GetSubpather() subpath.Interface {
+	return f.subpather
+}
+
 func (f *fakeVolumeHost) NewWrapperMounter(volName string, spec Spec, pod *v1.Pod, opts VolumeOptions) (Mounter, error) {
 	// The name of wrapper volume is set to "wrapped_{wrapped_volume_name}"
 	wrapperVolumeName := "wrapped_" + volName

pkg/volume/util/BUILD

@@ -88,6 +88,7 @@ filegroup(
         "//pkg/volume/util/nestedpendingoperations:all-srcs",
         "//pkg/volume/util/operationexecutor:all-srcs",
         "//pkg/volume/util/recyclerclient:all-srcs",
+        "//pkg/volume/util/subpath:all-srcs",
         "//pkg/volume/util/types:all-srcs",
         "//pkg/volume/util/volumepathhandler:all-srcs",
     ],

pkg/volume/util/operationexecutor/operation_generator.go

@@ -750,11 +750,11 @@ func (og *operationGenerator) GenerateUnmountVolumeFunc(
 	}
 
 	unmountVolumeFunc := func() (error, error) {
-		mounter := og.volumePluginMgr.Host.GetMounter(volumeToUnmount.PluginName)
+		subpather := og.volumePluginMgr.Host.GetSubpather()
 
 		// Remove all bind-mounts for subPaths
 		podDir := path.Join(podsDir, string(volumeToUnmount.PodUID))
-		if err := mounter.CleanSubPaths(podDir, volumeToUnmount.InnerVolumeSpecName); err != nil {
+		if err := subpather.CleanSubPaths(podDir, volumeToUnmount.InnerVolumeSpecName); err != nil {
 			return volumeToUnmount.GenerateError("error cleaning subPath mounts", err)
 		}
 

pkg/volume/util/subpath/BUILD

+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "subpath.go",
+        "subpath_linux.go",
+        "subpath_nsenter.go",
+        "subpath_unsupported.go",
+        "subpath_windows.go",
+    ],
+    importpath = "k8s.io/kubernetes/pkg/volume/util/subpath",
+    visibility = ["//visibility:public"],
+    deps = select({
+        "@io_bazel_rules_go//go/platform:android": [
+            "//pkg/util/mount:go_default_library",
+            "//vendor/k8s.io/utils/nsenter:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:darwin": [
+            "//pkg/util/mount:go_default_library",
+            "//vendor/k8s.io/utils/nsenter:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:dragonfly": [
+            "//pkg/util/mount:go_default_library",
+            "//vendor/k8s.io/utils/nsenter:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:freebsd": [
+            "//pkg/util/mount:go_default_library",
+            "//vendor/k8s.io/utils/nsenter:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:linux": [
+            "//pkg/util/mount:go_default_library",
+            "//vendor/golang.org/x/sys/unix:go_default_library",
+            "//vendor/k8s.io/klog:go_default_library",
+            "//vendor/k8s.io/utils/nsenter:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:nacl": [
+            "//pkg/util/mount:go_default_library",
+            "//vendor/k8s.io/utils/nsenter:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:netbsd": [
+            "//pkg/util/mount:go_default_library",
+            "//vendor/k8s.io/utils/nsenter:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:openbsd": [
+            "//pkg/util/mount:go_default_library",
+            "//vendor/k8s.io/utils/nsenter:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:plan9": [
+            "//pkg/util/mount:go_default_library",
+            "//vendor/k8s.io/utils/nsenter:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:solaris": [
+            "//pkg/util/mount:go_default_library",
+            "//vendor/k8s.io/utils/nsenter:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:windows": [
+            "//pkg/util/mount:go_default_library",
+            "//vendor/k8s.io/klog:go_default_library",
+            "//vendor/k8s.io/utils/nsenter:go_default_library",
+        ],
+        "//conditions:default": [],
+    }),
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "subpath_linux_test.go",
+        "subpath_nsenter_test.go",
+        "subpath_windows_test.go",
+    ],
+    embed = [":go_default_library"],
+    deps = select({
+        "@io_bazel_rules_go//go/platform:linux": [
+            "//pkg/util/mount:go_default_library",
+            "//vendor/golang.org/x/sys/unix:go_default_library",
+            "//vendor/k8s.io/klog:go_default_library",
+            "//vendor/k8s.io/utils/nsenter:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:windows": [
+            "//vendor/github.com/stretchr/testify/assert:go_default_library",
+        ],
+        "//conditions:default": [],
+    }),
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

pkg/volume/util/subpath/OWNERS

+# See the OWNERS docs at https://go.k8s.io/owners
+
+reviewers:
+  - jingxu97
+  - saad-ali
+  - jsafrane
+  - msau42
+  - andyzhangx
+approvers:
+  - jingxu97
+  - saad-ali
+  - jsafrane
+

pkg/volume/util/subpath/subpath.go

+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package subpath
+
+import "os"
+
+// Interface defines the set of methods all subpathers must implement
+type Interface interface {
+	// CleanSubPaths removes any bind-mounts created by PrepareSafeSubpath in given
+	// pod volume directory.
+	CleanSubPaths(poodDir string, volumeName string) error
+
+	// PrepareSafeSubpath does everything that's necessary to prepare a subPath
+	// that's 1) inside given volumePath and 2) immutable after this call.
+	//
+	// newHostPath - location of prepared subPath. It should be used instead of
+	// hostName when running the container.
+	// cleanupAction - action to run when the container is running or it failed to start.
+	//
+	// CleanupAction must be called immediately after the container with given
+	// subpath starts. On the other hand, Interface.CleanSubPaths must be called
+	// when the pod finishes.
+	PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error)
+
+	// SafeMakeDir creates subdir within given base. It makes sure that the
+	// created directory does not escape given base directory mis-using
+	// symlinks. Note that the function makes sure that it creates the directory
+	// somewhere under the base, nothing else. E.g. if the directory already
+	// exists, it may exist outside of the base due to symlinks.
+	// This method should be used if the directory to create is inside volume
+	// that's under user control. User must not be able to use symlinks to
+	// escape the volume to create directories somewhere else.
+	SafeMakeDir(subdir string, base string, perm os.FileMode) error
+}
+
+// Subpath defines the attributes of a subpath
+type Subpath struct {
+	// index of the VolumeMount for this container
+	VolumeMountIndex int
+
+	// Full path to the subpath directory on the host
+	Path string
+
+	// name of the volume that is a valid directory name.
+	VolumeName string
+
+	// Full path to the volume path
+	VolumePath string
+
+	// Path to the pod's directory, including pod UID
+	PodDir string
+
+	// Name of the container
+	ContainerName string
+}
+
+// Compile time-check for all implementers of subpath interface
+var _ Interface = &subpath{}
+var _ Interface = &FakeSubpath{}
+
+// FakeSubpath is a subpather implementation for testing
+type FakeSubpath struct{}
+
+// PrepareSafeSubpath is a fake implementation of PrepareSafeSubpath. Always returns
+// newHostPath == subPath.Path
+func (fs *FakeSubpath) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
+	return subPath.Path, nil, nil
+}
+
+// CleanSubPaths is a fake implementation of CleanSubPaths. It is a noop
+func (fs *FakeSubpath) CleanSubPaths(podDir string, volumeName string) error {
+	return nil
+}
+
+// SafeMakeDir is a fake implementation of SafeMakeDir. It is a noop
+func (fs *FakeSubpath) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
+	return nil
+}

pkg/volume/util/subpath/subpath_linux.go

+// +build linux
+
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package subpath
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"syscall"
+
+	"golang.org/x/sys/unix"
+
+	"k8s.io/klog"
+
+	"k8s.io/kubernetes/pkg/util/mount"
+)
+
+const (
+	// place for subpath mounts
+	// TODO: pass in directory using kubelet_getters instead
+	containerSubPathDirectoryName = "volume-subpaths"
+	// syscall.Openat flags used to traverse directories not following symlinks
+	nofollowFlags = unix.O_RDONLY | unix.O_NOFOLLOW
+	// flags for getting file descriptor without following the symlink
+	openFDFlags = unix.O_NOFOLLOW | unix.O_PATH
+)
+
+type subpath struct {
+	mounter mount.Interface
+}
+
+// New returns a subpath.Interface for the current system
+func New(mounter mount.Interface) Interface {
+	return &subpath{
+		mounter: mounter,
+	}
+}
+
+func (sp *subpath) CleanSubPaths(podDir string, volumeName string) error {
+	return doCleanSubPaths(sp.mounter, podDir, volumeName)
+}
+
+func (sp *subpath) SafeMakeDir(subdir string, base string, perm os.FileMode) error {
+	realBase, err := filepath.EvalSymlinks(base)
+	if err != nil {
+		return fmt.Errorf("error resolving symlinks in %s: %s", base, err)
+	}
+
+	realFullPath := filepath.Join(realBase, subdir)
+
+	return doSafeMakeDir(realFullPath, realBase, perm)
+}
+
+func (sp *subpath) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
+	newHostPath, err = doBindSubPath(sp.mounter, subPath)
+
+	// There is no action when the container starts. Bind-mount will be cleaned
+	// when container stops by CleanSubPaths.
+	cleanupAction = nil
+	return newHostPath, cleanupAction, err
+}
+
+// This implementation is shared between Linux and NsEnter
+func safeOpenSubPath(mounter mount.Interface, subpath Subpath) (int, error) {
+	if !mount.PathWithinBase(subpath.Path, subpath.VolumePath) {
+		return -1, fmt.Errorf("subpath %q not within volume path %q", subpath.Path, subpath.VolumePath)
+	}
+	fd, err := doSafeOpen(subpath.Path, subpath.VolumePath)
+	if err != nil {
+		return -1, fmt.Errorf("error opening subpath %v: %v", subpath.Path, err)
+	}
+	return fd, nil
+}
+
+// prepareSubpathTarget creates target for bind-mount of subpath. It returns
+// "true" when the target already exists and something is mounted there.
+// Given Subpath must have all paths with already resolved symlinks and with
+// paths relevant to kubelet (when it runs in a container).
+// This function is called also by NsEnterMounter. It works because
+// /var/lib/kubelet is mounted from the host into the container with Kubelet as
+// /var/lib/kubelet too.
+func prepareSubpathTarget(mounter mount.Interface, subpath Subpath) (bool, string, error) {
+	// Early check for already bind-mounted subpath.
+	bindPathTarget := getSubpathBindTarget(subpath)
+	notMount, err := mounter.IsNotMountPoint(bindPathTarget)
+	if err != nil {
+		if !os.IsNotExist(err) {
+			return false, "", fmt.Errorf("error checking path %s for mount: %s", bindPathTarget, err)
+		}
+		// Ignore ErrorNotExist: the file/directory will be created below if it does not exist yet.
+		notMount = true
+	}
+	if !notMount {
+		// It's already mounted
+		klog.V(5).Infof("Skipping bind-mounting subpath %s: already mounted", bindPathTarget)
+		return true, bindPathTarget, nil
+	}
+
+	// bindPathTarget is in /var/lib/kubelet and thus reachable without any
+	// translation even to containerized kubelet.
+	bindParent := filepath.Dir(bindPathTarget)
+	err = os.MkdirAll(bindParent, 0750)
+	if err != nil && !os.IsExist(err) {
+		return false, "", fmt.Errorf("error creating directory %s: %s", bindParent, err)
+	}
+
+	t, err := os.Lstat(subpath.Path)
+	if err != nil {
+		return false, "", fmt.Errorf("lstat %s failed: %s", subpath.Path, err)
+	}
+
+	if t.Mode()&os.ModeDir > 0 {
+		if err = os.Mkdir(bindPathTarget, 0750); err != nil && !os.IsExist(err) {
+			return false, "", fmt.Errorf("error creating directory %s: %s", bindPathTarget, err)
+		}
+	} else {
+		// "/bin/touch <bindPathTarget>".
+		// A file is enough for all possible targets (symlink, device, pipe,
+		// socket, ...), bind-mounting them into a file correctly changes type
+		// of the target file.
+		if err = ioutil.WriteFile(bindPathTarget, []byte{}, 0640); err != nil {
+			return false, "", fmt.Errorf("error creating file %s: %s", bindPathTarget, err)
+		}
+	}
+	return false, bindPathTarget, nil
+}
+
+func getSubpathBindTarget(subpath Subpath) string {
+	// containerName is DNS label, i.e. safe as a directory name.
+	return filepath.Join(subpath.PodDir, containerSubPathDirectoryName, subpath.VolumeName, subpath.ContainerName, strconv.Itoa(subpath.VolumeMountIndex))
+}
+
+func doBindSubPath(mounter mount.Interface, subpath Subpath) (hostPath string, err error) {
+	// Linux, kubelet runs on the host:
+	// - safely open the subpath
+	// - bind-mount /proc/<pid of kubelet>/fd/<fd> to subpath target
+	// User can't change /proc/<pid of kubelet>/fd/<fd> to point to a bad place.
+
+	// Evaluate all symlinks here once for all subsequent functions.
+	newVolumePath, err := filepath.EvalSymlinks(subpath.VolumePath)
+	if err != nil {
+		return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.VolumePath, err)
+	}
+	newPath, err := filepath.EvalSymlinks(subpath.Path)
+	if err != nil {
+		return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.Path, err)
+	}
+	klog.V(5).Infof("doBindSubPath %q (%q) for volumepath %q", subpath.Path, newPath, subpath.VolumePath)
+	subpath.VolumePath = newVolumePath
+	subpath.Path = newPath
+
+	fd, err := safeOpenSubPath(mounter, subpath)
+	if err != nil {
+		return "", err
+	}
+	defer syscall.Close(fd)
+
+	alreadyMounted, bindPathTarget, err := prepareSubpathTarget(mounter, subpath)
+	if err != nil {
+		return "", err
+	}
+	if alreadyMounted {
+		return bindPathTarget, nil
+	}
+
+	success := false
+	defer func() {
+		// Cleanup subpath on error
+		if !success {
+			klog.V(4).Infof("doBindSubPath() failed for %q, cleaning up subpath", bindPathTarget)
+			if cleanErr := cleanSubPath(mounter, subpath); cleanErr != nil {
+				klog.Errorf("Failed to clean subpath %q: %v", bindPathTarget, cleanErr)
+			}
+		}
+	}()
+
+	kubeletPid := os.Getpid()
+	mountSource := fmt.Sprintf("/proc/%d/fd/%v", kubeletPid, fd)
+
+	// Do the bind mount
+	options := []string{"bind"}
+	klog.V(5).Infof("bind mounting %q at %q", mountSource, bindPathTarget)
+	if err = mounter.Mount(mountSource, bindPathTarget, "" /*fstype*/, options); err != nil {
+		return "", fmt.Errorf("error mounting %s: %s", subpath.Path, err)
+	}
+	success = true
+
+	klog.V(3).Infof("Bound SubPath %s into %s", subpath.Path, bindPathTarget)
+	return bindPathTarget, nil
+}
+
+// This implementation is shared between Linux and NsEnter
+func doCleanSubPaths(mounter mount.Interface, podDir string, volumeName string) error {
+	// scan /var/lib/kubelet/pods/<uid>/volume-subpaths/<volume>/*
+	subPathDir := filepath.Join(podDir, containerSubPathDirectoryName, volumeName)
+	klog.V(4).Infof("Cleaning up subpath mounts for %s", subPathDir)
+
+	containerDirs, err := ioutil.ReadDir(subPathDir)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return fmt.Errorf("error reading %s: %s", subPathDir, err)
+	}
+
+	for _, containerDir := range containerDirs {
+		if !containerDir.IsDir() {
+			klog.V(4).Infof("Container file is not a directory: %s", containerDir.Name())
+			continue
+		}
+		klog.V(4).Infof("Cleaning up subpath mounts for container %s", containerDir.Name())
+
+		// scan /var/lib/kubelet/pods/<uid>/volume-subpaths/<volume>/<container name>/*
+		fullContainerDirPath := filepath.Join(subPathDir, containerDir.Name())
+		err = filepath.Walk(fullContainerDirPath, func(path string, info os.FileInfo, err error) error {
+			if path == fullContainerDirPath {
+				// Skip top level directory
+				return nil
+			}
+
+			// pass through errors and let doCleanSubPath handle them
+			if err = doCleanSubPath(mounter, fullContainerDirPath, filepath.Base(path)); err != nil {
+				return err
+			}
+			return nil
+		})
+		if err != nil {
+			return fmt.Errorf("error processing %s: %s", fullContainerDirPath, err)
+		}
+
+		// Whole container has been processed, remove its directory.
+		if err := os.Remove(fullContainerDirPath); err != nil {
+			return fmt.Errorf("error deleting %s: %s", fullContainerDirPath, err)
+		}
+		klog.V(5).Infof("Removed %s", fullContainerDirPath)
+	}
+	// Whole pod volume subpaths have been cleaned up, remove its subpath directory.
+	if err := os.Remove(subPathDir); err != nil {
+		return fmt.Errorf("error deleting %s: %s", subPathDir, err)
+	}
+	klog.V(5).Infof("Removed %s", subPathDir)
+
+	// Remove entire subpath directory if it's the last one
+	podSubPathDir := filepath.Join(podDir, containerSubPathDirectoryName)
+	if err := os.Remove(podSubPathDir); err != nil && !os.IsExist(err) {
+		return fmt.Errorf("error deleting %s: %s", podSubPathDir, err)
+	}
+	klog.V(5).Infof("Removed %s", podSubPathDir)
+	return nil
+}
+
+// doCleanSubPath tears down the single subpath bind mount
+func doCleanSubPath(mounter mount.Interface, fullContainerDirPath, subPathIndex string) error {
+	// process /var/lib/kubelet/pods/<uid>/volume-subpaths/<volume>/<container name>/<subPathName>
+	klog.V(4).Infof("Cleaning up subpath mounts for subpath %v", subPathIndex)
+	fullSubPath := filepath.Join(fullContainerDirPath, subPathIndex)
+
+	if err := mount.CleanupMountPoint(fullSubPath, mounter, true); err != nil {
+		return fmt.Errorf("error cleaning subpath mount %s: %s", fullSubPath, err)
+	}
+
+	klog.V(4).Infof("Successfully cleaned subpath directory %s", fullSubPath)
+	return nil
+}
+
+// cleanSubPath will teardown the subpath bind mount and any remove any directories if empty
+func cleanSubPath(mounter mount.Interface, subpath Subpath) error {
+	containerDir := filepath.Join(subpath.PodDir, containerSubPathDirectoryName, subpath.VolumeName, subpath.ContainerName)
+
+	// Clean subdir bindmount
+	if err := doCleanSubPath(mounter, containerDir, strconv.Itoa(subpath.VolumeMountIndex)); err != nil && !os.IsNotExist(err) {
+		return err
+	}
+
+	// Recusively remove directories if empty
+	if err := removeEmptyDirs(subpath.PodDir, containerDir); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// removeEmptyDirs works backwards from endDir to baseDir and removes each directory
+// if it is empty.  It stops once it encounters a directory that has content
+func removeEmptyDirs(baseDir, endDir string) error {
+	if !mount.PathWithinBase(endDir, baseDir) {
+		return fmt.Errorf("endDir %q is not within baseDir %q", endDir, baseDir)
+	}
+
+	for curDir := endDir; curDir != baseDir; curDir = filepath.Dir(curDir) {
+		s, err := os.Stat(curDir)
+		if err != nil {
+			if os.IsNotExist(err) {
+				klog.V(5).Infof("curDir %q doesn't exist, skipping", curDir)
+				continue
+			}
+			return fmt.Errorf("error stat %q: %v", curDir, err)
+		}
+		if !s.IsDir() {
+			return fmt.Errorf("path %q not a directory", curDir)
+		}
+
+		err = os.Remove(curDir)
+		if os.IsExist(err) {
+			klog.V(5).Infof("Directory %q not empty, not removing", curDir)
+			break
+		} else if err != nil {
+			return fmt.Errorf("error removing directory %q: %v", curDir, err)
+		}
+		klog.V(5).Infof("Removed directory %q", curDir)
+	}
+	return nil
+}
+
+// This implementation is shared between Linux and NsEnterMounter. Both pathname
+// and base must be either already resolved symlinks or thet will be resolved in
+// kubelet's mount namespace (in case it runs containerized).
+func doSafeMakeDir(pathname string, base string, perm os.FileMode) error {
+	klog.V(4).Infof("Creating directory %q within base %q", pathname, base)
+
+	if !mount.PathWithinBase(pathname, base) {
+		return fmt.Errorf("path %s is outside of allowed base %s", pathname, base)
+	}
+
+	// Quick check if the directory already exists
+	s, err := os.Stat(pathname)
+	if err == nil {
+		// Path exists
+		if s.IsDir() {
+			// The directory already exists. It can be outside of the parent,
+			// but there is no race-proof check.
+			klog.V(4).Infof("Directory %s already exists", pathname)
+			return nil
+		}
+		return &os.PathError{Op: "mkdir", Path: pathname, Err: syscall.ENOTDIR}
+	}
+
+	// Find all existing directories
+	existingPath, toCreate, err := findExistingPrefix(base, pathname)
+	if err != nil {
+		return fmt.Errorf("error opening directory %s: %s", pathname, err)
+	}
+	// Ensure the existing directory is inside allowed base
+	fullExistingPath, err := filepath.EvalSymlinks(existingPath)
+	if err != nil {
+		return fmt.Errorf("error opening directory %s: %s", existingPath, err)
+	}
+	if !mount.PathWithinBase(fullExistingPath, base) {
+		return fmt.Errorf("path %s is outside of allowed base %s", fullExistingPath, err)
+	}
+
+	klog.V(4).Infof("%q already exists, %q to create", fullExistingPath, filepath.Join(toCreate...))
+	parentFD, err := doSafeOpen(fullExistingPath, base)
+	if err != nil {
+		return fmt.Errorf("cannot open directory %s: %s", existingPath, err)
+	}
+	childFD := -1
+	defer func() {
+		if parentFD != -1 {
+			if err = syscall.Close(parentFD); err != nil {
+				klog.V(4).Infof("Closing FD %v failed for safemkdir(%v): %v", parentFD, pathname, err)
+			}
+		}
+		if childFD != -1 {
+			if err = syscall.Close(childFD); err != nil {
+				klog.V(4).Infof("Closing FD %v failed for safemkdir(%v): %v", childFD, pathname, err)
+			}
+		}
+	}()
+
+	currentPath := fullExistingPath
+	// create the directories one by one, making sure nobody can change
+	// created directory into symlink.
+	for _, dir := range toCreate {
+		currentPath = filepath.Join(currentPath, dir)
+		klog.V(4).Infof("Creating %s", dir)
+		err = syscall.Mkdirat(parentFD, currentPath, uint32(perm))
+		if err != nil {
+			return fmt.Errorf("cannot create directory %s: %s", currentPath, err)
+		}
+		// Dive into the created directory
+		childFD, err := syscall.Openat(parentFD, dir, nofollowFlags, 0)
+		if err != nil {
+			return fmt.Errorf("cannot open %s: %s", currentPath, err)
+		}
+		// We can be sure that childFD is safe to use. It could be changed
+		// by user after Mkdirat() and before Openat(), however:
+		// - it could not be changed to symlink - we use nofollowFlags
+		// - it could be changed to a file (or device, pipe, socket, ...)
+		//   but either subsequent Mkdirat() fails or we mount this file
+		//   to user's container. Security is no violated in both cases
+		//   and user either gets error or the file that it can already access.
+
+		if err = syscall.Close(parentFD); err != nil {
+			klog.V(4).Infof("Closing FD %v failed for safemkdir(%v): %v", parentFD, pathname, err)
+		}
+		parentFD = childFD
+		childFD = -1
+	}
+
+	// Everything was created. mkdirat(..., perm) above was affected by current
+	// umask and we must apply the right permissions to the last directory
+	// (that's the one that will be available to the container as subpath)
+	// so user can read/write it. This is the behavior of previous code.
+	// TODO: chmod all created directories, not just the last one.
+	// parentFD is the last created directory.
+
+	// Translate perm (os.FileMode) to uint32 that fchmod() expects
+	kernelPerm := uint32(perm & os.ModePerm)
+	if perm&os.ModeSetgid > 0 {
+		kernelPerm |= syscall.S_ISGID
+	}
+	if perm&os.ModeSetuid > 0 {
+		kernelPerm |= syscall.S_ISUID
+	}
+	if perm&os.ModeSticky > 0 {
+		kernelPerm |= syscall.S_ISVTX
+	}
+	if err = syscall.Fchmod(parentFD, kernelPerm); err != nil {
+		return fmt.Errorf("chmod %q failed: %s", currentPath, err)
+	}
+	return nil
+}
+
+// findExistingPrefix finds prefix of pathname that exists. In addition, it
+// returns list of remaining directories that don't exist yet.
+func findExistingPrefix(base, pathname string) (string, []string, error) {
+	rel, err := filepath.Rel(base, pathname)
+	if err != nil {
+		return base, nil, err
+	}
+	dirs := strings.Split(rel, string(filepath.Separator))
+
+	// Do OpenAt in a loop to find the first non-existing dir. Resolve symlinks.
+	// This should be faster than looping through all dirs and calling os.Stat()
+	// on each of them, as the symlinks are resolved only once with OpenAt().
+	currentPath := base
+	fd, err := syscall.Open(currentPath, syscall.O_RDONLY, 0)
+	if err != nil {
+		return pathname, nil, fmt.Errorf("error opening %s: %s", currentPath, err)
+	}
+	defer func() {
+		if err = syscall.Close(fd); err != nil {
+			klog.V(4).Infof("Closing FD %v failed for findExistingPrefix(%v): %v", fd, pathname, err)
+		}
+	}()
+	for i, dir := range dirs {
+		// Using O_PATH here will prevent hangs in case user replaces directory with
+		// fifo
+		childFD, err := syscall.Openat(fd, dir, unix.O_PATH, 0)
+		if err != nil {
+			if os.IsNotExist(err) {
+				return currentPath, dirs[i:], nil
+			}
+			return base, nil, err
+		}
+		if err = syscall.Close(fd); err != nil {
+			klog.V(4).Infof("Closing FD %v failed for findExistingPrefix(%v): %v", fd, pathname, err)
+		}
+		fd = childFD
+		currentPath = filepath.Join(currentPath, dir)
+	}
+	return pathname, []string{}, nil
+}
+
+// This implementation is shared between Linux and NsEnterMounter
+// Open path and return its fd.
+// Symlinks are disallowed (pathname must already resolve symlinks),
+// and the path must be within the base directory.
+func doSafeOpen(pathname string, base string) (int, error) {
+	pathname = filepath.Clean(pathname)
+	base = filepath.Clean(base)
+
+	// Calculate segments to follow
+	subpath, err := filepath.Rel(base, pathname)
+	if err != nil {
+		return -1, err
+	}
+	segments := strings.Split(subpath, string(filepath.Separator))
+
+	// Assumption: base is the only directory that we have under control.
+	// Base dir is not allowed to be a symlink.
+	parentFD, err := syscall.Open(base, nofollowFlags, 0)
+	if err != nil {
+		return -1, fmt.Errorf("cannot open directory %s: %s", base, err)
+	}
+	defer func() {
+		if parentFD != -1 {
+			if err = syscall.Close(parentFD); err != nil {
+				klog.V(4).Infof("Closing FD %v failed for safeopen(%v): %v", parentFD, pathname, err)
+			}
+		}
+	}()
+
+	childFD := -1
+	defer func() {
+		if childFD != -1 {
+			if err = syscall.Close(childFD); err != nil {
+				klog.V(4).Infof("Closing FD %v failed for safeopen(%v): %v", childFD, pathname, err)
+			}
+		}
+	}()
+
+	currentPath := base
+
+	// Follow the segments one by one using openat() to make
+	// sure the user cannot change already existing directories into symlinks.
+	for _, seg := range segments {
+		currentPath = filepath.Join(currentPath, seg)
+		if !mount.PathWithinBase(currentPath, base) {
+			return -1, fmt.Errorf("path %s is outside of allowed base %s", currentPath, base)
+		}
+
+		klog.V(5).Infof("Opening path %s", currentPath)
+		childFD, err = syscall.Openat(parentFD, seg, openFDFlags, 0)
+		if err != nil {
+			return -1, fmt.Errorf("cannot open %s: %s", currentPath, err)
+		}
+
+		var deviceStat unix.Stat_t
+		err := unix.Fstat(childFD, &deviceStat)
+		if err != nil {
+			return -1, fmt.Errorf("Error running fstat on %s with %v", currentPath, err)
+		}
+		fileFmt := deviceStat.Mode & syscall.S_IFMT
+		if fileFmt == syscall.S_IFLNK {
+			return -1, fmt.Errorf("Unexpected symlink found %s", currentPath)
+		}
+
+		// Close parentFD
+		if err = syscall.Close(parentFD); err != nil {
+			return -1, fmt.Errorf("closing fd for %q failed: %v", filepath.Dir(currentPath), err)
+		}
+		// Set child to new parent
+		parentFD = childFD
+		childFD = -1
+	}
+
+	// We made it to the end, return this fd, don't close it
+	finalFD := parentFD
+	parentFD = -1
+
+	return finalFD, nil
+}

pkg/volume/util/subpath/subpath_linux_test.go

+// +build linux
+
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package subpath
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net"
+	"os"
+	"path/filepath"
+	"reflect"
+	"strconv"
+	"syscall"
+	"testing"
+
+	"k8s.io/klog"
+
+	"k8s.io/kubernetes/pkg/util/mount"
+)
+
+func TestSafeMakeDir(t *testing.T) {
+	defaultPerm := os.FileMode(0750) + os.ModeDir
+	tests := []struct {
+		name string
+		// Function that prepares directory structure for the test under given
+		// base.
+		prepare     func(base string) error
+		path        string
+		checkPath   string
+		perm        os.FileMode
+		expectError bool
+	}{
+		{
+			"directory-does-not-exist",
+			func(base string) error {
+				return nil
+			},
+			"test/directory",
+			"test/directory",
+			defaultPerm,
+			false,
+		},
+		{
+			"directory-with-sgid",
+			func(base string) error {
+				return nil
+			},
+			"test/directory",
+			"test/directory",
+			os.FileMode(0777) + os.ModeDir + os.ModeSetgid,
+			false,
+		},
+		{
+			"directory-with-suid",
+			func(base string) error {
+				return nil
+			},
+			"test/directory",
+			"test/directory",
+			os.FileMode(0777) + os.ModeDir + os.ModeSetuid,
+			false,
+		},
+		{
+			"directory-with-sticky-bit",
+			func(base string) error {
+				return nil
+			},
+			"test/directory",
+			"test/directory",
+			os.FileMode(0777) + os.ModeDir + os.ModeSticky,
+			false,
+		},
+		{
+			"directory-exists",
+			func(base string) error {
+				return os.MkdirAll(filepath.Join(base, "test/directory"), 0750)
+			},
+			"test/directory",
+			"test/directory",
+			defaultPerm,
+			false,
+		},
+		{
+			"create-base",
+			func(base string) error {
+				return nil
+			},
+			"",
+			"",
+			defaultPerm,
+			false,
+		},
+		{
+			"escape-base-using-dots",
+			func(base string) error {
+				return nil
+			},
+			"..",
+			"",
+			defaultPerm,
+			true,
+		},
+		{
+			"escape-base-using-dots-2",
+			func(base string) error {
+				return nil
+			},
+			"test/../../..",
+			"",
+			defaultPerm,
+			true,
+		},
+		{
+			"follow-symlinks",
+			func(base string) error {
+				if err := os.MkdirAll(filepath.Join(base, "destination"), defaultPerm); err != nil {
+					return err
+				}
+				return os.Symlink("destination", filepath.Join(base, "test"))
+			},
+			"test/directory",
+			"destination/directory",
+			defaultPerm,
+			false,
+		},
+		{
+			"follow-symlink-loop",
+			func(base string) error {
+				return os.Symlink("test", filepath.Join(base, "test"))
+			},
+			"test/directory",
+			"",
+			defaultPerm,
+			true,
+		},
+		{
+			"follow-symlink-multiple follow",
+			func(base string) error {
+				/* test1/dir points to test2 and test2/dir points to test1 */
+				if err := os.MkdirAll(filepath.Join(base, "test1"), defaultPerm); err != nil {
+					return err
+				}
+				if err := os.MkdirAll(filepath.Join(base, "test2"), defaultPerm); err != nil {
+					return err
+				}
+				if err := os.Symlink(filepath.Join(base, "test2"), filepath.Join(base, "test1/dir")); err != nil {
+					return err
+				}
+				if err := os.Symlink(filepath.Join(base, "test1"), filepath.Join(base, "test2/dir")); err != nil {
+					return err
+				}
+				return nil
+			},
+			"test1/dir/dir/dir/dir/dir/dir/dir/foo",
+			"test2/foo",
+			defaultPerm,
+			false,
+		},
+		{
+			"danglink-symlink",
+			func(base string) error {
+				return os.Symlink("non-existing", filepath.Join(base, "test"))
+			},
+			"test/directory",
+			"",
+			defaultPerm,
+			true,
+		},
+		{
+			"non-directory",
+			func(base string) error {
+				return ioutil.WriteFile(filepath.Join(base, "test"), []byte{}, defaultPerm)
+			},
+			"test/directory",
+			"",
+			defaultPerm,
+			true,
+		},
+		{
+			"non-directory-final",
+			func(base string) error {
+				return ioutil.WriteFile(filepath.Join(base, "test"), []byte{}, defaultPerm)
+			},
+			"test",
+			"",
+			defaultPerm,
+			true,
+		},
+		{
+			"escape-with-relative-symlink",
+			func(base string) error {
+				if err := os.MkdirAll(filepath.Join(base, "dir"), defaultPerm); err != nil {
+					return err
+				}
+				if err := os.MkdirAll(filepath.Join(base, "exists"), defaultPerm); err != nil {
+					return err
+				}
+				return os.Symlink("../exists", filepath.Join(base, "dir/test"))
+			},
+			"dir/test",
+			"",
+			defaultPerm,
+			false,
+		},
+		{
+			"escape-with-relative-symlink-not-exists",
+			func(base string) error {
+				if err := os.MkdirAll(filepath.Join(base, "dir"), defaultPerm); err != nil {
+					return err
+				}
+				return os.Symlink("../not-exists", filepath.Join(base, "dir/test"))
+			},
+			"dir/test",
+			"",
+			defaultPerm,
+			true,
+		},
+		{
+			"escape-with-symlink",
+			func(base string) error {
+				return os.Symlink("/", filepath.Join(base, "test"))
+			},
+			"test/directory",
+			"",
+			defaultPerm,
+			true,
+		},
+	}
+
+	for i := range tests {
+		test := tests[i]
+		t.Run(test.name, func(t *testing.T) {
+			base, err := ioutil.TempDir("", "safe-make-dir-"+test.name+"-")
+			if err != nil {
+				t.Fatalf(err.Error())
+			}
+			defer os.RemoveAll(base)
+			test.prepare(base)
+			pathToCreate := filepath.Join(base, test.path)
+			err = doSafeMakeDir(pathToCreate, base, test.perm)
+			if err != nil && !test.expectError {
+				t.Fatal(err)
+			}
+			if err != nil {
+				t.Logf("got error: %s", err)
+			}
+			if err == nil && test.expectError {
+				t.Fatalf("expected error, got none")
+			}
+
+			if test.checkPath != "" {
+				st, err := os.Stat(filepath.Join(base, test.checkPath))
+				if err != nil {
+					t.Fatalf("cannot read path %s", test.checkPath)
+				}
+				actualMode := st.Mode()
+				if actualMode != test.perm {
+					if actualMode^test.perm == os.ModeSetgid && test.perm&os.ModeSetgid == 0 {
+						// when TMPDIR is a kubernetes emptydir, the sticky gid bit is set due to fsgroup
+						t.Logf("masking bit from %o", actualMode)
+					} else {
+						t.Errorf("expected permissions %o, got %o (%b)", test.perm, actualMode, test.perm^actualMode)
+					}
+				}
+			}
+		})
+	}
+}
+
+func TestRemoveEmptyDirs(t *testing.T) {
+	defaultPerm := os.FileMode(0750)
+	tests := []struct {
+		name string
+		// Function that prepares directory structure for the test under given
+		// base.
+		prepare func(base string) error
+		// Function that validates directory structure after the test
+		validate    func(base string) error
+		baseDir     string
+		endDir      string
+		expectError bool
+	}{
+		{
+			name: "all-empty",
+			prepare: func(base string) error {
+				return os.MkdirAll(filepath.Join(base, "a/b/c"), defaultPerm)
+			},
+			validate: func(base string) error {
+				return validateDirEmpty(filepath.Join(base, "a"))
+			},
+			baseDir:     "a",
+			endDir:      "a/b/c",
+			expectError: false,
+		},
+		{
+			name: "dir-not-empty",
+			prepare: func(base string) error {
+				if err := os.MkdirAll(filepath.Join(base, "a/b/c"), defaultPerm); err != nil {
+					return err
+				}
+				return os.Mkdir(filepath.Join(base, "a/b/d"), defaultPerm)
+			},
+			validate: func(base string) error {
+				if err := validateDirNotExists(filepath.Join(base, "a/b/c")); err != nil {
+					return err
+				}
+				return validateDirExists(filepath.Join(base, "a/b"))
+			},
+			baseDir:     "a",
+			endDir:      "a/b/c",
+			expectError: false,
+		},
+		{
+			name: "path-not-within-base",
+			prepare: func(base string) error {
+				return os.MkdirAll(filepath.Join(base, "a/b/c"), defaultPerm)
+			},
+			validate: func(base string) error {
+				return validateDirExists(filepath.Join(base, "a"))
+			},
+			baseDir:     "a",
+			endDir:      "b/c",
+			expectError: true,
+		},
+		{
+			name: "path-already-deleted",
+			prepare: func(base string) error {
+				return nil
+			},
+			validate: func(base string) error {
+				return nil
+			},
+			baseDir:     "a",
+			endDir:      "a/b/c",
+			expectError: false,
+		},
+		{
+			name: "path-not-dir",
+			prepare: func(base string) error {
+				if err := os.MkdirAll(filepath.Join(base, "a/b"), defaultPerm); err != nil {
+					return err
+				}
+				return ioutil.WriteFile(filepath.Join(base, "a/b", "c"), []byte{}, defaultPerm)
+			},
+			validate: func(base string) error {
+				if err := validateDirExists(filepath.Join(base, "a/b")); err != nil {
+					return err
+				}
+				return validateFileExists(filepath.Join(base, "a/b/c"))
+			},
+			baseDir:     "a",
+			endDir:      "a/b/c",
+			expectError: true,
+		},
+	}
+
+	for _, test := range tests {
+		klog.V(4).Infof("test %q", test.name)
+		base, err := ioutil.TempDir("", "remove-empty-dirs-"+test.name+"-")
+		if err != nil {
+			t.Fatalf(err.Error())
+		}
+		if err = test.prepare(base); err != nil {
+			os.RemoveAll(base)
+			t.Fatalf("failed to prepare test %q: %v", test.name, err.Error())
+		}
+
+		err = removeEmptyDirs(filepath.Join(base, test.baseDir), filepath.Join(base, test.endDir))
+		if err != nil && !test.expectError {
+			t.Errorf("test %q failed: %v", test.name, err)
+		}
+		if err == nil && test.expectError {
+			t.Errorf("test %q failed: expected error, got success", test.name)
+		}
+
+		if err = test.validate(base); err != nil {
+			t.Errorf("test %q failed validation: %v", test.name, err)
+		}
+
+		os.RemoveAll(base)
+	}
+}
+
+func TestCleanSubPaths(t *testing.T) {
+	defaultPerm := os.FileMode(0750)
+	testVol := "vol1"
+
+	tests := []struct {
+		name string
+		// Function that prepares directory structure for the test under given
+		// base.
+		prepare func(base string) ([]mount.MountPoint, error)
+		// Function that validates directory structure after the test
+		validate    func(base string) error
+		expectError bool
+	}{
+		{
+			name: "not-exists",
+			prepare: func(base string) ([]mount.MountPoint, error) {
+				return nil, nil
+			},
+			validate: func(base string) error {
+				return nil
+			},
+			expectError: false,
+		},
+		{
+			name: "subpath-not-mount",
+			prepare: func(base string) ([]mount.MountPoint, error) {
+				return nil, os.MkdirAll(filepath.Join(base, containerSubPathDirectoryName, testVol, "container1", "0"), defaultPerm)
+			},
+			validate: func(base string) error {
+				return validateDirNotExists(filepath.Join(base, containerSubPathDirectoryName))
+			},
+			expectError: false,
+		},
+		{
+			name: "subpath-file",
+			prepare: func(base string) ([]mount.MountPoint, error) {
+				path := filepath.Join(base, containerSubPathDirectoryName, testVol, "container1")
+				if err := os.MkdirAll(path, defaultPerm); err != nil {
+					return nil, err
+				}
+				return nil, ioutil.WriteFile(filepath.Join(path, "0"), []byte{}, defaultPerm)
+			},
+			validate: func(base string) error {
+				return validateDirNotExists(filepath.Join(base, containerSubPathDirectoryName))
+			},
+			expectError: false,
+		},
+		{
+			name: "subpath-container-not-dir",
+			prepare: func(base string) ([]mount.MountPoint, error) {
+				path := filepath.Join(base, containerSubPathDirectoryName, testVol)
+				if err := os.MkdirAll(path, defaultPerm); err != nil {
+					return nil, err
+				}
+				return nil, ioutil.WriteFile(filepath.Join(path, "container1"), []byte{}, defaultPerm)
+			},
+			validate: func(base string) error {
+				return validateDirExists(filepath.Join(base, containerSubPathDirectoryName, testVol))
+			},
+			expectError: true,
+		},
+		{
+			name: "subpath-multiple-container-not-dir",
+			prepare: func(base string) ([]mount.MountPoint, error) {
+				path := filepath.Join(base, containerSubPathDirectoryName, testVol)
+				if err := os.MkdirAll(filepath.Join(path, "container1"), defaultPerm); err != nil {
+					return nil, err
+				}
+				return nil, ioutil.WriteFile(filepath.Join(path, "container2"), []byte{}, defaultPerm)
+			},
+			validate: func(base string) error {
+				path := filepath.Join(base, containerSubPathDirectoryName, testVol)
+				if err := validateDirNotExists(filepath.Join(path, "container1")); err != nil {
+					return err
+				}
+				return validateFileExists(filepath.Join(path, "container2"))
+			},
+			expectError: true,
+		},
+		{
+			name: "subpath-mount",
+			prepare: func(base string) ([]mount.MountPoint, error) {
+				path := filepath.Join(base, containerSubPathDirectoryName, testVol, "container1", "0")
+				if err := os.MkdirAll(path, defaultPerm); err != nil {
+					return nil, err
+				}
+				mounts := []mount.MountPoint{{Device: "/dev/sdb", Path: path}}
+				return mounts, nil
+			},
+			validate: func(base string) error {
+				return validateDirNotExists(filepath.Join(base, containerSubPathDirectoryName))
+			},
+		},
+		{
+			name: "subpath-mount-multiple",
+			prepare: func(base string) ([]mount.MountPoint, error) {
+				path := filepath.Join(base, containerSubPathDirectoryName, testVol, "container1", "0")
+				path2 := filepath.Join(base, containerSubPathDirectoryName, testVol, "container1", "1")
+				path3 := filepath.Join(base, containerSubPathDirectoryName, testVol, "container2", "1")
+				if err := os.MkdirAll(path, defaultPerm); err != nil {
+					return nil, err
+				}
+				if err := os.MkdirAll(path2, defaultPerm); err != nil {
+					return nil, err
+				}
+				if err := os.MkdirAll(path3, defaultPerm); err != nil {
+					return nil, err
+				}
+				mounts := []mount.MountPoint{
+					{Device: "/dev/sdb", Path: path},
+					{Device: "/dev/sdb", Path: path3},
+				}
+				return mounts, nil
+			},
+			validate: func(base string) error {
+				return validateDirNotExists(filepath.Join(base, containerSubPathDirectoryName))
+			},
+		},
+		{
+			name: "subpath-mount-multiple-vols",
+			prepare: func(base string) ([]mount.MountPoint, error) {
+				path := filepath.Join(base, containerSubPathDirectoryName, testVol, "container1", "0")
+				path2 := filepath.Join(base, containerSubPathDirectoryName, "vol2", "container1", "1")
+				if err := os.MkdirAll(path, defaultPerm); err != nil {
+					return nil, err
+				}
+				if err := os.MkdirAll(path2, defaultPerm); err != nil {
+					return nil, err
+				}
+				mounts := []mount.MountPoint{
+					{Device: "/dev/sdb", Path: path},
+				}
+				return mounts, nil
+			},
+			validate: func(base string) error {
+				baseSubdir := filepath.Join(base, containerSubPathDirectoryName)
+				if err := validateDirNotExists(filepath.Join(baseSubdir, testVol)); err != nil {
+					return err
+				}
+				return validateDirExists(baseSubdir)
+			},
+		},
+	}
+
+	for _, test := range tests {
+		klog.V(4).Infof("test %q", test.name)
+		base, err := ioutil.TempDir("", "clean-subpaths-"+test.name+"-")
+		if err != nil {
+			t.Fatalf(err.Error())
+		}
+		mounts, err := test.prepare(base)
+		if err != nil {
+			os.RemoveAll(base)
+			t.Fatalf("failed to prepare test %q: %v", test.name, err.Error())
+		}
+
+		fm := &mount.FakeMounter{MountPoints: mounts}
+
+		err = doCleanSubPaths(fm, base, testVol)
+		if err != nil && !test.expectError {
+			t.Errorf("test %q failed: %v", test.name, err)
+		}
+		if err == nil && test.expectError {
+			t.Errorf("test %q failed: expected error, got success", test.name)
+		}
+		if err = test.validate(base); err != nil {
+			t.Errorf("test %q failed validation: %v", test.name, err)
+		}
+
+		os.RemoveAll(base)
+	}
+}
+
+var (
+	testVol       = "vol1"
+	testPod       = "pod0"
+	testContainer = "container0"
+	testSubpath   = 1
+)
+
+func setupFakeMounter(testMounts []string) *mount.FakeMounter {
+	mounts := []mount.MountPoint{}
+	for _, mountPoint := range testMounts {
+		mounts = append(mounts, mount.MountPoint{Device: "/foo", Path: mountPoint})
+	}
+	return &mount.FakeMounter{MountPoints: mounts}
+}
+
+func getTestPaths(base string) (string, string) {
+	return filepath.Join(base, testVol),
+		filepath.Join(base, testPod, containerSubPathDirectoryName, testVol, testContainer, strconv.Itoa(testSubpath))
+}
+
+func TestBindSubPath(t *testing.T) {
+	defaultPerm := os.FileMode(0750)
+
+	tests := []struct {
+		name string
+		// Function that prepares directory structure for the test under given
+		// base.
+		prepare     func(base string) ([]string, string, string, error)
+		expectError bool
+	}{
+		{
+			name: "subpath-dir",
+			prepare: func(base string) ([]string, string, string, error) {
+				volpath, _ := getTestPaths(base)
+				subpath := filepath.Join(volpath, "dir0")
+				return nil, volpath, subpath, os.MkdirAll(subpath, defaultPerm)
+			},
+			expectError: false,
+		},
+		{
+			name: "subpath-dir-symlink",
+			prepare: func(base string) ([]string, string, string, error) {
+				volpath, _ := getTestPaths(base)
+				subpath := filepath.Join(volpath, "dir0")
+				if err := os.MkdirAll(subpath, defaultPerm); err != nil {
+					return nil, "", "", err
+				}
+				subpathLink := filepath.Join(volpath, "dirLink")
+				return nil, volpath, subpath, os.Symlink(subpath, subpathLink)
+			},
+			expectError: false,
+		},
+		{
+			name: "subpath-file",
+			prepare: func(base string) ([]string, string, string, error) {
+				volpath, _ := getTestPaths(base)
+				subpath := filepath.Join(volpath, "file0")
+				if err := os.MkdirAll(volpath, defaultPerm); err != nil {
+					return nil, "", "", err
+				}
+				return nil, volpath, subpath, ioutil.WriteFile(subpath, []byte{}, defaultPerm)
+			},
+			expectError: false,
+		},
+		{
+			name: "subpath-not-exists",
+			prepare: func(base string) ([]string, string, string, error) {
+				volpath, _ := getTestPaths(base)
+				subpath := filepath.Join(volpath, "file0")
+				return nil, volpath, subpath, nil
+			},
+			expectError: true,
+		},
+		{
+			name: "subpath-outside",
+			prepare: func(base string) ([]string, string, string, error) {
+				volpath, _ := getTestPaths(base)
+				subpath := filepath.Join(volpath, "dir0")
+				if err := os.MkdirAll(volpath, defaultPerm); err != nil {
+					return nil, "", "", err
+				}
+				return nil, volpath, subpath, os.Symlink(base, subpath)
+			},
+			expectError: true,
+		},
+		{
+			name: "subpath-symlink-child-outside",
+			prepare: func(base string) ([]string, string, string, error) {
+				volpath, _ := getTestPaths(base)
+				subpathDir := filepath.Join(volpath, "dir0")
+				subpath := filepath.Join(subpathDir, "child0")
+				if err := os.MkdirAll(subpathDir, defaultPerm); err != nil {
+					return nil, "", "", err
+				}
+				return nil, volpath, subpath, os.Symlink(base, subpath)
+			},
+			expectError: true,
+		},
+		{
+			name: "subpath-child-outside-exists",
+			prepare: func(base string) ([]string, string, string, error) {
+				volpath, _ := getTestPaths(base)
+				subpathDir := filepath.Join(volpath, "dir0")
+				child := filepath.Join(base, "child0")
+				subpath := filepath.Join(subpathDir, "child0")
+				if err := os.MkdirAll(volpath, defaultPerm); err != nil {
+					return nil, "", "", err
+				}
+				// touch file outside
+				if err := ioutil.WriteFile(child, []byte{}, defaultPerm); err != nil {
+					return nil, "", "", err
+				}
+
+				// create symlink for subpath dir
+				return nil, volpath, subpath, os.Symlink(base, subpathDir)
+			},
+			expectError: true,
+		},
+		{
+			name: "subpath-child-outside-not-exists",
+			prepare: func(base string) ([]string, string, string, error) {
+				volpath, _ := getTestPaths(base)
+				subpathDir := filepath.Join(volpath, "dir0")
+				subpath := filepath.Join(subpathDir, "child0")
+				if err := os.MkdirAll(volpath, defaultPerm); err != nil {
+					return nil, "", "", err
+				}
+				// create symlink for subpath dir
+				return nil, volpath, subpath, os.Symlink(base, subpathDir)
+			},
+			expectError: true,
+		},
+		{
+			name: "subpath-child-outside-exists-middle-dir-symlink",
+			prepare: func(base string) ([]string, string, string, error) {
+				volpath, _ := getTestPaths(base)
+				subpathDir := filepath.Join(volpath, "dir0")
+				symlinkDir := filepath.Join(subpathDir, "linkDir0")
+				child := filepath.Join(base, "child0")
+				subpath := filepath.Join(symlinkDir, "child0")
+				if err := os.MkdirAll(subpathDir, defaultPerm); err != nil {
+					return nil, "", "", err
+				}
+				// touch file outside
+				if err := ioutil.WriteFile(child, []byte{}, defaultPerm); err != nil {
+					return nil, "", "", err
+				}
+
+				// create symlink for middle dir
+				return nil, volpath, subpath, os.Symlink(base, symlinkDir)
+			},
+			expectError: true,
+		},
+		{
+			name: "subpath-backstepping",
+			prepare: func(base string) ([]string, string, string, error) {
+				volpath, _ := getTestPaths(base)
+				subpath := filepath.Join(volpath, "dir0")
+				symlinkBase := filepath.Join(volpath, "..")
+				if err := os.MkdirAll(volpath, defaultPerm); err != nil {
+					return nil, "", "", err
+				}
+
+				// create symlink for subpath
+				return nil, volpath, subpath, os.Symlink(symlinkBase, subpath)
+			},
+			expectError: true,
+		},
+		{
+			name: "subpath-mountdir-already-exists",
+			prepare: func(base string) ([]string, string, string, error) {
+				volpath, subpathMount := getTestPaths(base)
+				if err := os.MkdirAll(subpathMount, defaultPerm); err != nil {
+					return nil, "", "", err
+				}
+
+				subpath := filepath.Join(volpath, "dir0")
+				return nil, volpath, subpath, os.MkdirAll(subpath, defaultPerm)
+			},
+			expectError: false,
+		},
+		{
+			name: "subpath-mount-already-exists",
+			prepare: func(base string) ([]string, string, string, error) {
+				volpath, subpathMount := getTestPaths(base)
+				mounts := []string{subpathMount}
+				if err := os.MkdirAll(subpathMount, defaultPerm); err != nil {
+					return nil, "", "", err
+				}
+
+				subpath := filepath.Join(volpath, "dir0")
+				return mounts, volpath, subpath, os.MkdirAll(subpath, defaultPerm)
+			},
+			expectError: false,
+		},
+		{
+			name: "mount-unix-socket",
+			prepare: func(base string) ([]string, string, string, error) {
+				volpath, subpathMount := getTestPaths(base)
+				mounts := []string{subpathMount}
+				if err := os.MkdirAll(volpath, defaultPerm); err != nil {
+					return nil, "", "", err
+				}
+
+				socketFile, socketCreateError := createSocketFile(volpath)
+
+				return mounts, volpath, socketFile, socketCreateError
+			},
+			expectError: false,
+		},
+		{
+			name: "subpath-mounting-fifo",
+			prepare: func(base string) ([]string, string, string, error) {
+				volpath, subpathMount := getTestPaths(base)
+				mounts := []string{subpathMount}
+				if err := os.MkdirAll(volpath, defaultPerm); err != nil {
+					return nil, "", "", err
+				}
+
+				testFifo := filepath.Join(volpath, "mount_test.fifo")
+				err := syscall.Mkfifo(testFifo, 0)
+				return mounts, volpath, testFifo, err
+			},
+			expectError: false,
+		},
+	}
+
+	for _, test := range tests {
+		klog.V(4).Infof("test %q", test.name)
+		base, err := ioutil.TempDir("", "bind-subpath-"+test.name+"-")
+		if err != nil {
+			t.Fatalf(err.Error())
+		}
+
+		mounts, volPath, subPath, err := test.prepare(base)
+		if err != nil {
+			os.RemoveAll(base)
+			t.Fatalf("failed to prepare test %q: %v", test.name, err.Error())
+		}
+
+		fm := setupFakeMounter(mounts)
+
+		subpath := Subpath{
+			VolumeMountIndex: testSubpath,
+			Path:             subPath,
+			VolumeName:       testVol,
+			VolumePath:       volPath,
+			PodDir:           filepath.Join(base, "pod0"),
+			ContainerName:    testContainer,
+		}
+
+		_, subpathMount := getTestPaths(base)
+		bindPathTarget, err := doBindSubPath(fm, subpath)
+		if test.expectError {
+			if err == nil {
+				t.Errorf("test %q failed: expected error, got success", test.name)
+			}
+			if bindPathTarget != "" {
+				t.Errorf("test %q failed: expected empty bindPathTarget, got %v", test.name, bindPathTarget)
+			}
+			if err = validateDirNotExists(subpathMount); err != nil {
+				t.Errorf("test %q failed: %v", test.name, err)
+			}
+		}
+		if !test.expectError {
+			if err != nil {
+				t.Errorf("test %q failed: %v", test.name, err)
+			}
+			if bindPathTarget != subpathMount {
+				t.Errorf("test %q failed: expected bindPathTarget %v, got %v", test.name, subpathMount, bindPathTarget)
+			}
+			if err = validateFileExists(subpathMount); err != nil {
+				t.Errorf("test %q failed: %v", test.name, err)
+			}
+		}
+
+		os.RemoveAll(base)
+	}
+}
+
+func TestSafeOpen(t *testing.T) {
+	defaultPerm := os.FileMode(0750)
+
+	tests := []struct {
+		name string
+		// Function that prepares directory structure for the test under given
+		// base.
+		prepare     func(base string) error
+		path        string
+		expectError bool
+	}{
+		{
+			"directory-does-not-exist",
+			func(base string) error {
+				return nil
+			},
+			"test/directory",
+			true,
+		},
+		{
+			"directory-exists",
+			func(base string) error {
+				return os.MkdirAll(filepath.Join(base, "test/directory"), 0750)
+			},
+			"test/directory",
+			false,
+		},
+		{
+			"escape-base-using-dots",
+			func(base string) error {
+				return nil
+			},
+			"..",
+			true,
+		},
+		{
+			"escape-base-using-dots-2",
+			func(base string) error {
+				return os.MkdirAll(filepath.Join(base, "test"), 0750)
+			},
+			"test/../../..",
+			true,
+		},
+		{
+			"symlink",
+			func(base string) error {
+				if err := os.MkdirAll(filepath.Join(base, "destination"), defaultPerm); err != nil {
+					return err
+				}
+				return os.Symlink("destination", filepath.Join(base, "test"))
+			},
+			"test",
+			true,
+		},
+		{
+			"symlink-nested",
+			func(base string) error {
+				if err := os.MkdirAll(filepath.Join(base, "dir1/dir2"), defaultPerm); err != nil {
+					return err
+				}
+				return os.Symlink("dir1", filepath.Join(base, "dir1/dir2/test"))
+			},
+			"test",
+			true,
+		},
+		{
+			"symlink-loop",
+			func(base string) error {
+				return os.Symlink("test", filepath.Join(base, "test"))
+			},
+			"test",
+			true,
+		},
+		{
+			"symlink-not-exists",
+			func(base string) error {
+				return os.Symlink("non-existing", filepath.Join(base, "test"))
+			},
+			"test",
+			true,
+		},
+		{
+			"non-directory",
+			func(base string) error {
+				return ioutil.WriteFile(filepath.Join(base, "test"), []byte{}, defaultPerm)
+			},
+			"test/directory",
+			true,
+		},
+		{
+			"non-directory-final",
+			func(base string) error {
+				return ioutil.WriteFile(filepath.Join(base, "test"), []byte{}, defaultPerm)
+			},
+			"test",
+			false,
+		},
+		{
+			"escape-with-relative-symlink",
+			func(base string) error {
+				if err := os.MkdirAll(filepath.Join(base, "dir"), defaultPerm); err != nil {
+					return err
+				}
+				if err := os.MkdirAll(filepath.Join(base, "exists"), defaultPerm); err != nil {
+					return err
+				}
+				return os.Symlink("../exists", filepath.Join(base, "dir/test"))
+			},
+			"dir/test",
+			true,
+		},
+		{
+			"escape-with-relative-symlink-not-exists",
+			func(base string) error {
+				if err := os.MkdirAll(filepath.Join(base, "dir"), defaultPerm); err != nil {
+					return err
+				}
+				return os.Symlink("../not-exists", filepath.Join(base, "dir/test"))
+			},
+			"dir/test",
+			true,
+		},
+		{
+			"escape-with-symlink",
+			func(base string) error {
+				return os.Symlink("/", filepath.Join(base, "test"))
+			},
+			"test",
+			true,
+		},
+		{
+			"mount-unix-socket",
+			func(base string) error {
+				socketFile, socketError := createSocketFile(base)
+
+				if socketError != nil {
+					return fmt.Errorf("Error preparing socket file %s with %v", socketFile, socketError)
+				}
+				return nil
+			},
+			"mt.sock",
+			false,
+		},
+		{
+			"mounting-unix-socket-in-middle",
+			func(base string) error {
+				testSocketFile, socketError := createSocketFile(base)
+
+				if socketError != nil {
+					return fmt.Errorf("Error preparing socket file %s with %v", testSocketFile, socketError)
+				}
+				return nil
+			},
+			"mt.sock/bar",
+			true,
+		},
+	}
+
+	for _, test := range tests {
+		klog.V(4).Infof("test %q", test.name)
+		base, err := ioutil.TempDir("", "safe-open-"+test.name+"-")
+		if err != nil {
+			t.Fatalf(err.Error())
+		}
+
+		test.prepare(base)
+		pathToCreate := filepath.Join(base, test.path)
+		fd, err := doSafeOpen(pathToCreate, base)
+		if err != nil && !test.expectError {
+			t.Errorf("test %q: %s", test.name, err)
+		}
+		if err != nil {
+			klog.Infof("got error: %s", err)
+		}
+		if err == nil && test.expectError {
+			t.Errorf("test %q: expected error, got none", test.name)
+		}
+
+		syscall.Close(fd)
+		os.RemoveAll(base)
+	}
+}
+
+func createSocketFile(socketDir string) (string, error) {
+	testSocketFile := filepath.Join(socketDir, "mt.sock")
+
+	// Switch to volume path and create the socket file
+	// socket file can not have length of more than 108 character
+	// and hence we must use relative path
+	oldDir, _ := os.Getwd()
+
+	err := os.Chdir(socketDir)
+	if err != nil {
+		return "", err
+	}
+	defer func() {
+		os.Chdir(oldDir)
+	}()
+	_, socketCreateError := net.Listen("unix", "mt.sock")
+	return testSocketFile, socketCreateError
+}
+
+func TestFindExistingPrefix(t *testing.T) {
+	defaultPerm := os.FileMode(0750)
+	tests := []struct {
+		name string
+		// Function that prepares directory structure for the test under given
+		// base.
+		prepare      func(base string) error
+		path         string
+		expectedPath string
+		expectedDirs []string
+		expectError  bool
+	}{
+		{
+			"directory-does-not-exist",
+			func(base string) error {
+				return nil
+			},
+			"directory",
+			"",
+			[]string{"directory"},
+			false,
+		},
+		{
+			"directory-exists",
+			func(base string) error {
+				return os.MkdirAll(filepath.Join(base, "test/directory"), 0750)
+			},
+			"test/directory",
+			"test/directory",
+			[]string{},
+			false,
+		},
+		{
+			"follow-symlinks",
+			func(base string) error {
+				if err := os.MkdirAll(filepath.Join(base, "destination/directory"), defaultPerm); err != nil {
+					return err
+				}
+				return os.Symlink("destination", filepath.Join(base, "test"))
+			},
+			"test/directory",
+			"test/directory",
+			[]string{},
+			false,
+		},
+		{
+			"follow-symlink-loop",
+			func(base string) error {
+				return os.Symlink("test", filepath.Join(base, "test"))
+			},
+			"test/directory",
+			"",
+			nil,
+			true,
+		},
+		{
+			"follow-symlink-multiple follow",
+			func(base string) error {
+				/* test1/dir points to test2 and test2/dir points to test1 */
+				if err := os.MkdirAll(filepath.Join(base, "test1"), defaultPerm); err != nil {
+					return err
+				}
+				if err := os.MkdirAll(filepath.Join(base, "test2"), defaultPerm); err != nil {
+					return err
+				}
+				if err := os.Symlink(filepath.Join(base, "test2"), filepath.Join(base, "test1/dir")); err != nil {
+					return err
+				}
+				if err := os.Symlink(filepath.Join(base, "test1"), filepath.Join(base, "test2/dir")); err != nil {
+					return err
+				}
+				return nil
+			},
+			"test1/dir/dir/foo/bar",
+			"test1/dir/dir",
+			[]string{"foo", "bar"},
+			false,
+		},
+		{
+			"danglink-symlink",
+			func(base string) error {
+				return os.Symlink("non-existing", filepath.Join(base, "test"))
+			},
+			// OS returns IsNotExist error both for dangling symlink and for
+			// non-existing directory.
+			"test/directory",
+			"",
+			[]string{"test", "directory"},
+			false,
+		},
+		{
+			"with-fifo-in-middle",
+			func(base string) error {
+				testFifo := filepath.Join(base, "mount_test.fifo")
+				return syscall.Mkfifo(testFifo, 0)
+			},
+			"mount_test.fifo/directory",
+			"",
+			nil,
+			true,
+		},
+	}
+
+	for _, test := range tests {
+		klog.V(4).Infof("test %q", test.name)
+		base, err := ioutil.TempDir("", "find-prefix-"+test.name+"-")
+		if err != nil {
+			t.Fatalf(err.Error())
+		}
+		test.prepare(base)
+		path := filepath.Join(base, test.path)
+		existingPath, dirs, err := findExistingPrefix(base, path)
+		if err != nil && !test.expectError {
+			t.Errorf("test %q: %s", test.name, err)
+		}
+		if err != nil {
+			klog.Infof("got error: %s", err)
+		}
+		if err == nil && test.expectError {
+			t.Errorf("test %q: expected error, got none", test.name)
+		}
+
+		fullExpectedPath := filepath.Join(base, test.expectedPath)
+		if existingPath != fullExpectedPath {
+			t.Errorf("test %q: expected path %q, got %q", test.name, fullExpectedPath, existingPath)
+		}
+		if !reflect.DeepEqual(dirs, test.expectedDirs) {
+			t.Errorf("test %q: expected dirs %v, got %v", test.name, test.expectedDirs, dirs)
+		}
+		os.RemoveAll(base)
+	}
+}
+
+func validateDirEmpty(dir string) error {
+	files, err := ioutil.ReadDir(dir)
+	if err != nil {
+		return err
+	}
+
+	if len(files) != 0 {
+		return fmt.Errorf("Directory %q is not empty", dir)
+	}
+	return nil
+}
+
+func validateDirExists(dir string) error {
+	_, err := ioutil.ReadDir(dir)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func validateDirNotExists(dir string) error {
+	_, err := ioutil.ReadDir(dir)
+	if os.IsNotExist(err) {
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+	return fmt.Errorf("dir %q still exists", dir)
+}
+
+func validateFileExists(file string) error {
+	if _, err := os.Stat(file); err != nil {
+		return err
+	}
+	return nil
+}

pkg/volume/util/subpath/subpath_nsenter.go

+// +build linux
+
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package subpath
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"syscall"
+
+	"golang.org/x/sys/unix"
+
+	"k8s.io/klog"
+	"k8s.io/utils/nsenter"
+
+	"k8s.io/kubernetes/pkg/util/mount"
+)
+
+type subpathNSE struct {
+	mounter mount.Interface
+	ne      *nsenter.Nsenter
+	rootDir string
+}
+
+// Compile time-check for all implementers of subpath interface
+var _ Interface = &subpathNSE{}
+
+// NewNSEnter returns a subpath.Interface that is to be used with the NsenterMounter
+// It is only valid on Linux systems
+func NewNSEnter(mounter mount.Interface, ne *nsenter.Nsenter, rootDir string) Interface {
+	return &subpathNSE{
+		mounter: mounter,
+		ne:      ne,
+		rootDir: rootDir,
+	}
+}
+
+func (sp *subpathNSE) CleanSubPaths(podDir string, volumeName string) error {
+	return doCleanSubPaths(sp.mounter, podDir, volumeName)
+}
+
+func (sp *subpathNSE) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
+	// Bind-mount the subpath to avoid using symlinks in subpaths.
+	newHostPath, err = sp.doNsEnterBindSubPath(subPath)
+
+	// There is no action when the container starts. Bind-mount will be cleaned
+	// when container stops by CleanSubPaths.
+	cleanupAction = nil
+	return newHostPath, cleanupAction, err
+}
+
+func (sp *subpathNSE) SafeMakeDir(subdir string, base string, perm os.FileMode) error {
+	fullSubdirPath := filepath.Join(base, subdir)
+	evaluatedSubdirPath, err := sp.ne.EvalSymlinks(fullSubdirPath, false /* mustExist */)
+	if err != nil {
+		return fmt.Errorf("error resolving symlinks in %s: %s", fullSubdirPath, err)
+	}
+	evaluatedSubdirPath = filepath.Clean(evaluatedSubdirPath)
+
+	evaluatedBase, err := sp.ne.EvalSymlinks(base, true /* mustExist */)
+	if err != nil {
+		return fmt.Errorf("error resolving symlinks in %s: %s", base, err)
+	}
+	evaluatedBase = filepath.Clean(evaluatedBase)
+
+	rootDir := filepath.Clean(sp.rootDir)
+	if mount.PathWithinBase(evaluatedBase, rootDir) {
+		// Base is in /var/lib/kubelet. This directory is shared between the
+		// container with kubelet and the host. We don't need to add '/rootfs'.
+		// This is useful when /rootfs is mounted as read-only - we can still
+		// create subpaths for paths in /var/lib/kubelet.
+		return doSafeMakeDir(evaluatedSubdirPath, evaluatedBase, perm)
+	}
+
+	// Base is somewhere on the host's filesystem. Add /rootfs and try to make
+	// the directory there.
+	// This requires /rootfs to be writable.
+	kubeletSubdirPath := sp.ne.KubeletPath(evaluatedSubdirPath)
+	kubeletBase := sp.ne.KubeletPath(evaluatedBase)
+	return doSafeMakeDir(kubeletSubdirPath, kubeletBase, perm)
+}
+
+func (sp *subpathNSE) doNsEnterBindSubPath(subpath Subpath) (hostPath string, err error) {
+	// Linux, kubelet runs in a container:
+	// - safely open the subpath
+	// - bind-mount the subpath to target (this can be unsafe)
+	// - check that we mounted the right thing by comparing device ID and inode
+	//   of the subpath (via safely opened fd) and the target (that's under our
+	//   control)
+
+	// Evaluate all symlinks here once for all subsequent functions.
+	evaluatedHostVolumePath, err := sp.ne.EvalSymlinks(subpath.VolumePath, true /*mustExist*/)
+	if err != nil {
+		return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.VolumePath, err)
+	}
+	evaluatedHostSubpath, err := sp.ne.EvalSymlinks(subpath.Path, true /*mustExist*/)
+	if err != nil {
+		return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.Path, err)
+	}
+	klog.V(5).Infof("doBindSubPath %q (%q) for volumepath %q", subpath.Path, evaluatedHostSubpath, subpath.VolumePath)
+	subpath.VolumePath = sp.ne.KubeletPath(evaluatedHostVolumePath)
+	subpath.Path = sp.ne.KubeletPath(evaluatedHostSubpath)
+
+	// Check the subpath is correct and open it
+	fd, err := safeOpenSubPath(sp.mounter, subpath)
+	if err != nil {
+		return "", err
+	}
+	defer syscall.Close(fd)
+
+	alreadyMounted, bindPathTarget, err := prepareSubpathTarget(sp.mounter, subpath)
+	if err != nil {
+		return "", err
+	}
+	if alreadyMounted {
+		return bindPathTarget, nil
+	}
+
+	success := false
+	defer func() {
+		// Cleanup subpath on error
+		if !success {
+			klog.V(4).Infof("doNsEnterBindSubPath() failed for %q, cleaning up subpath", bindPathTarget)
+			if cleanErr := cleanSubPath(sp.mounter, subpath); cleanErr != nil {
+				klog.Errorf("Failed to clean subpath %q: %v", bindPathTarget, cleanErr)
+			}
+		}
+	}()
+
+	// Leap of faith: optimistically expect that nobody has modified previously
+	// expanded evalSubPath with evil symlinks and bind-mount it.
+	// Mount is done on the host! don't use kubelet path!
+	klog.V(5).Infof("bind mounting %q at %q", evaluatedHostSubpath, bindPathTarget)
+	if err = sp.mounter.Mount(evaluatedHostSubpath, bindPathTarget, "" /*fstype*/, []string{"bind"}); err != nil {
+		return "", fmt.Errorf("error mounting %s: %s", evaluatedHostSubpath, err)
+	}
+
+	// Check that the bind-mount target is the same inode and device as the
+	// source that we keept open, i.e. we mounted the right thing.
+	err = checkDeviceInode(fd, bindPathTarget)
+	if err != nil {
+		return "", fmt.Errorf("error checking bind mount for subpath %s: %s", subpath.VolumePath, err)
+	}
+
+	success = true
+	klog.V(3).Infof("Bound SubPath %s into %s", subpath.Path, bindPathTarget)
+	return bindPathTarget, nil
+}
+
+// checkDeviceInode checks that opened file and path represent the same file.
+func checkDeviceInode(fd int, path string) error {
+	var srcStat, dstStat unix.Stat_t
+	err := unix.Fstat(fd, &srcStat)
+	if err != nil {
+		return fmt.Errorf("error running fstat on subpath FD: %v", err)
+	}
+
+	err = unix.Stat(path, &dstStat)
+	if err != nil {
+		return fmt.Errorf("error running fstat on %s: %v", path, err)
+	}
+
+	if srcStat.Dev != dstStat.Dev {
+		return fmt.Errorf("different device number")
+	}
+	if srcStat.Ino != dstStat.Ino {
+		return fmt.Errorf("different inode")
+	}
+	return nil
+}

pkg/volume/util/subpath/subpath_nsenter_test.go

+// +build linux
+
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package subpath
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"golang.org/x/sys/unix"
+
+	"k8s.io/utils/nsenter"
+
+	"k8s.io/kubernetes/pkg/util/mount"
+)
+
+func TestCheckDeviceInode(t *testing.T) {
+	testDir, err := ioutil.TempDir("", "nsenter-mounter-device-")
+	if err != nil {
+		t.Fatalf("Cannot create temporary directory: %s", err)
+	}
+	defer os.RemoveAll(testDir)
+
+	tests := []struct {
+		name        string
+		srcPath     string
+		dstPath     string
+		expectError string
+	}{
+		{
+			name:        "the same file",
+			srcPath:     filepath.Join(testDir, "1"),
+			dstPath:     filepath.Join(testDir, "1"),
+			expectError: "",
+		},
+		{
+			name:        "different file on the same FS",
+			srcPath:     filepath.Join(testDir, "2.1"),
+			dstPath:     filepath.Join(testDir, "2.2"),
+			expectError: "different inode",
+		},
+		{
+			name:    "different file on different device",
+			srcPath: filepath.Join(testDir, "3"),
+			// /proc is always on a different "device" than /tmp (or $TEMP)
+			dstPath:     "/proc/self/status",
+			expectError: "different device",
+		},
+	}
+
+	for _, test := range tests {
+		if err := ioutil.WriteFile(test.srcPath, []byte{}, 0644); err != nil {
+			t.Errorf("Test %q: cannot create srcPath %s: %s", test.name, test.srcPath, err)
+			continue
+		}
+
+		// Don't create dst if it exists
+		if _, err := os.Stat(test.dstPath); os.IsNotExist(err) {
+			if err := ioutil.WriteFile(test.dstPath, []byte{}, 0644); err != nil {
+				t.Errorf("Test %q: cannot create dstPath %s: %s", test.name, test.dstPath, err)
+				continue
+			}
+		} else if err != nil {
+			t.Errorf("Test %q: cannot check existence of dstPath %s: %s", test.name, test.dstPath, err)
+			continue
+		}
+
+		fd, err := unix.Open(test.srcPath, unix.O_CREAT, 0644)
+		if err != nil {
+			t.Errorf("Test %q: cannot open srcPath %s: %s", test.name, test.srcPath, err)
+			continue
+		}
+
+		err = checkDeviceInode(fd, test.dstPath)
+
+		if test.expectError == "" && err != nil {
+			t.Errorf("Test %q: expected no error, got %s", test.name, err)
+		}
+		if test.expectError != "" {
+			if err == nil {
+				t.Errorf("Test %q: expected error, got none", test.name)
+			} else {
+				if !strings.Contains(err.Error(), test.expectError) {
+					t.Errorf("Test %q: expected error %q, got %q", test.name, test.expectError, err)
+				}
+			}
+		}
+	}
+}
+
+func newFakeNsenterMounter(tmpdir string, t *testing.T) (*mount.NsenterMounter, string, string, *nsenter.Nsenter, error) {
+	rootfsPath := filepath.Join(tmpdir, "rootfs")
+	if err := os.Mkdir(rootfsPath, 0755); err != nil {
+		return nil, "", "", nil, err
+	}
+	ne, err := nsenter.NewFakeNsenter(rootfsPath)
+	if err != nil {
+		return nil, "", "", nil, err
+	}
+
+	varlibPath := filepath.Join(tmpdir, "/var/lib/kubelet")
+	if err := os.MkdirAll(varlibPath, 0755); err != nil {
+		return nil, "", "", nil, err
+	}
+
+	return mount.NewNsenterMounter(varlibPath, ne), rootfsPath, varlibPath, ne, nil
+}
+
+func TestNsenterSafeMakeDir(t *testing.T) {
+	tests := []struct {
+		name        string
+		prepare     func(base, rootfs, varlib string) (expectedDir string, err error)
+		subdir      string
+		expectError bool
+		// If true, "base" directory for SafeMakeDir will be /var/lib/kubelet
+		baseIsVarLib bool
+	}{
+		{
+			name: "simple directory",
+			// evaluated in base
+			subdir: "some/subdirectory/structure",
+			prepare: func(base, rootfs, varlib string) (expectedDir string, err error) {
+				// expected to be created in /roots/
+				expectedDir = filepath.Join(rootfs, base, "some/subdirectory/structure")
+				return expectedDir, nil
+			},
+		},
+		{
+			name: "simple existing directory",
+			// evaluated in base
+			subdir: "some/subdirectory/structure",
+			prepare: func(base, rootfs, varlib string) (expectedDir string, err error) {
+				// On the host: directory exists
+				hostPath := filepath.Join(base, "some/subdirectory/structure")
+				if err := os.MkdirAll(hostPath, 0755); err != nil {
+					return "", err
+				}
+				// In rootfs: directory exists
+				kubeletPath := filepath.Join(rootfs, hostPath)
+				if err := os.MkdirAll(kubeletPath, 0755); err != nil {
+					return "", err
+				}
+				// expected to be created in /roots/
+				expectedDir = kubeletPath
+				return expectedDir, nil
+			},
+		},
+		{
+			name: "absolute symlink into safe place",
+			// evaluated in base
+			subdir: "some/subdirectory/structure",
+			prepare: func(base, rootfs, varlib string) (expectedDir string, err error) {
+				// On the host: /base/other/subdirectory exists, /base/some is link to /base/other
+				hostPath := filepath.Join(base, "other/subdirectory")
+				if err := os.MkdirAll(hostPath, 0755); err != nil {
+					return "", err
+				}
+				somePath := filepath.Join(base, "some")
+				otherPath := filepath.Join(base, "other")
+				if err := os.Symlink(otherPath, somePath); err != nil {
+					return "", err
+				}
+
+				// In rootfs: /base/other/subdirectory exists
+				kubeletPath := filepath.Join(rootfs, hostPath)
+				if err := os.MkdirAll(kubeletPath, 0755); err != nil {
+					return "", err
+				}
+				// expected 'structure' to be created
+				expectedDir = filepath.Join(rootfs, hostPath, "structure")
+				return expectedDir, nil
+			},
+		},
+		{
+			name: "relative symlink into safe place",
+			// evaluated in base
+			subdir: "some/subdirectory/structure",
+			prepare: func(base, rootfs, varlib string) (expectedDir string, err error) {
+				// On the host: /base/other/subdirectory exists, /base/some is link to other
+				hostPath := filepath.Join(base, "other/subdirectory")
+				if err := os.MkdirAll(hostPath, 0755); err != nil {
+					return "", err
+				}
+				somePath := filepath.Join(base, "some")
+				if err := os.Symlink("other", somePath); err != nil {
+					return "", err
+				}
+
+				// In rootfs: /base/other/subdirectory exists
+				kubeletPath := filepath.Join(rootfs, hostPath)
+				if err := os.MkdirAll(kubeletPath, 0755); err != nil {
+					return "", err
+				}
+				// expected 'structure' to be created
+				expectedDir = filepath.Join(rootfs, hostPath, "structure")
+				return expectedDir, nil
+			},
+		},
+		{
+			name: "symlink into unsafe place",
+			// evaluated in base
+			subdir: "some/subdirectory/structure",
+			prepare: func(base, rootfs, varlib string) (expectedDir string, err error) {
+				// On the host: /base/some is link to /bin/other
+				somePath := filepath.Join(base, "some")
+				if err := os.Symlink("/bin", somePath); err != nil {
+					return "", err
+				}
+				return "", nil
+			},
+			expectError: true,
+		},
+		{
+			name: "simple directory in /var/lib/kubelet",
+			// evaluated in varlib
+			subdir:       "some/subdirectory/structure",
+			baseIsVarLib: true,
+			prepare: func(base, rootfs, varlib string) (expectedDir string, err error) {
+				// expected to be created in /base/var/lib/kubelet, not in /rootfs!
+				expectedDir = filepath.Join(varlib, "some/subdirectory/structure")
+				return expectedDir, nil
+			},
+		},
+		{
+			name: "safe symlink in /var/lib/kubelet",
+			// evaluated in varlib
+			subdir:       "some/subdirectory/structure",
+			baseIsVarLib: true,
+			prepare: func(base, rootfs, varlib string) (expectedDir string, err error) {
+				// On the host: /varlib/kubelet/other/subdirectory exists, /varlib/some is link to other
+				hostPath := filepath.Join(varlib, "other/subdirectory")
+				if err := os.MkdirAll(hostPath, 0755); err != nil {
+					return "", err
+				}
+				somePath := filepath.Join(varlib, "some")
+				if err := os.Symlink("other", somePath); err != nil {
+					return "", err
+				}
+
+				// expected to be created in /base/var/lib/kubelet, not in /rootfs!
+				expectedDir = filepath.Join(varlib, "other/subdirectory/structure")
+				return expectedDir, nil
+			},
+		},
+		{
+			name: "unsafe symlink in /var/lib/kubelet",
+			// evaluated in varlib
+			subdir:       "some/subdirectory/structure",
+			baseIsVarLib: true,
+			prepare: func(base, rootfs, varlib string) (expectedDir string, err error) {
+				// On the host: /varlib/some is link to /bin
+				somePath := filepath.Join(varlib, "some")
+				if err := os.Symlink("/bin", somePath); err != nil {
+					return "", err
+				}
+
+				return "", nil
+			},
+			expectError: true,
+		},
+	}
+	for _, test := range tests {
+		tmpdir, err := ioutil.TempDir("", "nsenter-get-safedir-")
+		if err != nil {
+			t.Error(err)
+			continue
+		}
+		defer os.RemoveAll(tmpdir)
+
+		mounter, rootfs, varlib, ne, err := newFakeNsenterMounter(tmpdir, t)
+		if err != nil {
+			t.Error(err)
+			continue
+		}
+
+		fsp := NewNSEnter(mounter, ne, varlib)
+
+		// Prepare base directory for the test
+		testBase := filepath.Join(tmpdir, "base")
+		if err := os.Mkdir(testBase, 0755); err != nil {
+			t.Error(err)
+			continue
+		}
+		// Prepare base directory also in /rootfs
+		rootfsBase := filepath.Join(rootfs, testBase)
+		if err := os.MkdirAll(rootfsBase, 0755); err != nil {
+			t.Error(err)
+			continue
+		}
+
+		expectedDir := ""
+		if test.prepare != nil {
+			expectedDir, err = test.prepare(testBase, rootfs, varlib)
+			if err != nil {
+				t.Error(err)
+				continue
+			}
+		}
+
+		if test.baseIsVarLib {
+			// use /var/lib/kubelet as the test base so we can test creating
+			// subdirs there directly in /var/lib/kubenet and not in
+			// /rootfs/var/lib/kubelet
+			testBase = varlib
+		}
+
+		err = fsp.SafeMakeDir(test.subdir, testBase, 0755)
+		if err != nil && !test.expectError {
+			t.Errorf("Test %q: unexpected error: %s", test.name, err)
+		}
+		if test.expectError {
+			if err == nil {
+				t.Errorf("Test %q: expected error, got none", test.name)
+			} else {
+				if !strings.Contains(err.Error(), "is outside of allowed base") {
+					t.Errorf("Test %q: expected error to contain \"is outside of allowed base\", got this one instead: %s", test.name, err)
+				}
+			}
+		}
+
+		if expectedDir != "" {
+			_, err := os.Stat(expectedDir)
+			if err != nil {
+				t.Errorf("Test %q: expected %q to exist, got error: %s", test.name, expectedDir, err)
+			}
+		}
+	}
+}

pkg/volume/util/subpath/subpath_unsupported.go

+// +build !linux,!windows
+
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package subpath
+
+import (
+	"errors"
+	"os"
+
+	"k8s.io/kubernetes/pkg/util/mount"
+	"k8s.io/utils/nsenter"
+)
+
+type subpath struct{}
+
+var errUnsupported = errors.New("util/subpath on this platform is not supported")
+
+// New returns a subpath.Interface for the current system.
+func New(mount.Interface) Interface {
+	return &subpath{}
+}
+
+// NewNSEnter is to satisfy the compiler for having NewSubpathNSEnter exist for all
+// OS choices. however, NSEnter is only valid on Linux
+func NewNSEnter(mounter mount.Interface, ne *nsenter.Nsenter, rootDir string) Interface {
+	return nil
+}
+
+func (sp *subpath) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
+	return subPath.Path, nil, errUnsupported
+}
+
+func (sp *subpath) CleanSubPaths(podDir string, volumeName string) error {
+	return errUnsupported
+}
+
+func (sp *subpath) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
+	return errUnsupported
+}

pkg/volume/util/subpath/subpath_windows.go

+// +build windows
+
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package subpath
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"syscall"
+
+	"k8s.io/klog"
+	"k8s.io/kubernetes/pkg/util/mount"
+	"k8s.io/utils/nsenter"
+)
+
+type subpath struct{}
+
+// New returns a subpath.Interface for the current system
+func New(mount.Interface) Interface {
+	return &subpath{}
+}
+
+// NewNSEnter is to satisfy the compiler for having NewSubpathNSEnter exist for all
+// OS choices. however, NSEnter is only valid on Linux
+func NewNSEnter(mounter mount.Interface, ne *nsenter.Nsenter, rootDir string) Interface {
+	return nil
+}
+
+// check whether hostPath is within volume path
+// this func will lock all intermediate subpath directories, need to close handle outside of this func after container started
+func lockAndCheckSubPath(volumePath, hostPath string) ([]uintptr, error) {
+	if len(volumePath) == 0 || len(hostPath) == 0 {
+		return []uintptr{}, nil
+	}
+
+	finalSubPath, err := filepath.EvalSymlinks(hostPath)
+	if err != nil {
+		return []uintptr{}, fmt.Errorf("cannot read link %s: %s", hostPath, err)
+	}
+	finalVolumePath, err := filepath.EvalSymlinks(volumePath)
+	if err != nil {
+		return []uintptr{}, fmt.Errorf("cannot read link %s: %s", volumePath, err)
+	}
+
+	return lockAndCheckSubPathWithoutSymlink(finalVolumePath, finalSubPath)
+}
+
+// lock all intermediate subPath directories and check they are all within volumePath
+// volumePath & subPath should not contain any symlink, otherwise it will return error
+func lockAndCheckSubPathWithoutSymlink(volumePath, subPath string) ([]uintptr, error) {
+	if len(volumePath) == 0 || len(subPath) == 0 {
+		return []uintptr{}, nil
+	}
+
+	// get relative path to volumePath
+	relSubPath, err := filepath.Rel(volumePath, subPath)
+	if err != nil {
+		return []uintptr{}, fmt.Errorf("Rel(%s, %s) error: %v", volumePath, subPath, err)
+	}
+	if mount.StartsWithBackstep(relSubPath) {
+		return []uintptr{}, fmt.Errorf("SubPath %q not within volume path %q", subPath, volumePath)
+	}
+
+	if relSubPath == "." {
+		// volumePath and subPath are equal
+		return []uintptr{}, nil
+	}
+
+	fileHandles := []uintptr{}
+	var errorResult error
+
+	currentFullPath := volumePath
+	dirs := strings.Split(relSubPath, string(os.PathSeparator))
+	for _, dir := range dirs {
+		// lock intermediate subPath directory first
+		currentFullPath = filepath.Join(currentFullPath, dir)
+		handle, err := lockPath(currentFullPath)
+		if err != nil {
+			errorResult = fmt.Errorf("cannot lock path %s: %s", currentFullPath, err)
+			break
+		}
+		fileHandles = append(fileHandles, handle)
+
+		// make sure intermediate subPath directory does not contain symlink any more
+		stat, err := os.Lstat(currentFullPath)
+		if err != nil {
+			errorResult = fmt.Errorf("Lstat(%q) error: %v", currentFullPath, err)
+			break
+		}
+		if stat.Mode()&os.ModeSymlink != 0 {
+			errorResult = fmt.Errorf("subpath %q is an unexpected symlink after EvalSymlinks", currentFullPath)
+			break
+		}
+
+		if !mount.PathWithinBase(currentFullPath, volumePath) {
+			errorResult = fmt.Errorf("SubPath %q not within volume path %q", currentFullPath, volumePath)
+			break
+		}
+	}
+
+	return fileHandles, errorResult
+}
+
+// unlockPath unlock directories
+func unlockPath(fileHandles []uintptr) {
+	if fileHandles != nil {
+		for _, handle := range fileHandles {
+			syscall.CloseHandle(syscall.Handle(handle))
+		}
+	}
+}
+
+// lockPath locks a directory or symlink, return handle, exec "syscall.CloseHandle(handle)" to unlock the path
+func lockPath(path string) (uintptr, error) {
+	if len(path) == 0 {
+		return uintptr(syscall.InvalidHandle), syscall.ERROR_FILE_NOT_FOUND
+	}
+	pathp, err := syscall.UTF16PtrFromString(path)
+	if err != nil {
+		return uintptr(syscall.InvalidHandle), err
+	}
+	access := uint32(syscall.GENERIC_READ)
+	sharemode := uint32(syscall.FILE_SHARE_READ)
+	createmode := uint32(syscall.OPEN_EXISTING)
+	flags := uint32(syscall.FILE_FLAG_BACKUP_SEMANTICS | syscall.FILE_FLAG_OPEN_REPARSE_POINT)
+	fd, err := syscall.CreateFile(pathp, access, sharemode, nil, createmode, flags, 0)
+	return uintptr(fd), err
+}
+
+// Lock all directories in subPath and check they're not symlinks.
+func (sp *subpath) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
+	handles, err := lockAndCheckSubPath(subPath.VolumePath, subPath.Path)
+
+	// Unlock the directories when the container starts
+	cleanupAction = func() {
+		unlockPath(handles)
+	}
+	return subPath.Path, cleanupAction, err
+}
+
+// No bind-mounts for subpaths are necessary on Windows
+func (sp *subpath) CleanSubPaths(podDir string, volumeName string) error {
+	return nil
+}
+
+// SafeMakeDir makes sure that the created directory does not escape given base directory mis-using symlinks.
+func (sp *subpath) SafeMakeDir(subdir string, base string, perm os.FileMode) error {
+	realBase, err := filepath.EvalSymlinks(base)
+	if err != nil {
+		return fmt.Errorf("error resolving symlinks in %s: %s", base, err)
+	}
+
+	realFullPath := filepath.Join(realBase, subdir)
+	return doSafeMakeDir(realFullPath, realBase, perm)
+}
+
+func doSafeMakeDir(pathname string, base string, perm os.FileMode) error {
+	klog.V(4).Infof("Creating directory %q within base %q", pathname, base)
+
+	if !mount.PathWithinBase(pathname, base) {
+		return fmt.Errorf("path %s is outside of allowed base %s", pathname, base)
+	}
+
+	// Quick check if the directory already exists
+	s, err := os.Stat(pathname)
+	if err == nil {
+		// Path exists
+		if s.IsDir() {
+			// The directory already exists. It can be outside of the parent,
+			// but there is no race-proof check.
+			klog.V(4).Infof("Directory %s already exists", pathname)
+			return nil
+		}
+		return &os.PathError{Op: "mkdir", Path: pathname, Err: syscall.ENOTDIR}
+	}
+
+	// Find all existing directories
+	existingPath, toCreate, err := findExistingPrefix(base, pathname)
+	if err != nil {
+		return fmt.Errorf("error opening directory %s: %s", pathname, err)
+	}
+	if len(toCreate) == 0 {
+		return nil
+	}
+
+	// Ensure the existing directory is inside allowed base
+	fullExistingPath, err := filepath.EvalSymlinks(existingPath)
+	if err != nil {
+		return fmt.Errorf("error opening existing directory %s: %s", existingPath, err)
+	}
+	fullBasePath, err := filepath.EvalSymlinks(base)
+	if err != nil {
+		return fmt.Errorf("cannot read link %s: %s", base, err)
+	}
+	if !mount.PathWithinBase(fullExistingPath, fullBasePath) {
+		return fmt.Errorf("path %s is outside of allowed base %s", fullExistingPath, err)
+	}
+
+	// lock all intermediate directories from fullBasePath to fullExistingPath (top to bottom)
+	fileHandles, err := lockAndCheckSubPathWithoutSymlink(fullBasePath, fullExistingPath)
+	defer unlockPath(fileHandles)
+	if err != nil {
+		return err
+	}
+
+	klog.V(4).Infof("%q already exists, %q to create", fullExistingPath, filepath.Join(toCreate...))
+	currentPath := fullExistingPath
+	// create the directories one by one, making sure nobody can change
+	// created directory into symlink by lock that directory immediately
+	for _, dir := range toCreate {
+		currentPath = filepath.Join(currentPath, dir)
+		klog.V(4).Infof("Creating %s", dir)
+		if err := os.Mkdir(currentPath, perm); err != nil {
+			return fmt.Errorf("cannot create directory %s: %s", currentPath, err)
+		}
+		handle, err := lockPath(currentPath)
+		if err != nil {
+			return fmt.Errorf("cannot lock path %s: %s", currentPath, err)
+		}
+		defer syscall.CloseHandle(syscall.Handle(handle))
+		// make sure newly created directory does not contain symlink after lock
+		stat, err := os.Lstat(currentPath)
+		if err != nil {
+			return fmt.Errorf("Lstat(%q) error: %v", currentPath, err)
+		}
+		if stat.Mode()&os.ModeSymlink != 0 {
+			return fmt.Errorf("subpath %q is an unexpected symlink after Mkdir", currentPath)
+		}
+	}
+
+	return nil
+}
+
+// findExistingPrefix finds prefix of pathname that exists. In addition, it
+// returns list of remaining directories that don't exist yet.
+func findExistingPrefix(base, pathname string) (string, []string, error) {
+	rel, err := filepath.Rel(base, pathname)
+	if err != nil {
+		return base, nil, err
+	}
+
+	if mount.StartsWithBackstep(rel) {
+		return base, nil, fmt.Errorf("pathname(%s) is not within base(%s)", pathname, base)
+	}
+
+	if rel == "." {
+		// base and pathname are equal
+		return pathname, []string{}, nil
+	}
+
+	dirs := strings.Split(rel, string(filepath.Separator))
+
+	parent := base
+	currentPath := base
+	for i, dir := range dirs {
+		parent = currentPath
+		currentPath = filepath.Join(parent, dir)
+		if _, err := os.Lstat(currentPath); err != nil {
+			if os.IsNotExist(err) {
+				return parent, dirs[i:], nil
+			}
+			return base, nil, err
+		}
+	}
+
+	return pathname, []string{}, nil
+}

pkg/volume/util/subpath/subpath_windows_test.go

+// +build windows
+
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package subpath
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func makeLink(link, target string) error {
+	if output, err := exec.Command("cmd", "/c", "mklink", "/D", link, target).CombinedOutput(); err != nil {
+		return fmt.Errorf("mklink failed: %v, link(%q) target(%q) output: %q", err, link, target, string(output))
+	}
+	return nil
+}
+
+func TestDoSafeMakeDir(t *testing.T) {
+	base, err := ioutil.TempDir("", "TestDoSafeMakeDir")
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+
+	defer os.RemoveAll(base)
+
+	testingVolumePath := filepath.Join(base, "testingVolumePath")
+	os.MkdirAll(testingVolumePath, 0755)
+	defer os.RemoveAll(testingVolumePath)
+
+	tests := []struct {
+		volumePath    string
+		subPath       string
+		expectError   bool
+		symlinkTarget string
+	}{
+		{
+			volumePath:    testingVolumePath,
+			subPath:       ``,
+			expectError:   true,
+			symlinkTarget: "",
+		},
+		{
+			volumePath:    testingVolumePath,
+			subPath:       filepath.Join(testingVolumePath, `x`),
+			expectError:   false,
+			symlinkTarget: "",
+		},
+		{
+			volumePath:    testingVolumePath,
+			subPath:       filepath.Join(testingVolumePath, `a\b\c\d`),
+			expectError:   false,
+			symlinkTarget: "",
+		},
+		{
+			volumePath:    testingVolumePath,
+			subPath:       filepath.Join(testingVolumePath, `symlink`),
+			expectError:   false,
+			symlinkTarget: base,
+		},
+		{
+			volumePath:    testingVolumePath,
+			subPath:       filepath.Join(testingVolumePath, `symlink\c\d`),
+			expectError:   true,
+			symlinkTarget: "",
+		},
+		{
+			volumePath:    testingVolumePath,
+			subPath:       filepath.Join(testingVolumePath, `symlink\y926`),
+			expectError:   true,
+			symlinkTarget: "",
+		},
+		{
+			volumePath:    testingVolumePath,
+			subPath:       filepath.Join(testingVolumePath, `a\b\symlink`),
+			expectError:   false,
+			symlinkTarget: base,
+		},
+		{
+			volumePath:    testingVolumePath,
+			subPath:       filepath.Join(testingVolumePath, `a\x\symlink`),
+			expectError:   false,
+			symlinkTarget: filepath.Join(testingVolumePath, `a`),
+		},
+	}
+
+	for _, test := range tests {
+		if len(test.volumePath) > 0 && len(test.subPath) > 0 && len(test.symlinkTarget) > 0 {
+			// make all parent sub directories
+			if parent := filepath.Dir(test.subPath); parent != "." {
+				os.MkdirAll(parent, 0755)
+			}
+
+			// make last element as symlink
+			linkPath := test.subPath
+			if _, err := os.Stat(linkPath); err != nil && os.IsNotExist(err) {
+				if err := makeLink(linkPath, test.symlinkTarget); err != nil {
+					t.Fatalf("unexpected error: %v", fmt.Errorf("mklink link(%q) target(%q) error: %q", linkPath, test.symlinkTarget, err))
+				}
+			}
+		}
+
+		err := doSafeMakeDir(test.subPath, test.volumePath, os.FileMode(0755))
+		if test.expectError {
+			assert.NotNil(t, err, "Expect error during doSafeMakeDir(%s, %s)", test.subPath, test.volumePath)
+			continue
+		}
+		assert.Nil(t, err, "Expect no error during doSafeMakeDir(%s, %s)", test.subPath, test.volumePath)
+		if _, err := os.Stat(test.subPath); os.IsNotExist(err) {
+			t.Errorf("subPath should exists after doSafeMakeDir(%s, %s)", test.subPath, test.volumePath)
+		}
+	}
+}
+
+func TestLockAndCheckSubPath(t *testing.T) {
+	base, err := ioutil.TempDir("", "TestLockAndCheckSubPath")
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+
+	defer os.RemoveAll(base)
+
+	testingVolumePath := filepath.Join(base, "testingVolumePath")
+
+	tests := []struct {
+		volumePath          string
+		subPath             string
+		expectedHandleCount int
+		expectError         bool
+		symlinkTarget       string
+	}{
+		{
+			volumePath:          `c:\`,
+			subPath:             ``,
+			expectedHandleCount: 0,
+			expectError:         false,
+			symlinkTarget:       "",
+		},
+		{
+			volumePath:          ``,
+			subPath:             `a`,
+			expectedHandleCount: 0,
+			expectError:         false,
+			symlinkTarget:       "",
+		},
+		{
+			volumePath:          testingVolumePath,
+			subPath:             filepath.Join(testingVolumePath, `a`),
+			expectedHandleCount: 1,
+			expectError:         false,
+			symlinkTarget:       "",
+		},
+		{
+			volumePath:          testingVolumePath,
+			subPath:             filepath.Join(testingVolumePath, `a\b\c\d`),
+			expectedHandleCount: 4,
+			expectError:         false,
+			symlinkTarget:       "",
+		},
+		{
+			volumePath:          testingVolumePath,
+			subPath:             filepath.Join(testingVolumePath, `symlink`),
+			expectedHandleCount: 0,
+			expectError:         true,
+			symlinkTarget:       base,
+		},
+		{
+			volumePath:          testingVolumePath,
+			subPath:             filepath.Join(testingVolumePath, `a\b\c\symlink`),
+			expectedHandleCount: 0,
+			expectError:         true,
+			symlinkTarget:       base,
+		},
+		{
+			volumePath:          testingVolumePath,
+			subPath:             filepath.Join(testingVolumePath, `a\b\c\d\symlink`),
+			expectedHandleCount: 2,
+			expectError:         false,
+			symlinkTarget:       filepath.Join(testingVolumePath, `a\b`),
+		},
+	}
+
+	for _, test := range tests {
+		if len(test.volumePath) > 0 && len(test.subPath) > 0 {
+			os.MkdirAll(test.volumePath, 0755)
+			if len(test.symlinkTarget) == 0 {
+				// make all intermediate sub directories
+				os.MkdirAll(test.subPath, 0755)
+			} else {
+				// make all parent sub directories
+				if parent := filepath.Dir(test.subPath); parent != "." {
+					os.MkdirAll(parent, 0755)
+				}
+
+				// make last element as symlink
+				linkPath := test.subPath
+				if _, err := os.Stat(linkPath); err != nil && os.IsNotExist(err) {
+					if err := makeLink(linkPath, test.symlinkTarget); err != nil {
+						t.Fatalf("unexpected error: %v", fmt.Errorf("mklink link(%q) target(%q) error: %q", linkPath, test.symlinkTarget, err))
+					}
+				}
+			}
+		}
+
+		fileHandles, err := lockAndCheckSubPath(test.volumePath, test.subPath)
+		unlockPath(fileHandles)
+		assert.Equal(t, test.expectedHandleCount, len(fileHandles))
+		if test.expectError {
+			assert.NotNil(t, err, "Expect error during LockAndCheckSubPath(%s, %s)", test.volumePath, test.subPath)
+			continue
+		}
+		assert.Nil(t, err, "Expect no error during LockAndCheckSubPath(%s, %s)", test.volumePath, test.subPath)
+	}
+
+	// remove dir will happen after closing all file handles
+	assert.Nil(t, os.RemoveAll(testingVolumePath), "Expect no error during remove dir %s", testingVolumePath)
+}
+
+func TestLockAndCheckSubPathWithoutSymlink(t *testing.T) {
+	base, err := ioutil.TempDir("", "TestLockAndCheckSubPathWithoutSymlink")
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+
+	defer os.RemoveAll(base)
+
+	testingVolumePath := filepath.Join(base, "testingVolumePath")
+
+	tests := []struct {
+		volumePath          string
+		subPath             string
+		expectedHandleCount int
+		expectError         bool
+		symlinkTarget       string
+	}{
+		{
+			volumePath:          `c:\`,
+			subPath:             ``,
+			expectedHandleCount: 0,
+			expectError:         false,
+			symlinkTarget:       "",
+		},
+		{
+			volumePath:          ``,
+			subPath:             `a`,
+			expectedHandleCount: 0,
+			expectError:         false,
+			symlinkTarget:       "",
+		},
+		{
+			volumePath:          testingVolumePath,
+			subPath:             filepath.Join(testingVolumePath, `a`),
+			expectedHandleCount: 1,
+			expectError:         false,
+			symlinkTarget:       "",
+		},
+		{
+			volumePath:          testingVolumePath,
+			subPath:             filepath.Join(testingVolumePath, `a\b\c\d`),
+			expectedHandleCount: 4,
+			expectError:         false,
+			symlinkTarget:       "",
+		},
+		{
+			volumePath:          testingVolumePath,
+			subPath:             filepath.Join(testingVolumePath, `symlink`),
+			expectedHandleCount: 1,
+			expectError:         true,
+			symlinkTarget:       base,
+		},
+		{
+			volumePath:          testingVolumePath,
+			subPath:             filepath.Join(testingVolumePath, `a\b\c\symlink`),
+			expectedHandleCount: 4,
+			expectError:         true,
+			symlinkTarget:       base,
+		},
+		{
+			volumePath:          testingVolumePath,
+			subPath:             filepath.Join(testingVolumePath, `a\b\c\d\symlink`),
+			expectedHandleCount: 5,
+			expectError:         true,
+			symlinkTarget:       filepath.Join(testingVolumePath, `a\b`),
+		},
+	}
+
+	for _, test := range tests {
+		if len(test.volumePath) > 0 && len(test.subPath) > 0 {
+			os.MkdirAll(test.volumePath, 0755)
+			if len(test.symlinkTarget) == 0 {
+				// make all intermediate sub directories
+				os.MkdirAll(test.subPath, 0755)
+			} else {
+				// make all parent sub directories
+				if parent := filepath.Dir(test.subPath); parent != "." {
+					os.MkdirAll(parent, 0755)
+				}
+
+				// make last element as symlink
+				linkPath := test.subPath
+				if _, err := os.Stat(linkPath); err != nil && os.IsNotExist(err) {
+					if err := makeLink(linkPath, test.symlinkTarget); err != nil {
+						t.Fatalf("unexpected error: %v", fmt.Errorf("mklink link(%q) target(%q) error: %q", linkPath, test.symlinkTarget, err))
+					}
+				}
+			}
+		}
+
+		fileHandles, err := lockAndCheckSubPathWithoutSymlink(test.volumePath, test.subPath)
+		unlockPath(fileHandles)
+		assert.Equal(t, test.expectedHandleCount, len(fileHandles))
+		if test.expectError {
+			assert.NotNil(t, err, "Expect error during LockAndCheckSubPath(%s, %s)", test.volumePath, test.subPath)
+			continue
+		}
+		assert.Nil(t, err, "Expect no error during LockAndCheckSubPath(%s, %s)", test.volumePath, test.subPath)
+	}
+
+	// remove dir will happen after closing all file handles
+	assert.Nil(t, os.RemoveAll(testingVolumePath), "Expect no error during remove dir %s", testingVolumePath)
+}
+
+func TestFindExistingPrefix(t *testing.T) {
+	base, err := ioutil.TempDir("", "TestFindExistingPrefix")
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+
+	defer os.RemoveAll(base)
+
+	testingVolumePath := filepath.Join(base, "testingVolumePath")
+
+	tests := []struct {
+		base                    string
+		pathname                string
+		expectError             bool
+		expectedExistingPath    string
+		expectedToCreateDirs    []string
+		createSubPathBeforeTest bool
+	}{
+		{
+			base:                    `c:\tmp\a`,
+			pathname:                `c:\tmp\b`,
+			expectError:             true,
+			expectedExistingPath:    "",
+			expectedToCreateDirs:    []string{},
+			createSubPathBeforeTest: false,
+		},
+		{
+			base:                    ``,
+			pathname:                `c:\tmp\b`,
+			expectError:             true,
+			expectedExistingPath:    "",
+			expectedToCreateDirs:    []string{},
+			createSubPathBeforeTest: false,
+		},
+		{
+			base:                    `c:\tmp\a`,
+			pathname:                `d:\tmp\b`,
+			expectError:             true,
+			expectedExistingPath:    "",
+			expectedToCreateDirs:    []string{},
+			createSubPathBeforeTest: false,
+		},
+		{
+			base:                    testingVolumePath,
+			pathname:                testingVolumePath,
+			expectError:             false,
+			expectedExistingPath:    testingVolumePath,
+			expectedToCreateDirs:    []string{},
+			createSubPathBeforeTest: false,
+		},
+		{
+			base:                    testingVolumePath,
+			pathname:                filepath.Join(testingVolumePath, `a\b`),
+			expectError:             false,
+			expectedExistingPath:    filepath.Join(testingVolumePath, `a\b`),
+			expectedToCreateDirs:    []string{},
+			createSubPathBeforeTest: true,
+		},
+		{
+			base:                    testingVolumePath,
+			pathname:                filepath.Join(testingVolumePath, `a\b\c\`),
+			expectError:             false,
+			expectedExistingPath:    filepath.Join(testingVolumePath, `a\b`),
+			expectedToCreateDirs:    []string{`c`},
+			createSubPathBeforeTest: false,
+		},
+		{
+			base:                    testingVolumePath,
+			pathname:                filepath.Join(testingVolumePath, `a\b\c\d`),
+			expectError:             false,
+			expectedExistingPath:    filepath.Join(testingVolumePath, `a\b`),
+			expectedToCreateDirs:    []string{`c`, `d`},
+			createSubPathBeforeTest: false,
+		},
+	}
+
+	for _, test := range tests {
+		if test.createSubPathBeforeTest {
+			os.MkdirAll(test.pathname, 0755)
+		}
+
+		existingPath, toCreate, err := findExistingPrefix(test.base, test.pathname)
+		if test.expectError {
+			assert.NotNil(t, err, "Expect error during findExistingPrefix(%s, %s)", test.base, test.pathname)
+			continue
+		}
+		assert.Nil(t, err, "Expect no error during findExistingPrefix(%s, %s)", test.base, test.pathname)
+
+		assert.Equal(t, test.expectedExistingPath, existingPath, "Expect result not equal with findExistingPrefix(%s, %s) return: %q, expected: %q",
+			test.base, test.pathname, existingPath, test.expectedExistingPath)
+
+		assert.Equal(t, test.expectedToCreateDirs, toCreate, "Expect result not equal with findExistingPrefix(%s, %s) return: %q, expected: %q",
+			test.base, test.pathname, toCreate, test.expectedToCreateDirs)
+
+	}
+	// remove dir will happen after closing all file handles
+	assert.Nil(t, os.RemoveAll(testingVolumePath), "Expect no error during remove dir %s", testingVolumePath)
+}