Merge pull request #61044 from liggitt/subpath-master

pkg/apis/core/validation/validation.go

@@ -2264,7 +2264,11 @@ func ValidateVolumeMounts(mounts []core.VolumeMount, voldevices map[string]strin
 		}
 
 		if len(mnt.SubPath) > 0 {
-			allErrs = append(allErrs, validateLocalDescendingPath(mnt.SubPath, fldPath.Child("subPath"))...)
+			if !utilfeature.DefaultFeatureGate.Enabled(features.VolumeSubpath) {
+				allErrs = append(allErrs, field.Forbidden(fldPath.Child("subPath"), "subPath is disabled by feature-gate"))
+			} else {
+				allErrs = append(allErrs, validateLocalDescendingPath(mnt.SubPath, fldPath.Child("subPath"))...)
+			}
 		}
 
 		if mnt.MountPropagation != nil {

pkg/apis/core/validation/validation_test.go

@@ -4766,6 +4766,68 @@ func TestValidateVolumeMounts(t *testing.T) {
 	}
 }
 
+func TestValidateDisabledSubpath(t *testing.T) {
+	utilfeature.DefaultFeatureGate.Set("VolumeSubpath=false")
+	defer utilfeature.DefaultFeatureGate.Set("VolumeSubpath=true")
+
+	volumes := []core.Volume{
+		{Name: "abc", VolumeSource: core.VolumeSource{PersistentVolumeClaim: &core.PersistentVolumeClaimVolumeSource{ClaimName: "testclaim1"}}},
+		{Name: "abc-123", VolumeSource: core.VolumeSource{PersistentVolumeClaim: &core.PersistentVolumeClaimVolumeSource{ClaimName: "testclaim2"}}},
+		{Name: "123", VolumeSource: core.VolumeSource{HostPath: &core.HostPathVolumeSource{Path: "/foo/baz", Type: newHostPathType(string(core.HostPathUnset))}}},
+	}
+	vols, v1err := ValidateVolumes(volumes, field.NewPath("field"))
+	if len(v1err) > 0 {
+		t.Errorf("Invalid test volume - expected success %v", v1err)
+		return
+	}
+
+	container := core.Container{
+		SecurityContext: nil,
+	}
+
+	goodVolumeDevices := []core.VolumeDevice{
+		{Name: "xyz", DevicePath: "/foofoo"},
+		{Name: "uvw", DevicePath: "/foofoo/share/test"},
+	}
+
+	cases := map[string]struct {
+		mounts      []core.VolumeMount
+		expectError bool
+	}{
+		"subpath not specified": {
+			[]core.VolumeMount{
+				{
+					Name:      "abc-123",
+					MountPath: "/bab",
+				},
+			},
+			false,
+		},
+		"subpath specified": {
+			[]core.VolumeMount{
+				{
+					Name:      "abc-123",
+					MountPath: "/bab",
+					SubPath:   "baz",
+				},
+			},
+			true,
+		},
+	}
+
+	for name, test := range cases {
+		errs := ValidateVolumeMounts(test.mounts, GetVolumeDeviceMap(goodVolumeDevices), vols, &container, field.NewPath("field"))
+
+		if len(errs) != 0 && !test.expectError {
+			t.Errorf("test %v failed: %+v", name, errs)
+		}
+
+		if len(errs) == 0 && test.expectError {
+			t.Errorf("test %v failed, expected error", name)
+		}
+	}
+}
+
 func TestValidateMountPropagation(t *testing.T) {
 	bTrue := true
 	bFalse := false

pkg/features/kube_features.go

@@ -267,6 +267,13 @@ const (
 	//
 	// Enables control over the primary group ID of containers' init processes.
 	RunAsGroup utilfeature.Feature = "RunAsGroup"
+
+	// owner: @saad-ali
+	// ga
+	//
+	// Allow mounting a subpath of a volume in a container
+	// Do not remove this feature gate even though it's GA
+	VolumeSubpath utilfeature.Feature = "VolumeSubpath"
 )
 
 func init() {
@@ -313,6 +320,7 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS
 	CRIContainerLogRotation:                     {Default: false, PreRelease: utilfeature.Alpha},
 	GCERegionalPersistentDisk:                   {Default: true, PreRelease: utilfeature.Beta},
 	RunAsGroup:                                  {Default: false, PreRelease: utilfeature.Alpha},
+	VolumeSubpath:                               {Default: true, PreRelease: utilfeature.GA},
 
 	// inherited features from generic apiserver, relisted here to get a conflict if it is changed
 	// unintentionally on either side:

pkg/kubelet/cm/container_manager_linux_test.go

@@ -95,6 +95,18 @@ func (mi *fakeMountInterface) ExistsPath(pathname string) bool {
 	return true
 }
 
+func (mi *fakeMountInterface) PrepareSafeSubpath(subPath mount.Subpath) (newHostPath string, cleanupAction func(), err error) {
+	return "", nil, nil
+}
+
+func (mi *fakeMountInterface) CleanSubPaths(_, _ string) error {
+	return nil
+}
+
+func (mi *fakeMountInterface) SafeMakeDir(_, _ string, _ os.FileMode) error {
+	return nil
+}
+
 func fakeContainerMgrMountInt() mount.Interface {
 	return &fakeMountInterface{
 		[]mount.MountPoint{

pkg/kubelet/container/helpers.go

@@ -46,7 +46,7 @@ type HandlerRunner interface {
 // RuntimeHelper wraps kubelet to make container runtime
 // able to get necessary informations like the RunContainerOptions, DNS settings, Host IP.
 type RuntimeHelper interface {
-	GenerateRunContainerOptions(pod *v1.Pod, container *v1.Container, podIP string) (contOpts *RunContainerOptions, err error)
+	GenerateRunContainerOptions(pod *v1.Pod, container *v1.Container, podIP string) (contOpts *RunContainerOptions, cleanupAction func(), err error)
 	GetPodDNS(pod *v1.Pod) (dnsConfig *runtimeapi.DNSConfig, err error)
 	// GetPodCgroupParent returns the CgroupName identifier, and its literal cgroupfs form on the host
 	// of a pod.

pkg/kubelet/container/testing/fake_runtime_helper.go

@@ -34,12 +34,12 @@ type FakeRuntimeHelper struct {
 	Err             error
 }
 
-func (f *FakeRuntimeHelper) GenerateRunContainerOptions(pod *v1.Pod, container *v1.Container, podIP string) (*kubecontainer.RunContainerOptions, error) {
+func (f *FakeRuntimeHelper) GenerateRunContainerOptions(pod *v1.Pod, container *v1.Container, podIP string) (*kubecontainer.RunContainerOptions, func(), error) {
 	var opts kubecontainer.RunContainerOptions
 	if len(container.TerminationMessagePath) != 0 {
 		opts.PodContainerDir = f.PodContainerDir
 	}
-	return &opts, nil
+	return &opts, nil, nil
 }
 
 func (f *FakeRuntimeHelper) GetPodCgroupParent(pod *v1.Pod) string {

pkg/kubelet/kubelet_pods.go

@@ -61,6 +61,7 @@ import (
 	kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
 	"k8s.io/kubernetes/pkg/kubelet/util/format"
 	utilfile "k8s.io/kubernetes/pkg/util/file"
+	mountutil "k8s.io/kubernetes/pkg/util/mount"
 	volumeutil "k8s.io/kubernetes/pkg/volume/util"
 	"k8s.io/kubernetes/pkg/volume/util/volumepathhandler"
 	volumevalidation "k8s.io/kubernetes/pkg/volume/validation"
@@ -160,7 +161,7 @@ func (kl *Kubelet) makeBlockVolumes(pod *v1.Pod, container *v1.Container, podVol
 }
 
 // makeMounts determines the mount points for the given container.
-func makeMounts(pod *v1.Pod, podDir string, container *v1.Container, hostName, hostDomain, podIP string, podVolumes kubecontainer.VolumeMap) ([]kubecontainer.Mount, error) {
+func makeMounts(pod *v1.Pod, podDir string, container *v1.Container, hostName, hostDomain, podIP string, podVolumes kubecontainer.VolumeMap, mounter mountutil.Interface) ([]kubecontainer.Mount, func(), error) {
 	// Kubernetes only mounts on /etc/hosts if:
 	// - container is not an infrastructure (pause) container
 	// - container is not already mounting on /etc/hosts
@@ -170,13 +171,14 @@ func makeMounts(pod *v1.Pod, podDir string, container *v1.Container, hostName, h
 	mountEtcHostsFile := len(podIP) > 0 && runtime.GOOS != "windows"
 	glog.V(3).Infof("container: %v/%v/%v podIP: %q creating hosts mount: %v", pod.Namespace, pod.Name, container.Name, podIP, mountEtcHostsFile)
 	mounts := []kubecontainer.Mount{}
-	for _, mount := range container.VolumeMounts {
+	var cleanupAction func() = nil
+	for i, mount := range container.VolumeMounts {
 		// do not mount /etc/hosts if container is already mounting on the path
 		mountEtcHostsFile = mountEtcHostsFile && (mount.MountPath != etcHostsPath)
 		vol, ok := podVolumes[mount.Name]
 		if !ok || vol.Mounter == nil {
 			glog.Errorf("Mount cannot be satisfied for container %q, because the volume is missing or the volume mounter is nil: %+v", container.Name, mount)
-			return nil, fmt.Errorf("cannot find volume %q to mount into container %q", mount.Name, container.Name)
+			return nil, cleanupAction, fmt.Errorf("cannot find volume %q to mount into container %q", mount.Name, container.Name)
 		}
 
 		relabelVolume := false
@@ -189,25 +191,33 @@ func makeMounts(pod *v1.Pod, podDir string, container *v1.Container, hostName, h
 		}
 		hostPath, err := volumeutil.GetPath(vol.Mounter)
 		if err != nil {
-			return nil, err
+			return nil, cleanupAction, err
 		}
 		if mount.SubPath != "" {
+			if !utilfeature.DefaultFeatureGate.Enabled(features.VolumeSubpath) {
+				return nil, cleanupAction, fmt.Errorf("volume subpaths are disabled")
+			}
+
 			if filepath.IsAbs(mount.SubPath) {
-				return nil, fmt.Errorf("error SubPath `%s` must not be an absolute path", mount.SubPath)
+				return nil, cleanupAction, fmt.Errorf("error SubPath `%s` must not be an absolute path", mount.SubPath)
 			}
 
 			err = volumevalidation.ValidatePathNoBacksteps(mount.SubPath)
 			if err != nil {
-				return nil, fmt.Errorf("unable to provision SubPath `%s`: %v", mount.SubPath, err)
+				return nil, cleanupAction, fmt.Errorf("unable to provision SubPath `%s`: %v", mount.SubPath, err)
 			}
 
 			fileinfo, err := os.Lstat(hostPath)
 			if err != nil {
-				return nil, err
+				return nil, cleanupAction, err
 			}
 			perm := fileinfo.Mode()
 
-			hostPath = filepath.Join(hostPath, mount.SubPath)
+			volumePath, err := filepath.EvalSymlinks(hostPath)
+			if err != nil {
+				return nil, cleanupAction, err
+			}
+			hostPath = filepath.Join(volumePath, mount.SubPath)
 
 			if subPathExists, err := utilfile.FileOrSymlinkExists(hostPath); err != nil {
 				glog.Errorf("Could not determine if subPath %s exists; will not attempt to change its permissions", hostPath)
@@ -216,17 +226,25 @@ func makeMounts(pod *v1.Pod, podDir string, container *v1.Container, hostName, h
 				// incorrect ownership and mode. For example, the sub path directory must have at least g+rwx
 				// when the pod specifies an fsGroup, and if the directory is not created here, Docker will
 				// later auto-create it with the incorrect mode 0750
-				if err := os.MkdirAll(hostPath, perm); err != nil {
-					glog.Errorf("failed to mkdir:%s", hostPath)
-					return nil, err
-				}
-
-				// chmod the sub path because umask may have prevented us from making the sub path with the same
-				// permissions as the mounter path
-				if err := os.Chmod(hostPath, perm); err != nil {
-					return nil, err
+				// Make extra care not to escape the volume!
+				if err := mounter.SafeMakeDir(hostPath, volumePath, perm); err != nil {
+					glog.Errorf("failed to mkdir %q: %v", hostPath, err)
+					return nil, cleanupAction, err
 				}
 			}
+			hostPath, cleanupAction, err = mounter.PrepareSafeSubpath(mountutil.Subpath{
+				VolumeMountIndex: i,
+				Path:             hostPath,
+				VolumeName:       mount.Name,
+				VolumePath:       volumePath,
+				PodDir:           podDir,
+				ContainerName:    container.Name,
+			})
+			if err != nil {
+				// Don't pass detailed error back to the user because it could give information about host filesystem
+				glog.Errorf("failed to prepare subPath for volumeMount %q of container %q: %v", mount.Name, container.Name, err)
+				return nil, cleanupAction, fmt.Errorf("failed to prepare subPath for volumeMount %q of container %q", mount.Name, container.Name)
+			}
 		}
 
 		// Docker Volume Mounts fail on Windows if it is not of the form C:/
@@ -242,7 +260,7 @@ func makeMounts(pod *v1.Pod, podDir string, container *v1.Container, hostName, h
 
 		propagation, err := translateMountPropagation(mount.MountPropagation)
 		if err != nil {
-			return nil, err
+			return nil, cleanupAction, err
 		}
 		glog.V(5).Infof("Pod %q container %q mount %q has propagation %q", format.Pod(pod), container.Name, mount.Name, propagation)
 
@@ -261,11 +279,11 @@ func makeMounts(pod *v1.Pod, podDir string, container *v1.Container, hostName, h
 		hostAliases := pod.Spec.HostAliases
 		hostsMount, err := makeHostsMount(podDir, podIP, hostName, hostDomain, hostAliases, pod.Spec.HostNetwork)
 		if err != nil {
-			return nil, err
+			return nil, cleanupAction, err
 		}
 		mounts = append(mounts, *hostsMount)
 	}
-	return mounts, nil
+	return mounts, cleanupAction, nil
 }
 
 // translateMountPropagation transforms v1.MountPropagationMode to
@@ -437,17 +455,17 @@ func (kl *Kubelet) GetPodCgroupParent(pod *v1.Pod) string {
 
 // GenerateRunContainerOptions generates the RunContainerOptions, which can be used by
 // the container runtime to set parameters for launching a container.
-func (kl *Kubelet) GenerateRunContainerOptions(pod *v1.Pod, container *v1.Container, podIP string) (*kubecontainer.RunContainerOptions, error) {
+func (kl *Kubelet) GenerateRunContainerOptions(pod *v1.Pod, container *v1.Container, podIP string) (*kubecontainer.RunContainerOptions, func(), error) {
 	opts, err := kl.containerManager.GetResources(pod, container)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
 	cgroupParent := kl.GetPodCgroupParent(pod)
 	opts.CgroupParent = cgroupParent
 	hostname, hostDomainName, err := kl.GeneratePodHostNameAndDomain(pod)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 	opts.Hostname = hostname
 	podName := volumeutil.GetUniquePodName(pod)
@@ -457,7 +475,7 @@ func (kl *Kubelet) GenerateRunContainerOptions(pod *v1.Pod, container *v1.Contai
 	// TODO(random-liu): Move following convert functions into pkg/kubelet/container
 	devices, err := kl.makeGPUDevices(pod, container)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 	opts.Devices = append(opts.Devices, devices...)
 
@@ -466,20 +484,20 @@ func (kl *Kubelet) GenerateRunContainerOptions(pod *v1.Pod, container *v1.Contai
 		blkutil := volumepathhandler.NewBlockVolumePathHandler()
 		blkVolumes, err := kl.makeBlockVolumes(pod, container, volumes, blkutil)
 		if err != nil {
-			return nil, err
+			return nil, nil, err
 		}
 		opts.Devices = append(opts.Devices, blkVolumes...)
 	}
 
-	mounts, err := makeMounts(pod, kl.getPodDir(pod.UID), container, hostname, hostDomainName, podIP, volumes)
+	mounts, cleanupAction, err := makeMounts(pod, kl.getPodDir(pod.UID), container, hostname, hostDomainName, podIP, volumes, kl.mounter)
 	if err != nil {
-		return nil, err
+		return nil, cleanupAction, err
 	}
 	opts.Mounts = append(opts.Mounts, mounts...)
 
 	envs, err := kl.makeEnvironmentVariables(pod, container, podIP)
 	if err != nil {
-		return nil, err
+		return nil, cleanupAction, err
 	}
 	opts.Envs = append(opts.Envs, envs...)
 
@@ -499,7 +517,7 @@ func (kl *Kubelet) GenerateRunContainerOptions(pod *v1.Pod, container *v1.Contai
 		opts.EnableHostUserNamespace = kl.enableHostUserNamespace(pod)
 	}
 
-	return opts, nil
+	return opts, cleanupAction, nil
 }
 
 var masterServices = sets.NewString("kubernetes")

pkg/kubelet/kubelet_pods_test.go

@@ -47,6 +47,7 @@ import (
 	containertest "k8s.io/kubernetes/pkg/kubelet/container/testing"
 	"k8s.io/kubernetes/pkg/kubelet/server/portforward"
 	"k8s.io/kubernetes/pkg/kubelet/server/remotecommand"
+	"k8s.io/kubernetes/pkg/util/mount"
 	volumetest "k8s.io/kubernetes/pkg/volume/testing"
 )
 
@@ -303,6 +304,7 @@ func TestMakeMounts(t *testing.T) {
 
 	for name, tc := range testCases {
 		t.Run(name, func(t *testing.T) {
+			fm := &mount.FakeMounter{}
 			pod := v1.Pod{
 				Spec: v1.PodSpec{
 					HostNetwork: true,
@@ -315,7 +317,7 @@ func TestMakeMounts(t *testing.T) {
 				return
 			}
 
-			mounts, err := makeMounts(&pod, "/pod", &tc.container, "fakepodname", "", "", tc.podVolumes)
+			mounts, _, err := makeMounts(&pod, "/pod", &tc.container, "fakepodname", "", "", tc.podVolumes, fm)
 
 			// validate only the error if we expect an error
 			if tc.expectErr {
@@ -338,7 +340,7 @@ func TestMakeMounts(t *testing.T) {
 				t.Errorf("Failed to enable feature gate for MountPropagation: %v", err)
 				return
 			}
-			mounts, err = makeMounts(&pod, "/pod", &tc.container, "fakepodname", "", "", tc.podVolumes)
+			mounts, _, err = makeMounts(&pod, "/pod", &tc.container, "fakepodname", "", "", tc.podVolumes, fm)
 			if !tc.expectErr {
 				expectedPrivateMounts := []kubecontainer.Mount{}
 				for _, mount := range tc.expectedMounts {
@@ -353,6 +355,62 @@ func TestMakeMounts(t *testing.T) {
 	}
 }
 
+func TestDisabledSubpath(t *testing.T) {
+	fm := &mount.FakeMounter{}
+	pod := v1.Pod{
+		Spec: v1.PodSpec{
+			HostNetwork: true,
+		},
+	}
+	podVolumes := kubecontainer.VolumeMap{
+		"disk": kubecontainer.VolumeInfo{Mounter: &stubVolume{path: "/mnt/disk"}},
+	}
+
+	cases := map[string]struct {
+		container   v1.Container
+		expectError bool
+	}{
+		"subpath not specified": {
+			v1.Container{
+				VolumeMounts: []v1.VolumeMount{
+					{
+						MountPath: "/mnt/path3",
+						Name:      "disk",
+						ReadOnly:  true,
+					},
+				},
+			},
+			false,
+		},
+		"subpath specified": {
+			v1.Container{
+				VolumeMounts: []v1.VolumeMount{
+					{
+						MountPath: "/mnt/path3",
+						SubPath:   "/must/not/be/absolute",
+						Name:      "disk",
+						ReadOnly:  true,
+					},
+				},
+			},
+			true,
+		},
+	}
+
+	utilfeature.DefaultFeatureGate.Set("VolumeSubpath=false")
+	defer utilfeature.DefaultFeatureGate.Set("VolumeSubpath=true")
+
+	for name, test := range cases {
+		_, _, err := makeMounts(&pod, "/pod", &test.container, "fakepodname", "", "", podVolumes, fm)
+		if err != nil && !test.expectError {
+			t.Errorf("test %v failed: %v", name, err)
+		}
+		if err == nil && test.expectError {
+			t.Errorf("test %v failed: expected error", name)
+		}
+	}
+}
+
 func TestMakeBlockVolumes(t *testing.T) {
 	testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */)
 	defer testKubelet.Cleanup()

pkg/kubelet/kubelet_pods_windows_test.go

@@ -24,6 +24,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"k8s.io/api/core/v1"
 	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
+	"k8s.io/kubernetes/pkg/util/mount"
 )
 
 func TestMakeMountsWindows(t *testing.T) {
@@ -64,7 +65,8 @@ func TestMakeMountsWindows(t *testing.T) {
 		},
 	}
 
-	mounts, _ := makeMounts(&pod, "/pod", &container, "fakepodname", "", "", podVolumes)
+	fm := &mount.FakeMounter{}
+	mounts, _, _ := makeMounts(&pod, "/pod", &container, "fakepodname", "", "", podVolumes, fm)
 
 	expectedMounts := []kubecontainer.Mount{
 		{

pkg/kubelet/kuberuntime/kuberuntime_container.go

@@ -106,11 +106,15 @@ func (m *kubeGenericRuntimeManager) startContainer(podSandboxID string, podSandb
 		restartCount = containerStatus.RestartCount + 1
 	}
 
-	containerConfig, err := m.generateContainerConfig(container, pod, restartCount, podIP, imageRef, containerType)
+	containerConfig, cleanupAction, err := m.generateContainerConfig(container, pod, restartCount, podIP, imageRef, containerType)
+	if cleanupAction != nil {
+		defer cleanupAction()
+	}
 	if err != nil {
 		m.recordContainerEvent(pod, container, "", v1.EventTypeWarning, events.FailedToCreateContainer, "Error: %v", grpc.ErrorDesc(err))
 		return grpc.ErrorDesc(err), ErrCreateContainerConfig
 	}
+
 	containerID, err := m.runtimeService.CreateContainer(podSandboxID, containerConfig, podSandboxConfig)
 	if err != nil {
 		m.recordContainerEvent(pod, container, containerID, v1.EventTypeWarning, events.FailedToCreateContainer, "Error: %v", grpc.ErrorDesc(err))
@@ -172,27 +176,27 @@ func (m *kubeGenericRuntimeManager) startContainer(podSandboxID string, podSandb
 }
 
 // generateContainerConfig generates container config for kubelet runtime v1.
-func (m *kubeGenericRuntimeManager) generateContainerConfig(container *v1.Container, pod *v1.Pod, restartCount int, podIP, imageRef string, containerType kubecontainer.ContainerType) (*runtimeapi.ContainerConfig, error) {
-	opts, err := m.runtimeHelper.GenerateRunContainerOptions(pod, container, podIP)
+func (m *kubeGenericRuntimeManager) generateContainerConfig(container *v1.Container, pod *v1.Pod, restartCount int, podIP, imageRef string, containerType kubecontainer.ContainerType) (*runtimeapi.ContainerConfig, func(), error) {
+	opts, cleanupAction, err := m.runtimeHelper.GenerateRunContainerOptions(pod, container, podIP)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
 	uid, username, err := m.getImageUser(container.Image)
 	if err != nil {
-		return nil, err
+		return nil, cleanupAction, err
 	}
 
 	// Verify RunAsNonRoot. Non-root verification only supports numeric user.
 	if err := verifyRunAsNonRoot(pod, container, uid, username); err != nil {
-		return nil, err
+		return nil, cleanupAction, err
 	}
 
 	command, args := kubecontainer.ExpandContainerCommandAndArgs(container, opts.Envs)
 	logDir := BuildContainerLogsDirectory(kubetypes.UID(pod.UID), container.Name)
 	err = m.osInterface.MkdirAll(logDir, 0755)
 	if err != nil {
-		return nil, fmt.Errorf("create container log directory for container %s failed: %v", container.Name, err)
+		return nil, cleanupAction, fmt.Errorf("create container log directory for container %s failed: %v", container.Name, err)
 	}
 	containerLogsPath := buildContainerLogsPath(container.Name, restartCount)
 	restartCountUint32 := uint32(restartCount)
@@ -217,7 +221,7 @@ func (m *kubeGenericRuntimeManager) generateContainerConfig(container *v1.Contai
 
 	// set platform specific configurations.
 	if err := m.applyPlatformSpecificContainerConfig(config, container, pod, uid, username); err != nil {
-		return nil, err
+		return nil, cleanupAction, err
 	}
 
 	// set environment variables
@@ -231,7 +235,7 @@ func (m *kubeGenericRuntimeManager) generateContainerConfig(container *v1.Contai
 	}
 	config.Envs = envs
 
-	return config, nil
+	return config, cleanupAction, nil
 }
 
 // makeDevices generates container devices for kubelet runtime v1.

pkg/kubelet/kuberuntime/kuberuntime_container_linux_test.go

@@ -32,7 +32,7 @@ func makeExpectedConfig(m *kubeGenericRuntimeManager, pod *v1.Pod, containerInde
 	container := &pod.Spec.Containers[containerIndex]
 	podIP := ""
 	restartCount := 0
-	opts, _ := m.runtimeHelper.GenerateRunContainerOptions(pod, container, podIP)
+	opts, _, _ := m.runtimeHelper.GenerateRunContainerOptions(pod, container, podIP)
 	containerLogsPath := buildContainerLogsPath(container.Name, restartCount)
 	restartCountUint32 := uint32(restartCount)
 	envs := make([]*runtimeapi.KeyValue, len(opts.Envs))
@@ -84,7 +84,7 @@ func TestGenerateContainerConfig(t *testing.T) {
 	}
 
 	expectedConfig := makeExpectedConfig(m, pod, 0)
-	containerConfig, err := m.generateContainerConfig(&pod.Spec.Containers[0], pod, 0, "", pod.Spec.Containers[0].Image, kubecontainer.ContainerTypeRegular)
+	containerConfig, _, err := m.generateContainerConfig(&pod.Spec.Containers[0], pod, 0, "", pod.Spec.Containers[0].Image, kubecontainer.ContainerTypeRegular)
 	assert.NoError(t, err)
 	assert.Equal(t, expectedConfig, containerConfig, "generate container config for kubelet runtime v1.")
 
@@ -113,7 +113,7 @@ func TestGenerateContainerConfig(t *testing.T) {
 		},
 	}
 
-	_, err = m.generateContainerConfig(&podWithContainerSecurityContext.Spec.Containers[0], podWithContainerSecurityContext, 0, "", podWithContainerSecurityContext.Spec.Containers[0].Image, kubecontainer.ContainerTypeRegular)
+	_, _, err = m.generateContainerConfig(&podWithContainerSecurityContext.Spec.Containers[0], podWithContainerSecurityContext, 0, "", podWithContainerSecurityContext.Spec.Containers[0].Image, kubecontainer.ContainerTypeRegular)
 	assert.Error(t, err)
 
 	imageId, _ := imageService.PullImage(&runtimeapi.ImageSpec{Image: "busybox"}, nil)
@@ -125,6 +125,6 @@ func TestGenerateContainerConfig(t *testing.T) {
 	podWithContainerSecurityContext.Spec.Containers[0].SecurityContext.RunAsUser = nil
 	podWithContainerSecurityContext.Spec.Containers[0].SecurityContext.RunAsNonRoot = &runAsNonRootTrue
 
-	_, err = m.generateContainerConfig(&podWithContainerSecurityContext.Spec.Containers[0], podWithContainerSecurityContext, 0, "", podWithContainerSecurityContext.Spec.Containers[0].Image, kubecontainer.ContainerTypeRegular)
+	_, _, err = m.generateContainerConfig(&podWithContainerSecurityContext.Spec.Containers[0], podWithContainerSecurityContext, 0, "", podWithContainerSecurityContext.Spec.Containers[0].Image, kubecontainer.ContainerTypeRegular)
 	assert.Error(t, err, "RunAsNonRoot should fail for non-numeric username")
 }

pkg/kubelet/kuberuntime/kuberuntime_manager_test.go

@@ -143,7 +143,7 @@ func makeFakeContainer(t *testing.T, m *kubeGenericRuntimeManager, template cont
 	sandboxConfig, err := m.generatePodSandboxConfig(template.pod, template.sandboxAttempt)
 	assert.NoError(t, err, "generatePodSandboxConfig for container template %+v", template)
 
-	containerConfig, err := m.generateContainerConfig(template.container, template.pod, template.attempt, "", template.container.Image, template.containerType)
+	containerConfig, _, err := m.generateContainerConfig(template.container, template.pod, template.attempt, "", template.container.Image, template.containerType)
 	assert.NoError(t, err, "generateContainerConfig for container template %+v", template)
 
 	podSandboxID := apitest.BuildSandboxName(sandboxConfig.Metadata)

pkg/kubelet/rkt/rkt.go

@@ -842,7 +842,7 @@ func (r *Runtime) newAppcRuntimeApp(pod *v1.Pod, podIP string, c v1.Container, r
 	}
 
 	// TODO: determine how this should be handled for rkt
-	opts, err := r.runtimeHelper.GenerateRunContainerOptions(pod, &c, podIP)
+	opts, _, err := r.runtimeHelper.GenerateRunContainerOptions(pod, &c, podIP)
 	if err != nil {
 		return err
 	}

pkg/kubelet/util/BUILD

@@ -8,9 +8,13 @@ load(
 
 go_test(
     name = "go_default_test",
-    srcs = ["util_test.go"],
+    srcs = [
+        "util_test.go",
+    ],
     embed = [":go_default_library"],
-    deps = ["//vendor/github.com/stretchr/testify/assert:go_default_library"],
+    deps = [
+        "//vendor/github.com/stretchr/testify/assert:go_default_library",
+    ],
 )
 
 go_library(

pkg/kubelet/util/util.go

@@ -19,6 +19,8 @@ package util
 import (
 	"fmt"
 	"net/url"
+	"path/filepath"
+	"strings"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
@@ -45,3 +47,14 @@ func parseEndpoint(endpoint string) (string, string, error) {
 		return u.Scheme, "", fmt.Errorf("protocol %q not supported", u.Scheme)
 	}
 }
+
+func pathWithinBase(fullPath, basePath string) bool {
+	rel, err := filepath.Rel(basePath, fullPath)
+	if err != nil {
+		return false
+	}
+	if strings.HasPrefix(rel, "..") {
+		return false
+	}
+	return true
+}

pkg/kubelet/util/util_unsupported.go

@@ -31,3 +31,12 @@ func CreateListener(endpoint string) (net.Listener, error) {
 func GetAddressAndDialer(endpoint string) (string, func(addr string, timeout time.Duration) (net.Conn, error), error) {
 	return "", nil, fmt.Errorf("GetAddressAndDialer is unsupported in this build")
 }
+
+// LockAndCheckSubPath empty implementation
+func LockAndCheckSubPath(volumePath, subPath string) ([]uintptr, error) {
+	return []uintptr{}, nil
+}
+
+// UnlockPath empty implementation
+func UnlockPath(fileHandles []uintptr) {
+}

pkg/kubelet/volumemanager/reconciler/reconciler.go

@@ -169,7 +169,7 @@ func (rc *reconciler) reconcile() {
 			// Volume is mounted, unmount it
 			glog.V(5).Infof(mountedVolume.GenerateMsgDetailed("Starting operationExecutor.UnmountVolume", ""))
 			err := rc.operationExecutor.UnmountVolume(
-				mountedVolume.MountedVolume, rc.actualStateOfWorld)
+				mountedVolume.MountedVolume, rc.actualStateOfWorld, rc.kubeletPodsDir)
 			if err != nil &&
 				!nestedpendingoperations.IsAlreadyExists(err) &&
 				!exponentialbackoff.IsExponentialBackoff(err) {
@@ -401,7 +401,7 @@ func (rc *reconciler) cleanupMounts(volume podVolume) {
 	}
 	// TODO: Currently cleanupMounts only includes UnmountVolume operation. In the next PR, we will add
 	// to unmount both volume and device in the same routine.
-	err := rc.operationExecutor.UnmountVolume(mountedVolume, rc.actualStateOfWorld)
+	err := rc.operationExecutor.UnmountVolume(mountedVolume, rc.actualStateOfWorld, rc.kubeletPodsDir)
 	if err != nil {
 		glog.Errorf(mountedVolume.GenerateErrorDetailed(fmt.Sprintf("volumeHandler.UnmountVolumeHandler for UnmountVolume failed"), err).Error())
 		return

pkg/kubelet/volumemanager/reconciler/reconciler_test.go

@@ -908,7 +908,7 @@ func Test_GenerateUnmapVolumeFunc_Plugin_Not_Found(t *testing.T) {
 			volumeToUnmount := operationexecutor.MountedVolume{
 				PluginName: "fake-file-plugin",
 				VolumeSpec: tmpSpec}
-			err := oex.UnmountVolume(volumeToUnmount, asw)
+			err := oex.UnmountVolume(volumeToUnmount, asw, "" /* podsDir */)
 			// Assert
 			if assert.Error(t, err) {
 				assert.Contains(t, err.Error(), tc.expectedErrMsg)

pkg/util/mount/BUILD

-package(default_visibility = ["//visibility:public"])
-
-load(
-    "@io_bazel_rules_go//go:def.bzl",
-    "go_library",
-    "go_test",
-)
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 
 go_library(
     name = "go_default_library",
@@ -72,6 +66,7 @@ go_library(
         "//conditions:default": [],
     }),
     importpath = "k8s.io/kubernetes/pkg/util/mount",
+    visibility = ["//visibility:public"],
     deps = [
         "//vendor/github.com/golang/glog:go_default_library",
         "//vendor/k8s.io/utils/exec:go_default_library",
@@ -104,7 +99,15 @@ go_test(
     embed = [":go_default_library"],
     deps = [
         "//vendor/k8s.io/utils/exec/testing:go_default_library",
-    ],
+    ] + select({
+        "@io_bazel_rules_go//go/platform:linux": [
+            "//vendor/github.com/golang/glog:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:windows": [
+            "//vendor/github.com/stretchr/testify/assert:go_default_library",
+        ],
+        "//conditions:default": [],
+    }),
 )
 
 filegroup(
@@ -118,4 +121,5 @@ filegroup(
     name = "all-srcs",
     srcs = [":package-srcs"],
     tags = ["automanaged"],
+    visibility = ["//visibility:public"],
 )

pkg/util/mount/exec_mount.go

@@ -20,6 +20,7 @@ package mount
 
 import (
 	"fmt"
+	"os"
 
 	"github.com/golang/glog"
 )
@@ -138,3 +139,15 @@ func (m *execMounter) MakeDir(pathname string) error {
 func (m *execMounter) ExistsPath(pathname string) bool {
 	return m.wrappedMounter.ExistsPath(pathname)
 }
+
+func (m *execMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
+	return m.wrappedMounter.PrepareSafeSubpath(subPath)
+}
+
+func (m *execMounter) CleanSubPaths(podDir string, volumeName string) error {
+	return m.wrappedMounter.CleanSubPaths(podDir, volumeName)
+}
+
+func (m *execMounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
+	return m.wrappedMounter.SafeMakeDir(pathname, base, perm)
+}

pkg/util/mount/exec_mount_test.go

@@ -20,6 +20,7 @@ package mount
 
 import (
 	"fmt"
+	"os"
 	"reflect"
 	"strings"
 	"testing"
@@ -151,3 +152,14 @@ func (fm *fakeMounter) ExistsPath(pathname string) bool {
 func (fm *fakeMounter) GetFileType(pathname string) (FileType, error) {
 	return FileTypeFile, nil
 }
+func (fm *fakeMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
+	return subPath.Path, nil, nil
+}
+
+func (fm *fakeMounter) CleanSubPaths(podDir string, volumeName string) error {
+	return nil
+}
+
+func (fm *fakeMounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
+	return nil
+}

pkg/util/mount/exec_mount_unsupported.go

@@ -20,6 +20,7 @@ package mount
 
 import (
 	"errors"
+	"os"
 )
 
 type execMounter struct{}
@@ -85,3 +86,15 @@ func (mounter *execMounter) MakeFile(pathname string) error {
 func (mounter *execMounter) ExistsPath(pathname string) bool {
 	return true
 }
+
+func (mounter *execMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
+	return subPath.Path, nil, nil
+}
+
+func (mounter *execMounter) CleanSubPaths(podDir string, volumeName string) error {
+	return nil
+}
+
+func (mounter *execMounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
+	return nil
+}

pkg/util/mount/fake.go

@@ -197,3 +197,14 @@ func (f *FakeMounter) MakeFile(pathname string) error {
 func (f *FakeMounter) ExistsPath(pathname string) bool {
 	return false
 }
+
+func (f *FakeMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
+	return subPath.Path, nil, nil
+}
+
+func (f *FakeMounter) CleanSubPaths(podDir string, volumeName string) error {
+	return nil
+}
+func (mounter *FakeMounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
+	return nil
+}

pkg/util/mount/mount.go

@@ -22,6 +22,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"syscall"
 )
 
 type FileType string
@@ -83,9 +84,45 @@ type Interface interface {
 	// MakeDir creates a new directory.
 	// Will operate in the host mount namespace if kubelet is running in a container
 	MakeDir(pathname string) error
+	// SafeMakeDir makes sure that the created directory does not escape given
+	// base directory mis-using symlinks. The directory is created in the same
+	// mount namespace as where kubelet is running. Note that the function makes
+	// sure that it creates the directory somewhere under the base, nothing
+	// else. E.g. if the directory already exists, it may exists outside of the
+	// base due to symlinks.
+	SafeMakeDir(pathname string, base string, perm os.FileMode) error
 	// ExistsPath checks whether the path exists.
 	// Will operate in the host mount namespace if kubelet is running in a container
 	ExistsPath(pathname string) bool
+	// CleanSubPaths removes any bind-mounts created by PrepareSafeSubpath in given
+	// pod volume directory.
+	CleanSubPaths(podDir string, volumeName string) error
+	// PrepareSafeSubpath does everything that's necessary to prepare a subPath
+	// that's 1) inside given volumePath and 2) immutable after this call.
+	//
+	// newHostPath - location of prepared subPath. It should be used instead of
+	// hostName when running the container.
+	// cleanupAction - action to run when the container is running or it failed to start.
+	//
+	// CleanupAction must be called immediately after the container with given
+	// subpath starts. On the other hand, Interface.CleanSubPaths must be called
+	// when the pod finishes.
+	PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error)
+}
+
+type Subpath struct {
+	// index of the VolumeMount for this container
+	VolumeMountIndex int
+	// Full path to the subpath directory on the host
+	Path string
+	// name of the volume that is a valid directory name.
+	VolumeName string
+	// Full path to the volume path
+	VolumePath string
+	// Path to the pod's directory, including pod UID
+	PodDir string
+	// Name of the container
+	ContainerName string
 }
 
 // Exec executes command where mount utilities are. This can be either the host,
@@ -195,6 +232,13 @@ func GetDeviceNameFromMount(mounter Interface, mountPath string) (string, int, e
 	return device, refCount, nil
 }
 
+func isNotDirErr(err error) bool {
+	if e, ok := err.(*os.PathError); ok && e.Err == syscall.ENOTDIR {
+		return true
+	}
+	return false
+}
+
 // IsNotMountPoint determines if a directory is a mountpoint.
 // It should return ErrNotExist when the directory does not exist.
 // This method uses the List() of all mountpoints
@@ -210,7 +254,7 @@ func IsNotMountPoint(mounter Interface, file string) (bool, error) {
 		notMnt = true
 		notMntErr = nil
 	}
-	if notMntErr != nil {
+	if notMntErr != nil && isNotDirErr(notMntErr) {
 		return notMnt, notMntErr
 	}
 	// identified as mountpoint, so return this fact
@@ -274,3 +318,16 @@ func HasMountRefs(mountPath string, mountRefs []string) bool {
 	}
 	return count > 0
 }
+
+// pathWithinBase checks if give path is within given base directory.
+func pathWithinBase(fullPath, basePath string) bool {
+	rel, err := filepath.Rel(basePath, fullPath)
+	if err != nil {
+		return false
+	}
+	if strings.HasPrefix(rel, "..") {
+		// Needed to escape the base path
+		return false
+	}
+	return true
+}

pkg/util/mount/mount_unsupported.go

@@ -20,6 +20,7 @@ package mount
 
 import (
 	"errors"
+	"os"
 )
 
 type Mounter struct {
@@ -108,3 +109,15 @@ func (mounter *Mounter) MakeFile(pathname string) error {
 func (mounter *Mounter) ExistsPath(pathname string) bool {
 	return true
 }
+
+func (mounter *Mounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
+	return subPath.Path, nil, nil
+}
+
+func (mounter *Mounter) CleanSubPaths(podDir string, volumeName string) error {
+	return nil
+}
+
+func (mounter *Mounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
+	return nil
+}

pkg/util/mount/mount_windows.go

@@ -260,6 +260,123 @@ func (mounter *Mounter) ExistsPath(pathname string) bool {
 	return true
 }
 
+// check whether hostPath is within volume path
+// this func will lock all intermediate subpath directories, need to close handle outside of this func after container started
+func lockAndCheckSubPath(volumePath, hostPath string) ([]uintptr, error) {
+	if len(volumePath) == 0 || len(hostPath) == 0 {
+		return []uintptr{}, nil
+	}
+
+	finalSubPath, err := filepath.EvalSymlinks(hostPath)
+	if err != nil {
+		return []uintptr{}, fmt.Errorf("cannot read link %s: %s", hostPath, err)
+	}
+	finalVolumePath, err := filepath.EvalSymlinks(volumePath)
+	if err != nil {
+		return []uintptr{}, fmt.Errorf("cannot read link %s: %s", volumePath, err)
+	}
+
+	return lockAndCheckSubPathWithoutSymlink(finalVolumePath, finalSubPath)
+}
+
+// lock all intermediate subPath directories and check they are all within volumePath
+// volumePath & subPath should not contain any symlink, otherwise it will return error
+func lockAndCheckSubPathWithoutSymlink(volumePath, subPath string) ([]uintptr, error) {
+	if len(volumePath) == 0 || len(subPath) == 0 {
+		return []uintptr{}, nil
+	}
+
+	// get relative path to volumePath
+	relSubPath, err := filepath.Rel(volumePath, subPath)
+	if err != nil {
+		return []uintptr{}, fmt.Errorf("Rel(%s, %s) error: %v", volumePath, subPath, err)
+	}
+	if strings.HasPrefix(relSubPath, "..") {
+		return []uintptr{}, fmt.Errorf("SubPath %q not within volume path %q", subPath, volumePath)
+	}
+
+	if relSubPath == "." {
+		// volumePath and subPath are equal
+		return []uintptr{}, nil
+	}
+
+	fileHandles := []uintptr{}
+	var errorResult error
+
+	currentFullPath := volumePath
+	dirs := strings.Split(relSubPath, string(os.PathSeparator))
+	for _, dir := range dirs {
+		// lock intermediate subPath directory first
+		currentFullPath = filepath.Join(currentFullPath, dir)
+		handle, err := lockPath(currentFullPath)
+		if err != nil {
+			errorResult = fmt.Errorf("cannot lock path %s: %s", currentFullPath, err)
+			break
+		}
+		fileHandles = append(fileHandles, handle)
+
+		// make sure intermediate subPath directory does not contain symlink any more
+		stat, err := os.Lstat(currentFullPath)
+		if err != nil {
+			errorResult = fmt.Errorf("Lstat(%q) error: %v", currentFullPath, err)
+			break
+		}
+		if stat.Mode()&os.ModeSymlink != 0 {
+			errorResult = fmt.Errorf("subpath %q is an unexpected symlink after EvalSymlinks", currentFullPath)
+			break
+		}
+
+		if !pathWithinBase(currentFullPath, volumePath) {
+			errorResult = fmt.Errorf("SubPath %q not within volume path %q", currentFullPath, volumePath)
+			break
+		}
+	}
+
+	return fileHandles, errorResult
+}
+
+// unlockPath unlock directories
+func unlockPath(fileHandles []uintptr) {
+	if fileHandles != nil {
+		for _, handle := range fileHandles {
+			syscall.CloseHandle(syscall.Handle(handle))
+		}
+	}
+}
+
+// lockPath locks a directory or symlink, return handle, exec "syscall.CloseHandle(handle)" to unlock the path
+func lockPath(path string) (uintptr, error) {
+	if len(path) == 0 {
+		return uintptr(syscall.InvalidHandle), syscall.ERROR_FILE_NOT_FOUND
+	}
+	pathp, err := syscall.UTF16PtrFromString(path)
+	if err != nil {
+		return uintptr(syscall.InvalidHandle), err
+	}
+	access := uint32(syscall.GENERIC_READ)
+	sharemode := uint32(syscall.FILE_SHARE_READ)
+	createmode := uint32(syscall.OPEN_EXISTING)
+	flags := uint32(syscall.FILE_FLAG_BACKUP_SEMANTICS | syscall.FILE_FLAG_OPEN_REPARSE_POINT)
+	fd, err := syscall.CreateFile(pathp, access, sharemode, nil, createmode, flags, 0)
+	return uintptr(fd), err
+}
+
+// Lock all directories in subPath and check they're not symlinks.
+func (mounter *Mounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
+	handles, err := lockAndCheckSubPath(subPath.VolumePath, subPath.Path)
+
+	// Unlock the directories when the container starts
+	cleanupAction = func() {
+		unlockPath(handles)
+	}
+	return subPath.Path, cleanupAction, err
+}
+
+// No bind-mounts for subpaths are necessary on Windows
+func (mounter *Mounter) CleanSubPaths(podDir string, volumeName string) error {
+	return nil
+}
+
 func (mounter *SafeFormatAndMount) formatAndMount(source string, target string, fstype string, options []string) error {
 	// Try to mount the disk
 	glog.V(4).Infof("Attempting to formatAndMount disk: %s %s %s", fstype, source, target)
@@ -344,3 +461,120 @@ func getAllParentLinks(path string) ([]string, error) {
 
 	return links, nil
 }
+
+// SafeMakeDir makes sure that the created directory does not escape given base directory mis-using symlinks.
+func (mounter *Mounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
+	return doSafeMakeDir(pathname, base, perm)
+}
+
+func doSafeMakeDir(pathname string, base string, perm os.FileMode) error {
+	glog.V(4).Infof("Creating directory %q within base %q", pathname, base)
+
+	if !pathWithinBase(pathname, base) {
+		return fmt.Errorf("path %s is outside of allowed base %s", pathname, base)
+	}
+
+	// Quick check if the directory already exists
+	s, err := os.Stat(pathname)
+	if err == nil {
+		// Path exists
+		if s.IsDir() {
+			// The directory already exists. It can be outside of the parent,
+			// but there is no race-proof check.
+			glog.V(4).Infof("Directory %s already exists", pathname)
+			return nil
+		}
+		return &os.PathError{Op: "mkdir", Path: pathname, Err: syscall.ENOTDIR}
+	}
+
+	// Find all existing directories
+	existingPath, toCreate, err := findExistingPrefix(base, pathname)
+	if err != nil {
+		return fmt.Errorf("error opening directory %s: %s", pathname, err)
+	}
+	if len(toCreate) == 0 {
+		return nil
+	}
+
+	// Ensure the existing directory is inside allowed base
+	fullExistingPath, err := filepath.EvalSymlinks(existingPath)
+	if err != nil {
+		return fmt.Errorf("error opening existing directory %s: %s", existingPath, err)
+	}
+	fullBasePath, err := filepath.EvalSymlinks(base)
+	if err != nil {
+		return fmt.Errorf("cannot read link %s: %s", base, err)
+	}
+	if !pathWithinBase(fullExistingPath, fullBasePath) {
+		return fmt.Errorf("path %s is outside of allowed base %s", fullExistingPath, err)
+	}
+
+	// lock all intermediate directories from fullBasePath to fullExistingPath (top to bottom)
+	fileHandles, err := lockAndCheckSubPathWithoutSymlink(fullBasePath, fullExistingPath)
+	defer unlockPath(fileHandles)
+	if err != nil {
+		return err
+	}
+
+	glog.V(4).Infof("%q already exists, %q to create", fullExistingPath, filepath.Join(toCreate...))
+	currentPath := fullExistingPath
+	// create the directories one by one, making sure nobody can change
+	// created directory into symlink by lock that directory immediately
+	for _, dir := range toCreate {
+		currentPath = filepath.Join(currentPath, dir)
+		glog.V(4).Infof("Creating %s", dir)
+		if err := os.Mkdir(currentPath, perm); err != nil {
+			return fmt.Errorf("cannot create directory %s: %s", currentPath, err)
+		}
+		handle, err := lockPath(currentPath)
+		if err != nil {
+			return fmt.Errorf("cannot lock path %s: %s", currentPath, err)
+		}
+		defer syscall.CloseHandle(syscall.Handle(handle))
+		// make sure newly created directory does not contain symlink after lock
+		stat, err := os.Lstat(currentPath)
+		if err != nil {
+			return fmt.Errorf("Lstat(%q) error: %v", currentPath, err)
+		}
+		if stat.Mode()&os.ModeSymlink != 0 {
+			return fmt.Errorf("subpath %q is an unexpected symlink after Mkdir", currentPath)
+		}
+	}
+
+	return nil
+}
+
+// findExistingPrefix finds prefix of pathname that exists. In addition, it
+// returns list of remaining directories that don't exist yet.
+func findExistingPrefix(base, pathname string) (string, []string, error) {
+	rel, err := filepath.Rel(base, pathname)
+	if err != nil {
+		return base, nil, err
+	}
+
+	if strings.HasPrefix(rel, "..") {
+		return base, nil, fmt.Errorf("pathname(%s) is not within base(%s)", pathname, base)
+	}
+
+	if rel == "." {
+		// base and pathname are equal
+		return pathname, []string{}, nil
+	}
+
+	dirs := strings.Split(rel, string(filepath.Separator))
+
+	parent := base
+	currentPath := base
+	for i, dir := range dirs {
+		parent = currentPath
+		currentPath = filepath.Join(parent, dir)
+		if _, err := os.Lstat(currentPath); err != nil {
+			if os.IsNotExist(err) {
+				return parent, dirs[i:], nil
+			}
+			return base, nil, err
+		}
+	}
+
+	return pathname, []string{}, nil
+}

pkg/util/mount/nsenter_mount.go

@@ -22,9 +22,12 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"regexp"
+	"strconv"
 	"strings"
 
 	"github.com/golang/glog"
+	utilio "k8s.io/kubernetes/pkg/util/io"
 	"k8s.io/kubernetes/pkg/util/nsenter"
 )
 
@@ -33,6 +36,13 @@ const (
 	hostProcMountsPath = "/rootfs/proc/1/mounts"
 	// hostProcMountinfoPath is the default mount info path for rootfs
 	hostProcMountinfoPath = "/rootfs/proc/1/mountinfo"
+	// hostProcSelfStatusPath is the default path to /proc/self/status on the host
+	hostProcSelfStatusPath = "/rootfs/proc/self/status"
+)
+
+var (
+	// pidRegExp matches "Pid:	<pid>" in /proc/self/status
+	pidRegExp = regexp.MustCompile(`\nPid:\t([0-9]*)\n`)
 )
 
 // Currently, all docker containers receive their own mount namespaces.
@@ -270,3 +280,41 @@ func (mounter *NsenterMounter) ExistsPath(pathname string) bool {
 	}
 	return false
 }
+
+func (mounter *NsenterMounter) CleanSubPaths(podDir string, volumeName string) error {
+	return doCleanSubPaths(mounter, podDir, volumeName)
+}
+
+// getPidOnHost returns kubelet's pid in the host pid namespace
+func (mounter *NsenterMounter) getPidOnHost(procStatusPath string) (int, error) {
+	// Get the PID from /rootfs/proc/self/status
+	statusBytes, err := utilio.ConsistentRead(procStatusPath, maxListTries)
+	if err != nil {
+		return 0, fmt.Errorf("error reading %s: %s", procStatusPath, err)
+	}
+	matches := pidRegExp.FindSubmatch(statusBytes)
+	if len(matches) < 2 {
+		return 0, fmt.Errorf("cannot parse %s: no Pid:", procStatusPath)
+	}
+	return strconv.Atoi(string(matches[1]))
+}
+
+func (mounter *NsenterMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
+	hostPid, err := mounter.getPidOnHost(hostProcSelfStatusPath)
+	if err != nil {
+		return "", nil, err
+	}
+	glog.V(4).Infof("Kubelet's PID on the host is %d", hostPid)
+
+	// Bind-mount the subpath to avoid using symlinks in subpaths.
+	newHostPath, err = doBindSubPath(mounter, subPath, hostPid)
+
+	// There is no action when the container starts. Bind-mount will be cleaned
+	// when container stops by CleanSubPaths.
+	cleanupAction = nil
+	return newHostPath, cleanupAction, err
+}
+
+func (mounter *NsenterMounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
+	return doSafeMakeDir(pathname, base, perm)
+}

pkg/util/mount/nsenter_mount_test.go

@@ -18,7 +18,13 @@ limitations under the License.
 
 package mount
 
-import "testing"
+import (
+	"io/ioutil"
+	"os"
+	"path"
+	"strconv"
+	"testing"
+)
 
 func TestParseFindMnt(t *testing.T) {
 	tests := []struct {
@@ -65,3 +71,121 @@ func TestParseFindMnt(t *testing.T) {
 		}
 	}
 }
+
+func TestGetPidOnHost(t *testing.T) {
+	tempDir, err := ioutil.TempDir("", "get_pid_on_host_tests")
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+	defer os.RemoveAll(tempDir)
+
+	tests := []struct {
+		name        string
+		procFile    string
+		expectedPid int
+		expectError bool
+	}{
+		{
+			name: "valid status file",
+			procFile: `Name:	cat
+Umask:	0002
+State:	R (running)
+Tgid:	15041
+Ngid:	0
+Pid:	15041
+PPid:	22699
+TracerPid:	0
+Uid:	1000	1000	1000	1000
+Gid:	1000	1000	1000	1000
+FDSize:	256
+Groups:	10 135 156 157 158 973 984 1000 1001
+NStgid:	15041
+NSpid:	15041
+NSpgid:	15041
+NSsid:	22699
+VmPeak:	  115016 kB
+VmSize:	  115016 kB
+VmLck:	       0 kB
+VmPin:	       0 kB
+VmHWM:	     816 kB
+VmRSS:	     816 kB
+RssAnon:	      64 kB
+RssFile:	     752 kB
+RssShmem:	       0 kB
+VmData:	     312 kB
+VmStk:	     136 kB
+VmExe:	      32 kB
+VmLib:	    2060 kB
+VmPTE:	      44 kB
+VmPMD:	      12 kB
+VmSwap:	       0 kB
+HugetlbPages:	       0 kB
+Threads:	1
+SigQ:	2/60752
+SigPnd:	0000000000000000
+ShdPnd:	0000000000000000
+SigBlk:	0000000000000000
+SigIgn:	0000000000000000
+SigCgt:	0000000000000000
+CapInh:	0000000000000000
+CapPrm:	0000000000000000
+CapEff:	0000000000000000
+CapBnd:	0000003fffffffff
+CapAmb:	0000000000000000
+NoNewPrivs:	0
+Seccomp:	0
+Cpus_allowed:	ff
+Cpus_allowed_list:	0-7
+Mems_allowed:	00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000001
+Mems_allowed_list:	0
+voluntary_ctxt_switches:	0
+nonvoluntary_ctxt_switches:	0
+`,
+			expectedPid: 15041,
+		},
+		{
+			name: "no Pid:",
+			procFile: `Name:	cat
+Umask:	0002
+State:	R (running)
+Tgid:	15041
+Ngid:	0
+PPid:	22699
+`,
+			expectedPid: 0,
+			expectError: true,
+		},
+		{
+			name: "invalid Pid:",
+			procFile: `Name:	cat
+Umask:	0002
+State:	R (running)
+Tgid:	15041
+Ngid:	0
+Pid:	invalid
+PPid:	22699
+`,
+			expectedPid: 0,
+			expectError: true,
+		},
+	}
+
+	for i, test := range tests {
+		filename := path.Join(tempDir, strconv.Itoa(i))
+		err := ioutil.WriteFile(filename, []byte(test.procFile), 0666)
+		if err != nil {
+			t.Fatalf(err.Error())
+		}
+		mounter := NsenterMounter{}
+		pid, err := mounter.getPidOnHost(filename)
+		if err != nil && !test.expectError {
+			t.Errorf("Test %q: unexpected error: %s", test.name, err)
+		}
+		if err == nil && test.expectError {
+			t.Errorf("Test %q: expected error, got none", test.name)
+		}
+		if pid != test.expectedPid {
+			t.Errorf("Test %q: expected pid %d, got %d", test.name, test.expectedPid, pid)
+		}
+	}
+}

pkg/util/mount/nsenter_mount_unsupported.go

@@ -20,6 +20,7 @@ package mount
 
 import (
 	"errors"
+	"os"
 )
 
 type NsenterMounter struct{}
@@ -85,3 +86,15 @@ func (*NsenterMounter) MakeFile(pathname string) error {
 func (*NsenterMounter) ExistsPath(pathname string) bool {
 	return true
 }
+
+func (*NsenterMounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
+	return nil
+}
+
+func (*NsenterMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
+	return subPath.Path, nil, nil
+}
+
+func (*NsenterMounter) CleanSubPaths(podDir string, volumeName string) error {
+	return nil
+}

pkg/util/removeall/removeall_test.go

@@ -79,6 +79,18 @@ func (mounter *fakeMounter) ExistsPath(pathname string) bool {
 	return true
 }
 
+func (mounter *fakeMounter) PrepareSafeSubpath(subPath mount.Subpath) (newHostPath string, cleanupAction func(), err error) {
+	return "", nil, nil
+}
+
+func (mounter *fakeMounter) CleanSubPaths(_, _ string) error {
+	return nil
+}
+
+func (mounter *fakeMounter) SafeMakeDir(_, _ string, _ os.FileMode) error {
+	return nil
+}
+
 func (mounter *fakeMounter) IsLikelyNotMountPoint(file string) (bool, error) {
 	name := path.Base(file)
 	if strings.HasPrefix(name, "mount") {

pkg/volume/host_path/host_path_test.go

@@ -376,6 +376,18 @@ func (fftc *fakeFileTypeChecker) GetFileType(_ string) (utilmount.FileType, erro
 	return utilmount.FileType(fftc.desiredType), nil
 }
 
+func (fftc *fakeFileTypeChecker) PrepareSafeSubpath(subPath utilmount.Subpath) (newHostPath string, cleanupAction func(), err error) {
+	return "", nil, nil
+}
+
+func (fftc *fakeFileTypeChecker) CleanSubPaths(_, _ string) error {
+	return nil
+}
+
+func (fftc *fakeFileTypeChecker) SafeMakeDir(_, _ string, _ os.FileMode) error {
+	return nil
+}
+
 func setUp() error {
 	err := os.MkdirAll("/tmp/ExistingFolder", os.FileMode(0755))
 	if err != nil {

pkg/volume/util/operationexecutor/operation_executor.go

@@ -112,7 +112,7 @@ type OperationExecutor interface {
 	// For 'Block' volumeMode, this method unmaps symbolic link to the volume
 	// from both the pod device map path in volumeToUnmount and global map path.
 	// And then, updates the actual state of the world to reflect that.
-	UnmountVolume(volumeToUnmount MountedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater) error
+	UnmountVolume(volumeToUnmount MountedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater, podsDir string) error
 
 	// If a volume has 'Filesystem' volumeMode, UnmountDevice unmounts the
 	// volumes global mount path from the device (for attachable volumes only,
@@ -746,7 +746,8 @@ func (oe *operationExecutor) MountVolume(
 
 func (oe *operationExecutor) UnmountVolume(
 	volumeToUnmount MountedVolume,
-	actualStateOfWorld ActualStateOfWorldMounterUpdater) error {
+	actualStateOfWorld ActualStateOfWorldMounterUpdater,
+	podsDir string) error {
 	fsVolume, err := util.CheckVolumeModeFilesystem(volumeToUnmount.VolumeSpec)
 	if err != nil {
 		return err
@@ -756,7 +757,7 @@ func (oe *operationExecutor) UnmountVolume(
 		// Filesystem volume case
 		// Unmount a volume if a volume is mounted
 		generatedOperations, err = oe.operationGenerator.GenerateUnmountVolumeFunc(
-			volumeToUnmount, actualStateOfWorld)
+			volumeToUnmount, actualStateOfWorld, podsDir)
 	} else {
 		// Block volume case
 		// Unmap a volume if a volume is mapped

pkg/volume/util/operationexecutor/operation_executor_test.go

@@ -124,7 +124,7 @@ func TestOperationExecutor_UnmountVolume_ConcurrentUnmountForAllPlugins(t *testi
 				PodUID:     pod.UID,
 			}
 		}
-		oe.UnmountVolume(volumesToUnmount[i], nil /* actualStateOfWorldMounterUpdater */)
+		oe.UnmountVolume(volumesToUnmount[i], nil /* actualStateOfWorldMounterUpdater */, "" /*podsDir*/)
 	}
 
 	// Assert
@@ -318,7 +318,7 @@ func TestOperationExecutor_UnmountVolume_ConcurrentUnmountForAllPlugins_VolumeMo
 				VolumeSpec: tmpSpec,
 			}
 		}
-		oe.UnmountVolume(volumesToUnmount[i], nil /* actualStateOfWorldMounterUpdater */)
+		oe.UnmountVolume(volumesToUnmount[i], nil /* actualStateOfWorldMounterUpdater */, "" /* podsDir */)
 	}
 
 	// Assert
@@ -372,7 +372,7 @@ func (fopg *fakeOperationGenerator) GenerateMountVolumeFunc(waitForAttachTimeout
 		OperationFunc: opFunc,
 	}, nil
 }
-func (fopg *fakeOperationGenerator) GenerateUnmountVolumeFunc(volumeToUnmount MountedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater) (volumetypes.GeneratedOperations, error) {
+func (fopg *fakeOperationGenerator) GenerateUnmountVolumeFunc(volumeToUnmount MountedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater, podsDir string) (volumetypes.GeneratedOperations, error) {
 	opFunc := func() (error, error) {
 		startOperationAndBlock(fopg.ch, fopg.quit)
 		return nil, nil

pkg/volume/util/operationexecutor/operation_generator.go

@@ -18,6 +18,7 @@ package operationexecutor
 
 import (
 	"fmt"
+	"path"
 	"strings"
 	"time"
 
@@ -85,7 +86,7 @@ type OperationGenerator interface {
 	GenerateMountVolumeFunc(waitForAttachTimeout time.Duration, volumeToMount VolumeToMount, actualStateOfWorldMounterUpdater ActualStateOfWorldMounterUpdater, isRemount bool) (volumetypes.GeneratedOperations, error)
 
 	// Generates the UnmountVolume function needed to perform the unmount of a volume plugin
-	GenerateUnmountVolumeFunc(volumeToUnmount MountedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater) (volumetypes.GeneratedOperations, error)
+	GenerateUnmountVolumeFunc(volumeToUnmount MountedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater, podsDir string) (volumetypes.GeneratedOperations, error)
 
 	// Generates the AttachVolume function needed to perform attach of a volume plugin
 	GenerateAttachVolumeFunc(volumeToAttach VolumeToAttach, actualStateOfWorld ActualStateOfWorldAttacherUpdater) (volumetypes.GeneratedOperations, error)
@@ -651,7 +652,8 @@ func (og *operationGenerator) resizeFileSystem(volumeToMount VolumeToMount, devi
 
 func (og *operationGenerator) GenerateUnmountVolumeFunc(
 	volumeToUnmount MountedVolume,
-	actualStateOfWorld ActualStateOfWorldMounterUpdater) (volumetypes.GeneratedOperations, error) {
+	actualStateOfWorld ActualStateOfWorldMounterUpdater,
+	podsDir string) (volumetypes.GeneratedOperations, error) {
 	// Get mountable plugin
 	volumePlugin, err :=
 		og.volumePluginMgr.FindPluginByName(volumeToUnmount.PluginName)
@@ -666,6 +668,14 @@ func (og *operationGenerator) GenerateUnmountVolumeFunc(
 	}
 
 	unmountVolumeFunc := func() (error, error) {
+		mounter := og.volumePluginMgr.Host.GetMounter(volumeToUnmount.PluginName)
+
+		// Remove all bind-mounts for subPaths
+		podDir := path.Join(podsDir, string(volumeToUnmount.PodUID))
+		if err := mounter.CleanSubPaths(podDir, volumeToUnmount.OuterVolumeSpecName); err != nil {
+			return volumeToUnmount.GenerateError("error cleaning subPath mounts", err)
+		}
+
 		// Execute unmount
 		unmountErr := volumeUnmounter.TearDown()
 		if unmountErr != nil {

test/e2e/framework/util.go

@@ -1368,23 +1368,23 @@ func WaitForPodRunningInNamespace(c clientset.Interface, pod *v1.Pod) error {
 	if pod.Status.Phase == v1.PodRunning {
 		return nil
 	}
-	return waitTimeoutForPodRunningInNamespace(c, pod.Name, pod.Namespace, PodStartTimeout)
+	return WaitTimeoutForPodRunningInNamespace(c, pod.Name, pod.Namespace, PodStartTimeout)
 }
 
 // Waits default amount of time (PodStartTimeout) for the specified pod to become running.
 // Returns an error if timeout occurs first, or pod goes in to failed state.
 func WaitForPodNameRunningInNamespace(c clientset.Interface, podName, namespace string) error {
-	return waitTimeoutForPodRunningInNamespace(c, podName, namespace, PodStartTimeout)
+	return WaitTimeoutForPodRunningInNamespace(c, podName, namespace, PodStartTimeout)
 }
 
 // Waits an extended amount of time (slowPodStartTimeout) for the specified pod to become running.
 // The resourceVersion is used when Watching object changes, it tells since when we care
 // about changes to the pod. Returns an error if timeout occurs first, or pod goes in to failed state.
 func waitForPodRunningInNamespaceSlow(c clientset.Interface, podName, namespace string) error {
-	return waitTimeoutForPodRunningInNamespace(c, podName, namespace, slowPodStartTimeout)
+	return WaitTimeoutForPodRunningInNamespace(c, podName, namespace, slowPodStartTimeout)
 }
 
-func waitTimeoutForPodRunningInNamespace(c clientset.Interface, podName, namespace string, timeout time.Duration) error {
+func WaitTimeoutForPodRunningInNamespace(c clientset.Interface, podName, namespace string, timeout time.Duration) error {
 	return wait.PollImmediate(Poll, timeout, podRunning(c, podName, namespace))
 }
 

test/e2e/storage/BUILD

-package(default_visibility = ["//visibility:public"])
-
-load(
-    "@io_bazel_rules_go//go:def.bzl",
-    "go_library",
-)
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
 
 go_library(
     name = "go_default_library",
@@ -21,6 +16,7 @@ go_library(
         "pv_protection.go",
         "pvc_protection.go",
         "regional_pd.go",
+        "subpath.go",
         "volume_expand.go",
         "volume_io.go",
         "volume_metrics.go",
@@ -28,6 +24,7 @@ go_library(
         "volumes.go",
     ],
     importpath = "k8s.io/kubernetes/test/e2e/storage",
+    visibility = ["//visibility:public"],
     deps = [
         "//pkg/api/testapi:go_default_library",
         "//pkg/api/v1/pod:go_default_library",
@@ -70,6 +67,7 @@ go_library(
         "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/rand:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
@@ -95,4 +93,5 @@ filegroup(
         "//test/e2e/storage/vsphere:all-srcs",
     ],
     tags = ["automanaged"],
+    visibility = ["//visibility:public"],
 )