Merge pull request #41944 from jingxu97/Feb/mounter

cluster/gce/gci/configure-helper.sh

@@ -652,7 +652,7 @@ function start-kubelet {
   flags+=" --cluster-dns=${DNS_SERVER_IP}"
   flags+=" --cluster-domain=${DNS_DOMAIN}"
   flags+=" --pod-manifest-path=/etc/kubernetes/manifests"
-  flags+=" --experimental-mounter-path=${KUBE_HOME}/bin/mounter"
+  flags+=" --experimental-mounter-path=${CONTAINERIZED_MOUNTER_HOME}/mounter"
   flags+=" --experimental-check-node-capabilities-before-mount=true"
 
   if [[ -n "${KUBELET_PORT:-}" ]]; then
@@ -928,6 +928,18 @@ function compute-master-manifest-variables {
   fi
 }
 
+# A helper function that bind mounts kubelet dirs for running mount in a chroot
+function prepare-mounter-rootfs {
+  echo "Prepare containerized mounter"
+  mount --bind "${CONTAINERIZED_MOUNTER_HOME}" "${CONTAINERIZED_MOUNTER_HOME}"
+  mount -o remount,exec "${CONTAINERIZED_MOUNTER_HOME}"
+  CONTAINERIZED_MOUNTER_ROOTFS="${CONTAINERIZED_MOUNTER_HOME}/rootfs"
+  mount --rbind /var/lib/kubelet/ "${CONTAINERIZED_MOUNTER_ROOTFS}/var/lib/kubelet"
+  mount --make-rshared "${CONTAINERIZED_MOUNTER_ROOTFS}/var/lib/kubelet"
+  mount --bind -o ro /proc "${CONTAINERIZED_MOUNTER_ROOTFS}/proc"
+  mount --bind -o ro /dev "${CONTAINERIZED_MOUNTER_ROOTFS}/dev"
+}
+
 # A helper function for removing salt configuration and comments from a file.
 # This is mainly for preparing a manifest file.
 #
@@ -1479,15 +1491,11 @@ function override-kubectl {
     echo "export PATH=${KUBE_HOME}/bin:\$PATH" > /etc/profile.d/kube_env.sh
 }
 
-function pre-warm-mounter {
-    echo "prewarming mounter"
-    ${KUBE_HOME}/bin/mounter &> /dev/null
-}
-
 ########### Main Function ###########
 echo "Start to configure instance for kubernetes"
 
 KUBE_HOME="/home/kubernetes"
+CONTAINERIZED_MOUNTER_HOME="${KUBE_HOME}/containerized_mounter"
 if [[ ! -e "${KUBE_HOME}/kube-env" ]]; then
   echo "The ${KUBE_HOME}/kube-env file does not exist!! Terminate cluster initialization."
   exit 1
@@ -1534,7 +1542,6 @@ fi
 
 override-kubectl
 # Run the containerized mounter once to pre-cache the container image.
-pre-warm-mounter
 assemble-docker-flags
 load-docker-images
 start-kubelet
@@ -1565,4 +1572,5 @@ else
   fi
 fi
 reset-motd
+prepare-mounter-rootfs
 echo "Done for the configuration for kubernetes"

cluster/gce/gci/configure.sh

@@ -115,19 +115,18 @@ function split-commas {
 }
 
 function install-gci-mounter-tools {
-    local -r rkt_version="v1.18.0"
-    local -r gci_mounter_version="v2"
-    local -r rkt_binary_sha1="75fc8f29c79bc9e505f3e7f6e8fadf2425c21967"
-    local -r rkt_stage1_fly_sha1="474df5a1f934960ba669b360ab713d0a54283091"
-    local -r gci_mounter_sha1="851e841d8640d6a05e64e22c493f5ac3c4cba561"
-    download-or-bust "${rkt_binary_sha1}" "https://storage.googleapis.com/kubernetes-release/rkt/${rkt_version}/rkt"
-    download-or-bust "${rkt_stage1_fly_sha1}" "https://storage.googleapis.com/kubernetes-release/rkt/${rkt_version}/stage1-fly.aci"
-    download-or-bust "${gci_mounter_sha1}" "https://storage.googleapis.com/kubernetes-release/gci-mounter/gci-mounter-${gci_mounter_version}.aci"
-    local -r rkt_dst="${KUBE_HOME}/bin/"
-    mv "${KUBE_HOME}/rkt" "${rkt_dst}/rkt"
-    mv "${KUBE_HOME}/stage1-fly.aci" "${rkt_dst}/stage1-fly.aci"
-    mv "${KUBE_HOME}/gci-mounter-${gci_mounter_version}.aci" "${rkt_dst}/gci-mounter-${gci_mounter_version}.aci"
-    chmod a+x "${rkt_dst}/rkt"
+    CONTAINERIZED_MOUNTER_HOME="${KUBE_HOME}/containerized_mounter"
+    mkdir "${CONTAINERIZED_MOUNTER_HOME}"
+    chmod a+x "${CONTAINERIZED_MOUNTER_HOME}"
+    mkdir "${CONTAINERIZED_MOUNTER_HOME}/rootfs"
+    local -r mounter_tar_sha="8003b798cf33c7f91320cd6ee5cec4fa22244571"
+    download-or-bust "${mounter_tar_sha}" "https://storage.googleapis.com/kubernetes-release/gci-mounter/mounter.tar"
+    cp "${dst_dir}/kubernetes/gci-trusty/gci-mounter" "${CONTAINERIZED_MOUNTER_HOME}/mounter"
+    chmod a+x "${CONTAINERIZED_MOUNTER_HOME}/mounter"
+    mv "${KUBE_HOME}/mounter.tar" /tmp/mounter.tar
+    tar xvf /tmp/mounter.tar -C "${CONTAINERIZED_MOUNTER_HOME}/rootfs"
+    rm /tmp/mounter.tar
+    mkdir "${CONTAINERIZED_MOUNTER_HOME}/rootfs/var/lib/kubelet"
 }
 
 # Install node problem detector binary.
@@ -222,7 +221,6 @@ function install-kube-binary-config {
       xargs sed -ri "s@(image\":\s+\")gcr.io/google_containers@\1${kube_addon_registry}@"
   fi
   cp "${dst_dir}/kubernetes/gci-trusty/gci-configure-helper.sh" "${KUBE_HOME}/bin/configure-helper.sh"
-  cp "${dst_dir}/kubernetes/gci-trusty/gci-mounter" "${KUBE_HOME}/bin/mounter"
   cp "${dst_dir}/kubernetes/gci-trusty/health-monitor.sh" "${KUBE_HOME}/bin/health-monitor.sh"
   chmod -R 755 "${kube_bin}"
 

cluster/gce/gci/master.yaml

@@ -34,7 +34,6 @@ write_files:
       Type=oneshot
       RemainAfterExit=yes
       ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure-helper.sh
-      ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/mounter
       ExecStart=/home/kubernetes/bin/configure-helper.sh
 
       [Install]

cluster/gce/gci/mounter/mounter.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+)
+
+const (
+	// Location of the mount file to use
+	chrootCmd        = "chroot"
+	mountCmd         = "mount"
+	rootfs           = "rootfs"
+	nfsRPCBindErrMsg = "mount.nfs: rpc.statd is not running but is required for remote locking.\nmount.nfs: Either use '-o nolock' to keep locks local, or start statd.\nmount.nfs: an incorrect mount option was specified\n"
+	rpcBindCmd       = "/sbin/rpcbind"
+	defaultRootfs    = "/home/kubernetes/containerized_mounter/rootfs"
+)
+
+func main() {
+
+	if len(os.Args) < 2 {
+		fmt.Fprintf(os.Stderr, "Command failed: must provide a command to run.\n")
+		return
+	}
+	path, _ := filepath.Split(os.Args[0])
+	rootfsPath := filepath.Join(path, rootfs)
+	if _, err := os.Stat(rootfsPath); os.IsNotExist(err) {
+		rootfsPath = defaultRootfs
+	}
+	command := os.Args[1]
+	switch command {
+	case mountCmd:
+		mountErr := mountInChroot(rootfsPath, os.Args[2:])
+		if mountErr != nil {
+			fmt.Fprintf(os.Stderr, "Mount failed: %v", mountErr)
+			os.Exit(1)
+		}
+	default:
+		fmt.Fprintf(os.Stderr, "Unknown command, must be %s", mountCmd)
+		os.Exit(1)
+
+	}
+}
+
+// MountInChroot is to run mount within chroot with the passing root directory
+func mountInChroot(rootfsPath string, args []string) error {
+	if _, err := os.Stat(rootfsPath); os.IsNotExist(err) {
+		return fmt.Errorf("Path <%s> does not exist.\n", rootfsPath)
+	}
+	args = append([]string{rootfsPath, mountCmd}, args...)
+	output, err := exec.Command(chrootCmd, args...).CombinedOutput()
+	if err == nil {
+		return err
+	}
+
+	if !strings.EqualFold(string(output), nfsRPCBindErrMsg) {
+		// Mount failed but not because of RPC bind error
+		return fmt.Errorf("Mount failed: %v\nMounting command: %s\nMounting arguments: %v\nOutput: %s\n", err, chrootCmd, args, string(output))
+	}
+
+	// Mount failed because it is NFS V3 and we need to run rpcBind
+	output, err = exec.Command(chrootCmd, rootfsPath, rpcBindCmd, "-w").CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("Mount issued for NFS V3 but unable to run rpcbind:\n Output: %s\n Error: %v", string(output), err)
+	}
+
+	// Rpcbind is running, try mounting again
+	output, err = exec.Command(chrootCmd, args...).CombinedOutput()
+
+	if err != nil {
+		return fmt.Errorf("Mount failed for NFS V3 even after running rpcBind %s, %v", string(output), err)
+	}
+
+	return nil
+}

cluster/gce/gci/node.yaml

@@ -34,7 +34,6 @@ write_files:
       Type=oneshot
       RemainAfterExit=yes
       ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure-helper.sh
-      ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/mounter
       ExecStart=/home/kubernetes/bin/configure-helper.sh
 
       [Install]

cluster/gce/trusty/configure.sh

@@ -137,7 +137,6 @@ install_kube_binary_config() {
     # This should be the case of GCI.
     readonly kube_bin="${kube_home}/bin"
     mkdir -p "${kube_bin}"
-    mount --bind "${kube_bin}" "${kube_bin}"
     mount -o remount,rw,exec "${kube_bin}"
     cp "${src_dir}/kubelet" "${kube_bin}"
     cp "${src_dir}/kubectl" "${kube_bin}"

cluster/saltbase/salt/e2e-image-puller/e2e-image-puller.manifest

@@ -27,7 +27,7 @@ spec:
     command:
     - /bin/sh
     - -c
-    - "for i in gcr.io/google_containers/busybox gcr.io/google_containers/busybox:1.24 gcr.io/google_containers/dnsutils:e2e gcr.io/google_containers/eptest:0.1 gcr.io/google_containers/fakegitserver:0.1 gcr.io/google_containers/hostexec:1.2 gcr.io/google_containers/iperf:e2e gcr.io/google_containers/jessie-dnsutils:e2e gcr.io/google_containers/liveness:e2e gcr.io/google_containers/mounttest:0.8 gcr.io/google_containers/mounttest-user:0.5 gcr.io/google_containers/netexec:1.4 gcr.io/google_containers/netexec:1.7 gcr.io/google_containers/nettest:1.7 gcr.io/google_containers/nettest:1.8 gcr.io/google_containers/nginx-slim:0.7 gcr.io/google_containers/nginx-slim:0.8 gcr.io/google_containers/n-way-http:1.0 gcr.io/google_containers/pause:2.0 gcr.io/google_containers/pause-amd64:3.0 gcr.io/google_containers/porter:cd5cb5791ebaa8641955f0e8c2a9bed669b1eaab gcr.io/google_containers/portforwardtester:1.2 gcr.io/google_containers/redis:e2e gcr.io/google_containers/resource_consumer:beta4 gcr.io/google_containers/resource_consumer/controller:beta4 gcr.io/google_containers/serve_hostname:v1.4 gcr.io/google_containers/test-webserver:e2e gcr.io/google_containers/ubuntu:14.04 gcr.io/google_containers/update-demo:kitten gcr.io/google_containers/update-demo:nautilus gcr.io/google_containers/volume-ceph:0.1 gcr.io/google_containers/volume-gluster:0.2 gcr.io/google_containers/volume-iscsi:0.1 gcr.io/google_containers/volume-nfs:0.6 gcr.io/google_containers/volume-rbd:0.1 gcr.io/google_samples/gb-redisslave:v1 gcr.io/google_containers/redis:v1; do echo $(date '+%X') pulling $i; docker pull $i 1>/dev/null; done; exit 0;"
+    - "for i in gcr.io/google_containers/busybox gcr.io/google_containers/busybox:1.24 gcr.io/google_containers/dnsutils:e2e gcr.io/google_containers/eptest:0.1 gcr.io/google_containers/fakegitserver:0.1 gcr.io/google_containers/hostexec:1.2 gcr.io/google_containers/iperf:e2e gcr.io/google_containers/jessie-dnsutils:e2e gcr.io/google_containers/liveness:e2e gcr.io/google_containers/mounttest:0.8 gcr.io/google_containers/mounttest-user:0.5 gcr.io/google_containers/netexec:1.4 gcr.io/google_containers/netexec:1.7 gcr.io/google_containers/nettest:1.7 gcr.io/google_containers/nettest:1.8 gcr.io/google_containers/nginx-slim:0.7 gcr.io/google_containers/nginx-slim:0.8 gcr.io/google_containers/n-way-http:1.0 gcr.io/google_containers/pause:2.0 gcr.io/google_containers/pause-amd64:3.0 gcr.io/google_containers/porter:cd5cb5791ebaa8641955f0e8c2a9bed669b1eaab gcr.io/google_containers/portforwardtester:1.2 gcr.io/google_containers/redis:e2e gcr.io/google_containers/resource_consumer:beta4 gcr.io/google_containers/resource_consumer/controller:beta4 gcr.io/google_containers/serve_hostname:v1.4 gcr.io/google_containers/test-webserver:e2e gcr.io/google_containers/ubuntu:14.04 gcr.io/google_containers/update-demo:kitten gcr.io/google_containers/update-demo:nautilus gcr.io/google_containers/volume-ceph:0.1 gcr.io/google_containers/volume-gluster:0.2 gcr.io/google_containers/volume-iscsi:0.1 gcr.io/google_containers/volume-nfs:0.8 gcr.io/google_containers/volume-rbd:0.1 gcr.io/google_samples/gb-redisslave:v1 gcr.io/google_containers/redis:v1; do echo $(date '+%X') pulling $i; docker pull $i 1>/dev/null; done; exit 0;"
     securityContext:
       privileged: true
     volumeMounts:

pkg/util/mount/mount.go

@@ -98,11 +98,6 @@ func (mounter *SafeFormatAndMount) FormatAndMount(source string, target string,
 // It provides options to override the default mounter behavior.
 // mounterPath allows using an alternative to `/bin/mount` for mounting.
 func New(mounterPath string) Interface {
-	// If mounter-path flag is not set, use default mount path
-	if mounterPath == "" {
-		mounterPath = defaultMountCommand
-	}
-
 	return &Mounter{
 		mounterPath: mounterPath,
 	}

pkg/util/mount/mount_linux.go

@@ -63,23 +63,23 @@ type Mounter struct {
 // currently come from mount(8), e.g. "ro", "remount", "bind", etc. If no more option is
 // required, call Mount with an empty string list or nil.
 func (mounter *Mounter) Mount(source string, target string, fstype string, options []string) error {
-	// Path to mounter binary. Set to mount accessible via $PATH by default.
+	// Path to mounter binary if containerized mounter is needed. Otherwise, it is set to empty.
 	// All Linux distros are expected to be shipped with a mount utility that an support bind mounts.
-	mounterPath := defaultMountCommand
+	mounterPath := ""
 	bind, bindRemountOpts := isBind(options)
 	if bind {
-		err := doMount(mounterPath, source, target, fstype, []string{"bind"})
+		err := doMount(mounterPath, defaultMountCommand, source, target, fstype, []string{"bind"})
 		if err != nil {
 			return err
 		}
-		return doMount(mounterPath, source, target, fstype, bindRemountOpts)
+		return doMount(mounterPath, defaultMountCommand, source, target, fstype, bindRemountOpts)
 	}
 	// The list of filesystems that require containerized mounter on GCI image cluster
-	fsTypesNeedMounter := sets.NewString("nfs", "glusterfs")
+	fsTypesNeedMounter := sets.NewString("nfs", "glusterfs", "ceph", "cifs")
 	if fsTypesNeedMounter.Has(fstype) {
 		mounterPath = mounter.mounterPath
 	}
-	return doMount(mounterPath, source, target, fstype, options)
+	return doMount(mounterPath, defaultMountCommand, source, target, fstype, options)
 }
 
 // isBind detects whether a bind mount is being requested and makes the remount options to
@@ -107,10 +107,13 @@ func isBind(options []string) (bool, []string) {
 	return bind, bindRemountOpts
 }
 
-// doMount runs the mount command.
-func doMount(mountCmd string, source string, target string, fstype string, options []string) error {
-	glog.V(4).Infof("Mounting %s %s %s %v with command: %q", source, target, fstype, options, mountCmd)
+// doMount runs the mount command. mounterPath is the path to mounter binary if containerized mounter is used.
+func doMount(mounterPath string, mountCmd string, source string, target string, fstype string, options []string) error {
 	mountArgs := makeMountArgs(source, target, fstype, options)
+	if len(mounterPath) > 0 {
+		mountArgs = append([]string{mountCmd}, mountArgs...)
+		mountCmd = mounterPath
+	}
 
 	glog.V(4).Infof("Mounting cmd (%s) with arguments (%s)", mountCmd, mountArgs)
 	command := exec.Command(mountCmd, mountArgs...)

test/e2e/common/volumes.go

@@ -393,6 +393,36 @@ var _ = framework.KubeDescribe("GCP Volumes", func() {
 		})
 	})
 
+	framework.KubeDescribe("NFSv3", func() {
+		It("should be mountable for NFSv3 [Volume]", func() {
+			config := VolumeTestConfig{
+				namespace:   namespace.Name,
+				prefix:      "nfs",
+				serverImage: "gcr.io/google_containers/volume-nfs:0.8",
+				serverPorts: []int{2049},
+			}
+
+			defer func() {
+				if clean {
+					volumeTestCleanup(f, config)
+				}
+			}()
+			pod := startVolumeServer(f, config)
+			serverIP := pod.Status.PodIP
+			framework.Logf("NFS server IP address: %v", serverIP)
+
+			volume := v1.VolumeSource{
+				NFS: &v1.NFSVolumeSource{
+					Server:   serverIP,
+					Path:     "/exports",
+					ReadOnly: true,
+				},
+			}
+			// Must match content of test/images/volume-tester/nfs/index.html
+			testVolumeClient(f, config, volume, nil, "Hello from NFS!")
+		})
+	})
+
 	////////////////////////////////////////////////////////////////////////
 	// Gluster
 	////////////////////////////////////////////////////////////////////////
@@ -488,10 +518,10 @@ func isTestEnabled(c clientset.Interface) bool {
 	}
 
 	// For cluster e2e test, because nodeName is empty, retrieve the node objects from api server
-	// and check their images. Only run NFSv4 and GlusterFS if nodes are using GCI image for now.
+	// and check their images. Only run NFS and GlusterFS tests if nodes are using GCI image for now.
 	nodes := framework.GetReadySchedulableNodesOrDie(c)
 	for _, node := range nodes.Items {
-		if !strings.Contains(node.Status.NodeInfo.OSImage, "Google Container-VM") {
+		if !strings.Contains(node.Status.NodeInfo.OSImage, "Container-Optimized OS") {
 			return false
 		}
 	}

test/e2e_node/jenkins/gci-init.yaml

@@ -3,10 +3,15 @@
 runcmd:
   - mount /tmp /tmp -o remount,exec,suid
   - usermod -a -G docker jenkins
-  - mkdir -p /home/kubernetes/bin/
-  - mount -B /home/kubernetes/bin /home/kubernetes/bin
-  - mount -B -o remount,exec /home/kubernetes/bin 
-  - wget https://storage.googleapis.com/kubernetes-release/rkt/v1.18.0/rkt -O /home/kubernetes/bin/rkt
-  - wget https://storage.googleapis.com/kubernetes-release/rkt/v1.18.0/stage1-fly.aci -O /home/kubernetes/bin/stage1-fly.aci
-  - wget https://storage.googleapis.com/kubernetes-release/gci-mounter/gci-mounter-v2.aci -O /home/kubernetes/bin/gci-mounter-v2.aci
-  - chmod a+x /home/kubernetes/bin/rkt
+  - mkdir -p /var/lib/kubelet
+  - mkdir -p /home/kubernetes/containerized_mounter/rootfs
+  - mount --bind /home/kubernetes/containerized_mounter/ /home/kubernetes/containerized_mounter/
+  - mount -o remount, exec /home/kubernetes/containerized_mounter/
+  - wget https://storage.googleapis.com/kubernetes-release/gci-mounter/mounter.tar -O /tmp/mounter.tar
+  - tar xvf /tmp/mounter.tar -C /home/kubernetes/containerized_mounter/rootfs
+  - mkdir -p /home/kubernetes/containerized_mounter/rootfs/var/lib/kubelet
+  - mount --rbind /var/lib/kubelet /home/kubernetes/containerized_mounter/rootfs/var/lib/kubelet
+  - mount --make-rshared /home/kubernetes/containerized_mounter/rootfs/var/lib/kubelet
+  - mount --bind /proc /home/kubernetes/containerized_mounter/rootfs/proc
+  - mount --bind /dev /home/kubernetes/containerized_mounter/rootfs/dev
+  - rm /tmp/mounter.tar
\ No newline at end of file