Merge pull request #35136 from dashpole/cadvisor_root_path_configuration

cluster/gce/config-default.sh

@@ -163,12 +163,7 @@ HAIRPIN_MODE="${HAIRPIN_MODE:-promiscuous-bridge}" # promiscuous-bridge, hairpin
 E2E_STORAGE_TEST_ENVIRONMENT=${KUBE_E2E_STORAGE_TEST_ENVIRONMENT:-false}
 
 # Evict pods whenever compute resource availability on the nodes gets below a threshold.
-# TODO: Get rid of the conditionals once https://github.com/kubernetes/kubernetes/issues/33444 is resolved.
-if [[ "${NODE_OS_DISTRIBUTION}" == "debian" ]]; then
-    EVICTION_HARD="${EVICTION_HARD:-memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%}"
-else
-    EVICTION_HARD="${EVICTION_HARD:-memory.available<100Mi,imagefs.available<10%,imagefs.inodesFree<5%}"
-fi
+EVICTION_HARD="${EVICTION_HARD:-memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%}"
 
 # Optional: custom scheduling algorithm
 SCHEDULING_ALGORITHM_PROVIDER="${SCHEDULING_ALGORITHM_PROVIDER:-}"

cluster/gce/config-test.sh

@@ -198,12 +198,7 @@ E2E_STORAGE_TEST_ENVIRONMENT=${KUBE_E2E_STORAGE_TEST_ENVIRONMENT:-false}
 PREPULL_E2E_IMAGES="${PREPULL_E2E_IMAGES:-true}"
 
 # Evict pods whenever compute resource availability on the nodes gets below a threshold.
-# TODO: Get rid of the conditionals once https://github.com/kubernetes/kubernetes/issues/33444 is resolved.
-if [[ "${NODE_OS_DISTRIBUTION}" == "debian" ]]; then
-    EVICTION_HARD="${EVICTION_HARD:-memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%}"
-else
-    EVICTION_HARD="${EVICTION_HARD:-memory.available<100Mi,imagefs.available<10%,imagefs.inodesFree<5%}"
-fi
+EVICTION_HARD="${EVICTION_HARD:-memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%}"
 
 # Optional: custom scheduling algorithm
 SCHEDULING_ALGORITHM_PROVIDER="${SCHEDULING_ALGORITHM_PROVIDER:-}"

cmd/kubelet/app/server.go

@@ -413,7 +413,7 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.KubeletDeps) (err error) {
 	}
 
 	if kubeDeps.CAdvisorInterface == nil {
-		kubeDeps.CAdvisorInterface, err = cadvisor.New(uint(s.CAdvisorPort), s.ContainerRuntime)
+		kubeDeps.CAdvisorInterface, err = cadvisor.New(uint(s.CAdvisorPort), s.ContainerRuntime, s.RootDirectory)
 		if err != nil {
 			return err
 		}

pkg/kubelet/cadvisor/cadvisor_linux.go

@@ -41,6 +41,7 @@ import (
 
 type cadvisorClient struct {
 	runtime  string
+	rootPath string
 	manager.Manager
 }
 
@@ -93,7 +94,7 @@ func containerLabels(c *cadvisorapi.ContainerInfo) map[string]string {
 }
 
 // New creates a cAdvisor and exports its API on the specified port if port > 0.
-func New(port uint, runtime string) (Interface, error) {
+func New(port uint, runtime string, rootPath string) (Interface, error) {
 	sysFs, err := sysfs.NewRealSysFs()
 	if err != nil {
 		return nil, err
@@ -107,6 +108,7 @@ func New(port uint, runtime string) (Interface, error) {
 
 	cadvisorClient := &cadvisorClient{
 		runtime:  runtime,
+		rootPath: rootPath,
 		Manager:  m,
 	}
 
@@ -202,7 +204,7 @@ func (cc *cadvisorClient) ImagesFsInfo() (cadvisorapiv2.FsInfo, error) {
 }
 
 func (cc *cadvisorClient) RootFsInfo() (cadvisorapiv2.FsInfo, error) {
-	return cc.getFsInfo(cadvisorfs.LabelSystemRoot)
+	return cc.GetDirFsInfo(cc.rootPath)
 }
 
 func (cc *cadvisorClient) getFsInfo(label string) (cadvisorapiv2.FsInfo, error) {

test/e2e/framework/test_context.go

@@ -212,7 +212,7 @@ func RegisterNodeFlags() {
 	flag.BoolVar(&TestContext.DisableKubenet, "disable-kubenet", false, "If true, start kubelet without kubenet. (default false)")
 	// TODO: uncomment this when the flag is re-enabled in kubelet
 	//flag.BoolVar(&TestContext.CgroupsPerQOS, "cgroups-per-qos", false, "Enable creation of QoS cgroup hierarchy, if true top level QoS and pod cgroups are created.")
-	flag.StringVar(&TestContext.EvictionHard, "eviction-hard", "memory.available<250Mi,imagefs.available<10%", "The hard eviction thresholds. If set, pods get evicted when the specified resources drop below the thresholds.")
+	flag.StringVar(&TestContext.EvictionHard, "eviction-hard", "memory.available<250Mi,nodefs.available<10%,nodefs.inodesFree<5%", "The hard eviction thresholds. If set, pods get evicted when the specified resources drop below the thresholds.")
 	flag.StringVar(&TestContext.ManifestPath, "manifest-path", "", "The path to the static pod manifest file.")
 	flag.BoolVar(&TestContext.PrepullImages, "prepull-images", true, "If true, prepull images so image pull failures do not cause test failures.")
 	flag.StringVar(&TestContext.RuntimeIntegrationType, "runtime-integration-type", "", "Choose the integration path for the container runtime, mainly used for CRI validation.")

test/e2e_node/environment/conformance.go

@@ -99,7 +99,7 @@ func containerRuntime() error {
 	}
 
 	// Setup cadvisor to check the container environment
-	c, err := cadvisor.New(0 /*don't start the http server*/, "docker")
+	c, err := cadvisor.New(0 /*don't start the http server*/, "docker", "/var/lib/kubelet")
 	if err != nil {
 		return printError("Container Runtime Check: %s Could not start cadvisor %v", failed, err)
 	}

vendor/github.com/google/cadvisor/container/common/fsHandler.go

@@ -16,6 +16,7 @@
 package common
 
 import (
+	"fmt"
 	"sync"
 	"time"
 
@@ -26,15 +27,20 @@ import (
 
 type FsHandler interface {
 	Start()
-	Usage() (baseUsageBytes uint64, totalUsageBytes uint64)
+	Usage() FsUsage
 	Stop()
 }
 
+type FsUsage struct {
+	BaseUsageBytes  uint64
+	TotalUsageBytes uint64
+	InodeUsage      uint64
+}
+
 type realFsHandler struct {
 	sync.RWMutex
 	lastUpdate time.Time
-	usageBytes     uint64
-	baseUsageBytes uint64
+	usage      FsUsage
 	period     time.Duration
 	minPeriod  time.Duration
 	rootfs     string
@@ -45,18 +51,19 @@ type realFsHandler struct {
 }
 
 const (
-	longDu             = time.Second
-	duTimeout          = time.Minute
-	maxDuBackoffFactor = 20
+	longOp           = time.Second
+	timeout          = 2 * time.Minute
+	maxBackoffFactor = 20
 )
 
+const DefaultPeriod = time.Minute
+
 var _ FsHandler = &realFsHandler{}
 
 func NewFsHandler(period time.Duration, rootfs, extraDir string, fsInfo fs.FsInfo) FsHandler {
 	return &realFsHandler{
 		lastUpdate: time.Time{},
-		usageBytes:     0,
-		baseUsageBytes: 0,
+		usage:      FsUsage{},
 		period:     period,
 		minPeriod:  period,
 		rootfs:     rootfs,
@@ -68,29 +75,37 @@ func NewFsHandler(period time.Duration, rootfs, extraDir string, fsInfo fs.FsInf
 
 func (fh *realFsHandler) update() error {
 	var (
-		baseUsage, extraDirUsage uint64
-		err                      error
+		baseUsage, extraDirUsage, inodeUsage    uint64
+		rootDiskErr, rootInodeErr, extraDiskErr error
 	)
 	// TODO(vishh): Add support for external mounts.
 	if fh.rootfs != "" {
-		baseUsage, err = fh.fsInfo.GetDirUsage(fh.rootfs, duTimeout)
-		if err != nil {
-			return err
-		}
+		baseUsage, rootDiskErr = fh.fsInfo.GetDirDiskUsage(fh.rootfs, timeout)
+		inodeUsage, rootInodeErr = fh.fsInfo.GetDirInodeUsage(fh.rootfs, timeout)
 	}
 
 	if fh.extraDir != "" {
-		extraDirUsage, err = fh.fsInfo.GetDirUsage(fh.extraDir, duTimeout)
-		if err != nil {
-			return err
-		}
+		extraDirUsage, extraDiskErr = fh.fsInfo.GetDirDiskUsage(fh.extraDir, timeout)
 	}
 
+	// Wait to handle errors until after all operartions are run.
+	// An error in one will not cause an early return, skipping others
 	fh.Lock()
 	defer fh.Unlock()
 	fh.lastUpdate = time.Now()
-	fh.usageBytes = baseUsage + extraDirUsage
-	fh.baseUsageBytes = baseUsage
+	if rootDiskErr == nil && fh.rootfs != "" {
+		fh.usage.InodeUsage = inodeUsage
+	}
+	if rootInodeErr == nil && fh.rootfs != "" {
+		fh.usage.TotalUsageBytes = baseUsage + extraDirUsage
+	}
+	if extraDiskErr == nil && fh.extraDir != "" {
+		fh.usage.BaseUsageBytes = baseUsage
+	}
+	// Combine errors into a single error to return
+	if rootDiskErr != nil || rootInodeErr != nil || extraDiskErr != nil {
+		return fmt.Errorf("rootDiskErr: %v, rootInodeErr: %v, extraDiskErr: %v", rootDiskErr, rootInodeErr, extraDiskErr)
+	}
 	return nil
 }
 
@@ -105,15 +120,15 @@ func (fh *realFsHandler) trackUsage() {
 			if err := fh.update(); err != nil {
 				glog.Errorf("failed to collect filesystem stats - %v", err)
 				fh.period = fh.period * 2
-				if fh.period > maxDuBackoffFactor*fh.minPeriod {
-					fh.period = maxDuBackoffFactor * fh.minPeriod
+				if fh.period > maxBackoffFactor*fh.minPeriod {
+					fh.period = maxBackoffFactor * fh.minPeriod
 				}
 			} else {
 				fh.period = fh.minPeriod
 			}
 			duration := time.Since(start)
-			if duration > longDu {
-				glog.V(2).Infof("`du` on following dirs took %v: %v", duration, []string{fh.rootfs, fh.extraDir})
+			if duration > longOp {
+				glog.V(2).Infof("du and find on following dirs took %v: %v", duration, []string{fh.rootfs, fh.extraDir})
 			}
 		}
 	}
@@ -127,8 +142,8 @@ func (fh *realFsHandler) Stop() {
 	close(fh.stopChan)
 }
 
-func (fh *realFsHandler) Usage() (baseUsageBytes, totalUsageBytes uint64) {
+func (fh *realFsHandler) Usage() FsUsage {
 	fh.RLock()
 	defer fh.RUnlock()
-	return fh.baseUsageBytes, fh.usageBytes
+	return fh.usage
 }

vendor/github.com/google/cadvisor/container/docker/handler.go

@@ -243,7 +243,7 @@ func newDockerContainerHandler(
 
 	if !ignoreMetrics.Has(container.DiskUsageMetrics) {
 		handler.fsHandler = &dockerFsHandler{
-			fsHandler:       common.NewFsHandler(time.Minute, rootfsStorageDir, otherStorageDir, fsInfo),
+			fsHandler:       common.NewFsHandler(common.DefaultPeriod, rootfsStorageDir, otherStorageDir, fsInfo),
 			thinPoolWatcher: thinPoolWatcher,
 			deviceID:        handler.deviceID,
 		}
@@ -283,8 +283,8 @@ func (h *dockerFsHandler) Stop() {
 	h.fsHandler.Stop()
 }
 
-func (h *dockerFsHandler) Usage() (uint64, uint64) {
-	baseUsage, usage := h.fsHandler.Usage()
+func (h *dockerFsHandler) Usage() common.FsUsage {
+	usage := h.fsHandler.Usage()
 
 	// When devicemapper is the storage driver, the base usage of the container comes from the thin pool.
 	// We still need the result of the fsHandler for any extra storage associated with the container.
@@ -299,12 +299,12 @@ func (h *dockerFsHandler) Usage() (uint64, uint64) {
 			// had at least 1 refresh and we still can't find the device.
 			glog.V(5).Infof("unable to get fs usage from thin pool for device %s: %v", h.deviceID, err)
 		} else {
-			baseUsage = thinPoolUsage
-			usage += thinPoolUsage
+			usage.BaseUsageBytes = thinPoolUsage
+			usage.TotalUsageBytes += thinPoolUsage
 		}
 	}
 
-	return baseUsage, usage
+	return usage
 }
 
 func (self *dockerContainerHandler) Start() {
@@ -387,7 +387,10 @@ func (self *dockerContainerHandler) getFsStats(stats *info.ContainerStats) error
 	}
 
 	fsStat := info.FsStats{Device: device, Type: fsType, Limit: limit}
-	fsStat.BaseUsage, fsStat.Usage = self.fsHandler.Usage()
+	usage := self.fsHandler.Usage()
+	fsStat.BaseUsage = usage.BaseUsageBytes
+	fsStat.Usage = usage.TotalUsageBytes
+	fsStat.Inodes = usage.InodeUsage
 
 	stats.Filesystem = append(stats.Filesystem, fsStat)
 

vendor/github.com/google/cadvisor/container/rkt/handler.go

@@ -18,7 +18,6 @@ package rkt
 import (
 	"fmt"
 	"os"
-	"time"
 
 	rktapi "github.com/coreos/rkt/api/v1alpha"
 	"github.com/google/cadvisor/container"
@@ -150,7 +149,7 @@ func newRktContainerHandler(name string, rktClient rktapi.PublicAPIClient, rktPa
 	}
 
 	if !ignoreMetrics.Has(container.DiskUsageMetrics) {
-		handler.fsHandler = common.NewFsHandler(time.Minute, rootfsStorageDir, "", fsInfo)
+		handler.fsHandler = common.NewFsHandler(common.DefaultPeriod, rootfsStorageDir, "", fsInfo)
 	}
 
 	return handler, nil
@@ -228,7 +227,10 @@ func (handler *rktContainerHandler) getFsStats(stats *info.ContainerStats) error
 
 	fsStat := info.FsStats{Device: deviceInfo.Device, Limit: limit}
 
-	fsStat.BaseUsage, fsStat.Usage = handler.fsHandler.Usage()
+	usage := handler.fsHandler.Usage()
+	fsStat.BaseUsage = usage.BaseUsageBytes
+	fsStat.Usage = usage.TotalUsageBytes
+	fsStat.Inodes = usage.InodeUsage
 
 	stats.Filesystem = append(stats.Filesystem, fsStat)
 

vendor/github.com/google/cadvisor/fs/fs.go

@@ -19,6 +19,7 @@ package fs
 
 import (
 	"bufio"
+	"bytes"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -44,24 +45,24 @@ const (
 	LabelRktImages    = "rkt-images"
 )
 
-// The maximum number of `du` tasks that can be running at once.
-const maxConsecutiveDus = 20
+// The maximum number of `du` and `find` tasks that can be running at once.
+const maxConcurrentOps = 20
 
-// A pool for restricting the number of consecutive `du` tasks running.
-var duPool = make(chan struct{}, maxConsecutiveDus)
+// A pool for restricting the number of consecutive `du` and `find` tasks running.
+var pool = make(chan struct{}, maxConcurrentOps)
 
 func init() {
-	for i := 0; i < maxConsecutiveDus; i++ {
-		releaseDuToken()
+	for i := 0; i < maxConcurrentOps; i++ {
+		releaseToken()
 	}
 }
 
-func claimDuToken() {
-	<-duPool
+func claimToken() {
+	<-pool
 }
 
-func releaseDuToken() {
-	duPool <- struct{}{}
+func releaseToken() {
+	pool <- struct{}{}
 }
 
 type partition struct {
@@ -428,12 +429,12 @@ func (self *RealFsInfo) GetDirFsDevice(dir string) (*DeviceInfo, error) {
 	return nil, fmt.Errorf("could not find device with major: %d, minor: %d in cached partitions map", major, minor)
 }
 
-func (self *RealFsInfo) GetDirUsage(dir string, timeout time.Duration) (uint64, error) {
+func (self *RealFsInfo) GetDirDiskUsage(dir string, timeout time.Duration) (uint64, error) {
 	if dir == "" {
 		return 0, fmt.Errorf("invalid directory")
 	}
-	claimDuToken()
-	defer releaseDuToken()
+	claimToken()
+	defer releaseToken()
 	cmd := exec.Command("nice", "-n", "19", "du", "-s", dir)
 	stdoutp, err := cmd.StdoutPipe()
 	if err != nil {
@@ -447,21 +448,21 @@ func (self *RealFsInfo) GetDirUsage(dir string, timeout time.Duration) (uint64,
 	if err := cmd.Start(); err != nil {
 		return 0, fmt.Errorf("failed to exec du - %v", err)
 	}
-	stdoutb, souterr := ioutil.ReadAll(stdoutp)
-	stderrb, _ := ioutil.ReadAll(stderrp)
 	timer := time.AfterFunc(timeout, func() {
 		glog.Infof("killing cmd %v due to timeout(%s)", cmd.Args, timeout.String())
 		cmd.Process.Kill()
 	})
+	stdoutb, souterr := ioutil.ReadAll(stdoutp)
+	if souterr != nil {
+		glog.Errorf("failed to read from stdout for cmd %v - %v", cmd.Args, souterr)
+	}
+	stderrb, _ := ioutil.ReadAll(stderrp)
 	err = cmd.Wait()
 	timer.Stop()
 	if err != nil {
 		return 0, fmt.Errorf("du command failed on %s with output stdout: %s, stderr: %s - %v", dir, string(stdoutb), string(stderrb), err)
 	}
 	stdout := string(stdoutb)
-	if souterr != nil {
-		glog.Errorf("failed to read from stdout for cmd %v - %v", cmd.Args, souterr)
-	}
 	usageInKb, err := strconv.ParseUint(strings.Fields(stdout)[0], 10, 64)
 	if err != nil {
 		return 0, fmt.Errorf("cannot parse 'du' output %s - %s", stdout, err)
@@ -469,6 +470,48 @@ func (self *RealFsInfo) GetDirUsage(dir string, timeout time.Duration) (uint64,
 	return usageInKb * 1024, nil
 }
 
+func (self *RealFsInfo) GetDirInodeUsage(dir string, timeout time.Duration) (uint64, error) {
+	if dir == "" {
+		return 0, fmt.Errorf("invalid directory")
+	}
+	var stdout, stdwcerr, stdfinderr bytes.Buffer
+	var err error
+	claimToken()
+	defer releaseToken()
+	findCmd := exec.Command("find", dir, "-xdev", "-printf", ".")
+	wcCmd := exec.Command("wc", "-c")
+	if wcCmd.Stdin, err = findCmd.StdoutPipe(); err != nil {
+		return 0, fmt.Errorf("failed to setup stdout for cmd %v - %v", findCmd.Args, err)
+	}
+	wcCmd.Stdout, wcCmd.Stderr, findCmd.Stderr = &stdout, &stdwcerr, &stdfinderr
+	if err = findCmd.Start(); err != nil {
+		return 0, fmt.Errorf("failed to exec cmd %v - %v; stderr: %v", findCmd.Args, err, stdfinderr.String())
+	}
+
+	if err = wcCmd.Start(); err != nil {
+		return 0, fmt.Errorf("failed to exec cmd %v - %v; stderr %v", wcCmd.Args, err, stdwcerr.String())
+	}
+	timer := time.AfterFunc(timeout, func() {
+		glog.Infof("killing cmd %v, and cmd %v due to timeout(%s)", findCmd.Args, wcCmd.Args, timeout.String())
+		wcCmd.Process.Kill()
+		findCmd.Process.Kill()
+	})
+	err = findCmd.Wait()
+	if err != nil {
+		return 0, fmt.Errorf("cmd %v failed. stderr: %s; err: %v", findCmd.Args, stdfinderr.String(), err)
+	}
+	err = wcCmd.Wait()
+	if err != nil {
+		return 0, fmt.Errorf("cmd %v failed. stderr: %s; err: %v", wcCmd.Args, stdwcerr.String(), err)
+	}
+	timer.Stop()
+	inodeUsage, err := strconv.ParseUint(strings.TrimSpace(stdout.String()), 10, 64)
+	if err != nil {
+		return 0, fmt.Errorf("cannot parse cmds: %v, %v output %s - %s", findCmd.Args, wcCmd.Args, stdout.String(), err)
+	}
+	return inodeUsage, nil
+}
+
 func getVfsStats(path string) (total uint64, free uint64, avail uint64, inodes uint64, inodesFree uint64, err error) {
 	var s syscall.Statfs_t
 	if err = syscall.Statfs(path, &s); err != nil {

vendor/github.com/google/cadvisor/fs/types.go

@@ -67,7 +67,10 @@ type FsInfo interface {
 	GetFsInfoForPath(mountSet map[string]struct{}) ([]Fs, error)
 
 	// Returns number of bytes occupied by 'dir'.
-	GetDirUsage(dir string, timeout time.Duration) (uint64, error)
+	GetDirDiskUsage(dir string, timeout time.Duration) (uint64, error)
+
+	// Returns number of inodes used by 'dir'.
+	GetDirInodeUsage(dir string, timeout time.Duration) (uint64, error)
 
 	// Returns the block device info of the filesystem on which 'dir' resides.
 	GetDirFsDevice(dir string) (*DeviceInfo, error)

vendor/github.com/google/cadvisor/info/v2/container.go

@@ -301,4 +301,8 @@ type FilesystemStats struct {
 	TotalUsageBytes *uint64 `json:"totalUsageBytes,omitempty"`
 	// Number of bytes consumed by a container through its root filesystem.
 	BaseUsageBytes *uint64 `json:"baseUsageBytes,omitempty"`
+	// Number of inodes used within the container's root filesystem.
+	// This only accounts for inodes that are shared across containers,
+	// and does not include inodes used in mounted directories.
+	InodeUsage *uint64 `json:"containter_inode_usage,omitempty"`
 }

vendor/github.com/google/cadvisor/info/v2/conversion.go

@@ -127,6 +127,7 @@ func ContainerStatsFromV1(spec *v1.ContainerSpec, stats []*v1.ContainerStats) []
 				stat.Filesystem = &FilesystemStats{
 					TotalUsageBytes: &val.Filesystem[0].Usage,
 					BaseUsageBytes:  &val.Filesystem[0].BaseUsage,
+					InodeUsage:      &val.Filesystem[0].Inodes,
 				}
 			} else if len(val.Filesystem) > 1 {
 				// Cannot handle multiple devices per container.

vendor/github.com/google/cadvisor/manager/manager.go

@@ -101,6 +101,9 @@ type Manager interface {
 	// Get version information about different components we depend on.
 	GetVersionInfo() (*info.VersionInfo, error)
 
+	// Get filesystem information for the filesystem that contains the given directory
+	GetDirFsInfo(dir string) (v2.FsInfo, error)
+
 	// Get filesystem information for a given label.
 	// Returns information for all global filesystems if label is empty.
 	GetFsInfo(label string) ([]v2.FsInfo, error)
@@ -657,6 +660,27 @@ func (self *manager) getRequestedContainers(containerName string, options v2.Req
 	return containersMap, nil
 }
 
+func (self *manager) GetDirFsInfo(dir string) (v2.FsInfo, error) {
+	dirDevice, err := self.fsInfo.GetDirFsDevice(dir)
+	if err != nil {
+		return v2.FsInfo{}, fmt.Errorf("error trying to get filesystem Device for dir %v: err: %v", dir, err)
+	}
+	dirMountpoint, err := self.fsInfo.GetMountpointForDevice(dirDevice.Device)
+	if err != nil {
+		return v2.FsInfo{}, fmt.Errorf("error trying to get MountPoint for Root Device: %v, err: %v", dirDevice, err)
+	}
+	infos, err := self.GetFsInfo("")
+	if err != nil {
+		return v2.FsInfo{}, err
+	}
+	for _, info := range infos {
+		if info.Mountpoint == dirMountpoint {
+			return info, nil
+		}
+	}
+	return v2.FsInfo{}, fmt.Errorf("did not find fs info for dir: %v", dir)
+}
+
 func (self *manager) GetFsInfo(label string) ([]v2.FsInfo, error) {
 	var empty time.Time
 	// Get latest data from filesystems hanging off root container.