Merge pull request #20169 from vishh/20117

pkg/kubelet/dockertools/manager.go

@@ -1357,6 +1357,38 @@ func containerAndPodFromLabels(inspect *docker.Container) (pod *api.Pod, contain
 	return
 }
 
+func (dm *DockerManager) applyOOMScoreAdj(container *api.Container, containerInfo *docker.Container) error {
+	cgroupName, err := dm.procFs.GetFullContainerName(containerInfo.State.Pid)
+	if err != nil {
+		if err == os.ErrNotExist {
+			// Container exited. We cannot do anything about it. Ignore this error.
+			glog.V(2).Infof("Failed to apply OOM score adj on container %q with ID %q. Init process does not exist.", containerInfo.Name, containerInfo.ID)
+			return nil
+		}
+		return err
+	}
+	// Set OOM score of the container based on the priority of the container.
+	// Processes in lower-priority pods should be killed first if the system runs out of memory.
+	// The main pod infrastructure container is considered high priority, since if it is killed the
+	// whole pod will die.
+	// TODO: Cache this value.
+	var oomScoreAdj int
+	if containerInfo.Name == PodInfraContainerName {
+		oomScoreAdj = qos.PodInfraOOMAdj
+	} else {
+		oomScoreAdj = qos.GetContainerOOMScoreAdjust(container, int64(dm.machineInfo.MemoryCapacity))
+	}
+	if err = dm.oomAdjuster.ApplyOOMScoreAdjContainer(cgroupName, oomScoreAdj, 5); err != nil {
+		if err == os.ErrNotExist {
+			// Container exited. We cannot do anything about it. Ignore this error.
+			glog.V(2).Infof("Failed to apply OOM score adj on container %q with ID %q. Init process does not exist.", containerInfo.Name, containerInfo.ID)
+			return nil
+		}
+		return err
+	}
+	return nil
+}
+
 // Run a single container from a pod. Returns the docker container ID
 // If do not need to pass labels, just pass nil.
 func (dm *DockerManager) runContainerInPod(pod *api.Pod, container *api.Container, netMode, ipcMode, pidMode string, restartCount int) (kubecontainer.ContainerID, error) {
@@ -1418,24 +1450,9 @@ func (dm *DockerManager) runContainerInPod(pod *api.Pod, container *api.Containe
 		return kubecontainer.ContainerID{}, fmt.Errorf("can't get init PID for container %q", id)
 	}
 
-	// Set OOM score of the container based on the priority of the container.
-	// Processes in lower-priority pods should be killed first if the system runs out of memory.
-	// The main pod infrastructure container is considered high priority, since if it is killed the
-	// whole pod will die.
-	var oomScoreAdj int
-	if container.Name == PodInfraContainerName {
-		oomScoreAdj = qos.PodInfraOOMAdj
-	} else {
-		oomScoreAdj = qos.GetContainerOOMScoreAdjust(container, int64(dm.machineInfo.MemoryCapacity))
-	}
-	cgroupName, err := dm.procFs.GetFullContainerName(containerInfo.State.Pid)
-	if err != nil {
-		return kubecontainer.ContainerID{}, fmt.Errorf("GetFullContainerName: %v", err)
-	}
-	if err = dm.oomAdjuster.ApplyOOMScoreAdjContainer(cgroupName, oomScoreAdj, 5); err != nil {
-		return kubecontainer.ContainerID{}, fmt.Errorf("ApplyOOMScoreAdjContainer: %v", err)
+	if err := dm.applyOOMScoreAdj(container, containerInfo); err != nil {
+		return kubecontainer.ContainerID{}, fmt.Errorf("failed to apply oom-score-adj to container %q- %v", err, containerInfo.Name)
 	}
-
 	// The addNDotsOption call appends the ndots option to the resolv.conf file generated by docker.
 	// This resolv.conf file is shared by all containers of the same pod, and needs to be modified only once per pod.
 	// we modify it when the pause container is created since it is the first container created in the pod since it holds

pkg/util/oom/oom_linux.go

@@ -21,6 +21,7 @@ package oom
 import (
 	"fmt"
 	"io/ioutil"
+	"os"
 	"path"
 	"strconv"
 
@@ -48,7 +49,18 @@ func getPids(cgroupName string) ([]int, error) {
 	return fsManager.GetPids()
 }
 
+func syscallNotExists(err error) bool {
+	if err == nil {
+		return false
+	}
+	if e, ok := err.(*os.SyscallError); ok && os.IsNotExist(e) {
+		return true
+	}
+	return false
+}
+
 // Writes 'value' to /proc/<pid>/oom_score_adj. PID = 0 means self
+// Returns os.ErrNotExist if the `pid` does not exist.
 func applyOOMScoreAdj(pid int, oomScoreAdj int) error {
 	if pid < 0 {
 		return fmt.Errorf("invalid PID %d specified for oom_score_adj", pid)
@@ -61,20 +73,18 @@ func applyOOMScoreAdj(pid int, oomScoreAdj int) error {
 		pidStr = strconv.Itoa(pid)
 	}
 
-	oomScoreAdjPath := path.Join("/proc", pidStr, "oom_score_adj")
 	maxTries := 2
+	oomScoreAdjPath := path.Join("/proc", pidStr, "oom_score_adj")
+	value := strconv.Itoa(oomScoreAdj)
 	var err error
 	for i := 0; i < maxTries; i++ {
-		_, readErr := ioutil.ReadFile(oomScoreAdjPath)
-		if readErr != nil {
-			err = fmt.Errorf("failed to read oom_score_adj: %v", readErr)
-		} else if writeErr := ioutil.WriteFile(oomScoreAdjPath, []byte(strconv.Itoa(oomScoreAdj)), 0700); writeErr != nil {
-			err = fmt.Errorf("failed to set oom_score_adj to %d: %v", oomScoreAdj, writeErr)
-		} else {
-			return nil
+		if err = ioutil.WriteFile(oomScoreAdjPath, []byte(value), 0700); err != nil {
+			if syscallNotExists(err) {
+				return os.ErrNotExist
+			}
+			err = fmt.Errorf("failed to apply oom-score-adj to pid %d (%v)", err)
 		}
 	}
-
 	return err
 }
 
@@ -86,6 +96,10 @@ func (oomAdjuster *OOMAdjuster) applyOOMScoreAdjContainer(cgroupName string, oom
 		continueAdjusting := false
 		pidList, err := oomAdjuster.pidLister(cgroupName)
 		if err != nil {
+			if syscallNotExists(err) {
+				// Nothing to do since the container doesn't exist anymore.
+				return os.ErrNotExist
+			}
 			continueAdjusting = true
 			glog.Errorf("Error getting process list for cgroup %s: %+v", cgroupName, err)
 		} else if len(pidList) == 0 {
@@ -97,6 +111,7 @@ func (oomAdjuster *OOMAdjuster) applyOOMScoreAdjContainer(cgroupName string, oom
 					if err = oomAdjuster.ApplyOOMScoreAdj(pid, oomScoreAdj); err == nil {
 						adjustedProcessSet[pid] = true
 					}
+					// Processes can come and go while we try to apply oom score adjust value. So ignore errors here.
 				}
 			}
 		}

pkg/util/procfs/procfs.go

@@ -19,6 +19,7 @@ package procfs
 import (
 	"fmt"
 	"io/ioutil"
+	"os"
 	"path"
 	"strconv"
 	"strings"
@@ -48,6 +49,9 @@ func (pfs *ProcFS) GetFullContainerName(pid int) (string, error) {
 	filePath := path.Join("/proc", strconv.Itoa(pid), "cgroup")
 	content, err := ioutil.ReadFile(filePath)
 	if err != nil {
+		if e, ok := err.(*os.SyscallError); ok && os.IsNotExist(e) {
+			return "", os.ErrNotExist
+		}
 		return "", err
 	}
 	return containerNameFromProcCgroup(string(content))