Merge pull request #35597 from feiskyer/gpu

pkg/kubelet/api/v1alpha1/runtime/api.proto

@@ -428,6 +428,19 @@ message ContainerMetadata {
     optional uint32 attempt = 2;
 }
 
+// Device specifies a host device to mount into a container.
+message Device {
+    // The path of the device within the container.
+    optional string container_path = 1;
+    // The path of the device on the host.
+    optional string host_path = 2;
+    // Cgroups permissions of the device, candidates are one or more of
+    // * r - allows container to read from the specified device.
+    // * w - allows container to write to the specified device.
+    // * m - allows container to create device files that do not yet exist.
+    optional string permissions = 3;
+}
+
 // ContainerConfig holds all the required and optional fields for creating a
 // container.
 message ContainerConfig {
@@ -444,25 +457,27 @@ message ContainerConfig {
     repeated string args = 4;
     // Current working directory of the command.
     optional string working_dir = 5;
-    // List of environment variable to set in the container
+    // List of environment variable to set in the container.
     repeated KeyValue envs = 6;
-    // Mounts specifies mounts for the container
+    // Mounts specifies mounts for the container.
     repeated Mount mounts = 7;
+    // Devices specifies devices for the container.
+    repeated Device devices = 8;
     // Labels are key value pairs that may be used to scope and select individual resources.
     // Label keys are of the form:
     //     label-key ::= prefixed-name | name
     //     prefixed-name ::= prefix '/' name
     //     prefix ::= DNS_SUBDOMAIN
     //     name ::= DNS_LABEL
-    map<string, string> labels = 8;
+    map<string, string> labels = 9;
     // Annotations is an unstructured key value map that may be set by external
     // tools to store and retrieve arbitrary metadata.
-    map<string, string> annotations = 9;
+    map<string, string> annotations = 10;
     // If set, run container in privileged mode.
     // Processes in privileged containers are essentially equivalent to root on the host.
-    optional bool privileged = 10;
+    optional bool privileged = 11;
     // If set, the root filesystem of the container is read-only.
-    optional bool readonly_rootfs = 11;
+    optional bool readonly_rootfs = 12;
     // Path relative to PodSandboxConfig.LogDirectory for container to store
     // the log (STDOUT and STDERR) on the host.
     // E.g.,
@@ -473,19 +488,19 @@ message ContainerConfig {
     // container logs are under active discussion in
     // https://issues.k8s.io/24677. There *may* be future change of direction
     // for logging as the discussion carries on.
-    optional string log_path = 12;
+    optional string log_path = 13;
     // The hash of container config
 
     // Variables for interactive containers, these have very specialized
     // use-cases (e.g. debugging).
     // TODO: Determine if we need to continue supporting these fields that are
     // part of Kubernetes's Container Spec.
-    optional bool stdin = 13;
-    optional bool stdin_once = 14;
-    optional bool tty = 15;
+    optional bool stdin = 14;
+    optional bool stdin_once = 15;
+    optional bool tty = 16;
 
     // Linux contains configuration specific to Linux containers.
-    optional LinuxContainerConfig linux = 16;
+    optional LinuxContainerConfig linux = 17;
 }
 
 message CreateContainerRequest {

pkg/kubelet/dockershim/docker_container.go

@@ -163,13 +163,23 @@ func (ds *dockerService) CreateContainer(podSandboxID string, config *runtimeApi
 				CPUShares:  rOpts.GetCpuShares(),
 				CPUQuota:   rOpts.GetCpuQuota(),
 				CPUPeriod:  rOpts.GetCpuPeriod(),
-				// TODO: Need to set devices.
 			}
 			hc.OomScoreAdj = int(rOpts.GetOomScoreAdj())
 		}
 		// Note: ShmSize is handled in kube_docker_client.go
 	}
 
+	// Set devices for container.
+	devices := make([]dockercontainer.DeviceMapping, len(config.Devices))
+	for i, device := range config.Devices {
+		devices[i] = dockercontainer.DeviceMapping{
+			PathOnHost:        device.GetHostPath(),
+			PathInContainer:   device.GetContainerPath(),
+			CgroupPermissions: device.GetPermissions(),
+		}
+	}
+	hc.Resources.Devices = devices
+
 	var err error
 	hc.SecurityOpt, err = getContainerSecurityOpts(config.Metadata.GetName(), sandboxConfig, ds.seccompProfileRoot)
 	if err != nil {

pkg/kubelet/dockertools/docker_manager.go

@@ -635,7 +635,6 @@ func (dm *DockerManager) runContainer(
 	memoryLimit := container.Resources.Limits.Memory().Value()
 	cpuRequest := container.Resources.Requests.Cpu()
 	cpuLimit := container.Resources.Limits.Cpu()
-	nvidiaGPULimit := container.Resources.Limits.NvidiaGPU()
 	var cpuShares int64
 	// If request is not specified, but limit is, we want request to default to limit.
 	// API server does this for new containers, but we repeat this logic in Kubelet
@@ -647,17 +646,18 @@ func (dm *DockerManager) runContainer(
 		// of CPU shares.
 		cpuShares = cm.MilliCPUToShares(cpuRequest.MilliValue())
 	}
-	var devices []dockercontainer.DeviceMapping
-	if nvidiaGPULimit.Value() != 0 {
-		// Experimental. For now, we hardcode /dev/nvidia0 no matter what the user asks for
-		// (we only support one device per node).
-		devices = []dockercontainer.DeviceMapping{
-			{PathOnHost: "/dev/nvidia0", PathInContainer: "/dev/nvidia0", CgroupPermissions: "mrw"},
-			{PathOnHost: "/dev/nvidiactl", PathInContainer: "/dev/nvidiactl", CgroupPermissions: "mrw"},
-			{PathOnHost: "/dev/nvidia-uvm", PathInContainer: "/dev/nvidia-uvm", CgroupPermissions: "mrw"},
+
+	// Set devices for container.
+	devices := make([]dockercontainer.DeviceMapping, len(opts.Devices))
+	for i, device := range opts.Devices {
+		devices[i] = dockercontainer.DeviceMapping{
+			PathOnHost:        device.PathOnHost,
+			PathInContainer:   device.PathInContainer,
+			CgroupPermissions: device.Permissions,
 		}
 	}
 	binds := makeMountBindings(opts.Mounts)
+
 	// The reason we create and mount the log file in here (not in kubelet) is because
 	// the file's location depends on the ID of the container, and we need to create and
 	// mount the file before actually starting the container.

pkg/kubelet/kubelet_pods.go

@@ -75,6 +75,23 @@ func (kl *Kubelet) getActivePods() []*api.Pod {
 	return activePods
 }
 
+// makeDevices determines the devices for the given container.
+// Experimental. For now, we hardcode /dev/nvidia0 no matter what the user asks for
+// (we only support one device per node).
+// TODO: add support for more than 1 GPU after #28216.
+func makeDevices(container *api.Container) []kubecontainer.DeviceInfo {
+	nvidiaGPULimit := container.Resources.Limits.NvidiaGPU()
+	if nvidiaGPULimit.Value() != 0 {
+		return []kubecontainer.DeviceInfo{
+			{PathOnHost: "/dev/nvidia0", PathInContainer: "/dev/nvidia0", Permissions: "mrw"},
+			{PathOnHost: "/dev/nvidiactl", PathInContainer: "/dev/nvidiactl", Permissions: "mrw"},
+			{PathOnHost: "/dev/nvidia-uvm", PathInContainer: "/dev/nvidia-uvm", Permissions: "mrw"},
+		}
+	}
+
+	return nil
+}
+
 // makeMounts determines the mount points for the given container.
 func makeMounts(pod *api.Pod, podDir string, container *api.Container, hostName, hostDomain, podIP string, podVolumes kubecontainer.VolumeMap) ([]kubecontainer.Mount, error) {
 	// Kubernetes only mounts on /etc/hosts if :
@@ -255,6 +272,7 @@ func (kl *Kubelet) GenerateRunContainerOptions(pod *api.Pod, container *api.Cont
 	volumes := kl.volumeManager.GetMountedVolumesForPod(podName)
 
 	opts.PortMappings = makePortMappings(container)
+	opts.Devices = makeDevices(container)
 
 	opts.Mounts, err = makeMounts(pod, kl.getPodDir(pod.UID), container, hostname, hostDomainName, podIP, volumes)
 	if err != nil {

pkg/kubelet/kubelet_pods_test.go

@@ -28,6 +28,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/api/resource"
 	"k8s.io/kubernetes/pkg/apimachinery/registered"
 	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
 	containertest "k8s.io/kubernetes/pkg/kubelet/container/testing"
@@ -1262,3 +1263,36 @@ func TestGetHostPortConflicts(t *testing.T) {
 	pods = append(pods, expected)
 	assert.True(t, hasHostPortConflicts(pods), "Should have port conflicts")
 }
+
+func TestMakeDevices(t *testing.T) {
+	testCases := []struct {
+		container *api.Container
+		devices   []kubecontainer.DeviceInfo
+		test      string
+	}{
+		{
+			test:      "no device",
+			container: &api.Container{},
+			devices:   nil,
+		},
+		{
+			test: "gpu",
+			container: &api.Container{
+				Resources: api.ResourceRequirements{
+					Limits: map[api.ResourceName]resource.Quantity{
+						api.ResourceNvidiaGPU: resource.MustParse("1000"),
+					},
+				},
+			},
+			devices: []kubecontainer.DeviceInfo{
+				{PathOnHost: "/dev/nvidia0", PathInContainer: "/dev/nvidia0", Permissions: "mrw"},
+				{PathOnHost: "/dev/nvidiactl", PathInContainer: "/dev/nvidiactl", Permissions: "mrw"},
+				{PathOnHost: "/dev/nvidia-uvm", PathInContainer: "/dev/nvidia-uvm", Permissions: "mrw"},
+			},
+		},
+	}
+
+	for _, test := range testCases {
+		assert.Equal(t, test.devices, makeDevices(test.container), "[test %q]", test.test)
+	}
+}

pkg/kubelet/kuberuntime/kuberuntime_container.go

@@ -151,6 +151,7 @@ func (m *kubeGenericRuntimeManager) generateContainerConfig(container *api.Conta
 		Labels:      newContainerLabels(container, pod),
 		Annotations: newContainerAnnotations(container, pod, restartCount),
 		Mounts:      m.makeMounts(opts, container, podHasSELinuxLabel),
+		Devices:     makeDevices(opts),
 		LogPath:     &containerLogsPath,
 		Stdin:       &container.Stdin,
 		StdinOnce:   &container.StdinOnce,
@@ -251,6 +252,22 @@ func (m *kubeGenericRuntimeManager) generateLinuxContainerConfig(container *api.
 	return linuxConfig
 }
 
+// makeDevices generates container devices for kubelet runtime api.
+func makeDevices(opts *kubecontainer.RunContainerOptions) []*runtimeApi.Device {
+	devices := make([]*runtimeApi.Device, len(opts.Devices))
+
+	for idx := range opts.Devices {
+		device := opts.Devices[idx]
+		devices[idx] = &runtimeApi.Device{
+			HostPath:      &device.PathOnHost,
+			ContainerPath: &device.PathInContainer,
+			Permissions:   &device.Permissions,
+		}
+	}
+
+	return devices
+}
+
 // makeMounts generates container volume mounts for kubelet runtime api.
 func (m *kubeGenericRuntimeManager) makeMounts(opts *kubecontainer.RunContainerOptions, container *api.Container, podHasSELinuxLabel bool) []*runtimeApi.Mount {
 	volumeMounts := []*runtimeApi.Mount{}