Skip to content

K
k3s
Commit bc9b6e93 authored by Yu-Ju Hong's avatar Yu-Ju Hong
Browse files
Options

Merge pull request #7868 from derekwaynecarr/bound_duration

Allow a pod to be actively killed by the kubelet if it runs too long
parents ee4bf2ba 82163326
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 398 additions and 2 deletions
pkg/api/conversion.go
View file @ bc9b6e93
......@@ -141,6 +141,10 @@ func init() {
out.TerminationGracePeriodSeconds = new(int64)
*out.TerminationGracePeriodSeconds = *in.TerminationGracePeriodSeconds
}
if in.ActiveDeadlineSeconds != nil {
out.ActiveDeadlineSeconds = new(int64)
*out.ActiveDeadlineSeconds = *in.ActiveDeadlineSeconds
}
out.DNSPolicy = in.DNSPolicy
out.Version = "v1beta2"
return nil
......@@ -159,6 +163,10 @@ func init() {
out.TerminationGracePeriodSeconds = new(int64)
*out.TerminationGracePeriodSeconds = *in.TerminationGracePeriodSeconds
}
if in.ActiveDeadlineSeconds != nil {
out.ActiveDeadlineSeconds = new(int64)
*out.ActiveDeadlineSeconds = *in.ActiveDeadlineSeconds
}
out.DNSPolicy = in.DNSPolicy
return nil
},
......
pkg/api/types.go
View file @ bc9b6e93
......@@ -811,6 +811,9 @@ type PodSpec struct {
// a termination signal and the time when the processes are forcibly halted with a kill signal.
// Set this value longer than the expected cleanup time for your process.
TerminationGracePeriodSeconds *int64 `json:"terminationGracePeriodSeconds,omitempty"`
// Optional duration in seconds relative to the StartTime that the pod may be active on a node
// before the system actively tries to terminate the pod; value must be positive integer
ActiveDeadlineSeconds *int64 `json:"activeDeadlineSeconds,omitempty"`
// Required: Set DNS policy.
DNSPolicy DNSPolicy `json:"dnsPolicy,omitempty"`
// NodeSelector is a selector which must be true for the pod to fit on a node
......@@ -841,6 +844,10 @@ type PodStatus struct {
HostIP string `json:"hostIP,omitempty"`
PodIP string `json:"podIP,omitempty"`
// Date and time at which the object was acknowledged by the Kubelet.
// This is before the Kubelet pulled the container image(s) for the pod.
StartTime *util.Time `json:"startTime,omitempty"`
// The list has one entry per container in the manifest. Each entry is
// currently the output of `docker inspect`. This output format is *not*
// final and should not be relied upon.
......@@ -1692,6 +1699,7 @@ type ContainerManifest struct {
Containers []Container `json:"containers"`
RestartPolicy RestartPolicy `json:"restartPolicy,omitempty"`
TerminationGracePeriodSeconds *int64 `json:"terminationGracePeriodSeconds,omitempty"`
ActiveDeadlineSeconds *int64 `json:"activeDeadlineSeconds,omitempty"`
// Required: Set DNS policy.
DNSPolicy DNSPolicy `json:"dnsPolicy"`
HostNetwork bool `json:"hostNetwork,omitempty"`
......
pkg/api/v1/conversion_generated.go
View file @ bc9b6e93
......@@ -2652,6 +2652,12 @@ func convert_v1_PodSpec_To_api_PodSpec(in *PodSpec, out *newer.PodSpec, s conver
} else {
out.TerminationGracePeriodSeconds = nil
}
if in.ActiveDeadlineSeconds != nil {
out.ActiveDeadlineSeconds = new(int64)
*out.ActiveDeadlineSeconds = *in.ActiveDeadlineSeconds
} else {
out.ActiveDeadlineSeconds = nil
}
out.DNSPolicy = newer.DNSPolicy(in.DNSPolicy)
if in.NodeSelector != nil {
out.NodeSelector = make(map[string]string)
......@@ -2698,6 +2704,12 @@ func convert_api_PodSpec_To_v1_PodSpec(in *newer.PodSpec, out *PodSpec, s conver
} else {
out.TerminationGracePeriodSeconds = nil
}
if in.ActiveDeadlineSeconds != nil {
out.ActiveDeadlineSeconds = new(int64)
*out.ActiveDeadlineSeconds = *in.ActiveDeadlineSeconds
} else {
out.ActiveDeadlineSeconds = nil
}
out.DNSPolicy = DNSPolicy(in.DNSPolicy)
if in.NodeSelector != nil {
out.NodeSelector = make(map[string]string)
......@@ -2731,6 +2743,13 @@ func convert_v1_PodStatus_To_api_PodStatus(in *PodStatus, out *newer.PodStatus,
out.Message = in.Message
out.HostIP = in.HostIP
out.PodIP = in.PodIP
if in.StartTime != nil {
if err := s.Convert(&in.StartTime, &out.StartTime, 0); err != nil {
return err
}
} else {
out.StartTime = nil
}
if in.ContainerStatuses != nil {
out.ContainerStatuses = make([]newer.ContainerStatus, len(in.ContainerStatuses))
for i := range in.ContainerStatuses {
......@@ -2762,6 +2781,13 @@ func convert_api_PodStatus_To_v1_PodStatus(in *newer.PodStatus, out *PodStatus,
out.Message = in.Message
out.HostIP = in.HostIP
out.PodIP = in.PodIP
if in.StartTime != nil {
if err := s.Convert(&in.StartTime, &out.StartTime, 0); err != nil {
return err
}
} else {
out.StartTime = nil
}
if in.ContainerStatuses != nil {
out.ContainerStatuses = make([]ContainerStatus, len(in.ContainerStatuses))
for i := range in.ContainerStatuses {
......
pkg/api/v1/types.go
View file @ bc9b6e93
......@@ -813,6 +813,7 @@ type PodSpec struct {
// a termination signal and the time when the processes are forcibly halted with a kill signal.
// Set this value longer than the expected cleanup time for your process.
TerminationGracePeriodSeconds *int64 `json:"terminationGracePeriodSeconds,omitempty" description:"optional duration in seconds the pod needs to terminate gracefully; may be decreased in delete request; value must be non-negative integer; the value zero indicates delete immediately; if this value is not set, the default grace period will be used instead; the grace period is the duration in seconds after the processes running in the pod are sent a termination signal and the time when the processes are forcibly halted with a kill signal; set this value longer than the expected cleanup time for your process"`
ActiveDeadlineSeconds *int64 `json:"activeDeadlineSeconds,omitempty" description:"optional duration in seconds the pod may be active on the node relative to StartTime before the system will actively try to mark it failed and kill associated containers; value must be a positive integer`
// Optional: Set DNS policy. Defaults to "ClusterFirst"
DNSPolicy DNSPolicy `json:"dnsPolicy,omitempty" description:"DNS policy for containers within the pod; one of 'ClusterFirst' or 'Default'"`
// NodeSelector is a selector which must be true for the pod to fit on a node
......@@ -842,6 +843,8 @@ type PodStatus struct {
HostIP string `json:"hostIP,omitempty" description:"IP address of the host to which the pod is assigned; empty if not yet scheduled"`
PodIP string `json:"podIP,omitempty" description:"IP address allocated to the pod; routable at least within the cluster; empty if not yet allocated"`
StartTime *util.Time `json:"startTime,omitempty" description:"RFC 3339 date and time at which the object was acknowledged by the Kubelet. This is before the Kubelet pulled the container image(s) for the pod."`
// The list has one entry per container in the manifest. Each entry is currently the output
// of `docker inspect`.
ContainerStatuses []ContainerStatus `json:"containerStatuses,omitempty" description:"list of container statuses"`
......
pkg/api/v1beta1/conversion.go
View file @ bc9b6e93
......@@ -710,6 +710,10 @@ func init() {
out.TerminationGracePeriodSeconds = new(int64)
*out.TerminationGracePeriodSeconds = *in.TerminationGracePeriodSeconds
}
if in.ActiveDeadlineSeconds != nil {
out.ActiveDeadlineSeconds = new(int64)
*out.ActiveDeadlineSeconds = *in.ActiveDeadlineSeconds
}
out.DNSPolicy = DNSPolicy(in.DNSPolicy)
out.Version = "v1beta2"
out.HostNetwork = in.HostNetwork
......@@ -729,6 +733,10 @@ func init() {
out.TerminationGracePeriodSeconds = new(int64)
*out.TerminationGracePeriodSeconds = *in.TerminationGracePeriodSeconds
}
if in.ActiveDeadlineSeconds != nil {
out.ActiveDeadlineSeconds = new(int64)
*out.ActiveDeadlineSeconds = *in.ActiveDeadlineSeconds
}
out.DNSPolicy = newer.DNSPolicy(in.DNSPolicy)
out.HostNetwork = in.HostNetwork
return nil
......
pkg/api/v1beta1/types.go
View file @ bc9b6e93
......@@ -68,6 +68,7 @@ type ContainerManifest struct {
// a termination signal and the time when the processes are forcibly halted with a kill signal.
// Set this value longer than the expected cleanup time for your process.
TerminationGracePeriodSeconds *int64 `json:"terminationGracePeriodSeconds,omitempty" description:"optional duration in seconds the pod needs to terminate gracefully; may be decreased in delete request; value must be non-negative integer; the value zero indicates delete immediately; if this value is not set, the default grace period will be used instead; the grace period is the duration in seconds after the processes running in the pod are sent a termination signal and the time when the processes are forcibly halted with a kill signal; set this value longer than the expected cleanup time for your process"`
ActiveDeadlineSeconds *int64 `json:"activeDeadlineSeconds,omitempty" description:"optional duration in seconds the pod may be active on the node relative to StartTime before the system will actively try to mark it failed and kill associated containers; value must be a positive integer`
// Optional: Set DNS policy. Defaults to "ClusterFirst"
DNSPolicy DNSPolicy `json:"dnsPolicy,omitempty" description:"DNS policy for containers within the pod; one of 'ClusterFirst' or 'Default'"`
// Uses the host's network namespace. If this option is set, the ports that will be
......
pkg/api/v1beta2/conversion.go
View file @ bc9b6e93
......@@ -490,6 +490,10 @@ func init() {
out.TerminationGracePeriodSeconds = new(int64)
*out.TerminationGracePeriodSeconds = *in.TerminationGracePeriodSeconds
}
if in.ActiveDeadlineSeconds != nil {
out.ActiveDeadlineSeconds = new(int64)
*out.ActiveDeadlineSeconds = *in.ActiveDeadlineSeconds
}
out.DNSPolicy = DNSPolicy(in.DNSPolicy)
out.Version = "v1beta2"
out.HostNetwork = in.HostNetwork
......@@ -509,6 +513,10 @@ func init() {
out.TerminationGracePeriodSeconds = new(int64)
*out.TerminationGracePeriodSeconds = *in.TerminationGracePeriodSeconds
}
if in.ActiveDeadlineSeconds != nil {
out.ActiveDeadlineSeconds = new(int64)
*out.ActiveDeadlineSeconds = *in.ActiveDeadlineSeconds
}
out.DNSPolicy = newer.DNSPolicy(in.DNSPolicy)
out.HostNetwork = in.HostNetwork
return nil
......
pkg/api/v1beta2/types.go
View file @ bc9b6e93
......@@ -1530,6 +1530,7 @@ type ContainerManifest struct {
// a termination signal and the time when the processes are forcibly halted with a kill signal.
// Set this value longer than the expected cleanup time for your process.
TerminationGracePeriodSeconds *int64 `json:"terminationGracePeriodSeconds,omitempty" description:"optional duration in seconds the pod needs to terminate gracefully; may be decreased in delete request; value must be non-negative integer; the value zero indicates delete immediately; if this value is not set, the default grace period will be used instead; the grace period is the duration in seconds after the processes running in the pod are sent a termination signal and the time when the processes are forcibly halted with a kill signal; set this value longer than the expected cleanup time for your process"`
ActiveDeadlineSeconds *int64 `json:"activeDeadlineSeconds,omitempty" description:"optional duration in seconds the pod may be active on the node relative to StartTime before the system will actively try to mark it failed and kill associated containers; value must be a positive integer`
// Optional: Set DNS policy. Defaults to "ClusterFirst"
DNSPolicy DNSPolicy `json:"dnsPolicy,omitempty" description:"DNS policy for containers within the pod; one of 'ClusterFirst' or 'Default'"`
// Uses the host's network namespace. If this option is set, the ports that will be
......
pkg/api/v1beta3/conversion_generated.go
View file @ bc9b6e93
......@@ -2652,6 +2652,12 @@ func convert_v1beta3_PodSpec_To_api_PodSpec(in *PodSpec, out *newer.PodSpec, s c
} else {
out.TerminationGracePeriodSeconds = nil
}
if in.ActiveDeadlineSeconds != nil {
out.ActiveDeadlineSeconds = new(int64)
*out.ActiveDeadlineSeconds = *in.ActiveDeadlineSeconds
} else {
out.ActiveDeadlineSeconds = nil
}
out.DNSPolicy = newer.DNSPolicy(in.DNSPolicy)
if in.NodeSelector != nil {
out.NodeSelector = make(map[string]string)
......@@ -2698,6 +2704,12 @@ func convert_api_PodSpec_To_v1beta3_PodSpec(in *newer.PodSpec, out *PodSpec, s c
} else {
out.TerminationGracePeriodSeconds = nil
}
if in.ActiveDeadlineSeconds != nil {
out.ActiveDeadlineSeconds = new(int64)
*out.ActiveDeadlineSeconds = *in.ActiveDeadlineSeconds
} else {
out.ActiveDeadlineSeconds = nil
}
out.DNSPolicy = DNSPolicy(in.DNSPolicy)
if in.NodeSelector != nil {
out.NodeSelector = make(map[string]string)
......@@ -2731,6 +2743,13 @@ func convert_v1beta3_PodStatus_To_api_PodStatus(in *PodStatus, out *newer.PodSta
out.Message = in.Message
out.HostIP = in.HostIP
out.PodIP = in.PodIP
if in.StartTime != nil {
if err := s.Convert(&in.StartTime, &out.StartTime, 0); err != nil {
return err
}
} else {
out.StartTime = nil
}
if in.ContainerStatuses != nil {
out.ContainerStatuses = make([]newer.ContainerStatus, len(in.ContainerStatuses))
for i := range in.ContainerStatuses {
......@@ -2762,6 +2781,13 @@ func convert_api_PodStatus_To_v1beta3_PodStatus(in *newer.PodStatus, out *PodSta
out.Message = in.Message
out.HostIP = in.HostIP
out.PodIP = in.PodIP
if in.StartTime != nil {
if err := s.Convert(&in.StartTime, &out.StartTime, 0); err != nil {
return err
}
} else {
out.StartTime = nil
}
if in.ContainerStatuses != nil {
out.ContainerStatuses = make([]ContainerStatus, len(in.ContainerStatuses))
for i := range in.ContainerStatuses {
......
pkg/api/v1beta3/types.go
View file @ bc9b6e93
......@@ -813,6 +813,7 @@ type PodSpec struct {
// a termination signal and the time when the processes are forcibly halted with a kill signal.
// Set this value longer than the expected cleanup time for your process.
TerminationGracePeriodSeconds *int64 `json:"terminationGracePeriodSeconds,omitempty" description:"optional duration in seconds the pod needs to terminate gracefully; may be decreased in delete request; value must be non-negative integer; the value zero indicates delete immediately; if this value is not set, the default grace period will be used instead; the grace period is the duration in seconds after the processes running in the pod are sent a termination signal and the time when the processes are forcibly halted with a kill signal; set this value longer than the expected cleanup time for your process"`
ActiveDeadlineSeconds *int64 `json:"activeDeadlineSeconds,omitempty" description:"optional duration in seconds the pod may be active on the node relative to StartTime before the system will actively try to mark it failed and kill associated containers; value must be a positive integer`
// Optional: Set DNS policy. Defaults to "ClusterFirst"
DNSPolicy DNSPolicy `json:"dnsPolicy,omitempty" description:"DNS policy for containers within the pod; one of 'ClusterFirst' or 'Default'"`
// NodeSelector is a selector which must be true for the pod to fit on a node
......@@ -842,6 +843,8 @@ type PodStatus struct {
HostIP string `json:"hostIP,omitempty" description:"IP address of the host to which the pod is assigned; empty if not yet scheduled"`
PodIP string `json:"podIP,omitempty" description:"IP address allocated to the pod; routable at least within the cluster; empty if not yet allocated"`
StartTime *util.Time `json:"startTime,omitempty" description:"RFC 3339 date and time at which the object was acknowledged by the Kubelet. This is before the Kubelet pulled the container image(s) for the pod."`
// The list has one entry per container in the manifest. Each entry is currently the output
// of `docker inspect`.
ContainerStatuses []ContainerStatus `json:"containerStatuses,omitempty" description:"list of container statuses"`
......
pkg/api/validation/validation.go
View file @ bc9b6e93
......@@ -911,6 +911,12 @@ func ValidatePodSpec(spec *api.PodSpec) errs.ValidationErrorList {
allErrs = append(allErrs, validateDNSPolicy(&spec.DNSPolicy).Prefix("dnsPolicy")...)
allErrs = append(allErrs, ValidateLabels(spec.NodeSelector, "nodeSelector")...)
allErrs = append(allErrs, validateHostNetwork(spec.HostNetwork, spec.Containers).Prefix("hostNetwork")...)
if spec.ActiveDeadlineSeconds != nil {
if *spec.ActiveDeadlineSeconds <= 0 {
allErrs = append(allErrs, errs.NewFieldInvalid("activeDeadlineSeconds", spec.ActiveDeadlineSeconds, "activeDeadlineSeconds must be a positive integer greater than 0"))
}
}
return allErrs
}
......
pkg/api/validation/validation_test.go
View file @ bc9b6e93
......@@ -989,6 +989,7 @@ func TestValidateDNSPolicy(t *testing.T) {
}
func TestValidatePodSpec(t *testing.T) {
activeDeadlineSeconds := int64(30)
successCases := []api.PodSpec{
{ // Populate basic fields, leave defaults for most.
Volumes: []api.Volume{{Name: "vol", VolumeSource: api.VolumeSource{EmptyDir: &api.EmptyDirVolumeSource{}}}},
......@@ -1005,8 +1006,9 @@ func TestValidatePodSpec(t *testing.T) {
NodeSelector: map[string]string{
"key": "value",
},
Host: "foobar",
DNSPolicy: api.DNSClusterFirst,
Host: "foobar",
DNSPolicy: api.DNSClusterFirst,
ActiveDeadlineSeconds: &activeDeadlineSeconds,
},
{ // Populate HostNetwork.
Containers: []api.Container{
......@@ -1025,6 +1027,7 @@ func TestValidatePodSpec(t *testing.T) {
}
}
activeDeadlineSeconds = int64(0)
failureCases := map[string]api.PodSpec{
"bad volume": {
Volumes: []api.Volume{{}},
......@@ -1061,6 +1064,19 @@ func TestValidatePodSpec(t *testing.T) {
RestartPolicy: api.RestartPolicyAlways,
DNSPolicy: api.DNSClusterFirst,
},
"bad-active-deadline-seconds": {
Volumes: []api.Volume{
{Name: "vol", VolumeSource: api.VolumeSource{EmptyDir: &api.EmptyDirVolumeSource{}}},
},
Containers: []api.Container{{Name: "ctr", Image: "image", ImagePullPolicy: "IfNotPresent"}},
RestartPolicy: api.RestartPolicyAlways,
NodeSelector: map[string]string{
"key": "value",
},
Host: "foobar",
DNSPolicy: api.DNSClusterFirst,
ActiveDeadlineSeconds: &activeDeadlineSeconds,
},
}
for k, v := range failureCases {
if errs := ValidatePodSpec(&v); len(errs) == 0 {
......
pkg/kubelet/kubelet.go
View file @ bc9b6e93
......@@ -1145,6 +1145,39 @@ func (kl *Kubelet) cleanupOrphanedVolumes(pods []*api.Pod, runningPods []*kubeco
return nil
}
// filterOutPodsPastActiveDeadline filters pods with an ActiveDeadlineSeconds value that has been exceeded.
// It records an event that the pod has been active longer than the allocated time, and updates the pod status as failed.
// By filtering the pod from the result set, the Kubelet will kill the pod's containers as part of normal SyncPods workflow.
func (kl *Kubelet) filterOutPodsPastActiveDeadline(allPods []*api.Pod) (pods []*api.Pod) {
now := util.Now()
for _, pod := range allPods {
keepPod := true
if pod.Spec.ActiveDeadlineSeconds != nil {
podStatus, ok := kl.statusManager.GetPodStatus(kubecontainer.GetPodFullName(pod))
if !ok {
podStatus = pod.Status
}
if !podStatus.StartTime.IsZero() {
startTime := podStatus.StartTime.Time
duration := now.Time.Sub(startTime)
allowedDuration := time.Duration(*pod.Spec.ActiveDeadlineSeconds) * time.Second
if duration >= allowedDuration {
keepPod = false
}
}
}
if keepPod {
pods = append(pods, pod)
} else {
kl.recorder.Eventf(pod, "deadline", "Pod was active on the node longer than specified deadline")
kl.statusManager.SetPodStatus(pod, api.PodStatus{
Phase: api.PodFailed,
Message: "Pod was active on the node longer than specified deadline"})
}
}
return pods
}
// Filter out pods in the terminated state ("Failed" or "Succeeded").
func (kl *Kubelet) filterOutTerminatedPods(allPods []*api.Pod) []*api.Pod {
var pods []*api.Pod
......@@ -1448,6 +1481,8 @@ func (kl *Kubelet) admitPods(allPods []*api.Pod, podSyncTypes map[types.UID]metr
// These two conditions could be alleviated by checkpointing kubelet.
pods := kl.filterOutTerminatedPods(allPods)
pods = kl.filterOutPodsPastActiveDeadline(pods)
// Respect the pod creation order when resolving conflicts.
sort.Sort(podsByCreationTime(pods))
......@@ -1895,6 +1930,7 @@ func (kl *Kubelet) generatePodStatus(pod *api.Pod) (api.PodStatus, error) {
glog.V(3).Infof("Generating status for %q", podFullName)
spec := &pod.Spec
podStatus, err := kl.containerRuntime.GetPodStatus(pod)
if err != nil {
......@@ -1923,6 +1959,7 @@ func (kl *Kubelet) generatePodStatus(pod *api.Pod) (api.PodStatus, error) {
}
}
}
podStatus.Conditions = append(podStatus.Conditions, getPodReadyCondition(spec, podStatus.ContainerStatuses)...)
hostIP, err := kl.GetHostIP()
......
pkg/kubelet/kubelet_test.go
View file @ bc9b6e93
......@@ -4501,3 +4501,183 @@ func TestMakePortMappings(t *testing.T) {
}
}
}
func TestFilterOutPodsPastActiveDeadline(t *testing.T) {
testKubelet := newTestKubelet(t)
kubelet := testKubelet.kubelet
pods := newTestPods(5)
exceededActiveDeadlineSeconds := int64(30)
notYetActiveDeadlineSeconds := int64(120)
now := util.Now()
startTime := util.NewTime(now.Time.Add(-1 * time.Minute))
pods[0].Status.StartTime = &startTime
pods[0].Spec.ActiveDeadlineSeconds = &exceededActiveDeadlineSeconds
pods[1].Status.StartTime = &startTime
pods[1].Spec.ActiveDeadlineSeconds = &notYetActiveDeadlineSeconds
expected := []*api.Pod{pods[1], pods[2], pods[3], pods[4]}
kubelet.podManager.SetPods(pods)
actual := kubelet.filterOutPodsPastActiveDeadline(pods)
if !reflect.DeepEqual(expected, actual) {
expectedNames := ""
for _, pod := range expected {
expectedNames = expectedNames + pod.Name + " "
}
actualNames := ""
for _, pod := range actual {
actualNames = actualNames + pod.Name + " "
}
t.Errorf("expected %#v, got %#v", expectedNames, actualNames)
}
}
func TestSyncPodsDeletesPodsThatRunTooLong(t *testing.T) {
testKubelet := newTestKubelet(t)
testKubelet.fakeCadvisor.On("MachineInfo").Return(&cadvisorApi.MachineInfo{}, nil)
kubelet := testKubelet.kubelet
fakeDocker := testKubelet.fakeDocker
now := util.Now()
startTime := util.NewTime(now.Time.Add(-1 * time.Minute))
exceededActiveDeadlineSeconds := int64(30)
pods := []*api.Pod{
{
ObjectMeta: api.ObjectMeta{
UID: "12345678",
Name: "bar",
Namespace: "new",
},
Spec: api.PodSpec{
Containers: []api.Container{
{Name: "foo"},
},
ActiveDeadlineSeconds: &exceededActiveDeadlineSeconds,
},
Status: api.PodStatus{
StartTime: &startTime,
},
},
}
fakeDocker.ContainerList = []docker.APIContainers{
{
// the k8s prefix is required for the kubelet to manage the container
Names: []string{"/k8s_foo_bar_new_12345678_1111"},
ID: "1234",
},
{
// pod infra container
Names: []string{"/k8s_POD." + strconv.FormatUint(generatePodInfraContainerHash(pods[0]), 16) + "_bar_new_12345678_2222"},
ID: "9876",
},
}
fakeDocker.ContainerMap = map[string]*docker.Container{
"1234": {
ID: "1234",
Config: &docker.Config{},
HostConfig: &docker.HostConfig{},
},
"9876": {
ID: "9876",
Config: &docker.Config{},
HostConfig: &docker.HostConfig{},
},
"9999": {
ID: "9999",
Config: &docker.Config{},
HostConfig: &docker.HostConfig{},
},
}
err := kubelet.SyncPods(pods, emptyPodUIDs, map[string]*api.Pod{}, time.Now())
if err != nil {
t.Errorf("unexpected error: %v", err)
}
verifyCalls(t, fakeDocker, []string{"list", "inspect_container", "stop", "inspect_container", "stop", "list"})
// A map iteration is used to delete containers, so must not depend on
// order here.
expectedToStop := map[string]bool{
"1234": true,
"9876": true,
}
if len(fakeDocker.Stopped) != 2 ||
!expectedToStop[fakeDocker.Stopped[0]] ||
!expectedToStop[fakeDocker.Stopped[1]] {
t.Errorf("Wrong containers were stopped: %v", fakeDocker.Stopped)
}
}
func TestSyncPodsDoesNotDeletePodsThatRunTooLong(t *testing.T) {
testKubelet := newTestKubelet(t)
testKubelet.fakeCadvisor.On("MachineInfo").Return(&cadvisorApi.MachineInfo{}, nil)
kubelet := testKubelet.kubelet
fakeDocker := testKubelet.fakeDocker
now := util.Now()
startTime := util.NewTime(now.Time.Add(-1 * time.Minute))
exceededActiveDeadlineSeconds := int64(300)
pods := []*api.Pod{
{
ObjectMeta: api.ObjectMeta{
UID: "12345678",
Name: "bar",
Namespace: "new",
},
Spec: api.PodSpec{
Containers: []api.Container{
{Name: "foo"},
},
ActiveDeadlineSeconds: &exceededActiveDeadlineSeconds,
},
Status: api.PodStatus{
StartTime: &startTime,
},
},
}
fakeDocker.ContainerList = []docker.APIContainers{
{
// the k8s prefix is required for the kubelet to manage the container
Names: []string{"/k8s_foo_bar_new_12345678_1111"},
ID: "1234",
},
{
// pod infra container
Names: []string{"/k8s_POD." + strconv.FormatUint(generatePodInfraContainerHash(pods[0]), 16) + "_bar_new_12345678_2222"},
ID: "9876",
},
}
fakeDocker.ContainerMap = map[string]*docker.Container{
"1234": {
ID: "1234",
Config: &docker.Config{},
HostConfig: &docker.HostConfig{},
},
"9876": {
ID: "9876",
Config: &docker.Config{},
HostConfig: &docker.HostConfig{},
},
"9999": {
ID: "9999",
Config: &docker.Config{},
HostConfig: &docker.HostConfig{},
},
}
kubelet.podManager.SetPods(pods)
err := kubelet.SyncPods(pods, emptyPodUIDs, map[string]*api.Pod{}, time.Now())
if err != nil {
t.Errorf("unexpected error: %v", err)
}
verifyCalls(t, fakeDocker, []string{
"list", "list", "list",
// Get pod status.
"inspect_container", "inspect_container",
// Check the pod infra container.
"inspect_container",
// Get pod status.
"list", "inspect_container", "inspect_container", "list"})
}
pkg/kubelet/status_manager.go
View file @ bc9b6e93
......@@ -74,6 +74,25 @@ func (s *statusManager) SetPodStatus(pod *api.Pod, status api.PodStatus) {
s.podStatusesLock.Lock()
defer s.podStatusesLock.Unlock()
oldStatus, found := s.podStatuses[podFullName]
// ensure that the start time does not change across updates.
if found && oldStatus.StartTime != nil {
status.StartTime = oldStatus.StartTime
}
// if the status has no start time, we need to set an initial time
if status.StartTime.IsZero() {
if pod.Status.StartTime.IsZero() {
// the pod did not have a previously recorded value so set to now
now := util.Now()
status.StartTime = &now
} else {
// the pod had a recorded value, but the kubelet restarted so we need to rebuild cache
// based on last observed value
status.StartTime = pod.Status.StartTime
}
}
if !found || !reflect.DeepEqual(oldStatus, status) {
s.podStatuses[podFullName] = status
s.podStatusChannel <- podStatusSyncRequest{pod, status}
......
pkg/kubelet/status_manager_test.go
View file @ bc9b6e93
......@@ -20,10 +20,13 @@ import (
"math/rand"
"strconv"
"testing"
"time"
"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
"github.com/GoogleCloudPlatform/kubernetes/pkg/client/testclient"
kubecontainer "github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/container"
"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
)
var testPod *api.Pod = &api.Pod{
......@@ -87,6 +90,32 @@ func TestNewStatus(t *testing.T) {
syncer := newTestStatusManager()
syncer.SetPodStatus(testPod, getRandomPodStatus())
verifyUpdates(t, syncer, 1)
status, _ := syncer.GetPodStatus(kubecontainer.GetPodFullName(testPod))
if status.StartTime.IsZero() {
t.Errorf("SetPodStatus did not set a proper start time value")
}
}
func TestNewStatusPreservesPodStartTime(t *testing.T) {
syncer := newTestStatusManager()
pod := &api.Pod{
ObjectMeta: api.ObjectMeta{
UID: "12345678",
Name: "foo",
Namespace: "new",
},
Status: api.PodStatus{},
}
now := util.Now()
startTime := util.NewTime(now.Time.Add(-1 * time.Minute))
pod.Status.StartTime = &startTime
syncer.SetPodStatus(pod, getRandomPodStatus())
status, _ := syncer.GetPodStatus(kubecontainer.GetPodFullName(pod))
if !status.StartTime.Time.Equal(startTime.Time) {
t.Errorf("Unexpected start time, expected %v, actual %v", startTime, status.StartTime)
}
}
func TestChangedStatus(t *testing.T) {
......@@ -96,6 +125,23 @@ func TestChangedStatus(t *testing.T) {
verifyUpdates(t, syncer, 2)
}
func TestChangedStatusKeepsStartTime(t *testing.T) {
syncer := newTestStatusManager()
now := util.Now()
firstStatus := getRandomPodStatus()
firstStatus.StartTime = &now
syncer.SetPodStatus(testPod, firstStatus)
syncer.SetPodStatus(testPod, getRandomPodStatus())
verifyUpdates(t, syncer, 2)
finalStatus, _ := syncer.GetPodStatus(kubecontainer.GetPodFullName(testPod))
if finalStatus.StartTime.IsZero() {
t.Errorf("StartTime should not be zero")
}
if !finalStatus.StartTime.Time.Equal(now.Time) {
t.Errorf("Expected %v, but got %v", now.Time, finalStatus.StartTime.Time)
}
}
func TestUnchangedStatus(t *testing.T) {
syncer := newTestStatusManager()
podStatus := getRandomPodStatus()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment