Skip to content

K
k3s
Commit 00938eac authored by Kubernetes Submit Queue's avatar Kubernetes Submit Queue Committed by GitHub
Browse files
Options

Merge pull request #42741 from kargakis/avoid-ns-skew

Automatic merge from submit-queue (batch tested with PRs 43481, 43419, 42741, 43480) controller: work around milliseconds skew in AddAfter AddAfter is not requeueing precisely after the provided time and may skew for some millieseconds. This is really important because controllers don't relist often so a missed check because of ms difference is essentially dropping the key. For example, in [1] the test requeues a Deployment for a progress check after 10s[2] but the Deployment is synced 9ms earlier ending up in the controller not recognizing the Deployment as failed thus dropping it from the queue w/o any error. The drop is fixed by forcing the controller to resync the Deployment but we are going to resync after the full duration. @deads2k if you don't like this I am going to handle this on a case by case basis [1] https://github.com/kubernetes/kubernetes/issues/39785#issuecomment-279959133 [2] https://github.com/kubernetes/kubernetes/blob/c48b2cab0f25abd374ae9308afb19735a4150e19/test/e2e/deployment.go#L1122
parents 5c262ab8 68b78282
Hide whitespace changes
Inline Side-by-side
Showing with 12 additions and 4 deletions
pkg/controller/daemon/daemoncontroller.go
View file @ 00938eac
...@@ -346,7 +346,9 @@ func (dsc *DaemonSetsController) updatePod(old, cur interface{}) { ...@@ -346,7 +346,9 @@ func (dsc *DaemonSetsController) updatePod(old, cur interface{}) {
dsc.enqueueDaemonSet(ds) dsc.enqueueDaemonSet(ds)
// See https://github.com/kubernetes/kubernetes/pull/38076 for more details // See https://github.com/kubernetes/kubernetes/pull/38076 for more details
if changedToReady && ds.Spec.MinReadySeconds > 0 { if changedToReady && ds.Spec.MinReadySeconds > 0 {
dsc.enqueueDaemonSetAfter(ds, time.Duration(ds.Spec.MinReadySeconds)*time.Second) // Add a second to avoid milliseconds skew in AddAfter.
// See https://github.com/kubernetes/kubernetes/issues/39785#issuecomment-279959133 for more info.
dsc.enqueueDaemonSetAfter(ds, (time.Duration(ds.Spec.MinReadySeconds)*time.Second)+time.Second)
} }
return return
} }
......
pkg/controller/deployment/progress.go
View file @ 00938eac
...@@ -233,6 +233,8 @@ func (dc *DeploymentController) requeueStuckDeployment(d *extensions.Deployment, ...@@ -233,6 +233,8 @@ func (dc *DeploymentController) requeueStuckDeployment(d *extensions.Deployment,
return time.Duration(0) return time.Duration(0)
} }
glog.V(4).Infof("Queueing up deployment %q for a progress check after %ds", d.Name, int(after.Seconds())) glog.V(4).Infof("Queueing up deployment %q for a progress check after %ds", d.Name, int(after.Seconds()))
dc.enqueueAfter(d, after) // Add a second to avoid milliseconds skew in AddAfter.
// See https://github.com/kubernetes/kubernetes/issues/39785#issuecomment-279959133 for more info.
dc.enqueueAfter(d, after+time.Second)
return after return after
} }
pkg/controller/replicaset/replica_set.go
View file @ 00938eac
...@@ -320,7 +320,9 @@ func (rsc *ReplicaSetController) updatePod(old, cur interface{}) { ...@@ -320,7 +320,9 @@ func (rsc *ReplicaSetController) updatePod(old, cur interface{}) {
// "closer" to kubelet (from the deployment to the replica set controller). // "closer" to kubelet (from the deployment to the replica set controller).
if !v1.IsPodReady(oldPod) && v1.IsPodReady(curPod) && rs.Spec.MinReadySeconds > 0 { if !v1.IsPodReady(oldPod) && v1.IsPodReady(curPod) && rs.Spec.MinReadySeconds > 0 {
glog.V(2).Infof("ReplicaSet %q will be enqueued after %ds for availability check", rs.Name, rs.Spec.MinReadySeconds) glog.V(2).Infof("ReplicaSet %q will be enqueued after %ds for availability check", rs.Name, rs.Spec.MinReadySeconds)
rsc.enqueueReplicaSetAfter(rs, time.Duration(rs.Spec.MinReadySeconds)*time.Second) // Add a second to avoid milliseconds skew in AddAfter.
// See https://github.com/kubernetes/kubernetes/issues/39785#issuecomment-279959133 for more info.
rsc.enqueueReplicaSetAfter(rs, (time.Duration(rs.Spec.MinReadySeconds)*time.Second)+time.Second)
} }
return return
} }
......
pkg/controller/replication/replication_controller.go
View file @ 00938eac
...@@ -319,7 +319,9 @@ func (rm *ReplicationManager) updatePod(old, cur interface{}) { ...@@ -319,7 +319,9 @@ func (rm *ReplicationManager) updatePod(old, cur interface{}) {
// "closer" to kubelet (from the deployment to the ReplicationController controller). // "closer" to kubelet (from the deployment to the ReplicationController controller).
if !v1.IsPodReady(oldPod) && v1.IsPodReady(curPod) && rc.Spec.MinReadySeconds > 0 { if !v1.IsPodReady(oldPod) && v1.IsPodReady(curPod) && rc.Spec.MinReadySeconds > 0 {
glog.V(2).Infof("ReplicationController %q will be enqueued after %ds for availability check", rc.Name, rc.Spec.MinReadySeconds) glog.V(2).Infof("ReplicationController %q will be enqueued after %ds for availability check", rc.Name, rc.Spec.MinReadySeconds)
rm.enqueueControllerAfter(rc, time.Duration(rc.Spec.MinReadySeconds)*time.Second) // Add a second to avoid milliseconds skew in AddAfter.
// See https://github.com/kubernetes/kubernetes/issues/39785#issuecomment-279959133 for more info.
rm.enqueueControllerAfter(rc, (time.Duration(rc.Spec.MinReadySeconds)*time.Second)+time.Second)
} }
return return
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment