Commit 7f1d9382 authored by Kubernetes Submit Queue's avatar Kubernetes Submit Queue Committed by GitHub

Merge pull request #48846 from dashpole/remove_ood

Automatic merge from submit-queue Remove flags low-diskspace-threshold-mb and outofdisk-transition-frequency issue: #48843 This removes two flags replaced by the eviction manager. These have been depreciated for two releases, which I believe correctly follows the kubernetes depreciation guidelines. ```release-note Remove depreciated flags: --low-diskspace-threshold-mb and --outofdisk-transition-frequency, which are replaced by --eviction-hard ``` cc @mtaufen since I am changing kubelet flags cc @vishh @derekwaynecarr /sig node
parents 72494353 9bfa28bd
......@@ -54219,7 +54219,7 @@
"type": "string"
},
"reason": {
"description": "A brief CamelCase message indicating details about why the pod is in this state. e.g. 'OutOfDisk'",
"description": "A brief CamelCase message indicating details about why the pod is in this state. e.g. 'Evicted'",
"type": "string"
},
"startTime": {
......@@ -20979,7 +20979,7 @@
},
"reason": {
"type": "string",
"description": "A brief CamelCase message indicating details about why the pod is in this state. e.g. 'OutOfDisk'"
"description": "A brief CamelCase message indicating details about why the pod is in this state. e.g. 'Evicted'"
},
"hostIP": {
"type": "string",
......
......@@ -159,7 +159,7 @@ TEST_CLUSTER_RESYNC_PERIOD="${TEST_CLUSTER_RESYNC_PERIOD:---min-resync-period=3m
# ContentType used by all components to communicate with apiserver.
TEST_CLUSTER_API_CONTENT_TYPE="${TEST_CLUSTER_API_CONTENT_TYPE:-}"
KUBELET_TEST_ARGS="${KUBELET_TEST_ARGS:-} --max-pods=110 --serialize-image-pulls=false --outofdisk-transition-frequency=0 ${TEST_CLUSTER_API_CONTENT_TYPE}"
KUBELET_TEST_ARGS="${KUBELET_TEST_ARGS:-} --max-pods=110 --serialize-image-pulls=false ${TEST_CLUSTER_API_CONTENT_TYPE}"
if [[ "${NODE_OS_DISTRIBUTION}" == "gci" ]] || [[ "${NODE_OS_DISTRIBUTION}" == "ubuntu" ]]; then
NODE_KUBELET_TEST_ARGS=" --experimental-kernel-memcg-notification=true"
fi
......
......@@ -226,8 +226,6 @@ func (c *kubeletConfiguration) addFlags(fs *pflag.FlagSet) {
fs.DurationVar(&c.ImageMinimumGCAge.Duration, "minimum-image-ttl-duration", c.ImageMinimumGCAge.Duration, "Minimum age for an unused image before it is garbage collected. Examples: '300ms', '10s' or '2h45m'.")
fs.Int32Var(&c.ImageGCHighThresholdPercent, "image-gc-high-threshold", c.ImageGCHighThresholdPercent, "The percent of disk usage after which image garbage collection is always run.")
fs.Int32Var(&c.ImageGCLowThresholdPercent, "image-gc-low-threshold", c.ImageGCLowThresholdPercent, "The percent of disk usage before which image garbage collection is never run. Lowest disk usage to garbage collect to.")
fs.Int32Var(&c.LowDiskSpaceThresholdMB, "low-diskspace-threshold-mb", c.LowDiskSpaceThresholdMB, "The absolute free disk space, in MB, to maintain. When disk space falls below this threshold, new pods would be rejected.")
fs.MarkDeprecated("low-diskspace-threshold-mb", "Use --eviction-hard instead. Will be removed in a future version.")
fs.DurationVar(&c.VolumeStatsAggPeriod.Duration, "volume-stats-agg-period", c.VolumeStatsAggPeriod.Duration, "Specifies interval for kubelet to calculate and cache the volume disk usage for all pods and volumes. To disable volume calculations, set to 0.")
fs.StringVar(&c.VolumePluginDir, "volume-plugin-dir", c.VolumePluginDir, "<Warning: Alpha feature> The full path of the directory in which to search for additional third party volume plugins")
fs.StringVar(&c.CloudProvider, "cloud-provider", c.CloudProvider, "The provider for cloud services. By default, kubelet will attempt to auto-detect the cloud provider. Specify empty string for running with no cloud provider.")
......@@ -269,8 +267,6 @@ func (c *kubeletConfiguration) addFlags(fs *pflag.FlagSet) {
fs.Int32Var(&c.KubeAPIQPS, "kube-api-qps", c.KubeAPIQPS, "QPS to use while talking with kubernetes apiserver")
fs.Int32Var(&c.KubeAPIBurst, "kube-api-burst", c.KubeAPIBurst, "Burst to use while talking with kubernetes apiserver")
fs.BoolVar(&c.SerializeImagePulls, "serialize-image-pulls", c.SerializeImagePulls, "Pull images one at a time. We recommend *not* changing the default value on nodes that run docker daemon with version < 1.9 or an Aufs storage backend. Issue #10959 has more details.")
fs.DurationVar(&c.OutOfDiskTransitionFrequency.Duration, "outofdisk-transition-frequency", c.OutOfDiskTransitionFrequency.Duration, "Duration for which the kubelet has to wait before transitioning out of out-of-disk node condition status.")
fs.MarkDeprecated("outofdisk-transition-frequency", "Use --eviction-pressure-transition-period instead. Will be removed in a future version.")
fs.BoolVar(&c.EnableCustomMetrics, "enable-custom-metrics", c.EnableCustomMetrics, "Support for gathering custom metrics.")
fs.StringVar(&c.RuntimeCgroups, "runtime-cgroups", c.RuntimeCgroups, "Optional absolute name of cgroups to create and run the runtime in.")
......
......@@ -4998,7 +4998,7 @@ The resulting set of endpoints can be viewed as:<br>
</tr>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">reason</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">A brief CamelCase message indicating details about why the pod is in this state. e.g. <em>OutOfDisk</em></p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">A brief CamelCase message indicating details about why the pod is in this state. e.g. <em>Evicted</em></p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">false</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">string</p></td>
<td class="tableblock halign-left valign-top"></td>
......
......@@ -449,7 +449,6 @@ log-flush-frequency
log-lines-total
logexporter-gcs-path
long-running-request-regexp
low-diskspace-threshold-mb
make-iptables-util-chains
make-symlinks
manifest-url
......@@ -521,7 +520,6 @@ oidc-issuer-url
oidc-username-claim
only-idl
oom-score-adj
outofdisk-transition-frequency
output-base
output-directory
output-file-base
......
......@@ -2334,7 +2334,7 @@ type PodStatus struct {
// A human readable message indicating details about why the pod is in this state.
// +optional
Message string
// A brief CamelCase message indicating details about why the pod is in this state. e.g. 'OutOfDisk'
// A brief CamelCase message indicating details about why the pod is in this state. e.g. 'Evicted'
// +optional
Reason string
......
......@@ -309,10 +309,6 @@ type KubeletConfiguration struct {
// image garbage collection is never run. Lowest disk usage to garbage
// collect to.
ImageGCLowThresholdPercent int32
// lowDiskSpaceThresholdMB is the absolute free disk space, in MB, to
// maintain. When disk space falls below this threshold, new pods would
// be rejected.
LowDiskSpaceThresholdMB int32
// How frequently to calculate and cache volume disk usage for all pods
VolumeStatsAggPeriod metav1.Duration
// volumePluginDir is the full path of the directory in which to search
......@@ -412,10 +408,6 @@ type KubeletConfiguration struct {
// run docker daemon with version < 1.9 or an Aufs storage backend.
// Issue #10959 has more details.
SerializeImagePulls bool
// outOfDiskTransitionFrequency is duration for which the kubelet has to
// wait before transitioning out of out-of-disk node condition status.
// +optional
OutOfDiskTransitionFrequency metav1.Duration
// nodeLabels to add when registering the node in the cluster.
NodeLabels map[string]string
// nonMasqueradeCIDR configures masquerading: traffic to IPs outside this range will use IP masquerade.
......
......@@ -363,9 +363,6 @@ func SetDefaults_KubeletConfiguration(obj *KubeletConfiguration) {
if obj.KubeAPIBurst == 0 {
obj.KubeAPIBurst = 10
}
if obj.OutOfDiskTransitionFrequency == zeroDuration {
obj.OutOfDiskTransitionFrequency = metav1.Duration{Duration: 5 * time.Minute}
}
if string(obj.HairpinMode) == "" {
obj.HairpinMode = PromiscuousBridge
}
......
......@@ -386,10 +386,6 @@ type KubeletConfiguration struct {
// image garbage collection is never run. Lowest disk usage to garbage
// collect to. The percent is calculated as this field value out of 100.
ImageGCLowThresholdPercent *int32 `json:"imageGCLowThresholdPercent"`
// lowDiskSpaceThresholdMB is the absolute free disk space, in MB, to
// maintain. When disk space falls below this threshold, new pods would
// be rejected.
LowDiskSpaceThresholdMB int32 `json:"lowDiskSpaceThresholdMB"`
// How frequently to calculate and cache volume disk usage for all pods
VolumeStatsAggPeriod metav1.Duration `json:"volumeStatsAggPeriod"`
// volumePluginDir is the full path of the directory in which to search
......@@ -483,9 +479,6 @@ type KubeletConfiguration struct {
// run docker daemon with version < 1.9 or an Aufs storage backend.
// Issue #10959 has more details.
SerializeImagePulls *bool `json:"serializeImagePulls"`
// outOfDiskTransitionFrequency is duration for which the kubelet has to
// wait before transitioning out of out-of-disk node condition status.
OutOfDiskTransitionFrequency metav1.Duration `json:"outOfDiskTransitionFrequency"`
// nodeLabels to add when registering the node in the cluster.
NodeLabels map[string]string `json:"nodeLabels"`
// nonMasqueradeCIDR configures masquerading: traffic to IPs outside this range will use IP masquerade.
......
......@@ -429,7 +429,6 @@ func autoConvert_v1alpha1_KubeletConfiguration_To_componentconfig_KubeletConfigu
if err := v1.Convert_Pointer_int32_To_int32(&in.ImageGCLowThresholdPercent, &out.ImageGCLowThresholdPercent, s); err != nil {
return err
}
out.LowDiskSpaceThresholdMB = in.LowDiskSpaceThresholdMB
out.VolumeStatsAggPeriod = in.VolumeStatsAggPeriod
out.VolumePluginDir = in.VolumePluginDir
out.CloudProvider = in.CloudProvider
......@@ -474,7 +473,6 @@ func autoConvert_v1alpha1_KubeletConfiguration_To_componentconfig_KubeletConfigu
if err := v1.Convert_Pointer_bool_To_bool(&in.SerializeImagePulls, &out.SerializeImagePulls, s); err != nil {
return err
}
out.OutOfDiskTransitionFrequency = in.OutOfDiskTransitionFrequency
out.NodeLabels = *(*map[string]string)(unsafe.Pointer(&in.NodeLabels))
out.NonMasqueradeCIDR = in.NonMasqueradeCIDR
out.EnableCustomMetrics = in.EnableCustomMetrics
......@@ -609,7 +607,6 @@ func autoConvert_componentconfig_KubeletConfiguration_To_v1alpha1_KubeletConfigu
if err := v1.Convert_int32_To_Pointer_int32(&in.ImageGCLowThresholdPercent, &out.ImageGCLowThresholdPercent, s); err != nil {
return err
}
out.LowDiskSpaceThresholdMB = in.LowDiskSpaceThresholdMB
out.VolumeStatsAggPeriod = in.VolumeStatsAggPeriod
out.VolumePluginDir = in.VolumePluginDir
out.CloudProvider = in.CloudProvider
......@@ -658,7 +655,6 @@ func autoConvert_componentconfig_KubeletConfiguration_To_v1alpha1_KubeletConfigu
if err := v1.Convert_bool_To_Pointer_bool(&in.SerializeImagePulls, &out.SerializeImagePulls, s); err != nil {
return err
}
out.OutOfDiskTransitionFrequency = in.OutOfDiskTransitionFrequency
out.NodeLabels = *(*map[string]string)(unsafe.Pointer(&in.NodeLabels))
out.NonMasqueradeCIDR = in.NonMasqueradeCIDR
out.EnableCustomMetrics = in.EnableCustomMetrics
......
......@@ -493,7 +493,6 @@ func (in *KubeletConfiguration) DeepCopyInto(out *KubeletConfiguration) {
**out = **in
}
}
out.OutOfDiskTransitionFrequency = in.OutOfDiskTransitionFrequency
if in.NodeLabels != nil {
in, out := &in.NodeLabels, &out.NodeLabels
*out = make(map[string]string, len(*in))
......
......@@ -434,7 +434,6 @@ func (in *KubeletConfiguration) DeepCopyInto(out *KubeletConfiguration) {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
out.OutOfDiskTransitionFrequency = in.OutOfDiskTransitionFrequency
if in.NodeLabels != nil {
in, out := &in.NodeLabels, &out.NodeLabels
*out = make(map[string]string, len(*in))
......
......@@ -12,7 +12,6 @@ go_library(
name = "go_default_library",
srcs = [
"active_deadline.go",
"disk_manager.go",
"doc.go",
"kubelet.go",
"kubelet_cadvisor.go",
......@@ -148,7 +147,6 @@ go_test(
name = "go_default_test",
srcs = [
"active_deadline_test.go",
"disk_manager_test.go",
"kubelet_cadvisor_test.go",
"kubelet_getters_test.go",
"kubelet_network_test.go",
......
/*
Copyright 2015 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package kubelet
import (
"fmt"
"sync"
"time"
"github.com/golang/glog"
cadvisorapi "github.com/google/cadvisor/info/v2"
"k8s.io/kubernetes/pkg/kubelet/cadvisor"
)
// Manages policy for diskspace management for disks holding docker images and root fs.
// mb is used to easily convert an int to an mb
const mb = 1024 * 1024
// Implementation is thread-safe.
type diskSpaceManager interface {
// Checks the available disk space
IsRootDiskSpaceAvailable() (bool, error)
IsRuntimeDiskSpaceAvailable() (bool, error)
}
// DiskSpacePolicy defines the free disk for Docker and Root.
type DiskSpacePolicy struct {
// free disk space threshold for filesystem holding docker images.
DockerFreeDiskMB int
// free disk space threshold for root filesystem. Host volumes are created on root fs.
RootFreeDiskMB int
}
type fsInfo struct {
Usage int64
Capacity int64
Available int64
Timestamp time.Time
}
type realDiskSpaceManager struct {
cadvisor cadvisor.Interface
cachedInfo map[string]fsInfo // cache of filesystem info.
lock sync.Mutex // protecting cachedInfo.
policy DiskSpacePolicy // thresholds. Set at creation time.
}
func (dm *realDiskSpaceManager) getFsInfo(fsType string, f func() (cadvisorapi.FsInfo, error)) (fsInfo, error) {
dm.lock.Lock()
defer dm.lock.Unlock()
fsi := fsInfo{}
if info, ok := dm.cachedInfo[fsType]; ok {
timeLimit := time.Now().Add(-2 * time.Second)
if info.Timestamp.After(timeLimit) {
fsi = info
}
}
if fsi.Timestamp.IsZero() {
fs, err := f()
if err != nil {
return fsInfo{}, err
}
fsi.Timestamp = time.Now()
fsi.Usage = int64(fs.Usage)
fsi.Capacity = int64(fs.Capacity)
fsi.Available = int64(fs.Available)
dm.cachedInfo[fsType] = fsi
}
return fsi, nil
}
func (dm *realDiskSpaceManager) IsRuntimeDiskSpaceAvailable() (bool, error) {
return dm.isSpaceAvailable("runtime", dm.policy.DockerFreeDiskMB, dm.cadvisor.ImagesFsInfo)
}
func (dm *realDiskSpaceManager) IsRootDiskSpaceAvailable() (bool, error) {
return dm.isSpaceAvailable("root", dm.policy.RootFreeDiskMB, dm.cadvisor.RootFsInfo)
}
func (dm *realDiskSpaceManager) isSpaceAvailable(fsType string, threshold int, f func() (cadvisorapi.FsInfo, error)) (bool, error) {
fsInfo, err := dm.getFsInfo(fsType, f)
if err != nil {
return true, fmt.Errorf("failed to get fs info for %q: %v", fsType, err)
}
if fsInfo.Capacity == 0 {
return true, fmt.Errorf("could not determine capacity for %q fs. Info: %+v", fsType, fsInfo)
}
if fsInfo.Available < 0 {
return true, fmt.Errorf("wrong available space for %q: %+v", fsType, fsInfo)
}
if fsInfo.Available < int64(threshold)*mb {
glog.Infof("Running out of space on disk for %q: available %d MB, threshold %d MB", fsType, fsInfo.Available/mb, threshold)
return false, nil
}
return true, nil
}
func validatePolicy(policy DiskSpacePolicy) error {
if policy.DockerFreeDiskMB < 0 {
return fmt.Errorf("free disk space should be non-negative; invalid value %d for docker disk space threshold", policy.DockerFreeDiskMB)
}
if policy.RootFreeDiskMB < 0 {
return fmt.Errorf("free disk space should be non-negative; invalid value %d for root disk space threshold", policy.RootFreeDiskMB)
}
return nil
}
func newDiskSpaceManager(cadvisorInterface cadvisor.Interface, policy DiskSpacePolicy) (diskSpaceManager, error) {
// validate policy
err := validatePolicy(policy)
if err != nil {
return nil, err
}
dm := &realDiskSpaceManager{
cadvisor: cadvisorInterface,
policy: policy,
cachedInfo: map[string]fsInfo{},
}
return dm, nil
}
/*
Copyright 2015 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package kubelet
import (
"fmt"
"testing"
cadvisorapi "github.com/google/cadvisor/info/v2"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
cadvisortest "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing"
)
func testPolicy() DiskSpacePolicy {
return DiskSpacePolicy{
DockerFreeDiskMB: 250,
RootFreeDiskMB: 250,
}
}
func setUp(t *testing.T) (*assert.Assertions, DiskSpacePolicy, *cadvisortest.Mock) {
assert := assert.New(t)
policy := testPolicy()
c := new(cadvisortest.Mock)
return assert, policy, c
}
func TestValidPolicy(t *testing.T) {
assert, policy, c := setUp(t)
_, err := newDiskSpaceManager(c, policy)
assert.NoError(err)
policy = testPolicy()
policy.DockerFreeDiskMB = -1
_, err = newDiskSpaceManager(c, policy)
assert.Error(err)
policy = testPolicy()
policy.RootFreeDiskMB = -1
_, err = newDiskSpaceManager(c, policy)
assert.Error(err)
}
func TestSpaceAvailable(t *testing.T) {
assert, policy, mockCadvisor := setUp(t)
dm, err := newDiskSpaceManager(mockCadvisor, policy)
assert.NoError(err)
mockCadvisor.On("ImagesFsInfo").Return(cadvisorapi.FsInfo{
Usage: 400 * mb,
Capacity: 1000 * mb,
Available: 600 * mb,
}, nil)
mockCadvisor.On("RootFsInfo").Return(cadvisorapi.FsInfo{
Usage: 9 * mb,
Capacity: 10 * mb,
}, nil)
ok, err := dm.IsRuntimeDiskSpaceAvailable()
assert.NoError(err)
assert.True(ok)
ok, err = dm.IsRootDiskSpaceAvailable()
assert.NoError(err)
assert.False(ok)
}
// TestIsRuntimeDiskSpaceAvailableWithSpace verifies IsRuntimeDiskSpaceAvailable results when
// space is available.
func TestIsRuntimeDiskSpaceAvailableWithSpace(t *testing.T) {
assert, policy, mockCadvisor := setUp(t)
dm, err := newDiskSpaceManager(mockCadvisor, policy)
require.NoError(t, err)
// 500MB available
mockCadvisor.On("ImagesFsInfo").Return(cadvisorapi.FsInfo{
Usage: 9500 * mb,
Capacity: 10000 * mb,
Available: 500 * mb,
}, nil)
ok, err := dm.IsRuntimeDiskSpaceAvailable()
assert.NoError(err)
assert.True(ok)
}
// TestIsRuntimeDiskSpaceAvailableWithoutSpace verifies IsRuntimeDiskSpaceAvailable results when
// space is not available.
func TestIsRuntimeDiskSpaceAvailableWithoutSpace(t *testing.T) {
// 1MB available
assert, policy, mockCadvisor := setUp(t)
mockCadvisor.On("ImagesFsInfo").Return(cadvisorapi.FsInfo{
Usage: 999 * mb,
Capacity: 1000 * mb,
Available: 1 * mb,
}, nil)
dm, err := newDiskSpaceManager(mockCadvisor, policy)
require.NoError(t, err)
ok, err := dm.IsRuntimeDiskSpaceAvailable()
assert.NoError(err)
assert.False(ok)
}
// TestIsRootDiskSpaceAvailableWithSpace verifies IsRootDiskSpaceAvailable results when
// space is available.
func TestIsRootDiskSpaceAvailableWithSpace(t *testing.T) {
assert, policy, mockCadvisor := setUp(t)
policy.RootFreeDiskMB = 10
dm, err := newDiskSpaceManager(mockCadvisor, policy)
assert.NoError(err)
// 999MB available
mockCadvisor.On("RootFsInfo").Return(cadvisorapi.FsInfo{
Usage: 1 * mb,
Capacity: 1000 * mb,
Available: 999 * mb,
}, nil)
ok, err := dm.IsRootDiskSpaceAvailable()
assert.NoError(err)
assert.True(ok)
}
// TestIsRootDiskSpaceAvailableWithoutSpace verifies IsRootDiskSpaceAvailable results when
// space is not available.
func TestIsRootDiskSpaceAvailableWithoutSpace(t *testing.T) {
assert, policy, mockCadvisor := setUp(t)
policy.RootFreeDiskMB = 10
dm, err := newDiskSpaceManager(mockCadvisor, policy)
assert.NoError(err)
// 9MB available
mockCadvisor.On("RootFsInfo").Return(cadvisorapi.FsInfo{
Usage: 990 * mb,
Capacity: 1000 * mb,
Available: 9 * mb,
}, nil)
ok, err := dm.IsRootDiskSpaceAvailable()
assert.NoError(err)
assert.False(ok)
}
// TestCache verifies that caching works properly with DiskSpaceAvailable calls
func TestCache(t *testing.T) {
assert, policy, mockCadvisor := setUp(t)
dm, err := newDiskSpaceManager(mockCadvisor, policy)
assert.NoError(err)
mockCadvisor.On("ImagesFsInfo").Return(cadvisorapi.FsInfo{
Usage: 400 * mb,
Capacity: 1000 * mb,
Available: 300 * mb,
}, nil).Once()
mockCadvisor.On("RootFsInfo").Return(cadvisorapi.FsInfo{
Usage: 500 * mb,
Capacity: 1000 * mb,
Available: 500 * mb,
}, nil).Once()
// Initial calls which should be recorded in mockCadvisor
ok, err := dm.IsRuntimeDiskSpaceAvailable()
assert.NoError(err)
assert.True(ok)
ok, err = dm.IsRootDiskSpaceAvailable()
assert.NoError(err)
assert.True(ok)
// Get the current count of calls to mockCadvisor
cadvisorCallCount := len(mockCadvisor.Calls)
// Checking for space again shouldn't need to mock as cache would serve it.
ok, err = dm.IsRuntimeDiskSpaceAvailable()
assert.NoError(err)
assert.True(ok)
ok, err = dm.IsRootDiskSpaceAvailable()
assert.NoError(err)
assert.True(ok)
// Ensure no more calls to the mockCadvisor occurred
assert.Equal(cadvisorCallCount, len(mockCadvisor.Calls))
}
// TestFsInfoError verifies errors are returned by DiskSpaceAvailable calls
// when FsInfo calls return an error
func TestFsInfoError(t *testing.T) {
assert, policy, mockCadvisor := setUp(t)
policy.RootFreeDiskMB = 10
dm, err := newDiskSpaceManager(mockCadvisor, policy)
assert.NoError(err)
mockCadvisor.On("ImagesFsInfo").Return(cadvisorapi.FsInfo{}, fmt.Errorf("can't find fs"))
mockCadvisor.On("RootFsInfo").Return(cadvisorapi.FsInfo{}, fmt.Errorf("EBUSY"))
ok, err := dm.IsRuntimeDiskSpaceAvailable()
assert.Error(err)
assert.True(ok)
ok, err = dm.IsRootDiskSpaceAvailable()
assert.Error(err)
assert.True(ok)
}
// Test_getFSInfo verifies multiple possible cases for getFsInfo.
func Test_getFsInfo(t *testing.T) {
assert, policy, mockCadvisor := setUp(t)
// Sunny day case
mockCadvisor.On("RootFsInfo").Return(cadvisorapi.FsInfo{
Usage: 10 * mb,
Capacity: 100 * mb,
Available: 90 * mb,
}, nil).Once()
dm := &realDiskSpaceManager{
cadvisor: mockCadvisor,
policy: policy,
cachedInfo: map[string]fsInfo{},
}
available, err := dm.isSpaceAvailable("root", 10, dm.cadvisor.RootFsInfo)
assert.True(available)
assert.NoError(err)
// Threshold case
mockCadvisor = new(cadvisortest.Mock)
mockCadvisor.On("RootFsInfo").Return(cadvisorapi.FsInfo{
Usage: 9 * mb,
Capacity: 100 * mb,
Available: 9 * mb,
}, nil).Once()
dm = &realDiskSpaceManager{
cadvisor: mockCadvisor,
policy: policy,
cachedInfo: map[string]fsInfo{},
}
available, err = dm.isSpaceAvailable("root", 10, dm.cadvisor.RootFsInfo)
assert.False(available)
assert.NoError(err)
// Frozen case
mockCadvisor.On("RootFsInfo").Return(cadvisorapi.FsInfo{
Usage: 9 * mb,
Capacity: 10 * mb,
Available: 500 * mb,
}, nil).Once()
dm = &realDiskSpaceManager{
cadvisor: mockCadvisor,
policy: policy,
cachedInfo: map[string]fsInfo{},
}
available, err = dm.isSpaceAvailable("root", 10, dm.cadvisor.RootFsInfo)
assert.True(available)
assert.NoError(err)
// Capacity error case
mockCadvisor = new(cadvisortest.Mock)
mockCadvisor.On("RootFsInfo").Return(cadvisorapi.FsInfo{
Usage: 9 * mb,
Capacity: 0,
Available: 500 * mb,
}, nil).Once()
dm = &realDiskSpaceManager{
cadvisor: mockCadvisor,
policy: policy,
cachedInfo: map[string]fsInfo{},
}
available, err = dm.isSpaceAvailable("root", 10, dm.cadvisor.RootFsInfo)
assert.True(available)
assert.Error(err)
assert.Contains(fmt.Sprintf("%s", err), "could not determine capacity")
// Available error case skipped as v2.FSInfo uses uint64 and this
// can not be less than 0
}
......@@ -53,7 +53,6 @@ const (
NodeSelectorMismatching = "NodeSelectorMismatching"
InsufficientFreeCPU = "InsufficientFreeCPU"
InsufficientFreeMemory = "InsufficientFreeMemory"
OutOfDisk = "OutOfDisk"
HostNetworkNotSupported = "HostNetworkNotSupported"
UndefinedShaper = "NilShaper"
NodeRebooted = "Rebooted"
......
......@@ -1363,12 +1363,12 @@ func TestHasNodeConditions(t *testing.T) {
result bool
}{
"has-condition": {
inputs: []v1.NodeConditionType{v1.NodeReady, v1.NodeOutOfDisk, v1.NodeMemoryPressure},
inputs: []v1.NodeConditionType{v1.NodeReady, v1.NodeDiskPressure, v1.NodeMemoryPressure},
item: v1.NodeMemoryPressure,
result: true,
},
"does-not-have-condition": {
inputs: []v1.NodeConditionType{v1.NodeReady, v1.NodeOutOfDisk},
inputs: []v1.NodeConditionType{v1.NodeReady, v1.NodeDiskPressure},
item: v1.NodeMemoryPressure,
result: false,
},
......
......@@ -368,11 +368,6 @@ func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Dep
LowThresholdPercent: int(kubeCfg.ImageGCLowThresholdPercent),
}
diskSpacePolicy := DiskSpacePolicy{
DockerFreeDiskMB: int(kubeCfg.LowDiskSpaceThresholdMB),
RootFreeDiskMB: int(kubeCfg.LowDiskSpaceThresholdMB),
}
enforceNodeAllocatable := kubeCfg.EnforceNodeAllocatable
if kubeCfg.ExperimentalNodeAllocatableIgnoreEvictionThreshold {
// Do not provide kubeCfg.EnforceNodeAllocatable to eviction threshold parsing if we are not enforcing Evictions
......@@ -416,10 +411,6 @@ func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Dep
Namespace: "",
}
diskSpaceManager, err := newDiskSpaceManager(kubeDeps.CAdvisorInterface, diskSpacePolicy)
if err != nil {
return nil, fmt.Errorf("failed to initialize disk manager: %v", err)
}
containerRefManager := kubecontainer.NewRefManager()
oomWatcher := NewOOMWatcher(kubeDeps.CAdvisorInterface, kubeDeps.Recorder)
......@@ -453,7 +444,6 @@ func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Dep
streamingConnectionIdleTimeout: kubeCfg.StreamingConnectionIdleTimeout.Duration,
recorder: kubeDeps.Recorder,
cadvisor: kubeDeps.CAdvisorInterface,
diskSpaceManager: diskSpaceManager,
cloud: kubeDeps.Cloud,
autoDetectCloudProvider: (componentconfigv1alpha1.AutoDetectCloudProvider == kubeCfg.CloudProvider),
externalCloudProvider: cloudprovider.IsExternal(kubeCfg.CloudProvider),
......@@ -475,7 +465,6 @@ func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Dep
containerManager: kubeDeps.ContainerManager,
nodeIP: net.ParseIP(nodeIP),
clock: clock.RealClock{},
outOfDiskTransitionFrequency: kubeCfg.OutOfDiskTransitionFrequency.Duration,
enableControllerAttachDetach: kubeCfg.EnableControllerAttachDetach,
iptClient: utilipt.New(utilexec.New(), utildbus.New(), utilipt.ProtocolIpv4),
makeIPTablesUtilChains: kubeCfg.MakeIPTablesUtilChains,
......@@ -924,9 +913,6 @@ type Kubelet struct {
// Manager for image garbage collection.
imageManager images.ImageGCManager
// Diskspace manager.
diskSpaceManager diskSpaceManager
// Secret manager.
secretManager secret.Manager
......@@ -1044,12 +1030,6 @@ type Kubelet struct {
// easy to test the code.
clock clock.Clock
// outOfDiskTransitionFrequency specifies the amount of time the kubelet has to be actually
// not out of disk before it can transition the node condition status from out-of-disk to
// not-out-of-disk. This prevents a pod that causes out-of-disk condition from repeatedly
// getting rescheduled onto the node.
outOfDiskTransitionFrequency time.Duration
// handlers called during the tryUpdateNodeStatus cycle
setNodeStatusFuncs []func(*v1.Node) error
......@@ -1650,27 +1630,6 @@ func (kl *Kubelet) deletePod(pod *v1.Pod) error {
return nil
}
// isOutOfDisk detects if pods can't fit due to lack of disk space.
func (kl *Kubelet) isOutOfDisk() bool {
// Check disk space once globally and reject or accept all new pods.
withinBounds, err := kl.diskSpaceManager.IsRuntimeDiskSpaceAvailable()
// Assume enough space in case of errors.
if err != nil {
glog.Errorf("Failed to check if disk space is available for the runtime: %v", err)
} else if !withinBounds {
return true
}
withinBounds, err = kl.diskSpaceManager.IsRootDiskSpaceAvailable()
// Assume enough space in case of errors.
if err != nil {
glog.Errorf("Failed to check if disk space is available on the root partition: %v", err)
} else if !withinBounds {
return true
}
return false
}
// rejectPod records an event about the pod with the given reason and message,
// and updates the pod to the failed phase in the status manage.
func (kl *Kubelet) rejectPod(pod *v1.Pod, reason, message string) {
......@@ -1697,12 +1656,6 @@ func (kl *Kubelet) canAdmitPod(pods []*v1.Pod, pod *v1.Pod) (bool, string, strin
return false, result.Reason, result.Message
}
}
// TODO: When disk space scheduling is implemented (#11976), remove the out-of-disk check here and
// add the disk space predicate to predicates.GeneralPredicates.
if kl.isOutOfDisk() {
glog.Warningf("Failed to admit pod %v - %s", format.Pod(pod), "predicate fails due to OutOfDisk")
return false, "OutOfDisk", "cannot be started due to lack of disk space."
}
return true, "", ""
}
......
......@@ -852,70 +852,6 @@ func (kl *Kubelet) setNodeDiskPressureCondition(node *v1.Node) {
}
}
// Set OODCondition for the node.
func (kl *Kubelet) setNodeOODCondition(node *v1.Node) {
currentTime := metav1.NewTime(kl.clock.Now())
var nodeOODCondition *v1.NodeCondition
// Check if NodeOutOfDisk condition already exists and if it does, just pick it up for update.
for i := range node.Status.Conditions {
if node.Status.Conditions[i].Type == v1.NodeOutOfDisk {
nodeOODCondition = &node.Status.Conditions[i]
}
}
newOODCondition := false
// If the NodeOutOfDisk condition doesn't exist, create one.
if nodeOODCondition == nil {
nodeOODCondition = &v1.NodeCondition{
Type: v1.NodeOutOfDisk,
Status: v1.ConditionUnknown,
}
// nodeOODCondition cannot be appended to node.Status.Conditions here because it gets
// copied to the slice. So if we append nodeOODCondition to the slice here none of the
// updates we make to nodeOODCondition below are reflected in the slice.
newOODCondition = true
}
// Update the heartbeat time irrespective of all the conditions.
nodeOODCondition.LastHeartbeatTime = currentTime
// Note: The conditions below take care of the case when a new NodeOutOfDisk condition is
// created and as well as the case when the condition already exists. When a new condition
// is created its status is set to v1.ConditionUnknown which matches either
// nodeOODCondition.Status != v1.ConditionTrue or
// nodeOODCondition.Status != v1.ConditionFalse in the conditions below depending on whether
// the kubelet is out of disk or not.
if kl.isOutOfDisk() {
if nodeOODCondition.Status != v1.ConditionTrue {
nodeOODCondition.Status = v1.ConditionTrue
nodeOODCondition.Reason = "KubeletOutOfDisk"
nodeOODCondition.Message = "out of disk space"
nodeOODCondition.LastTransitionTime = currentTime
kl.recordNodeStatusEvent(v1.EventTypeNormal, "NodeOutOfDisk")
}
} else {
if nodeOODCondition.Status != v1.ConditionFalse {
// Update the out of disk condition when the condition status is unknown even if we
// are within the outOfDiskTransitionFrequency duration. We do this to set the
// condition status correctly at kubelet startup.
if nodeOODCondition.Status == v1.ConditionUnknown || kl.clock.Since(nodeOODCondition.LastTransitionTime.Time) >= kl.outOfDiskTransitionFrequency {
nodeOODCondition.Status = v1.ConditionFalse
nodeOODCondition.Reason = "KubeletHasSufficientDisk"
nodeOODCondition.Message = "kubelet has sufficient disk space available"
nodeOODCondition.LastTransitionTime = currentTime
kl.recordNodeStatusEvent(v1.EventTypeNormal, "NodeHasSufficientDisk")
} else {
glog.Infof("Node condition status for OutOfDisk is false, but last transition time is less than %s", kl.outOfDiskTransitionFrequency)
}
}
}
if newOODCondition {
node.Status.Conditions = append(node.Status.Conditions, *nodeOODCondition)
}
}
// Maintains Node.Spec.Unschedulable value from previous run of tryUpdateNodeStatus()
// TODO: why is this a package var?
var oldNodeUnschedulable bool
......@@ -966,7 +902,6 @@ func (kl *Kubelet) defaultNodeStatusFuncs() []func(*v1.Node) error {
return []func(*v1.Node) error{
kl.setNodeAddress,
withoutError(kl.setNodeStatusInfo),
withoutError(kl.setNodeOODCondition),
withoutError(kl.setNodeMemoryPressureCondition),
withoutError(kl.setNodeDiskPressureCondition),
withoutError(kl.setNodeReadyCondition),
......
......@@ -213,11 +213,6 @@ func newTestKubeletWithImageList(
kubelet.configMapManager = configMapManager
kubelet.podManager = kubepod.NewBasicPodManager(fakeMirrorClient, kubelet.secretManager, kubelet.configMapManager)
kubelet.statusManager = status.NewManager(fakeKubeClient, kubelet.podManager, &statustest.FakePodDeletionSafetyProvider{})
diskSpaceManager, err := newDiskSpaceManager(mockCadvisor, DiskSpacePolicy{})
if err != nil {
t.Fatalf("can't initialize disk space manager: %v", err)
}
kubelet.diskSpaceManager = diskSpaceManager
kubelet.containerRuntime = fakeRuntime
kubelet.runtimeCache = containertest.NewFakeRuntimeCache(kubelet.containerRuntime)
......@@ -737,25 +732,6 @@ func TestValidateContainerLogStatus(t *testing.T) {
}
}
// updateDiskSpacePolicy creates a new DiskSpaceManager with a new policy. This new manager along
// with the mock FsInfo values added to Cadvisor should make the kubelet report that it has
// sufficient disk space or it is out of disk, depending on the capacity, availability and
// threshold values.
func updateDiskSpacePolicy(kubelet *Kubelet, mockCadvisor *cadvisortest.Mock, rootCap, dockerCap, rootAvail, dockerAvail uint64, rootThreshold, dockerThreshold int) error {
dockerimagesFsInfo := cadvisorapiv2.FsInfo{Capacity: rootCap * mb, Available: rootAvail * mb}
rootFsInfo := cadvisorapiv2.FsInfo{Capacity: dockerCap * mb, Available: dockerAvail * mb}
mockCadvisor.On("ImagesFsInfo").Return(dockerimagesFsInfo, nil)
mockCadvisor.On("RootFsInfo").Return(rootFsInfo, nil)
dsp := DiskSpacePolicy{DockerFreeDiskMB: rootThreshold, RootFreeDiskMB: dockerThreshold}
diskSpaceManager, err := newDiskSpaceManager(mockCadvisor, dsp)
if err != nil {
return err
}
kubelet.diskSpaceManager = diskSpaceManager
return nil
}
func TestCreateMirrorPod(t *testing.T) {
for _, updateType := range []kubetypes.SyncPodType{kubetypes.SyncPodCreate, kubetypes.SyncPodUpdate} {
testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */)
......
......@@ -55,19 +55,18 @@ func TestRunOnce(t *testing.T) {
cadvisor := &cadvisortest.Mock{}
cadvisor.On("MachineInfo").Return(&cadvisorapi.MachineInfo{}, nil)
cadvisor.On("ImagesFsInfo").Return(cadvisorapiv2.FsInfo{
Usage: 400 * mb,
Capacity: 1000 * mb,
Available: 600 * mb,
Usage: 400,
Capacity: 1000,
Available: 600,
}, nil)
cadvisor.On("RootFsInfo").Return(cadvisorapiv2.FsInfo{
Usage: 9 * mb,
Capacity: 10 * mb,
Usage: 9,
Capacity: 10,
}, nil)
fakeSecretManager := secret.NewFakeManager()
fakeConfigMapManager := configmap.NewFakeManager()
podManager := kubepod.NewBasicPodManager(
podtest.NewFakeMirrorClient(), fakeSecretManager, fakeConfigMapManager)
diskSpaceManager, _ := newDiskSpaceManager(cadvisor, DiskSpacePolicy{})
fakeRuntime := &containertest.FakeRuntime{}
basePath, err := utiltesting.MkTmpdir("kubelet")
if err != nil {
......@@ -82,7 +81,6 @@ func TestRunOnce(t *testing.T) {
statusManager: status.NewManager(nil, podManager, &statustest.FakePodDeletionSafetyProvider{}),
podManager: podManager,
os: &containertest.FakeOS{},
diskSpaceManager: diskSpaceManager,
containerRuntime: fakeRuntime,
reasonCache: NewReasonCache(),
clock: clock.RealClock{},
......
......@@ -134,14 +134,12 @@ func GetHollowKubeletConfig(
c.MinimumGCAge.Duration = 1 * time.Minute
c.NodeStatusUpdateFrequency.Duration = 10 * time.Second
c.SyncFrequency.Duration = 10 * time.Second
c.OutOfDiskTransitionFrequency.Duration = 5 * time.Minute
c.EvictionPressureTransitionPeriod.Duration = 5 * time.Minute
c.MaxPods = int32(maxPods)
c.PodsPerCore = int32(podsPerCore)
c.ClusterDNS = []string{}
c.ImageGCHighThresholdPercent = 90
c.ImageGCLowThresholdPercent = 80
c.LowDiskSpaceThresholdMB = 256
c.VolumeStatsAggPeriod.Duration = time.Minute
c.CgroupRoot = ""
c.ContainerRuntime = kubetypes.DockerContainerRuntime
......
......@@ -1588,7 +1588,7 @@ func TestPrintPod(t *testing.T) {
ObjectMeta: metav1.ObjectMeta{Name: "test5"},
Spec: api.PodSpec{Containers: make([]api.Container, 2)},
Status: api.PodStatus{
Reason: "OutOfDisk",
Reason: "podReason",
Phase: "podPhase",
ContainerStatuses: []api.ContainerStatus{
{Ready: true, RestartCount: 3, State: api.ContainerState{Running: &api.ContainerStateRunning{}}},
......@@ -1596,7 +1596,7 @@ func TestPrintPod(t *testing.T) {
},
},
},
[]metav1alpha1.TableRow{{Cells: []interface{}{"test5", "1/2", "OutOfDisk", 6, "<unknown>"}}},
[]metav1alpha1.TableRow{{Cells: []interface{}{"test5", "1/2", "podReason", 6, "<unknown>"}}},
},
}
......
......@@ -2830,7 +2830,7 @@ message PodStatus {
optional string message = 3;
// A brief CamelCase message indicating details about why the pod is in this state.
// e.g. 'OutOfDisk'
// e.g. 'Evicted'
// +optional
optional string reason = 4;
......
......@@ -2624,7 +2624,7 @@ type PodStatus struct {
// +optional
Message string `json:"message,omitempty" protobuf:"bytes,3,opt,name=message"`
// A brief CamelCase message indicating details about why the pod is in this state.
// e.g. 'OutOfDisk'
// e.g. 'Evicted'
// +optional
Reason string `json:"reason,omitempty" protobuf:"bytes,4,opt,name=reason"`
......
......@@ -1389,7 +1389,7 @@ var map_PodStatus = map[string]string{
"phase": "Current condition of the pod. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#pod-phase",
"conditions": "Current service state of pod. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#pod-conditions",
"message": "A human readable message indicating details about why the pod is in this condition.",
"reason": "A brief CamelCase message indicating details about why the pod is in this state. e.g. 'OutOfDisk'",
"reason": "A brief CamelCase message indicating details about why the pod is in this state. e.g. 'Evicted'",
"hostIP": "IP address of the host to which the pod is assigned. Empty if not yet scheduled.",
"podIP": "IP address allocated to the pod. Routable at least within the cluster. Empty if not yet allocated.",
"startTime": "RFC 3339 date and time at which the object was acknowledged by the Kubelet. This is before the Kubelet pulled the container image(s) for the pod.",
......
......@@ -66,7 +66,6 @@ go_library(
"limit_range.go",
"network_partition.go",
"no-snat.go",
"nodeoutofdisk.go",
"pod_gc.go",
"podpreset.go",
"pods.go",
......@@ -102,7 +101,6 @@ go_library(
"//test/e2e_federation:go_default_library",
"//test/utils:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/github.com/google/cadvisor/info/v1:go_default_library",
"//vendor/github.com/onsi/ginkgo:go_default_library",
"//vendor/github.com/onsi/ginkgo/config:go_default_library",
"//vendor/github.com/onsi/ginkgo/reporters:go_default_library",
......
......@@ -336,7 +336,6 @@ NoExecuteTaintManager doesn't evict pod with tolerations from tainted nodes,free
NoExecuteTaintManager eventually evict pod with finite tolerations from tainted nodes,freehan,0,scheduling
NoExecuteTaintManager evicts pods from tainted nodes,freehan,0,scheduling
NoExecuteTaintManager removing taint cancels eviction,freehan,0,scheduling
NodeOutOfDisk runs out of disk space,vishh,0,node
NodeProblemDetector KernelMonitor should generate node condition and events for corresponding errors,Random-Liu,0,node
Nodes Resize should be able to add nodes,piosz,1,cluster-lifecycle
Nodes Resize should be able to delete nodes,zmerlynn,1,cluster-lifecycle
......
......@@ -83,7 +83,6 @@
"Logging",
"MemoryEviction",
"MirrorPod",
"NodeOutOfDisk",
"NodeProblemDetector",
"Opaque",
"Pod garbage",
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment