fix #51135 make CFS quota period configurable, adds a cli flag and config option…

cmd/kubeadm/app/util/config/testdata/conversion/master/internal.yaml

@@ -78,6 +78,7 @@ ComponentConfigs:
         CacheAuthorizedTTL: 5m0s
         CacheUnauthorizedTTL: 30s
     CPUCFSQuota: true
+    CPUCFSQuotaPeriod: 0s
     CPUManagerPolicy: none
     CPUManagerReconcilePeriod: 10s
     CgroupDriver: cgroupfs

cmd/kubeadm/app/util/config/testdata/conversion/master/v1alpha2.yaml

@@ -91,6 +91,7 @@ kubeletConfiguration:
     containerLogMaxSize: 10Mi
     contentType: application/vnd.kubernetes.protobuf
     cpuCFSQuota: true
+    cpuCFSQuotaPeriod: 0s
     cpuManagerPolicy: none
     cpuManagerReconcilePeriod: 10s
     enableControllerAttachDetach: true

cmd/kubeadm/app/util/config/testdata/conversion/master/v1alpha3.yaml

@@ -107,6 +107,7 @@ containerLogMaxFiles: 5
 containerLogMaxSize: 10Mi
 contentType: application/vnd.kubernetes.protobuf
 cpuCFSQuota: true
+cpuCFSQuotaPeriod: 0s
 cpuManagerPolicy: none
 cpuManagerReconcilePeriod: 10s
 enableControllerAttachDetach: true

cmd/kubeadm/app/util/config/testdata/defaulting/master/defaulted.yaml

@@ -102,6 +102,7 @@ containerLogMaxFiles: 5
 containerLogMaxSize: 10Mi
 contentType: application/vnd.kubernetes.protobuf
 cpuCFSQuota: true
+cpuCFSQuotaPeriod: 0s
 cpuManagerPolicy: none
 cpuManagerReconcilePeriod: 10s
 enableControllerAttachDetach: true

cmd/kubelet/app/options/options.go

@@ -530,6 +530,7 @@ func AddKubeletConfigFlags(mainfs *pflag.FlagSet, c *kubeletconfig.KubeletConfig
 
 	fs.StringVar(&c.ResolverConfig, "resolv-conf", c.ResolverConfig, "Resolver configuration file used as the basis for the container DNS resolution configuration.")
 	fs.BoolVar(&c.CPUCFSQuota, "cpu-cfs-quota", c.CPUCFSQuota, "Enable CPU CFS quota enforcement for containers that specify CPU limits")
+	fs.DurationVar(&c.CPUCFSQuotaPeriod.Duration, "cpu-cfs-quota-period", c.CPUCFSQuotaPeriod.Duration, "Sets CPU CFS quota period value, cpu.cfs_period_us, defaults to Linux Kernel default")
 	fs.BoolVar(&c.EnableControllerAttachDetach, "enable-controller-attach-detach", c.EnableControllerAttachDetach, "Enables the Attach/Detach controller to manage attachment/detachment of volumes scheduled to this node, and disables kubelet from executing any attach/detach operations")
 	fs.BoolVar(&c.MakeIPTablesUtilChains, "make-iptables-util-chains", c.MakeIPTablesUtilChains, "If true, kubelet will ensure iptables utility rules are present on host.")
 	fs.Int32Var(&c.IPTablesMasqueradeBit, "iptables-masquerade-bit", c.IPTablesMasqueradeBit, "The bit of the fwmark space to mark packets for SNAT. Must be within the range [0, 31]. Please match this parameter with corresponding parameter in kube-proxy.")

cmd/kubelet/app/server.go

@@ -708,6 +708,7 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies, stopCh <-chan
 				ExperimentalCPUManagerReconcilePeriod: s.CPUManagerReconcilePeriod.Duration,
 				ExperimentalPodPidsLimit:              s.PodPidsLimit,
 				EnforceCPULimits:                      s.CPUCFSQuota,
+				CPUCFSQuotaPeriod:                     s.CPUCFSQuotaPeriod.Duration,
 			},
 			s.FailSwapOn,
 			devicePluginEnabled,

pkg/features/kube_features.go

@@ -155,6 +155,12 @@ const (
 	// Alternative container-level CPU affinity policies.
 	CPUManager utilfeature.Feature = "CPUManager"
 
+	// owner: @szuecs
+	// alpha: v1.12
+	//
+	// Enable nodes to change CPUCFSQuotaPeriod
+	CPUCFSQuotaPeriod utilfeature.Feature = "CustomCPUCFSQuotaPeriod"
+
 	// owner: @derekwaynecarr
 	// beta: v1.10
 	//
@@ -408,6 +414,7 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS
 	ExpandInUsePersistentVolumes:                {Default: false, PreRelease: utilfeature.Alpha},
 	AttachVolumeLimit:                           {Default: false, PreRelease: utilfeature.Alpha},
 	CPUManager:                                  {Default: true, PreRelease: utilfeature.Beta},
+	CPUCFSQuotaPeriod:                           {Default: false, PreRelease: utilfeature.Alpha},
 	ServiceNodeExclusion:                        {Default: false, PreRelease: utilfeature.Alpha},
 	MountContainers:                             {Default: false, PreRelease: utilfeature.Alpha},
 	VolumeScheduling:                            {Default: true, PreRelease: utilfeature.Beta},

pkg/kubelet/apis/config/helpers_test.go

@@ -145,6 +145,7 @@ var (
 		"Authorization.Webhook.CacheAuthorizedTTL.Duration",
 		"Authorization.Webhook.CacheUnauthorizedTTL.Duration",
 		"CPUCFSQuota",
+		"CPUCFSQuotaPeriod.Duration",
 		"CPUManagerPolicy",
 		"CPUManagerReconcilePeriod.Duration",
 		"QOSReserved[*]",

pkg/kubelet/apis/config/types.go

@@ -220,6 +220,8 @@ type KubeletConfiguration struct {
 	// cpuCFSQuota enables CPU CFS quota enforcement for containers that
 	// specify CPU limits
 	CPUCFSQuota bool
+	// CPUCFSQuotaPeriod sets the CPU CFS quota period value, cpu.cfs_period_us, defaults to 100ms
+	CPUCFSQuotaPeriod metav1.Duration
 	// maxOpenFiles is Number of files that can be opened by Kubelet process.
 	MaxOpenFiles int64
 	// contentType is contentType of requests sent to apiserver.

pkg/kubelet/apis/config/v1beta1/BUILD

@@ -18,6 +18,7 @@ go_library(
     ],
     importpath = "k8s.io/kubernetes/pkg/kubelet/apis/config/v1beta1",
     deps = [
+        "//pkg/features:go_default_library",
         "//pkg/kubelet/apis/config:go_default_library",
         "//pkg/kubelet/qos:go_default_library",
         "//pkg/kubelet/types:go_default_library",

pkg/kubelet/apis/config/v1beta1/defaults.go

@@ -21,6 +21,7 @@ import (
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	kruntime "k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/kubernetes/pkg/features"
 	"k8s.io/kubernetes/pkg/kubelet/qos"
 	kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
 	"k8s.io/kubernetes/pkg/master/ports"
@@ -154,6 +155,9 @@ func SetDefaults_KubeletConfiguration(obj *KubeletConfiguration) {
 	if obj.CPUCFSQuota == nil {
 		obj.CPUCFSQuota = utilpointer.BoolPtr(true)
 	}
+	if obj.CPUCFSQuotaPeriod == nil && obj.FeatureGates[string(features.CPUCFSQuotaPeriod)] {
+		obj.CPUCFSQuotaPeriod = &metav1.Duration{Duration: 100 * time.Millisecond}
+	}
 	if obj.MaxOpenFiles == 0 {
 		obj.MaxOpenFiles = 1000000
 	}

pkg/kubelet/apis/config/v1beta1/types.go

@@ -466,6 +466,13 @@ type KubeletConfiguration struct {
 	// Default: true
 	// +optional
 	CPUCFSQuota *bool `json:"cpuCFSQuota,omitempty"`
+	// CPUCFSQuotaPeriod is the CPU CFS quota period value, cpu.cfs_period_us.
+	// Dynamic Kubelet Config (beta): If dynamically updating this field, consider that
+	// limits set for containers will result in different cpu.cfs_quota settings. This
+	// will trigger container restarts on the node being reconfigured.
+	// Default: "100ms"
+	// +optional
+	CPUCFSQuotaPeriod *metav1.Duration `json:"cpuCFSQuotaPeriod,omitempty"`
 	// maxOpenFiles is Number of files that can be opened by Kubelet process.
 	// Dynamic Kubelet Config (beta): If dynamically updating this field, consider that
 	// it may impact the ability of the Kubelet to interact with the node's filesystem.

pkg/kubelet/apis/config/v1beta1/zz_generated.conversion.go

@@ -280,6 +280,9 @@ func autoConvert_v1beta1_KubeletConfiguration_To_config_KubeletConfiguration(in
 	if err := v1.Convert_Pointer_bool_To_bool(&in.CPUCFSQuota, &out.CPUCFSQuota, s); err != nil {
 		return err
 	}
+	if err := v1.Convert_Pointer_v1_Duration_To_v1_Duration(&in.CPUCFSQuotaPeriod, &out.CPUCFSQuotaPeriod, s); err != nil {
+		return err
+	}
 	out.MaxOpenFiles = in.MaxOpenFiles
 	out.ContentType = in.ContentType
 	if err := v1.Convert_Pointer_int32_To_int32(&in.KubeAPIQPS, &out.KubeAPIQPS, s); err != nil {
@@ -406,6 +409,9 @@ func autoConvert_config_KubeletConfiguration_To_v1beta1_KubeletConfiguration(in
 	if err := v1.Convert_bool_To_Pointer_bool(&in.CPUCFSQuota, &out.CPUCFSQuota, s); err != nil {
 		return err
 	}
+	if err := v1.Convert_v1_Duration_To_Pointer_v1_Duration(&in.CPUCFSQuotaPeriod, &out.CPUCFSQuotaPeriod, s); err != nil {
+		return err
+	}
 	out.MaxOpenFiles = in.MaxOpenFiles
 	out.ContentType = in.ContentType
 	if err := v1.Convert_int32_To_Pointer_int32(&in.KubeAPIQPS, &out.KubeAPIQPS, s); err != nil {

pkg/kubelet/apis/config/v1beta1/zz_generated.deepcopy.go

@@ -21,6 +21,7 @@ limitations under the License.
 package v1beta1
 
 import (
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	runtime "k8s.io/apimachinery/pkg/runtime"
 )
 
@@ -178,6 +179,11 @@ func (in *KubeletConfiguration) DeepCopyInto(out *KubeletConfiguration) {
 		*out = new(bool)
 		**out = **in
 	}
+	if in.CPUCFSQuotaPeriod != nil {
+		in, out := &in.CPUCFSQuotaPeriod, &out.CPUCFSQuotaPeriod
+		*out = new(v1.Duration)
+		**out = **in
+	}
 	if in.KubeAPIQPS != nil {
 		in, out := &in.KubeAPIQPS, &out.KubeAPIQPS
 		*out = new(int32)

pkg/kubelet/apis/config/validation/BUILD

@@ -43,6 +43,7 @@ go_test(
     embed = [":go_default_library"],
     deps = [
         "//pkg/kubelet/apis/config:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library",
     ],
 )

pkg/kubelet/apis/config/validation/validation.go

@@ -18,6 +18,7 @@ package validation
 
 import (
 	"fmt"
+	"time"
 
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	utilvalidation "k8s.io/apimachinery/pkg/util/validation"
@@ -54,6 +55,9 @@ func ValidateKubeletConfiguration(kc *kubeletconfig.KubeletConfiguration) error
 	if kc.HealthzPort != 0 && utilvalidation.IsValidPortNum(int(kc.HealthzPort)) != nil {
 		allErrors = append(allErrors, fmt.Errorf("invalid configuration: HealthzPort (--healthz-port) %v must be between 1 and 65535, inclusive", kc.HealthzPort))
 	}
+	if localFeatureGate.Enabled(features.CPUCFSQuotaPeriod) && utilvalidation.IsInRange(int(kc.CPUCFSQuotaPeriod.Duration), int(1*time.Microsecond), int(time.Second)) != nil {
+		allErrors = append(allErrors, fmt.Errorf("invalid configuration: CPUCFSQuotaPeriod (--cpu-cfs-quota-period) %v must be between 1usec and 1sec, inclusive", kc.CPUCFSQuotaPeriod))
+	}
 	if utilvalidation.IsInRange(int(kc.ImageGCHighThresholdPercent), 0, 100) != nil {
 		allErrors = append(allErrors, fmt.Errorf("invalid configuration: ImageGCHighThresholdPercent (--image-gc-high-threshold) %v must be between 0 and 100, inclusive", kc.ImageGCHighThresholdPercent))
 	}

pkg/kubelet/apis/config/validation/validation_test.go

@@ -18,7 +18,9 @@ package validation
 
 import (
 	"testing"
+	"time"
 
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
 )
@@ -48,6 +50,7 @@ func TestValidateKubeletConfiguration(t *testing.T) {
 		RegistryPullQPS:             5,
 		HairpinMode:                 kubeletconfig.PromiscuousBridge,
 		NodeLeaseDurationSeconds:    1,
+		CPUCFSQuotaPeriod:           metav1.Duration{Duration: 100 * time.Millisecond},
 	}
 	if allErrors := ValidateKubeletConfiguration(successCase); allErrors != nil {
 		t.Errorf("expect no errors, got %v", allErrors)
@@ -77,6 +80,7 @@ func TestValidateKubeletConfiguration(t *testing.T) {
 		RegistryPullQPS:             -10,
 		HairpinMode:                 "foo",
 		NodeLeaseDurationSeconds:    -1,
+		CPUCFSQuotaPeriod:           metav1.Duration{Duration: 0},
 	}
 	const numErrs = 23
 	if allErrors := ValidateKubeletConfiguration(errorCase); len(allErrors.(utilerrors.Aggregate).Errors()) != numErrs {

pkg/kubelet/apis/config/zz_generated.deepcopy.go

@@ -123,6 +123,7 @@ func (in *KubeletConfiguration) DeepCopyInto(out *KubeletConfiguration) {
 		}
 	}
 	out.RuntimeRequestTimeout = in.RuntimeRequestTimeout
+	out.CPUCFSQuotaPeriod = in.CPUCFSQuotaPeriod
 	if in.EvictionHard != nil {
 		in, out := &in.EvictionHard, &out.EvictionHard
 		*out = make(map[string]string, len(*in))

pkg/kubelet/cm/BUILD

@@ -134,11 +134,14 @@ go_test(
     embed = [":go_default_library"],
     deps = select({
         "@io_bazel_rules_go//go/platform:linux": [
+            "//pkg/features:go_default_library",
             "//pkg/kubelet/eviction/api:go_default_library",
             "//pkg/util/mount:go_default_library",
             "//staging/src/k8s.io/api/core/v1:go_default_library",
             "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
             "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
+            "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
+            "//staging/src/k8s.io/apiserver/pkg/util/feature/testing:go_default_library",
             "//vendor/github.com/stretchr/testify/assert:go_default_library",
             "//vendor/github.com/stretchr/testify/require:go_default_library",
         ],

pkg/kubelet/cm/container_manager.go

@@ -114,6 +114,7 @@ type NodeConfig struct {
 	ExperimentalCPUManagerReconcilePeriod time.Duration
 	ExperimentalPodPidsLimit              int64
 	EnforceCPULimits                      bool
+	CPUCFSQuotaPeriod                     time.Duration
 }
 
 type NodeAllocatableConfig struct {

pkg/kubelet/cm/container_manager_linux.go

@@ -307,6 +307,7 @@ func (cm *containerManagerImpl) NewPodContainerManager() PodContainerManager {
 			cgroupManager:     cm.cgroupManager,
 			podPidsLimit:      cm.ExperimentalPodPidsLimit,
 			enforceCPULimits:  cm.EnforceCPULimits,
+			cpuCFSQuotaPeriod: uint64(cm.CPUCFSQuotaPeriod / time.Microsecond),
 		}
 	}
 	return &podContainerManagerNoop{

pkg/kubelet/cm/helpers_linux.go

@@ -27,9 +27,11 @@ import (
 
 	"k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/kubernetes/pkg/api/v1/resource"
 	v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper"
 	v1qos "k8s.io/kubernetes/pkg/apis/core/v1/helper/qos"
+	kubefeatures "k8s.io/kubernetes/pkg/features"
 )
 
 const (
@@ -44,28 +46,29 @@ const (
 )
 
 // MilliCPUToQuota converts milliCPU to CFS quota and period values.
-func MilliCPUToQuota(milliCPU int64) (quota int64, period uint64) {
+func MilliCPUToQuota(milliCPU int64, period int64) (quota int64) {
 	// CFS quota is measured in two values:
-	//  - cfs_period_us=100ms (the amount of time to measure usage across)
+	//  - cfs_period_us=100ms (the amount of time to measure usage across given by period)
 	//  - cfs_quota=20ms (the amount of cpu time allowed to be used across a period)
 	// so in the above example, you are limited to 20% of a single CPU
 	// for multi-cpu environments, you just scale equivalent amounts
+	// see https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt for details
 
 	if milliCPU == 0 {
 		return
 	}
 
-	// we set the period to 100ms by default
-	period = QuotaPeriod
+	if !utilfeature.DefaultFeatureGate.Enabled(kubefeatures.CPUCFSQuotaPeriod) {
+		period = QuotaPeriod
+	}
 
 	// we then convert your milliCPU to a value normalized over a period
-	quota = (milliCPU * QuotaPeriod) / MilliCPUToCPU
+	quota = (milliCPU * period) / MilliCPUToCPU
 
 	// quota needs to be a minimum of 1ms.
 	if quota < MinQuotaPeriod {
 		quota = MinQuotaPeriod
 	}
-
 	return
 }
 
@@ -103,7 +106,7 @@ func HugePageLimits(resourceList v1.ResourceList) map[int64]int64 {
 }
 
 // ResourceConfigForPod takes the input pod and outputs the cgroup resource config.
-func ResourceConfigForPod(pod *v1.Pod, enforceCPULimits bool) *ResourceConfig {
+func ResourceConfigForPod(pod *v1.Pod, enforceCPULimits bool, cpuPeriod uint64) *ResourceConfig {
 	// sum requests and limits.
 	reqs, limits := resource.PodRequestsAndLimits(pod)
 
@@ -122,7 +125,7 @@ func ResourceConfigForPod(pod *v1.Pod, enforceCPULimits bool) *ResourceConfig {
 
 	// convert to CFS values
 	cpuShares := MilliCPUToShares(cpuRequests)
-	cpuQuota, cpuPeriod := MilliCPUToQuota(cpuLimits)
+	cpuQuota := MilliCPUToQuota(cpuLimits, int64(cpuPeriod))
 
 	// track if limits were applied for each resource.
 	memoryLimitsDeclared := true

pkg/kubelet/cm/helpers_unsupported.go

@@ -28,13 +28,12 @@ const (
 	SharesPerCPU  = 0
 	MilliCPUToCPU = 0
 
-	QuotaPeriod    = 0
 	MinQuotaPeriod = 0
 )
 
-// MilliCPUToQuota converts milliCPU to CFS quota and period values.
-func MilliCPUToQuota(milliCPU int64) (int64, int64) {
-	return 0, 0
+// MilliCPUToQuota converts milliCPU and period to CFS quota values.
+func MilliCPUToQuota(milliCPU, period int64) int64 {
+	return 0
 }
 
 // MilliCPUToShares converts the milliCPU to CFS shares.
@@ -43,7 +42,7 @@ func MilliCPUToShares(milliCPU int64) int64 {
 }
 
 // ResourceConfigForPod takes the input pod and outputs the cgroup resource config.
-func ResourceConfigForPod(pod *v1.Pod, enforceCPULimit bool) *ResourceConfig {
+func ResourceConfigForPod(pod *v1.Pod, enforceCPULimit bool, cpuPeriod uint64) *ResourceConfig {
 	return nil
 }
 

pkg/kubelet/cm/pod_container_manager_linux.go

@@ -51,6 +51,9 @@ type podContainerManagerImpl struct {
 	podPidsLimit int64
 	// enforceCPULimits controls whether cfs quota is enforced or not
 	enforceCPULimits bool
+	// cpuCFSQuotaPeriod is the cfs period value, cfs_period_us, setting per
+	// node for all containers in usec
+	cpuCFSQuotaPeriod uint64
 }
 
 // Make sure that podContainerManagerImpl implements the PodContainerManager interface
@@ -81,7 +84,7 @@ func (m *podContainerManagerImpl) EnsureExists(pod *v1.Pod) error {
 		// Create the pod container
 		containerConfig := &CgroupConfig{
 			Name:               podContainerName,
-			ResourceParameters: ResourceConfigForPod(pod, m.enforceCPULimits),
+			ResourceParameters: ResourceConfigForPod(pod, m.enforceCPULimits, m.cpuCFSQuotaPeriod),
 		}
 		if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.SupportPodPidsLimit) && m.podPidsLimit > 0 {
 			containerConfig.ResourceParameters.PodPidsLimit = &m.podPidsLimit

pkg/kubelet/kubelet.go

@@ -668,6 +668,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
 		float32(kubeCfg.RegistryPullQPS),
 		int(kubeCfg.RegistryBurst),
 		kubeCfg.CPUCFSQuota,
+		kubeCfg.CPUCFSQuotaPeriod,
 		runtimeService,
 		imageService,
 		kubeDeps.ContainerManager.InternalContainerLifecycle(),

pkg/kubelet/kuberuntime/BUILD

@@ -83,6 +83,7 @@ go_library(
 go_test(
     name = "go_default_test",
     srcs = [
+        "helpers_linux_test.go",
         "helpers_test.go",
         "instrumented_services_test.go",
         "kuberuntime_container_linux_test.go",

pkg/kubelet/kuberuntime/fake_kuberuntime_manager.go

@@ -21,6 +21,7 @@ import (
 	"time"
 
 	cadvisorapi "github.com/google/cadvisor/info/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/tools/record"
 	"k8s.io/client-go/util/flowcontrol"
@@ -74,6 +75,7 @@ func NewFakeKubeRuntimeManager(runtimeService internalapi.RuntimeService, imageS
 	kubeRuntimeManager := &kubeGenericRuntimeManager{
 		recorder:            recorder,
 		cpuCFSQuota:         false,
+		cpuCFSQuotaPeriod:   metav1.Duration{Duration: time.Microsecond * 100},
 		livenessManager:     proberesults.NewManager(),
 		containerRefManager: kubecontainer.NewRefManager(),
 		machineInfo:         machineInfo,

pkg/kubelet/kuberuntime/helpers_linux.go

@@ -18,6 +18,11 @@ limitations under the License.
 
 package kuberuntime
 
+import (
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	kubefeatures "k8s.io/kubernetes/pkg/features"
+)
+
 const (
 	// Taken from lmctfy https://github.com/google/lmctfy/blob/master/lmctfy/controllers/cpu_controller.cc
 	minShares     = 2
@@ -25,7 +30,7 @@ const (
 	milliCPUToCPU = 1000
 
 	// 100000 is equivalent to 100ms
-	quotaPeriod    = 100 * minQuotaPeriod
+	quotaPeriod    = 100000
 	minQuotaPeriod = 1000
 )
 
@@ -44,21 +49,22 @@ func milliCPUToShares(milliCPU int64) int64 {
 }
 
 // milliCPUToQuota converts milliCPU to CFS quota and period values
-func milliCPUToQuota(milliCPU int64) (quota int64, period int64) {
+func milliCPUToQuota(milliCPU int64, period int64) (quota int64) {
 	// CFS quota is measured in two values:
 	//  - cfs_period_us=100ms (the amount of time to measure usage across)
 	//  - cfs_quota=20ms (the amount of cpu time allowed to be used across a period)
 	// so in the above example, you are limited to 20% of a single CPU
 	// for multi-cpu environments, you just scale equivalent amounts
+	// see https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt for details
 	if milliCPU == 0 {
 		return
 	}
-
-	// we set the period to 100ms by default
-	period = quotaPeriod
+	if !utilfeature.DefaultFeatureGate.Enabled(kubefeatures.CPUCFSQuotaPeriod) {
+		period = quotaPeriod
+	}
 
 	// we then convert your milliCPU to a value normalized over a period
-	quota = (milliCPU * quotaPeriod) / milliCPUToCPU
+	quota = (milliCPU * period) / milliCPUToCPU
 
 	// quota needs to be a minimum of 1ms.
 	if quota < minQuotaPeriod {

pkg/kubelet/kuberuntime/helpers_linux_test.go

+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kuberuntime
+
+import (
+	"testing"
+
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
+	"k8s.io/kubernetes/pkg/features"
+)
+
+func TestMilliCPUToQuota(t *testing.T) {
+	for _, testCase := range []struct {
+		msg      string
+		input    int64
+		expected int64
+		period   uint64
+	}{
+		{
+			msg:      "all-zero",
+			input:    int64(0),
+			expected: int64(0),
+			period:   uint64(0),
+		},
+		{
+			msg:      "5 input default quota and period",
+			input:    int64(5),
+			expected: int64(1000),
+			period:   uint64(100000),
+		},
+		{
+			msg:      "9 input default quota and period",
+			input:    int64(9),
+			expected: int64(1000),
+			period:   uint64(100000),
+		},
+		{
+			msg:      "10 input default quota and period",
+			input:    int64(10),
+			expected: int64(1000),
+			period:   uint64(100000),
+		},
+		{
+			msg:      "200 input 20k quota and default period",
+			input:    int64(200),
+			expected: int64(20000),
+			period:   uint64(100000),
+		},
+		{
+			msg:      "500 input 50k quota and default period",
+			input:    int64(500),
+			expected: int64(50000),
+			period:   uint64(100000),
+		},
+		{
+			msg:      "1k input 100k quota and default period",
+			input:    int64(1000),
+			expected: int64(100000),
+			period:   uint64(100000),
+		},
+		{
+			msg:      "1500 input 150k quota and default period",
+			input:    int64(1500),
+			expected: int64(150000),
+			period:   uint64(100000),
+		}} {
+		t.Run(testCase.msg, func(t *testing.T) {
+			quota := milliCPUToQuota(testCase.input, int64(testCase.period))
+			if quota != testCase.expected {
+				t.Errorf("Input %v and %v, expected quota %v, but got quota %v", testCase.input, testCase.period, testCase.expected, quota)
+			}
+		})
+	}
+}
+
+func TestMilliCPUToQuotaWithCustomCPUCFSQuotaPeriod(t *testing.T) {
+	utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CPUCFSQuotaPeriod, true)
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CPUCFSQuotaPeriod, false)
+
+	for _, testCase := range []struct {
+		msg      string
+		input    int64
+		expected int64
+		period   uint64
+	}{
+		{
+			msg:      "all-zero",
+			input:    int64(0),
+			expected: int64(0),
+			period:   uint64(0),
+		},
+		{
+			msg:      "5 input default quota and period",
+			input:    int64(5),
+			expected: minQuotaPeriod,
+			period:   uint64(100000),
+		},
+		{
+			msg:      "9 input default quota and period",
+			input:    int64(9),
+			expected: minQuotaPeriod,
+			period:   uint64(100000),
+		},
+		{
+			msg:      "10 input default quota and period",
+			input:    int64(10),
+			expected: minQuotaPeriod,
+			period:   uint64(100000),
+		},
+		{
+			msg:      "200 input 20k quota and default period",
+			input:    int64(200),
+			expected: int64(20000),
+			period:   uint64(100000),
+		},
+		{
+			msg:      "500 input 50k quota and default period",
+			input:    int64(500),
+			expected: int64(50000),
+			period:   uint64(100000),
+		},
+		{
+			msg:      "1k input 100k quota and default period",
+			input:    int64(1000),
+			expected: int64(100000),
+			period:   uint64(100000),
+		},
+		{
+			msg:      "1500 input 150k quota and default period",
+			input:    int64(1500),
+			expected: int64(150000),
+			period:   uint64(100000),
+		},
+		{
+			msg:      "5 input 10k period and default quota expected",
+			input:    int64(5),
+			period:   uint64(10000),
+			expected: minQuotaPeriod,
+		},
+		{
+			msg:      "5 input 5k period and default quota expected",
+			input:    int64(5),
+			period:   uint64(5000),
+			expected: minQuotaPeriod,
+		},
+		{
+			msg:      "9 input 10k period and default quota expected",
+			input:    int64(9),
+			period:   uint64(10000),
+			expected: minQuotaPeriod,
+		},
+		{
+			msg:      "10 input 200k period and 2000 quota expected",
+			input:    int64(10),
+			period:   uint64(200000),
+			expected: int64(2000),
+		},
+		{
+			msg:      "200 input 200k period and 40k quota",
+			input:    int64(200),
+			period:   uint64(200000),
+			expected: int64(40000),
+		},
+		{
+			msg:      "500 input 20k period and 20k expected quota",
+			input:    int64(500),
+			period:   uint64(20000),
+			expected: int64(10000),
+		},
+		{
+			msg:      "1000 input 10k period and 10k expected quota",
+			input:    int64(1000),
+			period:   uint64(10000),
+			expected: int64(10000),
+		},
+		{
+			msg:      "1500 input 5000 period and 7500 expected quota",
+			input:    int64(1500),
+			period:   uint64(5000),
+			expected: int64(7500),
+		}} {
+		t.Run(testCase.msg, func(t *testing.T) {
+			quota := milliCPUToQuota(testCase.input, int64(testCase.period))
+			if quota != testCase.expected {
+				t.Errorf("Input %v and %v, expected quota %v, but got quota %v", testCase.input, testCase.period, testCase.expected, quota)
+			}
+		})
+	}
+}

pkg/kubelet/kuberuntime/kuberuntime_container_linux.go

@@ -19,6 +19,8 @@ limitations under the License.
 package kuberuntime
 
 import (
+	"time"
+
 	"k8s.io/api/core/v1"
 	runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
 	"k8s.io/kubernetes/pkg/kubelet/qos"
@@ -65,7 +67,8 @@ func (m *kubeGenericRuntimeManager) generateLinuxContainerConfig(container *v1.C
 	if m.cpuCFSQuota {
 		// if cpuLimit.Amount is nil, then the appropriate default value is returned
 		// to allow full usage of cpu resource.
-		cpuQuota, cpuPeriod := milliCPUToQuota(cpuLimit.MilliValue())
+		cpuPeriod := int64(m.cpuCFSQuotaPeriod.Duration / time.Microsecond)
+		cpuQuota := milliCPUToQuota(cpuLimit.MilliValue(), cpuPeriod)
 		lc.Resources.CpuQuota = cpuQuota
 		lc.Resources.CpuPeriod = cpuPeriod
 	}

pkg/kubelet/kuberuntime/kuberuntime_manager.go

@@ -98,6 +98,9 @@ type kubeGenericRuntimeManager struct {
 	// If true, enforce container cpu limits with CFS quota support
 	cpuCFSQuota bool
 
+	// CPUCFSQuotaPeriod sets the CPU CFS quota period value, cpu.cfs_period_us, defaults to 100ms
+	cpuCFSQuotaPeriod metav1.Duration
+
 	// wrapped image puller.
 	imagePuller images.ImageManager
 
@@ -146,6 +149,7 @@ func NewKubeGenericRuntimeManager(
 	imagePullQPS float32,
 	imagePullBurst int,
 	cpuCFSQuota bool,
+	cpuCFSQuotaPeriod metav1.Duration,
 	runtimeService internalapi.RuntimeService,
 	imageService internalapi.ImageManagerService,
 	internalLifecycle cm.InternalContainerLifecycle,
@@ -154,6 +158,7 @@ func NewKubeGenericRuntimeManager(
 	kubeRuntimeManager := &kubeGenericRuntimeManager{
 		recorder:            recorder,
 		cpuCFSQuota:         cpuCFSQuota,
+		cpuCFSQuotaPeriod:   cpuCFSQuotaPeriod,
 		seccompProfileRoot:  seccompProfileRoot,
 		livenessManager:     livenessManager,
 		containerRefManager: containerRefManager,