Commit f7ad8286 authored by Erik Wilson's avatar Erik Wilson

Update vendor v1.14.3-k3s.1

parent f078d799
...@@ -63,7 +63,7 @@ import: ...@@ -63,7 +63,7 @@ import:
- package: github.com/coreos/go-iptables - package: github.com/coreos/go-iptables
version: 47f22b0dd3355c0ba570ba12b0b8a36bf214c04b version: 47f22b0dd3355c0ba570ba12b0b8a36bf214c04b
- package: github.com/coreos/go-semver - package: github.com/coreos/go-semver
version: v0.2.0-9-ge214231b295a8e version: v0.3.0
- package: github.com/coreos/go-systemd - package: github.com/coreos/go-systemd
version: 48702e0da86bd25e76cfef347e2adeb434a0d0a6 version: 48702e0da86bd25e76cfef347e2adeb434a0d0a6
- package: github.com/coreos/pkg - package: github.com/coreos/pkg
...@@ -95,7 +95,7 @@ import: ...@@ -95,7 +95,7 @@ import:
- package: github.com/euank/go-kmsg-parser - package: github.com/euank/go-kmsg-parser
version: v2.0.0 version: v2.0.0
- package: github.com/evanphx/json-patch - package: github.com/evanphx/json-patch
version: v4.1.0-19-g5858425f75500d version: v4.2.0
- package: github.com/exponent-io/jsonpath - package: github.com/exponent-io/jsonpath
version: d6023ce2651d8eafb5c75bb0c7167536102ec9f5 version: d6023ce2651d8eafb5c75bb0c7167536102ec9f5
- package: github.com/fatih/camelcase - package: github.com/fatih/camelcase
...@@ -142,7 +142,7 @@ import: ...@@ -142,7 +142,7 @@ import:
- package: github.com/hashicorp/golang-lru - package: github.com/hashicorp/golang-lru
version: v0.5.0 version: v0.5.0
- package: github.com/ibuildthecloud/kvsql - package: github.com/ibuildthecloud/kvsql
version: d37dd2b0829b44a4964e48c9396e14b0536fefb6 version: 1afc2d8ad7d7e263c1971b05cb37e83aa5562561
repo: https://github.com/erikwilson/rancher-kvsql.git repo: https://github.com/erikwilson/rancher-kvsql.git
- package: github.com/imdario/mergo - package: github.com/imdario/mergo
version: v0.3.5 version: v0.3.5
...@@ -310,7 +310,7 @@ import: ...@@ -310,7 +310,7 @@ import:
- package: k8s.io/klog - package: k8s.io/klog
version: v0.2.0-14-g8e90cee79f8237 version: v0.2.0-14-g8e90cee79f8237
- package: k8s.io/kubernetes - package: k8s.io/kubernetes
version: v1.14.1-k3s.4 version: v1.14.3-k3s.1
repo: https://github.com/rancher/k3s.git repo: https://github.com/rancher/k3s.git
transitive: true transitive: true
staging: true staging: true
......
...@@ -9,7 +9,7 @@ package=github.com/opencontainers/runc/libcontainer/nsenter ...@@ -9,7 +9,7 @@ package=github.com/opencontainers/runc/libcontainer/nsenter
package=github.com/opencontainers/runc/libcontainer/specconv package=github.com/opencontainers/runc/libcontainer/specconv
package=github.com/opencontainers/runc/contrib/cmd/recvtty package=github.com/opencontainers/runc/contrib/cmd/recvtty
k8s.io/kubernetes v1.14.1-k3s.4 https://github.com/rancher/k3s.git transitive=true,staging=true k8s.io/kubernetes v1.14.3-k3s.1 https://github.com/rancher/k3s.git transitive=true,staging=true
github.com/rancher/wrangler 4202dbfa88013c19238bb004d82e013f0593493d github.com/rancher/wrangler 4202dbfa88013c19238bb004d82e013f0593493d
github.com/rancher/wrangler-api efe26ac6a9d720e1bfa5a8cc5f8dce5ad598ce26 github.com/rancher/wrangler-api efe26ac6a9d720e1bfa5a8cc5f8dce5ad598ce26
......
...@@ -268,6 +268,7 @@ func NewCacherFromConfig(config Config) *Cacher { ...@@ -268,6 +268,7 @@ func NewCacherFromConfig(config Config) *Cacher {
cacher.stopWg.Add(1) cacher.stopWg.Add(1)
go func() { go func() {
defer cacher.stopWg.Done() defer cacher.stopWg.Done()
defer cacher.terminateAllWatchers()
wait.Until( wait.Until(
func() { func() {
if !cacher.isStopped() { if !cacher.isStopped() {
......
...@@ -10,6 +10,7 @@ go_test( ...@@ -10,6 +10,7 @@ go_test(
name = "go_default_test", name = "go_default_test",
srcs = [ srcs = [
"compact_test.go", "compact_test.go",
"event_test.go",
"lease_manager_test.go", "lease_manager_test.go",
"store_test.go", "store_test.go",
"watcher_test.go", "watcher_test.go",
...@@ -36,7 +37,10 @@ go_test( ...@@ -36,7 +37,10 @@ go_test(
"//vendor/github.com/coreos/etcd/clientv3:go_default_library", "//vendor/github.com/coreos/etcd/clientv3:go_default_library",
"//vendor/github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes:go_default_library", "//vendor/github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes:go_default_library",
"//vendor/github.com/coreos/etcd/integration:go_default_library", "//vendor/github.com/coreos/etcd/integration:go_default_library",
"//vendor/github.com/coreos/etcd/mvcc/mvccpb:go_default_library",
"//vendor/github.com/coreos/pkg/capnslog:go_default_library", "//vendor/github.com/coreos/pkg/capnslog:go_default_library",
"//vendor/github.com/stretchr/testify/assert:go_default_library",
"//vendor/github.com/stretchr/testify/require:go_default_library",
], ],
) )
......
...@@ -17,6 +17,7 @@ limitations under the License. ...@@ -17,6 +17,7 @@ limitations under the License.
package etcd3 package etcd3
import ( import (
"fmt"
"github.com/coreos/etcd/clientv3" "github.com/coreos/etcd/clientv3"
"github.com/coreos/etcd/mvcc/mvccpb" "github.com/coreos/etcd/mvcc/mvccpb"
) )
...@@ -42,7 +43,12 @@ func parseKV(kv *mvccpb.KeyValue) *event { ...@@ -42,7 +43,12 @@ func parseKV(kv *mvccpb.KeyValue) *event {
} }
} }
func parseEvent(e *clientv3.Event) *event { func parseEvent(e *clientv3.Event) (*event, error) {
if !e.IsCreate() && e.PrevKv == nil {
// If the previous value is nil, error. One example of how this is possible is if the previous value has been compacted already.
return nil, fmt.Errorf("etcd event received with PrevKv=nil (key=%q, modRevision=%d, type=%s)", string(e.Kv.Key), e.Kv.ModRevision, e.Type.String())
}
ret := &event{ ret := &event{
key: string(e.Kv.Key), key: string(e.Kv.Key),
value: e.Kv.Value, value: e.Kv.Value,
...@@ -53,5 +59,5 @@ func parseEvent(e *clientv3.Event) *event { ...@@ -53,5 +59,5 @@ func parseEvent(e *clientv3.Event) *event {
if e.PrevKv != nil { if e.PrevKv != nil {
ret.prevValue = e.PrevKv.Value ret.prevValue = e.PrevKv.Value
} }
return ret return ret, nil
} }
...@@ -288,19 +288,20 @@ func (s *store) GuaranteedUpdate( ...@@ -288,19 +288,20 @@ func (s *store) GuaranteedUpdate(
ret, ttl, err := s.updateState(origState, tryUpdate) ret, ttl, err := s.updateState(origState, tryUpdate)
if err != nil { if err != nil {
// It's possible we were working with stale data // If our data is already up to date, return the error
if mustCheckData && apierrors.IsConflict(err) { if !mustCheckData {
// Actually fetch return err
origState, err = getCurrentState()
if err != nil {
return err
}
mustCheckData = false
// Retry
continue
} }
return err // It's possible we were working with stale data
// Actually fetch
origState, err = getCurrentState()
if err != nil {
return err
}
mustCheckData = false
// Retry
continue
} }
data, err := runtime.Encode(s.codec, ret) data, err := runtime.Encode(s.codec, ret)
......
...@@ -210,7 +210,13 @@ func (wc *watchChan) startWatching(watchClosedCh chan struct{}) { ...@@ -210,7 +210,13 @@ func (wc *watchChan) startWatching(watchClosedCh chan struct{}) {
return return
} }
for _, e := range wres.Events { for _, e := range wres.Events {
wc.sendEvent(parseEvent(e)) parsedEvent, err := parseEvent(e)
if err != nil {
klog.Errorf("watch chan error: %v", err)
wc.sendError(err)
return
}
wc.sendEvent(parsedEvent)
} }
} }
// When we come to this point, it's only possible that client side ends the watch. // When we come to this point, it's only possible that client side ends the watch.
......
...@@ -171,6 +171,7 @@ func restConfigFromKubeconfig(configAuthInfo *clientcmdapi.AuthInfo) (*rest.Conf ...@@ -171,6 +171,7 @@ func restConfigFromKubeconfig(configAuthInfo *clientcmdapi.AuthInfo) (*rest.Conf
// blindly overwrite existing values based on precedence // blindly overwrite existing values based on precedence
if len(configAuthInfo.Token) > 0 { if len(configAuthInfo.Token) > 0 {
config.BearerToken = configAuthInfo.Token config.BearerToken = configAuthInfo.Token
config.BearerTokenFile = configAuthInfo.TokenFile
} else if len(configAuthInfo.TokenFile) > 0 { } else if len(configAuthInfo.TokenFile) > 0 {
tokenBytes, err := ioutil.ReadFile(configAuthInfo.TokenFile) tokenBytes, err := ioutil.ReadFile(configAuthInfo.TokenFile)
if err != nil { if err != nil {
......
...@@ -172,7 +172,7 @@ func (d *CachedDiscoveryClient) getCachedFile(filename string) ([]byte, error) { ...@@ -172,7 +172,7 @@ func (d *CachedDiscoveryClient) getCachedFile(filename string) ([]byte, error) {
} }
func (d *CachedDiscoveryClient) writeCachedFile(filename string, obj runtime.Object) error { func (d *CachedDiscoveryClient) writeCachedFile(filename string, obj runtime.Object) error {
if err := os.MkdirAll(filepath.Dir(filename), 0755); err != nil { if err := os.MkdirAll(filepath.Dir(filename), 0750); err != nil {
return err return err
} }
...@@ -191,7 +191,7 @@ func (d *CachedDiscoveryClient) writeCachedFile(filename string, obj runtime.Obj ...@@ -191,7 +191,7 @@ func (d *CachedDiscoveryClient) writeCachedFile(filename string, obj runtime.Obj
return err return err
} }
err = os.Chmod(f.Name(), 0755) err = os.Chmod(f.Name(), 0660)
if err != nil { if err != nil {
return err return err
} }
......
...@@ -18,6 +18,7 @@ package disk ...@@ -18,6 +18,7 @@ package disk
import ( import (
"net/http" "net/http"
"os"
"path/filepath" "path/filepath"
"github.com/gregjones/httpcache" "github.com/gregjones/httpcache"
...@@ -35,6 +36,8 @@ type cacheRoundTripper struct { ...@@ -35,6 +36,8 @@ type cacheRoundTripper struct {
// corresponding requests. // corresponding requests.
func newCacheRoundTripper(cacheDir string, rt http.RoundTripper) http.RoundTripper { func newCacheRoundTripper(cacheDir string, rt http.RoundTripper) http.RoundTripper {
d := diskv.New(diskv.Options{ d := diskv.New(diskv.Options{
PathPerm: os.FileMode(0750),
FilePerm: os.FileMode(0660),
BasePath: cacheDir, BasePath: cacheDir,
TempDir: filepath.Join(cacheDir, ".diskv-temp"), TempDir: filepath.Join(cacheDir, ".diskv-temp"),
}) })
......
...@@ -17,6 +17,7 @@ limitations under the License. ...@@ -17,6 +17,7 @@ limitations under the License.
package dynamic package dynamic
import ( import (
"fmt"
"io" "io"
"k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/api/meta"
...@@ -94,6 +95,9 @@ func (c *dynamicResourceClient) Create(obj *unstructured.Unstructured, opts meta ...@@ -94,6 +95,9 @@ func (c *dynamicResourceClient) Create(obj *unstructured.Unstructured, opts meta
return nil, err return nil, err
} }
name = accessor.GetName() name = accessor.GetName()
if len(name) == 0 {
return nil, fmt.Errorf("name is required")
}
} }
result := c.client.client. result := c.client.client.
...@@ -122,6 +126,10 @@ func (c *dynamicResourceClient) Update(obj *unstructured.Unstructured, opts meta ...@@ -122,6 +126,10 @@ func (c *dynamicResourceClient) Update(obj *unstructured.Unstructured, opts meta
if err != nil { if err != nil {
return nil, err return nil, err
} }
name := accessor.GetName()
if len(name) == 0 {
return nil, fmt.Errorf("name is required")
}
outBytes, err := runtime.Encode(unstructured.UnstructuredJSONScheme, obj) outBytes, err := runtime.Encode(unstructured.UnstructuredJSONScheme, obj)
if err != nil { if err != nil {
return nil, err return nil, err
...@@ -129,7 +137,7 @@ func (c *dynamicResourceClient) Update(obj *unstructured.Unstructured, opts meta ...@@ -129,7 +137,7 @@ func (c *dynamicResourceClient) Update(obj *unstructured.Unstructured, opts meta
result := c.client.client. result := c.client.client.
Put(). Put().
AbsPath(append(c.makeURLSegments(accessor.GetName()), subresources...)...). AbsPath(append(c.makeURLSegments(name), subresources...)...).
Body(outBytes). Body(outBytes).
SpecificallyVersionedParams(&opts, dynamicParameterCodec, versionV1). SpecificallyVersionedParams(&opts, dynamicParameterCodec, versionV1).
Do() Do()
...@@ -153,6 +161,10 @@ func (c *dynamicResourceClient) UpdateStatus(obj *unstructured.Unstructured, opt ...@@ -153,6 +161,10 @@ func (c *dynamicResourceClient) UpdateStatus(obj *unstructured.Unstructured, opt
if err != nil { if err != nil {
return nil, err return nil, err
} }
name := accessor.GetName()
if len(name) == 0 {
return nil, fmt.Errorf("name is required")
}
outBytes, err := runtime.Encode(unstructured.UnstructuredJSONScheme, obj) outBytes, err := runtime.Encode(unstructured.UnstructuredJSONScheme, obj)
if err != nil { if err != nil {
...@@ -161,7 +173,7 @@ func (c *dynamicResourceClient) UpdateStatus(obj *unstructured.Unstructured, opt ...@@ -161,7 +173,7 @@ func (c *dynamicResourceClient) UpdateStatus(obj *unstructured.Unstructured, opt
result := c.client.client. result := c.client.client.
Put(). Put().
AbsPath(append(c.makeURLSegments(accessor.GetName()), "status")...). AbsPath(append(c.makeURLSegments(name), "status")...).
Body(outBytes). Body(outBytes).
SpecificallyVersionedParams(&opts, dynamicParameterCodec, versionV1). SpecificallyVersionedParams(&opts, dynamicParameterCodec, versionV1).
Do() Do()
...@@ -181,6 +193,9 @@ func (c *dynamicResourceClient) UpdateStatus(obj *unstructured.Unstructured, opt ...@@ -181,6 +193,9 @@ func (c *dynamicResourceClient) UpdateStatus(obj *unstructured.Unstructured, opt
} }
func (c *dynamicResourceClient) Delete(name string, opts *metav1.DeleteOptions, subresources ...string) error { func (c *dynamicResourceClient) Delete(name string, opts *metav1.DeleteOptions, subresources ...string) error {
if len(name) == 0 {
return fmt.Errorf("name is required")
}
if opts == nil { if opts == nil {
opts = &metav1.DeleteOptions{} opts = &metav1.DeleteOptions{}
} }
...@@ -216,6 +231,9 @@ func (c *dynamicResourceClient) DeleteCollection(opts *metav1.DeleteOptions, lis ...@@ -216,6 +231,9 @@ func (c *dynamicResourceClient) DeleteCollection(opts *metav1.DeleteOptions, lis
} }
func (c *dynamicResourceClient) Get(name string, opts metav1.GetOptions, subresources ...string) (*unstructured.Unstructured, error) { func (c *dynamicResourceClient) Get(name string, opts metav1.GetOptions, subresources ...string) (*unstructured.Unstructured, error) {
if len(name) == 0 {
return nil, fmt.Errorf("name is required")
}
result := c.client.client.Get().AbsPath(append(c.makeURLSegments(name), subresources...)...).SpecificallyVersionedParams(&opts, dynamicParameterCodec, versionV1).Do() result := c.client.client.Get().AbsPath(append(c.makeURLSegments(name), subresources...)...).SpecificallyVersionedParams(&opts, dynamicParameterCodec, versionV1).Do()
if err := result.Error(); err != nil { if err := result.Error(); err != nil {
return nil, err return nil, err
...@@ -284,6 +302,9 @@ func (c *dynamicResourceClient) Watch(opts metav1.ListOptions) (watch.Interface, ...@@ -284,6 +302,9 @@ func (c *dynamicResourceClient) Watch(opts metav1.ListOptions) (watch.Interface,
} }
func (c *dynamicResourceClient) Patch(name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (*unstructured.Unstructured, error) { func (c *dynamicResourceClient) Patch(name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (*unstructured.Unstructured, error) {
if len(name) == 0 {
return nil, fmt.Errorf("name is required")
}
result := c.client.client. result := c.client.client.
Patch(pt). Patch(pt).
AbsPath(append(c.makeURLSegments(name), subresources...)...). AbsPath(append(c.makeURLSegments(name), subresources...)...).
......
...@@ -3,8 +3,8 @@ package version ...@@ -3,8 +3,8 @@ package version
var ( var (
gitMajor = "1" gitMajor = "1"
gitMinor = "14" gitMinor = "14"
gitVersion = "v1.14.1-k3s.4" gitVersion = "v1.14.3-k3s.1"
gitCommit = "52f3b42401c93c36467f1fd6d294a3aba26c7def" gitCommit = "8343999292c55c807be4406fcaa9f047e8751ffd"
gitTreeState = "clean" gitTreeState = "clean"
buildDate = "2019-04-15T22:13+00:00Z" buildDate = "2019-06-12T04:56+00:00Z"
) )
...@@ -73,9 +73,10 @@ func (c *Config) TransportConfig() (*transport.Config, error) { ...@@ -73,9 +73,10 @@ func (c *Config) TransportConfig() (*transport.Config, error) {
KeyFile: c.KeyFile, KeyFile: c.KeyFile,
KeyData: c.KeyData, KeyData: c.KeyData,
}, },
Username: c.Username, Username: c.Username,
Password: c.Password, Password: c.Password,
BearerToken: c.BearerToken, BearerToken: c.BearerToken,
BearerTokenFile: c.BearerTokenFile,
Impersonate: transport.ImpersonationConfig{ Impersonate: transport.ImpersonationConfig{
UserName: c.Impersonate.UserName, UserName: c.Impersonate.UserName,
Groups: c.Impersonate.Groups, Groups: c.Impersonate.Groups,
......
...@@ -228,6 +228,7 @@ func (config *DirectClientConfig) getUserIdentificationPartialConfig(configAuthI ...@@ -228,6 +228,7 @@ func (config *DirectClientConfig) getUserIdentificationPartialConfig(configAuthI
// blindly overwrite existing values based on precedence // blindly overwrite existing values based on precedence
if len(configAuthInfo.Token) > 0 { if len(configAuthInfo.Token) > 0 {
mergedConfig.BearerToken = configAuthInfo.Token mergedConfig.BearerToken = configAuthInfo.Token
mergedConfig.BearerTokenFile = configAuthInfo.TokenFile
} else if len(configAuthInfo.TokenFile) > 0 { } else if len(configAuthInfo.TokenFile) > 0 {
tokenBytes, err := ioutil.ReadFile(configAuthInfo.TokenFile) tokenBytes, err := ioutil.ReadFile(configAuthInfo.TokenFile)
if err != nil { if err != nil {
...@@ -499,8 +500,9 @@ func (config *inClusterClientConfig) ClientConfig() (*restclient.Config, error) ...@@ -499,8 +500,9 @@ func (config *inClusterClientConfig) ClientConfig() (*restclient.Config, error)
if server := config.overrides.ClusterInfo.Server; len(server) > 0 { if server := config.overrides.ClusterInfo.Server; len(server) > 0 {
icc.Host = server icc.Host = server
} }
if token := config.overrides.AuthInfo.Token; len(token) > 0 { if len(config.overrides.AuthInfo.Token) > 0 || len(config.overrides.AuthInfo.TokenFile) > 0 {
icc.BearerToken = token icc.BearerToken = config.overrides.AuthInfo.Token
icc.BearerTokenFile = config.overrides.AuthInfo.TokenFile
} }
if certificateAuthorityFile := config.overrides.ClusterInfo.CertificateAuthority; len(certificateAuthorityFile) > 0 { if certificateAuthorityFile := config.overrides.ClusterInfo.CertificateAuthority; len(certificateAuthorityFile) > 0 {
icc.TLSClientConfig.CAFile = certificateAuthorityFile icc.TLSClientConfig.CAFile = certificateAuthorityFile
......
...@@ -51,6 +51,9 @@ type KubeProxyIPVSConfiguration struct { ...@@ -51,6 +51,9 @@ type KubeProxyIPVSConfiguration struct {
// excludeCIDRs is a list of CIDR's which the ipvs proxier should not touch // excludeCIDRs is a list of CIDR's which the ipvs proxier should not touch
// when cleaning up ipvs services. // when cleaning up ipvs services.
ExcludeCIDRs []string `json:"excludeCIDRs"` ExcludeCIDRs []string `json:"excludeCIDRs"`
// strict ARP configure arp_ignore and arp_announce to avoid answering ARP queries
// from kube-ipvs0 interface
StrictARP bool `json:"strictARP"`
} }
// KubeProxyConntrackConfiguration contains conntrack settings for // KubeProxyConntrackConfiguration contains conntrack settings for
......
...@@ -92,6 +92,7 @@ const ( ...@@ -92,6 +92,7 @@ const (
// proxyRun defines the interface to run a specified ProxyServer // proxyRun defines the interface to run a specified ProxyServer
type proxyRun interface { type proxyRun interface {
Run() error Run() error
CleanupAndExit() error
} }
// Options contains everything necessary to create and run a proxy server. // Options contains everything necessary to create and run a proxy server.
...@@ -166,6 +167,7 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) { ...@@ -166,6 +167,7 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) {
fs.DurationVar(&o.config.IPVS.SyncPeriod.Duration, "ipvs-sync-period", o.config.IPVS.SyncPeriod.Duration, "The maximum interval of how often ipvs rules are refreshed (e.g. '5s', '1m', '2h22m'). Must be greater than 0.") fs.DurationVar(&o.config.IPVS.SyncPeriod.Duration, "ipvs-sync-period", o.config.IPVS.SyncPeriod.Duration, "The maximum interval of how often ipvs rules are refreshed (e.g. '5s', '1m', '2h22m'). Must be greater than 0.")
fs.DurationVar(&o.config.IPVS.MinSyncPeriod.Duration, "ipvs-min-sync-period", o.config.IPVS.MinSyncPeriod.Duration, "The minimum interval of how often the ipvs rules can be refreshed as endpoints and services change (e.g. '5s', '1m', '2h22m').") fs.DurationVar(&o.config.IPVS.MinSyncPeriod.Duration, "ipvs-min-sync-period", o.config.IPVS.MinSyncPeriod.Duration, "The minimum interval of how often the ipvs rules can be refreshed as endpoints and services change (e.g. '5s', '1m', '2h22m').")
fs.StringSliceVar(&o.config.IPVS.ExcludeCIDRs, "ipvs-exclude-cidrs", o.config.IPVS.ExcludeCIDRs, "A comma-separated list of CIDR's which the ipvs proxier should not touch when cleaning up IPVS rules.") fs.StringSliceVar(&o.config.IPVS.ExcludeCIDRs, "ipvs-exclude-cidrs", o.config.IPVS.ExcludeCIDRs, "A comma-separated list of CIDR's which the ipvs proxier should not touch when cleaning up IPVS rules.")
fs.BoolVar(&o.config.IPVS.StrictARP, "ipvs-strict-arp", o.config.IPVS.StrictARP, "Enable strict ARP by setting arp_ignore to 1 and arp_announce to 2")
fs.DurationVar(&o.config.ConfigSyncPeriod.Duration, "config-sync-period", o.config.ConfigSyncPeriod.Duration, "How often configuration from the apiserver is refreshed. Must be greater than 0.") fs.DurationVar(&o.config.ConfigSyncPeriod.Duration, "config-sync-period", o.config.ConfigSyncPeriod.Duration, "How often configuration from the apiserver is refreshed. Must be greater than 0.")
fs.BoolVar(&o.config.IPTables.MasqueradeAll, "masquerade-all", o.config.IPTables.MasqueradeAll, "If using the pure iptables proxy, SNAT all traffic sent via Service cluster IPs (this not commonly needed)") fs.BoolVar(&o.config.IPTables.MasqueradeAll, "masquerade-all", o.config.IPTables.MasqueradeAll, "If using the pure iptables proxy, SNAT all traffic sent via Service cluster IPs (this not commonly needed)")
fs.StringVar(&o.config.ClusterCIDR, "cluster-cidr", o.config.ClusterCIDR, "The CIDR range of pods in the cluster. When configured, traffic sent to a Service cluster IP from outside this range will be masqueraded and traffic sent from pods to an external LoadBalancer IP will be directed to the respective cluster IP instead") fs.StringVar(&o.config.ClusterCIDR, "cluster-cidr", o.config.ClusterCIDR, "The CIDR range of pods in the cluster. When configured, traffic sent to a Service cluster IP from outside this range will be masqueraded and traffic sent from pods to an external LoadBalancer IP will be directed to the respective cluster IP instead")
...@@ -305,6 +307,11 @@ func (o *Options) Run() error { ...@@ -305,6 +307,11 @@ func (o *Options) Run() error {
if err != nil { if err != nil {
return err return err
} }
if o.CleanupAndExit {
return proxyServer.CleanupAndExit()
}
o.proxyServer = proxyServer o.proxyServer = proxyServer
return o.runLoop() return o.runLoop()
} }
...@@ -498,7 +505,6 @@ type ProxyServer struct { ...@@ -498,7 +505,6 @@ type ProxyServer struct {
Conntracker Conntracker // if nil, ignored Conntracker Conntracker // if nil, ignored
ProxyMode string ProxyMode string
NodeRef *v1.ObjectReference NodeRef *v1.ObjectReference
CleanupAndExit bool
CleanupIPVS bool CleanupIPVS bool
MetricsBindAddress string MetricsBindAddress string
EnableProfiling bool EnableProfiling bool
...@@ -550,19 +556,10 @@ func createClients(config componentbaseconfig.ClientConnectionConfiguration, mas ...@@ -550,19 +556,10 @@ func createClients(config componentbaseconfig.ClientConnectionConfiguration, mas
} }
// Run runs the specified ProxyServer. This should never exit (unless CleanupAndExit is set). // Run runs the specified ProxyServer. This should never exit (unless CleanupAndExit is set).
// TODO: At the moment, Run() cannot return a nil error, otherwise it's caller will never exit. Update callers of Run to handle nil errors.
func (s *ProxyServer) Run() error { func (s *ProxyServer) Run() error {
// To help debugging, immediately log version // To help debugging, immediately log version
klog.Infof("Version: %+v", version.Get()) klog.Infof("Version: %+v", version.Get())
// remove iptables rules and exit
if s.CleanupAndExit {
encounteredError := userspace.CleanupLeftovers(s.IptInterface)
encounteredError = iptables.CleanupLeftovers(s.IptInterface) || encounteredError
encounteredError = ipvs.CleanupLeftovers(s.IpvsInterface, s.IptInterface, s.IpsetInterface, s.CleanupIPVS) || encounteredError
if encounteredError {
return errors.New("encountered an error while tearing down rules.")
}
return nil
}
// TODO(vmarmol): Use container config for this. // TODO(vmarmol): Use container config for this.
var oomAdjuster *oom.OOMAdjuster var oomAdjuster *oom.OOMAdjuster
...@@ -708,3 +705,15 @@ func getConntrackMax(config kubeproxyconfig.KubeProxyConntrackConfiguration) (in ...@@ -708,3 +705,15 @@ func getConntrackMax(config kubeproxyconfig.KubeProxyConntrackConfiguration) (in
} }
return 0, nil return 0, nil
} }
// CleanupAndExit remove iptables rules and exit if success return nil
func (s *ProxyServer) CleanupAndExit() error {
encounteredError := userspace.CleanupLeftovers(s.IptInterface)
encounteredError = iptables.CleanupLeftovers(s.IptInterface) || encounteredError
encounteredError = ipvs.CleanupLeftovers(s.IpvsInterface, s.IptInterface, s.IpsetInterface, s.CleanupIPVS) || encounteredError
if encounteredError {
return errors.New("encountered an error while tearing down rules")
}
return nil
}
...@@ -104,7 +104,6 @@ func newProxyServer( ...@@ -104,7 +104,6 @@ func newProxyServer(
IptInterface: iptInterface, IptInterface: iptInterface,
IpvsInterface: ipvsInterface, IpvsInterface: ipvsInterface,
IpsetInterface: ipsetInterface, IpsetInterface: ipsetInterface,
CleanupAndExit: cleanupAndExit,
}, nil }, nil
} }
...@@ -185,6 +184,7 @@ func newProxyServer( ...@@ -185,6 +184,7 @@ func newProxyServer(
config.IPVS.SyncPeriod.Duration, config.IPVS.SyncPeriod.Duration,
config.IPVS.MinSyncPeriod.Duration, config.IPVS.MinSyncPeriod.Duration,
config.IPVS.ExcludeCIDRs, config.IPVS.ExcludeCIDRs,
config.IPVS.StrictARP,
config.IPTables.MasqueradeAll, config.IPTables.MasqueradeAll,
int(*config.IPTables.MasqueradeBit), int(*config.IPTables.MasqueradeBit),
config.ClusterCIDR, config.ClusterCIDR,
......
...@@ -24,6 +24,8 @@ import ( ...@@ -24,6 +24,8 @@ import (
"errors" "errors"
"fmt" "fmt"
"net" "net"
// Enable pprof HTTP handlers.
_ "net/http/pprof" _ "net/http/pprof"
"k8s.io/api/core/v1" "k8s.io/api/core/v1"
...@@ -63,7 +65,7 @@ func newProxyServer(config *proxyconfigapi.KubeProxyConfiguration, cleanupAndExi ...@@ -63,7 +65,7 @@ func newProxyServer(config *proxyconfigapi.KubeProxyConfiguration, cleanupAndExi
// We omit creation of pretty much everything if we run in cleanup mode // We omit creation of pretty much everything if we run in cleanup mode
if cleanupAndExit { if cleanupAndExit {
return &ProxyServer{CleanupAndExit: cleanupAndExit}, nil return &ProxyServer{}, nil
} }
client, eventClient, err := createClients(config.ClientConnection, master) client, eventClient, err := createClients(config.ClientConnection, master)
......
...@@ -133,6 +133,7 @@ go_library( ...@@ -133,6 +133,7 @@ go_library(
"//staging/src/k8s.io/client-go/tools/record:go_default_library", "//staging/src/k8s.io/client-go/tools/record:go_default_library",
"//staging/src/k8s.io/client-go/util/cert:go_default_library", "//staging/src/k8s.io/client-go/util/cert:go_default_library",
"//staging/src/k8s.io/client-go/util/certificate:go_default_library", "//staging/src/k8s.io/client-go/util/certificate:go_default_library",
"//staging/src/k8s.io/client-go/util/connrotation:go_default_library",
"//staging/src/k8s.io/client-go/util/keyutil:go_default_library", "//staging/src/k8s.io/client-go/util/keyutil:go_default_library",
"//staging/src/k8s.io/cloud-provider:go_default_library", "//staging/src/k8s.io/cloud-provider:go_default_library",
"//staging/src/k8s.io/component-base/cli/flag:go_default_library", "//staging/src/k8s.io/component-base/cli/flag:go_default_library",
......
...@@ -53,6 +53,7 @@ import ( ...@@ -53,6 +53,7 @@ import (
"k8s.io/client-go/tools/record" "k8s.io/client-go/tools/record"
certutil "k8s.io/client-go/util/cert" certutil "k8s.io/client-go/util/cert"
"k8s.io/client-go/util/certificate" "k8s.io/client-go/util/certificate"
"k8s.io/client-go/util/connrotation"
"k8s.io/client-go/util/keyutil" "k8s.io/client-go/util/keyutil"
cliflag "k8s.io/component-base/cli/flag" cliflag "k8s.io/component-base/cli/flag"
kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1"
...@@ -526,6 +527,9 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies, stopCh <-chan ...@@ -526,6 +527,9 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies, stopCh <-chan
if err != nil { if err != nil {
return err return err
} }
if closeAllConns == nil {
return errors.New("closeAllConns must be a valid function other than nil")
}
kubeDeps.OnHeartbeatFailure = closeAllConns kubeDeps.OnHeartbeatFailure = closeAllConns
kubeDeps.KubeClient, err = clientset.NewForConfig(clientConfig) kubeDeps.KubeClient, err = clientset.NewForConfig(clientConfig)
...@@ -761,8 +765,21 @@ func buildKubeletClientConfig(s *options.KubeletServer, nodeName types.NodeName) ...@@ -761,8 +765,21 @@ func buildKubeletClientConfig(s *options.KubeletServer, nodeName types.NodeName)
} }
kubeClientConfigOverrides(s, clientConfig) kubeClientConfigOverrides(s, clientConfig)
closeAllConns, err := updateDialer(clientConfig)
if err != nil {
return nil, nil, err
}
return clientConfig, closeAllConns, nil
}
return clientConfig, nil, nil // updateDialer instruments a restconfig with a dial. the returned function allows forcefully closing all active connections.
func updateDialer(clientConfig *restclient.Config) (func(), error) {
if clientConfig.Transport != nil || clientConfig.Dial != nil {
return nil, fmt.Errorf("there is already a transport or dialer configured")
}
d := connrotation.NewDialer((&net.Dialer{Timeout: 30 * time.Second, KeepAlive: 30 * time.Second}).DialContext)
clientConfig.Dial = d.DialContext
return d.CloseAll, nil
} }
// buildClientCertificateManager creates a certificate manager that will use certConfig to request a client certificate // buildClientCertificateManager creates a certificate manager that will use certConfig to request a client certificate
......
...@@ -964,10 +964,10 @@ func (dsc *DaemonSetsController) manage(ds *apps.DaemonSet, hash string) error { ...@@ -964,10 +964,10 @@ func (dsc *DaemonSetsController) manage(ds *apps.DaemonSet, hash string) error {
failedPodsObserved += failedPodsObservedOnNode failedPodsObserved += failedPodsObservedOnNode
} }
// Remove pods assigned to not existing nodes when daemonset pods are scheduled by default scheduler. // Remove unscheduled pods assigned to not existing nodes when daemonset pods are scheduled by scheduler.
// If node doesn't exist then pods are never scheduled and can't be deleted by PodGCController. // If node doesn't exist then pods are never scheduled and can't be deleted by PodGCController.
if utilfeature.DefaultFeatureGate.Enabled(features.ScheduleDaemonSetPods) { if utilfeature.DefaultFeatureGate.Enabled(features.ScheduleDaemonSetPods) {
podsToDelete = append(podsToDelete, getPodsWithoutNode(nodeList, nodeToDaemonPods)...) podsToDelete = append(podsToDelete, getUnscheduledPodsWithoutNode(nodeList, nodeToDaemonPods)...)
} }
// Label new pods using the hash label value of the current history when creating them // Label new pods using the hash label value of the current history when creating them
...@@ -1531,8 +1531,9 @@ func failedPodsBackoffKey(ds *apps.DaemonSet, nodeName string) string { ...@@ -1531,8 +1531,9 @@ func failedPodsBackoffKey(ds *apps.DaemonSet, nodeName string) string {
return fmt.Sprintf("%s/%d/%s", ds.UID, ds.Status.ObservedGeneration, nodeName) return fmt.Sprintf("%s/%d/%s", ds.UID, ds.Status.ObservedGeneration, nodeName)
} }
// getPodsWithoutNode returns list of pods assigned to not existing nodes. // getUnscheduledPodsWithoutNode returns list of unscheduled pods assigned to not existing nodes.
func getPodsWithoutNode(runningNodesList []*v1.Node, nodeToDaemonPods map[string][]*v1.Pod) []string { // Returned pods can't be deleted by PodGCController so they should be deleted by DaemonSetController.
func getUnscheduledPodsWithoutNode(runningNodesList []*v1.Node, nodeToDaemonPods map[string][]*v1.Pod) []string {
var results []string var results []string
isNodeRunning := make(map[string]bool) isNodeRunning := make(map[string]bool)
for _, node := range runningNodesList { for _, node := range runningNodesList {
...@@ -1541,7 +1542,9 @@ func getPodsWithoutNode(runningNodesList []*v1.Node, nodeToDaemonPods map[string ...@@ -1541,7 +1542,9 @@ func getPodsWithoutNode(runningNodesList []*v1.Node, nodeToDaemonPods map[string
for n, pods := range nodeToDaemonPods { for n, pods := range nodeToDaemonPods {
if !isNodeRunning[n] { if !isNodeRunning[n] {
for _, pod := range pods { for _, pod := range pods {
results = append(results, pod.Name) if len(pod.Spec.NodeName) == 0 {
results = append(results, pod.Name)
}
} }
} }
} }
......
...@@ -32,6 +32,7 @@ go_test( ...@@ -32,6 +32,7 @@ go_test(
"//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
"//staging/src/k8s.io/cli-runtime/pkg/genericclioptions:go_default_library", "//staging/src/k8s.io/cli-runtime/pkg/genericclioptions:go_default_library",
"//staging/src/k8s.io/client-go/rest/fake:go_default_library", "//staging/src/k8s.io/client-go/rest/fake:go_default_library",
"//vendor/github.com/stretchr/testify/require:go_default_library",
], ],
) )
......
...@@ -418,9 +418,7 @@ func clean(fileName string) string { ...@@ -418,9 +418,7 @@ func clean(fileName string) string {
return path.Clean(string(os.PathSeparator) + fileName) return path.Clean(string(os.PathSeparator) + fileName)
} }
func (o *CopyOptions) untarAll(reader io.Reader, destFile, prefix string) error { func (o *CopyOptions) untarAll(reader io.Reader, destDir, prefix string) error {
entrySeq := -1
// TODO: use compression here? // TODO: use compression here?
tarReader := tar.NewReader(reader) tarReader := tar.NewReader(reader)
for { for {
...@@ -431,52 +429,60 @@ func (o *CopyOptions) untarAll(reader io.Reader, destFile, prefix string) error ...@@ -431,52 +429,60 @@ func (o *CopyOptions) untarAll(reader io.Reader, destFile, prefix string) error
} }
break break
} }
entrySeq++
mode := header.FileInfo().Mode() // All the files will start with the prefix, which is the directory where
// all the files will start with the prefix, which is the directory where
// they were located on the pod, we need to strip down that prefix, but // they were located on the pod, we need to strip down that prefix, but
// if the prefix is missing it means the tar was tempered with // if the prefix is missing it means the tar was tempered with.
// For the case where prefix is empty we need to ensure that the path
// is not absolute, which also indicates the tar file was tempered with.
if !strings.HasPrefix(header.Name, prefix) { if !strings.HasPrefix(header.Name, prefix) {
return fmt.Errorf("tar contents corrupted") return fmt.Errorf("tar contents corrupted")
} }
outFileName := path.Join(destFile, clean(header.Name[len(prefix):]))
baseName := path.Dir(outFileName) // basic file information
mode := header.FileInfo().Mode()
destFileName := path.Join(destDir, header.Name[len(prefix):])
baseName := path.Dir(destFileName)
if err := os.MkdirAll(baseName, 0755); err != nil { if err := os.MkdirAll(baseName, 0755); err != nil {
return err return err
} }
if header.FileInfo().IsDir() { if header.FileInfo().IsDir() {
if err := os.MkdirAll(outFileName, 0755); err != nil { if err := os.MkdirAll(destFileName, 0755); err != nil {
return err return err
} }
continue continue
} }
// handle coping remote file into local directory // We need to ensure that the destination file is always within boundries
if entrySeq == 0 && !header.FileInfo().IsDir() { // of the destination directory. This prevents any kind of path traversal
exists, err := dirExists(outFileName) // from within tar archive.
if err != nil { dir, file := filepath.Split(destFileName)
return err evaledPath, err := filepath.EvalSymlinks(dir)
} if err != nil {
if exists { return err
outFileName = filepath.Join(outFileName, path.Base(clean(header.Name))) }
} // For scrutiny we verify both the actual destination as well as we follow
// all the links that might lead outside of the destination directory.
if !isDestRelative(destDir, destFileName) || !isDestRelative(destDir, filepath.Join(evaledPath, file)) {
fmt.Fprintf(o.IOStreams.ErrOut, "warning: link %q is pointing to %q which is outside target destination, skipping\n", destFileName, header.Linkname)
continue
} }
if mode&os.ModeSymlink != 0 { if mode&os.ModeSymlink != 0 {
linkname := header.Linkname linkname := header.Linkname
// error is returned if linkname can't be made relative to destFile, // We need to ensure that the link destination is always within boundries
// but relative can end up being ../dir that's why we also need to // of the destination directory. This prevents any kind of path traversal
// verify if relative path is the same after Clean-ing // from within tar archive.
relative, err := filepath.Rel(destFile, linkname) if !isDestRelative(destDir, linkJoin(destFileName, linkname)) {
if path.IsAbs(linkname) && (err != nil || relative != stripPathShortcuts(relative)) { fmt.Fprintf(o.IOStreams.ErrOut, "warning: link %q is pointing to %q which is outside target destination, skipping\n", destFileName, header.Linkname)
fmt.Fprintf(o.IOStreams.ErrOut, "warning: link %q is pointing to %q which is outside target destination, skipping\n", outFileName, header.Linkname)
continue continue
} }
if err := os.Symlink(linkname, outFileName); err != nil { if err := os.Symlink(linkname, destFileName); err != nil {
return err return err
} }
} else { } else {
outFile, err := os.Create(outFileName) outFile, err := os.Create(destFileName)
if err != nil { if err != nil {
return err return err
} }
...@@ -490,14 +496,32 @@ func (o *CopyOptions) untarAll(reader io.Reader, destFile, prefix string) error ...@@ -490,14 +496,32 @@ func (o *CopyOptions) untarAll(reader io.Reader, destFile, prefix string) error
} }
} }
if entrySeq == -1 {
//if no file was copied
errInfo := fmt.Sprintf("error: %s no such file or directory", prefix)
return errors.New(errInfo)
}
return nil return nil
} }
// linkJoin joins base and link to get the final path to be created.
// It will consider whether link is an absolute path or not when returning result.
func linkJoin(base, link string) string {
if filepath.IsAbs(link) {
return link
}
return filepath.Join(base, link)
}
// isDestRelative returns true if dest is pointing outside the base directory,
// false otherwise.
func isDestRelative(base, dest string) bool {
fullPath := dest
if !filepath.IsAbs(dest) {
fullPath = filepath.Join(base, dest)
}
relative, err := filepath.Rel(base, fullPath)
if err != nil {
return false
}
return relative == "." || relative == stripPathShortcuts(relative)
}
func getPrefix(file string) string { func getPrefix(file string) string {
// tar strips the leading '/' if it's there, so we will too // tar strips the leading '/' if it's there, so we will too
return strings.TrimLeft(file, "/") return strings.TrimLeft(file, "/")
...@@ -524,15 +548,3 @@ func (o *CopyOptions) execute(options *exec.ExecOptions) error { ...@@ -524,15 +548,3 @@ func (o *CopyOptions) execute(options *exec.ExecOptions) error {
} }
return nil return nil
} }
// dirExists checks if a path exists and is a directory.
func dirExists(path string) (bool, error) {
fi, err := os.Stat(path)
if err == nil && fi.IsDir() {
return true, nil
}
if os.IsNotExist(err) {
return false, nil
}
return false, err
}
...@@ -632,7 +632,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, ...@@ -632,7 +632,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
} }
klet.runtimeService = runtimeService klet.runtimeService = runtimeService
if utilfeature.DefaultFeatureGate.Enabled(features.RuntimeClass) { if utilfeature.DefaultFeatureGate.Enabled(features.RuntimeClass) && kubeDeps.KubeClient != nil {
klet.runtimeClassManager = runtimeclass.NewManager(kubeDeps.KubeClient) klet.runtimeClassManager = runtimeclass.NewManager(kubeDeps.KubeClient)
} }
......
...@@ -89,6 +89,7 @@ go_test( ...@@ -89,6 +89,7 @@ go_test(
"//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
"//vendor/github.com/golang/mock/gomock:go_default_library", "//vendor/github.com/golang/mock/gomock:go_default_library",
"//vendor/github.com/google/cadvisor/fs:go_default_library", "//vendor/github.com/google/cadvisor/fs:go_default_library",
"//vendor/github.com/google/cadvisor/info/v1:go_default_library", "//vendor/github.com/google/cadvisor/info/v1:go_default_library",
......
...@@ -137,6 +137,7 @@ func (p *criStatsProvider) listPodStats(updateCPUNanoCoreUsage bool) ([]statsapi ...@@ -137,6 +137,7 @@ func (p *criStatsProvider) listPodStats(updateCPUNanoCoreUsage bool) ([]statsapi
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to list all pod sandboxes: %v", err) return nil, fmt.Errorf("failed to list all pod sandboxes: %v", err)
} }
podSandboxes = removeTerminatedPods(podSandboxes)
for _, s := range podSandboxes { for _, s := range podSandboxes {
podSandboxMap[s.Id] = s podSandboxMap[s.Id] = s
} }
...@@ -153,7 +154,7 @@ func (p *criStatsProvider) listPodStats(updateCPUNanoCoreUsage bool) ([]statsapi ...@@ -153,7 +154,7 @@ func (p *criStatsProvider) listPodStats(updateCPUNanoCoreUsage bool) ([]statsapi
return nil, fmt.Errorf("failed to list all container stats: %v", err) return nil, fmt.Errorf("failed to list all container stats: %v", err)
} }
containers = removeTerminatedContainer(containers) containers = removeTerminatedContainers(containers)
// Creates container map. // Creates container map.
containerMap := make(map[string]*runtimeapi.Container) containerMap := make(map[string]*runtimeapi.Container)
for _, c := range containers { for _, c := range containers {
...@@ -233,6 +234,7 @@ func (p *criStatsProvider) ListPodCPUAndMemoryStats() ([]statsapi.PodStats, erro ...@@ -233,6 +234,7 @@ func (p *criStatsProvider) ListPodCPUAndMemoryStats() ([]statsapi.PodStats, erro
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to list all pod sandboxes: %v", err) return nil, fmt.Errorf("failed to list all pod sandboxes: %v", err)
} }
podSandboxes = removeTerminatedPods(podSandboxes)
for _, s := range podSandboxes { for _, s := range podSandboxes {
podSandboxMap[s.Id] = s podSandboxMap[s.Id] = s
} }
...@@ -245,7 +247,7 @@ func (p *criStatsProvider) ListPodCPUAndMemoryStats() ([]statsapi.PodStats, erro ...@@ -245,7 +247,7 @@ func (p *criStatsProvider) ListPodCPUAndMemoryStats() ([]statsapi.PodStats, erro
return nil, fmt.Errorf("failed to list all container stats: %v", err) return nil, fmt.Errorf("failed to list all container stats: %v", err)
} }
containers = removeTerminatedContainer(containers) containers = removeTerminatedContainers(containers)
// Creates container map. // Creates container map.
containerMap := make(map[string]*runtimeapi.Container) containerMap := make(map[string]*runtimeapi.Container)
for _, c := range containers { for _, c := range containers {
...@@ -690,9 +692,51 @@ func (p *criStatsProvider) cleanupOutdatedCaches() { ...@@ -690,9 +692,51 @@ func (p *criStatsProvider) cleanupOutdatedCaches() {
} }
} }
// removeTerminatedContainer returns the specified container but with // removeTerminatedPods returns pods with terminated ones removed.
// the stats of the terminated containers removed. // It only removes a terminated pod when there is a running instance
func removeTerminatedContainer(containers []*runtimeapi.Container) []*runtimeapi.Container { // of the pod with the same name and namespace.
// This is needed because:
// 1) PodSandbox may be recreated;
// 2) Pod may be recreated with the same name and namespace.
func removeTerminatedPods(pods []*runtimeapi.PodSandbox) []*runtimeapi.PodSandbox {
podMap := make(map[statsapi.PodReference][]*runtimeapi.PodSandbox)
// Sort order by create time
sort.Slice(pods, func(i, j int) bool {
return pods[i].CreatedAt < pods[j].CreatedAt
})
for _, pod := range pods {
refID := statsapi.PodReference{
Name: pod.GetMetadata().GetName(),
Namespace: pod.GetMetadata().GetNamespace(),
// UID is intentionally left empty.
}
podMap[refID] = append(podMap[refID], pod)
}
result := make([]*runtimeapi.PodSandbox, 0)
for _, refs := range podMap {
if len(refs) == 1 {
result = append(result, refs[0])
continue
}
found := false
for i := 0; i < len(refs); i++ {
if refs[i].State == runtimeapi.PodSandboxState_SANDBOX_READY {
found = true
result = append(result, refs[i])
}
}
if !found {
result = append(result, refs[len(refs)-1])
}
}
return result
}
// removeTerminatedContainers returns containers with terminated ones.
// It only removes a terminated container when there is a running instance
// of the container.
func removeTerminatedContainers(containers []*runtimeapi.Container) []*runtimeapi.Container {
containerMap := make(map[containerID][]*runtimeapi.Container) containerMap := make(map[containerID][]*runtimeapi.Container)
// Sort order by create time // Sort order by create time
sort.Slice(containers, func(i, j int) bool { sort.Slice(containers, func(i, j int) bool {
......
...@@ -51,7 +51,7 @@ func cadvisorInfoToCPUandMemoryStats(info *cadvisorapiv2.ContainerInfo) (*statsa ...@@ -51,7 +51,7 @@ func cadvisorInfoToCPUandMemoryStats(info *cadvisorapiv2.ContainerInfo) (*statsa
cpuStats.UsageCoreNanoSeconds = &cstat.Cpu.Usage.Total cpuStats.UsageCoreNanoSeconds = &cstat.Cpu.Usage.Total
} }
} }
if info.Spec.HasMemory { if info.Spec.HasMemory && cstat.Memory != nil {
pageFaults := cstat.Memory.ContainerData.Pgfault pageFaults := cstat.Memory.ContainerData.Pgfault
majorPageFaults := cstat.Memory.ContainerData.Pgmajfault majorPageFaults := cstat.Memory.ContainerData.Pgmajfault
memoryStats = &statsapi.MemoryStats{ memoryStats = &statsapi.MemoryStats{
......
...@@ -242,13 +242,15 @@ func (dswp *desiredStateOfWorldPopulator) findAndRemoveDeletedPods() { ...@@ -242,13 +242,15 @@ func (dswp *desiredStateOfWorldPopulator) findAndRemoveDeletedPods() {
if runningContainers { if runningContainers {
klog.V(4).Infof( klog.V(4).Infof(
"Pod %q has been removed from pod manager. However, it still has one or more containers in the non-exited state. Therefore, it will not be removed from volume manager.", "Pod %q still has one or more containers in the non-exited state. Therefore, it will not be removed from desired state.",
format.Pod(volumeToMount.Pod)) format.Pod(volumeToMount.Pod))
continue continue
} }
exists, _, _ := dswp.actualStateOfWorld.PodExistsInVolume(volumeToMount.PodName, volumeToMount.VolumeName)
if !dswp.actualStateOfWorld.VolumeExists(volumeToMount.VolumeName) && podExists { if !exists && podExists {
klog.V(4).Infof(volumeToMount.GenerateMsgDetailed("Actual state has not yet has this information skip removing volume from desired state", "")) klog.V(4).Infof(
volumeToMount.GenerateMsgDetailed(fmt.Sprintf("Actual state has not yet has this volume mounted information and pod (%q) still exists in pod manager, skip removing volume from desired state",
format.Pod(volumeToMount.Pod)), ""))
continue continue
} }
klog.V(4).Infof(volumeToMount.GenerateMsgDetailed("Removing volume from desired state", "")) klog.V(4).Infof(volumeToMount.GenerateMsgDetailed("Removing volume from desired state", ""))
......
...@@ -55,6 +55,9 @@ type KubeProxyIPVSConfiguration struct { ...@@ -55,6 +55,9 @@ type KubeProxyIPVSConfiguration struct {
// excludeCIDRs is a list of CIDR's which the ipvs proxier should not touch // excludeCIDRs is a list of CIDR's which the ipvs proxier should not touch
// when cleaning up ipvs services. // when cleaning up ipvs services.
ExcludeCIDRs []string ExcludeCIDRs []string
// strict ARP configure arp_ignore and arp_announce to avoid answering ARP queries
// from kube-ipvs0 interface
StrictARP bool
} }
// KubeProxyConntrackConfiguration contains conntrack settings for // KubeProxyConntrackConfiguration contains conntrack settings for
......
...@@ -226,6 +226,7 @@ func autoConvert_v1alpha1_KubeProxyIPVSConfiguration_To_config_KubeProxyIPVSConf ...@@ -226,6 +226,7 @@ func autoConvert_v1alpha1_KubeProxyIPVSConfiguration_To_config_KubeProxyIPVSConf
out.MinSyncPeriod = in.MinSyncPeriod out.MinSyncPeriod = in.MinSyncPeriod
out.Scheduler = in.Scheduler out.Scheduler = in.Scheduler
out.ExcludeCIDRs = *(*[]string)(unsafe.Pointer(&in.ExcludeCIDRs)) out.ExcludeCIDRs = *(*[]string)(unsafe.Pointer(&in.ExcludeCIDRs))
out.StrictARP = in.StrictARP
return nil return nil
} }
...@@ -239,6 +240,7 @@ func autoConvert_config_KubeProxyIPVSConfiguration_To_v1alpha1_KubeProxyIPVSConf ...@@ -239,6 +240,7 @@ func autoConvert_config_KubeProxyIPVSConfiguration_To_v1alpha1_KubeProxyIPVSConf
out.MinSyncPeriod = in.MinSyncPeriod out.MinSyncPeriod = in.MinSyncPeriod
out.Scheduler = in.Scheduler out.Scheduler = in.Scheduler
out.ExcludeCIDRs = *(*[]string)(unsafe.Pointer(&in.ExcludeCIDRs)) out.ExcludeCIDRs = *(*[]string)(unsafe.Pointer(&in.ExcludeCIDRs))
out.StrictARP = in.StrictARP
return nil return nil
} }
......
...@@ -17,10 +17,10 @@ limitations under the License. ...@@ -17,10 +17,10 @@ limitations under the License.
package ipvs package ipvs
import ( import (
"fmt"
"sync" "sync"
"time" "time"
"fmt"
"k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/util/wait"
"k8s.io/klog" "k8s.io/klog"
utilipvs "k8s.io/kubernetes/pkg/util/ipvs" utilipvs "k8s.io/kubernetes/pkg/util/ipvs"
...@@ -164,10 +164,10 @@ func (m *GracefulTerminationManager) deleteRsFunc(rsToDelete *listItem) (bool, e ...@@ -164,10 +164,10 @@ func (m *GracefulTerminationManager) deleteRsFunc(rsToDelete *listItem) (bool, e
} }
for _, rs := range rss { for _, rs := range rss {
if rsToDelete.RealServer.Equal(rs) { if rsToDelete.RealServer.Equal(rs) {
// Delete RS with no connections // For UDP traffic, no graceful termination, we immediately delete the RS
// For UDP, ActiveConn is always 0 // (existing connections will be deleted on the next packet because sysctlExpireNoDestConn=1)
// For TCP, InactiveConn are connections not in ESTABLISHED state // For other protocols, don't delete until all connections have expired)
if rs.ActiveConn+rs.InactiveConn != 0 { if rsToDelete.VirtualServer.Protocol != "udp" && rs.ActiveConn+rs.InactiveConn != 0 {
klog.Infof("Not deleting, RS %v: %v ActiveConn, %v InactiveConn", rsToDelete.String(), rs.ActiveConn, rs.InactiveConn) klog.Infof("Not deleting, RS %v: %v ActiveConn, %v InactiveConn", rsToDelete.String(), rs.ActiveConn, rs.InactiveConn)
return false, nil return false, nil
} }
......
...@@ -194,7 +194,9 @@ type Proxier struct { ...@@ -194,7 +194,9 @@ type Proxier struct {
syncPeriod time.Duration syncPeriod time.Duration
minSyncPeriod time.Duration minSyncPeriod time.Duration
// Values are CIDR's to exclude when cleaning up IPVS rules. // Values are CIDR's to exclude when cleaning up IPVS rules.
excludeCIDRs []string excludeCIDRs []string
// Set to true to set sysctls arp_ignore and arp_announce
strictARP bool
iptables utiliptables.Interface iptables utiliptables.Interface
ipvs utilipvs.Interface ipvs utilipvs.Interface
ipset utilipset.Interface ipset utilipset.Interface
...@@ -285,6 +287,7 @@ func NewProxier(ipt utiliptables.Interface, ...@@ -285,6 +287,7 @@ func NewProxier(ipt utiliptables.Interface,
syncPeriod time.Duration, syncPeriod time.Duration,
minSyncPeriod time.Duration, minSyncPeriod time.Duration,
excludeCIDRs []string, excludeCIDRs []string,
strictARP bool,
masqueradeAll bool, masqueradeAll bool,
masqueradeBit int, masqueradeBit int,
clusterCIDR string, clusterCIDR string,
...@@ -344,17 +347,19 @@ func NewProxier(ipt utiliptables.Interface, ...@@ -344,17 +347,19 @@ func NewProxier(ipt utiliptables.Interface,
} }
} }
// Set the arp_ignore sysctl we need for if strictARP {
if val, _ := sysctl.GetSysctl(sysctlArpIgnore); val != 1 { // Set the arp_ignore sysctl we need for
if err := sysctl.SetSysctl(sysctlArpIgnore, 1); err != nil { if val, _ := sysctl.GetSysctl(sysctlArpIgnore); val != 1 {
return nil, fmt.Errorf("can't set sysctl %s: %v", sysctlArpIgnore, err) if err := sysctl.SetSysctl(sysctlArpIgnore, 1); err != nil {
return nil, fmt.Errorf("can't set sysctl %s: %v", sysctlArpIgnore, err)
}
} }
}
// Set the arp_announce sysctl we need for // Set the arp_announce sysctl we need for
if val, _ := sysctl.GetSysctl(sysctlArpAnnounce); val != 2 { if val, _ := sysctl.GetSysctl(sysctlArpAnnounce); val != 2 {
if err := sysctl.SetSysctl(sysctlArpAnnounce, 2); err != nil { if err := sysctl.SetSysctl(sysctlArpAnnounce, 2); err != nil {
return nil, fmt.Errorf("can't set sysctl %s: %v", sysctlArpAnnounce, err) return nil, fmt.Errorf("can't set sysctl %s: %v", sysctlArpAnnounce, err)
}
} }
} }
...@@ -1654,15 +1659,17 @@ func (proxier *Proxier) syncEndpoint(svcPortName proxy.ServicePortName, onlyNode ...@@ -1654,15 +1659,17 @@ func (proxier *Proxier) syncEndpoint(svcPortName proxy.ServicePortName, onlyNode
func (proxier *Proxier) cleanLegacyService(activeServices map[string]bool, currentServices map[string]*utilipvs.VirtualServer, legacyBindAddrs map[string]bool) { func (proxier *Proxier) cleanLegacyService(activeServices map[string]bool, currentServices map[string]*utilipvs.VirtualServer, legacyBindAddrs map[string]bool) {
for cs := range currentServices { for cs := range currentServices {
svc := currentServices[cs] svc := currentServices[cs]
if proxier.isIPInExcludeCIDRs(svc.Address) {
continue
}
if _, ok := activeServices[cs]; !ok { if _, ok := activeServices[cs]; !ok {
// This service was not processed in the latest sync loop so before deleting it,
okayToDelete := true
rsList, _ := proxier.ipvs.GetRealServers(svc) rsList, _ := proxier.ipvs.GetRealServers(svc)
// If we still have real servers graceful termination is not done // If we still have real servers graceful termination is not done
if len(rsList) > 0 { if len(rsList) > 0 {
okayToDelete = false continue
} }
// Applying graceful termination to all real servers // Applying graceful termination to all real servers
for _, rs := range rsList { for _, rs := range rsList {
uniqueRS := GetUniqueRSName(svc, rs) uniqueRS := GetUniqueRSName(svc, rs)
...@@ -1675,35 +1682,36 @@ func (proxier *Proxier) cleanLegacyService(activeServices map[string]bool, curre ...@@ -1675,35 +1682,36 @@ func (proxier *Proxier) cleanLegacyService(activeServices map[string]bool, curre
klog.Errorf("Failed to delete destination: %v, error: %v", uniqueRS, err) klog.Errorf("Failed to delete destination: %v, error: %v", uniqueRS, err)
} }
} }
// make sure it does not fall within an excluded CIDR range. klog.V(4).Infof("Delete service %s", svc.String())
for _, excludedCIDR := range proxier.excludeCIDRs { if err := proxier.ipvs.DeleteVirtualServer(svc); err != nil {
// Any validation of this CIDR already should have occurred. klog.Errorf("Failed to delete service %s, error: %v", svc.String(), err)
_, n, _ := net.ParseCIDR(excludedCIDR)
if n.Contains(svc.Address) {
okayToDelete = false
break
}
} }
if okayToDelete { addr := svc.Address.String()
klog.V(4).Infof("Delete service %s", svc.String()) if _, ok := legacyBindAddrs[addr]; ok {
if err := proxier.ipvs.DeleteVirtualServer(svc); err != nil { klog.V(4).Infof("Unbinding address %s", addr)
klog.Errorf("Failed to delete service %s, error: %v", svc.String(), err) if err := proxier.netlinkHandle.UnbindAddress(addr, DefaultDummyDevice); err != nil {
} klog.Errorf("Failed to unbind service addr %s from dummy interface %s: %v", addr, DefaultDummyDevice, err)
addr := svc.Address.String() } else {
if _, ok := legacyBindAddrs[addr]; ok { // In case we delete a multi-port service, avoid trying to unbind multiple times
klog.V(4).Infof("Unbinding address %s", addr) delete(legacyBindAddrs, addr)
if err := proxier.netlinkHandle.UnbindAddress(addr, DefaultDummyDevice); err != nil {
klog.Errorf("Failed to unbind service addr %s from dummy interface %s: %v", addr, DefaultDummyDevice, err)
} else {
// In case we delete a multi-port service, avoid trying to unbind multiple times
delete(legacyBindAddrs, addr)
}
} }
} }
} }
} }
} }
func (proxier *Proxier) isIPInExcludeCIDRs(ip net.IP) bool {
// make sure it does not fall within an excluded CIDR range.
for _, excludedCIDR := range proxier.excludeCIDRs {
// Any validation of this CIDR already should have occurred.
_, n, _ := net.ParseCIDR(excludedCIDR)
if n.Contains(ip) {
return true
}
}
return false
}
func (proxier *Proxier) getLegacyBindAddr(activeBindAddrs map[string]bool, currentBindAddrs []string) map[string]bool { func (proxier *Proxier) getLegacyBindAddr(activeBindAddrs map[string]bool, currentBindAddrs []string) map[string]bool {
legacyAddrs := make(map[string]bool) legacyAddrs := make(map[string]bool)
isIpv6 := utilnet.IsIPv6(proxier.nodeIP) isIpv6 := utilnet.IsIPv6(proxier.nodeIP)
......
package(default_visibility = ["//visibility:public"]) load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
"go_test",
)
go_test( go_test(
name = "go_default_test", name = "go_default_test",
...@@ -20,6 +14,7 @@ go_test( ...@@ -20,6 +14,7 @@ go_test(
"//pkg/registry/registrytest:go_default_library", "//pkg/registry/registrytest:go_default_library",
"//pkg/securitycontext:go_default_library", "//pkg/securitycontext:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/apitesting:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/equality:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/equality:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
...@@ -27,7 +22,9 @@ go_test( ...@@ -27,7 +22,9 @@ go_test(
"//staging/src/k8s.io/apimachinery/pkg/fields:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/fields:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/apis/example/v1:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/endpoints/request:go_default_library", "//staging/src/k8s.io/apiserver/pkg/endpoints/request:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/registry/generic:go_default_library", "//staging/src/k8s.io/apiserver/pkg/registry/generic:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/registry/generic/registry:go_default_library", "//staging/src/k8s.io/apiserver/pkg/registry/generic/registry:go_default_library",
...@@ -46,6 +43,7 @@ go_library( ...@@ -46,6 +43,7 @@ go_library(
"storage.go", "storage.go",
], ],
importpath = "k8s.io/kubernetes/pkg/registry/core/pod/storage", importpath = "k8s.io/kubernetes/pkg/registry/core/pod/storage",
visibility = ["//visibility:public"],
deps = [ deps = [
"//pkg/api/pod:go_default_library", "//pkg/api/pod:go_default_library",
"//pkg/apis/core:go_default_library", "//pkg/apis/core:go_default_library",
...@@ -85,4 +83,5 @@ filegroup( ...@@ -85,4 +83,5 @@ filegroup(
name = "all-srcs", name = "all-srcs",
srcs = [":package-srcs"], srcs = [":package-srcs"],
tags = ["automanaged"], tags = ["automanaged"],
visibility = ["//visibility:public"],
) )
...@@ -19,6 +19,7 @@ package storage ...@@ -19,6 +19,7 @@ package storage
import ( import (
"context" "context"
"fmt" "fmt"
"reflect"
"time" "time"
"k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/errors"
...@@ -29,6 +30,7 @@ import ( ...@@ -29,6 +30,7 @@ import (
"k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/util/wait"
genericregistry "k8s.io/apiserver/pkg/registry/generic/registry" genericregistry "k8s.io/apiserver/pkg/registry/generic/registry"
"k8s.io/apiserver/pkg/registry/rest" "k8s.io/apiserver/pkg/registry/rest"
"k8s.io/apiserver/pkg/util/dryrun"
"k8s.io/client-go/util/retry" "k8s.io/client-go/util/retry"
api "k8s.io/kubernetes/pkg/apis/core" api "k8s.io/kubernetes/pkg/apis/core"
"k8s.io/kubernetes/pkg/apis/policy" "k8s.io/kubernetes/pkg/apis/policy"
...@@ -77,11 +79,36 @@ func (r *EvictionREST) New() runtime.Object { ...@@ -77,11 +79,36 @@ func (r *EvictionREST) New() runtime.Object {
return &policy.Eviction{} return &policy.Eviction{}
} }
// Propagate dry-run takes the dry-run option from the request and pushes it into the eviction object.
// It returns an error if they have non-matching dry-run options.
func propagateDryRun(eviction *policy.Eviction, options *metav1.CreateOptions) (*metav1.DeleteOptions, error) {
if eviction.DeleteOptions == nil {
return &metav1.DeleteOptions{DryRun: options.DryRun}, nil
}
if len(eviction.DeleteOptions.DryRun) == 0 {
eviction.DeleteOptions.DryRun = options.DryRun
return eviction.DeleteOptions, nil
}
if len(options.DryRun) == 0 {
return eviction.DeleteOptions, nil
}
if !reflect.DeepEqual(options.DryRun, eviction.DeleteOptions.DryRun) {
return nil, fmt.Errorf("Non-matching dry-run options in request and content: %v and %v", options.DryRun, eviction.DeleteOptions.DryRun)
}
return eviction.DeleteOptions, nil
}
// Create attempts to create a new eviction. That is, it tries to evict a pod. // Create attempts to create a new eviction. That is, it tries to evict a pod.
func (r *EvictionREST) Create(ctx context.Context, obj runtime.Object, createValidation rest.ValidateObjectFunc, options *metav1.CreateOptions) (runtime.Object, error) { func (r *EvictionREST) Create(ctx context.Context, obj runtime.Object, createValidation rest.ValidateObjectFunc, options *metav1.CreateOptions) (runtime.Object, error) {
eviction := obj.(*policy.Eviction) eviction := obj.(*policy.Eviction)
obj, err := r.store.Get(ctx, eviction.Name, &metav1.GetOptions{}) deletionOptions, err := propagateDryRun(eviction, options)
if err != nil {
return nil, err
}
obj, err = r.store.Get(ctx, eviction.Name, &metav1.GetOptions{})
if err != nil { if err != nil {
return nil, err return nil, err
} }
...@@ -89,7 +116,7 @@ func (r *EvictionREST) Create(ctx context.Context, obj runtime.Object, createVal ...@@ -89,7 +116,7 @@ func (r *EvictionREST) Create(ctx context.Context, obj runtime.Object, createVal
// Evicting a terminal pod should result in direct deletion of pod as it already caused disruption by the time we are evicting. // Evicting a terminal pod should result in direct deletion of pod as it already caused disruption by the time we are evicting.
// There is no need to check for pdb. // There is no need to check for pdb.
if pod.Status.Phase == api.PodSucceeded || pod.Status.Phase == api.PodFailed { if pod.Status.Phase == api.PodSucceeded || pod.Status.Phase == api.PodFailed {
_, _, err = r.store.Delete(ctx, eviction.Name, eviction.DeleteOptions) _, _, err = r.store.Delete(ctx, eviction.Name, deletionOptions)
if err != nil { if err != nil {
return nil, err return nil, err
} }
...@@ -118,7 +145,7 @@ func (r *EvictionREST) Create(ctx context.Context, obj runtime.Object, createVal ...@@ -118,7 +145,7 @@ func (r *EvictionREST) Create(ctx context.Context, obj runtime.Object, createVal
// If it was false already, or if it becomes false during the course of our retries, // If it was false already, or if it becomes false during the course of our retries,
// raise an error marked as a 429. // raise an error marked as a 429.
if err := r.checkAndDecrement(pod.Namespace, pod.Name, pdb); err != nil { if err := r.checkAndDecrement(pod.Namespace, pod.Name, pdb, dryrun.IsDryRun(deletionOptions.DryRun)); err != nil {
return err return err
} }
} }
...@@ -138,11 +165,6 @@ func (r *EvictionREST) Create(ctx context.Context, obj runtime.Object, createVal ...@@ -138,11 +165,6 @@ func (r *EvictionREST) Create(ctx context.Context, obj runtime.Object, createVal
// At this point there was either no PDB or we succeeded in decrementing // At this point there was either no PDB or we succeeded in decrementing
// Try the delete // Try the delete
deletionOptions := eviction.DeleteOptions
if deletionOptions == nil {
// default to non-nil to trigger graceful deletion
deletionOptions = &metav1.DeleteOptions{}
}
_, _, err = r.store.Delete(ctx, eviction.Name, deletionOptions) _, _, err = r.store.Delete(ctx, eviction.Name, deletionOptions)
if err != nil { if err != nil {
return nil, err return nil, err
...@@ -153,7 +175,7 @@ func (r *EvictionREST) Create(ctx context.Context, obj runtime.Object, createVal ...@@ -153,7 +175,7 @@ func (r *EvictionREST) Create(ctx context.Context, obj runtime.Object, createVal
} }
// checkAndDecrement checks if the provided PodDisruptionBudget allows any disruption. // checkAndDecrement checks if the provided PodDisruptionBudget allows any disruption.
func (r *EvictionREST) checkAndDecrement(namespace string, podName string, pdb policy.PodDisruptionBudget) error { func (r *EvictionREST) checkAndDecrement(namespace string, podName string, pdb policy.PodDisruptionBudget, dryRun bool) error {
if pdb.Status.ObservedGeneration < pdb.Generation { if pdb.Status.ObservedGeneration < pdb.Generation {
// TODO(mml): Add a Retry-After header. Once there are time-based // TODO(mml): Add a Retry-After header. Once there are time-based
// budgets, we can sometimes compute a sensible suggested value. But // budgets, we can sometimes compute a sensible suggested value. But
...@@ -179,6 +201,12 @@ func (r *EvictionREST) checkAndDecrement(namespace string, podName string, pdb p ...@@ -179,6 +201,12 @@ func (r *EvictionREST) checkAndDecrement(namespace string, podName string, pdb p
if pdb.Status.DisruptedPods == nil { if pdb.Status.DisruptedPods == nil {
pdb.Status.DisruptedPods = make(map[string]metav1.Time) pdb.Status.DisruptedPods = make(map[string]metav1.Time)
} }
// If this is a dry-run, we don't need to go any further than that.
if dryRun == true {
return nil
}
// Eviction handler needs to inform the PDB controller that it is about to delete a pod // Eviction handler needs to inform the PDB controller that it is about to delete a pod
// so it should not consider it as available in calculations when updating PodDisruptions allowed. // so it should not consider it as available in calculations when updating PodDisruptions allowed.
// If the pod is not deleted within a reasonable time limit PDB controller will assume that it won't // If the pod is not deleted within a reasonable time limit PDB controller will assume that it won't
......
...@@ -3,8 +3,8 @@ package version ...@@ -3,8 +3,8 @@ package version
var ( var (
gitMajor = "1" gitMajor = "1"
gitMinor = "14" gitMinor = "14"
gitVersion = "v1.14.1-k3s.4" gitVersion = "v1.14.3-k3s.1"
gitCommit = "52f3b42401c93c36467f1fd6d294a3aba26c7def" gitCommit = "8343999292c55c807be4406fcaa9f047e8751ffd"
gitTreeState = "clean" gitTreeState = "clean"
buildDate = "2019-04-15T22:13+00:00Z" buildDate = "2019-06-12T04:56+00:00Z"
) )
...@@ -24,11 +24,16 @@ import ( ...@@ -24,11 +24,16 @@ import (
"k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume"
) )
const (
statusSuccess = "success"
statusFailUnknown = "fail-unknown"
)
var storageOperationMetric = prometheus.NewHistogramVec( var storageOperationMetric = prometheus.NewHistogramVec(
prometheus.HistogramOpts{ prometheus.HistogramOpts{
Name: "storage_operation_duration_seconds", Name: "storage_operation_duration_seconds",
Help: "Storage operation duration", Help: "Storage operation duration",
Buckets: []float64{.1, .25, .5, 1, 2.5, 5, 10, 15, 25, 50}, Buckets: []float64{.1, .25, .5, 1, 2.5, 5, 10, 15, 25, 50, 120, 300, 600},
}, },
[]string{"volume_plugin", "operation_name"}, []string{"volume_plugin", "operation_name"},
) )
...@@ -41,6 +46,14 @@ var storageOperationErrorMetric = prometheus.NewCounterVec( ...@@ -41,6 +46,14 @@ var storageOperationErrorMetric = prometheus.NewCounterVec(
[]string{"volume_plugin", "operation_name"}, []string{"volume_plugin", "operation_name"},
) )
var storageOperationStatusMetric = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "storage_operation_status_count",
Help: "Storage operation return statuses count",
},
[]string{"volume_plugin", "operation_name", "status"},
)
func init() { func init() {
registerMetrics() registerMetrics()
} }
...@@ -48,6 +61,7 @@ func init() { ...@@ -48,6 +61,7 @@ func init() {
func registerMetrics() { func registerMetrics() {
prometheus.MustRegister(storageOperationMetric) prometheus.MustRegister(storageOperationMetric)
prometheus.MustRegister(storageOperationErrorMetric) prometheus.MustRegister(storageOperationErrorMetric)
prometheus.MustRegister(storageOperationStatusMetric)
} }
// OperationCompleteHook returns a hook to call when an operation is completed // OperationCompleteHook returns a hook to call when an operation is completed
...@@ -56,11 +70,16 @@ func OperationCompleteHook(plugin, operationName string) func(*error) { ...@@ -56,11 +70,16 @@ func OperationCompleteHook(plugin, operationName string) func(*error) {
opComplete := func(err *error) { opComplete := func(err *error) {
timeTaken := time.Since(requestTime).Seconds() timeTaken := time.Since(requestTime).Seconds()
// Create metric with operation name and plugin name // Create metric with operation name and plugin name
status := statusSuccess
if *err != nil { if *err != nil {
// TODO: Establish well-known error codes to be able to distinguish
// user configuration errors from system errors.
status = statusFailUnknown
storageOperationErrorMetric.WithLabelValues(plugin, operationName).Inc() storageOperationErrorMetric.WithLabelValues(plugin, operationName).Inc()
} else { } else {
storageOperationMetric.WithLabelValues(plugin, operationName).Observe(timeTaken) storageOperationMetric.WithLabelValues(plugin, operationName).Observe(timeTaken)
} }
storageOperationStatusMetric.WithLabelValues(plugin, operationName, status).Inc()
} }
return opComplete return opComplete
} }
......
...@@ -468,29 +468,6 @@ func CheckRequest(quotas []corev1.ResourceQuota, a admission.Attributes, evaluat ...@@ -468,29 +468,6 @@ func CheckRequest(quotas []corev1.ResourceQuota, a admission.Attributes, evaluat
restrictedResourcesSet.Insert(localRestrictedResourcesSet.List()...) restrictedResourcesSet.Insert(localRestrictedResourcesSet.List()...)
} }
// verify that for every resource that had limited by default consumption
// enabled that there was a corresponding quota that covered its use.
// if not, we reject the request.
hasNoCoveringQuota := limitedResourceNamesSet.Difference(restrictedResourcesSet)
if len(hasNoCoveringQuota) > 0 {
return quotas, admission.NewForbidden(a, fmt.Errorf("insufficient quota to consume: %v", strings.Join(hasNoCoveringQuota.List(), ",")))
}
// verify that for every scope that had limited access enabled
// that there was a corresponding quota that covered it.
// if not, we reject the request.
scopesHasNoCoveringQuota, err := evaluator.UncoveredQuotaScopes(limitedScopes, restrictedScopes)
if err != nil {
return quotas, err
}
if len(scopesHasNoCoveringQuota) > 0 {
return quotas, fmt.Errorf("insufficient quota to match these scopes: %v", scopesHasNoCoveringQuota)
}
if len(interestingQuotaIndexes) == 0 {
return quotas, nil
}
// Usage of some resources cannot be counted in isolation. For example, when // Usage of some resources cannot be counted in isolation. For example, when
// the resource represents a number of unique references to external // the resource represents a number of unique references to external
// resource. In such a case an evaluator needs to process other objects in // resource. In such a case an evaluator needs to process other objects in
...@@ -537,6 +514,29 @@ func CheckRequest(quotas []corev1.ResourceQuota, a admission.Attributes, evaluat ...@@ -537,6 +514,29 @@ func CheckRequest(quotas []corev1.ResourceQuota, a admission.Attributes, evaluat
return quotas, nil return quotas, nil
} }
// verify that for every resource that had limited by default consumption
// enabled that there was a corresponding quota that covered its use.
// if not, we reject the request.
hasNoCoveringQuota := limitedResourceNamesSet.Difference(restrictedResourcesSet)
if len(hasNoCoveringQuota) > 0 {
return quotas, admission.NewForbidden(a, fmt.Errorf("insufficient quota to consume: %v", strings.Join(hasNoCoveringQuota.List(), ",")))
}
// verify that for every scope that had limited access enabled
// that there was a corresponding quota that covered it.
// if not, we reject the request.
scopesHasNoCoveringQuota, err := evaluator.UncoveredQuotaScopes(limitedScopes, restrictedScopes)
if err != nil {
return quotas, err
}
if len(scopesHasNoCoveringQuota) > 0 {
return quotas, fmt.Errorf("insufficient quota to match these scopes: %v", scopesHasNoCoveringQuota)
}
if len(interestingQuotaIndexes) == 0 {
return quotas, nil
}
outQuotas, err := copyQuotas(quotas) outQuotas, err := copyQuotas(quotas)
if err != nil { if err != nil {
return nil, err return nil, err
......
...@@ -228,6 +228,7 @@ func (config *DirectClientConfig) getUserIdentificationPartialConfig(configAuthI ...@@ -228,6 +228,7 @@ func (config *DirectClientConfig) getUserIdentificationPartialConfig(configAuthI
// blindly overwrite existing values based on precedence // blindly overwrite existing values based on precedence
if len(configAuthInfo.Token) > 0 { if len(configAuthInfo.Token) > 0 {
mergedConfig.BearerToken = configAuthInfo.Token mergedConfig.BearerToken = configAuthInfo.Token
mergedConfig.BearerTokenFile = configAuthInfo.TokenFile
} else if len(configAuthInfo.TokenFile) > 0 { } else if len(configAuthInfo.TokenFile) > 0 {
tokenBytes, err := ioutil.ReadFile(configAuthInfo.TokenFile) tokenBytes, err := ioutil.ReadFile(configAuthInfo.TokenFile)
if err != nil { if err != nil {
...@@ -499,8 +500,9 @@ func (config *inClusterClientConfig) ClientConfig() (*restclient.Config, error) ...@@ -499,8 +500,9 @@ func (config *inClusterClientConfig) ClientConfig() (*restclient.Config, error)
if server := config.overrides.ClusterInfo.Server; len(server) > 0 { if server := config.overrides.ClusterInfo.Server; len(server) > 0 {
icc.Host = server icc.Host = server
} }
if token := config.overrides.AuthInfo.Token; len(token) > 0 { if len(config.overrides.AuthInfo.Token) > 0 || len(config.overrides.AuthInfo.TokenFile) > 0 {
icc.BearerToken = token icc.BearerToken = config.overrides.AuthInfo.Token
icc.BearerTokenFile = config.overrides.AuthInfo.TokenFile
} }
if certificateAuthorityFile := config.overrides.ClusterInfo.CertificateAuthority; len(certificateAuthorityFile) > 0 { if certificateAuthorityFile := config.overrides.ClusterInfo.CertificateAuthority; len(certificateAuthorityFile) > 0 {
icc.TLSClientConfig.CAFile = certificateAuthorityFile icc.TLSClientConfig.CAFile = certificateAuthorityFile
......
...@@ -6,7 +6,7 @@ cloud.google.com/go v0.1.0-115-g3b1ae45394a234 ...@@ -6,7 +6,7 @@ cloud.google.com/go v0.1.0-115-g3b1ae45394a234
github.com/armon/circbuf bbbad097214e2918d8543d5201d12bfd7bca254d github.com/armon/circbuf bbbad097214e2918d8543d5201d12bfd7bca254d
github.com/asaskevich/govalidator v9-26-gf9ffefc3facfbe github.com/asaskevich/govalidator v9-26-gf9ffefc3facfbe
github.com/aws/aws-sdk-go v1.16.26 github.com/aws/aws-sdk-go v1.16.26
github.com/Azure/azure-sdk-for-go v21.3.0 github.com/Azure/azure-sdk-for-go v21.4.0
github.com/Azure/go-ansiterm d6e3b3328b783f23731bc4d058875b0371ff8109 github.com/Azure/go-ansiterm d6e3b3328b783f23731bc4d058875b0371ff8109
github.com/Azure/go-autorest v11.1.0 github.com/Azure/go-autorest v11.1.0
github.com/bazelbuild/bazel-gazelle 0.15.0 github.com/bazelbuild/bazel-gazelle 0.15.0
...@@ -26,7 +26,7 @@ github.com/container-storage-interface/spec v1.1.0 ...@@ -26,7 +26,7 @@ github.com/container-storage-interface/spec v1.1.0
github.com/coreos/bbolt v1.3.1-coreos.6 github.com/coreos/bbolt v1.3.1-coreos.6
github.com/coreos/etcd v3.3.10 github.com/coreos/etcd v3.3.10
github.com/coreos/go-oidc 065b426bd41667456c1a924468f507673629c46b github.com/coreos/go-oidc 065b426bd41667456c1a924468f507673629c46b
github.com/coreos/go-semver v0.2.0-9-ge214231b295a8e github.com/coreos/go-semver v0.3.0
github.com/coreos/go-systemd v17 github.com/coreos/go-systemd v17
github.com/coreos/pkg v4 github.com/coreos/pkg v4
github.com/coreos/rkt v1.25.0 github.com/coreos/rkt v1.25.0
...@@ -47,7 +47,7 @@ github.com/docker/spdystream 449fdfce4d962303d702fec724ef0ad181c92528 ...@@ -47,7 +47,7 @@ github.com/docker/spdystream 449fdfce4d962303d702fec724ef0ad181c92528
github.com/elazarl/goproxy v1.0-104-gc4fc26588b6ef8 github.com/elazarl/goproxy v1.0-104-gc4fc26588b6ef8
github.com/emicklei/go-restful 2.2.0-4-gff4f55a206334e github.com/emicklei/go-restful 2.2.0-4-gff4f55a206334e
github.com/euank/go-kmsg-parser v2.0.0 github.com/euank/go-kmsg-parser v2.0.0
github.com/evanphx/json-patch v4.1.0-19-g5858425f75500d github.com/evanphx/json-patch v4.2.0
github.com/exponent-io/jsonpath d6023ce2651d8eafb5c75bb0c7167536102ec9f5 github.com/exponent-io/jsonpath d6023ce2651d8eafb5c75bb0c7167536102ec9f5
github.com/fatih/camelcase f6a740d52f961c60348ebb109adde9f4635d7540 github.com/fatih/camelcase f6a740d52f961c60348ebb109adde9f4635d7540
github.com/fsnotify/fsnotify v1.3.1-1-gf12c6236fe7b5c github.com/fsnotify/fsnotify v1.3.1-1-gf12c6236fe7b5c
...@@ -173,7 +173,7 @@ github.com/tmc/grpc-websocket-proxy 89b8d40f7ca833297db804fcb3be53a76d01c238 ...@@ -173,7 +173,7 @@ github.com/tmc/grpc-websocket-proxy 89b8d40f7ca833297db804fcb3be53a76d01c238
github.com/ugorji/go bdcc60b419d136a85cdf2e7cbcac34b3f1cd6e57 github.com/ugorji/go bdcc60b419d136a85cdf2e7cbcac34b3f1cd6e57
github.com/vishvananda/netlink b2de5d10e38ecce8607e6b438b6d174f389a004e github.com/vishvananda/netlink b2de5d10e38ecce8607e6b438b6d174f389a004e
github.com/vishvananda/netns be1fbeda19366dea804f00efff2dd73a1642fdcc github.com/vishvananda/netns be1fbeda19366dea804f00efff2dd73a1642fdcc
github.com/vmware/govmomi v0.20.0 github.com/vmware/govmomi v0.20.1
github.com/vmware/photon-controller-go-sdk PROMOTED-488 github.com/vmware/photon-controller-go-sdk PROMOTED-488
github.com/xanzy/go-cloudstack v2.1.1-1-g1e2cbf647e57fa github.com/xanzy/go-cloudstack v2.1.1-1-g1e2cbf647e57fa
github.com/xiang90/probing 0.0.1 github.com/xiang90/probing 0.0.1
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment