Skip to content

K
k3s
Commit 2d9b2d9f authored by leigh schrandt's avatar leigh schrandt
Browse files
Options

Switch to a dedicated CA for kubeadm etcd identities

parent be2e7028
Hide whitespace changes
Inline Side-by-side
Showing with 123 additions and 43 deletions
cmd/kubeadm/app/cmd/phases/certs.go
View file @ 2d9b2d9f
...@@ -52,7 +52,7 @@ var ( ...@@ -52,7 +52,7 @@ var (
`) `)
caCertLongDesc = fmt.Sprintf(normalizer.LongDesc(` caCertLongDesc = fmt.Sprintf(normalizer.LongDesc(`
Generates the self-signed certificate authority and related key, and saves them into %s and %s files. Generates the self-signed kubernetes certificate authority and related key, and saves them into %s and %s files.
If both files already exist, kubeadm skips the generation step and existing files will be used. If both files already exist, kubeadm skips the generation step and existing files will be used.
`+cmdutil.AlphaDisclaimer), kubeadmconstants.CACertName, kubeadmconstants.CAKeyName) `+cmdutil.AlphaDisclaimer), kubeadmconstants.CACertName, kubeadmconstants.CAKeyName)
...@@ -74,6 +74,12 @@ var ( ...@@ -74,6 +74,12 @@ var (
If both files already exist, kubeadm skips the generation step and existing files will be used. If both files already exist, kubeadm skips the generation step and existing files will be used.
`+cmdutil.AlphaDisclaimer), kubeadmconstants.APIServerKubeletClientCertName, kubeadmconstants.APIServerKubeletClientKeyName) `+cmdutil.AlphaDisclaimer), kubeadmconstants.APIServerKubeletClientCertName, kubeadmconstants.APIServerKubeletClientKeyName)
etcdCaCertLongDesc = fmt.Sprintf(normalizer.LongDesc(`
Generates the self-signed etcd certificate authority and related key and saves them into %s and %s files.
If both files already exist, kubeadm skips the generation step and existing files will be used.
`+cmdutil.AlphaDisclaimer), kubeadmconstants.EtcdCACertName, kubeadmconstants.EtcdCAKeyName)
etcdServerCertLongDesc = fmt.Sprintf(normalizer.LongDesc(` etcdServerCertLongDesc = fmt.Sprintf(normalizer.LongDesc(`
Generates the etcd serving certificate and key and saves them into %s and %s files. Generates the etcd serving certificate and key and saves them into %s and %s files.
...@@ -166,7 +172,7 @@ func getCertsSubCommands(defaultKubernetesVersion string) []*cobra.Command { ...@@ -166,7 +172,7 @@ func getCertsSubCommands(defaultKubernetesVersion string) []*cobra.Command {
}, },
{ {
use: "ca", use: "ca",
short: "Generates self-signed CA to provision identities for each component in the cluster", short: "Generates self-signed kubernetes CA to provision identities for components of the cluster",
long: caCertLongDesc, long: caCertLongDesc,
cmdFunc: certsphase.CreateCACertAndKeyfiles, cmdFunc: certsphase.CreateCACertAndKeyfiles,
}, },
...@@ -183,6 +189,12 @@ func getCertsSubCommands(defaultKubernetesVersion string) []*cobra.Command { ...@@ -183,6 +189,12 @@ func getCertsSubCommands(defaultKubernetesVersion string) []*cobra.Command {
cmdFunc: certsphase.CreateAPIServerKubeletClientCertAndKeyFiles, cmdFunc: certsphase.CreateAPIServerKubeletClientCertAndKeyFiles,
}, },
{ {
use: "etcd-ca",
short: "Generates self-signed CA to provision identities for etcd",
long: etcdCaCertLongDesc,
cmdFunc: certsphase.CreateEtcdCACertAndKeyFiles,
},
{
use: "etcd-server", use: "etcd-server",
short: "Generates etcd serving certificate and key", short: "Generates etcd serving certificate and key",
long: etcdServerCertLongDesc, long: etcdServerCertLongDesc,
......
cmd/kubeadm/app/constants/constants.go
View file @ 2d9b2d9f
...@@ -65,6 +65,13 @@ const ( ...@@ -65,6 +65,13 @@ const (
// APIServerKubeletClientCertCommonName defines kubelet client certificate common name (CN) // APIServerKubeletClientCertCommonName defines kubelet client certificate common name (CN)
APIServerKubeletClientCertCommonName = "kube-apiserver-kubelet-client" APIServerKubeletClientCertCommonName = "kube-apiserver-kubelet-client"
// EtcdCACertAndKeyBaseName defines etcd's CA certificate and key base name
EtcdCACertAndKeyBaseName = "etcd/ca"
// EtcdCACertName defines etcd's CA certificate name
EtcdCACertName = "etcd/ca.crt"
// EtcdCAKeyName defines etcd's CA key name
EtcdCAKeyName = "etcd/ca.key"
// EtcdServerCertAndKeyBaseName defines etcd's server certificate and key base name // EtcdServerCertAndKeyBaseName defines etcd's server certificate and key base name
EtcdServerCertAndKeyBaseName = "etcd/server" EtcdServerCertAndKeyBaseName = "etcd/server"
// EtcdServerCertName defines etcd's server certificate name // EtcdServerCertName defines etcd's server certificate name
......
cmd/kubeadm/app/phases/certs/certs.go
View file @ 2d9b2d9f
...@@ -37,6 +37,7 @@ func CreatePKIAssets(cfg *kubeadmapi.MasterConfiguration) error { ...@@ -37,6 +37,7 @@ func CreatePKIAssets(cfg *kubeadmapi.MasterConfiguration) error {
CreateCACertAndKeyfiles, CreateCACertAndKeyfiles,
CreateAPIServerCertAndKeyFiles, CreateAPIServerCertAndKeyFiles,
CreateAPIServerKubeletClientCertAndKeyFiles, CreateAPIServerKubeletClientCertAndKeyFiles,
CreateEtcdCACertAndKeyFiles,
CreateEtcdServerCertAndKeyFiles, CreateEtcdServerCertAndKeyFiles,
CreateEtcdPeerCertAndKeyFiles, CreateEtcdPeerCertAndKeyFiles,
CreateAPIServerEtcdClientCertAndKeyFiles, CreateAPIServerEtcdClientCertAndKeyFiles,
...@@ -122,17 +123,36 @@ func CreateAPIServerKubeletClientCertAndKeyFiles(cfg *kubeadmapi.MasterConfigura ...@@ -122,17 +123,36 @@ func CreateAPIServerKubeletClientCertAndKeyFiles(cfg *kubeadmapi.MasterConfigura
) )
} }
// CreateEtcdCACertAndKeyFiles create a self signed etcd CA certificate and key files.
// The etcd CA and client certs are used to secure communication between etcd peers and connections to etcd from the API server.
// This is a separate CA, so that kubernetes client identities cannot connect to etcd directly or peer with the etcd cluster.
// If the etcd CA certificate and key files already exists in the target folder, they are used only if evaluated equals; otherwise an error is returned.
func CreateEtcdCACertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error {
etcdCACert, etcdCAKey, err := NewEtcdCACertAndKey()
if err != nil {
return err
}
return writeCertificateAuthorithyFilesIfNotExist(
cfg.CertificatesDir,
kubeadmconstants.EtcdCACertAndKeyBaseName,
etcdCACert,
etcdCAKey,
)
}
// CreateEtcdServerCertAndKeyFiles create a new certificate and key file for etcd. // CreateEtcdServerCertAndKeyFiles create a new certificate and key file for etcd.
// If the etcd serving certificate and key file already exist in the target folder, they are used only if evaluated equal; otherwise an error is returned. // If the etcd serving certificate and key file already exist in the target folder, they are used only if evaluated equal; otherwise an error is returned.
// It assumes the cluster CA certificate and key file exist in the CertificatesDir // It assumes the etcd CA certificate and key file exist in the CertificatesDir
func CreateEtcdServerCertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error { func CreateEtcdServerCertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error {
caCert, caKey, err := loadCertificateAuthority(cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName) etcdCACert, etcdCAKey, err := loadCertificateAuthority(cfg.CertificatesDir, kubeadmconstants.EtcdCACertAndKeyBaseName)
if err != nil { if err != nil {
return err return err
} }
etcdServerCert, etcdServerKey, err := NewEtcdServerCertAndKey(cfg, caCert, caKey) etcdServerCert, etcdServerKey, err := NewEtcdServerCertAndKey(cfg, etcdCACert, etcdCAKey)
if err != nil { if err != nil {
return err return err
} }
...@@ -140,7 +160,7 @@ func CreateEtcdServerCertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error ...@@ -140,7 +160,7 @@ func CreateEtcdServerCertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error
return writeCertificateFilesIfNotExist( return writeCertificateFilesIfNotExist(
cfg.CertificatesDir, cfg.CertificatesDir,
kubeadmconstants.EtcdServerCertAndKeyBaseName, kubeadmconstants.EtcdServerCertAndKeyBaseName,
caCert, etcdCACert,
etcdServerCert, etcdServerCert,
etcdServerKey, etcdServerKey,
) )
...@@ -148,15 +168,15 @@ func CreateEtcdServerCertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error ...@@ -148,15 +168,15 @@ func CreateEtcdServerCertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error
// CreateEtcdPeerCertAndKeyFiles create a new certificate and key file for etcd peering. // CreateEtcdPeerCertAndKeyFiles create a new certificate and key file for etcd peering.
// If the etcd peer certificate and key file already exist in the target folder, they are used only if evaluated equal; otherwise an error is returned. // If the etcd peer certificate and key file already exist in the target folder, they are used only if evaluated equal; otherwise an error is returned.
// It assumes the cluster CA certificate and key file exist in the CertificatesDir // It assumes the etcd CA certificate and key file exist in the CertificatesDir
func CreateEtcdPeerCertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error { func CreateEtcdPeerCertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error {
caCert, caKey, err := loadCertificateAuthority(cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName) etcdCACert, etcdCAKey, err := loadCertificateAuthority(cfg.CertificatesDir, kubeadmconstants.EtcdCACertAndKeyBaseName)
if err != nil { if err != nil {
return err return err
} }
etcdPeerCert, etcdPeerKey, err := NewEtcdPeerCertAndKey(cfg, caCert, caKey) etcdPeerCert, etcdPeerKey, err := NewEtcdPeerCertAndKey(cfg, etcdCACert, etcdCAKey)
if err != nil { if err != nil {
return err return err
} }
...@@ -164,7 +184,7 @@ func CreateEtcdPeerCertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error { ...@@ -164,7 +184,7 @@ func CreateEtcdPeerCertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error {
return writeCertificateFilesIfNotExist( return writeCertificateFilesIfNotExist(
cfg.CertificatesDir, cfg.CertificatesDir,
kubeadmconstants.EtcdPeerCertAndKeyBaseName, kubeadmconstants.EtcdPeerCertAndKeyBaseName,
caCert, etcdCACert,
etcdPeerCert, etcdPeerCert,
etcdPeerKey, etcdPeerKey,
) )
...@@ -172,15 +192,15 @@ func CreateEtcdPeerCertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error { ...@@ -172,15 +192,15 @@ func CreateEtcdPeerCertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error {
// CreateAPIServerEtcdClientCertAndKeyFiles create a new client certificate for the apiserver calling etcd // CreateAPIServerEtcdClientCertAndKeyFiles create a new client certificate for the apiserver calling etcd
// If the apiserver-etcd-client certificate and key file already exist in the target folder, they are used only if evaluated equal; otherwise an error is returned. // If the apiserver-etcd-client certificate and key file already exist in the target folder, they are used only if evaluated equal; otherwise an error is returned.
// It assumes the cluster CA certificate and key file exist in the CertificatesDir // It assumes the etcd CA certificate and key file exist in the CertificatesDir
func CreateAPIServerEtcdClientCertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error { func CreateAPIServerEtcdClientCertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error {
caCert, caKey, err := loadCertificateAuthority(cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName) etcdCACert, etcdCAKey, err := loadCertificateAuthority(cfg.CertificatesDir, kubeadmconstants.EtcdCACertAndKeyBaseName)
if err != nil { if err != nil {
return err return err
} }
apiEtcdClientCert, apiEtcdClientKey, err := NewAPIServerEtcdClientCertAndKey(caCert, caKey) apiEtcdClientCert, apiEtcdClientKey, err := NewAPIServerEtcdClientCertAndKey(etcdCACert, etcdCAKey)
if err != nil { if err != nil {
return err return err
} }
...@@ -188,7 +208,7 @@ func CreateAPIServerEtcdClientCertAndKeyFiles(cfg *kubeadmapi.MasterConfiguratio ...@@ -188,7 +208,7 @@ func CreateAPIServerEtcdClientCertAndKeyFiles(cfg *kubeadmapi.MasterConfiguratio
return writeCertificateFilesIfNotExist( return writeCertificateFilesIfNotExist(
cfg.CertificatesDir, cfg.CertificatesDir,
kubeadmconstants.APIServerEtcdClientCertAndKeyBaseName, kubeadmconstants.APIServerEtcdClientCertAndKeyBaseName,
caCert, etcdCACert,
apiEtcdClientCert, apiEtcdClientCert,
apiEtcdClientKey, apiEtcdClientKey,
) )
...@@ -302,6 +322,17 @@ func NewAPIServerKubeletClientCertAndKey(caCert *x509.Certificate, caKey *rsa.Pr ...@@ -302,6 +322,17 @@ func NewAPIServerKubeletClientCertAndKey(caCert *x509.Certificate, caKey *rsa.Pr
return apiClientCert, apiClientKey, nil return apiClientCert, apiClientKey, nil
} }
// NewEtcdCACertAndKey generate a self signed etcd CA.
func NewEtcdCACertAndKey() (*x509.Certificate, *rsa.PrivateKey, error) {
etcdCACert, etcdCAKey, err := pkiutil.NewCertificateAuthority()
if err != nil {
return nil, nil, fmt.Errorf("failure while generating etcd CA certificate and key: %v", err)
}
return etcdCACert, etcdCAKey, nil
}
// NewEtcdServerCertAndKey generate CA certificate for etcd, signed by the given CA. // NewEtcdServerCertAndKey generate CA certificate for etcd, signed by the given CA.
func NewEtcdServerCertAndKey(cfg *kubeadmapi.MasterConfiguration, caCert *x509.Certificate, caKey *rsa.PrivateKey) (*x509.Certificate, *rsa.PrivateKey, error) { func NewEtcdServerCertAndKey(cfg *kubeadmapi.MasterConfiguration, caCert *x509.Certificate, caKey *rsa.PrivateKey) (*x509.Certificate, *rsa.PrivateKey, error) {
......
cmd/kubeadm/app/phases/certs/certs_test.go
View file @ 2d9b2d9f
...@@ -310,6 +310,15 @@ func TestNewAPIServerKubeletClientCertAndKey(t *testing.T) { ...@@ -310,6 +310,15 @@ func TestNewAPIServerKubeletClientCertAndKey(t *testing.T) {
certstestutil.AssertCertificateHasOrganizations(t, apiKubeletClientCert, kubeadmconstants.MastersGroup) certstestutil.AssertCertificateHasOrganizations(t, apiKubeletClientCert, kubeadmconstants.MastersGroup)
} }
func TestNewEtcdCACertAndKey(t *testing.T) {
etcdCACert, _, err := NewEtcdCACertAndKey()
if err != nil {
t.Fatalf("failed creation of cert and key: %v", err)
}
certstestutil.AssertCertificateIsCa(t, etcdCACert)
}
func TestNewEtcdServerCertAndKey(t *testing.T) { func TestNewEtcdServerCertAndKey(t *testing.T) {
proxy := "user-etcd-proxy" proxy := "user-etcd-proxy"
proxyIP := "10.10.10.100" proxyIP := "10.10.10.100"
...@@ -583,6 +592,7 @@ func TestCreateCertificateFilesMethods(t *testing.T) { ...@@ -583,6 +592,7 @@ func TestCreateCertificateFilesMethods(t *testing.T) {
kubeadmconstants.CACertName, kubeadmconstants.CAKeyName, kubeadmconstants.CACertName, kubeadmconstants.CAKeyName,
kubeadmconstants.APIServerCertName, kubeadmconstants.APIServerKeyName, kubeadmconstants.APIServerCertName, kubeadmconstants.APIServerKeyName,
kubeadmconstants.APIServerKubeletClientCertName, kubeadmconstants.APIServerKubeletClientKeyName, kubeadmconstants.APIServerKubeletClientCertName, kubeadmconstants.APIServerKubeletClientKeyName,
kubeadmconstants.EtcdCACertName, kubeadmconstants.EtcdCAKeyName,
kubeadmconstants.EtcdServerCertName, kubeadmconstants.EtcdServerKeyName, kubeadmconstants.EtcdServerCertName, kubeadmconstants.EtcdServerKeyName,
kubeadmconstants.EtcdPeerCertName, kubeadmconstants.EtcdPeerKeyName, kubeadmconstants.EtcdPeerCertName, kubeadmconstants.EtcdPeerKeyName,
kubeadmconstants.APIServerEtcdClientCertName, kubeadmconstants.APIServerEtcdClientKeyName, kubeadmconstants.APIServerEtcdClientCertName, kubeadmconstants.APIServerEtcdClientKeyName,
...@@ -606,17 +616,21 @@ func TestCreateCertificateFilesMethods(t *testing.T) { ...@@ -606,17 +616,21 @@ func TestCreateCertificateFilesMethods(t *testing.T) {
expectedFiles: []string{kubeadmconstants.APIServerKubeletClientCertName, kubeadmconstants.APIServerKubeletClientKeyName}, expectedFiles: []string{kubeadmconstants.APIServerKubeletClientCertName, kubeadmconstants.APIServerKubeletClientKeyName},
}, },
{ {
setupFunc: CreateCACertAndKeyfiles, createFunc: CreateEtcdCACertAndKeyFiles,
expectedFiles: []string{kubeadmconstants.EtcdCACertName, kubeadmconstants.EtcdCAKeyName},
},
{
setupFunc: CreateEtcdCACertAndKeyFiles,
createFunc: CreateEtcdServerCertAndKeyFiles, createFunc: CreateEtcdServerCertAndKeyFiles,
expectedFiles: []string{kubeadmconstants.EtcdServerCertName, kubeadmconstants.EtcdServerKeyName}, expectedFiles: []string{kubeadmconstants.EtcdServerCertName, kubeadmconstants.EtcdServerKeyName},
}, },
{ {
setupFunc: CreateCACertAndKeyfiles, setupFunc: CreateEtcdCACertAndKeyFiles,
createFunc: CreateEtcdPeerCertAndKeyFiles, createFunc: CreateEtcdPeerCertAndKeyFiles,
expectedFiles: []string{kubeadmconstants.EtcdPeerCertName, kubeadmconstants.EtcdPeerKeyName}, expectedFiles: []string{kubeadmconstants.EtcdPeerCertName, kubeadmconstants.EtcdPeerKeyName},
}, },
{ {
setupFunc: CreateCACertAndKeyfiles, setupFunc: CreateEtcdCACertAndKeyFiles,
createFunc: CreateAPIServerEtcdClientCertAndKeyFiles, createFunc: CreateAPIServerEtcdClientCertAndKeyFiles,
expectedFiles: []string{kubeadmconstants.APIServerEtcdClientCertName, kubeadmconstants.APIServerEtcdClientKeyName}, expectedFiles: []string{kubeadmconstants.APIServerEtcdClientCertName, kubeadmconstants.APIServerEtcdClientKeyName},
}, },
......
cmd/kubeadm/app/phases/certs/doc.go
View file @ 2d9b2d9f
...@@ -40,6 +40,8 @@ package certs ...@@ -40,6 +40,8 @@ package certs
- apiserver-kubelet-client.key - apiserver-kubelet-client.key
- apiserver-etcd-client.crt - apiserver-etcd-client.crt
- apiserver-etcd-client.key - apiserver-etcd-client.key
- etcd/ca.crt
- etcd/ca.key
- etcd/server.crt - etcd/server.crt
- etcd/server.key - etcd/server.key
- etcd/peer.crt - etcd/peer.crt
......
cmd/kubeadm/app/phases/controlplane/manifests.go
View file @ 2d9b2d9f
...@@ -206,7 +206,7 @@ func getAPIServerCommand(cfg *kubeadmapi.MasterConfiguration, k8sVersion *versio ...@@ -206,7 +206,7 @@ func getAPIServerCommand(cfg *kubeadmapi.MasterConfiguration, k8sVersion *versio
} else { } else {
// Default to etcd static pod on localhost // Default to etcd static pod on localhost
etcdEndpointsArg := "--etcd-servers=https://127.0.0.1:2379" etcdEndpointsArg := "--etcd-servers=https://127.0.0.1:2379"
etcdCAFileArg := fmt.Sprintf("--etcd-cafile=%s", filepath.Join(cfg.CertificatesDir, kubeadmconstants.CACertName)) etcdCAFileArg := fmt.Sprintf("--etcd-cafile=%s", filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdCACertName))
etcdClientFileArg := fmt.Sprintf("--etcd-certfile=%s", filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerEtcdClientCertName)) etcdClientFileArg := fmt.Sprintf("--etcd-certfile=%s", filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerEtcdClientCertName))
etcdKeyFileArg := fmt.Sprintf("--etcd-keyfile=%s", filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerEtcdClientKeyName)) etcdKeyFileArg := fmt.Sprintf("--etcd-keyfile=%s", filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerEtcdClientKeyName))
command = append(command, etcdEndpointsArg, etcdCAFileArg, etcdClientFileArg, etcdKeyFileArg) command = append(command, etcdEndpointsArg, etcdCAFileArg, etcdClientFileArg, etcdKeyFileArg)
......
cmd/kubeadm/app/phases/controlplane/manifests_test.go
View file @ 2d9b2d9f
...@@ -225,7 +225,7 @@ func TestGetAPIServerCommand(t *testing.T) { ...@@ -225,7 +225,7 @@ func TestGetAPIServerCommand(t *testing.T) {
"--authorization-mode=Node,RBAC", "--authorization-mode=Node,RBAC",
"--advertise-address=1.2.3.4", "--advertise-address=1.2.3.4",
"--etcd-servers=https://127.0.0.1:2379", "--etcd-servers=https://127.0.0.1:2379",
"--etcd-cafile=" + testCertsDir + "/ca.crt", "--etcd-cafile=" + testCertsDir + "/etcd/ca.crt",
"--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt", "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt",
"--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key", "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key",
}, },
...@@ -262,7 +262,7 @@ func TestGetAPIServerCommand(t *testing.T) { ...@@ -262,7 +262,7 @@ func TestGetAPIServerCommand(t *testing.T) {
"--authorization-mode=Node,RBAC", "--authorization-mode=Node,RBAC",
"--advertise-address=1.2.3.4", "--advertise-address=1.2.3.4",
"--etcd-servers=https://127.0.0.1:2379", "--etcd-servers=https://127.0.0.1:2379",
"--etcd-cafile=" + testCertsDir + "/ca.crt", "--etcd-cafile=" + testCertsDir + "/etcd/ca.crt",
"--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt", "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt",
"--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key", "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key",
}, },
...@@ -299,7 +299,7 @@ func TestGetAPIServerCommand(t *testing.T) { ...@@ -299,7 +299,7 @@ func TestGetAPIServerCommand(t *testing.T) {
"--authorization-mode=Node,RBAC", "--authorization-mode=Node,RBAC",
"--advertise-address=4.3.2.1", "--advertise-address=4.3.2.1",
"--etcd-servers=https://127.0.0.1:2379", "--etcd-servers=https://127.0.0.1:2379",
"--etcd-cafile=" + testCertsDir + "/ca.crt", "--etcd-cafile=" + testCertsDir + "/etcd/ca.crt",
"--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt", "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt",
"--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key", "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key",
}, },
...@@ -337,7 +337,7 @@ func TestGetAPIServerCommand(t *testing.T) { ...@@ -337,7 +337,7 @@ func TestGetAPIServerCommand(t *testing.T) {
"--authorization-mode=Node,RBAC", "--authorization-mode=Node,RBAC",
"--advertise-address=4.3.2.1", "--advertise-address=4.3.2.1",
"--etcd-servers=https://127.0.0.1:2379", "--etcd-servers=https://127.0.0.1:2379",
"--etcd-cafile=" + testCertsDir + "/ca.crt", "--etcd-cafile=" + testCertsDir + "/etcd/ca.crt",
"--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt", "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt",
"--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key", "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key",
}, },
...@@ -380,7 +380,7 @@ func TestGetAPIServerCommand(t *testing.T) { ...@@ -380,7 +380,7 @@ func TestGetAPIServerCommand(t *testing.T) {
"--authorization-mode=Node,RBAC", "--authorization-mode=Node,RBAC",
"--advertise-address=4.3.2.1", "--advertise-address=4.3.2.1",
"--etcd-servers=https://127.0.0.1:2379", "--etcd-servers=https://127.0.0.1:2379",
"--etcd-cafile=" + testCertsDir + "/ca.crt", "--etcd-cafile=" + testCertsDir + "/etcd/ca.crt",
"--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt", "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt",
"--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key", "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key",
}, },
...@@ -418,7 +418,7 @@ func TestGetAPIServerCommand(t *testing.T) { ...@@ -418,7 +418,7 @@ func TestGetAPIServerCommand(t *testing.T) {
"--authorization-mode=Node,RBAC", "--authorization-mode=Node,RBAC",
"--advertise-address=2001:db8::1", "--advertise-address=2001:db8::1",
"--etcd-servers=https://127.0.0.1:2379", "--etcd-servers=https://127.0.0.1:2379",
"--etcd-cafile=" + testCertsDir + "/ca.crt", "--etcd-cafile=" + testCertsDir + "/etcd/ca.crt",
"--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt", "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt",
"--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key", "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key",
}, },
...@@ -456,7 +456,7 @@ func TestGetAPIServerCommand(t *testing.T) { ...@@ -456,7 +456,7 @@ func TestGetAPIServerCommand(t *testing.T) {
"--authorization-mode=Node,RBAC", "--authorization-mode=Node,RBAC",
"--advertise-address=2001:db8::1", "--advertise-address=2001:db8::1",
"--etcd-servers=https://127.0.0.1:2379", "--etcd-servers=https://127.0.0.1:2379",
"--etcd-cafile=" + testCertsDir + "/ca.crt", "--etcd-cafile=" + testCertsDir + "/etcd/ca.crt",
"--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt", "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt",
"--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key", "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key",
}, },
...@@ -569,7 +569,7 @@ func TestGetAPIServerCommand(t *testing.T) { ...@@ -569,7 +569,7 @@ func TestGetAPIServerCommand(t *testing.T) {
"--authorization-mode=Node,RBAC", "--authorization-mode=Node,RBAC",
"--advertise-address=2001:db8::1", "--advertise-address=2001:db8::1",
"--etcd-servers=https://127.0.0.1:2379", "--etcd-servers=https://127.0.0.1:2379",
"--etcd-cafile=" + testCertsDir + "/ca.crt", "--etcd-cafile=" + testCertsDir + "/etcd/ca.crt",
"--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt", "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt",
"--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key", "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key",
}, },
...@@ -610,7 +610,7 @@ func TestGetAPIServerCommand(t *testing.T) { ...@@ -610,7 +610,7 @@ func TestGetAPIServerCommand(t *testing.T) {
"--authorization-mode=Node,RBAC", "--authorization-mode=Node,RBAC",
"--advertise-address=2001:db8::1", "--advertise-address=2001:db8::1",
"--etcd-servers=https://127.0.0.1:2379", "--etcd-servers=https://127.0.0.1:2379",
"--etcd-cafile=" + testCertsDir + "/ca.crt", "--etcd-cafile=" + testCertsDir + "/etcd/ca.crt",
"--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt", "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt",
"--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key", "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key",
fmt.Sprintf("--endpoint-reconciler-type=%s", reconcilers.LeaseEndpointReconcilerType), fmt.Sprintf("--endpoint-reconciler-type=%s", reconcilers.LeaseEndpointReconcilerType),
...@@ -652,7 +652,7 @@ func TestGetAPIServerCommand(t *testing.T) { ...@@ -652,7 +652,7 @@ func TestGetAPIServerCommand(t *testing.T) {
"--authorization-mode=Node,RBAC", "--authorization-mode=Node,RBAC",
"--advertise-address=1.2.3.4", "--advertise-address=1.2.3.4",
"--etcd-servers=https://127.0.0.1:2379", "--etcd-servers=https://127.0.0.1:2379",
"--etcd-cafile=" + testCertsDir + "/ca.crt", "--etcd-cafile=" + testCertsDir + "/etcd/ca.crt",
"--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt", "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt",
"--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key", "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key",
"--cloud-provider=gce", "--cloud-provider=gce",
...@@ -691,7 +691,7 @@ func TestGetAPIServerCommand(t *testing.T) { ...@@ -691,7 +691,7 @@ func TestGetAPIServerCommand(t *testing.T) {
"--authorization-mode=Node,RBAC", "--authorization-mode=Node,RBAC",
"--advertise-address=1.2.3.4", "--advertise-address=1.2.3.4",
"--etcd-servers=https://127.0.0.1:2379", "--etcd-servers=https://127.0.0.1:2379",
"--etcd-cafile=" + testCertsDir + "/ca.crt", "--etcd-cafile=" + testCertsDir + "/etcd/ca.crt",
"--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt", "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt",
"--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key", "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key",
"--cloud-provider=aws", "--cloud-provider=aws",
......
cmd/kubeadm/app/phases/etcd/local.go
View file @ 2d9b2d9f
...@@ -30,7 +30,7 @@ import ( ...@@ -30,7 +30,7 @@ import (
const ( const (
etcdVolumeName = "etcd-data" etcdVolumeName = "etcd-data"
certsVolumeName = "k8s-certs" certsVolumeName = "etcd-certs"
) )
// CreateLocalEtcdStaticPodManifestFile will write local etcd static pod manifest file. // CreateLocalEtcdStaticPodManifestFile will write local etcd static pod manifest file.
...@@ -53,7 +53,7 @@ func GetEtcdPodSpec(cfg *kubeadmapi.MasterConfiguration) v1.Pod { ...@@ -53,7 +53,7 @@ func GetEtcdPodSpec(cfg *kubeadmapi.MasterConfiguration) v1.Pod {
pathType := v1.HostPathDirectoryOrCreate pathType := v1.HostPathDirectoryOrCreate
etcdMounts := map[string]v1.Volume{ etcdMounts := map[string]v1.Volume{
etcdVolumeName: staticpodutil.NewVolume(etcdVolumeName, cfg.Etcd.DataDir, &pathType), etcdVolumeName: staticpodutil.NewVolume(etcdVolumeName, cfg.Etcd.DataDir, &pathType),
certsVolumeName: staticpodutil.NewVolume(certsVolumeName, cfg.CertificatesDir, &pathType), certsVolumeName: staticpodutil.NewVolume(certsVolumeName, cfg.CertificatesDir+"/etcd", &pathType),
} }
return staticpodutil.ComponentPod(v1.Container{ return staticpodutil.ComponentPod(v1.Container{
Name: kubeadmconstants.Etcd, Name: kubeadmconstants.Etcd,
...@@ -63,7 +63,7 @@ func GetEtcdPodSpec(cfg *kubeadmapi.MasterConfiguration) v1.Pod { ...@@ -63,7 +63,7 @@ func GetEtcdPodSpec(cfg *kubeadmapi.MasterConfiguration) v1.Pod {
// Mount the etcd datadir path read-write so etcd can store data in a more persistent manner // Mount the etcd datadir path read-write so etcd can store data in a more persistent manner
VolumeMounts: []v1.VolumeMount{ VolumeMounts: []v1.VolumeMount{
staticpodutil.NewVolumeMount(etcdVolumeName, cfg.Etcd.DataDir, false), staticpodutil.NewVolumeMount(etcdVolumeName, cfg.Etcd.DataDir, false),
staticpodutil.NewVolumeMount(certsVolumeName, cfg.CertificatesDir, false), staticpodutil.NewVolumeMount(certsVolumeName, cfg.CertificatesDir+"/etcd", false),
}, },
LivenessProbe: staticpodutil.ComponentProbe(cfg, kubeadmconstants.Etcd, 2379, "/health", v1.URISchemeHTTP), LivenessProbe: staticpodutil.ComponentProbe(cfg, kubeadmconstants.Etcd, 2379, "/health", v1.URISchemeHTTP),
}, etcdMounts) }, etcdMounts)
...@@ -77,11 +77,11 @@ func getEtcdCommand(cfg *kubeadmapi.MasterConfiguration) []string { ...@@ -77,11 +77,11 @@ func getEtcdCommand(cfg *kubeadmapi.MasterConfiguration) []string {
"data-dir": cfg.Etcd.DataDir, "data-dir": cfg.Etcd.DataDir,
"cert-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdServerCertName), "cert-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdServerCertName),
"key-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdServerKeyName), "key-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdServerKeyName),
"trusted-ca-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.CACertName), "trusted-ca-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdCACertName),
"client-cert-auth": "true", "client-cert-auth": "true",
"peer-cert-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdPeerCertName), "peer-cert-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdPeerCertName),
"peer-key-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdPeerKeyName), "peer-key-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdPeerKeyName),
"peer-trusted-ca-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.CACertName), "peer-trusted-ca-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdCACertName),
"peer-client-cert-auth": "true", "peer-client-cert-auth": "true",
} }
......
cmd/kubeadm/app/phases/etcd/local_test.go
View file @ 2d9b2d9f
...@@ -84,11 +84,11 @@ func TestGetEtcdCommand(t *testing.T) { ...@@ -84,11 +84,11 @@ func TestGetEtcdCommand(t *testing.T) {
"--data-dir=/var/lib/etcd", "--data-dir=/var/lib/etcd",
"--cert-file=" + kubeadmconstants.EtcdServerCertName, "--cert-file=" + kubeadmconstants.EtcdServerCertName,
"--key-file=" + kubeadmconstants.EtcdServerKeyName, "--key-file=" + kubeadmconstants.EtcdServerKeyName,
"--trusted-ca-file=" + kubeadmconstants.CACertName, "--trusted-ca-file=" + kubeadmconstants.EtcdCACertName,
"--client-cert-auth=true", "--client-cert-auth=true",
"--peer-cert-file=" + kubeadmconstants.EtcdPeerCertName, "--peer-cert-file=" + kubeadmconstants.EtcdPeerCertName,
"--peer-key-file=" + kubeadmconstants.EtcdPeerKeyName, "--peer-key-file=" + kubeadmconstants.EtcdPeerKeyName,
"--peer-trusted-ca-file=" + kubeadmconstants.CACertName, "--peer-trusted-ca-file=" + kubeadmconstants.EtcdCACertName,
"--peer-client-cert-auth=true", "--peer-client-cert-auth=true",
}, },
}, },
...@@ -109,11 +109,11 @@ func TestGetEtcdCommand(t *testing.T) { ...@@ -109,11 +109,11 @@ func TestGetEtcdCommand(t *testing.T) {
"--data-dir=/var/lib/etcd", "--data-dir=/var/lib/etcd",
"--cert-file=" + kubeadmconstants.EtcdServerCertName, "--cert-file=" + kubeadmconstants.EtcdServerCertName,
"--key-file=" + kubeadmconstants.EtcdServerKeyName, "--key-file=" + kubeadmconstants.EtcdServerKeyName,
"--trusted-ca-file=" + kubeadmconstants.CACertName, "--trusted-ca-file=" + kubeadmconstants.EtcdCACertName,
"--client-cert-auth=true", "--client-cert-auth=true",
"--peer-cert-file=" + kubeadmconstants.EtcdPeerCertName, "--peer-cert-file=" + kubeadmconstants.EtcdPeerCertName,
"--peer-key-file=" + kubeadmconstants.EtcdPeerKeyName, "--peer-key-file=" + kubeadmconstants.EtcdPeerKeyName,
"--peer-trusted-ca-file=" + kubeadmconstants.CACertName, "--peer-trusted-ca-file=" + kubeadmconstants.EtcdCACertName,
"--peer-client-cert-auth=true", "--peer-client-cert-auth=true",
}, },
}, },
...@@ -128,11 +128,11 @@ func TestGetEtcdCommand(t *testing.T) { ...@@ -128,11 +128,11 @@ func TestGetEtcdCommand(t *testing.T) {
"--data-dir=/etc/foo", "--data-dir=/etc/foo",
"--cert-file=" + kubeadmconstants.EtcdServerCertName, "--cert-file=" + kubeadmconstants.EtcdServerCertName,
"--key-file=" + kubeadmconstants.EtcdServerKeyName, "--key-file=" + kubeadmconstants.EtcdServerKeyName,
"--trusted-ca-file=" + kubeadmconstants.CACertName, "--trusted-ca-file=" + kubeadmconstants.EtcdCACertName,
"--client-cert-auth=true", "--client-cert-auth=true",
"--peer-cert-file=" + kubeadmconstants.EtcdPeerCertName, "--peer-cert-file=" + kubeadmconstants.EtcdPeerCertName,
"--peer-key-file=" + kubeadmconstants.EtcdPeerKeyName, "--peer-key-file=" + kubeadmconstants.EtcdPeerKeyName,
"--peer-trusted-ca-file=" + kubeadmconstants.CACertName, "--peer-trusted-ca-file=" + kubeadmconstants.EtcdCACertName,
"--peer-client-cert-auth=true", "--peer-client-cert-auth=true",
}, },
}, },
......
cmd/kubeadm/app/phases/upgrade/staticpods.go
View file @ 2d9b2d9f
...@@ -136,17 +136,22 @@ func upgradeComponent(component string, waiter apiclient.Waiter, pathMgr StaticP ...@@ -136,17 +136,22 @@ func upgradeComponent(component string, waiter apiclient.Waiter, pathMgr StaticP
} }
// ensure etcd certs are generated for etcd and kube-apiserver // ensure etcd certs are generated for etcd and kube-apiserver
if component == constants.Etcd || component == constants.KubeAPIServer {
if err := certsphase.CreateEtcdCACertAndKeyFiles(cfg); err != nil {
return fmt.Errorf("failed to upgrade the %s CA certificate and key: %v", constants.Etcd, err)
}
}
if component == constants.Etcd { if component == constants.Etcd {
if err := certsphase.CreateEtcdServerCertAndKeyFiles(cfg); err != nil { if err := certsphase.CreateEtcdServerCertAndKeyFiles(cfg); err != nil {
return fmt.Errorf("failed to upgrade the %s certificate: %v", constants.Etcd, err) return fmt.Errorf("failed to upgrade the %s certificate and key: %v", constants.Etcd, err)
} }
if err := certsphase.CreateEtcdPeerCertAndKeyFiles(cfg); err != nil { if err := certsphase.CreateEtcdPeerCertAndKeyFiles(cfg); err != nil {
return fmt.Errorf("failed to upgrade the %s peer certificate: %v", constants.Etcd, err) return fmt.Errorf("failed to upgrade the %s peer certificate and key: %v", constants.Etcd, err)
} }
} }
if component == constants.KubeAPIServer { if component == constants.KubeAPIServer {
if err := certsphase.CreateAPIServerEtcdClientCertAndKeyFiles(cfg); err != nil { if err := certsphase.CreateAPIServerEtcdClientCertAndKeyFiles(cfg); err != nil {
return fmt.Errorf("failed to upgrade the %s %s-client certificate: %v", constants.KubeAPIServer, constants.Etcd, err) return fmt.Errorf("failed to upgrade the %s %s-client certificate and key: %v", constants.KubeAPIServer, constants.Etcd, err)
} }
} }
......
cmd/kubeadm/app/phases/upgrade/staticpods_test.go
View file @ 2d9b2d9f
...@@ -324,6 +324,7 @@ func TestStaticPodControlPlane(t *testing.T) { ...@@ -324,6 +324,7 @@ func TestStaticPodControlPlane(t *testing.T) {
certsphase.CreateCACertAndKeyfiles, certsphase.CreateCACertAndKeyfiles,
certsphase.CreateAPIServerCertAndKeyFiles, certsphase.CreateAPIServerCertAndKeyFiles,
certsphase.CreateAPIServerKubeletClientCertAndKeyFiles, certsphase.CreateAPIServerKubeletClientCertAndKeyFiles,
// certsphase.CreateEtcdCACertAndKeyFiles,
// certsphase.CreateEtcdServerCertAndKeyFiles, // certsphase.CreateEtcdServerCertAndKeyFiles,
// certsphase.CreateEtcdPeerCertAndKeyFiles, // certsphase.CreateEtcdPeerCertAndKeyFiles,
// certsphase.CreateAPIServerEtcdClientCertAndKeyFiles, // certsphase.CreateAPIServerEtcdClientCertAndKeyFiles,
......
docs/.generated_docs
View file @ 2d9b2d9f
...@@ -24,6 +24,7 @@ docs/admin/kubeadm_alpha_phase_certs_apiserver-etcd-client.md ...@@ -24,6 +24,7 @@ docs/admin/kubeadm_alpha_phase_certs_apiserver-etcd-client.md
docs/admin/kubeadm_alpha_phase_certs_apiserver-kubelet-client.md docs/admin/kubeadm_alpha_phase_certs_apiserver-kubelet-client.md
docs/admin/kubeadm_alpha_phase_certs_apiserver.md docs/admin/kubeadm_alpha_phase_certs_apiserver.md
docs/admin/kubeadm_alpha_phase_certs_ca.md docs/admin/kubeadm_alpha_phase_certs_ca.md
docs/admin/kubeadm_alpha_phase_certs_etcd-ca.md
docs/admin/kubeadm_alpha_phase_certs_etcd-peer.md docs/admin/kubeadm_alpha_phase_certs_etcd-peer.md
docs/admin/kubeadm_alpha_phase_certs_etcd-server.md docs/admin/kubeadm_alpha_phase_certs_etcd-server.md
docs/admin/kubeadm_alpha_phase_certs_front-proxy-ca.md docs/admin/kubeadm_alpha_phase_certs_front-proxy-ca.md
...@@ -90,6 +91,7 @@ docs/man/man1/kubeadm-alpha-phase-certs-apiserver-etcd-client.1 ...@@ -90,6 +91,7 @@ docs/man/man1/kubeadm-alpha-phase-certs-apiserver-etcd-client.1
docs/man/man1/kubeadm-alpha-phase-certs-apiserver-kubelet-client.1 docs/man/man1/kubeadm-alpha-phase-certs-apiserver-kubelet-client.1
docs/man/man1/kubeadm-alpha-phase-certs-apiserver.1 docs/man/man1/kubeadm-alpha-phase-certs-apiserver.1
docs/man/man1/kubeadm-alpha-phase-certs-ca.1 docs/man/man1/kubeadm-alpha-phase-certs-ca.1
docs/man/man1/kubeadm-alpha-phase-certs-etcd-ca.1
docs/man/man1/kubeadm-alpha-phase-certs-etcd-peer.1 docs/man/man1/kubeadm-alpha-phase-certs-etcd-peer.1
docs/man/man1/kubeadm-alpha-phase-certs-etcd-server.1 docs/man/man1/kubeadm-alpha-phase-certs-etcd-server.1
docs/man/man1/kubeadm-alpha-phase-certs-front-proxy-ca.1 docs/man/man1/kubeadm-alpha-phase-certs-front-proxy-ca.1
......
docs/admin/kubeadm_alpha_phase_certs_etcd-ca.md 0 → 100644
View file @ 2d9b2d9f
This file is autogenerated, but we've stopped checking such files into the
repository to reduce the need for rebases. Please run hack/generate-docs.sh to
populate this file.
docs/man/man1/kubeadm-alpha-phase-certs-etcd-ca.1 0 → 100644
View file @ 2d9b2d9f
This file is autogenerated, but we've stopped checking such files into the
repository to reduce the need for rebases. Please run hack/generate-docs.sh to
populate this file.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment