Skip to content

K
k3s
Commit 5d69d6e7 authored by Vitor Savian's avatar Vitor Savian
Browse files
Options

Add tls for kine

Signed-off-by: 's avatarVitor Savian <vitor.savian@suse.com> Bump kine Signed-off-by: 's avatarVitor Savian <vitor.savian@suse.com> Add integration tests for kine with tls Signed-off-by: 's avatarVitor Savian <vitor.savian@suse.com>
parent c51d7bfb
Hide whitespace changes
Inline Side-by-side
Showing with 131 additions and 45 deletions
go.mod
View file @ 5d69d6e7
......@@ -123,8 +123,8 @@ require (
github.com/ipfs/go-log/v2 v2.5.1
github.com/json-iterator/go v1.1.12
github.com/k3s-io/helm-controller v0.15.9
github.com/k3s-io/kine v0.11.4
github.com/klauspost/compress v1.17.2
github.com/k3s-io/kine v0.11.7
github.com/klauspost/compress v1.17.7
github.com/kubernetes-sigs/cri-tools v0.0.0-00010101000000-000000000000
github.com/lib/pq v1.10.2
github.com/libp2p/go-libp2p v0.30.0
......@@ -159,10 +159,10 @@ require (
go.etcd.io/etcd/etcdutl/v3 v3.5.9
go.etcd.io/etcd/server/v3 v3.5.10
go.uber.org/zap v1.26.0
golang.org/x/crypto v0.18.0
golang.org/x/net v0.20.0
golang.org/x/crypto v0.21.0
golang.org/x/net v0.21.0
golang.org/x/sync v0.6.0
golang.org/x/sys v0.16.0
golang.org/x/sys v0.18.0
google.golang.org/grpc v1.60.1
gopkg.in/yaml.v2 v2.4.0
inet.af/tcpproxy v0.0.0-20200125044825-b6bb9b5b8252
......@@ -330,7 +330,8 @@ require (
github.com/jackc/pgerrcode v0.0.0-20220416144525-469b46aa5efa // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
github.com/jackc/pgx/v5 v5.4.2 // indirect
github.com/jackc/pgx/v5 v5.5.4 // indirect
github.com/jackc/puddle/v2 v2.2.1 // indirect
github.com/jackpal/go-nat-pmp v1.0.2 // indirect
github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect
github.com/jbenet/goprocess v0.1.4 // indirect
......@@ -404,10 +405,10 @@ require (
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
github.com/nats-io/jsm.go v0.0.31-0.20220317133147-fe318f464eee // indirect
github.com/nats-io/jwt/v2 v2.5.3 // indirect
github.com/nats-io/nats-server/v2 v2.10.5 // indirect
github.com/nats-io/nats.go v1.31.0 // indirect
github.com/nats-io/nkeys v0.4.6 // indirect
github.com/nats-io/jwt/v2 v2.5.5 // indirect
github.com/nats-io/nats-server/v2 v2.10.12 // indirect
github.com/nats-io/nats.go v1.34.0 // indirect
github.com/nats-io/nkeys v0.4.7 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.1.0-rc5 // indirect
......
go.sum
View file @ 5d69d6e7
......@@ -905,8 +905,10 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk=
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
github.com/jackc/pgx/v5 v5.4.2 h1:u1gmGDwbdRUZiwisBm/Ky2M14uQyUP65bG8+20nnyrg=
github.com/jackc/pgx/v5 v5.4.2/go.mod h1:q6iHT8uDNXWiFNOlRqJzBTaSH3+2xCXkokxHZC5qWFY=
github.com/jackc/pgx/v5 v5.5.4 h1:Xp2aQS8uXButQdnCMWNmvx6UysWQQC+u1EoizjguY+8=
github.com/jackc/pgx/v5 v5.5.4/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A=
github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk=
github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus=
github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc=
github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA=
......@@ -959,8 +961,8 @@ github.com/k3s-io/etcd/server/v3 v3.5.9-k3s1 h1:B3039IkTPnwQEt4tIMjC6yd6b1Q3Z9ZZ
github.com/k3s-io/etcd/server/v3 v3.5.9-k3s1/go.mod h1:GgI1fQClQCFIzuVjlvdbMxNbnISt90gdfYyqiAIt65g=
github.com/k3s-io/helm-controller v0.15.9 h1:eBZq0KkZCDyWh4og+tyI43Nt9T5TNjc7QCFhAt1aR64=
github.com/k3s-io/helm-controller v0.15.9/go.mod h1:AYitg40howLjKloL/zdjDDOPL1jg/K5R4af0tQcyPR8=
github.com/k3s-io/kine v0.11.4 h1:ZIXQT4vPPKNL9DwLF4dQ11tWtpJ1C/7OKNIpFmTkImo=
github.com/k3s-io/kine v0.11.4/go.mod h1:NmwOWsWgB3aScq5+LEYytAaceqkG7lmCLLjjrWug8v4=
github.com/k3s-io/kine v0.11.7 h1:+I4TrxozQv4cdmD8RULI35r4o5G+A7gOD3F75lfjDP0=
github.com/k3s-io/kine v0.11.7/go.mod h1:4C/zNVwl3FU1EubA2ju1Hq36JIjp8gAZaM+Hfnuvqt4=
github.com/k3s-io/klog/v2 v2.100.1-k3s1 h1:xb/Ta8dpQuIZueQEw2YTZUYrKoILdBmPiITVkNmYPa0=
github.com/k3s-io/klog/v2 v2.100.1-k3s1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
github.com/k3s-io/kube-router/v2 v2.0.1 h1:UCsdkQjSfOkVakixilRDDkG9yq775GBSKxBfsyUj8ng=
......@@ -1039,8 +1041,8 @@ github.com/klauspost/compress v1.14.4/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47e
github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
github.com/klauspost/compress v1.16.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4=
github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg=
github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
......@@ -1271,17 +1273,17 @@ github.com/natefinch/lumberjack v2.0.0+incompatible/go.mod h1:Wi9p2TTF5DG5oU+6Yf
github.com/nats-io/jsm.go v0.0.31-0.20220317133147-fe318f464eee h1:+l6i7zS8N1LOokm7dzShezI9STRGrzp0O49Pw8Jetdk=
github.com/nats-io/jsm.go v0.0.31-0.20220317133147-fe318f464eee/go.mod h1:EKSYvbvWAoh0hIfuZ+ieWm8u0VOTRTeDfuQvNPKRqEg=
github.com/nats-io/jwt/v2 v2.2.1-0.20220113022732-58e87895b296/go.mod h1:0tqz9Hlu6bCBFLWAASKhE5vUA4c24L9KPUUgvwumE/k=
github.com/nats-io/jwt/v2 v2.5.3 h1:/9SWvzc6hTfamcgXJ3uYRpgj+QuY2aLNqRiqrKcrpEo=
github.com/nats-io/jwt/v2 v2.5.3/go.mod h1:iysuPemFcc7p4IoYots3IuELSI4EDe9Y0bQMe+I3Bf4=
github.com/nats-io/jwt/v2 v2.5.5 h1:ROfXb50elFq5c9+1ztaUbdlrArNFl2+fQWP6B8HGEq4=
github.com/nats-io/jwt/v2 v2.5.5/go.mod h1:ZdWS1nZa6WMZfFwwgpEaqBV8EPGVgOTDHN/wTbz0Y5A=
github.com/nats-io/nats-server/v2 v2.7.5-0.20220309212130-5c0d1999ff72/go.mod h1:1vZ2Nijh8tcyNe8BDVyTviCd9NYzRbubQYiEHsvOQWc=
github.com/nats-io/nats-server/v2 v2.10.5 h1:hhWt6m9ja/mNnm6ixc85jCthDaiUFPaeJI79K/MD980=
github.com/nats-io/nats-server/v2 v2.10.5/go.mod h1:xUMTU4kS//SDkJCSvFwN9SyJ9nUuLhSkzB/Qz0dvjjg=
github.com/nats-io/nats-server/v2 v2.10.12 h1:G6u+RDrHkw4bkwn7I911O5jqys7jJVRY6MwgndyUsnE=
github.com/nats-io/nats-server/v2 v2.10.12/go.mod h1:H1n6zXtYLFCgXcf/SF8QNTSIFuS8tyZQMN9NguUHdEs=
github.com/nats-io/nats.go v1.13.1-0.20220308171302-2f2f6968e98d/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w=
github.com/nats-io/nats.go v1.31.0 h1:/WFBHEc/dOKBF6qf1TZhrdEfTmOZ5JzdJ+Y3m6Y/p7E=
github.com/nats-io/nats.go v1.31.0/go.mod h1:di3Bm5MLsoB4Bx61CBTsxuarI36WbhAwOm8QrW39+i8=
github.com/nats-io/nats.go v1.34.0 h1:fnxnPCNiwIG5w08rlMcEKTUw4AV/nKyGCOJE8TdhSPk=
github.com/nats-io/nats.go v1.34.0/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8=
github.com/nats-io/nkeys v0.3.0/go.mod h1:gvUNGjVcM2IPr5rCsRsC6Wb3Hr2CQAm08dsxtV6A5y4=
github.com/nats-io/nkeys v0.4.6 h1:IzVe95ru2CT6ta874rt9saQRkWfe2nFj1NtvYSLqMzY=
github.com/nats-io/nkeys v0.4.6/go.mod h1:4DxZNzenSVd1cYQoAa8948QY3QDjrHfcfVADymtkpts=
github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI=
github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo=
......
pkg/cli/cmds/server.go
View file @ 5d69d6e7
......@@ -60,6 +60,7 @@ type Server struct {
DatastoreCAFile string
DatastoreCertFile string
DatastoreKeyFile string
KineTLS bool
AdvertiseIP string
AdvertisePort int
DisableScheduler bool
......@@ -310,6 +311,12 @@ var ServerFlags = []cli.Flag{
Usage: "(flags) Customized flag for kube-cloud-controller-manager process",
Value: &ServerConfig.ExtraCloudControllerArgs,
},
&cli.BoolFlag{
Name: "kine-tls",
Usage: "(experimental/db) Enable TLS on the kine etcd server socket",
Destination: &ServerConfig.KineTLS,
Hidden: true,
},
&cli.StringFlag{
Name: "datastore-endpoint",
Usage: "(db) Specify etcd, NATS, MySQL, Postgres, or SQLite (default) data source name",
......
pkg/cli/server/server.go
View file @ 5d69d6e7
......@@ -152,6 +152,7 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
serverConfig.ControlConfig.Datastore.BackendTLSConfig.CAFile = cfg.DatastoreCAFile
serverConfig.ControlConfig.Datastore.BackendTLSConfig.CertFile = cfg.DatastoreCertFile
serverConfig.ControlConfig.Datastore.BackendTLSConfig.KeyFile = cfg.DatastoreKeyFile
serverConfig.ControlConfig.KineTLS = cfg.KineTLS
serverConfig.ControlConfig.AdvertiseIP = cfg.AdvertiseIP
serverConfig.ControlConfig.AdvertisePort = cfg.AdvertisePort
serverConfig.ControlConfig.FlannelBackend = cfg.FlannelBackend
......
pkg/cluster/cluster.go
View file @ 5d69d6e7
......@@ -84,7 +84,7 @@ func (c *Cluster) Start(ctx context.Context) (<-chan struct{}, error) {
return nil, err
}
if err := c.startStorage(ctx); err != nil {
if err := c.startStorage(ctx, false); err != nil {
return nil, err
}
......@@ -132,12 +132,19 @@ func (c *Cluster) Start(ctx context.Context) (<-chan struct{}, error) {
// This calls into the kine endpoint code, which sets up the database client
// and unix domain socket listener if using an external database. In the case of an etcd
// backend it just returns the user-provided etcd endpoints and tls config.
func (c *Cluster) startStorage(ctx context.Context) error {
if c.storageStarted {
func (c *Cluster) startStorage(ctx context.Context, bootstrap bool) error {
if c.storageStarted && !c.config.KineTLS {
return nil
}
c.storageStarted = true
if !bootstrap {
// set the tls config for the kine storage
c.config.Datastore.ServerTLSConfig.CAFile = c.config.Runtime.ETCDServerCA
c.config.Datastore.ServerTLSConfig.CertFile = c.config.Runtime.ServerETCDCert
c.config.Datastore.ServerTLSConfig.KeyFile = c.config.Runtime.ServerETCDKey
}
// start listening on the kine socket as an etcd endpoint, or return the external etcd endpoints
etcdConfig, err := endpoint.Listen(ctx, c.config.Datastore)
if err != nil {
......@@ -148,9 +155,15 @@ func (c *Cluster) startStorage(ctx context.Context) error {
// based on what the kine wrapper tells us about the datastore. Single-node datastores like sqlite don't require
// leader election, while basically all others (etcd, external database, etc) do since they allow multiple servers.
c.config.Runtime.EtcdConfig = etcdConfig
c.config.Datastore.BackendTLSConfig = etcdConfig.TLSConfig
c.config.Datastore.Endpoint = strings.Join(etcdConfig.Endpoints, ",")
c.config.NoLeaderElect = !etcdConfig.LeaderElect
// after the bootstrap we need to set the args for api-server with kine in unixs or just set the
// values if the datastoreTLS is not enabled
if !bootstrap || !c.config.KineTLS {
c.config.Datastore.BackendTLSConfig = etcdConfig.TLSConfig
c.config.Datastore.Endpoint = strings.Join(etcdConfig.Endpoints, ",")
c.config.NoLeaderElect = !etcdConfig.LeaderElect
}
return nil
}
......
pkg/cluster/storage.go
View file @ 5d69d6e7
......@@ -20,7 +20,6 @@ import (
const maxBootstrapWaitAttempts = 5
func RotateBootstrapToken(ctx context.Context, config *config.Control, oldToken string) error {
token, err := util.ReadTokenFromFile(config.Runtime.ServerToken, config.Runtime.ServerCA, config.DataDir)
if err != nil {
return err
......@@ -151,8 +150,21 @@ func bootstrapKeyData(ctx context.Context, storageClient client.Client) (*client
// bootstrap key as a lock. This function will not return successfully until either the
// bootstrap key has been locked, or data is read into the struct.
func (c *Cluster) storageBootstrap(ctx context.Context) error {
if err := c.startStorage(ctx); err != nil {
return err
if c.config.KineTLS {
bootstrapCtx, cancel := context.WithCancel(ctx)
defer func() {
time.Sleep(time.Second)
cancel()
}()
logrus.Info("Starting temporary kine to reconcile with datastore")
if err := c.startStorage(bootstrapCtx, true); err != nil {
return err
}
} else {
if err := c.startStorage(ctx, true); err != nil {
return err
}
}
storageClient, err := client.New(c.config.Runtime.EtcdConfig)
......
pkg/daemons/config/types.go
View file @ 5d69d6e7
......@@ -179,6 +179,7 @@ type Control struct {
KubeConfigMode string
HelmJobImage string
DataDir string
KineTLS bool
Datastore endpoint.Config `json:"-"`
Disables map[string]bool
DisableAgent bool
......
pkg/daemons/control/deps/deps.go
View file @ 5d69d6e7
......@@ -42,8 +42,7 @@ const (
RequestHeaderCN = "system:auth-proxy"
)
var (
kubeconfigTemplate = template.Must(template.New("kubeconfig").Parse(`apiVersion: v1
var kubeconfigTemplate = template.Must(template.New("kubeconfig").Parse(`apiVersion: v1
clusters:
- cluster:
server: {{.URL}}
......@@ -64,7 +63,6 @@ users:
client-certificate: {{.ClientCert}}
client-key: {{.ClientKey}}
`))
)
func migratePassword(p *passwd.Passwd) error {
server, _ := p.Pass("server")
......@@ -283,9 +281,7 @@ func genEncryptedNetworkInfo(controlConfig *config.Control) error {
}
func getServerPass(passwd *passwd.Passwd, config *config.Control) (string, error) {
var (
err error
)
var err error
serverPass := config.Token
if serverPass == "" {
......@@ -448,14 +444,16 @@ func genServerCerts(config *config.Control) error {
}
func genETCDCerts(config *config.Control) error {
runtime := config.Runtime
regen, err := createSigningCertKey("etcd-server", runtime.ETCDServerCA, runtime.ETCDServerCAKey)
if err != nil {
return err
}
altNames := &certutil.AltNames{}
altNames := &certutil.AltNames{
DNSNames: []string{"kine.sock"},
}
addSANs(altNames, config.SANs)
if _, err := createClientCertKey(regen, "etcd-client", nil,
......@@ -845,5 +843,4 @@ func genCloudConfig(controlConfig *config.Control) error {
return err
}
return os.WriteFile(controlConfig.Runtime.CloudControllerConfig, b, 0600)
}
pkg/daemons/control/server.go
View file @ 5d69d6e7
......@@ -286,7 +286,6 @@ func prepare(ctx context.Context, config *config.Control) error {
deps.CreateRuntimeCertFiles(config)
cluster := cluster.New(config)
if err := cluster.Bootstrap(ctx, config.ClusterReset); err != nil {
return err
}
......
pkg/etcd/etcd.go
View file @ 5d69d6e7
......@@ -379,7 +379,6 @@ func (e *ETCD) Reset(ctx context.Context, rebootstrap func() error) error {
continue
}
}
}
}()
......@@ -442,7 +441,7 @@ func (e *ETCD) Start(ctx context.Context, clientAccessInfo *clientaccess.Info) e
go e.manageLearners(ctx)
if isInitialized {
//check etcd dir permission
// check etcd dir permission
etcdDir := dbDir(e.config)
info, err := os.Stat(etcdDir)
if err != nil {
......
tests/integration/startup/startup_int_test.go
View file @ 5d69d6e7
......@@ -40,6 +40,59 @@ var _ = Describe("startup tests", Ordered, func() {
return testutil.K3sDefaultDeployments()
}, "120s", "5s").Should(Succeed())
})
It("has kine without tls", func() {
Eventually(func() error {
match, err := testutil.SearchK3sLog(startupServer, "Kine available at unix://kine.sock")
if err != nil {
return err
}
if match {
return nil
}
return errors.New("error finding kine sock")
}, "30s", "2s").Should(Succeed())
})
It("does not use kine with tls after bootstrap", func() {
Eventually(func() error {
match, err := testutil.SearchK3sLog(startupServer, "Kine available at unixs://kine.sock")
if err != nil {
return err
}
if match {
return errors.New("Kine with tls when the kine-tls is not set")
}
return nil
}, "30s", "2s").Should(Succeed())
})
It("dies cleanly", func() {
Expect(testutil.K3sKillServer(startupServer)).To(Succeed())
Expect(testutil.K3sCleanup(-1, "")).To(Succeed())
})
})
When("a server with kine-tls is created", func() {
It("is created with kine-tls", func() {
var err error
startupServerArgs = []string{"--kine-tls"}
startupServer, err = testutil.K3sStartServer(startupServerArgs...)
Expect(err).ToNot(HaveOccurred())
})
It("has the default pods deployed", func() {
Eventually(func() error {
return testutil.K3sDefaultDeployments()
}, "120s", "5s").Should(Succeed())
})
It("set kine to use tls", func() {
Eventually(func() error {
match, err := testutil.SearchK3sLog(startupServer, "Kine available at unixs://kine.sock")
if err != nil {
return err
}
if match {
return nil
}
return errors.New("error finding unixs://kine.sock")
}, "30s", "2s").Should(Succeed())
})
It("dies cleanly", func() {
Expect(testutil.K3sKillServer(startupServer)).To(Succeed())
Expect(testutil.K3sCleanup(-1, "")).To(Succeed())
......@@ -312,6 +365,7 @@ var _ = Describe("startup tests", Ordered, func() {
Expect(testutil.K3sCleanup(-1, "")).To(Succeed())
})
})
})
var failed bool
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment