[Release-1.32] Fix upgrade test container version (#12000)

scripts/test

@@ -26,25 +26,25 @@ export K3S_IMAGE="rancher/k3s:${VERSION_TAG}${SUFFIX}"
 # Run all tests on tag events, as we want test failures to block the release
 if [ "$ARCH" == 'arm' ] ||  [ "$DRONE_BUILD_EVENT" = 'tag' ]; then
 
-  go test ./tests/docker/basics/basics_test.go -k3sImage="$K3S_IMAGE"
+  go test ./tests/docker/basics/basics_test.go -k3sImage="$K3S_IMAGE" -ci
   echo "Did go test basics $?"
 
   # Extract v1.XX minor version for skew and upgrade tests
   minor_version=$(echo $VERSION_K8S | cut -d '.' -f1,2)
 
-  go test ./tests/docker/cacerts/cacerts_test.go -k3sImage="$K3S_IMAGE"
+  go test ./tests/docker/cacerts/cacerts_test.go -k3sImage="$K3S_IMAGE" -ci
   echo "Did go test cacerts $?"
 
-  go test ./tests/docker/skew/skew_test.go -k3sImage="$K3S_IMAGE" -channel="$minor_version"
+  go test ./tests/docker/skew/skew_test.go -k3sImage="$K3S_IMAGE" -channel="$minor_version" -ci
   echo "Did go test skew $?"
 
-  go test ./tests/docker/bootstraptoken/bootstraptoken_test.go -k3sImage="$K3S_IMAGE"
+  go test ./tests/docker/bootstraptoken/bootstraptoken_test.go -k3sImage="$K3S_IMAGE" -ci
   echo "Did go test bootstraptoken $?"
 
-  go test ./tests/docker/upgrade/upgrade_test.go -k3sImage="$K3S_IMAGE" -channel="$minor_version"
+  go test ./tests/docker/upgrade/upgrade_test.go -k3sImage="$K3S_IMAGE" -channel="$minor_version" -ci
   echo "Did go test upgrade $?"
   
-  go test ./tests/docker/lazypull/lazypull_test.go -k3sImage="$K3S_IMAGE"
+  go test ./tests/docker/lazypull/lazypull_test.go -k3sImage="$K3S_IMAGE" -ci
   echo "Did go test lazypull $?"
 
 fi
@@ -67,9 +67,9 @@ fi
 # ---
 
 if [ "$DRONE_BUILD_EVENT" = 'cron' ]; then
-  run-go-test ./tests/docker/conformance/conformance_test.go -k3sImage="$K3S_IMAGE" -db sqlite -serial -ginkgo.v
+  run-go-test ./tests/docker/conformance/conformance_test.go -k3sImage="$K3S_IMAGE" -db sqlite -serial -ginkgo.v -ci
   echo "Did go conformance sqlite serial $?"
-  run-go-test ./tests/docker/conformance/conformance_test.go -k3sImage="$K3S_IMAGE" -db etcd -serial -ginkgo.v
+  run-go-test ./tests/docker/conformance/conformance_test.go -k3sImage="$K3S_IMAGE" -db etcd -serial -ginkgo.v -ci
   echo "Did go conformance etcd serial $?"
   test-run-sonobuoy mysql serial
   echo "Did test-run-sonobuoy-mysqk serial $?"
@@ -87,9 +87,9 @@ if [ "$DRONE_BUILD_EVENT" = 'cron' ]; then
 
   E2E_OUTPUT=$artifacts test-run-sonobuoy parallel
   echo "Did test-run-sonobuoy parallel $?"
-  run-go-test ./tests/docker/conformance/conformance_test.go -k3sImage="$K3S_IMAGE" -db sqlite -ginkgo.v
+  run-go-test ./tests/docker/conformance/conformance_test.go -k3sImage="$K3S_IMAGE" -db sqlite -ginkgo.v -ci
   echo "Did go conformance sqlite parallel $?"
-  run-go-test ./tests/docker/conformance/conformance_test.go -k3sImage="$K3S_IMAGE" -db etcd -ginkgo.v
+  run-go-test ./tests/docker/conformance/conformance_test.go -k3sImage="$K3S_IMAGE" -db etcd -ginkgo.v -ci
   echo "Did go test conformance etcd parallel $?"
   test-run-sonobuoy mysql parallel
   echo "Did test-run-sonobuoy-mysql parallel $?"

tests/docker/basics/basics_test.go

@@ -14,6 +14,7 @@ import (
 )
 
 var k3sImage = flag.String("k3sImage", "", "The image used to provision containers")
+var ci = flag.Bool("ci", false, "running on CI, forced cleanup")
 var config *tester.TestConfig
 
 func Test_DockerBasic(t *testing.T) {
@@ -74,8 +75,8 @@ var _ = AfterEach(func() {
 })
 
 var _ = AfterSuite(func() {
-	if config != nil && !failed {
-		config.Cleanup()
+	if config != nil && (*ci || !failed) {
+		Expect(config.Cleanup()).To(Succeed())
 	}
 })
 

tests/docker/bootstraptoken/bootstraptoken_test.go

@@ -12,6 +12,7 @@ import (
 )
 
 var k3sImage = flag.String("k3sImage", "", "The k3s image used to provision containers")
+var ci = flag.Bool("ci", false, "running on CI, forced cleanup")
 var config *tester.TestConfig
 
 func Test_DockerBootstrapToken(t *testing.T) {
@@ -62,7 +63,7 @@ var _ = AfterEach(func() {
 })
 
 var _ = AfterSuite(func() {
-	if config != nil && !failed {
-		config.Cleanup()
+	if config != nil && (*ci || !failed) {
+		Expect(config.Cleanup()).To(Succeed())
 	}
 })

tests/docker/cacerts/cacerts_test.go

@@ -15,6 +15,7 @@ import (
 )
 
 var k3sImage = flag.String("k3sImage", "", "The k3s image used to provision containers")
+var ci = flag.Bool("ci", false, "running on CI, forced cleanup")
 var config *tester.TestConfig
 var testID string
 

tests/docker/conformance/conformance_test.go

@@ -20,6 +20,7 @@ import (
 var k3sImage = flag.String("k3sImage", "", "The k3s image used to provision containers")
 var db = flag.String("db", "", "The database to use for the tests (sqlite, etcd, mysql, postgres)")
 var serial = flag.Bool("serial", false, "Run the Serial Conformance Tests")
+var ci = flag.Bool("ci", false, "running on CI, forced cleanup")
 var config *tester.TestConfig
 
 func Test_DockerConformance(t *testing.T) {
@@ -123,8 +124,8 @@ var _ = AfterEach(func() {
 })
 
 var _ = AfterSuite(func() {
-	if config != nil && !failed {
-		config.Cleanup()
+	if config != nil && (*ci || !failed) {
+		Expect(config.Cleanup()).To(Succeed())
 	}
 })
 

tests/docker/etcd/etcd_test.go

@@ -12,6 +12,7 @@ import (
 )
 
 var k3sImage = flag.String("k3sImage", "", "The k3s image used to provision containers")
+var ci = flag.Bool("ci", false, "running on CI, forced cleanup")
 var config *tester.TestConfig
 
 func Test_DockerEtcd(t *testing.T) {
@@ -72,7 +73,7 @@ var _ = AfterEach(func() {
 })
 
 var _ = AfterSuite(func() {
-	if config != nil && !failed {
-		config.Cleanup()
+	if config != nil && (*ci || !failed) {
+		Expect(config.Cleanup()).To(Succeed())
 	}
 })

tests/docker/lazypull/lazypull_test.go

@@ -13,6 +13,7 @@ import (
 )
 
 var k3sImage = flag.String("k3sImage", "", "The k3s image used to provision containers")
+var ci = flag.Bool("ci", false, "running on CI, forced cleanup")
 var config *tester.TestConfig
 
 func Test_DockerLazyPull(t *testing.T) {
@@ -71,8 +72,8 @@ var _ = AfterEach(func() {
 })
 
 var _ = AfterSuite(func() {
-	if config != nil && !failed {
-		config.Cleanup()
+	if config != nil && (*ci || !failed) {
+		Expect(config.Cleanup()).To(Succeed())
 	}
 })
 

tests/docker/skew/skew_test.go

@@ -17,6 +17,7 @@ import (
 // the current commit build of K3s defined by <k3sImage>
 var k3sImage = flag.String("k3sImage", "", "The current commit build of K3s")
 var channel = flag.String("channel", "latest", "The release channel to test")
+var ci = flag.Bool("ci", false, "running on CI, forced cleanup")
 var config *tester.TestConfig
 
 func Test_DockerSkew(t *testing.T) {
@@ -55,7 +56,7 @@ var _ = Describe("Skew Tests", Ordered, func() {
 			Expect(config.ProvisionAgents(1)).To(Succeed())
 			Eventually(func() error {
 				return tests.CheckDeployments([]string{"coredns", "local-path-provisioner", "metrics-server", "traefik"}, config.KubeconfigFile)
-			}, "120s", "5s").Should(Succeed())
+			}, "180s", "5s").Should(Succeed())
 		})
 		It("should match respective versions", func() {
 			for _, server := range config.Servers {
@@ -100,7 +101,7 @@ var _ = Describe("Skew Tests", Ordered, func() {
 			Expect(config.ProvisionServers(3)).To(Succeed())
 			Eventually(func() error {
 				return tests.CheckDeployments([]string{"coredns", "local-path-provisioner", "metrics-server", "traefik"}, config.KubeconfigFile)
-			}, "120s", "5s").Should(Succeed())
+			}, "180s", "5s").Should(Succeed())
 			Eventually(func(g Gomega) {
 				g.Expect(tests.ParseNodes(config.KubeconfigFile)).To(HaveLen(3))
 				g.Expect(tests.NodesReady(config.KubeconfigFile, config.GetNodeNames())).To(Succeed())
@@ -134,7 +135,7 @@ var _ = AfterEach(func() {
 })
 
 var _ = AfterSuite(func() {
-	if config != nil && !failed {
-		config.Cleanup()
+	if config != nil && (*ci || !failed) {
+		Expect(config.Cleanup()).To(Succeed())
 	}
 })

tests/docker/test-helpers.go

@@ -390,6 +390,7 @@ func (config *TestConfig) RemoveNode(nodeName string) error {
 	if _, err := RunCommand(cmd); err != nil {
 		return fmt.Errorf("failed to remove node %s: %v", nodeName, err)
 	}
+	fmt.Println("Stopped and removed ", nodeName)
 	return nil
 }
 
@@ -428,6 +429,7 @@ func (config *TestConfig) Cleanup() error {
 			errs = append(errs, err)
 		}
 	}
+	config.Servers = nil
 
 	// Stop and remove all agents
 	for _, agent := range config.Agents {
@@ -435,6 +437,7 @@ func (config *TestConfig) Cleanup() error {
 			errs = append(errs, err)
 		}
 	}
+	config.Agents = nil
 
 	// Stop DB if it was started
 	if config.DBType == "mysql" || config.DBType == "postgres" {
@@ -456,8 +459,6 @@ func (config *TestConfig) Cleanup() error {
 	if config.TestDir != "" {
 		return os.RemoveAll(config.TestDir)
 	}
-	config.Agents = nil
-	config.Servers = nil
 	return nil
 }
 
@@ -640,10 +641,47 @@ func FetchExternalIPs(kubeconfig string, servicename string) ([]string, error) {
 // RestartCluster restarts the k3s service on each node given
 func RestartCluster(nodes []DockerNode) error {
 	for _, node := range nodes {
-		cmd := "systemctl restart k3s* --all"
+		// Wait 60 seconds for the restart to succeed.
+		// If k3s doesn't report started to systemd within 60 seconds something is wrong.
+		cmd := "timeout -v 60 systemctl restart k3s* --all"
 		if _, err := node.RunCmdOnNode(cmd); err != nil {
 			return err
 		}
 	}
 	return nil
 }
+
+func DescribeNodesAndPods(config *TestConfig) string {
+	cmd := "kubectl describe node,pod -A --kubeconfig=" + config.KubeconfigFile
+	out, err := RunCommand(cmd)
+	if err != nil {
+		return fmt.Sprintf("** %v **\n%s", err, out)
+	}
+	return out
+}
+
+func TailDockerLogs(lines int, nodes []DockerNode) string {
+	logs := &strings.Builder{}
+	for _, node := range nodes {
+		cmd := fmt.Sprintf("docker logs %s --tail=%d", node.Name, lines)
+		if l, err := RunCommand(cmd); err != nil {
+			fmt.Fprintf(logs, "** failed to read docker logs for node %s ***\n%v\n", node.Name, err)
+		} else {
+			fmt.Fprintf(logs, "** docker logs for node %s ***\n%s\n", node.Name, l)
+		}
+	}
+	return logs.String()
+}
+
+func TailJournalLogs(lines int, nodes []DockerNode) string {
+	logs := &strings.Builder{}
+	for _, node := range nodes {
+		cmd := fmt.Sprintf("journalctl -u k3s* --no-pager --lines=%d", lines)
+		if l, err := node.RunCmdOnNode(cmd); err != nil {
+			fmt.Fprintf(logs, "** failed to read journald log for node %s ***\n%v\n", node.Name, err)
+		} else {
+			fmt.Fprintf(logs, "** journald log for node %s ***\n%s\n", node.Name, l)
+		}
+	}
+	return logs.String()
+}

tests/docker/upgrade/upgrade_test.go

@@ -19,6 +19,7 @@ import (
 // the current commit build of K3s defined by <k3sImage>
 var k3sImage = flag.String("k3sImage", "", "The current commit build of K3s")
 var channel = flag.String("channel", "latest", "The release channel to test")
+var ci = flag.Bool("ci", false, "running on CI, forced cleanup")
 var config *tester.TestConfig
 
 var numServers = 1
@@ -127,7 +128,7 @@ var _ = Describe("Upgrade Tests", Ordered, func() {
 				cVersion = strings.Replace(cVersion, "-amd64", "", 1)
 				cVersion = strings.Replace(cVersion, "-arm64", "", 1)
 				cVersion = strings.Replace(cVersion, "-arm", "", 1)
-				cVersion = strings.Replace(cVersion, "-", "+", 1)
+				cVersion = strings.Replace(cVersion, "-k3s", "+k3s", 1)
 				Expect(out).To(ContainSubstring(cVersion))
 			}
 		})
@@ -146,7 +147,7 @@ var _ = AfterEach(func() {
 })
 
 var _ = AfterSuite(func() {
-	if config != nil && !failed {
-		config.Cleanup()
+	if config != nil && (*ci || !failed) {
+		Expect(config.Cleanup()).To(Succeed())
 	}
 })