Run Docker test on Github Actions arm64 runners (#11705)

.github/workflows/build-k3s.yaml

@@ -3,6 +3,10 @@ name: Build K3s
 on: 
   workflow_call:
    inputs:
+    arch:
+      type: string
+      description: 'Architecture to build'
+      default: 'ubuntu-latest'
     upload-repo:
       type: boolean
       required: false
@@ -18,7 +22,7 @@ permissions:
 jobs:
   build:
     name: Build
-    runs-on: ubuntu-latest
+    runs-on: ${{ inputs.arch }} # defaults to ubuntu-latest, for arm64 use ubuntu-24.04-arm
     timeout-minutes: 20
     steps:
     - name: Checkout K3s
@@ -44,9 +48,15 @@ jobs:
     - name: "Save K3s image"
       if: inputs.upload-image == true
       run: docker image save rancher/k3s -o ./dist/artifacts/k3s-image.tar
-    - name: "Upload K3s binary"
-      if: inputs.upload-repo == false
+    - name: "Upload K3s Artifacts"
+      if: inputs.upload-repo == false && inputs.arch == 'ubuntu-latest'
       uses: actions/upload-artifact@v4
       with:
         name: k3s
+        path: dist/artifacts/k3s*
+    - name: "Upload K3s arm64 Artifacts"
+      if: contains(inputs.arch, 'arm')
+      uses: actions/upload-artifact@v4
+      with:
+        name: k3s-arm64
         path: dist/artifacts/k3s*
\ No newline at end of file

.github/workflows/e2e.yaml

@@ -32,6 +32,11 @@ jobs:
     uses: ./.github/workflows/build-k3s.yaml
     with:
       upload-image: true
+  build-arm64:
+    uses: ./.github/workflows/build-k3s.yaml
+    with:
+      arch: ubuntu-24.04-arm
+      upload-image: true
   e2e:
     name: "E2E Tests"
     needs: build
@@ -100,40 +105,15 @@ jobs:
           files: tests/e2e/${{ matrix.etest }}/coverage.out
           flags: e2etests # optional
           verbose: true # optional (default = false)
-  docker:
-    needs: build
-    name: Docker Tests
-    runs-on: ubuntu-latest
-    timeout-minutes: 20
-    strategy:
-      fail-fast: false
-      matrix:
-        dtest: [basics, bootstraptoken, cacerts, compat, lazypull, upgrade]
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v4
-      with:
-        fetch-depth: 1
-    - name: "Download k3s image"
-      uses: actions/download-artifact@v4
-      with:
-        name: k3s
-        path: ./dist/artifacts
-    - name: Load k3s image
-      run: docker image load -i ./dist/artifacts/k3s-image.tar
-    - name: Run ${{ matrix.dtest }} Test
-      run: | 
-        chmod +x ./dist/artifacts/k3s
-        . ./scripts/version.sh
-        . ./tests/docker/test-helpers
-        . ./tests/docker/test-run-${{ matrix.dtest }}
-        echo "Did test-run-${{ matrix.dtest }} pass $?"
   
   build-go-tests:
     name: "Build Go Tests"
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        arch: [amd64, arm64]
     outputs:
-      branch_name: ${{ steps.branch_step.outputs.BRANCH_NAME }}
+      channel: ${{ steps.channel_step.outputs.channel }}
     steps:
     - name: Checkout
       uses: actions/checkout@v4
@@ -142,66 +122,62 @@ jobs:
     - name: Build Go Tests
       run: |
         mkdir -p ./dist/artifacts
-        go test -c -ldflags="-w -s" -o ./dist/artifacts ./tests/docker/...
+        GOOS=linux GOARCH=${{ matrix.arch }} go test -c -ldflags="-w -s" -o ./dist/artifacts ./tests/docker/...
     - name: Upload Go Tests
       uses: actions/upload-artifact@v4
       with:
-        name: docker-go-tests
+        name: docker-go-tests-${{ matrix.arch }}
         path: ./dist/artifacts/*.test
         compression-level: 9
         retention-days: 1
-    # For upgrade and skew tests, we need to know the branch name this run is based off.
-    # Since this is predetermined, we can run this step before the docker-go job, saving time.
-    # For PRs we can use the base_ref (ie the target branch of the PR).
-    # For pushes to k3s-io/k3s, the branch_name is a valid ref, master or release-x.y.
-    # For pushes to a fork, we need to determine the branch name by finding the parent branch from git show-branch history.
-    - name: Determine branch name
-      id: branch_step
+    # For upgrade and skew tests, we need to know the channel this run is based off.
+    # Since this is predetermined, we can run this step before the actual test job, saving time.
+    - name: Determine channel
+      id: channel_step
       run: |
-        if [ ${{ github.repository }} = "k3s-io/k3s" ]; then 
-          BRANCH_NAME=$(echo ${{ github.base_ref || github.ref_name }})
-        elif [ -z "${{ github.base_ref }}" ]; then
-          # We are in a fork, and need some git history to determine the branch name
-          # For some reason, the first fetch doesn't always get the full history, so we sleep and fetch again
-          git fetch origin --depth=100 +refs/heads/*:refs/remotes/origin/*
-          sleep 5
-          git fetch origin --depth=100 +refs/heads/*:refs/remotes/origin/*
-          BRANCH_NAME=$(git show-branch -a 2> /dev/null  | grep '\*' | grep -v `git rev-parse --abbrev-ref HEAD` | head -n1 |  sed 's/.*\[\(.*\/\)\(.*\)\].*/\2/' | sed 's/[\^~].*//')
-        else
-          BRANCH_NAME=${{ github.base_ref }}
-        fi
-        echo "Branch Name is $BRANCH_NAME"
-        echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_OUTPUT
-    # branch name should be either master or release-1.XX
-    - name: Fail if branch name does not match pattern
+        . ./scripts/version.sh
+        MINOR_VER=$(echo $VERSION_TAG | cut -d'.' -f1,2)
+        echo "CHANNEL=$MINOR_VER" >> $GITHUB_OUTPUT 
+    # channel name should be v1.XX or latest
+    - name: Fail if channel name does not match pattern
       run: |
-        if [[ ! ${{ steps.branch_step.outputs.branch_name }} =~ ^(master|release-[0-9]+\.[0-9]+)$ ]]; then
-          echo "Branch name ${{ steps.branch_step.outputs.branch_name }} does not match pattern"
-          echo "If this is a PR/fork, ensure you have recently rebased off master/release-1.XX branch"
+        if [[ ! ${{ steps.channel_step.outputs.channel }} =~ ^v1\.[0-9]+$|latest$ ]]; then
+          echo "Channel name ${{ steps.channel_step.outputs.channel }} does not match pattern"
           exit 1
         fi
-
+  
   docker-go:
-    needs: [build, build-go-tests]
-    name: Docker Tests In GO
-    runs-on: ubuntu-latest
-    timeout-minutes: 20
+    needs: [build, build-arm64, build-go-tests]
+    name: Docker
+    timeout-minutes: 30
     strategy:
       fail-fast: false
       matrix:
         dtest: [basics, bootstraptoken, cacerts, etcd, lazypull, skew, snapshotrestore, upgrade]
+        arch: [amd64, arm64]
+    runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }}
     env:
-      BRANCH_NAME: ${{ needs.build-go-tests.outputs.branch_name }}
+      CHANNEL: ${{ needs.build-go-tests.outputs.channel }}
     steps:
     - name: Checkout
       uses: actions/checkout@v4
-    - name: "Download K3s image"
+    - name: "Download K3s image (amd64)"
+      if: ${{ matrix.arch == 'amd64' }}
       uses: actions/download-artifact@v4
       with:
         name: k3s
         path: ./dist/artifacts
+    - name: "Download K3s image (arm64)"
+      if: ${{ matrix.arch == 'arm64' }}
+      uses: actions/download-artifact@v4
+      with:
+        name: k3s-arm64
+        path: ./dist/artifacts
     - name: Load and set K3s image
       run: |
+        if [ ${{ matrix.arch }} = "arm64" ]; then
+          mv ./dist/artifacts/k3s-arm64 ./dist/artifacts/k3s
+        fi
         chmod +x ./dist/artifacts/k3s
         docker image load -i ./dist/artifacts/k3s-image.tar
         IMAGE_TAG=$(docker image ls --format '{{.Repository}}:{{.Tag}}' | grep 'rancher/k3s')
@@ -209,7 +185,7 @@ jobs:
     - name: Download Go Tests
       uses: actions/download-artifact@v4
       with:
-        name: docker-go-tests
+        name: docker-go-tests-${{ matrix.arch }}
         path: ./dist/artifacts
     - name: Run ${{ matrix.dtest }} Test
       # Put the compied test binary back in the same place as the test source
@@ -218,7 +194,7 @@ jobs:
         mv ./dist/artifacts/${{ matrix.dtest }}.test ./tests/docker/${{ matrix.dtest }}/
         cd ./tests/docker/${{ matrix.dtest }}
         if [ ${{ matrix.dtest }} = "upgrade" ] || [ ${{ matrix.dtest }} = "skew" ]; then
-          ./${{ matrix.dtest }}.test -k3sImage=$K3S_IMAGE -branch=$BRANCH_NAME
+          ./${{ matrix.dtest }}.test -k3sImage=$K3S_IMAGE -channel=$CHANNEL
         elif [ ${{ matrix.dtest }} = "snapshotrestore" ]; then
           ./${{ matrix.dtest }}.test -ci 
         else

scripts/test

@@ -20,8 +20,9 @@ mkdir -p $artifacts
 docker ps
 
 # ---
-# Only run basic tests on non amd64 archs, we use GitHub Actions for amd64
-if [ "$ARCH" != 'amd64' ]; then
+# Only run PR tests on arm arch, we use GitHub Actions for amd64 and arm64
+# Run all tests on tag events, as we want test failures to block the release
+if [ "$ARCH" == 'arm' ] ||  [ "$DRONE_BUILD_EVENT" = 'tag' ]; then
 
   export K3S_IMAGE="rancher/k3s:${VERSION_TAG}${SUFFIX}"
   go test ./tests/docker/basics/basics_test.go -k3sImage="$K3S_IMAGE"
@@ -47,13 +48,10 @@ if [ "$ARCH" != 'amd64' ]; then
 fi
 
 
-
+#TODO convert this to new go test framework
 . ./tests/docker/test-run-hardened
 echo "Did test-run-hardened $?"
 
-. ./tests/docker/test-run-etcd
-echo "Did test-run-etcd $?"
-
 # ---
 
 [ "$ARCH" != 'amd64' ] && \

tests/docker/skew/skew_test.go

@@ -15,7 +15,7 @@ import (
 // Using these two flags, we upgrade from the latest release of <branch> to
 // the current commit build of K3s defined by <k3sImage>
 var k3sImage = flag.String("k3sImage", "", "The current commit build of K3s")
-var branch = flag.String("branch", "master", "The release branch to test")
+var channel = flag.String("channel", "latest", "The release channel to test")
 var config *tester.TestConfig
 
 func Test_DockerSkew(t *testing.T) {
@@ -30,14 +30,17 @@ var _ = BeforeSuite(func() {
 	// For master and unreleased branches, we want the latest stable release
 	var upgradeChannel string
 	var err error
-	if *branch == "master" {
+	if *channel == "latest" || *channel == "v1.32" {
+		// disabled: AuthorizeNodeWithSelectors is now on by default, which breaks compat with agents < v1.32.
+		// This can be ren-enabled once the previous branch is v1.32 or higher, or when RBAC changes have been backported.
+		// ref: https://github.com/kubernetes/kubernetes/pull/128168
+		Skip("Skipping version skew tests for " + *channel + " due to AuthorizeNodeWithSelectors")
+
 		upgradeChannel = "stable"
 	} else {
-		upgradeChannel = strings.Replace(*branch, "release-", "v", 1)
-		// now that it is in v1.1 format, we want to substract one from the minor version
-		// to get the previous release
-		sV, err := semver.ParseTolerant(upgradeChannel)
-		Expect(err).NotTo(HaveOccurred(), "failed to parse version from "+upgradeChannel)
+		// We want to substract one from the minor version to get the previous release
+		sV, err := semver.ParseTolerant(*channel)
+		Expect(err).NotTo(HaveOccurred(), "failed to parse version from "+*channel)
 		sV.Minor--
 		upgradeChannel = fmt.Sprintf("v%d.%d", sV.Major, sV.Minor)
 	}

tests/docker/upgrade/upgrade_test.go

@@ -17,7 +17,7 @@ import (
 // Using these two flags, we upgrade from the latest release of <branch> to
 // the current commit build of K3s defined by <k3sImage>
 var k3sImage = flag.String("k3sImage", "", "The current commit build of K3s")
-var branch = flag.String("branch", "master", "The release branch to test")
+var channel = flag.String("channel", "latest", "The release channel to test")
 var config *tester.TestConfig
 
 var numServers = 1
@@ -34,22 +34,15 @@ var _ = Describe("Upgrade Tests", Ordered, func() {
 	Context("Setup Cluster with Lastest Release", func() {
 		var latestVersion string
 		It("should determine latest branch version", func() {
-			var upgradeChannel string
-			var err error
-			if *branch == "master" {
-				upgradeChannel = "latest"
-			} else {
-				upgradeChannel = strings.Replace(*branch, "release-", "v", 1)
-				url := fmt.Sprintf("https://update.k3s.io/v1-release/channels/%s", upgradeChannel)
-				resp, err := http.Head(url)
-				// Cover the case where the branch does not exist yet,
-				// such as a new unreleased minor version
-				if err != nil || resp.StatusCode != http.StatusOK {
-					upgradeChannel = "latest"
-				}
+			url := fmt.Sprintf("https://update.k3s.io/v1-release/channels/%s", *channel)
+			resp, err := http.Head(url)
+			// Cover the case where the branch does not exist yet,
+			// such as a new unreleased minor version
+			if err != nil || resp.StatusCode != http.StatusOK {
+				*channel = "latest"
 			}
 
-			latestVersion, err = tester.GetVersionFromChannel(upgradeChannel)
+			latestVersion, err = tester.GetVersionFromChannel(*channel)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(latestVersion).To(ContainSubstring("v1."))
 			fmt.Println("Using latest version: ", latestVersion)
@@ -131,6 +124,7 @@ var _ = Describe("Upgrade Tests", Ordered, func() {
 				Expect(err).NotTo(HaveOccurred())
 				cVersion := strings.Split(*k3sImage, ":")[1]
 				cVersion = strings.Replace(cVersion, "-amd64", "", 1)
+				cVersion = strings.Replace(cVersion, "-arm64", "", 1)
 				cVersion = strings.Replace(cVersion, "-", "+", 1)
 				Expect(out).To(ContainSubstring(cVersion))
 			}