Migrate E2E tests to GitHub Actions

.drone.yml

@@ -644,10 +644,6 @@ steps:
   - vagrant destroy -f
   - go test -v -timeout=30m ./secretsencryption_test.go -ci -local
   - cp ./coverage.out /tmp/artifacts/se-coverage.out
-  - cd ../startup
-  - vagrant destroy -f
-  - go test -v -timeout=30m ./startup_test.go -ci -local
-  - cp ./coverage.out /tmp/artifacts/startup-coverage.out
   - |
     if [ "$DRONE_BUILD_EVENT" = "pull_request" ]; then
       cd ../upgradecluster
@@ -673,7 +669,6 @@ steps:
     files:
       - /tmp/artifacts/validate-coverage.out
       - /tmp/artifacts/se-coverage.out
-      - /tmp/artifacts/startup-coverage.out
       - /tmp/artifacts/upgrade-coverage.out
     flags: 
       - e2etests

.github/workflows/e2e.yaml

+name: E2E Test Coverage
+on: 
+  push:
+    paths-ignore:
+      - "**.md"
+      - "channel.yaml"
+      - "install.sh"
+      - "tests/**"
+      - "!tests/e2e**"
+      - ".github/**"
+      - "!.github/workflows/e2e.yaml"
+  pull_request:
+    paths-ignore:
+      - "**.md"
+      - "channel.yaml"
+      - "install.sh"
+      - "tests/**"
+      - "!tests/e2e**"
+      - ".github/**"
+      - "!.github/workflows/e2e.yaml"
+  workflow_dispatch: {}
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    uses: ./.github/workflows/build-k3s.yaml
+  test:
+    name: "E2E Test"
+    needs: build
+    runs-on: ubuntu-latest
+    timeout-minutes: 40
+    strategy:
+      fail-fast: false
+      matrix:
+        # TODO fix embeddedmirror and add it to the matrix
+        etest: [startup, s3, externalip, privateregistry]
+      max-parallel: 3
+    steps:
+      - name: "Checkout"
+        uses: actions/checkout@v4
+        with: {fetch-depth: 1}
+      
+      - name: Set up vagrant and libvirt
+        uses: ./.github/actions/vagrant-setup
+      - name: "Vagrant Cache"
+        uses: actions/cache@v4
+        with:
+          path: |
+             ~/.vagrant.d/boxes
+          key: vagrant-box-${{ matrix.vm }}
+      - name: "Vagrant Plugin(s)"
+        run: vagrant plugin install vagrant-k3s vagrant-reload vagrant-scp
+      
+      - name: Install Go
+        uses: actions/setup-go@v5
+        with: 
+          go-version-file: 'go.mod'
+          cache: false
+      - name: "Download k3s binary"
+        uses: actions/download-artifact@v3
+        with:
+          name: k3s
+          path: ./dist/artifacts
+      
+      - name: Run E2E Tests
+        env:
+          E2E_GOCOVER: "true"
+        run: | 
+          chmod +x ./dist/artifacts/k3s
+          cd tests/e2e/${{ matrix.etest }}
+          go test -v -timeout=45m ./${{ matrix.etest}}_test.go -ci -local
+      - name: On Failure, Launch Debug Session
+        uses: lhotari/action-upterm@v1
+        if: ${{ failure() }}
+        with:
+          ## If no one connects after 5 minutes, shut down server.
+          wait-timeout-minutes: 5
+      - name: Upload Results To Codecov
+        uses: codecov/codecov-action@v4
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: tests/e2e/${{ matrix.etest }}/coverage.out
+          flags: e2etests # optional
+          verbose: true # optional (default = false)
\ No newline at end of file

.github/workflows/snapshotter.yaml

@@ -79,4 +79,4 @@ jobs:
       - name: Copy out vagrant boxes for cache
         run: |
           sudo mv -f /root/.vagrant.d/boxes /tmp/boxes
-          sudo chmod -R 777 /tmp/boxes
\ No newline at end of file
+          sudo chmod -R 777 /tmp/boxes

tests/e2e/embeddedmirror/Vagrantfile

@@ -27,7 +27,6 @@ def provision(vm, role, role_num, node_num)
   addCoverageDir(vm, role, GOCOVER)
   install_type = getInstallType(vm, RELEASE_VERSION, GITHUB_BRANCH)
   
-  vm.provision "shell", inline: "ping -c 2 k3s.io"
  
   # The formatting on this is a little weird, but it allows inserting variables
   # and still using the heredoc formatting with escapped quotes

tests/e2e/privateregistry/Vagrantfile

@@ -27,7 +27,6 @@ def provision(vm, role, role_num, node_num)
   addCoverageDir(vm, role, GOCOVER)
   install_type = getInstallType(vm, RELEASE_VERSION, GITHUB_BRANCH)
   
-  vm.provision "shell", inline: "ping -c 2 k3s.io"
  
   # The formatting on this is a little weird, but it allows inserting variables
   # and still using the heredoc formatting with escapped quotes

tests/e2e/s3/Vagrantfile

@@ -27,7 +27,6 @@ def provision(vm, role, role_num, node_num)
   addCoverageDir(vm, role, GOCOVER)
   install_type = getInstallType(vm, RELEASE_VERSION, GITHUB_BRANCH)
 
-  vm.provision "shell", inline: "ping -c 2 k3s.io"
 
   runS3mock = <<~'SCRIPT'
     docker run -p 9090:9090 -p 9191:9191 -d -e initialBuckets=test -e debug=true -t adobe/s3mock

tests/e2e/startup/Vagrantfile

@@ -29,7 +29,6 @@ def provision(vm, role, role_num, node_num)
   node_ip = "#{NETWORK_PREFIX}.#{100+node_num}"
   
 
-  vm.provision "shell", inline: "ping -c 2 k3s.io"
 
   if role.include?("server")
     vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s|

tests/e2e/testutils.go

@@ -111,11 +111,11 @@ func CreateCluster(nodeOS string, serverCount, agentCount int) ([]string, []stri
 	}
 	// Bring up the first server node
 	cmd := fmt.Sprintf(`%s %s vagrant up %s &> vagrant.log`, nodeEnvs, testOptions, serverNodeNames[0])
-
 	fmt.Println(cmd)
 	if _, err := RunCommand(cmd); err != nil {
 		return nil, nil, newNodeError(cmd, serverNodeNames[0], err)
 	}
+
 	// Bring up the rest of the nodes in parallel
 	errg, _ := errgroup.WithContext(context.Background())
 	for _, node := range append(serverNodeNames[1:], agentNodeNames...) {
@@ -170,17 +170,21 @@ func CreateLocalCluster(nodeOS string, serverCount, agentCount int) ([]string, [
 	}
 	testOptions += " E2E_RELEASE_VERSION=skip"
 
-	// Bring up the all of the nodes in parallel
+	// Provision the first server node. In GitHub Actions, this also imports the VM image into libvirt, which
+	// takes time and can cause the next vagrant up to fail if it is not given enough time to complete.
+	cmd = fmt.Sprintf(`%s %s vagrant up --no-provision %s &> vagrant.log`, nodeEnvs, testOptions, serverNodeNames[0])
+	fmt.Println(cmd)
+	if _, err := RunCommand(cmd); err != nil {
+		return nil, nil, newNodeError(cmd, serverNodeNames[0], err)
+	}
+
+	// Bring up the rest of the nodes in parallel
 	errg, _ := errgroup.WithContext(context.Background())
-	for i, node := range append(serverNodeNames, agentNodeNames...) {
-		if i == 0 {
-			cmd = fmt.Sprintf(`%s %s vagrant up --no-provision %s &> vagrant.log`, nodeEnvs, testOptions, node)
-		} else {
-			cmd = fmt.Sprintf(`%s %s vagrant up --no-provision %s &>> vagrant.log`, nodeEnvs, testOptions, node)
-		}
+	for _, node := range append(serverNodeNames[1:], agentNodeNames...) {
+		cmd := fmt.Sprintf(`%s %s vagrant up --no-provision %s &>> vagrant.log`, nodeEnvs, testOptions, node)
 		errg.Go(func() error {
 			if _, err := RunCommand(cmd); err != nil {
-				return fmt.Errorf("failed initializing nodes: %s: %v", cmd, err)
+				return newNodeError(cmd, node, err)
 			}
 			return nil
 		})
@@ -190,10 +194,10 @@ func CreateLocalCluster(nodeOS string, serverCount, agentCount int) ([]string, [
 	if err := errg.Wait(); err != nil {
 		return nil, nil, err
 	}
+
 	if err := scpK3sBinary(append(serverNodeNames, agentNodeNames...)); err != nil {
 		return nil, nil, err
 	}
-
 	// Install K3s on all nodes in parallel
 	errg, _ = errgroup.WithContext(context.Background())
 	for _, node := range append(serverNodeNames, agentNodeNames...) {
@@ -471,6 +475,9 @@ func RunCommand(cmd string) (string, error) {
 		c.Env = append(os.Environ(), "KUBECONFIG="+kc)
 	}
 	out, err := c.CombinedOutput()
+	if err != nil {
+		return string(out), fmt.Errorf("failed to run command: %s, %v", cmd, err)
+	}
 	return string(out), err
 }