Commit 1d5684a5 authored by Kubernetes Submit Queue's avatar Kubernetes Submit Queue Committed by GitHub

Merge pull request #30728 from Random-Liu/fix-node-e2e-vm-leak

Automatic merge from submit-queue Node E2E: Stop throwing fatal error in buildGo to fix vm leak. Fixes https://github.com/kubernetes/kubernetes/issues/30719. Avoid throwing out fatal error during `buildGo`. As is pointed out in https://github.com/kubernetes/kubernetes/issues/30719#issuecomment-240283268, we delete vm instances in `defer`. However, `defer` will not be called if there is a fatal error (process exits immediately) http://stackoverflow.com/questions/17888610/are-deferred-functions-called-when-calling-log-fatalln. This PR: * changes all fatal error during create/delete instance to normal error so as to make sure we have a chance to delete the instance. * adds log for instance deletion so as to better surface similar error in the future. @dchen1107 @yujuhong /cc @kubernetes/sig-node
parents fc7a7f5f 1c5bd554
...@@ -37,11 +37,11 @@ var buildTargets = []string{ ...@@ -37,11 +37,11 @@ var buildTargets = []string{
"vendor/github.com/onsi/ginkgo/ginkgo", "vendor/github.com/onsi/ginkgo/ginkgo",
} }
func buildGo() { func buildGo() error {
glog.Infof("Building k8s binaries...") glog.Infof("Building k8s binaries...")
k8sRoot, err := getK8sRootDir() k8sRoot, err := getK8sRootDir()
if err != nil { if err != nil {
glog.Fatalf("Failed to locate kubernetes root directory %v.", err) return fmt.Errorf("failed to locate kubernetes root directory %v.", err)
} }
targets := strings.Join(buildTargets, " ") targets := strings.Join(buildTargets, " ")
cmd := exec.Command("make", "-C", k8sRoot, fmt.Sprintf("WHAT=%s", targets)) cmd := exec.Command("make", "-C", k8sRoot, fmt.Sprintf("WHAT=%s", targets))
...@@ -49,8 +49,9 @@ func buildGo() { ...@@ -49,8 +49,9 @@ func buildGo() {
cmd.Stderr = os.Stderr cmd.Stderr = os.Stderr
err = cmd.Run() err = cmd.Run()
if err != nil { if err != nil {
glog.Fatalf("Failed to build go packages %v\n", err) return fmt.Errorf("failed to build go packages %v\n", err)
} }
return nil
} }
func getK8sBin(bin string) (string, error) { func getK8sBin(bin string) (string, error) {
......
...@@ -101,7 +101,7 @@ func TestE2eNode(t *testing.T) { ...@@ -101,7 +101,7 @@ func TestE2eNode(t *testing.T) {
// Setup the kubelet on the node // Setup the kubelet on the node
var _ = SynchronizedBeforeSuite(func() []byte { var _ = SynchronizedBeforeSuite(func() []byte {
if *buildServices { if *buildServices {
buildGo() Expect(buildGo()).To(Succeed())
} }
if framework.TestContext.NodeName == "" { if framework.TestContext.NodeName == "" {
hostname, err := os.Hostname() hostname, err := os.Hostname()
......
...@@ -82,12 +82,14 @@ func GetHostnameOrIp(hostname string) string { ...@@ -82,12 +82,14 @@ func GetHostnameOrIp(hostname string) string {
// the binaries k8s required for node e2e tests // the binaries k8s required for node e2e tests
func CreateTestArchive() (string, error) { func CreateTestArchive() (string, error) {
// Build the executables // Build the executables
buildGo() if err := buildGo(); err != nil {
return "", fmt.Errorf("failed to build the depedencies: %v", err)
}
// Make sure we can find the newly built binaries // Make sure we can find the newly built binaries
buildOutputDir, err := getK8sBuildOutputDir() buildOutputDir, err := getK8sBuildOutputDir()
if err != nil { if err != nil {
glog.Fatalf("Failed to locate kubernetes build output directory %v", err) return "", fmt.Errorf("failed to locate kubernetes build output directory %v", err)
} }
ginkgoTest := filepath.Join(buildOutputDir, "e2e_node.test") ginkgoTest := filepath.Join(buildOutputDir, "e2e_node.test")
......
...@@ -502,9 +502,11 @@ func getComputeClient() (*compute.Service, error) { ...@@ -502,9 +502,11 @@ func getComputeClient() (*compute.Service, error) {
} }
func deleteInstance(image string) { func deleteInstance(image string) {
_, err := computeService.Instances.Delete(*project, *zone, imageToInstanceName(image)).Do() instanceName := imageToInstanceName(image)
glog.Infof("Deleting instance %q", instanceName)
_, err := computeService.Instances.Delete(*project, *zone, instanceName).Do()
if err != nil { if err != nil {
glog.Infof("Error deleting instance %s", imageToInstanceName(image)) glog.Errorf("Error deleting instance %q: %v", instanceName, err)
} }
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment