Consume new config value for network project id

pkg/cloudprovider/providers/gce/gce.go

@@ -144,12 +144,14 @@ type GCEServiceManager struct {
 	gce *GCECloud
 }
 
-// ConfigFile is the struct used to parse the /etc/gce.conf configuration file.
-type ConfigFile struct {
-	Global struct {
+type ConfigGlobal struct {
 	TokenURL  string `gcfg:"token-url"`
 	TokenBody string `gcfg:"token-body"`
+	// ProjectID and NetworkProjectID can either be the numeric or string-based
+	// unique identifier that starts with [a-z].
 	ProjectID string `gcfg:"project-id"`
+	// NetworkProjectID refers to the project which owns the network being used.
+	NetworkProjectID string `gcfg:"network-project-id"`
 	NetworkName      string `gcfg:"network-name"`
 	SubnetworkName   string `gcfg:"subnetwork-name"`
 	// SecondaryRangeName is the name of the secondary range to allocate IP
@@ -166,20 +168,27 @@ type ConfigFile struct {
 	// located in (i.e. where the controller will be running). If this is
 	// blank, then the local zone will be discovered via the metadata server.
 	LocalZone string `gcfg:"local-zone"`
-		// AlphaFeatures is a list of API flags to be enabled. Defaults to none.
-		// Example API name format: "MyFeatureFlag"
+	// Possible values: List of api names separated by comma. Default to none.
+	// For example: MyFeatureFlag
 	AlphaFeatures []string `gcfg:"alpha-features"`
-	}
+}
+
+// ConfigFile is the struct used to parse the /etc/gce.conf configuration file.
+type ConfigFile struct {
+	Global ConfigGlobal `gcfg:"global"`
 }
 
 // CloudConfig includes all the necessary configuration for creating GCECloud
 type CloudConfig struct {
 	ApiEndpoint        string
 	ProjectID          string
+	NetworkProjectID   string
 	Region             string
 	Zone               string
 	ManagedZones       []string
+	NetworkName        string
 	NetworkURL         string
+	SubnetworkName     string
 	SubnetworkURL      string
 	SecondaryRangeName string
 	NodeTags           []string
@@ -207,11 +216,6 @@ func (g *GCECloud) GetKMSService() *cloudkms.Service {
 	return g.cloudkmsService
 }
 
-// Returns the ProjectID corresponding to the project this cloud is in.
-func (g *GCECloud) GetProjectID() string {
-	return g.projectID
-}
-
 // newGCECloud creates a new instance of GCECloud.
 func newGCECloud(config io.Reader) (gceCloud *GCECloud, err error) {
 	var cloudConfig *CloudConfig
@@ -280,6 +284,7 @@ func generateCloudConfig(configFile *ConfigFile) (cloudConfig *CloudConfig, err
 			return nil, err
 		}
 	}
+
 	if configFile != nil {
 		if configFile.Global.ProjectID != "" {
 			cloudConfig.ProjectID = configFile.Global.ProjectID
@@ -287,6 +292,9 @@ func generateCloudConfig(configFile *ConfigFile) (cloudConfig *CloudConfig, err
 		if configFile.Global.LocalZone != "" {
 			cloudConfig.Zone = configFile.Global.LocalZone
 		}
+		if configFile.Global.NetworkProjectID != "" {
+			cloudConfig.NetworkProjectID = configFile.Global.NetworkProjectID
+		}
 	}
 
 	// retrieve region
@@ -301,27 +309,27 @@ func generateCloudConfig(configFile *ConfigFile) (cloudConfig *CloudConfig, err
 		cloudConfig.ManagedZones = nil // Use all zones in region
 	}
 
-	// generate networkURL
+	// Determine if network parameter is URL or Name
 	if configFile != nil && configFile.Global.NetworkName != "" {
 		if strings.Contains(configFile.Global.NetworkName, "/") {
 			cloudConfig.NetworkURL = configFile.Global.NetworkName
 		} else {
-			cloudConfig.NetworkURL = gceNetworkURL(cloudConfig.ApiEndpoint, cloudConfig.ProjectID, configFile.Global.NetworkName)
+			cloudConfig.NetworkName = configFile.Global.NetworkName
 		}
 	} else {
-		networkName, err := getNetworkNameViaMetadata()
+		cloudConfig.NetworkName, err = getNetworkNameViaMetadata()
 		if err != nil {
 			return nil, err
 		}
-		cloudConfig.NetworkURL = gceNetworkURL("", cloudConfig.ProjectID, networkName)
 	}
 
-	// generate subnetworkURL
+	// Determine if subnetwork parameter is URL or Name
+	// If cluster is on a GCP network of mode=custom, then `SubnetName` must be specified in config file.
 	if configFile != nil && configFile.Global.SubnetworkName != "" {
 		if strings.Contains(configFile.Global.SubnetworkName, "/") {
 			cloudConfig.SubnetworkURL = configFile.Global.SubnetworkName
 		} else {
-			cloudConfig.SubnetworkURL = gceSubnetworkURL(cloudConfig.ApiEndpoint, cloudConfig.ProjectID, cloudConfig.Region, configFile.Global.SubnetworkName)
+			cloudConfig.SubnetworkName = configFile.Global.SubnetworkName
 		}
 	}
 
@@ -332,11 +340,15 @@ func generateCloudConfig(configFile *ConfigFile) (cloudConfig *CloudConfig, err
 	return cloudConfig, err
 }
 
-// Creates a GCECloud object using the specified parameters.
+// CreateGCECloud creates a GCECloud object using the specified parameters.
 // If no networkUrl is specified, loads networkName via rest call.
 // If no tokenSource is specified, uses oauth2.DefaultTokenSource.
 // If managedZones is nil / empty all zones in the region will be managed.
 func CreateGCECloud(config *CloudConfig) (*GCECloud, error) {
+	// Use ProjectID for NetworkProjectID, if it wasn't explicitly set.
+	if config.NetworkProjectID == "" {
+		config.NetworkProjectID = config.ProjectID
+	}
 
 	client, err := newOauthClient(config.TokenSource)
 	if err != nil {
@@ -385,19 +397,49 @@ func CreateGCECloud(config *CloudConfig) (*GCECloud, error) {
 		return nil, err
 	}
 
-	if config.NetworkURL == "" {
-		networkName, err := getNetworkNameViaAPICall(service, config.ProjectID)
+	// config.ProjectID may be the id or project number
+	// In gce_routes.go, the generated networkURL is compared with a URL within a route
+	// therefore, we need to make sure the URL is constructed with the string ID.
+	projID, err := getProjectID(service, config.ProjectID)
 	if err != nil {
-			return nil, err
+		return nil, fmt.Errorf("failed to get project %v, err: %v", config.ProjectID, err)
+	}
+
+	// config.NetworkProjectID may be the id or project number. In order to compare project ID
+	// to network project ID to determine XPN status, we need to verify both are actual IDs.
+	netProjID := projID
+	if config.NetworkProjectID != config.ProjectID {
+		netProjID, err = getProjectID(service, config.NetworkProjectID)
+		if err != nil {
+			return nil, fmt.Errorf("failed to get network project %v, err: %v", config.NetworkProjectID, err)
 		}
-		config.NetworkURL = gceNetworkURL(config.ApiEndpoint, config.ProjectID, networkName)
 	}
 
-	networkProjectID, err := getProjectIDInURL(config.NetworkURL)
+	onXPN := projID != netProjID
+
+	var networkURL string
+	var subnetURL string
+
+	if config.NetworkName == "" && config.NetworkURL == "" {
+		// TODO: Stop using this call and return an error.
+		// This function returns the first network in a list of networks for a project. The project
+		// should be set via configuration instead of randomly taking the first.
+		networkName, err := getNetworkNameViaAPICall(service, config.NetworkProjectID)
 		if err != nil {
 			return nil, err
 		}
-	onXPN := networkProjectID != config.ProjectID
+		networkURL = gceNetworkURL(config.ApiEndpoint, netProjID, networkName)
+	} else if config.NetworkURL != "" {
+		networkURL = config.NetworkURL
+	} else {
+		networkURL = gceNetworkURL(config.ApiEndpoint, netProjID, config.NetworkName)
+	}
+
+	if config.SubnetworkURL != "" {
+		subnetURL = config.SubnetworkURL
+	} else if config.SubnetworkName != "" {
+		subnetURL = gceSubnetworkURL(config.ApiEndpoint, netProjID, config.Region, config.SubnetworkName)
+	}
 
 	if len(config.ManagedZones) == 0 {
 		config.ManagedZones, err = getZonesForRegion(service, config.ProjectID, config.Region)
@@ -417,14 +459,14 @@ func CreateGCECloud(config *CloudConfig) (*GCECloud, error) {
 		serviceBeta:              serviceBeta,
 		containerService:         containerService,
 		cloudkmsService:          cloudkmsService,
-		projectID:                config.ProjectID,
-		networkProjectID:         networkProjectID,
+		projectID:                projID,
+		networkProjectID:         netProjID,
 		onXPN:                    onXPN,
 		region:                   config.Region,
 		localZone:                config.Zone,
 		managedZones:             config.ManagedZones,
-		networkURL:               config.NetworkURL,
-		subnetworkURL:            config.SubnetworkURL,
+		networkURL:               networkURL,
+		subnetworkURL:            subnetURL,
 		secondaryRangeName:       config.SecondaryRangeName,
 		nodeTags:                 config.NodeTags,
 		nodeInstancePrefix:       config.NodeInstancePrefix,
@@ -473,6 +515,16 @@ func (gce *GCECloud) ProviderName() string {
 	return ProviderName
 }
 
+// ProjectID returns the ProjectID corresponding to the project this cloud is in.
+func (g *GCECloud) ProjectID() string {
+	return g.projectID
+}
+
+// NetworkProjectID returns the ProjectID corresponding to the project this cluster's network is in.
+func (g *GCECloud) NetworkProjectID() string {
+	return g.networkProjectID
+}
+
 // Region returns the region
 func (gce *GCECloud) Region() string {
 	return gce.region
@@ -569,6 +621,16 @@ func getNetworkNameViaAPICall(svc *compute.Service, projectID string) (string, e
 	return networkList.Items[0].Name, nil
 }
 
+// getProjectID returns the project's string ID given a project number or string
+func getProjectID(svc *compute.Service, projectNumberOrID string) (string, error) {
+	proj, err := svc.Projects.Get(projectNumberOrID).Do()
+	if err != nil {
+		return "", err
+	}
+
+	return proj.Name, nil
+}
+
 func getZonesForRegion(svc *compute.Service, projectID, region string) ([]string, error) {
 	// TODO: use PageToken to list all not just the first 500
 	listCall := svc.Zones.List(projectID)

pkg/cloudprovider/providers/gce/gce_firewall.go

@@ -27,38 +27,38 @@ func newFirewallMetricContext(request string) *metricContext {
 // GetFirewall returns the Firewall by name.
 func (gce *GCECloud) GetFirewall(name string) (*compute.Firewall, error) {
 	mc := newFirewallMetricContext("get")
-	v, err := gce.service.Firewalls.Get(gce.projectID, name).Do()
+	v, err := gce.service.Firewalls.Get(gce.NetworkProjectID(), name).Do()
 	return v, mc.Observe(err)
 }
 
 // CreateFirewall creates the passed firewall
 func (gce *GCECloud) CreateFirewall(f *compute.Firewall) error {
 	mc := newFirewallMetricContext("create")
-	op, err := gce.service.Firewalls.Insert(gce.projectID, f).Do()
+	op, err := gce.service.Firewalls.Insert(gce.NetworkProjectID(), f).Do()
 	if err != nil {
 		return mc.Observe(err)
 	}
 
-	return gce.waitForGlobalOp(op, mc)
+	return gce.waitForGlobalOpInProject(op, gce.NetworkProjectID(), mc)
 }
 
 // DeleteFirewall deletes the given firewall rule.
 func (gce *GCECloud) DeleteFirewall(name string) error {
 	mc := newFirewallMetricContext("delete")
-	op, err := gce.service.Firewalls.Delete(gce.projectID, name).Do()
+	op, err := gce.service.Firewalls.Delete(gce.NetworkProjectID(), name).Do()
 	if err != nil {
 		return mc.Observe(err)
 	}
-	return gce.waitForGlobalOp(op, mc)
+	return gce.waitForGlobalOpInProject(op, gce.NetworkProjectID(), mc)
 }
 
 // UpdateFirewall applies the given firewall as an update to an existing service.
 func (gce *GCECloud) UpdateFirewall(f *compute.Firewall) error {
 	mc := newFirewallMetricContext("update")
-	op, err := gce.service.Firewalls.Update(gce.projectID, f.Name, f).Do()
+	op, err := gce.service.Firewalls.Update(gce.NetworkProjectID(), f.Name, f).Do()
 	if err != nil {
 		return mc.Observe(err)
 	}
 
-	return gce.waitForGlobalOp(op, mc)
+	return gce.waitForGlobalOpInProject(op, gce.NetworkProjectID(), mc)
 }

pkg/cloudprovider/providers/gce/gce_loadbalancer_external.go

@@ -729,7 +729,7 @@ func (gce *GCECloud) firewallNeedsUpdate(name, serviceName, region, ipAddress st
 		return false, false, nil
 	}
 
-	fw, err := gce.service.Firewalls.Get(gce.projectID, makeFirewallName(name)).Do()
+	fw, err := gce.service.Firewalls.Get(gce.NetworkProjectID(), makeFirewallName(name)).Do()
 	if err != nil {
 		if isHTTPErrorCode(err, http.StatusNotFound) {
 			return false, true, nil
@@ -776,7 +776,7 @@ func (gce *GCECloud) ensureHttpHealthCheckFirewall(serviceName, ipAddress, regio
 	ports := []v1.ServicePort{{Protocol: "tcp", Port: hcPort}}
 
 	fwName := MakeHealthCheckFirewallName(clusterID, hcName, isNodesHealthCheck)
-	fw, err := gce.service.Firewalls.Get(gce.projectID, fwName).Do()
+	fw, err := gce.service.Firewalls.Get(gce.NetworkProjectID(), fwName).Do()
 	if err != nil {
 		if !isHTTPErrorCode(err, http.StatusNotFound) {
 			return fmt.Errorf("error getting firewall for health checks: %v", err)

pkg/cloudprovider/providers/gce/gce_op.go

@@ -93,62 +93,74 @@ func getErrorFromOp(op *computev1.Operation) error {
 }
 
 func (gce *GCECloud) waitForGlobalOp(op gceObject, mc *metricContext) error {
+	return gce.waitForGlobalOpInProject(op, gce.ProjectID(), mc)
+}
+
+func (gce *GCECloud) waitForRegionOp(op gceObject, region string, mc *metricContext) error {
+	return gce.waitForRegionOpInProject(op, gce.ProjectID(), region, mc)
+}
+
+func (gce *GCECloud) waitForZoneOp(op gceObject, zone string, mc *metricContext) error {
+	return gce.waitForZoneOpInProject(op, gce.ProjectID(), zone, mc)
+}
+
+func (gce *GCECloud) waitForGlobalOpInProject(op gceObject, projectID string, mc *metricContext) error {
 	switch v := op.(type) {
 	case *computealpha.Operation:
 		return gce.waitForOp(convertToV1Operation(op), func(operationName string) (*computev1.Operation, error) {
-			op, err := gce.serviceAlpha.GlobalOperations.Get(gce.projectID, operationName).Do()
+			op, err := gce.serviceAlpha.GlobalOperations.Get(projectID, operationName).Do()
 			return convertToV1Operation(op), err
 		}, mc)
 	case *computebeta.Operation:
 		return gce.waitForOp(convertToV1Operation(op), func(operationName string) (*computev1.Operation, error) {
-			op, err := gce.serviceBeta.GlobalOperations.Get(gce.projectID, operationName).Do()
+			op, err := gce.serviceBeta.GlobalOperations.Get(projectID, operationName).Do()
 			return convertToV1Operation(op), err
 		}, mc)
 	case *computev1.Operation:
 		return gce.waitForOp(op.(*computev1.Operation), func(operationName string) (*computev1.Operation, error) {
-			return gce.service.GlobalOperations.Get(gce.projectID, operationName).Do()
+			return gce.service.GlobalOperations.Get(projectID, operationName).Do()
 		}, mc)
 	default:
 		return fmt.Errorf("unexpected type: %T", v)
 	}
 }
 
-func (gce *GCECloud) waitForRegionOp(op gceObject, region string, mc *metricContext) error {
+func (gce *GCECloud) waitForRegionOpInProject(op gceObject, projectID, region string, mc *metricContext) error {
 	switch v := op.(type) {
 	case *computealpha.Operation:
 		return gce.waitForOp(convertToV1Operation(op), func(operationName string) (*computev1.Operation, error) {
-			op, err := gce.serviceAlpha.RegionOperations.Get(gce.projectID, region, operationName).Do()
+			op, err := gce.serviceAlpha.RegionOperations.Get(projectID, region, operationName).Do()
 			return convertToV1Operation(op), err
 		}, mc)
 	case *computebeta.Operation:
 		return gce.waitForOp(convertToV1Operation(op), func(operationName string) (*computev1.Operation, error) {
-			op, err := gce.serviceBeta.RegionOperations.Get(gce.projectID, region, operationName).Do()
+			op, err := gce.serviceBeta.RegionOperations.Get(projectID, region, operationName).Do()
 			return convertToV1Operation(op), err
 		}, mc)
 	case *computev1.Operation:
 		return gce.waitForOp(op.(*computev1.Operation), func(operationName string) (*computev1.Operation, error) {
-			return gce.service.RegionOperations.Get(gce.projectID, region, operationName).Do()
+			return gce.service.RegionOperations.Get(projectID, region, operationName).Do()
 		}, mc)
 	default:
 		return fmt.Errorf("unexpected type: %T", v)
 	}
 }
 
-func (gce *GCECloud) waitForZoneOp(op gceObject, zone string, mc *metricContext) error {
+func (gce *GCECloud) waitForZoneOpInProject(op gceObject, projectID, zone string, mc *metricContext) error {
 	switch v := op.(type) {
 	case *computealpha.Operation:
 		return gce.waitForOp(convertToV1Operation(op), func(operationName string) (*computev1.Operation, error) {
-			op, err := gce.serviceAlpha.ZoneOperations.Get(gce.projectID, zone, operationName).Do()
+			op, err := gce.serviceAlpha.ZoneOperations.Get(projectID, zone, operationName).Do()
 			return convertToV1Operation(op), err
 		}, mc)
 	case *computebeta.Operation:
 		return gce.waitForOp(convertToV1Operation(op), func(operationName string) (*computev1.Operation, error) {
-			op, err := gce.serviceBeta.ZoneOperations.Get(gce.projectID, zone, operationName).Do()
+			op, err := gce.serviceBeta.ZoneOperations.Get(projectID, zone, operationName).Do()
 			return convertToV1Operation(op), err
 		}, mc)
 	case *computev1.Operation:
 		return gce.waitForOp(op.(*computev1.Operation), func(operationName string) (*computev1.Operation, error) {
-			return gce.service.ZoneOperations.Get(gce.projectID, zone, operationName).Do()
+			return gce.service.ZoneOperations.Get(projectID, zone, operationName).Do()
 		}, mc)
 	default:
 		return fmt.Errorf("unexpected type: %T", v)

pkg/cloudprovider/providers/gce/gce_routes.go

@@ -38,13 +38,13 @@ func (gce *GCECloud) ListRoutes(clusterName string) ([]*cloudprovider.Route, err
 	page := 0
 	for ; page == 0 || (pageToken != "" && page < maxPages); page++ {
 		mc := newRoutesMetricContext("list_page")
-		listCall := gce.service.Routes.List(gce.projectID)
+		listCall := gce.service.Routes.List(gce.NetworkProjectID())
 
 		prefix := truncateClusterName(clusterName)
 		// Filter for routes starting with clustername AND belonging to the
 		// relevant gcp network AND having description = "k8s-node-route".
 		filter := "(name eq " + prefix + "-.*) "
-		filter = filter + "(network eq " + gce.networkURL + ") "
+		filter = filter + "(network eq " + gce.NetworkURL() + ") "
 		filter = filter + "(description eq " + k8sNodeRouteTag + ")"
 		listCall = listCall.Filter(filter)
 		if pageToken != "" {
@@ -80,11 +80,11 @@ func (gce *GCECloud) CreateRoute(clusterName string, nameHint string, route *clo
 	}
 
 	mc := newRoutesMetricContext("create")
-	insertOp, err := gce.service.Routes.Insert(gce.projectID, &compute.Route{
+	insertOp, err := gce.service.Routes.Insert(gce.NetworkProjectID(), &compute.Route{
 		Name:            routeName,
 		DestRange:       route.DestinationCIDR,
 		NextHopInstance: fmt.Sprintf("zones/%s/instances/%s", targetInstance.Zone, targetInstance.Name),
-		Network:         gce.networkURL,
+		Network:         gce.NetworkURL(),
 		Priority:        1000,
 		Description:     k8sNodeRouteTag,
 	}).Do()
@@ -96,16 +96,16 @@ func (gce *GCECloud) CreateRoute(clusterName string, nameHint string, route *clo
 			return mc.Observe(err)
 		}
 	}
-	return gce.waitForGlobalOp(insertOp, mc)
+	return gce.waitForGlobalOpInProject(insertOp, gce.NetworkProjectID(), mc)
 }
 
 func (gce *GCECloud) DeleteRoute(clusterName string, route *cloudprovider.Route) error {
 	mc := newRoutesMetricContext("delete")
-	deleteOp, err := gce.service.Routes.Delete(gce.projectID, route.Name).Do()
+	deleteOp, err := gce.service.Routes.Delete(gce.NetworkProjectID(), route.Name).Do()
 	if err != nil {
 		return mc.Observe(err)
 	}
-	return gce.waitForGlobalOp(deleteOp, mc)
+	return gce.waitForGlobalOpInProject(deleteOp, gce.NetworkProjectID(), mc)
 }
 
 func truncateClusterName(clusterName string) string {

test/e2e/e2e.go

@@ -84,8 +84,8 @@ func setupProviderConfig() error {
 			Region:             region,
 			Zone:               zone,
 			ManagedZones:       managedZones,
-			NetworkURL:         "",
-			SubnetworkURL:      "",
+			NetworkName:        "", // TODO: Change this to use framework.TestContext.CloudConfig.Network?
+			SubnetworkName:     "",
 			NodeTags:           nil,
 			NodeInstancePrefix: "",
 			TokenSource:        nil,