Update winkernel proxy for overlay

pkg/proxy/winkernel/BUILD

-load("@io_bazel_rules_go//go:def.bzl", "go_library")
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 
 go_library(
     name = "go_default_library",
     srcs = [
+        "hnsV1.go",
+        "hnsV2.go",
         "metrics.go",
         "proxier.go",
     ],
@@ -15,6 +17,7 @@ go_library(
             "//pkg/api/v1/service:go_default_library",
             "//pkg/apis/core/v1/helper:go_default_library",
             "//pkg/proxy:go_default_library",
+            "//pkg/proxy/apis/config:go_default_library",
             "//pkg/proxy/healthcheck:go_default_library",
             "//pkg/util/async:go_default_library",
             "//staging/src/k8s.io/api/core/v1:go_default_library",
@@ -23,6 +26,7 @@ go_library(
             "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
             "//staging/src/k8s.io/client-go/tools/record:go_default_library",
             "//vendor/github.com/Microsoft/hcsshim:go_default_library",
+            "//vendor/github.com/Microsoft/hcsshim/hcn:go_default_library",
             "//vendor/github.com/davecgh/go-spew/spew:go_default_library",
             "//vendor/k8s.io/klog:go_default_library",
         ],
@@ -43,3 +47,22 @@ filegroup(
     tags = ["automanaged"],
     visibility = ["//visibility:public"],
 )
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "hns_test.go",
+        "proxier_test.go",
+    ],
+    embed = [":go_default_library"],
+    deps = select({
+        "@io_bazel_rules_go//go/platform:windows": [
+            "//pkg/proxy:go_default_library",
+            "//staging/src/k8s.io/api/core/v1:go_default_library",
+            "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+            "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
+            "//vendor/github.com/Microsoft/hcsshim/hcn:go_default_library",
+        ],
+        "//conditions:default": [],
+    }),
+)

pkg/proxy/winkernel/hnsV1.go

+// +build windows
+
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package winkernel
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/Microsoft/hcsshim"
+	"k8s.io/klog"
+	"net"
+	"strings"
+)
+
+type HostNetworkService interface {
+	getNetworkByName(name string) (*hnsNetworkInfo, error)
+	getEndpointByID(id string) (*endpointsInfo, error)
+	getEndpointByIpAddress(ip string, networkName string) (*endpointsInfo, error)
+	createEndpoint(ep *endpointsInfo, networkName string) (*endpointsInfo, error)
+	deleteEndpoint(hnsID string) error
+	getLoadBalancer(endpoints []endpointsInfo, isILB bool, isDSR bool, sourceVip string, vip string, protocol uint16, internalPort uint16, externalPort uint16) (*loadBalancerInfo, error)
+	deleteLoadBalancer(hnsID string) error
+}
+
+// V1 HNS API
+type hnsV1 struct{}
+
+func (hns hnsV1) getNetworkByName(name string) (*hnsNetworkInfo, error) {
+	hnsnetwork, err := hcsshim.GetHNSNetworkByName(name)
+	if err != nil {
+		klog.Errorf("%v", err)
+		return nil, err
+	}
+
+	return &hnsNetworkInfo{
+		id:          hnsnetwork.Id,
+		name:        hnsnetwork.Name,
+		networkType: hnsnetwork.Type,
+	}, nil
+}
+func (hns hnsV1) getEndpointByID(id string) (*endpointsInfo, error) {
+	hnsendpoint, err := hcsshim.GetHNSEndpointByID(id)
+	if err != nil {
+		klog.Errorf("%v", err)
+		return nil, err
+	}
+	return &endpointsInfo{
+		ip:         hnsendpoint.IPAddress.String(),
+		isLocal:    !hnsendpoint.IsRemoteEndpoint, //TODO: Change isLocal to isRemote
+		macAddress: hnsendpoint.MacAddress,
+		hnsID:      hnsendpoint.Id,
+		hns:        hns,
+	}, nil
+}
+func (hns hnsV1) getEndpointByIpAddress(ip string, networkName string) (*endpointsInfo, error) {
+	hnsnetwork, err := hcsshim.GetHNSNetworkByName(networkName)
+	if err != nil {
+		klog.Errorf("%v", err)
+		return nil, err
+	}
+
+	endpoints, err := hcsshim.HNSListEndpointRequest()
+	for _, endpoint := range endpoints {
+		equal := false
+		if endpoint.IPAddress != nil {
+			equal = endpoint.IPAddress.String() == ip
+		}
+		if equal && strings.EqualFold(endpoint.VirtualNetwork, hnsnetwork.Id) {
+			return &endpointsInfo{
+				ip:         endpoint.IPAddress.String(),
+				isLocal:    !endpoint.IsRemoteEndpoint,
+				macAddress: endpoint.MacAddress,
+				hnsID:      endpoint.Id,
+				hns:        hns,
+			}, nil
+		}
+	}
+
+	return nil, fmt.Errorf("Endpoint %v not found on network %s", ip, networkName)
+}
+func (hns hnsV1) createEndpoint(ep *endpointsInfo, networkName string) (*endpointsInfo, error) {
+	hnsNetwork, err := hcsshim.GetHNSNetworkByName(networkName)
+	if err != nil {
+		return nil, fmt.Errorf("Could not find network %s: %v", networkName, err)
+	}
+	hnsEndpoint := &hcsshim.HNSEndpoint{
+		MacAddress: ep.macAddress,
+		IPAddress:  net.ParseIP(ep.ip),
+	}
+
+	var createdEndpoint *hcsshim.HNSEndpoint
+	if !ep.isLocal {
+		if len(ep.providerAddress) != 0 {
+			paPolicy := hcsshim.PaPolicy{
+				Type: hcsshim.PA,
+				PA:   ep.providerAddress,
+			}
+			paPolicyJson, err := json.Marshal(paPolicy)
+			if err != nil {
+				return nil, fmt.Errorf("PA Policy creation failed: %v", err)
+			}
+			hnsEndpoint.Policies = append(hnsEndpoint.Policies, paPolicyJson)
+		}
+		createdEndpoint, err = hnsNetwork.CreateRemoteEndpoint(hnsEndpoint)
+		if err != nil {
+			return nil, fmt.Errorf("Remote endpoint creation failed: %v", err)
+		}
+
+	} else {
+		createdEndpoint, err = hnsNetwork.CreateEndpoint(hnsEndpoint)
+		if err != nil {
+			return nil, fmt.Errorf("Local endpoint creation failed: %v", err)
+		}
+	}
+	return &endpointsInfo{
+		ip:              createdEndpoint.IPAddress.String(),
+		isLocal:         createdEndpoint.IsRemoteEndpoint,
+		macAddress:      createdEndpoint.MacAddress,
+		hnsID:           createdEndpoint.Id,
+		providerAddress: ep.providerAddress, //TODO get from createdEndpoint
+		hns:             hns,
+	}, nil
+}
+func (hns hnsV1) deleteEndpoint(hnsID string) error {
+	hnsendpoint, err := hcsshim.GetHNSEndpointByID(hnsID)
+	if err != nil {
+		return err
+	}
+	_, err = hnsendpoint.Delete()
+	if err == nil {
+		klog.V(3).Infof("Remote endpoint resource deleted id %s", hnsID)
+	}
+	return err
+}
+
+func (hns hnsV1) getLoadBalancer(endpoints []endpointsInfo, isILB bool, isDSR bool, sourceVip string, vip string, protocol uint16, internalPort uint16, externalPort uint16) (*loadBalancerInfo, error) {
+	plists, err := hcsshim.HNSListPolicyListRequest()
+	if err != nil {
+		return nil, err
+	}
+
+	if isDSR {
+		klog.V(3).Info("DSR is not supported in V1. Using non DSR instead")
+	}
+
+	for _, plist := range plists {
+		if len(plist.EndpointReferences) != len(endpoints) {
+			continue
+		}
+		// Validate if input meets any of the policy lists
+		elbPolicy := hcsshim.ELBPolicy{}
+		if err = json.Unmarshal(plist.Policies[0], &elbPolicy); err != nil {
+			continue
+		}
+		if elbPolicy.Protocol == protocol && elbPolicy.InternalPort == internalPort && elbPolicy.ExternalPort == externalPort && elbPolicy.ILB == isILB {
+			if len(vip) > 0 {
+				if len(elbPolicy.VIPs) == 0 || elbPolicy.VIPs[0] != vip {
+					continue
+				}
+			}
+			LogJson(plist, "Found existing Hns loadbalancer policy resource", 1)
+			return &loadBalancerInfo{
+				hnsID: plist.ID,
+			}, nil
+		}
+	}
+
+	var hnsEndpoints []hcsshim.HNSEndpoint
+	for _, ep := range endpoints {
+		endpoint, err := hcsshim.GetHNSEndpointByID(ep.hnsID)
+		if err != nil {
+			return nil, err
+		}
+		hnsEndpoints = append(hnsEndpoints, *endpoint)
+	}
+	lb, err := hcsshim.AddLoadBalancer(
+		hnsEndpoints,
+		isILB,
+		sourceVip,
+		vip,
+		protocol,
+		internalPort,
+		externalPort,
+	)
+
+	if err == nil {
+		LogJson(lb, "Hns loadbalancer policy resource", 1)
+	} else {
+		return nil, err
+	}
+	return &loadBalancerInfo{
+		hnsID: lb.ID,
+	}, err
+}
+func (hns hnsV1) deleteLoadBalancer(hnsID string) error {
+	if len(hnsID) == 0 {
+		// Return silently
+		return nil
+	}
+
+	// Cleanup HNS policies
+	hnsloadBalancer, err := hcsshim.GetPolicyListByID(hnsID)
+	if err != nil {
+		return err
+	}
+	LogJson(hnsloadBalancer, "Removing Policy", 2)
+
+	_, err = hnsloadBalancer.Delete()
+	return err
+}

pkg/proxy/winkernel/hnsV2.go

+// +build windows
+
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package winkernel
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/Microsoft/hcsshim/hcn"
+	"k8s.io/klog"
+
+	"strings"
+)
+
+type hnsV2 struct{}
+
+func (hns hnsV2) getNetworkByName(name string) (*hnsNetworkInfo, error) {
+	hnsnetwork, err := hcn.GetNetworkByName(name)
+	if err != nil {
+		klog.Errorf("%v", err)
+		return nil, err
+	}
+
+	var remoteSubnets []*remoteSubnetInfo
+	for _, policy := range hnsnetwork.Policies {
+		if policy.Type == hcn.RemoteSubnetRoute {
+			policySettings := hcn.RemoteSubnetRoutePolicySetting{}
+			err = json.Unmarshal(policy.Settings, &policySettings)
+			if err != nil {
+				return nil, fmt.Errorf("Failed to unmarshal Remote Subnet policy settings")
+			}
+			rs := &remoteSubnetInfo{
+				destinationPrefix: policySettings.DestinationPrefix,
+				isolationId:       policySettings.IsolationId,
+				providerAddress:   policySettings.ProviderAddress,
+				drMacAddress:      policySettings.DistributedRouterMacAddress,
+			}
+			remoteSubnets = append(remoteSubnets, rs)
+		}
+	}
+
+	return &hnsNetworkInfo{
+		id:            hnsnetwork.Id,
+		name:          hnsnetwork.Name,
+		networkType:   string(hnsnetwork.Type),
+		remoteSubnets: remoteSubnets,
+	}, nil
+}
+func (hns hnsV2) getEndpointByID(id string) (*endpointsInfo, error) {
+	hnsendpoint, err := hcn.GetEndpointByID(id)
+	if err != nil {
+		return nil, err
+	}
+	return &endpointsInfo{ //TODO: fill out PA
+		ip:         hnsendpoint.IpConfigurations[0].IpAddress,
+		isLocal:    uint32(hnsendpoint.Flags&hcn.EndpointFlagsRemoteEndpoint) == 0, //TODO: Change isLocal to isRemote
+		macAddress: hnsendpoint.MacAddress,
+		hnsID:      hnsendpoint.Id,
+		hns:        hns,
+	}, nil
+}
+func (hns hnsV2) getEndpointByIpAddress(ip string, networkName string) (*endpointsInfo, error) {
+	hnsnetwork, err := hcn.GetNetworkByName(networkName)
+	if err != nil {
+		klog.Errorf("%v", err)
+		return nil, err
+	}
+
+	endpoints, err := hcn.ListEndpoints()
+	for _, endpoint := range endpoints {
+		equal := false
+		if endpoint.IpConfigurations != nil && len(endpoint.IpConfigurations) > 0 {
+			equal = endpoint.IpConfigurations[0].IpAddress == ip
+		}
+		if equal && strings.EqualFold(endpoint.HostComputeNetwork, hnsnetwork.Id) {
+			return &endpointsInfo{
+				ip:         endpoint.IpConfigurations[0].IpAddress,
+				isLocal:    uint32(endpoint.Flags&hcn.EndpointFlagsRemoteEndpoint) == 0, //TODO: Change isLocal to isRemote
+				macAddress: endpoint.MacAddress,
+				hnsID:      endpoint.Id,
+				hns:        hns,
+			}, nil
+		}
+	}
+
+	return nil, fmt.Errorf("Endpoint %v not found on network %s", ip, networkName)
+}
+func (hns hnsV2) createEndpoint(ep *endpointsInfo, networkName string) (*endpointsInfo, error) {
+	hnsNetwork, err := hcn.GetNetworkByName(networkName)
+	if err != nil {
+		return nil, fmt.Errorf("Could not find network %s: %v", networkName, err)
+	}
+	var flags hcn.EndpointFlags
+	if !ep.isLocal {
+		flags |= hcn.EndpointFlagsRemoteEndpoint
+	}
+	ipConfig := &hcn.IpConfig{
+		IpAddress: ep.ip,
+	}
+	hnsEndpoint := &hcn.HostComputeEndpoint{
+		IpConfigurations: []hcn.IpConfig{*ipConfig},
+		MacAddress:       ep.macAddress,
+		Flags:            flags,
+		SchemaVersion: hcn.SchemaVersion{
+			Major: 2,
+			Minor: 0,
+		},
+	}
+
+	var createdEndpoint *hcn.HostComputeEndpoint
+	if !ep.isLocal {
+		if len(ep.providerAddress) != 0 {
+			policySettings := hcn.ProviderAddressEndpointPolicySetting{
+				ProviderAddress: ep.providerAddress,
+			}
+			policySettingsJson, err := json.Marshal(policySettings)
+			if err != nil {
+				return nil, fmt.Errorf("PA Policy creation failed: %v", err)
+			}
+			paPolicy := hcn.EndpointPolicy{
+				Type:     hcn.NetworkProviderAddress,
+				Settings: policySettingsJson,
+			}
+			hnsEndpoint.Policies = append(hnsEndpoint.Policies, paPolicy)
+		}
+		createdEndpoint, err = hnsNetwork.CreateRemoteEndpoint(hnsEndpoint)
+		if err != nil {
+			return nil, fmt.Errorf("Remote endpoint creation failed: %v", err)
+		}
+	} else {
+		createdEndpoint, err = hnsNetwork.CreateEndpoint(hnsEndpoint)
+		if err != nil {
+			return nil, fmt.Errorf("Local endpoint creation failed: %v", err)
+		}
+	}
+	return &endpointsInfo{
+		ip:              createdEndpoint.IpConfigurations[0].IpAddress,
+		isLocal:         uint32(createdEndpoint.Flags&hcn.EndpointFlagsRemoteEndpoint) == 0,
+		macAddress:      createdEndpoint.MacAddress,
+		hnsID:           createdEndpoint.Id,
+		providerAddress: ep.providerAddress, //TODO get from createdEndpoint
+		hns:             hns,
+	}, nil
+}
+func (hns hnsV2) deleteEndpoint(hnsID string) error {
+	hnsendpoint, err := hcn.GetEndpointByID(hnsID)
+	if err != nil {
+		return err
+	}
+	err = hnsendpoint.Delete()
+	if err == nil {
+		klog.V(3).Infof("Remote endpoint resource deleted id %s", hnsID)
+	}
+	return err
+}
+func (hns hnsV2) getLoadBalancer(endpoints []endpointsInfo, isILB bool, isDSR bool, sourceVip string, vip string, protocol uint16, internalPort uint16, externalPort uint16) (*loadBalancerInfo, error) {
+	plists, err := hcn.ListLoadBalancers()
+	if err != nil {
+		return nil, err
+	}
+
+	for _, plist := range plists {
+		if len(plist.HostComputeEndpoints) != len(endpoints) {
+			continue
+		}
+		// Validate if input meets any of the policy lists
+		lbPortMapping := plist.PortMappings[0]
+		if lbPortMapping.Protocol == uint32(protocol) && lbPortMapping.InternalPort == internalPort && lbPortMapping.ExternalPort == externalPort && (lbPortMapping.Flags&1 != 0) == isILB {
+			if len(vip) > 0 {
+				if len(plist.FrontendVIPs) == 0 || plist.FrontendVIPs[0] != vip {
+					continue
+				}
+			}
+			LogJson(plist, "Found existing Hns loadbalancer policy resource", 1)
+			return &loadBalancerInfo{
+				hnsID: plist.Id,
+			}, nil
+		}
+	}
+
+	var hnsEndpoints []hcn.HostComputeEndpoint
+	for _, ep := range endpoints {
+		endpoint, err := hcn.GetEndpointByID(ep.hnsID)
+		if err != nil {
+			return nil, err
+		}
+		hnsEndpoints = append(hnsEndpoints, *endpoint)
+	}
+
+	vips := []string{}
+	if len(vip) > 0 {
+		vips = append(vips, vip)
+	}
+	lb, err := hcn.AddLoadBalancer(
+		hnsEndpoints,
+		isILB,
+		isDSR,
+		sourceVip,
+		vips,
+		protocol,
+		internalPort,
+		externalPort,
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	LogJson(lb, "Hns loadbalancer policy resource", 1)
+
+	return &loadBalancerInfo{
+		hnsID: lb.Id,
+	}, err
+}
+func (hns hnsV2) deleteLoadBalancer(hnsID string) error {
+	lb, err := hcn.GetLoadBalancerByID(hnsID)
+	if err != nil {
+		// Return silently
+		return nil
+	}
+
+	err = lb.Delete()
+	return err
+}