Merge pull request #8643 from vishh/headless

cluster/addons/dns/kube2sky/Changelog

 * Fri May 15 2015  Tim Hockin <thockin@google.com>
   - First Changelog entry
   - Current version is 1.4
+
+
+## Version 1.7 (May 25 2015 Vishnu Kannan <vishnuk@google.com>)
+- Adding support for headless services. All pods backing a headless service is addressible via DNS RR.

cluster/addons/dns/kube2sky/Makefile

@@ -4,13 +4,13 @@
 
 .PHONY: all kube2sky container push clean test
 
-TAG = 1.6
+TAG = 1.7
 PREFIX = gcr.io/google_containers
 
 all: container
 
 kube2sky: kube2sky.go
-	CGO_ENABLED=0 godep go build -a -installsuffix cgo --ldflags '-w' ./kube2sky.go
+	GOOS=linux GOARCH=amd64 CGO_ENABLED=0 godep go build -a -installsuffix cgo --ldflags '-w' ./kube2sky.go
 
 container: kube2sky
 	docker build -t $(PREFIX)/kube2sky:$(TAG) .

cluster/addons/dns/kube2sky/README.md

@@ -21,8 +21,13 @@ example, if this is set to `kubernetes.io`, then a service named "nifty" in the
 
 `-verbose`: Log additional information.
 
-'-etcd_mutation_timeout': For how long the application will keep retrying etcd 
+`-etcd_mutation_timeout`: For how long the application will keep retrying etcd 
 mutation (insertion or removal of a dns entry) before giving up and crashing.
 
+`--etcd-server`: The etcd server that is being used by skydns.
+
+`--kube_master_url`: URL of kubernetes master. Reuired if `--kubecfg_file` is not set.
+
+`--kubecfg_file`: Path to kubecfg file that contains the master URL and tokens to authenticate with the master.
 
 [![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/cluster/addons/dns/kube2sky/README.md?pixel)]()

cluster/addons/dns/kube2sky/kube2sky_test.go

@@ -18,12 +18,14 @@ package main
 
 import (
 	"encoding/json"
+	"net/http"
 	"path"
 	"strings"
 	"testing"
 	"time"
 
 	kapi "github.com/GoogleCloudPlatform/kubernetes/pkg/api"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/client/cache"
 	"github.com/coreos/go-etcd/etcd"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -34,22 +36,35 @@ type fakeEtcdClient struct {
 	writes map[string]string
 }
 
-func (ec *fakeEtcdClient) Set(path, value string, ttl uint64) (*etcd.Response, error) {
-	ec.writes[path] = value
+func (ec *fakeEtcdClient) Set(key, value string, ttl uint64) (*etcd.Response, error) {
+	ec.writes[key] = value
 	return nil, nil
 }
 
-func (ec *fakeEtcdClient) Delete(path string, recursive bool) (*etcd.Response, error) {
+func (ec *fakeEtcdClient) Delete(key string, recursive bool) (*etcd.Response, error) {
 	for p := range ec.writes {
-		if (recursive && strings.HasPrefix(p, path)) || (!recursive && p == path) {
+		if (recursive && strings.HasPrefix(p, key)) || (!recursive && p == key) {
 			delete(ec.writes, p)
 		}
 	}
 	return nil, nil
 }
 
+func (ec *fakeEtcdClient) RawGet(key string, sort, recursive bool) (*etcd.RawResponse, error) {
+	count := 0
+	for path := range ec.writes {
+		if strings.HasPrefix(path, key) {
+			count++
+		}
+	}
+	if count == 0 {
+		return &etcd.RawResponse{StatusCode: http.StatusNotFound}, nil
+	}
+	return &etcd.RawResponse{StatusCode: http.StatusOK}, nil
+}
+
 const (
-	testDomain       = "cluster.local"
+	testDomain       = "cluster.local."
 	basePath         = "/skydns/local/cluster"
 	serviceSubDomain = "svc"
 )
@@ -59,6 +74,8 @@ func newKube2Sky(ec etcdClient) *kube2sky {
 		etcdClient:          ec,
 		domain:              testDomain,
 		etcdMutationTimeout: time.Second,
+		endpointsStore:      cache.NewStore(cache.MetaNamespaceKeyFunc),
+		servicesStore:       cache.NewStore(cache.MetaNamespaceKeyFunc),
 	}
 }
 
@@ -113,14 +130,166 @@ func TestHeadlessService(t *testing.T) {
 	k2s := newKube2Sky(ec)
 	service := kapi.Service{
 		ObjectMeta: kapi.ObjectMeta{
-			Name:      testNamespace,
+			Name:      testService,
+			Namespace: testNamespace,
+		},
+		Spec: kapi.ServiceSpec{
+			PortalIP: "None",
+			Ports: []kapi.ServicePort{
+				{Port: 80},
+			},
+		},
+	}
+	assert.NoError(t, k2s.servicesStore.Add(&service))
+	endpoints := kapi.Endpoints{
+		ObjectMeta: service.ObjectMeta,
+		Subsets: []kapi.EndpointSubset{
+			{
+				Addresses: []kapi.EndpointAddress{
+					{IP: "10.0.0.1"},
+					{IP: "10.0.0.2"},
+				},
+				Ports: []kapi.EndpointPort{
+					{Port: 80},
+				},
+			},
+			{
+				Addresses: []kapi.EndpointAddress{
+					{IP: "10.0.0.3"},
+					{IP: "10.0.0.4"},
+				},
+				Ports: []kapi.EndpointPort{
+					{Port: 8080},
+				},
+			},
+		},
+	}
+	// We expect 4 records with "svc" subdomain and 4 records without
+	// "svc" subdomain.
+	expectedDNSRecords := 8
+	assert.NoError(t, k2s.endpointsStore.Add(&endpoints))
+	k2s.newService(&service)
+	assert.Equal(t, expectedDNSRecords, len(ec.writes))
+	k2s.removeService(&service)
+	assert.Empty(t, ec.writes)
+}
+
+func TestHeadlessServiceEndpointsUpdate(t *testing.T) {
+	const (
+		testService   = "testService"
+		testNamespace = "default"
+	)
+	ec := &fakeEtcdClient{make(map[string]string)}
+	k2s := newKube2Sky(ec)
+	service := kapi.Service{
+		ObjectMeta: kapi.ObjectMeta{
+			Name:      testService,
+			Namespace: testNamespace,
+		},
+		Spec: kapi.ServiceSpec{
+			PortalIP: "None",
+			Ports: []kapi.ServicePort{
+				{Port: 80},
+			},
+		},
+	}
+	assert.NoError(t, k2s.servicesStore.Add(&service))
+	endpoints := kapi.Endpoints{
+		ObjectMeta: service.ObjectMeta,
+		Subsets: []kapi.EndpointSubset{
+			{
+				Addresses: []kapi.EndpointAddress{
+					{IP: "10.0.0.1"},
+					{IP: "10.0.0.2"},
+				},
+				Ports: []kapi.EndpointPort{
+					{Port: 80},
+				},
+			},
+		},
+	}
+	expectedDNSRecords := 4
+	assert.NoError(t, k2s.endpointsStore.Add(&endpoints))
+	k2s.newService(&service)
+	assert.Equal(t, expectedDNSRecords, len(ec.writes))
+	endpoints.Subsets = append(endpoints.Subsets,
+		kapi.EndpointSubset{
+			Addresses: []kapi.EndpointAddress{
+				{IP: "10.0.0.3"},
+				{IP: "10.0.0.4"},
+			},
+			Ports: []kapi.EndpointPort{
+				{Port: 8080},
+			},
+		},
+	)
+	expectedDNSRecords = 8
+	k2s.handleEndpointAdd(&endpoints)
+
+	assert.Equal(t, expectedDNSRecords, len(ec.writes))
+	k2s.removeService(&service)
+	assert.Empty(t, ec.writes)
+}
+
+func TestHeadlessServiceWithDelayedEndpointsAddition(t *testing.T) {
+	const (
+		testService   = "testService"
+		testNamespace = "default"
+	)
+	ec := &fakeEtcdClient{make(map[string]string)}
+	k2s := newKube2Sky(ec)
+	service := kapi.Service{
+		ObjectMeta: kapi.ObjectMeta{
+			Name:      testService,
 			Namespace: testNamespace,
 		},
+		Spec: kapi.ServiceSpec{
+			PortalIP: "None",
+			Ports: []kapi.ServicePort{
+				{Port: 80},
+			},
+		},
 	}
+	assert.NoError(t, k2s.servicesStore.Add(&service))
+	// Headless service DNS records should not be created since
+	// corresponding endpoints object doesn't exist.
 	k2s.newService(&service)
 	assert.Empty(t, ec.writes)
+
+	// Add an endpoints object for the service.
+	endpoints := kapi.Endpoints{
+		ObjectMeta: service.ObjectMeta,
+		Subsets: []kapi.EndpointSubset{
+			{
+				Addresses: []kapi.EndpointAddress{
+					{IP: "10.0.0.1"},
+					{IP: "10.0.0.2"},
+				},
+				Ports: []kapi.EndpointPort{
+					{Port: 80},
+				},
+			},
+			{
+				Addresses: []kapi.EndpointAddress{
+					{IP: "10.0.0.3"},
+					{IP: "10.0.0.4"},
+				},
+				Ports: []kapi.EndpointPort{
+					{Port: 8080},
+				},
+			},
+		},
+	}
+	// We expect 4 records with "svc" subdomain and 4 records without
+	// "svc" subdomain.
+	expectedDNSRecords := 8
+	k2s.handleEndpointAdd(&endpoints)
+	assert.Equal(t, expectedDNSRecords, len(ec.writes))
 }
 
+// TODO: Test service updates for headless services.
+// TODO: Test headless service addition with delayed endpoints addition
+
 func TestAddSinglePortService(t *testing.T) {
 	const (
 		testService   = "testService"
@@ -206,3 +375,10 @@ func TestDeleteSinglePortService(t *testing.T) {
 	k2s.removeService(&service)
 	assert.Empty(t, ec.writes)
 }
+
+func TestBuildDNSName(t *testing.T) {
+	expectedDNSName := "name.ns.svc.cluster.local."
+	assert.Equal(t, expectedDNSName, buildDNSNameString("local.", "cluster", "svc", "ns", "name"))
+	newExpectedDNSName := "00.name.ns.svc.cluster.local."
+	assert.Equal(t, newExpectedDNSName, buildDNSNameString(expectedDNSName, "00"))
+}

cluster/addons/dns/skydns-rc.yaml.in

@@ -30,7 +30,7 @@ spec:
         - -initial-cluster-token
         - skydns-etcd
       - name: kube2sky
-        image: gcr.io/google_containers/kube2sky:1.6
+        image: gcr.io/google_containers/kube2sky:1.7
         args:
         # command = "/kube2sky"
         - -domain={{ pillar['dns_domain'] }}

test/e2e/dns.go

@@ -178,4 +178,166 @@ var _ = Describe("DNS", func() {
 
 		Logf("DNS probes using %s succeeded\n", pod.Name)
 	})
+	It("should provide DNS for headless services", func() {
+		if providerIs("vagrant") {
+			By("Skipping test which is broken for vagrant (See https://github.com/GoogleCloudPlatform/kubernetes/issues/3580)")
+			return
+		}
+
+		podClient := f.Client.Pods(api.NamespaceDefault)
+
+		By("Waiting for DNS Service to be Running")
+		dnsPods, err := podClient.List(dnsServiceLableSelector, fields.Everything())
+		if err != nil {
+			Failf("Failed to list all dns service pods")
+		}
+		if len(dnsPods.Items) != 1 {
+			Failf("Unexpected number of pods (%d) matches the label selector %v", len(dnsPods.Items), dnsServiceLableSelector.String())
+		}
+		expectNoError(waitForPodRunning(f.Client, dnsPods.Items[0].Name))
+
+		// Create a test headless service.
+		By("Creating a test headless service")
+		testServiceName := "test-service"
+		testServiceSelector := map[string]string{
+			"dns-test": "true",
+		}
+		svc := &api.Service{
+			ObjectMeta: api.ObjectMeta{
+				Name: testServiceName,
+			},
+			Spec: api.ServiceSpec{
+				PortalIP: "None",
+				Ports: []api.ServicePort{
+					{Port: 80},
+				},
+				Selector: testServiceSelector,
+			},
+		}
+
+		_, err = f.Client.Services(f.Namespace.Name).Create(svc)
+		Expect(err).NotTo(HaveOccurred())
+		defer func() {
+			By("deleting the test headless service")
+			defer GinkgoRecover()
+			f.Client.Services(f.Namespace.Name).Delete(svc.Name)
+		}()
+
+		// All the names we need to be able to resolve.
+		// TODO: Create more endpoints and ensure that multiple A records are returned
+		// for headless service.
+		namesToResolve := []string{
+			fmt.Sprintf("%s", testServiceName),
+			fmt.Sprintf("%s.%s", testServiceName, f.Namespace.Name),
+			fmt.Sprintf("%s.%s.svc", testServiceName, f.Namespace.Name),
+		}
+
+		probeCmd := "for i in `seq 1 600`; do "
+		for _, name := range namesToResolve {
+			// Resolve by TCP and UDP DNS.
+			probeCmd += fmt.Sprintf(`test -n "$(dig +notcp +noall +answer +search %s)" && echo OK > /results/udp@%s;`, name, name)
+			probeCmd += fmt.Sprintf(`test -n "$(dig +tcp +noall +answer +search %s)" && echo OK > /results/tcp@%s;`, name, name)
+		}
+		probeCmd += "sleep 1; done"
+		Logf("vishh: 1")
+		// Run a pod which probes DNS and exposes the results by HTTP.
+		By("creating a pod to probe DNS")
+		pod := &api.Pod{
+			TypeMeta: api.TypeMeta{
+				Kind:       "Pod",
+				APIVersion: latest.Version,
+			},
+			ObjectMeta: api.ObjectMeta{
+				Name:   "dns-test",
+				Labels: testServiceSelector,
+			},
+			Spec: api.PodSpec{
+				Volumes: []api.Volume{
+					{
+						Name: "results",
+						VolumeSource: api.VolumeSource{
+							EmptyDir: &api.EmptyDirVolumeSource{},
+						},
+					},
+				},
+				Containers: []api.Container{
+					// TODO: Consider scraping logs instead of running a webserver.
+					{
+						Name:  "webserver",
+						Image: "gcr.io/google_containers/test-webserver",
+						VolumeMounts: []api.VolumeMount{
+							{
+								Name:      "results",
+								MountPath: "/results",
+							},
+						},
+					},
+					{
+						Name:    "querier",
+						Image:   "gcr.io/google_containers/dnsutils",
+						Command: []string{"sh", "-c", probeCmd},
+						VolumeMounts: []api.VolumeMount{
+							{
+								Name:      "results",
+								MountPath: "/results",
+							},
+						},
+					},
+				},
+			},
+		}
+
+		By("submitting the pod to kubernetes")
+		podClient = f.Client.Pods(f.Namespace.Name)
+		defer func() {
+			By("deleting the pod")
+			defer GinkgoRecover()
+			podClient.Delete(pod.Name, nil)
+		}()
+		if _, err := podClient.Create(pod); err != nil {
+			Failf("Failed to create %s pod: %v", pod.Name, err)
+		}
+
+		expectNoError(f.WaitForPodRunning(pod.Name))
+
+		By("retrieving the pod")
+		pod, err = podClient.Get(pod.Name)
+		if err != nil {
+			Failf("Failed to get pod %s: %v", pod.Name, err)
+		}
+
+		// Try to find results for each expected name.
+		By("looking for the results for each expected name")
+		var failed []string
+
+		expectNoError(wait.Poll(time.Second*2, time.Second*60, func() (bool, error) {
+			failed = []string{}
+			for _, name := range namesToResolve {
+				for _, proto := range []string{"udp", "tcp"} {
+					testCase := fmt.Sprintf("%s@%s", proto, name)
+					_, err := f.Client.Get().
+						Prefix("proxy").
+						Resource("pods").
+						Namespace(f.Namespace.Name).
+						Name(pod.Name).
+						Suffix("results", testCase).
+						Do().Raw()
+					if err != nil {
+						failed = append(failed, testCase)
+					}
+				}
+			}
+			if len(failed) == 0 {
+				return true, nil
+			}
+			Logf("Lookups using %s failed for: %v\n", pod.Name, failed)
+			return false, nil
+		}))
+		Expect(len(failed)).To(Equal(0))
+
+		// TODO: probe from the host, too.
+
+		Logf("DNS probes using %s succeeded\n", pod.Name)
+	})
+
 })