Core support for ip-per-service

cluster/gce/config-default.sh

@@ -31,3 +31,4 @@ MINION_IP_RANGES=($(eval echo "10.244.{1..${NUM_MINIONS}}.0/24"))
 MINION_SCOPES="compute-rw"
 # Increase the sleep interval value if concerned about API rate limits. 3, in seconds, is the default.
 POLL_SLEEP_INTERVAL=3
+PORTAL_NET="10.0.0.0/16"

cluster/gce/config-test.sh

@@ -31,3 +31,4 @@ MINION_IP_RANGES=($(eval echo "10.245.{1..${NUM_MINIONS}}.0/24"))
 MINION_SCOPES=""
 # Increase the sleep interval value if concerned about API rate limits. 3, in seconds, is the default.
 POLL_SLEEP_INTERVAL=3
+PORTAL_NET="10.0.0.0/16"

cluster/gce/templates/create-dynamic-salt-files.sh

@@ -21,6 +21,7 @@
 mkdir -p /srv/salt-overlay/pillar
 cat <<EOF >/srv/salt-overlay/pillar/cluster-params.sls
 node_instance_prefix: $NODE_INSTANCE_PREFIX
+portal_net: $PORTAL_NET
 EOF
 
 mkdir -p /srv/salt-overlay/salt/nginx

cluster/gce/util.sh

@@ -263,6 +263,7 @@ function kube-up {
     echo "readonly SERVER_BINARY_TAR_URL='${SERVER_BINARY_TAR_URL}'"
     echo "readonly SALT_TAR_URL='${SALT_TAR_URL}'"
     echo "readonly MASTER_HTPASSWD='${htpasswd}'"
+    echo "readonly PORTAL_NET='${PORTAL_NET}'"
     grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/create-dynamic-salt-files.sh"
     grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/download-release.sh"
     grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/salt-master.sh"

cluster/saltbase/salt/apiserver/default

@@ -55,5 +55,8 @@
   {%- set minion_regexp = "" %}
 {% endif %}
 {% endif %}
+{% if pillar['portal_net'] is defined %}
+  {% set portal_net = "-portal_net=" + pillar['portal_net'] %}
+{% endif %}
 
-DAEMON_ARGS="{{daemon_args}} {{address}} {{machines}} {{etcd_servers}} {{ minion_regexp }} {{ cloud_provider }} --allow_privileged={{pillar['allow_privileged']}}"
+DAEMON_ARGS="{{daemon_args}} {{address}} {{machines}} {{etcd_servers}} {{ minion_regexp }} {{ cloud_provider }} --allow_privileged={{pillar['allow_privileged']}} {{portal_net}}"

cluster/saltbase/salt/kube-proxy/initd

@@ -22,7 +22,7 @@ DAEMON_ARGS=""
 DAEMON_LOG_FILE=/var/log/$NAME.log
 PIDFILE=/var/run/$NAME.pid
 SCRIPTNAME=/etc/init.d/$NAME
-DAEMON_USER=kube-proxy
+DAEMON_USER=root
 
 # Exit if the package is not installed
 [ -x "$DAEMON" ] || exit 0

cmd/apiserver/apiserver.go

@@ -64,6 +64,7 @@ var (
 	machineList           util.StringList
 	corsAllowedOriginList util.StringList
 	allowPrivileged       = flag.Bool("allow_privileged", false, "If true, allow privileged containers.")
+	portalNet             util.IPNet // TODO: make this a list
 	// TODO: Discover these by pinging the host machines, and rip out these flags.
 	nodeMilliCPU      = flag.Int("node_milli_cpu", 1000, "The amount of MilliCPU provisioned on each node")
 	nodeMemory        = flag.Int("node_memory", 3*1024*1024*1024, "The amount of memory (in bytes) provisioned on each node")
@@ -75,6 +76,7 @@ func init() {
 	flag.Var(&etcdServerList, "etcd_servers", "List of etcd servers to watch (http://ip:port), comma separated. Mutually exclusive with -etcd_config")
 	flag.Var(&machineList, "machines", "List of machines to schedule onto, comma separated.")
 	flag.Var(&corsAllowedOriginList, "cors_allowed_origins", "List of allowed origins for CORS, comma separated.  An allowed origin can be a regular expression to support subdomain matching.  If this list is empty CORS will not be enabled.")
+	flag.Var(&portalNet, "portal_net", "A CIDR notation IP range from which to assign portal IPs. This must not overlap with any IP ranges assigned to nodes for pods.")
 }
 
 func verifyMinionFlags() {
@@ -89,6 +91,13 @@ func verifyMinionFlags() {
 	}
 }
 
+// TODO: Longer term we should read this from some config store, rather than a flag.
+func verifyPortalFlags() {
+	if portalNet.IP == nil {
+		glog.Fatal("No -portal_net specified")
+	}
+}
+
 func initCloudProvider(name string, configFilePath string) cloudprovider.Interface {
 	var config *os.File
 
@@ -141,6 +150,7 @@ func main() {
 
 	verflag.PrintAndExitIfRequested()
 	verifyMinionFlags()
+	verifyPortalFlags()
 
 	if (*etcdConfigFile != "" && len(etcdServerList) != 0) || (*etcdConfigFile == "" && len(etcdServerList) == 0) {
 		glog.Fatalf("specify either -etcd_servers or -etcd_config")
@@ -172,6 +182,7 @@ func main() {
 		glog.Fatalf("Invalid storage version or misconfigured etcd: %v", err)
 	}
 
+	n := net.IPNet(portalNet)
 	m := master.New(&master.Config{
 		Client:             client,
 		Cloud:              cloud,
@@ -188,6 +199,7 @@ func main() {
 				resources.Memory: util.NewIntOrStringFromInt(*nodeMemory),
 			},
 		},
+		PortalNet: &n,
 	})
 
 	mux := http.NewServeMux()

cmd/integration/integration.go

@@ -123,11 +123,16 @@ func startComponents(manifestURL string) (apiServerURL string) {
 	}
 
 	// Master
+	_, portalNet, err := net.ParseCIDR("10.0.0.0/24")
+	if err != nil {
+		glog.Fatalf("Unable to parse CIDR: %v", err)
+	}
 	m := master.New(&master.Config{
 		Client:        cl,
 		EtcdHelper:    helper,
 		Minions:       machineList,
 		PodInfoGetter: fakePodInfoGetter{},
+		PortalNet:     portalNet,
 	})
 	mux := http.NewServeMux()
 	apiserver.NewAPIGroup(m.API_v1beta1()).InstallREST(mux, "/api/v1beta1")
@@ -349,18 +354,33 @@ func runServiceTest(client *client.Client) {
 	if err := wait.Poll(time.Second, time.Second*20, podExists(client, ctx, pod.ID)); err != nil {
 		glog.Fatalf("FAILED: pod never started running %v", err)
 	}
-	svc := api.Service{
+	svc1 := api.Service{
 		TypeMeta: api.TypeMeta{ID: "service1"},
 		Selector: map[string]string{
 			"name": "thisisalonglabel",
 		},
 		Port: 8080,
 	}
-	_, err = client.CreateService(ctx, &svc)
+	_, err = client.CreateService(ctx, &svc1)
+	if err != nil {
+		glog.Fatalf("Failed to create service: %v, %v", svc1, err)
+	}
+	if err := wait.Poll(time.Second, time.Second*20, endpointsSet(client, ctx, svc1.ID, 1)); err != nil {
+		glog.Fatalf("FAILED: unexpected endpoints: %v", err)
+	}
+	// A second service with the same port.
+	svc2 := api.Service{
+		TypeMeta: api.TypeMeta{ID: "service2"},
+		Selector: map[string]string{
+			"name": "thisisalonglabel",
+		},
+		Port: 8080,
+	}
+	_, err = client.CreateService(ctx, &svc2)
 	if err != nil {
-		glog.Fatalf("Failed to create service: %v, %v", svc, err)
+		glog.Fatalf("Failed to create service: %v, %v", svc2, err)
 	}
-	if err := wait.Poll(time.Second, time.Second*10, endpointsSet(client, ctx, svc.ID, 1)); err != nil {
+	if err := wait.Poll(time.Second, time.Second*20, endpointsSet(client, ctx, svc2.ID, 1)); err != nil {
 		glog.Fatalf("FAILED: unexpected endpoints: %v", err)
 	}
 	glog.Info("Service test passed.")

cmd/proxy/proxy.go

@@ -25,6 +25,8 @@ import (
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/proxy"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/proxy/config"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/util/exec"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/util/iptables"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/version/verflag"
 	"github.com/coreos/go-etcd/etcd"
 	"github.com/golang/glog"
@@ -40,7 +42,7 @@ var (
 func init() {
 	client.BindClientConfigFlags(flag.CommandLine, clientConfig)
 	flag.Var(&etcdServerList, "etcd_servers", "List of etcd servers to watch (http://ip:port), comma separated (optional). Mutually exclusive with -etcd_config")
-	flag.Var(&bindAddress, "bind_address", "The address for the proxy server to serve on (set to 0.0.0.0 for all interfaces)")
+	flag.Var(&bindAddress, "bind_address", "The IP address for the proxy server to serve on (set to 0.0.0.0 for all interfaces)")
 }
 
 func main() {
@@ -97,12 +99,12 @@ func main() {
 	}
 
 	loadBalancer := proxy.NewLoadBalancerRR()
-	proxier := proxy.NewProxier(loadBalancer, net.IP(bindAddress))
+	proxier := proxy.NewProxier(loadBalancer, net.IP(bindAddress), iptables.New(exec.New()))
 	// Wire proxier to handle changes to services
 	serviceConfig.RegisterHandler(proxier)
 	// And wire loadBalancer to handle changes to endpoints to services
 	endpointsConfig.RegisterHandler(loadBalancer)
 
 	// Just loop forever for now...
-	select {}
+	proxier.SyncLoop()
 }

examples/guestbook-go/README.md

@@ -85,7 +85,7 @@ redis-slave-controller  gurpartap/redis  name=redis,role=slave  2
 The redis slave configures itself by looking for the Kubernetes service environment variables in the container environment. In particular, the redis slave is started with the following command:
 
 ```shell
-redis-server --slaveof $SERVICE_HOST $REDIS_MASTER_SERVICE_PORT
+redis-server --slaveof $REDIS_MASTER_SERVICE_HOST $REDIS_MASTER_SERVICE_PORT
 ```
 
 Once that's up you can list the pods in the cluster, to verify that the master and slaves are running:

examples/guestbook-go/_src/main.go

@@ -71,7 +71,7 @@ func HandleError(result interface{}, err error) (r interface{}) {
 }
 
 func main() {
-	pool = simpleredis.NewConnectionPoolHost(os.Getenv("SERVICE_HOST") + ":" + os.Getenv("REDIS_MASTER_SERVICE_PORT"))
+	pool = simpleredis.NewConnectionPoolHost(os.Getenv("REDIS_MASTER_SERVICE_HOST") + ":" + os.Getenv("REDIS_MASTER_SERVICE_PORT"))
 	defer pool.Close()
 
 	r := mux.NewRouter()

hack/test-cmd.sh

@@ -71,7 +71,8 @@ ${GO_OUT}/apiserver \
   --port="${API_PORT}" \
   --etcd_servers="http://${ETCD_HOST}:${ETCD_PORT}" \
   --machines="127.0.0.1" \
-  --minion_port=${KUBELET_PORT} 1>&2 &
+  --minion_port=${KUBELET_PORT} \
+  --portal_net="10.0.0.0/24" 1>&2 &
 APISERVER_PID=$!
 
 wait_for_url "http://127.0.0.1:${API_PORT}/healthz" "apiserver: "

pkg/api/types.go

@@ -423,6 +423,13 @@ type Service struct {
 	// ContainerPort is the name of the port on the container to direct traffic to.
 	// Optional, if unspecified use the first port on the container.
 	ContainerPort util.IntOrString `json:"containerPort,omitempty" yaml:"containerPort,omitempty"`
+
+	// PortalIP is assigned by the master.  If specified by the user it will be ignored.
+	// TODO: This is awkward - if we had a BoundService, it would be better factored.
+	PortalIP string `json:"portalIP,omitempty" yaml:"portalIP,omitempty"`
+
+	// ProxyPort is assigned by the master.  If specified by the user it will be ignored.
+	ProxyPort int `json:"proxyPort,omitempty" yaml:"proxyPort,omitempty"`
 }
 
 // Endpoints is a collection of endpoints that implement the actual service, for example:

pkg/api/v1beta1/types.go

@@ -451,6 +451,12 @@ type Service struct {
 	// ContainerPort is the name of the port on the container to direct traffic to.
 	// Optional, if unspecified use the first port on the container.
 	ContainerPort util.IntOrString `json:"containerPort,omitempty" yaml:"containerPort,omitempty"`
+
+	// PortalIP is assigned by the master.  If specified by the user it will be ignored.
+	PortalIP string `json:"portalIP,omitempty" yaml:"portalIP,omitempty"`
+
+	// ProxyPort is assigned by the master.  If specified by the user it will be ignored.
+	ProxyPort int `json:"proxyPort,omitempty" yaml:"proxyPort,omitempty"`
 }
 
 // Endpoints is a collection of endpoints that implement the actual service, for example:

pkg/api/v1beta2/types.go

@@ -416,6 +416,12 @@ type Service struct {
 	// ContainerPort is the name of the port on the container to direct traffic to.
 	// Optional, if unspecified use the first port on the container.
 	ContainerPort util.IntOrString `json:"containerPort,omitempty" yaml:"containerPort,omitempty"`
+
+	// PortalIP is assigned by the master.  If specified by the user it will be ignored.
+	PortalIP string `json:"portalIP,omitempty" yaml:"portalIP,omitempty"`
+
+	// ProxyPort is assigned by the master.  If specified by the user it will be ignored.
+	ProxyPort int `json:"proxyPort,omitempty" yaml:"proxyPort,omitempty"`
 }
 
 // Endpoints is a collection of endpoints that implement the actual service, for example:

pkg/api/v1beta3/types.go

@@ -570,6 +570,11 @@ type ReplicationControllerList struct {
 
 // ServiceStatus represents the current status of a service
 type ServiceStatus struct {
+	// PortalIP is assigned by the master.
+	PortalIP string `json:"portalIP,omitempty" yaml:"portalIP,omitempty"`
+
+	// ProxyPort is assigned by the master.  If 0, the proxy will choose an ephemeral port.
+	ProxyPort int `json:"proxyPort,omitempty" yaml:"proxyPort,omitempty"`
 }
 
 // ServiceSpec describes the attributes that a user creates on a service

pkg/kubecfg/resource_printer.go

@@ -140,7 +140,7 @@ func (h *HumanReadablePrinter) validatePrintHandlerFunc(printFunc reflect.Value)
 
 var podColumns = []string{"ID", "Image(s)", "Host", "Labels", "Status"}
 var replicationControllerColumns = []string{"ID", "Image(s)", "Selector", "Replicas"}
-var serviceColumns = []string{"ID", "Labels", "Selector", "Port"}
+var serviceColumns = []string{"ID", "Labels", "Selector", "IP", "Port"}
 var minionColumns = []string{"Minion identifier"}
 var statusColumns = []string{"Status"}
 var eventColumns = []string{"Name", "Kind", "Status", "Reason", "Message"}
@@ -226,8 +226,8 @@ func printReplicationControllerList(list *api.ReplicationControllerList, w io.Wr
 }
 
 func printService(svc *api.Service, w io.Writer) error {
-	_, err := fmt.Fprintf(w, "%s\t%s\t%s\t%d\n", svc.ID, labels.Set(svc.Labels),
-		labels.Set(svc.Selector), svc.Port)
+	_, err := fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%d\n", svc.ID, labels.Set(svc.Labels),
+		labels.Set(svc.Selector), svc.PortalIP, svc.Port)
 	return err
 }
 

pkg/kubectl/resource_printer.go

@@ -152,7 +152,7 @@ func (h *HumanReadablePrinter) validatePrintHandlerFunc(printFunc reflect.Value)
 
 var podColumns = []string{"ID", "IMAGE(S)", "HOST", "LABELS", "STATUS"}
 var replicationControllerColumns = []string{"ID", "IMAGE(S)", "SELECTOR", "REPLICAS"}
-var serviceColumns = []string{"ID", "LABELS", "SELECTOR", "PORT"}
+var serviceColumns = []string{"ID", "LABELS", "SELECTOR", "IP", "PORT"}
 var minionColumns = []string{"ID"}
 var statusColumns = []string{"STATUS"}
 
@@ -222,8 +222,8 @@ func printReplicationControllerList(list *api.ReplicationControllerList, w io.Wr
 }
 
 func printService(svc *api.Service, w io.Writer) error {
-	_, err := fmt.Fprintf(w, "%s\t%s\t%s\t%d\n", svc.ID, labels.Set(svc.Labels),
-		labels.Set(svc.Selector), svc.Port)
+	_, err := fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%d\n", svc.ID, labels.Set(svc.Labels),
+		labels.Set(svc.Selector), svc.PortalIP, svc.Port)
 	return err
 }
 

pkg/master/master.go

@@ -17,6 +17,7 @@ limitations under the License.
 package master
 
 import (
+	"net"
 	"net/http"
 	"time"
 
@@ -54,6 +55,7 @@ type Config struct {
 	MinionRegexp       string
 	PodInfoGetter      client.PodInfoGetter
 	NodeResources      api.NodeResources
+	PortalNet          *net.IPNet
 }
 
 // Master contains state for a Kubernetes cluster master/api server.
@@ -67,6 +69,7 @@ type Master struct {
 	eventRegistry      generic.Registry
 	storage            map[string]apiserver.RESTStorage
 	client             *client.Client
+	portalNet          *net.IPNet
 }
 
 // NewEtcdHelper returns an EtcdHelper for the provided arguments or an error if the version
@@ -98,6 +101,7 @@ func New(c *Config) *Master {
 		eventRegistry:      event.NewEtcdRegistry(c.EtcdHelper, uint64(c.EventTTL.Seconds())),
 		minionRegistry:     minionRegistry,
 		client:             c.Client,
+		portalNet:          c.PortalNet,
 	}
 	m.init(c)
 	return m
@@ -137,7 +141,7 @@ func (m *Master) init(c *Config) {
 			Minions:       m.client,
 		}),
 		"replicationControllers": controller.NewREST(m.controllerRegistry, m.podRegistry),
-		"services":               service.NewREST(m.serviceRegistry, c.Cloud, m.minionRegistry),
+		"services":               service.NewREST(m.serviceRegistry, c.Cloud, m.minionRegistry, m.portalNet),
 		"endpoints":              endpoint.NewREST(m.endpointRegistry),
 		"minions":                minion.NewREST(m.minionRegistry),
 		"events":                 event.NewREST(m.eventRegistry),

pkg/registry/pod/manifest_factory_test.go

@@ -48,10 +48,8 @@ func TestMakeManifestNoServices(t *testing.T) {
 	}
 
 	container := manifest.Containers[0]
-	if len(container.Env) != 1 ||
-		container.Env[0].Name != "SERVICE_HOST" ||
-		container.Env[0].Value != "machine" {
-		t.Errorf("Expected one env vars, got: %#v", manifest)
+	if len(container.Env) != 0 {
+		t.Errorf("Expected zero env vars, got: %#v", manifest)
 	}
 	if manifest.ID != "foobar" {
 		t.Errorf("Failed to assign ID to manifest: %#v", manifest.ID)
@@ -69,6 +67,7 @@ func TestMakeManifestServices(t *testing.T) {
 						Kind:   util.IntstrInt,
 						IntVal: 900,
 					},
+					PortalIP: "1.2.3.4",
 				},
 			},
 		},
@@ -96,7 +95,7 @@ func TestMakeManifestServices(t *testing.T) {
 	envs := []api.EnvVar{
 		{
 			Name:  "TEST_SERVICE_HOST",
-			Value: "machine",
+			Value: "1.2.3.4",
 		},
 		{
 			Name:  "TEST_SERVICE_PORT",
@@ -104,11 +103,11 @@ func TestMakeManifestServices(t *testing.T) {
 		},
 		{
 			Name:  "TEST_PORT",
-			Value: "tcp://machine:8080",
+			Value: "tcp://1.2.3.4:8080",
 		},
 		{
 			Name:  "TEST_PORT_8080_TCP",
-			Value: "tcp://machine:8080",
+			Value: "tcp://1.2.3.4:8080",
 		},
 		{
 			Name:  "TEST_PORT_8080_TCP_PROTO",
@@ -120,11 +119,7 @@ func TestMakeManifestServices(t *testing.T) {
 		},
 		{
 			Name:  "TEST_PORT_8080_TCP_ADDR",
-			Value: "machine",
-		},
-		{
-			Name:  "SERVICE_HOST",
-			Value: "machine",
+			Value: "1.2.3.4",
 		},
 	}
 	if len(container.Env) != len(envs) {
@@ -149,6 +144,7 @@ func TestMakeManifestServicesExistingEnvVar(t *testing.T) {
 						Kind:   util.IntstrInt,
 						IntVal: 900,
 					},
+					PortalIP: "1.2.3.4",
 				},
 			},
 		},
@@ -186,7 +182,7 @@ func TestMakeManifestServicesExistingEnvVar(t *testing.T) {
 		},
 		{
 			Name:  "TEST_SERVICE_HOST",
-			Value: "machine",
+			Value: "1.2.3.4",
 		},
 		{
 			Name:  "TEST_SERVICE_PORT",
@@ -194,11 +190,11 @@ func TestMakeManifestServicesExistingEnvVar(t *testing.T) {
 		},
 		{
 			Name:  "TEST_PORT",
-			Value: "tcp://machine:8080",
+			Value: "tcp://1.2.3.4:8080",
 		},
 		{
 			Name:  "TEST_PORT_8080_TCP",
-			Value: "tcp://machine:8080",
+			Value: "tcp://1.2.3.4:8080",
 		},
 		{
 			Name:  "TEST_PORT_8080_TCP_PROTO",
@@ -210,11 +206,7 @@ func TestMakeManifestServicesExistingEnvVar(t *testing.T) {
 		},
 		{
 			Name:  "TEST_PORT_8080_TCP_ADDR",
-			Value: "machine",
-		},
-		{
-			Name:  "SERVICE_HOST",
-			Value: "machine",
+			Value: "1.2.3.4",
 		},
 	}
 	if len(container.Env) != len(envs) {

pkg/registry/registrytest/service.go

@@ -17,6 +17,8 @@ limitations under the License.
 package registrytest
 
 import (
+	"sync"
+
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/labels"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/watch"
@@ -27,6 +29,7 @@ func NewServiceRegistry() *ServiceRegistry {
 }
 
 type ServiceRegistry struct {
+	mu            sync.Mutex
 	List          api.ServiceList
 	Service       *api.Service
 	Err           error
@@ -39,48 +42,84 @@ type ServiceRegistry struct {
 }
 
 func (r *ServiceRegistry) ListServices(ctx api.Context) (*api.ServiceList, error) {
-	return &r.List, r.Err
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	// Return by copy to avoid data races
+	res := new(api.ServiceList)
+	*res = r.List
+	res.Items = append([]api.Service{}, r.List.Items...)
+	return res, r.Err
 }
 
 func (r *ServiceRegistry) CreateService(ctx api.Context, svc *api.Service) error {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
 	r.Service = svc
 	r.List.Items = append(r.List.Items, *svc)
 	return r.Err
 }
 
 func (r *ServiceRegistry) GetService(ctx api.Context, id string) (*api.Service, error) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
 	r.GottenID = id
 	return r.Service, r.Err
 }
 
 func (r *ServiceRegistry) DeleteService(ctx api.Context, id string) error {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
 	r.DeletedID = id
+	r.Service = nil
 	return r.Err
 }
 
 func (r *ServiceRegistry) UpdateService(ctx api.Context, svc *api.Service) error {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
 	r.UpdatedID = svc.ID
+	r.Service = svc
 	return r.Err
 }
 
 func (r *ServiceRegistry) WatchServices(ctx api.Context, label labels.Selector, field labels.Selector, resourceVersion string) (watch.Interface, error) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
 	return nil, r.Err
 }
 
 func (r *ServiceRegistry) ListEndpoints(ctx api.Context) (*api.EndpointsList, error) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
 	return &r.EndpointsList, r.Err
 }
 
 func (r *ServiceRegistry) GetEndpoints(ctx api.Context, id string) (*api.Endpoints, error) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
 	r.GottenID = id
 	return &r.Endpoints, r.Err
 }
 
 func (r *ServiceRegistry) UpdateEndpoints(ctx api.Context, e *api.Endpoints) error {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
 	r.Endpoints = *e
 	return r.Err
 }
 
 func (r *ServiceRegistry) WatchEndpoints(ctx api.Context, label, field labels.Selector, resourceVersion string) (watch.Interface, error) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
 	return nil, r.Err
 }

pkg/registry/service/ip_allocator.go

@@ -32,7 +32,6 @@ type ipAllocator struct {
 }
 
 // newIPAllocator creates and intializes a new ipAllocator object.
-// FIXME: resync from storage at startup.
 func newIPAllocator(subnet *net.IPNet) *ipAllocator {
 	if subnet == nil || subnet.IP == nil || subnet.Mask == nil {
 		return nil

pkg/registry/service/rest.go

@@ -19,6 +19,7 @@ package service
 import (
 	"fmt"
 	"math/rand"
+	"net"
 	"strconv"
 	"strings"
 
@@ -32,21 +33,49 @@ import (
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/runtime"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/watch"
+	"github.com/golang/glog"
 )
 
 // REST adapts a service registry into apiserver's RESTStorage model.
 type REST struct {
-	registry Registry
-	cloud    cloudprovider.Interface
-	machines minion.Registry
+	registry  Registry
+	cloud     cloudprovider.Interface
+	machines  minion.Registry
+	portalMgr *ipAllocator
 }
 
 // NewREST returns a new REST.
-func NewREST(registry Registry, cloud cloudprovider.Interface, machines minion.Registry) *REST {
+func NewREST(registry Registry, cloud cloudprovider.Interface, machines minion.Registry, portalNet *net.IPNet) *REST {
+	// TODO: Before we can replicate masters, this has to be synced (e.g. lives in etcd)
+	ipa := newIPAllocator(portalNet)
+	reloadIPsFromStorage(ipa, registry)
+
 	return &REST{
-		registry: registry,
-		cloud:    cloud,
-		machines: machines,
+		registry:  registry,
+		cloud:     cloud,
+		machines:  machines,
+		portalMgr: ipa,
+	}
+}
+
+// Helper: mark all previously allocated IPs in the allocator.
+func reloadIPsFromStorage(ipa *ipAllocator, registry Registry) {
+	services, err := registry.ListServices(api.NewContext())
+	if err != nil {
+		// This is really bad.
+		glog.Errorf("can't list services to init service REST: %s", err)
+		return
+	}
+	for i := range services.Items {
+		s := &services.Items[i]
+		if s.PortalIP == "" {
+			glog.Warningf("service %q has no PortalIP", s.ID)
+			continue
+		}
+		if err := ipa.Allocate(net.ParseIP(s.PortalIP)); err != nil {
+			// This is really bad.
+			glog.Errorf("service %q PortalIP %s could not be allocated: %s", s.ID, s.PortalIP, err)
+		}
 	}
 }
 
@@ -61,9 +90,16 @@ func (rs *REST) Create(ctx api.Context, obj runtime.Object) (<-chan runtime.Obje
 
 	srv.CreationTimestamp = util.Now()
 
+	if ip, err := rs.portalMgr.AllocateNext(); err != nil {
+		return nil, err
+	} else {
+		srv.PortalIP = ip.String()
+	}
+
 	return apiserver.MakeAsync(func() (runtime.Object, error) {
 		// TODO: Consider moving this to a rectification loop, so that we make/remove external load balancers
 		// correctly no matter what http operations happen.
+		srv.ProxyPort = 0
 		if srv.CreateExternalLoadBalancer {
 			if rs.cloud == nil {
 				return nil, fmt.Errorf("requested an external service, but no cloud provider supplied.")
@@ -88,6 +124,9 @@ func (rs *REST) Create(ctx api.Context, obj runtime.Object) (<-chan runtime.Obje
 			if err != nil {
 				return nil, err
 			}
+			// External load-balancers require a known port for the service proxy.
+			// TODO: If we end up brokering HostPorts between Pods and Services, this can be any port.
+			srv.ProxyPort = srv.Port
 		}
 		err := rs.registry.CreateService(ctx, srv)
 		if err != nil {
@@ -110,6 +149,7 @@ func (rs *REST) Delete(ctx api.Context, id string) (<-chan runtime.Object, error
 	if err != nil {
 		return nil, err
 	}
+	rs.portalMgr.Release(net.ParseIP(service.PortalIP))
 	return apiserver.MakeAsync(func() (runtime.Object, error) {
 		rs.deleteExternalLoadBalancer(service)
 		return &api.Status{Status: api.StatusSuccess}, rs.registry.DeleteService(ctx, id)
@@ -161,16 +201,13 @@ func GetServiceEnvironmentVariables(ctx api.Context, registry Registry, machine
 	for _, service := range services.Items {
 		// Host
 		name := makeEnvVariableName(service.ID) + "_SERVICE_HOST"
-		result = append(result, api.EnvVar{Name: name, Value: machine})
+		result = append(result, api.EnvVar{Name: name, Value: service.PortalIP})
 		// Port
 		name = makeEnvVariableName(service.ID) + "_SERVICE_PORT"
 		result = append(result, api.EnvVar{Name: name, Value: strconv.Itoa(service.Port)})
 		// Docker-compatible vars.
-		result = append(result, makeLinkVariables(service, machine)...)
+		result = append(result, makeLinkVariables(service)...)
 	}
-	// The 'SERVICE_HOST' variable is deprecated.
-	// TODO(thockin): get rid of it once ip-per-service is in and "deployed".
-	result = append(result, api.EnvVar{Name: "SERVICE_HOST", Value: machine})
 	return result, nil
 }
 
@@ -183,8 +220,15 @@ func (rs *REST) Update(ctx api.Context, obj runtime.Object) (<-chan runtime.Obje
 		return nil, errors.NewInvalid("service", srv.ID, errs)
 	}
 	return apiserver.MakeAsync(func() (runtime.Object, error) {
+		cur, err := rs.registry.GetService(ctx, srv.ID)
+		if err != nil {
+			return nil, err
+		}
+		// Copy over non-user fields.
+		srv.PortalIP = cur.PortalIP
+		srv.ProxyPort = cur.ProxyPort
 		// TODO: check to see if external load balancer status changed
-		err := rs.registry.UpdateService(ctx, srv)
+		err = rs.registry.UpdateService(ctx, srv)
 		if err != nil {
 			return nil, err
 		}
@@ -234,7 +278,7 @@ func makeEnvVariableName(str string) string {
 	return strings.ToUpper(strings.Replace(str, "-", "_", -1))
 }
 
-func makeLinkVariables(service api.Service, machine string) []api.EnvVar {
+func makeLinkVariables(service api.Service) []api.EnvVar {
 	prefix := makeEnvVariableName(service.ID)
 	protocol := string(api.ProtocolTCP)
 	if service.Protocol != "" {
@@ -244,11 +288,11 @@ func makeLinkVariables(service api.Service, machine string) []api.EnvVar {
 	return []api.EnvVar{
 		{
 			Name:  prefix + "_PORT",
-			Value: fmt.Sprintf("%s://%s:%d", strings.ToLower(protocol), machine, service.Port),
+			Value: fmt.Sprintf("%s://%s:%d", strings.ToLower(protocol), service.PortalIP, service.Port),
 		},
 		{
 			Name:  portPrefix,
-			Value: fmt.Sprintf("%s://%s:%d", strings.ToLower(protocol), machine, service.Port),
+			Value: fmt.Sprintf("%s://%s:%d", strings.ToLower(protocol), service.PortalIP, service.Port),
 		},
 		{
 			Name:  portPrefix + "_PROTO",
@@ -260,7 +304,7 @@ func makeLinkVariables(service api.Service, machine string) []api.EnvVar {
 		},
 		{
 			Name:  portPrefix + "_ADDR",
-			Value: machine,
+			Value: service.PortalIP,
 		},
 	}
 }