Port forward over websockets

cluster/saltbase/salt/e2e-image-puller/e2e-image-puller.manifest

@@ -27,7 +27,7 @@ spec:
     command:
     - /bin/sh
     - -c
-    - "for i in gcr.io/google_containers/busybox gcr.io/google_containers/busybox:1.24 gcr.io/google_containers/dnsutils:e2e gcr.io/google_containers/eptest:0.1 gcr.io/google_containers/fakegitserver:0.1 gcr.io/google_containers/hostexec:1.2 gcr.io/google_containers/iperf:e2e gcr.io/google_containers/jessie-dnsutils:e2e gcr.io/google_containers/liveness:e2e gcr.io/google_containers/mounttest:0.7 gcr.io/google_containers/mounttest-user:0.3 gcr.io/google_containers/netexec:1.4 gcr.io/google_containers/netexec:1.7 gcr.io/google_containers/nettest:1.7 gcr.io/google_containers/nettest:1.8 gcr.io/google_containers/nginx-slim:0.7 gcr.io/google_containers/nginx-slim:0.8 gcr.io/google_containers/n-way-http:1.0 gcr.io/google_containers/pause:2.0 gcr.io/google_containers/pause-amd64:3.0 gcr.io/google_containers/porter:cd5cb5791ebaa8641955f0e8c2a9bed669b1eaab gcr.io/google_containers/portforwardtester:1.0 gcr.io/google_containers/redis:e2e gcr.io/google_containers/resource_consumer:beta4 gcr.io/google_containers/resource_consumer/controller:beta4 gcr.io/google_containers/serve_hostname:v1.4 gcr.io/google_containers/test-webserver:e2e gcr.io/google_containers/ubuntu:14.04 gcr.io/google_containers/update-demo:kitten gcr.io/google_containers/update-demo:nautilus gcr.io/google_containers/volume-ceph:0.1 gcr.io/google_containers/volume-gluster:0.2 gcr.io/google_containers/volume-iscsi:0.1 gcr.io/google_containers/volume-nfs:0.6 gcr.io/google_containers/volume-rbd:0.1 gcr.io/google_samples/gb-redisslave:v1 gcr.io/google_containers/redis:v1; do echo $(date '+%X') pulling $i; docker pull $i 1>/dev/null; done; exit 0;"
+    - "for i in gcr.io/google_containers/busybox gcr.io/google_containers/busybox:1.24 gcr.io/google_containers/dnsutils:e2e gcr.io/google_containers/eptest:0.1 gcr.io/google_containers/fakegitserver:0.1 gcr.io/google_containers/hostexec:1.2 gcr.io/google_containers/iperf:e2e gcr.io/google_containers/jessie-dnsutils:e2e gcr.io/google_containers/liveness:e2e gcr.io/google_containers/mounttest:0.7 gcr.io/google_containers/mounttest-user:0.3 gcr.io/google_containers/netexec:1.4 gcr.io/google_containers/netexec:1.7 gcr.io/google_containers/nettest:1.7 gcr.io/google_containers/nettest:1.8 gcr.io/google_containers/nginx-slim:0.7 gcr.io/google_containers/nginx-slim:0.8 gcr.io/google_containers/n-way-http:1.0 gcr.io/google_containers/pause:2.0 gcr.io/google_containers/pause-amd64:3.0 gcr.io/google_containers/porter:cd5cb5791ebaa8641955f0e8c2a9bed669b1eaab gcr.io/google_containers/portforwardtester:1.2 gcr.io/google_containers/redis:e2e gcr.io/google_containers/resource_consumer:beta4 gcr.io/google_containers/resource_consumer/controller:beta4 gcr.io/google_containers/serve_hostname:v1.4 gcr.io/google_containers/test-webserver:e2e gcr.io/google_containers/ubuntu:14.04 gcr.io/google_containers/update-demo:kitten gcr.io/google_containers/update-demo:nautilus gcr.io/google_containers/volume-ceph:0.1 gcr.io/google_containers/volume-gluster:0.2 gcr.io/google_containers/volume-iscsi:0.1 gcr.io/google_containers/volume-nfs:0.6 gcr.io/google_containers/volume-rbd:0.1 gcr.io/google_samples/gb-redisslave:v1 gcr.io/google_containers/redis:v1; do echo $(date '+%X') pulling $i; docker pull $i 1>/dev/null; done; exit 0;"
     securityContext:
       privileged: true
     volumeMounts:

pkg/client/tests/portfoward_test.go

@@ -84,7 +84,7 @@ func fakePortForwardServer(t *testing.T, testName string, serverSends, expectedF
 			received: make(map[uint16]string),
 			send:     serverSends,
 		}
-		portforward.ServePortForward(w, req, pf, "pod", "uid", 0, 10*time.Second)
+		portforward.ServePortForward(w, req, pf, "pod", "uid", 0, 10*time.Second, portforward.SupportedProtocols)
 
 		for port, expected := range expectedFromClient {
 			actual, ok := pf.received[port]

pkg/genericapiserver/endpoints/installer.go

@@ -1047,10 +1047,14 @@ func typeToJSON(typeName string) string {
 		return "string"
 	case "byte", "*byte":
 		return "string"
-	case "[]string", "[]*string":
-		// TODO: Fix this when go-restful supports a way to specify an array query param:
+
+	// TODO: Fix these when go-restful supports a way to specify an array query param:
 	// https://github.com/emicklei/go-restful/issues/225
+	case "[]string", "[]*string":
 		return "string"
+	case "[]int32", "[]*int32":
+		return "integer"
+
 	default:
 		return typeName
 	}

pkg/kubelet/kubelet.go

@@ -2173,7 +2173,8 @@ func getStreamingConfig(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps
 		},
 		StreamIdleTimeout:               kubeCfg.StreamingConnectionIdleTimeout.Duration,
 		StreamCreationTimeout:           streaming.DefaultConfig.StreamCreationTimeout,
-		SupportedProtocols:    streaming.DefaultConfig.SupportedProtocols,
+		SupportedRemoteCommandProtocols: streaming.DefaultConfig.SupportedRemoteCommandProtocols,
+		SupportedPortForwardProtocols:   streaming.DefaultConfig.SupportedPortForwardProtocols,
 	}
 	if kubeDeps.TLSOptions != nil {
 		config.TLSConfig = kubeDeps.TLSOptions.Config

pkg/kubelet/server/portforward/constants.go

@@ -18,4 +18,6 @@ limitations under the License.
 package portforward
 
 // The subprotocol "portforward.k8s.io" is used for port forwarding.
-const PortForwardProtocolV1Name = "portforward.k8s.io"
+const ProtocolV1Name = "portforward.k8s.io"
+
+var SupportedProtocols = []string{ProtocolV1Name}

pkg/kubelet/server/portforward/portforward_test.go → pkg/kubelet/server/portforward/httpstream_test.go

@@ -25,7 +25,7 @@ import (
 	"k8s.io/kubernetes/pkg/api"
 )
 
-func TestPortForwardStreamReceived(t *testing.T) {
+func TestHTTPStreamReceived(t *testing.T) {
 	tests := map[string]struct {
 		port          string
 		streamType    string
@@ -62,7 +62,7 @@ func TestPortForwardStreamReceived(t *testing.T) {
 	}
 	for name, test := range tests {
 		streams := make(chan httpstream.Stream, 1)
-		f := portForwardStreamReceived(streams)
+		f := httpStreamReceived(streams)
 		stream := newFakeHttpStream()
 		if len(test.port) > 0 {
 			stream.headers.Set("port", test.port)
@@ -92,48 +92,11 @@ func TestPortForwardStreamReceived(t *testing.T) {
 	}
 }
 
-type fakeHttpStream struct {
-	headers http.Header
-	id      uint32
-}
-
-func newFakeHttpStream() *fakeHttpStream {
-	return &fakeHttpStream{
-		headers: make(http.Header),
-	}
-}
-
-var _ httpstream.Stream = &fakeHttpStream{}
-
-func (s *fakeHttpStream) Read(data []byte) (int, error) {
-	return 0, nil
-}
-
-func (s *fakeHttpStream) Write(data []byte) (int, error) {
-	return 0, nil
-}
-
-func (s *fakeHttpStream) Close() error {
-	return nil
-}
-
-func (s *fakeHttpStream) Reset() error {
-	return nil
-}
-
-func (s *fakeHttpStream) Headers() http.Header {
-	return s.headers
-}
-
-func (s *fakeHttpStream) Identifier() uint32 {
-	return s.id
-}
-
 func TestGetStreamPair(t *testing.T) {
 	timeout := make(chan time.Time)
 
-	h := &portForwardStreamHandler{
-		streamPairs: make(map[string]*portForwardStreamPair),
+	h := &httpStreamHandler{
+		streamPairs: make(map[string]*httpStreamPair),
 	}
 
 	// test adding a new entry
@@ -223,7 +186,7 @@ func TestGetStreamPair(t *testing.T) {
 }
 
 func TestRequestID(t *testing.T) {
-	h := &portForwardStreamHandler{}
+	h := &httpStreamHandler{}
 
 	s := newFakeHttpStream()
 	s.headers.Set(api.StreamType, api.StreamTypeError)
@@ -244,3 +207,40 @@ func TestRequestID(t *testing.T) {
 		t.Errorf("expected %q, got %q", e, a)
 	}
 }
+
+type fakeHttpStream struct {
+	headers http.Header
+	id      uint32
+}
+
+func newFakeHttpStream() *fakeHttpStream {
+	return &fakeHttpStream{
+		headers: make(http.Header),
+	}
+}
+
+var _ httpstream.Stream = &fakeHttpStream{}
+
+func (s *fakeHttpStream) Read(data []byte) (int, error) {
+	return 0, nil
+}
+
+func (s *fakeHttpStream) Write(data []byte) (int, error) {
+	return 0, nil
+}
+
+func (s *fakeHttpStream) Close() error {
+	return nil
+}
+
+func (s *fakeHttpStream) Reset() error {
+	return nil
+}
+
+func (s *fakeHttpStream) Headers() http.Header {
+	return s.headers
+}
+
+func (s *fakeHttpStream) Identifier() uint32 {
+	return s.id
+}

pkg/kubelet/server/portforward/websocket.go

+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package portforward
+
+import (
+	"encoding/binary"
+	"fmt"
+	"io"
+	"net/http"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/golang/glog"
+
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apiserver/pkg/server/httplog"
+	"k8s.io/apiserver/pkg/util/wsstream"
+	"k8s.io/kubernetes/pkg/api"
+)
+
+const (
+	dataChannel = iota
+	errorChannel
+
+	v4BinaryWebsocketProtocol = "v4." + wsstream.ChannelWebSocketProtocol
+	v4Base64WebsocketProtocol = "v4." + wsstream.Base64ChannelWebSocketProtocol
+)
+
+// options contains details about which streams are required for
+// port forwarding.
+type v4Options struct {
+	ports []uint16
+}
+
+// newOptions creates a new options from the Request.
+func newV4Options(req *http.Request) (*v4Options, error) {
+	portStrings := req.URL.Query()[api.PortHeader]
+	if len(portStrings) == 0 {
+		return nil, fmt.Errorf("%q is required", api.PortHeader)
+	}
+
+	ports := make([]uint16, 0, len(portStrings))
+	for _, portString := range portStrings {
+		if len(portString) == 0 {
+			return nil, fmt.Errorf("%q is cannot be empty", api.PortHeader)
+		}
+		for _, p := range strings.Split(portString, ",") {
+			port, err := strconv.ParseUint(p, 10, 16)
+			if err != nil {
+				return nil, fmt.Errorf("unable to parse %q as a port: %v", portString, err)
+			}
+			if port < 1 {
+				return nil, fmt.Errorf("port %q must be > 0", portString)
+			}
+			ports = append(ports, uint16(port))
+		}
+	}
+
+	return &v4Options{
+		ports: ports,
+	}, nil
+}
+
+func handleWebSocketStreams(req *http.Request, w http.ResponseWriter, portForwarder PortForwarder, podName string, uid types.UID, supportedPortForwardProtocols []string, idleTimeout, streamCreationTimeout time.Duration) error {
+	opts, err := newV4Options(req)
+	if err != nil {
+		w.WriteHeader(http.StatusBadRequest)
+		fmt.Fprint(w, err.Error())
+		return err
+	}
+
+	channels := make([]wsstream.ChannelType, 0, len(opts.ports)*2)
+	for i := 0; i < len(opts.ports); i++ {
+		channels = append(channels, wsstream.ReadWriteChannel, wsstream.WriteChannel)
+	}
+	conn := wsstream.NewConn(map[string]wsstream.ChannelProtocolConfig{
+		"": {
+			Binary:   true,
+			Channels: channels,
+		},
+		v4BinaryWebsocketProtocol: {
+			Binary:   true,
+			Channels: channels,
+		},
+		v4Base64WebsocketProtocol: {
+			Binary:   false,
+			Channels: channels,
+		},
+	})
+	conn.SetIdleTimeout(idleTimeout)
+	_, streams, err := conn.Open(httplog.Unlogged(w), req)
+	if err != nil {
+		err = fmt.Errorf("Unable to upgrade websocket connection: %v", err)
+		return err
+	}
+	defer conn.Close()
+	streamPairs := make([]*websocketStreamPair, len(opts.ports))
+	for i := range streamPairs {
+		streamPair := websocketStreamPair{
+			port:        opts.ports[i],
+			dataStream:  streams[i*2+dataChannel],
+			errorStream: streams[i*2+errorChannel],
+		}
+		streamPairs[i] = &streamPair
+
+		portBytes := make([]byte, 2)
+		binary.LittleEndian.PutUint16(portBytes, streamPair.port)
+		streamPair.dataStream.Write(portBytes)
+		streamPair.errorStream.Write(portBytes)
+	}
+	h := &websocketStreamHandler{
+		conn:        conn,
+		streamPairs: streamPairs,
+		pod:         podName,
+		uid:         uid,
+		forwarder:   portForwarder,
+	}
+	h.run()
+
+	return nil
+}
+
+// websocketStreamPair represents the error and data streams for a port
+// forwarding request.
+type websocketStreamPair struct {
+	port        uint16
+	dataStream  io.ReadWriteCloser
+	errorStream io.WriteCloser
+}
+
+// websocketStreamHandler is capable of processing a single port forward
+// request over a websocket connection
+type websocketStreamHandler struct {
+	conn        *wsstream.Conn
+	ports       []uint16
+	streamPairs []*websocketStreamPair
+	pod         string
+	uid         types.UID
+	forwarder   PortForwarder
+}
+
+// run invokes the websocketStreamHandler's forwarder.PortForward
+// function for the given stream pair.
+func (h *websocketStreamHandler) run() {
+	wg := sync.WaitGroup{}
+	wg.Add(len(h.streamPairs))
+
+	for _, pair := range h.streamPairs {
+		p := pair
+		go func() {
+			defer wg.Done()
+			h.portForward(p)
+		}()
+	}
+
+	wg.Wait()
+}
+
+func (h *websocketStreamHandler) portForward(p *websocketStreamPair) {
+	defer p.dataStream.Close()
+	defer p.errorStream.Close()
+
+	glog.V(5).Infof("(conn=%p) invoking forwarder.PortForward for port %d", h.conn, p.port)
+	err := h.forwarder.PortForward(h.pod, h.uid, p.port, p.dataStream)
+	glog.V(5).Infof("(conn=%p) done invoking forwarder.PortForward for port %d", h.conn, p.port)
+
+	if err != nil {
+		msg := fmt.Errorf("error forwarding port %d to pod %s, uid %v: %v", p.port, h.pod, h.uid, err)
+		runtime.HandleError(msg)
+		fmt.Fprint(p.errorStream, msg.Error())
+	}
+}

pkg/kubelet/server/server.go

@@ -335,9 +335,15 @@ func (s *Server) InstallDebuggingHandlers(criHandler http.Handler) {
 	ws = new(restful.WebService)
 	ws.
 		Path("/portForward")
+	ws.Route(ws.GET("/{podNamespace}/{podID}").
+		To(s.getPortForward).
+		Operation("getPortForward"))
 	ws.Route(ws.POST("/{podNamespace}/{podID}").
 		To(s.getPortForward).
 		Operation("getPortForward"))
+	ws.Route(ws.GET("/{podNamespace}/{podID}/{uid}").
+		To(s.getPortForward).
+		Operation("getPortForward"))
 	ws.Route(ws.POST("/{podNamespace}/{podID}/{uid}").
 		To(s.getPortForward).
 		Operation("getPortForward"))
@@ -720,7 +726,8 @@ func (s *Server) getPortForward(request *restful.Request, response *restful.Resp
 		kubecontainer.GetPodFullName(pod),
 		params.podUID,
 		s.host.StreamingConnectionIdleTimeout(),
-		remotecommand.DefaultStreamCreationTimeout)
+		remotecommand.DefaultStreamCreationTimeout,
+		portforward.SupportedProtocols)
 }
 
 // ServeHTTP responds to HTTP requests on the Kubelet.

pkg/kubelet/server/server_websocket_test.go

+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package server
+
+import (
+	"encoding/binary"
+	"fmt"
+	"io"
+	"strconv"
+	"sync"
+	"testing"
+	"time"
+
+	"golang.org/x/net/websocket"
+
+	"k8s.io/apimachinery/pkg/types"
+)
+
+const (
+	dataChannel = iota
+	errorChannel
+)
+
+func TestServeWSPortForward(t *testing.T) {
+	tests := []struct {
+		port          string
+		uid           bool
+		clientData    string
+		containerData string
+		shouldError   bool
+	}{
+		{port: "", shouldError: true},
+		{port: "abc", shouldError: true},
+		{port: "-1", shouldError: true},
+		{port: "65536", shouldError: true},
+		{port: "0", shouldError: true},
+		{port: "1", shouldError: false},
+		{port: "8000", shouldError: false},
+		{port: "8000", clientData: "client data", containerData: "container data", shouldError: false},
+		{port: "65535", shouldError: false},
+		{port: "65535", uid: true, shouldError: false},
+	}
+
+	podNamespace := "other"
+	podName := "foo"
+	expectedPodName := getPodName(podName, podNamespace)
+	expectedUid := "9b01b80f-8fb4-11e4-95ab-4200af06647"
+
+	for i, test := range tests {
+		fw := newServerTest()
+		defer fw.testHTTPServer.Close()
+
+		fw.fakeKubelet.streamingConnectionIdleTimeoutFunc = func() time.Duration {
+			return 0
+		}
+
+		portForwardFuncDone := make(chan struct{})
+
+		fw.fakeKubelet.portForwardFunc = func(name string, uid types.UID, port uint16, stream io.ReadWriteCloser) error {
+			defer close(portForwardFuncDone)
+
+			if e, a := expectedPodName, name; e != a {
+				t.Fatalf("%d: pod name: expected '%v', got '%v'", i, e, a)
+			}
+
+			if e, a := expectedUid, uid; test.uid && e != string(a) {
+				t.Fatalf("%d: uid: expected '%v', got '%v'", i, e, a)
+			}
+
+			p, err := strconv.ParseUint(test.port, 10, 16)
+			if err != nil {
+				t.Fatalf("%d: error parsing port string '%s': %v", i, test.port, err)
+			}
+			if e, a := uint16(p), port; e != a {
+				t.Fatalf("%d: port: expected '%v', got '%v'", i, e, a)
+			}
+
+			if test.clientData != "" {
+				fromClient := make([]byte, 32)
+				n, err := stream.Read(fromClient)
+				if err != nil {
+					t.Fatalf("%d: error reading client data: %v", i, err)
+				}
+				if e, a := test.clientData, string(fromClient[0:n]); e != a {
+					t.Fatalf("%d: client data: expected to receive '%v', got '%v'", i, e, a)
+				}
+			}
+
+			if test.containerData != "" {
+				_, err := stream.Write([]byte(test.containerData))
+				if err != nil {
+					t.Fatalf("%d: error writing container data: %v", i, err)
+				}
+			}
+
+			return nil
+		}
+
+		var url string
+		if test.uid {
+			url = fmt.Sprintf("ws://%s/portForward/%s/%s/%s?port=%s", fw.testHTTPServer.Listener.Addr().String(), podNamespace, podName, expectedUid, test.port)
+		} else {
+			url = fmt.Sprintf("ws://%s/portForward/%s/%s?port=%s", fw.testHTTPServer.Listener.Addr().String(), podNamespace, podName, test.port)
+		}
+
+		ws, err := websocket.Dial(url, "", "http://127.0.0.1/")
+		if test.shouldError {
+			if err == nil {
+				t.Fatalf("%d: websocket dial expected err", i)
+			}
+			continue
+		} else if err != nil {
+			t.Fatalf("%d: websocket dial unexpected err: %v", i, err)
+		}
+
+		defer ws.Close()
+
+		p, err := strconv.ParseUint(test.port, 10, 16)
+		if err != nil {
+			t.Fatalf("%d: error parsing port string '%s': %v", i, test.port, err)
+		}
+		p16 := uint16(p)
+
+		channel, data, err := wsRead(ws)
+		if err != nil {
+			t.Fatalf("%d: read failed: expected no error: got %v", i, err)
+		}
+		if channel != dataChannel {
+			t.Fatalf("%d: wrong channel: got %q: expected %q", i, channel, dataChannel)
+		}
+		if len(data) != binary.Size(p16) {
+			t.Fatalf("%d: wrong data size: got %q: expected %d", i, data, binary.Size(p16))
+		}
+		if e, a := p16, binary.LittleEndian.Uint16(data); e != a {
+			t.Fatalf("%d: wrong data: got %q: expected %s", i, data, test.port)
+		}
+
+		channel, data, err = wsRead(ws)
+		if err != nil {
+			t.Fatalf("%d: read succeeded: expected no error: got %v", i, err)
+		}
+		if channel != errorChannel {
+			t.Fatalf("%d: wrong channel: got %q: expected %q", i, channel, errorChannel)
+		}
+		if len(data) != binary.Size(p16) {
+			t.Fatalf("%d: wrong data size: got %q: expected %d", i, data, binary.Size(p16))
+		}
+		if e, a := p16, binary.LittleEndian.Uint16(data); e != a {
+			t.Fatalf("%d: wrong data: got %q: expected %s", i, data, test.port)
+		}
+
+		if test.clientData != "" {
+			println("writing the client data")
+			err := wsWrite(ws, dataChannel, []byte(test.clientData))
+			if err != nil {
+				t.Fatalf("%d: unexpected error writing client data: %v", i, err)
+			}
+		}
+
+		if test.containerData != "" {
+			channel, data, err = wsRead(ws)
+			if err != nil {
+				t.Fatalf("%d: unexpected error reading container data: %v", i, err)
+			}
+			if e, a := test.containerData, string(data); e != a {
+				t.Fatalf("%d: expected to receive '%v' from container, got '%v'", i, e, a)
+			}
+		}
+
+		<-portForwardFuncDone
+	}
+}
+
+func TestServeWSMultiplePortForward(t *testing.T) {
+	portsText := []string{"7000,8000", "9000"}
+	ports := []uint16{7000, 8000, 9000}
+	podNamespace := "other"
+	podName := "foo"
+	expectedPodName := getPodName(podName, podNamespace)
+
+	fw := newServerTest()
+	defer fw.testHTTPServer.Close()
+
+	fw.fakeKubelet.streamingConnectionIdleTimeoutFunc = func() time.Duration {
+		return 0
+	}
+
+	portForwardWG := sync.WaitGroup{}
+	portForwardWG.Add(len(ports))
+
+	portsMutex := sync.Mutex{}
+	portsForwarded := map[uint16]struct{}{}
+
+	fw.fakeKubelet.portForwardFunc = func(name string, uid types.UID, port uint16, stream io.ReadWriteCloser) error {
+		defer portForwardWG.Done()
+
+		if e, a := expectedPodName, name; e != a {
+			t.Fatalf("%d: pod name: expected '%v', got '%v'", port, e, a)
+		}
+
+		portsMutex.Lock()
+		portsForwarded[port] = struct{}{}
+		portsMutex.Unlock()
+
+		fromClient := make([]byte, 32)
+		n, err := stream.Read(fromClient)
+		if err != nil {
+			t.Fatalf("%d: error reading client data: %v", port, err)
+		}
+		if e, a := fmt.Sprintf("client data on port %d", port), string(fromClient[0:n]); e != a {
+			t.Fatalf("%d: client data: expected to receive '%v', got '%v'", port, e, a)
+		}
+
+		_, err = stream.Write([]byte(fmt.Sprintf("container data on port %d", port)))
+		if err != nil {
+			t.Fatalf("%d: error writing container data: %v", port, err)
+		}
+
+		return nil
+	}
+
+	url := fmt.Sprintf("ws://%s/portForward/%s/%s?", fw.testHTTPServer.Listener.Addr().String(), podNamespace, podName)
+	for _, port := range portsText {
+		url = url + fmt.Sprintf("port=%s&", port)
+	}
+
+	ws, err := websocket.Dial(url, "", "http://127.0.0.1/")
+	if err != nil {
+		t.Fatalf("websocket dial unexpected err: %v", err)
+	}
+
+	defer ws.Close()
+
+	for i, port := range ports {
+		channel, data, err := wsRead(ws)
+		if err != nil {
+			t.Fatalf("%d: read failed: expected no error: got %v", i, err)
+		}
+		if int(channel) != i*2+dataChannel {
+			t.Fatalf("%d: wrong channel: got %q: expected %q", i, channel, i*2+dataChannel)
+		}
+		if len(data) != binary.Size(port) {
+			t.Fatalf("%d: wrong data size: got %q: expected %d", i, data, binary.Size(port))
+		}
+		if e, a := port, binary.LittleEndian.Uint16(data); e != a {
+			t.Fatalf("%d: wrong data: got %q: expected %d", i, data, port)
+		}
+
+		channel, data, err = wsRead(ws)
+		if err != nil {
+			t.Fatalf("%d: read succeeded: expected no error: got %v", i, err)
+		}
+		if int(channel) != i*2+errorChannel {
+			t.Fatalf("%d: wrong channel: got %q: expected %q", i, channel, i*2+errorChannel)
+		}
+		if len(data) != binary.Size(port) {
+			t.Fatalf("%d: wrong data size: got %q: expected %d", i, data, binary.Size(port))
+		}
+		if e, a := port, binary.LittleEndian.Uint16(data); e != a {
+			t.Fatalf("%d: wrong data: got %q: expected %d", i, data, port)
+		}
+	}
+
+	for i, port := range ports {
+		println("writing the client data", port)
+		err := wsWrite(ws, byte(i*2+dataChannel), []byte(fmt.Sprintf("client data on port %d", port)))
+		if err != nil {
+			t.Fatalf("%d: unexpected error writing client data: %v", i, err)
+		}
+
+		channel, data, err := wsRead(ws)
+		if err != nil {
+			t.Fatalf("%d: unexpected error reading container data: %v", i, err)
+		}
+
+		if int(channel) != i*2+dataChannel {
+			t.Fatalf("%d: wrong channel: got %q: expected %q", port, channel, i*2+dataChannel)
+		}
+		if e, a := fmt.Sprintf("container data on port %d", port), string(data); e != a {
+			t.Fatalf("%d: expected to receive '%v' from container, got '%v'", i, e, a)
+		}
+	}
+
+	portForwardWG.Wait()
+
+	portsMutex.Lock()
+	defer portsMutex.Unlock()
+	if len(ports) != len(portsForwarded) {
+		t.Fatalf("expected to forward %d ports; got %v", len(ports), portsForwarded)
+	}
+}
+func wsWrite(conn *websocket.Conn, channel byte, data []byte) error {
+	frame := make([]byte, len(data)+1)
+	frame[0] = channel
+	copy(frame[1:], data)
+	err := websocket.Message.Send(conn, frame)
+	return err
+}
+
+func wsRead(conn *websocket.Conn) (byte, []byte, error) {
+	for {
+		var data []byte
+		err := websocket.Message.Receive(conn, &data)
+		if err != nil {
+			return 0, nil, err
+		}
+
+		if len(data) == 0 {
+			continue
+		}
+
+		channel := data[0]
+		data = data[1:]
+
+		return channel, data, err
+	}
+}

pkg/kubelet/server/streaming/server.go

@@ -80,7 +80,12 @@ type Config struct {
 	// The streaming protocols the server supports (understands and permits).  See
 	// k8s.io/kubernetes/pkg/kubelet/server/remotecommand/constants.go for available protocols.
 	// Only used for SPDY streaming.
-	SupportedProtocols []string
+	SupportedRemoteCommandProtocols []string
+
+	// The streaming protocols the server supports (understands and permits).  See
+	// k8s.io/kubernetes/pkg/kubelet/server/portforward/constants.go for available protocols.
+	// Only used for SPDY streaming.
+	SupportedPortForwardProtocols []string
 
 	// The config for serving over TLS. If nil, TLS will not be used.
 	TLSConfig *tls.Config
@@ -91,7 +96,8 @@ type Config struct {
 var DefaultConfig = Config{
 	StreamIdleTimeout:               4 * time.Hour,
 	StreamCreationTimeout:           remotecommand.DefaultStreamCreationTimeout,
-	SupportedProtocols:    remotecommand.SupportedStreamingProtocols,
+	SupportedRemoteCommandProtocols: remotecommand.SupportedStreamingProtocols,
+	SupportedPortForwardProtocols:   portforward.SupportedProtocols,
 }
 
 // TODO(timstclair): Add auth(n/z) interface & handling.
@@ -248,7 +254,7 @@ func (s *server) serveExec(req *restful.Request, resp *restful.Response) {
 		streamOpts,
 		s.config.StreamIdleTimeout,
 		s.config.StreamCreationTimeout,
-		s.config.SupportedProtocols)
+		s.config.SupportedRemoteCommandProtocols)
 }
 
 func (s *server) serveAttach(req *restful.Request, resp *restful.Response) {
@@ -280,7 +286,7 @@ func (s *server) serveAttach(req *restful.Request, resp *restful.Response) {
 		streamOpts,
 		s.config.StreamIdleTimeout,
 		s.config.StreamCreationTimeout,
-		s.config.SupportedProtocols)
+		s.config.SupportedRemoteCommandProtocols)
 }
 
 func (s *server) servePortForward(req *restful.Request, resp *restful.Response) {
@@ -303,7 +309,8 @@ func (s *server) servePortForward(req *restful.Request, resp *restful.Response)
 		pf.PodSandboxId,
 		"", // unused: podUID
 		s.config.StreamIdleTimeout,
-		s.config.StreamCreationTimeout)
+		s.config.StreamCreationTimeout,
+		s.config.SupportedPortForwardProtocols)
 }
 
 // criAdapter wraps the Runtime functions to conform to the remotecommand interfaces.

pkg/kubelet/server/streaming/server_test.go

@@ -240,7 +240,7 @@ func TestServePortForward(t *testing.T) {
 
 	exec, err := remotecommand.NewExecutor(&restclient.Config{}, "POST", reqURL)
 	require.NoError(t, err)
-	streamConn, _, err := exec.Dial(kubeletportforward.PortForwardProtocolV1Name)
+	streamConn, _, err := exec.Dial(kubeletportforward.ProtocolV1Name)
 	require.NoError(t, err)
 	defer streamConn.Close()
 

pkg/registry/core/pod/rest/subresources.go

@@ -165,9 +165,10 @@ func (r *PortForwardREST) New() runtime.Object {
 	return &api.Pod{}
 }
 
-// NewConnectOptions returns nil since portforward doesn't take additional parameters
+// NewConnectOptions returns the versioned object that represents the
+// portforward parameters
 func (r *PortForwardREST) NewConnectOptions() (runtime.Object, bool, string) {
-	return nil, false, ""
+	return &api.PodPortForwardOptions{}, false, ""
 }
 
 // ConnectMethods returns the methods supported by portforward
@@ -177,7 +178,11 @@ func (r *PortForwardREST) ConnectMethods() []string {
 
 // Connect returns a handler for the pod portforward proxy
 func (r *PortForwardREST) Connect(ctx genericapirequest.Context, name string, opts runtime.Object, responder rest.Responder) (http.Handler, error) {
-	location, transport, err := pod.PortForwardLocation(r.Store, r.KubeletConn, ctx, name)
+	portForwardOpts, ok := opts.(*api.PodPortForwardOptions)
+	if !ok {
+		return nil, fmt.Errorf("invalid options object: %#v", opts)
+	}
+	location, transport, err := pod.PortForwardLocation(r.Store, r.KubeletConn, ctx, name, portForwardOpts)
 	if err != nil {
 		return nil, err
 	}

pkg/registry/core/pod/strategy.go

@@ -383,6 +383,15 @@ func streamParams(params url.Values, opts runtime.Object) error {
 		if opts.TTY {
 			params.Add(api.ExecTTYParam, "1")
 		}
+	case *api.PodPortForwardOptions:
+		if len(opts.Ports) == 0 {
+			return errors.NewBadRequest("at least one port must be specified")
+		}
+		ports := make([]string, len(opts.Ports))
+		for i, p := range opts.Ports {
+			ports[i] = strconv.FormatInt(int64(p), 10)
+		}
+		params.Add(api.PortHeader, strings.Join(ports, ","))
 	default:
 		return fmt.Errorf("Unknown object for streaming: %v", opts)
 	}
@@ -477,6 +486,7 @@ func PortForwardLocation(
 	connInfo client.ConnectionInfoGetter,
 	ctx genericapirequest.Context,
 	name string,
+	opts *api.PodPortForwardOptions,
 ) (*url.URL, http.RoundTripper, error) {
 	pod, err := getPod(getter, ctx, name)
 	if err != nil {
@@ -492,10 +502,15 @@ func PortForwardLocation(
 	if err != nil {
 		return nil, nil, err
 	}
+	params := url.Values{}
+	if err := streamParams(params, opts); err != nil {
+		return nil, nil, err
+	}
 	loc := &url.URL{
 		Scheme:   nodeInfo.Scheme,
 		Host:     net.JoinHostPort(nodeInfo.Hostname, nodeInfo.Port),
 		Path:     fmt.Sprintf("/portForward/%s/%s", pod.Namespace, pod.Name),
+		RawQuery: params.Encode(),
 	}
 	return loc, nodeInfo.Transport, nil
 }

pkg/registry/core/pod/strategy_test.go

@@ -17,6 +17,7 @@ limitations under the License.
 package pod
 
 import (
+	"net/url"
 	"reflect"
 	"testing"
 
@@ -26,9 +27,11 @@ import (
 	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
 	genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
 	"k8s.io/kubernetes/pkg/api"
 	apitesting "k8s.io/kubernetes/pkg/api/testing"
+	"k8s.io/kubernetes/pkg/kubelet/client"
 )
 
 func TestMatchPod(t *testing.T) {
@@ -333,3 +336,64 @@ func TestSelectableFieldLabelConversions(t *testing.T) {
 		nil,
 	)
 }
+
+type mockConnectionInfoGetter struct {
+	info *client.ConnectionInfo
+}
+
+func (g mockConnectionInfoGetter) GetConnectionInfo(nodeName types.NodeName) (*client.ConnectionInfo, error) {
+	return g.info, nil
+}
+
+func TestPortForwardLocation(t *testing.T) {
+	ctx := genericapirequest.NewDefaultContext()
+	tcs := []struct {
+		in          *api.Pod
+		info        *client.ConnectionInfo
+		opts        *api.PodPortForwardOptions
+		expectedErr error
+		expectedURL *url.URL
+	}{
+		{
+			in: &api.Pod{
+				Spec: api.PodSpec{},
+			},
+			opts:        &api.PodPortForwardOptions{},
+			expectedErr: errors.NewBadRequest("pod test does not have a host assigned"),
+		},
+		{
+			in: &api.Pod{
+				Spec: api.PodSpec{
+					NodeName: "node1",
+				},
+			},
+			opts:        &api.PodPortForwardOptions{},
+			expectedErr: errors.NewBadRequest("at least one port must be specified"),
+		},
+		{
+			in: &api.Pod{
+				ObjectMeta: api.ObjectMeta{
+					Namespace: "ns",
+					Name:      "pod1",
+				},
+				Spec: api.PodSpec{
+					NodeName: "node1",
+				},
+			},
+			info:        &client.ConnectionInfo{},
+			opts:        &api.PodPortForwardOptions{Ports: []int32{80}},
+			expectedURL: &url.URL{Host: ":", Path: "/portForward/ns/pod1", RawQuery: "port=80"},
+		},
+	}
+	for _, tc := range tcs {
+		getter := &mockPodGetter{tc.in}
+		connectionGetter := &mockConnectionInfoGetter{tc.info}
+		loc, _, err := PortForwardLocation(getter, connectionGetter, ctx, "test", tc.opts)
+		if !reflect.DeepEqual(err, tc.expectedErr) {
+			t.Errorf("expected %v, got %v", tc.expectedErr, err)
+		}
+		if !reflect.DeepEqual(loc, tc.expectedURL) {
+			t.Errorf("expected %v, got %v", tc.expectedURL, loc)
+		}
+	}
+}

test/e2e/portforward.go

@@ -17,6 +17,8 @@ limitations under the License.
 package e2e
 
 import (
+	"bytes"
+	"encoding/binary"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -28,6 +30,7 @@ import (
 	"syscall"
 	"time"
 
+	"golang.org/x/net/websocket"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/kubernetes/pkg/api/v1"
@@ -36,6 +39,7 @@ import (
 	testutils "k8s.io/kubernetes/test/utils"
 
 	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
 )
 
 const (
@@ -368,10 +372,107 @@ func doTestMustConnectSendDisconnect(bindAddress string, f *framework.Framework)
 	verifyLogMessage(logOutput, "^Done$")
 }
 
+func doTestOverWebSockets(bindAddress string, f *framework.Framework) {
+	config, err := framework.LoadConfig()
+	Expect(err).NotTo(HaveOccurred(), "unable to get base config")
+
+	By("creating the pod")
+	pod := pfPod("def", "10", "10", "100", fmt.Sprintf("%s", bindAddress))
+	if _, err := f.ClientSet.Core().Pods(f.Namespace.Name).Create(pod); err != nil {
+		framework.Failf("Couldn't create pod: %v", err)
+	}
+	if err := f.WaitForPodReady(pod.Name); err != nil {
+		framework.Failf("Pod did not start running: %v", err)
+	}
+	defer func() {
+		logs, err := framework.GetPodLogs(f.ClientSet, f.Namespace.Name, pod.Name, "portforwardtester")
+		if err != nil {
+			framework.Logf("Error getting pod log: %v", err)
+		} else {
+			framework.Logf("Pod log:\n%s", logs)
+		}
+	}()
+
+	req := f.ClientSet.Core().RESTClient().Get().
+		Namespace(f.Namespace.Name).
+		Resource("pods").
+		Name(pod.Name).
+		Suffix("portforward").
+		Param("ports", "80")
+
+	url := req.URL()
+	ws, err := framework.OpenWebSocketForURL(url, config, []string{"v4.channel.k8s.io"})
+	if err != nil {
+		framework.Failf("Failed to open websocket to %s: %v", url.String(), err)
+	}
+	defer ws.Close()
+
+	Eventually(func() error {
+		channel, msg, err := wsRead(ws)
+		if err != nil {
+			return fmt.Errorf("Failed to read completely from websocket %s: %v", url.String(), err)
+		}
+		if channel != 0 {
+			return fmt.Errorf("Got message from server that didn't start with channel 0 (data): %v", msg)
+		}
+		if p := binary.LittleEndian.Uint16(msg); p != 80 {
+			return fmt.Errorf("Received the wrong port: %d", p)
+		}
+		return nil
+	}, time.Minute, 10*time.Second).Should(BeNil())
+
+	Eventually(func() error {
+		channel, msg, err := wsRead(ws)
+		if err != nil {
+			return fmt.Errorf("Failed to read completely from websocket %s: %v", url.String(), err)
+		}
+		if channel != 1 {
+			return fmt.Errorf("Got message from server that didn't start with channel 1 (error): %v", msg)
+		}
+		if p := binary.LittleEndian.Uint16(msg); p != 80 {
+			return fmt.Errorf("Received the wrong port: %d", p)
+		}
+		return nil
+	}, time.Minute, 10*time.Second).Should(BeNil())
+
+	By("sending the expected data to the local port")
+	err = wsWrite(ws, 0, []byte("def"))
+	if err != nil {
+		framework.Failf("Failed to write to websocket %s: %v", url.String(), err)
+	}
+
+	By("reading data from the local port")
+	buf := bytes.Buffer{}
+	expectedData := bytes.Repeat([]byte("x"), 100)
+	Eventually(func() error {
+		channel, msg, err := wsRead(ws)
+		if err != nil {
+			return fmt.Errorf("Failed to read completely from websocket %s: %v", url.String(), err)
+		}
+		if channel != 0 {
+			return fmt.Errorf("Got message from server that didn't start with channel 0 (data): %v", msg)
+		}
+		buf.Write(msg)
+		if bytes.Equal(expectedData, buf.Bytes()) {
+			return fmt.Errorf("Expected %q from server, got %q", expectedData, buf.Bytes())
+		}
+		return nil
+	}, time.Minute, 10*time.Second).Should(BeNil())
+
+	By("verifying logs")
+	logOutput, err := framework.GetPodLogs(f.ClientSet, f.Namespace.Name, pod.Name, "portforwardtester")
+	if err != nil {
+		framework.Failf("Error retrieving pod logs: %v", err)
+	}
+	verifyLogMessage(logOutput, "^Accepted client connection$")
+	verifyLogMessage(logOutput, "^Received expected client data$")
+}
+
 var _ = framework.KubeDescribe("Port forwarding", func() {
 	f := framework.NewDefaultFramework("port-forwarding")
 
-	framework.KubeDescribe("With a server  listening on 0.0.0.0 that expects a client request", func() {
+	framework.KubeDescribe("With a server listening on 0.0.0.0", func() {
+		framework.KubeDescribe("that expects a client request", func() {
 			It("should support a client that connects, sends no data, and disconnects", func() {
 				doTestMustConnectSendNothing("0.0.0.0", f)
 			})
@@ -380,13 +481,19 @@ var _ = framework.KubeDescribe("Port forwarding", func() {
 			})
 		})
 
-	framework.KubeDescribe("With a server  listening on 0.0.0.0 that expects no client request", func() {
+		framework.KubeDescribe("that expects no client request", func() {
 			It("should support a client that connects, sends data, and disconnects", func() {
 				doTestConnectSendDisconnect("0.0.0.0", f)
 			})
 		})
 
-	framework.KubeDescribe("With a server  listening on localhost that expects a client request", func() {
+		It("should support forwarding over websockets", func() {
+			doTestOverWebSockets("0.0.0.0", f)
+		})
+	})
+
+	framework.KubeDescribe("With a server listening on localhost", func() {
+		framework.KubeDescribe("that expects a client request", func() {
 			It("should support a client that connects, sends no data, and disconnects [Conformance]", func() {
 				doTestMustConnectSendNothing("localhost", f)
 			})
@@ -395,11 +502,16 @@ var _ = framework.KubeDescribe("Port forwarding", func() {
 			})
 		})
 
-	framework.KubeDescribe("With a server  listening on localhost that expects no client request", func() {
+		framework.KubeDescribe("that expects no client request", func() {
 			It("should support a client that connects, sends data, and disconnects [Conformance]", func() {
 				doTestConnectSendDisconnect("localhost", f)
 			})
 		})
+
+		It("should support forwarding over websockets", func() {
+			doTestOverWebSockets("localhost", f)
+		})
+	})
 })
 
 func verifyLogMessage(log, expected string) {
@@ -412,3 +524,30 @@ func verifyLogMessage(log, expected string) {
 	}
 	framework.Failf("Missing %q from log: %s", expected, log)
 }
+
+func wsRead(conn *websocket.Conn) (byte, []byte, error) {
+	for {
+		var data []byte
+		err := websocket.Message.Receive(conn, &data)
+		if err != nil {
+			return 0, nil, err
+		}
+
+		if len(data) == 0 {
+			continue
+		}
+
+		channel := data[0]
+		data = data[1:]
+
+		return channel, data, err
+	}
+}
+
+func wsWrite(conn *websocket.Conn, channel byte, data []byte) error {
+	frame := make([]byte, len(data)+1)
+	frame[0] = channel
+	copy(frame[1:], data)
+	err := websocket.Message.Send(conn, frame)
+	return err
+}