TUN-4168: Transparently proxy websocket connections using stdlib HTTP client instead of gorilla/websocket; move websocket client code into carrier package since it's only used by access subcommands now (#345).

carrier/websocket.go

@@ -4,6 +4,7 @@ import (
 	"io"
 	"net/http"
 	"net/http/httputil"
+	"net/url"
 
 	"github.com/gorilla/websocket"
 	"github.com/rs/zerolog"
@@ -60,7 +61,7 @@ func createWebsocketStream(options *StartOptions, log *zerolog.Logger) (*cfwebso
 		TLSClientConfig: options.TLSClientConfig,
 		Proxy:           http.ProxyFromEnvironment,
 	}
-	wsConn, resp, err := cfwebsocket.ClientConnect(req, dialer)
+	wsConn, resp, err := clientConnect(req, dialer)
 	defer closeRespBody(resp)
 
 	if err != nil && IsAccessResponse(resp) {
@@ -87,6 +88,63 @@ func createWebsocketStream(options *StartOptions, log *zerolog.Logger) (*cfwebso
 	return &cfwebsocket.GorillaConn{Conn: wsConn}, nil
 }
 
+var stripWebsocketHeaders = []string{
+	"Upgrade",
+	"Connection",
+	"Sec-Websocket-Key",
+	"Sec-Websocket-Version",
+	"Sec-Websocket-Extensions",
+}
+
+// the gorilla websocket library sets its own Upgrade, Connection, Sec-WebSocket-Key,
+// Sec-WebSocket-Version and Sec-Websocket-Extensions headers.
+// https://github.com/gorilla/websocket/blob/master/client.go#L189-L194.
+func websocketHeaders(req *http.Request) http.Header {
+	wsHeaders := make(http.Header)
+	for key, val := range req.Header {
+		wsHeaders[key] = val
+	}
+	// Assume the header keys are in canonical format.
+	for _, header := range stripWebsocketHeaders {
+		wsHeaders.Del(header)
+	}
+	wsHeaders.Set("Host", req.Host) // See TUN-1097
+	return wsHeaders
+}
+
+// clientConnect creates a WebSocket client connection for provided request. Caller is responsible for closing
+// the connection. The response body may not contain the entire response and does
+// not need to be closed by the application.
+func clientConnect(req *http.Request, dialler *websocket.Dialer) (*websocket.Conn, *http.Response, error) {
+	req.URL.Scheme = changeRequestScheme(req.URL)
+	wsHeaders := websocketHeaders(req)
+	if dialler == nil {
+		dialler = &websocket.Dialer{
+			Proxy: http.ProxyFromEnvironment,
+		}
+	}
+	conn, response, err := dialler.Dial(req.URL.String(), wsHeaders)
+	if err != nil {
+		return nil, response, err
+	}
+	return conn, response, nil
+}
+
+// changeRequestScheme is needed as the gorilla websocket library requires the ws scheme.
+// (even though it changes it back to http/https, but ¯\_(ツ)_/¯.)
+func changeRequestScheme(reqURL *url.URL) string {
+	switch reqURL.Scheme {
+	case "https":
+		return "wss"
+	case "http":
+		return "ws"
+	case "":
+		return "ws"
+	default:
+		return reqURL.Scheme
+	}
+}
+
 // createAccessAuthenticatedStream will try load a token from storage and make
 // a connection with the token set on the request. If it still get redirect,
 // this probably means the token in storage is invalid (expired/revoked). If that
@@ -126,7 +184,7 @@ func createAccessWebSocketStream(options *StartOptions, log *zerolog.Logger) (*w
 	dump, err := httputil.DumpRequest(req, false)
 	log.Debug().Msgf("Access Websocket request: %s", string(dump))
 
-	conn, resp, err := cfwebsocket.ClientConnect(req, nil)
+	conn, resp, err := clientConnect(req, nil)
 
 	if resp != nil {
 		r, err := httputil.DumpResponse(resp, true)

carrier/websocket_test.go

@@ -0,0 +1,123 @@
+package carrier
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"math/rand"
+	"testing"
+	"time"
+
+	gws "github.com/gorilla/websocket"
+	"github.com/rs/zerolog"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/net/websocket"
+
+	"github.com/cloudflare/cloudflared/hello"
+	"github.com/cloudflare/cloudflared/tlsconfig"
+	cfwebsocket "github.com/cloudflare/cloudflared/websocket"
+)
+
+func websocketClientTLSConfig(t *testing.T) *tls.Config {
+	certPool := x509.NewCertPool()
+	helloCert, err := tlsconfig.GetHelloCertificateX509()
+	assert.NoError(t, err)
+	certPool.AddCert(helloCert)
+	assert.NotNil(t, certPool)
+	return &tls.Config{RootCAs: certPool}
+}
+
+func TestWebsocketHeaders(t *testing.T) {
+	req := testRequest(t, "http://example.com", nil)
+	wsHeaders := websocketHeaders(req)
+	for _, header := range stripWebsocketHeaders {
+		assert.Empty(t, wsHeaders[header])
+	}
+	assert.Equal(t, "curl/7.59.0", wsHeaders.Get("User-Agent"))
+}
+
+func TestServe(t *testing.T) {
+	log := zerolog.Nop()
+	shutdownC := make(chan struct{})
+	errC := make(chan error)
+	listener, err := hello.CreateTLSListener("localhost:1111")
+	assert.NoError(t, err)
+	defer listener.Close()
+
+	go func() {
+		errC <- hello.StartHelloWorldServer(&log, listener, shutdownC)
+	}()
+
+	req := testRequest(t, "https://localhost:1111/ws", nil)
+
+	tlsConfig := websocketClientTLSConfig(t)
+	assert.NotNil(t, tlsConfig)
+	d := gws.Dialer{TLSClientConfig: tlsConfig}
+	conn, resp, err := clientConnect(req, &d)
+	assert.NoError(t, err)
+	assert.Equal(t, "websocket", resp.Header.Get("Upgrade"))
+
+	for i := 0; i < 1000; i++ {
+		messageSize := rand.Int()%2048 + 1
+		clientMessage := make([]byte, messageSize)
+		// rand.Read always returns len(clientMessage) and a nil error
+		rand.Read(clientMessage)
+		err = conn.WriteMessage(websocket.BinaryFrame, clientMessage)
+		assert.NoError(t, err)
+
+		messageType, message, err := conn.ReadMessage()
+		assert.NoError(t, err)
+		assert.Equal(t, websocket.BinaryFrame, messageType)
+		assert.Equal(t, clientMessage, message)
+	}
+
+	_ = conn.Close()
+	close(shutdownC)
+	<-errC
+}
+
+func TestWebsocketWrapper(t *testing.T) {
+	listener, err := hello.CreateTLSListener("localhost:0")
+	require.NoError(t, err)
+
+	serverErrorChan := make(chan error)
+	helloSvrCtx, cancelHelloSvr := context.WithCancel(context.Background())
+	defer func() { <-serverErrorChan }()
+	defer cancelHelloSvr()
+	go func() {
+		log := zerolog.Nop()
+		serverErrorChan <- hello.StartHelloWorldServer(&log, listener, helloSvrCtx.Done())
+	}()
+
+	tlsConfig := websocketClientTLSConfig(t)
+	d := gws.Dialer{TLSClientConfig: tlsConfig, HandshakeTimeout: time.Minute}
+	testAddr := fmt.Sprintf("https://%s/ws", listener.Addr().String())
+	req := testRequest(t, testAddr, nil)
+	conn, resp, err := clientConnect(req, &d)
+	require.NoError(t, err)
+	assert.Equal(t, "websocket", resp.Header.Get("Upgrade"))
+
+	// Websocket now connected to test server so lets check our wrapper
+	wrapper := cfwebsocket.GorillaConn{Conn: conn}
+	buf := make([]byte, 100)
+	wrapper.Write([]byte("abc"))
+	n, err := wrapper.Read(buf)
+	require.NoError(t, err)
+	require.Equal(t, n, 3)
+	require.Equal(t, "abc", string(buf[:n]))
+
+	// Test partial read, read 1 of 3 bytes in one read and the other 2 in another read
+	wrapper.Write([]byte("abc"))
+	buf = buf[:1]
+	n, err = wrapper.Read(buf)
+	require.NoError(t, err)
+	require.Equal(t, n, 1)
+	require.Equal(t, "a", string(buf[:n]))
+	buf = buf[:cap(buf)]
+	n, err = wrapper.Read(buf)
+	require.NoError(t, err)
+	require.Equal(t, n, 2)
+	require.Equal(t, "bc", string(buf[:n]))
+}