TUN-6530: Implement ICMPv4 proxy

This proxy uses unprivileged datagram-oriented endpoint and is shared by all quic connections

supervisor/supervisor.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"net"
 	"strings"
 	"time"
 
@@ -15,6 +16,7 @@ import (
 	"github.com/cloudflare/cloudflared/edgediscovery"
 	"github.com/cloudflare/cloudflared/edgediscovery/allregions"
 	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/ingress"
 	"github.com/cloudflare/cloudflared/orchestration"
 	"github.com/cloudflare/cloudflared/retry"
 	"github.com/cloudflare/cloudflared/signal"
@@ -44,7 +46,7 @@ type Supervisor struct {
 	config                  *TunnelConfig
 	orchestrator            *orchestration.Orchestrator
 	edgeIPs                 *edgediscovery.Edge
-	edgeTunnelServer        EdgeTunnelServer
+	edgeTunnelServer        *EdgeTunnelServer
 	tunnelErrors            chan tunnelError
 	tunnelsConnecting       map[int]chan struct{}
 	tunnelsProtocolFallback map[int]*protocolFallback
@@ -114,6 +116,15 @@ func NewSupervisor(config *TunnelConfig, orchestrator *orchestration.Orchestrato
 		gracefulShutdownC: gracefulShutdownC,
 		connAwareLogger:   log,
 	}
+	if useDatagramV2(config) {
+		// For non-privileged datagram-oriented ICMP endpoints, network must be "udp4" or "udp6"
+		// TODO: TUN-6654 listen for IPv6 and decide if it should listen on specific IP
+		icmpProxy, err := ingress.NewICMPProxy("udp4", net.IPv4zero, config.Log)
+		if err != nil {
+			return nil, err
+		}
+		edgeTunnelServer.icmpProxy = icmpProxy
+	}
 
 	useReconnectToken := false
 	if config.ClassicTunnel != nil {
@@ -125,7 +136,7 @@ func NewSupervisor(config *TunnelConfig, orchestrator *orchestration.Orchestrato
 		config:                     config,
 		orchestrator:               orchestrator,
 		edgeIPs:                    edgeIPs,
-		edgeTunnelServer:           edgeTunnelServer,
+		edgeTunnelServer:           &edgeTunnelServer,
 		tunnelErrors:               make(chan tunnelError),
 		tunnelsConnecting:          map[int]chan struct{}{},
 		tunnelsProtocolFallback:    map[int]*protocolFallback{},
@@ -142,6 +153,14 @@ func (s *Supervisor) Run(
 	ctx context.Context,
 	connectedSignal *signal.Signal,
 ) error {
+	if s.edgeTunnelServer.icmpProxy != nil {
+		go func() {
+			if err := s.edgeTunnelServer.icmpProxy.ListenResponse(ctx); err != nil {
+				s.log.Logger().Err(err).Msg("icmp proxy terminated")
+			}
+		}()
+	}
+
 	if err := s.initialize(ctx, connectedSignal); err != nil {
 		if err == errEarlyShutdown {
 			return nil
@@ -413,3 +432,15 @@ func (s *Supervisor) authenticate(ctx context.Context, numPreviousAttempts int)
 	registrationOptions.NumPreviousAttempts = uint8(numPreviousAttempts)
 	return rpcClient.Authenticate(ctx, s.config.ClassicTunnel, registrationOptions)
 }
+
+func useDatagramV2(config *TunnelConfig) bool {
+	if config.NamedTunnel == nil {
+		return false
+	}
+	for _, feature := range config.NamedTunnel.Client.Features {
+		if feature == FeatureDatagramV2 {
+			return true
+		}
+	}
+	return false
+}

supervisor/tunnel.go

@@ -20,6 +20,7 @@ import (
 	"github.com/cloudflare/cloudflared/edgediscovery"
 	"github.com/cloudflare/cloudflared/edgediscovery/allregions"
 	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/ingress"
 	"github.com/cloudflare/cloudflared/orchestration"
 	quicpogs "github.com/cloudflare/cloudflared/quic"
 	"github.com/cloudflare/cloudflared/retry"
@@ -193,11 +194,12 @@ type EdgeTunnelServer struct {
 	reconnectCh       chan ReconnectSignal
 	gracefulShutdownC <-chan struct{}
 	tracker           *tunnelstate.ConnTracker
+	icmpProxy         ingress.ICMPProxy
 
 	connAwareLogger *ConnAwareLogger
 }
 
-func (e EdgeTunnelServer) Serve(ctx context.Context, connIndex uint8, protocolFallback *protocolFallback, connectedSignal *signal.Signal) error {
+func (e *EdgeTunnelServer) Serve(ctx context.Context, connIndex uint8, protocolFallback *protocolFallback, connectedSignal *signal.Signal) error {
 	haConnections.Inc()
 	defer haConnections.Dec()
 
@@ -229,20 +231,14 @@ func (e EdgeTunnelServer) Serve(ctx context.Context, connIndex uint8, protocolFa
 	// to another protocol when a particular metal doesn't support new protocol
 	// Each connection can also have it's own IP version because individual connections might fallback
 	// to another IP version.
-	err, recoverable := ServeTunnel(
+	err, recoverable := e.serveTunnel(
 		ctx,
 		connLog,
-		e.credentialManager,
-		e.config,
-		e.orchestrator,
 		addr,
 		connIndex,
 		connectedFuse,
 		protocolFallback,
-		e.cloudflaredUUID,
-		e.reconnectCh,
 		protocolFallback.protocol,
-		e.gracefulShutdownC,
 	)
 
 	// If the connection is recoverable, we want to maintain the same IP
@@ -361,20 +357,14 @@ func selectNextProtocol(
 
 // ServeTunnel runs a single tunnel connection, returns nil on graceful shutdown,
 // on error returns a flag indicating if error can be retried
-func ServeTunnel(
+func (e *EdgeTunnelServer) serveTunnel(
 	ctx context.Context,
 	connLog *ConnAwareLogger,
-	credentialManager *reconnectCredentialManager,
-	config *TunnelConfig,
-	orchestrator *orchestration.Orchestrator,
 	addr *allregions.EdgeAddr,
 	connIndex uint8,
 	fuse *h2mux.BooleanFuse,
 	backoff *protocolFallback,
-	cloudflaredUUID uuid.UUID,
-	reconnectCh chan ReconnectSignal,
 	protocol connection.Protocol,
-	gracefulShutdownC <-chan struct{},
 ) (err error, recoverable bool) {
 	// Treat panics as recoverable errors
 	defer func() {
@@ -389,21 +379,15 @@ func ServeTunnel(
 		}
 	}()
 
-	defer config.Observer.SendDisconnect(connIndex)
-	err, recoverable = serveTunnel(
+	defer e.config.Observer.SendDisconnect(connIndex)
+	err, recoverable = e.serveConnection(
 		ctx,
 		connLog,
-		credentialManager,
-		config,
-		orchestrator,
 		addr,
 		connIndex,
 		fuse,
 		backoff,
-		cloudflaredUUID,
-		reconnectCh,
 		protocol,
-		gracefulShutdownC,
 	)
 
 	if err != nil {
@@ -416,7 +400,7 @@ func ServeTunnel(
 			connLog.ConnAwareLogger().Err(err).Msg("Register tunnel error from server side")
 			// Don't send registration error return from server to Sentry. They are
 			// logged on server side
-			if incidents := config.IncidentLookup.ActiveIncidents(); len(incidents) > 0 {
+			if incidents := e.config.IncidentLookup.ActiveIncidents(); len(incidents) > 0 {
 				connLog.ConnAwareLogger().Msg(activeIncidentsMsg(incidents))
 			}
 			return err.Cause, !err.Permanent
@@ -442,93 +426,73 @@ func ServeTunnel(
 	return nil, false
 }
 
-func serveTunnel(
+func (e *EdgeTunnelServer) serveConnection(
 	ctx context.Context,
 	connLog *ConnAwareLogger,
-	credentialManager *reconnectCredentialManager,
-	config *TunnelConfig,
-	orchestrator *orchestration.Orchestrator,
 	addr *allregions.EdgeAddr,
 	connIndex uint8,
 	fuse *h2mux.BooleanFuse,
 	backoff *protocolFallback,
-	cloudflaredUUID uuid.UUID,
-	reconnectCh chan ReconnectSignal,
 	protocol connection.Protocol,
-	gracefulShutdownC <-chan struct{},
 ) (err error, recoverable bool) {
 	connectedFuse := &connectedFuse{
 		fuse:    fuse,
 		backoff: backoff,
 	}
 	controlStream := connection.NewControlStream(
-		config.Observer,
+		e.config.Observer,
 		connectedFuse,
-		config.NamedTunnel,
+		e.config.NamedTunnel,
 		connIndex,
 		addr.UDP.IP,
 		nil,
-		gracefulShutdownC,
-		config.GracePeriod,
+		e.gracefulShutdownC,
+		e.config.GracePeriod,
 		protocol,
 	)
 
 	switch protocol {
 	case connection.QUIC, connection.QUICWarp:
-		connOptions := config.connectionOptions(addr.UDP.String(), uint8(backoff.Retries()))
-		return ServeQUIC(ctx,
+		connOptions := e.config.connectionOptions(addr.UDP.String(), uint8(backoff.Retries()))
+		return e.serveQUIC(ctx,
 			addr.UDP,
-			config,
-			orchestrator,
 			connLog,
 			connOptions,
 			controlStream,
-			connIndex,
-			reconnectCh,
-			gracefulShutdownC)
+			connIndex)
 
 	case connection.HTTP2, connection.HTTP2Warp:
-		edgeConn, err := edgediscovery.DialEdge(ctx, dialTimeout, config.EdgeTLSConfigs[protocol], addr.TCP)
+		edgeConn, err := edgediscovery.DialEdge(ctx, dialTimeout, e.config.EdgeTLSConfigs[protocol], addr.TCP)
 		if err != nil {
 			connLog.ConnAwareLogger().Err(err).Msg("Unable to establish connection with Cloudflare edge")
 			return err, true
 		}
 
-		connOptions := config.connectionOptions(edgeConn.LocalAddr().String(), uint8(backoff.Retries()))
-		if err := ServeHTTP2(
+		connOptions := e.config.connectionOptions(edgeConn.LocalAddr().String(), uint8(backoff.Retries()))
+		if err := e.serveHTTP2(
 			ctx,
 			connLog,
-			config,
-			orchestrator,
 			edgeConn,
 			connOptions,
 			controlStream,
 			connIndex,
-			gracefulShutdownC,
-			reconnectCh,
 		); err != nil {
 			return err, false
 		}
 
 	default:
-		edgeConn, err := edgediscovery.DialEdge(ctx, dialTimeout, config.EdgeTLSConfigs[protocol], addr.TCP)
+		edgeConn, err := edgediscovery.DialEdge(ctx, dialTimeout, e.config.EdgeTLSConfigs[protocol], addr.TCP)
 		if err != nil {
 			connLog.ConnAwareLogger().Err(err).Msg("Unable to establish connection with Cloudflare edge")
 			return err, true
 		}
 
-		if err := ServeH2mux(
+		if err := e.serveH2mux(
 			ctx,
 			connLog,
-			credentialManager,
-			config,
-			orchestrator,
 			edgeConn,
 			connIndex,
 			connectedFuse,
-			cloudflaredUUID,
-			reconnectCh,
-			gracefulShutdownC,
 		); err != nil {
 			return err, false
 		}
@@ -544,30 +508,24 @@ func (r unrecoverableError) Error() string {
 	return r.err.Error()
 }
 
-func ServeH2mux(
+func (e *EdgeTunnelServer) serveH2mux(
 	ctx context.Context,
 	connLog *ConnAwareLogger,
-	credentialManager *reconnectCredentialManager,
-	config *TunnelConfig,
-	orchestrator *orchestration.Orchestrator,
 	edgeConn net.Conn,
 	connIndex uint8,
 	connectedFuse *connectedFuse,
-	cloudflaredUUID uuid.UUID,
-	reconnectCh chan ReconnectSignal,
-	gracefulShutdownC <-chan struct{},
 ) error {
 	connLog.Logger().Debug().Msgf("Connecting via h2mux")
 	// Returns error from parsing the origin URL or handshake errors
 	handler, err, recoverable := connection.NewH2muxConnection(
-		orchestrator,
-		config.GracePeriod,
-		config.MuxerConfig,
+		e.orchestrator,
+		e.config.GracePeriod,
+		e.config.MuxerConfig,
 		edgeConn,
 		connIndex,
-		config.Observer,
-		gracefulShutdownC,
-		config.Log,
+		e.config.Observer,
+		e.gracefulShutdownC,
+		e.config.Log,
 	)
 	if err != nil {
 		if !recoverable {
@@ -579,42 +537,38 @@ func ServeH2mux(
 	errGroup, serveCtx := errgroup.WithContext(ctx)
 
 	errGroup.Go(func() error {
-		if config.NamedTunnel != nil {
-			connOptions := config.connectionOptions(edgeConn.LocalAddr().String(), uint8(connectedFuse.backoff.Retries()))
-			return handler.ServeNamedTunnel(serveCtx, config.NamedTunnel, connOptions, connectedFuse)
+		if e.config.NamedTunnel != nil {
+			connOptions := e.config.connectionOptions(edgeConn.LocalAddr().String(), uint8(connectedFuse.backoff.Retries()))
+			return handler.ServeNamedTunnel(serveCtx, e.config.NamedTunnel, connOptions, connectedFuse)
 		}
-		registrationOptions := config.registrationOptions(connIndex, edgeConn.LocalAddr().String(), cloudflaredUUID)
-		return handler.ServeClassicTunnel(serveCtx, config.ClassicTunnel, credentialManager, registrationOptions, connectedFuse)
+		registrationOptions := e.config.registrationOptions(connIndex, edgeConn.LocalAddr().String(), e.cloudflaredUUID)
+		return handler.ServeClassicTunnel(serveCtx, e.config.ClassicTunnel, e.credentialManager, registrationOptions, connectedFuse)
 	})
 
 	errGroup.Go(func() error {
-		return listenReconnect(serveCtx, reconnectCh, gracefulShutdownC)
+		return listenReconnect(serveCtx, e.reconnectCh, e.gracefulShutdownC)
 	})
 
 	return errGroup.Wait()
 }
 
-func ServeHTTP2(
+func (e *EdgeTunnelServer) serveHTTP2(
 	ctx context.Context,
 	connLog *ConnAwareLogger,
-	config *TunnelConfig,
-	orchestrator *orchestration.Orchestrator,
 	tlsServerConn net.Conn,
 	connOptions *tunnelpogs.ConnectionOptions,
 	controlStreamHandler connection.ControlStreamHandler,
 	connIndex uint8,
-	gracefulShutdownC <-chan struct{},
-	reconnectCh chan ReconnectSignal,
 ) error {
 	connLog.Logger().Debug().Msgf("Connecting via http2")
 	h2conn := connection.NewHTTP2Connection(
 		tlsServerConn,
-		orchestrator,
+		e.orchestrator,
 		connOptions,
-		config.Observer,
+		e.config.Observer,
 		connIndex,
 		controlStreamHandler,
-		config.Log,
+		e.config.Log,
 	)
 
 	errGroup, serveCtx := errgroup.WithContext(ctx)
@@ -623,7 +577,7 @@ func ServeHTTP2(
 	})
 
 	errGroup.Go(func() error {
-		err := listenReconnect(serveCtx, reconnectCh, gracefulShutdownC)
+		err := listenReconnect(serveCtx, e.reconnectCh, e.gracefulShutdownC)
 		if err != nil {
 			// forcefully break the connection (this is only used for testing)
 			connLog.Logger().Debug().Msg("Forcefully breaking http2 connection")
@@ -635,19 +589,15 @@ func ServeHTTP2(
 	return errGroup.Wait()
 }
 
-func ServeQUIC(
+func (e *EdgeTunnelServer) serveQUIC(
 	ctx context.Context,
 	edgeAddr *net.UDPAddr,
-	config *TunnelConfig,
-	orchestrator *orchestration.Orchestrator,
 	connLogger *ConnAwareLogger,
 	connOptions *tunnelpogs.ConnectionOptions,
 	controlStreamHandler connection.ControlStreamHandler,
 	connIndex uint8,
-	reconnectCh chan ReconnectSignal,
-	gracefulShutdownC <-chan struct{},
 ) (err error, recoverable bool) {
-	tlsConfig := config.EdgeTLSConfigs[connection.QUIC]
+	tlsConfig := e.config.EdgeTLSConfigs[connection.QUIC]
 	quicConfig := &quic.Config{
 		HandshakeIdleTimeout:  quicpogs.HandshakeIdleTimeout,
 		MaxIdleTimeout:        quicpogs.MaxIdleTimeout,
@@ -663,10 +613,11 @@ func ServeQUIC(
 		quicConfig,
 		edgeAddr,
 		tlsConfig,
-		orchestrator,
+		e.orchestrator,
 		connOptions,
 		controlStreamHandler,
-		connLogger.Logger())
+		connLogger.Logger(),
+		e.icmpProxy)
 	if err != nil {
 		connLogger.ConnAwareLogger().Err(err).Msgf("Failed to create new quic connection")
 		return err, true
@@ -682,7 +633,7 @@ func ServeQUIC(
 	})
 
 	errGroup.Go(func() error {
-		err := listenReconnect(serveCtx, reconnectCh, gracefulShutdownC)
+		err := listenReconnect(serveCtx, e.reconnectCh, e.gracefulShutdownC)
 		if err != nil {
 			// forcefully break the connection (this is only used for testing)
 			connLogger.Logger().Debug().Msg("Forcefully breaking quic connection")