TUN-6530: Implement ICMPv4 proxy

This proxy uses unprivileged datagram-oriented endpoint and is shared by all quic connections

supervisor/tunnel.go

@@ -20,6 +20,7 @@ import (
 	"github.com/cloudflare/cloudflared/edgediscovery"
 	"github.com/cloudflare/cloudflared/edgediscovery/allregions"
 	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/ingress"
 	"github.com/cloudflare/cloudflared/orchestration"
 	quicpogs "github.com/cloudflare/cloudflared/quic"
 	"github.com/cloudflare/cloudflared/retry"
@@ -193,11 +194,12 @@ type EdgeTunnelServer struct {
 	reconnectCh       chan ReconnectSignal
 	gracefulShutdownC <-chan struct{}
 	tracker           *tunnelstate.ConnTracker
+	icmpProxy         ingress.ICMPProxy
 
 	connAwareLogger *ConnAwareLogger
 }
 
-func (e EdgeTunnelServer) Serve(ctx context.Context, connIndex uint8, protocolFallback *protocolFallback, connectedSignal *signal.Signal) error {
+func (e *EdgeTunnelServer) Serve(ctx context.Context, connIndex uint8, protocolFallback *protocolFallback, connectedSignal *signal.Signal) error {
 	haConnections.Inc()
 	defer haConnections.Dec()
 
@@ -229,20 +231,14 @@ func (e EdgeTunnelServer) Serve(ctx context.Context, connIndex uint8, protocolFa
 	// to another protocol when a particular metal doesn't support new protocol
 	// Each connection can also have it's own IP version because individual connections might fallback
 	// to another IP version.
-	err, recoverable := ServeTunnel(
+	err, recoverable := e.serveTunnel(
 		ctx,
 		connLog,
-		e.credentialManager,
-		e.config,
-		e.orchestrator,
 		addr,
 		connIndex,
 		connectedFuse,
 		protocolFallback,
-		e.cloudflaredUUID,
-		e.reconnectCh,
 		protocolFallback.protocol,
-		e.gracefulShutdownC,
 	)
 
 	// If the connection is recoverable, we want to maintain the same IP
@@ -361,20 +357,14 @@ func selectNextProtocol(
 
 // ServeTunnel runs a single tunnel connection, returns nil on graceful shutdown,
 // on error returns a flag indicating if error can be retried
-func ServeTunnel(
+func (e *EdgeTunnelServer) serveTunnel(
 	ctx context.Context,
 	connLog *ConnAwareLogger,
-	credentialManager *reconnectCredentialManager,
-	config *TunnelConfig,
-	orchestrator *orchestration.Orchestrator,
 	addr *allregions.EdgeAddr,
 	connIndex uint8,
 	fuse *h2mux.BooleanFuse,
 	backoff *protocolFallback,
-	cloudflaredUUID uuid.UUID,
-	reconnectCh chan ReconnectSignal,
 	protocol connection.Protocol,
-	gracefulShutdownC <-chan struct{},
 ) (err error, recoverable bool) {
 	// Treat panics as recoverable errors
 	defer func() {
@@ -389,21 +379,15 @@ func ServeTunnel(
 		}
 	}()
 
-	defer config.Observer.SendDisconnect(connIndex)
-	err, recoverable = serveTunnel(
+	defer e.config.Observer.SendDisconnect(connIndex)
+	err, recoverable = e.serveConnection(
 		ctx,
 		connLog,
-		credentialManager,
-		config,
-		orchestrator,
 		addr,
 		connIndex,
 		fuse,
 		backoff,
-		cloudflaredUUID,
-		reconnectCh,
 		protocol,
-		gracefulShutdownC,
 	)
 
 	if err != nil {
@@ -416,7 +400,7 @@ func ServeTunnel(
 			connLog.ConnAwareLogger().Err(err).Msg("Register tunnel error from server side")
 			// Don't send registration error return from server to Sentry. They are
 			// logged on server side
-			if incidents := config.IncidentLookup.ActiveIncidents(); len(incidents) > 0 {
+			if incidents := e.config.IncidentLookup.ActiveIncidents(); len(incidents) > 0 {
 				connLog.ConnAwareLogger().Msg(activeIncidentsMsg(incidents))
 			}
 			return err.Cause, !err.Permanent
@@ -442,93 +426,73 @@ func ServeTunnel(
 	return nil, false
 }
 
-func serveTunnel(
+func (e *EdgeTunnelServer) serveConnection(
 	ctx context.Context,
 	connLog *ConnAwareLogger,
-	credentialManager *reconnectCredentialManager,
-	config *TunnelConfig,
-	orchestrator *orchestration.Orchestrator,
 	addr *allregions.EdgeAddr,
 	connIndex uint8,
 	fuse *h2mux.BooleanFuse,
 	backoff *protocolFallback,
-	cloudflaredUUID uuid.UUID,
-	reconnectCh chan ReconnectSignal,
 	protocol connection.Protocol,
-	gracefulShutdownC <-chan struct{},
 ) (err error, recoverable bool) {
 	connectedFuse := &connectedFuse{
 		fuse:    fuse,
 		backoff: backoff,
 	}
 	controlStream := connection.NewControlStream(
-		config.Observer,
+		e.config.Observer,
 		connectedFuse,
-		config.NamedTunnel,
+		e.config.NamedTunnel,
 		connIndex,
 		addr.UDP.IP,
 		nil,
-		gracefulShutdownC,
-		config.GracePeriod,
+		e.gracefulShutdownC,
+		e.config.GracePeriod,
 		protocol,
 	)
 
 	switch protocol {
 	case connection.QUIC, connection.QUICWarp:
-		connOptions := config.connectionOptions(addr.UDP.String(), uint8(backoff.Retries()))
-		return ServeQUIC(ctx,
+		connOptions := e.config.connectionOptions(addr.UDP.String(), uint8(backoff.Retries()))
+		return e.serveQUIC(ctx,
 			addr.UDP,
-			config,
-			orchestrator,
 			connLog,
 			connOptions,
 			controlStream,
-			connIndex,
-			reconnectCh,
-			gracefulShutdownC)
+			connIndex)
 
 	case connection.HTTP2, connection.HTTP2Warp:
-		edgeConn, err := edgediscovery.DialEdge(ctx, dialTimeout, config.EdgeTLSConfigs[protocol], addr.TCP)
+		edgeConn, err := edgediscovery.DialEdge(ctx, dialTimeout, e.config.EdgeTLSConfigs[protocol], addr.TCP)
 		if err != nil {
 			connLog.ConnAwareLogger().Err(err).Msg("Unable to establish connection with Cloudflare edge")
 			return err, true
 		}
 
-		connOptions := config.connectionOptions(edgeConn.LocalAddr().String(), uint8(backoff.Retries()))
-		if err := ServeHTTP2(
+		connOptions := e.config.connectionOptions(edgeConn.LocalAddr().String(), uint8(backoff.Retries()))
+		if err := e.serveHTTP2(
 			ctx,
 			connLog,
-			config,
-			orchestrator,
 			edgeConn,
 			connOptions,
 			controlStream,
 			connIndex,
-			gracefulShutdownC,
-			reconnectCh,
 		); err != nil {
 			return err, false
 		}
 
 	default:
-		edgeConn, err := edgediscovery.DialEdge(ctx, dialTimeout, config.EdgeTLSConfigs[protocol], addr.TCP)
+		edgeConn, err := edgediscovery.DialEdge(ctx, dialTimeout, e.config.EdgeTLSConfigs[protocol], addr.TCP)
 		if err != nil {
 			connLog.ConnAwareLogger().Err(err).Msg("Unable to establish connection with Cloudflare edge")
 			return err, true
 		}
 
-		if err := ServeH2mux(
+		if err := e.serveH2mux(
 			ctx,
 			connLog,
-			credentialManager,
-			config,
-			orchestrator,
 			edgeConn,
 			connIndex,
 			connectedFuse,
-			cloudflaredUUID,
-			reconnectCh,
-			gracefulShutdownC,
 		); err != nil {
 			return err, false
 		}
@@ -544,30 +508,24 @@ func (r unrecoverableError) Error() string {
 	return r.err.Error()
 }
 
-func ServeH2mux(
+func (e *EdgeTunnelServer) serveH2mux(
 	ctx context.Context,
 	connLog *ConnAwareLogger,
-	credentialManager *reconnectCredentialManager,
-	config *TunnelConfig,
-	orchestrator *orchestration.Orchestrator,
 	edgeConn net.Conn,
 	connIndex uint8,
 	connectedFuse *connectedFuse,
-	cloudflaredUUID uuid.UUID,
-	reconnectCh chan ReconnectSignal,
-	gracefulShutdownC <-chan struct{},
 ) error {
 	connLog.Logger().Debug().Msgf("Connecting via h2mux")
 	// Returns error from parsing the origin URL or handshake errors
 	handler, err, recoverable := connection.NewH2muxConnection(
-		orchestrator,
-		config.GracePeriod,
-		config.MuxerConfig,
+		e.orchestrator,
+		e.config.GracePeriod,
+		e.config.MuxerConfig,
 		edgeConn,
 		connIndex,
-		config.Observer,
-		gracefulShutdownC,
-		config.Log,
+		e.config.Observer,
+		e.gracefulShutdownC,
+		e.config.Log,
 	)
 	if err != nil {
 		if !recoverable {
@@ -579,42 +537,38 @@ func ServeH2mux(
 	errGroup, serveCtx := errgroup.WithContext(ctx)
 
 	errGroup.Go(func() error {
-		if config.NamedTunnel != nil {
-			connOptions := config.connectionOptions(edgeConn.LocalAddr().String(), uint8(connectedFuse.backoff.Retries()))
-			return handler.ServeNamedTunnel(serveCtx, config.NamedTunnel, connOptions, connectedFuse)
+		if e.config.NamedTunnel != nil {
+			connOptions := e.config.connectionOptions(edgeConn.LocalAddr().String(), uint8(connectedFuse.backoff.Retries()))
+			return handler.ServeNamedTunnel(serveCtx, e.config.NamedTunnel, connOptions, connectedFuse)
 		}
-		registrationOptions := config.registrationOptions(connIndex, edgeConn.LocalAddr().String(), cloudflaredUUID)
-		return handler.ServeClassicTunnel(serveCtx, config.ClassicTunnel, credentialManager, registrationOptions, connectedFuse)
+		registrationOptions := e.config.registrationOptions(connIndex, edgeConn.LocalAddr().String(), e.cloudflaredUUID)
+		return handler.ServeClassicTunnel(serveCtx, e.config.ClassicTunnel, e.credentialManager, registrationOptions, connectedFuse)
 	})
 
 	errGroup.Go(func() error {
-		return listenReconnect(serveCtx, reconnectCh, gracefulShutdownC)
+		return listenReconnect(serveCtx, e.reconnectCh, e.gracefulShutdownC)
 	})
 
 	return errGroup.Wait()
 }
 
-func ServeHTTP2(
+func (e *EdgeTunnelServer) serveHTTP2(
 	ctx context.Context,
 	connLog *ConnAwareLogger,
-	config *TunnelConfig,
-	orchestrator *orchestration.Orchestrator,
 	tlsServerConn net.Conn,
 	connOptions *tunnelpogs.ConnectionOptions,
 	controlStreamHandler connection.ControlStreamHandler,
 	connIndex uint8,
-	gracefulShutdownC <-chan struct{},
-	reconnectCh chan ReconnectSignal,
 ) error {
 	connLog.Logger().Debug().Msgf("Connecting via http2")
 	h2conn := connection.NewHTTP2Connection(
 		tlsServerConn,
-		orchestrator,
+		e.orchestrator,
 		connOptions,
-		config.Observer,
+		e.config.Observer,
 		connIndex,
 		controlStreamHandler,
-		config.Log,
+		e.config.Log,
 	)
 
 	errGroup, serveCtx := errgroup.WithContext(ctx)
@@ -623,7 +577,7 @@ func ServeHTTP2(
 	})
 
 	errGroup.Go(func() error {
-		err := listenReconnect(serveCtx, reconnectCh, gracefulShutdownC)
+		err := listenReconnect(serveCtx, e.reconnectCh, e.gracefulShutdownC)
 		if err != nil {
 			// forcefully break the connection (this is only used for testing)
 			connLog.Logger().Debug().Msg("Forcefully breaking http2 connection")
@@ -635,19 +589,15 @@ func ServeHTTP2(
 	return errGroup.Wait()
 }
 
-func ServeQUIC(
+func (e *EdgeTunnelServer) serveQUIC(
 	ctx context.Context,
 	edgeAddr *net.UDPAddr,
-	config *TunnelConfig,
-	orchestrator *orchestration.Orchestrator,
 	connLogger *ConnAwareLogger,
 	connOptions *tunnelpogs.ConnectionOptions,
 	controlStreamHandler connection.ControlStreamHandler,
 	connIndex uint8,
-	reconnectCh chan ReconnectSignal,
-	gracefulShutdownC <-chan struct{},
 ) (err error, recoverable bool) {
-	tlsConfig := config.EdgeTLSConfigs[connection.QUIC]
+	tlsConfig := e.config.EdgeTLSConfigs[connection.QUIC]
 	quicConfig := &quic.Config{
 		HandshakeIdleTimeout:  quicpogs.HandshakeIdleTimeout,
 		MaxIdleTimeout:        quicpogs.MaxIdleTimeout,
@@ -663,10 +613,11 @@ func ServeQUIC(
 		quicConfig,
 		edgeAddr,
 		tlsConfig,
-		orchestrator,
+		e.orchestrator,
 		connOptions,
 		controlStreamHandler,
-		connLogger.Logger())
+		connLogger.Logger(),
+		e.icmpProxy)
 	if err != nil {
 		connLogger.ConnAwareLogger().Err(err).Msgf("Failed to create new quic connection")
 		return err, true
@@ -682,7 +633,7 @@ func ServeQUIC(
 	})
 
 	errGroup.Go(func() error {
-		err := listenReconnect(serveCtx, reconnectCh, gracefulShutdownC)
+		err := listenReconnect(serveCtx, e.reconnectCh, e.gracefulShutdownC)
 		if err != nil {
 			// forcefully break the connection (this is only used for testing)
 			connLogger.Logger().Debug().Msg("Forcefully breaking quic connection")