TUN-3738: Refactor observer to avoid potential of blocking on tunnel notifications

metrics/metrics.go

@@ -10,8 +10,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/cloudflare/cloudflared/connection"
-
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/rs/zerolog"
@@ -23,21 +21,22 @@ const (
 	startupTime     = time.Millisecond * 500
 )
 
-func newMetricsHandler(connectionEvents <-chan connection.Event, log *zerolog.Logger) *http.ServeMux {
-	readyServer := NewReadyServer(connectionEvents, log)
+func newMetricsHandler(readyServer *ReadyServer) *http.ServeMux {
 	mux := http.NewServeMux()
 	mux.Handle("/metrics", promhttp.Handler())
 	mux.HandleFunc("/healthcheck", func(w http.ResponseWriter, r *http.Request) {
 		_, _ = fmt.Fprintf(w, "OK\n")
 	})
-	mux.Handle("/ready", readyServer)
+	if readyServer != nil {
+		mux.Handle("/ready", readyServer)
+	}
 	return mux
 }
 
 func ServeMetrics(
 	l net.Listener,
 	shutdownC <-chan struct{},
-	connectionEvents <-chan connection.Event,
+	readyServer *ReadyServer,
 	log *zerolog.Logger,
 ) (err error) {
 	var wg sync.WaitGroup
@@ -45,7 +44,7 @@ func ServeMetrics(
 	trace.AuthRequest = func(*http.Request) (bool, bool) { return true, true }
 	// TODO: parameterize ReadTimeout and WriteTimeout. The maximum time we can
 	// profile CPU usage depends on WriteTimeout
-	h := newMetricsHandler(connectionEvents, log)
+	h := newMetricsHandler(readyServer)
 	server := &http.Server{
 		ReadTimeout:  10 * time.Second,
 		WriteTimeout: 10 * time.Second,

metrics/readiness.go

@@ -19,30 +19,28 @@ type ReadyServer struct {
 }
 
 // NewReadyServer initializes a ReadyServer and starts listening for dis/connection events.
-func NewReadyServer(connectionEvents <-chan conn.Event, log *zerolog.Logger) *ReadyServer {
-	rs := ReadyServer{
+func NewReadyServer(log *zerolog.Logger) *ReadyServer {
+	return &ReadyServer{
 		isConnected: make(map[int]bool, 0),
 		log:         log,
 	}
-	go func() {
-		for c := range connectionEvents {
-			switch c.EventType {
-			case conn.Connected:
-				rs.Lock()
-				rs.isConnected[int(c.Index)] = true
-				rs.Unlock()
-			case conn.Disconnected, conn.Reconnecting, conn.RegisteringTunnel:
-				rs.Lock()
-				rs.isConnected[int(c.Index)] = false
-				rs.Unlock()
-			case conn.SetURL:
-				continue
-			default:
-				rs.log.Error().Msgf("Unknown connection event case %v", c)
-			}
-		}
-	}()
-	return &rs
+}
+
+func (rs *ReadyServer) OnTunnelEvent(c conn.Event) {
+	switch c.EventType {
+	case conn.Connected:
+		rs.Lock()
+		rs.isConnected[int(c.Index)] = true
+		rs.Unlock()
+	case conn.Disconnected, conn.Reconnecting, conn.RegisteringTunnel:
+		rs.Lock()
+		rs.isConnected[int(c.Index)] = false
+		rs.Unlock()
+	case conn.SetURL:
+		break
+	default:
+		rs.log.Error().Msgf("Unknown connection event case %v", c)
+	}
 }
 
 type body struct {

metrics/readiness_test.go

@@ -3,6 +3,11 @@ package metrics
 import (
 	"net/http"
 	"testing"
+
+	"github.com/rs/zerolog"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/cloudflare/cloudflared/connection"
 )
 
 func TestReadyServer_makeResponse(t *testing.T) {
@@ -56,3 +61,49 @@ func TestReadyServer_makeResponse(t *testing.T) {
 		})
 	}
 }
+
+func TestReadinessEventHandling(t *testing.T) {
+	nopLogger := zerolog.Nop()
+	rs := NewReadyServer(&nopLogger)
+
+	// start not ok
+	code, ready := rs.makeResponse()
+	assert.NotEqualValues(t, http.StatusOK, code)
+	assert.Zero(t, ready)
+
+	// one connected => ok
+	rs.OnTunnelEvent(connection.Event{
+		Index:     1,
+		EventType: connection.Connected,
+	})
+	code, ready = rs.makeResponse()
+	assert.EqualValues(t, http.StatusOK, code)
+	assert.EqualValues(t, 1, ready)
+
+	// another connected => still ok
+	rs.OnTunnelEvent(connection.Event{
+		Index:     2,
+		EventType: connection.Connected,
+	})
+	code, ready = rs.makeResponse()
+	assert.EqualValues(t, http.StatusOK, code)
+	assert.EqualValues(t, 2,  ready)
+
+	// one reconnecting => still ok
+	rs.OnTunnelEvent(connection.Event{
+		Index:     2,
+		EventType: connection.Reconnecting,
+	})
+	code, ready = rs.makeResponse()
+	assert.EqualValues(t, http.StatusOK, code)
+	assert.EqualValues(t, 1, ready)
+
+	// other disconnected => not ok
+	rs.OnTunnelEvent(connection.Event{
+		Index:     1,
+		EventType: connection.Disconnected,
+	})
+	code, ready = rs.makeResponse()
+	assert.NotEqualValues(t, http.StatusOK, code)
+	assert.Zero(t, ready)
+}