mirror of
https://github.com/cloudflare/cloudflared.git
synced 2025-07-27 19:29:57 +00:00
TUN-8737: update metrics server port selection
## Summary Update how metrics server binds to a listener by using a known set of ports whenever the default address is used with the fallback to a random port in case all address are already in use. The default address changes at compile time in order to bind to a different default address when the final deliverable is a docker image. Refactor ReadyServer tests. Closes TUN-8737
This commit is contained in:
Luis Neto
@@ -10,6 +10,7 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/facebookgo/grace/gracenet"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
"github.com/rs/zerolog"
|
||||
@@ -21,6 +22,34 @@ const (
|
||||
defaultShutdownTimeout = time.Second * 15
|
||||
)
|
||||
|
||||
// This variable is set at compile time to allow the default local address to change.
|
||||
var Runtime = "host"
|
||||
|
||||
func GetMetricsDefaultAddress(runtimeType string) string {
|
||||
// When issuing the diagnostic command we may have to reach a server that is
|
||||
// running in a virtual enviroment and in that case we must bind to 0.0.0.0
|
||||
// otherwise the server won't be reachable.
|
||||
switch runtimeType {
|
||||
case "virtual":
|
||||
return "0.0.0.0:0"
|
||||
default:
|
||||
return "localhost:0"
|
||||
}
|
||||
}
|
||||
|
||||
// GetMetricsKnownAddresses returns the addresses used by the metrics server to bind at
|
||||
// startup time to allow a semi-deterministic approach to know where the server is listening at.
|
||||
// The ports were selected because at the time we are in 2024 and they do not collide with any
|
||||
// know/registered port according https://en.wikipedia.org/wiki/List_of_TCP_and_UDP_port_numbers.
|
||||
func GetMetricsKnownAddresses(runtimeType string) [5]string {
|
||||
switch Runtime {
|
||||
case "virtual":
|
||||
return [5]string{"0.0.0.0:20241", "0.0.0.0:20242", "0.0.0.0:20243", "0.0.0.0:20244", "0.0.0.0:20245"}
|
||||
default:
|
||||
return [5]string{"localhost:20241", "localhost:20242", "localhost:20243", "localhost:20244", "localhost:20245"}
|
||||
}
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
ReadyServer *ReadyServer
|
||||
QuickTunnelHostname string
|
||||
@@ -65,6 +94,42 @@ func newMetricsHandler(
|
||||
return router
|
||||
}
|
||||
|
||||
// CreateMetricsListener will create a new [net.Listener] by using an
|
||||
// known set of ports when the default address is passed with the fallback
|
||||
// of choosing a random port when none is available.
|
||||
//
|
||||
// In case the provided address is not the default one then it will be used
|
||||
// as is.
|
||||
func CreateMetricsListener(listeners *gracenet.Net, laddr string) (net.Listener, error) {
|
||||
if laddr == GetMetricsDefaultAddress(Runtime) {
|
||||
// On the presence of the default address select
|
||||
// a port from the known set of addresses iteratively.
|
||||
addresses := GetMetricsKnownAddresses(Runtime)
|
||||
for _, address := range addresses {
|
||||
listener, err := listeners.Listen("tcp", address)
|
||||
if err == nil {
|
||||
return listener, nil
|
||||
}
|
||||
}
|
||||
|
||||
// When no port is available then bind to a random one
|
||||
listener, err := listeners.Listen("tcp", laddr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to listen to default metrics address: %w", err)
|
||||
}
|
||||
|
||||
return listener, nil
|
||||
}
|
||||
|
||||
// Explicitly got a local address then bind to it
|
||||
listener, err := listeners.Listen("tcp", laddr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to bind to address (%s): %w", laddr, err)
|
||||
}
|
||||
|
||||
return listener, nil
|
||||
}
|
||||
|
||||
func ServeMetrics(
|
||||
l net.Listener,
|
||||
ctx context.Context,
|
||||
|
||||
52
metrics/metrics_test.go
Normal file
52
metrics/metrics_test.go
Normal file
@@ -0,0 +1,52 @@
|
||||
package metrics_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/facebookgo/grace/gracenet"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/cloudflare/cloudflared/metrics"
|
||||
)
|
||||
|
||||
func TestMetricsListenerCreation(t *testing.T) {
|
||||
t.Parallel()
|
||||
listeners := gracenet.Net{}
|
||||
listener1, err := metrics.CreateMetricsListener(&listeners, metrics.GetMetricsDefaultAddress("host"))
|
||||
assert.Equal(t, "127.0.0.1:20241", listener1.Addr().String())
|
||||
require.NoError(t, err)
|
||||
listener2, err := metrics.CreateMetricsListener(&listeners, metrics.GetMetricsDefaultAddress("host"))
|
||||
assert.Equal(t, "127.0.0.1:20242", listener2.Addr().String())
|
||||
require.NoError(t, err)
|
||||
listener3, err := metrics.CreateMetricsListener(&listeners, metrics.GetMetricsDefaultAddress("host"))
|
||||
assert.Equal(t, "127.0.0.1:20243", listener3.Addr().String())
|
||||
require.NoError(t, err)
|
||||
listener4, err := metrics.CreateMetricsListener(&listeners, metrics.GetMetricsDefaultAddress("host"))
|
||||
assert.Equal(t, "127.0.0.1:20244", listener4.Addr().String())
|
||||
require.NoError(t, err)
|
||||
listener5, err := metrics.CreateMetricsListener(&listeners, metrics.GetMetricsDefaultAddress("host"))
|
||||
assert.Equal(t, "127.0.0.1:20245", listener5.Addr().String())
|
||||
require.NoError(t, err)
|
||||
listener6, err := metrics.CreateMetricsListener(&listeners, metrics.GetMetricsDefaultAddress("host"))
|
||||
addresses := [5]string{"127.0.0.1:20241", "127.0.0.1:20242", "127.0.0.1:20243", "127.0.0.1:20244", "127.0.0.1:20245"}
|
||||
assert.NotContains(t, addresses, listener6.Addr().String())
|
||||
require.NoError(t, err)
|
||||
listener7, err := metrics.CreateMetricsListener(&listeners, "localhost:12345")
|
||||
assert.Equal(t, "127.0.0.1:12345", listener7.Addr().String())
|
||||
require.NoError(t, err)
|
||||
err = listener1.Close()
|
||||
require.NoError(t, err)
|
||||
err = listener2.Close()
|
||||
require.NoError(t, err)
|
||||
err = listener3.Close()
|
||||
require.NoError(t, err)
|
||||
err = listener4.Close()
|
||||
require.NoError(t, err)
|
||||
err = listener5.Close()
|
||||
require.NoError(t, err)
|
||||
err = listener6.Close()
|
||||
require.NoError(t, err)
|
||||
err = listener7.Close()
|
||||
require.NoError(t, err)
|
||||
}
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
"net/http"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/rs/zerolog"
|
||||
|
||||
conn "github.com/cloudflare/cloudflared/connection"
|
||||
"github.com/cloudflare/cloudflared/tunnelstate"
|
||||
@@ -19,10 +18,13 @@ type ReadyServer struct {
|
||||
}
|
||||
|
||||
// NewReadyServer initializes a ReadyServer and starts listening for dis/connection events.
|
||||
func NewReadyServer(log *zerolog.Logger, clientID uuid.UUID) *ReadyServer {
|
||||
func NewReadyServer(
|
||||
clientID uuid.UUID,
|
||||
tracker *tunnelstate.ConnTracker,
|
||||
) *ReadyServer {
|
||||
return &ReadyServer{
|
||||
clientID: clientID,
|
||||
tracker: tunnelstate.NewConnTracker(log),
|
||||
clientID,
|
||||
tracker,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,136 +1,106 @@
|
||||
package metrics
|
||||
package metrics_test
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/rs/zerolog"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/cloudflare/cloudflared/connection"
|
||||
"github.com/cloudflare/cloudflared/metrics"
|
||||
"github.com/cloudflare/cloudflared/tunnelstate"
|
||||
)
|
||||
|
||||
func TestReadyServer_makeResponse(t *testing.T) {
|
||||
type fields struct {
|
||||
isConnected map[uint8]tunnelstate.ConnectionInfo
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
fields fields
|
||||
wantOK bool
|
||||
wantReadyConnections uint
|
||||
}{
|
||||
{
|
||||
name: "One connection online => HTTP 200",
|
||||
fields: fields{
|
||||
isConnected: map[uint8]tunnelstate.ConnectionInfo{
|
||||
0: {IsConnected: false},
|
||||
1: {IsConnected: false},
|
||||
2: {IsConnected: true},
|
||||
3: {IsConnected: false},
|
||||
},
|
||||
},
|
||||
wantOK: true,
|
||||
wantReadyConnections: 1,
|
||||
},
|
||||
{
|
||||
name: "No connections online => no HTTP 200",
|
||||
fields: fields{
|
||||
isConnected: map[uint8]tunnelstate.ConnectionInfo{
|
||||
0: {IsConnected: false},
|
||||
1: {IsConnected: false},
|
||||
2: {IsConnected: false},
|
||||
3: {IsConnected: false},
|
||||
},
|
||||
},
|
||||
wantReadyConnections: 0,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
rs := &ReadyServer{
|
||||
tracker: tunnelstate.MockedConnTracker(tt.fields.isConnected),
|
||||
}
|
||||
gotStatusCode, gotReadyConnections := rs.makeResponse()
|
||||
if tt.wantOK && gotStatusCode != http.StatusOK {
|
||||
t.Errorf("ReadyServer.makeResponse() gotStatusCode = %v, want ok = %v", gotStatusCode, tt.wantOK)
|
||||
}
|
||||
if gotReadyConnections != tt.wantReadyConnections {
|
||||
t.Errorf("ReadyServer.makeResponse() gotReadyConnections = %v, want %v", gotReadyConnections, tt.wantReadyConnections)
|
||||
}
|
||||
})
|
||||
func mockRequest(t *testing.T, readyServer *metrics.ReadyServer) (int, uint) {
|
||||
t.Helper()
|
||||
|
||||
var readyreadyConnections struct {
|
||||
Status int `json:"status"`
|
||||
ReadyConnections uint `json:"readyConnections"`
|
||||
ConnectorID uuid.UUID `json:"connectorId"`
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
readyServer.ServeHTTP(rec, nil)
|
||||
|
||||
decoder := json.NewDecoder(rec.Body)
|
||||
err := decoder.Decode(&readyreadyConnections)
|
||||
require.NoError(t, err)
|
||||
return rec.Code, readyreadyConnections.ReadyConnections
|
||||
}
|
||||
|
||||
func TestReadinessEventHandling(t *testing.T) {
|
||||
nopLogger := zerolog.Nop()
|
||||
rs := NewReadyServer(&nopLogger, uuid.Nil)
|
||||
tracker := tunnelstate.NewConnTracker(&nopLogger)
|
||||
rs := metrics.NewReadyServer(uuid.Nil, tracker)
|
||||
|
||||
// start not ok
|
||||
code, ready := rs.makeResponse()
|
||||
code, readyConnections := mockRequest(t, rs)
|
||||
assert.NotEqualValues(t, http.StatusOK, code)
|
||||
assert.Zero(t, ready)
|
||||
assert.Zero(t, readyConnections)
|
||||
|
||||
// one connected => ok
|
||||
rs.OnTunnelEvent(connection.Event{
|
||||
Index: 1,
|
||||
EventType: connection.Connected,
|
||||
})
|
||||
code, ready = rs.makeResponse()
|
||||
code, readyConnections = mockRequest(t, rs)
|
||||
assert.EqualValues(t, http.StatusOK, code)
|
||||
assert.EqualValues(t, 1, ready)
|
||||
assert.EqualValues(t, 1, readyConnections)
|
||||
|
||||
// another connected => still ok
|
||||
rs.OnTunnelEvent(connection.Event{
|
||||
Index: 2,
|
||||
EventType: connection.Connected,
|
||||
})
|
||||
code, ready = rs.makeResponse()
|
||||
code, readyConnections = mockRequest(t, rs)
|
||||
assert.EqualValues(t, http.StatusOK, code)
|
||||
assert.EqualValues(t, 2, ready)
|
||||
assert.EqualValues(t, 2, readyConnections)
|
||||
|
||||
// one reconnecting => still ok
|
||||
rs.OnTunnelEvent(connection.Event{
|
||||
Index: 2,
|
||||
EventType: connection.Reconnecting,
|
||||
})
|
||||
code, ready = rs.makeResponse()
|
||||
code, readyConnections = mockRequest(t, rs)
|
||||
assert.EqualValues(t, http.StatusOK, code)
|
||||
assert.EqualValues(t, 1, ready)
|
||||
assert.EqualValues(t, 1, readyConnections)
|
||||
|
||||
// Regression test for TUN-3777
|
||||
rs.OnTunnelEvent(connection.Event{
|
||||
Index: 1,
|
||||
EventType: connection.RegisteringTunnel,
|
||||
})
|
||||
code, ready = rs.makeResponse()
|
||||
code, readyConnections = mockRequest(t, rs)
|
||||
assert.NotEqualValues(t, http.StatusOK, code)
|
||||
assert.Zero(t, ready)
|
||||
assert.Zero(t, readyConnections)
|
||||
|
||||
// other connected then unregistered => not ok
|
||||
rs.OnTunnelEvent(connection.Event{
|
||||
Index: 1,
|
||||
EventType: connection.Connected,
|
||||
})
|
||||
code, ready = rs.makeResponse()
|
||||
code, readyConnections = mockRequest(t, rs)
|
||||
assert.EqualValues(t, http.StatusOK, code)
|
||||
assert.EqualValues(t, 1, ready)
|
||||
assert.EqualValues(t, 1, readyConnections)
|
||||
rs.OnTunnelEvent(connection.Event{
|
||||
Index: 1,
|
||||
EventType: connection.Unregistering,
|
||||
})
|
||||
code, ready = rs.makeResponse()
|
||||
code, readyConnections = mockRequest(t, rs)
|
||||
assert.NotEqualValues(t, http.StatusOK, code)
|
||||
assert.Zero(t, ready)
|
||||
assert.Zero(t, readyConnections)
|
||||
|
||||
// other disconnected => not ok
|
||||
rs.OnTunnelEvent(connection.Event{
|
||||
Index: 1,
|
||||
EventType: connection.Disconnected,
|
||||
})
|
||||
code, ready = rs.makeResponse()
|
||||
code, readyConnections = mockRequest(t, rs)
|
||||
assert.NotEqualValues(t, http.StatusOK, code)
|
||||
assert.Zero(t, ready)
|
||||
assert.Zero(t, readyConnections)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user