hoelee/cloudflared
1
0
Fork 0
mirror of https://github.com/cloudflare/cloudflared.git synced 2025-07-27 17:19:58 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

TUN-8424: Refactor capnp registration server

Browse Source 
Move RegistrationServer and RegistrationClient into tunnelrpc module
to properly abstract out the capnp aspects internal to the module only.
This commit is contained in:
Devin Carr
2024-05-24 11:40:10 -07:00
parent 43446bc692
commit 654a326098
13 changed files with 197 additions and 167 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
connection/connection.go
View File

@@ -42,7 +42,7 @@ type Orchestrator interface {
GetOriginProxy() (OriginProxy, error)
}
type NamedTunnelProperties struct {
type TunnelProperties struct {
Credentials Credentials
Client pogs.ClientInfo
QuickTunnelUrl string

80
connection/control.go
View File

@@ -6,25 +6,25 @@ import (
"net"
"time"
"github.com/rs/zerolog"
"github.com/cloudflare/cloudflared/management"
"github.com/cloudflare/cloudflared/tunnelrpc"
tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
)
// RPCClientFunc derives a named tunnel rpc client that can then be used to register and unregister connections.
type RPCClientFunc func(context.Context, io.ReadWriteCloser, *zerolog.Logger) NamedTunnelRPCClient
// registerClient derives a named tunnel rpc client that can then be used to register and unregister connections.
type registerClientFunc func(context.Context, io.ReadWriteCloser, time.Duration) tunnelrpc.RegistrationClient
type controlStream struct {
observer *Observer
connectedFuse ConnectedFuse
namedTunnelProperties *NamedTunnelProperties
connIndex uint8
edgeAddress net.IP
protocol Protocol
connectedFuse ConnectedFuse
tunnelProperties *TunnelProperties
connIndex uint8
edgeAddress net.IP
protocol Protocol
newRPCClientFunc RPCClientFunc
registerClientFunc registerClientFunc
registerTimeout time.Duration
gracefulShutdownC <-chan struct{}
gracePeriod time.Duration
@@ -47,27 +47,29 @@ type TunnelConfigJSONGetter interface {
func NewControlStream(
observer *Observer,
connectedFuse ConnectedFuse,
namedTunnelConfig *NamedTunnelProperties,
tunnelProperties *TunnelProperties,
connIndex uint8,
edgeAddress net.IP,
newRPCClientFunc RPCClientFunc,
registerClientFunc registerClientFunc,
registerTimeout time.Duration,
gracefulShutdownC <-chan struct{},
gracePeriod time.Duration,
protocol Protocol,
) ControlStreamHandler {
if newRPCClientFunc == nil {
newRPCClientFunc = newRegistrationRPCClient
if registerClientFunc == nil {
registerClientFunc = tunnelrpc.NewRegistrationClient
}
return &controlStream{
observer: observer,
connectedFuse: connectedFuse,
namedTunnelProperties: namedTunnelConfig,
newRPCClientFunc: newRPCClientFunc,
connIndex: connIndex,
edgeAddress: edgeAddress,
gracefulShutdownC: gracefulShutdownC,
gracePeriod: gracePeriod,
protocol: protocol,
observer: observer,
connectedFuse: connectedFuse,
tunnelProperties: tunnelProperties,
registerClientFunc: registerClientFunc,
registerTimeout: registerTimeout,
connIndex: connIndex,
edgeAddress: edgeAddress,
gracefulShutdownC: gracefulShutdownC,
gracePeriod: gracePeriod,
protocol: protocol,
}
}
@@ -77,13 +79,25 @@ func (c *controlStream) ServeControlStream(
connOptions *tunnelpogs.ConnectionOptions,
tunnelConfigGetter TunnelConfigJSONGetter,
) error {
rpcClient := c.newRPCClientFunc(ctx, rw, c.observer.log)
registrationClient := c.registerClientFunc(ctx, rw, c.registerTimeout)
registrationDetails, err := rpcClient.RegisterConnection(ctx, c.namedTunnelProperties, connOptions, c.connIndex, c.edgeAddress, c.observer)
registrationDetails, err := registrationClient.RegisterConnection(
ctx,
c.tunnelProperties.Credentials.Auth(),
c.tunnelProperties.Credentials.TunnelID,
connOptions,
c.connIndex,
c.edgeAddress)
if err != nil {
rpcClient.Close()
return err
defer registrationClient.Close()
if err.Error() == DuplicateConnectionError {
c.observer.metrics.regFail.WithLabelValues("dup_edge_conn", "registerConnection").Inc()
return errDuplicationConnection
}
c.observer.metrics.regFail.WithLabelValues("server_error", "registerConnection").Inc()
return serverRegistrationErrorFromRPC(err)
}
c.observer.metrics.regSuccess.WithLabelValues("registerConnection").Inc()
c.observer.logConnected(registrationDetails.UUID, c.connIndex, registrationDetails.Location, c.edgeAddress, c.protocol)
c.observer.sendConnectedEvent(c.connIndex, c.protocol, registrationDetails.Location)
@@ -92,21 +106,23 @@ func (c *controlStream) ServeControlStream(
// if conn index is 0 and tunnel is not remotely managed, then send local ingress rules configuration
if c.connIndex == 0 && !registrationDetails.TunnelIsRemotelyManaged {
if tunnelConfig, err := tunnelConfigGetter.GetConfigJSON(); err == nil {
if err := rpcClient.SendLocalConfiguration(ctx, tunnelConfig, c.observer); err != nil {
if err := registrationClient.SendLocalConfiguration(ctx, tunnelConfig); err != nil {
c.observer.metrics.localConfigMetrics.pushesErrors.Inc()
c.observer.log.Err(err).Msg("unable to send local configuration")
}
c.observer.metrics.localConfigMetrics.pushes.Inc()
} else {
c.observer.log.Err(err).Msg("failed to obtain current configuration")
}
}
c.waitForUnregister(ctx, rpcClient)
c.waitForUnregister(ctx, registrationClient)
return nil
}
func (c *controlStream) waitForUnregister(ctx context.Context, rpcClient NamedTunnelRPCClient) {
func (c *controlStream) waitForUnregister(ctx context.Context, registrationClient tunnelrpc.RegistrationClient) {
// wait for connection termination or start of graceful shutdown
defer rpcClient.Close()
defer registrationClient.Close()
select {
case <-ctx.Done():
break
@@ -115,7 +131,7 @@ func (c *controlStream) waitForUnregister(ctx context.Context, rpcClient NamedTu
}
c.observer.sendUnregisteringEvent(c.connIndex)
rpcClient.GracefulShutdown(ctx, c.gracePeriod)
registrationClient.GracefulShutdown(ctx, c.gracePeriod)
c.observer.log.Info().
Int(management.EventTypeKey, int(management.Cloudflared)).
Uint8(LogFieldConnIndex, c.connIndex).

3
connection/http2.go
View File

@@ -40,8 +40,6 @@ type HTTP2Connection struct {
connOptions *tunnelpogs.ConnectionOptions
observer *Observer
connIndex uint8
// newRPCClientFunc allows us to mock RPCs during testing
newRPCClientFunc func(context.Context, io.ReadWriteCloser, *zerolog.Logger) NamedTunnelRPCClient
log *zerolog.Logger
activeRequestsWG sync.WaitGroup
@@ -69,7 +67,6 @@ func NewHTTP2Connection(
connOptions: connOptions,
observer: observer,
connIndex: connIndex,
newRPCClientFunc: newRegistrationRPCClient,
controlStreamHandler: controlStreamHandler,
log: log,
}

32
connection/http2_test.go
View File

@@ -20,8 +20,8 @@ import (
"github.com/stretchr/testify/require"
"golang.org/x/net/http2"
"github.com/cloudflare/cloudflared/tunnelrpc"
"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
)
var (
@@ -36,10 +36,11 @@ func newTestHTTP2Connection() (*HTTP2Connection, net.Conn) {
controlStream := NewControlStream(
obs,
mockConnectedFuse{},
&NamedTunnelProperties{},
&TunnelProperties{},
connIndex,
nil,
nil,
1*time.Second,
nil,
1*time.Second,
HTTP2,
@@ -168,23 +169,23 @@ type mockNamedTunnelRPCClient struct {
unregistered chan struct{}
}
func (mc mockNamedTunnelRPCClient) SendLocalConfiguration(c context.Context, config []byte, observer *Observer) error {
func (mc mockNamedTunnelRPCClient) SendLocalConfiguration(c context.Context, config []byte) error {
return nil
}
func (mc mockNamedTunnelRPCClient) RegisterConnection(
c context.Context,
properties *NamedTunnelProperties,
options *tunnelpogs.ConnectionOptions,
ctx context.Context,
auth pogs.TunnelAuth,
tunnelID uuid.UUID,
options *pogs.ConnectionOptions,
connIndex uint8,
edgeAddress net.IP,
observer *Observer,
) (*tunnelpogs.ConnectionDetails, error) {
) (*pogs.ConnectionDetails, error) {
if mc.shouldFail != nil {
return nil, mc.shouldFail
}
close(mc.registered)
return &tunnelpogs.ConnectionDetails{
return &pogs.ConnectionDetails{
Location: "LIS",
UUID: uuid.New(),
TunnelIsRemotelyManaged: false,
@@ -203,8 +204,8 @@ type mockRPCClientFactory struct {
unregistered chan struct{}
}
func (mf *mockRPCClientFactory) newMockRPCClient(context.Context, io.ReadWriteCloser, *zerolog.Logger) NamedTunnelRPCClient {
return mockNamedTunnelRPCClient{
func (mf *mockRPCClientFactory) newMockRPCClient(context.Context, io.ReadWriteCloser, time.Duration) tunnelrpc.RegistrationClient {
return &mockNamedTunnelRPCClient{
shouldFail: mf.shouldFail,
registered: mf.registered,
unregistered: mf.unregistered,
@@ -360,10 +361,11 @@ func TestServeControlStream(t *testing.T) {
controlStream := NewControlStream(
obs,
mockConnectedFuse{},
&NamedTunnelProperties{},
&TunnelProperties{},
1,
nil,
rpcClientFactory.newMockRPCClient,
1*time.Second,
nil,
1*time.Second,
HTTP2,
@@ -412,10 +414,11 @@ func TestFailRegistration(t *testing.T) {
controlStream := NewControlStream(
obs,
mockConnectedFuse{},
&NamedTunnelProperties{},
&TunnelProperties{},
http2Conn.connIndex,
nil,
rpcClientFactory.newMockRPCClient,
1*time.Second,
nil,
1*time.Second,
HTTP2,
@@ -460,10 +463,11 @@ func TestGracefulShutdownHTTP2(t *testing.T) {
controlStream := NewControlStream(
obs,
mockConnectedFuse{},
&NamedTunnelProperties{},
&TunnelProperties{},
http2Conn.connIndex,
nil,
rpcClientFactory.newMockRPCClient,
1*time.Second,
shutdownC,
1*time.Second,
HTTP2,

111
connection/rpc.go
View File

@@ -1,111 +0,0 @@
package connection
import (
"context"
"io"
"net"
"time"
"github.com/rs/zerolog"
"zombiezen.com/go/capnproto2/rpc"
tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
)
type NamedTunnelRPCClient interface {
RegisterConnection(
c context.Context,
config *NamedTunnelProperties,
options *tunnelpogs.ConnectionOptions,
connIndex uint8,
edgeAddress net.IP,
observer *Observer,
) (*tunnelpogs.ConnectionDetails, error)
SendLocalConfiguration(
c context.Context,
config []byte,
observer *Observer,
) error
GracefulShutdown(ctx context.Context, gracePeriod time.Duration)
Close()
}
type registrationServerClient struct {
client tunnelpogs.RegistrationServer_PogsClient
transport rpc.Transport
}
func newRegistrationRPCClient(
ctx context.Context,
stream io.ReadWriteCloser,
log *zerolog.Logger,
) NamedTunnelRPCClient {
transport := rpc.StreamTransport(stream)
conn := rpc.NewConn(transport)
return &registrationServerClient{
client: tunnelpogs.RegistrationServer_PogsClient{Client: conn.Bootstrap(ctx), Conn: conn},
transport: transport,
}
}
func (rsc *registrationServerClient) RegisterConnection(
ctx context.Context,
properties *NamedTunnelProperties,
options *tunnelpogs.ConnectionOptions,
connIndex uint8,
edgeAddress net.IP,
observer *Observer,
) (*tunnelpogs.ConnectionDetails, error) {
conn, err := rsc.client.RegisterConnection(
ctx,
properties.Credentials.Auth(),
properties.Credentials.TunnelID,
connIndex,
options,
)
if err != nil {
if err.Error() == DuplicateConnectionError {
observer.metrics.regFail.WithLabelValues("dup_edge_conn", "registerConnection").Inc()
return nil, errDuplicationConnection
}
observer.metrics.regFail.WithLabelValues("server_error", "registerConnection").Inc()
return nil, serverRegistrationErrorFromRPC(err)
}
observer.metrics.regSuccess.WithLabelValues("registerConnection").Inc()
return conn, nil
}
func (rsc *registrationServerClient) SendLocalConfiguration(ctx context.Context, config []byte, observer *Observer) (err error) {
observer.metrics.localConfigMetrics.pushes.Inc()
defer func() {
if err != nil {
observer.metrics.localConfigMetrics.pushesErrors.Inc()
}
}()
return rsc.client.SendLocalConfiguration(ctx, config)
}
func (rsc *registrationServerClient) GracefulShutdown(ctx context.Context, gracePeriod time.Duration) {
ctx, cancel := context.WithTimeout(ctx, gracePeriod)
defer cancel()
_ = rsc.client.UnregisterConnection(ctx)
}
func (rsc *registrationServerClient) Close() {
// Closing the client will also close the connection
_ = rsc.client.Close()
// Closing the transport also closes the stream
_ = rsc.transport.Close()
}
type rpcName string
const (
register rpcName = "register"
reconnect rpcName = "reconnect"
unregister rpcName = "unregister"
authenticate rpcName = " authenticate"
)