From 3e0ff3a7713378a170b34863ec630aa4d72ea41a Mon Sep 17 00:00:00 2001 From: Chung-Ting Huang Date: Mon, 29 Aug 2022 18:49:07 +0100 Subject: [PATCH] TUN-6531: Implement ICMP proxy for Windows using IcmpSendEcho --- ingress/icmp_generic.go | 2 +- ingress/icmp_linux.go | 9 +- ingress/icmp_windows.go | 317 ++++++++++++++++++++++++++++++ ingress/icmp_windows_test.go | 139 +++++++++++++ ingress/origin_icmp_proxy.go | 9 + ingress/origin_icmp_proxy_test.go | 11 +- quic/datagramv2.go | 2 +- 7 files changed, 472 insertions(+), 17 deletions(-) create mode 100644 ingress/icmp_windows.go create mode 100644 ingress/icmp_windows_test.go diff --git a/ingress/icmp_generic.go b/ingress/icmp_generic.go index e2c9aae7..976387f9 100644 --- a/ingress/icmp_generic.go +++ b/ingress/icmp_generic.go @@ -1,4 +1,4 @@ -//go:build !darwin && !linux +//go:build !darwin && !linux && !windows package ingress diff --git a/ingress/icmp_linux.go b/ingress/icmp_linux.go index 9803bb37..f7e2d936 100644 --- a/ingress/icmp_linux.go +++ b/ingress/icmp_linux.go @@ -58,12 +58,11 @@ func (ip *icmpProxy) Request(pk *packet.ICMP, responder packet.FlowResponder) er if pk == nil { return errPacketNil } - switch body := pk.Message.Body.(type) { - case *icmp.Echo: - return ip.sendICMPEchoRequest(pk, body, responder) - default: - return fmt.Errorf("sending ICMP %s is not implemented", pk.Type) + echo, err := getICMPEcho(pk) + if err != nil { + return err } + return ip.sendICMPEchoRequest(pk, echo, responder) } func (ip *icmpProxy) Serve(ctx context.Context) error { diff --git a/ingress/icmp_windows.go b/ingress/icmp_windows.go new file mode 100644 index 00000000..36eb90d5 --- /dev/null +++ b/ingress/icmp_windows.go @@ -0,0 +1,317 @@ +//go:build windows + +package ingress + +/* +#include +#include +*/ +import "C" +import ( + "context" + "encoding/binary" + "fmt" + "net/netip" + "runtime/debug" + "sync" + "syscall" + "unsafe" + + "github.com/google/gopacket/layers" + "github.com/pkg/errors" + "github.com/rs/zerolog" + "golang.org/x/net/icmp" + "golang.org/x/net/ipv4" + "golang.org/x/net/ipv6" + + "github.com/cloudflare/cloudflared/packet" +) + +const ( + icmpEchoReplyCode = 0 +) + +var ( + Iphlpapi = syscall.NewLazyDLL("Iphlpapi.dll") + IcmpCreateFile_proc = Iphlpapi.NewProc("IcmpCreateFile") + IcmpSendEcho_proc = Iphlpapi.NewProc("IcmpSendEcho") + echoReplySize = unsafe.Sizeof(echoReply{}) + endian = binary.LittleEndian +) + +// IP_STATUS code, see https://docs.microsoft.com/en-us/windows/win32/api/ipexport/ns-ipexport-icmp_echo_reply32#members +// for possible values +type ipStatus uint32 + +const ( + success ipStatus = 0 + bufTooSmall = iota + 11000 + destNetUnreachable + destHostUnreachable + destProtocolUnreachable + destPortUnreachable + noResources + badOption + hwError + packetTooBig + reqTimedOut + badReq + badRoute + ttlExpiredTransit + ttlExpiredReassembly + paramProblem + sourceQuench + optionTooBig + badDestination + // Can be returned for malformed ICMP packets + generalFailure = 11050 +) + +func (is ipStatus) String() string { + switch is { + case success: + return "Success" + case bufTooSmall: + return "The reply buffer too small" + case destNetUnreachable: + return "The destination network was unreachable" + case destHostUnreachable: + return "The destination host was unreachable" + case destProtocolUnreachable: + return "The destination protocol was unreachable" + case destPortUnreachable: + return "The destination port was unreachable" + case noResources: + return "Insufficient IP resources were available" + case badOption: + return "A bad IP option was specified" + case hwError: + return "A hardware error occurred" + case packetTooBig: + return "The packet was too big" + case reqTimedOut: + return "The request timed out" + case badReq: + return "Bad request" + case badRoute: + return "Bad route" + case ttlExpiredTransit: + return "The TTL expired in transit" + case ttlExpiredReassembly: + return "The TTL expired during fragment reassembly" + case paramProblem: + return "A parameter problem" + case sourceQuench: + return "Datagrams are arriving too fast to be processed and datagrams may have been discarded" + case optionTooBig: + return "The IP option was too big" + case badDestination: + return "Bad destination" + case generalFailure: + return "The ICMP packet might be malformed" + default: + return fmt.Sprintf("Unknown ip status %d", is) + } +} + +// https://docs.microsoft.com/en-us/windows/win32/api/ipexport/ns-ipexport-ip_option_information +type ipOption struct { + TTL uint8 + Tos uint8 + Flags uint8 + OptionsSize uint8 + OptionsData uintptr +} + +// https://docs.microsoft.com/en-us/windows/win32/api/ipexport/ns-ipexport-icmp_echo_reply +type echoReply struct { + Address uint32 + Status ipStatus + RoundTripTime uint32 + DataSize uint16 + Reserved uint16 + // The pointer size defers between 32-bit and 64-bit platforms + DataPointer *byte + Options ipOption +} + +type icmpProxy struct { + // An open handle that can send ICMP requests https://docs.microsoft.com/en-us/windows/win32/api/icmpapi/nf-icmpapi-icmpcreatefile + handle uintptr + logger *zerolog.Logger + // A pool of reusable *packet.Encoder + encoderPool sync.Pool +} + +func newICMPProxy(listenIP netip.Addr, logger *zerolog.Logger) (ICMPProxy, error) { + handle, _, err := IcmpCreateFile_proc.Call() + // Windows procedure calls always return non-nil error constructed from the result of GetLastError. + // Caller need to inspect the primary returned value + if syscall.Handle(handle) == syscall.InvalidHandle { + return nil, errors.Wrap(err, "invalid ICMP handle") + } + return &icmpProxy{ + handle: handle, + logger: logger, + encoderPool: sync.Pool{ + New: func() any { + return packet.NewEncoder() + }, + }, + }, nil +} + +func (ip *icmpProxy) Serve(ctx context.Context) error { + <-ctx.Done() + syscall.CloseHandle(syscall.Handle(ip.handle)) + return ctx.Err() +} + +func (ip *icmpProxy) Request(pk *packet.ICMP, responder packet.FlowResponder) error { + if pk == nil { + return errPacketNil + } + defer func() { + if r := recover(); r != nil { + ip.logger.Error().Interface("error", r).Msgf("Recover panic from sending icmp request/response, error %s", debug.Stack()) + } + }() + echo, err := getICMPEcho(pk) + if err != nil { + return err + } + + resp, err := ip.icmpSendEcho(pk.Dst, echo) + if err != nil { + return errors.Wrap(err, "failed to send/receive ICMP echo") + } + + err = ip.handleEchoResponse(pk, echo, resp, responder) + if err != nil { + return errors.Wrap(err, "failed to handle ICMP echo reply") + } + return nil +} + +func (ip *icmpProxy) handleEchoResponse(request *packet.ICMP, echoReq *icmp.Echo, resp *echoResp, responder packet.FlowResponder) error { + var replyType icmp.Type + if request.Dst.Is4() { + replyType = ipv4.ICMPTypeEchoReply + } else { + replyType = ipv6.ICMPTypeEchoReply + } + + pk := packet.ICMP{ + IP: &packet.IP{ + Src: request.Dst, + Dst: request.Src, + Protocol: layers.IPProtocol(request.Type.Protocol()), + }, + Message: &icmp.Message{ + Type: replyType, + Code: icmpEchoReplyCode, + Body: &icmp.Echo{ + ID: echoReq.ID, + Seq: echoReq.Seq, + Data: resp.data, + }, + }, + } + + serializedPacket, err := ip.encodeICMPReply(&pk) + if err != nil { + return err + } + return responder.SendPacket(serializedPacket) +} + +func (ip *icmpProxy) encodeICMPReply(pk *packet.ICMP) (packet.RawPacket, error) { + cachedEncoder := ip.encoderPool.Get() + defer ip.encoderPool.Put(cachedEncoder) + encoder, ok := cachedEncoder.(*packet.Encoder) + if !ok { + return packet.RawPacket{}, fmt.Errorf("encoderPool returned %T, expect *packet.Encoder", cachedEncoder) + } + return encoder.Encode(pk) +} + +/* + Wrapper to call https://docs.microsoft.com/en-us/windows/win32/api/icmpapi/nf-icmpapi-icmpsendecho + Parameters: + - IcmpHandle: + - DestinationAddress: IPv4 in the form of https://docs.microsoft.com/en-us/windows/win32/api/inaddr/ns-inaddr-in_addr#syntax + - RequestData: A pointer to echo data + - RequestSize: Number of bytes in buffer pointed by echo data + - RequestOptions: IP header options + - ReplyBuffer: A pointer to the buffer for echoReply, options and data + - ReplySize: Number of bytes allocated for ReplyBuffer + - Timeout: Timeout in milliseconds to wait for a reply + Returns: + - the number of replies in uint32 https://docs.microsoft.com/en-us/windows/win32/api/icmpapi/nf-icmpapi-icmpsendecho#return-value + To retain the reference allocated objects, conversion from pointer to uintptr must happen as arguments to the + syscall function +*/ +func (ip *icmpProxy) icmpSendEcho(dst netip.Addr, echo *icmp.Echo) (*echoResp, error) { + dataSize := len(echo.Data) + replySize := echoReplySize + uintptr(dataSize) + replyBuf := make([]byte, replySize) + noIPHeaderOption := uintptr(0) + inAddr, err := inAddrV4(dst) + if err != nil { + return nil, err + } + replyCount, _, err := IcmpSendEcho_proc.Call(ip.handle, uintptr(inAddr), uintptr(unsafe.Pointer(&echo.Data[0])), + uintptr(dataSize), noIPHeaderOption, uintptr(unsafe.Pointer(&replyBuf[0])), + replySize, icmpTimeoutMs) + if replyCount == 0 { + // status is returned in 5th to 8th byte of reply buffer + if status, err := unmarshalIPStatus(replyBuf[4:8]); err == nil { + return nil, fmt.Errorf("received ip status: %s", status) + } + return nil, errors.Wrap(err, "did not receive ICMP echo reply") + } else if replyCount > 1 { + ip.logger.Warn().Msgf("Received %d ICMP echo replies, only sending 1 back", replyCount) + } + return newEchoResp(replyBuf) +} + +type echoResp struct { + reply *echoReply + data []byte +} + +func newEchoResp(replyBuf []byte) (*echoResp, error) { + if len(replyBuf) == 0 { + return nil, fmt.Errorf("reply buffer is empty") + } + // This is pattern 1 of https://pkg.go.dev/unsafe@master#Pointer, conversion of *replyBuf to *echoReply + // replyBuf size is larger than echoReply + reply := *(*echoReply)(unsafe.Pointer(&replyBuf[0])) + if reply.Status != success { + return nil, fmt.Errorf("status %d", reply.Status) + } + dataBufStart := len(replyBuf) - int(reply.DataSize) + if dataBufStart < int(echoReplySize) { + return nil, fmt.Errorf("reply buffer size %d is too small to hold data of size %d", len(replyBuf), int(reply.DataSize)) + } + return &echoResp{ + reply: &reply, + data: replyBuf[dataBufStart:], + }, nil +} + +// Third definition of https://docs.microsoft.com/en-us/windows/win32/api/inaddr/ns-inaddr-in_addr#syntax is address in uint32 +func inAddrV4(ip netip.Addr) (uint32, error) { + if !ip.Is4() { + return 0, fmt.Errorf("%s is not IPv4", ip) + } + v4 := ip.As4() + return endian.Uint32(v4[:]), nil +} + +func unmarshalIPStatus(replyBuf []byte) (ipStatus, error) { + if len(replyBuf) != 4 { + return 0, fmt.Errorf("ipStatus needs to be 4 bytes, got %d", len(replyBuf)) + } + return ipStatus(endian.Uint32(replyBuf)), nil +} diff --git a/ingress/icmp_windows_test.go b/ingress/icmp_windows_test.go new file mode 100644 index 00000000..633ddf47 --- /dev/null +++ b/ingress/icmp_windows_test.go @@ -0,0 +1,139 @@ +//go:build windows + +package ingress + +import ( + "bytes" + "encoding/binary" + "fmt" + "io" + "net/netip" + "testing" + "unsafe" + + "golang.org/x/net/icmp" + + "github.com/stretchr/testify/require" +) + +// TestParseEchoReply tests parsing raw bytes from icmpSendEcho into echoResp +func TestParseEchoReply(t *testing.T) { + dst, err := inAddrV4(netip.MustParseAddr("192.168.10.20")) + require.NoError(t, err) + + validReplyData := []byte(t.Name()) + validReply := echoReply{ + Address: dst, + Status: success, + RoundTripTime: uint32(20), + DataSize: uint16(len(validReplyData)), + DataPointer: &validReplyData[0], + Options: ipOption{ + TTL: 59, + }, + } + + destHostUnreachableReply := validReply + destHostUnreachableReply.Status = destHostUnreachable + + tests := []struct { + testCase string + replyBuf []byte + expectedReply *echoReply + expectedData []byte + }{ + { + testCase: "empty buffer", + }, + { + testCase: "status not success", + replyBuf: destHostUnreachableReply.marshal(t, []byte{}), + }, + { + testCase: "valid reply", + replyBuf: validReply.marshal(t, validReplyData), + expectedReply: &validReply, + expectedData: validReplyData, + }, + } + + for _, test := range tests { + resp, err := newEchoResp(test.replyBuf) + if test.expectedReply == nil { + require.Error(t, err) + require.Nil(t, resp) + } else { + require.NoError(t, err) + require.Equal(t, resp.reply, test.expectedReply) + require.True(t, bytes.Equal(resp.data, test.expectedData)) + } + } +} + +// TestSendEchoErrors makes sure icmpSendEcho handles error cases +func TestSendEchoErrors(t *testing.T) { + proxy, err := newICMPProxy(localhostIP, &noopLogger) + require.NoError(t, err) + winProxy := proxy.(*icmpProxy) + + echo := icmp.Echo{ + ID: 6193, + Seq: 25712, + Data: []byte(t.Name()), + } + documentIP := netip.MustParseAddr("192.0.2.200") + resp, err := winProxy.icmpSendEcho(documentIP, &echo) + require.Error(t, err) + require.Nil(t, resp) +} + +func (er *echoReply) marshal(t *testing.T, data []byte) []byte { + buf := new(bytes.Buffer) + + for _, field := range []any{ + er.Address, + er.Status, + er.RoundTripTime, + er.DataSize, + er.Reserved, + } { + require.NoError(t, binary.Write(buf, endian, field)) + } + + require.NoError(t, marshalPointer(buf, uintptr(unsafe.Pointer(er.DataPointer)))) + + for _, field := range []any{ + er.Options.TTL, + er.Options.Tos, + er.Options.Flags, + er.Options.OptionsSize, + } { + require.NoError(t, binary.Write(buf, endian, field)) + } + + require.NoError(t, marshalPointer(buf, er.Options.OptionsData)) + + padSize := buf.Len() % int(unsafe.Alignof(er)) + padding := make([]byte, padSize) + n, err := buf.Write(padding) + require.NoError(t, err) + require.Equal(t, padSize, n) + + n, err = buf.Write(data) + require.NoError(t, err) + require.Equal(t, len(data), n) + + return buf.Bytes() +} + +func marshalPointer(buf io.Writer, ptr uintptr) error { + size := unsafe.Sizeof(ptr) + switch size { + case 4: + return binary.Write(buf, endian, uint32(ptr)) + case 8: + return binary.Write(buf, endian, uint64(ptr)) + default: + return fmt.Errorf("unexpected pointer size %d", size) + } +} diff --git a/ingress/origin_icmp_proxy.go b/ingress/origin_icmp_proxy.go index 0ac851be..f2a4180c 100644 --- a/ingress/origin_icmp_proxy.go +++ b/ingress/origin_icmp_proxy.go @@ -15,6 +15,7 @@ import ( const ( defaultCloseAfterIdle = time.Second * 15 mtu = 1500 + icmpTimeoutMs = 1000 ) var ( @@ -42,3 +43,11 @@ func newICMPConn(listenIP netip.Addr) (*icmp.PacketConn, error) { } return icmp.ListenPacket(network, listenIP.String()) } + +func getICMPEcho(pk *packet.ICMP) (*icmp.Echo, error) { + echo, ok := pk.Message.Body.(*icmp.Echo) + if !ok { + return nil, fmt.Errorf("expect ICMP echo, got %s", pk.Type) + } + return echo, nil +} diff --git a/ingress/origin_icmp_proxy_test.go b/ingress/origin_icmp_proxy_test.go index efb7d489..7a1ebc89 100644 --- a/ingress/origin_icmp_proxy_test.go +++ b/ingress/origin_icmp_proxy_test.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "net/netip" - "runtime" "testing" "github.com/google/gopacket/layers" @@ -28,7 +27,6 @@ var ( // is allowed in ping_group_range. See the following gist for how to do that: // https://github.com/ValentinBELYN/icmplib/blob/main/docs/6-use-icmplib-without-privileges.md func TestICMPProxyEcho(t *testing.T) { - onlyDarwinOrLinux(t) const ( echoID = 36571 endSeq = 100 @@ -46,7 +44,7 @@ func TestICMPProxyEcho(t *testing.T) { responder := echoFlowResponder{ decoder: packet.NewICMPDecoder(), - respChan: make(chan []byte), + respChan: make(chan []byte, 1), } ip := packet.IP{ @@ -76,7 +74,6 @@ func TestICMPProxyEcho(t *testing.T) { // TestICMPProxyRejectNotEcho makes sure it rejects messages other than echo func TestICMPProxyRejectNotEcho(t *testing.T) { - onlyDarwinOrLinux(t) msgs := []icmp.Message{ { Type: ipv4.ICMPTypeDestinationUnreachable, @@ -121,12 +118,6 @@ func TestICMPProxyRejectNotEcho(t *testing.T) { } } -func onlyDarwinOrLinux(t *testing.T) { - if runtime.GOOS != "darwin" && runtime.GOOS != "linux" { - t.Skip("Cannot create non-privileged datagram-oriented ICMP endpoint on Windows") - } -} - type echoFlowResponder struct { decoder *packet.ICMPDecoder respChan chan []byte diff --git a/quic/datagramv2.go b/quic/datagramv2.go index 9bc38f40..3f1c8f0e 100644 --- a/quic/datagramv2.go +++ b/quic/datagramv2.go @@ -21,7 +21,7 @@ const ( const ( typeIDLen = 1 // Same as sessionDemuxChan capacity - packetChanCapacity = 16 + packetChanCapacity = 128 ) func suffixType(b []byte, datagramType datagramV2Type) ([]byte, error) {