TUN-8052: Update go to 1.21.5

Also update golang.org/x/net and google.golang.org/grpc to fix vulnerabilities,
although cloudflared is using them in a way that is not exposed to those risks
This commit is contained in:
Chung-Ting
2023-12-11 11:58:19 +00:00
parent b901d73d9b
commit 12dd91ada1
974 changed files with 28780 additions and 25799 deletions

View File

@@ -1,10 +1,10 @@
//go:build gofuzz
// +build gofuzz
package httprule
func Fuzz(data []byte) int {
_, err := Parse(string(data))
if err != nil {
if _, err := Parse(string(data)); err != nil {
return 0
}
return 0

View File

@@ -1,6 +1,7 @@
package httprule
import (
"errors"
"fmt"
"strings"
)
@@ -164,9 +165,9 @@ func (p *parser) segment() (segment, error) {
v, err := p.variable()
if err != nil {
return nil, fmt.Errorf("segment neither wildcards, literal or variable: %v", err)
return nil, fmt.Errorf("segment neither wildcards, literal or variable: %w", err)
}
return v, err
return v, nil
}
func (p *parser) literal() (segment, error) {
@@ -191,7 +192,7 @@ func (p *parser) variable() (segment, error) {
if _, err := p.accept("="); err == nil {
segs, err = p.segments()
if err != nil {
return nil, fmt.Errorf("invalid segment in variable %q: %v", path, err)
return nil, fmt.Errorf("invalid segment in variable %q: %w", path, err)
}
} else {
segs = []segment{wildcard{}}
@@ -213,12 +214,12 @@ func (p *parser) fieldPath() (string, error) {
}
components := []string{c}
for {
if _, err = p.accept("."); err != nil {
if _, err := p.accept("."); err != nil {
return strings.Join(components, "."), nil
}
c, err := p.accept(typeIdent)
if err != nil {
return "", fmt.Errorf("invalid field path component: %v", err)
return "", fmt.Errorf("invalid field path component: %w", err)
}
components = append(components, c)
}
@@ -237,10 +238,8 @@ const (
typeEOF = termType("$")
)
const (
// eof is the terminal symbol which always appears at the end of token sequence.
eof = "\u0000"
)
// eof is the terminal symbol which always appears at the end of token sequence.
const eof = "\u0000"
// accept tries to accept a token in "p".
// This function consumes a token and returns it if it matches to the specified "term".
@@ -275,11 +274,12 @@ func (p *parser) accept(term termType) (string, error) {
// expectPChars determines if "t" consists of only pchars defined in RFC3986.
//
// https://www.ietf.org/rfc/rfc3986.txt, P.49
// pchar = unreserved / pct-encoded / sub-delims / ":" / "@"
// unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~"
// sub-delims = "!" / "$" / "&" / "'" / "(" / ")"
// / "*" / "+" / "," / ";" / "="
// pct-encoded = "%" HEXDIG HEXDIG
//
// pchar = unreserved / pct-encoded / sub-delims / ":" / "@"
// unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~"
// sub-delims = "!" / "$" / "&" / "'" / "(" / ")"
// / "*" / "+" / "," / ";" / "="
// pct-encoded = "%" HEXDIG HEXDIG
func expectPChars(t string) error {
const (
init = iota
@@ -333,7 +333,7 @@ func expectPChars(t string) error {
// expectIdent determines if "ident" is a valid identifier in .proto schema ([[:alpha:]_][[:alphanum:]_]*).
func expectIdent(ident string) error {
if ident == "" {
return fmt.Errorf("empty identifier")
return errors.New("empty identifier")
}
for pos, r := range ident {
switch {

View File

@@ -27,9 +27,9 @@ go_library(
"//internal/httprule",
"//utilities",
"@go_googleapis//google/api:httpbody_go_proto",
"@io_bazel_rules_go//proto/wkt:field_mask_go_proto",
"@org_golang_google_grpc//codes",
"@org_golang_google_grpc//grpclog",
"@org_golang_google_grpc//health/grpc_health_v1",
"@org_golang_google_grpc//metadata",
"@org_golang_google_grpc//status",
"@org_golang_google_protobuf//encoding/protojson",
@@ -37,6 +37,8 @@ go_library(
"@org_golang_google_protobuf//reflect/protoreflect",
"@org_golang_google_protobuf//reflect/protoregistry",
"@org_golang_google_protobuf//types/known/durationpb",
"@org_golang_google_protobuf//types/known/fieldmaskpb",
"@org_golang_google_protobuf//types/known/structpb",
"@org_golang_google_protobuf//types/known/timestamppb",
"@org_golang_google_protobuf//types/known/wrapperspb",
],
@@ -56,8 +58,10 @@ go_test(
"marshal_jsonpb_test.go",
"marshal_proto_test.go",
"marshaler_registry_test.go",
"mux_internal_test.go",
"mux_test.go",
"pattern_test.go",
"query_fuzz_test.go",
"query_test.go",
],
embed = [":runtime"],
@@ -69,8 +73,9 @@ go_test(
"@go_googleapis//google/api:httpbody_go_proto",
"@go_googleapis//google/rpc:errdetails_go_proto",
"@go_googleapis//google/rpc:status_go_proto",
"@io_bazel_rules_go//proto/wkt:field_mask_go_proto",
"@org_golang_google_grpc//:go_default_library",
"@org_golang_google_grpc//codes",
"@org_golang_google_grpc//health/grpc_health_v1",
"@org_golang_google_grpc//metadata",
"@org_golang_google_grpc//status",
"@org_golang_google_protobuf//encoding/protojson",
@@ -78,6 +83,7 @@ go_test(
"@org_golang_google_protobuf//testing/protocmp",
"@org_golang_google_protobuf//types/known/durationpb",
"@org_golang_google_protobuf//types/known/emptypb",
"@org_golang_google_protobuf//types/known/fieldmaskpb",
"@org_golang_google_protobuf//types/known/structpb",
"@org_golang_google_protobuf//types/known/timestamppb",
"@org_golang_google_protobuf//types/known/wrapperspb",

View File

@@ -13,6 +13,7 @@ import (
"time"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/grpclog"
"google.golang.org/grpc/metadata"
"google.golang.org/grpc/status"
)
@@ -35,11 +36,15 @@ const metadataHeaderBinarySuffix = "-Bin"
const xForwardedFor = "X-Forwarded-For"
const xForwardedHost = "X-Forwarded-Host"
var (
// DefaultContextTimeout is used for gRPC call context.WithTimeout whenever a Grpc-Timeout inbound
// header isn't present. If the value is 0 the sent `context` will not have a timeout.
DefaultContextTimeout = 0 * time.Second
)
// DefaultContextTimeout is used for gRPC call context.WithTimeout whenever a Grpc-Timeout inbound
// header isn't present. If the value is 0 the sent `context` will not have a timeout.
var DefaultContextTimeout = 0 * time.Second
// malformedHTTPHeaders lists the headers that the gRPC server may reject outright as malformed.
// See https://github.com/grpc/grpc-go/pull/4803#issuecomment-986093310 for more context.
var malformedHTTPHeaders = map[string]struct{}{
"connection": {},
}
type (
rpcMethodKey struct{}
@@ -95,12 +100,43 @@ func AnnotateIncomingContext(ctx context.Context, mux *ServeMux, req *http.Reque
return metadata.NewIncomingContext(ctx, md), nil
}
func isValidGRPCMetadataKey(key string) bool {
// Must be a valid gRPC "Header-Name" as defined here:
// https://github.com/grpc/grpc/blob/4b05dc88b724214d0c725c8e7442cbc7a61b1374/doc/PROTOCOL-HTTP2.md
// This means 0-9 a-z _ - .
// Only lowercase letters are valid in the wire protocol, but the client library will normalize
// uppercase ASCII to lowercase, so uppercase ASCII is also acceptable.
bytes := []byte(key) // gRPC validates strings on the byte level, not Unicode.
for _, ch := range bytes {
validLowercaseLetter := ch >= 'a' && ch <= 'z'
validUppercaseLetter := ch >= 'A' && ch <= 'Z'
validDigit := ch >= '0' && ch <= '9'
validOther := ch == '.' || ch == '-' || ch == '_'
if !validLowercaseLetter && !validUppercaseLetter && !validDigit && !validOther {
return false
}
}
return true
}
func isValidGRPCMetadataTextValue(textValue string) bool {
// Must be a valid gRPC "ASCII-Value" as defined here:
// https://github.com/grpc/grpc/blob/4b05dc88b724214d0c725c8e7442cbc7a61b1374/doc/PROTOCOL-HTTP2.md
// This means printable ASCII (including/plus spaces); 0x20 to 0x7E inclusive.
bytes := []byte(textValue) // gRPC validates strings on the byte level, not Unicode.
for _, ch := range bytes {
if ch < 0x20 || ch > 0x7E {
return false
}
}
return true
}
func annotateContext(ctx context.Context, mux *ServeMux, req *http.Request, rpcMethodName string, options ...AnnotateContextOption) (context.Context, metadata.MD, error) {
ctx = withRPCMethod(ctx, rpcMethodName)
for _, o := range options {
ctx = o(ctx)
}
var pairs []string
timeout := DefaultContextTimeout
if tm := req.Header.Get(metadataGrpcTimeout); tm != "" {
var err error
@@ -109,7 +145,7 @@ func annotateContext(ctx context.Context, mux *ServeMux, req *http.Request, rpcM
return nil, nil, status.Errorf(codes.InvalidArgument, "invalid grpc-timeout: %s", tm)
}
}
var pairs []string
for key, vals := range req.Header {
key = textproto.CanonicalMIMEHeaderKey(key)
for _, val := range vals {
@@ -118,6 +154,10 @@ func annotateContext(ctx context.Context, mux *ServeMux, req *http.Request, rpcM
pairs = append(pairs, "authorization", val)
}
if h, ok := mux.incomingHeaderMatcher(key); ok {
if !isValidGRPCMetadataKey(h) {
grpclog.Errorf("HTTP header name %q is not valid as gRPC metadata key; skipping", h)
continue
}
// Handles "-bin" metadata in grpc, since grpc will do another base64
// encode before sending to server, we need to decode it first.
if strings.HasSuffix(key, metadataHeaderBinarySuffix) {
@@ -127,6 +167,9 @@ func annotateContext(ctx context.Context, mux *ServeMux, req *http.Request, rpcM
}
val = string(b)
} else if !isValidGRPCMetadataTextValue(val) {
grpclog.Errorf("Value of HTTP header %q contains non-ASCII value (not valid as gRPC metadata): skipping", h)
continue
}
pairs = append(pairs, h, val)
}
@@ -172,11 +215,17 @@ type serverMetadataKey struct{}
// NewServerMetadataContext creates a new context with ServerMetadata
func NewServerMetadataContext(ctx context.Context, md ServerMetadata) context.Context {
if ctx == nil {
ctx = context.Background()
}
return context.WithValue(ctx, serverMetadataKey{}, md)
}
// ServerMetadataFromContext returns the ServerMetadata in ctx
func ServerMetadataFromContext(ctx context.Context) (md ServerMetadata, ok bool) {
if ctx == nil {
return md, false
}
md, ok = ctx.Value(serverMetadataKey{}).(ServerMetadata)
return
}
@@ -269,8 +318,8 @@ func timeoutUnitToDuration(u uint8) (d time.Duration, ok bool) {
case 'n':
return time.Nanosecond, true
default:
return
}
return
}
// isPermanentHTTPHeader checks whether hdr belongs to the list of
@@ -308,6 +357,13 @@ func isPermanentHTTPHeader(hdr string) bool {
return false
}
// isMalformedHTTPHeader checks whether header belongs to the list of
// "malformed headers" and would be rejected by the gRPC server.
func isMalformedHTTPHeader(header string) bool {
_, isMalformed := malformedHTTPHeaders[strings.ToLower(header)]
return isMalformed
}
// RPCMethod returns the method string for the server context. The returned
// string is in the format of "/package.service/method".
func RPCMethod(ctx context.Context) (string, bool) {

View File

@@ -37,7 +37,7 @@ func BoolSlice(val, sep string) ([]bool, error) {
for i, v := range s {
value, err := Bool(v)
if err != nil {
return values, err
return nil, err
}
values[i] = value
}
@@ -57,7 +57,7 @@ func Float64Slice(val, sep string) ([]float64, error) {
for i, v := range s {
value, err := Float64(v)
if err != nil {
return values, err
return nil, err
}
values[i] = value
}
@@ -81,7 +81,7 @@ func Float32Slice(val, sep string) ([]float32, error) {
for i, v := range s {
value, err := Float32(v)
if err != nil {
return values, err
return nil, err
}
values[i] = value
}
@@ -101,7 +101,7 @@ func Int64Slice(val, sep string) ([]int64, error) {
for i, v := range s {
value, err := Int64(v)
if err != nil {
return values, err
return nil, err
}
values[i] = value
}
@@ -125,7 +125,7 @@ func Int32Slice(val, sep string) ([]int32, error) {
for i, v := range s {
value, err := Int32(v)
if err != nil {
return values, err
return nil, err
}
values[i] = value
}
@@ -145,7 +145,7 @@ func Uint64Slice(val, sep string) ([]uint64, error) {
for i, v := range s {
value, err := Uint64(v)
if err != nil {
return values, err
return nil, err
}
values[i] = value
}
@@ -169,7 +169,7 @@ func Uint32Slice(val, sep string) ([]uint32, error) {
for i, v := range s {
value, err := Uint32(v)
if err != nil {
return values, err
return nil, err
}
values[i] = value
}
@@ -197,7 +197,7 @@ func BytesSlice(val, sep string) ([][]byte, error) {
for i, v := range s {
value, err := Bytes(v)
if err != nil {
return values, err
return nil, err
}
values[i] = value
}
@@ -209,8 +209,7 @@ func Timestamp(val string) (*timestamppb.Timestamp, error) {
var r timestamppb.Timestamp
val = strconv.Quote(strings.Trim(val, `"`))
unmarshaler := &protojson.UnmarshalOptions{}
err := unmarshaler.Unmarshal([]byte(val), &r)
if err != nil {
if err := unmarshaler.Unmarshal([]byte(val), &r); err != nil {
return nil, err
}
return &r, nil
@@ -221,8 +220,7 @@ func Duration(val string) (*durationpb.Duration, error) {
var r durationpb.Duration
val = strconv.Quote(strings.Trim(val, `"`))
unmarshaler := &protojson.UnmarshalOptions{}
err := unmarshaler.Unmarshal([]byte(val), &r)
if err != nil {
if err := unmarshaler.Unmarshal([]byte(val), &r); err != nil {
return nil, err
}
return &r, nil
@@ -257,66 +255,64 @@ func EnumSlice(val, sep string, enumValMap map[string]int32) ([]int32, error) {
for i, v := range s {
value, err := Enum(v, enumValMap)
if err != nil {
return values, err
return nil, err
}
values[i] = value
}
return values, nil
}
/*
Support fot google.protobuf.wrappers on top of primitive types
*/
// Support for google.protobuf.wrappers on top of primitive types
// StringValue well-known type support as wrapper around string type
func StringValue(val string) (*wrapperspb.StringValue, error) {
return &wrapperspb.StringValue{Value: val}, nil
return wrapperspb.String(val), nil
}
// FloatValue well-known type support as wrapper around float32 type
func FloatValue(val string) (*wrapperspb.FloatValue, error) {
parsedVal, err := Float32(val)
return &wrapperspb.FloatValue{Value: parsedVal}, err
return wrapperspb.Float(parsedVal), err
}
// DoubleValue well-known type support as wrapper around float64 type
func DoubleValue(val string) (*wrapperspb.DoubleValue, error) {
parsedVal, err := Float64(val)
return &wrapperspb.DoubleValue{Value: parsedVal}, err
return wrapperspb.Double(parsedVal), err
}
// BoolValue well-known type support as wrapper around bool type
func BoolValue(val string) (*wrapperspb.BoolValue, error) {
parsedVal, err := Bool(val)
return &wrapperspb.BoolValue{Value: parsedVal}, err
return wrapperspb.Bool(parsedVal), err
}
// Int32Value well-known type support as wrapper around int32 type
func Int32Value(val string) (*wrapperspb.Int32Value, error) {
parsedVal, err := Int32(val)
return &wrapperspb.Int32Value{Value: parsedVal}, err
return wrapperspb.Int32(parsedVal), err
}
// UInt32Value well-known type support as wrapper around uint32 type
func UInt32Value(val string) (*wrapperspb.UInt32Value, error) {
parsedVal, err := Uint32(val)
return &wrapperspb.UInt32Value{Value: parsedVal}, err
return wrapperspb.UInt32(parsedVal), err
}
// Int64Value well-known type support as wrapper around int64 type
func Int64Value(val string) (*wrapperspb.Int64Value, error) {
parsedVal, err := Int64(val)
return &wrapperspb.Int64Value{Value: parsedVal}, err
return wrapperspb.Int64(parsedVal), err
}
// UInt64Value well-known type support as wrapper around uint64 type
func UInt64Value(val string) (*wrapperspb.UInt64Value, error) {
parsedVal, err := Uint64(val)
return &wrapperspb.UInt64Value{Value: parsedVal}, err
return wrapperspb.UInt64(parsedVal), err
}
// BytesValue well-known type support as wrapper around bytes[] type
func BytesValue(val string) (*wrapperspb.BytesValue, error) {
parsedVal, err := Bytes(val)
return &wrapperspb.BytesValue{Value: parsedVal}, err
return wrapperspb.Bytes(parsedVal), err
}

View File

@@ -38,7 +38,7 @@ func HTTPStatusFromCode(code codes.Code) int {
case codes.OK:
return http.StatusOK
case codes.Canceled:
return http.StatusRequestTimeout
return 499
case codes.Unknown:
return http.StatusInternalServerError
case codes.InvalidArgument:
@@ -70,10 +70,10 @@ func HTTPStatusFromCode(code codes.Code) int {
return http.StatusServiceUnavailable
case codes.DataLoss:
return http.StatusInternalServerError
default:
grpclog.Infof("Unknown gRPC error code: %v", code)
return http.StatusInternalServerError
}
grpclog.Infof("Unknown gRPC error code: %v", code)
return http.StatusInternalServerError
}
// HTTPError uses the mux-configured error handler.
@@ -162,10 +162,11 @@ func DefaultStreamErrorHandler(_ context.Context, err error) *status.Status {
// DefaultRoutingErrorHandler is our default handler for routing errors.
// By default http error codes mapped on the following error codes:
// NotFound -> grpc.NotFound
// StatusBadRequest -> grpc.InvalidArgument
// MethodNotAllowed -> grpc.Unimplemented
// Other -> grpc.Internal, method is not expecting to be called for anything else
//
// NotFound -> grpc.NotFound
// StatusBadRequest -> grpc.InvalidArgument
// MethodNotAllowed -> grpc.Unimplemented
// Other -> grpc.Internal, method is not expecting to be called for anything else
func DefaultRoutingErrorHandler(ctx context.Context, mux *ServeMux, marshaler Marshaler, w http.ResponseWriter, r *http.Request, httpStatus int) {
sterr := status.Error(codes.Internal, "Unexpected routing error")
switch httpStatus {

View File

@@ -2,13 +2,14 @@ package runtime
import (
"encoding/json"
"errors"
"fmt"
"io"
"sort"
"google.golang.org/genproto/protobuf/field_mask"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/reflect/protoreflect"
field_mask "google.golang.org/protobuf/types/known/fieldmaskpb"
)
func getFieldByName(fields protoreflect.FieldDescriptors, name string) protoreflect.FieldDescriptor {
@@ -44,7 +45,7 @@ func FieldMaskFromRequestBody(r io.Reader, msg proto.Message) (*field_mask.Field
// if the item is an object, then enqueue all of its children
for k, v := range m {
if item.msg == nil {
return nil, fmt.Errorf("JSON structure did not match request type")
return nil, errors.New("JSON structure did not match request type")
}
fd := getFieldByName(item.msg.Descriptor().Fields(), k)
@@ -53,7 +54,7 @@ func FieldMaskFromRequestBody(r io.Reader, msg proto.Message) (*field_mask.Field
}
if isDynamicProtoMessage(fd.Message()) {
for _, p := range buildPathsBlindly(k, v) {
for _, p := range buildPathsBlindly(string(fd.FullName().Name()), v) {
newPath := p
if item.path != "" {
newPath = item.path + "." + newPath
@@ -63,7 +64,7 @@ func FieldMaskFromRequestBody(r io.Reader, msg proto.Message) (*field_mask.Field
continue
}
if isProtobufAnyMessage(fd.Message()) {
if isProtobufAnyMessage(fd.Message()) && !fd.IsList() {
_, hasTypeField := v.(map[string]interface{})["@type"]
if hasTypeField {
queue = append(queue, fieldMaskPathItem{path: k})

View File

@@ -52,11 +52,11 @@ func ForwardResponseStream(ctx context.Context, mux *ServeMux, marshaler Marshal
return
}
if err != nil {
handleForwardResponseStreamError(ctx, wroteHeader, marshaler, w, req, mux, err)
handleForwardResponseStreamError(ctx, wroteHeader, marshaler, w, req, mux, err, delimiter)
return
}
if err := handleForwardResponseOptions(ctx, w, resp, opts); err != nil {
handleForwardResponseStreamError(ctx, wroteHeader, marshaler, w, req, mux, err)
handleForwardResponseStreamError(ctx, wroteHeader, marshaler, w, req, mux, err, delimiter)
return
}
@@ -82,15 +82,15 @@ func ForwardResponseStream(ctx context.Context, mux *ServeMux, marshaler Marshal
if err != nil {
grpclog.Infof("Failed to marshal response chunk: %v", err)
handleForwardResponseStreamError(ctx, wroteHeader, marshaler, w, req, mux, err)
handleForwardResponseStreamError(ctx, wroteHeader, marshaler, w, req, mux, err, delimiter)
return
}
if _, err = w.Write(buf); err != nil {
if _, err := w.Write(buf); err != nil {
grpclog.Infof("Failed to send response chunk: %v", err)
return
}
wroteHeader = true
if _, err = w.Write(delimiter); err != nil {
if _, err := w.Write(delimiter); err != nil {
grpclog.Infof("Failed to send delimiter chunk: %v", err)
return
}
@@ -200,20 +200,24 @@ func handleForwardResponseOptions(ctx context.Context, w http.ResponseWriter, re
return nil
}
func handleForwardResponseStreamError(ctx context.Context, wroteHeader bool, marshaler Marshaler, w http.ResponseWriter, req *http.Request, mux *ServeMux, err error) {
func handleForwardResponseStreamError(ctx context.Context, wroteHeader bool, marshaler Marshaler, w http.ResponseWriter, req *http.Request, mux *ServeMux, err error, delimiter []byte) {
st := mux.streamErrorHandler(ctx, err)
msg := errorChunk(st)
if !wroteHeader {
w.Header().Set("Content-Type", marshaler.ContentType(msg))
w.WriteHeader(HTTPStatusFromCode(st.Code()))
}
buf, merr := marshaler.Marshal(msg)
if merr != nil {
grpclog.Infof("Failed to marshal an error: %v", merr)
buf, err := marshaler.Marshal(msg)
if err != nil {
grpclog.Infof("Failed to marshal an error: %v", err)
return
}
if _, werr := w.Write(buf); werr != nil {
grpclog.Infof("Failed to notify error to client: %v", werr)
if _, err := w.Write(buf); err != nil {
grpclog.Infof("Failed to notify error to client: %v", err)
return
}
if _, err := w.Write(delimiter); err != nil {
grpclog.Infof("Failed to send delimiter chunk: %v", err)
return
}
}

View File

@@ -92,23 +92,20 @@ func (j *JSONPb) marshalNonProtoField(v interface{}) ([]byte, error) {
if rv.Type().Elem().Implements(protoMessageType) {
var buf bytes.Buffer
err := buf.WriteByte('[')
if err != nil {
if err := buf.WriteByte('['); err != nil {
return nil, err
}
for i := 0; i < rv.Len(); i++ {
if i != 0 {
err = buf.WriteByte(',')
if err != nil {
if err := buf.WriteByte(','); err != nil {
return nil, err
}
}
if err = j.marshalTo(&buf, rv.Index(i).Interface().(proto.Message)); err != nil {
if err := j.marshalTo(&buf, rv.Index(i).Interface().(proto.Message)); err != nil {
return nil, err
}
}
err = buf.WriteByte(']')
if err != nil {
if err := buf.WriteByte(']'); err != nil {
return nil, err
}
@@ -117,17 +114,16 @@ func (j *JSONPb) marshalNonProtoField(v interface{}) ([]byte, error) {
if rv.Type().Elem().Implements(typeProtoEnum) {
var buf bytes.Buffer
err := buf.WriteByte('[')
if err != nil {
if err := buf.WriteByte('['); err != nil {
return nil, err
}
for i := 0; i < rv.Len(); i++ {
if i != 0 {
err = buf.WriteByte(',')
if err != nil {
if err := buf.WriteByte(','); err != nil {
return nil, err
}
}
var err error
if j.UseEnumNumbers {
_, err = buf.WriteString(strconv.FormatInt(rv.Index(i).Int(), 10))
} else {
@@ -137,8 +133,7 @@ func (j *JSONPb) marshalNonProtoField(v interface{}) ([]byte, error) {
return nil, err
}
}
err = buf.WriteByte(']')
if err != nil {
if err := buf.WriteByte(']'); err != nil {
return nil, err
}
@@ -219,8 +214,7 @@ func decodeJSONPb(d *json.Decoder, unmarshaler protojson.UnmarshalOptions, v int
// Decode into bytes for marshalling
var b json.RawMessage
err := d.Decode(&b)
if err != nil {
if err := d.Decode(&b); err != nil {
return err
}
@@ -239,8 +233,7 @@ func decodeNonProtoField(d *json.Decoder, unmarshaler protojson.UnmarshalOptions
if rv.Type().ConvertibleTo(typeProtoMessage) {
// Decode into bytes for marshalling
var b json.RawMessage
err := d.Decode(&b)
if err != nil {
if err := d.Decode(&b); err != nil {
return err
}
@@ -280,6 +273,17 @@ func decodeNonProtoField(d *json.Decoder, unmarshaler protojson.UnmarshalOptions
return nil
}
if rv.Kind() == reflect.Slice {
if rv.Type().Elem().Kind() == reflect.Uint8 {
var sl []byte
if err := d.Decode(&sl); err != nil {
return err
}
if sl != nil {
rv.SetBytes(sl)
}
return nil
}
var sl []json.RawMessage
if err := d.Decode(&sl); err != nil {
return err

View File

@@ -1,10 +1,8 @@
package runtime
import (
"io"
"errors"
"io/ioutil"
"io"
"google.golang.org/protobuf/proto"
)
@@ -38,7 +36,7 @@ func (*ProtoMarshaller) Unmarshal(data []byte, value interface{}) error {
// NewDecoder returns a Decoder which reads proto stream from "reader".
func (marshaller *ProtoMarshaller) NewDecoder(reader io.Reader) Decoder {
return DecoderFunc(func(value interface{}) error {
buffer, err := ioutil.ReadAll(reader)
buffer, err := io.ReadAll(reader)
if err != nil {
return err
}
@@ -53,8 +51,7 @@ func (marshaller *ProtoMarshaller) NewEncoder(writer io.Writer) Encoder {
if err != nil {
return err
}
_, err = writer.Write(buffer)
if err != nil {
if _, err := writer.Write(buffer); err != nil {
return err
}

View File

@@ -6,10 +6,13 @@ import (
"fmt"
"net/http"
"net/textproto"
"regexp"
"strings"
"github.com/grpc-ecosystem/grpc-gateway/v2/internal/httprule"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/grpclog"
"google.golang.org/grpc/health/grpc_health_v1"
"google.golang.org/grpc/metadata"
"google.golang.org/grpc/status"
"google.golang.org/protobuf/proto"
@@ -23,15 +26,15 @@ const (
// path string before doing any routing.
UnescapingModeLegacy UnescapingMode = iota
// EscapingTypeExceptReserved unescapes all path parameters except RFC 6570
// UnescapingModeAllExceptReserved unescapes all path parameters except RFC 6570
// reserved characters.
UnescapingModeAllExceptReserved
// EscapingTypeExceptSlash unescapes URL path parameters except path
// seperators, which will be left as "%2F".
// UnescapingModeAllExceptSlash unescapes URL path parameters except path
// separators, which will be left as "%2F".
UnescapingModeAllExceptSlash
// URL path parameters will be fully decoded.
// UnescapingModeAllCharacters unescapes all URL path parameters.
UnescapingModeAllCharacters
// UnescapingModeDefault is the default escaping type.
@@ -40,6 +43,8 @@ const (
UnescapingModeDefault = UnescapingModeLegacy
)
var encodedPathSplitter = regexp.MustCompile("(/|%2F)")
// A HandlerFunc handles a specific pair of path pattern and HTTP method.
type HandlerFunc func(w http.ResponseWriter, r *http.Request, pathParams map[string]string)
@@ -75,7 +80,7 @@ func WithForwardResponseOption(forwardResponseOption func(context.Context, http.
}
}
// WithEscapingType sets the escaping type. See the definitions of UnescapingMode
// WithUnescapingMode sets the escaping type. See the definitions of UnescapingMode
// for more information.
func WithUnescapingMode(mode UnescapingMode) ServeMuxOption {
return func(serveMux *ServeMux) {
@@ -96,13 +101,14 @@ func SetQueryParameterParser(queryParameterParser QueryParameterParser) ServeMux
type HeaderMatcherFunc func(string) (string, bool)
// DefaultHeaderMatcher is used to pass http request headers to/from gRPC context. This adds permanent HTTP header
// keys (as specified by the IANA) to gRPC context with grpcgateway- prefix. HTTP headers that start with
// 'Grpc-Metadata-' are mapped to gRPC metadata after removing prefix 'Grpc-Metadata-'.
// keys (as specified by the IANA, e.g: Accept, Cookie, Host) to the gRPC metadata with the grpcgateway- prefix. If you want to know which headers are considered permanent, you can view the isPermanentHTTPHeader function.
// HTTP headers that start with 'Grpc-Metadata-' are mapped to gRPC metadata after removing the prefix 'Grpc-Metadata-'.
// Other headers are not added to the gRPC metadata.
func DefaultHeaderMatcher(key string) (string, bool) {
key = textproto.CanonicalMIMEHeaderKey(key)
if isPermanentHTTPHeader(key) {
switch key = textproto.CanonicalMIMEHeaderKey(key); {
case isPermanentHTTPHeader(key):
return MetadataPrefix + key, true
} else if strings.HasPrefix(key, MetadataHeaderPrefix) {
case strings.HasPrefix(key, MetadataHeaderPrefix):
return key[len(MetadataHeaderPrefix):], true
}
return "", false
@@ -113,11 +119,30 @@ func DefaultHeaderMatcher(key string) (string, bool) {
// This matcher will be called with each header in http.Request. If matcher returns true, that header will be
// passed to gRPC context. To transform the header before passing to gRPC context, matcher should return modified header.
func WithIncomingHeaderMatcher(fn HeaderMatcherFunc) ServeMuxOption {
for _, header := range fn.matchedMalformedHeaders() {
grpclog.Warningf("The configured forwarding filter would allow %q to be sent to the gRPC server, which will likely cause errors. See https://github.com/grpc/grpc-go/pull/4803#issuecomment-986093310 for more information.", header)
}
return func(mux *ServeMux) {
mux.incomingHeaderMatcher = fn
}
}
// matchedMalformedHeaders returns the malformed headers that would be forwarded to gRPC server.
func (fn HeaderMatcherFunc) matchedMalformedHeaders() []string {
if fn == nil {
return nil
}
headers := make([]string, 0)
for header := range malformedHTTPHeaders {
out, accept := fn(header)
if accept && isMalformedHTTPHeader(out) {
headers = append(headers, out)
}
}
return headers
}
// WithOutgoingHeaderMatcher returns a ServeMuxOption representing a headerMatcher for outgoing response from gateway.
//
// This matcher will be called with each header in response header metadata. If matcher returns true, that header will be
@@ -179,6 +204,56 @@ func WithDisablePathLengthFallback() ServeMuxOption {
}
}
// WithHealthEndpointAt returns a ServeMuxOption that will add an endpoint to the created ServeMux at the path specified by endpointPath.
// When called the handler will forward the request to the upstream grpc service health check (defined in the
// gRPC Health Checking Protocol).
//
// See here https://grpc-ecosystem.github.io/grpc-gateway/docs/operations/health_check/ for more information on how
// to setup the protocol in the grpc server.
//
// If you define a service as query parameter, this will also be forwarded as service in the HealthCheckRequest.
func WithHealthEndpointAt(healthCheckClient grpc_health_v1.HealthClient, endpointPath string) ServeMuxOption {
return func(s *ServeMux) {
// error can be ignored since pattern is definitely valid
_ = s.HandlePath(
http.MethodGet, endpointPath, func(w http.ResponseWriter, r *http.Request, _ map[string]string,
) {
_, outboundMarshaler := MarshalerForRequest(s, r)
resp, err := healthCheckClient.Check(r.Context(), &grpc_health_v1.HealthCheckRequest{
Service: r.URL.Query().Get("service"),
})
if err != nil {
s.errorHandler(r.Context(), s, outboundMarshaler, w, r, err)
return
}
w.Header().Set("Content-Type", "application/json")
if resp.GetStatus() != grpc_health_v1.HealthCheckResponse_SERVING {
switch resp.GetStatus() {
case grpc_health_v1.HealthCheckResponse_NOT_SERVING, grpc_health_v1.HealthCheckResponse_UNKNOWN:
err = status.Error(codes.Unavailable, resp.String())
case grpc_health_v1.HealthCheckResponse_SERVICE_UNKNOWN:
err = status.Error(codes.NotFound, resp.String())
}
s.errorHandler(r.Context(), s, outboundMarshaler, w, r, err)
return
}
_ = outboundMarshaler.NewEncoder(w).Encode(resp)
})
}
}
// WithHealthzEndpoint returns a ServeMuxOption that will add a /healthz endpoint to the created ServeMux.
//
// See WithHealthEndpointAt for the general implementation.
func WithHealthzEndpoint(healthCheckClient grpc_health_v1.HealthClient) ServeMuxOption {
return WithHealthEndpointAt(healthCheckClient, "/healthz")
}
// NewServeMux returns a new ServeMux whose internal mapping is empty.
func NewServeMux(opts ...ServeMuxOption) *ServeMux {
serveMux := &ServeMux{
@@ -229,7 +304,7 @@ func (s *ServeMux) HandlePath(meth string, pathPattern string, h HandlerFunc) er
return nil
}
// ServeHTTP dispatches the request to the first handler whose pattern matches to r.Method and r.Path.
// ServeHTTP dispatches the request to the first handler whose pattern matches to r.Method and r.URL.Path.
func (s *ServeMux) ServeHTTP(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
@@ -245,8 +320,6 @@ func (s *ServeMux) ServeHTTP(w http.ResponseWriter, r *http.Request) {
path = r.URL.RawPath
}
components := strings.Split(path[1:], "/")
if override := r.Header.Get("X-HTTP-Method-Override"); override != "" && s.isPathLengthFallback(r) {
r.Method = strings.ToUpper(override)
if err := r.ParseForm(); err != nil {
@@ -257,8 +330,18 @@ func (s *ServeMux) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
}
// Verb out here is to memoize for the fallback case below
var verb string
var pathComponents []string
// since in UnescapeModeLegacy, the URL will already have been fully unescaped, if we also split on "%2F"
// in this escaping mode we would be double unescaping but in UnescapingModeAllCharacters, we still do as the
// path is the RawPath (i.e. unescaped). That does mean that the behavior of this function will change its default
// behavior when the UnescapingModeDefault gets changed from UnescapingModeLegacy to UnescapingModeAllExceptReserved
if s.unescapingMode == UnescapingModeAllCharacters {
pathComponents = encodedPathSplitter.Split(path[1:], -1)
} else {
pathComponents = strings.Split(path[1:], "/")
}
lastPathComponent := pathComponents[len(pathComponents)-1]
for _, h := range s.handlers[r.Method] {
// If the pattern has a verb, explicitly look for a suffix in the last
@@ -269,23 +352,28 @@ func (s *ServeMux) ServeHTTP(w http.ResponseWriter, r *http.Request) {
// parser because we know what verb we're looking for, however, there
// are still some cases that the parser itself cannot disambiguate. See
// the comment there if interested.
var verb string
patVerb := h.pat.Verb()
l := len(components)
lastComponent := components[l-1]
var idx int = -1
if patVerb != "" && strings.HasSuffix(lastComponent, ":"+patVerb) {
idx = len(lastComponent) - len(patVerb) - 1
idx := -1
if patVerb != "" && strings.HasSuffix(lastPathComponent, ":"+patVerb) {
idx = len(lastPathComponent) - len(patVerb) - 1
}
if idx == 0 {
_, outboundMarshaler := MarshalerForRequest(s, r)
s.routingErrorHandler(ctx, s, outboundMarshaler, w, r, http.StatusNotFound)
return
}
comps := make([]string, len(pathComponents))
copy(comps, pathComponents)
if idx > 0 {
components[l-1], verb = lastComponent[:idx], lastComponent[idx+1:]
comps[len(comps)-1], verb = lastPathComponent[:idx], lastPathComponent[idx+1:]
}
pathParams, err := h.pat.MatchAndEscape(components, verb, s.unescapingMode)
pathParams, err := h.pat.MatchAndEscape(comps, verb, s.unescapingMode)
if err != nil {
var mse MalformedSequenceError
if ok := errors.As(err, &mse); ok {
@@ -301,14 +389,33 @@ func (s *ServeMux) ServeHTTP(w http.ResponseWriter, r *http.Request) {
return
}
// lookup other methods to handle fallback from GET to POST and
// to determine if it is NotImplemented or NotFound.
// if no handler has found for the request, lookup for other methods
// to handle POST -> GET fallback if the request is subject to path
// length fallback.
// Note we are not eagerly checking the request here as we want to return the
// right HTTP status code, and we need to process the fallback candidates in
// order to do that.
for m, handlers := range s.handlers {
if m == r.Method {
continue
}
for _, h := range handlers {
pathParams, err := h.pat.MatchAndEscape(components, verb, s.unescapingMode)
var verb string
patVerb := h.pat.Verb()
idx := -1
if patVerb != "" && strings.HasSuffix(lastPathComponent, ":"+patVerb) {
idx = len(lastPathComponent) - len(patVerb) - 1
}
comps := make([]string, len(pathComponents))
copy(comps, pathComponents)
if idx > 0 {
comps[len(comps)-1], verb = lastPathComponent[:idx], lastPathComponent[idx+1:]
}
pathParams, err := h.pat.MatchAndEscape(comps, verb, s.unescapingMode)
if err != nil {
var mse MalformedSequenceError
if ok := errors.As(err, &mse); ok {
@@ -320,8 +427,11 @@ func (s *ServeMux) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
continue
}
// X-HTTP-Method-Override is optional. Always allow fallback to POST.
if s.isPathLengthFallback(r) {
// Also, only consider POST -> GET fallbacks, and avoid falling back to
// potentially dangerous operations like DELETE.
if s.isPathLengthFallback(r) && m == http.MethodGet {
if err := r.ParseForm(); err != nil {
_, outboundMarshaler := MarshalerForRequest(s, r)
sterr := status.Error(codes.InvalidArgument, err.Error())

View File

@@ -15,8 +15,6 @@ var (
ErrNotMatch = errors.New("not match to the path pattern")
// ErrInvalidPattern indicates that the given definition of Pattern is not valid.
ErrInvalidPattern = errors.New("invalid pattern")
// ErrMalformedSequence indicates that an escape sequence was malformed.
ErrMalformedSequence = errors.New("malformed escape sequence")
)
type MalformedSequenceError string

View File

@@ -1,7 +1,6 @@
package runtime
import (
"encoding/base64"
"errors"
"fmt"
"net/url"
@@ -11,19 +10,21 @@ import (
"time"
"github.com/grpc-ecosystem/grpc-gateway/v2/utilities"
"google.golang.org/genproto/protobuf/field_mask"
"google.golang.org/grpc/grpclog"
"google.golang.org/protobuf/encoding/protojson"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/reflect/protoreflect"
"google.golang.org/protobuf/reflect/protoregistry"
"google.golang.org/protobuf/types/known/durationpb"
field_mask "google.golang.org/protobuf/types/known/fieldmaskpb"
"google.golang.org/protobuf/types/known/structpb"
"google.golang.org/protobuf/types/known/timestamppb"
"google.golang.org/protobuf/types/known/wrapperspb"
)
var valuesKeyRegexp = regexp.MustCompile(`^(.*)\[(.*)\]$`)
var currentQueryParser QueryParameterParser = &defaultQueryParser{}
var currentQueryParser QueryParameterParser = &DefaultQueryParser{}
// QueryParameterParser defines interface for all query parameter parsers
type QueryParameterParser interface {
@@ -36,14 +37,17 @@ func PopulateQueryParameters(msg proto.Message, values url.Values, filter *utili
return currentQueryParser.Parse(msg, values, filter)
}
type defaultQueryParser struct{}
// DefaultQueryParser is a QueryParameterParser which implements the default
// query parameters parsing behavior.
//
// See https://github.com/grpc-ecosystem/grpc-gateway/issues/2632 for more context.
type DefaultQueryParser struct{}
// Parse populates "values" into "msg".
// A value is ignored if its key starts with one of the elements in "filter".
func (*defaultQueryParser) Parse(msg proto.Message, values url.Values, filter *utilities.DoubleArray) error {
func (*DefaultQueryParser) Parse(msg proto.Message, values url.Values, filter *utilities.DoubleArray) error {
for key, values := range values {
match := valuesKeyRegexp.FindStringSubmatch(key)
if len(match) == 3 {
if match := valuesKeyRegexp.FindStringSubmatch(key); len(match) == 3 {
key = match[1]
values = append([]string{match[2]}, values...)
}
@@ -175,10 +179,10 @@ func parseField(fieldDescriptor protoreflect.FieldDescriptor, value string) (pro
return protoreflect.ValueOfBool(v), nil
case protoreflect.EnumKind:
enum, err := protoregistry.GlobalTypes.FindEnumByName(fieldDescriptor.Enum().FullName())
switch {
case errors.Is(err, protoregistry.NotFound):
return protoreflect.Value{}, fmt.Errorf("enum %q is not registered", fieldDescriptor.Enum().FullName())
case err != nil:
if err != nil {
if errors.Is(err, protoregistry.NotFound) {
return protoreflect.Value{}, fmt.Errorf("enum %q is not registered", fieldDescriptor.Enum().FullName())
}
return protoreflect.Value{}, fmt.Errorf("failed to look up enum: %w", err)
}
// Look for enum by name
@@ -189,8 +193,7 @@ func parseField(fieldDescriptor protoreflect.FieldDescriptor, value string) (pro
return protoreflect.Value{}, fmt.Errorf("%q is not a valid value", value)
}
// Look for enum by number
v = enum.Descriptor().Values().ByNumber(protoreflect.EnumNumber(i))
if v == nil {
if v = enum.Descriptor().Values().ByNumber(protoreflect.EnumNumber(i)); v == nil {
return protoreflect.Value{}, fmt.Errorf("%q is not a valid value", value)
}
}
@@ -234,7 +237,7 @@ func parseField(fieldDescriptor protoreflect.FieldDescriptor, value string) (pro
case protoreflect.StringKind:
return protoreflect.ValueOfString(value), nil
case protoreflect.BytesKind:
v, err := base64.URLEncoding.DecodeString(value)
v, err := Bytes(value)
if err != nil {
return protoreflect.Value{}, err
}
@@ -250,18 +253,12 @@ func parseMessage(msgDescriptor protoreflect.MessageDescriptor, value string) (p
var msg proto.Message
switch msgDescriptor.FullName() {
case "google.protobuf.Timestamp":
if value == "null" {
break
}
t, err := time.Parse(time.RFC3339Nano, value)
if err != nil {
return protoreflect.Value{}, err
}
msg = timestamppb.New(t)
case "google.protobuf.Duration":
if value == "null" {
break
}
d, err := time.ParseDuration(value)
if err != nil {
return protoreflect.Value{}, err
@@ -272,55 +269,67 @@ func parseMessage(msgDescriptor protoreflect.MessageDescriptor, value string) (p
if err != nil {
return protoreflect.Value{}, err
}
msg = &wrapperspb.DoubleValue{Value: v}
msg = wrapperspb.Double(v)
case "google.protobuf.FloatValue":
v, err := strconv.ParseFloat(value, 32)
if err != nil {
return protoreflect.Value{}, err
}
msg = &wrapperspb.FloatValue{Value: float32(v)}
msg = wrapperspb.Float(float32(v))
case "google.protobuf.Int64Value":
v, err := strconv.ParseInt(value, 10, 64)
if err != nil {
return protoreflect.Value{}, err
}
msg = &wrapperspb.Int64Value{Value: v}
msg = wrapperspb.Int64(v)
case "google.protobuf.Int32Value":
v, err := strconv.ParseInt(value, 10, 32)
if err != nil {
return protoreflect.Value{}, err
}
msg = &wrapperspb.Int32Value{Value: int32(v)}
msg = wrapperspb.Int32(int32(v))
case "google.protobuf.UInt64Value":
v, err := strconv.ParseUint(value, 10, 64)
if err != nil {
return protoreflect.Value{}, err
}
msg = &wrapperspb.UInt64Value{Value: v}
msg = wrapperspb.UInt64(v)
case "google.protobuf.UInt32Value":
v, err := strconv.ParseUint(value, 10, 32)
if err != nil {
return protoreflect.Value{}, err
}
msg = &wrapperspb.UInt32Value{Value: uint32(v)}
msg = wrapperspb.UInt32(uint32(v))
case "google.protobuf.BoolValue":
v, err := strconv.ParseBool(value)
if err != nil {
return protoreflect.Value{}, err
}
msg = &wrapperspb.BoolValue{Value: v}
msg = wrapperspb.Bool(v)
case "google.protobuf.StringValue":
msg = &wrapperspb.StringValue{Value: value}
msg = wrapperspb.String(value)
case "google.protobuf.BytesValue":
v, err := base64.URLEncoding.DecodeString(value)
v, err := Bytes(value)
if err != nil {
return protoreflect.Value{}, err
}
msg = &wrapperspb.BytesValue{Value: v}
msg = wrapperspb.Bytes(v)
case "google.protobuf.FieldMask":
fm := &field_mask.FieldMask{}
fm.Paths = append(fm.Paths, strings.Split(value, ",")...)
msg = fm
case "google.protobuf.Value":
var v structpb.Value
if err := protojson.Unmarshal([]byte(value), &v); err != nil {
return protoreflect.Value{}, err
}
msg = &v
case "google.protobuf.Struct":
var v structpb.Struct
if err := protojson.Unmarshal([]byte(value), &v); err != nil {
return protoreflect.Value{}, err
}
msg = &v
default:
return protoreflect.Value{}, fmt.Errorf("unsupported message type: %q", string(msgDescriptor.FullName()))
}

View File

@@ -8,6 +8,7 @@ go_library(
"doc.go",
"pattern.go",
"readerfactory.go",
"string_array_flag.go",
"trie.go",
],
importpath = "github.com/grpc-ecosystem/grpc-gateway/v2/utilities",
@@ -16,7 +17,10 @@ go_library(
go_test(
name = "utilities_test",
size = "small",
srcs = ["trie_test.go"],
srcs = [
"string_array_flag_test.go",
"trie_test.go",
],
deps = [":utilities"],
)

View File

@@ -3,13 +3,12 @@ package utilities
import (
"bytes"
"io"
"io/ioutil"
)
// IOReaderFactory takes in an io.Reader and returns a function that will allow you to create a new reader that begins
// at the start of the stream
func IOReaderFactory(r io.Reader) (func() io.Reader, error) {
b, err := ioutil.ReadAll(r)
b, err := io.ReadAll(r)
if err != nil {
return nil, err
}

View File

@@ -0,0 +1,33 @@
package utilities
import (
"flag"
"strings"
)
// flagInterface is an cut down interface to `flag`
type flagInterface interface {
Var(value flag.Value, name string, usage string)
}
// StringArrayFlag defines a flag with the specified name and usage string.
// The return value is the address of a `StringArrayFlags` variable that stores the repeated values of the flag.
func StringArrayFlag(f flagInterface, name string, usage string) *StringArrayFlags {
value := &StringArrayFlags{}
f.Var(value, name, usage)
return value
}
// StringArrayFlags is a wrapper of `[]string` to provider an interface for `flag.Var`
type StringArrayFlags []string
// String returns a string representation of `StringArrayFlags`
func (i *StringArrayFlags) String() string {
return strings.Join(*i, ",")
}
// Set appends a value to `StringArrayFlags`
func (i *StringArrayFlags) Set(value string) error {
*i = append(*i, value)
return nil
}

View File

@@ -40,7 +40,7 @@ func NewDoubleArray(seqs [][]string) *DoubleArray {
func registerTokens(da *DoubleArray, seqs [][]string) [][]int {
var result [][]int
for _, seq := range seqs {
var encoded []int
encoded := make([]int, 0, len(seq))
for _, token := range seq {
if _, ok := da.Encoding[token]; !ok {
da.Encoding[token] = len(da.Encoding)