STOR-519: Add db-connect, a SQL over HTTPS server

vendor/github.com/kshvakov/clickhouse/lib/data/block.go

@@ -0,0 +1,294 @@
+package data
+
+import (
+	"database/sql/driver"
+	"fmt"
+	"io"
+	"reflect"
+	"strings"
+
+	"github.com/kshvakov/clickhouse/lib/binary"
+	"github.com/kshvakov/clickhouse/lib/column"
+	wb "github.com/kshvakov/clickhouse/lib/writebuffer"
+)
+
+type offset [][]int
+
+type Block struct {
+	Values     [][]interface{}
+	Columns    []column.Column
+	NumRows    uint64
+	NumColumns uint64
+	offsets    []offset
+	buffers    []*buffer
+	info       blockInfo
+}
+
+func (block *Block) Copy() *Block {
+	return &Block{
+		Columns:    block.Columns,
+		NumColumns: block.NumColumns,
+		info:       block.info,
+	}
+}
+
+func (block *Block) ColumnNames() []string {
+	names := make([]string, 0, len(block.Columns))
+	for _, column := range block.Columns {
+		names = append(names, column.Name())
+	}
+	return names
+}
+
+func (block *Block) Read(serverInfo *ServerInfo, decoder *binary.Decoder) (err error) {
+	if err = block.info.read(decoder); err != nil {
+		return err
+	}
+
+	if block.NumColumns, err = decoder.Uvarint(); err != nil {
+		return err
+	}
+	if block.NumRows, err = decoder.Uvarint(); err != nil {
+		return err
+	}
+	block.Values = make([][]interface{}, block.NumColumns)
+	if block.NumRows > 10 {
+		for i := 0; i < int(block.NumColumns); i++ {
+			block.Values[i] = make([]interface{}, 0, block.NumRows)
+		}
+	}
+	for i := 0; i < int(block.NumColumns); i++ {
+		var (
+			value      interface{}
+			columnName string
+			columnType string
+		)
+		if columnName, err = decoder.String(); err != nil {
+			return err
+		}
+		if columnType, err = decoder.String(); err != nil {
+			return err
+		}
+		c, err := column.Factory(columnName, columnType, serverInfo.Timezone)
+		if err != nil {
+			return err
+		}
+		block.Columns = append(block.Columns, c)
+		switch column := c.(type) {
+		case *column.Array:
+			if block.Values[i], err = column.ReadArray(decoder, int(block.NumRows)); err != nil {
+				return err
+			}
+		case *column.Nullable:
+			if block.Values[i], err = column.ReadNull(decoder, int(block.NumRows)); err != nil {
+				return err
+			}
+		default:
+			for row := 0; row < int(block.NumRows); row++ {
+				if value, err = column.Read(decoder); err != nil {
+					return err
+				}
+				block.Values[i] = append(block.Values[i], value)
+			}
+		}
+	}
+	return nil
+}
+
+func (block *Block) writeArray(column column.Column, value reflect.Value, num, level int) error {
+	switch {
+	case value.Kind() == reflect.Slice:
+		if len(block.offsets[num]) < level {
+			block.offsets[num] = append(block.offsets[num], []int{value.Len()})
+		} else {
+			block.offsets[num][level-1] = append(
+				block.offsets[num][level-1],
+				block.offsets[num][level-1][len(block.offsets[num][level-1])-1]+value.Len(),
+			)
+		}
+		for i := 0; i < value.Len(); i++ {
+			if err := block.writeArray(column, value.Index(i), num, level+1); err != nil {
+				return err
+			}
+		}
+	default:
+		if err := column.Write(block.buffers[num].Column, value.Interface()); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (block *Block) AppendRow(args []driver.Value) error {
+	if len(block.Columns) != len(args) {
+		return fmt.Errorf("block: expected %d arguments (columns: %s), got %d", len(block.Columns), strings.Join(block.ColumnNames(), ", "), len(args))
+	}
+	block.Reserve()
+	{
+		block.NumRows++
+	}
+	for num, c := range block.Columns {
+		switch column := c.(type) {
+		case *column.Array:
+			value := reflect.ValueOf(args[num])
+			if value.Kind() != reflect.Slice {
+				return fmt.Errorf("unsupported Array(T) type [%T]", value.Interface())
+			}
+			if err := block.writeArray(c, value, num, 1); err != nil {
+				return err
+			}
+		case *column.Nullable:
+			if err := column.WriteNull(block.buffers[num].Offset, block.buffers[num].Column, args[num]); err != nil {
+				return err
+			}
+		default:
+			if err := column.Write(block.buffers[num].Column, args[num]); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (block *Block) Reserve() {
+	if len(block.buffers) == 0 {
+		block.buffers = make([]*buffer, len(block.Columns))
+		block.offsets = make([]offset, len(block.Columns))
+		for i := 0; i < len(block.Columns); i++ {
+			var (
+				offsetBuffer = wb.New(wb.InitialSize)
+				columnBuffer = wb.New(wb.InitialSize)
+			)
+			block.buffers[i] = &buffer{
+				Offset:       binary.NewEncoder(offsetBuffer),
+				Column:       binary.NewEncoder(columnBuffer),
+				offsetBuffer: offsetBuffer,
+				columnBuffer: columnBuffer,
+			}
+		}
+	}
+}
+
+func (block *Block) Reset() {
+	block.NumRows = 0
+	block.NumColumns = 0
+	for _, buffer := range block.buffers {
+		buffer.reset()
+	}
+	{
+		block.offsets = nil
+		block.buffers = nil
+	}
+}
+
+func (block *Block) Write(serverInfo *ServerInfo, encoder *binary.Encoder) error {
+	if err := block.info.write(encoder); err != nil {
+		return err
+	}
+	encoder.Uvarint(block.NumColumns)
+	encoder.Uvarint(block.NumRows)
+	defer func() {
+		block.NumRows = 0
+		for i := range block.offsets {
+			block.offsets[i] = offset{}
+		}
+	}()
+	for i, column := range block.Columns {
+		encoder.String(column.Name())
+		encoder.String(column.CHType())
+		if len(block.buffers) == len(block.Columns) {
+			for _, offsets := range block.offsets[i] {
+				for _, offset := range offsets {
+					if err := encoder.UInt64(uint64(offset)); err != nil {
+						return err
+					}
+				}
+			}
+			if _, err := block.buffers[i].WriteTo(encoder); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+type blockInfo struct {
+	num1        uint64
+	isOverflows bool
+	num2        uint64
+	bucketNum   int32
+	num3        uint64
+}
+
+func (info *blockInfo) read(decoder *binary.Decoder) error {
+	var err error
+	if info.num1, err = decoder.Uvarint(); err != nil {
+		return err
+	}
+	if info.isOverflows, err = decoder.Bool(); err != nil {
+		return err
+	}
+	if info.num2, err = decoder.Uvarint(); err != nil {
+		return err
+	}
+	if info.bucketNum, err = decoder.Int32(); err != nil {
+		return err
+	}
+	if info.num3, err = decoder.Uvarint(); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (info *blockInfo) write(encoder *binary.Encoder) error {
+	if err := encoder.Uvarint(1); err != nil {
+		return err
+	}
+	if err := encoder.Bool(info.isOverflows); err != nil {
+		return err
+	}
+	if err := encoder.Uvarint(2); err != nil {
+		return err
+	}
+	if info.bucketNum == 0 {
+		info.bucketNum = -1
+	}
+	if err := encoder.Int32(info.bucketNum); err != nil {
+		return err
+	}
+	if err := encoder.Uvarint(0); err != nil {
+		return err
+	}
+	return nil
+}
+
+type buffer struct {
+	Offset       *binary.Encoder
+	Column       *binary.Encoder
+	offsetBuffer *wb.WriteBuffer
+	columnBuffer *wb.WriteBuffer
+}
+
+func (buf *buffer) WriteTo(w io.Writer) (int64, error) {
+	var size int64
+	{
+		ln, err := buf.offsetBuffer.WriteTo(w)
+		if err != nil {
+			return size, err
+		}
+		size += ln
+	}
+	{
+		ln, err := buf.columnBuffer.WriteTo(w)
+		if err != nil {
+			return size, err
+		}
+		size += ln
+	}
+	return size, nil
+}
+
+func (buf *buffer) reset() {
+	buf.offsetBuffer.Reset()
+	buf.columnBuffer.Reset()
+}

vendor/github.com/kshvakov/clickhouse/lib/data/block_write_column.go

@@ -0,0 +1,98 @@
+package data
+
+import (
+	"fmt"
+	"reflect"
+	"time"
+
+	"github.com/kshvakov/clickhouse/lib/binary"
+)
+
+func (block *Block) WriteDate(c int, v time.Time) error {
+	_, offset := v.Zone()
+	nday := (v.Unix() + int64(offset)) / 24 / 3600
+	return block.buffers[c].Column.UInt16(uint16(nday))
+}
+
+func (block *Block) WriteDateTime(c int, v time.Time) error {
+	return block.buffers[c].Column.UInt32(uint32(v.Unix()))
+}
+
+func (block *Block) WriteBool(c int, v bool) error {
+	if v {
+		return block.buffers[c].Column.UInt8(1)
+	}
+	return block.buffers[c].Column.UInt8(0)
+}
+
+func (block *Block) WriteInt8(c int, v int8) error {
+	return block.buffers[c].Column.Int8(v)
+}
+
+func (block *Block) WriteInt16(c int, v int16) error {
+	return block.buffers[c].Column.Int16(v)
+}
+
+func (block *Block) WriteInt32(c int, v int32) error {
+	return block.buffers[c].Column.Int32(v)
+}
+
+func (block *Block) WriteInt64(c int, v int64) error {
+	return block.buffers[c].Column.Int64(v)
+}
+
+func (block *Block) WriteUInt8(c int, v uint8) error {
+	return block.buffers[c].Column.UInt8(v)
+}
+
+func (block *Block) WriteUInt16(c int, v uint16) error {
+	return block.buffers[c].Column.UInt16(v)
+}
+
+func (block *Block) WriteUInt32(c int, v uint32) error {
+	return block.buffers[c].Column.UInt32(v)
+}
+
+func (block *Block) WriteUInt64(c int, v uint64) error {
+	return block.buffers[c].Column.UInt64(v)
+}
+
+func (block *Block) WriteFloat32(c int, v float32) error {
+	return block.buffers[c].Column.Float32(v)
+}
+
+func (block *Block) WriteFloat64(c int, v float64) error {
+	return block.buffers[c].Column.Float64(v)
+}
+
+func (block *Block) WriteBytes(c int, v []byte) error {
+	if err := block.buffers[c].Column.Uvarint(uint64(len(v))); err != nil {
+		return err
+	}
+	if _, err := block.buffers[c].Column.Write(v); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (block *Block) WriteString(c int, v string) error {
+	if err := block.buffers[c].Column.Uvarint(uint64(len(v))); err != nil {
+		return err
+	}
+	if _, err := block.buffers[c].Column.Write(binary.Str2Bytes(v)); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (block *Block) WriteFixedString(c int, v []byte) error {
+	return block.Columns[c].Write(block.buffers[c].Column, v)
+}
+
+func (block *Block) WriteArray(c int, v interface{}) error {
+	value := reflect.ValueOf(v)
+	if value.Kind() != reflect.Slice {
+		return fmt.Errorf("unsupported Array(T) type [%T]", value.Interface())
+	}
+	return block.writeArray(block.Columns[c], value, c, 1)
+}

vendor/github.com/kshvakov/clickhouse/lib/data/client_info.go

@@ -0,0 +1,29 @@
+package data
+
+import (
+	"fmt"
+
+	"github.com/kshvakov/clickhouse/lib/binary"
+)
+
+const ClientName = "Golang SQLDriver"
+
+const (
+	ClickHouseRevision         = 54213
+	ClickHouseDBMSVersionMajor = 1
+	ClickHouseDBMSVersionMinor = 1
+)
+
+type ClientInfo struct{}
+
+func (ClientInfo) Write(encoder *binary.Encoder) error {
+	encoder.String(ClientName)
+	encoder.Uvarint(ClickHouseDBMSVersionMajor)
+	encoder.Uvarint(ClickHouseDBMSVersionMinor)
+	encoder.Uvarint(ClickHouseRevision)
+	return nil
+}
+
+func (ClientInfo) String() string {
+	return fmt.Sprintf("%s %d.%d.%d", ClientName, ClickHouseDBMSVersionMajor, ClickHouseDBMSVersionMinor, ClickHouseRevision)
+}

vendor/github.com/kshvakov/clickhouse/lib/data/server_info.go

@@ -0,0 +1,47 @@
+package data
+
+import (
+	"fmt"
+	//"io"
+	"time"
+
+	"github.com/kshvakov/clickhouse/lib/binary"
+	"github.com/kshvakov/clickhouse/lib/protocol"
+)
+
+type ServerInfo struct {
+	Name         string
+	Revision     uint64
+	MinorVersion uint64
+	MajorVersion uint64
+	Timezone     *time.Location
+}
+
+func (srv *ServerInfo) Read(decoder *binary.Decoder) (err error) {
+	if srv.Name, err = decoder.String(); err != nil {
+		return fmt.Errorf("could not read server name: %v", err)
+	}
+	if srv.MajorVersion, err = decoder.Uvarint(); err != nil {
+		return fmt.Errorf("could not read server major version: %v", err)
+	}
+	if srv.MinorVersion, err = decoder.Uvarint(); err != nil {
+		return fmt.Errorf("could not read server minor version: %v", err)
+	}
+	if srv.Revision, err = decoder.Uvarint(); err != nil {
+		return fmt.Errorf("could not read server revision: %v", err)
+	}
+	if srv.Revision >= protocol.DBMS_MIN_REVISION_WITH_SERVER_TIMEZONE {
+		timezone, err := decoder.String()
+		if err != nil {
+			return fmt.Errorf("could not read server timezone: %v", err)
+		}
+		if srv.Timezone, err = time.LoadLocation(timezone); err != nil {
+			return fmt.Errorf("could not load time location: %v", err)
+		}
+	}
+	return nil
+}
+
+func (srv ServerInfo) String() string {
+	return fmt.Sprintf("%s %d.%d.%d (%s)", srv.Name, srv.MajorVersion, srv.MinorVersion, srv.Revision, srv.Timezone)
+}