Add db-connect, a SQL over HTTPS server

vendor/github.com/kshvakov/clickhouse/lib/binary/compress_reader.go

@@ -0,0 +1,107 @@
+// +build !clz4
+
+package binary
+
+import (
+	"encoding/binary"
+	"fmt"
+	"io"
+
+	"github.com/kshvakov/clickhouse/lib/lz4"
+)
+
+type compressReader struct {
+	reader io.Reader
+	// data uncompressed
+	data []byte
+	// data position
+	pos int
+	// data compressed
+	zdata []byte
+	// lz4 headers
+	header []byte
+}
+
+// NewCompressReader wrap the io.Reader
+func NewCompressReader(r io.Reader) *compressReader {
+	p := &compressReader{
+		reader: r,
+		header: make([]byte, HeaderSize),
+	}
+	p.data = make([]byte, BlockMaxSize, BlockMaxSize)
+
+	zlen := lz4.CompressBound(BlockMaxSize) + HeaderSize
+	p.zdata = make([]byte, zlen, zlen)
+
+	p.pos = len(p.data)
+	return p
+}
+
+func (cr *compressReader) Read(buf []byte) (n int, err error) {
+	var bytesRead = 0
+	n = len(buf)
+
+	if cr.pos < len(cr.data) {
+		copyedSize := copy(buf, cr.data[cr.pos:])
+
+		bytesRead += copyedSize
+		cr.pos += copyedSize
+	}
+
+	for bytesRead < n {
+		if err = cr.readCompressedData(); err != nil {
+			return bytesRead, err
+		}
+		copyedSize := copy(buf[bytesRead:], cr.data)
+
+		bytesRead += copyedSize
+		cr.pos = copyedSize
+	}
+	return n, nil
+}
+
+func (cr *compressReader) readCompressedData() (err error) {
+	cr.pos = 0
+	var n int
+	n, err = cr.reader.Read(cr.header)
+	if err != nil {
+		return
+	}
+	if n != len(cr.header) {
+		return fmt.Errorf("Lz4 decompression header EOF")
+	}
+
+	compressedSize := int(binary.LittleEndian.Uint32(cr.header[17:])) - 9
+	decompressedSize := int(binary.LittleEndian.Uint32(cr.header[21:]))
+
+	if compressedSize > cap(cr.zdata) {
+		cr.zdata = make([]byte, compressedSize)
+	}
+	if decompressedSize > cap(cr.data) {
+		cr.data = make([]byte, decompressedSize)
+	}
+
+	cr.zdata = cr.zdata[:compressedSize]
+	cr.data = cr.data[:decompressedSize]
+
+	// @TODO checksum
+	if cr.header[16] == LZ4 {
+		n, err = cr.reader.Read(cr.zdata)
+		if err != nil {
+			return
+		}
+
+		if n != len(cr.zdata) {
+			return fmt.Errorf("Decompress read size not match")
+		}
+
+		_, err = lz4.Decode(cr.data, cr.zdata)
+		if err != nil {
+			return
+		}
+	} else {
+		return fmt.Errorf("Unknown compression method: 0x%02x ", cr.header[16])
+	}
+
+	return nil
+}

vendor/github.com/kshvakov/clickhouse/lib/binary/compress_reader_clz4.go

@@ -0,0 +1,107 @@
+// +build clz4
+
+package binary
+
+import (
+	"encoding/binary"
+	"fmt"
+	"io"
+
+	lz4 "github.com/cloudflare/golz4"
+)
+
+type compressReader struct {
+	reader io.Reader
+	// data uncompressed
+	data []byte
+	// data position
+	pos int
+	// data compressed
+	zdata []byte
+	// lz4 headers
+	header []byte
+}
+
+// NewCompressReader wrap the io.Reader
+func NewCompressReader(r io.Reader) *compressReader {
+	p := &compressReader{
+		reader: r,
+		header: make([]byte, HeaderSize),
+	}
+	p.data = make([]byte, BlockMaxSize, BlockMaxSize)
+
+	zlen := lz4.CompressBound(p.data) + HeaderSize
+	p.zdata = make([]byte, zlen, zlen)
+
+	p.pos = len(p.data)
+	return p
+}
+
+func (cr *compressReader) Read(buf []byte) (n int, err error) {
+	var bytesRead = 0
+	n = len(buf)
+
+	if cr.pos < len(cr.data) {
+		copyedSize := copy(buf, cr.data[cr.pos:])
+
+		bytesRead += copyedSize
+		cr.pos += copyedSize
+	}
+
+	for bytesRead < n {
+		if err = cr.readCompressedData(); err != nil {
+			return bytesRead, err
+		}
+		copyedSize := copy(buf[bytesRead:], cr.data)
+
+		bytesRead += copyedSize
+		cr.pos = copyedSize
+	}
+	return n, nil
+}
+
+func (cr *compressReader) readCompressedData() (err error) {
+	cr.pos = 0
+	var n int
+	n, err = cr.reader.Read(cr.header)
+	if err != nil {
+		return
+	}
+	if n != len(cr.header) {
+		return fmt.Errorf("Lz4 decompression header EOF")
+	}
+
+	compressedSize := int(binary.LittleEndian.Uint32(cr.header[17:])) - 9
+	decompressedSize := int(binary.LittleEndian.Uint32(cr.header[21:]))
+
+	if compressedSize > cap(cr.zdata) {
+		cr.zdata = make([]byte, compressedSize)
+	}
+	if decompressedSize > cap(cr.data) {
+		cr.data = make([]byte, decompressedSize)
+	}
+
+	cr.zdata = cr.zdata[:compressedSize]
+	cr.data = cr.data[:decompressedSize]
+
+	// @TODO checksum
+	if cr.header[16] == LZ4 {
+		n, err = cr.reader.Read(cr.zdata)
+		if err != nil {
+			return
+		}
+
+		if n != len(cr.zdata) {
+			return fmt.Errorf("Decompress read size not match")
+		}
+
+		err = lz4.Uncompress(cr.zdata, cr.data)
+		if err != nil {
+			return
+		}
+	} else {
+		return fmt.Errorf("Unknown compression method: 0x%02x ", cr.header[16])
+	}
+
+	return nil
+}

vendor/github.com/kshvakov/clickhouse/lib/binary/compress_settings.go

@@ -0,0 +1,21 @@
+package binary
+
+type CompressionMethodByte byte
+
+const (
+	NONE CompressionMethodByte = 0x02
+	LZ4                        = 0x82
+	ZSTD                       = 0x90
+)
+
+const (
+	// ChecksumSize is 128bits for cityhash102 checksum
+	ChecksumSize = 16
+	// CompressHeader magic + compressed_size + uncompressed_size
+	CompressHeaderSize = 1 + 4 + 4
+
+	// HeaderSize
+	HeaderSize = ChecksumSize + CompressHeaderSize
+	// BlockMaxSize 1MB
+	BlockMaxSize = 1 << 10
+)

vendor/github.com/kshvakov/clickhouse/lib/binary/compress_writer.go

@@ -0,0 +1,79 @@
+// +build !clz4
+
+package binary
+
+import (
+	"encoding/binary"
+	"io"
+
+	"github.com/kshvakov/clickhouse/lib/cityhash102"
+	"github.com/kshvakov/clickhouse/lib/lz4"
+)
+
+type compressWriter struct {
+	writer io.Writer
+	// data uncompressed
+	data []byte
+	// data position
+	pos int
+	// data compressed
+	zdata []byte
+}
+
+// NewCompressWriter wrap the io.Writer
+func NewCompressWriter(w io.Writer) *compressWriter {
+	p := &compressWriter{writer: w}
+	p.data = make([]byte, BlockMaxSize, BlockMaxSize)
+
+	zlen := lz4.CompressBound(BlockMaxSize) + HeaderSize
+	p.zdata = make([]byte, zlen, zlen)
+	return p
+}
+
+func (cw *compressWriter) Write(buf []byte) (int, error) {
+	var n int
+	for len(buf) > 0 {
+		// Accumulate the data to be compressed.
+		m := copy(cw.data[cw.pos:], buf)
+		cw.pos += m
+		buf = buf[m:]
+
+		if cw.pos == len(cw.data) {
+			err := cw.Flush()
+			if err != nil {
+				return n, err
+			}
+		}
+		n += m
+	}
+	return n, nil
+}
+
+func (cw *compressWriter) Flush() (err error) {
+	if cw.pos == 0 {
+		return
+	}
+
+	// write the headers
+	compressedSize, err := lz4.Encode(cw.zdata[HeaderSize:], cw.data[:cw.pos])
+	if err != nil {
+		return err
+	}
+	compressedSize += CompressHeaderSize
+	// fill the header, compressed_size_32 + uncompressed_size_32
+	cw.zdata[16] = LZ4
+	binary.LittleEndian.PutUint32(cw.zdata[17:], uint32(compressedSize))
+	binary.LittleEndian.PutUint32(cw.zdata[21:], uint32(cw.pos))
+
+	// fill the checksum
+	checkSum := cityhash102.CityHash128(cw.zdata[16:], uint32(compressedSize))
+	binary.LittleEndian.PutUint64(cw.zdata[0:], checkSum.Lower64())
+	binary.LittleEndian.PutUint64(cw.zdata[8:], checkSum.Higher64())
+
+	cw.writer.Write(cw.zdata[:compressedSize+ChecksumSize])
+	if w, ok := cw.writer.(WriteFlusher); ok {
+		err = w.Flush()
+	}
+	cw.pos = 0
+	return
+}

vendor/github.com/kshvakov/clickhouse/lib/binary/compress_writer_clz4.go

@@ -0,0 +1,78 @@
+// +build clz4
+
+package binary
+
+import (
+	"encoding/binary"
+	"io"
+
+	lz4 "github.com/cloudflare/golz4"
+	"github.com/kshvakov/clickhouse/lib/cityhash102"
+)
+
+type compressWriter struct {
+	writer io.Writer
+	// data uncompressed
+	data []byte
+	// data position
+	pos int
+	// data compressed
+	zdata []byte
+}
+
+// NewCompressWriter wrap the io.Writer
+func NewCompressWriter(w io.Writer) *compressWriter {
+	p := &compressWriter{writer: w}
+	p.data = make([]byte, BlockMaxSize, BlockMaxSize)
+
+	zlen := lz4.CompressBound(p.data) + HeaderSize
+	p.zdata = make([]byte, zlen, zlen)
+	return p
+}
+
+func (cw *compressWriter) Write(buf []byte) (int, error) {
+	var n int
+	for len(buf) > 0 {
+		// Accumulate the data to be compressed.
+		m := copy(cw.data[cw.pos:], buf)
+		cw.pos += m
+		buf = buf[m:]
+
+		if cw.pos == len(cw.data) {
+			err := cw.Flush()
+			if err != nil {
+				return n, err
+			}
+		}
+		n += m
+	}
+	return n, nil
+}
+
+func (cw *compressWriter) Flush() (err error) {
+	if cw.pos == 0 {
+		return
+	}
+	// write the headers
+	compressedSize, err := lz4.Compress(cw.data[:cw.pos], cw.zdata[HeaderSize:])
+	if err != nil {
+		return err
+	}
+	compressedSize += CompressHeaderSize
+	// fill the header, compressed_size_32 + uncompressed_size_32
+	cw.zdata[16] = LZ4
+	binary.LittleEndian.PutUint32(cw.zdata[17:], uint32(compressedSize))
+	binary.LittleEndian.PutUint32(cw.zdata[21:], uint32(cw.pos))
+
+	// fill the checksum
+	checkSum := cityhash102.CityHash128(cw.zdata[16:], uint32(compressedSize))
+	binary.LittleEndian.PutUint64(cw.zdata[0:], checkSum.Lower64())
+	binary.LittleEndian.PutUint64(cw.zdata[8:], checkSum.Higher64())
+
+	cw.writer.Write(cw.zdata[:compressedSize+ChecksumSize])
+	if w, ok := cw.writer.(WriteFlusher); ok {
+		err = w.Flush()
+	}
+	cw.pos = 0
+	return
+}

vendor/github.com/kshvakov/clickhouse/lib/binary/decoder.go

@@ -0,0 +1,165 @@
+package binary
+
+import (
+	"encoding/binary"
+	"io"
+	"math"
+)
+
+func NewDecoder(input io.Reader) *Decoder {
+	return &Decoder{
+		input:         input,
+		compressInput: NewCompressReader(input),
+	}
+}
+
+type Decoder struct {
+	compress      bool
+	input         io.Reader
+	compressInput io.Reader
+	scratch       [binary.MaxVarintLen64]byte
+}
+
+func (decoder *Decoder) SelectCompress(compress bool) {
+	decoder.compress = compress
+}
+
+func (decoder *Decoder) Get() io.Reader {
+	if decoder.compress {
+		return decoder.compressInput
+	}
+	return decoder.input
+}
+
+func (decoder *Decoder) Bool() (bool, error) {
+	v, err := decoder.ReadByte()
+	if err != nil {
+		return false, err
+	}
+	return v == 1, nil
+}
+
+func (decoder *Decoder) Uvarint() (uint64, error) {
+	return binary.ReadUvarint(decoder)
+}
+
+func (decoder *Decoder) Int8() (int8, error) {
+	v, err := decoder.ReadByte()
+	if err != nil {
+		return 0, err
+	}
+	return int8(v), nil
+}
+
+func (decoder *Decoder) Int16() (int16, error) {
+	v, err := decoder.UInt16()
+	if err != nil {
+		return 0, err
+	}
+	return int16(v), nil
+}
+
+func (decoder *Decoder) Int32() (int32, error) {
+	v, err := decoder.UInt32()
+	if err != nil {
+		return 0, err
+	}
+	return int32(v), nil
+}
+
+func (decoder *Decoder) Int64() (int64, error) {
+	v, err := decoder.UInt64()
+	if err != nil {
+		return 0, err
+	}
+	return int64(v), nil
+}
+
+func (decoder *Decoder) UInt8() (uint8, error) {
+	v, err := decoder.ReadByte()
+	if err != nil {
+		return 0, err
+	}
+	return uint8(v), nil
+}
+
+func (decoder *Decoder) UInt16() (uint16, error) {
+	if _, err := decoder.Get().Read(decoder.scratch[:2]); err != nil {
+		return 0, err
+	}
+	return uint16(decoder.scratch[0]) | uint16(decoder.scratch[1])<<8, nil
+}
+
+func (decoder *Decoder) UInt32() (uint32, error) {
+	if _, err := decoder.Get().Read(decoder.scratch[:4]); err != nil {
+		return 0, err
+	}
+	return uint32(decoder.scratch[0]) |
+		uint32(decoder.scratch[1])<<8 |
+		uint32(decoder.scratch[2])<<16 |
+		uint32(decoder.scratch[3])<<24, nil
+}
+
+func (decoder *Decoder) UInt64() (uint64, error) {
+	if _, err := decoder.Get().Read(decoder.scratch[:8]); err != nil {
+		return 0, err
+	}
+	return uint64(decoder.scratch[0]) |
+		uint64(decoder.scratch[1])<<8 |
+		uint64(decoder.scratch[2])<<16 |
+		uint64(decoder.scratch[3])<<24 |
+		uint64(decoder.scratch[4])<<32 |
+		uint64(decoder.scratch[5])<<40 |
+		uint64(decoder.scratch[6])<<48 |
+		uint64(decoder.scratch[7])<<56, nil
+}
+
+func (decoder *Decoder) Float32() (float32, error) {
+	v, err := decoder.UInt32()
+	if err != nil {
+		return 0, err
+	}
+	return math.Float32frombits(v), nil
+}
+
+func (decoder *Decoder) Float64() (float64, error) {
+	v, err := decoder.UInt64()
+	if err != nil {
+		return 0, err
+	}
+	return math.Float64frombits(v), nil
+}
+
+func (decoder *Decoder) Fixed(ln int) ([]byte, error) {
+	if reader, ok := decoder.Get().(FixedReader); ok {
+		return reader.Fixed(ln)
+	}
+	buf := make([]byte, ln)
+	if _, err := decoder.Get().Read(buf); err != nil {
+		return nil, err
+	}
+	return buf, nil
+}
+
+func (decoder *Decoder) String() (string, error) {
+	strlen, err := decoder.Uvarint()
+	if err != nil {
+		return "", err
+	}
+	str, err := decoder.Fixed(int(strlen))
+	if err != nil {
+		return "", err
+	}
+	return string(str), nil
+}
+
+func (decoder *Decoder) ReadByte() (byte, error) {
+	if _, err := decoder.Get().Read(decoder.scratch[:1]); err != nil {
+		return 0x0, err
+	}
+	return decoder.scratch[0], nil
+}
+
+type FixedReader interface {
+	Fixed(ln int) ([]byte, error)
+}

vendor/github.com/kshvakov/clickhouse/lib/binary/encoder.go

@@ -0,0 +1,162 @@
+package binary
+
+import (
+	"encoding/binary"
+	"io"
+	"math"
+	"reflect"
+	"unsafe"
+)
+
+func NewEncoder(w io.Writer) *Encoder {
+	return &Encoder{
+		output:         w,
+		compressOutput: NewCompressWriter(w),
+	}
+}
+
+type Encoder struct {
+	compress       bool
+	output         io.Writer
+	compressOutput io.Writer
+	scratch        [binary.MaxVarintLen64]byte
+}
+
+func (enc *Encoder) SelectCompress(compress bool) {
+	if enc.compress && !compress {
+		enc.Flush()
+	}
+	enc.compress = compress
+}
+
+func (enc *Encoder) Get() io.Writer {
+	if enc.compress {
+		return enc.compressOutput
+	}
+	return enc.output
+}
+
+func (enc *Encoder) Uvarint(v uint64) error {
+	ln := binary.PutUvarint(enc.scratch[:binary.MaxVarintLen64], v)
+	if _, err := enc.Get().Write(enc.scratch[0:ln]); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (enc *Encoder) Bool(v bool) error {
+	if v {
+		return enc.UInt8(1)
+	}
+	return enc.UInt8(0)
+}
+
+func (enc *Encoder) Int8(v int8) error {
+	return enc.UInt8(uint8(v))
+}
+
+func (enc *Encoder) Int16(v int16) error {
+	return enc.UInt16(uint16(v))
+}
+
+func (enc *Encoder) Int32(v int32) error {
+	return enc.UInt32(uint32(v))
+}
+
+func (enc *Encoder) Int64(v int64) error {
+	return enc.UInt64(uint64(v))
+}
+
+func (enc *Encoder) UInt8(v uint8) error {
+	enc.scratch[0] = v
+	if _, err := enc.Get().Write(enc.scratch[:1]); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (enc *Encoder) UInt16(v uint16) error {
+	enc.scratch[0] = byte(v)
+	enc.scratch[1] = byte(v >> 8)
+	if _, err := enc.Get().Write(enc.scratch[:2]); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (enc *Encoder) UInt32(v uint32) error {
+	enc.scratch[0] = byte(v)
+	enc.scratch[1] = byte(v >> 8)
+	enc.scratch[2] = byte(v >> 16)
+	enc.scratch[3] = byte(v >> 24)
+	if _, err := enc.Get().Write(enc.scratch[:4]); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (enc *Encoder) UInt64(v uint64) error {
+	enc.scratch[0] = byte(v)
+	enc.scratch[1] = byte(v >> 8)
+	enc.scratch[2] = byte(v >> 16)
+	enc.scratch[3] = byte(v >> 24)
+	enc.scratch[4] = byte(v >> 32)
+	enc.scratch[5] = byte(v >> 40)
+	enc.scratch[6] = byte(v >> 48)
+	enc.scratch[7] = byte(v >> 56)
+	if _, err := enc.Get().Write(enc.scratch[:8]); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (enc *Encoder) Float32(v float32) error {
+	return enc.UInt32(math.Float32bits(v))
+}
+
+func (enc *Encoder) Float64(v float64) error {
+	return enc.UInt64(math.Float64bits(v))
+}
+
+func (enc *Encoder) String(v string) error {
+	str := Str2Bytes(v)
+	if err := enc.Uvarint(uint64(len(str))); err != nil {
+		return err
+	}
+	if _, err := enc.Get().Write(str); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (enc *Encoder) RawString(str []byte) error {
+	if err := enc.Uvarint(uint64(len(str))); err != nil {
+		return err
+	}
+	if _, err := enc.Get().Write(str); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (enc *Encoder) Write(b []byte) (int, error) {
+	return enc.Get().Write(b)
+}
+
+func (enc *Encoder) Flush() error {
+	if w, ok := enc.Get().(WriteFlusher); ok {
+		return w.Flush()
+	}
+	return nil
+}
+
+type WriteFlusher interface {
+	Flush() error
+}
+
+func Str2Bytes(str string) []byte {
+	header := (*reflect.SliceHeader)(unsafe.Pointer(&str))
+	header.Len = len(str)
+	header.Cap = header.Len
+	return *(*[]byte)(unsafe.Pointer(header))
+}