TUN-2243: Revert "STOR-519: Add db-connect, a SQL over HTTPS server"

This reverts commit 5da2109811.
This commit is contained in:
Adam Chalmers
2019-08-26 16:45:49 -05:00
parent c3c88cc31e
commit 4e1df1a211
410 changed files with 666 additions and 362649 deletions

View File

@@ -1,107 +0,0 @@
// +build !clz4
package binary
import (
"encoding/binary"
"fmt"
"io"
"github.com/kshvakov/clickhouse/lib/lz4"
)
type compressReader struct {
reader io.Reader
// data uncompressed
data []byte
// data position
pos int
// data compressed
zdata []byte
// lz4 headers
header []byte
}
// NewCompressReader wrap the io.Reader
func NewCompressReader(r io.Reader) *compressReader {
p := &compressReader{
reader: r,
header: make([]byte, HeaderSize),
}
p.data = make([]byte, BlockMaxSize, BlockMaxSize)
zlen := lz4.CompressBound(BlockMaxSize) + HeaderSize
p.zdata = make([]byte, zlen, zlen)
p.pos = len(p.data)
return p
}
func (cr *compressReader) Read(buf []byte) (n int, err error) {
var bytesRead = 0
n = len(buf)
if cr.pos < len(cr.data) {
copyedSize := copy(buf, cr.data[cr.pos:])
bytesRead += copyedSize
cr.pos += copyedSize
}
for bytesRead < n {
if err = cr.readCompressedData(); err != nil {
return bytesRead, err
}
copyedSize := copy(buf[bytesRead:], cr.data)
bytesRead += copyedSize
cr.pos = copyedSize
}
return n, nil
}
func (cr *compressReader) readCompressedData() (err error) {
cr.pos = 0
var n int
n, err = cr.reader.Read(cr.header)
if err != nil {
return
}
if n != len(cr.header) {
return fmt.Errorf("Lz4 decompression header EOF")
}
compressedSize := int(binary.LittleEndian.Uint32(cr.header[17:])) - 9
decompressedSize := int(binary.LittleEndian.Uint32(cr.header[21:]))
if compressedSize > cap(cr.zdata) {
cr.zdata = make([]byte, compressedSize)
}
if decompressedSize > cap(cr.data) {
cr.data = make([]byte, decompressedSize)
}
cr.zdata = cr.zdata[:compressedSize]
cr.data = cr.data[:decompressedSize]
// @TODO checksum
if cr.header[16] == LZ4 {
n, err = cr.reader.Read(cr.zdata)
if err != nil {
return
}
if n != len(cr.zdata) {
return fmt.Errorf("Decompress read size not match")
}
_, err = lz4.Decode(cr.data, cr.zdata)
if err != nil {
return
}
} else {
return fmt.Errorf("Unknown compression method: 0x%02x ", cr.header[16])
}
return nil
}

View File

@@ -1,107 +0,0 @@
// +build clz4
package binary
import (
"encoding/binary"
"fmt"
"io"
lz4 "github.com/cloudflare/golz4"
)
type compressReader struct {
reader io.Reader
// data uncompressed
data []byte
// data position
pos int
// data compressed
zdata []byte
// lz4 headers
header []byte
}
// NewCompressReader wrap the io.Reader
func NewCompressReader(r io.Reader) *compressReader {
p := &compressReader{
reader: r,
header: make([]byte, HeaderSize),
}
p.data = make([]byte, BlockMaxSize, BlockMaxSize)
zlen := lz4.CompressBound(p.data) + HeaderSize
p.zdata = make([]byte, zlen, zlen)
p.pos = len(p.data)
return p
}
func (cr *compressReader) Read(buf []byte) (n int, err error) {
var bytesRead = 0
n = len(buf)
if cr.pos < len(cr.data) {
copyedSize := copy(buf, cr.data[cr.pos:])
bytesRead += copyedSize
cr.pos += copyedSize
}
for bytesRead < n {
if err = cr.readCompressedData(); err != nil {
return bytesRead, err
}
copyedSize := copy(buf[bytesRead:], cr.data)
bytesRead += copyedSize
cr.pos = copyedSize
}
return n, nil
}
func (cr *compressReader) readCompressedData() (err error) {
cr.pos = 0
var n int
n, err = cr.reader.Read(cr.header)
if err != nil {
return
}
if n != len(cr.header) {
return fmt.Errorf("Lz4 decompression header EOF")
}
compressedSize := int(binary.LittleEndian.Uint32(cr.header[17:])) - 9
decompressedSize := int(binary.LittleEndian.Uint32(cr.header[21:]))
if compressedSize > cap(cr.zdata) {
cr.zdata = make([]byte, compressedSize)
}
if decompressedSize > cap(cr.data) {
cr.data = make([]byte, decompressedSize)
}
cr.zdata = cr.zdata[:compressedSize]
cr.data = cr.data[:decompressedSize]
// @TODO checksum
if cr.header[16] == LZ4 {
n, err = cr.reader.Read(cr.zdata)
if err != nil {
return
}
if n != len(cr.zdata) {
return fmt.Errorf("Decompress read size not match")
}
err = lz4.Uncompress(cr.zdata, cr.data)
if err != nil {
return
}
} else {
return fmt.Errorf("Unknown compression method: 0x%02x ", cr.header[16])
}
return nil
}

View File

@@ -1,21 +0,0 @@
package binary
type CompressionMethodByte byte
const (
NONE CompressionMethodByte = 0x02
LZ4 = 0x82
ZSTD = 0x90
)
const (
// ChecksumSize is 128bits for cityhash102 checksum
ChecksumSize = 16
// CompressHeader magic + compressed_size + uncompressed_size
CompressHeaderSize = 1 + 4 + 4
// HeaderSize
HeaderSize = ChecksumSize + CompressHeaderSize
// BlockMaxSize 1MB
BlockMaxSize = 1 << 10
)

View File

@@ -1,79 +0,0 @@
// +build !clz4
package binary
import (
"encoding/binary"
"io"
"github.com/kshvakov/clickhouse/lib/cityhash102"
"github.com/kshvakov/clickhouse/lib/lz4"
)
type compressWriter struct {
writer io.Writer
// data uncompressed
data []byte
// data position
pos int
// data compressed
zdata []byte
}
// NewCompressWriter wrap the io.Writer
func NewCompressWriter(w io.Writer) *compressWriter {
p := &compressWriter{writer: w}
p.data = make([]byte, BlockMaxSize, BlockMaxSize)
zlen := lz4.CompressBound(BlockMaxSize) + HeaderSize
p.zdata = make([]byte, zlen, zlen)
return p
}
func (cw *compressWriter) Write(buf []byte) (int, error) {
var n int
for len(buf) > 0 {
// Accumulate the data to be compressed.
m := copy(cw.data[cw.pos:], buf)
cw.pos += m
buf = buf[m:]
if cw.pos == len(cw.data) {
err := cw.Flush()
if err != nil {
return n, err
}
}
n += m
}
return n, nil
}
func (cw *compressWriter) Flush() (err error) {
if cw.pos == 0 {
return
}
// write the headers
compressedSize, err := lz4.Encode(cw.zdata[HeaderSize:], cw.data[:cw.pos])
if err != nil {
return err
}
compressedSize += CompressHeaderSize
// fill the header, compressed_size_32 + uncompressed_size_32
cw.zdata[16] = LZ4
binary.LittleEndian.PutUint32(cw.zdata[17:], uint32(compressedSize))
binary.LittleEndian.PutUint32(cw.zdata[21:], uint32(cw.pos))
// fill the checksum
checkSum := cityhash102.CityHash128(cw.zdata[16:], uint32(compressedSize))
binary.LittleEndian.PutUint64(cw.zdata[0:], checkSum.Lower64())
binary.LittleEndian.PutUint64(cw.zdata[8:], checkSum.Higher64())
cw.writer.Write(cw.zdata[:compressedSize+ChecksumSize])
if w, ok := cw.writer.(WriteFlusher); ok {
err = w.Flush()
}
cw.pos = 0
return
}

View File

@@ -1,78 +0,0 @@
// +build clz4
package binary
import (
"encoding/binary"
"io"
lz4 "github.com/cloudflare/golz4"
"github.com/kshvakov/clickhouse/lib/cityhash102"
)
type compressWriter struct {
writer io.Writer
// data uncompressed
data []byte
// data position
pos int
// data compressed
zdata []byte
}
// NewCompressWriter wrap the io.Writer
func NewCompressWriter(w io.Writer) *compressWriter {
p := &compressWriter{writer: w}
p.data = make([]byte, BlockMaxSize, BlockMaxSize)
zlen := lz4.CompressBound(p.data) + HeaderSize
p.zdata = make([]byte, zlen, zlen)
return p
}
func (cw *compressWriter) Write(buf []byte) (int, error) {
var n int
for len(buf) > 0 {
// Accumulate the data to be compressed.
m := copy(cw.data[cw.pos:], buf)
cw.pos += m
buf = buf[m:]
if cw.pos == len(cw.data) {
err := cw.Flush()
if err != nil {
return n, err
}
}
n += m
}
return n, nil
}
func (cw *compressWriter) Flush() (err error) {
if cw.pos == 0 {
return
}
// write the headers
compressedSize, err := lz4.Compress(cw.data[:cw.pos], cw.zdata[HeaderSize:])
if err != nil {
return err
}
compressedSize += CompressHeaderSize
// fill the header, compressed_size_32 + uncompressed_size_32
cw.zdata[16] = LZ4
binary.LittleEndian.PutUint32(cw.zdata[17:], uint32(compressedSize))
binary.LittleEndian.PutUint32(cw.zdata[21:], uint32(cw.pos))
// fill the checksum
checkSum := cityhash102.CityHash128(cw.zdata[16:], uint32(compressedSize))
binary.LittleEndian.PutUint64(cw.zdata[0:], checkSum.Lower64())
binary.LittleEndian.PutUint64(cw.zdata[8:], checkSum.Higher64())
cw.writer.Write(cw.zdata[:compressedSize+ChecksumSize])
if w, ok := cw.writer.(WriteFlusher); ok {
err = w.Flush()
}
cw.pos = 0
return
}

View File

@@ -1,165 +0,0 @@
package binary
import (
"encoding/binary"
"io"
"math"
)
func NewDecoder(input io.Reader) *Decoder {
return &Decoder{
input: input,
compressInput: NewCompressReader(input),
}
}
type Decoder struct {
compress bool
input io.Reader
compressInput io.Reader
scratch [binary.MaxVarintLen64]byte
}
func (decoder *Decoder) SelectCompress(compress bool) {
decoder.compress = compress
}
func (decoder *Decoder) Get() io.Reader {
if decoder.compress {
return decoder.compressInput
}
return decoder.input
}
func (decoder *Decoder) Bool() (bool, error) {
v, err := decoder.ReadByte()
if err != nil {
return false, err
}
return v == 1, nil
}
func (decoder *Decoder) Uvarint() (uint64, error) {
return binary.ReadUvarint(decoder)
}
func (decoder *Decoder) Int8() (int8, error) {
v, err := decoder.ReadByte()
if err != nil {
return 0, err
}
return int8(v), nil
}
func (decoder *Decoder) Int16() (int16, error) {
v, err := decoder.UInt16()
if err != nil {
return 0, err
}
return int16(v), nil
}
func (decoder *Decoder) Int32() (int32, error) {
v, err := decoder.UInt32()
if err != nil {
return 0, err
}
return int32(v), nil
}
func (decoder *Decoder) Int64() (int64, error) {
v, err := decoder.UInt64()
if err != nil {
return 0, err
}
return int64(v), nil
}
func (decoder *Decoder) UInt8() (uint8, error) {
v, err := decoder.ReadByte()
if err != nil {
return 0, err
}
return uint8(v), nil
}
func (decoder *Decoder) UInt16() (uint16, error) {
if _, err := decoder.Get().Read(decoder.scratch[:2]); err != nil {
return 0, err
}
return uint16(decoder.scratch[0]) | uint16(decoder.scratch[1])<<8, nil
}
func (decoder *Decoder) UInt32() (uint32, error) {
if _, err := decoder.Get().Read(decoder.scratch[:4]); err != nil {
return 0, err
}
return uint32(decoder.scratch[0]) |
uint32(decoder.scratch[1])<<8 |
uint32(decoder.scratch[2])<<16 |
uint32(decoder.scratch[3])<<24, nil
}
func (decoder *Decoder) UInt64() (uint64, error) {
if _, err := decoder.Get().Read(decoder.scratch[:8]); err != nil {
return 0, err
}
return uint64(decoder.scratch[0]) |
uint64(decoder.scratch[1])<<8 |
uint64(decoder.scratch[2])<<16 |
uint64(decoder.scratch[3])<<24 |
uint64(decoder.scratch[4])<<32 |
uint64(decoder.scratch[5])<<40 |
uint64(decoder.scratch[6])<<48 |
uint64(decoder.scratch[7])<<56, nil
}
func (decoder *Decoder) Float32() (float32, error) {
v, err := decoder.UInt32()
if err != nil {
return 0, err
}
return math.Float32frombits(v), nil
}
func (decoder *Decoder) Float64() (float64, error) {
v, err := decoder.UInt64()
if err != nil {
return 0, err
}
return math.Float64frombits(v), nil
}
func (decoder *Decoder) Fixed(ln int) ([]byte, error) {
if reader, ok := decoder.Get().(FixedReader); ok {
return reader.Fixed(ln)
}
buf := make([]byte, ln)
if _, err := decoder.Get().Read(buf); err != nil {
return nil, err
}
return buf, nil
}
func (decoder *Decoder) String() (string, error) {
strlen, err := decoder.Uvarint()
if err != nil {
return "", err
}
str, err := decoder.Fixed(int(strlen))
if err != nil {
return "", err
}
return string(str), nil
}
func (decoder *Decoder) ReadByte() (byte, error) {
if _, err := decoder.Get().Read(decoder.scratch[:1]); err != nil {
return 0x0, err
}
return decoder.scratch[0], nil
}
type FixedReader interface {
Fixed(ln int) ([]byte, error)
}

View File

@@ -1,162 +0,0 @@
package binary
import (
"encoding/binary"
"io"
"math"
"reflect"
"unsafe"
)
func NewEncoder(w io.Writer) *Encoder {
return &Encoder{
output: w,
compressOutput: NewCompressWriter(w),
}
}
type Encoder struct {
compress bool
output io.Writer
compressOutput io.Writer
scratch [binary.MaxVarintLen64]byte
}
func (enc *Encoder) SelectCompress(compress bool) {
if enc.compress && !compress {
enc.Flush()
}
enc.compress = compress
}
func (enc *Encoder) Get() io.Writer {
if enc.compress {
return enc.compressOutput
}
return enc.output
}
func (enc *Encoder) Uvarint(v uint64) error {
ln := binary.PutUvarint(enc.scratch[:binary.MaxVarintLen64], v)
if _, err := enc.Get().Write(enc.scratch[0:ln]); err != nil {
return err
}
return nil
}
func (enc *Encoder) Bool(v bool) error {
if v {
return enc.UInt8(1)
}
return enc.UInt8(0)
}
func (enc *Encoder) Int8(v int8) error {
return enc.UInt8(uint8(v))
}
func (enc *Encoder) Int16(v int16) error {
return enc.UInt16(uint16(v))
}
func (enc *Encoder) Int32(v int32) error {
return enc.UInt32(uint32(v))
}
func (enc *Encoder) Int64(v int64) error {
return enc.UInt64(uint64(v))
}
func (enc *Encoder) UInt8(v uint8) error {
enc.scratch[0] = v
if _, err := enc.Get().Write(enc.scratch[:1]); err != nil {
return err
}
return nil
}
func (enc *Encoder) UInt16(v uint16) error {
enc.scratch[0] = byte(v)
enc.scratch[1] = byte(v >> 8)
if _, err := enc.Get().Write(enc.scratch[:2]); err != nil {
return err
}
return nil
}
func (enc *Encoder) UInt32(v uint32) error {
enc.scratch[0] = byte(v)
enc.scratch[1] = byte(v >> 8)
enc.scratch[2] = byte(v >> 16)
enc.scratch[3] = byte(v >> 24)
if _, err := enc.Get().Write(enc.scratch[:4]); err != nil {
return err
}
return nil
}
func (enc *Encoder) UInt64(v uint64) error {
enc.scratch[0] = byte(v)
enc.scratch[1] = byte(v >> 8)
enc.scratch[2] = byte(v >> 16)
enc.scratch[3] = byte(v >> 24)
enc.scratch[4] = byte(v >> 32)
enc.scratch[5] = byte(v >> 40)
enc.scratch[6] = byte(v >> 48)
enc.scratch[7] = byte(v >> 56)
if _, err := enc.Get().Write(enc.scratch[:8]); err != nil {
return err
}
return nil
}
func (enc *Encoder) Float32(v float32) error {
return enc.UInt32(math.Float32bits(v))
}
func (enc *Encoder) Float64(v float64) error {
return enc.UInt64(math.Float64bits(v))
}
func (enc *Encoder) String(v string) error {
str := Str2Bytes(v)
if err := enc.Uvarint(uint64(len(str))); err != nil {
return err
}
if _, err := enc.Get().Write(str); err != nil {
return err
}
return nil
}
func (enc *Encoder) RawString(str []byte) error {
if err := enc.Uvarint(uint64(len(str))); err != nil {
return err
}
if _, err := enc.Get().Write(str); err != nil {
return err
}
return nil
}
func (enc *Encoder) Write(b []byte) (int, error) {
return enc.Get().Write(b)
}
func (enc *Encoder) Flush() error {
if w, ok := enc.Get().(WriteFlusher); ok {
return w.Flush()
}
return nil
}
type WriteFlusher interface {
Flush() error
}
func Str2Bytes(str string) []byte {
header := (*reflect.SliceHeader)(unsafe.Pointer(&str))
header.Len = len(str)
header.Cap = header.Len
return *(*[]byte)(unsafe.Pointer(header))
}