From 68d73bda2e8ca0ffda268afee2a4145e3f91cba7 Mon Sep 17 00:00:00 2001 From: victor Date: Tue, 19 Aug 2025 19:19:02 +0800 Subject: [PATCH 01/24] feat(golang): 1. array support 2. protocol version option, binary protocol for doubles --- buffer.go | 240 +++++++++++++++++++++++++- buffer_test.go | 327 ++++++++++++++++++++++++++++++++++-- conf_parse.go | 12 ++ conf_test.go | 40 ++++- examples/from-conf/main.go | 29 ++++ examples/http/basic/main.go | 30 ++++ examples/tcp/basic/main.go | 33 +++- export_test.go | 48 +++++- http_sender.go | 195 ++++++++++++++++++++- http_sender_test.go | 173 +++++++++++++++++-- integration_test.go | 127 ++++++++++---- interop_test.go | 4 +- ndarray.go | 194 +++++++++++++++++++++ ndarray_test.go | 319 +++++++++++++++++++++++++++++++++++ sender.go | 80 ++++++++- sender_pool.go | 22 ++- tcp_integration_test.go | 89 +++++++++- tcp_sender.go | 99 ++++++++++- tcp_sender_test.go | 90 +++++++++- utils_test.go | 71 ++++++-- 20 files changed, 2113 insertions(+), 109 deletions(-) create mode 100644 ndarray.go create mode 100644 ndarray_test.go diff --git a/buffer.go b/buffer.go index c5df6f2c..ccdcf116 100644 --- a/buffer.go +++ b/buffer.go @@ -26,12 +26,14 @@ package questdb import ( "bytes" + "encoding/binary" "errors" "fmt" "math" "math/big" "strconv" "time" + "unsafe" ) // errInvalidMsg indicates a failed attempt to construct an ILP @@ -39,6 +41,52 @@ import ( // chars found in table or column name. var errInvalidMsg = errors.New("invalid message") +type binaryFlag byte + +const ( + arrayBinaryFlag binaryFlag = 14 + float64BinaryFlag binaryFlag = 16 +) + +// isLittleEndian checks if the current machine uses little-endian byte order +func isLittleEndian() bool { + var i int32 = 0x01020304 + return *(*byte)(unsafe.Pointer(&i)) == 0x04 +} + +// writeFloat64Data optimally writes float64 slice data to buffer +// Uses batch memory copy on little-endian machines for better performance +func (b *buffer) writeFloat64Data(data []float64) { + if isLittleEndian() && len(data) > 0 { + b.Write(unsafe.Slice((*byte)(unsafe.Pointer(&data[0])), len(data)*8)) + } else { + bytes := make([]byte, 8) + for _, val := range data { + binary.LittleEndian.PutUint64(bytes[0:], math.Float64bits(val)) + b.Write(bytes) + } + } +} + +// writeUint32 optimally writes a single uint32 value +func (b *buffer) writeUint32(val uint32) { + if isLittleEndian() { + // On little-endian machines, we can directly write the uint32 as bytes + b.Write((*[4]byte)(unsafe.Pointer(&val))[:]) + } else { + // On big-endian machines, use the standard conversion + data := make([]byte, 4) + binary.LittleEndian.PutUint32(data, val) + b.Write(data) + } +} + +type arrayElemType byte + +const ( + arrayElemDouble arrayElemType = 10 +) + // buffer is a wrapper on top of bytes.Buffer. It extends the // original struct with methods for writing int64 and float64 // numbers without unnecessary allocations. @@ -90,6 +138,12 @@ func (b *buffer) ClearLastErr() { b.lastErr = nil } +func (b *buffer) SetLastErr(err error) { + if b.lastErr == nil { + b.lastErr = err + } +} + func (b *buffer) writeInt(i int64) { // We need up to 20 bytes to fit an int64, including a sign. var a [20]byte @@ -393,8 +447,8 @@ func (b *buffer) resetMsgFlags() { b.hasFields = false } -func (b *buffer) Messages() string { - return b.String() +func (b *buffer) Messages() []byte { + return b.Buffer.Bytes() } func (b *buffer) Table(name string) *buffer { @@ -517,6 +571,188 @@ func (b *buffer) Float64Column(name string, val float64) *buffer { return b } +func (b *buffer) Float64ColumnBinaryFormat(name string, val float64) *buffer { + if !b.prepareForField() { + return b + } + b.lastErr = b.writeColumnName(name) + if b.lastErr != nil { + return b + } + b.WriteByte('=') + // binary format flag + b.WriteByte('=') + b.WriteByte(byte(float64BinaryFlag)) + if isLittleEndian() { + b.Write((*[8]byte)(unsafe.Pointer(&val))[:]) + } else { + data := make([]byte, 8) + binary.LittleEndian.PutUint64(data, math.Float64bits(val)) + b.Write(data) + } + b.hasFields = true + return b +} + +func (b *buffer) writeFloat64ArrayHeader(dims byte) { + b.WriteByte('=') + b.WriteByte('=') + b.WriteByte(byte(arrayBinaryFlag)) + b.WriteByte(byte(arrayElemDouble)) + b.WriteByte(dims) +} + +func (b *buffer) Float641DArrayColumn(name string, values []float64) *buffer { + if !b.prepareForField() { + return b + } + b.lastErr = b.writeColumnName(name) + if b.lastErr != nil { + return b + } + + dim1 := len(values) + b.writeFloat64ArrayHeader(1) + + // Write shape + b.writeUint32(uint32(dim1)) + + // Write values + if len(values) > 0 { + b.writeFloat64Data(values) + } + + b.hasFields = true + return b +} + +func (b *buffer) Float642DArrayColumn(name string, values [][]float64) *buffer { + if !b.prepareForField() { + return b + } + b.lastErr = b.writeColumnName(name) + if b.lastErr != nil { + return b + } + + // Validate array shape + dim1 := len(values) + var dim2 int + if dim1 > 0 { + dim2 = len(values[0]) + for i, row := range values { + if len(row) != dim2 { + b.lastErr = fmt.Errorf("irregular 2D array shape: row %d has length %d, expected %d", i, len(row), dim2) + return b + } + } + } + + b.writeFloat64ArrayHeader(2) + + // Write shape + b.writeUint32(uint32(dim1)) + b.writeUint32(uint32(dim2)) + + // Write values + for _, row := range values { + if len(row) > 0 { + b.writeFloat64Data(row) + } + } + + b.hasFields = true + return b +} + +func (b *buffer) Float643DArrayColumn(name string, values [][][]float64) *buffer { + if !b.prepareForField() { + return b + } + b.lastErr = b.writeColumnName(name) + if b.lastErr != nil { + return b + } + + // Validate array shape + dim1 := len(values) + var dim2, dim3 int + if dim1 > 0 { + dim2 = len(values[0]) + if dim2 > 0 { + dim3 = len(values[0][0]) + } + + for i, level1 := range values { + if len(level1) != dim2 { + b.lastErr = fmt.Errorf("irregular 3D array shape: level1[%d] has length %d, expected %d", i, len(level1), dim2) + return b + } + for j, level2 := range level1 { + if len(level2) != dim3 { + b.lastErr = fmt.Errorf("irregular 3D array shape: level2[%d][%d] has length %d, expected %d", i, j, len(level2), dim3) + return b + } + } + } + } + + b.writeFloat64ArrayHeader(3) + + // Write shape + b.writeUint32(uint32(dim1)) + b.writeUint32(uint32(dim2)) + b.writeUint32(uint32(dim3)) + + // Write values + for _, level1 := range values { + for _, level2 := range level1 { + if len(level2) > 0 { + b.writeFloat64Data(level2) + } + } + } + + b.hasFields = true + return b +} + +func (b *buffer) Float64NDArrayColumn(name string, value *NdArray[float64]) *buffer { + if !b.prepareForField() { + return b + } + b.lastErr = b.writeColumnName(name) + if b.lastErr != nil { + return b + } + + // Validate the NdArray + if value == nil { + b.lastErr = fmt.Errorf("NDArray cannot be nil") + return b + } + + shape := value.Shape() + numDims := value.NDims() + + // Write nDims + b.writeFloat64ArrayHeader(byte(numDims)) + + // Write shape + for _, dim := range shape { + b.writeUint32(uint32(dim)) + } + + // Write data + data := value.GetData() + if len(data) > 0 { + b.writeFloat64Data(data) + } + + b.hasFields = true + return b +} + func (b *buffer) StringColumn(name, val string) *buffer { if !b.prepareForField() { return b diff --git a/buffer_test.go b/buffer_test.go index 515d9a9f..8e461c71 100644 --- a/buffer_test.go +++ b/buffer_test.go @@ -25,6 +25,7 @@ package questdb_test import ( + "encoding/binary" "math" "math/big" "strconv" @@ -46,7 +47,7 @@ func TestValidWrites(t *testing.T) { testCases := []struct { name string writerFn bufWriterFn - expectedLines []string + expectedLines [][]byte }{ { "multiple rows", @@ -61,9 +62,9 @@ func TestValidWrites(t *testing.T) { } return nil }, - []string{ - "my_test_table str_col=\"foo\",long_col=42i", - "my_test_table str_col=\"bar\",long_col=-42i 42000", + [][]byte{ + []byte("my_test_table str_col=\"foo\",long_col=42i"), + []byte("my_test_table str_col=\"bar\",long_col=-42i 42000"), }, }, { @@ -71,8 +72,8 @@ func TestValidWrites(t *testing.T) { func(s *qdb.Buffer) error { return s.Table("таблица").StringColumn("колонка", "значение").At(time.Time{}, false) }, - []string{ - "таблица колонка=\"значение\"", + [][]byte{ + []byte("таблица колонка=\"значение\""), }, }, } @@ -85,8 +86,15 @@ func TestValidWrites(t *testing.T) { assert.NoError(t, err) // Check the buffer - assert.Equal(t, strings.Join(tc.expectedLines, "\n")+"\n", buf.Messages()) - + var expectedLines []byte + for i, line := range tc.expectedLines { + if i != 0 { + expectedLines = append(expectedLines, '\n') + } + expectedLines = append(expectedLines, line...) + } + expectedLines = append(expectedLines, '\n') + assert.Equal(t, expectedLines, buf.Messages()) }) } } @@ -109,8 +117,8 @@ func TestTimestampSerialization(t *testing.T) { assert.NoError(t, err) // Check the buffer - expectedLines := []string{"my_test_table a_col=" + strconv.FormatInt(tc.val.UnixMicro(), 10) + "t"} - assert.Equal(t, strings.Join(expectedLines, "\n")+"\n", buf.Messages()) + expected := []byte("my_test_table a_col=" + strconv.FormatInt(tc.val.UnixMicro(), 10) + "t\n") + assert.Equal(t, expected, buf.Messages()) }) } } @@ -135,8 +143,8 @@ func TestInt64Serialization(t *testing.T) { assert.NoError(t, err) // Check the buffer - expectedLines := []string{"my_test_table a_col=" + strconv.FormatInt(tc.val, 10) + "i"} - assert.Equal(t, strings.Join(expectedLines, "\n")+"\n", buf.Messages()) + expected := []byte("my_test_table a_col=" + strconv.FormatInt(tc.val, 10) + "i\n") + assert.Equal(t, expected, buf.Messages()) }) } } @@ -163,8 +171,8 @@ func TestLong256Column(t *testing.T) { assert.NoError(t, err) // Check the buffer - expectedLines := []string{"my_test_table a_col=" + tc.expected + "i"} - assert.Equal(t, strings.Join(expectedLines, "\n")+"\n", buf.Messages()) + expected := []byte("my_test_table a_col=" + tc.expected + "i\n") + assert.Equal(t, expected, buf.Messages()) }) } } @@ -196,8 +204,8 @@ func TestFloat64Serialization(t *testing.T) { assert.NoError(t, err) // Check the buffer - expectedLines := []string{"my_test_table a_col=" + tc.expected} - assert.Equal(t, strings.Join(expectedLines, "\n")+"\n", buf.Messages()) + expected := []byte("my_test_table a_col=" + tc.expected + "\n") + assert.Equal(t, expected, buf.Messages()) }) } } @@ -428,8 +436,8 @@ func TestInvalidMessageGetsDiscarded(t *testing.T) { assert.Error(t, err) // The second message should be discarded. - expectedLines := []string{testTable + " foo=\"bar\""} - assert.Equal(t, strings.Join(expectedLines, "\n")+"\n", buf.Messages()) + expected := []byte(testTable + " foo=\"bar\"\n") + assert.Equal(t, expected, buf.Messages()) } func TestInvalidTableName(t *testing.T) { @@ -447,3 +455,286 @@ func TestInvalidColumnName(t *testing.T) { assert.ErrorContains(t, err, "column name contains an illegal char") assert.Empty(t, buf.Messages()) } + +func TestFloat64ColumnBinaryFormat(t *testing.T) { + testCases := []struct { + name string + val float64 + }{ + {"positive number", 42.3}, + {"negative number", -42.3}, + {"zero", 0.0}, + {"NaN", math.NaN()}, + {"positive infinity", math.Inf(1)}, + {"negative infinity", math.Inf(-1)}, + {"smallest value", math.SmallestNonzeroFloat64}, + {"max value", math.MaxFloat64}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + buf := newTestBuffer() + err := buf.Table(testTable).Float64ColumnBinaryFormat("a_col", tc.val).At(time.Time{}, false) + assert.NoError(t, err) + assert.Equal(t, buf.Messages(), float64ToByte(testTable, "a_col", tc.val)) + }) + } +} + +func TestFloat641DArrayColumn(t *testing.T) { + testCases := []struct { + name string + values []float64 + }{ + {"single value", []float64{42.5}}, + {"multiple values", []float64{1.1, 2.2, 3.3}}, + {"empty array", []float64{}}, + {"with special values", []float64{math.NaN(), math.Inf(1), math.Inf(-1), 0.0}}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + buf := newTestBuffer() + + err := buf.Table(testTable).Float641DArrayColumn("array_col", tc.values).At(time.Time{}, false) + assert.NoError(t, err) + assert.Equal(t, float641DArrayToByte(testTable, "array_col", tc.values), buf.Messages()) + }) + } +} + +func TestFloat642DArrayColumn(t *testing.T) { + testCases := []struct { + name string + values [][]float64 + }{ + {"2x2 array", [][]float64{{1.1, 2.2}, {3.3, 4.4}}}, + {"1x3 array", [][]float64{{1.0, 2.0, 3.0}}}, + {"3x1 array", [][]float64{{1.0}, {2.0}, {3.0}}}, + {"empty array", [][]float64{}}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + buf := newTestBuffer() + + err := buf.Table(testTable).Float642DArrayColumn("array_col", tc.values).At(time.Time{}, false) + assert.NoError(t, err) + assert.Equal(t, float642DArrayToByte(testTable, "array_col", tc.values), buf.Messages()) + }) + } +} + +func TestFloat642DArrayColumnIrregularShape(t *testing.T) { + buf := newTestBuffer() + + irregularArray := [][]float64{{1.0, 2.0}, {3.0, 4.0, 5.0}} + err := buf.Table(testTable).Float642DArrayColumn("array_col", irregularArray).At(time.Time{}, false) + + assert.ErrorContains(t, err, "irregular 2D array shape") + assert.Empty(t, buf.Messages()) +} + +func TestFloat643DArrayColumn(t *testing.T) { + testCases := []struct { + name string + values [][][]float64 + }{ + {"2x2x2 array", [][][]float64{{{1.1, 2.2}, {3.3, 4.4}}, {{5.5, 6.6}, {7.7, 8.8}}}}, + {"1x1x3 array", [][][]float64{{{1.0, 2.0, 3.0}}}}, + {"empty array", [][][]float64{}}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + buf := newTestBuffer() + + err := buf.Table(testTable).Float643DArrayColumn("array_col", tc.values).At(time.Time{}, false) + assert.NoError(t, err) + assert.Equal(t, float643DArrayToByte(testTable, "array_col", tc.values), buf.Messages()) + }) + } +} + +func TestFloat643DArrayColumnIrregularShape(t *testing.T) { + buf := newTestBuffer() + irregularArray := [][][]float64{{{1.0, 2.0}, {3.0}}, {{4.0, 5.0}, {6.0, 7.0}}} + err := buf.Table(testTable).Float643DArrayColumn("array_col", irregularArray).At(time.Time{}, false) + + assert.ErrorContains(t, err, "irregular 3D array shape") + assert.Empty(t, buf.Messages()) +} + +func TestFloat64NDArrayColumn(t *testing.T) { + testCases := []struct { + name string + shape []uint + data []float64 + }{ + {"1D array", []uint{3}, []float64{1.0, 2.0, 3.0}}, + {"2D array", []uint{2, 2}, []float64{1.0, 2.0, 3.0, 4.0}}, + {"3D array", []uint{2, 1, 2}, []float64{1.0, 2.0, 3.0, 4.0}}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + buf := newTestBuffer() + ndArray, err := qdb.NewNDArray[float64](tc.shape...) + assert.NoError(t, err) + for _, val := range tc.data { + ndArray.Append(val) + } + + err = buf.Table(testTable).Float64NDArrayColumn("ndarray_col", ndArray).At(time.Time{}, false) + assert.NoError(t, err) + assert.Equal(t, float64NDArrayToByte(testTable, "ndarray_col", ndArray), buf.Messages()) + }) + } +} + +func TestFloat64NDArrayColumnNil(t *testing.T) { + buf := newTestBuffer() + err := buf.Table(testTable).Float64NDArrayColumn("ndarray_col", nil).At(time.Time{}, false) + assert.ErrorContains(t, err, "NDArray cannot be nil") + assert.Empty(t, buf.Messages()) +} + +func float64ToByte(table, col string, val float64) []byte { + buf := make([]byte, 0, 128) + buf = append(buf, ([]byte)(table)...) + buf = append(buf, ' ') + buf = append(buf, ([]byte)(col)...) + buf = append(buf, '=') + buf = append(buf, '=') + buf = append(buf, 16) + buf1 := make([]byte, 8) + binary.LittleEndian.PutUint64(buf1, math.Float64bits(val)) + buf = append(buf, buf1...) + buf = append(buf, '\n') + return buf +} + +func float641DArrayToByte(table, col string, vals []float64) []byte { + buf := make([]byte, 0, 128) + buf = append(buf, ([]byte)(table)...) + buf = append(buf, ' ') + buf = append(buf, ([]byte)(col)...) + buf = append(buf, '=') + buf = append(buf, '=') + buf = append(buf, 14) + buf = append(buf, 10) + buf = append(buf, 1) + + shapeData := make([]byte, 4) + binary.LittleEndian.PutUint32(shapeData, uint32(len(vals))) + buf = append(buf, shapeData...) + + // Write values + for _, val := range vals { + valData := make([]byte, 8) + binary.LittleEndian.PutUint64(valData, math.Float64bits(val)) + buf = append(buf, valData...) + } + buf = append(buf, '\n') + return buf +} + +func float642DArrayToByte(table, col string, vals [][]float64) []byte { + buf := make([]byte, 0, 256) + buf = append(buf, ([]byte)(table)...) + buf = append(buf, ' ') + buf = append(buf, ([]byte)(col)...) + buf = append(buf, '=') + buf = append(buf, '=') + buf = append(buf, 14) + buf = append(buf, 10) + buf = append(buf, 2) + + dim1 := len(vals) + var dim2 int + if dim1 > 0 { + dim2 = len(vals[0]) + } + + shapeData := make([]byte, 8) + binary.LittleEndian.PutUint32(shapeData[:4], uint32(dim1)) + binary.LittleEndian.PutUint32(shapeData[4:8], uint32(dim2)) + buf = append(buf, shapeData...) + + for _, row := range vals { + for _, val := range row { + valData := make([]byte, 8) + binary.LittleEndian.PutUint64(valData, math.Float64bits(val)) + buf = append(buf, valData...) + } + } + buf = append(buf, '\n') + return buf +} + +func float643DArrayToByte(table, col string, vals [][][]float64) []byte { + buf := make([]byte, 0, 512) + buf = append(buf, ([]byte)(table)...) + buf = append(buf, ' ') + buf = append(buf, ([]byte)(col)...) + buf = append(buf, '=') + buf = append(buf, '=') + buf = append(buf, 14) + buf = append(buf, 10) + buf = append(buf, 3) + + dim1 := len(vals) + var dim2, dim3 int + if dim1 > 0 { + dim2 = len(vals[0]) + if dim2 > 0 { + dim3 = len(vals[0][0]) + } + } + + shapeData := make([]byte, 12) + binary.LittleEndian.PutUint32(shapeData[:4], uint32(dim1)) + binary.LittleEndian.PutUint32(shapeData[4:8], uint32(dim2)) + binary.LittleEndian.PutUint32(shapeData[8:12], uint32(dim3)) + buf = append(buf, shapeData...) + + for _, level1 := range vals { + for _, level2 := range level1 { + for _, val := range level2 { + valData := make([]byte, 8) + binary.LittleEndian.PutUint64(valData, math.Float64bits(val)) + buf = append(buf, valData...) + } + } + } + buf = append(buf, '\n') + return buf +} + +func float64NDArrayToByte(table, col string, ndarray *qdb.NdArray[float64]) []byte { + buf := make([]byte, 0, 512) + buf = append(buf, ([]byte)(table)...) + buf = append(buf, ' ') + buf = append(buf, ([]byte)(col)...) + buf = append(buf, '=') + buf = append(buf, '=') + buf = append(buf, 14) + buf = append(buf, 10) + buf = append(buf, byte(ndarray.NDims())) + + shape := ndarray.Shape() + for _, dim := range shape { + shapeData := make([]byte, 4) + binary.LittleEndian.PutUint32(shapeData, uint32(dim)) + buf = append(buf, shapeData...) + } + + data := ndarray.GetData() + for _, val := range data { + valData := make([]byte, 8) + binary.LittleEndian.PutUint64(valData, math.Float64bits(val)) + buf = append(buf, valData...) + } + buf = append(buf, '\n') + return buf +} diff --git a/conf_parse.go b/conf_parse.go index 5337a145..2f186974 100644 --- a/conf_parse.go +++ b/conf_parse.go @@ -162,6 +162,18 @@ func confFromStr(conf string) (*lineSenderConfig, error) { return nil, NewInvalidConfigStrError("tls_roots is not available in the go client") case "tls_roots_password": return nil, NewInvalidConfigStrError("tls_roots_password is not available in the go client") + case "protocol_version": + if v != "auto" { + version, err := strconv.Atoi(v) + if err != nil { + return nil, NewInvalidConfigStrError("invalid %s value, %q is not a valid int", k, v) + } + pVersion := protocolVersion(version) + if pVersion < ProtocolVersion1 || pVersion > ProtocolVersion2 { + return nil, NewInvalidConfigStrError("current client only supports protocol version 1(text format for all datatypes), 2(binary format for part datatypes) or explicitly unset") + } + senderConf.protocolVersion = pVersion + } default: return nil, NewInvalidConfigStrError("unsupported option %q", k) } diff --git a/conf_test.go b/conf_test.go index 4aab4a9c..37018dd6 100644 --- a/conf_test.go +++ b/conf_test.go @@ -201,6 +201,17 @@ func TestParserHappyCases(t *testing.T) { }, }, }, + { + name: "protocol version", + config: fmt.Sprintf("http::addr=%s;protocol_version=1;", addr), + expected: qdb.ConfigData{ + Schema: "http", + KeyValuePairs: map[string]string{ + "addr": addr, + "protocol_version": "1", + }, + }, + }, { name: "equal sign in password", config: fmt.Sprintf("http::addr=%s;username=%s;password=pass=word;", addr, user), @@ -288,15 +299,16 @@ type configTestCase struct { func TestHappyCasesFromConf(t *testing.T) { var ( - addr = "localhost:1111" - user = "test-user" - pass = "test-pass" - token = "test-token" - minThroughput = 999 - requestTimeout = time.Second * 88 - retryTimeout = time.Second * 99 - initBufSize = 256 - maxBufSize = 1024 + addr = "localhost:1111" + user = "test-user" + pass = "test-pass" + token = "test-token" + minThroughput = 999 + requestTimeout = time.Second * 88 + retryTimeout = time.Second * 99 + initBufSize = 256 + maxBufSize = 1024 + protocolVersion = qdb.ProtocolVersion2 ) testCases := []configTestCase{ @@ -382,6 +394,16 @@ func TestHappyCasesFromConf(t *testing.T) { qdb.WithMinThroughput(minThroughput), }, }, + { + name: "protocol_version", + config: fmt.Sprintf("http::addr=%s;protocol_version=%d;", + addr, protocolVersion), + expectedOpts: []qdb.LineSenderOption{ + qdb.WithHttp(), + qdb.WithAddress(addr), + qdb.WithProtocolVersion(protocolVersion), + }, + }, { name: "bearer token", config: fmt.Sprintf("http::addr=%s;token=%s", diff --git a/examples/from-conf/main.go b/examples/from-conf/main.go index 39ae0949..fa78cd55 100644 --- a/examples/from-conf/main.go +++ b/examples/from-conf/main.go @@ -27,12 +27,28 @@ func main() { if err != nil { log.Fatal(err) } + // Prepare array data. + // QuestDB server version 9.0.0 or later is required for array support. + array, err := qdb.NewNDArray[float64](2, 3, 2) + if err != nil { + log.Fatal(err) + } + hasMore := true + val := 100.0 + for hasMore { + hasMore, err = array.Append(val + 1) + val = val + 1 + if err != nil { + log.Fatal(err) + } + } err = sender. Table("trades"). Symbol("symbol", "ETH-USD"). Symbol("side", "sell"). Float64Column("price", 2615.54). Float64Column("amount", 0.00044). + Float64NDArrayColumn("price_history", array). At(ctx, tradedTs) if err != nil { log.Fatal(err) @@ -42,12 +58,25 @@ func main() { if err != nil { log.Fatal(err) } + + // Reuse array + hasMore = true + array.ResetAppendIndex() + val = 200.0 + for hasMore { + hasMore, err = array.Append(val + 1) + val = val + 1 + if err != nil { + log.Fatal(err) + } + } err = sender. Table("trades"). Symbol("symbol", "BTC-USD"). Symbol("side", "sell"). Float64Column("price", 39269.98). Float64Column("amount", 0.001). + Float64NDArrayColumn("price_history", array). At(ctx, tradedTs) if err != nil { log.Fatal(err) diff --git a/examples/http/basic/main.go b/examples/http/basic/main.go index 4d84f17d..130ee479 100644 --- a/examples/http/basic/main.go +++ b/examples/http/basic/main.go @@ -28,17 +28,46 @@ func main() { if err != nil { log.Fatal(err) } + + // Prepare array data. + // QuestDB server version 9.0.0 or later is required for array support. + array, err := qdb.NewNDArray[float64](2, 3, 2) + if err != nil { + log.Fatal(err) + } + hasMore := true + val := 100.0 + for hasMore { + hasMore, err = array.Append(val + 1) + val = val + 1 + if err != nil { + log.Fatal(err) + } + } + err = sender. Table("trades"). Symbol("symbol", "ETH-USD"). Symbol("side", "sell"). Float64Column("price", 2615.54). Float64Column("amount", 0.00044). + Float64NDArrayColumn("price_history", array). At(ctx, tradedTs) if err != nil { log.Fatal(err) } + // Reuse array + hasMore = true + array.ResetAppendIndex() + val = 200.0 + for hasMore { + hasMore, err = array.Append(val + 1) + val = val + 1 + if err != nil { + log.Fatal(err) + } + } tradedTs, err = time.Parse(time.RFC3339, "2022-08-06T15:04:06.987654Z") if err != nil { log.Fatal(err) @@ -49,6 +78,7 @@ func main() { Symbol("side", "sell"). Float64Column("price", 39269.98). Float64Column("amount", 0.001). + Float64NDArrayColumn("price_history", array). At(ctx, tradedTs) if err != nil { log.Fatal(err) diff --git a/examples/tcp/basic/main.go b/examples/tcp/basic/main.go index 716c4de8..6ce7ff4a 100644 --- a/examples/tcp/basic/main.go +++ b/examples/tcp/basic/main.go @@ -11,7 +11,7 @@ import ( func main() { ctx := context.TODO() // Connect to QuestDB running on 127.0.0.1:9009 - sender, err := qdb.NewLineSender(ctx, qdb.WithTcp()) + sender, err := qdb.NewLineSender(ctx, qdb.WithTcp(), qdb.WithProtocolVersion(qdb.ProtocolVersion2)) if err != nil { log.Fatal(err) } @@ -23,17 +23,47 @@ func main() { if err != nil { log.Fatal(err) } + + // Prepare array data. + // QuestDB server version 9.0.0 or later is required for array support. + array, err := qdb.NewNDArray[float64](2, 3, 2) + if err != nil { + log.Fatal(err) + } + hasMore := true + val := 100.0 + for hasMore { + hasMore, err = array.Append(val + 1) + val = val + 1 + if err != nil { + log.Fatal(err) + } + } + err = sender. Table("trades"). Symbol("symbol", "ETH-USD"). Symbol("side", "sell"). Float64Column("price", 2615.54). Float64Column("amount", 0.00044). + Float64NDArrayColumn("price_history", array). At(ctx, tradedTs) if err != nil { log.Fatal(err) } + // Reuse array + hasMore = true + array.ResetAppendIndex() + val = 200.0 + for hasMore { + hasMore, err = array.Append(val + 1) + val = val + 1 + if err != nil { + log.Fatal(err) + } + } + tradedTs, err = time.Parse(time.RFC3339, "2022-08-06T15:04:06.987654Z") if err != nil { log.Fatal(err) @@ -44,6 +74,7 @@ func main() { Symbol("side", "sell"). Float64Column("price", 39269.98). Float64Column("amount", 0.001). + Float64NDArrayColumn("price_history", array). At(ctx, tradedTs) if err != nil { log.Fatal(err) diff --git a/export_test.go b/export_test.go index eebd655a..21cce501 100644 --- a/export_test.go +++ b/export_test.go @@ -53,37 +53,79 @@ func ConfFromStr(conf string) (*LineSenderConfig, error) { return confFromStr(conf) } -func Messages(s LineSender) string { +func Messages(s LineSender) []byte { + if ps, ok := s.(*pooledSender); ok { + s = ps.wrapped + } if hs, ok := s.(*httpLineSender); ok { return hs.Messages() } + if hs, ok := s.(*httpLineSenderV2); ok { + return hs.Messages() + } if ts, ok := s.(*tcpLineSender); ok { return ts.Messages() } + if ts, ok := s.(*tcpLineSenderV2); ok { + return ts.Messages() + } panic("unexpected struct") } func MsgCount(s LineSender) int { if ps, ok := s.(*pooledSender); ok { - hs, _ := ps.wrapped.(*httpLineSender) - return hs.MsgCount() + s = ps } if hs, ok := s.(*httpLineSender); ok { return hs.MsgCount() } + if hs, ok := s.(*httpLineSenderV2); ok { + return hs.MsgCount() + } if ts, ok := s.(*tcpLineSender); ok { return ts.MsgCount() } + if ts, ok := s.(*tcpLineSenderV2); ok { + return ts.MsgCount() + } panic("unexpected struct") } func BufLen(s LineSender) int { + if ps, ok := s.(*pooledSender); ok { + s = ps + } if hs, ok := s.(*httpLineSender); ok { return hs.BufLen() } + if hs, ok := s.(*httpLineSenderV2); ok { + return hs.BufLen() + } if ts, ok := s.(*tcpLineSender); ok { return ts.BufLen() } + if ts, ok := s.(*tcpLineSenderV2); ok { + return ts.BufLen() + } + panic("unexpected struct") +} + +func ProtocolVersion(s LineSender) protocolVersion { + if ps, ok := s.(*pooledSender); ok { + s = ps + } + if _, ok := s.(*httpLineSender); ok { + return ProtocolVersion1 + } + if _, ok := s.(*httpLineSenderV2); ok { + return ProtocolVersion2 + } + if _, ok := s.(*tcpLineSender); ok { + return ProtocolVersion1 + } + if _, ok := s.(*tcpLineSenderV2); ok { + return ProtocolVersion2 + } panic("unexpected struct") } diff --git a/http_sender.go b/http_sender.go index 21429610..ab44388c 100644 --- a/http_sender.go +++ b/http_sender.go @@ -29,6 +29,7 @@ import ( "context" "crypto/tls" "encoding/json" + "errors" "fmt" "io" "math/big" @@ -114,9 +115,23 @@ type httpLineSender struct { globalTransport *globalHttpTransport } -func newHttpLineSender(conf *lineSenderConfig) (*httpLineSender, error) { - var transport *http.Transport +type httpLineSenderV2 struct { + httpLineSender +} + +func newHttpLineSender(ctx context.Context, conf *lineSenderConfig) (LineSender, error) { + + // auto detect server line protocol version + pVersion := conf.protocolVersion + if pVersion == protocolVersionUnset { + var err error + pVersion, err = detectProtocolVersion(ctx, conf) + if err != nil { + return nil, err + } + } + var transport *http.Transport s := &httpLineSender{ address: conf.address, minThroughputBytesPerSecond: conf.minThroughput, @@ -161,7 +176,13 @@ func newHttpLineSender(conf *lineSenderConfig) (*httpLineSender, error) { } s.uri += fmt.Sprintf("://%s/write", s.address) - return s, nil + if pVersion == ProtocolVersion1 { + return s, nil + } else { + return &httpLineSenderV2{ + *s, + }, nil + } } func (s *httpLineSender) Flush(ctx context.Context) error { @@ -282,6 +303,26 @@ func (s *httpLineSender) BoolColumn(name string, val bool) LineSender { return s } +func (s *httpLineSender) Float641DArrayColumn(name string, values []float64) LineSender { + s.buf.SetLastErr(errors.New("current protocol version does not support double-array")) + return s +} + +func (s *httpLineSender) Float642DArrayColumn(name string, values [][]float64) LineSender { + s.buf.SetLastErr(errors.New("current protocol version does not support double-array")) + return s +} + +func (s *httpLineSender) Float643DArrayColumn(name string, values [][][]float64) LineSender { + s.buf.SetLastErr(errors.New("current protocol version does not support double-array")) + return s +} + +func (s *httpLineSender) Float64NDArrayColumn(name string, values *NdArray[float64]) LineSender { + s.buf.SetLastErr(errors.New("current protocol version does not support double-array")) + return s +} + func (s *httpLineSender) Close(ctx context.Context) error { if s.closed { return errDoubleSenderClose @@ -399,6 +440,90 @@ func (s *httpLineSender) makeRequest(ctx context.Context) (bool, error) { } +func detectProtocolVersion(ctx context.Context, conf *lineSenderConfig) (protocolVersion, error) { + tmpClient := http.Client{ + Transport: globalTransport.transport, + Timeout: 0, + } + globalTransport.RegisterClient() + defer globalTransport.UnregisterClient() + + scheme := "http" + if conf.tlsMode != tlsDisabled { + scheme = "https" + } + settingsUri := fmt.Sprintf("%s://%s/settings", scheme, conf.address) + + req, err := http.NewRequest(http.MethodGet, settingsUri, nil) + if err != nil { + return protocolVersionUnset, err + } + + reqCtx, cancel := context.WithTimeout(ctx, conf.requestTimeout) + defer cancel() + req = req.WithContext(reqCtx) + + resp, err := tmpClient.Do(req) + if err != nil { + return protocolVersionUnset, err + } + defer resp.Body.Close() + + switch resp.StatusCode { + case 404: + return ProtocolVersion1, nil + case 200: + return parseServerSettings(resp, conf) + default: + buf, _ := io.ReadAll(resp.Body) + return protocolVersionUnset, fmt.Errorf("failed to detect server line protocol version [http-status=%d, http-message=%s]", + resp.StatusCode, string(buf)) + } +} + +func parseServerSettings(resp *http.Response, conf *lineSenderConfig) (protocolVersion, error) { + buf, err := io.ReadAll(resp.Body) + if err != nil { + return protocolVersionUnset, fmt.Errorf("%d: %s", resp.StatusCode, resp.Status) + } + + var settings struct { + Config struct { + LineProtoSupportVersions []int `json:"line.proto.support.versions"` + MaxFileNameLength int `json:"cairo.max.file.name.length"` + } `json:"config"` + } + + if err := json.Unmarshal(buf, &settings); err != nil { + return ProtocolVersion1, nil + } + + // Update file name limit if provided by server + if settings.Config.MaxFileNameLength != 0 { + conf.fileNameLimit = settings.Config.MaxFileNameLength + } + + // Determine protocol version based on server support + versions := settings.Config.LineProtoSupportVersions + if len(versions) == 0 { + return ProtocolVersion1, nil + } + + for _, version := range versions { + if version == 2 { + return ProtocolVersion2, nil + } + } + + for _, version := range versions { + if version == 1 { + return ProtocolVersion1, nil + } + } + + return protocolVersionUnset, errors.New("server does not support current client") +} + func isRetryableError(statusCode int) bool { switch statusCode { case 500, // Internal Server Error @@ -416,9 +541,9 @@ func isRetryableError(statusCode int) bool { } } -// Messages returns a copy of accumulated ILP messages that are not +// Messages returns the accumulated ILP messages that are not // flushed to the TCP connection yet. Useful for debugging purposes. -func (s *httpLineSender) Messages() string { +func (s *httpLineSender) Messages() []byte { return s.buf.Messages() } @@ -431,3 +556,63 @@ func (s *httpLineSender) MsgCount() int { func (s *httpLineSender) BufLen() int { return s.buf.Len() } + +func (s *httpLineSenderV2) Table(name string) LineSender { + s.buf.Table(name) + return s +} + +func (s *httpLineSenderV2) Symbol(name, val string) LineSender { + s.buf.Symbol(name, val) + return s +} + +func (s *httpLineSenderV2) Int64Column(name string, val int64) LineSender { + s.buf.Int64Column(name, val) + return s +} + +func (s *httpLineSenderV2) Long256Column(name string, val *big.Int) LineSender { + s.buf.Long256Column(name, val) + return s +} + +func (s *httpLineSenderV2) TimestampColumn(name string, ts time.Time) LineSender { + s.buf.TimestampColumn(name, ts) + return s +} + +func (s *httpLineSenderV2) StringColumn(name, val string) LineSender { + s.buf.StringColumn(name, val) + return s +} + +func (s *httpLineSenderV2) BoolColumn(name string, val bool) LineSender { + s.buf.BoolColumn(name, val) + return s +} + +func (s *httpLineSenderV2) Float64Column(name string, val float64) LineSender { + s.buf.Float64ColumnBinaryFormat(name, val) + return s +} + +func (s *httpLineSenderV2) Float641DArrayColumn(name string, values []float64) LineSender { + s.buf.Float641DArrayColumn(name, values) + return s +} + +func (s *httpLineSenderV2) Float642DArrayColumn(name string, values [][]float64) LineSender { + s.buf.Float642DArrayColumn(name, values) + return s +} + +func (s *httpLineSenderV2) Float643DArrayColumn(name string, values [][][]float64) LineSender { + s.buf.Float643DArrayColumn(name, values) + return s +} + +func (s *httpLineSenderV2) Float64NDArrayColumn(name string, values *NdArray[float64]) LineSender { + s.buf.Float64NDArrayColumn(name, values) + return s +} diff --git a/http_sender_test.go b/http_sender_test.go index 67d12adb..af2b2cdf 100644 --- a/http_sender_test.go +++ b/http_sender_test.go @@ -57,27 +57,27 @@ func TestHttpHappyCasesFromConf(t *testing.T) { testCases := []httpConfigTestCase{ { name: "request_timeout and retry_timeout milli conversion", - config: fmt.Sprintf("http::addr=%s;request_timeout=%d;retry_timeout=%d;", + config: fmt.Sprintf("http::addr=%s;request_timeout=%d;retry_timeout=%d;protocol_version=2;", addr, request_timeout.Milliseconds(), retry_timeout.Milliseconds()), }, { name: "pass before user", - config: fmt.Sprintf("http::addr=%s;password=%s;username=%s;", + config: fmt.Sprintf("http::addr=%s;password=%s;username=%s;protocol_version=2;", addr, pass, user), }, { name: "request_min_throughput", - config: fmt.Sprintf("http::addr=%s;request_min_throughput=%d;", + config: fmt.Sprintf("http::addr=%s;request_min_throughput=%d;protocol_version=2;", addr, min_throughput), }, { name: "bearer token", - config: fmt.Sprintf("http::addr=%s;token=%s;", + config: fmt.Sprintf("http::addr=%s;token=%s;protocol_version=2;", addr, token), }, { name: "auto flush", - config: fmt.Sprintf("http::addr=%s;auto_flush_rows=100;auto_flush_interval=1000;", + config: fmt.Sprintf("http::addr=%s;auto_flush_rows=100;auto_flush_interval=1000;protocol_version=2;", addr), }, } @@ -100,11 +100,11 @@ func TestHttpHappyCasesFromEnv(t *testing.T) { testCases := []httpConfigTestCase{ { name: "addr only", - config: fmt.Sprintf("http::addr=%s", addr), + config: fmt.Sprintf("http::addr=%s;protocol_version=1;", addr), }, { name: "auto flush", - config: fmt.Sprintf("http::addr=%s;auto_flush_rows=100;auto_flush_interval=1000;", + config: fmt.Sprintf("http::addr=%s;auto_flush_rows=100;auto_flush_interval=1000;protocol_version=2;", addr), }, } @@ -168,6 +168,11 @@ func TestHttpPathologicalCasesFromConf(t *testing.T) { config: "hTtp::addr=localhost:1234;", expectedErr: "invalid schema", }, + { + name: "protocol version", + config: "http::protocol_version=abc;", + expectedErr: "invalid protocol_version value", + }, } for _, tc := range testCases { @@ -729,13 +734,13 @@ func TestBufferClearAfterFlush(t *testing.T) { func TestCustomTransportAndTlsInit(t *testing.T) { ctx := context.Background() - s1, err := qdb.NewLineSender(ctx, qdb.WithHttp()) + s1, err := qdb.NewLineSender(ctx, qdb.WithHttp(), qdb.WithProtocolVersion(qdb.ProtocolVersion1)) assert.NoError(t, err) - s2, err := qdb.NewLineSender(ctx, qdb.WithHttp(), qdb.WithTls()) + s2, err := qdb.NewLineSender(ctx, qdb.WithHttp(), qdb.WithTls(), qdb.WithProtocolVersion(qdb.ProtocolVersion2)) assert.NoError(t, err) - s3, err := qdb.NewLineSender(ctx, qdb.WithHttp(), qdb.WithTlsInsecureSkipVerify()) + s3, err := qdb.NewLineSender(ctx, qdb.WithHttp(), qdb.WithTlsInsecureSkipVerify(), qdb.WithProtocolVersion(qdb.ProtocolVersion2)) assert.NoError(t, err) transport := http.Transport{} @@ -744,6 +749,7 @@ func TestCustomTransportAndTlsInit(t *testing.T) { qdb.WithHttp(), qdb.WithHttpTransport(&transport), qdb.WithTls(), + qdb.WithProtocolVersion(qdb.ProtocolVersion2), ) assert.NoError(t, err) @@ -763,6 +769,133 @@ func TestCustomTransportAndTlsInit(t *testing.T) { assert.Equal(t, int64(0), qdb.GlobalTransport.ClientCount()) } +func TestAutoDetectProtocolVersionOldServer1(t *testing.T) { + ctx := context.Background() + + srv, err := newTestServerWithProtocol(readAndDiscard, "http", nil) + assert.NoError(t, err) + defer srv.Close() + sender, err := qdb.NewLineSender(ctx, qdb.WithHttp(), qdb.WithAddress(srv.Addr())) + assert.Equal(t, qdb.ProtocolVersion(sender), qdb.ProtocolVersion1) +} + +func TestAutoDetectProtocolVersionOldServer2(t *testing.T) { + ctx := context.Background() + + srv, err := newTestServerWithProtocol(readAndDiscard, "http", []int{}) + assert.NoError(t, err) + defer srv.Close() + sender, err := qdb.NewLineSender(ctx, qdb.WithHttp(), qdb.WithAddress(srv.Addr())) + assert.Equal(t, qdb.ProtocolVersion(sender), qdb.ProtocolVersion1) +} + +func TestAutoDetectProtocolVersionOldServer3(t *testing.T) { + ctx := context.Background() + + srv, err := newTestServerWithProtocol(readAndDiscard, "http", []int{1}) + assert.NoError(t, err) + defer srv.Close() + sender, err := qdb.NewLineSender(ctx, qdb.WithHttp(), qdb.WithAddress(srv.Addr())) + assert.Equal(t, qdb.ProtocolVersion(sender), qdb.ProtocolVersion1) +} + +func TestAutoDetectProtocolVersionNewServer1(t *testing.T) { + ctx := context.Background() + + srv, err := newTestServerWithProtocol(readAndDiscard, "http", []int{1, 2}) + assert.NoError(t, err) + defer srv.Close() + sender, err := qdb.NewLineSender(ctx, qdb.WithHttp(), qdb.WithAddress(srv.Addr())) + assert.Equal(t, qdb.ProtocolVersion(sender), qdb.ProtocolVersion2) +} + +func TestAutoDetectProtocolVersionNewServer2(t *testing.T) { + ctx := context.Background() + + srv, err := newTestServerWithProtocol(readAndDiscard, "http", []int{2}) + assert.NoError(t, err) + defer srv.Close() + sender, err := qdb.NewLineSender(ctx, qdb.WithHttp(), qdb.WithAddress(srv.Addr())) + assert.Equal(t, qdb.ProtocolVersion(sender), qdb.ProtocolVersion2) +} + +func TestAutoDetectProtocolVersionNewServer3(t *testing.T) { + ctx := context.Background() + + srv, err := newTestServerWithProtocol(readAndDiscard, "http", []int{2, 3}) + assert.NoError(t, err) + defer srv.Close() + sender, err := qdb.NewLineSender(ctx, qdb.WithHttp(), qdb.WithAddress(srv.Addr())) + assert.Equal(t, qdb.ProtocolVersion(sender), qdb.ProtocolVersion2) +} + +func TestAutoDetectProtocolVersionNewServer4(t *testing.T) { + ctx := context.Background() + + srv, err := newTestServerWithProtocol(readAndDiscard, "http", []int{3}) + assert.NoError(t, err) + defer srv.Close() + _, err = qdb.NewLineSender(ctx, qdb.WithHttp(), qdb.WithAddress(srv.Addr())) + assert.ErrorContains(t, err, "server does not support current client") +} + +func TestSpecifyProtocolVersion(t *testing.T) { + ctx := context.Background() + + srv, err := newTestServerWithProtocol(readAndDiscard, "http", []int{1, 2}) + assert.NoError(t, err) + defer srv.Close() + sender, err := qdb.NewLineSender(ctx, qdb.WithHttp(), qdb.WithAddress(srv.Addr()), qdb.WithProtocolVersion(qdb.ProtocolVersion1)) + assert.Equal(t, qdb.ProtocolVersion(sender), qdb.ProtocolVersion1) +} + +func TestArrayColumnUnsupportedInHttpProtocolV1(t *testing.T) { + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(50*time.Millisecond)) + defer cancel() + + srv, err := newTestServerWithProtocol(readAndDiscard, "http", []int{1}) + assert.NoError(t, err) + defer srv.Close() + sender, err := qdb.NewLineSender(ctx, qdb.WithHttp(), qdb.WithAddress(srv.Addr())) + assert.NoError(t, err) + defer sender.Close(ctx) + + values1D := []float64{1.0, 2.0, 3.0, 4.0, 5.0} + values2D := [][]float64{{1.0, 2.0}, {3.0, 4.0}, {5.0, 6.0}} + values3D := [][][]float64{{{1.0, 2.0}, {3.0, 4.0}}, {{5.0, 6.0}, {7.0, 8.0}}} + arrayND, err := qdb.NewNDArray[float64](2, 2, 1, 2) + assert.NoError(t, err) + arrayND.Fill(11.0) + + err = sender. + Table(testTable). + Float641DArrayColumn("array_1d", values1D). + At(ctx, time.UnixMicro(1)) + assert.Error(t, err) + assert.Contains(t, err.Error(), "current protocol version does not support double-array") + + err = sender. + Table(testTable). + Float642DArrayColumn("array_2d", values2D). + At(ctx, time.UnixMicro(2)) + assert.Error(t, err) + assert.Contains(t, err.Error(), "current protocol version does not support double-array") + + err = sender. + Table(testTable). + Float643DArrayColumn("array_3d", values3D). + At(ctx, time.UnixMicro(3)) + assert.Error(t, err) + assert.Contains(t, err.Error(), "current protocol version does not support double-array") + + err = sender. + Table(testTable). + Float64NDArrayColumn("array_nd", arrayND). + At(ctx, time.UnixMicro(4)) + assert.Error(t, err) + assert.Contains(t, err.Error(), "current protocol version does not support double-array") +} + func BenchmarkHttpLineSenderBatch1000(b *testing.B) { ctx := context.Background() @@ -773,6 +906,12 @@ func BenchmarkHttpLineSenderBatch1000(b *testing.B) { sender, err := qdb.NewLineSender(ctx, qdb.WithHttp(), qdb.WithAddress(srv.Addr())) assert.NoError(b, err) + values1D := []float64{1.0, 2.0, 3.0, 4.0, 5.0} + values2D := [][]float64{{1.0, 2.0}, {3.0, 4.0}, {5.0, 6.0}} + values3D := [][][]float64{{{1.0, 2.0}, {3.0, 4.0}}, {{5.0, 6.0}, {7.0, 8.0}}} + arrayND, _ := qdb.NewNDArray[float64](2, 3) + arrayND.Fill(10.0) + b.ResetTimer() for i := 0; i < b.N; i++ { for j := 0; j < 1000; j++ { @@ -784,6 +923,10 @@ func BenchmarkHttpLineSenderBatch1000(b *testing.B) { StringColumn("str_col", "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua"). BoolColumn("bool_col", true). TimestampColumn("timestamp_col", time.UnixMicro(42)). + Float641DArrayColumn("array_1d", values1D). + Float642DArrayColumn("array_2d", values2D). + Float643DArrayColumn("array_3d", values3D). + Float64NDArrayColumn("array_nd", arrayND). At(ctx, time.UnixMicro(int64(1000*i))) } sender.Flush(ctx) @@ -801,6 +944,12 @@ func BenchmarkHttpLineSenderNoFlush(b *testing.B) { sender, err := qdb.NewLineSender(ctx, qdb.WithHttp(), qdb.WithAddress(srv.Addr())) assert.NoError(b, err) + values1D := []float64{1.0, 2.0, 3.0, 4.0, 5.0} + values2D := [][]float64{{1.0, 2.0}, {3.0, 4.0}, {5.0, 6.0}} + values3D := [][][]float64{{{1.0, 2.0}, {3.0, 4.0}}, {{5.0, 6.0}, {7.0, 8.0}}} + arrayND, _ := qdb.NewNDArray[float64](2, 3) + arrayND.Fill(10) + b.ResetTimer() for i := 0; i < b.N; i++ { sender. @@ -811,6 +960,10 @@ func BenchmarkHttpLineSenderNoFlush(b *testing.B) { StringColumn("str_col", "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua"). BoolColumn("bool_col", true). TimestampColumn("timestamp_col", time.UnixMicro(42)). + Float641DArrayColumn("array_1d", values1D). + Float642DArrayColumn("array_2d", values2D). + Float643DArrayColumn("array_3d", values3D). + Float64NDArrayColumn("array_nd", arrayND). At(ctx, time.UnixMicro(int64(1000*i))) } sender.Flush(ctx) diff --git a/integration_test.go b/integration_test.go index 09df41d5..2442db64 100644 --- a/integration_test.go +++ b/integration_test.go @@ -132,7 +132,7 @@ func setupQuestDB0(ctx context.Context, auth ilpAuthType, setupProxy bool) (*que return nil, err } req := testcontainers.ContainerRequest{ - Image: "questdb/questdb:7.4.2", + Image: "questdb/questdb:9.0.1", ExposedPorts: []string{"9000/tcp", "9009/tcp"}, WaitingFor: wait.ForHTTP("/").WithPort("9000"), Networks: []string{networkName}, @@ -298,7 +298,7 @@ func (suite *integrationTestSuite) TestE2EValidWrites() { {"double_col", "DOUBLE"}, {"long_col", "LONG"}, {"long256_col", "LONG256"}, - {"str_col", "STRING"}, + {"str_col", "VARCHAR"}, {"bool_col", "BOOLEAN"}, {"timestamp_col", "TIMESTAMP"}, {"timestamp", "TIMESTAMP"}, @@ -312,10 +312,10 @@ func (suite *integrationTestSuite) TestE2EValidWrites() { }, { "escaped chars", - "my-awesome_test 1=2.csv", + "m y-awesome_test 1=2.csv", func(s qdb.LineSender) error { return s. - Table("my-awesome_test 1=2.csv"). + Table("m y-awesome_test 1=2.csv"). Symbol("sym_name 1=2", "value 1,2=3\n4\r5\"6\\7"). StringColumn("str_name 1=2", "value 1,2=3\n4\r5\"6\\7"). At(ctx, time.UnixMicro(1)) @@ -323,7 +323,7 @@ func (suite *integrationTestSuite) TestE2EValidWrites() { tableData{ Columns: []column{ {"sym_name 1=2", "SYMBOL"}, - {"str_name 1=2", "STRING"}, + {"str_name 1=2", "VARCHAR"}, {"timestamp", "TIMESTAMP"}, }, Dataset: [][]interface{}{ @@ -413,44 +413,103 @@ func (suite *integrationTestSuite) TestE2EValidWrites() { Count: 1, }, }, + { + "double array", + testTable, + func(s qdb.LineSender) error { + values1D := []float64{1.0, 2.0, 3.0, 4.0, 5.0} + values2D := [][]float64{{1.0, 2.0}, {3.0, 4.0}, {5.0, 6.0}} + values3D := [][][]float64{{{1.0, 2.0}, {3.0, 4.0}}, {{5.0, 6.0}, {7.0, 8.0}}} + arrayND, _ := qdb.NewNDArray[float64](2, 2, 1, 2) + arrayND.Fill(11.0) + + return s. + Table(testTable). + Float641DArrayColumn("array_1d", values1D). + Float642DArrayColumn("array_2d", values2D). + Float643DArrayColumn("array_3d", values3D). + Float64NDArrayColumn("array_nd", arrayND). + At(ctx, time.UnixMicro(1)) + }, + tableData{ + Columns: []column{ + {"array_1d", "ARRAY"}, + {"array_2d", "ARRAY"}, + {"array_3d", "ARRAY"}, + {"array_nd", "ARRAY"}, + {"timestamp", "TIMESTAMP"}, + }, + Dataset: [][]interface{}{ + { + []interface{}{float64(1), float64(2), float64(3), float64(4), float64(5)}, + []interface{}{[]interface{}{float64(1), float64(2)}, []interface{}{float64(3), float64(4)}, []interface{}{float64(5), float64(6)}}, + []interface{}{[]interface{}{[]interface{}{float64(1), float64(2)}, []interface{}{float64(3), float64(4)}}, []interface{}{[]interface{}{float64(5), float64(6)}, []interface{}{float64(7), float64(8)}}}, + []interface{}{[]interface{}{[]interface{}{[]interface{}{float64(11), float64(11)}}, []interface{}{[]interface{}{float64(11), float64(11)}}}, []interface{}{[]interface{}{[]interface{}{float64(11), float64(11)}}, []interface{}{[]interface{}{float64(11), float64(11)}}}}, + "1970-01-01T00:00:00.000001Z"}, + }, + Count: 1, + }, + }, } for _, tc := range testCases { for _, protocol := range []string{"tcp", "http"} { - suite.T().Run(fmt.Sprintf("%s: %s", tc.name, protocol), func(t *testing.T) { - var ( - sender qdb.LineSender - err error - ) - - questdbC, err := setupQuestDB(ctx, noAuth) - assert.NoError(t, err) - - switch protocol { - case "tcp": - sender, err = qdb.NewLineSender(ctx, qdb.WithTcp(), qdb.WithAddress(questdbC.ilpAddress)) - assert.NoError(t, err) - case "http": - sender, err = qdb.NewLineSender(ctx, qdb.WithHttp(), qdb.WithAddress(questdbC.httpAddress)) + for _, pVersion := range []int{0, 1, 2} { + suite.T().Run(fmt.Sprintf("%s: %s", tc.name, protocol), func(t *testing.T) { + var ( + sender qdb.LineSender + err error + ) + + ignoreArray := false + questdbC, err := setupQuestDB(ctx, noAuth) assert.NoError(t, err) - default: - panic(protocol) - } - err = tc.writerFn(sender) - assert.NoError(t, err) + switch protocol { + case "tcp": + if pVersion == 0 { + sender, err = qdb.NewLineSender(ctx, qdb.WithTcp(), qdb.WithAddress(questdbC.ilpAddress)) + ignoreArray = true + } else if pVersion == 1 { + sender, err = qdb.NewLineSender(ctx, qdb.WithTcp(), qdb.WithAddress(questdbC.ilpAddress), qdb.WithProtocolVersion(qdb.ProtocolVersion1)) + ignoreArray = true + } else if pVersion == 2 { + sender, err = qdb.NewLineSender(ctx, qdb.WithTcp(), qdb.WithAddress(questdbC.ilpAddress), qdb.WithProtocolVersion(qdb.ProtocolVersion2)) + } + assert.NoError(t, err) + case "http": + if pVersion == 0 { + sender, err = qdb.NewLineSender(ctx, qdb.WithHttp(), qdb.WithAddress(questdbC.httpAddress)) + } else if pVersion == 1 { + sender, err = qdb.NewLineSender(ctx, qdb.WithHttp(), qdb.WithAddress(questdbC.httpAddress), qdb.WithProtocolVersion(qdb.ProtocolVersion1)) + ignoreArray = true + } else if pVersion == 2 { + sender, err = qdb.NewLineSender(ctx, qdb.WithHttp(), qdb.WithAddress(questdbC.httpAddress), qdb.WithProtocolVersion(qdb.ProtocolVersion2)) + } + assert.NoError(t, err) + default: + panic(protocol) + } + if ignoreArray && tc.name == "double array" { + return + } + + dropTable(t, tc.tableName, questdbC.httpAddress) + err = tc.writerFn(sender) + assert.NoError(t, err) - err = sender.Flush(ctx) - assert.NoError(t, err) + err = sender.Flush(ctx) + assert.NoError(t, err) - assert.Eventually(t, func() bool { - data := queryTableData(t, tc.tableName, questdbC.httpAddress) - return reflect.DeepEqual(tc.expected, data) - }, eventualDataTimeout, 100*time.Millisecond) + assert.Eventually(t, func() bool { + data := queryTableData(t, tc.tableName, questdbC.httpAddress) + return reflect.DeepEqual(tc.expected, data) + }, eventualDataTimeout, 100*time.Millisecond) - sender.Close(ctx) - questdbC.Stop(ctx) - }) + sender.Close(ctx) + questdbC.Stop(ctx) + }) + } } } } diff --git a/interop_test.go b/interop_test.go index 089c777c..88f0ec02 100644 --- a/interop_test.go +++ b/interop_test.go @@ -75,7 +75,7 @@ func TestTcpClientInterop(t *testing.T) { srv, err := newTestTcpServer(sendToBackChannel) assert.NoError(t, err) - sender, err := qdb.NewLineSender(ctx, qdb.WithTcp(), qdb.WithAddress(srv.Addr())) + sender, err := qdb.NewLineSender(ctx, qdb.WithTcp(), qdb.WithAddress(srv.Addr()), qdb.WithProtocolVersion(qdb.ProtocolVersion1)) assert.NoError(t, err) sender.Table(tc.Table) @@ -129,7 +129,7 @@ func TestHttpClientInterop(t *testing.T) { srv, err := newTestHttpServer(sendToBackChannel) assert.NoError(t, err) - sender, err := qdb.NewLineSender(ctx, qdb.WithHttp(), qdb.WithAddress(srv.Addr())) + sender, err := qdb.NewLineSender(ctx, qdb.WithHttp(), qdb.WithAddress(srv.Addr()), qdb.WithProtocolVersion(qdb.ProtocolVersion1)) assert.NoError(t, err) sender.Table(tc.Table) diff --git a/ndarray.go b/ndarray.go new file mode 100644 index 00000000..ff6d3ad0 --- /dev/null +++ b/ndarray.go @@ -0,0 +1,194 @@ +package questdb + +import ( + "errors" + "fmt" +) + +const ( + // MaxDimensions defines the maximum dims of NdArray + MaxDimensions = 32 + + // MaxElements defines the maximum total number of elements of NdArray + MaxElements = (1 << 28) - 1 +) + +// Numeric represents the constraint for numeric types that can be used in NdArray +type Numeric interface { + ~float64 +} + +// NdArray represents a generic n-dimensional array with shape validation +type NdArray[T Numeric] struct { + data []T + shape []uint + appendIndex uint +} + +// NewNDArray creates a new NdArray with the specified shape +func NewNDArray[T Numeric](shape ...uint) (*NdArray[T], error) { + if err := validateShape(shape); err != nil { + return nil, fmt.Errorf("invalid shape: %w", err) + } + totalElements := product(shape) + data := make([]T, totalElements) + shapeSlice := make([]uint, len(shape)) + copy(shapeSlice, shape) + return &NdArray[T]{ + shape: shapeSlice, + data: data, + appendIndex: 0, + }, nil +} + +// Shape returns a copy of the array's shape +func (n *NdArray[T]) Shape() []uint { + shape := make([]uint, len(n.shape)) + copy(shape, n.shape) + return shape +} + +// NDims returns the number of dimensions +func (n *NdArray[T]) NDims() int { + return len(n.shape) +} + +// Size returns the total number of elements +func (n *NdArray[T]) Size() int { + return len(n.data) +} + +// Set sets a value at the specified multi-dimensional position +func (n *NdArray[T]) Set(v T, positions ...uint) error { + if len(positions) != n.NDims() { + return fmt.Errorf("position dimensions (%d) don't match array dimensions (%d)", len(positions), n.NDims()) + } + + index, err := n.positionsToIndex(positions) + if err != nil { + return err + } + + n.data[index] = v + return nil +} + +// Get retrieves a value at the specified multi-dimensional position +func (n *NdArray[T]) Get(positions ...uint) (T, error) { + var zero T + if len(positions) != n.NDims() { + return zero, fmt.Errorf("position dimensions (%d) don't match array dimensions (%d)", len(positions), n.NDims()) + } + + index, err := n.positionsToIndex(positions) + if err != nil { + return zero, err + } + + return n.data[index], nil +} + +// Reshape creates a new NdArray with a different shape but same data +func (n *NdArray[T]) Reshape(newShape ...uint) (*NdArray[T], error) { + if err := validateShape(newShape); err != nil { + return nil, fmt.Errorf("invalid new shape: %v", err) + } + + if uint(len(n.data)) != product(newShape) { + return nil, fmt.Errorf("new shape size (%d) doesn't match data size (%d)", + product(newShape), len(n.data)) + } + + // Create new array sharing the same data + newArray := &NdArray[T]{ + shape: make([]uint, len(newShape)), + data: n.data, // Share the same underlying data + } + copy(newArray.shape, newShape) + + return newArray, nil +} + +// Append adds a value to the array sequentially +func (n *NdArray[T]) Append(val T) (bool, error) { + if n.appendIndex >= uint(len(n.data)) { + return false, errors.New("array is full") + } + n.data[n.appendIndex] = val + n.appendIndex++ + return n.appendIndex < uint(len(n.data)), nil +} + +// ResetAppendIndex resets the append index to 0 +func (n *NdArray[T]) ResetAppendIndex() { + n.appendIndex = 0 +} + +// GetData returns the underlying data slice +func (n *NdArray[T]) GetData() []T { + return n.data +} + +// Fill fills the entire array with the specified value +func (n *NdArray[T]) Fill(value T) { + for i := range n.data { + n.data[i] = value + } + n.appendIndex = uint(len(n.data)) // Mark as full +} + +// positionsToIndex converts multi-dimensional positions to a flat index +func (n *NdArray[T]) positionsToIndex(positions []uint) (int, error) { + // Validate positions are within bounds + for i, pos := range positions { + if pos >= n.shape[i] { + return 0, fmt.Errorf("position[%d]=%d is out of bounds for dimension size %d", + i, pos, n.shape[i]) + } + } + + // Calculate flat index + index := 0 + for i, pos := range positions { + index += int(pos) * int(product(n.shape[i+1:])) + } + return index, nil +} + +// validateShape validates that shape dimensions are valid +func validateShape(shape []uint) error { + if len(shape) == 0 { + return errors.New("shape cannot be empty") + } + + // Check maximum dimensions limit + if len(shape) > MaxDimensions { + return fmt.Errorf("too many dimensions: %d exceeds maximum of %d", + len(shape), MaxDimensions) + } + + // Check maximum elements limit + totalElements := product(shape) + if totalElements > MaxElements { + return fmt.Errorf("array too large: %d elements exceeds maximum of %d", + totalElements, MaxElements) + } + + return nil +} + +// product calculates the product of slice elements with overflow protection +func product(s []uint) uint { + if len(s) == 0 { + return 1 + } + p := uint(1) + for _, v := range s { + // Check for potential overflow before multiplication + if v > 0 && p > MaxElements/v { + return MaxElements + 1 // Return a value that will trigger validation error + } + p *= v + } + return p +} diff --git a/ndarray_test.go b/ndarray_test.go new file mode 100644 index 00000000..14f73bb5 --- /dev/null +++ b/ndarray_test.go @@ -0,0 +1,319 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2022 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package questdb_test + +import ( + "testing" + + qdb "github.com/questdb/go-questdb-client/v3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNew_ValidShapes(t *testing.T) { + testCases := []struct { + name string + shape []uint + expected []uint + }{ + {"1D array", []uint{5}, []uint{5}}, + {"2D array", []uint{3, 4}, []uint{3, 4}}, + {"3D array", []uint{2, 3, 4}, []uint{2, 3, 4}}, + {"single element", []uint{1}, []uint{1}}, + {"large array", []uint{100, 200}, []uint{100, 200}}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + arr, err := qdb.NewNDArray[float64](tc.shape...) + require.NoError(t, err) + assert.Equal(t, tc.expected, arr.Shape()) + assert.Equal(t, len(tc.shape), arr.NDims()) + + expectedSize := uint(1) + for _, dim := range tc.shape { + expectedSize *= dim + } + assert.Equal(t, int(expectedSize), arr.Size()) + }) + } +} + +func TestNew_InvalidShapes(t *testing.T) { + testCases := []struct { + name string + shape []uint + }{ + {"empty shape", []uint{}}, + {"too many dimensions", make([]uint, qdb.MaxDimensions+1)}, + {"too many elements", []uint{qdb.MaxElements + 1}}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Initialize shape with 1 for "too many dimensions" test + if tc.name == "too many dimensions" { + for i := range tc.shape { + tc.shape[i] = 1 + } + } + + arr, err := qdb.NewNDArray[float64](tc.shape...) + assert.Error(t, err) + assert.Nil(t, arr) + }) + } +} + +func TestSetGet_ValidPositions(t *testing.T) { + arr, err := qdb.NewNDArray[float64](3, 4) + require.NoError(t, err) + + testCases := []struct { + positions []uint + value float64 + }{ + {[]uint{0, 0}, 1.5}, + {[]uint{2, 3}, 42.0}, + {[]uint{1, 2}, -7.5}, + } + + for _, tc := range testCases { + err := arr.Set(tc.value, tc.positions...) + require.NoError(t, err) + + retrieved, err := arr.Get(tc.positions...) + require.NoError(t, err) + assert.Equal(t, tc.value, retrieved) + } +} + +func TestSetGet_InvalidPositions(t *testing.T) { + arr, err := qdb.NewNDArray[float64](3, 4) + require.NoError(t, err) + + testCases := []struct { + name string + positions []uint + }{ + {"wrong dimensions", []uint{0}}, + {"out of bounds first dim", []uint{3, 0}}, + {"out of bounds second dim", []uint{0, 4}}, + {"too many dimensions", []uint{0, 0, 0}}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + err := arr.Set(1.0, tc.positions...) + assert.Error(t, err) + + _, err = arr.Get(tc.positions...) + assert.Error(t, err) + }) + } +} + +func TestReshape_ValidShapes(t *testing.T) { + arr, err := qdb.NewNDArray[float64](2, 3) + require.NoError(t, err) + + value := 1.0 + for i := uint(0); i < 2; i++ { + for j := uint(0); j < 3; j++ { + err := arr.Set(value, i, j) + require.NoError(t, err) + value++ + } + } + + testCases := []struct { + name string + newShape []uint + }{ + {"1D reshape", []uint{6}}, + {"3D reshape", []uint{1, 2, 3}}, + {"different 2D", []uint{3, 2}}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + reshaped, err := arr.Reshape(tc.newShape...) + require.NoError(t, err) + assert.Equal(t, tc.newShape, reshaped.Shape()) + assert.Equal(t, arr.Size(), reshaped.Size()) + assert.Equal(t, arr.GetData(), reshaped.GetData()) + }) + } +} + +func TestReshape_InvalidShapes(t *testing.T) { + arr, err := qdb.NewNDArray[float64](2, 3) + require.NoError(t, err) + + testCases := []struct { + name string + newShape []uint + }{ + {"wrong size", []uint{5}}, + {"empty shape", []uint{}}, + {"too large", []uint{qdb.MaxElements + 1}}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + reshaped, err := arr.Reshape(tc.newShape...) + assert.Error(t, err) + assert.Nil(t, reshaped) + }) + } +} + +func TestAppend(t *testing.T) { + arr, err := qdb.NewNDArray[float64](2, 2) + require.NoError(t, err) + + values := []float64{1.0, 2.0, 3.0, 4.0} + for i, val := range values { + hasMore, err := arr.Append(val) + require.NoError(t, err) + + if i < len(values)-1 { + assert.True(t, hasMore, "should have more space") + } else { + assert.False(t, hasMore, "should be full") + } + } + + assert.Equal(t, values, arr.GetData()) + _, err = arr.Append(5.0) + assert.Error(t, err) + assert.Contains(t, err.Error(), "array is full") +} + +func TestResetAppendIndex(t *testing.T) { + arr, err := qdb.NewNDArray[float64](2, 2) + require.NoError(t, err) + for i := 0; i < 4; i++ { + _, err := arr.Append(float64(i)) + require.NoError(t, err) + } + + // Reset + arr.ResetAppendIndex() + hasMore, err := arr.Append(10.0) + require.NoError(t, err) + assert.True(t, hasMore) + + // The first element was overwritten + val, err := arr.Get(0, 0) + require.NoError(t, err) + assert.Equal(t, 10.0, val) +} + +func TestFill(t *testing.T) { + arr, err := qdb.NewNDArray[float64](2, 3) + require.NoError(t, err) + + fillValue := 42.0 + arr.Fill(fillValue) + + for i := uint(0); i < 2; i++ { + for j := uint(0); j < 3; j++ { + val, err := arr.Get(i, j) + require.NoError(t, err) + assert.Equal(t, fillValue, val) + } + } + + _, err = arr.Append(1.0) + assert.Error(t, err) + assert.Contains(t, err.Error(), "array is full") +} + +func TestShape_ReturnsImmutableCopy(t *testing.T) { + originalShape := []uint{3, 4} + arr, err := qdb.NewNDArray[float64](originalShape...) + require.NoError(t, err) + + shape := arr.Shape() + shape[0] = 999 + actualShape := arr.Shape() + assert.Equal(t, originalShape, actualShape) + assert.NotEqual(t, uint(999), actualShape[0]) +} + +func TestGetData_SharedReference(t *testing.T) { + arr, err := qdb.NewNDArray[float64](2, 2) + require.NoError(t, err) + + data := arr.GetData() + data[0] = 42.0 + val, err := arr.Get(0, 0) + require.NoError(t, err) + assert.Equal(t, 42.0, val) +} + +func TestMaxLimits(t *testing.T) { + t.Run("max dimensions", func(t *testing.T) { + shape := make([]uint, qdb.MaxDimensions) + for i := range shape { + shape[i] = 1 + } + + arr, err := qdb.NewNDArray[float64](shape...) + require.NoError(t, err) + assert.Equal(t, qdb.MaxDimensions, arr.NDims()) + }) + + t.Run("max elements", func(t *testing.T) { + arr, err := qdb.NewNDArray[float64](qdb.MaxElements) + require.NoError(t, err) + assert.Equal(t, qdb.MaxElements, arr.Size()) + }) +} + +func TestPositionsToIndex(t *testing.T) { + arr, err := qdb.NewNDArray[float64](3, 4) + require.NoError(t, err) + + testCases := []struct { + positions []uint + value float64 + }{ + {[]uint{0, 0}, 100.0}, + {[]uint{0, 1}, 101.0}, + {[]uint{1, 0}, 102.0}, + {[]uint{2, 3}, 103.0}, + } + + for _, tc := range testCases { + err := arr.Set(tc.value, tc.positions...) + require.NoError(t, err) + + retrieved, err := arr.Get(tc.positions...) + require.NoError(t, err) + assert.Equal(t, tc.value, retrieved) + } +} diff --git a/sender.go b/sender.go index 87b636fb..b0b57975 100644 --- a/sender.go +++ b/sender.go @@ -120,6 +120,64 @@ type LineSender interface { // '-', '*' '%%', '~', or a non-printable char. BoolColumn(name string, val bool) LineSender + // Float641DArrayColumn adds an array of 64-bit floats (double array) to the ILP message. + // + // Column name cannot contain any of the following characters: + // '\n', '\r', '?', '.', ',', "', '"', '\', '/', ':', ')', '(', '+', + // '-', '*' '%%', '~', or a non-printable char. + Float641DArrayColumn(name string, values []float64) LineSender + + // Float642DArrayColumn adds a 2D array of 64-bit floats (double 2D array) to the ILP message. + // + // The values parameter must have a regular (rectangular) shape - all rows must have + // exactly the same length. If the array has irregular shape, this method returns an error. + // + // Example of valid input: + // values := [][]float64{{1.0, 2.0}, {3.0, 4.0}, {5.0, 6.0}} // 3x2 regular shape + // + // Example of invalid input: + // values := [][]float64{{1.0, 2.0}, {3.0}, {4.0, 5.0, 6.0}} // irregular shape - returns error + // + // Column name cannot contain any of the following characters: + // '\n', '\r', '?', '.', ',', "', '"', '\', '/', ':', ')', '(', '+', + // '-', '*' '%%', '~', or a non-printable char. + Float642DArrayColumn(name string, values [][]float64) LineSender + + // Float643DArrayColumn adds a 3D array of 64-bit floats (double 3D array) to the ILP message. + // + // The values parameter must have a regular (cuboid) shape - all dimensions must have + // consistent sizes throughout. If the array has irregular shape, this method returns an error. + // + // Example of valid input: + // values := [][][]float64{ + // {{1.0, 2.0}, {3.0, 4.0}}, // 2x2 matrix + // {{5.0, 6.0}, {7.0, 8.0}}, // 2x2 matrix (same shape) + // } // 2x2x2 regular shape + // + // Example of invalid input: + // values := [][][]float64{ + // {{1.0, 2.0}, {3.0, 4.0}}, // 2x2 matrix + // {{5.0}, {6.0, 7.0, 8.0}}, // irregular matrix - returns error + // } + // + // Column name cannot contain any of the following characters: + // '\n', '\r', '?', '.', ',', "', '"', '\', '/', ':', ')', '(', '+', + // '-', '*' '%%', '~', or a non-printable char. + Float643DArrayColumn(name string, values [][][]float64) LineSender + + // Float64NDArrayColumn adds an n-dimensional array of 64-bit floats (double n-D array) to the ILP message. + // + // Example usage: + // // Create a 2x3x4 array + // arr, _ := questdb.NewNDArray[float64](2, 3, 4) + // arr.Fill(1.5) + // sender.Float64NDArrayColumn("ndarray_col", arr) + // + // Column name cannot contain any of the following characters: + // '\n', '\r', '?', '.', ',', "', '"', '\', '/', ':', ')', '(', '+', + // '-', '*' '%%', '~', or a non-printable char. + Float64NDArrayColumn(name string, values *NdArray[float64]) LineSender + // At sets the designated timestamp value and finalizes the ILP // message. // @@ -189,6 +247,14 @@ const ( tlsInsecureSkipVerify tlsMode = 2 ) +type protocolVersion int64 + +const ( + protocolVersionUnset protocolVersion = 0 + ProtocolVersion1 protocolVersion = 1 + ProtocolVersion2 protocolVersion = 2 +) + type lineSenderConfig struct { senderType senderType address string @@ -213,6 +279,8 @@ type lineSenderConfig struct { // Auto-flush fields autoFlushRows int autoFlushInterval time.Duration + + protocolVersion protocolVersion } // LineSenderOption defines line sender config option. @@ -404,6 +472,12 @@ func WithAutoFlushInterval(interval time.Duration) LineSenderOption { } } +func WithProtocolVersion(version protocolVersion) LineSenderOption { + return func(s *lineSenderConfig) { + s.protocolVersion = version + } +} + // LineSenderFromEnv creates a LineSender with a config string defined by the QDB_CLIENT_CONF // environment variable. See LineSenderFromConf for the config string format. // @@ -556,7 +630,7 @@ func newLineSender(ctx context.Context, conf *lineSenderConfig) (LineSender, err if err != nil { return nil, err } - return newHttpLineSender(conf) + return newHttpLineSender(ctx, conf) } return nil, errors.New("sender type is not specified: use WithHttp or WithTcp") } @@ -638,6 +712,10 @@ func validateConf(conf *lineSenderConfig) error { if conf.autoFlushInterval < 0 { return fmt.Errorf("auto flush interval is negative: %d", conf.autoFlushInterval) } + if conf.protocolVersion < protocolVersionUnset || conf.protocolVersion > ProtocolVersion2 { + return errors.New("current client only supports protocol version 1(text format for all datatypes), " + + "2(binary format for part datatypes) or explicitly unset") + } return nil } diff --git a/sender_pool.go b/sender_pool.go index a59a81ac..cba9f3c0 100644 --- a/sender_pool.go +++ b/sender_pool.go @@ -181,7 +181,7 @@ func (p *LineSenderPool) Sender(ctx context.Context) (LineSender, error) { return nil, errHttpOnlySender } } - s, err = newHttpLineSender(conf) + s, err = newHttpLineSender(ctx, conf) } if err != nil { @@ -324,6 +324,26 @@ func (ps *pooledSender) BoolColumn(name string, val bool) LineSender { return ps } +func (ps *pooledSender) Float641DArrayColumn(name string, values []float64) LineSender { + ps.wrapped.Float641DArrayColumn(name, values) + return ps +} + +func (ps *pooledSender) Float642DArrayColumn(name string, values [][]float64) LineSender { + ps.wrapped.Float642DArrayColumn(name, values) + return ps +} + +func (ps *pooledSender) Float643DArrayColumn(name string, values [][][]float64) LineSender { + ps.wrapped.Float643DArrayColumn(name, values) + return ps +} + +func (ps *pooledSender) Float64NDArrayColumn(name string, values *NdArray[float64]) LineSender { + ps.wrapped.Float64NDArrayColumn(name, values) + return ps +} + func (ps *pooledSender) AtNow(ctx context.Context) error { err := ps.wrapped.AtNow(ctx) if err != nil { diff --git a/tcp_integration_test.go b/tcp_integration_test.go index 2b0f3c30..07832a81 100644 --- a/tcp_integration_test.go +++ b/tcp_integration_test.go @@ -59,6 +59,7 @@ func (suite *integrationTestSuite) TestE2EWriteInBatches() { sender, err := qdb.NewLineSender(ctx, qdb.WithTcp(), qdb.WithAddress(questdbC.ilpAddress)) assert.NoError(suite.T(), err) defer sender.Close(ctx) + dropTable(suite.T(), testTable, questdbC.httpAddress) for i := 0; i < n; i++ { for j := 0; j < nBatch; j++ { @@ -112,6 +113,7 @@ func (suite *integrationTestSuite) TestE2EImplicitFlush() { sender, err := qdb.NewLineSender(ctx, qdb.WithTcp(), qdb.WithAddress(questdbC.ilpAddress), qdb.WithInitBufferSize(bufCap)) assert.NoError(suite.T(), err) defer sender.Close(ctx) + dropTable(suite.T(), testTable, questdbC.httpAddress) for i := 0; i < 10*bufCap; i++ { err = sender. @@ -147,6 +149,7 @@ func (suite *integrationTestSuite) TestE2ESuccessfulAuth() { ) assert.NoError(suite.T(), err) + dropTable(suite.T(), testTable, questdbC.httpAddress) err = sender. Table(testTable). StringColumn("str_col", "foobar"). @@ -169,7 +172,7 @@ func (suite *integrationTestSuite) TestE2ESuccessfulAuth() { expected := tableData{ Columns: []column{ - {"str_col", "STRING"}, + {"str_col", "VARCHAR"}, {"timestamp", "TIMESTAMP"}, }, Dataset: [][]interface{}{ @@ -214,6 +217,7 @@ func (suite *integrationTestSuite) TestE2EFailedAuth() { return } + dropTable(suite.T(), testTable, questdbC.httpAddress) err = sender. Table(testTable). StringColumn("str_col", "barbaz"). @@ -252,6 +256,7 @@ func (suite *integrationTestSuite) TestE2EWritesWithTlsProxy() { ) assert.NoError(suite.T(), err) defer sender.Close(ctx) + dropTable(suite.T(), testTable, questdbC.httpAddress) err = sender. Table(testTable). @@ -270,7 +275,7 @@ func (suite *integrationTestSuite) TestE2EWritesWithTlsProxy() { expected := tableData{ Columns: []column{ - {"str_col", "STRING"}, + {"str_col", "VARCHAR"}, {"timestamp", "TIMESTAMP"}, }, Dataset: [][]interface{}{ @@ -305,6 +310,7 @@ func (suite *integrationTestSuite) TestE2ESuccessfulAuthWithTlsProxy() { qdb.WithTlsInsecureSkipVerify(), ) assert.NoError(suite.T(), err) + dropTable(suite.T(), testTable, questdbC.httpAddress) err = sender. Table(testTable). @@ -328,7 +334,7 @@ func (suite *integrationTestSuite) TestE2ESuccessfulAuthWithTlsProxy() { expected := tableData{ Columns: []column{ - {"str_col", "STRING"}, + {"str_col", "VARCHAR"}, {"timestamp", "TIMESTAMP"}, }, Dataset: [][]interface{}{ @@ -344,6 +350,66 @@ func (suite *integrationTestSuite) TestE2ESuccessfulAuthWithTlsProxy() { }, eventualDataTimeout, 100*time.Millisecond) } +func (suite *integrationTestSuite) TestDoubleArrayColumn() { + if testing.Short() { + suite.T().Skip("skipping integration test") + } + + ctx := context.Background() + questdbC, err := setupQuestDB(ctx, noAuth) + assert.NoError(suite.T(), err) + defer questdbC.Stop(ctx) + + sender, err := qdb.NewLineSender(ctx, qdb.WithTcp(), qdb.WithAddress(questdbC.ilpAddress), qdb.WithProtocolVersion(qdb.ProtocolVersion2)) + assert.NoError(suite.T(), err) + defer sender.Close(ctx) + dropTable(suite.T(), testTable, questdbC.httpAddress) + + values1D := []float64{1.0, 2.0, 3.0, 4.0, 5.0} + values2D := [][]float64{{1.0, 2.0}, {3.0, 4.0}, {5.0, 6.0}} + values3D := [][][]float64{{{1.0, 2.0}, {3.0, 4.0}}, {{5.0, 6.0}, {7.0, 8.0}}} + arrayND, err := qdb.NewNDArray[float64](2, 2, 1, 2) + assert.NoError(suite.T(), err) + arrayND.Fill(11.0) + + err = sender. + Table(testTable). + Float641DArrayColumn("array_1d", values1D). + Float642DArrayColumn("array_2d", values2D). + Float643DArrayColumn("array_3d", values3D). + Float64NDArrayColumn("array_nd", arrayND). + At(ctx, time.UnixMicro(1)) + assert.NoError(suite.T(), err) + + err = sender.Flush(ctx) + assert.NoError(suite.T(), err) + + // Expected results + expected := tableData{ + Columns: []column{ + {"array_1d", "ARRAY"}, + {"array_2d", "ARRAY"}, + {"array_3d", "ARRAY"}, + {"array_nd", "ARRAY"}, + {"timestamp", "TIMESTAMP"}, + }, + Dataset: [][]interface{}{ + { + []interface{}{float64(1), float64(2), float64(3), float64(4), float64(5)}, + []interface{}{[]interface{}{float64(1), float64(2)}, []interface{}{float64(3), float64(4)}, []interface{}{float64(5), float64(6)}}, + []interface{}{[]interface{}{[]interface{}{float64(1), float64(2)}, []interface{}{float64(3), float64(4)}}, []interface{}{[]interface{}{float64(5), float64(6)}, []interface{}{float64(7), float64(8)}}}, + []interface{}{[]interface{}{[]interface{}{[]interface{}{float64(11), float64(11)}}, []interface{}{[]interface{}{float64(11), float64(11)}}}, []interface{}{[]interface{}{[]interface{}{float64(11), float64(11)}}, []interface{}{[]interface{}{float64(11), float64(11)}}}}, + "1970-01-01T00:00:00.000001Z"}, + }, + Count: 1, + } + + assert.Eventually(suite.T(), func() bool { + data := queryTableData(suite.T(), testTable, questdbC.httpAddress) + return reflect.DeepEqual(expected, data) + }, eventualDataTimeout, 100*time.Millisecond) +} + type tableData struct { Columns []column `json:"columns"` Dataset [][]interface{} `json:"dataset"` @@ -355,6 +421,23 @@ type column struct { Type string `json:"type"` } +func dropTable(t *testing.T, tableName, address string) { + // We always query data using the QuestDB container over http + address = "http://" + address + u, err := url.Parse(address) + assert.NoError(t, err) + + u.Path += "exec" + params := url.Values{} + params.Add("query", "drop table if exists '"+tableName+"'") + u.RawQuery = params.Encode() + url := fmt.Sprintf("%v", u) + + res, err := http.Get(url) + assert.NoError(t, err) + defer res.Body.Close() +} + func queryTableData(t *testing.T, tableName, address string) tableData { // We always query data using the QuestDB container over http address = "http://" + address diff --git a/tcp_sender.go b/tcp_sender.go index d4bf86d3..1f7a93bc 100644 --- a/tcp_sender.go +++ b/tcp_sender.go @@ -33,6 +33,7 @@ import ( "crypto/rand" "crypto/tls" "encoding/base64" + "errors" "fmt" "math/big" "net" @@ -45,7 +46,11 @@ type tcpLineSender struct { conn net.Conn } -func newTcpLineSender(ctx context.Context, conf *lineSenderConfig) (*tcpLineSender, error) { +type tcpLineSenderV2 struct { + tcpLineSender +} + +func newTcpLineSender(ctx context.Context, conf *lineSenderConfig) (LineSender, error) { var ( d net.Dialer key *ecdsa.PrivateKey @@ -131,7 +136,13 @@ func newTcpLineSender(ctx context.Context, conf *lineSenderConfig) (*tcpLineSend s.conn = conn - return s, nil + if conf.protocolVersion == protocolVersionUnset || conf.protocolVersion == ProtocolVersion1 { + return s, nil + } else { + return &tcpLineSenderV2{ + *s, + }, nil + } } func (s *tcpLineSender) Close(_ context.Context) error { @@ -183,6 +194,26 @@ func (s *tcpLineSender) BoolColumn(name string, val bool) LineSender { return s } +func (s *tcpLineSender) Float641DArrayColumn(name string, values []float64) LineSender { + s.buf.SetLastErr(errors.New("current protocol version does not support double-array")) + return s +} + +func (s *tcpLineSender) Float642DArrayColumn(name string, values [][]float64) LineSender { + s.buf.SetLastErr(errors.New("current protocol version does not support double-array")) + return s +} + +func (s *tcpLineSender) Float643DArrayColumn(name string, values [][][]float64) LineSender { + s.buf.SetLastErr(errors.New("current protocol version does not support double-array")) + return s +} + +func (s *tcpLineSender) Float64NDArrayColumn(name string, values *NdArray[float64]) LineSender { + s.buf.SetLastErr(errors.New("current protocol version does not support double-array")) + return s +} + func (s *tcpLineSender) Flush(ctx context.Context) error { err := s.buf.LastErr() s.buf.ClearLastErr() @@ -240,9 +271,9 @@ func (s *tcpLineSender) At(ctx context.Context, ts time.Time) error { return nil } -// Messages returns a copy of accumulated ILP messages that are not +// Messages returns the accumulated ILP messages that are not // flushed to the TCP connection yet. Useful for debugging purposes. -func (s *tcpLineSender) Messages() string { +func (s *tcpLineSender) Messages() []byte { return s.buf.Messages() } @@ -255,3 +286,63 @@ func (s *tcpLineSender) MsgCount() int { func (s *tcpLineSender) BufLen() int { return s.buf.Len() } + +func (s *tcpLineSenderV2) Table(name string) LineSender { + s.buf.Table(name) + return s +} + +func (s *tcpLineSenderV2) Symbol(name, val string) LineSender { + s.buf.Symbol(name, val) + return s +} + +func (s *tcpLineSenderV2) Int64Column(name string, val int64) LineSender { + s.buf.Int64Column(name, val) + return s +} + +func (s *tcpLineSenderV2) Long256Column(name string, val *big.Int) LineSender { + s.buf.Long256Column(name, val) + return s +} + +func (s *tcpLineSenderV2) TimestampColumn(name string, ts time.Time) LineSender { + s.buf.TimestampColumn(name, ts) + return s +} + +func (s *tcpLineSenderV2) StringColumn(name, val string) LineSender { + s.buf.StringColumn(name, val) + return s +} + +func (s *tcpLineSenderV2) BoolColumn(name string, val bool) LineSender { + s.buf.BoolColumn(name, val) + return s +} + +func (s *tcpLineSenderV2) Float64Column(name string, val float64) LineSender { + s.buf.Float64ColumnBinaryFormat(name, val) + return s +} + +func (s *tcpLineSenderV2) Float641DArrayColumn(name string, values []float64) LineSender { + s.buf.Float641DArrayColumn(name, values) + return s +} + +func (s *tcpLineSenderV2) Float642DArrayColumn(name string, values [][]float64) LineSender { + s.buf.Float642DArrayColumn(name, values) + return s +} + +func (s *tcpLineSenderV2) Float643DArrayColumn(name string, values [][][]float64) LineSender { + s.buf.Float643DArrayColumn(name, values) + return s +} + +func (s *tcpLineSenderV2) Float64NDArrayColumn(name string, values *NdArray[float64]) LineSender { + s.buf.Float64NDArrayColumn(name, values) + return s +} diff --git a/tcp_sender_test.go b/tcp_sender_test.go index d738d93d..02949e65 100644 --- a/tcp_sender_test.go +++ b/tcp_sender_test.go @@ -67,6 +67,10 @@ func TestTcpHappyCasesFromConf(t *testing.T) { config: fmt.Sprintf("tcp::addr=%s;init_buf_size=%d;", addr, initBufSize), }, + { + name: "protocol_version", + config: fmt.Sprintf("tcp::addr=%s;protocol_version=2;", addr), + }, } for _, tc := range testCases { @@ -100,6 +104,10 @@ func TestTcpHappyCasesFromEnv(t *testing.T) { config: fmt.Sprintf("tcp::addr=%s;init_buf_size=%d;", addr, initBufSize), }, + { + name: "protocol_version", + config: fmt.Sprintf("tcp::addr=%s;protocol_version=2;", addr), + }, } for _, tc := range testCases { @@ -133,6 +141,11 @@ func TestTcpPathologicalCasesFromEnv(t *testing.T) { config: "tcp::auto_flush_rows=5;", expectedErr: "autoFlushRows setting is not available", }, + { + name: "protocol_version", + config: "tcp::protocol_version=3;", + expectedErr: "current client only supports protocol version 1(text format for all datatypes), 2(binary format for part datatypes) or explicitly unset", + }, } for _, tc := range testCases { @@ -197,6 +210,11 @@ func TestTcpPathologicalCasesFromConf(t *testing.T) { config: "tCp::addr=localhost:1234;", expectedErr: "invalid schema", }, + { + name: "protocol version", + config: "tcp::protocol_version=abc;", + expectedErr: "invalid protocol_version value", + }, } for _, tc := range testCases { @@ -301,6 +319,53 @@ func TestErrorOnContextDeadline(t *testing.T) { t.Fail() } +func TestArrayColumnUnsupportedInTCPProtocolV1(t *testing.T) { + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(50*time.Millisecond)) + defer cancel() + + srv, err := newTestTcpServer(readAndDiscard) + assert.NoError(t, err) + defer srv.Close() + sender, err := qdb.NewLineSender(ctx, qdb.WithTcp(), qdb.WithAddress(srv.Addr()), qdb.WithProtocolVersion(qdb.ProtocolVersion1)) + assert.NoError(t, err) + defer sender.Close(ctx) + + values1D := []float64{1.0, 2.0, 3.0, 4.0, 5.0} + values2D := [][]float64{{1.0, 2.0}, {3.0, 4.0}, {5.0, 6.0}} + values3D := [][][]float64{{{1.0, 2.0}, {3.0, 4.0}}, {{5.0, 6.0}, {7.0, 8.0}}} + arrayND, err := qdb.NewNDArray[float64](2, 2, 1, 2) + assert.NoError(t, err) + arrayND.Fill(11.0) + + err = sender. + Table(testTable). + Float641DArrayColumn("array_1d", values1D). + At(ctx, time.UnixMicro(1)) + assert.Error(t, err) + assert.Contains(t, err.Error(), "current protocol version does not support double-array") + + err = sender. + Table(testTable). + Float642DArrayColumn("array_2d", values2D). + At(ctx, time.UnixMicro(2)) + assert.Error(t, err) + assert.Contains(t, err.Error(), "current protocol version does not support double-array") + + err = sender. + Table(testTable). + Float643DArrayColumn("array_3d", values3D). + At(ctx, time.UnixMicro(3)) + assert.Error(t, err) + assert.Contains(t, err.Error(), "current protocol version does not support double-array") + + err = sender. + Table(testTable). + Float64NDArrayColumn("array_nd", arrayND). + At(ctx, time.UnixMicro(4)) + assert.Error(t, err) + assert.Contains(t, err.Error(), "current protocol version does not support double-array") +} + func BenchmarkLineSenderBatch1000(b *testing.B) { ctx := context.Background() @@ -308,10 +373,17 @@ func BenchmarkLineSenderBatch1000(b *testing.B) { assert.NoError(b, err) defer srv.Close() - sender, err := qdb.NewLineSender(ctx, qdb.WithTcp(), qdb.WithAddress(srv.Addr())) + sender, err := qdb.NewLineSender(ctx, qdb.WithTcp(), qdb.WithAddress(srv.Addr()), qdb.WithProtocolVersion(qdb.ProtocolVersion2)) assert.NoError(b, err) defer sender.Close(ctx) + // Prepare test array data + values1D := []float64{1.0, 2.0, 3.0, 4.0, 5.0} + values2D := [][]float64{{1.0, 2.0}, {3.0, 4.0}, {5.0, 6.0}} + values3D := [][][]float64{{{1.0, 2.0}, {3.0, 4.0}}, {{5.0, 6.0}, {7.0, 8.0}}} + arrayND, _ := qdb.NewNDArray[float64](2, 3) + arrayND.Fill(1.5) + b.ResetTimer() for i := 0; i < b.N; i++ { for j := 0; j < 1000; j++ { @@ -323,6 +395,10 @@ func BenchmarkLineSenderBatch1000(b *testing.B) { StringColumn("str_col", "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua"). BoolColumn("bool_col", true). TimestampColumn("timestamp_col", time.UnixMicro(42)). + Float641DArrayColumn("array_1d", values1D). + Float642DArrayColumn("array_2d", values2D). + Float643DArrayColumn("array_3d", values3D). + Float64NDArrayColumn("array_nd", arrayND). At(ctx, time.UnixMicro(int64(1000*i))) } sender.Flush(ctx) @@ -336,10 +412,16 @@ func BenchmarkLineSenderNoFlush(b *testing.B) { assert.NoError(b, err) defer srv.Close() - sender, err := qdb.NewLineSender(ctx, qdb.WithTcp(), qdb.WithAddress(srv.Addr())) + sender, err := qdb.NewLineSender(ctx, qdb.WithTcp(), qdb.WithAddress(srv.Addr()), qdb.WithProtocolVersion(qdb.ProtocolVersion2)) assert.NoError(b, err) defer sender.Close(ctx) + values1D := []float64{1.0, 2.0, 3.0, 4.0, 5.0} + values2D := [][]float64{{1.0, 2.0}, {3.0, 4.0}, {5.0, 6.0}} + values3D := [][][]float64{{{1.0, 2.0}, {3.0, 4.0}}, {{5.0, 6.0}, {7.0, 8.0}}} + arrayND, _ := qdb.NewNDArray[float64](2, 3) + arrayND.Fill(1.5) + b.ResetTimer() for i := 0; i < b.N; i++ { sender. @@ -350,6 +432,10 @@ func BenchmarkLineSenderNoFlush(b *testing.B) { StringColumn("str_col", "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua"). BoolColumn("bool_col", true). TimestampColumn("timestamp_col", time.UnixMicro(42)). + Float641DArrayColumn("array_1d", values1D). + Float642DArrayColumn("array_2d", values2D). + Float643DArrayColumn("array_3d", values3D). + Float64NDArrayColumn("array_nd", arrayND). At(ctx, time.UnixMicro(int64(1000*i))) } sender.Flush(ctx) diff --git a/utils_test.go b/utils_test.go index 8ecef818..b197fb77 100644 --- a/utils_test.go +++ b/utils_test.go @@ -53,12 +53,13 @@ const ( ) type testServer struct { - addr string - tcpListener net.Listener - serverType serverType - BackCh chan string - closeCh chan struct{} - wg sync.WaitGroup + addr string + tcpListener net.Listener + serverType serverType + protocolVersions []int + BackCh chan string + closeCh chan struct{} + wg sync.WaitGroup } func (t *testServer) Addr() string { @@ -66,24 +67,25 @@ func (t *testServer) Addr() string { } func newTestTcpServer(serverType serverType) (*testServer, error) { - return newTestServerWithProtocol(serverType, "tcp") + return newTestServerWithProtocol(serverType, "tcp", []int{}) } func newTestHttpServer(serverType serverType) (*testServer, error) { - return newTestServerWithProtocol(serverType, "http") + return newTestServerWithProtocol(serverType, "http", []int{1, 2}) } -func newTestServerWithProtocol(serverType serverType, protocol string) (*testServer, error) { +func newTestServerWithProtocol(serverType serverType, protocol string, protocolVersions []int) (*testServer, error) { tcp, err := net.Listen("tcp", "127.0.0.1:") if err != nil { return nil, err } s := &testServer{ - addr: tcp.Addr().String(), - tcpListener: tcp, - serverType: serverType, - BackCh: make(chan string, 1000), - closeCh: make(chan struct{}), + addr: tcp.Addr().String(), + tcpListener: tcp, + serverType: serverType, + BackCh: make(chan string, 1000), + closeCh: make(chan struct{}), + protocolVersions: protocolVersions, } switch protocol { @@ -193,6 +195,47 @@ func (s *testServer) serveHttp() { var ( err error ) + if r.Method == "GET" && r.URL.Path == "/settings" { + if s.protocolVersions == nil { + w.WriteHeader(http.StatusNotFound) + data, err := json.Marshal(map[string]interface{}{ + "code": "404", + "message": "Not Found", + }) + if err != nil { + panic(err) + } + w.Write(data) + return + } + w.Header().Set("Content-Type", "application/json") + var data []byte + if len(s.protocolVersions) == 0 { + data, err = json.Marshal(map[string]interface{}{ + "version": "8.1.2", + }) + } else { + data, err = json.Marshal(map[string]interface{}{ + "config": map[string]interface{}{ + "release.type": "OSS", + "release.version": "[DEVELOPMENT]", + "http.settings.readonly": false, + "line.proto.support.versions": s.protocolVersions, + "ilp.proto.transports": []string{"tcp", "http"}, + "posthog.enabled": false, + "posthog.api.key": nil, + "cairo.max.file.name.length": 256, + }, + "preferences.version": 0, + "preferences": map[string]interface{}{}, + }) + } + if err != nil { + panic(err) + } + w.Write(data) + return + } switch s.serverType { case failFirstThenSendToBackChannel: From 8b3716a312219abee3202d1fc5de48e28056b1cd Mon Sep 17 00:00:00 2001 From: victor Date: Wed, 20 Aug 2025 11:09:59 +0800 Subject: [PATCH 02/24] update readme and docs --- README.md | 90 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ ndarray.go | 8 ++--- sender.go | 9 ++++++ 3 files changed, 101 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 20ec0f28..6d15551a 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ Features: New in v3: * Supports ILP over HTTP using the same client semantics +* Supports n-dimensional arrays of doubles for QuestDB servers 9.0.0 and up Documentation is available [here](https://pkg.go.dev/github.com/questdb/go-questdb-client/v3). @@ -99,6 +100,95 @@ HTTP is the recommended transport to use. To connect via TCP, set the configurat // ... ``` +## N-dimensional arrays + +QuestDB server version 9.0.0 and newer supports n-dimensional arrays of double precision floating point numbers. +The Go client provides several methods to send arrays to QuestDB: + +### 1D Arrays + +```go +// Send a 1D array of doubles +values1D := []float64{1.1, 2.2, 3.3, 4.4} +err = sender. + Table("measurements"). + Symbol("sensor", "temp_probe_1"). + Float641DArrayColumn("readings", values1D). + AtNow(ctx) +``` + +### 2D Arrays + +```go +// Send a 2D array of doubles (must be rectangular) +values2D := [][]float64{ + {1.1, 2.2, 3.3}, + {4.4, 5.5, 6.6}, + {7.7, 8.8, 9.9}, +} +err = sender. + Table("matrix_data"). + Symbol("experiment", "test_001"). + Float642DArrayColumn("matrix", values2D). + AtNow(ctx) +``` + +### 3D Arrays + +```go +// Send a 3D array of doubles (must be regular cuboid shape) +values3D := [][][]float64{ + {{1.0, 2.0}, {3.0, 4.0}}, + {{5.0, 6.0}, {7.0, 8.0}}, +} +err = sender. + Table("tensor_data"). + Symbol("model", "neural_net_v1"). + Float643DArrayColumn("weights", values3D). + AtNow(ctx) +``` + +### N-dimensional Arrays + +For higher dimensions, use the `NewNDArray` function: + +```go +// Create a 2x3x4 array +arr, err := qdb.NewNDArray[float64](2, 3, 4) +if err != nil { + log.Fatal(err) +} + +// Fill with values +arr.Fill(1.5) + +// Or set individual values +arr.Set([]uint{0, 1, 2}, 42.0) + +err = sender. + Table("ndarray_data"). + Symbol("dataset", "training_batch_1"). + Float64NDArrayColumn("features", arr). + AtNow(ctx) +``` + +The array data is sent over a new protocol version (2) that is auto-negotiated +when using HTTP(s), or can be specified explicitly via the ``protocol_version=2`` +parameter when using TCP(s). + +We recommend using HTTP(s), but here is an TCP example, should you need it:: + +```go +sender, err := qdb.NewLineSender(ctx, + qdb.WithTcp(), + qdb.WithProtocolVersion(qdb.ProtocolVersion2)) +``` + +When using ``protocol_version=2`` (with either TCP(s) or HTTP(s)), the sender +will now also serialize ``float64`` (double-precision) columns as binary. +You might see a performance uplift if this is a dominant data type in your +ingestion workload. + ## Pooled Line Senders **Warning: Experimental feature designed for use with HTTP senders ONLY** diff --git a/ndarray.go b/ndarray.go index ff6d3ad0..d6e09093 100644 --- a/ndarray.go +++ b/ndarray.go @@ -177,17 +177,13 @@ func validateShape(shape []uint) error { return nil } -// product calculates the product of slice elements with overflow protection +// product calculates the product of slice elements func product(s []uint) uint { if len(s) == 0 { - return 1 + return 0 } p := uint(1) for _, v := range s { - // Check for potential overflow before multiplication - if v > 0 && p > MaxElements/v { - return MaxElements + 1 // Return a value that will trigger validation error - } p *= v } return p diff --git a/sender.go b/sender.go index b0b57975..c0d34016 100644 --- a/sender.go +++ b/sender.go @@ -472,6 +472,15 @@ func WithAutoFlushInterval(interval time.Duration) LineSenderOption { } } +// WithProtocolVersion sets the ingestion protocol version. +// +// - HTTP transport automatically negotiates the protocol version by default(unset, STRONGLY RECOMMENDED). +// You can explicitly configure the protocol version to avoid the slight latency cost at connection time. +// - TCP transport does not negotiate the protocol version and uses [ProtocolVersion1] by +// default. You must explicitly set [ProtocolVersion2] in order to ingest +// arrays. +// +// NOTE: QuestDB server version 9.0.0 or later is required for [ProtocolVersion2]. func WithProtocolVersion(version protocolVersion) LineSenderOption { return func(s *lineSenderConfig) { s.protocolVersion = version From 9d702b020cc8ea515e1c17a80a0dcfd2251bdbe5 Mon Sep 17 00:00:00 2001 From: victor Date: Thu, 21 Aug 2025 13:30:00 +0800 Subject: [PATCH 03/24] code reviews. --- README.md | 10 +-- buffer.go | 110 +++++++++++++++--------- buffer_test.go | 161 +++++++++++++++++++++++++++--------- conf_parse.go | 2 +- examples/from-conf/main.go | 20 ++--- examples/http/basic/main.go | 21 ++--- examples/tcp/basic/main.go | 20 ++--- export_test.go | 4 + http_sender.go | 74 ++++++++--------- http_sender_test.go | 34 +++++--- integration_test.go | 49 +++++++++-- ndarray.go | 64 ++++++++++---- ndarray_test.go | 14 ++-- sender.go | 18 ++-- sender_pool.go | 16 ++-- tcp_integration_test.go | 8 +- tcp_sender.go | 26 +++--- tcp_sender_test.go | 26 +++--- utils_test.go | 45 ++++++++-- 19 files changed, 462 insertions(+), 260 deletions(-) diff --git a/README.md b/README.md index 6d15551a..fdfb7a85 100644 --- a/README.md +++ b/README.md @@ -113,7 +113,7 @@ values1D := []float64{1.1, 2.2, 3.3, 4.4} err = sender. Table("measurements"). Symbol("sensor", "temp_probe_1"). - Float641DArrayColumn("readings", values1D). + Float64Array1DColumn("readings", values1D). AtNow(ctx) ``` @@ -129,7 +129,7 @@ values2D := [][]float64{ err = sender. Table("matrix_data"). Symbol("experiment", "test_001"). - Float642DArrayColumn("matrix", values2D). + Float64Array2DColumn("matrix", values2D). AtNow(ctx) ``` @@ -144,7 +144,7 @@ values3D := [][][]float64{ err = sender. Table("tensor_data"). Symbol("model", "neural_net_v1"). - Float643DArrayColumn("weights", values3D). + Float64Array3DColumn("weights", values3D). AtNow(ctx) ``` @@ -168,7 +168,7 @@ arr.Set([]uint{0, 1, 2}, 42.0) err = sender. Table("ndarray_data"). Symbol("dataset", "training_batch_1"). - Float64NDArrayColumn("features", arr). + Float64ArrayNDColumn("features", arr). AtNow(ctx) ``` @@ -176,7 +176,7 @@ The array data is sent over a new protocol version (2) that is auto-negotiated when using HTTP(s), or can be specified explicitly via the ``protocol_version=2`` parameter when using TCP(s). -We recommend using HTTP(s), but here is an TCP example, should you need it:: +We recommend using HTTP(s), but here is an TCP example, should you need it: ```go sender, err := qdb.NewLineSender(ctx, diff --git a/buffer.go b/buffer.go index ccdcf116..88345e29 100644 --- a/buffer.go +++ b/buffer.go @@ -41,23 +41,25 @@ import ( // chars found in table or column name. var errInvalidMsg = errors.New("invalid message") -type binaryFlag byte +type binaryCode byte const ( - arrayBinaryFlag binaryFlag = 14 - float64BinaryFlag binaryFlag = 16 + arrayCode binaryCode = 14 + float64Code binaryCode = 16 ) -// isLittleEndian checks if the current machine uses little-endian byte order -func isLittleEndian() bool { +var isLittleEndian = func() bool { var i int32 = 0x01020304 return *(*byte)(unsafe.Pointer(&i)) == 0x04 -} +}() + +// MaxArrayElements defines the maximum total number of elements of Array +const MaxArrayElements = (1 << 28) - 1 // writeFloat64Data optimally writes float64 slice data to buffer // Uses batch memory copy on little-endian machines for better performance func (b *buffer) writeFloat64Data(data []float64) { - if isLittleEndian() && len(data) > 0 { + if isLittleEndian && len(data) > 0 { b.Write(unsafe.Slice((*byte)(unsafe.Pointer(&data[0])), len(data)*8)) } else { bytes := make([]byte, 8) @@ -68,15 +70,14 @@ func (b *buffer) writeFloat64Data(data []float64) { } } -// writeUint32 optimally writes a single uint32 value -func (b *buffer) writeUint32(val uint32) { - if isLittleEndian() { +func (b *buffer) writeInt32(val int32) { + if isLittleEndian { // On little-endian machines, we can directly write the uint32 as bytes b.Write((*[4]byte)(unsafe.Pointer(&val))[:]) } else { // On big-endian machines, use the standard conversion data := make([]byte, 4) - binary.LittleEndian.PutUint32(data, val) + binary.LittleEndian.PutUint32(data, uint32(val)) b.Write(data) } } @@ -85,6 +86,7 @@ type arrayElemType byte const ( arrayElemDouble arrayElemType = 10 + arrayElemNull = 33 ) // buffer is a wrapper on top of bytes.Buffer. It extends the @@ -571,7 +573,7 @@ func (b *buffer) Float64Column(name string, val float64) *buffer { return b } -func (b *buffer) Float64ColumnBinaryFormat(name string, val float64) *buffer { +func (b *buffer) Float64ColumnBinary(name string, val float64) *buffer { if !b.prepareForField() { return b } @@ -582,8 +584,8 @@ func (b *buffer) Float64ColumnBinaryFormat(name string, val float64) *buffer { b.WriteByte('=') // binary format flag b.WriteByte('=') - b.WriteByte(byte(float64BinaryFlag)) - if isLittleEndian() { + b.WriteByte(byte(float64Code)) + if isLittleEndian { b.Write((*[8]byte)(unsafe.Pointer(&val))[:]) } else { data := make([]byte, 8) @@ -594,15 +596,7 @@ func (b *buffer) Float64ColumnBinaryFormat(name string, val float64) *buffer { return b } -func (b *buffer) writeFloat64ArrayHeader(dims byte) { - b.WriteByte('=') - b.WriteByte('=') - b.WriteByte(byte(arrayBinaryFlag)) - b.WriteByte(byte(arrayElemDouble)) - b.WriteByte(dims) -} - -func (b *buffer) Float641DArrayColumn(name string, values []float64) *buffer { +func (b *buffer) Float64Array1DColumn(name string, values []float64) *buffer { if !b.prepareForField() { return b } @@ -610,12 +604,20 @@ func (b *buffer) Float641DArrayColumn(name string, values []float64) *buffer { if b.lastErr != nil { return b } + if values == nil { + b.writeNullArray() + return b + } dim1 := len(values) + if dim1 > MaxArrayElements { + b.lastErr = fmt.Errorf("array size %d exceeds maximum limit %d", dim1, MaxArrayElements) + return b + } b.writeFloat64ArrayHeader(1) // Write shape - b.writeUint32(uint32(dim1)) + b.writeInt32(int32(dim1)) // Write values if len(values) > 0 { @@ -626,7 +628,7 @@ func (b *buffer) Float641DArrayColumn(name string, values []float64) *buffer { return b } -func (b *buffer) Float642DArrayColumn(name string, values [][]float64) *buffer { +func (b *buffer) Float64Array2DColumn(name string, values [][]float64) *buffer { if !b.prepareForField() { return b } @@ -635,11 +637,21 @@ func (b *buffer) Float642DArrayColumn(name string, values [][]float64) *buffer { return b } + if values == nil { + b.writeNullArray() + return b + } + // Validate array shape dim1 := len(values) var dim2 int if dim1 > 0 { dim2 = len(values[0]) + totalElements := dim1 * dim2 + if dim1 > MaxArrayElements || dim2 > MaxArrayElements || totalElements > MaxArrayElements || totalElements < 0 { + b.lastErr = fmt.Errorf("array size %d exceeds maximum limit %d", totalElements, MaxArrayElements) + return b + } for i, row := range values { if len(row) != dim2 { b.lastErr = fmt.Errorf("irregular 2D array shape: row %d has length %d, expected %d", i, len(row), dim2) @@ -651,8 +663,8 @@ func (b *buffer) Float642DArrayColumn(name string, values [][]float64) *buffer { b.writeFloat64ArrayHeader(2) // Write shape - b.writeUint32(uint32(dim1)) - b.writeUint32(uint32(dim2)) + b.writeInt32(int32(dim1)) + b.writeInt32(int32(dim2)) // Write values for _, row := range values { @@ -665,7 +677,7 @@ func (b *buffer) Float642DArrayColumn(name string, values [][]float64) *buffer { return b } -func (b *buffer) Float643DArrayColumn(name string, values [][][]float64) *buffer { +func (b *buffer) Float64Array3DColumn(name string, values [][][]float64) *buffer { if !b.prepareForField() { return b } @@ -674,6 +686,11 @@ func (b *buffer) Float643DArrayColumn(name string, values [][][]float64) *buffer return b } + if values == nil { + b.writeNullArray() + return b + } + // Validate array shape dim1 := len(values) var dim2, dim3 int @@ -682,6 +699,11 @@ func (b *buffer) Float643DArrayColumn(name string, values [][][]float64) *buffer if dim2 > 0 { dim3 = len(values[0][0]) } + totalElements := dim1 * dim2 * dim3 + if dim1 > MaxArrayElements || dim2 > MaxArrayElements || dim3 > MaxArrayElements || totalElements > MaxArrayElements || totalElements < 0 { + b.lastErr = fmt.Errorf("array size %d exceeds maximum limit %d", totalElements, MaxArrayElements) + return b + } for i, level1 := range values { if len(level1) != dim2 { @@ -700,9 +722,9 @@ func (b *buffer) Float643DArrayColumn(name string, values [][][]float64) *buffer b.writeFloat64ArrayHeader(3) // Write shape - b.writeUint32(uint32(dim1)) - b.writeUint32(uint32(dim2)) - b.writeUint32(uint32(dim3)) + b.writeInt32(int32(dim1)) + b.writeInt32(int32(dim2)) + b.writeInt32(int32(dim3)) // Write values for _, level1 := range values { @@ -717,7 +739,7 @@ func (b *buffer) Float643DArrayColumn(name string, values [][][]float64) *buffer return b } -func (b *buffer) Float64NDArrayColumn(name string, value *NdArray[float64]) *buffer { +func (b *buffer) Float64ArrayNDColumn(name string, value *NdArray[float64]) *buffer { if !b.prepareForField() { return b } @@ -726,25 +748,23 @@ func (b *buffer) Float64NDArrayColumn(name string, value *NdArray[float64]) *buf return b } - // Validate the NdArray if value == nil { - b.lastErr = fmt.Errorf("NDArray cannot be nil") + b.writeNullArray() return b } shape := value.Shape() numDims := value.NDims() - // Write nDims b.writeFloat64ArrayHeader(byte(numDims)) // Write shape for _, dim := range shape { - b.writeUint32(uint32(dim)) + b.writeInt32(int32(dim)) } // Write data - data := value.GetData() + data := value.Data() if len(data) > 0 { b.writeFloat64Data(data) } @@ -827,3 +847,19 @@ func (b *buffer) At(ts time.Time, sendTs bool) error { b.resetMsgFlags() return nil } + +func (b *buffer) writeFloat64ArrayHeader(dims byte) { + b.WriteByte('=') + b.WriteByte('=') + b.WriteByte(byte(arrayCode)) + b.WriteByte(byte(arrayElemDouble)) + b.WriteByte(dims) +} + +func (b *buffer) writeNullArray() { + b.WriteByte('=') + b.WriteByte('=') + b.WriteByte(byte(arrayCode)) + b.WriteByte(byte(arrayElemNull)) + b.hasFields = true +} diff --git a/buffer_test.go b/buffer_test.go index 8e461c71..02413f9c 100644 --- a/buffer_test.go +++ b/buffer_test.go @@ -456,7 +456,7 @@ func TestInvalidColumnName(t *testing.T) { assert.Empty(t, buf.Messages()) } -func TestFloat64ColumnBinaryFormat(t *testing.T) { +func TestFloat64ColumnBinary(t *testing.T) { testCases := []struct { name string val float64 @@ -474,14 +474,14 @@ func TestFloat64ColumnBinaryFormat(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { buf := newTestBuffer() - err := buf.Table(testTable).Float64ColumnBinaryFormat("a_col", tc.val).At(time.Time{}, false) + err := buf.Table(testTable).Float64ColumnBinary("a_col", tc.val).At(time.Time{}, false) assert.NoError(t, err) assert.Equal(t, buf.Messages(), float64ToByte(testTable, "a_col", tc.val)) }) } } -func TestFloat641DArrayColumn(t *testing.T) { +func TestFloat64Array1DColumn(t *testing.T) { testCases := []struct { name string values []float64 @@ -490,20 +490,24 @@ func TestFloat641DArrayColumn(t *testing.T) { {"multiple values", []float64{1.1, 2.2, 3.3}}, {"empty array", []float64{}}, {"with special values", []float64{math.NaN(), math.Inf(1), math.Inf(-1), 0.0}}, + {"null array", nil}, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - buf := newTestBuffer() + for _, littleEndian := range []bool{true, false} { + qdb.SetLittleEndian(littleEndian) + buf := newTestBuffer() - err := buf.Table(testTable).Float641DArrayColumn("array_col", tc.values).At(time.Time{}, false) - assert.NoError(t, err) - assert.Equal(t, float641DArrayToByte(testTable, "array_col", tc.values), buf.Messages()) + err := buf.Table(testTable).Float64Array1DColumn("array_col", tc.values).At(time.Time{}, false) + assert.NoError(t, err) + assert.Equal(t, float641DArrayToByte(testTable, "array_col", tc.values), buf.Messages()) + } }) } } -func TestFloat642DArrayColumn(t *testing.T) { +func TestFloat64Array2DColumn(t *testing.T) { testCases := []struct { name string values [][]float64 @@ -512,30 +516,33 @@ func TestFloat642DArrayColumn(t *testing.T) { {"1x3 array", [][]float64{{1.0, 2.0, 3.0}}}, {"3x1 array", [][]float64{{1.0}, {2.0}, {3.0}}}, {"empty array", [][]float64{}}, + {"null array", nil}, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - buf := newTestBuffer() - - err := buf.Table(testTable).Float642DArrayColumn("array_col", tc.values).At(time.Time{}, false) - assert.NoError(t, err) - assert.Equal(t, float642DArrayToByte(testTable, "array_col", tc.values), buf.Messages()) + for _, littleEndian := range []bool{true, false} { + qdb.SetLittleEndian(littleEndian) + buf := newTestBuffer() + err := buf.Table(testTable).Float64Array2DColumn("array_col", tc.values).At(time.Time{}, false) + assert.NoError(t, err) + assert.Equal(t, float642DArrayToByte(testTable, "array_col", tc.values), buf.Messages()) + } }) } } -func TestFloat642DArrayColumnIrregularShape(t *testing.T) { +func TestFloat64Array2DColumnIrregularShape(t *testing.T) { buf := newTestBuffer() irregularArray := [][]float64{{1.0, 2.0}, {3.0, 4.0, 5.0}} - err := buf.Table(testTable).Float642DArrayColumn("array_col", irregularArray).At(time.Time{}, false) + err := buf.Table(testTable).Float64Array2DColumn("array_col", irregularArray).At(time.Time{}, false) assert.ErrorContains(t, err, "irregular 2D array shape") assert.Empty(t, buf.Messages()) } -func TestFloat643DArrayColumn(t *testing.T) { +func TestFloat64Array3DColumn(t *testing.T) { testCases := []struct { name string values [][][]float64 @@ -543,29 +550,35 @@ func TestFloat643DArrayColumn(t *testing.T) { {"2x2x2 array", [][][]float64{{{1.1, 2.2}, {3.3, 4.4}}, {{5.5, 6.6}, {7.7, 8.8}}}}, {"1x1x3 array", [][][]float64{{{1.0, 2.0, 3.0}}}}, {"empty array", [][][]float64{}}, + {"null array", nil}, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - buf := newTestBuffer() - - err := buf.Table(testTable).Float643DArrayColumn("array_col", tc.values).At(time.Time{}, false) - assert.NoError(t, err) - assert.Equal(t, float643DArrayToByte(testTable, "array_col", tc.values), buf.Messages()) + for _, littleEndian := range []bool{true, false} { + qdb.SetLittleEndian(littleEndian) + buf := newTestBuffer() + err := buf.Table(testTable).Float64Array3DColumn("array_col", tc.values).At(time.Time{}, false) + assert.NoError(t, err) + assert.Equal(t, float643DArrayToByte(testTable, "array_col", tc.values), buf.Messages()) + } }) } } -func TestFloat643DArrayColumnIrregularShape(t *testing.T) { +func TestFloat64Array3DColumnIrregularShape(t *testing.T) { buf := newTestBuffer() irregularArray := [][][]float64{{{1.0, 2.0}, {3.0}}, {{4.0, 5.0}, {6.0, 7.0}}} - err := buf.Table(testTable).Float643DArrayColumn("array_col", irregularArray).At(time.Time{}, false) - - assert.ErrorContains(t, err, "irregular 3D array shape") + err := buf.Table(testTable).Float64Array3DColumn("array_col", irregularArray).At(time.Time{}, false) + assert.ErrorContains(t, err, "irregular 3D array shape: level2[0][1] has length 1, expected 2") + assert.Empty(t, buf.Messages()) + irregularArray2 := [][][]float64{{{1.0, 2.0}, {3.0, 4.0}}, {{4.0, 5.0}}} + err = buf.Table(testTable).Float64Array3DColumn("array_col", irregularArray2).At(time.Time{}, false) + assert.ErrorContains(t, err, "irregular 3D array shape: level1[1] has length 1, expected 2") assert.Empty(t, buf.Messages()) } -func TestFloat64NDArrayColumn(t *testing.T) { +func TestFloat64ArrayNDColumn(t *testing.T) { testCases := []struct { name string shape []uint @@ -574,28 +587,73 @@ func TestFloat64NDArrayColumn(t *testing.T) { {"1D array", []uint{3}, []float64{1.0, 2.0, 3.0}}, {"2D array", []uint{2, 2}, []float64{1.0, 2.0, 3.0, 4.0}}, {"3D array", []uint{2, 1, 2}, []float64{1.0, 2.0, 3.0, 4.0}}, + {"null array", nil, nil}, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - buf := newTestBuffer() - ndArray, err := qdb.NewNDArray[float64](tc.shape...) - assert.NoError(t, err) - for _, val := range tc.data { - ndArray.Append(val) + for _, littleEndian := range []bool{true, false} { + qdb.SetLittleEndian(littleEndian) + buf := newTestBuffer() + var err error + if tc.data == nil { + err = buf.Table(testTable).Float64ArrayNDColumn("ndarray_col", nil).At(time.Time{}, false) + assert.NoError(t, err) + assert.Equal(t, float64NDArrayToByte(testTable, "ndarray_col", nil), buf.Messages()) + } else { + ndArray, err := qdb.NewNDArray[float64](tc.shape...) + assert.NoError(t, err) + for _, val := range tc.data { + ndArray.Append(val) + } + err = buf.Table(testTable).Float64ArrayNDColumn("ndarray_col", ndArray).At(time.Time{}, false) + assert.NoError(t, err) + assert.Equal(t, float64NDArrayToByte(testTable, "ndarray_col", ndArray), buf.Messages()) + } } - - err = buf.Table(testTable).Float64NDArrayColumn("ndarray_col", ndArray).At(time.Time{}, false) - assert.NoError(t, err) - assert.Equal(t, float64NDArrayToByte(testTable, "ndarray_col", ndArray), buf.Messages()) }) } } -func TestFloat64NDArrayColumnNil(t *testing.T) { +func TestFloat64Array1DColumnExceedsMaxElements(t *testing.T) { buf := newTestBuffer() - err := buf.Table(testTable).Float64NDArrayColumn("ndarray_col", nil).At(time.Time{}, false) - assert.ErrorContains(t, err, "NDArray cannot be nil") + largeSize := qdb.MaxArrayElements + 1 + values := make([]float64, largeSize) + + err := buf.Table(testTable).Float64Array1DColumn("array_col", values).At(time.Time{}, false) + assert.ErrorContains(t, err, "array size 268435456 exceeds maximum limit 268435455") + assert.Empty(t, buf.Messages()) +} + +func TestFloat64Array2DColumnExceedsMaxElements(t *testing.T) { + buf := newTestBuffer() + dim1 := 65536 + dim2 := 4097 + values := make([][]float64, dim1) + for i := range values { + values[i] = make([]float64, dim2) + } + + err := buf.Table(testTable).Float64Array2DColumn("array_col", values).At(time.Time{}, false) + assert.ErrorContains(t, err, "array size 268500992 exceeds maximum limit 268435455") + assert.Empty(t, buf.Messages()) +} + +func TestFloat64Array3DColumnExceedsMaxElements(t *testing.T) { + buf := newTestBuffer() + dim1 := 1024 + dim2 := 1024 + dim3 := 256 + values := make([][][]float64, dim1) + for i := range values { + values[i] = make([][]float64, dim2) + for j := range values[i] { + values[i][j] = make([]float64, dim3) + } + } + + err := buf.Table(testTable).Float64Array3DColumn("array_col", values).At(time.Time{}, false) + assert.ErrorContains(t, err, "array size 268435456 exceeds maximum limit 268435455") assert.Empty(t, buf.Messages()) } @@ -614,7 +672,23 @@ func float64ToByte(table, col string, val float64) []byte { return buf } +func nullArrayToByte(table, col string) []byte { + buf := make([]byte, 0, 128) + buf = append(buf, ([]byte)(table)...) + buf = append(buf, ' ') + buf = append(buf, ([]byte)(col)...) + buf = append(buf, '=') + buf = append(buf, '=') + buf = append(buf, 14) + buf = append(buf, 33) + buf = append(buf, '\n') + return buf +} + func float641DArrayToByte(table, col string, vals []float64) []byte { + if vals == nil { + return nullArrayToByte(table, col) + } buf := make([]byte, 0, 128) buf = append(buf, ([]byte)(table)...) buf = append(buf, ' ') @@ -640,6 +714,9 @@ func float641DArrayToByte(table, col string, vals []float64) []byte { } func float642DArrayToByte(table, col string, vals [][]float64) []byte { + if vals == nil { + return nullArrayToByte(table, col) + } buf := make([]byte, 0, 256) buf = append(buf, ([]byte)(table)...) buf = append(buf, ' ') @@ -673,6 +750,9 @@ func float642DArrayToByte(table, col string, vals [][]float64) []byte { } func float643DArrayToByte(table, col string, vals [][][]float64) []byte { + if vals == nil { + return nullArrayToByte(table, col) + } buf := make([]byte, 0, 512) buf = append(buf, ([]byte)(table)...) buf = append(buf, ' ') @@ -712,6 +792,9 @@ func float643DArrayToByte(table, col string, vals [][][]float64) []byte { } func float64NDArrayToByte(table, col string, ndarray *qdb.NdArray[float64]) []byte { + if ndarray == nil { + return nullArrayToByte(table, col) + } buf := make([]byte, 0, 512) buf = append(buf, ([]byte)(table)...) buf = append(buf, ' ') @@ -729,7 +812,7 @@ func float64NDArrayToByte(table, col string, ndarray *qdb.NdArray[float64]) []by buf = append(buf, shapeData...) } - data := ndarray.GetData() + data := ndarray.Data() for _, val := range data { valData := make([]byte, 8) binary.LittleEndian.PutUint64(valData, math.Float64bits(val)) diff --git a/conf_parse.go b/conf_parse.go index 2f186974..7957dc51 100644 --- a/conf_parse.go +++ b/conf_parse.go @@ -170,7 +170,7 @@ func confFromStr(conf string) (*lineSenderConfig, error) { } pVersion := protocolVersion(version) if pVersion < ProtocolVersion1 || pVersion > ProtocolVersion2 { - return nil, NewInvalidConfigStrError("current client only supports protocol version 1(text format for all datatypes), 2(binary format for part datatypes) or explicitly unset") + return nil, NewInvalidConfigStrError("current client only supports protocol version 1 (text format for all datatypes), 2 (binary format for part datatypes) or explicitly unset") } senderConf.protocolVersion = pVersion } diff --git a/examples/from-conf/main.go b/examples/from-conf/main.go index fa78cd55..a3b993e6 100644 --- a/examples/from-conf/main.go +++ b/examples/from-conf/main.go @@ -33,22 +33,15 @@ func main() { if err != nil { log.Fatal(err) } - hasMore := true - val := 100.0 - for hasMore { - hasMore, err = array.Append(val + 1) - val = val + 1 - if err != nil { - log.Fatal(err) - } - } + // Fill array with value 87.2 + array.Fill(87.2) err = sender. Table("trades"). Symbol("symbol", "ETH-USD"). Symbol("side", "sell"). Float64Column("price", 2615.54). Float64Column("amount", 0.00044). - Float64NDArrayColumn("price_history", array). + Float64ArrayNDColumn("price_history", array). At(ctx, tradedTs) if err != nil { log.Fatal(err) @@ -60,9 +53,10 @@ func main() { } // Reuse array - hasMore = true + // `Append` value to array + hasMore := true array.ResetAppendIndex() - val = 200.0 + val := 200.0 for hasMore { hasMore, err = array.Append(val + 1) val = val + 1 @@ -76,7 +70,7 @@ func main() { Symbol("side", "sell"). Float64Column("price", 39269.98). Float64Column("amount", 0.001). - Float64NDArrayColumn("price_history", array). + Float64ArrayNDColumn("price_history", array). At(ctx, tradedTs) if err != nil { log.Fatal(err) diff --git a/examples/http/basic/main.go b/examples/http/basic/main.go index 130ee479..c1a52d9d 100644 --- a/examples/http/basic/main.go +++ b/examples/http/basic/main.go @@ -35,32 +35,25 @@ func main() { if err != nil { log.Fatal(err) } - hasMore := true - val := 100.0 - for hasMore { - hasMore, err = array.Append(val + 1) - val = val + 1 - if err != nil { - log.Fatal(err) - } - } - + // Fill array with value 87.2 + array.Fill(87.2) err = sender. Table("trades"). Symbol("symbol", "ETH-USD"). Symbol("side", "sell"). Float64Column("price", 2615.54). Float64Column("amount", 0.00044). - Float64NDArrayColumn("price_history", array). + Float64ArrayNDColumn("price_history", array). At(ctx, tradedTs) if err != nil { log.Fatal(err) } // Reuse array - hasMore = true + // `Append` value to array + hasMore := true array.ResetAppendIndex() - val = 200.0 + val := 200.0 for hasMore { hasMore, err = array.Append(val + 1) val = val + 1 @@ -78,7 +71,7 @@ func main() { Symbol("side", "sell"). Float64Column("price", 39269.98). Float64Column("amount", 0.001). - Float64NDArrayColumn("price_history", array). + Float64ArrayNDColumn("price_history", array). At(ctx, tradedTs) if err != nil { log.Fatal(err) diff --git a/examples/tcp/basic/main.go b/examples/tcp/basic/main.go index 6ce7ff4a..ca26ad66 100644 --- a/examples/tcp/basic/main.go +++ b/examples/tcp/basic/main.go @@ -30,15 +30,8 @@ func main() { if err != nil { log.Fatal(err) } - hasMore := true - val := 100.0 - for hasMore { - hasMore, err = array.Append(val + 1) - val = val + 1 - if err != nil { - log.Fatal(err) - } - } + // Fill array with value 87.2 + array.Fill(87.2) err = sender. Table("trades"). @@ -46,16 +39,17 @@ func main() { Symbol("side", "sell"). Float64Column("price", 2615.54). Float64Column("amount", 0.00044). - Float64NDArrayColumn("price_history", array). + Float64ArrayNDColumn("price_history", array). At(ctx, tradedTs) if err != nil { log.Fatal(err) } // Reuse array - hasMore = true + // `Append` value to array + hasMore := true array.ResetAppendIndex() - val = 200.0 + val := 200.0 for hasMore { hasMore, err = array.Append(val + 1) val = val + 1 @@ -74,7 +68,7 @@ func main() { Symbol("side", "sell"). Float64Column("price", 39269.98). Float64Column("amount", 0.001). - Float64NDArrayColumn("price_history", array). + Float64ArrayNDColumn("price_history", array). At(ctx, tradedTs) if err != nil { log.Fatal(err) diff --git a/export_test.go b/export_test.go index 21cce501..640c4735 100644 --- a/export_test.go +++ b/export_test.go @@ -132,3 +132,7 @@ func ProtocolVersion(s LineSender) protocolVersion { func NewLineSenderConfig(t SenderType) *LineSenderConfig { return newLineSenderConfig(t) } + +func SetLittleEndian(littleEndian bool) { + isLittleEndian = littleEndian +} diff --git a/http_sender.go b/http_sender.go index ab44388c..69f22fff 100644 --- a/http_sender.go +++ b/http_sender.go @@ -120,17 +120,6 @@ type httpLineSenderV2 struct { } func newHttpLineSender(ctx context.Context, conf *lineSenderConfig) (LineSender, error) { - - // auto detect server line protocol version - pVersion := conf.protocolVersion - if pVersion == protocolVersionUnset { - var err error - pVersion, err = detectProtocolVersion(ctx, conf) - if err != nil { - return nil, err - } - } - var transport *http.Transport s := &httpLineSender{ address: conf.address, @@ -170,6 +159,16 @@ func newHttpLineSender(ctx context.Context, conf *lineSenderConfig) (LineSender, s.globalTransport.RegisterClient() } + // auto detect server line protocol version + pVersion := conf.protocolVersion + if pVersion == protocolVersionUnset { + var err error + pVersion, err = s.detectProtocolVersion(ctx, conf) + if err != nil { + return nil, err + } + } + s.uri = "http" if conf.tlsMode != tlsDisabled { s.uri += "s" @@ -303,22 +302,22 @@ func (s *httpLineSender) BoolColumn(name string, val bool) LineSender { return s } -func (s *httpLineSender) Float641DArrayColumn(name string, values []float64) LineSender { +func (s *httpLineSender) Float64Array1DColumn(name string, values []float64) LineSender { s.buf.SetLastErr(errors.New("current protocol version does not support double-array")) return s } -func (s *httpLineSender) Float642DArrayColumn(name string, values [][]float64) LineSender { +func (s *httpLineSender) Float64Array2DColumn(name string, values [][]float64) LineSender { s.buf.SetLastErr(errors.New("current protocol version does not support double-array")) return s } -func (s *httpLineSender) Float643DArrayColumn(name string, values [][][]float64) LineSender { +func (s *httpLineSender) Float64Array3DColumn(name string, values [][][]float64) LineSender { s.buf.SetLastErr(errors.New("current protocol version does not support double-array")) return s } -func (s *httpLineSender) Float64NDArrayColumn(name string, values *NdArray[float64]) LineSender { +func (s *httpLineSender) Float64ArrayNDColumn(name string, values *NdArray[float64]) LineSender { s.buf.SetLastErr(errors.New("current protocol version does not support double-array")) return s } @@ -440,30 +439,23 @@ func (s *httpLineSender) makeRequest(ctx context.Context) (bool, error) { } -func detectProtocolVersion(ctx context.Context, conf *lineSenderConfig) (protocolVersion, error) { - tmpClient := http.Client{ - Transport: globalTransport.transport, - Timeout: 0, - } - globalTransport.RegisterClient() - defer globalTransport.UnregisterClient() - +func (s *httpLineSender) detectProtocolVersion(ctx context.Context, conf *lineSenderConfig) (protocolVersion, error) { scheme := "http" if conf.tlsMode != tlsDisabled { scheme = "https" } - settingsUri := fmt.Sprintf("%s://%s/settings", scheme, conf.address) + settingsUri := fmt.Sprintf("%s://%s/settings", scheme, s.address) req, err := http.NewRequest(http.MethodGet, settingsUri, nil) if err != nil { return protocolVersionUnset, err } - reqCtx, cancel := context.WithTimeout(ctx, conf.requestTimeout) + reqCtx, cancel := context.WithTimeout(ctx, s.requestTimeout) defer cancel() req = req.WithContext(reqCtx) - resp, err := tmpClient.Do(req) + resp, err := s.client.Do(req) if err != nil { return protocolVersionUnset, err } @@ -476,6 +468,9 @@ func detectProtocolVersion(ctx context.Context, conf *lineSenderConfig) (protoco return parseServerSettings(resp, conf) default: buf, _ := io.ReadAll(resp.Body) + if len(buf) > 1024 { + buf = append(buf[:1024], []byte("...")...) + } return protocolVersionUnset, fmt.Errorf("failed to detect server line protocol version [http-status=%d, http-message=%s]", resp.StatusCode, string(buf)) } @@ -509,17 +504,18 @@ func parseServerSettings(resp *http.Response, conf *lineSenderConfig) (protocolV return ProtocolVersion1, nil } + hasProtocolVersion1 := false for _, version := range versions { if version == 2 { return ProtocolVersion2, nil } - } - - for _, version := range versions { if version == 1 { - return ProtocolVersion1, nil + hasProtocolVersion1 = true } } + if hasProtocolVersion1 { + return ProtocolVersion1, nil + } return protocolVersionUnset, errors.New("server does not support current client") } @@ -593,26 +589,26 @@ func (s *httpLineSenderV2) BoolColumn(name string, val bool) LineSender { } func (s *httpLineSenderV2) Float64Column(name string, val float64) LineSender { - s.buf.Float64ColumnBinaryFormat(name, val) + s.buf.Float64ColumnBinary(name, val) return s } -func (s *httpLineSenderV2) Float641DArrayColumn(name string, values []float64) LineSender { - s.buf.Float641DArrayColumn(name, values) +func (s *httpLineSenderV2) Float64Array1DColumn(name string, values []float64) LineSender { + s.buf.Float64Array1DColumn(name, values) return s } -func (s *httpLineSenderV2) Float642DArrayColumn(name string, values [][]float64) LineSender { - s.buf.Float642DArrayColumn(name, values) +func (s *httpLineSenderV2) Float64Array2DColumn(name string, values [][]float64) LineSender { + s.buf.Float64Array2DColumn(name, values) return s } -func (s *httpLineSenderV2) Float643DArrayColumn(name string, values [][][]float64) LineSender { - s.buf.Float643DArrayColumn(name, values) +func (s *httpLineSenderV2) Float64Array3DColumn(name string, values [][][]float64) LineSender { + s.buf.Float64Array3DColumn(name, values) return s } -func (s *httpLineSenderV2) Float64NDArrayColumn(name string, values *NdArray[float64]) LineSender { - s.buf.Float64NDArrayColumn(name, values) +func (s *httpLineSenderV2) Float64ArrayNDColumn(name string, values *NdArray[float64]) LineSender { + s.buf.Float64ArrayNDColumn(name, values) return s } diff --git a/http_sender_test.go b/http_sender_test.go index af2b2cdf..008ac6bf 100644 --- a/http_sender_test.go +++ b/http_sender_test.go @@ -839,6 +839,16 @@ func TestAutoDetectProtocolVersionNewServer4(t *testing.T) { assert.ErrorContains(t, err, "server does not support current client") } +func TestAutoDetectProtocolVersionError(t *testing.T) { + ctx := context.Background() + + srv, err := newTestHttpServerWithErrMsg(readAndDiscard, "Internal error") + assert.NoError(t, err) + defer srv.Close() + _, err = qdb.NewLineSender(ctx, qdb.WithHttp(), qdb.WithAddress(srv.Addr())) + assert.ErrorContains(t, err, "failed to detect server line protocol version [http-status=500, http-message={\"code\":\"500\",\"message\":\"Internal error\"}]") +} + func TestSpecifyProtocolVersion(t *testing.T) { ctx := context.Background() @@ -869,28 +879,28 @@ func TestArrayColumnUnsupportedInHttpProtocolV1(t *testing.T) { err = sender. Table(testTable). - Float641DArrayColumn("array_1d", values1D). + Float64Array1DColumn("array_1d", values1D). At(ctx, time.UnixMicro(1)) assert.Error(t, err) assert.Contains(t, err.Error(), "current protocol version does not support double-array") err = sender. Table(testTable). - Float642DArrayColumn("array_2d", values2D). + Float64Array2DColumn("array_2d", values2D). At(ctx, time.UnixMicro(2)) assert.Error(t, err) assert.Contains(t, err.Error(), "current protocol version does not support double-array") err = sender. Table(testTable). - Float643DArrayColumn("array_3d", values3D). + Float64Array3DColumn("array_3d", values3D). At(ctx, time.UnixMicro(3)) assert.Error(t, err) assert.Contains(t, err.Error(), "current protocol version does not support double-array") err = sender. Table(testTable). - Float64NDArrayColumn("array_nd", arrayND). + Float64ArrayNDColumn("array_nd", arrayND). At(ctx, time.UnixMicro(4)) assert.Error(t, err) assert.Contains(t, err.Error(), "current protocol version does not support double-array") @@ -923,10 +933,10 @@ func BenchmarkHttpLineSenderBatch1000(b *testing.B) { StringColumn("str_col", "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua"). BoolColumn("bool_col", true). TimestampColumn("timestamp_col", time.UnixMicro(42)). - Float641DArrayColumn("array_1d", values1D). - Float642DArrayColumn("array_2d", values2D). - Float643DArrayColumn("array_3d", values3D). - Float64NDArrayColumn("array_nd", arrayND). + Float64Array1DColumn("array_1d", values1D). + Float64Array2DColumn("array_2d", values2D). + Float64Array3DColumn("array_3d", values3D). + Float64ArrayNDColumn("array_nd", arrayND). At(ctx, time.UnixMicro(int64(1000*i))) } sender.Flush(ctx) @@ -960,10 +970,10 @@ func BenchmarkHttpLineSenderNoFlush(b *testing.B) { StringColumn("str_col", "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua"). BoolColumn("bool_col", true). TimestampColumn("timestamp_col", time.UnixMicro(42)). - Float641DArrayColumn("array_1d", values1D). - Float642DArrayColumn("array_2d", values2D). - Float643DArrayColumn("array_3d", values3D). - Float64NDArrayColumn("array_nd", arrayND). + Float64Array1DColumn("array_1d", values1D). + Float64Array2DColumn("array_2d", values2D). + Float64Array3DColumn("array_3d", values3D). + Float64ArrayNDColumn("array_nd", arrayND). At(ctx, time.UnixMicro(int64(1000*i))) } sender.Flush(ctx) diff --git a/integration_test.go b/integration_test.go index 2442db64..25356341 100644 --- a/integration_test.go +++ b/integration_test.go @@ -132,7 +132,7 @@ func setupQuestDB0(ctx context.Context, auth ilpAuthType, setupProxy bool) (*que return nil, err } req := testcontainers.ContainerRequest{ - Image: "questdb/questdb:9.0.1", + Image: "questdb/questdb:9.0.2", ExposedPorts: []string{"9000/tcp", "9009/tcp"}, WaitingFor: wait.ForHTTP("/").WithPort("9000"), Networks: []string{networkName}, @@ -423,13 +423,36 @@ func (suite *integrationTestSuite) TestE2EValidWrites() { arrayND, _ := qdb.NewNDArray[float64](2, 2, 1, 2) arrayND.Fill(11.0) - return s. + err := s. Table(testTable). - Float641DArrayColumn("array_1d", values1D). - Float642DArrayColumn("array_2d", values2D). - Float643DArrayColumn("array_3d", values3D). - Float64NDArrayColumn("array_nd", arrayND). + Float64Array1DColumn("array_1d", values1D). + Float64Array2DColumn("array_2d", values2D). + Float64Array3DColumn("array_3d", values3D). + Float64ArrayNDColumn("array_nd", arrayND). At(ctx, time.UnixMicro(1)) + if err != nil { + return err + } + // empty array + emptyNdArray, _ := qdb.NewNDArray[float64](2, 2, 0, 2) + err = s. + Table(testTable). + Float64Array1DColumn("array_1d", []float64{}). + Float64Array2DColumn("array_2d", [][]float64{{}}). + Float64Array3DColumn("array_3d", [][][]float64{{{}}}). + Float64ArrayNDColumn("array_nd", emptyNdArray). + At(ctx, time.UnixMicro(2)) + if err != nil { + return err + } + // null array + return s. + Table(testTable). + Float64Array1DColumn("array_1d", nil). + Float64Array2DColumn("array_2d", nil). + Float64Array3DColumn("array_3d", nil). + Float64ArrayNDColumn("array_nd", nil). + At(ctx, time.UnixMicro(3)) }, tableData{ Columns: []column{ @@ -446,8 +469,20 @@ func (suite *integrationTestSuite) TestE2EValidWrites() { []interface{}{[]interface{}{[]interface{}{float64(1), float64(2)}, []interface{}{float64(3), float64(4)}}, []interface{}{[]interface{}{float64(5), float64(6)}, []interface{}{float64(7), float64(8)}}}, []interface{}{[]interface{}{[]interface{}{[]interface{}{float64(11), float64(11)}}, []interface{}{[]interface{}{float64(11), float64(11)}}}, []interface{}{[]interface{}{[]interface{}{float64(11), float64(11)}}, []interface{}{[]interface{}{float64(11), float64(11)}}}}, "1970-01-01T00:00:00.000001Z"}, + { + []interface{}{}, + []interface{}{}, + []interface{}{}, + []interface{}{}, + "1970-01-01T00:00:00.000002Z"}, + { + nil, + nil, + nil, + nil, + "1970-01-01T00:00:00.000003Z"}, }, - Count: 1, + Count: 3, }, }, } diff --git a/ndarray.go b/ndarray.go index d6e09093..1cfcce7d 100644 --- a/ndarray.go +++ b/ndarray.go @@ -8,9 +8,6 @@ import ( const ( // MaxDimensions defines the maximum dims of NdArray MaxDimensions = 32 - - // MaxElements defines the maximum total number of elements of NdArray - MaxElements = (1 << 28) - 1 ) // Numeric represents the constraint for numeric types that can be used in NdArray @@ -18,14 +15,23 @@ type Numeric interface { ~float64 } -// NdArray represents a generic n-dimensional array with shape validation +// NdArray represents a generic n-dimensional array with shape validation. +// It's designed to be used with the [LineSender.Float64ArrayNDColumn] method for sending +// multi-dimensional arrays to QuestDB via the ILP protocol. +// +// NdArray instances are meant to be reused across multiple calls to the sender +// to avoid memory allocations. Use Append to populate data and +// ResetAppendIndex to reset the array for reuse after sending data. +// +// By default, all values in the array are initialized to zero. type NdArray[T Numeric] struct { data []T shape []uint appendIndex uint } -// NewNDArray creates a new NdArray with the specified shape +// NewNDArray creates a new NdArray with the specified shape. +// All elements are initialized to zero by default. func NewNDArray[T Numeric](shape ...uint) (*NdArray[T], error) { if err := validateShape(shape); err != nil { return nil, fmt.Errorf("invalid shape: %w", err) @@ -109,7 +115,22 @@ func (n *NdArray[T]) Reshape(newShape ...uint) (*NdArray[T], error) { return newArray, nil } -// Append adds a value to the array sequentially +// Append adds a value to the array sequentially at the current append index. +// Returns true if there's more space for additional values, false if the array is now full. +// Use ResetAppendIndex() to reuse the array for multiple ILP messages. +// +// Example: +// +// arr, _ := NewNDArray[float64](2, 3) // 2x3 array (6 elements total) +// hasMore, _ := arr.Append(1.0) // hasMore = true, index now at 1 +// hasMore, _ = arr.Append(2.0) // hasMore = true, index now at 2 +// // ... append 4 more values +// hasMore, _ = arr.Append(6.0) // hasMore = false, array is full +// +// // To reuse the array: +// arr.ResetAppendIndex() +// arr.Append(10.0) // overwrites +// ... func (n *NdArray[T]) Append(val T) (bool, error) { if n.appendIndex >= uint(len(n.data)) { return false, errors.New("array is full") @@ -119,13 +140,27 @@ func (n *NdArray[T]) Append(val T) (bool, error) { return n.appendIndex < uint(len(n.data)), nil } -// ResetAppendIndex resets the append index to 0 +// ResetAppendIndex resets the append index to 0, allowing the NdArray to be reused +// for multiple append operations. This is useful for reusing arrays across multiple +// messages/rows ingestion without reallocating memory. +// +// Example: +// +// arr, _ := NewNDArray[float64](2) // 1D array with 3 elements +// arr.Append(2.0) +// arr.Append(3.0) // array is now full +// +// // sender.Float64ArrayNDColumn(arr) +// +// arr.ResetAppendIndex() // reset for reuse +// arr.Append(4.0) +// arr.Append(5.0) func (n *NdArray[T]) ResetAppendIndex() { n.appendIndex = 0 } -// GetData returns the underlying data slice -func (n *NdArray[T]) GetData() []T { +// Data returns the underlying data slice +func (n *NdArray[T]) Data() []T { return n.data } @@ -137,9 +172,7 @@ func (n *NdArray[T]) Fill(value T) { n.appendIndex = uint(len(n.data)) // Mark as full } -// positionsToIndex converts multi-dimensional positions to a flat index func (n *NdArray[T]) positionsToIndex(positions []uint) (int, error) { - // Validate positions are within bounds for i, pos := range positions { if pos >= n.shape[i] { return 0, fmt.Errorf("position[%d]=%d is out of bounds for dimension size %d", @@ -147,7 +180,6 @@ func (n *NdArray[T]) positionsToIndex(positions []uint) (int, error) { } } - // Calculate flat index index := 0 for i, pos := range positions { index += int(pos) * int(product(n.shape[i+1:])) @@ -155,29 +187,25 @@ func (n *NdArray[T]) positionsToIndex(positions []uint) (int, error) { return index, nil } -// validateShape validates that shape dimensions are valid func validateShape(shape []uint) error { if len(shape) == 0 { return errors.New("shape cannot be empty") } - // Check maximum dimensions limit if len(shape) > MaxDimensions { return fmt.Errorf("too many dimensions: %d exceeds maximum of %d", len(shape), MaxDimensions) } - // Check maximum elements limit totalElements := product(shape) - if totalElements > MaxElements { + if totalElements > MaxArrayElements { return fmt.Errorf("array too large: %d elements exceeds maximum of %d", - totalElements, MaxElements) + totalElements, MaxArrayElements) } return nil } -// product calculates the product of slice elements func product(s []uint) uint { if len(s) == 0 { return 0 diff --git a/ndarray_test.go b/ndarray_test.go index 14f73bb5..14779de5 100644 --- a/ndarray_test.go +++ b/ndarray_test.go @@ -68,7 +68,7 @@ func TestNew_InvalidShapes(t *testing.T) { }{ {"empty shape", []uint{}}, {"too many dimensions", make([]uint, qdb.MaxDimensions+1)}, - {"too many elements", []uint{qdb.MaxElements + 1}}, + {"too many elements", []uint{qdb.MaxArrayElements + 1}}, } for _, tc := range testCases { @@ -163,7 +163,7 @@ func TestReshape_ValidShapes(t *testing.T) { require.NoError(t, err) assert.Equal(t, tc.newShape, reshaped.Shape()) assert.Equal(t, arr.Size(), reshaped.Size()) - assert.Equal(t, arr.GetData(), reshaped.GetData()) + assert.Equal(t, arr.Data(), reshaped.Data()) }) } } @@ -178,7 +178,7 @@ func TestReshape_InvalidShapes(t *testing.T) { }{ {"wrong size", []uint{5}}, {"empty shape", []uint{}}, - {"too large", []uint{qdb.MaxElements + 1}}, + {"too large", []uint{qdb.MaxArrayElements + 1}}, } for _, tc := range testCases { @@ -206,7 +206,7 @@ func TestAppend(t *testing.T) { } } - assert.Equal(t, values, arr.GetData()) + assert.Equal(t, values, arr.Data()) _, err = arr.Append(5.0) assert.Error(t, err) assert.Contains(t, err.Error(), "array is full") @@ -268,7 +268,7 @@ func TestGetData_SharedReference(t *testing.T) { arr, err := qdb.NewNDArray[float64](2, 2) require.NoError(t, err) - data := arr.GetData() + data := arr.Data() data[0] = 42.0 val, err := arr.Get(0, 0) require.NoError(t, err) @@ -288,9 +288,9 @@ func TestMaxLimits(t *testing.T) { }) t.Run("max elements", func(t *testing.T) { - arr, err := qdb.NewNDArray[float64](qdb.MaxElements) + arr, err := qdb.NewNDArray[float64](qdb.MaxArrayElements) require.NoError(t, err) - assert.Equal(t, qdb.MaxElements, arr.Size()) + assert.Equal(t, qdb.MaxArrayElements, arr.Size()) }) } diff --git a/sender.go b/sender.go index c0d34016..b6f38f28 100644 --- a/sender.go +++ b/sender.go @@ -120,14 +120,14 @@ type LineSender interface { // '-', '*' '%%', '~', or a non-printable char. BoolColumn(name string, val bool) LineSender - // Float641DArrayColumn adds an array of 64-bit floats (double array) to the ILP message. + // Float64Array1DColumn adds an array of 64-bit floats (double array) to the ILP message. // // Column name cannot contain any of the following characters: // '\n', '\r', '?', '.', ',', "', '"', '\', '/', ':', ')', '(', '+', // '-', '*' '%%', '~', or a non-printable char. - Float641DArrayColumn(name string, values []float64) LineSender + Float64Array1DColumn(name string, values []float64) LineSender - // Float642DArrayColumn adds a 2D array of 64-bit floats (double 2D array) to the ILP message. + // Float64Array2DColumn adds a 2D array of 64-bit floats (double 2D array) to the ILP message. // // The values parameter must have a regular (rectangular) shape - all rows must have // exactly the same length. If the array has irregular shape, this method returns an error. @@ -141,9 +141,9 @@ type LineSender interface { // Column name cannot contain any of the following characters: // '\n', '\r', '?', '.', ',', "', '"', '\', '/', ':', ')', '(', '+', // '-', '*' '%%', '~', or a non-printable char. - Float642DArrayColumn(name string, values [][]float64) LineSender + Float64Array2DColumn(name string, values [][]float64) LineSender - // Float643DArrayColumn adds a 3D array of 64-bit floats (double 3D array) to the ILP message. + // Float64Array3DColumn adds a 3D array of 64-bit floats (double 3D array) to the ILP message. // // The values parameter must have a regular (cuboid) shape - all dimensions must have // consistent sizes throughout. If the array has irregular shape, this method returns an error. @@ -163,20 +163,20 @@ type LineSender interface { // Column name cannot contain any of the following characters: // '\n', '\r', '?', '.', ',', "', '"', '\', '/', ':', ')', '(', '+', // '-', '*' '%%', '~', or a non-printable char. - Float643DArrayColumn(name string, values [][][]float64) LineSender + Float64Array3DColumn(name string, values [][][]float64) LineSender - // Float64NDArrayColumn adds an n-dimensional array of 64-bit floats (double n-D array) to the ILP message. + // Float64ArrayNDColumn adds an n-dimensional array of 64-bit floats (double n-D array) to the ILP message. // // Example usage: // // Create a 2x3x4 array // arr, _ := questdb.NewNDArray[float64](2, 3, 4) // arr.Fill(1.5) - // sender.Float64NDArrayColumn("ndarray_col", arr) + // sender.Float64ArrayNDColumn("ndarray_col", arr) // // Column name cannot contain any of the following characters: // '\n', '\r', '?', '.', ',', "', '"', '\', '/', ':', ')', '(', '+', // '-', '*' '%%', '~', or a non-printable char. - Float64NDArrayColumn(name string, values *NdArray[float64]) LineSender + Float64ArrayNDColumn(name string, values *NdArray[float64]) LineSender // At sets the designated timestamp value and finalizes the ILP // message. diff --git a/sender_pool.go b/sender_pool.go index cba9f3c0..7da85726 100644 --- a/sender_pool.go +++ b/sender_pool.go @@ -324,23 +324,23 @@ func (ps *pooledSender) BoolColumn(name string, val bool) LineSender { return ps } -func (ps *pooledSender) Float641DArrayColumn(name string, values []float64) LineSender { - ps.wrapped.Float641DArrayColumn(name, values) +func (ps *pooledSender) Float64Array1DColumn(name string, values []float64) LineSender { + ps.wrapped.Float64Array1DColumn(name, values) return ps } -func (ps *pooledSender) Float642DArrayColumn(name string, values [][]float64) LineSender { - ps.wrapped.Float642DArrayColumn(name, values) +func (ps *pooledSender) Float64Array2DColumn(name string, values [][]float64) LineSender { + ps.wrapped.Float64Array2DColumn(name, values) return ps } -func (ps *pooledSender) Float643DArrayColumn(name string, values [][][]float64) LineSender { - ps.wrapped.Float643DArrayColumn(name, values) +func (ps *pooledSender) Float64Array3DColumn(name string, values [][][]float64) LineSender { + ps.wrapped.Float64Array3DColumn(name, values) return ps } -func (ps *pooledSender) Float64NDArrayColumn(name string, values *NdArray[float64]) LineSender { - ps.wrapped.Float64NDArrayColumn(name, values) +func (ps *pooledSender) Float64ArrayNDColumn(name string, values *NdArray[float64]) LineSender { + ps.wrapped.Float64ArrayNDColumn(name, values) return ps } diff --git a/tcp_integration_test.go b/tcp_integration_test.go index 07832a81..9ac2ceff 100644 --- a/tcp_integration_test.go +++ b/tcp_integration_test.go @@ -374,10 +374,10 @@ func (suite *integrationTestSuite) TestDoubleArrayColumn() { err = sender. Table(testTable). - Float641DArrayColumn("array_1d", values1D). - Float642DArrayColumn("array_2d", values2D). - Float643DArrayColumn("array_3d", values3D). - Float64NDArrayColumn("array_nd", arrayND). + Float64Array1DColumn("array_1d", values1D). + Float64Array2DColumn("array_2d", values2D). + Float64Array3DColumn("array_3d", values3D). + Float64ArrayNDColumn("array_nd", arrayND). At(ctx, time.UnixMicro(1)) assert.NoError(suite.T(), err) diff --git a/tcp_sender.go b/tcp_sender.go index 1f7a93bc..f46c5de1 100644 --- a/tcp_sender.go +++ b/tcp_sender.go @@ -194,22 +194,22 @@ func (s *tcpLineSender) BoolColumn(name string, val bool) LineSender { return s } -func (s *tcpLineSender) Float641DArrayColumn(name string, values []float64) LineSender { +func (s *tcpLineSender) Float64Array1DColumn(name string, values []float64) LineSender { s.buf.SetLastErr(errors.New("current protocol version does not support double-array")) return s } -func (s *tcpLineSender) Float642DArrayColumn(name string, values [][]float64) LineSender { +func (s *tcpLineSender) Float64Array2DColumn(name string, values [][]float64) LineSender { s.buf.SetLastErr(errors.New("current protocol version does not support double-array")) return s } -func (s *tcpLineSender) Float643DArrayColumn(name string, values [][][]float64) LineSender { +func (s *tcpLineSender) Float64Array3DColumn(name string, values [][][]float64) LineSender { s.buf.SetLastErr(errors.New("current protocol version does not support double-array")) return s } -func (s *tcpLineSender) Float64NDArrayColumn(name string, values *NdArray[float64]) LineSender { +func (s *tcpLineSender) Float64ArrayNDColumn(name string, values *NdArray[float64]) LineSender { s.buf.SetLastErr(errors.New("current protocol version does not support double-array")) return s } @@ -323,26 +323,26 @@ func (s *tcpLineSenderV2) BoolColumn(name string, val bool) LineSender { } func (s *tcpLineSenderV2) Float64Column(name string, val float64) LineSender { - s.buf.Float64ColumnBinaryFormat(name, val) + s.buf.Float64ColumnBinary(name, val) return s } -func (s *tcpLineSenderV2) Float641DArrayColumn(name string, values []float64) LineSender { - s.buf.Float641DArrayColumn(name, values) +func (s *tcpLineSenderV2) Float64Array1DColumn(name string, values []float64) LineSender { + s.buf.Float64Array1DColumn(name, values) return s } -func (s *tcpLineSenderV2) Float642DArrayColumn(name string, values [][]float64) LineSender { - s.buf.Float642DArrayColumn(name, values) +func (s *tcpLineSenderV2) Float64Array2DColumn(name string, values [][]float64) LineSender { + s.buf.Float64Array2DColumn(name, values) return s } -func (s *tcpLineSenderV2) Float643DArrayColumn(name string, values [][][]float64) LineSender { - s.buf.Float643DArrayColumn(name, values) +func (s *tcpLineSenderV2) Float64Array3DColumn(name string, values [][][]float64) LineSender { + s.buf.Float64Array3DColumn(name, values) return s } -func (s *tcpLineSenderV2) Float64NDArrayColumn(name string, values *NdArray[float64]) LineSender { - s.buf.Float64NDArrayColumn(name, values) +func (s *tcpLineSenderV2) Float64ArrayNDColumn(name string, values *NdArray[float64]) LineSender { + s.buf.Float64ArrayNDColumn(name, values) return s } diff --git a/tcp_sender_test.go b/tcp_sender_test.go index 02949e65..688494e1 100644 --- a/tcp_sender_test.go +++ b/tcp_sender_test.go @@ -144,7 +144,7 @@ func TestTcpPathologicalCasesFromEnv(t *testing.T) { { name: "protocol_version", config: "tcp::protocol_version=3;", - expectedErr: "current client only supports protocol version 1(text format for all datatypes), 2(binary format for part datatypes) or explicitly unset", + expectedErr: "current client only supports protocol version 1 (text format for all datatypes), 2 (binary format for part datatypes) or explicitly unset", }, } @@ -339,28 +339,28 @@ func TestArrayColumnUnsupportedInTCPProtocolV1(t *testing.T) { err = sender. Table(testTable). - Float641DArrayColumn("array_1d", values1D). + Float64Array1DColumn("array_1d", values1D). At(ctx, time.UnixMicro(1)) assert.Error(t, err) assert.Contains(t, err.Error(), "current protocol version does not support double-array") err = sender. Table(testTable). - Float642DArrayColumn("array_2d", values2D). + Float64Array2DColumn("array_2d", values2D). At(ctx, time.UnixMicro(2)) assert.Error(t, err) assert.Contains(t, err.Error(), "current protocol version does not support double-array") err = sender. Table(testTable). - Float643DArrayColumn("array_3d", values3D). + Float64Array3DColumn("array_3d", values3D). At(ctx, time.UnixMicro(3)) assert.Error(t, err) assert.Contains(t, err.Error(), "current protocol version does not support double-array") err = sender. Table(testTable). - Float64NDArrayColumn("array_nd", arrayND). + Float64ArrayNDColumn("array_nd", arrayND). At(ctx, time.UnixMicro(4)) assert.Error(t, err) assert.Contains(t, err.Error(), "current protocol version does not support double-array") @@ -395,10 +395,10 @@ func BenchmarkLineSenderBatch1000(b *testing.B) { StringColumn("str_col", "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua"). BoolColumn("bool_col", true). TimestampColumn("timestamp_col", time.UnixMicro(42)). - Float641DArrayColumn("array_1d", values1D). - Float642DArrayColumn("array_2d", values2D). - Float643DArrayColumn("array_3d", values3D). - Float64NDArrayColumn("array_nd", arrayND). + Float64Array1DColumn("array_1d", values1D). + Float64Array2DColumn("array_2d", values2D). + Float64Array3DColumn("array_3d", values3D). + Float64ArrayNDColumn("array_nd", arrayND). At(ctx, time.UnixMicro(int64(1000*i))) } sender.Flush(ctx) @@ -432,10 +432,10 @@ func BenchmarkLineSenderNoFlush(b *testing.B) { StringColumn("str_col", "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua"). BoolColumn("bool_col", true). TimestampColumn("timestamp_col", time.UnixMicro(42)). - Float641DArrayColumn("array_1d", values1D). - Float642DArrayColumn("array_2d", values2D). - Float643DArrayColumn("array_3d", values3D). - Float64NDArrayColumn("array_nd", arrayND). + Float64Array1DColumn("array_1d", values1D). + Float64Array2DColumn("array_2d", values2D). + Float64Array3DColumn("array_3d", values3D). + Float64ArrayNDColumn("array_nd", arrayND). At(ctx, time.UnixMicro(int64(1000*i))) } sender.Flush(ctx) diff --git a/utils_test.go b/utils_test.go index b197fb77..d37cf5d5 100644 --- a/utils_test.go +++ b/utils_test.go @@ -53,13 +53,14 @@ const ( ) type testServer struct { - addr string - tcpListener net.Listener - serverType serverType - protocolVersions []int - BackCh chan string - closeCh chan struct{} - wg sync.WaitGroup + addr string + tcpListener net.Listener + serverType serverType + protocolVersions []int + BackCh chan string + closeCh chan struct{} + wg sync.WaitGroup + settingsReqErrMsg string } func (t *testServer) Addr() string { @@ -74,6 +75,23 @@ func newTestHttpServer(serverType serverType) (*testServer, error) { return newTestServerWithProtocol(serverType, "http", []int{1, 2}) } +func newTestHttpServerWithErrMsg(serverType serverType, errMsg string) (*testServer, error) { + tcp, err := net.Listen("tcp", "127.0.0.1:") + if err != nil { + return nil, err + } + s := &testServer{ + addr: tcp.Addr().String(), + tcpListener: tcp, + serverType: serverType, + BackCh: make(chan string, 1000), + closeCh: make(chan struct{}), + settingsReqErrMsg: errMsg, + } + go s.serveHttp() + return s, nil +} + func newTestServerWithProtocol(serverType serverType, protocol string, protocolVersions []int) (*testServer, error) { tcp, err := net.Listen("tcp", "127.0.0.1:") if err != nil { @@ -196,7 +214,18 @@ func (s *testServer) serveHttp() { err error ) if r.Method == "GET" && r.URL.Path == "/settings" { - if s.protocolVersions == nil { + if len(s.settingsReqErrMsg) != 0 { + w.WriteHeader(http.StatusInternalServerError) + data, err := json.Marshal(map[string]interface{}{ + "code": "500", + "message": s.settingsReqErrMsg, + }) + if err != nil { + panic(err) + } + w.Write(data) + return + } else if s.protocolVersions == nil { w.WriteHeader(http.StatusNotFound) data, err := json.Marshal(map[string]interface{}{ "code": "404", From e408045379e53a340ae9ded42c6c299eb217335d Mon Sep 17 00:00:00 2001 From: victor Date: Thu, 21 Aug 2025 13:46:26 +0800 Subject: [PATCH 04/24] rename --- ndarray.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ndarray.go b/ndarray.go index 1cfcce7d..6aee33ba 100644 --- a/ndarray.go +++ b/ndarray.go @@ -10,8 +10,8 @@ const ( MaxDimensions = 32 ) -// Numeric represents the constraint for numeric types that can be used in NdArray -type Numeric interface { +// NdArrayElementType represents the constraint for numeric types that can be used in NdArray +type NdArrayElementType interface { ~float64 } @@ -24,7 +24,7 @@ type Numeric interface { // ResetAppendIndex to reset the array for reuse after sending data. // // By default, all values in the array are initialized to zero. -type NdArray[T Numeric] struct { +type NdArray[T NdArrayElementType] struct { data []T shape []uint appendIndex uint @@ -32,7 +32,7 @@ type NdArray[T Numeric] struct { // NewNDArray creates a new NdArray with the specified shape. // All elements are initialized to zero by default. -func NewNDArray[T Numeric](shape ...uint) (*NdArray[T], error) { +func NewNDArray[T NdArrayElementType](shape ...uint) (*NdArray[T], error) { if err := validateShape(shape); err != nil { return nil, fmt.Errorf("invalid shape: %w", err) } From 99d1a6c94e8bfe8d5bad0c055c7f13d8e6dbd2e0 Mon Sep 17 00:00:00 2001 From: victor Date: Thu, 21 Aug 2025 22:54:16 +0800 Subject: [PATCH 05/24] code reviews. --- buffer.go | 8 ++++---- integration_test.go | 16 +++++++++------- ndarray.go | 5 ++++- ndarray_test.go | 3 ++- 4 files changed, 19 insertions(+), 13 deletions(-) diff --git a/buffer.go b/buffer.go index 88345e29..f4dd1bfa 100644 --- a/buffer.go +++ b/buffer.go @@ -647,8 +647,8 @@ func (b *buffer) Float64Array2DColumn(name string, values [][]float64) *buffer { var dim2 int if dim1 > 0 { dim2 = len(values[0]) - totalElements := dim1 * dim2 - if dim1 > MaxArrayElements || dim2 > MaxArrayElements || totalElements > MaxArrayElements || totalElements < 0 { + totalElements := product([]uint{uint(dim1), uint(dim2)}) + if totalElements > MaxArrayElements { b.lastErr = fmt.Errorf("array size %d exceeds maximum limit %d", totalElements, MaxArrayElements) return b } @@ -699,8 +699,8 @@ func (b *buffer) Float64Array3DColumn(name string, values [][][]float64) *buffer if dim2 > 0 { dim3 = len(values[0][0]) } - totalElements := dim1 * dim2 * dim3 - if dim1 > MaxArrayElements || dim2 > MaxArrayElements || dim3 > MaxArrayElements || totalElements > MaxArrayElements || totalElements < 0 { + totalElements := product([]uint{uint(dim1), uint(dim2), uint(dim3)}) + if totalElements > MaxArrayElements { b.lastErr = fmt.Errorf("array size %d exceeds maximum limit %d", totalElements, MaxArrayElements) return b } diff --git a/integration_test.go b/integration_test.go index 25356341..a81d745d 100644 --- a/integration_test.go +++ b/integration_test.go @@ -27,6 +27,7 @@ package questdb_test import ( "context" "fmt" + "math" "math/big" "path/filepath" "reflect" @@ -417,11 +418,12 @@ func (suite *integrationTestSuite) TestE2EValidWrites() { "double array", testTable, func(s qdb.LineSender) error { - values1D := []float64{1.0, 2.0, 3.0, 4.0, 5.0} - values2D := [][]float64{{1.0, 2.0}, {3.0, 4.0}, {5.0, 6.0}} - values3D := [][][]float64{{{1.0, 2.0}, {3.0, 4.0}}, {{5.0, 6.0}, {7.0, 8.0}}} + values1D := []float64{1.0, 2.0, 3.0, 4.0, 5.0, math.NaN()} + values2D := [][]float64{{1.0, 2.0}, {3.0, 4.0}, {5.0, 6.0}, {math.NaN(), math.NaN()}} + values3D := [][][]float64{{{1.0, 2.0}, {3.0, 4.0}}, {{5.0, 6.0}, {7.0, math.NaN()}}} arrayND, _ := qdb.NewNDArray[float64](2, 2, 1, 2) arrayND.Fill(11.0) + arrayND.Set(math.NaN(), 1, 1, 0, 1) err := s. Table(testTable). @@ -464,10 +466,10 @@ func (suite *integrationTestSuite) TestE2EValidWrites() { }, Dataset: [][]interface{}{ { - []interface{}{float64(1), float64(2), float64(3), float64(4), float64(5)}, - []interface{}{[]interface{}{float64(1), float64(2)}, []interface{}{float64(3), float64(4)}, []interface{}{float64(5), float64(6)}}, - []interface{}{[]interface{}{[]interface{}{float64(1), float64(2)}, []interface{}{float64(3), float64(4)}}, []interface{}{[]interface{}{float64(5), float64(6)}, []interface{}{float64(7), float64(8)}}}, - []interface{}{[]interface{}{[]interface{}{[]interface{}{float64(11), float64(11)}}, []interface{}{[]interface{}{float64(11), float64(11)}}}, []interface{}{[]interface{}{[]interface{}{float64(11), float64(11)}}, []interface{}{[]interface{}{float64(11), float64(11)}}}}, + []interface{}{float64(1), float64(2), float64(3), float64(4), float64(5), nil}, + []interface{}{[]interface{}{float64(1), float64(2)}, []interface{}{float64(3), float64(4)}, []interface{}{float64(5), float64(6)}, []interface{}{nil, nil}}, + []interface{}{[]interface{}{[]interface{}{float64(1), float64(2)}, []interface{}{float64(3), float64(4)}}, []interface{}{[]interface{}{float64(5), float64(6)}, []interface{}{float64(7), nil}}}, + []interface{}{[]interface{}{[]interface{}{[]interface{}{float64(11), float64(11)}}, []interface{}{[]interface{}{float64(11), float64(11)}}}, []interface{}{[]interface{}{[]interface{}{float64(11), float64(11)}}, []interface{}{[]interface{}{float64(11), nil}}}}, "1970-01-01T00:00:00.000001Z"}, { []interface{}{}, diff --git a/ndarray.go b/ndarray.go index 6aee33ba..70040b78 100644 --- a/ndarray.go +++ b/ndarray.go @@ -208,10 +208,13 @@ func validateShape(shape []uint) error { func product(s []uint) uint { if len(s) == 0 { - return 0 + return 1 } p := uint(1) for _, v := range s { + if v != 0 && p > MaxArrayElements/v { + return MaxArrayElements + 1 + } p *= v } return p diff --git a/ndarray_test.go b/ndarray_test.go index 14779de5..acd0fdef 100644 --- a/ndarray_test.go +++ b/ndarray_test.go @@ -178,7 +178,8 @@ func TestReshape_InvalidShapes(t *testing.T) { }{ {"wrong size", []uint{5}}, {"empty shape", []uint{}}, - {"too large", []uint{qdb.MaxArrayElements + 1}}, + {"too large1", []uint{qdb.MaxArrayElements + 1}}, + {"too large2", []uint{4294967296, 4294967296}}, } for _, tc := range testCases { From b4669b2885bbb6f391eb12918ea000e5ccd4bddc Mon Sep 17 00:00:00 2001 From: victor Date: Fri, 22 Aug 2025 09:01:15 +0800 Subject: [PATCH 06/24] better example comments. --- examples/from-conf/main.go | 3 +-- examples/http/basic/main.go | 3 +-- examples/tcp/basic/main.go | 3 +-- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/examples/from-conf/main.go b/examples/from-conf/main.go index a3b993e6..08437b14 100644 --- a/examples/from-conf/main.go +++ b/examples/from-conf/main.go @@ -52,8 +52,7 @@ func main() { log.Fatal(err) } - // Reuse array - // `Append` value to array + // Reuse array by resetting index and appending new values sequentially hasMore := true array.ResetAppendIndex() val := 200.0 diff --git a/examples/http/basic/main.go b/examples/http/basic/main.go index c1a52d9d..83c03707 100644 --- a/examples/http/basic/main.go +++ b/examples/http/basic/main.go @@ -49,8 +49,7 @@ func main() { log.Fatal(err) } - // Reuse array - // `Append` value to array + // Reuse array by resetting index and appending new values sequentially hasMore := true array.ResetAppendIndex() val := 200.0 diff --git a/examples/tcp/basic/main.go b/examples/tcp/basic/main.go index ca26ad66..5fdc632c 100644 --- a/examples/tcp/basic/main.go +++ b/examples/tcp/basic/main.go @@ -45,8 +45,7 @@ func main() { log.Fatal(err) } - // Reuse array - // `Append` value to array + // Reuse array by resetting index and appending new values sequentially hasMore := true array.ResetAppendIndex() val := 200.0 From 6d25e299019f23958dee3f746165e0fb6e7d2c1f Mon Sep 17 00:00:00 2001 From: victor Date: Mon, 25 Aug 2025 16:49:58 +0800 Subject: [PATCH 07/24] update staticcheck command and fix staticcheck code. --- .github/workflows/build.yml | 8 ++------ buffer.go | 2 +- http_sender_test.go | 7 +++++++ 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b3e25f02..999ce8f0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -24,12 +24,8 @@ jobs: - name: Run vet run: go vet ./... - - name: Run staticcheck - uses: dominikh/staticcheck-action@v1.3.0 - with: - version: "2023.1.2" - install-go: false - cache-key: ${{ matrix.go-version }} + - name: Run Staticcheck + run: go run honnef.co/go/tools/cmd/staticcheck@latest ./... - name: Run tests run: go test -v ./... diff --git a/buffer.go b/buffer.go index f4dd1bfa..85d586f1 100644 --- a/buffer.go +++ b/buffer.go @@ -86,7 +86,7 @@ type arrayElemType byte const ( arrayElemDouble arrayElemType = 10 - arrayElemNull = 33 + arrayElemNull arrayElemType = 33 ) // buffer is a wrapper on top of bytes.Buffer. It extends the diff --git a/http_sender_test.go b/http_sender_test.go index 008ac6bf..47d9a431 100644 --- a/http_sender_test.go +++ b/http_sender_test.go @@ -777,6 +777,7 @@ func TestAutoDetectProtocolVersionOldServer1(t *testing.T) { defer srv.Close() sender, err := qdb.NewLineSender(ctx, qdb.WithHttp(), qdb.WithAddress(srv.Addr())) assert.Equal(t, qdb.ProtocolVersion(sender), qdb.ProtocolVersion1) + assert.NoError(t, err) } func TestAutoDetectProtocolVersionOldServer2(t *testing.T) { @@ -787,6 +788,7 @@ func TestAutoDetectProtocolVersionOldServer2(t *testing.T) { defer srv.Close() sender, err := qdb.NewLineSender(ctx, qdb.WithHttp(), qdb.WithAddress(srv.Addr())) assert.Equal(t, qdb.ProtocolVersion(sender), qdb.ProtocolVersion1) + assert.NoError(t, err) } func TestAutoDetectProtocolVersionOldServer3(t *testing.T) { @@ -797,6 +799,7 @@ func TestAutoDetectProtocolVersionOldServer3(t *testing.T) { defer srv.Close() sender, err := qdb.NewLineSender(ctx, qdb.WithHttp(), qdb.WithAddress(srv.Addr())) assert.Equal(t, qdb.ProtocolVersion(sender), qdb.ProtocolVersion1) + assert.NoError(t, err) } func TestAutoDetectProtocolVersionNewServer1(t *testing.T) { @@ -807,6 +810,7 @@ func TestAutoDetectProtocolVersionNewServer1(t *testing.T) { defer srv.Close() sender, err := qdb.NewLineSender(ctx, qdb.WithHttp(), qdb.WithAddress(srv.Addr())) assert.Equal(t, qdb.ProtocolVersion(sender), qdb.ProtocolVersion2) + assert.NoError(t, err) } func TestAutoDetectProtocolVersionNewServer2(t *testing.T) { @@ -817,6 +821,7 @@ func TestAutoDetectProtocolVersionNewServer2(t *testing.T) { defer srv.Close() sender, err := qdb.NewLineSender(ctx, qdb.WithHttp(), qdb.WithAddress(srv.Addr())) assert.Equal(t, qdb.ProtocolVersion(sender), qdb.ProtocolVersion2) + assert.NoError(t, err) } func TestAutoDetectProtocolVersionNewServer3(t *testing.T) { @@ -827,6 +832,7 @@ func TestAutoDetectProtocolVersionNewServer3(t *testing.T) { defer srv.Close() sender, err := qdb.NewLineSender(ctx, qdb.WithHttp(), qdb.WithAddress(srv.Addr())) assert.Equal(t, qdb.ProtocolVersion(sender), qdb.ProtocolVersion2) + assert.NoError(t, err) } func TestAutoDetectProtocolVersionNewServer4(t *testing.T) { @@ -857,6 +863,7 @@ func TestSpecifyProtocolVersion(t *testing.T) { defer srv.Close() sender, err := qdb.NewLineSender(ctx, qdb.WithHttp(), qdb.WithAddress(srv.Addr()), qdb.WithProtocolVersion(qdb.ProtocolVersion1)) assert.Equal(t, qdb.ProtocolVersion(sender), qdb.ProtocolVersion1) + assert.NoError(t, err) } func TestArrayColumnUnsupportedInHttpProtocolV1(t *testing.T) { From 317e4c5fd1cbe02524793dde9ee71263d6e5c1f3 Mon Sep 17 00:00:00 2001 From: victor Date: Mon, 25 Aug 2025 16:55:49 +0800 Subject: [PATCH 08/24] update staticcheck version to 0.4.3 in build configuration --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 999ce8f0..3d35507f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -25,7 +25,7 @@ jobs: run: go vet ./... - name: Run Staticcheck - run: go run honnef.co/go/tools/cmd/staticcheck@latest ./... + run: go run honnef.co/go/tools/cmd/staticcheck@0.4.3 ./... - name: Run tests run: go test -v ./... From f9631dc90a983c115e4620a85eb0607fe0081109 Mon Sep 17 00:00:00 2001 From: victor Date: Mon, 25 Aug 2025 17:04:25 +0800 Subject: [PATCH 09/24] version typo --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3d35507f..a73f324d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -25,7 +25,7 @@ jobs: run: go vet ./... - name: Run Staticcheck - run: go run honnef.co/go/tools/cmd/staticcheck@0.4.3 ./... + run: go run honnef.co/go/tools/cmd/staticcheck@v0.4.3 ./... - name: Run tests run: go test -v ./... From 14b5e61a82528dfca749983cdf9414d019a71a56 Mon Sep 17 00:00:00 2001 From: victor Date: Mon, 25 Aug 2025 17:16:01 +0800 Subject: [PATCH 10/24] add suffix to network name for unique identification in integration tests --- integration_test.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/integration_test.go b/integration_test.go index a81d745d..e4264314 100644 --- a/integration_test.go +++ b/integration_test.go @@ -100,6 +100,8 @@ const ( bearerToken = "testToken1" ) +var networkNameSuffix = 1 + func setupQuestDB(ctx context.Context, auth ilpAuthType) (*questdbContainer, error) { return setupQuestDB0(ctx, auth, false) } @@ -132,12 +134,15 @@ func setupQuestDB0(ctx context.Context, auth ilpAuthType, setupProxy bool) (*que if err != nil { return nil, err } + + networkNameWithSuffix := fmt.Sprintf("%s_%d", networkName, networkNameSuffix) + networkNameSuffix++ req := testcontainers.ContainerRequest{ Image: "questdb/questdb:9.0.2", ExposedPorts: []string{"9000/tcp", "9009/tcp"}, WaitingFor: wait.ForHTTP("/").WithPort("9000"), - Networks: []string{networkName}, - NetworkAliases: map[string][]string{networkName: {"questdb"}}, + Networks: []string{networkNameWithSuffix}, + NetworkAliases: map[string][]string{networkNameWithSuffix: {"questdb"}}, Env: env, Mounts: testcontainers.Mounts(testcontainers.ContainerMount{ Source: testcontainers.GenericBindMountSource{ @@ -149,7 +154,7 @@ func setupQuestDB0(ctx context.Context, auth ilpAuthType, setupProxy bool) (*que newNetwork, err := testcontainers.GenericNetwork(ctx, testcontainers.GenericNetworkRequest{ NetworkRequest: testcontainers.NetworkRequest{ - Name: networkName, + Name: networkNameWithSuffix, CheckDuplicate: true, }, }) @@ -196,7 +201,7 @@ func setupQuestDB0(ctx context.Context, auth ilpAuthType, setupProxy bool) (*que Image: "haproxy:2.6.4", ExposedPorts: []string{"8443/tcp", "8444/tcp", "8445/tcp", "8888/tcp"}, WaitingFor: wait.ForHTTP("/").WithPort("8888"), - Networks: []string{networkName}, + Networks: []string{networkNameWithSuffix}, Mounts: testcontainers.Mounts(testcontainers.ContainerMount{ Source: testcontainers.GenericBindMountSource{ HostPath: path, From df4e34fd97c6a89c1612801b42493c9ca0a21ac2 Mon Sep 17 00:00:00 2001 From: victor Date: Mon, 25 Aug 2025 19:32:10 +0800 Subject: [PATCH 11/24] debug ci --- integration_test.go | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/integration_test.go b/integration_test.go index e4264314..a81d745d 100644 --- a/integration_test.go +++ b/integration_test.go @@ -100,8 +100,6 @@ const ( bearerToken = "testToken1" ) -var networkNameSuffix = 1 - func setupQuestDB(ctx context.Context, auth ilpAuthType) (*questdbContainer, error) { return setupQuestDB0(ctx, auth, false) } @@ -134,15 +132,12 @@ func setupQuestDB0(ctx context.Context, auth ilpAuthType, setupProxy bool) (*que if err != nil { return nil, err } - - networkNameWithSuffix := fmt.Sprintf("%s_%d", networkName, networkNameSuffix) - networkNameSuffix++ req := testcontainers.ContainerRequest{ Image: "questdb/questdb:9.0.2", ExposedPorts: []string{"9000/tcp", "9009/tcp"}, WaitingFor: wait.ForHTTP("/").WithPort("9000"), - Networks: []string{networkNameWithSuffix}, - NetworkAliases: map[string][]string{networkNameWithSuffix: {"questdb"}}, + Networks: []string{networkName}, + NetworkAliases: map[string][]string{networkName: {"questdb"}}, Env: env, Mounts: testcontainers.Mounts(testcontainers.ContainerMount{ Source: testcontainers.GenericBindMountSource{ @@ -154,7 +149,7 @@ func setupQuestDB0(ctx context.Context, auth ilpAuthType, setupProxy bool) (*que newNetwork, err := testcontainers.GenericNetwork(ctx, testcontainers.GenericNetworkRequest{ NetworkRequest: testcontainers.NetworkRequest{ - Name: networkNameWithSuffix, + Name: networkName, CheckDuplicate: true, }, }) @@ -201,7 +196,7 @@ func setupQuestDB0(ctx context.Context, auth ilpAuthType, setupProxy bool) (*que Image: "haproxy:2.6.4", ExposedPorts: []string{"8443/tcp", "8444/tcp", "8445/tcp", "8888/tcp"}, WaitingFor: wait.ForHTTP("/").WithPort("8888"), - Networks: []string{networkNameWithSuffix}, + Networks: []string{networkName}, Mounts: testcontainers.Mounts(testcontainers.ContainerMount{ Source: testcontainers.GenericBindMountSource{ HostPath: path, From e2621b81b74959a2f2da5a13295dd040413001ed Mon Sep 17 00:00:00 2001 From: victor Date: Mon, 25 Aug 2025 23:13:58 +0800 Subject: [PATCH 12/24] debug ci --- tcp_integration_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tcp_integration_test.go b/tcp_integration_test.go index 9ac2ceff..1ff8f062 100644 --- a/tcp_integration_test.go +++ b/tcp_integration_test.go @@ -54,6 +54,9 @@ func (suite *integrationTestSuite) TestE2EWriteInBatches() { questdbC, err := setupQuestDB(ctx, noAuth) assert.NoError(suite.T(), err) + if questdbC == nil { + assert.Fail(suite.T(), "fail to start QuestDB") + } defer questdbC.Stop(ctx) sender, err := qdb.NewLineSender(ctx, qdb.WithTcp(), qdb.WithAddress(questdbC.ilpAddress)) From 061b169ba944f142ed1f7c31eac199138a537393 Mon Sep 17 00:00:00 2001 From: victor Date: Mon, 25 Aug 2025 23:18:49 +0800 Subject: [PATCH 13/24] debug ci --- tcp_integration_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/tcp_integration_test.go b/tcp_integration_test.go index 1ff8f062..21dca97f 100644 --- a/tcp_integration_test.go +++ b/tcp_integration_test.go @@ -56,6 +56,7 @@ func (suite *integrationTestSuite) TestE2EWriteInBatches() { assert.NoError(suite.T(), err) if questdbC == nil { assert.Fail(suite.T(), "fail to start QuestDB") + return } defer questdbC.Stop(ctx) From 0aed8f0d88d93d425c01cb2bf92a7c6481f4f2b2 Mon Sep 17 00:00:00 2001 From: victor Date: Mon, 25 Aug 2025 23:24:40 +0800 Subject: [PATCH 14/24] debug ci --- integration_test.go | 17 +++++++++++------ tcp_integration_test.go | 4 ---- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/integration_test.go b/integration_test.go index a81d745d..d462b8fa 100644 --- a/integration_test.go +++ b/integration_test.go @@ -74,14 +74,19 @@ func (c *questdbContainer) Stop(ctx context.Context) error { return err } } - err := c.Terminate(ctx) - if err != nil { - return err + if c.Container != nil { + err := c.Terminate(ctx) + if err != nil { + return err + } } - err = c.network.Remove(ctx) - if err != nil { - return err + if c.network != nil { + err := c.network.Remove(ctx) + if err != nil { + return err + } } + return nil } diff --git a/tcp_integration_test.go b/tcp_integration_test.go index 21dca97f..9ac2ceff 100644 --- a/tcp_integration_test.go +++ b/tcp_integration_test.go @@ -54,10 +54,6 @@ func (suite *integrationTestSuite) TestE2EWriteInBatches() { questdbC, err := setupQuestDB(ctx, noAuth) assert.NoError(suite.T(), err) - if questdbC == nil { - assert.Fail(suite.T(), "fail to start QuestDB") - return - } defer questdbC.Stop(ctx) sender, err := qdb.NewLineSender(ctx, qdb.WithTcp(), qdb.WithAddress(questdbC.ilpAddress)) From da58a9cfed2a440cc7b940d4c4b9ba341d739a74 Mon Sep 17 00:00:00 2001 From: victor Date: Mon, 25 Aug 2025 23:32:01 +0800 Subject: [PATCH 15/24] debug ci --- buffer_test.go | 2 +- integration_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/buffer_test.go b/buffer_test.go index 02413f9c..5e7dcfe9 100644 --- a/buffer_test.go +++ b/buffer_test.go @@ -635,7 +635,7 @@ func TestFloat64Array2DColumnExceedsMaxElements(t *testing.T) { } err := buf.Table(testTable).Float64Array2DColumn("array_col", values).At(time.Time{}, false) - assert.ErrorContains(t, err, "array size 268500992 exceeds maximum limit 268435455") + assert.ErrorContains(t, err, "array size 268435456 exceeds maximum limit 268435455") assert.Empty(t, buf.Messages()) } diff --git a/integration_test.go b/integration_test.go index d462b8fa..a69e28b1 100644 --- a/integration_test.go +++ b/integration_test.go @@ -155,7 +155,7 @@ func setupQuestDB0(ctx context.Context, auth ilpAuthType, setupProxy bool) (*que newNetwork, err := testcontainers.GenericNetwork(ctx, testcontainers.GenericNetworkRequest{ NetworkRequest: testcontainers.NetworkRequest{ Name: networkName, - CheckDuplicate: true, + CheckDuplicate: false, }, }) if err != nil { From 8988e0909a88bc901881618899cd2ee9f62e97a9 Mon Sep 17 00:00:00 2001 From: victor Date: Mon, 25 Aug 2025 23:37:13 +0800 Subject: [PATCH 16/24] debug ci --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index eb411c1b..50e3baff 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.19 require ( github.com/stretchr/testify v1.9.0 - github.com/testcontainers/testcontainers-go v0.26.0 + github.com/testcontainers/testcontainers-go v0.38.0 ) require ( From d7b1b0f70f019b640a7bf84fe8ca3a90ed608775 Mon Sep 17 00:00:00 2001 From: victor Date: Mon, 25 Aug 2025 23:37:33 +0800 Subject: [PATCH 17/24] debug ci --- go.mod | 26 ++++++++++++++------------ go.sum | 12 ++++++++++++ 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 50e3baff..5c841ff9 100644 --- a/go.mod +++ b/go.mod @@ -1,40 +1,42 @@ module github.com/questdb/go-questdb-client/v3 -go 1.19 +go 1.23.0 + +toolchain go1.24.3 require ( - github.com/stretchr/testify v1.9.0 + github.com/stretchr/testify v1.10.0 github.com/testcontainers/testcontainers-go v0.38.0 ) require ( - dario.cat/mergo v1.0.0 // indirect + dario.cat/mergo v1.0.1 // indirect github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect - github.com/Microsoft/go-winio v0.6.1 // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect github.com/Microsoft/hcsshim v0.11.4 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/containerd/containerd v1.7.12 // indirect github.com/containerd/log v0.1.0 // indirect - github.com/cpuguy83/dockercfg v0.3.1 // indirect + github.com/cpuguy83/dockercfg v0.3.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/docker/distribution v2.8.2+incompatible // indirect - github.com/docker/docker v24.0.9+incompatible // indirect + github.com/docker/docker v28.2.2+incompatible // indirect github.com/docker/go-connections v0.5.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/uuid v1.6.0 // indirect - github.com/klauspost/compress v1.17.0 // indirect + github.com/klauspost/compress v1.18.0 // indirect github.com/kr/text v0.2.0 // indirect github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a // indirect - github.com/magiconair/properties v1.8.7 // indirect + github.com/magiconair/properties v1.8.10 // indirect github.com/moby/patternmatcher v0.6.0 // indirect - github.com/moby/sys/sequential v0.5.0 // indirect + github.com/moby/sys/sequential v0.6.0 // indirect github.com/moby/term v0.5.0 // indirect github.com/morikuni/aec v1.0.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.1.0-rc5 // indirect + github.com/opencontainers/image-spec v1.1.1 // indirect github.com/opencontainers/runc v1.1.5 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect @@ -44,10 +46,10 @@ require ( github.com/sirupsen/logrus v1.9.3 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect - github.com/yusufpapurcu/wmi v1.2.3 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect golang.org/x/exp v0.0.0-20231005195138-3e424a577f31 // indirect golang.org/x/mod v0.13.0 // indirect - golang.org/x/sys v0.16.0 // indirect + golang.org/x/sys v0.32.0 // indirect golang.org/x/tools v0.14.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97 // indirect google.golang.org/grpc v1.58.3 // indirect diff --git a/go.sum b/go.sum index 0e359b42..67dc8f79 100644 --- a/go.sum +++ b/go.sum @@ -1,11 +1,13 @@ dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/Microsoft/hcsshim v0.11.4 h1:68vKo2VN8DE9AdN4tnkWnmdhqdbpUFM8OF3Airm7fz8= github.com/Microsoft/hcsshim v0.11.4/go.mod h1:smjE4dvqPX9Zldna+t5FG3rnoHhaB7QYxPRqGcpAD9w= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= @@ -20,6 +22,7 @@ github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3 github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/dockercfg v0.3.1 h1:/FpZ+JaygUR/lZP2NlFI2DVfrOEMAIKP5wWEJdoYe9E= github.com/cpuguy83/dockercfg v0.3.1/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= +github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= @@ -31,6 +34,7 @@ github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m3 github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v24.0.9+incompatible h1:HPGzNmwfLZWdxHqK9/II92pyi1EpYKsAqcl4G0Of9v0= github.com/docker/docker v24.0.9+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v28.2.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= @@ -59,6 +63,7 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM= github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -70,11 +75,13 @@ github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a h1:N9zuLhTvBSRt0g github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a/go.mod h1:JKx41uQRwqlTZabZc+kILPrO/3jlKnQ2Z8b7YiVw5cE= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc= github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo= +github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= @@ -84,6 +91,7 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0-rc5 h1:Ygwkfw9bpDvs+c9E34SdgGOj41dX/cbdlwvlWt0pnFI= github.com/opencontainers/image-spec v1.1.0-rc5/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= +github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= github.com/opencontainers/runc v1.1.5 h1:L44KXEpKmfWDcS02aeGm8QNTFXTo2D+8MYGDIJ/GDEs= github.com/opencontainers/runc v1.1.5/go.mod h1:1J5XiS+vdZ3wCyZybsuxXZWGrgSr8fFJHLXuG2PsnNg= github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= @@ -118,9 +126,11 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/testcontainers/testcontainers-go v0.26.0 h1:uqcYdoOHBy1ca7gKODfBd9uTHVK3a7UL848z09MVZ0c= github.com/testcontainers/testcontainers-go v0.26.0/go.mod h1:ICriE9bLX5CLxL9OFQ2N+2N+f+803LNJ1utJb1+Inx0= +github.com/testcontainers/testcontainers-go v0.38.0/go.mod h1:C52c9MoHpWO+C4aqmgSU+hxlR5jlEayWtgYrb8Pzz1w= github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= @@ -132,6 +142,7 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -172,6 +183,7 @@ golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= From 238b12205c987be07d3dfb5093473adc526e0bd1 Mon Sep 17 00:00:00 2001 From: victor Date: Tue, 26 Aug 2025 14:25:57 +0800 Subject: [PATCH 18/24] debug ci --- go.mod | 28 ++++---- http_integration_test.go | 50 +++++++++++--- integration_test.go | 4 +- tcp_integration_test.go | 145 +++++++++++++++++++++++++++++++-------- 4 files changed, 171 insertions(+), 56 deletions(-) diff --git a/go.mod b/go.mod index 5c841ff9..eb411c1b 100644 --- a/go.mod +++ b/go.mod @@ -1,42 +1,40 @@ module github.com/questdb/go-questdb-client/v3 -go 1.23.0 - -toolchain go1.24.3 +go 1.19 require ( - github.com/stretchr/testify v1.10.0 - github.com/testcontainers/testcontainers-go v0.38.0 + github.com/stretchr/testify v1.9.0 + github.com/testcontainers/testcontainers-go v0.26.0 ) require ( - dario.cat/mergo v1.0.1 // indirect + dario.cat/mergo v1.0.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect - github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/Microsoft/go-winio v0.6.1 // indirect github.com/Microsoft/hcsshim v0.11.4 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/containerd/containerd v1.7.12 // indirect github.com/containerd/log v0.1.0 // indirect - github.com/cpuguy83/dockercfg v0.3.2 // indirect + github.com/cpuguy83/dockercfg v0.3.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/docker/distribution v2.8.2+incompatible // indirect - github.com/docker/docker v28.2.2+incompatible // indirect + github.com/docker/docker v24.0.9+incompatible // indirect github.com/docker/go-connections v0.5.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/uuid v1.6.0 // indirect - github.com/klauspost/compress v1.18.0 // indirect + github.com/klauspost/compress v1.17.0 // indirect github.com/kr/text v0.2.0 // indirect github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a // indirect - github.com/magiconair/properties v1.8.10 // indirect + github.com/magiconair/properties v1.8.7 // indirect github.com/moby/patternmatcher v0.6.0 // indirect - github.com/moby/sys/sequential v0.6.0 // indirect + github.com/moby/sys/sequential v0.5.0 // indirect github.com/moby/term v0.5.0 // indirect github.com/morikuni/aec v1.0.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.1.1 // indirect + github.com/opencontainers/image-spec v1.1.0-rc5 // indirect github.com/opencontainers/runc v1.1.5 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect @@ -46,10 +44,10 @@ require ( github.com/sirupsen/logrus v1.9.3 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect - github.com/yusufpapurcu/wmi v1.2.4 // indirect + github.com/yusufpapurcu/wmi v1.2.3 // indirect golang.org/x/exp v0.0.0-20231005195138-3e424a577f31 // indirect golang.org/x/mod v0.13.0 // indirect - golang.org/x/sys v0.32.0 // indirect + golang.org/x/sys v0.16.0 // indirect golang.org/x/tools v0.14.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97 // indirect google.golang.org/grpc v1.58.3 // indirect diff --git a/http_integration_test.go b/http_integration_test.go index 246e6cb6..87e9a010 100644 --- a/http_integration_test.go +++ b/http_integration_test.go @@ -42,7 +42,10 @@ func (suite *integrationTestSuite) TestE2ESuccessfulHttpBasicAuthWithTlsProxy() ctx := context.Background() questdbC, err := setupQuestDB(ctx, httpBasicAuth) - assert.NoError(suite.T(), err) + if err != nil { + assert.Fail(suite.T(), err.Error()) + return + } defer questdbC.Stop(ctx) sender, err := qdb.NewLineSender( @@ -52,23 +55,35 @@ func (suite *integrationTestSuite) TestE2ESuccessfulHttpBasicAuthWithTlsProxy() qdb.WithBasicAuth(basicAuthUser, basicAuthPass), qdb.WithTlsInsecureSkipVerify(), ) - assert.NoError(suite.T(), err) + if err != nil { + assert.Fail(suite.T(), err.Error()) + return + } defer sender.Close(ctx) err = sender. Table(testTable). StringColumn("str_col", "foobar"). At(ctx, time.UnixMicro(1)) - assert.NoError(suite.T(), err) + if err != nil { + assert.Fail(suite.T(), err.Error()) + return + } err = sender. Table(testTable). StringColumn("str_col", "barbaz"). At(ctx, time.UnixMicro(2)) - assert.NoError(suite.T(), err) + if err != nil { + assert.Fail(suite.T(), err.Error()) + return + } err = sender.Flush(ctx) - assert.NoError(suite.T(), err) + if err != nil { + assert.Fail(suite.T(), err.Error()) + return + } expected := tableData{ Columns: []column{ @@ -101,19 +116,34 @@ func (suite *integrationTestSuite) TestServerSideError() { ) questdbC, err := setupQuestDB(ctx, noAuth) - assert.NoError(suite.T(), err) + if err != nil { + assert.Fail(suite.T(), err.Error()) + return + } sender, err = qdb.NewLineSender(ctx, qdb.WithHttp(), qdb.WithAddress(questdbC.httpAddress)) - assert.NoError(suite.T(), err) + if err != nil { + assert.Fail(suite.T(), err.Error()) + return + } err = sender.Table(testTable).Int64Column("long_col", 42).AtNow(ctx) - assert.NoError(suite.T(), err) + if err != nil { + assert.Fail(suite.T(), err.Error()) + return + } err = sender.Flush(ctx) - assert.NoError(suite.T(), err) + if err != nil { + assert.Fail(suite.T(), err.Error()) + return + } // Now, use wrong type for the long_col. err = sender.Table(testTable).StringColumn("long_col", "42").AtNow(ctx) - assert.NoError(suite.T(), err) + if err != nil { + assert.Fail(suite.T(), err.Error()) + return + } err = sender.Flush(ctx) assert.Error(suite.T(), err) assert.ErrorContains(suite.T(), err, "my_test_table, column: long_col; cast error from protocol type: STRING to column type") diff --git a/integration_test.go b/integration_test.go index a69e28b1..c0a1586c 100644 --- a/integration_test.go +++ b/integration_test.go @@ -140,7 +140,7 @@ func setupQuestDB0(ctx context.Context, auth ilpAuthType, setupProxy bool) (*que req := testcontainers.ContainerRequest{ Image: "questdb/questdb:9.0.2", ExposedPorts: []string{"9000/tcp", "9009/tcp"}, - WaitingFor: wait.ForHTTP("/").WithPort("9000"), + WaitingFor: wait.ForHTTP("/ping").WithPort("9000"), Networks: []string{networkName}, NetworkAliases: map[string][]string{networkName: {"questdb"}}, Env: env, @@ -155,7 +155,7 @@ func setupQuestDB0(ctx context.Context, auth ilpAuthType, setupProxy bool) (*que newNetwork, err := testcontainers.GenericNetwork(ctx, testcontainers.GenericNetworkRequest{ NetworkRequest: testcontainers.NetworkRequest{ Name: networkName, - CheckDuplicate: false, + CheckDuplicate: true, }, }) if err != nil { diff --git a/tcp_integration_test.go b/tcp_integration_test.go index 9ac2ceff..fc8b5faa 100644 --- a/tcp_integration_test.go +++ b/tcp_integration_test.go @@ -53,11 +53,17 @@ func (suite *integrationTestSuite) TestE2EWriteInBatches() { ctx := context.Background() questdbC, err := setupQuestDB(ctx, noAuth) - assert.NoError(suite.T(), err) + if err != nil { + assert.Fail(suite.T(), err.Error()) + return + } defer questdbC.Stop(ctx) sender, err := qdb.NewLineSender(ctx, qdb.WithTcp(), qdb.WithAddress(questdbC.ilpAddress)) - assert.NoError(suite.T(), err) + if err != nil { + assert.Fail(suite.T(), err.Error()) + return + } defer sender.Close(ctx) dropTable(suite.T(), testTable, questdbC.httpAddress) @@ -67,10 +73,16 @@ func (suite *integrationTestSuite) TestE2EWriteInBatches() { Table(testTable). Int64Column("long_col", int64(j)). At(ctx, time.UnixMicro(int64(i*nBatch+j))) - assert.NoError(suite.T(), err) + if err != nil { + assert.Fail(suite.T(), err.Error()) + return + } } err = sender.Flush(ctx) - assert.NoError(suite.T(), err) + if err != nil { + assert.Fail(suite.T(), err.Error()) + return + } } expected := tableData{ @@ -107,11 +119,17 @@ func (suite *integrationTestSuite) TestE2EImplicitFlush() { ctx := context.Background() questdbC, err := setupQuestDB(ctx, noAuth) - assert.NoError(suite.T(), err) + if err != nil { + assert.Fail(suite.T(), err.Error()) + return + } defer questdbC.Stop(ctx) sender, err := qdb.NewLineSender(ctx, qdb.WithTcp(), qdb.WithAddress(questdbC.ilpAddress), qdb.WithInitBufferSize(bufCap)) - assert.NoError(suite.T(), err) + if err != nil { + assert.Fail(suite.T(), err.Error()) + return + } defer sender.Close(ctx) dropTable(suite.T(), testTable, questdbC.httpAddress) @@ -120,7 +138,10 @@ func (suite *integrationTestSuite) TestE2EImplicitFlush() { Table(testTable). BoolColumn("b", true). AtNow(ctx) - assert.NoError(suite.T(), err) + if err != nil { + assert.Fail(suite.T(), err.Error()) + return + } } assert.Eventually(suite.T(), func() bool { @@ -138,7 +159,10 @@ func (suite *integrationTestSuite) TestE2ESuccessfulAuth() { ctx := context.Background() questdbC, err := setupQuestDB(ctx, authEnabled) - assert.NoError(suite.T(), err) + if err != nil { + assert.Fail(suite.T(), err.Error()) + return + } defer questdbC.Stop(ctx) sender, err := qdb.NewLineSender( @@ -147,23 +171,35 @@ func (suite *integrationTestSuite) TestE2ESuccessfulAuth() { qdb.WithAddress(questdbC.ilpAddress), qdb.WithAuth("testUser1", "5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48"), ) - assert.NoError(suite.T(), err) + if err != nil { + assert.Fail(suite.T(), err.Error()) + return + } dropTable(suite.T(), testTable, questdbC.httpAddress) err = sender. Table(testTable). StringColumn("str_col", "foobar"). At(ctx, time.UnixMicro(1)) - assert.NoError(suite.T(), err) + if err != nil { + assert.Fail(suite.T(), err.Error()) + return + } err = sender. Table(testTable). StringColumn("str_col", "barbaz"). At(ctx, time.UnixMicro(2)) - assert.NoError(suite.T(), err) + if err != nil { + assert.Fail(suite.T(), err.Error()) + return + } err = sender.Flush(ctx) - assert.NoError(suite.T(), err) + if err != nil { + assert.Fail(suite.T(), err.Error()) + return + } // Close the connection to make sure that ILP messages are written. That's because // the server may not write messages that are received immediately after the signed @@ -196,7 +232,10 @@ func (suite *integrationTestSuite) TestE2EFailedAuth() { ctx := context.Background() questdbC, err := setupQuestDB(ctx, authEnabled) - assert.NoError(suite.T(), err) + if err != nil { + assert.Fail(suite.T(), err.Error()) + return + } defer questdbC.Stop(ctx) sender, err := qdb.NewLineSender( @@ -205,7 +244,10 @@ func (suite *integrationTestSuite) TestE2EFailedAuth() { qdb.WithAddress(questdbC.ilpAddress), qdb.WithAuth("wrongKeyId", "5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48"), ) - assert.NoError(suite.T(), err) + if err != nil { + assert.Fail(suite.T(), err.Error()) + return + } defer sender.Close(ctx) err = sender. @@ -245,7 +287,10 @@ func (suite *integrationTestSuite) TestE2EWritesWithTlsProxy() { ctx := context.Background() questdbC, err := setupQuestDBWithProxy(ctx, noAuth) - assert.NoError(suite.T(), err) + if err != nil { + assert.Fail(suite.T(), err.Error()) + return + } defer questdbC.Stop(ctx) sender, err := qdb.NewLineSender( @@ -254,7 +299,10 @@ func (suite *integrationTestSuite) TestE2EWritesWithTlsProxy() { qdb.WithAddress(questdbC.proxyIlpTcpAddress), // We're sending data through proxy. qdb.WithTlsInsecureSkipVerify(), ) - assert.NoError(suite.T(), err) + if err != nil { + assert.Fail(suite.T(), err.Error()) + return + } defer sender.Close(ctx) dropTable(suite.T(), testTable, questdbC.httpAddress) @@ -262,16 +310,25 @@ func (suite *integrationTestSuite) TestE2EWritesWithTlsProxy() { Table(testTable). StringColumn("str_col", "foobar"). At(ctx, time.UnixMicro(1)) - assert.NoError(suite.T(), err) + if err != nil { + assert.Fail(suite.T(), err.Error()) + return + } err = sender. Table(testTable). StringColumn("str_col", "barbaz"). At(ctx, time.UnixMicro(2)) - assert.NoError(suite.T(), err) + if err != nil { + assert.Fail(suite.T(), err.Error()) + return + } err = sender.Flush(ctx) - assert.NoError(suite.T(), err) + if err != nil { + assert.Fail(suite.T(), err.Error()) + return + } expected := tableData{ Columns: []column{ @@ -299,7 +356,10 @@ func (suite *integrationTestSuite) TestE2ESuccessfulAuthWithTlsProxy() { ctx := context.Background() questdbC, err := setupQuestDBWithProxy(ctx, authEnabled) - assert.NoError(suite.T(), err) + if err != nil { + assert.Fail(suite.T(), err.Error()) + return + } defer questdbC.Stop(ctx) sender, err := qdb.NewLineSender( @@ -309,23 +369,35 @@ func (suite *integrationTestSuite) TestE2ESuccessfulAuthWithTlsProxy() { qdb.WithAuth("testUser1", "5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48"), qdb.WithTlsInsecureSkipVerify(), ) - assert.NoError(suite.T(), err) + if err != nil { + assert.Fail(suite.T(), err.Error()) + return + } dropTable(suite.T(), testTable, questdbC.httpAddress) err = sender. Table(testTable). StringColumn("str_col", "foobar"). At(ctx, time.UnixMicro(1)) - assert.NoError(suite.T(), err) + if err != nil { + assert.Fail(suite.T(), err.Error()) + return + } err = sender. Table(testTable). StringColumn("str_col", "barbaz"). At(ctx, time.UnixMicro(2)) - assert.NoError(suite.T(), err) + if err != nil { + assert.Fail(suite.T(), err.Error()) + return + } err = sender.Flush(ctx) - assert.NoError(suite.T(), err) + if err != nil { + assert.Fail(suite.T(), err.Error()) + return + } // Close the connection to make sure that ILP messages are written. That's because // the server may not write messages that are received immediately after the signed @@ -357,11 +429,17 @@ func (suite *integrationTestSuite) TestDoubleArrayColumn() { ctx := context.Background() questdbC, err := setupQuestDB(ctx, noAuth) - assert.NoError(suite.T(), err) + if err != nil { + assert.Fail(suite.T(), err.Error()) + return + } defer questdbC.Stop(ctx) sender, err := qdb.NewLineSender(ctx, qdb.WithTcp(), qdb.WithAddress(questdbC.ilpAddress), qdb.WithProtocolVersion(qdb.ProtocolVersion2)) - assert.NoError(suite.T(), err) + if err != nil { + assert.Fail(suite.T(), err.Error()) + return + } defer sender.Close(ctx) dropTable(suite.T(), testTable, questdbC.httpAddress) @@ -369,7 +447,10 @@ func (suite *integrationTestSuite) TestDoubleArrayColumn() { values2D := [][]float64{{1.0, 2.0}, {3.0, 4.0}, {5.0, 6.0}} values3D := [][][]float64{{{1.0, 2.0}, {3.0, 4.0}}, {{5.0, 6.0}, {7.0, 8.0}}} arrayND, err := qdb.NewNDArray[float64](2, 2, 1, 2) - assert.NoError(suite.T(), err) + if err != nil { + assert.Fail(suite.T(), err.Error()) + return + } arrayND.Fill(11.0) err = sender. @@ -379,10 +460,16 @@ func (suite *integrationTestSuite) TestDoubleArrayColumn() { Float64Array3DColumn("array_3d", values3D). Float64ArrayNDColumn("array_nd", arrayND). At(ctx, time.UnixMicro(1)) - assert.NoError(suite.T(), err) + if err != nil { + assert.Fail(suite.T(), err.Error()) + return + } err = sender.Flush(ctx) - assert.NoError(suite.T(), err) + if err != nil { + assert.Fail(suite.T(), err.Error()) + return + } // Expected results expected := tableData{ From 1c185fff2e556e85c59f041e43d5d57705c35f91 Mon Sep 17 00:00:00 2001 From: victor Date: Tue, 26 Aug 2025 15:09:50 +0800 Subject: [PATCH 19/24] fix ci --- integration_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_test.go b/integration_test.go index c0a1586c..16253ae2 100644 --- a/integration_test.go +++ b/integration_test.go @@ -140,7 +140,7 @@ func setupQuestDB0(ctx context.Context, auth ilpAuthType, setupProxy bool) (*que req := testcontainers.ContainerRequest{ Image: "questdb/questdb:9.0.2", ExposedPorts: []string{"9000/tcp", "9009/tcp"}, - WaitingFor: wait.ForHTTP("/ping").WithPort("9000"), + WaitingFor: wait.ForHTTP("/settings").WithPort("9000"), Networks: []string{networkName}, NetworkAliases: map[string][]string{networkName: {"questdb"}}, Env: env, From ad7d6f2a41ff1630e40c5a280f08c4e71f57cae2 Mon Sep 17 00:00:00 2001 From: victor Date: Tue, 26 Aug 2025 15:18:18 +0800 Subject: [PATCH 20/24] fix ci --- integration_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_test.go b/integration_test.go index 16253ae2..d5e60a5a 100644 --- a/integration_test.go +++ b/integration_test.go @@ -155,7 +155,7 @@ func setupQuestDB0(ctx context.Context, auth ilpAuthType, setupProxy bool) (*que newNetwork, err := testcontainers.GenericNetwork(ctx, testcontainers.GenericNetworkRequest{ NetworkRequest: testcontainers.NetworkRequest{ Name: networkName, - CheckDuplicate: true, + CheckDuplicate: false, }, }) if err != nil { From f62ad8053c31c2cf6a5623bcb82c5cd07925b2d5 Mon Sep 17 00:00:00 2001 From: victor Date: Tue, 26 Aug 2025 15:24:20 +0800 Subject: [PATCH 21/24] uniq networkName --- integration_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/integration_test.go b/integration_test.go index d5e60a5a..0ebb1713 100644 --- a/integration_test.go +++ b/integration_test.go @@ -137,12 +137,13 @@ func setupQuestDB0(ctx context.Context, auth ilpAuthType, setupProxy bool) (*que if err != nil { return nil, err } + uniqueNetworkName := fmt.Sprintf("%s-%d", networkName, time.Now().UnixNano()) req := testcontainers.ContainerRequest{ Image: "questdb/questdb:9.0.2", ExposedPorts: []string{"9000/tcp", "9009/tcp"}, WaitingFor: wait.ForHTTP("/settings").WithPort("9000"), - Networks: []string{networkName}, - NetworkAliases: map[string][]string{networkName: {"questdb"}}, + Networks: []string{uniqueNetworkName}, + NetworkAliases: map[string][]string{uniqueNetworkName: {"questdb"}}, Env: env, Mounts: testcontainers.Mounts(testcontainers.ContainerMount{ Source: testcontainers.GenericBindMountSource{ @@ -151,11 +152,10 @@ func setupQuestDB0(ctx context.Context, auth ilpAuthType, setupProxy bool) (*que Target: testcontainers.ContainerMountTarget("/root/.questdb/auth"), }), } - newNetwork, err := testcontainers.GenericNetwork(ctx, testcontainers.GenericNetworkRequest{ NetworkRequest: testcontainers.NetworkRequest{ - Name: networkName, - CheckDuplicate: false, + Name: uniqueNetworkName, + CheckDuplicate: true, }, }) if err != nil { @@ -201,7 +201,7 @@ func setupQuestDB0(ctx context.Context, auth ilpAuthType, setupProxy bool) (*que Image: "haproxy:2.6.4", ExposedPorts: []string{"8443/tcp", "8444/tcp", "8445/tcp", "8888/tcp"}, WaitingFor: wait.ForHTTP("/").WithPort("8888"), - Networks: []string{networkName}, + Networks: []string{uniqueNetworkName}, Mounts: testcontainers.Mounts(testcontainers.ContainerMount{ Source: testcontainers.GenericBindMountSource{ HostPath: path, From 9fde7326453427270303d1a9a15d7c4933e62d98 Mon Sep 17 00:00:00 2001 From: victor Date: Tue, 26 Aug 2025 16:17:42 +0800 Subject: [PATCH 22/24] tests --- http_integration_test.go | 2 +- http_sender.go | 5 +++++ integration_test.go | 2 +- sender_pool_test.go | 2 +- 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/http_integration_test.go b/http_integration_test.go index 87e9a010..f1815f81 100644 --- a/http_integration_test.go +++ b/http_integration_test.go @@ -87,7 +87,7 @@ func (suite *integrationTestSuite) TestE2ESuccessfulHttpBasicAuthWithTlsProxy() expected := tableData{ Columns: []column{ - {"str_col", "STRING"}, + {"str_col", "VARCHAR"}, {"timestamp", "TIMESTAMP"}, }, Dataset: [][]interface{}{ diff --git a/http_sender.go b/http_sender.go index 69f22fff..7d070fc4 100644 --- a/http_sender.go +++ b/http_sender.go @@ -450,6 +450,11 @@ func (s *httpLineSender) detectProtocolVersion(ctx context.Context, conf *lineSe if err != nil { return protocolVersionUnset, err } + if s.user != "" && s.pass != "" { + req.SetBasicAuth(s.user, s.pass) + } else if s.token != "" { + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", s.token)) + } reqCtx, cancel := context.WithTimeout(ctx, s.requestTimeout) defer cancel() diff --git a/integration_test.go b/integration_test.go index 0ebb1713..e5b6f421 100644 --- a/integration_test.go +++ b/integration_test.go @@ -200,7 +200,7 @@ func setupQuestDB0(ctx context.Context, auth ilpAuthType, setupProxy bool) (*que req = testcontainers.ContainerRequest{ Image: "haproxy:2.6.4", ExposedPorts: []string{"8443/tcp", "8444/tcp", "8445/tcp", "8888/tcp"}, - WaitingFor: wait.ForHTTP("/").WithPort("8888"), + WaitingFor: wait.ForHTTP("/settings").WithPort("8888"), Networks: []string{uniqueNetworkName}, Mounts: testcontainers.Mounts(testcontainers.ContainerMount{ Source: testcontainers.GenericBindMountSource{ diff --git a/sender_pool_test.go b/sender_pool_test.go index 9cea0b29..978d993f 100644 --- a/sender_pool_test.go +++ b/sender_pool_test.go @@ -37,7 +37,7 @@ import ( ) func TestBasicBehavior(t *testing.T) { - p, err := qdb.PoolFromConf("http::addr=localhost:1234") + p, err := qdb.PoolFromConf("http::addr=localhost:1234;protocol_version=2;") require.NoError(t, err) ctx := context.Background() From 46fd0371b3d8d902bc58cb7cb45ede9e7ff47b94 Mon Sep 17 00:00:00 2001 From: victor Date: Tue, 26 Aug 2025 16:37:25 +0800 Subject: [PATCH 23/24] fix tests --- export_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/export_test.go b/export_test.go index 640c4735..bcd02a66 100644 --- a/export_test.go +++ b/export_test.go @@ -74,7 +74,7 @@ func Messages(s LineSender) []byte { func MsgCount(s LineSender) int { if ps, ok := s.(*pooledSender); ok { - s = ps + s = ps.wrapped } if hs, ok := s.(*httpLineSender); ok { return hs.MsgCount() @@ -93,7 +93,7 @@ func MsgCount(s LineSender) int { func BufLen(s LineSender) int { if ps, ok := s.(*pooledSender); ok { - s = ps + s = ps.wrapped } if hs, ok := s.(*httpLineSender); ok { return hs.BufLen() @@ -112,7 +112,7 @@ func BufLen(s LineSender) int { func ProtocolVersion(s LineSender) protocolVersion { if ps, ok := s.(*pooledSender); ok { - s = ps + s = ps.wrapped } if _, ok := s.(*httpLineSender); ok { return ProtocolVersion1 From bdcd67606becfa92898a953500982b7a5f3f5960 Mon Sep 17 00:00:00 2001 From: victor Date: Tue, 26 Aug 2025 16:49:06 +0800 Subject: [PATCH 24/24] fix tests. --- sender_pool_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sender_pool_test.go b/sender_pool_test.go index 978d993f..a98e4892 100644 --- a/sender_pool_test.go +++ b/sender_pool_test.go @@ -104,7 +104,7 @@ func TestFlushOnClose(t *testing.T) { } func TestPooledSenderDoubleClose(t *testing.T) { - p, err := qdb.PoolFromConf("http::addr=localhost:1234") + p, err := qdb.PoolFromConf("http::addr=localhost:1234;protocol_version=1;") require.NoError(t, err) ctx := context.Background() @@ -122,7 +122,7 @@ func TestPooledSenderDoubleClose(t *testing.T) { func TestMaxPoolSize(t *testing.T) { // Create a pool with 2 max senders - p, err := qdb.PoolFromConf("http::addr=localhost:1234", qdb.WithMaxSenders(3)) + p, err := qdb.PoolFromConf("http::addr=localhost:1234;protocol_version=1;", qdb.WithMaxSenders(3)) require.NoError(t, err) ctx := context.Background()