diff --git a/.github/workflows/chdb.yml b/.github/workflows/chdb.yml index 1c07297..2f09caf 100644 --- a/.github/workflows/chdb.yml +++ b/.github/workflows/chdb.yml @@ -32,7 +32,7 @@ jobs: run: ./chdb-go "SELECT 12345" build_mac: - runs-on: macos-12 + runs-on: macos-15 steps: - uses: actions/checkout@v3 - name: Fetch library diff --git a/README.md b/README.md index 3570958..f703248 100644 --- a/README.md +++ b/README.md @@ -67,9 +67,9 @@ func main() { fmt.Println(result) tmp_path := filepath.Join(os.TempDir(), "chdb_test") - defer os.RemoveAll(tmp_path) // Stateful Query (persistent) session, _ := chdb.NewSession(tmp_path) + // session cleanup will also delete the folder defer session.Cleanup() _, err = session.Query("CREATE DATABASE IF NOT EXISTS testdb; " + diff --git a/chdb-purego/binding.go b/chdb-purego/binding.go new file mode 100644 index 0000000..9c684a6 --- /dev/null +++ b/chdb-purego/binding.go @@ -0,0 +1,62 @@ +package chdbpurego + +import ( + "os" + "os/exec" + + "github.com/ebitengine/purego" +) + +func findLibrary() string { + // Env var + if envPath := os.Getenv("CHDB_LIB_PATH"); envPath != "" { + return envPath + } + + // ldconfig with Linux + if path, err := exec.LookPath("libchdb.so"); err == nil { + return path + } + + // default path + commonPaths := []string{ + "/usr/local/lib/libchdb.so", + "/opt/homebrew/lib/libchdb.dylib", + } + + for _, p := range commonPaths { + if _, err := os.Stat(p); err == nil { + return p + } + } + + //should be an error ? + return "libchdb.so" +} + +var ( + queryStable func(argc int, argv []string) *local_result + freeResult func(result *local_result) + queryStableV2 func(argc int, argv []string) *local_result_v2 + freeResultV2 func(result *local_result_v2) + connectChdb func(argc int, argv []string) **chdb_conn + closeConn func(conn **chdb_conn) + queryConn func(conn *chdb_conn, query string, format string) *local_result_v2 +) + +func init() { + path := findLibrary() + libchdb, err := purego.Dlopen(path, purego.RTLD_NOW|purego.RTLD_GLOBAL) + if err != nil { + panic(err) + } + purego.RegisterLibFunc(&queryStable, libchdb, "query_stable") + purego.RegisterLibFunc(&freeResult, libchdb, "free_result") + purego.RegisterLibFunc(&queryStableV2, libchdb, "query_stable_v2") + + purego.RegisterLibFunc(&freeResultV2, libchdb, "free_result_v2") + purego.RegisterLibFunc(&connectChdb, libchdb, "connect_chdb") + purego.RegisterLibFunc(&closeConn, libchdb, "close_conn") + purego.RegisterLibFunc(&queryConn, libchdb, "query_conn") + +} diff --git a/chdb-purego/chdb.go b/chdb-purego/chdb.go new file mode 100644 index 0000000..c4171ba --- /dev/null +++ b/chdb-purego/chdb.go @@ -0,0 +1,178 @@ +package chdbpurego + +import ( + "errors" + "fmt" + "unsafe" +) + +type result struct { + localResv2 *local_result_v2 +} + +func newChdbResult(cRes *local_result_v2) ChdbResult { + res := &result{ + localResv2: cRes, + } + // runtime.SetFinalizer(res, res.Free) + return res + +} + +// Buf implements ChdbResult. +func (c *result) Buf() []byte { + if c.localResv2 != nil { + if c.localResv2.buf != nil && c.localResv2.len > 0 { + return unsafe.Slice(c.localResv2.buf, c.localResv2.len) + } + } + return nil +} + +// BytesRead implements ChdbResult. +func (c *result) BytesRead() uint64 { + if c.localResv2 != nil { + return c.localResv2.bytes_read + } + return 0 +} + +// Elapsed implements ChdbResult. +func (c *result) Elapsed() float64 { + if c.localResv2 != nil { + return c.localResv2.elapsed + } + return 0 +} + +// Error implements ChdbResult. +func (c *result) Error() error { + if c.localResv2 != nil { + if c.localResv2.error_message != nil { + return errors.New(ptrToGoString(c.localResv2.error_message)) + } + } + return nil +} + +// Free implements ChdbResult. +func (c *result) Free() { + if c.localResv2 != nil { + freeResultV2(c.localResv2) + c.localResv2 = nil + } + +} + +// Len implements ChdbResult. +func (c *result) Len() int { + if c.localResv2 != nil { + return int(c.localResv2.len) + } + return 0 +} + +// RowsRead implements ChdbResult. +func (c *result) RowsRead() uint64 { + if c.localResv2 != nil { + return c.localResv2.rows_read + } + return 0 +} + +// String implements ChdbResult. +func (c *result) String() string { + ret := c.Buf() + if ret == nil { + return "" + } + return string(ret) +} + +type connection struct { + conn **chdb_conn +} + +func newChdbConn(conn **chdb_conn) ChdbConn { + c := &connection{ + conn: conn, + } + // runtime.SetFinalizer(c, c.Close) + return c +} + +// Close implements ChdbConn. +func (c *connection) Close() { + if c.conn != nil { + closeConn(c.conn) + } +} + +// Query implements ChdbConn. +func (c *connection) Query(queryStr string, formatStr string) (result ChdbResult, err error) { + + if c.conn == nil { + return nil, fmt.Errorf("invalid connection") + } + + rawConn := *c.conn + + res := queryConn(rawConn, queryStr, formatStr) + if res == nil { + // According to the C ABI of chDB v1.2.0, the C function query_stable_v2 + // returns nil if the query returns no data. This is not an error. We + // will change this behavior in the future. + return newChdbResult(res), nil + } + if res.error_message != nil { + return nil, errors.New(ptrToGoString(res.error_message)) + } + + return newChdbResult(res), nil +} + +func (c *connection) Ready() bool { + if c.conn != nil { + deref := *c.conn + if deref != nil { + return deref.connected + } + } + return false +} + +// Session will keep the state of query. +// If path is None, it will create a temporary directory and use it as the database path +// and the temporary directory will be removed when the session is closed. +// You can also pass in a path to create a database at that path where will keep your data. +// +// You can also use a connection string to pass in the path and other parameters. +// Examples: +// - ":memory:" (for in-memory database) +// - "test.db" (for relative path) +// - "file:test.db" (same as above) +// - "/path/to/test.db" (for absolute path) +// - "file:/path/to/test.db" (same as above) +// - "file:test.db?param1=value1¶m2=value2" (for relative path with query params) +// - "file::memory:?verbose&log-level=test" (for in-memory database with query params) +// - "///path/to/test.db?param1=value1¶m2=value2" (for absolute path) +// +// Connection string args handling: +// +// Connection string can contain query params like "file:test.db?param1=value1¶m2=value2" +// "param1=value1" will be passed to ClickHouse engine as start up args. +// +// For more details, see `clickhouse local --help --verbose` +// Some special args handling: +// - "mode=ro" would be "--readonly=1" for clickhouse (read-only mode) +// +// Important: +// - There can be only one session at a time. If you want to create a new session, you need to close the existing one. +// - Creating a new session will close the existing one. +func NewConnection(argc int, argv []string) (ChdbConn, error) { + conn := connectChdb(argc, argv) + if conn == nil { + return nil, fmt.Errorf("could not create a chdb connection") + } + return newChdbConn(conn), nil +} diff --git a/chdb-purego/helpers.go b/chdb-purego/helpers.go new file mode 100644 index 0000000..7c78274 --- /dev/null +++ b/chdb-purego/helpers.go @@ -0,0 +1,21 @@ +package chdbpurego + +import ( + "unsafe" +) + +func ptrToGoString(ptr *byte) string { + if ptr == nil { + return "" + } + + var length int + for { + if *(*byte)(unsafe.Pointer(uintptr(unsafe.Pointer(ptr)) + uintptr(length))) == 0 { + break + } + length++ + } + + return string(unsafe.Slice(ptr, length)) +} diff --git a/chdb-purego/types.go b/chdb-purego/types.go new file mode 100644 index 0000000..ea953cb --- /dev/null +++ b/chdb-purego/types.go @@ -0,0 +1,60 @@ +package chdbpurego + +import "unsafe" + +// old local result struct. for reference: +// https://github.com/chdb-io/chdb/blob/main/programs/local/chdb.h#L29 +type local_result struct { + buf *byte + len uintptr + _vec unsafe.Pointer + elapsed float64 + rows_read uint64 + bytes_read uint64 +} + +// new local result struct. for reference: https://github.com/chdb-io/chdb/blob/main/programs/local/chdb.h#L40 +type local_result_v2 struct { + buf *byte + len uintptr + _vec unsafe.Pointer + elapsed float64 + rows_read uint64 + bytes_read uint64 + error_message *byte +} + +// clickhouse background server connection.for reference: https://github.com/chdb-io/chdb/blob/main/programs/local/chdb.h#L82 +type chdb_conn struct { + server unsafe.Pointer + connected bool + queue unsafe.Pointer +} + +type ChdbResult interface { + // Raw bytes result buffer, used for reading the result of clickhouse query + Buf() []byte + // String rapresentation of the the buffer + String() string + // Lenght in bytes of the buffer + Len() int + // Number of seconds elapsed for the query execution + Elapsed() float64 + // Amount of rows returned by the query + RowsRead() uint64 + // Amount of bytes returned by the query + BytesRead() uint64 + // If the query had any error during execution, here you can retrieve the cause. + Error() error + // Free the query result and all the allocated memory + Free() +} + +type ChdbConn interface { + //Query executes the given queryStr in the underlying clickhouse connection, and output the result in the given formatStr + Query(queryStr string, formatStr string) (result ChdbResult, err error) + //Ready returns a boolean indicating if the connections is successfully established. + Ready() bool + //Close the connection and free the underlying allocated memory + Close() +} diff --git a/chdb.h b/chdb.h new file mode 100644 index 0000000..1188128 --- /dev/null +++ b/chdb.h @@ -0,0 +1,123 @@ +#pragma once + +#ifdef __cplusplus +# include +# include +# include +# include +# include +# include +extern "C" { +#else +# include +# include +# include +#endif + +#define CHDB_EXPORT __attribute__((visibility("default"))) +struct local_result +{ + char * buf; + size_t len; + void * _vec; // std::vector *, for freeing + double elapsed; + uint64_t rows_read; + uint64_t bytes_read; +}; + +#ifdef __cplusplus +struct local_result_v2 +{ + char * buf = nullptr; + size_t len = 0; + void * _vec = nullptr; // std::vector *, for freeing + double elapsed = 0.0; + uint64_t rows_read = 0; + uint64_t bytes_read = 0; + char * error_message = nullptr; +}; +#else +struct local_result_v2 +{ + char * buf; + size_t len; + void * _vec; // std::vector *, for freeing + double elapsed; + uint64_t rows_read; + uint64_t bytes_read; + char * error_message; +}; +#endif + +CHDB_EXPORT struct local_result * query_stable(int argc, char ** argv); +CHDB_EXPORT void free_result(struct local_result * result); + +CHDB_EXPORT struct local_result_v2 * query_stable_v2(int argc, char ** argv); +CHDB_EXPORT void free_result_v2(struct local_result_v2 * result); + +#ifdef __cplusplus +struct query_request +{ + std::string query; + std::string format; +}; + +struct query_queue +{ + std::mutex mutex; + std::condition_variable query_cv; // For query submission + std::condition_variable result_cv; // For query result retrieval + query_request current_query; + local_result_v2 * current_result = nullptr; + bool has_query = false; + bool shutdown = false; + bool cleanup_done = false; +}; +#endif + +/** + * Connection structure for chDB + * Contains server instance, connection state, and query processing queue + */ +struct chdb_conn +{ + void * server; /* ClickHouse LocalServer instance */ + bool connected; /* Connection state flag */ + void * queue; /* Query processing queue */ +}; + +/** + * Creates a new chDB connection. + * Only one active connection is allowed per process. + * Creating a new connection with different path requires closing existing connection. + * + * @param argc Number of command-line arguments + * @param argv Command-line arguments array (--path= to specify database location) + * @return Pointer to connection pointer, or NULL on failure + * @note Default path is ":memory:" if not specified + */ +CHDB_EXPORT struct chdb_conn ** connect_chdb(int argc, char ** argv); + +/** + * Closes an existing chDB connection and cleans up resources. + * Thread-safe function that handles connection shutdown and cleanup. + * + * @param conn Pointer to connection pointer to close + */ +CHDB_EXPORT void close_conn(struct chdb_conn ** conn); + +/** + * Executes a query on the given connection. + * Thread-safe function that handles query execution in a separate thread. + * + * @param conn Connection to execute query on + * @param query SQL query string to execute + * @param format Output format string (e.g., "CSV", default format) + * @return Query result structure containing output or error message + * @note Returns error result if connection is invalid or closed + */ +CHDB_EXPORT struct local_result_v2 * query_conn(struct chdb_conn * conn, const char * query, const char * format); + +#ifdef __cplusplus +} +#endif diff --git a/chdb.md b/chdb.md index 32b2abf..426a847 100644 --- a/chdb.md +++ b/chdb.md @@ -8,27 +8,28 @@ import "github.com/chdb-io/chdb-go/chdb" ## Index -- [func Query\(queryStr string, outputFormats ...string\) \*chdbstable.LocalResult](<#Query>) +- [func Query\(queryStr string, outputFormats ...string\) \(result chdbpurego.ChdbResult, err error\)](<#Query>) - [type Session](<#Session>) - [func NewSession\(paths ...string\) \(\*Session, error\)](<#NewSession>) - [func \(s \*Session\) Cleanup\(\)](<#Session.Cleanup>) - [func \(s \*Session\) Close\(\)](<#Session.Close>) + - [func \(s \*Session\) ConnStr\(\) string](<#Session.ConnStr>) - [func \(s \*Session\) IsTemp\(\) bool](<#Session.IsTemp>) - [func \(s \*Session\) Path\(\) string](<#Session.Path>) - - [func \(s \*Session\) Query\(queryStr string, outputFormats ...string\) \*chdbstable.LocalResult](<#Session.Query>) + - [func \(s \*Session\) Query\(queryStr string, outputFormats ...string\) \(result chdbpurego.ChdbResult, err error\)](<#Session.Query>) ## func [Query]() ```go -func Query(queryStr string, outputFormats ...string) *chdbstable.LocalResult +func Query(queryStr string, outputFormats ...string) (result chdbpurego.ChdbResult, err error) ``` -Query calls queryToBuffer with a default output format of "CSV" if not provided. +Query calls query\_conn with a default in\-memory session and default output format of "CSV" if not provided. -## type [Session]() +## type [Session]() @@ -39,7 +40,7 @@ type Session struct { ``` -### func [NewSession]() +### func [NewSession]() ```go func NewSession(paths ...string) (*Session, error) @@ -48,7 +49,7 @@ func NewSession(paths ...string) (*Session, error) NewSession creates a new session with the given path. If path is empty, a temporary directory is created. Note: The temporary directory is removed when Close is called. -### func \(\*Session\) [Cleanup]() +### func \(\*Session\) [Cleanup]() ```go func (s *Session) Cleanup() @@ -57,7 +58,7 @@ func (s *Session) Cleanup() Cleanup closes the session and removes the directory. -### func \(\*Session\) [Close]() +### func \(\*Session\) [Close]() ```go func (s *Session) Close() @@ -69,8 +70,17 @@ Close closes the session and removes the temporary directory temporary directory is created when NewSession was called with an empty path. ``` + +### func \(\*Session\) [ConnStr]() + +```go +func (s *Session) ConnStr() string +``` + +ConnStr returns the current connection string used for the underlying connection + -### func \(\*Session\) [IsTemp]() +### func \(\*Session\) [IsTemp]() ```go func (s *Session) IsTemp() bool @@ -79,7 +89,7 @@ func (s *Session) IsTemp() bool IsTemp returns whether the session is temporary. -### func \(\*Session\) [Path]() +### func \(\*Session\) [Path]() ```go func (s *Session) Path() string @@ -88,12 +98,12 @@ func (s *Session) Path() string Path returns the path of the session. -### func \(\*Session\) [Query]() +### func \(\*Session\) [Query]() ```go -func (s *Session) Query(queryStr string, outputFormats ...string) *chdbstable.LocalResult +func (s *Session) Query(queryStr string, outputFormats ...string) (result chdbpurego.ChdbResult, err error) ``` -Query calls queryToBuffer with a default output format of "CSV" if not provided. +Query calls \`query\_conn\` function with the current connection and a default output format of "CSV" if not provided. Generated by [gomarkdoc]() diff --git a/chdb/doc.md b/chdb/doc.md new file mode 100644 index 0000000..968986a --- /dev/null +++ b/chdb/doc.md @@ -0,0 +1,109 @@ + + +# chdb + +```go +import "github.com/chdb-io/chdb-go/chdb" +``` + +## Index + +- [func Query\(queryStr string, outputFormats ...string\) \(result chdbpurego.ChdbResult, err error\)](<#Query>) +- [type Session](<#Session>) + - [func NewSession\(paths ...string\) \(\*Session, error\)](<#NewSession>) + - [func \(s \*Session\) Cleanup\(\)](<#Session.Cleanup>) + - [func \(s \*Session\) Close\(\)](<#Session.Close>) + - [func \(s \*Session\) ConnStr\(\) string](<#Session.ConnStr>) + - [func \(s \*Session\) IsTemp\(\) bool](<#Session.IsTemp>) + - [func \(s \*Session\) Path\(\) string](<#Session.Path>) + - [func \(s \*Session\) Query\(queryStr string, outputFormats ...string\) \(result chdbpurego.ChdbResult, err error\)](<#Session.Query>) + + + +## func [Query]() + +```go +func Query(queryStr string, outputFormats ...string) (result chdbpurego.ChdbResult, err error) +``` + +Query calls query\_conn with a default in\-memory session and default output format of "CSV" if not provided. + + +## type [Session]() + + + +```go +type Session struct { + // contains filtered or unexported fields +} +``` + + +### func [NewSession]() + +```go +func NewSession(paths ...string) (*Session, error) +``` + +NewSession creates a new session with the given path. If path is empty, a temporary directory is created. Note: The temporary directory is removed when Close is called. + + +### func \(\*Session\) [Cleanup]() + +```go +func (s *Session) Cleanup() +``` + +Cleanup closes the session and removes the directory. + + +### func \(\*Session\) [Close]() + +```go +func (s *Session) Close() +``` + +Close closes the session and removes the temporary directory + +``` +temporary directory is created when NewSession was called with an empty path. +``` + + +### func \(\*Session\) [ConnStr]() + +```go +func (s *Session) ConnStr() string +``` + + + + +### func \(\*Session\) [IsTemp]() + +```go +func (s *Session) IsTemp() bool +``` + +IsTemp returns whether the session is temporary. + + +### func \(\*Session\) [Path]() + +```go +func (s *Session) Path() string +``` + +Path returns the path of the session. + + +### func \(\*Session\) [Query]() + +```go +func (s *Session) Query(queryStr string, outputFormats ...string) (result chdbpurego.ChdbResult, err error) +``` + +Query calls queryToBuffer with a default output format of "CSV" if not provided. + +Generated by [gomarkdoc]() diff --git a/chdb/driver/arrow.go b/chdb/driver/arrow.go deleted file mode 100644 index 3066ca0..0000000 --- a/chdb/driver/arrow.go +++ /dev/null @@ -1,175 +0,0 @@ -package chdbdriver - -import ( - "database/sql/driver" - "fmt" - "reflect" - "time" - - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/ipc" - "github.com/chdb-io/chdb-go/chdbstable" -) - -type arrowRows struct { - localResult *chdbstable.LocalResult - reader *ipc.FileReader - curRecord arrow.Record - curRow int64 -} - -func (r *arrowRows) Columns() (out []string) { - sch := r.reader.Schema() - for i := 0; i < sch.NumFields(); i++ { - out = append(out, sch.Field(i).Name) - } - return -} - -func (r *arrowRows) Close() error { - if r.curRecord != nil { - r.curRecord = nil - } - // ignore reader close - _ = r.reader.Close() - r.reader = nil - r.localResult = nil - return nil -} - -func (r *arrowRows) Next(dest []driver.Value) error { - if r.curRecord != nil && r.curRow == r.curRecord.NumRows() { - r.curRecord = nil - } - for r.curRecord == nil { - record, err := r.reader.Read() - if err != nil { - return err - } - if record.NumRows() == 0 { - continue - } - r.curRecord = record - r.curRow = 0 - } - - for i, col := range r.curRecord.Columns() { - if col.IsNull(int(r.curRow)) { - dest[i] = nil - continue - } - switch col := col.(type) { - case *array.Boolean: - dest[i] = col.Value(int(r.curRow)) - case *array.Int8: - dest[i] = col.Value(int(r.curRow)) - case *array.Uint8: - dest[i] = col.Value(int(r.curRow)) - case *array.Int16: - dest[i] = col.Value(int(r.curRow)) - case *array.Uint16: - dest[i] = col.Value(int(r.curRow)) - case *array.Int32: - dest[i] = col.Value(int(r.curRow)) - case *array.Uint32: - dest[i] = col.Value(int(r.curRow)) - case *array.Int64: - dest[i] = col.Value(int(r.curRow)) - case *array.Uint64: - dest[i] = col.Value(int(r.curRow)) - case *array.Float32: - dest[i] = col.Value(int(r.curRow)) - case *array.Float64: - dest[i] = col.Value(int(r.curRow)) - case *array.String: - dest[i] = col.Value(int(r.curRow)) - case *array.LargeString: - dest[i] = col.Value(int(r.curRow)) - case *array.Binary: - dest[i] = col.Value(int(r.curRow)) - case *array.LargeBinary: - dest[i] = col.Value(int(r.curRow)) - case *array.Date32: - dest[i] = col.Value(int(r.curRow)).ToTime() - case *array.Date64: - dest[i] = col.Value(int(r.curRow)).ToTime() - case *array.Time32: - dest[i] = col.Value(int(r.curRow)).ToTime(col.DataType().(*arrow.Time32Type).Unit) - case *array.Time64: - dest[i] = col.Value(int(r.curRow)).ToTime(col.DataType().(*arrow.Time64Type).Unit) - case *array.Timestamp: - dest[i] = col.Value(int(r.curRow)).ToTime(col.DataType().(*arrow.TimestampType).Unit) - case *array.Decimal128: - dest[i] = col.Value(int(r.curRow)) - case *array.Decimal256: - dest[i] = col.Value(int(r.curRow)) - default: - return fmt.Errorf( - "not yet implemented populating from columns of type " + col.DataType().String(), - ) - } - } - - r.curRow++ - return nil -} - -func (r *arrowRows) ColumnTypeDatabaseTypeName(index int) string { - return r.reader.Schema().Field(index).Type.String() -} - -func (r *arrowRows) ColumnTypeNullable(index int) (nullable, ok bool) { - return r.reader.Schema().Field(index).Nullable, true -} - -func (r *arrowRows) ColumnTypePrecisionScale(index int) (precision, scale int64, ok bool) { - typ := r.reader.Schema().Field(index).Type - switch dt := typ.(type) { - case *arrow.Decimal128Type: - return int64(dt.Precision), int64(dt.Scale), true - case *arrow.Decimal256Type: - return int64(dt.Precision), int64(dt.Scale), true - } - return 0, 0, false -} - -func (r *arrowRows) ColumnTypeScanType(index int) reflect.Type { - switch r.reader.Schema().Field(index).Type.ID() { - case arrow.BOOL: - return reflect.TypeOf(false) - case arrow.INT8: - return reflect.TypeOf(int8(0)) - case arrow.UINT8: - return reflect.TypeOf(uint8(0)) - case arrow.INT16: - return reflect.TypeOf(int16(0)) - case arrow.UINT16: - return reflect.TypeOf(uint16(0)) - case arrow.INT32: - return reflect.TypeOf(int32(0)) - case arrow.UINT32: - return reflect.TypeOf(uint32(0)) - case arrow.INT64: - return reflect.TypeOf(int64(0)) - case arrow.UINT64: - return reflect.TypeOf(uint64(0)) - case arrow.FLOAT32: - return reflect.TypeOf(float32(0)) - case arrow.FLOAT64: - return reflect.TypeOf(float64(0)) - case arrow.DECIMAL128: - return reflect.TypeOf(decimal128.Num{}) - case arrow.DECIMAL256: - return reflect.TypeOf(decimal256.Num{}) - case arrow.BINARY: - return reflect.TypeOf([]byte{}) - case arrow.STRING: - return reflect.TypeOf(string("")) - case arrow.TIME32, arrow.TIME64, arrow.DATE32, arrow.DATE64, arrow.TIMESTAMP: - return reflect.TypeOf(time.Time{}) - } - return nil -} diff --git a/chdb/driver/arrow_test.go b/chdb/driver/arrow_test.go deleted file mode 100644 index 769db00..0000000 --- a/chdb/driver/arrow_test.go +++ /dev/null @@ -1,106 +0,0 @@ -package chdbdriver - -import ( - "database/sql" - "fmt" - "os" - "testing" - - "github.com/chdb-io/chdb-go/chdb" -) - -func TestDbWithArrow(t *testing.T) { - - db, err := sql.Open("chdb", fmt.Sprintf("driverType=%s", "ARROW")) - if err != nil { - t.Errorf("open db fail, err:%s", err) - } - if db.Ping() != nil { - t.Errorf("ping db fail") - } - rows, err := db.Query(`SELECT 1,'abc'`) - if err != nil { - t.Errorf("run Query fail, err:%s", err) - } - cols, err := rows.Columns() - if err != nil { - t.Errorf("get result columns fail, err: %s", err) - } - if len(cols) != 2 { - t.Errorf("select result columns length should be 2") - } - var ( - bar int - foo string - ) - defer rows.Close() - for rows.Next() { - err := rows.Scan(&bar, &foo) - if err != nil { - t.Errorf("scan fail, err: %s", err) - } - if bar != 1 { - t.Errorf("expected error") - } - if foo != "abc" { - t.Errorf("expected error") - } - } -} - -func TestDBWithArrowSession(t *testing.T) { - sessionDir, err := os.MkdirTemp("", "unittest-sessiondata") - if err != nil { - t.Fatalf("create temp directory fail, err: %s", err) - } - defer os.RemoveAll(sessionDir) - session, err := chdb.NewSession(sessionDir) - if err != nil { - t.Fatalf("new session fail, err: %s", err) - } - defer session.Cleanup() - - session.Query("CREATE DATABASE IF NOT EXISTS testdb; " + - "CREATE TABLE IF NOT EXISTS testdb.testtable (id UInt32) ENGINE = MergeTree() ORDER BY id;") - - session.Query("INSERT INTO testdb.testtable VALUES (1), (2), (3);") - - ret, err := session.Query("SELECT * FROM testdb.testtable;") - if err != nil { - t.Fatalf("Query fail, err: %s", err) - } - if string(ret.Buf()) != "1\n2\n3\n" { - t.Errorf("Query result should be 1\n2\n3\n, got %s", string(ret.Buf())) - } - db, err := sql.Open("chdb", fmt.Sprintf("session=%s;driverType=%s", sessionDir, "ARROW")) - if err != nil { - t.Fatalf("open db fail, err: %s", err) - } - if db.Ping() != nil { - t.Fatalf("ping db fail, err: %s", err) - } - rows, err := db.Query("select * from testdb.testtable;") - if err != nil { - t.Fatalf("exec create function fail, err: %s", err) - } - defer rows.Close() - cols, err := rows.Columns() - if err != nil { - t.Fatalf("get result columns fail, err: %s", err) - } - if len(cols) != 1 { - t.Fatalf("result columns length shoule be 3, actual: %d", len(cols)) - } - var bar = 0 - var count = 1 - for rows.Next() { - err = rows.Scan(&bar) - if err != nil { - t.Fatalf("scan fail, err: %s", err) - } - if bar != count { - t.Fatalf("result is not match, want: %d actual: %d", count, bar) - } - count++ - } -} diff --git a/chdb/driver/driver.go b/chdb/driver/driver.go index 22f7a57..29e8347 100644 --- a/chdb/driver/driver.go +++ b/chdb/driver/driver.go @@ -10,11 +10,9 @@ import ( "strings" "github.com/chdb-io/chdb-go/chdb" - "github.com/chdb-io/chdb-go/chdbstable" + chdbpurego "github.com/chdb-io/chdb-go/chdb-purego" "github.com/huandu/go-sqlbuilder" "github.com/parquet-go/parquet-go" - - "github.com/apache/arrow/go/v15/arrow/ipc" ) type DriverType int @@ -46,14 +44,8 @@ func (d DriverType) String() string { return "" } -func (d DriverType) PrepareRows(result *chdbstable.LocalResult, buf []byte, bufSize int, useUnsafe bool) (driver.Rows, error) { +func (d DriverType) PrepareRows(result chdbpurego.ChdbResult, buf []byte, bufSize int, useUnsafe bool) (driver.Rows, error) { switch d { - case ARROW: - reader, err := ipc.NewFileReader(bytes.NewReader(buf)) - if err != nil { - return nil, err - } - return &arrowRows{localResult: result, reader: reader}, nil case PARQUET: reader := parquet.NewGenericReader[any](bytes.NewReader(buf)) return &parquetRows{ @@ -67,8 +59,8 @@ func (d DriverType) PrepareRows(result *chdbstable.LocalResult, buf []byte, bufS func parseDriverType(s string) DriverType { switch strings.ToUpper(s) { - case "ARROW": - return ARROW + // case "ARROW": + // return ARROW case "PARQUET": return PARQUET } @@ -133,7 +125,7 @@ func (e *execResult) RowsAffected() (int64, error) { return -1, fmt.Errorf("does not support RowsAffected") } -type queryHandle func(string, ...string) (*chdbstable.LocalResult, error) +type queryHandle func(string, ...string) (chdbpurego.ChdbResult, error) type connector struct { udfPath string @@ -190,7 +182,7 @@ func NewConnect(opts map[string]string) (ret *connector, err error) { if ok { ret.driverType = parseDriverType(driverType) } else { - ret.driverType = ARROW //default to arrow + ret.driverType = PARQUET //default to parquet } bufferSize, ok := opts[driverBufferSizeKey] if ok { @@ -214,6 +206,13 @@ func NewConnect(opts map[string]string) (ret *connector, err error) { if ok { ret.udfPath = udfPath } + if ret.session == nil { + + ret.session, err = chdb.NewSession() + if err != nil { + return nil, err + } + } return } @@ -243,7 +242,8 @@ type conn struct { bufferSize int useUnsafe bool session *chdb.Session - QueryFun queryHandle + + QueryFun queryHandle } func prepareValues(values []driver.Value) []driver.NamedValue { @@ -267,6 +267,7 @@ func (c *conn) SetupQueryFun() { if c.session != nil { c.QueryFun = c.session.Query } + } func (c *conn) Query(query string, values []driver.Value) (driver.Rows, error) { @@ -334,7 +335,7 @@ func (c *conn) QueryContext(ctx context.Context, query string, args []driver.Nam } buf := result.Buf() - if buf == nil { + if len(buf) == 0 { return nil, fmt.Errorf("result is nil") } return c.driverType.PrepareRows(result, buf, c.bufferSize, c.useUnsafe) diff --git a/chdb/driver/driver_test.go b/chdb/driver/driver_test.go index ebbfcb1..8321e20 100644 --- a/chdb/driver/driver_test.go +++ b/chdb/driver/driver_test.go @@ -9,24 +9,57 @@ import ( "github.com/chdb-io/chdb-go/chdb" ) +var ( + session *chdb.Session +) + +func globalSetup() error { + sess, err := chdb.NewSession() + if err != nil { + return err + } + session = sess + return nil +} + +func globalTeardown() { + session.Cleanup() + session.Close() +} + +func TestMain(m *testing.M) { + if err := globalSetup(); err != nil { + fmt.Println("Global setup failed:", err) + os.Exit(1) + } + // Run all tests. + exitCode := m.Run() + + // Global teardown: clean up any resources here. + globalTeardown() + + // Exit with the code returned by m.Run(). + os.Exit(exitCode) +} + func TestDb(t *testing.T) { db, err := sql.Open("chdb", "") if err != nil { - t.Errorf("open db fail, err:%s", err) + t.Fatalf("open db fail, err:%s", err) } if db.Ping() != nil { - t.Errorf("ping db fail") + t.Fatalf("ping db fail") } rows, err := db.Query(`SELECT 1,'abc'`) if err != nil { - t.Errorf("run Query fail, err:%s", err) + t.Fatalf("run Query fail, err:%s", err) } cols, err := rows.Columns() if err != nil { - t.Errorf("get result columns fail, err: %s", err) + t.Fatalf("get result columns fail, err: %s", err) } if len(cols) != 2 { - t.Errorf("select result columns length should be 2") + t.Fatalf("select result columns length should be 2") } var ( bar int @@ -112,37 +145,74 @@ func TestDbWithOpt(t *testing.T) { } func TestDbWithSession(t *testing.T) { - sessionDir, err := os.MkdirTemp("", "unittest-sessiondata") + + session.Query( + "CREATE TABLE IF NOT EXISTS TestDbWithSession (id UInt32) ENGINE = MergeTree() ORDER BY id;") + + session.Query("INSERT INTO TestDbWithSession VALUES (1), (2), (3);") + + ret, err := session.Query("SELECT * FROM TestDbWithSession;") if err != nil { - t.Fatalf("create temp directory fail, err: %s", err) + t.Fatalf("Query fail, err: %s", err) + } + if string(ret.Buf()) != "1\n2\n3\n" { + t.Errorf("Query result should be 1\n2\n3\n, got %s", string(ret.Buf())) } - defer os.RemoveAll(sessionDir) - session, err := chdb.NewSession(sessionDir) + db, err := sql.Open("chdb", fmt.Sprintf("session=%s", session.ConnStr())) if err != nil { - t.Fatalf("new session fail, err: %s", err) + t.Fatalf("open db fail, err: %s", err) + } + if db.Ping() != nil { + t.Fatalf("ping db fail, err: %s", err) + } + rows, err := db.Query("select * from TestDbWithSession;") + if err != nil { + t.Fatalf("exec create function fail, err: %s", err) } - defer session.Cleanup() + defer rows.Close() + cols, err := rows.Columns() + if err != nil { + t.Fatalf("get result columns fail, err: %s", err) + } + if len(cols) != 1 { + t.Fatalf("result columns length shoule be 3, actual: %d", len(cols)) + } + var bar = 0 + var count = 1 + for rows.Next() { + err = rows.Scan(&bar) + if err != nil { + t.Fatalf("scan fail, err: %s", err) + } + if bar != count { + t.Fatalf("result is not match, want: %d actual: %d", count, bar) + } + count++ + } +} + +func TestDbWithConnection(t *testing.T) { - session.Query("CREATE DATABASE IF NOT EXISTS testdb; " + - "CREATE TABLE IF NOT EXISTS testdb.testtable (id UInt32) ENGINE = MergeTree() ORDER BY id;") + session.Query( + "CREATE TABLE IF NOT EXISTS TestDbWithConnection (id UInt32) ENGINE = MergeTree() ORDER BY id;") - session.Query("USE testdb; INSERT INTO testtable VALUES (1), (2), (3);") + session.Query("INSERT INTO TestDbWithConnection VALUES (1), (2), (3);") - ret, err := session.Query("SELECT * FROM testtable;") + ret, err := session.Query("SELECT * FROM TestDbWithConnection;") if err != nil { t.Fatalf("Query fail, err: %s", err) } if string(ret.Buf()) != "1\n2\n3\n" { t.Errorf("Query result should be 1\n2\n3\n, got %s", string(ret.Buf())) } - db, err := sql.Open("chdb", fmt.Sprintf("session=%s", sessionDir)) + db, err := sql.Open("chdb", fmt.Sprintf("session=%s", session.ConnStr())) if err != nil { t.Fatalf("open db fail, err: %s", err) } if db.Ping() != nil { t.Fatalf("ping db fail, err: %s", err) } - rows, err := db.Query("select * from testtable;") + rows, err := db.Query("select * from TestDbWithConnection;") if err != nil { t.Fatalf("exec create function fail, err: %s", err) } @@ -168,37 +238,73 @@ func TestDbWithSession(t *testing.T) { } } -func TestQueryRow(t *testing.T) { - sessionDir, err := os.MkdirTemp("", "unittest-sessiondata") +func TestDbWithConnectionSqlDriverOnly(t *testing.T) { + db, err := sql.Open("chdb", fmt.Sprintf("session=%s", session.ConnStr())) if err != nil { - t.Fatalf("create temp directory fail, err: %s", err) + t.Fatalf("open db fail, err: %s", err) } - defer os.RemoveAll(sessionDir) - session, err := chdb.NewSession(sessionDir) + if db.Ping() != nil { + t.Fatalf("ping db fail, err: %s", err) + } + + _, err = db.Exec( + "CREATE TABLE IF NOT EXISTS TestDbWithConnectionSqlDriverOnly (id UInt32) ENGINE = MergeTree() ORDER BY id;") if err != nil { - t.Fatalf("new session fail, err: %s", err) + t.Fatalf("could not create database & table: %s", err) + } + _, err = db.Exec("INSERT INTO TestDbWithConnectionSqlDriverOnly VALUES (1), (2), (3);") + if err != nil { + t.Fatalf("could not insert rows in the table: %s", err) + } + + rows, err := db.Query("select * from TestDbWithConnectionSqlDriverOnly;") + if err != nil { + t.Fatalf("exec create function fail, err: %s", err) + } + defer rows.Close() + cols, err := rows.Columns() + if err != nil { + t.Fatalf("get result columns fail, err: %s", err) + } + if len(cols) != 1 { + t.Fatalf("result columns length shoule be 3, actual: %d", len(cols)) + } + var bar = 0 + var count = 1 + for rows.Next() { + err = rows.Scan(&bar) + if err != nil { + t.Fatalf("scan fail, err: %s", err) + } + if bar != count { + t.Fatalf("result is not match, want: %d actual: %d", count, bar) + } + count++ } - defer session.Cleanup() - session.Query("CREATE DATABASE IF NOT EXISTS testdb; " + - "CREATE TABLE IF NOT EXISTS testdb.testtable (id UInt32) ENGINE = MergeTree() ORDER BY id;") +} - session.Query("USE testdb; INSERT INTO testtable VALUES (1), (2), (3);") +func TestQueryRow(t *testing.T) { - ret, err := session.Query("SELECT * FROM testtable;") + session.Query( + "CREATE TABLE IF NOT EXISTS TestQueryRow (id UInt32) ENGINE = MergeTree() ORDER BY id;") + + session.Query(" INSERT INTO TestQueryRow VALUES (1), (2), (3);") + + ret, err := session.Query("SELECT * FROM TestQueryRow;") if err != nil { t.Fatalf("Query fail, err: %s", err) } if string(ret.Buf()) != "1\n2\n3\n" { t.Errorf("Query result should be 1\n2\n3\n, got %s", string(ret.Buf())) } - db, err := sql.Open("chdb", fmt.Sprintf("session=%s", sessionDir)) + db, err := sql.Open("chdb", fmt.Sprintf("session=%s", session.ConnStr())) if err != nil { t.Fatalf("open db fail, err: %s", err) } if db.Ping() != nil { t.Fatalf("ping db fail, err: %s", err) } - rows := db.QueryRow("select * from testtable;") + rows := db.QueryRow("select * from TestQueryRow;") var bar = 0 var count = 1 @@ -217,20 +323,11 @@ func TestQueryRow(t *testing.T) { } func TestExec(t *testing.T) { - sessionDir, err := os.MkdirTemp("", "unittest-sessiondata") - if err != nil { - t.Fatalf("create temp directory fail, err: %s", err) - } - defer os.RemoveAll(sessionDir) - session, err := chdb.NewSession(sessionDir) - if err != nil { - t.Fatalf("new session fail, err: %s", err) - } - defer session.Cleanup() - session.Query("CREATE DATABASE IF NOT EXISTS testdb; " + - "CREATE TABLE IF NOT EXISTS testdb.testtable (id UInt32) ENGINE = MergeTree() ORDER BY id;") - db, err := sql.Open("chdb", fmt.Sprintf("session=%s", sessionDir)) + session.Query( + "CREATE TABLE IF NOT EXISTS TestExec (id UInt32) ENGINE = MergeTree() ORDER BY id;") + + db, err := sql.Open("chdb", fmt.Sprintf("session=%s", session.ConnStr())) if err != nil { t.Fatalf("open db fail, err: %s", err) } @@ -238,11 +335,25 @@ func TestExec(t *testing.T) { t.Fatalf("ping db fail, err: %s", err) } - _, err = db.Exec("INSERT INTO testdb.testtable VALUES (1), (2), (3);") + tables, err := db.Query("SHOW TABLES;") + if err != nil { + t.Fatalf(err.Error()) + } + defer tables.Close() + for tables.Next() { + var tblName string + if err := tables.Scan(&tblName); err != nil { + t.Fatal(err) + } + t.Log(tblName) + fmt.Printf("tblName: %v\n", tblName) + } + + _, err = db.Exec("INSERT INTO TestExec VALUES (1), (2), (3);") if err != nil { t.Fatalf("exec failed, err: %s", err) } - rows := db.QueryRow("select * from testdb.testtable;") + rows := db.QueryRow("select * from TestExec;") var bar = 0 var count = 1 diff --git a/chdb/driver/parquet.go b/chdb/driver/parquet.go index 5b1cce8..700468e 100644 --- a/chdb/driver/parquet.go +++ b/chdb/driver/parquet.go @@ -9,7 +9,7 @@ import ( "reflect" - "github.com/chdb-io/chdb-go/chdbstable" + chdbpurego "github.com/chdb-io/chdb-go/chdb-purego" "github.com/parquet-go/parquet-go" ) @@ -24,7 +24,7 @@ func getStringFromBytes(v parquet.Value) string { } type parquetRows struct { - localResult *chdbstable.LocalResult // result from clickhouse + localResult chdbpurego.ChdbResult // result from clickhouse reader *parquet.GenericReader[any] // parquet reader curRecord parquet.Row // TODO: delete this? buffer []parquet.Row // record buffer @@ -51,7 +51,9 @@ func (r *parquetRows) Close() error { // ignore reader close _ = r.reader.Close() r.reader = nil + r.localResult.Free() r.localResult = nil + r.buffer = nil return nil } diff --git a/chdb/driver/parquet_test.go b/chdb/driver/parquet_test.go index 44fdcc1..d21293c 100644 --- a/chdb/driver/parquet_test.go +++ b/chdb/driver/parquet_test.go @@ -3,10 +3,7 @@ package chdbdriver import ( "database/sql" "fmt" - "os" "testing" - - "github.com/chdb-io/chdb-go/chdb" ) func TestDbWithParquet(t *testing.T) { @@ -49,37 +46,74 @@ func TestDbWithParquet(t *testing.T) { } func TestDBWithParquetSession(t *testing.T) { - sessionDir, err := os.MkdirTemp("", "unittest-sessiondata") + + session.Query( + "CREATE TABLE IF NOT EXISTS TestDBWithParquetSession (id UInt32) ENGINE = MergeTree() ORDER BY id;") + + session.Query("INSERT INTO TestDBWithParquetSession VALUES (1), (2), (3);") + + ret, err := session.Query("SELECT * FROM TestDBWithParquetSession;") if err != nil { - t.Fatalf("create temp directory fail, err: %s", err) + t.Fatalf("Query fail, err: %s", err) } - defer os.RemoveAll(sessionDir) - session, err := chdb.NewSession(sessionDir) + if string(ret.Buf()) != "1\n2\n3\n" { + t.Errorf("Query result should be 1\n2\n3\n, got %s", string(ret.Buf())) + } + db, err := sql.Open("chdb", fmt.Sprintf("session=%s;driverType=%s", session.ConnStr(), "PARQUET")) if err != nil { - t.Fatalf("new session fail, err: %s", err) + t.Fatalf("open db fail, err: %s", err) } - defer session.Cleanup() + if db.Ping() != nil { + t.Fatalf("ping db fail, err: %s", err) + } + rows, err := db.Query("select * from TestDBWithParquetSession;") + if err != nil { + t.Fatalf("exec create function fail, err: %s", err) + } + defer rows.Close() + cols, err := rows.Columns() + if err != nil { + t.Fatalf("get result columns fail, err: %s", err) + } + if len(cols) != 1 { + t.Fatalf("result columns length shoule be 3, actual: %d", len(cols)) + } + var bar = 0 + var count = 1 + for rows.Next() { + err = rows.Scan(&bar) + if err != nil { + t.Fatalf("scan fail, err: %s", err) + } + if bar != count { + t.Fatalf("result is not match, want: %d actual: %d", count, bar) + } + count++ + } +} + +func TestDBWithParquetConnection(t *testing.T) { - session.Query("CREATE DATABASE IF NOT EXISTS testdb; " + - "CREATE TABLE IF NOT EXISTS testdb.testtable (id UInt32) ENGINE = MergeTree() ORDER BY id;") + session.Query( + "CREATE TABLE IF NOT EXISTS TestDBWithParquetConnection (id UInt32) ENGINE = MergeTree() ORDER BY id;") - session.Query("INSERT INTO testdb.testtable VALUES (1), (2), (3);") + session.Query("INSERT INTO TestDBWithParquetConnection VALUES (1), (2), (3);") - ret, err := session.Query("SELECT * FROM testdb.testtable;") + ret, err := session.Query("SELECT * FROM TestDBWithParquetConnection;") if err != nil { t.Fatalf("Query fail, err: %s", err) } if string(ret.Buf()) != "1\n2\n3\n" { t.Errorf("Query result should be 1\n2\n3\n, got %s", string(ret.Buf())) } - db, err := sql.Open("chdb", fmt.Sprintf("session=%s;driverType=%s", sessionDir, "PARQUET")) + db, err := sql.Open("chdb", fmt.Sprintf("session=%s;driverType=%s", session.ConnStr(), "PARQUET")) if err != nil { t.Fatalf("open db fail, err: %s", err) } if db.Ping() != nil { t.Fatalf("ping db fail, err: %s", err) } - rows, err := db.Query("select * from testdb.testtable;") + rows, err := db.Query("select * from TestDBWithParquetConnection;") if err != nil { t.Fatalf("exec create function fail, err: %s", err) } diff --git a/chdb/session.go b/chdb/session.go index 93d5fcb..3ed1128 100644 --- a/chdb/session.go +++ b/chdb/session.go @@ -1,47 +1,64 @@ package chdb import ( - "io/ioutil" + "fmt" "os" "path/filepath" - "github.com/chdb-io/chdb-go/chdbstable" + chdbpurego "github.com/chdb-io/chdb-go/chdb-purego" +) + +var ( + globalSession *Session ) type Session struct { - path string - isTemp bool + conn chdbpurego.ChdbConn + connStr string + path string + isTemp bool } // NewSession creates a new session with the given path. // If path is empty, a temporary directory is created. // Note: The temporary directory is removed when Close is called. func NewSession(paths ...string) (*Session, error) { + if globalSession != nil { + return globalSession, nil + } + path := "" if len(paths) > 0 { path = paths[0] } - + isTemp := false if path == "" { // Create a temporary directory - tempDir, err := ioutil.TempDir("", "chdb_") + tempDir, err := os.MkdirTemp("", "chdb_") if err != nil { return nil, err } path = tempDir - return &Session{path: path, isTemp: true}, nil + isTemp = true + } + connStr := fmt.Sprintf("file:%s/chdb.db", path) - return &Session{path: path, isTemp: false}, nil + conn, err := initConnection(connStr) + if err != nil { + return nil, err + } + globalSession = &Session{connStr: connStr, path: path, isTemp: isTemp, conn: conn} + return globalSession, nil } -// Query calls queryToBuffer with a default output format of "CSV" if not provided. -func (s *Session) Query(queryStr string, outputFormats ...string) (result *chdbstable.LocalResult, err error) { +// Query calls `query_conn` function with the current connection and a default output format of "CSV" if not provided. +func (s *Session) Query(queryStr string, outputFormats ...string) (result chdbpurego.ChdbResult, err error) { outputFormat := "CSV" // Default value if len(outputFormats) > 0 { outputFormat = outputFormats[0] } - return queryToBuffer(queryStr, outputFormat, s.path, "") + return s.conn.Query(queryStr, outputFormat) } // Close closes the session and removes the temporary directory @@ -49,9 +66,11 @@ func (s *Session) Query(queryStr string, outputFormats ...string) (result *chdbs // temporary directory is created when NewSession was called with an empty path. func (s *Session) Close() { // Remove the temporary directory if it starts with "chdb_" + s.conn.Close() if s.isTemp && filepath.Base(s.path)[:5] == "chdb_" { s.Cleanup() } + globalSession = nil } // Cleanup closes the session and removes the directory. @@ -65,6 +84,11 @@ func (s *Session) Path() string { return s.path } +// ConnStr returns the current connection string used for the underlying connection +func (s *Session) ConnStr() string { + return s.connStr +} + // IsTemp returns whether the session is temporary. func (s *Session) IsTemp() bool { return s.isTemp diff --git a/chdb/session_test.go b/chdb/session_test.go index f99ea62..dee42ce 100644 --- a/chdb/session_test.go +++ b/chdb/session_test.go @@ -1,18 +1,47 @@ package chdb import ( + "fmt" "os" "path/filepath" "testing" ) -// TestNewSession tests the creation of a new session. -func TestNewSession(t *testing.T) { - session, err := NewSession() +var ( + session *Session +) + +func globalSetup() error { + sess, err := NewSession() if err != nil { - t.Fatalf("Failed to create new session: %s", err) + return err } - defer session.Cleanup() + session = sess + return nil +} + +func globalTeardown() { + session.Cleanup() + session.Close() +} + +func TestMain(m *testing.M) { + if err := globalSetup(); err != nil { + fmt.Println("Global setup failed:", err) + os.Exit(1) + } + // Run all tests. + exitCode := m.Run() + + // Global teardown: clean up any resources here. + globalTeardown() + + // Exit with the code returned by m.Run(). + os.Exit(exitCode) +} + +// TestNewSession tests the creation of a new session. +func TestNewSession(t *testing.T) { // Check if the session directory exists if _, err := os.Stat(session.Path()); os.IsNotExist(err) { @@ -25,58 +54,33 @@ func TestNewSession(t *testing.T) { } } -// TestSessionClose tests the Close method of the session. -func TestSessionClose(t *testing.T) { - session, _ := NewSession() - defer session.Cleanup() // Cleanup in case Close fails +// This test is currently flaky because of this: https://github.com/chdb-io/chdb/pull/299/commits/91b0aedd8c17e74a4bb213e885d89cc9a77c99ad +func TestQuery(t *testing.T) { - // Close the session - session.Close() + // time.Sleep(time.Second * 5) - // Check if the session directory has been removed - if _, err := os.Stat(session.Path()); !os.IsNotExist(err) { - t.Errorf("Session directory should be removed after Close: %s", session.Path()) + _, err := session.Query("CREATE TABLE IF NOT EXISTS TestQuery (id UInt32) ENGINE = MergeTree() ORDER BY id;") + if err != nil { + t.Fatal(err) } -} -// TestSessionCleanup tests the Cleanup method of the session. -func TestSessionCleanup(t *testing.T) { - session, _ := NewSession() - - // Cleanup the session - session.Cleanup() - - // Check if the session directory has been removed - if _, err := os.Stat(session.Path()); !os.IsNotExist(err) { - t.Errorf("Session directory should be removed after Cleanup: %s", session.Path()) + _, err = session.Query("INSERT INTO TestQuery VALUES (1), (2), (3);") + if err != nil { + t.Fatal(err) } -} - -// TestQuery tests the Query method of the session. -func TestQuery(t *testing.T) { - path := filepath.Join(os.TempDir(), "chdb_test") - defer os.RemoveAll(path) - session, _ := NewSession(path) - defer session.Cleanup() - - session.Query("CREATE DATABASE IF NOT EXISTS testdb; " + - "CREATE TABLE IF NOT EXISTS testdb.testtable (id UInt32) ENGINE = MergeTree() ORDER BY id;") - - session.Query("USE testdb; INSERT INTO testtable VALUES (1), (2), (3);") - - ret, err := session.Query("SELECT * FROM testtable;") + ret, err := session.Query("SELECT * FROM TestQuery;") if err != nil { - t.Errorf("Query failed: %s", err) + t.Fatalf("Query failed: %s", err) } + if string(ret.Buf()) != "1\n2\n3\n" { - t.Errorf("Query result should be 1\n2\n3\n, got %s", string(ret.Buf())) + t.Fatalf("Query result should be 1\n2\n3\n, got %s", string(ret.Buf())) } } func TestSessionPathAndIsTemp(t *testing.T) { // Create a new session and check its Path and IsTemp session, _ := NewSession() - defer session.Cleanup() if session.Path() == "" { t.Errorf("Session path should not be empty") @@ -85,6 +89,7 @@ func TestSessionPathAndIsTemp(t *testing.T) { if !session.IsTemp() { t.Errorf("Session should be temporary") } + session.Close() // Create a new session with a specific path and check its Path and IsTemp path := filepath.Join(os.TempDir(), "chdb_test2") @@ -100,3 +105,30 @@ func TestSessionPathAndIsTemp(t *testing.T) { t.Errorf("Session should not be temporary") } } + +// TestSessionClose tests the Close method of the session. +func TestSessionClose(t *testing.T) { + session, _ := NewSession() + defer session.Cleanup() // Cleanup in case Close fails + + // Close the session + session.Close() + + // Check if the session directory has been removed + if _, err := os.Stat(session.Path()); !os.IsNotExist(err) { + t.Errorf("Session directory should be removed after Close: %s", session.Path()) + } +} + +// TestSessionCleanup tests the Cleanup method of the session. +func TestSessionCleanup(t *testing.T) { + session, _ := NewSession() + + // Cleanup the session + session.Cleanup() + + // Check if the session directory has been removed + if _, err := os.Stat(session.Path()); !os.IsNotExist(err) { + t.Errorf("Session directory should be removed after Cleanup: %s", session.Path()) + } +} diff --git a/chdb/wrapper.go b/chdb/wrapper.go index be5ab63..9c19fed 100644 --- a/chdb/wrapper.go +++ b/chdb/wrapper.go @@ -1,42 +1,25 @@ package chdb import ( - "github.com/chdb-io/chdb-go/chdbstable" + chdbpurego "github.com/chdb-io/chdb-go/chdb-purego" ) -// Query calls queryToBuffer with a default output format of "CSV" if not provided. -func Query(queryStr string, outputFormats ...string) (result *chdbstable.LocalResult, err error) { +// Query calls query_conn with a default in-memory session and default output format of "CSV" if not provided. +func Query(queryStr string, outputFormats ...string) (result chdbpurego.ChdbResult, err error) { outputFormat := "CSV" // Default value if len(outputFormats) > 0 { outputFormat = outputFormats[0] } - return queryToBuffer(queryStr, outputFormat, "", "") -} - -// queryToBuffer constructs the arguments for QueryStable and calls it. -func queryToBuffer(queryStr, outputFormat, path, udfPath string) (result *chdbstable.LocalResult, err error) { - argv := []string{"clickhouse", "--multiquery"} - - // Handle output format - if outputFormat == "Debug" || outputFormat == "debug" { - argv = append(argv, "--verbose", "--log-level=trace", "--output-format=CSV") - } else { - argv = append(argv, "--output-format="+outputFormat) - } - - // Handle path - if path != "" { - argv = append(argv, "--path="+path) - } - - // Add query string - argv = append(argv, "--query="+queryStr) - - // Handle user-defined functions path - if udfPath != "" { - argv = append(argv, "--", "--user_scripts_path="+udfPath, "--user_defined_executable_functions_config="+udfPath+"/*.xml") + tempSession, err := initConnection(":memory:") + if err != nil { + return nil, err } + defer tempSession.Close() + return tempSession.Query(queryStr, outputFormat) +} - // Call QueryStable with the constructed arguments - return chdbstable.QueryStable(len(argv), argv) +func initConnection(connStr string) (result chdbpurego.ChdbConn, err error) { + argv := []string{connStr} + // Call NewConnection with the constructed arguments + return chdbpurego.NewConnection(len(argv), argv) } diff --git a/chdb/wrapper_test.go b/chdb/wrapper_test.go index 10ae5bc..c97680a 100644 --- a/chdb/wrapper_test.go +++ b/chdb/wrapper_test.go @@ -1,69 +1,37 @@ package chdb import ( - "os" - "path/filepath" "testing" ) func TestQueryToBuffer(t *testing.T) { // Create a temporary directory - tempDir := filepath.Join(os.TempDir(), "chdb_test") - defer os.RemoveAll(tempDir) // Define test cases testCases := []struct { - name string - queryStr string - outputFormat string - path string + name string + queryStr string + outputFormat string + udfPath string expectedErrMsg string expectedResult string }{ { - name: "Basic Query", - queryStr: "SELECT 123", - outputFormat: "CSV", - path: "", + name: "Basic Query", + queryStr: "SELECT 123", + outputFormat: "CSV", + udfPath: "", expectedErrMsg: "", expectedResult: "123\n", }, - // Session - { - name: "Session Query 1", - queryStr: "CREATE DATABASE IF NOT EXISTS testdb; " + - "CREATE TABLE IF NOT EXISTS testdb.testtable (id UInt32) ENGINE = MergeTree() ORDER BY id;", - outputFormat: "CSV", - path: tempDir, - udfPath: "", - expectedErrMsg: "", - expectedResult: "", - }, - // { - // name: "Session Query 2", - // queryStr: "USE testdb; INSERT INTO testtable VALUES (1), (2), (3);", - // outputFormat: "CSV", - // path: tempDir, - // udfPath: "", - // expectedErrMsg: "", - // expectedResult: "", - // }, - // { - // name: "Session Query 3", - // queryStr: "SELECT * FROM testtable;", - // outputFormat: "CSV", - // path: tempDir, - // udfPath: "", - // expectedErrMsg: "", - // expectedResult: "1\n2\n3\n", - // }, + { - name: "Error Query", - queryStr: "SELECT * FROM nonexist; ", - outputFormat: "CSV", - path: tempDir, + name: "Error Query", + queryStr: "SELECT * FROM nonexist; ", + outputFormat: "CSV", + udfPath: "", expectedErrMsg: "Code: 60. DB::Exception: Unknown table expression identifier 'nonexist' in scope SELECT * FROM nonexist. (UNKNOWN_TABLE)", expectedResult: "", @@ -73,23 +41,24 @@ func TestQueryToBuffer(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { // Call queryToBuffer - result, err := queryToBuffer(tc.queryStr, tc.outputFormat, tc.path, tc.udfPath) + + result, err := Query(tc.queryStr, tc.outputFormat) // Verify if tc.expectedErrMsg != "" { if err == nil { - t.Errorf("%v queryToBuffer() with queryStr %v, outputFormat %v, path %v, udfPath %v, expect error message: %v, got no error", - tc.name, tc.queryStr, tc.outputFormat, tc.path, tc.udfPath, tc.expectedErrMsg) + t.Errorf("%v queryToBuffer() with queryStr %v, outputFormat %v, udfPath %v, expect error message: %v, got no error", + tc.name, tc.queryStr, tc.outputFormat, tc.udfPath, tc.expectedErrMsg) } else { if err.Error() != tc.expectedErrMsg { - t.Errorf("%v queryToBuffer() with queryStr %v, outputFormat %v, path %v, udfPath %v, expect error message: %v, got error message: %v", - tc.name, tc.queryStr, tc.outputFormat, tc.path, tc.udfPath, tc.expectedErrMsg, err.Error()) + t.Errorf("%v queryToBuffer() with queryStr %v, outputFormat %v, udfPath %v, expect error message: %v, got error message: %v", + tc.name, tc.queryStr, tc.outputFormat, tc.udfPath, tc.expectedErrMsg, err.Error()) } } } else { if string(result.Buf()) != tc.expectedResult { - t.Errorf("%v queryToBuffer() with queryStr %v, outputFormat %v, path %v, udfPath %v, expect result: %v, got result: %v", - tc.name, tc.queryStr, tc.outputFormat, tc.path, tc.udfPath, tc.expectedResult, string(result.Buf())) + t.Errorf("%v queryToBuffer() with queryStr %v, outputFormat %v, udfPath %v, expect result: %v, got result: %v", + tc.name, tc.queryStr, tc.outputFormat, tc.udfPath, tc.expectedResult, string(result.Buf())) } } }) diff --git a/chdbstable/chdb.go b/chdbstable/chdb.go deleted file mode 100644 index c02aba2..0000000 --- a/chdbstable/chdb.go +++ /dev/null @@ -1,111 +0,0 @@ -package chdbstable - -/* -#cgo LDFLAGS: -L/usr/local/lib -lchdb -#include // Include the C standard library for C.free -#include "chdb.h" -*/ -import "C" -import ( - "errors" - "runtime" - "unsafe" -) - -// ChdbError is returned when the C function returns an error. -type ChdbError struct { - msg string -} - -func (e *ChdbError) Error() string { - return e.msg -} - -// ErrNilResult is returned when the C function returns a nil pointer. -var ErrNilResult = errors.New("chDB C function returned nil pointer") - -// LocalResult mirrors the C struct local_result_v2 in Go. -type LocalResult struct { - cResult *C.struct_local_result_v2 -} - -// newLocalResult creates a new LocalResult and sets a finalizer to free C memory. -func newLocalResult(cResult *C.struct_local_result_v2) *LocalResult { - result := &LocalResult{cResult: cResult} - runtime.SetFinalizer(result, freeLocalResult) - return result -} - -// freeLocalResult is called by the garbage collector. -func freeLocalResult(result *LocalResult) { - C.free_result_v2(result.cResult) -} - -// QueryStable calls the C function query_stable_v2. -func QueryStable(argc int, argv []string) (result *LocalResult, err error) { - cArgv := make([]*C.char, len(argv)) - for i, s := range argv { - cArgv[i] = C.CString(s) - defer C.free(unsafe.Pointer(cArgv[i])) - } - - cResult := C.query_stable_v2(C.int(argc), &cArgv[0]) - if cResult == nil { - // According to the C ABI of chDB v1.2.0, the C function query_stable_v2 - // returns nil if the query returns no data. This is not an error. We - // will change this behavior in the future. - return newLocalResult(cResult), nil - } - if cResult.error_message != nil { - return nil, &ChdbError{msg: C.GoString(cResult.error_message)} - } - return newLocalResult(cResult), nil -} - -// Accessor methods to access fields of the local_result_v2 struct. -func (r *LocalResult) Buf() []byte { - if r.cResult == nil { - return nil - } - if r.cResult.buf == nil { - return nil - } - return C.GoBytes(unsafe.Pointer(r.cResult.buf), C.int(r.cResult.len)) -} - -// Stringer interface for LocalResult -func (r LocalResult) String() string { - ret := r.Buf() - if ret == nil { - return "" - } - return string(ret) -} - -func (r *LocalResult) Len() int { - if r.cResult == nil { - return 0 - } - return int(r.cResult.len) -} - -func (r *LocalResult) Elapsed() float64 { - if r.cResult == nil { - return 0 - } - return float64(r.cResult.elapsed) -} - -func (r *LocalResult) RowsRead() uint64 { - if r.cResult == nil { - return 0 - } - return uint64(r.cResult.rows_read) -} - -func (r *LocalResult) BytesRead() uint64 { - if r.cResult == nil { - return 0 - } - return uint64(r.cResult.bytes_read) -} diff --git a/chdbstable/chdb.h b/chdbstable/chdb.h deleted file mode 100644 index 821e3e8..0000000 --- a/chdbstable/chdb.h +++ /dev/null @@ -1,42 +0,0 @@ -#pragma once - -#ifdef __cplusplus -# include -# include -extern "C" { -#else -# include -# include -#endif - -#define CHDB_EXPORT __attribute__((visibility("default"))) -struct local_result -{ - char * buf; - size_t len; - void * _vec; // std::vector *, for freeing - double elapsed; - uint64_t rows_read; - uint64_t bytes_read; -}; - -struct local_result_v2 -{ - char * buf; - size_t len; - void * _vec; // std::vector *, for freeing - double elapsed; - uint64_t rows_read; - uint64_t bytes_read; - char * error_message; -}; - -CHDB_EXPORT struct local_result * query_stable(int argc, char ** argv); -CHDB_EXPORT void free_result(struct local_result * result); - -CHDB_EXPORT struct local_result_v2 * query_stable_v2(int argc, char ** argv); -CHDB_EXPORT void free_result_v2(struct local_result_v2 * result); - -#ifdef __cplusplus -} -#endif diff --git a/chdbstable/chdb_test.go b/chdbstable/chdb_test.go deleted file mode 100644 index 1cacb84..0000000 --- a/chdbstable/chdb_test.go +++ /dev/null @@ -1,75 +0,0 @@ -package chdbstable - -import ( - "testing" -) - -// TestCase defines the structure of a test case -type TestCase struct { - name string // Name of the test case - argv []string // Arguments to pass to QueryStable - expectError bool // Whether an error is expected - expectOutput string // Expected output -} - -func TestQueryStableMultipleCases(t *testing.T) { - // Define a series of test cases - testCases := []TestCase{ - { - name: "Single Query", - argv: []string{"clickhouse", "--multiquery", "--output-format=CSV", "--query=SELECT 123;"}, - expectError: false, - expectOutput: "123\n", - }, - { - name: "Single Queries", - argv: []string{"clickhouse", "--multiquery", "--output-format=CSV", "--query=SELECT 'abc';"}, - expectError: false, - expectOutput: "\"abc\"\n", - }, - { - name: "Error Query", - argv: []string{"clickhouse", "--multiquery", "--output-format=CSV", "--query=XXX;"}, - expectError: true, - expectOutput: "", - }, - } - - // Iterate over the test cases - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - result, err := QueryStable(len(tc.argv), tc.argv) - - // Assert based on the expected outcome of the test case - if tc.expectError { - if err == nil { - t.Errorf("Expected error, but got nil") - } - } else { - if err != nil { - t.Errorf("Expected no error, but got %v", err) - } else { - if result == nil { - t.Errorf("Expected non-nil result, but got nil") - } else { - if result.cResult == nil { - t.Errorf("Expected non-nil cResult, but got nil") - } else { - if result.cResult.error_message != nil { - t.Errorf("Expected nil error_message, but got %v", result.cResult.error_message) - } else { - if result.cResult.buf == nil { - t.Errorf("Expected non-nil output, but got nil") - } else { - if tc.expectOutput != string(result.String()) { - t.Errorf("Expected output %v, but got %v", tc.expectOutput, string(result.String())) - } - } - } - } - } - } - } - }) - } -} diff --git a/go.mod b/go.mod index c97fee5..e716833 100644 --- a/go.mod +++ b/go.mod @@ -3,20 +3,17 @@ module github.com/chdb-io/chdb-go go 1.21 require ( - github.com/apache/arrow/go/v15 v15.0.2 github.com/c-bata/go-prompt v0.2.6 + github.com/ebitengine/purego v0.8.2 + github.com/huandu/go-sqlbuilder v1.27.3 github.com/parquet-go/parquet-go v0.23.0 ) require ( github.com/andybalholm/brotli v1.1.0 // indirect - github.com/goccy/go-json v0.10.3 // indirect - github.com/google/flatbuffers v24.3.25+incompatible // indirect github.com/google/uuid v1.6.0 // indirect - github.com/huandu/go-sqlbuilder v1.27.3 // indirect github.com/huandu/xstrings v1.4.0 // indirect github.com/klauspost/compress v1.17.9 // indirect - github.com/klauspost/cpuid/v2 v2.2.8 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect @@ -26,11 +23,5 @@ require ( github.com/pkg/term v1.2.0-beta.2 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/segmentio/encoding v0.4.0 // indirect - github.com/zeebo/xxh3 v1.0.2 // indirect - golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect - golang.org/x/mod v0.19.0 // indirect - golang.org/x/sync v0.7.0 // indirect golang.org/x/sys v0.22.0 // indirect - golang.org/x/tools v0.23.0 // indirect - golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9 // indirect ) diff --git a/go.sum b/go.sum index 444020a..fdca1d0 100644 --- a/go.sum +++ b/go.sum @@ -1,19 +1,16 @@ github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= -github.com/apache/arrow/go/v15 v15.0.2 h1:60IliRbiyTWCWjERBCkO1W4Qun9svcYoZrSLcyOsMLE= -github.com/apache/arrow/go/v15 v15.0.2/go.mod h1:DGXsR3ajT524njufqf95822i+KTh+yea1jass9YXgjA= github.com/c-bata/go-prompt v0.2.6 h1:POP+nrHE+DfLYx370bedwNhsqmpCUynWPxuHi0C5vZI= github.com/c-bata/go-prompt v0.2.6/go.mod h1:/LMAke8wD2FsNu9EXNdHxNLbd9MedkPnCdfpU9wwHfY= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= -github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= -github.com/google/flatbuffers v24.3.25+incompatible h1:CX395cjN9Kke9mmalRoL3d81AtFUxJM+yDthflgJGkI= -github.com/google/flatbuffers v24.3.25+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/ebitengine/purego v0.8.2 h1:jPPGWs2sZ1UgOSgD2bClL0MJIqu58nOmIcBuXr62z1I= +github.com/ebitengine/purego v0.8.2/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= +github.com/huandu/go-assert v1.1.6 h1:oaAfYxq9KNDi9qswn/6aE0EydfxSa+tWZC1KabNitYs= github.com/huandu/go-assert v1.1.6/go.mod h1:JuIfbmYG9ykwvuxoJ3V8TB5QP+3+ajIA54Y44TmkMxs= github.com/huandu/go-sqlbuilder v1.27.3 h1:cNVF9vQP4i7rTk6XXJIEeMbGkZbxfjcITeJzobJK44k= github.com/huandu/go-sqlbuilder v1.27.3/go.mod h1:mS0GAtrtW+XL6nM2/gXHRJax2RwSW1TraavWDFAc1JA= @@ -21,8 +18,6 @@ github.com/huandu/xstrings v1.4.0 h1:D17IlohoQq4UcpqD7fDk80P7l+lwAmlFaBHgOipl2FU github.com/huandu/xstrings v1.4.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= -github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= -github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= @@ -58,17 +53,7 @@ github.com/segmentio/encoding v0.4.0 h1:MEBYvRqiUB2nfR2criEXWqwdY6HJOUrCn5hboVOV github.com/segmentio/encoding v0.4.0/go.mod h1:/d03Cd8PoaDeceuhUUUQWjU0KhWjrmYrWPgtJHYZSnI= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= -github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= -github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= -github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= -golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= -golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= -golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8= -golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -77,16 +62,9 @@ golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200918174421-af09f7315aff/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg= -golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI= -golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9 h1:LLhsEBxRTBLuKlQxFBYUOU8xyFgXv6cOTp2HASDlsDk= -golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= -gonum.org/v1/gonum v0.12.0 h1:xKuo6hzt+gMav00meVPUlXwSdoEJP46BR+wdxQEFK2o= -gonum.org/v1/gonum v0.12.0/go.mod h1:73TDxJfAAHeA8Mk9mf8NlIppyhQNo5GLTcYeqgo2lvY= google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/lowApi.md b/lowApi.md index 08c019a..4829877 100644 --- a/lowApi.md +++ b/lowApi.md @@ -1,95 +1,104 @@ -# chdbstable +# chdbpurego ```go -import "github.com/chdb-io/chdb-go/chdbstable" +import "github.com/chdb-io/chdb-go/chdb-purego" ``` ## Index -- [type LocalResult](<#LocalResult>) - - [func QueryStable\(argc int, argv \[\]string\) \*LocalResult](<#QueryStable>) - - [func \(r \*LocalResult\) Buf\(\) \[\]byte](<#LocalResult.Buf>) - - [func \(r \*LocalResult\) BytesRead\(\) uint64](<#LocalResult.BytesRead>) - - [func \(r \*LocalResult\) Elapsed\(\) float64](<#LocalResult.Elapsed>) - - [func \(r \*LocalResult\) Len\(\) int](<#LocalResult.Len>) - - [func \(r \*LocalResult\) RowsRead\(\) uint64](<#LocalResult.RowsRead>) - - [func \(r LocalResult\) String\(\) string](<#LocalResult.String>) +- [type ChdbConn](<#ChdbConn>) + - [func NewConnection\(argc int, argv \[\]string\) \(ChdbConn, error\)](<#NewConnection>) +- [type ChdbResult](<#ChdbResult>) + - [func RawQuery\(argc int, argv \[\]string\) \(result ChdbResult, err error\)](<#RawQuery>) - -## type [LocalResult]() + +## type [ChdbConn]() + -LocalResult mirrors the C struct local\_result in Go. ```go -type LocalResult struct { - // contains filtered or unexported fields +type ChdbConn interface { + //Query executes the given queryStr in the underlying clickhouse connection, and output the result in the given formatStr + Query(queryStr string, formatStr string) (result ChdbResult, err error) + //Ready returns a boolean indicating if the connections is successfully established. + Ready() bool + //Close the connection and free the underlying allocated memory + Close() } ``` - -### func [QueryStable]() + +### func [NewConnection]() ```go -func QueryStable(argc int, argv []string) *LocalResult +func NewConnection(argc int, argv []string) (ChdbConn, error) ``` -QueryStable calls the C function query\_stable. - - -### func \(\*LocalResult\) [Buf]() +Session will keep the state of query. If path is None, it will create a temporary directory and use it as the database path and the temporary directory will be removed when the session is closed. You can also pass in a path to create a database at that path where will keep your data. -```go -func (r *LocalResult) Buf() []byte -``` +You can also use a connection string to pass in the path and other parameters. Examples: -Accessor methods to access fields of the local\_result struct. +- ":memory:" \(for in\-memory database\) +- "test.db" \(for relative path\) +- "file:test.db" \(same as above\) +- "/path/to/test.db" \(for absolute path\) +- "file:/path/to/test.db" \(same as above\) +- "file:test.db?param1=value1¶m2=value2" \(for relative path with query params\) +- "file::memory:?verbose&log-level=test" \(for in\-memory database with query params\) +- "///path/to/test.db?param1=value1¶m2=value2" \(for absolute path\) - -### func \(\*LocalResult\) [BytesRead]() +Connection string args handling: -```go -func (r *LocalResult) BytesRead() uint64 ``` +Connection string can contain query params like "file:test.db?param1=value1¶m2=value2" +"param1=value1" will be passed to ClickHouse engine as start up args. - - - -### func \(\*LocalResult\) [Elapsed]() - -```go -func (r *LocalResult) Elapsed() float64 +For more details, see `clickhouse local --help --verbose` +Some special args handling: +- "mode=ro" would be "--readonly=1" for clickhouse (read-only mode) ``` +Important: +- There can be only one session at a time. If you want to create a new session, you need to close the existing one. +- Creating a new session will close the existing one. - -### func \(\*LocalResult\) [Len]() - -```go -func (r *LocalResult) Len() int -``` - + +## type [ChdbResult]() - -### func \(\*LocalResult\) [RowsRead]() ```go -func (r *LocalResult) RowsRead() uint64 +type ChdbResult interface { + // Raw bytes result buffer, used for reading the result of clickhouse query + Buf() []byte + // String rapresentation of the the buffer + String() string + // Lenght in bytes of the buffer + Len() int + // Number of seconds elapsed for the query execution + Elapsed() float64 + // Amount of rows returned by the query + RowsRead() uint64 + // Amount of bytes returned by the query + BytesRead() uint64 + // If the query had any error during execution, here you can retrieve the cause. + Error() error + // Free the query result and all the allocated memory + Free() error +} ``` - - - -### func \(LocalResult\) [String]() + +### func [RawQuery]() ```go -func (r LocalResult) String() string +func RawQuery(argc int, argv []string) (result ChdbResult, err error) ``` -Stringer interface for LocalResult +RawQuery will execute the given clickouse query without using any session. Generated by [gomarkdoc]() diff --git a/main.go b/main.go index 0865e8b..b4fd461 100644 --- a/main.go +++ b/main.go @@ -39,6 +39,7 @@ data will lost after exit. If you want to keep the data, specify a path to a dir // If path is specified or no additional arguments, enter interactive mode if len(flag.Args()) == 0 { + var err error var session *chdb.Session if *pathFlag != "" {