Support complex type such as map(string, array(map(string, float64)))

[gangtao]

following query can be successfully running on proton but failed to run in neutron.

with t1 as (
SELECT 
  '1' as granularity, to_float64(_tp_time) as ts , product_id as symbol, cast((['ts', 'price'], [ts, price]), 'map(string, float64)') AS value
FROM 
  btc_usd_coinbase
),
t2 as (
  select symbol, granularity, group_array(value) AS value_array from t1 group by symbol, granularity order by symbol, granularity
),
t3 as (
  select symbol, group_array(granularity) as granularities, group_array(value_array) as values from t2 group by symbol
)
select symbol, cast((granularities, values), 'map(string, array(map(string, float64)))') from t3

neutron error is

proton: unsupported column type "map(string, array(map(string, float64)))".

I think it is because go-driver does not support this complex type as of now, which is required .

[ye11ow]

The error is from https://github.com/timeplus-io/proton-go-driver/blob/v2.0.15/lib/column/column.go#L67

[leo-cai-timeplus]

clickhouse support this

package main

import (
	"context"
	"fmt"
	"github.com/ClickHouse/clickhouse-go/v2"
	"log"
	"time"
)

func example() error {
	var (
		ctx       = context.Background()
		conn, err = clickhouse.Open(&clickhouse.Options{
			Addr: []string{"127.0.0.1:9000"},
			Auth: clickhouse.Auth{
				Database: "default",
				Username: "default",
				Password: "",
			},
			//Debug:           true,
			DialTimeout:     time.Second,
			MaxOpenConns:    10,
			MaxIdleConns:    5,
			ConnMaxLifetime: time.Hour,
		})
	)
	if err != nil {
		return err
	}
	if err := conn.Exec(ctx, "set allow_experimental_map_type = 1;"); err != nil {
		return err
	}
	if err := conn.Exec(ctx, `DROP TABLE IF EXISTS table_map;`); err != nil {
		return err
	}
	err = conn.Exec(ctx, `
		CREATE TABLE table_map (a Map(String, Array(Map(String, Float64)))) ENGINE = Memory;
	`)
	if err != nil {
		return err
	}

	batch, err := conn.PrepareBatch(ctx, "INSERT INTO table_map")
	if err != nil {
		return err
	}
	for i := 0; i < 100; i++ {
		err := batch.Append(
			map[string][]map[string]float64{
				"key1": {
					{"sub_key1": 10.9, "sub_key2": float64(i)},
					{"sub_key3": float64(i) + 0.1},
				},
			},
		)
		if err != nil {
			return err
		}
	}
	err = batch.Send()
	if err != nil {
		return err
	}
	rows, err := conn.Query(ctx, "SELECT * FROM table_map")
	for rows.Next() {
		a := &map[string][]map[string]float64{}
		err := rows.Scan(a)
		if err != nil {
			return err
		}
		fmt.Println(a)
	}
	if err != nil {
		return err
	}
	return err
}

func main() {
	start := time.Now()
	if err := example(); err != nil {
		log.Fatal(err)
	}
	fmt.Println(time.Since(start))
}

[ye11ow]

Yeah, we were too behind the offical clickhouse go driver. It has been years since we rebased. This can be an issue

[jovezhong]

(Jove Github Bot) added it to the current sprint.

[jovezhong]

(Jove Github Bot) moved this ticket out of the GitHub project(up to 1200 tickets for one project).