Skip to content

Commit

Permalink
refactor: find new columns to improve write performance (#918)
Browse files Browse the repository at this point in the history
## Related Issues
Closes #

## Detailed Changes
Refactor find new columns to improve write performance.
* Skip building column schema when columns already exists.

## Test Plan 
TSBS.

## Benchmark
| | auto_create_table = false | auto_create_table = true| performance
loss |

|:--------:|:-------------------------:|:------------------------:|:-----------------:|
| Before | 171273.03 rows/s | 152436.35 rows/s | 11% |
| After | 171699.82 rows/s | 164791.73 rows/s | 4.1% |
  • Loading branch information
chunshao90 authored May 24, 2023
1 parent 391ea85 commit d90a94d
Show file tree
Hide file tree
Showing 4 changed files with 260 additions and 175 deletions.
2 changes: 1 addition & 1 deletion integration_tests/sdk/go/issue-779.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import (
)

func checkAutoAddColumnsWithCreateTable(ctx context.Context, client ceresdb.Client) error {
timestampName := "t"
timestampName := "timestamp"

err := dropTable(ctx, client)
if err != nil {
Expand Down
16 changes: 8 additions & 8 deletions integration_tests/sdk/go/util.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import (
const table = "godemo"

func createTable(ctx context.Context, client ceresdb.Client, timestampName string) error {
_, err := ddl(ctx, client, fmt.Sprintf("create table %s (%s timestamp not null, name string tag, value int64,TIMESTAMP KEY(t))", table, timestampName))
_, err := ddl(ctx, client, fmt.Sprintf("create table %s (`%s` timestamp not null, name string tag, value int64,TIMESTAMP KEY(%s))", table, timestampName, timestampName))
return err
}

Expand Down Expand Up @@ -60,9 +60,13 @@ func ensureRow(expectedVals []ceresdb.Value, actualRow []ceresdb.Column) error {
}

func query(ctx context.Context, client ceresdb.Client, ts int64, timestampName string, addNewColumn bool) error {
sql := fmt.Sprintf("select timestamp, name, value from %s where %s = %d order by name", table, timestampName, ts)
if addNewColumn {
sql = fmt.Sprintf("select timestamp, name, value, new_tag, new_field from %s where %s = %d order by name", table, timestampName, ts)
}
resp, err := client.SQLQuery(ctx, ceresdb.SQLQueryRequest{
Tables: []string{table},
SQL: fmt.Sprintf("select * from %s where %s = %d order by name", table, timestampName, ts),
SQL: sql,
})
if err != nil {
return err
Expand All @@ -73,23 +77,19 @@ func query(ctx context.Context, client ceresdb.Client, ts int64, timestampName s
}

row0 := []ceresdb.Value{
ceresdb.NewUint64Value(4024844655630594205),
ceresdb.NewInt64Value(ts),
ceresdb.NewStringValue("tag-0"),
ceresdb.NewInt64Value(0)}

row1 := []ceresdb.Value{
ceresdb.NewUint64Value(14230010170561829440),
ceresdb.NewInt64Value(ts),
ceresdb.NewStringValue("tag-1"),
ceresdb.NewInt64Value(1),
}

if addNewColumn {
row0[0] = ceresdb.NewUint64Value(8341999341185504339)
row1[0] = ceresdb.NewUint64Value(4452331151453582498)
row0 = append(row0, ceresdb.NewInt64Value(0), ceresdb.NewStringValue("new-tag-0"))
row1 = append(row1, ceresdb.NewInt64Value(1), ceresdb.NewStringValue("new-tag-1"))
row0 = append(row0, ceresdb.NewStringValue("new-tag-0"), ceresdb.NewInt64Value(0))
row1 = append(row1, ceresdb.NewStringValue("new-tag-1"), ceresdb.NewInt64Value(1))
}

if err := ensureRow(row0,
Expand Down
Loading

0 comments on commit d90a94d

Please sign in to comment.