Skip to content

Commit

Permalink
Merge pull request #283 from goccy/update-zetasqlite
Browse files Browse the repository at this point in the history
Update zetasqlite to v0.19.1
  • Loading branch information
goccy authored Mar 10, 2024
2 parents 369e12b + 6a2a833 commit e5c0d87
Show file tree
Hide file tree
Showing 4 changed files with 54 additions and 31 deletions.
8 changes: 4 additions & 4 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ require (
github.com/goccy/go-json v0.10.0
github.com/goccy/go-yaml v1.9.5
github.com/goccy/go-zetasql v0.5.5
github.com/goccy/go-zetasqlite v0.19.0
github.com/goccy/go-zetasqlite v0.19.1
github.com/google/go-cmp v0.5.9
github.com/googleapis/gax-go/v2 v2.11.0
github.com/gorilla/mux v1.8.0
Expand Down Expand Up @@ -75,11 +75,11 @@ require (
go.opencensus.io v0.24.0 // indirect
go.uber.org/atomic v1.7.0 // indirect
go.uber.org/multierr v1.6.0 // indirect
golang.org/x/crypto v0.16.0 // indirect
golang.org/x/crypto v0.21.0 // indirect
golang.org/x/mod v0.10.0 // indirect
golang.org/x/net v0.19.0 // indirect
golang.org/x/net v0.21.0 // indirect
golang.org/x/oauth2 v0.11.0 // indirect
golang.org/x/sys v0.15.0 // indirect
golang.org/x/sys v0.18.0 // indirect
golang.org/x/text v0.14.0 // indirect
golang.org/x/tools v0.9.1 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
Expand Down
16 changes: 8 additions & 8 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -94,8 +94,8 @@ github.com/goccy/go-yaml v1.9.5 h1:Eh/+3uk9kLxG4koCX6lRMAPS1OaMSAi+FJcya0INdB0=
github.com/goccy/go-yaml v1.9.5/go.mod h1:U/jl18uSupI5rdI2jmuCswEA2htH9eXfferR3KfscvA=
github.com/goccy/go-zetasql v0.5.5 h1:3JpXt3p2533rnZMu09upCcI7YJ2KfjKgdo2Lu0xo+fU=
github.com/goccy/go-zetasql v0.5.5/go.mod h1:xvvooX2RG404vnbdFZuAM8bTFksYwVUlqeIUrUNuo40=
github.com/goccy/go-zetasqlite v0.19.0 h1:pvWFw/QThPP5a3UMkgirIsb9/5tlfUq25DggFfhI86o=
github.com/goccy/go-zetasqlite v0.19.0/go.mod h1:ThavIQcmI6a/sVTpvJtj+idUui32gwbnXe1jeb7pISY=
github.com/goccy/go-zetasqlite v0.19.1 h1:yebtcyHgSNnNywan6FbI93uxJ6GXDpVMoqTlfT/Bdm4=
github.com/goccy/go-zetasqlite v0.19.1/go.mod h1:ThavIQcmI6a/sVTpvJtj+idUui32gwbnXe1jeb7pISY=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
Expand Down Expand Up @@ -250,8 +250,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY=
golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20220827204233-334a2380cb91 h1:tnebWN09GYg9OLPss1KXj8txwZc6X6uMr6VFdcGNbHw=
golang.org/x/exp v0.0.0-20220827204233-334a2380cb91/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE=
Expand All @@ -278,8 +278,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c=
golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U=
golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU=
Expand Down Expand Up @@ -314,8 +314,8 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
Expand Down
59 changes: 41 additions & 18 deletions internal/contentdata/repository.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import (
"database/sql"
"fmt"
"reflect"
"sort"
"strings"

"github.com/goccy/go-zetasqlite"
Expand Down Expand Up @@ -371,34 +370,58 @@ func (r *Repository) AddTableData(ctx context.Context, tx *connection.Tx, projec
_ = tx.MetadataRepoMode()
}()

var columns []string
var columns []*types.Column
for _, col := range table.Columns {
columns = append(columns, col.Name)
}
sort.Strings(columns)
rows := make([]string, 0, len(table.Data))
values := make([]interface{}, 0, len(table.Data)*len(columns))
for _, data := range table.Data {
placeholders := make([]string, 0, len(data))
for _, col := range columns {
values = append(values, data[col])
placeholders = append(placeholders, "?")
}
rows = append(rows, fmt.Sprintf("(%s)", strings.Join(placeholders, ",")))
columns = append(columns, col)
}

placeholders := make([]string, 0, len(columns))
columnsWithEscape := make([]string, 0, len(columns))
for _, col := range columns {
columnsWithEscape = append(columnsWithEscape, fmt.Sprintf("`%s`", col))
placeholders = append(placeholders, "?")
columnsWithEscape = append(columnsWithEscape, fmt.Sprintf("`%s`", col.Name))
}

query := fmt.Sprintf(
"INSERT `%s` (%s) VALUES %s",
"INSERT `%s` (%s) VALUES (%s)",
r.tablePath(projectID, datasetID, table.ID),
strings.Join(columnsWithEscape, ","),
strings.Join(rows, ","),
strings.Join(placeholders, ","),
)
if _, err := tx.Tx().ExecContext(ctx, query, values...); err != nil {

stmt, err := tx.Tx().PrepareContext(ctx, query)
if err != nil {
return err
}

for _, data := range table.Data {
values := make([]interface{}, 0, len(table.Columns))

for _, column := range columns {
if value, found := data[column.Name]; found {
isTimestampColumn := column.Type == types.TIMESTAMP
inputString, isInputString := value.(string)

if isInputString && isTimestampColumn {
parsedTimestamp, err := zetasqlite.TimeFromTimestampValue(inputString)
// If we could parse the timestamp, use it when inserting, otherwise fallback to the supplied value
if err == nil {
values = append(values, parsedTimestamp)
continue
}
}

values = append(values, value)
} else {
values = append(values, nil)
}
}

if _, err := stmt.ExecContext(ctx, values...); err != nil {
return err
}
}

return nil
}

Expand Down
2 changes: 1 addition & 1 deletion internal/types/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -192,7 +192,7 @@ func Format(schema *bigqueryv2.TableSchema, rows []*TableRow, useInt64Timestamp
for _, row := range rows {
cells := make([]*TableCell, 0, len(row.F))
for colIdx, cell := range row.F {
if schema.Fields[colIdx].Type == "TIMESTAMP" {
if schema.Fields[colIdx].Type == "TIMESTAMP" && cell.V != nil {
t, _ := zetasqlite.TimeFromTimestampValue(cell.V.(string))
microsec := t.UnixNano() / int64(time.Microsecond)
cells = append(cells, &TableCell{
Expand Down

0 comments on commit e5c0d87

Please sign in to comment.