improve data converting for DB.DoInsert/DoUpdate

contrib/drivers/clickhouse/clickhouse.go

@@ -18,6 +18,9 @@ import (
 	"time"
 
 	"github.com/ClickHouse/clickhouse-go/v2"
+	"github.com/google/uuid"
+	"github.com/shopspring/decimal"
+
 	"github.com/gogf/gf/v2/database/gdb"
 	"github.com/gogf/gf/v2/errors/gcode"
 	"github.com/gogf/gf/v2/errors/gerror"
@@ -27,8 +30,6 @@ import (
 	"github.com/gogf/gf/v2/util/gconv"
 	"github.com/gogf/gf/v2/util/gtag"
 	"github.com/gogf/gf/v2/util/gutil"
-	"github.com/google/uuid"
-	"github.com/shopspring/decimal"
 )
 
 // Driver is the driver for postgresql database.
@@ -50,7 +51,6 @@ const (
 	filterTypePattern                = `(?i)^UPDATE|DELETE`
 	replaceSchemaPattern             = `@(.+?)/([\w\.\-]+)+`
 	needParsedSqlInCtx   gctx.StrKey = "NeedParsedSql"
-	OrmTagForStruct                  = gtag.ORM
 	driverName                       = "clickhouse"
 )
 
@@ -298,13 +298,20 @@ func (d *Driver) DoInsert(
 		keysStr      = charL + strings.Join(keys, charR+","+charL) + charR
 		holderStr    = strings.Join(valueHolder, ",")
 		tx           gdb.TX
-		stdSqlResult sql.Result
 		stmt         *gdb.Stmt
 	)
 	tx, err = d.Core.Begin(ctx)
 	if err != nil {
 		return
 	}
+	// It here uses defer to guarantee transaction be committed or roll-backed.
+	defer func() {
+		if err == nil {
+			_ = tx.Commit()
+		} else {
+			_ = tx.Rollback()
+		}
+	}()
 	stmt, err = tx.Prepare(fmt.Sprintf(
 		"INSERT INTO %s(%s) VALUES (%s)",
 		d.QuotePrefixTableName(table), keysStr,
@@ -314,22 +321,23 @@ func (d *Driver) DoInsert(
 		return
 	}
 	for i := 0; i < len(list); i++ {
-		params := make([]interface{}, 0) // Values that will be committed to underlying database driver.
+		// Values that will be committed to underlying database driver.
+		params := make([]interface{}, 0)
 		for _, k := range keys {
 			params = append(params, list[i][k])
 		}
 		// Prepare is allowed to execute only once in a transaction opened by clickhouse
-		stdSqlResult, err = stmt.ExecContext(ctx, params...)
+		result, err = stmt.ExecContext(ctx, params...)
 		if err != nil {
-			return stdSqlResult, err
+			return
 		}
 	}
-	return stdSqlResult, tx.Commit()
+	return
 }
 
 // ConvertDataForRecord converting for any data that will be inserted into table/collection as a record.
 func (d *Driver) ConvertDataForRecord(ctx context.Context, value interface{}) (map[string]interface{}, error) {
-	m := gconv.Map(value, OrmTagForStruct)
+	m := gconv.Map(value, gtag.ORM)
 
 	// transforms a value of a particular type
 	for k, v := range m {

contrib/drivers/clickhouse/clickhouse_test.go

@@ -263,6 +263,35 @@ func TestDriverClickhouse_InsertOne(t *testing.T) {
 	gtest.AssertNil(err)
 }
 
+func TestDriverClickhouse_InsertOneAutoDateTimeWrite(t *testing.T) {
+	connect, err := gdb.New(gdb.ConfigNode{
+		Host:      "127.0.0.1",
+		Port:      "9000",
+		User:      "default",
+		Name:      "default",
+		Type:      "clickhouse",
+		Debug:     false,
+		CreatedAt: "created",
+	})
+	gtest.AssertNil(err)
+	gtest.AssertNE(connect, nil)
+	gtest.AssertEQ(createClickhouseTableVisits(connect), nil)
+	defer dropClickhouseTableVisits(connect)
+	beforeInsertTime := time.Now()
+	_, err = connect.Model("visits").Data(g.Map{
+		"duration": float64(grand.Intn(999)),
+		"url":      gconv.String(grand.Intn(999)),
+	}).Insert()
+	gtest.AssertNil(err)
+	// Query the inserted data to get the time field value
+	data, err := connect.Model("visits").One()
+	gtest.AssertNil(err)
+	// Get the time value from the inserted data
+	createdTime := data["created"].Time()
+	// Assert the time field value is equal to or after the beforeInsertTime
+	gtest.AssertGE(createdTime.Unix(), beforeInsertTime.Unix())
+}
+
 func TestDriverClickhouse_InsertMany(t *testing.T) {
 	connect := clickhouseConfigDB()
 	gtest.AssertEQ(createClickhouseTableVisits(connect), nil)

contrib/drivers/dm/dm.go

@@ -15,15 +15,19 @@ import (
 	"reflect"
 	"strconv"
 	"strings"
+	"time"
 
 	_ "gitee.com/chunanyong/dm"
+
 	"github.com/gogf/gf/v2/database/gdb"
 	"github.com/gogf/gf/v2/errors/gcode"
 	"github.com/gogf/gf/v2/errors/gerror"
 	"github.com/gogf/gf/v2/frame/g"
+	"github.com/gogf/gf/v2/os/gtime"
 	"github.com/gogf/gf/v2/text/gregex"
 	"github.com/gogf/gf/v2/text/gstr"
 	"github.com/gogf/gf/v2/util/gconv"
+	"github.com/gogf/gf/v2/util/gtag"
 	"github.com/gogf/gf/v2/util/gutil"
 )
 
@@ -169,6 +173,35 @@ func (d *Driver) TableFields(
 	return fields, nil
 }
 
+// ConvertDataForRecord converting for any data that will be inserted into table/collection as a record.
+func (d *Driver) ConvertDataForRecord(ctx context.Context, value interface{}) (map[string]interface{}, error) {
+	m := gconv.Map(value, gtag.ORM)
+
+	// transforms a value of a particular type
+	for k, v := range m {
+		switch itemValue := v.(type) {
+		// dm does not support time.Time, it so here converts it to time string that it supports.
+		case time.Time:
+			m[k] = gtime.New(itemValue).String()
+			// If the time is zero, it then updates it to nil,
+			// which will insert/update the value to database as "null".
+			if itemValue.IsZero() {
+				m[k] = nil
+			}
+
+		// dm does not support time.Time, it so here converts it to time string that it supports.
+		case *time.Time:
+			m[k] = gtime.New(itemValue).String()
+			// If the time is zero, it then updates it to nil,
+			// which will insert/update the value to database as "null".
+			if itemValue == nil || itemValue.IsZero() {
+				m[k] = nil
+			}
+		}
+	}
+	return m, nil
+}
+
 // DoFilter deals with the sql string before commits it to underlying sql driver.
 func (d *Driver) DoFilter(ctx context.Context, link gdb.Link, sql string, args []interface{}) (newSql string, newArgs []interface{}, err error) {
 	defer func() {

database/gdb/gdb_core.go

@@ -684,7 +684,12 @@ func (c *Core) DoUpdate(ctx context.Context, link Link, table string, data inter
 			return nil, err
 		}
 	}
-	return c.db.DoExec(ctx, link, fmt.Sprintf("UPDATE %s SET %s%s", table, updates, condition), args...)
+	return c.db.DoExec(ctx, link, fmt.Sprintf(
+		"UPDATE %s SET %s%s",
+		table, updates, condition,
+	),
+		args...,
+	)
 }
 
 // Delete does "DELETE FROM ... " statement for the table.

database/gdb/gdb_driver_wrapper_db.go

@@ -89,3 +89,26 @@ func (d *DriverWrapperDB) TableFields(
 	}
 	return
 }
+
+// DoInsert inserts or updates data forF given table.
+// This function is usually used for custom interface definition, you do not need call it manually.
+// The parameter `data` can be type of map/gmap/struct/*struct/[]map/[]struct, etc.
+// Eg:
+// Data(g.Map{"uid": 10000, "name":"john"})
+// Data(g.Slice{g.Map{"uid": 10000, "name":"john"}, g.Map{"uid": 20000, "name":"smith"})
+//
+// The parameter `option` values are as follows:
+// 0: insert:  just insert, if there's unique/primary key in the data, it returns error;
+// 1: replace: if there's unique/primary key in the data, it deletes it from table and inserts a new one;
+// 2: save:    if there's unique/primary key in the data, it updates it or else inserts a new one;
+// 3: ignore:  if there's unique/primary key in the data, it ignores the inserting;
+func (d *DriverWrapperDB) DoInsert(ctx context.Context, link Link, table string, list List, option DoInsertOption) (result sql.Result, err error) {
+	// Convert data type before commit it to underlying db driver.
+	for i, item := range list {
+		list[i], err = d.DB.ConvertDataForRecord(ctx, item)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return d.DB.DoInsert(ctx, link, table, list, option)
+}

database/gdb/gdb_func.go

@@ -209,6 +209,10 @@ func GetInsertOperationByOption(option InsertOption) string {
 	return operator
 }
 
+func anyValueToMapBeforeToRecord(value interface{}) map[string]interface{} {
+	return gconv.Map(value, structTagPriority...)
+}
+
 // DataToMapDeep converts `value` to map type recursively(if attribute struct is embedded).
 // The parameter `value` should be type of *map/map/*struct/struct.
 // It supports embedded struct definition for struct.

database/gdb/gdb_model_insert.go

@@ -39,11 +39,7 @@ func (m *Model) Batch(batch int) *Model {
 // Data(g.Map{"uid": 10000, "name":"john"})
 // Data(g.Slice{g.Map{"uid": 10000, "name":"john"}, g.Map{"uid": 20000, "name":"smith"}).
 func (m *Model) Data(data ...interface{}) *Model {
-	var (
-		err   error
-		ctx   = m.GetCtx()
-		model = m.getModel()
-	)
+	var model = m.getModel()
 	if len(data) > 1 {
 		if s := gconv.String(data[0]); gstr.Contains(s, "?") {
 			model.data = s
@@ -88,10 +84,7 @@ func (m *Model) Data(data ...interface{}) *Model {
 				}
 				list := make(List, reflectInfo.OriginValue.Len())
 				for i := 0; i < reflectInfo.OriginValue.Len(); i++ {
-					list[i], err = m.db.ConvertDataForRecord(ctx, reflectInfo.OriginValue.Index(i).Interface())
-					if err != nil {
-						panic(err)
-					}
+					list[i] = anyValueToMapBeforeToRecord(reflectInfo.OriginValue.Index(i).Interface())
 				}
 				model.data = list
 
@@ -108,24 +101,15 @@ func (m *Model) Data(data ...interface{}) *Model {
 						list  = make(List, len(array))
 					)
 					for i := 0; i < len(array); i++ {
-						list[i], err = m.db.ConvertDataForRecord(ctx, array[i])
-						if err != nil {
-							panic(err)
-						}
+						list[i] = anyValueToMapBeforeToRecord(array[i])
 					}
 					model.data = list
 				} else {
-					model.data, err = m.db.ConvertDataForRecord(ctx, data[0])
-					if err != nil {
-						panic(err)
-					}
+					model.data = anyValueToMapBeforeToRecord(data[0])
 				}
 
 			case reflect.Map:
-				model.data, err = m.db.ConvertDataForRecord(ctx, data[0])
-				if err != nil {
-					panic(err)
-				}
+				model.data = anyValueToMapBeforeToRecord(data[0])
 
 			default:
 				model.data = data[0]
@@ -278,53 +262,34 @@ func (m *Model) doInsertWithOption(ctx context.Context, insertOption InsertOptio
 
 	case List:
 		list = value
-		for i, v := range list {
-			list[i], err = m.db.ConvertDataForRecord(ctx, v)
-			if err != nil {
-				return nil, err
-			}
-		}
 
 	case Map:
-		var listItem map[string]interface{}
-		if listItem, err = m.db.ConvertDataForRecord(ctx, value); err != nil {
-			return nil, err
-		}
-		list = List{listItem}
+		list = List{value}
 
 	default:
+		// It uses gconv.Map here to simply fo the type converting from interface{} to map[string]interface{},
+		// as there's another DataToMapDeep in next logic to do the deep converting.
 		reflectInfo := reflection.OriginValueAndKind(newData)
 		switch reflectInfo.OriginKind {
 		// If it's slice type, it then converts it to List type.
 		case reflect.Slice, reflect.Array:
 			list = make(List, reflectInfo.OriginValue.Len())
 			for i := 0; i < reflectInfo.OriginValue.Len(); i++ {
-				list[i], err = m.db.ConvertDataForRecord(ctx, reflectInfo.OriginValue.Index(i).Interface())
+				list[i] = anyValueToMapBeforeToRecord(reflectInfo.OriginValue.Index(i).Interface())
 			}
 
 		case reflect.Map:
-			var listItem map[string]interface{}
-			if listItem, err = m.db.ConvertDataForRecord(ctx, value); err != nil {
-				return nil, err
-			}
-			list = List{listItem}
+			list = List{anyValueToMapBeforeToRecord(value)}
 
 		case reflect.Struct:
 			if v, ok := value.(iInterfaces); ok {
 				array := v.Interfaces()
 				list = make(List, len(array))
 				for i := 0; i < len(array); i++ {
-					list[i], err = m.db.ConvertDataForRecord(ctx, array[i])
-					if err != nil {
-						return nil, err
-					}
+					list[i] = anyValueToMapBeforeToRecord(array[i])
 				}
 			} else {
-				var listItem map[string]interface{}
-				if listItem, err = m.db.ConvertDataForRecord(ctx, value); err != nil {
-					return nil, err
-				}
-				list = List{listItem}
+				list = List{anyValueToMapBeforeToRecord(value)}
 			}
 
 		default:

database/gdb/gdb_model_update.go

@@ -9,9 +9,10 @@ package gdb
 import (
 	"database/sql"
 	"fmt"
-	"github.com/gogf/gf/v2/internal/intlog"
 	"reflect"
 
+	"github.com/gogf/gf/v2/internal/intlog"
+
 	"github.com/gogf/gf/v2/errors/gcode"
 	"github.com/gogf/gf/v2/errors/gerror"
 	"github.com/gogf/gf/v2/internal/reflection"
@@ -57,11 +58,7 @@ func (m *Model) Update(dataAndWhere ...interface{}) (result sql.Result, err erro
 
 	switch reflectInfo.OriginKind {
 	case reflect.Map, reflect.Struct:
-		var dataMap map[string]interface{}
-		dataMap, err = m.db.ConvertDataForRecord(ctx, m.data)
-		if err != nil {
-			return nil, err
-		}
+		var dataMap = anyValueToMapBeforeToRecord(m.data)
 		// Automatically update the record updating time.
 		if fieldNameUpdate != "" {
 			dataMap[fieldNameUpdate] = gtime.Now()