support append mode

Database.go

@@ -137,7 +137,10 @@ func string2interface(s []string) []interface{} {
 }
 
 func GetRemoteDatabases() (dbsettings []Model_DBSetting) {
-	sql := "SELECT * FROM tenant_master where imported=''"
+	sql := "SELECT * FROM tenant_master where isactive=1"
+	if RunMode == "append" {
+		sql += " and imported ='' "
+	}
 	// logrus.Fatal("Ping", sql, localdb)
 
 	res, err := localdb.Query(sql)

ImportDB.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"reflect"
 	"strings"
+	"time"
 
 	"github.com/sirupsen/logrus"
 )
@@ -65,6 +66,7 @@ func ImportData(dbsetting Model_DBSetting, tables []string) (err error) {
 	db, err := ConnectDB(dbsetting)
 	internaldb, _ := ConnectDB(localdbsetting)
 	//loop through all table
+	// tables1 := []string{"acc_payment"}
 	for _, tablename := range tables {
 		// get all data
 
@@ -80,7 +82,9 @@ func ImportData(dbsetting Model_DBSetting, tables []string) (err error) {
 			logrus.Fatal(err)
 		}
 	}
+	currentTime := time.Now()
 
+	internaldb.Exec("Update tenant_master set imported= ? WHERE tenant_id=?", dbsetting.Tenant_id, currentTime.Format("2006.01.02 15:04:05"))
 	return
 }
 
@@ -136,19 +140,22 @@ func InsertRecord(tenant_id string, internaldb *sql.DB, tablename string, rows *
 						value = "0"
 					}
 					tmp += ", " + value
-				} else { // else if fieldtype == "date" {
-					// 	if value == "" {
-					// 		value = "0000-00-00"
-					// 	}
-					// 	tmp += ", '" + value + "'"
-
-					// } else if fieldtype == "datetime" {
-					// 	if value == "" {
-					// 		value = "0000-00-00 00:00:00"
-					// 	}
-					// 	tmp += ", '" + value + "'"
-
-					// }else {
+				} else if fieldtype == "date" {
+					if value == "" {
+						value = "0000-00-00"
+					}
+					tmp += ", '" + value + "'"
+
+				} else if fieldtype == "datetime" {
+					if value == "" {
+						value = "0000-00-00 00:00:00"
+					}
+					if f == "document_trackupdated" && value != "0000-00-00 00:00:00" {
+						logrus.Warn(tenant_id, "=> ", row[50], " => document_trackupdated: ", value)
+					}
+					tmp += ", '" + value + "'"
+
+				} else {
 					tmp += ", '" + MysqlRealEscapeString(value) + "'"
 				}
 

README.md

@@ -1,5 +1,47 @@
 # project goal
-easily consolidate different database into single multitenant database
+easily consolidate different mysql/mariadb database into single multitenant mysql/mariadb database
+
+# how to use:
+[First time]
+1. Download binary `consolidatedb-linux.bin` from https://github.com/SIMITGROUP/consolidatedb/releases into you linux server
+2. create .env with below content (same directory as consolidatedb-linux.bin):
+```
+dbname=db0
+dbhost=127.0.0.1
+dbuser=dbuser
+dbpass=dbpassword
+```
+2. Create table as below at database `db0` (or any)
+```sql
+CREATE TABLE `tenant_master` (
+  `tenant_id` int(11) NOT NULL,
+  `tenant_name` varchar(50) DEFAULT NULL,
+  `host` varchar(50) DEFAULT NULL,
+  `db` varchar(50) DEFAULT NULL,
+  `user` varchar(50) DEFAULT NULL,
+  `pass` varchar(100) DEFAULT NULL,
+  `description` text DEFAULT NULL,
+  `imported` varchar(50) DEFAULT NULL,
+  `isactive` int(11) DEFAULT NULL,
+  PRIMARY KEY (`tenant_id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
+```
+3. Insert all targeted database database settings into table `tenant_master`
+    - isactive shall define 1 else it wont effect
+
+4. Run command line `consolidatedb-linux.bin --mode=init` for first time. it will 
+    - copy schema from 1st tenant into db0 (no indexes, no primary key)
+    - every table added additional column `tenant_id`
+    - and merge all database content stated in tenant_master into db0, with condition:
+        * imported empty
+        * isactive = 1
+    - job completed, all tenant record imported and column `imported` filled in current date/time
+
+[2nd times onwards]
+1. add more tenant record into `tenant_master`
+2 Run command line `consolidatedb-linux.bin --mode=append`
+
+
 
 
 # process

main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"database/sql"
+	"flag"
 	"fmt"
 	"log"
 	"math/big"
@@ -27,11 +28,22 @@ var Delimiter = ','
 var mapfields = make(map[string]*hashmap.Map)
 var mapfieldstr = make(map[string]string)
 var excludedtables = []string{"tenant_master", "gps_event", "system_event"}
+var RunMode = ""
 
-const MAX_CONCURRENT_JOBS = 4
+// const MAX_CONCURRENT_JOBS = 4
 
 func main() {
 	// sql_tablelist := "SELECT table_name FROM INFORMATION_SCHEMA.TABLES WHERE table_type = 'BASE TABLE' and table_schema=?"
+
+	flag.StringVar(&RunMode, "mode", "", "Run mode: init/append/addindex")
+	flag.Parse()
+
+	if RunMode == "append" || RunMode == "init" {
+		//ok
+	} else {
+		logrus.Fatal("mode '", RunMode, "' is not supported. Please add flat --mode=init/append")
+	}
+
 	var wg sync.WaitGroup
 	err := godotenv.Load()
 	start := time.Now()
@@ -49,7 +61,7 @@ func main() {
 	localdbsetting.User = localdbuser
 	localdbsetting.Pass = localdbpass
 
-	fmt.Println("Welcome mysql db consoler:")
+	fmt.Println("Welcome mysql db consoler:", RunMode)
 
 	// CreateFolderIfNotExists(datafolder)
 
@@ -58,21 +70,21 @@ func main() {
 	defer localdb.Close()
 
 	if err == nil {
-		fmt.Println("connected")
+		logrus.Info(localdbname, " connected")
 
-		localtables := GetAllTables(localdb, localdbname) // GetLocalTables()
+		// localtables := GetAllTables(localdb, localdbname) // GetLocalTables()
 
-		// names := lo.Uniq[string]([]string{"Samuel", "John", "Samuel"})
-		tablecount := len(localtables)
-		logrus.Info("table count at local:", tablecount)
+		// // names := lo.Uniq[string]([]string{"Samuel", "John", "Samuel"})
+		// tablecount := len(localtables)
+		// logrus.Info("table count at local:", tablecount)
 
-		if tablecount == 0 { //
-			logrus.Fatal(localdbname + " does not have table tenant_master")
-		}
+		// if tablecount == 0 { //
+		// 	logrus.Fatal(localdbname + " does not have table tenant_master")
+		// }
 
-		if tablecount == 1 {
-			//run create tables
-		}
+		// if tablecount == 1 {
+		// 	//run create tables
+		// }
 		// logrus.Fatal("GetRemoteDatabases")
 		dbsettings := GetRemoteDatabases()
 
@@ -145,6 +157,11 @@ func GenerateTables(dbsetting Model_DBSetting) (tables []string) {
 	},
 	{}]
 	*/
+	if RunMode != "init" {
+		logrus.Info("Skip generate table schemes")
+		return
+	}
+	// tables1 := []string{"acc_payment"}
 	for _, tablename := range tables {
 		//drop local table if exists
 		dropsql := "DROP TABLE IF EXISTS " + tablename