Merge pull request #808 from taosdata/3.0

taos-tools 3.0 branch to main
taosdata · Oct 17, 2024 · 5a7814f · 5a7814f
2 parents 45ee09e + 905c66e
commit 5a7814f
Show file tree

Hide file tree

Showing 38 changed files with 1,774 additions and 337 deletions.
diff --git a/.github/workflows/3.0-coveralls.yml b/.github/workflows/3.0-coveralls.yml
@@ -216,7 +216,7 @@ jobs:
           ./build/bin/taosBenchmark --help > /dev/null
           echo "TEST: taosBenchmark -V"
           ./build/bin/taosBenchmark -V
-          echo "TEST: taoBenchmark -w 16375"
+          echo "TEST: taoBenchmark -w 16375 -t 100 -y"
           ./build/bin/taosBenchmark -b binary -A bool -w 16375 > /dev/null || :
           echo "TEST: taoBenchmark -D NotANumber"
           ./build/bin/taosBenchmark -D NotANumber > /dev/null || :

diff --git a/.github/workflows/3.0-taosbenchmark-debug.yml b/.github/workflows/3.0-taosbenchmark-debug.yml
@@ -5,12 +5,10 @@ on:
     - cron: "10 16 * * *"
   push:
     branches:
-      - develop
       - 3.0
       - main
   pull_request:
     branches:
-      - develop
       - 3.0
       - main
 
@@ -122,7 +120,7 @@ jobs:
           sudo apt install libgflags2.2 libgflags-dev -y > /dev/null
           cd TDengine && mkdir debug && cd debug \
             && cmake .. -DBUILD_HTTP=false -DWEBSOCKET=true > /dev/null \
-            && make -j2 > /dev/null && sudo make install > /dev/null \
+            && make -j8 > /dev/null && sudo make install > /dev/null \
             && cd .. && rm -rf debug
           if [[ ! -f /usr/local/taos/bin/taosd ]] || [[ ! -f /usr/local/taos/bin/taosadapter ]]
           then

diff --git a/.github/workflows/3.0-taosbenchmark-release.yml b/.github/workflows/3.0-taosbenchmark-release.yml
@@ -5,12 +5,10 @@ on:
     - cron: "10 16 * * *"
   push:
     branches:
-      - develop
       - 3.0
       - main
   pull_request:
     branches:
-      - develop
       - 3.0
       - main
 
@@ -122,7 +120,7 @@ jobs:
           sudo apt install libgflags2.2 libgflags-dev -y > /dev/null
           cd TDengine && mkdir debug && cd debug \
             && cmake .. -DBUILD_HTTP=false -DWEBSOCKET=true > /dev/null \
-            && make -j2 > /dev/null && sudo make install > /dev/null \
+            && make -j8 > /dev/null && sudo make install > /dev/null \
             && cd .. && rm -rf debug
           if [[ ! -f /usr/local/taos/bin/taosd ]] || [[ ! -f /usr/local/taos/bin/taosadapter ]]
           then
@@ -296,3 +294,4 @@ jobs:
           elif [ -f ~/taosbenchmark-ws-success.txt ]; then
             cat ~/taosbenchmark-ws-success.txt
           fi
+
diff --git a/.github/workflows/3.0-windows-build.yml b/.github/workflows/3.0-windows-build.yml
@@ -93,7 +93,7 @@ jobs:
         run: |
           cl.exe
           cd TDengine/debug
-          cmake .. -G "NMake Makefiles JOM" -DBUILD_TOOLS=true && jom /j 2
+          cmake .. -G "NMake Makefiles JOM" -DBUILD_TOOLS=true && jom /j 8
 
       - name: Checkout TDengine's taos-tools to PR number
         if: |
@@ -120,6 +120,10 @@ jobs:
         with:
           toolchain: stable
 
+      - name : Delete TDengine/debug folder
+        run: |
+          Get-ChildItem -Path "TDengine/debug" -Recurse -Force | Remove-Item -Force -Recurse
+
       - uses: ilammy/msvc-dev-cmd@v1
 
       - name: Build & Install TDengine
@@ -131,7 +135,7 @@ jobs:
         run: |
           cl.exe
           cd TDengine/debug
-          cmake .. -G "NMake Makefiles JOM" -DBUILD_TOOLS=true && jom /j 2 && jom install
+          cmake .. -G "NMake Makefiles JOM" -DBUILD_TOOLS=true && jom /j 8 && jom install
           #cmake .. -G "NMake Makefiles" -DBUILD_HTTP=false -DBUILD_TOOLS=true -DWEBSOCKET=true && nmake && nmake install
 
           #      - name: taosdump codec Test

diff --git a/case/insertStmt2.json b/case/insertStmt2.json
@@ -0,0 +1,50 @@
+{
+    "filetype": "insert",
+    "cfgdir": "/etc/taos",
+    "host": "127.0.0.1",
+    "port": 6030,
+    "user": "root",
+    "password": "taosdata",
+    "num_of_records_per_req": 3,
+    "prepared_rand": 10,
+    "thread_count": 1,
+    "thread_bind_vgroup": "yes",
+    "create_table_thread_count": 1,
+    "confirm_parameter_prompt": "no",
+    "databases": [
+        {
+            "dbinfo": {
+                "name": "dbstmt2",
+                "drop": "yes",
+                "vgroups": 1
+            },
+            "super_tables": [
+                {
+                    "name": "meters",
+                    "child_table_exists": "no",
+                    "childtable_count": 10,
+                    "insert_rows": 20,
+                    "interlace_rows": 0,
+                    "childtable_prefix": "d",
+                    "insert_mode": "stmt2",
+                    "timestamp_step": 1000,
+                    "start_timestamp":1700000000000,
+                    "auto_create_table": "yes",
+                    "columns": [
+                        {"type": "FLOAT", "name": "current", "count": 1, "max": 12, "min": 8 },
+                        { "type": "INT", "name": "voltage", "max": 225, "min": 215 },
+                        { "type": "FLOAT", "name": "phase", "max": 1, "min": 0 }
+                    ],
+                    "tags": [
+                        {"type": "tinyint", "name": "groupid","max": 10,"min": 1},
+                        {"type": "binary",  "name": "location", "len": 16,
+                            "values": ["San Francisco", "Los Angles", "San Diego",
+                                "San Jose", "Palo Alto", "Campbell", "Mountain View",
+                                "Sunnyvale", "Santa Clara", "Cupertino"]
+                        }
+                    ]
+                }
+            ]
+        }
+    ]
+}
diff --git a/example/insertDemo.json b/example/insertDemo.json
@@ -45,8 +45,8 @@
                     "non_stop_mode": "no",
                     "line_protocol": "line",
                     "insert_rows": 100000,
-                    "childtable_limit": 10,
-                    "childtable_offset": 100,
+                    "childtable_limit": 0,
+                    "childtable_offset": 0,
                     "interlace_rows": 0,
                     "insert_interval": 0,
                     "partial_col_num": 0,

diff --git a/example/insert_csv.json b/example/insert_csv.json
@@ -35,8 +35,8 @@
             "data_source": "sample",
             "insert_mode": "taosc",
             "line_protocol": "line",
-            "childtable_limit": -10,
-            "childtable_offset": 10,
+            "childtable_limit": 0,
+            "childtable_offset": 0,
             "insert_rows": 10000,
             "insert_interval": 0,
             "interlace_rows": 0,

diff --git a/example/insert_math.json b/example/insert_math.json
@@ -36,8 +36,8 @@
                     "non_stop_mode": "no",
                     "line_protocol": "line",
                     "insert_rows": 1000,
-                    "childtable_limit": 10,
-                    "childtable_offset": 100,
+                    "childtable_limit": 0,
+                    "childtable_offset": 0,
                     "interlace_rows": 0,
                     "insert_interval": 0,
                     "partial_col_num": 0,

diff --git a/inc/bench.h b/inc/bench.h
@@ -138,9 +138,12 @@ typedef unsigned __int32 uint32_t;
 #define BOOL_BUFF_LEN       6
 #define FLOAT_BUFF_LEN      22
 #define DOUBLE_BUFF_LEN     42
-#define JSON_BUFF_LEN       20
 #define TIMESTAMP_BUFF_LEN  21
 #define PRINT_STAT_INTERVAL 30 * 1000
+#define DEFAULT_HOST        "localhost"
+
+// json tag type fixed length
+#define JSON_FIXED_LENGTH   4095
 
 #define MAX_QUERY_SQL_COUNT 100
 
@@ -161,8 +164,8 @@ typedef unsigned __int32 uint32_t;
 #define DEFAULT_BINWIDTH       64
 #define DEFAULT_REPLICA        1
 #define DEFAULT_CFGNAME_LEN    10
-#define DEFAULT_PREPARED_RAND  10000
-#define DEFAULT_REQ_PER_REQ    30000
+#define DEFAULT_PREPARED_RAND  20000
+#define DEFAULT_REQ_PER_REQ    10000
 #define DEFAULT_INSERT_ROWS    10000
 #define DEFAULT_DISORDER_RANGE 1000
 #define DEFAULT_CREATE_BATCH   10
@@ -191,7 +194,7 @@ typedef unsigned __int32 uint32_t;
 #define BENCH_PORT                \
     "The TCP/IP port number to use for the connection, default is 6030."
 #define BENCH_MODE                \
-    "insert mode, default is taosc, options: taosc|rest|stmt|sml"
+    "insert mode, default is taosc, options: taosc|rest|stmt|stmt2|sml"
 #define BENCH_USER                \
     "The user name to use when connecting to the server, default is root."
 #define BENCH_PASS                \
@@ -466,6 +469,13 @@ typedef unsigned __int32 uint32_t;
         }                                                                   \
     } while (0)
 
+
+#define IS_VAR_DATA_TYPE(t)                                                                                 \
+  (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_VARBINARY) || ((t) == TSDB_DATA_TYPE_NCHAR) || \
+   ((t) == TSDB_DATA_TYPE_JSON) || ((t) == TSDB_DATA_TYPE_GEOMETRY))
+
+
+
 enum TEST_MODE {
     INSERT_TEST,     // 0
     QUERY_TEST,      // 1
@@ -479,6 +489,7 @@ enum enum_TAOS_INTERFACE {
     TAOSC_IFACE,
     REST_IFACE,
     STMT_IFACE,
+    STMT2_IFACE,
     SML_IFACE,
     SML_REST_IFACE,
     INTERFACE_BUT
@@ -550,6 +561,7 @@ static const int OFF_CAP     = -1;
 typedef struct SStmtData {
     void    *data;
     char    *is_null;
+    int32_t *lengths;
 } StmtData;
 
 typedef struct SChildField {
@@ -977,6 +989,7 @@ typedef struct SBenchConn {
     TAOS* taos;
     TAOS* ctaos;  // check taos
     TAOS_STMT* stmt;
+    TAOS_STMT2* stmt2;
 #ifdef WEBSOCKET
     WS_TAOS* taos_ws;
     WS_STMT* stmt_ws;
@@ -1041,6 +1054,9 @@ typedef struct SThreadInfo_S {
     char        *csql;
     int32_t     clen;  // csql current write position
     bool        stmtBind;
+
+    // stmt2
+    BArray      *tagsStmt;
 } threadInfo;
 
 typedef struct SQueryThreadInfo_S {
@@ -1119,7 +1135,9 @@ void* benchArrayPush(BArray* pArray, void* pData);
 void* benchArrayDestroy(BArray* pArray);
 void benchArrayClear(BArray* pArray);
 void* benchArrayGet(const BArray* pArray, size_t index);
-void* benchArrayAddBatch(BArray* pArray, void* pData, int32_t elems);
+void* benchArrayAddBatch(BArray* pArray, void* pData, int32_t elems, bool free);
+BArray * copyBArray(BArray *pArray);
+bool searchBArray(BArray *pArray, const char *field_name, int32_t name_len, uint8_t field_type);
 
 #ifdef LINUX
 int32_t bsem_wait(sem_t* sem);
@@ -1185,5 +1203,17 @@ float tmpFloatImpl(Field *field, int i, int32_t angle, int32_t k);
 double tmpDoubleImpl(Field *field, int32_t angle, int32_t k);
 int tmpStr(char *tmp, int iface, Field *field, int64_t k);
 int tmpGeometry(char *tmp, int iface, Field *field, int64_t k);
-
+int tmpInt32ImplTag(Field *field, int i, int k);
+
+char* genQMark( int32_t QCnt);
+// stmt2
+TAOS_STMT2_BINDV* createBindV(int32_t count, int32_t tagCnt, int32_t colCnt);
+// clear bindv table count tables tag and column
+void resetBindV(TAOS_STMT2_BINDV *bindv, int32_t capacity, int32_t tagCnt, int32_t colCnt);
+void clearBindV(TAOS_STMT2_BINDV *bindv);
+void freeBindV(TAOS_STMT2_BINDV *bindv);
+void showBindV(TAOS_STMT2_BINDV *bindv, BArray *tags, BArray *cols);
+
+// IFace is rest return True
+bool isRest(int32_t iface);
 #endif   // INC_BENCH_H_
diff --git a/inc/benchData.h b/inc/benchData.h
@@ -28,7 +28,10 @@ int generateRandData(SSuperTable *stbInfo, char *sampleDataBuf,
         int64_t bufLen,
         int lenOfOneRow, BArray * fields, int64_t loop,
         bool tag, BArray *childCols);
-int prepareStmt(SSuperTable *stbInfo, TAOS_STMT *stmt, char* tagData, uint64_t tableSeq);
+// prepare
+int prepareStmt (TAOS_STMT  *stmt,  SSuperTable *stbInfo, char* tagData, uint64_t tableSeq);
+int prepareStmt2(TAOS_STMT2 *stmt2, SSuperTable *stbInfo, char* tagData, uint64_t tableSeq);
+
 uint32_t bindParamBatch(threadInfo *pThreadInfo,
         uint32_t batch, int64_t startTime, int64_t pos,
         SChildTable *childTbl, int32_t *pkCur, int32_t *pkCnt, int32_t *n, int64_t *delay2, int64_t *delay3);
@@ -51,7 +54,30 @@ void generateSmlJsonValues(
         char **sml_tags_json_array, SSuperTable *stbInfo, int tableSeq);
 
 // generateTag data from random or csv file, cnt is get count for each
-bool generateTagData(SSuperTable *stbInfo, char *buf, int64_t cnt, FILE* csv);
+bool generateTagData(SSuperTable *stbInfo, char *buf, int64_t cnt, FILE* csv, BArray* tagsStmt);
 // get tag from csv file
 FILE* openTagCsv(SSuperTable* stbInfo);
+
+//
+// STMT2 bind cols param progressive
+//
+uint32_t bindVTags(TAOS_STMT2_BINDV *bindv, int32_t tbIndex, int32_t w, BArray* fields);
+
+uint32_t bindVCols(TAOS_STMT2_BINDV *bindv, int32_t tbIndex,
+                 threadInfo *pThreadInfo,
+                 uint32_t batch, int64_t startTime, int64_t pos,
+                 SChildTable *childTbl, int32_t *pkCur, int32_t *pkCnt, int32_t *n);
+
+
+uint32_t bindVColsInterlace(TAOS_STMT2_BINDV *bindv, int32_t tbIndex,
+                 threadInfo *pThreadInfo,
+                 uint32_t batch, int64_t startTime, int64_t pos,
+                 SChildTable *childTbl, int32_t *pkCur, int32_t *pkCnt, int32_t *n);
+
+
+uint32_t bindVColsProgressive(TAOS_STMT2_BINDV *bindv, int32_t tbIndex,
+                 threadInfo *pThreadInfo,
+                 uint32_t batch, int64_t startTime, int64_t pos,
+                 SChildTable *childTbl, int32_t *pkCur, int32_t *pkCnt, int32_t *n);
+
 #endif  // INC_BENCHDATA_H_
diff --git a/src/benchCommandOpt.c b/src/benchCommandOpt.c
@@ -60,7 +60,7 @@ void parseFieldDatatype(char *dataType, BArray *fields, bool isTag) {
         benchArrayPush(fields, field);
         field = benchArrayGet(fields, 0);
         if (1 == regexMatch(dataType,
-                    "^(BINARY|NCHAR|JSON)(\\([1-9][0-9]*\\))$",
+                    "^(BINARY|NCHAR|GEOMETRY|VARBINARY|VARCHAR)(\\([1-9][0-9]*\\))$",
                     REG_ICASE | REG_EXTENDED)) {
             char type[DATATYPE_BUFF_LEN];
             char length[BIGINT_BUFF_LEN];
@@ -182,7 +182,7 @@ static void initStable() {
     tstrncpy(t2->name, "location", TSDB_COL_NAME_LEN + 1);
 
     t1->min = 1;
-    t1->max = 10;
+    t1->max = 100000;
 
 
     stbInfo->insert_interval = 0;
@@ -293,7 +293,7 @@ void modifyArgument() {
 #else
             taos_options(TSDB_OPTION_CONFIGDIR, g_configDir);
 #endif
-            g_arguments->host = NULL;
+            g_arguments->host = DEFAULT_HOST;
             g_arguments->port = 0;
         }
 #ifdef WEBSOCKET
@@ -346,18 +346,6 @@ void modifyArgument() {
         superTable->keep_trying = g_arguments->keep_trying;
         superTable->trying_interval = g_arguments->trying_interval;
     }
-
-    if (REST_IFACE == g_arguments->iface) {
-        if (0 != convertServAddr(g_arguments->iface,
-                                 false,
-                                 1)) {
-            errorPrint("%s", "Failed to convert server address\n");
-            return;
-        }
-        encodeAuthBase64();
-        g_arguments->rest_server_ver_major =
-            getServerVersionRest(g_arguments->port);
-    }
 }
 
 static void *queryStableAggrFunc(void *sarg) {