api: introduce keep_null option

Introduce keep_null option to return ffi's NULL instead of nil
in conn:execute().

Co-authored-by: Maxim Galaganov <voltage@lanport.net>

We can specify how lua_mysql_push_value() works with data == NULL and
use this fact to simplify the code.

Author: romanhabibov

README.md

@@ -91,6 +91,8 @@ Connect to a database.
  - `db` - database name
  - `use_numeric_result` - provide result of the "conn:execute" as ordered list
    (true/false); default value: false
+ - `keep_null` - provide printing null fields in the result of the
+   "conn:execute" (true/false); default value: false
 
 Throws an error on failure.
 
@@ -142,11 +144,26 @@ Throws an error on failure.
 ```
 
 *Example*:
+
+(when `keep_null = false` or is not set on a pool/connection creation)
+
+```
+tarantool> conn:execute("SELECT ? AS a, 'xx' AS b", NULL AS c , 42)
+---
+- - - a: 42
+      b: xx
+- true
+...
+```
+
+(when `keep_null = true` on a pool/connection creation)
+
 ```
-tarantool> conn:execute("SELECT ? AS a, 'xx' AS b", 42)
+tarantool> conn:execute("SELECT ? AS a, 'xx' AS b", NULL AS c, 42)
 ---
 - - - a: 42
       b: xx
+      c: null
 - true
 ...
 ```
@@ -222,6 +239,8 @@ Create a connection pool with count of size established connections.
  - `size` - count of connections in pool
  - `use_numeric_result` - provide result of the "conn:execute" as ordered list
    (true/false); default value: false
+ - `keep_null` - provide printing null fields in the result of the
+   "conn:execute" (true/false); default value: false
 
 Throws an error on failure.
 

mysql/driver.c

@@ -42,9 +42,23 @@
 #define TIMEOUT_INFINITY 365 * 86400 * 100.0
 static const char mysql_driver_label[] = "__tnt_mysql_driver";
 
+static int luaL_nil_ref = LUA_REFNIL;
+
+/**
+ * Push ffi's NULL (cdata<void *>: NULL) onto the stack.
+ * Can be used as replacement of nil in Lua tables.
+ * @param L stack
+ */
+static inline void
+luaL_pushnull(struct lua_State *L)
+{
+	lua_rawgeti(L, LUA_REGISTRYINDEX, luaL_nil_ref);
+}
+
 struct mysql_connection {
 	MYSQL *raw_conn;
 	int use_numeric_result;
+	int keep_null;
 };
 
 /*
@@ -169,11 +183,23 @@ lua_mysql_field_type_to_string(enum enum_field_types type)
 	return mysql_field_type_strs[hash];
 }
 
-/* Push value retrieved from mysql field to lua stack */
+/**
+ * Push value retrieved from mysql field to lua stack.
+ *
+ * When `data` is NULL, `field` and len` parameters are
+ * ignored and Lua nil or LuaJIT FFI NULL is pushed.
+ */
 static void
-lua_mysql_push_value(struct lua_State *L, MYSQL_FIELD *field,
-	void *data, unsigned long len)
+lua_mysql_push_value(struct lua_State *L, MYSQL_FIELD *field, void *data,
+		     unsigned long len, int keep_null)
 {
+	/*
+	 * Field type isn't MYSQL_TYPE_NULL actually in case of
+	 * Lua's nil passed as value.
+	 * Example: 'conn:execute('SELECT ? AS x', nil)'.
+	 */
+	if (data == NULL)
+		field->type = MYSQL_TYPE_NULL;
 	switch (field->type) {
 		case MYSQL_TYPE_TINY:
 		case MYSQL_TYPE_SHORT:
@@ -188,7 +214,10 @@ lua_mysql_push_value(struct lua_State *L, MYSQL_FIELD *field,
 		}
 
 		case MYSQL_TYPE_NULL:
-			lua_pushnil(L);
+			if (keep_null == 1)
+				luaL_pushnull(L);
+			else
+				lua_pushnil(L);
 			break;
 
 		case MYSQL_TYPE_LONGLONG: {
@@ -239,10 +268,8 @@ lua_mysql_fetch_result(struct lua_State *L)
 		unsigned long *len = mysql_fetch_lengths(result);
 		unsigned col_no;
 		for (col_no = 0; col_no < num_fields; ++col_no) {
-			if (!row[col_no])
-				continue;
-			lua_mysql_push_value(L, fields + col_no,
-					     row[col_no], len[col_no]);
+			lua_mysql_push_value(L, fields + col_no, row[col_no],
+					     len[col_no], conn->keep_null);
 			if (conn->use_numeric_result) {
 				/* Assign to a column number. */
 				lua_rawseti(L, -2, col_no + 1);
@@ -350,16 +377,16 @@ lua_mysql_stmt_push_row(struct lua_State *L)
 	unsigned long col_count = lua_tonumber(L, 1);
 	MYSQL_BIND *results = (MYSQL_BIND *)lua_topointer(L, 2);
 	MYSQL_FIELD *fields = (MYSQL_FIELD *)lua_topointer(L, 3);
+	int keep_null = lua_tointeger(L, 4);
 
 	lua_newtable(L);
 	unsigned col_no;
 	for (col_no = 0; col_no < col_count; ++col_no) {
-		if (*results[col_no].is_null)
-			continue;
+		void *data = *results[col_no].is_null ? NULL :
+			results[col_no].buffer;
 		lua_pushstring(L, fields[col_no].name);
-		lua_mysql_push_value(L, fields + col_no,
-				     results[col_no].buffer,
-				     *results[col_no].length);
+		lua_mysql_push_value(L, fields + col_no, data,
+				     *results[col_no].length, keep_null);
 		lua_settable(L, -3);
 	}
 	return 1;
@@ -371,7 +398,7 @@ lua_mysql_stmt_push_row(struct lua_State *L)
 static int
 lua_mysql_execute_prepared(struct lua_State *L)
 {
-	MYSQL *raw_conn = lua_check_mysqlconn(L, 1)->raw_conn;
+	struct mysql_connection *conn = lua_check_mysqlconn(L, 1);
 	size_t len;
 	const char *sql = lua_tolstring(L, 2, &len);
 	int ret_count = 0, fail = 0, error = 0;
@@ -389,7 +416,7 @@ lua_mysql_execute_prepared(struct lua_State *L)
 	lua_pushnumber(L, 0);
 	lua_newtable(L);
 	ret_count = 2;
-	stmt = mysql_stmt_init(raw_conn);
+	stmt = mysql_stmt_init(conn->raw_conn);
 	if ((error = !stmt))
 		goto done;
 	error = mysql_stmt_prepare(stmt, sql, len);
@@ -467,7 +494,8 @@ lua_mysql_execute_prepared(struct lua_State *L)
 		lua_pushnumber(L, col_count);
 		lua_pushlightuserdata(L, result_binds);
 		lua_pushlightuserdata(L, fields);
-		if ((fail = lua_pcall(L, 3, 1, 0)))
+		lua_pushinteger(L, conn->keep_null);
+		if ((fail = lua_pcall(L, 4, 1, 0)))
 			goto done;
 		lua_settable(L, -3);
 		++row_idx;
@@ -476,7 +504,7 @@ lua_mysql_execute_prepared(struct lua_State *L)
 
 done:
 	if (error)
-		ret_count = lua_mysql_push_error(L, raw_conn);
+		ret_count = lua_mysql_push_error(L, conn->raw_conn);
 	if (values)
 		free(values);
 	if (param_binds)
@@ -592,9 +620,9 @@ mysql_wait_for_io(my_socket socket, my_bool is_read, int timeout)
 static int
 lua_mysql_connect(struct lua_State *L)
 {
-	if (lua_gettop(L) < 6) {
+	if (lua_gettop(L) < 7) {
 		luaL_error(L, "Usage: mysql.connect(host, port, user, "
-			   "password, db, use_numeric_result)");
+			   "password, db, use_numeric_result, keep_null)");
 	}
 
 	const char *host = lua_tostring(L, 1);
@@ -603,6 +631,7 @@ lua_mysql_connect(struct lua_State *L)
 	const char *pass = lua_tostring(L, 4);
 	const char *db = lua_tostring(L, 5);
 	const int use_numeric_result = lua_toboolean(L, 6);
+	const int keep_null = lua_toboolean(L, 7);
 
 	MYSQL *raw_conn, *tmp_raw_conn = mysql_init(NULL);
 	if (!tmp_raw_conn) {
@@ -650,6 +679,7 @@ lua_mysql_connect(struct lua_State *L)
 	*conn_p = conn;
 	(*conn_p)->raw_conn = raw_conn;
 	(*conn_p)->use_numeric_result = use_numeric_result;
+	(*conn_p)->keep_null = keep_null;
 	luaL_getmetatable(L, mysql_driver_label);
 	lua_setmetatable(L, -2);
 
@@ -679,6 +709,10 @@ luaopen_mysql_driver(lua_State *L)
 	if (mysql_library_init(0, NULL, NULL))
 		luaL_error(L, "Failed to initialize mysql library");
 
+	/* Create NULL constant. */
+	*(void **) luaL_pushcdata(L, luaL_ctypeid(L, "void *")) = NULL;
+	luaL_nil_ref = luaL_ref(L, LUA_REGISTRYINDEX);
+
 	static const struct luaL_Reg methods [] = {
 		{"execute_prepared", lua_mysql_execute_prepared},
 		{"execute",	lua_mysql_execute},

mysql/init.lua

@@ -36,7 +36,8 @@ local function conn_get(pool, timeout)
         local status
         status, mysql_conn = driver.connect(pool.host, pool.port or 0,
                                             pool.user, pool.pass,
-                                            pool.db, pool.use_numeric_result)
+                                            pool.db, pool.use_numeric_result,
+                                            pool.keep_null)
         if status < 0 then
             error(mysql_conn)
         end
@@ -165,7 +166,8 @@ local function pool_create(opts)
     for i = 1, opts.size do
         local status, conn = driver.connect(opts.host, opts.port or 0,
                                             opts.user, opts.password,
-                                            opts.db, opts.use_numeric_result)
+                                            opts.db, opts.use_numeric_result,
+                                            opts.keep_null)
         if status < 0 then
             while queue:count() > 0 do
                 local mysql_conn = queue:get()
@@ -185,6 +187,7 @@ local function pool_create(opts)
         db          = opts.db,
         size        = opts.size,
         use_numeric_result = opts.use_numeric_result,
+        keep_null   = opts.keep_null,
 
         -- private variables
         queue       = queue,
@@ -244,7 +247,8 @@ local function connect(opts)
 
     local status, mysql_conn = driver.connect(opts.host, opts.port or 0,
                                               opts.user, opts.password,
-                                              opts.db, opts.use_numeric_result)
+                                              opts.db, opts.use_numeric_result,
+                                              opts.keep_null)
     if status < 0 then
         error(mysql_conn)
     end

test/mysql.test.lua

@@ -516,8 +516,32 @@ local function test_underlying_conn_closed_during_gc(test)
     test:ok(ffi.C.fcntl(handle, F_GETFD) == -1, 'descriptor is closed')
 end
 
+local function test_ffi_null_printing(test, pool)
+    test:plan(4)
+    local function json_result(keep_null, prepared)
+        local conn, err = mysql.connect({host = host, port = port, user = user,
+            password = password, db = db, keep_null = keep_null})
+        if conn == nil then error(err) end
+        local rows
+        if prepared then
+            rows = conn:execute('SELECT 1 AS w, ? AS x', nil)
+        else
+            rows = conn:execute('SELECT 1 AS w, NULL AS x')
+        end
+        return json.encode(rows)
+    end
+    local res = json_result(true, true)
+    test:ok(res == '[[{"x":null,"w":1}]]', 'execute_prepared keep_null enabled')
+    res = json_result(false, true)
+    test:ok(res == '[[{"w":1}]]', 'execute_prepared keep_null disabled')
+    res = json_result(true, false)
+    test:ok(res == '[[{"x":null,"w":1}]]', 'execute keep_null enabled')
+    res = json_result(false, false)
+    test:ok(res == '[[{"w":1}]]', 'execute keep_null disabled')
+end
+
 local test = tap.test('mysql connector')
-test:plan(8)
+test:plan(9)
 
 test:test('connection old api', test_old_api, conn)
 local pool_conn = p:get()
@@ -529,6 +553,7 @@ test:test('connection pool', test_connection_pool, p)
 test:test('connection reset', test_connection_reset, p)
 test:test('test_underlying_conn_closed_during_gc',
           test_underlying_conn_closed_during_gc, p)
+test:test('ffi null printing', test_ffi_null_printing, p)
 p:close()
 
 os.exit(test:check() and 0 or 1)