Merge pull request tursodatabase#1660 from tursodatabase/vector-searc…

…h-fix-null vector search: fix delete of rows with NULL vector value
Shopify · Aug 12, 2024 · 26c49a8 · 26c49a8
2 parents a5db99b + c135391
commit 26c49a8
Show file tree

Hide file tree

Showing 4 changed files with 33 additions and 8 deletions.
diff --git a/libsql-ffi/bundled/SQLite3MultipleCiphers/src/sqlite3.c b/libsql-ffi/bundled/SQLite3MultipleCiphers/src/sqlite3.c
@@ -213389,7 +213389,12 @@ int diskAnnDelete(
   DiskAnnTrace(("diskAnnDelete started: rowid=%lld\n", nodeRowid));
 
   rc = blobSpotCreate(pIndex, &pNodeBlob, nodeRowid, pIndex->nBlockSize, DISKANN_BLOB_WRITABLE);
-  if( rc != SQLITE_OK ){
+  if( rc == DISKANN_ROW_NOT_FOUND ){
+    // as we omit rows with NULL values during insert, it can be the case that there is nothing to delete in the index, while row exists in the base table
+    // so, we must simply silently stop delete process as there is nothing to delete from index
+    rc = SQLITE_OK;
+    goto out;
+  }else if( rc != SQLITE_OK ){
     *pzErrMsg = sqlite3_mprintf("vector index(delete): failed to create blob for node row");
     goto out;
   }

diff --git a/libsql-ffi/bundled/src/sqlite3.c b/libsql-ffi/bundled/src/sqlite3.c
@@ -213389,7 +213389,12 @@ int diskAnnDelete(
   DiskAnnTrace(("diskAnnDelete started: rowid=%lld\n", nodeRowid));
 
   rc = blobSpotCreate(pIndex, &pNodeBlob, nodeRowid, pIndex->nBlockSize, DISKANN_BLOB_WRITABLE);
-  if( rc != SQLITE_OK ){
+  if( rc == DISKANN_ROW_NOT_FOUND ){
+    // as we omit rows with NULL values during insert, it can be the case that there is nothing to delete in the index, while row exists in the base table
+    // so, we must simply silently stop delete process as there is nothing to delete from index
+    rc = SQLITE_OK;
+    goto out;
+  }else if( rc != SQLITE_OK ){
     *pzErrMsg = sqlite3_mprintf("vector index(delete): failed to create blob for node row");
     goto out;
   }

diff --git a/libsql-sqlite3/src/vectordiskann.c b/libsql-sqlite3/src/vectordiskann.c
@@ -1633,7 +1633,12 @@ int diskAnnDelete(
   DiskAnnTrace(("diskAnnDelete started: rowid=%lld\n", nodeRowid));
 
   rc = blobSpotCreate(pIndex, &pNodeBlob, nodeRowid, pIndex->nBlockSize, DISKANN_BLOB_WRITABLE);
-  if( rc != SQLITE_OK ){
+  if( rc == DISKANN_ROW_NOT_FOUND ){
+    // as we omit rows with NULL values during insert, it can be the case that there is nothing to delete in the index, while row exists in the base table
+    // so, we must simply silently stop delete process as there is nothing to delete from index
+    rc = SQLITE_OK;
+    goto out;
+  }else if( rc != SQLITE_OK ){
     *pzErrMsg = sqlite3_mprintf("vector index(delete): failed to create blob for node row");
     goto out;
   }

diff --git a/libsql-sqlite3/test/libsql_vector_index.test b/libsql-sqlite3/test/libsql_vector_index.test
@@ -131,13 +131,23 @@ do_execsql_test vector-empty {
 
 
 do_execsql_test vector-null {
-  CREATE TABLE t_null( v FLOAT32(3));
+  CREATE TABLE t_null( v FLOAT32(2));
   CREATE INDEX t_null_idx ON t_null( libsql_vector_idx(v) );
-  INSERT INTO t_null VALUES(vector('[1,2,3]'));
+  INSERT INTO t_null VALUES(vector('[1,-1]'));
   INSERT INTO t_null VALUES(NULL);
-  INSERT INTO t_null VALUES(vector('[2,3,4]'));
-  SELECT * FROM vector_top_k('t_null_idx', '[1,2,3]', 2);
-} {1 3}
+  INSERT INTO t_null VALUES(vector('[-2,1]'));
+  SELECT * FROM vector_top_k('t_null_idx', '[1,1]', 2);
+  UPDATE t_null SET v = vector('[1,1]') WHERE rowid = 2;
+  SELECT rowid FROM vector_top_k('t_null_idx', vector('[1,1]'), 3);
+  UPDATE t_null SET v = NULL WHERE rowid = 3;
+  SELECT rowid FROM vector_top_k('t_null_idx', vector('[1,1]'), 3);
+  UPDATE t_null SET v = NULL;
+  SELECT rowid FROM vector_top_k('t_null_idx', vector('[1,1]'), 3);
+} {
+  1 3
+  2 1 3
+  2 1
+}
 
 do_execsql_test vector-sql {
   CREATE TABLE t_sql( v FLOAT32(3));