Move futures inside executor to avoid races

.gitignore

@@ -24,4 +24,5 @@ image.png
 scratch/
 
 *-test.sh
-*.hdf5
+*.hdf5
+*.jsonl

integration/test_batch_v4.py

@@ -596,8 +596,8 @@ def batch_insert(batch: BatchClient) -> None:
     with concurrent.futures.ThreadPoolExecutor() as executor:
         with client.batch.dynamic() as batch:
             futures = [executor.submit(batch_insert, batch) for _ in range(nr_threads)]
-    for future in concurrent.futures.as_completed(futures):
-        future.result()
+        for future in concurrent.futures.as_completed(futures):
+            future.result()
     objs = client.collections.get(name).query.fetch_objects(limit=nr_objects * nr_threads).objects
     assert len(objs) == nr_objects * nr_threads
 

profiling/test_sphere.py

@@ -9,7 +9,7 @@
 
 
 def test_sphere(collection_factory: CollectionFactory) -> None:
-    sphere_file = get_file_path("sphere.100k.jsonl")
+    sphere_file = get_file_path("sphere.1m.jsonl")
 
     collection = collection_factory(
         properties=[
@@ -26,7 +26,7 @@ def test_sphere(collection_factory: CollectionFactory) -> None:
     )
     start = time.time()
 
-    import_objects = 50000
+    import_objects = 1000000
     with collection.batch.dynamic() as batch:
         with open(sphere_file) as jsonl_file:
             for i, jsonl in enumerate(jsonl_file):
@@ -45,7 +45,7 @@ def test_sphere(collection_factory: CollectionFactory) -> None:
                     vector=json_parsed["vector"],
                 )
                 if i % 1000 == 0:
-                    print(f"Imported {i} objects")
+                    print(f"Imported {len(collection)} objects")
     assert len(collection.batch.failed_objects) == 0
     assert len(collection) == import_objects
     print(f"Imported {import_objects} objects in {time.time() - start}")