Merge pull request #299 from weaviate/feat/add-vector

Support adding named vectors to a collection with existing named vectors.

Author: bevzzz

.github/workflows/main.yaml

@@ -14,6 +14,7 @@ env:
   WEAVIATE_128: 1.28.11
   WEAVIATE_129: 1.29.1
   WEAVIATE_130: 1.30.1
+  WEAVIATE_131: 1.31.0
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}

src/collections/config/index.ts

@@ -4,11 +4,13 @@ import { WeaviateShardStatus } from '../../openapi/types.js';
 import ClassUpdater from '../../schema/classUpdater.js';
 import { ClassGetter, PropertyCreator, ShardUpdater } from '../../schema/index.js';
 import ShardsGetter from '../../schema/shardsGetter.js';
+import VectorAdder from '../../schema/vectorAdder.js';
 import { DbVersionSupport } from '../../utils/dbVersion.js';
 import {
   PropertyConfigCreate,
   ReferenceMultiTargetConfigCreate,
   ReferenceSingleTargetConfigCreate,
+  VectorizersConfigAdd,
 } from '../configure/types/index.js';
 import { MergeWithExisting } from './classes.js';
 import {
@@ -23,7 +25,7 @@ import {
   VectorIndexConfigFlat,
   VectorIndexConfigHNSW,
 } from './types/index.js';
-import { classToCollection, resolveProperty, resolveReference } from './utils.js';
+import { classToCollection, makeVectorsConfig, resolveProperty, resolveReference } from './utils.js';
 
 const config = <T>(
   connection: Connection,
@@ -47,6 +49,11 @@ const config = <T>(
         .withProperty(resolveReference<any>(reference))
         .do()
         .then(() => {}),
+    addVector: async (vectors: VectorizersConfigAdd<T>) => {
+      const supportsDynamicVectorIndex = await dbVersionSupport.supportsDynamicVectorIndex();
+      const { vectorsConfig } = makeVectorsConfig(vectors, supportsDynamicVectorIndex);
+      return new VectorAdder(connection).withClassName(name).withVectors(vectorsConfig).do();
+    },
     get: () => getRaw().then(classToCollection<T>),
     getShards: () => {
       let builder = new ShardsGetter(connection).withClassName(name);
@@ -114,6 +121,17 @@ export interface Config<T> {
   addReference: (
     reference: ReferenceSingleTargetConfigCreate<T> | ReferenceMultiTargetConfigCreate<T>
   ) => Promise<void>;
+  /**
+   * Add one or more named vectors to the collection in Weaviate.
+   * Named vectors can be added to collections with existing named vectors only.
+   *
+   * Existing named vectors are immutable in Weaviate. The client will not include
+   * any of those in the request.
+   *
+   * @param {VectorizersConfigAdd<any>} vectors Vector configurations.
+   * @returns {Promise<void>} A promise that resolves when the named vector has been created.
+   */
+  addVector: (vectors: VectorizersConfigAdd<T>) => Promise<void>;
   /**
    * Get the configuration for this collection from Weaviate.
    *

src/collections/config/integration.test.ts

@@ -1,4 +1,5 @@
 /* eslint-disable @typescript-eslint/no-non-null-assertion */
+import { requireAtLeast } from '../../../test/version.js';
 import { WeaviateUnsupportedFeatureError } from '../../errors.js';
 import weaviate, { WeaviateClient, weaviateV2 } from '../../index.js';
 import {
@@ -386,6 +387,43 @@ describe('Testing of the collection.config namespace', () => {
     ]);
   });
 
+  requireAtLeast(
+    1,
+    31,
+    0
+  )('Mutable named vectors', () => {
+    it('should be able to add named vectors to a collection', async () => {
+      const collectionName = 'TestCollectionConfigAddVector' as const;
+      const collection = await client.collections.create({
+        name: collectionName,
+        vectorizers: weaviate.configure.vectorizer.none(),
+      });
+      // Add a single named vector
+      await collection.config.addVector(weaviate.configure.vectorizer.none({ name: 'vector-a' }));
+
+      // Add several named vectors
+      await collection.config.addVector([
+        weaviate.configure.vectorizer.none({ name: 'vector-b' }),
+        weaviate.configure.vectorizer.none({ name: 'vector-c' }),
+      ]);
+
+      // Trying to update 'default' vector -- should be omitted from request.
+      await collection.config.addVector(
+        weaviate.configure.vectorizer.none({
+          name: 'default',
+          vectorIndexConfig: weaviate.configure.vectorIndex.flat(),
+        })
+      );
+
+      const config = await collection.config.get();
+      expect(config.vectorizers).toHaveProperty('vector-a');
+      expect(config.vectorizers).toHaveProperty('vector-b');
+      expect(config.vectorizers).toHaveProperty('vector-c');
+
+      expect(config.vectorizers.default).toHaveProperty('indexType', 'hnsw');
+    });
+  });
+
   it('should get the shards of a sharded collection', async () => {
     const shards = await client.collections
       .create({

src/collections/config/utils.ts

@@ -1,5 +1,10 @@
-import { WeaviateDeserializationError } from '../../errors.js';
 import {
+  WeaviateDeserializationError,
+  WeaviateInvalidInputError,
+  WeaviateUnsupportedFeatureError,
+} from '../../errors.js';
+import {
+  Properties,
   WeaviateBM25Config,
   WeaviateClass,
   WeaviateInvertedIndexConfig,
@@ -13,11 +18,19 @@ import {
   WeaviateVectorIndexConfig,
   WeaviateVectorsConfig,
 } from '../../openapi/types.js';
+import { DbVersionSupport } from '../../utils/dbVersion.js';
+import { QuantizerGuards } from '../configure/parsing.js';
 import {
   PropertyConfigCreate,
   ReferenceConfigCreate,
   ReferenceMultiTargetConfigCreate,
   ReferenceSingleTargetConfigCreate,
+  VectorIndexConfigCreate,
+  VectorIndexConfigDynamicCreate,
+  VectorIndexConfigFlatCreate,
+  VectorIndexConfigHNSWCreate,
+  VectorizersConfigAdd,
+  VectorizersConfigCreate,
 } from '../configure/types/index.js';
 import {
   BQConfig,
@@ -46,6 +59,7 @@ import {
   VectorIndexConfigHNSW,
   VectorIndexConfigType,
   VectorIndexFilterStrategy,
+  VectorIndexType,
   VectorizerConfig,
 } from './types/index.js';
 
@@ -123,6 +137,91 @@ export const classToCollection = <T>(cls: WeaviateClass): CollectionConfig => {
   };
 };
 
+export const parseVectorIndex = (module: ModuleConfig<VectorIndexType, VectorIndexConfigCreate>): any => {
+  if (module.config === undefined) return undefined;
+  if (module.name === 'dynamic') {
+    const { hnsw, flat, ...conf } = module.config as VectorIndexConfigDynamicCreate;
+    return {
+      ...conf,
+      hnsw: parseVectorIndex({ name: 'hnsw', config: hnsw }),
+      flat: parseVectorIndex({ name: 'flat', config: flat }),
+    };
+  }
+  const { quantizer, ...conf } = module.config as
+    | VectorIndexConfigFlatCreate
+    | VectorIndexConfigHNSWCreate
+    | Record<string, any>;
+  if (quantizer === undefined) return conf;
+  if (QuantizerGuards.isBQCreate(quantizer)) {
+    const { type, ...quant } = quantizer;
+    return {
+      ...conf,
+      bq: {
+        ...quant,
+        enabled: true,
+      },
+    };
+  }
+  if (QuantizerGuards.isPQCreate(quantizer)) {
+    const { type, ...quant } = quantizer;
+    return {
+      ...conf,
+      pq: {
+        ...quant,
+        enabled: true,
+      },
+    };
+  }
+};
+
+export const parseVectorizerConfig = (config?: VectorizerConfig): any => {
+  if (config === undefined) return {};
+  const { vectorizeCollectionName, ...rest } = config as any;
+  return {
+    ...rest,
+    vectorizeClassName: vectorizeCollectionName,
+  };
+};
+
+export const makeVectorsConfig = <TProperties extends Properties | undefined = undefined>(
+  configVectorizers: VectorizersConfigCreate<TProperties> | VectorizersConfigAdd<TProperties>,
+  supportsDynamicVectorIndex: Awaited<ReturnType<DbVersionSupport['supportsDynamicVectorIndex']>>
+) => {
+  let vectorizers: string[] = [];
+  const vectorsConfig: Record<string, any> = {};
+  const vectorizersConfig = Array.isArray(configVectorizers)
+    ? configVectorizers
+    : [
+        {
+          ...configVectorizers,
+          name: configVectorizers.name || 'default',
+        },
+      ];
+  vectorizersConfig.forEach((v) => {
+    if (v.vectorIndex.name === 'dynamic' && !supportsDynamicVectorIndex.supports) {
+      throw new WeaviateUnsupportedFeatureError(supportsDynamicVectorIndex.message);
+    }
+    const vectorConfig: any = {
+      vectorIndexConfig: parseVectorIndex(v.vectorIndex),
+      vectorIndexType: v.vectorIndex.name,
+      vectorizer: {},
+    };
+    const vectorizer = v.vectorizer.name === 'text2vec-azure-openai' ? 'text2vec-openai' : v.vectorizer.name;
+    vectorizers = [...vectorizers, vectorizer];
+    vectorConfig.vectorizer[vectorizer] = {
+      properties: v.properties,
+      ...parseVectorizerConfig(v.vectorizer.config),
+    };
+    if (v.name === undefined) {
+      throw new WeaviateInvalidInputError(
+        'vectorName is required for each vectorizer when specifying more than one vectorizer'
+      );
+    }
+    vectorsConfig[v.name] = vectorConfig;
+  });
+  return { vectorsConfig, vectorizers };
+};
+
 function populated<T>(v: T | null | undefined): v is T {
   return v !== undefined && v !== null;
 }

src/collections/configure/types/vectorizer.ts

@@ -55,7 +55,11 @@ export type VectorConfigUpdate<N extends string | undefined, I extends VectorInd
 };
 
 export type VectorizersConfigCreate<T> =
-  | VectorConfigCreate<PrimitiveKeys<T>, undefined, VectorIndexType, Vectorizer>
+  | VectorConfigCreate<PrimitiveKeys<T>, string | undefined, VectorIndexType, Vectorizer>
+  | VectorConfigCreate<PrimitiveKeys<T>, string, VectorIndexType, Vectorizer>[];
+
+export type VectorizersConfigAdd<T> =
+  | VectorConfigCreate<PrimitiveKeys<T>, string, VectorIndexType, Vectorizer>
   | VectorConfigCreate<PrimitiveKeys<T>, string, VectorIndexType, Vectorizer>[];
 
 export type ConfigureNonTextVectorizerOptions<