From 12b170174c97bcf792ae995731c478ee721de3cf Mon Sep 17 00:00:00 2001 From: ashishshinde Date: Thu, 21 Dec 2023 17:13:45 +0530 Subject: [PATCH 1/5] Client changes for gRPC proto --- proto/transact.proto | 5 ++ proto/types.proto | 86 ++++++++++++++++++++--- src/aerospike_vector/auth_pb2.py | 5 +- src/aerospike_vector/conversions.py | 13 ++-- src/aerospike_vector/index_pb2.py | 5 +- src/aerospike_vector/transact_pb2.py | 17 ++--- src/aerospike_vector/types_pb2.py | 91 ++++++++++++++----------- src/aerospike_vector/vector_db_pb2.py | 13 ++-- src/aerospike_vector/vectordb_admin.py | 48 +++++++------ src/aerospike_vector/vectordb_client.py | 5 +- 10 files changed, 191 insertions(+), 97 deletions(-) diff --git a/proto/transact.proto b/proto/transact.proto index d5507fdc..b771ef80 100644 --- a/proto/transact.proto +++ b/proto/transact.proto @@ -53,6 +53,11 @@ message VectorSearchRequest { // The bin selector. BinSelector binSelector = 4; + + // Optional parameters to tune the search. + oneof searchParams { + HnswSearchParams hnswSearchParams = 5; + } } // Record transaction services. diff --git a/proto/types.proto b/proto/types.proto index 7298b0f1..d2f8808b 100644 --- a/proto/types.proto +++ b/proto/types.proto @@ -126,7 +126,6 @@ message RecordWithKey { } // Unique identifier for an index. -// TODO: Include optional set name? message IndexId { // The name of the index. string name = 1; @@ -147,26 +146,97 @@ enum IndexType { HNSW = 0; } +// Params for the HNSW index +message HnswParams { + // Maximum number bi-directional links per HNSW vertex. Greater values of + // 'm' in general provide better recall for data with high dimensionality, while + // lower values work well for data with lower dimensionality. + // The storage space required for the index increases proportionally with 'm'. + // The default value is 16. + optional uint32 m = 1; + + // The number of candidate nearest neighbors shortlisted during index creation. + // Larger values provide better recall at the cost of longer index update times. + // The default is 100. + optional uint32 efConstruction = 2; + + // The default number of candidate nearest neighbors shortlisted during search. + // Larger values provide better recall at the cost of longer search times. + // The default is 100. + optional uint32 ef = 3; + + // Configures batching behaviour for batch based index update. + HnswBatchingParams batchingParams = 4; +} + +// Params for the HNSW index search +message HnswSearchParams { + // The default number of candidate nearest neighbors shortlisted during search. + // Larger values provide better recall at the cost of longer search times. + // The default is value set in HnswParams for the index. + optional uint32 ef = 1; +} + +// Configures batching behaviour for batch based index update. +message HnswBatchingParams { + // Maximum number of records to fit in a batch. + // The default value is 10000. + optional uint32 maxRecords = 1; + + // The maximum amount of time in milliseconds to wait before finalizing a batch. + // The default value is 10000. + optional uint32 interval = 2; + + // Disables batching for index updates. + // Default is false meaning batching is enabled. + optional bool disabled = 3; +} + +// Index in Aerospike storage configuration +message AerospikeIndexStorage { + // Optional Aerospike namespace where the index is stored. + // Defaults to the index namespace. + optional string namespace = 8; + + // Optional Aerospike set where the index is stored. + // Defaults to the index name. + optional string set = 9; +} + // An index definition. message IndexDefinition { // The index identifier. IndexId id = 1; + // The type of index. IndexType type = 2; - VectorDistanceMetric vectorDistanceMetric = 3; + // Number of dimensions in data. + // Vectors not matching the dimension count will not be indexed. + uint32 dimensions = 3; - // Optional Aerospike set name. - optional string set = 4; + // Optional The distance metric to use. Defaults to SQUARED_EUCLIDEAN + VectorDistanceMetric vectorDistanceMetric = 4; // Indexed vector bin. string bin = 5; - // Number of dimensions in data. - // Vectors not matching the dimension count will not be indexed. - uint32 dimensions = 6; + // Optional filter on Aerospike set name from which records will be indexed. + // If not specified all sets in the index namespace will be indexed. + optional string setFilter = 6; + + // Index parameters. + oneof params { + HnswParams hnswParams = 7; + } + + // Index storage. + oneof storage { + AerospikeIndexStorage aerospikeStorage = 8; + } - map params = 7; + // Optional labels associated with the index. + map labels = 9; } // List of index definitions. diff --git a/src/aerospike_vector/auth_pb2.py b/src/aerospike_vector/auth_pb2.py index 5cea9dd3..73c24b51 100644 --- a/src/aerospike_vector/auth_pb2.py +++ b/src/aerospike_vector/auth_pb2.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: auth.proto +# Protobuf Python Version: 4.25.0 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool @@ -19,8 +20,8 @@ _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'auth_pb2', _globals) if _descriptor._USE_C_DESCRIPTORS == False: - DESCRIPTOR._options = None - DESCRIPTOR._serialized_options = b'\n\033com.aerospike.vector.clientP\001Z\034aerospike.com/vector/protos/' + _globals['DESCRIPTOR']._options = None + _globals['DESCRIPTOR']._serialized_options = b'\n\033com.aerospike.vector.clientP\001Z\034aerospike.com/vector/protos/' _globals['_AEROSPIKEAUTHREQUEST']._serialized_start=32 _globals['_AEROSPIKEAUTHREQUEST']._serialized_end=90 _globals['_AEROSPIKEAUTHRESPONSE']._serialized_start=92 diff --git a/src/aerospike_vector/conversions.py b/src/aerospike_vector/conversions.py index adf112a2..3d725e76 100644 --- a/src/aerospike_vector/conversions.py +++ b/src/aerospike_vector/conversions.py @@ -18,21 +18,25 @@ def toVectorDbValue(value: Any) -> types_pb2.Value: if isinstance(value[0], float): return types_pb2.Value( vectorValue=types_pb2.Vector( - floatArray={"value": [float(x) for x in value]})) + floatData={"value": [float(x) for x in value]})) elif isinstance(value[0], bool): return types_pb2.Value( vectorValue=types_pb2.Vector( - boolArray={"value": [True if x else False for x in value]})) + boolData={"value": [True if x else False for x in value]})) else: return types_pb2.Value( - listValue=types_pb2.List(entries = [toVectorDbValue(x) for x in value])) + listValue=types_pb2.List( + entries=[toVectorDbValue(x) for x in value])) elif isinstance(value, dict): d = types_pb2.Value( - mapValue=types_pb2.Map(entries = [types_pb2.MapEntry(key=toMapKey(k), value=toVectorDbValue(v)) for k,v in value.items()])) + mapValue=types_pb2.Map(entries=[ + types_pb2.MapEntry(key=toMapKey(k), value=toVectorDbValue(v)) + for k, v in value.items()])) return d else: raise Exception("Invalid type " + str(type(value))) + def toMapKey(value): if isinstance(value, str): return types_pb2.MapKey(stringValue=value) @@ -45,6 +49,7 @@ def toMapKey(value): else: raise Exception("Invalid map key type " + str(type(value))) + def fromVectorDbKey(key: types_pb2.Key) -> types.Key: keyValue = None if key.HasField("stringValue"): diff --git a/src/aerospike_vector/index_pb2.py b/src/aerospike_vector/index_pb2.py index bf5eacec..8a9ad0b9 100644 --- a/src/aerospike_vector/index_pb2.py +++ b/src/aerospike_vector/index_pb2.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: index.proto +# Protobuf Python Version: 4.25.0 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool @@ -21,8 +22,8 @@ _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'index_pb2', _globals) if _descriptor._USE_C_DESCRIPTORS == False: - DESCRIPTOR._options = None - DESCRIPTOR._serialized_options = b'\n\033com.aerospike.vector.clientP\001Z\034aerospike.com/vector/protos/' + _globals['DESCRIPTOR']._options = None + _globals['DESCRIPTOR']._serialized_options = b'\n\033com.aerospike.vector.clientP\001Z\034aerospike.com/vector/protos/' _globals['_INDEXSTATUSRESPONSE']._serialized_start=75 _globals['_INDEXSTATUSRESPONSE']._serialized_end=125 _globals['_INDEXSERVICE']._serialized_start=128 diff --git a/src/aerospike_vector/transact_pb2.py b/src/aerospike_vector/transact_pb2.py index 7df08891..2f9dd8ab 100644 --- a/src/aerospike_vector/transact_pb2.py +++ b/src/aerospike_vector/transact_pb2.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: transact.proto +# Protobuf Python Version: 4.25.0 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool @@ -15,16 +16,16 @@ from . import types_pb2 as types__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0etransact.proto\x12\x10\x61\x65rospike.vector\x1a\x1bgoogle/protobuf/empty.proto\x1a\x0btypes.proto\"U\n\nPutRequest\x12\"\n\x03key\x18\x01 \x01(\x0b\x32\x15.aerospike.vector.Key\x12#\n\x04\x62ins\x18\x03 \x03(\x0b\x32\x15.aerospike.vector.Bin\"d\n\nGetRequest\x12\"\n\x03key\x18\x01 \x01(\x0b\x32\x15.aerospike.vector.Key\x12\x32\n\x0b\x62inSelector\x18\x02 \x01(\x0b\x32\x1d.aerospike.vector.BinSelector\"P\n\x0b\x42inSelector\x12/\n\x04type\x18\x01 \x01(\x0e\x32!.aerospike.vector.BinSelectorType\x12\x10\n\x08\x62inNames\x18\x02 \x03(\t\"\xb1\x01\n\x13VectorSearchRequest\x12(\n\x05index\x18\x01 \x01(\x0b\x32\x19.aerospike.vector.IndexId\x12-\n\x0bqueryVector\x18\x02 \x01(\x0b\x32\x18.aerospike.vector.Vector\x12\r\n\x05limit\x18\x03 \x01(\r\x12\x32\n\x0b\x62inSelector\x18\x04 \x01(\x0b\x32\x1d.aerospike.vector.BinSelector*3\n\x0f\x42inSelectorType\x12\x07\n\x03\x41LL\x10\x00\x12\x08\n\x04NONE\x10\x01\x12\r\n\tSPECIFIED\x10\x02\x32\xa4\x02\n\x08Transact\x12=\n\x03Put\x12\x1c.aerospike.vector.PutRequest\x1a\x16.google.protobuf.Empty\"\x00\x12?\n\x03Get\x12\x1c.aerospike.vector.GetRequest\x1a\x18.aerospike.vector.Record\"\x00\x12<\n\x06\x45xists\x12\x15.aerospike.vector.Key\x1a\x19.aerospike.vector.Boolean\"\x00\x12Z\n\x0cVectorSearch\x12%.aerospike.vector.VectorSearchRequest\x1a\x1f.aerospike.vector.RecordWithKey\"\x00\x30\x01\x42=\n\x1b\x63om.aerospike.vector.clientP\x01Z\x1c\x61\x65rospike.com/vector/protos/b\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0etransact.proto\x12\x10\x61\x65rospike.vector\x1a\x1bgoogle/protobuf/empty.proto\x1a\x0btypes.proto\"U\n\nPutRequest\x12\"\n\x03key\x18\x01 \x01(\x0b\x32\x15.aerospike.vector.Key\x12#\n\x04\x62ins\x18\x03 \x03(\x0b\x32\x15.aerospike.vector.Bin\"d\n\nGetRequest\x12\"\n\x03key\x18\x01 \x01(\x0b\x32\x15.aerospike.vector.Key\x12\x32\n\x0b\x62inSelector\x18\x02 \x01(\x0b\x32\x1d.aerospike.vector.BinSelector\"P\n\x0b\x42inSelector\x12/\n\x04type\x18\x01 \x01(\x0e\x32!.aerospike.vector.BinSelectorType\x12\x10\n\x08\x62inNames\x18\x02 \x03(\t\"\x81\x02\n\x13VectorSearchRequest\x12(\n\x05index\x18\x01 \x01(\x0b\x32\x19.aerospike.vector.IndexId\x12-\n\x0bqueryVector\x18\x02 \x01(\x0b\x32\x18.aerospike.vector.Vector\x12\r\n\x05limit\x18\x03 \x01(\r\x12\x32\n\x0b\x62inSelector\x18\x04 \x01(\x0b\x32\x1d.aerospike.vector.BinSelector\x12>\n\x10hnswSearchParams\x18\x05 \x01(\x0b\x32\".aerospike.vector.HnswSearchParamsH\x00\x42\x0e\n\x0csearchParams*3\n\x0f\x42inSelectorType\x12\x07\n\x03\x41LL\x10\x00\x12\x08\n\x04NONE\x10\x01\x12\r\n\tSPECIFIED\x10\x02\x32\xa4\x02\n\x08Transact\x12=\n\x03Put\x12\x1c.aerospike.vector.PutRequest\x1a\x16.google.protobuf.Empty\"\x00\x12?\n\x03Get\x12\x1c.aerospike.vector.GetRequest\x1a\x18.aerospike.vector.Record\"\x00\x12<\n\x06\x45xists\x12\x15.aerospike.vector.Key\x1a\x19.aerospike.vector.Boolean\"\x00\x12Z\n\x0cVectorSearch\x12%.aerospike.vector.VectorSearchRequest\x1a\x1f.aerospike.vector.RecordWithKey\"\x00\x30\x01\x42=\n\x1b\x63om.aerospike.vector.clientP\x01Z\x1c\x61\x65rospike.com/vector/protos/b\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'transact_pb2', _globals) if _descriptor._USE_C_DESCRIPTORS == False: - DESCRIPTOR._options = None - DESCRIPTOR._serialized_options = b'\n\033com.aerospike.vector.clientP\001Z\034aerospike.com/vector/protos/' - _globals['_BINSELECTORTYPE']._serialized_start=529 - _globals['_BINSELECTORTYPE']._serialized_end=580 + _globals['DESCRIPTOR']._options = None + _globals['DESCRIPTOR']._serialized_options = b'\n\033com.aerospike.vector.clientP\001Z\034aerospike.com/vector/protos/' + _globals['_BINSELECTORTYPE']._serialized_start=609 + _globals['_BINSELECTORTYPE']._serialized_end=660 _globals['_PUTREQUEST']._serialized_start=78 _globals['_PUTREQUEST']._serialized_end=163 _globals['_GETREQUEST']._serialized_start=165 @@ -32,7 +33,7 @@ _globals['_BINSELECTOR']._serialized_start=267 _globals['_BINSELECTOR']._serialized_end=347 _globals['_VECTORSEARCHREQUEST']._serialized_start=350 - _globals['_VECTORSEARCHREQUEST']._serialized_end=527 - _globals['_TRANSACT']._serialized_start=583 - _globals['_TRANSACT']._serialized_end=875 + _globals['_VECTORSEARCHREQUEST']._serialized_end=607 + _globals['_TRANSACT']._serialized_start=663 + _globals['_TRANSACT']._serialized_end=955 # @@protoc_insertion_point(module_scope) diff --git a/src/aerospike_vector/types_pb2.py b/src/aerospike_vector/types_pb2.py index a4f47f00..20f5eba9 100644 --- a/src/aerospike_vector/types_pb2.py +++ b/src/aerospike_vector/types_pb2.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: types.proto +# Protobuf Python Version: 4.25.0 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool @@ -13,52 +14,60 @@ -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0btypes.proto\x12\x10\x61\x65rospike.vector\"\xa1\x01\n\x03Key\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x10\n\x03set\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x0e\n\x06\x64igest\x18\x03 \x01(\x0c\x12\x15\n\x0bstringValue\x18\x04 \x01(\tH\x00\x12\x14\n\nbytesValue\x18\x05 \x01(\x0cH\x00\x12\x12\n\x08intValue\x18\x06 \x01(\x05H\x00\x12\x13\n\tlongValue\x18\x07 \x01(\x03H\x00\x42\x07\n\x05valueB\x06\n\x04_set\"\x1a\n\tBoolArray\x12\r\n\x05value\x18\x01 \x03(\x08\"\x1b\n\nFloatArray\x12\r\n\x05value\x18\x01 \x03(\x02\"\x94\x01\n\x06MapKey\x12\x15\n\x0bstringValue\x18\x01 \x01(\tH\x00\x12\x14\n\nbytesValue\x18\x02 \x01(\x0cH\x00\x12\x12\n\x08intValue\x18\x03 \x01(\x05H\x00\x12\x13\n\tlongValue\x18\x04 \x01(\x03H\x00\x12\x14\n\nfloatValue\x18\x05 \x01(\x02H\x00\x12\x15\n\x0b\x64oubleValue\x18\x06 \x01(\x01H\x00\x42\x07\n\x05value\"Y\n\x08MapEntry\x12%\n\x03key\x18\x01 \x01(\x0b\x32\x18.aerospike.vector.MapKey\x12&\n\x05value\x18\x02 \x01(\x0b\x32\x17.aerospike.vector.Value\"2\n\x03Map\x12+\n\x07\x65ntries\x18\x01 \x03(\x0b\x32\x1a.aerospike.vector.MapEntry\"0\n\x04List\x12(\n\x07\x65ntries\x18\x01 \x03(\x0b\x32\x17.aerospike.vector.Value\"v\n\x06Vector\x12\x30\n\tboolArray\x18\x01 \x01(\x0b\x32\x1b.aerospike.vector.BoolArrayH\x00\x12\x32\n\nfloatArray\x18\x02 \x01(\x0b\x32\x1c.aerospike.vector.FloatArrayH\x00\x42\x06\n\x04\x64\x61ta\"\x9c\x02\n\x05Value\x12\x15\n\x0bstringValue\x18\x01 \x01(\tH\x00\x12\x14\n\nbytesValue\x18\x02 \x01(\x0cH\x00\x12\x12\n\x08intValue\x18\x03 \x01(\x05H\x00\x12\x13\n\tlongValue\x18\x04 \x01(\x03H\x00\x12\x14\n\nfloatValue\x18\x05 \x01(\x02H\x00\x12\x15\n\x0b\x64oubleValue\x18\x06 \x01(\x01H\x00\x12)\n\x08mapValue\x18\x07 \x01(\x0b\x32\x15.aerospike.vector.MapH\x00\x12+\n\tlistValue\x18\x08 \x01(\x0b\x32\x16.aerospike.vector.ListH\x00\x12/\n\x0bvectorValue\x18\t \x01(\x0b\x32\x18.aerospike.vector.VectorH\x00\x42\x07\n\x05value\";\n\x03\x42in\x12\x0c\n\x04name\x18\x01 \x01(\t\x12&\n\x05value\x18\x02 \x01(\x0b\x32\x17.aerospike.vector.Value\"U\n\x06Record\x12\x12\n\ngeneration\x18\x01 \x01(\r\x12\x12\n\nexpiration\x18\x02 \x01(\r\x12#\n\x04\x62ins\x18\x03 \x03(\x0b\x32\x15.aerospike.vector.Bin\"m\n\rRecordWithKey\x12\"\n\x03key\x18\x01 \x01(\x0b\x32\x15.aerospike.vector.Key\x12-\n\x06record\x18\x02 \x01(\x0b\x32\x18.aerospike.vector.RecordH\x00\x88\x01\x01\x42\t\n\x07_record\"*\n\x07IndexId\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x11\n\tnamespace\x18\x02 \x01(\t\"\xe8\x02\n\x0fIndexDefinition\x12%\n\x02id\x18\x01 \x01(\x0b\x32\x19.aerospike.vector.IndexId\x12)\n\x04type\x18\x02 \x01(\x0e\x32\x1b.aerospike.vector.IndexType\x12\x41\n\x10similarityMetric\x18\x03 \x01(\x0e\x32\'.aerospike.vector.IndexSimilarityMetric\x12\x10\n\x03set\x18\x04 \x01(\tH\x00\x88\x01\x01\x12\x0b\n\x03\x62in\x18\x05 \x01(\t\x12\x12\n\ndimensions\x18\x06 \x01(\r\x12=\n\x06params\x18\x07 \x03(\x0b\x32-.aerospike.vector.IndexDefinition.ParamsEntry\x1a\x46\n\x0bParamsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12&\n\x05value\x18\x02 \x01(\x0b\x32\x17.aerospike.vector.Value:\x02\x38\x01\x42\x06\n\x04_set\"I\n\x13IndexDefinitionList\x12\x32\n\x07indices\x18\x01 \x03(\x0b\x32!.aerospike.vector.IndexDefinition\"\x18\n\x07\x42oolean\x12\r\n\x05value\x18\x01 \x01(\x08*2\n\x15IndexSimilarityMetric\x12\r\n\tEUCLIDEAN\x10\x00\x12\n\n\x06\x43OSINE\x10\x01*\x15\n\tIndexType\x12\x08\n\x04HNSW\x10\x00\x42=\n\x1b\x63om.aerospike.vector.clientP\x01Z\x1c\x61\x65rospike.com/vector/protos/b\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0btypes.proto\x12\x10\x61\x65rospike.vector\"\xa1\x01\n\x03Key\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x10\n\x03set\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x0e\n\x06\x64igest\x18\x03 \x01(\x0c\x12\x15\n\x0bstringValue\x18\x04 \x01(\tH\x00\x12\x14\n\nbytesValue\x18\x05 \x01(\x0cH\x00\x12\x12\n\x08intValue\x18\x06 \x01(\x05H\x00\x12\x13\n\tlongValue\x18\x07 \x01(\x03H\x00\x42\x07\n\x05valueB\x06\n\x04_set\"\x19\n\x08\x42oolData\x12\r\n\x05value\x18\x01 \x03(\x08\"\x1a\n\tFloatData\x12\r\n\x05value\x18\x01 \x03(\x02\"\x94\x01\n\x06MapKey\x12\x15\n\x0bstringValue\x18\x01 \x01(\tH\x00\x12\x14\n\nbytesValue\x18\x02 \x01(\x0cH\x00\x12\x12\n\x08intValue\x18\x03 \x01(\x05H\x00\x12\x13\n\tlongValue\x18\x04 \x01(\x03H\x00\x12\x14\n\nfloatValue\x18\x05 \x01(\x02H\x00\x12\x15\n\x0b\x64oubleValue\x18\x06 \x01(\x01H\x00\x42\x07\n\x05value\"Y\n\x08MapEntry\x12%\n\x03key\x18\x01 \x01(\x0b\x32\x18.aerospike.vector.MapKey\x12&\n\x05value\x18\x02 \x01(\x0b\x32\x17.aerospike.vector.Value\"2\n\x03Map\x12+\n\x07\x65ntries\x18\x01 \x03(\x0b\x32\x1a.aerospike.vector.MapEntry\"0\n\x04List\x12(\n\x07\x65ntries\x18\x01 \x03(\x0b\x32\x17.aerospike.vector.Value\"r\n\x06Vector\x12.\n\x08\x62oolData\x18\x01 \x01(\x0b\x32\x1a.aerospike.vector.BoolDataH\x00\x12\x30\n\tfloatData\x18\x02 \x01(\x0b\x32\x1b.aerospike.vector.FloatDataH\x00\x42\x06\n\x04\x64\x61ta\"\x9c\x02\n\x05Value\x12\x15\n\x0bstringValue\x18\x01 \x01(\tH\x00\x12\x14\n\nbytesValue\x18\x02 \x01(\x0cH\x00\x12\x12\n\x08intValue\x18\x03 \x01(\x05H\x00\x12\x13\n\tlongValue\x18\x04 \x01(\x03H\x00\x12\x14\n\nfloatValue\x18\x05 \x01(\x02H\x00\x12\x15\n\x0b\x64oubleValue\x18\x06 \x01(\x01H\x00\x12)\n\x08mapValue\x18\x07 \x01(\x0b\x32\x15.aerospike.vector.MapH\x00\x12+\n\tlistValue\x18\x08 \x01(\x0b\x32\x16.aerospike.vector.ListH\x00\x12/\n\x0bvectorValue\x18\t \x01(\x0b\x32\x18.aerospike.vector.VectorH\x00\x42\x07\n\x05value\";\n\x03\x42in\x12\x0c\n\x04name\x18\x01 \x01(\t\x12&\n\x05value\x18\x02 \x01(\x0b\x32\x17.aerospike.vector.Value\"U\n\x06Record\x12\x12\n\ngeneration\x18\x01 \x01(\r\x12\x12\n\nexpiration\x18\x02 \x01(\r\x12#\n\x04\x62ins\x18\x03 \x03(\x0b\x32\x15.aerospike.vector.Bin\"m\n\rRecordWithKey\x12\"\n\x03key\x18\x01 \x01(\x0b\x32\x15.aerospike.vector.Key\x12-\n\x06record\x18\x02 \x01(\x0b\x32\x18.aerospike.vector.RecordH\x00\x88\x01\x01\x42\t\n\x07_record\"*\n\x07IndexId\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x11\n\tnamespace\x18\x02 \x01(\t\"\xa8\x01\n\nHnswParams\x12\x0e\n\x01m\x18\x01 \x01(\rH\x00\x88\x01\x01\x12\x1b\n\x0e\x65\x66\x43onstruction\x18\x02 \x01(\rH\x01\x88\x01\x01\x12\x0f\n\x02\x65\x66\x18\x03 \x01(\rH\x02\x88\x01\x01\x12<\n\x0e\x62\x61tchingParams\x18\x04 \x01(\x0b\x32$.aerospike.vector.HnswBatchingParamsB\x04\n\x02_mB\x11\n\x0f_efConstructionB\x05\n\x03_ef\"*\n\x10HnswSearchParams\x12\x0f\n\x02\x65\x66\x18\x01 \x01(\rH\x00\x88\x01\x01\x42\x05\n\x03_ef\"\x84\x01\n\x12HnswBatchingParams\x12\x17\n\nmaxRecords\x18\x01 \x01(\rH\x00\x88\x01\x01\x12\x15\n\x08interval\x18\x02 \x01(\rH\x01\x88\x01\x01\x12\x15\n\x08\x64isabled\x18\x03 \x01(\x08H\x02\x88\x01\x01\x42\r\n\x0b_maxRecordsB\x0b\n\t_intervalB\x0b\n\t_disabled\"W\n\x15\x41\x65rospikeIndexStorage\x12\x16\n\tnamespace\x18\x08 \x01(\tH\x00\x88\x01\x01\x12\x10\n\x03set\x18\t \x01(\tH\x01\x88\x01\x01\x42\x0c\n\n_namespaceB\x06\n\x04_set\"\xec\x03\n\x0fIndexDefinition\x12%\n\x02id\x18\x01 \x01(\x0b\x32\x19.aerospike.vector.IndexId\x12)\n\x04type\x18\x02 \x01(\x0e\x32\x1b.aerospike.vector.IndexType\x12\x12\n\ndimensions\x18\x03 \x01(\r\x12\x44\n\x14vectorDistanceMetric\x18\x04 \x01(\x0e\x32&.aerospike.vector.VectorDistanceMetric\x12\x0b\n\x03\x62in\x18\x05 \x01(\t\x12\x16\n\tsetFilter\x18\x06 \x01(\tH\x02\x88\x01\x01\x12\x32\n\nhnswParams\x18\x07 \x01(\x0b\x32\x1c.aerospike.vector.HnswParamsH\x00\x12\x43\n\x10\x61\x65rospikeStorage\x18\x08 \x01(\x0b\x32\'.aerospike.vector.AerospikeIndexStorageH\x01\x12=\n\x06labels\x18\t \x03(\x0b\x32-.aerospike.vector.IndexDefinition.LabelsEntry\x1a-\n\x0bLabelsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42\x08\n\x06paramsB\t\n\x07storageB\x0c\n\n_setFilter\"I\n\x13IndexDefinitionList\x12\x32\n\x07indices\x18\x01 \x03(\x0b\x32!.aerospike.vector.IndexDefinition\"\x18\n\x07\x42oolean\x12\r\n\x05value\x18\x01 \x01(\x08*f\n\x14VectorDistanceMetric\x12\x15\n\x11SQUARED_EUCLIDEAN\x10\x00\x12\n\n\x06\x43OSINE\x10\x01\x12\x0f\n\x0b\x44OT_PRODUCT\x10\x02\x12\r\n\tMANHATTAN\x10\x03\x12\x0b\n\x07HAMMING\x10\x04*\x15\n\tIndexType\x12\x08\n\x04HNSW\x10\x00\x42=\n\x1b\x63om.aerospike.vector.clientP\x01Z\x1c\x61\x65rospike.com/vector/protos/b\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'types_pb2', _globals) if _descriptor._USE_C_DESCRIPTORS == False: - DESCRIPTOR._options = None - DESCRIPTOR._serialized_options = b'\n\033com.aerospike.vector.clientP\001Z\034aerospike.com/vector/protos/' - _INDEXDEFINITION_PARAMSENTRY._options = None - _INDEXDEFINITION_PARAMSENTRY._serialized_options = b'8\001' - _globals['_INDEXSIMILARITYMETRIC']._serialized_start=1772 - _globals['_INDEXSIMILARITYMETRIC']._serialized_end=1822 - _globals['_INDEXTYPE']._serialized_start=1824 - _globals['_INDEXTYPE']._serialized_end=1845 + _globals['DESCRIPTOR']._options = None + _globals['DESCRIPTOR']._serialized_options = b'\n\033com.aerospike.vector.clientP\001Z\034aerospike.com/vector/protos/' + _globals['_INDEXDEFINITION_LABELSENTRY']._options = None + _globals['_INDEXDEFINITION_LABELSENTRY']._serialized_options = b'8\001' + _globals['_VECTORDISTANCEMETRIC']._serialized_start=2337 + _globals['_VECTORDISTANCEMETRIC']._serialized_end=2439 + _globals['_INDEXTYPE']._serialized_start=2441 + _globals['_INDEXTYPE']._serialized_end=2462 _globals['_KEY']._serialized_start=34 _globals['_KEY']._serialized_end=195 - _globals['_BOOLARRAY']._serialized_start=197 - _globals['_BOOLARRAY']._serialized_end=223 - _globals['_FLOATARRAY']._serialized_start=225 - _globals['_FLOATARRAY']._serialized_end=252 - _globals['_MAPKEY']._serialized_start=255 - _globals['_MAPKEY']._serialized_end=403 - _globals['_MAPENTRY']._serialized_start=405 - _globals['_MAPENTRY']._serialized_end=494 - _globals['_MAP']._serialized_start=496 - _globals['_MAP']._serialized_end=546 - _globals['_LIST']._serialized_start=548 - _globals['_LIST']._serialized_end=596 - _globals['_VECTOR']._serialized_start=598 - _globals['_VECTOR']._serialized_end=716 - _globals['_VALUE']._serialized_start=719 - _globals['_VALUE']._serialized_end=1003 - _globals['_BIN']._serialized_start=1005 - _globals['_BIN']._serialized_end=1064 - _globals['_RECORD']._serialized_start=1066 - _globals['_RECORD']._serialized_end=1151 - _globals['_RECORDWITHKEY']._serialized_start=1153 - _globals['_RECORDWITHKEY']._serialized_end=1262 - _globals['_INDEXID']._serialized_start=1264 - _globals['_INDEXID']._serialized_end=1306 - _globals['_INDEXDEFINITION']._serialized_start=1309 - _globals['_INDEXDEFINITION']._serialized_end=1669 - _globals['_INDEXDEFINITION_PARAMSENTRY']._serialized_start=1591 - _globals['_INDEXDEFINITION_PARAMSENTRY']._serialized_end=1661 - _globals['_INDEXDEFINITIONLIST']._serialized_start=1671 - _globals['_INDEXDEFINITIONLIST']._serialized_end=1744 - _globals['_BOOLEAN']._serialized_start=1746 - _globals['_BOOLEAN']._serialized_end=1770 + _globals['_BOOLDATA']._serialized_start=197 + _globals['_BOOLDATA']._serialized_end=222 + _globals['_FLOATDATA']._serialized_start=224 + _globals['_FLOATDATA']._serialized_end=250 + _globals['_MAPKEY']._serialized_start=253 + _globals['_MAPKEY']._serialized_end=401 + _globals['_MAPENTRY']._serialized_start=403 + _globals['_MAPENTRY']._serialized_end=492 + _globals['_MAP']._serialized_start=494 + _globals['_MAP']._serialized_end=544 + _globals['_LIST']._serialized_start=546 + _globals['_LIST']._serialized_end=594 + _globals['_VECTOR']._serialized_start=596 + _globals['_VECTOR']._serialized_end=710 + _globals['_VALUE']._serialized_start=713 + _globals['_VALUE']._serialized_end=997 + _globals['_BIN']._serialized_start=999 + _globals['_BIN']._serialized_end=1058 + _globals['_RECORD']._serialized_start=1060 + _globals['_RECORD']._serialized_end=1145 + _globals['_RECORDWITHKEY']._serialized_start=1147 + _globals['_RECORDWITHKEY']._serialized_end=1256 + _globals['_INDEXID']._serialized_start=1258 + _globals['_INDEXID']._serialized_end=1300 + _globals['_HNSWPARAMS']._serialized_start=1303 + _globals['_HNSWPARAMS']._serialized_end=1471 + _globals['_HNSWSEARCHPARAMS']._serialized_start=1473 + _globals['_HNSWSEARCHPARAMS']._serialized_end=1515 + _globals['_HNSWBATCHINGPARAMS']._serialized_start=1518 + _globals['_HNSWBATCHINGPARAMS']._serialized_end=1650 + _globals['_AEROSPIKEINDEXSTORAGE']._serialized_start=1652 + _globals['_AEROSPIKEINDEXSTORAGE']._serialized_end=1739 + _globals['_INDEXDEFINITION']._serialized_start=1742 + _globals['_INDEXDEFINITION']._serialized_end=2234 + _globals['_INDEXDEFINITION_LABELSENTRY']._serialized_start=2154 + _globals['_INDEXDEFINITION_LABELSENTRY']._serialized_end=2199 + _globals['_INDEXDEFINITIONLIST']._serialized_start=2236 + _globals['_INDEXDEFINITIONLIST']._serialized_end=2309 + _globals['_BOOLEAN']._serialized_start=2311 + _globals['_BOOLEAN']._serialized_end=2335 # @@protoc_insertion_point(module_scope) diff --git a/src/aerospike_vector/vector_db_pb2.py b/src/aerospike_vector/vector_db_pb2.py index da8bc15a..9ed551cd 100644 --- a/src/aerospike_vector/vector_db_pb2.py +++ b/src/aerospike_vector/vector_db_pb2.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: vector-db.proto +# Protobuf Python Version: 4.25.0 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool @@ -20,12 +21,12 @@ _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'vector_db_pb2', _globals) if _descriptor._USE_C_DESCRIPTORS == False: - DESCRIPTOR._options = None - DESCRIPTOR._serialized_options = b'\n\033com.aerospike.vector.clientP\001Z\034aerospike.com/vector/protos/' - _CLUSTERNODEENDPOINTS_ENDPOINTSENTRY._options = None - _CLUSTERNODEENDPOINTS_ENDPOINTSENTRY._serialized_options = b'8\001' - _CLUSTERPARTITIONS_PARTITIONSENTRY._options = None - _CLUSTERPARTITIONS_PARTITIONSENTRY._serialized_options = b'8\001' + _globals['DESCRIPTOR']._options = None + _globals['DESCRIPTOR']._serialized_options = b'\n\033com.aerospike.vector.clientP\001Z\034aerospike.com/vector/protos/' + _globals['_CLUSTERNODEENDPOINTS_ENDPOINTSENTRY']._options = None + _globals['_CLUSTERNODEENDPOINTS_ENDPOINTSENTRY']._serialized_options = b'8\001' + _globals['_CLUSTERPARTITIONS_PARTITIONSENTRY']._options = None + _globals['_CLUSTERPARTITIONS_PARTITIONSENTRY']._serialized_options = b'8\001' _globals['_ABOUTREQUEST']._serialized_start=66 _globals['_ABOUTREQUEST']._serialized_end=80 _globals['_ABOUTRESPONSE']._serialized_start=82 diff --git a/src/aerospike_vector/vectordb_admin.py b/src/aerospike_vector/vectordb_admin.py index 98c120a0..287cc226 100644 --- a/src/aerospike_vector/vectordb_admin.py +++ b/src/aerospike_vector/vectordb_admin.py @@ -27,32 +27,37 @@ def __init__(self, seeds: types.HostPort | tuple[types.HostPort, ...], self.channelProvider = vectordb_channel_provider.VectorDbChannelProvider( seeds, listener_name) - def indexCreate(self, namespace: str, name: str, set: str, + def indexCreate(self, namespace: str, name: str, vector_bin_name: str, dimensions: int, - params: dict[str, Any] = None, - index_similarity_metric: types_pb2.VectorDistanceMetric = - types_pb2.VectorDistanceMetric.SQUARED_EUCLIDEAN): + vector_distance_metric: types_pb2.VectorDistanceMetric = + types_pb2.VectorDistanceMetric.SQUARED_EUCLIDEAN, + setFilter: str = None, + indexParams: types_pb2.HnswParams = None, + labels: dict[str, str] = None): """Create an index""" index_stub = index_pb2_grpc.IndexServiceStub( self.channelProvider.getChannel()) - if params is None: - params = {} + if setFilter and not setFilter.strip(): + setFilter = None - grpcParams = {} - for k, v in params.items(): - grpcParams[k] = conversions.toVectorDbValue(v) + print("Index @@@@"+str(types_pb2.IndexDefinition( + id=types_pb2.IndexId(namespace=namespace, name=name), + vectorDistanceMetric=vector_distance_metric, + setFilter=setFilter, + hnswParams=indexParams, + bin=vector_bin_name, + dimensions=dimensions, + labels=labels))) - if not set.strip(): - set = None - index_stub.Create( types_pb2.IndexDefinition( id=types_pb2.IndexId(namespace=namespace, name=name), - set=set, - vectorDistanceMetric=index_similarity_metric, + vectorDistanceMetric=vector_distance_metric, + setFilter=setFilter, + hnswParams=indexParams, bin=vector_bin_name, dimensions=dimensions, - params=grpcParams)) + labels=labels)) def indexDrop(self, namespace: str, name: str): index_stub = index_pb2_grpc.IndexServiceStub( @@ -87,16 +92,9 @@ def waitForIndexCompletion(self, namespace: str, name: str, index_definition = self.indexGet(namespace, name) # Fetch batch wait interval - batching_config = \ - conversions.fromVectorDbValue(index_definition.params[ - "batching-config"]) - - # TODO: server should return defaults for batching-config if missing. - if not batching_config: - batching_config = {} - if "batch-interval" not in batching_config: - batching_config["batch-interval"] = 10_000 - batch_interval = batching_config["batch-interval"] + batch_interval = 10000 + if index_definition.hnswParams.batchingParams.interval == 0: + batch_interval = index_definition.hnswParams.batchingParams.interval unmerged_record_count = self.indexGetStatus(namespace, name).unmergedRecordCount diff --git a/src/aerospike_vector/vectordb_client.py b/src/aerospike_vector/vectordb_client.py index fc6e69d5..e065603d 100644 --- a/src/aerospike_vector/vectordb_client.py +++ b/src/aerospike_vector/vectordb_client.py @@ -13,7 +13,8 @@ class VectorDbClient(object): """Vector DB client""" - def __init__(self, seeds: types.HostPort | tuple[types.HostPort, ...], listener_name: str = None): + def __init__(self, seeds: types.HostPort | tuple[types.HostPort, ...], + listener_name: str = None): if not seeds: raise Exception("at least one seed host needed") @@ -58,6 +59,7 @@ def exists(self, namespace: str, set: str, key: Any) -> bool: def vectorSearch(self, namespace: str, index_name: str, query: list[bool | float], limit: int, + searchParams: types_pb2.HnswSearchParams = None, *bin_names: str) -> list[types.RecordWithKey]: transact_stub = transact_pb2_grpc.TransactStub( self._channelProvider.getChannel()) @@ -66,6 +68,7 @@ def vectorSearch(self, namespace: str, index_name: str, index=types_pb2.IndexId(namespace=namespace, name=index_name), queryVector=(conversions.toVectorDbValue(query).vectorValue), limit=limit, + searchParams=searchParams, binSelector=self._getBinSelector(bin_names) ) ) From 3df93aec631e8760766dea10c79068ab5ef096b6 Mon Sep 17 00:00:00 2001 From: ashishshinde Date: Thu, 21 Dec 2023 17:18:06 +0530 Subject: [PATCH 2/5] Removed debug print --- src/aerospike_vector/vectordb_admin.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/aerospike_vector/vectordb_admin.py b/src/aerospike_vector/vectordb_admin.py index 287cc226..322508f3 100644 --- a/src/aerospike_vector/vectordb_admin.py +++ b/src/aerospike_vector/vectordb_admin.py @@ -40,15 +40,6 @@ def indexCreate(self, namespace: str, name: str, if setFilter and not setFilter.strip(): setFilter = None - print("Index @@@@"+str(types_pb2.IndexDefinition( - id=types_pb2.IndexId(namespace=namespace, name=name), - vectorDistanceMetric=vector_distance_metric, - setFilter=setFilter, - hnswParams=indexParams, - bin=vector_bin_name, - dimensions=dimensions, - labels=labels))) - index_stub.Create( types_pb2.IndexDefinition( id=types_pb2.IndexId(namespace=namespace, name=name), From ebd664af31a63aa29e6d03fd4c4aa491b2ba5cd7 Mon Sep 17 00:00:00 2001 From: ashishshinde Date: Thu, 21 Dec 2023 17:39:02 +0530 Subject: [PATCH 3/5] Fixed search errors --- src/aerospike_vector/vectordb_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aerospike_vector/vectordb_client.py b/src/aerospike_vector/vectordb_client.py index e065603d..580add9c 100644 --- a/src/aerospike_vector/vectordb_client.py +++ b/src/aerospike_vector/vectordb_client.py @@ -68,7 +68,7 @@ def vectorSearch(self, namespace: str, index_name: str, index=types_pb2.IndexId(namespace=namespace, name=index_name), queryVector=(conversions.toVectorDbValue(query).vectorValue), limit=limit, - searchParams=searchParams, + hnswSearchParams=searchParams, binSelector=self._getBinSelector(bin_names) ) ) From 5431702918bfc4a5148137045237bd6979322c10 Mon Sep 17 00:00:00 2001 From: ashishshinde Date: Fri, 22 Dec 2023 13:19:16 +0530 Subject: [PATCH 4/5] Fixed typo in proto ids --- proto/types.proto | 4 ++-- src/aerospike_vector/auth_pb2.py | 5 ++--- src/aerospike_vector/index_pb2.py | 5 ++--- src/aerospike_vector/transact_pb2.py | 5 ++--- src/aerospike_vector/types_pb2.py | 11 +++++------ src/aerospike_vector/vector_db_pb2.py | 13 ++++++------- 6 files changed, 19 insertions(+), 24 deletions(-) diff --git a/proto/types.proto b/proto/types.proto index d2f8808b..eac830c6 100644 --- a/proto/types.proto +++ b/proto/types.proto @@ -196,11 +196,11 @@ message HnswBatchingParams { message AerospikeIndexStorage { // Optional Aerospike namespace where the index is stored. // Defaults to the index namespace. - optional string namespace = 8; + optional string namespace = 1; // Optional Aerospike set where the index is stored. // Defaults to the index name. - optional string set = 9; + optional string set = 2; } // An index definition. diff --git a/src/aerospike_vector/auth_pb2.py b/src/aerospike_vector/auth_pb2.py index 73c24b51..5cea9dd3 100644 --- a/src/aerospike_vector/auth_pb2.py +++ b/src/aerospike_vector/auth_pb2.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: auth.proto -# Protobuf Python Version: 4.25.0 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool @@ -20,8 +19,8 @@ _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'auth_pb2', _globals) if _descriptor._USE_C_DESCRIPTORS == False: - _globals['DESCRIPTOR']._options = None - _globals['DESCRIPTOR']._serialized_options = b'\n\033com.aerospike.vector.clientP\001Z\034aerospike.com/vector/protos/' + DESCRIPTOR._options = None + DESCRIPTOR._serialized_options = b'\n\033com.aerospike.vector.clientP\001Z\034aerospike.com/vector/protos/' _globals['_AEROSPIKEAUTHREQUEST']._serialized_start=32 _globals['_AEROSPIKEAUTHREQUEST']._serialized_end=90 _globals['_AEROSPIKEAUTHRESPONSE']._serialized_start=92 diff --git a/src/aerospike_vector/index_pb2.py b/src/aerospike_vector/index_pb2.py index 8a9ad0b9..bf5eacec 100644 --- a/src/aerospike_vector/index_pb2.py +++ b/src/aerospike_vector/index_pb2.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: index.proto -# Protobuf Python Version: 4.25.0 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool @@ -22,8 +21,8 @@ _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'index_pb2', _globals) if _descriptor._USE_C_DESCRIPTORS == False: - _globals['DESCRIPTOR']._options = None - _globals['DESCRIPTOR']._serialized_options = b'\n\033com.aerospike.vector.clientP\001Z\034aerospike.com/vector/protos/' + DESCRIPTOR._options = None + DESCRIPTOR._serialized_options = b'\n\033com.aerospike.vector.clientP\001Z\034aerospike.com/vector/protos/' _globals['_INDEXSTATUSRESPONSE']._serialized_start=75 _globals['_INDEXSTATUSRESPONSE']._serialized_end=125 _globals['_INDEXSERVICE']._serialized_start=128 diff --git a/src/aerospike_vector/transact_pb2.py b/src/aerospike_vector/transact_pb2.py index 2f9dd8ab..ac5a9215 100644 --- a/src/aerospike_vector/transact_pb2.py +++ b/src/aerospike_vector/transact_pb2.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: transact.proto -# Protobuf Python Version: 4.25.0 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool @@ -22,8 +21,8 @@ _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'transact_pb2', _globals) if _descriptor._USE_C_DESCRIPTORS == False: - _globals['DESCRIPTOR']._options = None - _globals['DESCRIPTOR']._serialized_options = b'\n\033com.aerospike.vector.clientP\001Z\034aerospike.com/vector/protos/' + DESCRIPTOR._options = None + DESCRIPTOR._serialized_options = b'\n\033com.aerospike.vector.clientP\001Z\034aerospike.com/vector/protos/' _globals['_BINSELECTORTYPE']._serialized_start=609 _globals['_BINSELECTORTYPE']._serialized_end=660 _globals['_PUTREQUEST']._serialized_start=78 diff --git a/src/aerospike_vector/types_pb2.py b/src/aerospike_vector/types_pb2.py index 20f5eba9..afdabc8f 100644 --- a/src/aerospike_vector/types_pb2.py +++ b/src/aerospike_vector/types_pb2.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: types.proto -# Protobuf Python Version: 4.25.0 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool @@ -14,16 +13,16 @@ -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0btypes.proto\x12\x10\x61\x65rospike.vector\"\xa1\x01\n\x03Key\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x10\n\x03set\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x0e\n\x06\x64igest\x18\x03 \x01(\x0c\x12\x15\n\x0bstringValue\x18\x04 \x01(\tH\x00\x12\x14\n\nbytesValue\x18\x05 \x01(\x0cH\x00\x12\x12\n\x08intValue\x18\x06 \x01(\x05H\x00\x12\x13\n\tlongValue\x18\x07 \x01(\x03H\x00\x42\x07\n\x05valueB\x06\n\x04_set\"\x19\n\x08\x42oolData\x12\r\n\x05value\x18\x01 \x03(\x08\"\x1a\n\tFloatData\x12\r\n\x05value\x18\x01 \x03(\x02\"\x94\x01\n\x06MapKey\x12\x15\n\x0bstringValue\x18\x01 \x01(\tH\x00\x12\x14\n\nbytesValue\x18\x02 \x01(\x0cH\x00\x12\x12\n\x08intValue\x18\x03 \x01(\x05H\x00\x12\x13\n\tlongValue\x18\x04 \x01(\x03H\x00\x12\x14\n\nfloatValue\x18\x05 \x01(\x02H\x00\x12\x15\n\x0b\x64oubleValue\x18\x06 \x01(\x01H\x00\x42\x07\n\x05value\"Y\n\x08MapEntry\x12%\n\x03key\x18\x01 \x01(\x0b\x32\x18.aerospike.vector.MapKey\x12&\n\x05value\x18\x02 \x01(\x0b\x32\x17.aerospike.vector.Value\"2\n\x03Map\x12+\n\x07\x65ntries\x18\x01 \x03(\x0b\x32\x1a.aerospike.vector.MapEntry\"0\n\x04List\x12(\n\x07\x65ntries\x18\x01 \x03(\x0b\x32\x17.aerospike.vector.Value\"r\n\x06Vector\x12.\n\x08\x62oolData\x18\x01 \x01(\x0b\x32\x1a.aerospike.vector.BoolDataH\x00\x12\x30\n\tfloatData\x18\x02 \x01(\x0b\x32\x1b.aerospike.vector.FloatDataH\x00\x42\x06\n\x04\x64\x61ta\"\x9c\x02\n\x05Value\x12\x15\n\x0bstringValue\x18\x01 \x01(\tH\x00\x12\x14\n\nbytesValue\x18\x02 \x01(\x0cH\x00\x12\x12\n\x08intValue\x18\x03 \x01(\x05H\x00\x12\x13\n\tlongValue\x18\x04 \x01(\x03H\x00\x12\x14\n\nfloatValue\x18\x05 \x01(\x02H\x00\x12\x15\n\x0b\x64oubleValue\x18\x06 \x01(\x01H\x00\x12)\n\x08mapValue\x18\x07 \x01(\x0b\x32\x15.aerospike.vector.MapH\x00\x12+\n\tlistValue\x18\x08 \x01(\x0b\x32\x16.aerospike.vector.ListH\x00\x12/\n\x0bvectorValue\x18\t \x01(\x0b\x32\x18.aerospike.vector.VectorH\x00\x42\x07\n\x05value\";\n\x03\x42in\x12\x0c\n\x04name\x18\x01 \x01(\t\x12&\n\x05value\x18\x02 \x01(\x0b\x32\x17.aerospike.vector.Value\"U\n\x06Record\x12\x12\n\ngeneration\x18\x01 \x01(\r\x12\x12\n\nexpiration\x18\x02 \x01(\r\x12#\n\x04\x62ins\x18\x03 \x03(\x0b\x32\x15.aerospike.vector.Bin\"m\n\rRecordWithKey\x12\"\n\x03key\x18\x01 \x01(\x0b\x32\x15.aerospike.vector.Key\x12-\n\x06record\x18\x02 \x01(\x0b\x32\x18.aerospike.vector.RecordH\x00\x88\x01\x01\x42\t\n\x07_record\"*\n\x07IndexId\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x11\n\tnamespace\x18\x02 \x01(\t\"\xa8\x01\n\nHnswParams\x12\x0e\n\x01m\x18\x01 \x01(\rH\x00\x88\x01\x01\x12\x1b\n\x0e\x65\x66\x43onstruction\x18\x02 \x01(\rH\x01\x88\x01\x01\x12\x0f\n\x02\x65\x66\x18\x03 \x01(\rH\x02\x88\x01\x01\x12<\n\x0e\x62\x61tchingParams\x18\x04 \x01(\x0b\x32$.aerospike.vector.HnswBatchingParamsB\x04\n\x02_mB\x11\n\x0f_efConstructionB\x05\n\x03_ef\"*\n\x10HnswSearchParams\x12\x0f\n\x02\x65\x66\x18\x01 \x01(\rH\x00\x88\x01\x01\x42\x05\n\x03_ef\"\x84\x01\n\x12HnswBatchingParams\x12\x17\n\nmaxRecords\x18\x01 \x01(\rH\x00\x88\x01\x01\x12\x15\n\x08interval\x18\x02 \x01(\rH\x01\x88\x01\x01\x12\x15\n\x08\x64isabled\x18\x03 \x01(\x08H\x02\x88\x01\x01\x42\r\n\x0b_maxRecordsB\x0b\n\t_intervalB\x0b\n\t_disabled\"W\n\x15\x41\x65rospikeIndexStorage\x12\x16\n\tnamespace\x18\x08 \x01(\tH\x00\x88\x01\x01\x12\x10\n\x03set\x18\t \x01(\tH\x01\x88\x01\x01\x42\x0c\n\n_namespaceB\x06\n\x04_set\"\xec\x03\n\x0fIndexDefinition\x12%\n\x02id\x18\x01 \x01(\x0b\x32\x19.aerospike.vector.IndexId\x12)\n\x04type\x18\x02 \x01(\x0e\x32\x1b.aerospike.vector.IndexType\x12\x12\n\ndimensions\x18\x03 \x01(\r\x12\x44\n\x14vectorDistanceMetric\x18\x04 \x01(\x0e\x32&.aerospike.vector.VectorDistanceMetric\x12\x0b\n\x03\x62in\x18\x05 \x01(\t\x12\x16\n\tsetFilter\x18\x06 \x01(\tH\x02\x88\x01\x01\x12\x32\n\nhnswParams\x18\x07 \x01(\x0b\x32\x1c.aerospike.vector.HnswParamsH\x00\x12\x43\n\x10\x61\x65rospikeStorage\x18\x08 \x01(\x0b\x32\'.aerospike.vector.AerospikeIndexStorageH\x01\x12=\n\x06labels\x18\t \x03(\x0b\x32-.aerospike.vector.IndexDefinition.LabelsEntry\x1a-\n\x0bLabelsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42\x08\n\x06paramsB\t\n\x07storageB\x0c\n\n_setFilter\"I\n\x13IndexDefinitionList\x12\x32\n\x07indices\x18\x01 \x03(\x0b\x32!.aerospike.vector.IndexDefinition\"\x18\n\x07\x42oolean\x12\r\n\x05value\x18\x01 \x01(\x08*f\n\x14VectorDistanceMetric\x12\x15\n\x11SQUARED_EUCLIDEAN\x10\x00\x12\n\n\x06\x43OSINE\x10\x01\x12\x0f\n\x0b\x44OT_PRODUCT\x10\x02\x12\r\n\tMANHATTAN\x10\x03\x12\x0b\n\x07HAMMING\x10\x04*\x15\n\tIndexType\x12\x08\n\x04HNSW\x10\x00\x42=\n\x1b\x63om.aerospike.vector.clientP\x01Z\x1c\x61\x65rospike.com/vector/protos/b\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0btypes.proto\x12\x10\x61\x65rospike.vector\"\xa1\x01\n\x03Key\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x10\n\x03set\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x0e\n\x06\x64igest\x18\x03 \x01(\x0c\x12\x15\n\x0bstringValue\x18\x04 \x01(\tH\x00\x12\x14\n\nbytesValue\x18\x05 \x01(\x0cH\x00\x12\x12\n\x08intValue\x18\x06 \x01(\x05H\x00\x12\x13\n\tlongValue\x18\x07 \x01(\x03H\x00\x42\x07\n\x05valueB\x06\n\x04_set\"\x19\n\x08\x42oolData\x12\r\n\x05value\x18\x01 \x03(\x08\"\x1a\n\tFloatData\x12\r\n\x05value\x18\x01 \x03(\x02\"\x94\x01\n\x06MapKey\x12\x15\n\x0bstringValue\x18\x01 \x01(\tH\x00\x12\x14\n\nbytesValue\x18\x02 \x01(\x0cH\x00\x12\x12\n\x08intValue\x18\x03 \x01(\x05H\x00\x12\x13\n\tlongValue\x18\x04 \x01(\x03H\x00\x12\x14\n\nfloatValue\x18\x05 \x01(\x02H\x00\x12\x15\n\x0b\x64oubleValue\x18\x06 \x01(\x01H\x00\x42\x07\n\x05value\"Y\n\x08MapEntry\x12%\n\x03key\x18\x01 \x01(\x0b\x32\x18.aerospike.vector.MapKey\x12&\n\x05value\x18\x02 \x01(\x0b\x32\x17.aerospike.vector.Value\"2\n\x03Map\x12+\n\x07\x65ntries\x18\x01 \x03(\x0b\x32\x1a.aerospike.vector.MapEntry\"0\n\x04List\x12(\n\x07\x65ntries\x18\x01 \x03(\x0b\x32\x17.aerospike.vector.Value\"r\n\x06Vector\x12.\n\x08\x62oolData\x18\x01 \x01(\x0b\x32\x1a.aerospike.vector.BoolDataH\x00\x12\x30\n\tfloatData\x18\x02 \x01(\x0b\x32\x1b.aerospike.vector.FloatDataH\x00\x42\x06\n\x04\x64\x61ta\"\x9c\x02\n\x05Value\x12\x15\n\x0bstringValue\x18\x01 \x01(\tH\x00\x12\x14\n\nbytesValue\x18\x02 \x01(\x0cH\x00\x12\x12\n\x08intValue\x18\x03 \x01(\x05H\x00\x12\x13\n\tlongValue\x18\x04 \x01(\x03H\x00\x12\x14\n\nfloatValue\x18\x05 \x01(\x02H\x00\x12\x15\n\x0b\x64oubleValue\x18\x06 \x01(\x01H\x00\x12)\n\x08mapValue\x18\x07 \x01(\x0b\x32\x15.aerospike.vector.MapH\x00\x12+\n\tlistValue\x18\x08 \x01(\x0b\x32\x16.aerospike.vector.ListH\x00\x12/\n\x0bvectorValue\x18\t \x01(\x0b\x32\x18.aerospike.vector.VectorH\x00\x42\x07\n\x05value\";\n\x03\x42in\x12\x0c\n\x04name\x18\x01 \x01(\t\x12&\n\x05value\x18\x02 \x01(\x0b\x32\x17.aerospike.vector.Value\"U\n\x06Record\x12\x12\n\ngeneration\x18\x01 \x01(\r\x12\x12\n\nexpiration\x18\x02 \x01(\r\x12#\n\x04\x62ins\x18\x03 \x03(\x0b\x32\x15.aerospike.vector.Bin\"m\n\rRecordWithKey\x12\"\n\x03key\x18\x01 \x01(\x0b\x32\x15.aerospike.vector.Key\x12-\n\x06record\x18\x02 \x01(\x0b\x32\x18.aerospike.vector.RecordH\x00\x88\x01\x01\x42\t\n\x07_record\"*\n\x07IndexId\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x11\n\tnamespace\x18\x02 \x01(\t\"\xa8\x01\n\nHnswParams\x12\x0e\n\x01m\x18\x01 \x01(\rH\x00\x88\x01\x01\x12\x1b\n\x0e\x65\x66\x43onstruction\x18\x02 \x01(\rH\x01\x88\x01\x01\x12\x0f\n\x02\x65\x66\x18\x03 \x01(\rH\x02\x88\x01\x01\x12<\n\x0e\x62\x61tchingParams\x18\x04 \x01(\x0b\x32$.aerospike.vector.HnswBatchingParamsB\x04\n\x02_mB\x11\n\x0f_efConstructionB\x05\n\x03_ef\"*\n\x10HnswSearchParams\x12\x0f\n\x02\x65\x66\x18\x01 \x01(\rH\x00\x88\x01\x01\x42\x05\n\x03_ef\"\x84\x01\n\x12HnswBatchingParams\x12\x17\n\nmaxRecords\x18\x01 \x01(\rH\x00\x88\x01\x01\x12\x15\n\x08interval\x18\x02 \x01(\rH\x01\x88\x01\x01\x12\x15\n\x08\x64isabled\x18\x03 \x01(\x08H\x02\x88\x01\x01\x42\r\n\x0b_maxRecordsB\x0b\n\t_intervalB\x0b\n\t_disabled\"W\n\x15\x41\x65rospikeIndexStorage\x12\x16\n\tnamespace\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x10\n\x03set\x18\x02 \x01(\tH\x01\x88\x01\x01\x42\x0c\n\n_namespaceB\x06\n\x04_set\"\xec\x03\n\x0fIndexDefinition\x12%\n\x02id\x18\x01 \x01(\x0b\x32\x19.aerospike.vector.IndexId\x12)\n\x04type\x18\x02 \x01(\x0e\x32\x1b.aerospike.vector.IndexType\x12\x12\n\ndimensions\x18\x03 \x01(\r\x12\x44\n\x14vectorDistanceMetric\x18\x04 \x01(\x0e\x32&.aerospike.vector.VectorDistanceMetric\x12\x0b\n\x03\x62in\x18\x05 \x01(\t\x12\x16\n\tsetFilter\x18\x06 \x01(\tH\x02\x88\x01\x01\x12\x32\n\nhnswParams\x18\x07 \x01(\x0b\x32\x1c.aerospike.vector.HnswParamsH\x00\x12\x43\n\x10\x61\x65rospikeStorage\x18\x08 \x01(\x0b\x32\'.aerospike.vector.AerospikeIndexStorageH\x01\x12=\n\x06labels\x18\t \x03(\x0b\x32-.aerospike.vector.IndexDefinition.LabelsEntry\x1a-\n\x0bLabelsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42\x08\n\x06paramsB\t\n\x07storageB\x0c\n\n_setFilter\"I\n\x13IndexDefinitionList\x12\x32\n\x07indices\x18\x01 \x03(\x0b\x32!.aerospike.vector.IndexDefinition\"\x18\n\x07\x42oolean\x12\r\n\x05value\x18\x01 \x01(\x08*f\n\x14VectorDistanceMetric\x12\x15\n\x11SQUARED_EUCLIDEAN\x10\x00\x12\n\n\x06\x43OSINE\x10\x01\x12\x0f\n\x0b\x44OT_PRODUCT\x10\x02\x12\r\n\tMANHATTAN\x10\x03\x12\x0b\n\x07HAMMING\x10\x04*\x15\n\tIndexType\x12\x08\n\x04HNSW\x10\x00\x42=\n\x1b\x63om.aerospike.vector.clientP\x01Z\x1c\x61\x65rospike.com/vector/protos/b\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'types_pb2', _globals) if _descriptor._USE_C_DESCRIPTORS == False: - _globals['DESCRIPTOR']._options = None - _globals['DESCRIPTOR']._serialized_options = b'\n\033com.aerospike.vector.clientP\001Z\034aerospike.com/vector/protos/' - _globals['_INDEXDEFINITION_LABELSENTRY']._options = None - _globals['_INDEXDEFINITION_LABELSENTRY']._serialized_options = b'8\001' + DESCRIPTOR._options = None + DESCRIPTOR._serialized_options = b'\n\033com.aerospike.vector.clientP\001Z\034aerospike.com/vector/protos/' + _INDEXDEFINITION_LABELSENTRY._options = None + _INDEXDEFINITION_LABELSENTRY._serialized_options = b'8\001' _globals['_VECTORDISTANCEMETRIC']._serialized_start=2337 _globals['_VECTORDISTANCEMETRIC']._serialized_end=2439 _globals['_INDEXTYPE']._serialized_start=2441 diff --git a/src/aerospike_vector/vector_db_pb2.py b/src/aerospike_vector/vector_db_pb2.py index 9ed551cd..da8bc15a 100644 --- a/src/aerospike_vector/vector_db_pb2.py +++ b/src/aerospike_vector/vector_db_pb2.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: vector-db.proto -# Protobuf Python Version: 4.25.0 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool @@ -21,12 +20,12 @@ _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'vector_db_pb2', _globals) if _descriptor._USE_C_DESCRIPTORS == False: - _globals['DESCRIPTOR']._options = None - _globals['DESCRIPTOR']._serialized_options = b'\n\033com.aerospike.vector.clientP\001Z\034aerospike.com/vector/protos/' - _globals['_CLUSTERNODEENDPOINTS_ENDPOINTSENTRY']._options = None - _globals['_CLUSTERNODEENDPOINTS_ENDPOINTSENTRY']._serialized_options = b'8\001' - _globals['_CLUSTERPARTITIONS_PARTITIONSENTRY']._options = None - _globals['_CLUSTERPARTITIONS_PARTITIONSENTRY']._serialized_options = b'8\001' + DESCRIPTOR._options = None + DESCRIPTOR._serialized_options = b'\n\033com.aerospike.vector.clientP\001Z\034aerospike.com/vector/protos/' + _CLUSTERNODEENDPOINTS_ENDPOINTSENTRY._options = None + _CLUSTERNODEENDPOINTS_ENDPOINTSENTRY._serialized_options = b'8\001' + _CLUSTERPARTITIONS_PARTITIONSENTRY._options = None + _CLUSTERPARTITIONS_PARTITIONSENTRY._serialized_options = b'8\001' _globals['_ABOUTREQUEST']._serialized_start=66 _globals['_ABOUTREQUEST']._serialized_end=80 _globals['_ABOUTRESPONSE']._serialized_start=82 From 94916859d799aa06bdaa6117f8d45ab3d305f5da Mon Sep 17 00:00:00 2001 From: ashishshinde Date: Tue, 26 Dec 2023 14:28:02 +0530 Subject: [PATCH 5/5] Added instructions to publish to jfrog proto --- README.md | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 459b7f6f..da143837 100644 --- a/README.md +++ b/README.md @@ -55,19 +55,35 @@ Create or update `$HOME/.pypirc` with following contents ```ini [distutils] -index-servers = ecosystem-python-dev-local +index-servers = + ecosystem-python-dev-local + ecosystem-python-prod-local + [ecosystem-python-dev-local] repository: https://aerospike.jfrog.io/artifactory/api/pypi/ecosystem-python-dev-local username: password: + +[ecosystem-python-prod-local] +repository: https://aerospike.jfrog.io/artifactory/api/pypi/ecosystem-python-prod-local +username: +password: ``` ### Upload the packages to the repository + +To upload dev packages run ```shell python3 -m pip install twine python3 -m twine upload --repository ecosystem-python-dev-local dist/* ``` +To upload release packages run +```shell +python3 -m pip install twine +python3 -m twine upload --repository ecosystem-python-prod-local dist/* +``` + ## Examples See [examples](https://github.com/aerospike/proximus-examples) for working samples.