diff --git a/v2.4.x/Variables.json b/v2.4.x/Variables.json index 7e9aa9985b..2aeff50a41 100644 --- a/v2.4.x/Variables.json +++ b/v2.4.x/Variables.json @@ -1,6 +1,6 @@ { - "milvus_release_version": "2.4.17", - "milvus_release_tag": "2.4.17", + "milvus_release_version": "2.5.x", + "milvus_release_tag": "2.5.x", "milvus_deb_name": "milvus_2.2.0-1_amd64", "milvus_rpm_name": "milvus-2.2.0-1.el7.x86_64", "milvus_python_sdk_version": "2.4.x", diff --git a/v2.4.x/assets/analyzer-overview.png b/v2.4.x/assets/analyzer-overview.png new file mode 100644 index 0000000000..70df5aa880 Binary files /dev/null and b/v2.4.x/assets/analyzer-overview.png differ diff --git a/v2.4.x/assets/ann-search.png b/v2.4.x/assets/ann-search.png new file mode 100644 index 0000000000..7983cf5eb8 Binary files /dev/null and b/v2.4.x/assets/ann-search.png differ diff --git a/v2.4.x/assets/batch-data-and-streaming-data.png b/v2.4.x/assets/batch-data-and-streaming-data.png new file mode 100644 index 0000000000..f513939169 Binary files /dev/null and b/v2.4.x/assets/batch-data-and-streaming-data.png differ diff --git a/v2.4.x/assets/binary-vector.png b/v2.4.x/assets/binary-vector.png new file mode 100644 index 0000000000..7432e69805 Binary files /dev/null and b/v2.4.x/assets/binary-vector.png differ diff --git a/v2.4.x/assets/bitmap.png b/v2.4.x/assets/bitmap.png new file mode 100644 index 0000000000..aca6297254 Binary files /dev/null and b/v2.4.x/assets/bitmap.png differ diff --git a/v2.4.x/assets/bm25.png b/v2.4.x/assets/bm25.png new file mode 100644 index 0000000000..b3a62d42d0 Binary files /dev/null and b/v2.4.x/assets/bm25.png differ diff --git a/v2.4.x/assets/collection-explained.png b/v2.4.x/assets/collection-explained.png new file mode 100644 index 0000000000..e3baf8468a Binary files /dev/null and b/v2.4.x/assets/collection-explained.png differ diff --git a/v2.4.x/assets/consistency-level-illustrated.png b/v2.4.x/assets/consistency-level-illustrated.png new file mode 100644 index 0000000000..f833080140 Binary files /dev/null and b/v2.4.x/assets/consistency-level-illustrated.png differ diff --git a/v2.4.x/assets/dense-vector.png b/v2.4.x/assets/dense-vector.png new file mode 100644 index 0000000000..fa236c2e87 Binary files /dev/null and b/v2.4.x/assets/dense-vector.png differ diff --git a/v2.4.x/assets/filtered-search.png b/v2.4.x/assets/filtered-search.png new file mode 100644 index 0000000000..5fe1c94fd5 Binary files /dev/null and b/v2.4.x/assets/filtered-search.png differ diff --git a/v2.4.x/assets/full-text-search.png b/v2.4.x/assets/full-text-search.png new file mode 100644 index 0000000000..330e94506f Binary files /dev/null and b/v2.4.x/assets/full-text-search.png differ diff --git a/v2.4.x/assets/grouping-search.png b/v2.4.x/assets/grouping-search.png new file mode 100644 index 0000000000..74736f0e14 Binary files /dev/null and b/v2.4.x/assets/grouping-search.png differ diff --git a/v2.4.x/assets/hybrid-search.png b/v2.4.x/assets/hybrid-search.png new file mode 100644 index 0000000000..887ab9fd53 Binary files /dev/null and b/v2.4.x/assets/hybrid-search.png differ diff --git a/v2.4.x/assets/idf.png b/v2.4.x/assets/idf.png new file mode 100644 index 0000000000..9e3ac9cbde Binary files /dev/null and b/v2.4.x/assets/idf.png differ diff --git a/v2.4.x/assets/keyword-match.png b/v2.4.x/assets/keyword-match.png new file mode 100644 index 0000000000..f56818e338 Binary files /dev/null and b/v2.4.x/assets/keyword-match.png differ diff --git a/v2.4.x/assets/partition-vs-partition-key.png b/v2.4.x/assets/partition-vs-partition-key.png new file mode 100644 index 0000000000..b18e696177 Binary files /dev/null and b/v2.4.x/assets/partition-vs-partition-key.png differ diff --git a/v2.4.x/assets/range-search.png b/v2.4.x/assets/range-search.png new file mode 100644 index 0000000000..91da340a8d Binary files /dev/null and b/v2.4.x/assets/range-search.png differ diff --git a/v2.4.x/assets/schema-explained.PNG b/v2.4.x/assets/schema-explained.PNG new file mode 100644 index 0000000000..cb3779da79 Binary files /dev/null and b/v2.4.x/assets/schema-explained.PNG differ diff --git a/v2.4.x/assets/schema-hands-on.png b/v2.4.x/assets/schema-hands-on.png new file mode 100644 index 0000000000..cb3779da79 Binary files /dev/null and b/v2.4.x/assets/schema-hands-on.png differ diff --git a/v2.4.x/assets/service-time-and-guarantee-time.png b/v2.4.x/assets/service-time-and-guarantee-time.png new file mode 100644 index 0000000000..3603f2395f Binary files /dev/null and b/v2.4.x/assets/service-time-and-guarantee-time.png differ diff --git a/v2.4.x/assets/sparse-vector.png b/v2.4.x/assets/sparse-vector.png new file mode 100644 index 0000000000..8c19af1354 Binary files /dev/null and b/v2.4.x/assets/sparse-vector.png differ diff --git a/v2.4.x/assets/upsert-entities.png b/v2.4.x/assets/upsert-entities.png new file mode 100644 index 0000000000..afdbcfa6f0 Binary files /dev/null and b/v2.4.x/assets/upsert-entities.png differ diff --git a/v2.4.x/assets/use-binary-vector.png b/v2.4.x/assets/use-binary-vector.png new file mode 100644 index 0000000000..52837e5ea1 Binary files /dev/null and b/v2.4.x/assets/use-binary-vector.png differ diff --git a/v2.4.x/assets/use-dense-vector.png b/v2.4.x/assets/use-dense-vector.png new file mode 100644 index 0000000000..82447e52f4 Binary files /dev/null and b/v2.4.x/assets/use-dense-vector.png differ diff --git a/v2.4.x/assets/use-sparse-vector.png b/v2.4.x/assets/use-sparse-vector.png new file mode 100644 index 0000000000..e6d4dac6d9 Binary files /dev/null and b/v2.4.x/assets/use-sparse-vector.png differ diff --git a/v2.4.x/assets/with-and-without-partition-key.png b/v2.4.x/assets/with-and-without-partition-key.png new file mode 100644 index 0000000000..5cf04df648 Binary files /dev/null and b/v2.4.x/assets/with-and-without-partition-key.png differ diff --git a/v2.4.x/format.js b/v2.4.x/format.js new file mode 100644 index 0000000000..8ff0987ec6 --- /dev/null +++ b/v2.4.x/format.js @@ -0,0 +1,107 @@ +const fs = require('fs'); +const { + join +} = require('path'); + +const formatMenuStructure = list => { + const newList = list.map(v => { + const { + id, + title, + isMenu = false, + outLink = '', + order = 0, + label1, + label2, + label3, + } = v; + + const parentId = label3 || label2 || label1 || ''; + const parentIds = [label1, label2, label3].filter(v => !!v); + const level = [label1, label2, label3].filter(v => !!v).length + 1; + + const baseInfo = { + id: id, + label: title, + } + + if (isMenu) { + baseInfo.isMenu = true; + } + + if (outLink) { + baseInfo.externalLink = outLink; + } + + return { + ...baseInfo, + parentId, + parentIds, + level, + order, + children: [], + }; + }); + + newList.sort((x, y) => y.level - x.level); + + const resultList = newList.slice(); + + newList.forEach(v => { + const { + parentId + } = v; + const parentIndex = resultList.findIndex(v => v.id === parentId); + if (parentIndex !== -1) { + + let childInfo = { + label: v.label, + id: v.id, + + // href: v.href, + // parentId: parentId, + // parentIds: v.parentIds, + // level: v.level, + order: v.order, + + } + + if(v.isMenu){ + childInfo.isMenu = true; + } + + if(v.externalLink){ + childInfo.externalLink = v.externalLink + } + + childInfo.children = v.children; + + resultList[parentIndex].children.push(childInfo); + } + }); + + return resultList.filter(v => v.level === 1).map(v => ({ + label: v.label, + id: v.id, + isMenu: v.isMenu, + externalLink: v.externalLink, + // href: v.href, + // parentId: parentId, + // parentIds: v.parentIds, + // level: v.level, + order: v.order, + children: v.children, + })); +}; + +const formatMenu = () => { + + + const menuPath = join(__dirname, 'site/en/menuStructure/en.json') + + const menu = fs.readFileSync(menuPath, 'utf-8'); + const newMenuStructure = formatMenuStructure(JSON.parse(menu).menuList); + + fs.writeFileSync(menuPath, JSON.stringify(newMenuStructure)); +}; +formatMenu(); diff --git a/v2.4.x/site/en/about/overview.md b/v2.4.x/site/en/about/overview.md index f0eeb5323e..dac67b419d 100644 --- a/v2.4.x/site/en/about/overview.md +++ b/v2.4.x/site/en/about/overview.md @@ -80,7 +80,7 @@ In addition to primitive data types, Milvus supports various advanced data types - [Sparse Vectors](sparse_vector.md) - [Binary Vectors](index-vector-fields.md) - [JSON Support](use-json-fields.md) -- [Array Support](array_data_type.md) +- [Array Support](array.md) - [Distance Metrics](metric.md) ### Acceleration diff --git a/v2.4.x/site/en/about/roadmap.md b/v2.4.x/site/en/about/roadmap.md index a96754adca..d1279c386d 100644 --- a/v2.4.x/site/en/about/roadmap.md +++ b/v2.4.x/site/en/about/roadmap.md @@ -15,35 +15,35 @@ Welcome to the Milvus Roadmap! Join us on our continuous journey to enhance and Category - Milvus 2.4.0 (Recently Achieved) - Milvus 2.5.0 (Upcoming in Mid-CY24) - Future Roadmap (Milvus 3.0 Expected within CY24) + Milvus 2.5.0 (Achieved in recent releases) + Next Release (Middle of CY25) + Future Roadmap (Within 1 year) - AI-developer Friendly
A developer-friendly technology stack, enhanced with the latest AI innovations - Multi-Vectors & Hybrid Search
Framework for multiplex recall and fusion

GPU Index Acceleration
Support for higher QPS and faster index creation

Model Library in PyMilvus
Integrated embedding models for Milvus - Sparse Vector (GA)
Local feature extraction and keyword search

Milvus Lite (GA)
A lightweight, in-memory version of Milvus

Embedding Models Gallery
Support for image and multi-modal embeddings and reranker models in model libraries - Original Data-In and Data-Out
Support for Blob data types

Data Clustering
Data co-locality

Scenario-oriented Vector Search
e.g. Multi-target search & NN filtering

Support Embedding & Reranker Endpoint + AI-Driven Unstructured Data Processing
Strengthening the ability to process and analyze unstructured data using AI models and advanced technologies. + Full Text Search
Support full text search with Sparse-BM25. The new API accepts text as input and automatically generates sparse vector inside Milvus

Sparse Vector(GA)
Support efficient storage and indexing method for sparse vector
+ Data-In and Data-Out
Support major model services to ingest original data

Advanced Reranker
Support model-based rerankers and user-defined scoring function

JSON Enhancement
JSON indexing and parsing to accelerate processing + Original Data-In and Data-Out
Support Blob and url reference to process original data

Support More Data Types
e.g. Datetime, Map, GIS

Support Tensors
Support list of vectors, typical usage like Colbert, Copali etc. - Rich Functionality
Enhanced retrieval and data management features - Support for FP16, BF16 Datatypes
These ML datatypes can help reduce memory usage

Grouping Search
Aggregate split embeddings

Fuzzy Match and Inverted Index
Support for fuzzy matching and inverted indexing for scalar types like varchar and int - Inverted Index for Array & JSON
Indexing for array and partial support JSON

Bitset Index
Improved execution speed and future data aggregation

Truncate Collection
Allows data clearance while preserving metadata

Support for NULL and Default Values - Support for More Datatypes
e.g. Datetime, GIS

Advanced Text Filtering
e.g. Match Phrase

Primary Key Deduplication + Search Quality & Performance
Deliver accurate, relevant, and fast results by optimizing architecture, algorithms and APIs + Text Match Function
Quickly filter keywords/tokens in text/varchar

Grouping Search Enhancement
Introduce group_size and add group by support in hybrid search

Bitmap Index & Inverted Index
Accelerate filtering on tags + Advanced Match
e.g. Match Phrase, Fuzzy Match, and more tokenizers

Aggregations
Scalar field aggregations, e.g. min, max, count, distinct.
+ Partial Update
Support updates to a specific field's value

Sorting Capability
Sort by scalar fields during execution

Support Data Clustering
Data co-locality - Cost Efficiency & Architecture
Advanced systems emphasizing stability, cost efficiency, scalability, and performance - Support for More Collections/Partitions
Handles over 10,000 collections in smaller clusters

Mmap Optimization
Balances reduced memory consumption with latency

Bulk Insert Optimazation
Simplifies importing large datasets - Lazy Load
Data is loaded on-demand through read operations

Major Compaction
Re-distributes data based on configuration to enhance read performance

Mmap for Growing Data
Mmap files for expanding data segments - Memory Control
Reduces out-of-memory issues and provides global memory management

LogNode Introduction
Ensures global consistency and addresses the single-point bottleneck in root coordination

Storage Format V2
Universal format design lays the groundwork for disk-based data access + Rich Functionality & Management
Developer-friendly and robust data management features + Support CSV files in data import
Bulkinsert supports CSV format

Support Null and Default Value
Null and Default types make importing data from other DBMS easier

Milvus WebUI (Beta)
Visual management tools for DBAs + Primary Key Deduplication
By using the global pk index

Online Schema Change
e.g. Add/delete field, modify varchar length

Data Versioning & Restore
Support data versioning by snapshot + Rust and C++ SDK
Support more clients

Support UDF
User-defined function - Enterprise Ready
Designed to meet the needs of enterprise production environments - Milvus CDC
Capability for data replication

Accesslog Enhancement
Detailed recording for audit and tracing - New Resource Group
Enhanced resource management

Storage Hook
Support for Bring Your Own Key (BYOK) encryption - Dynamic Replica Number Adjustment
Facilitates dynamic changes to the number of replicas

Dynamic Schema Modification
e.g., Add/delete fields, modify varchar lengths

Rust and C# SDKs + Cost Efficiency & Achitecture
State-of-the-art systems, prioritizing stability, cost-efficiency and scalability + Load by Field
Choose part of collection to load

Memory Optimization
Reduce OOM and load enhancement

Streaming Node (Beta)
Provide global consistency and solve performance bottleneck on root coordinator

Storage Format V2 (Beta)
Universal formats design and foundation for disk-based data access

Clustering Compaction
Data re-distribution based on configuration to accelerate read performance + Lazy Load
Load can be initiated by the first read operation without explicitly calling load()

Tiered Storage
Support hot and cold storage for cost optimization

Release by Field
Release part of collection to reduce memory usage

Streaming Node (GA)
Process streaming data and simplify the architecture + Remove dependencies
Reduce or eliminate dependencies on external components like pulsar, etcd

Merge coord logic into MixCoord
Simplify the architecture diff --git a/v2.4.x/site/en/adminGuide/configure-docker.md b/v2.4.x/site/en/adminGuide/configure-docker.md index faa8a14b74..184f9c20ea 100644 --- a/v2.4.x/site/en/adminGuide/configure-docker.md +++ b/v2.4.x/site/en/adminGuide/configure-docker.md @@ -16,10 +16,10 @@ In current release, all parameters take effect only after Milvus restarts. ## Download a configuration file -[Download](https://raw.githubusercontent.com/milvus-io/milvus/v2.4.17/configs/milvus.yaml) `milvus.yaml` directly or with the following command. +[Download](https://raw.githubusercontent.com/milvus-io/milvus/v2.5.x/configs/milvus.yaml) `milvus.yaml` directly or with the following command. ``` -$ wget https://raw.githubusercontent.com/milvus-io/milvus/v2.4.17/configs/milvus.yaml +$ wget https://raw.githubusercontent.com/milvus-io/milvus/v2.5.x/configs/milvus.yaml ``` ## Modify the configuration file @@ -178,13 +178,13 @@ Sorted by: ## Download an installation file -Download the installation file for Milvus [standalone](https://github.com/milvus-io/milvus/releases/download/v2.4.17/milvus-standalone-docker-compose.yml), and save it as `docker-compose.yml`. +Download the installation file for Milvus [standalone](https://github.com/milvus-io/milvus/releases/download/v2.5.x/milvus-standalone-docker-compose.yml), and save it as `docker-compose.yml`. You can also simply run the following command. ``` # For Milvus standalone -$ wget https://github.com/milvus-io/milvus/releases/download/v2.4.17/milvus-standalone-docker-compose.yml -O docker-compose.yml +$ wget https://github.com/milvus-io/milvus/releases/download/v2.5.x/milvus-standalone-docker-compose.yml -O docker-compose.yml ``` ## Modify the installation file diff --git a/v2.4.x/site/en/adminGuide/operator.md b/v2.4.x/site/en/adminGuide/operator.md index 756373838f..350f49a0e2 100644 --- a/v2.4.x/site/en/adminGuide/operator.md +++ b/v2.4.x/site/en/adminGuide/operator.md @@ -12,7 +12,7 @@ Milvus cluster depends on components including object storage, etcd, and Pulsar. This topic assumes that you have deployed Milvus Operator. -
See Deploy Milvus Operator for more information.
+
See Deploy Milvus Operator for more information.
You need to specify a configuration file for using Milvus Operator to start a Milvus cluster. diff --git a/v2.4.x/site/en/adminGuide/resource_group.md b/v2.4.x/site/en/adminGuide/resource_group.md index a906b744e0..d3b3d928f4 100644 --- a/v2.4.x/site/en/adminGuide/resource_group.md +++ b/v2.4.x/site/en/adminGuide/resource_group.md @@ -56,7 +56,7 @@ All code samples on this page are in PyMilvus 2.4.9. Upgrade your PyMilvus insta To create a resource group, run the following after you connect to a Milvus instance. The following snippet assumes that `default` is the alias of your Milvus connection. - ```Python + ```python import pymilvus # A resource group name should be a string of 1 to 255 characters, starting with a letter or an underscore (_) and containing only numbers, letters, and underscores (_). @@ -80,7 +80,7 @@ All code samples on this page are in PyMilvus 2.4.9. Upgrade your PyMilvus insta To view the list of resource groups in a Milvus instance, do as follows: - ```Python + ```python rgs = utility.list_resource_groups(using='default') print(f"Resource group list: {rgs}") @@ -91,7 +91,7 @@ All code samples on this page are in PyMilvus 2.4.9. Upgrade your PyMilvus insta You can have Milvus describe a resource group in concern as follows: - ```Python + ```python info = utility.describe_resource_group(name, using="default") print(f"Resource group description: {info}") @@ -110,7 +110,7 @@ All code samples on this page are in PyMilvus 2.4.9. Upgrade your PyMilvus insta Assuming there are currently 1 QueryNodes in the **__default_resource_group** of the cluster, and we want to transfer one node into created **rg**. `update_resource_groups` ensures atomicity for multiple configuration changes, so no intermediate states will be visible to Milvus. - ```Python + ```python source = '__default_resource_group' target = 'rg' expected_num_nodes_in_default = 0 @@ -138,7 +138,7 @@ All code samples on this page are in PyMilvus 2.4.9. Upgrade your PyMilvus insta Once there are query nodes in a resource group, you can load collections to this resource group. The following snippet assumes that a collection named `demo` already exists. - ```Python + ```python from pymilvus import Collection collection = Collection('demo') @@ -154,7 +154,7 @@ All code samples on this page are in PyMilvus 2.4.9. Upgrade your PyMilvus insta Also, you can just load a partition into a resource group and have its replicas distributed among several resource groups. The following assumes that a collection named `Books` already exists and it has a partition named `Novels`. - ```Python + ```python collection = Collection("Books") # Use the load method of a collection to load one of its partition @@ -173,7 +173,7 @@ All code samples on this page are in PyMilvus 2.4.9. Upgrade your PyMilvus insta Milvus uses [replicas](replica.md) to achieve load-balancing among [segments](glossary.md#Segment) distributed across several query nodes. You can move certain replicas of a collection from one resource group to another as follows: - ```Python + ```python source = '__default_resource_group' target = 'rg' collection_name = 'c' @@ -192,7 +192,7 @@ All code samples on this page are in PyMilvus 2.4.9. Upgrade your PyMilvus insta You can drop a resource group that hold no query node (`limits.node_num = 0`) at any time. In this guide, resource group `rg` now has one query node. You need to change the configuration `limits.node_num` of resource group into zero first. - ```Python + ```python try: utility.update_resource_groups({ "rg": utility.ResourceGroupConfig( @@ -218,7 +218,7 @@ Here is a good practice for managing QueryNodes in a cloud environment: Additionally, if we strictly enforce the constraint `sum(.requests.nodeNum) <= queryNodeNum`, we can precisely control the assignment of QueryNodes in the cluster. Let's assume there is currently only one QueryNode in the cluster and initialize the cluster. Here is an example setup: - ```Python + ```python from pymilvus import utility from pymilvus.client.types import ResourceGroupConfig @@ -261,7 +261,7 @@ Here is a good practice for managing QueryNodes in a cloud environment: 2. Cluster scale out Assuming we have the following scaling function: - ```Python + ```python def scale_to(node_num: int): # scale the querynode number in Milvus into node_num. @@ -269,7 +269,7 @@ Here is a good practice for managing QueryNodes in a cloud environment: ``` We can use the API to scale a specific resource group to a designated number of QueryNodes without affecting any other resource groups. - ```Python + ```python # scale rg1 into 3 nodes, rg2 into 1 nodes utility.update_resource_groups({ "rg1": ResourceGroupConfig( @@ -293,7 +293,7 @@ Here is a good practice for managing QueryNodes in a cloud environment: Similarly, we can establish scaling-in rules that prioritize selecting QueryNodes from **__pending_nodes** resource group. This information can be obtained through the `describe_resource_group` API. Achieving the goal of scaling-in specified resource group. - ```Python + ```python # scale rg1 from 3 nodes into 2 nodes utility.update_resource_groups({ "rg1": ResourceGroupConfig( diff --git a/v2.4.x/site/en/adminGuide/upgrade_milvus_cluster-docker.md b/v2.4.x/site/en/adminGuide/upgrade_milvus_cluster-docker.md index 04d3b2323d..0e6ed29198 100644 --- a/v2.4.x/site/en/adminGuide/upgrade_milvus_cluster-docker.md +++ b/v2.4.x/site/en/adminGuide/upgrade_milvus_cluster-docker.md @@ -10,7 +10,7 @@ title: Upgrade Milvus Cluster with Docker Compose This topic describes how to upgrade your Milvus using Docker Compose. -In normal cases, you can [upgrade Milvus by changing its image](#Upgrade-Milvus-by-changing-its-image). However, you need to [migrate the metadata](#Migrate-the-metadata) before any upgrade from v2.1.x to v2.4.17. +In normal cases, you can [upgrade Milvus by changing its image](#Upgrade-Milvus-by-changing-its-image). However, you need to [migrate the metadata](#Migrate-the-metadata) before any upgrade from v2.1.x to v2.5.x. ## Upgrade Milvus by changing its image @@ -24,35 +24,35 @@ In normal cases, you can upgrade Milvus as follows: ... rootcoord: container_name: milvus-rootcoord - image: milvusdb/milvus:v2.4.17 + image: milvusdb/milvus:v2.5.x ... proxy: container_name: milvus-proxy - image: milvusdb/milvus:v2.4.17 + image: milvusdb/milvus:v2.5.x ... querycoord: container_name: milvus-querycoord - image: milvusdb/milvus:v2.4.17 + image: milvusdb/milvus:v2.5.x ... querynode: container_name: milvus-querynode - image: milvusdb/milvus:v2.4.17 + image: milvusdb/milvus:v2.5.x ... indexcoord: container_name: milvus-indexcoord - image: milvusdb/milvus:v2.4.17 + image: milvusdb/milvus:v2.5.x ... indexnode: container_name: milvus-indexnode - image: milvusdb/milvus:v2.4.17 + image: milvusdb/milvus:v2.5.x ... datacoord: container_name: milvus-datacoord - image: milvusdb/milvus:v2.4.17 + image: milvusdb/milvus:v2.5.x ... datanode: container_name: milvus-datanode - image: milvusdb/milvus:v2.4.17 + image: milvusdb/milvus:v2.5.x ``` 2. Run the following commands to perform the upgrade. @@ -80,7 +80,7 @@ In normal cases, you can upgrade Milvus as follows: runWithBackup: true config: sourceVersion: 2.1.4 # Specify your milvus version - targetVersion: 2.4.17 + targetVersion: 2.5.x backupFilePath: /tmp/migration.bak metastore: type: etcd diff --git a/v2.4.x/site/en/adminGuide/upgrade_milvus_cluster-helm.md b/v2.4.x/site/en/adminGuide/upgrade_milvus_cluster-helm.md index 8b3298f3ee..047a10d9e9 100644 --- a/v2.4.x/site/en/adminGuide/upgrade_milvus_cluster-helm.md +++ b/v2.4.x/site/en/adminGuide/upgrade_milvus_cluster-helm.md @@ -85,11 +85,11 @@ zilliztech/milvus 4.1.0 2.3.0 Milvus is an ope You can choose the upgrade path for your Milvus as follows: -
- [Conduct a rolling upgrade](#conduct-a-rolling-upgrade) from Milvus v2.2.3 and later releases to v2.4.17.
+
- [Conduct a rolling upgrade](#conduct-a-rolling-upgrade) from Milvus v2.2.3 and later releases to v2.5.x.
-- [Upgrade Milvus using Helm](#Upgrade-Milvus-using-Helm) for an upgrade from a minor release before v2.2.3 to v2.4.17. +- [Upgrade Milvus using Helm](#Upgrade-Milvus-using-Helm) for an upgrade from a minor release before v2.2.3 to v2.5.x. -- [Migrate the metadata](#Migrate-the-metadata) before the upgrade from Milvus v2.1.x to v2.4.17. +- [Migrate the metadata](#Migrate-the-metadata) before the upgrade from Milvus v2.1.x to v2.5.x.
@@ -111,10 +111,10 @@ The script applies only to the upgrade of Milvus installed with Helm. The follow | `w` | New Milvus image tag | `milvusdb/milvus:v2.2.3` | True | | `o` | Operation | `update` | False | -Once you have ensured that all deployments in your Milvus instance are in their normal status. You can run the following command to upgrade the Milvus instance to 2.4.17. +Once you have ensured that all deployments in your Milvus instance are in their normal status. You can run the following command to upgrade the Milvus instance to 2.5.x. ```shell -sh rollingUpdate.sh -n default -i my-release -o update -t 2.4.17 -w 'milvusdb/milvus:v2.4.17' +sh rollingUpdate.sh -n default -i my-release -o update -t 2.5.x -w 'milvusdb/milvus:v2.5.x' ```
diff --git a/v2.4.x/site/en/adminGuide/upgrade_milvus_cluster-operator.md b/v2.4.x/site/en/adminGuide/upgrade_milvus_cluster-operator.md index e201122ac4..2837ef8268 100644 --- a/v2.4.x/site/en/adminGuide/upgrade_milvus_cluster-operator.md +++ b/v2.4.x/site/en/adminGuide/upgrade_milvus_cluster-operator.md @@ -26,9 +26,9 @@ helm -n milvus-operator upgrade milvus-operator zilliztech-milvus-operator/milvu Once you have upgraded your Milvus operator to the latest version, you have the following choices: -- To upgrade Milvus from v2.2.3 or later releases to 2.4.17, you can [conduct a rolling upgrade](#Conduct-a-rolling-upgrade). -- To upgrade Milvus from a minor release before v2.2.3 to 2.4.17, you are advised to [upgrade Milvus by changing its image version](#Upgrade-Milvus-by-changing-its-image). -- To upgrade Milvus from v2.1.x to 2.4.17, you need to [migrate the metadata](#Migrate-the-metadata) before the actual upgrade. +- To upgrade Milvus from v2.2.3 or later releases to 2.5.x, you can [conduct a rolling upgrade](#Conduct-a-rolling-upgrade). +- To upgrade Milvus from a minor release before v2.2.3 to 2.5.x, you are advised to [upgrade Milvus by changing its image version](#Upgrade-Milvus-by-changing-its-image). +- To upgrade Milvus from v2.1.x to 2.5.x, you need to [migrate the metadata](#Migrate-the-metadata) before the actual upgrade. ## Conduct a rolling upgrade @@ -47,7 +47,7 @@ spec: components: enableRollingUpdate: true imageUpdateMode: rollingUpgrade # Default value, can be omitted - image: milvusdb/milvus:v2.4.17 + image: milvusdb/milvus:v2.5.x ``` In this above configuration file, set `spec.components.enableRollingUpdate` to `true` and set `spec.components.image` to the desired Milvus version. @@ -63,7 +63,7 @@ spec: components: enableRollingUpdate: true imageUpdateMode: all - image: milvusdb/milvus:v2.4.17 + image: milvusdb/milvus:v2.5.x ``` You can set `spec.components.imageUpdateMode` to `rollingDowngrade` to have Milvus replace coordinator pod images with a lower version. @@ -101,7 +101,7 @@ metadata: spec: # Omit other fields ... components: - image: milvusdb/milvus:v2.4.17 + image: milvusdb/milvus:v2.5.x ``` Then run the following to perform the upgrade: @@ -112,11 +112,11 @@ kubectl patch -f milvusupgrade.yaml ## Migrate the metadata -Since Milvus 2.2.0, the metadata is incompatible with that in previous releases. The following example snippets assume an upgrade from Milvus 2.1.4 to Milvus 2.4.17. +Since Milvus 2.2.0, the metadata is incompatible with that in previous releases. The following example snippets assume an upgrade from Milvus 2.1.4 to Milvus 2.5.x. ### 1. Create a `.yaml` file for metadata migration -Create a metadata migration file. The following is an example. You need to specify the `name`, `sourceVersion`, and `targetVersion` in the configuration file. The following example sets the `name` to `my-release-upgrade`, `sourceVersion` to `v2.1.4`, and `targetVersion` to `v2.4.17`. This means that your Milvus cluster will be upgraded from v2.1.4 to v2.4.17. +Create a metadata migration file. The following is an example. You need to specify the `name`, `sourceVersion`, and `targetVersion` in the configuration file. The following example sets the `name` to `my-release-upgrade`, `sourceVersion` to `v2.1.4`, and `targetVersion` to `v2.5.x`. This means that your Milvus cluster will be upgraded from v2.1.4 to v2.5.x. ``` apiVersion: milvus.io/v1beta1 @@ -128,9 +128,9 @@ spec: namespace: default name: my-release sourceVersion: "v2.1.4" - targetVersion: "v2.4.17" + targetVersion: "v2.5.x" # below are some omit default values: - # targetImage: "milvusdb/milvus:v2.4.17" + # targetImage: "milvusdb/milvus:v2.5.x" # toolImage: "milvusdb/meta-migration:v2.2.0" # operation: upgrade # rollbackIfFailed: true diff --git a/v2.4.x/site/en/adminGuide/upgrade_milvus_standalone-docker.md b/v2.4.x/site/en/adminGuide/upgrade_milvus_standalone-docker.md index 5c8669584b..be187faccf 100644 --- a/v2.4.x/site/en/adminGuide/upgrade_milvus_standalone-docker.md +++ b/v2.4.x/site/en/adminGuide/upgrade_milvus_standalone-docker.md @@ -14,7 +14,7 @@ title: Upgrade Milvus Standalone with Docker Compose This topic describes how to upgrade your Milvus using Docker Compose. -In normal cases, you can [upgrade Milvus by changing its image](#Upgrade-Milvus-by-changing-its-image). However, you need to [migrate the metadata](#Migrate-the-metadata) before any upgrade from v2.1.x to v2.4.17. +In normal cases, you can [upgrade Milvus by changing its image](#Upgrade-Milvus-by-changing-its-image). However, you need to [migrate the metadata](#Migrate-the-metadata) before any upgrade from v2.1.x to v2.5.x.
@@ -32,7 +32,7 @@ In normal cases, you can upgrade Milvus as follows: ... standalone: container_name: milvus-standalone - image: milvusdb/milvus:v2.4.17 + image: milvusdb/milvus:v2.5.x ``` 2. Run the following commands to perform the upgrade. @@ -60,7 +60,7 @@ In normal cases, you can upgrade Milvus as follows: runWithBackup: true config: sourceVersion: 2.1.4 # Specify your milvus version - targetVersion: 2.4.17 + targetVersion: 2.5.x backupFilePath: /tmp/migration.bak metastore: type: etcd diff --git a/v2.4.x/site/en/adminGuide/upgrade_milvus_standalone-helm.md b/v2.4.x/site/en/adminGuide/upgrade_milvus_standalone-helm.md index 0afd40ee44..e140276d16 100644 --- a/v2.4.x/site/en/adminGuide/upgrade_milvus_standalone-helm.md +++ b/v2.4.x/site/en/adminGuide/upgrade_milvus_standalone-helm.md @@ -86,11 +86,11 @@ zilliztech/milvus 4.1.0 2.3.0 Milvus is an ope You can choose the upgrade path for your Milvus as follows: -
- [Conduct a rolling upgrade](#conduct-a-rolling-upgrade) from Milvus v2.2.3 and later releases to v2.4.17.
+
- [Conduct a rolling upgrade](#conduct-a-rolling-upgrade) from Milvus v2.2.3 and later releases to v2.5.x.
-- [Upgrade Milvus using Helm](#Upgrade-Milvus-using-Helm) for an upgrade from a minor release before v2.2.3 to v2.4.17. +- [Upgrade Milvus using Helm](#Upgrade-Milvus-using-Helm) for an upgrade from a minor release before v2.2.3 to v2.5.x. -- [Migrate the metadata](#Migrate-the-metadata) before the upgrade from Milvus v2.1.x to v2.4.17. +- [Migrate the metadata](#Migrate-the-metadata) before the upgrade from Milvus v2.1.x to v2.5.x.
@@ -112,10 +112,10 @@ The script applies only to the upgrade of Milvus installed with Helm. The follow | `w` | New Milvus image tag | `milvusdb/milvus:v2.2.3` | True | | `o` | Operation | `update` | False | -Once you have ensured that all deployments in your Milvus instance are in their normal status. You can run the following command to upgrade the Milvus instance to 2.4.17. +Once you have ensured that all deployments in your Milvus instance are in their normal status. You can run the following command to upgrade the Milvus instance to 2.5.x. ```shell -sh rollingUpdate.sh -n default -i my-release -o update -t 2.4.17 -w 'milvusdb/milvus:v2.4.17' +sh rollingUpdate.sh -n default -i my-release -o update -t 2.5.x -w 'milvusdb/milvus:v2.5.x' ```
@@ -203,42 +203,42 @@ The following table lists the operations you can do for meta migration. 4. Migrate the Milvus metadata. 5. Start Milvus components with a new image. -#### 2. Upgrade Milvus from v2.1.x to 2.4.17 +#### 2. Upgrade Milvus from v2.1.x to 2.5.x -The following commands assume that you upgrade Milvus from v2.1.4 to 2.4.17. Change them to the versions that fit your needs. +The following commands assume that you upgrade Milvus from v2.1.4 to 2.5.x. Change them to the versions that fit your needs. 1. Specify Milvus instance name, source Milvus version, and target Milvus version. ``` - ./migrate.sh -i my-release -s 2.1.4 -t 2.4.17 + ./migrate.sh -i my-release -s 2.1.4 -t 2.5.x ``` 2. Specify the namespace with `-n` if your Milvus is not installed in the default K8s namespace. ``` - ./migrate.sh -i my-release -n milvus -s 2.1.4 -t 2.4.17 + ./migrate.sh -i my-release -n milvus -s 2.1.4 -t 2.5.x ``` 3. Specify the root path with `-r` if your Milvus is installed with the custom `rootpath`. ``` - ./migrate.sh -i my-release -n milvus -s 2.1.4 -t 2.4.17 -r by-dev + ./migrate.sh -i my-release -n milvus -s 2.1.4 -t 2.5.x -r by-dev ``` 4. Specify the image tag with `-w` if your Milvus is installed with a custom `image`. ``` - ./migrate.sh -i my-release -n milvus -s 2.1.4 -t 2.4.17 -r by-dev -w milvusdb/milvus:v2.4.17 + ./migrate.sh -i my-release -n milvus -s 2.1.4 -t 2.5.x -r by-dev -w milvusdb/milvus:v2.5.x ``` 5. Set `-d true` if you want to automatically remove the migration pod after the migration is completed. ``` - ./migrate.sh -i my-release -n milvus -s 2.1.4 -t 2.4.17 -w milvusdb/milvus:v2.4.17 -d true + ./migrate.sh -i my-release -n milvus -s 2.1.4 -t 2.5.x -w milvusdb/milvus:v2.5.x -d true ``` 6. Rollback and migrate again if the migration fails. ``` - ./migrate.sh -i my-release -n milvus -s 2.1.4 -t 2.4.17 -r by-dev -o rollback -w milvusdb/milvus:v2.1.1 - ./migrate.sh -i my-release -n milvus -s 2.1.4 -t 2.4.17 -r by-dev -o migrate -w milvusdb/milvus:v2.4.17 + ./migrate.sh -i my-release -n milvus -s 2.1.4 -t 2.5.x -r by-dev -o rollback -w milvusdb/milvus:v2.1.1 + ./migrate.sh -i my-release -n milvus -s 2.1.4 -t 2.5.x -r by-dev -o migrate -w milvusdb/milvus:v2.5.x diff --git a/v2.4.x/site/en/adminGuide/upgrade_milvus_standalone-operator.md b/v2.4.x/site/en/adminGuide/upgrade_milvus_standalone-operator.md index 0faa4d4b9e..fd4f884f3e 100644 --- a/v2.4.x/site/en/adminGuide/upgrade_milvus_standalone-operator.md +++ b/v2.4.x/site/en/adminGuide/upgrade_milvus_standalone-operator.md @@ -26,9 +26,9 @@ helm -n milvus-operator upgrade milvus-operator zilliztech-milvus-operator/milvu Once you have upgraded your Milvus operator to the latest version, you have the following choices: -- To upgrade Milvus from v2.2.3 or later releases to 2.4.17, you can [conduct a rolling upgrade](#Conduct-a-rolling-upgrade). -- To upgrade Milvus from a minor release before v2.2.3 to 2.4.17, you are advised to [upgrade Milvus by changing its image version](#Upgrade-Milvus-by-changing-its-image). -- To upgrade Milvus from v2.1.x to 2.4.17, you need to [migrate the metadata](#Migrate-the-metadata) before the actual upgrade. +- To upgrade Milvus from v2.2.3 or later releases to 2.5.x, you can [conduct a rolling upgrade](#Conduct-a-rolling-upgrade). +- To upgrade Milvus from a minor release before v2.2.3 to 2.5.x, you are advised to [upgrade Milvus by changing its image version](#Upgrade-Milvus-by-changing-its-image). +- To upgrade Milvus from v2.1.x to 2.5.x, you need to [migrate the metadata](#Migrate-the-metadata) before the actual upgrade. ## Conduct a rolling upgrade @@ -47,7 +47,7 @@ spec: components: enableRollingUpdate: true imageUpdateMode: rollingUpgrade # Default value, can be omitted - image: milvusdb/milvus:v2.4.17 + image: milvusdb/milvus:v2.5.x ``` In this above configuration file, set `spec.components.enableRollingUpdate` to `true` and set `spec.components.image` to the desired Milvus version. @@ -63,7 +63,7 @@ spec: components: enableRollingUpdate: true imageUpdateMode: all - image: milvusdb/milvus:v2.4.17 + image: milvusdb/milvus:v2.5.x ``` You can set `spec.components.imageUpdateMode` to `rollingDowngrade` to have Milvus replace coordinator pod images with a lower version. @@ -102,7 +102,7 @@ labels: spec: # Omit other fields ... components: - image: milvusdb/milvus:v2.4.17 + image: milvusdb/milvus:v2.5.x ``` Then run the following to perform the upgrade: @@ -113,11 +113,11 @@ kubectl patch -f milvusupgrade.yaml ## Migrate the metadata -Since Milvus 2.2.0, the metadata is incompatible with that in previous releases. The following example snippets assume an upgrade from Milvus 2.1.4 to Milvus v2.4.17. +Since Milvus 2.2.0, the metadata is incompatible with that in previous releases. The following example snippets assume an upgrade from Milvus 2.1.4 to Milvus v2.5.x. ### 1. Create a `.yaml` file for metadata migration -Create a metadata migration file. The following is an example. You need to specify the `name`, `sourceVersion`, and `targetVersion` in the configuration file. The following example sets the `name` to `my-release-upgrade`, `sourceVersion` to `v2.1.4`, and `targetVersion` to `v2.4.17`. This means that your Milvus instance will be upgraded from v2.1.4 to v2.4.17. +Create a metadata migration file. The following is an example. You need to specify the `name`, `sourceVersion`, and `targetVersion` in the configuration file. The following example sets the `name` to `my-release-upgrade`, `sourceVersion` to `v2.1.4`, and `targetVersion` to `v2.5.x`. This means that your Milvus instance will be upgraded from v2.1.4 to v2.5.x. ``` apiVersion: milvus.io/v1beta1 @@ -129,9 +129,9 @@ spec: namespace: default name: my-release sourceVersion: "v2.1.4" - targetVersion: "v2.4.17" + targetVersion: "v2.5.x" # below are some omit default values: - # targetImage: "milvusdb/milvus:v2.4.17" + # targetImage: "milvusdb/milvus:v2.5.x" # toolImage: "milvusdb/meta-migration:v2.2.0" # operation: upgrade # rollbackIfFailed: true diff --git a/v2.4.x/site/en/getstarted/run-milvus-docker/install_standalone-docker-compose.md b/v2.4.x/site/en/getstarted/run-milvus-docker/install_standalone-docker-compose.md index 57d0e1a529..14bfea6c8f 100644 --- a/v2.4.x/site/en/getstarted/run-milvus-docker/install_standalone-docker-compose.md +++ b/v2.4.x/site/en/getstarted/run-milvus-docker/install_standalone-docker-compose.md @@ -21,7 +21,7 @@ Milvus provides a Docker Compose configuration file in the Milvus repository. To ```shell # Download the configuration file -$ wget https://github.com/milvus-io/milvus/releases/download/v2.4.17/milvus-standalone-docker-compose.yml -O docker-compose.yml +$ wget https://github.com/milvus-io/milvus/releases/download/v2.5.x/milvus-standalone-docker-compose.yml -O docker-compose.yml # Start Milvus $ sudo docker-compose up -d diff --git a/v2.4.x/site/en/getstarted/run-milvus-gpu/install_standalone-docker-compose-gpu.md b/v2.4.x/site/en/getstarted/run-milvus-gpu/install_standalone-docker-compose-gpu.md index 6117f70db5..367fa24f80 100644 --- a/v2.4.x/site/en/getstarted/run-milvus-gpu/install_standalone-docker-compose-gpu.md +++ b/v2.4.x/site/en/getstarted/run-milvus-gpu/install_standalone-docker-compose-gpu.md @@ -27,10 +27,10 @@ To install Milvus with GPU support using Docker Compose, follow these steps. ### 1. Download and configure the YAML file -Download [`milvus-standalone-docker-compose-gpu.yml`](https://github.com/milvus-io/milvus/releases/download/v2.4.17/milvus-standalone-docker-compose-gpu.yml) and save it as docker-compose.yml manually, or with the following command. +Download [`milvus-standalone-docker-compose-gpu.yml`](https://github.com/milvus-io/milvus/releases/download/v2.5.x/milvus-standalone-docker-compose-gpu.yml) and save it as docker-compose.yml manually, or with the following command. ```shell -$ wget https://github.com/milvus-io/milvus/releases/download/v2.4.17/milvus-standalone-docker-compose-gpu.yml -O docker-compose.yml +$ wget https://github.com/milvus-io/milvus/releases/download/v2.5.x/milvus-standalone-docker-compose-gpu.yml -O docker-compose.yml ``` You need to make some changes to the environment variables of the standalone service in the YAML file as follows: diff --git a/v2.4.x/site/en/menuStructure/en.json b/v2.4.x/site/en/menuStructure/en.json index e97797e2d8..fcb4f0a213 100644 --- a/v2.4.x/site/en/menuStructure/en.json +++ b/v2.4.x/site/en/menuStructure/en.json @@ -325,171 +325,446 @@ "order": 3, "children": [ { - "label": "Manage Databases", + "label": "Database", "id": "manage_databases.md", "order": 0, "children": [] }, { - "label": "Manage Schema", - "id": "schema.md", + "label": "Collections", + "id": "collections", "order": 1, - "children": [] + "children": [ + { + "label": "Collection Explained", + "id": "manage-collections.md", + "order": 0, + "children": [] + }, + { + "label": "Create Collection", + "id": "create-collection.md", + "order": 1, + "children": [] + }, + { + "label": "Create Collection Instantly", + "id": "create-collection-instantly.md", + "order": 2, + "children": [] + }, + { + "label": "View Collections", + "id": "view-collections.md", + "order": 3, + "children": [] + }, + { + "label": "Modify Collection", + "id": "modify-collection.md", + "order": 4, + "children": [] + }, + { + "label": "Load & Release", + "id": "load-and-release.md", + "order": 5, + "children": [] + }, + { + "label": "Manage Partitions", + "id": "manage-partitions.md", + "order": 6, + "children": [] + }, + { + "label": "Manage Aliases", + "id": "manage-aliases.md", + "order": 7, + "children": [] + }, + { + "label": "Drop Collection", + "id": "drop-collection.md", + "order": 8, + "children": [] + } + ] }, { - "label": "Manage Collections", - "id": "manage-collections.md", + "label": "Schema & Data Fields", + "id": "schema", "order": 2, - "children": [] - }, - { - "label": "Manage Indexes", - "id": "manage_indexes", - "order": 3, - "isMenu": true, "children": [ { - "label": "Index Vector Fields", - "id": "index-vector-fields.md", + "label": "Schema Explained", + "id": "schema.md", "order": 0, "children": [] }, { - "label": "Index Scalar Fields", - "id": "index-scalar-fields.md", + "label": "Primary Field & AutoID", + "id": "primary-field.md", "order": 1, "children": [] }, { - "label": "Index with GPU", - "id": "index-with-gpu.md", + "label": "Dense Vector", + "id": "dense-vector.md", "order": 2, "children": [] + }, + { + "label": "Binary Vector", + "id": "binary-vector.md", + "order": 3, + "children": [] + }, + { + "label": "Sparse Vector", + "id": "sparse_vector.md", + "order": 4, + "children": [] + }, + { + "label": "String Field", + "id": "string.md", + "order": 5, + "children": [] + }, + { + "label": "Number Field", + "id": "number.md", + "order": 6, + "children": [] + }, + { + "label": "JSON Field", + "id": "use-json-fields.md", + "order": 7, + "children": [] + }, + { + "label": "Array Field", + "id": "array.md", + "order": 8, + "children": [] + }, + { + "label": "Dynamic Field", + "id": "enable-dynamic-field.md", + "order": 9, + "children": [] + }, + { + "label": "Nullable & Default", + "id": "nullable-and-default.md", + "order": 10, + "children": [] + }, + { + "label": "Analyzer", + "id": "analyzer", + "order": 11, + "children": [ + { + "label": "Analyzer Overview", + "id": "analyzer-overview.md", + "order": 0, + "children": [] + }, + { + "label": "Built-in Analyzers", + "id": "built-in-analyzers", + "order": 1, + "children": [ + { + "label": "Standard", + "id": "standard-analyzer.md", + "order": 0, + "children": [] + }, + { + "label": "English", + "id": "english-analyzer.md", + "order": 1, + "children": [] + }, + { + "label": "Chinese", + "id": "chinese-analyzer.md", + "order": 2, + "children": [] + } + ] + }, + { + "label": "Tokenizers", + "id": "tokenizers", + "order": 2, + "children": [ + { + "label": "Standard", + "id": "standard-tokenizer.md", + "order": 0, + "children": [] + }, + { + "label": "Whitespace", + "id": "whitespace-tokenizer.md", + "order": 1, + "children": [] + }, + { + "label": "Jieba", + "id": "jieba-tokenizer.md", + "order": 1, + "children": [] + } + ] + }, + { + "label": "Filters", + "id": "filters", + "order": 3, + "children": [ + { + "label": "Lowercase", + "id": "lowercase-filter.md", + "order": 0, + "children": [] + }, + { + "label": "ASCII Folding", + "id": "ascii-folding-filter.md", + "order": 1, + "children": [] + }, + { + "label": "Alphanumonly", + "id": "alphanumonly-filter.md", + "order": 2, + "children": [] + }, + { + "label": "Cnalphanumonly", + "id": "cnalphanumonly-filter.md", + "order": 3, + "children": [] + }, + { + "label": "Cncharonly", + "id": "cncharonly-filter.md", + "order": 4, + "children": [] + }, + { + "label": "Length", + "id": "length-filter.md", + "order": 5, + "children": [] + }, + { + "label": "Stop", + "id": "stop-filter.md", + "order": 6, + "children": [] + }, + { + "label": "Decompounder", + "id": "decompounder-filter.md", + "order": 7, + "children": [] + }, + { + "label": "Stemmer", + "id": "stemmer-filter.md", + "order": 8, + "children": [] + } + ] + } + ] + }, + { + "label": "Hands-On", + "id": "schema-hands-on.md", + "order": 12, + "children": [] } ] }, { - "label": "Manage Partitions", - "id": "manage-partitions.md", - "order": 4, - "children": [] - }, - { - "label": "Insert, Upsert & Delete", - "id": "insert-update-delete.md", - "order": 5, - "children": [] - }, - { - "label": "Data Import", - "id": "data_import", - "order": 6, - "isMenu": true, + "label": "Insert & Delete", + "id": "insert-and-delete", + "order": 3, "children": [ { - "label": "Prepare Source Data", - "id": "prepare-source-data.md", + "label": "Insert Entities", + "id": "insert-update-delete.md", "order": 0, "children": [] }, { - "label": "Import Data", - "id": "import-data.md", + "label": "Upsert Entities", + "id": "upsert-entities.md", "order": 1, "children": [] + }, + { + "label": "Delete Entities", + "id": "delete-entities.md", + "order": 2, + "children": [] } ] }, { - "label": "Search, Query & Get", - "id": "search_query_get", - "order": 7, + "label": "Manage Indexes", + "id": "manage_indexes", + "order": 4, "isMenu": true, "children": [ { - "label": "Single-Vector Search", - "id": "single-vector-search.md", + "label": "Index Vector Fields", + "id": "index-vector-fields.md", "order": 0, "children": [] }, { - "label": "Hybrid Search", - "id": "multi-vector-search.md", + "label": "Index Scalar Fields", + "id": "index-scalar-fields.md", "order": 1, "children": [] }, { - "label": "Get & Scalar Query", - "id": "get-and-scalar-query.md", + "label": "Index with GPU", + "id": "index-with-gpu.md", "order": 2, "children": [] - }, - { - "label": "With Iterators", - "id": "with-iterators.md", - "order": 3, - "children": [] } ] }, { - "label": "Advanced Features", - "id": "advanced_features", - "order": 8, + "label": "Search & Rerank", + "id": "search-rerank", + "order": 5, "isMenu": true, "children": [ { - "label": "Dynamic Field", - "id": "enable-dynamic-field.md", + "label": "Basic ANN Search", + "id": "single-vector-search.md", "order": 0, "children": [] }, { - "label": "Partition Key", - "id": "use-partition-key.md", + "label": "Filtered Search", + "id": "filtered-search.md", "order": 1, "children": [] }, { - "label": "JSON Field", - "id": "use-json-fields.md", + "label": "Range Search", + "id": "range-search.md", "order": 2, "children": [] }, { - "label": "Array Field", - "id": "array_data_type.md", + "label": "Grouping Search", + "id": "grouping-search.md", "order": 3, "children": [] }, { - "label": "Sparse Vector", - "id": "sparse_vector.md", + "label": "Hybrid Search", + "id": "multi-vector-search.md", "order": 4, "children": [] }, { - "label": "Scalar Filtering Rules", - "id": "boolean.md", + "label": "Query", + "id": "get-and-scalar-query.md", "order": 5, "children": [] }, { - "label": "MMap Storage", - "id": "mmap.md", + "label": "Full Text Search", + "id": "full-text-search.md", "order": 6, "children": [] }, { - "label": "Manage Resource Groups", - "id": "resource_group.md", + "label": "Keyword Match", + "id": "keyword-match.md", "order": 7, "children": [] }, + { + "label": "Search Iterators", + "id": "with-iterators.md", + "order": 8, + "children": [] + }, + { + "label": "Use Partition Key", + "id": "use-partition-key.md", + "order": 9, + "children": [] + }, + { + "label": "Use mmap", + "id": "mmap.md", + "order": 10, + "children": [] + }, { "label": "Clustering Compaction", "id": "clustering-compaction.md", - "order": 8, + "order": 11, + "children": [] + }, + { + "label": "Consistency Level", + "id": "consistency.md", + "order": 12, + "children": [] + }, + { + "label": "Metric Types", + "id": "metric.md", + "order": 13, + "children": [] + }, + { + "label": "Metadata Filtering", + "id": "boolean.md", + "order": 14, + "children": [] + }, + { + "label": "Reranking", + "id": "reranking.md", + "order": 15, + "children": [] + } + ] + }, + { + "label": "Data Import", + "id": "data_import", + "order": 6, + "isMenu": true, + "children": [ + { + "label": "Prepare Source Data", + "id": "prepare-source-data.md", + "order": 0, + "children": [] + }, + { + "label": "Import Data", + "id": "import-data.md", + "order": 1, "children": [] } ] @@ -1039,10 +1314,23 @@ } ] }, + { + "label": "Resource Groups", + "id": "resource_groups", + "order": 9, + "children": [ + { + "label": "Manage Resource Groups", + "id": "resource_group.md", + "order": 0, + "children": [] + } + ] + }, { "label": "Security", "id": "security", - "order": 9, + "order": 10, "isMenu": true, "children": [ { diff --git a/v2.4.x/site/en/reference/array_data_type.md b/v2.4.x/site/en/reference/array_data_type.md deleted file mode 100644 index a75a7f5d54..0000000000 --- a/v2.4.x/site/en/reference/array_data_type.md +++ /dev/null @@ -1,1265 +0,0 @@ ---- -id: array_data_type.md -title: Use Array Fields ---- - -# Use Array Fields - -This guide explains how to use the array fields, such as inserting array values, creating indexes on vector and array fields, as well as searching and querying in array fields with basic and advanced operators. - -## Prerequisites - -Ensure you have the following: - -- Milvus installed and running. For information on how to install Milvus, refer to [Install Milvus](install-overview.md). -- One of Milvus SDKs installed in your environment. For details, refer to [Install SDKs](install-pymilvus.md). - -## Prepare data with an array field - -Milvus supports arrays as one of the field data types. An array in a Milvus collection should always have elements of the same data type, and the data type for array elements can be any of the supported data types in Milvus. For a list of supported data types, refer to [Supported data types](https://milvus.io/docs/schema.md#Supported-data-types). - -The following code snippet generates a random dataset containing an array field named `color_coord`, with all elements of the interger data type. - -
- Python - Java - Node.js -
- -```python -import random - -colors = ["green", "blue", "yellow", "red", "black", "white", "purple", "pink", "orange", "brown", "grey"] -data = [] - -for i in range(1000): - current_color = random.choice(colors) - current_tag = random.randint(1000, 9999) - current_coord = [ random.randint(0, 40) for _ in range(random.randint(3, 5)) ] - data.append({ - "id": i, - "vector": [ random.uniform(-1, 1) for _ in range(5) ], - "color": current_color, - "color_tag": current_tag, - "color_coord": current_coord, - }) - -print(data[0]) -``` - -```java -List colors = Arrays.asList("green", "blue", "yellow", "red", "black", "white", "purple", "pink", "orange", "brown", "grey"); -List data = new ArrayList<>(); - -for (int i=0; i<1000; i++) { - Random rand = new Random(); - String current_color = colors.get(rand.nextInt(colors.size()-1)); - Long current_tag = rand.nextLong(8999L) + 1000L; - - // Generate an random-sized array - Long capacity = rand.nextLong(5L) + 1L; - List current_coord = new ArrayList<>(); - current_coord.add(rand.nextLong(40L) + 1L); - current_coord.add(rand.nextLong(40L) + 1L); - for (int j=3; j Math.floor(Math.random() * 40)); - - data.push({ - id: i, - vector: Array(5).fill(0).map(() => Math.random()), - color: current_color, - color_tag: current_tag, - color_coord: current_coord, - }); -} - -console.log(data[0]); -``` - -This code snippet prepares a list of random colors and generates a dataset containing 1,000 entities. Each entity has an ID, a vector of five floating-point numbers, a color, a color tag, and an array field `color_coord` containing between 3 to 5 integer values. The sample data is printed to verify its structure. - -Output structure: - -```json -{ - id: 0, - vector: [ - 0.0338537420906162, - 0.6844108238358322, - 0.28410588909961754, - 0.09752595400212116, - 0.22671013058761114 - ], - color: 'orange', - color_tag: 5677, - color_coord: [ 3, 0, 18, 29 ] -} -``` - -## Set up MilvusClient - -To interact with Milvus, set up the Milvus client by specifying the server address. - -
- Python - Java - Node.js -
- -```python -from pymilvus import MilvusClient, DataType - -SERVER_ADDR = "http://localhost:19530" - -client = MilvusClient(uri=SERVER_ADDR) -``` - -```java -import io.milvus.v2.client.ConnectConfig; -import io.milvus.v2.client.MilvusClientV2; -import io.milvus.v2.common.DataType; - -String SERVER_ADDR = "http://localhost:19530"; - -// 1. Connect to Milvus server -ConnectConfig connectConfig = ConnectConfig.builder() - .uri(SERVER_ADDR) - .build(); - -MilvusClientV2 client = new MilvusClientV2(connectConfig); -``` - -```javascript -import { MilvusClient, DataType } from "@zilliz/milvus2-sdk-node"; - -// Connect to Milvus server -const address = "http://localhost:19530"; - -const milvusClient = new MilvusClient({address: address}); -``` - -## Create a collection with an array field - -### Define collection schema - -A schema defines the structure of the collection, including the fields and their data types. The example below defines a collection schema matching the sample data generated in the [previous section](#prepare-data-with-an-array-field). - -To configure an array field in a collection: - -
- -1. Set the `datatype`: Configure it as `DataType.ARRAY`. -2. Specify the `element_type`: Choose the data type for the elements in the array. Elements in an array field should all have the same data type. In this example, the `element_type` is set to `DataType.INT64`. -3. Define the `max_capacity`: Set this parameter to specify the maximum number of elements the array field can hold. - -
- -
- -1. Set the `dataType`: Configure it as `DataType.Array`. -2. Specify the `elementType`: Choose the data type for the elements in the array. Elements in an array field should all have the same data type. In this example, the `elementType` is set to `DataType.Int64`. -3. Define the `maxCapacity`: Set this parameter to specify the maximum number of elements the array field can hold. - -
- -
- -1. Set the `data_type`: Configure it as `DataType.Array`. -2. Specify the `element_type`: Choose the data type for the elements in the array. Elements in an array field should all have the same data type. In this example, the `element_type` is set to `DataType.Int64`. -3. Define the `max_capacity`: Set this parameter to specify the maximum number of elements the array field can hold. - -
- -The example code below defines the collection schema with an array field `color_coord`, with a maximum of 5 elements and each element of the integer data type. - -
- Python - Java - Node.js -
- -```python -schema = client.create_schema(auto_id=False, enable_dynamic_field=False) - -schema.add_field(field_name="id", datatype=DataType.INT64, is_primary=True) -schema.add_field(field_name="vector", datatype=DataType.FLOAT_VECTOR, dim=5) -schema.add_field(field_name="color", datatype=DataType.VARCHAR, max_length=512) -schema.add_field(field_name="color_tag", datatype=DataType.INT64) -schema.add_field(field_name="color_coord", datatype=DataType.ARRAY, element_type=DataType.INT64, max_capacity=5) -``` - -```java -// Create schema -CreateCollectionReq.CollectionSchema schema = client.createSchema(); - -// Add fields to schema -schema.addField(AddFieldReq.builder() - .fieldName("id") - .dataType(DataType.Int64) - .isPrimaryKey(true) - .autoID(false) - .build()); - -schema.addField(AddFieldReq.builder() - .fieldName("vector") - .dataType(DataType.FloatVector) - .dimension(5) - .build()); - -schema.addField(AddFieldReq.builder() - .fieldName("color") - .dataType(DataType.VarChar) - .maxLength(512) - .build()); - -schema.addField(AddFieldReq.builder() - .fieldName("color_tag") - .dataType(DataType.Int64) - .build()); - -schema.addField(AddFieldReq.builder() - .fieldName("color_coord") - .dataType(DataType.Array) - .elementType(DataType.Int64) - .maxCapacity(5) - .build()); -``` - -```javascript -const fields = [ - { - name: "id", - data_type: DataType.Int64, - is_primary_key: true, - auto_id: false - }, - { - name: "vector", - data_type: DataType.FloatVector, - dim: 5 - }, - { - name: "color", - data_type: DataType.VarChar, - max_length: 512 - }, - { - name: "color_tag", - data_type: DataType.Int64, - }, - { - name: "color_coord", - data_type: DataType.Array, - element_type: DataType.Int64, - max_capacity: 5 - } -]; -``` - -
- -For more information on methods and parameters, refer to [create_schema](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Collections/create_schema.md) and [add_field](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/CollectionSchema/add_field.md). - -
- -
- -For more information on methods and parameters, refer to [createSchema](https://milvus.io/api-reference/java/v2.4.x/v2/Collections/createSchema.md) and [addField](https://milvus.io/api-reference/java/v2.4.x/v2/CollectionSchema/addField.md). - -
- -
- -For more information on methods and parameters, refer to [createCollection](https://milvus.io/api-reference/node/v2.4.x/Collections/createCollection.md). - -
- -### Create the collection - -Then, create the collection using the defined schema. - -
- Python - Java - Node.js -
- -```python -client.create_collection(collection_name="test_collection", schema=schema) -client.list_collections() - -# Output: -# ['test_collection'] -``` - -```java -CreateCollectionReq customizedSetupReq = CreateCollectionReq.builder() - .collectionName("test_collection") - .collectionSchema(schema) - .build(); - -client.createCollection(customizedSetupReq); -``` - -```javascript -await client.createCollection({ - collection_name: "test_collection", - fields: fields -}); - -const res = await client.listCollections({collection_name: "test_collection"}); - -console.log("Existing collections: " + res.collection_names); - -// Output: -// Existing collections: test_collection -``` - -
- -For more information on methods and parameters, refer to [create_collection](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Collections/create_collection.md) and [list_collections](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Collections/list_collections.md). - -
- -
- -For more information on methods and parameters, refer to [createCollection](https://milvus.io/api-reference/java/v2.4.x/v2/Collections/createCollection.md). - -
- -
- -For more information on methods and parameters, refer to [createCollection](https://milvus.io/api-reference/node/v2.4.x/Collections/createCollection.md) and [listCollections](https://milvus.io/api-reference/node/v2.4.x/Collections/listCollections.md). - -
- -## Create indexes - -Indexes improve the performance of search and query operations. In Milvus, you can create indexes on both vector fields and scalar fields. In this example, we'll create an `IVF_FLAT` index on the vector field `vector` and an `INVERTED` index on the array field `color_coord`. For more information on indexes, refer to [Index Vector Fields](https://milvus.io/docs/index-vector-fields.md?tab=floating) and [Index Scalar Fields](https://milvus.io/docs/index-scalar-fields.md). - -### Index vector field - -Creating an index on a vector field can improve the performance of vector similarity search, which is necessary for each search operation. - -The example below creates an index of type `IVF_FLAT` on the vector field `vector`. - -
- Python - Java - Node.js -
- -```python -index_params = client.prepare_index_params() - -index_params.add_index( - field_name="vector", - metric_type="COSINE", - index_type="IVF_FLAT", - index_name="vector_index", - params={"nlist": 128} -) - -client.create_index(collection_name="test_collection", index_params=index_params) -client.describe_index(collection_name="test_collection", index_name="vector_index") - -# Output: -# {'nlist': '128', -# 'index_type': 'IVF_FLAT', -# 'metric_type': 'COSINE', -# 'field_name': 'vector', -# 'index_name': 'vector_index'} -``` - -```java -IndexParam indexParam = IndexParam.builder() - .metricType(IndexParam.MetricType.COSINE) - .indexType(IndexParam.IndexType.IVF_FLAT) - .fieldName("vector") - .indexName("vector_index") - .build(); -CreateIndexReq createIndexReq = CreateIndexReq.builder() - .collectionName("test_collection") - .indexParams(Collections.singletonList(indexParam)) - .build(); -client.createIndex(createIndexReq); -``` - -```javascript -await client.createIndex({ - collection_name: "test_collection", - field_name: "vector", - index_type: "IVF_FLAT", - metric_type: "COSINE", - index_name: "vector_index", - params: { "nlist": 128 } -}); - -res = await client.describeIndex({ - collection_name: "test_collection", - index_name: "vector_index" -}); - -console.log("Vector index description: " + JSON.stringify(res)); - -// Output: -// Vector index description: {"index_descriptions":[{"params":[{"key":"params","value":"{\"nlist\":128}"},{"key":"index_type","value":"IVF_FLAT"},{"key":"metric_type","value":"COSINE"}],"index_name":"vector_index","indexID":"451543183233666062","field_name":"vector","indexed_rows":"0","total_rows":"0","state":"Finished","index_state_fail_reason":"","pending_index_rows":"0"}],"status":{"extra_info":{},"error_code":"Success","reason":"","code":0,"retriable":false,"detail":""}} -``` - -
- -For more information on methods and parameters, refer to [prepare_index_params](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Management/prepare_index_params.md), [create_index](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Management/create_index.md), and [describe_index](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Management/describe_index.md). - -
- -
- -For more information on methods and parameters, refer to [IndexParam](https://milvus.io/api-reference/java/v2.4.x/v2/Management/IndexParam.md) and [createIndex](https://milvus.io/api-reference/java/v2.4.x/v2/Management/createIndex.md). - -
- -
- -For more information on methods and parameters, refer to [createIndex](https://milvus.io/api-reference/node/v2.4.x/Management/createIndex.md), and [describeIndex](https://milvus.io/api-reference/node/v2.4.x/Management/describeIndex.md). - -
- -### Index array field - -Creating an index on a scalar field can improve the retrieval performance of queries on that field, which is optional but recommended for large datasets. - -In this example, we'll create an inverted index on the `color_coord` array field. This will allow us to speed up filtering based on this field. The inverted index demonstrates excellent overall performance, significantly outperforming brute force filtering using raw data when data is not frequently retrieved, and maintaining comparable performance with frequent retrieval operations. For more information on inverted indexes, refer to [Scalar Index](scalar_index.md#Inverted-indexing). - -
- Python - Java - Node.js -
- -```python -index_params = client.prepare_index_params() - -index_params.add_index( - field_name="color_coord", - index_type="INVERTED", - index_name="inverted_index" -) - -client.create_index(collection_name="test_collection", index_params=index_params) -client.describe_index(collection_name="test_collection", index_name="inverted_index") - -# Output: -# {'index_type': 'INVERTED', -# 'field_name': 'color_coord', -# 'index_name': 'inverted_index'} -``` - -```java -IndexParam indexParam = IndexParam.builder() - .indexType(IndexParam.IndexType.INVERTED) - .fieldName("color_coord") - .indexName("inverted_index") - .build(); -CreateIndexReq createIndexReq = CreateIndexReq.builder() - .collectionName("test_collection") - .indexParams(Collections.singletonList(indexParam)) - .build(); -client.createIndex(createIndexReq); -``` - -```javascript -await client.createIndex({ - collection_name: "test_collection", - field_name: "color_coord", - index_type: "INVERTED", - index_name: "inverted_index" -}); - -res = await client.describeIndex({ - collection_name: "test_collection", - index_name: "inverted_index" -}); - -console.log("Array index description: " + JSON.stringify(res)); - -// Output: -// Array index description: {"index_descriptions":[{"params":[{"key":"index_type","value":"INVERTED"}],"index_name":"inverted_index","indexID":"451543183233667243","field_name":"color_coord","indexed_rows":"0","total_rows":"0","state":"Finished","index_state_fail_reason":"","pending_index_rows":"0"}],"status":{"extra_info":{},"error_code":"Success","reason":"","code":0,"retriable":false,"detail":""}} -``` - -
- -For more information on methods and parameters, refer to [prepare_index_params](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Management/prepare_index_params.md), [create_index](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Management/create_index.md), and [describe_index](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Management/describe_index.md). - -
- -
- -For more information on methods and parameters, refer to [IndexParam](https://milvus.io/api-reference/java/v2.4.x/v2/Management/IndexParam.md) and [createIndex](https://milvus.io/api-reference/java/v2.4.x/v2/Management/createIndex.md). - -
- -
- -For more information on methods and parameters, refer to [createIndex](https://milvus.io/api-reference/node/v2.4.x/Management/createIndex.md), and [describeIndex](https://milvus.io/api-reference/node/v2.4.x/Management/describeIndex.md). - -
- -## Insert data - -Once the collection and indexes are created, we can insert the data into the collection. This step inserts 1,000 entities into the `test_collection`. - -
- Python - Java - Node.js -
- -```python -res = client.insert(collection_name="test_collection", data=data) -print(res) - -# Output: -# {'insert_count': 1000, 'ids': [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, 329, 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, 340, 341, 342, 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, 426, 427, 428, 429, 430, 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, 455, 456, 457, 458, 459, 460, 461, 462, 463, 464, 465, 466, 467, 468, 469, 470, 471, 472, 473, 474, 475, 476, 477, 478, 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, 527, 528, 529, 530, 531, 532, 533, 534, 535, 536, 537, 538, 539, 540, 541, 542, 543, 544, 545, 546, 547, 548, 549, 550, 551, 552, 553, 554, 555, 556, 557, 558, 559, 560, 561, 562, 563, 564, 565, 566, 567, 568, 569, 570, 571, 572, 573, 574, 575, 576, 577, 578, 579, 580, 581, 582, 583, 584, 585, 586, 587, 588, 589, 590, 591, 592, 593, 594, 595, 596, 597, 598, 599, 600, 601, 602, 603, 604, 605, 606, 607, 608, 609, 610, 611, 612, 613, 614, 615, 616, 617, 618, 619, 620, 621, 622, 623, 624, 625, 626, 627, 628, 629, 630, 631, 632, 633, 634, 635, 636, 637, 638, 639, 640, 641, 642, 643, 644, 645, 646, 647, 648, 649, 650, 651, 652, 653, 654, 655, 656, 657, 658, 659, 660, 661, 662, 663, 664, 665, 666, 667, 668, 669, 670, 671, 672, 673, 674, 675, 676, 677, 678, 679, 680, 681, 682, 683, 684, 685, 686, 687, 688, 689, 690, 691, 692, 693, 694, 695, 696, 697, 698, 699, 700, 701, 702, 703, 704, 705, 706, 707, 708, 709, 710, 711, 712, 713, 714, 715, 716, 717, 718, 719, 720, 721, 722, 723, 724, 725, 726, 727, 728, 729, 730, 731, 732, 733, 734, 735, 736, 737, 738, 739, 740, 741, 742, 743, 744, 745, 746, 747, 748, 749, 750, 751, 752, 753, 754, 755, 756, 757, 758, 759, 760, 761, 762, 763, 764, 765, 766, 767, 768, 769, 770, 771, 772, 773, 774, 775, 776, 777, 778, 779, 780, 781, 782, 783, 784, 785, 786, 787, 788, 789, 790, 791, 792, 793, 794, 795, 796, 797, 798, 799, 800, 801, 802, 803, 804, 805, 806, 807, 808, 809, 810, 811, 812, 813, 814, 815, 816, 817, 818, 819, 820, 821, 822, 823, 824, 825, 826, 827, 828, 829, 830, 831, 832, 833, 834, 835, 836, 837, 838, 839, 840, 841, 842, 843, 844, 845, 846, 847, 848, 849, 850, 851, 852, 853, 854, 855, 856, 857, 858, 859, 860, 861, 862, 863, 864, 865, 866, 867, 868, 869, 870, 871, 872, 873, 874, 875, 876, 877, 878, 879, 880, 881, 882, 883, 884, 885, 886, 887, 888, 889, 890, 891, 892, 893, 894, 895, 896, 897, 898, 899, 900, 901, 902, 903, 904, 905, 906, 907, 908, 909, 910, 911, 912, 913, 914, 915, 916, 917, 918, 919, 920, 921, 922, 923, 924, 925, 926, 927, 928, 929, 930, 931, 932, 933, 934, 935, 936, 937, 938, 939, 940, 941, 942, 943, 944, 945, 946, 947, 948, 949, 950, 951, 952, 953, 954, 955, 956, 957, 958, 959, 960, 961, 962, 963, 964, 965, 966, 967, 968, 969, 970, 971, 972, 973, 974, 975, 976, 977, 978, 979, 980, 981, 982, 983, 984, 985, 986, 987, 988, 989, 990, 991, 992, 993, 994, 995, 996, 997, 998, 999], 'cost': 0} -``` - -```java -InsertReq insertReq = InsertReq.builder() - .collectionName("test_collection") - .data(data) - .build(); - -InsertResp insertResp = client.insert(insertReq); -``` - -```javascript -res = await client.insert({ - collection_name: "test_collection", - data: data -}); - -console.log(`Inserted ${res.insert_cnt} entities`); - -// Output: -// Inserted 1000 entities -``` - -## Load the collection - -After inserting data, we need to load the collection to make it available for search and query operations. - -
- Python - Java - Node.js -
- -```python -client.load_collection('test_collection') -``` - -```java -LoadCollectionReq loadCollectionReq = LoadCollectionReq.builder() - .collectionName("test_collection") - .build(); -client.loadCollection(loadCollectionReq); -``` - -```javascript -await client.loadCollection({ - collection_name: "test_collection" -}); - -res = await client.getLoadState({ - collection_name: "test_collection" -}); - -console.log("Collection load state: " + res.state); - -// Output: -// Collection load state: LoadStateLoaded -``` - -## Basic scalar filtering - -Once all of your data has been added, you can conduct searches and queries using the elements in the array field in the same manner as you would with a standard scalar field. - -
- -For more information on parameters, refer to [`search()`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Vector/search.md) in the SDK reference. - -
- -
- -For more information on parameters, refer to [`search()`](https://milvus.io/api-reference/java/v2.4.x/v2/Vector/search.md) in the SDK reference. - -
- -
- -For more information on parameters, refer to [`search()`](https://milvus.io/api-reference/node/v2.4.x/Vector/search.md) in the SDK reference. - -
- -
- Python - Java - Node.js -
- -```python -# 4. Basic search with the array field -query_vectors = [ [ random.uniform(-1, 1) for _ in range(5) ]] - -res = client.search( - collection_name="test_collection", - data=query_vectors, - filter="color_coord[0] < 10", - search_params={ - "metric_type": "COSINE", - "params": {"nprobe": 16} - }, - output_fields=["id", "color", "color_tag", "color_coord"], - limit=3 -) - -print(res) - -# Output: -# data: ["[{'id': 918, 'distance': 0.974249541759491, 'entity': {'color_coord': [4, 34, 9, 18, 29], 'id': 918, 'color': 'purple', 'color_tag': 2940}}, {'id': 822, 'distance': 0.9177230000495911, 'entity': {'color_coord': [7, 36, 32], 'id': 822, 'color': 'red', 'color_tag': 8519}}, {'id': 981, 'distance': 0.9116519689559937, 'entity': {'color_coord': [7, 16, 40, 32, 32], 'id': 981, 'color': 'pink', 'color_tag': 2992}}]"] , extra_info: {'cost': 0} -``` - -```java -// 4. Basic search with an Array field - -QueryReq queryReq = QueryReq.builder() - .collectionName("test_collection") - .filter("color_coord[0] in [7, 8, 9]") - .outputFields(Arrays.asList("id", "color", "color_tag", "color_coord")) - .limit(3L) - .build(); - -QueryResp queryResp = client.query(queryReq); - -System.out.println(JSONObject.toJSON(queryResp)); - -// Output: -// {"queryResults": [ -// {"entity": { -// "color": "orange", -// "color_tag": 2464, -// "id": 18, -// "color_coord": [ -// 9, -// 30 -// ] -// }}, -// {"entity": { -// "color": "pink", -// "color_tag": 2602, -// "id": 22, -// "color_coord": [ -// 8, -// 34, -// 16 -// ] -// }}, -// {"entity": { -// "color": "pink", -// "color_tag": 1243, -// "id": 42, -// "color_coord": [ -// 9, -// 20 -// ] -// }} -// ]} -``` - -```javascript -const query_vectors = [Array(5).fill(0).map(() => Math.random())]; - -res = await client.search({ - collection_name: "test_collection", - data: query_vectors, - filter: "color_coord[0] < 10", - output_fields: ["id", "color", "color_tag", "color_coord"], - limit: 3, - metric_type: "COSINE" -}); - -console.log("Search result: " + JSON.stringify(res)); - -// Output: -// Search result: [ -// { -// "score": 0.9969238042831421, -// "id": "212", -// "color": "green", -// "color_tag": "5603", -// "color_coord": [ -// "9", -// "14", -// "22", -// "4", -// "35" -// ] -// }, -// { -// "score": 0.9952742457389832, -// "id": "339", -// "color": "yellow", -// "color_tag": "8867", -// "color_coord": [ -// "8", -// "0", -// "6", -// "19", -// "23" -// ] -// }, -// { -// "score": 0.9944050312042236, -// "id": "24", -// "color": "red", -// "color_tag": "7686", -// "color_coord": [ -// "6", -// "17", -// "6", -// "32" -// ] -// } -// ] -``` - -## Advanced filtering - -As what we have in a JSON field, Milvus also provides advanced filtering operators for arrays, namely `ARRAY_CONTAINS`, `ARRAY_CONTAINS_ALL`, `ARRAY_CONTAINS_ANY`, and `ARRAY_LENGTH`. For more information on operators, refer to [Reference on array filters](#reference-on-array-filters). - -- Filters all entities having a `10` in their `color_coord` values. - -
- Python - Java - Node.js -
- - ```python - # 5. Advanced query within the array field - - res = client.query( - collection_name="test_collection", - filter="ARRAY_CONTAINS(color_coord, 10)", - output_fields=["id", "color", "color_tag", "color_coord"], - limit=3 - ) - - print(res) - - # Output: - # data: ["{'id': 2, 'color': 'green', 'color_tag': 3676, 'color_coord': [26, 37, 30, 10]}", "{'id': 28, 'color': 'red', 'color_tag': 4735, 'color_coord': [30, 10, 40, 34]}", "{'id': 32, 'color': 'green', 'color_tag': 8816, 'color_coord': [10, 9, 24, 39]}"] , extra_info: {'cost': 0} - ``` - - ```java - // 5. Advanced query within an Array field - queryReq = QueryReq.builder() - .collectionName("test_collection") - .filter("ARRAY_CONTAINS(color_coord, 10)") - .outputFields(Arrays.asList("id", "color", "color_tag", "color_coord")) - .limit(3) - .build(); - - queryResp = client.query(queryReq); - - System.out.println(JSONObject.toJSON(queryResp)); - - // Output: - // {"queryResults": [ - // {"entity": { - // "color": "blue", - // "color_tag": 4337, - // "id": 17, - // "color_coord": [ - // 11, - // 33, - // 10, - // 20 - // ] - // }}, - // {"entity": { - // "color": "white", - // "color_tag": 5219, - // "id": 25, - // "color_coord": [ - // 10, - // 15 - // ] - // }}, - // {"entity": { - // "color": "red", - // "color_tag": 7120, - // "id": 35, - // "color_coord": [ - // 19, - // 10, - // 10, - // 14 - // ] - // }} - // ]} - ``` - - ```javascript - // 5. Advanced search within the array field - res = await client.search({ - collection_name: "test_collection", - data: query_vectors, - filter: "ARRAY_CONTAINS(color_coord, 10)", - output_fields: ["id", "color", "color_tag", "color_coord"], - limit: 3 - }) - - console.log(JSON.stringify(res.results, null, 4)) - - // Output - // - // [ - // { - // "score": 1.7962548732757568, - // "id": "696", - // "color": "red", - // "color_tag": "1798", - // "color_coord": [ - // "33", - // "10", - // "37" - // ] - // }, - // { - // "score": 1.7126177549362183, - // "id": "770", - // "color": "red", - // "color_tag": "1962", - // "color_coord": [ - // "21", - // "23", - // "10" - // ] - // }, - // { - // "score": 1.6707111597061157, - // "id": "981", - // "color": "yellow", - // "color_tag": "3100", - // "color_coord": [ - // "28", - // "39", - // "10", - // "6" - // ] - // } - // ] - // - ``` - -- Filters all entities having a `7` and an `8` in their `color_coord` values. - -
- Python - Java - Node.js -
- - ```python - res = client.query( - collection_name="test_collection", - filter="ARRAY_CONTAINS_ALL(color_coord, [7, 8])", - output_fields=["id", "color", "color_tag", "color_coord"], - limit=3 - ) - - print(res) - - # Output: - # data: ["{'id': 147, 'color': 'brown', 'color_tag': 1287, 'color_coord': [7, 8, 11, 0]}", "{'id': 257, 'color': 'white', 'color_tag': 3641, 'color_coord': [2, 8, 31, 7]}", "{'id': 280, 'color': 'orange', 'color_tag': 1072, 'color_coord': [22, 7, 8]}"] , extra_info: {'cost': 0} - ``` - - ```java - queryReq = QueryReq.builder() - .collectionName("test_collection") - .filter("ARRAY_CONTAINS_ALL(color_coord, [7, 8, 9])") - .outputFields(Arrays.asList("id", "color", "color_tag", "color_coord")) - .limit(3) - .build(); - - queryResp = client.query(queryReq); - - System.out.println(JSONObject.toJSON(queryResp)); - - // Output: - // {"queryResults": [{"entity": { - // "color": "red", - // "color_tag": 6986, - // "id": 423, - // "color_coord": [ - // 26, - // 7, - // 8, - // 9 - // ] - // }}]} - ``` - - ```javascript - res = await client.search({ - collection_name: "test_collection", - data: query_vectors, - filter: "ARRAY_CONTAINS_ALL(color_coord, [7, 8])", - output_fields: ["id", "color", "color_tag", "color_coord"], - limit: 3 - }) - - console.log(JSON.stringify(res.results, null, 4)) - - // Output - // - // [ - // { - // "score": 0.8267516493797302, - // "id": "913", - // "color": "brown", - // "color_tag": "8897", - // "color_coord": [ - // "39", - // "31", - // "8", - // "29", - // "7" - // ] - // }, - // { - // "score": 0.6889009475708008, - // "id": "826", - // "color": "blue", - // "color_tag": "4903", - // "color_coord": [ - // "7", - // "25", - // "5", - // "12", - // "8" - // ] - // }, - // { - // "score": 0.5851659774780273, - // "id": "167", - // "color": "blue", - // "color_tag": "1550", - // "color_coord": [ - // "8", - // "27", - // "7" - // ] - // } - // ] - // - ``` - -- Filters all entities having either 7, 8, or 9 in their `color_coord` values. - -
- Python - Java - Node.js -
- - ```python - res = client.query( - collection_name="test_collection", - filter="ARRAY_CONTAINS_ANY(color_coord, [7, 8, 9])", - output_fields=["id", "color", "color_tag", "color_coord"], - limit=3 - ) - - print(res) - - # Output: - # data: ["{'id': 0, 'color': 'white', 'color_tag': 2081, 'color_coord': [16, 7, 35, 5, 25]}", "{'id': 1, 'color': 'purple', 'color_tag': 4669, 'color_coord': [11, 9, 15, 38, 21]}", "{'id': 3, 'color': 'yellow', 'color_tag': 2612, 'color_coord': [0, 12, 22, 7]}"] , extra_info: {'cost': 0} - ``` - - ```java - queryReq = QueryReq.builder() - .collectionName("test_collection") - .filter("ARRAY_CONTAINS_ANY(color_coord, [7, 8, 9])") - .outputFields(Arrays.asList("id", "color", "color_tag", "color_coord")) - .limit(3) - .build(); - - queryResp = client.query(queryReq); - - System.out.println(JSONObject.toJSON(queryResp)); - - // Output: - // {"queryResults": [ - // {"entity": { - // "color": "orange", - // "color_tag": 2464, - // "id": 18, - // "color_coord": [ - // 9, - // 30 - // ] - // }}, - // {"entity": { - // "color": "pink", - // "color_tag": 2602, - // "id": 22, - // "color_coord": [ - // 8, - // 34, - // 16 - // ] - // }}, - // {"entity": { - // "color": "pink", - // "color_tag": 1243, - // "id": 42, - // "color_coord": [ - // 9, - // 20 - // ] - // }} - // ]} - ``` - - ```javascript - res = await client.search({ - collection_name: "test_collection", - data: query_vectors, - filter: "ARRAY_CONTAINS_ANY(color_coord, [7, 8, 9])", - output_fields: ["id", "color", "color_tag", "color_coord"], - limit: 3 - }) - - console.log(JSON.stringify(res.results, null, 4)) - - // Output - // - // [ - // { - // "score": 2.015894889831543, - // "id": "260", - // "color": "green", - // "color_tag": "5320", - // "color_coord": [ - // "1", - // "7", - // "33", - // "13", - // "23" - // ] - // }, - // { - // "score": 1.783075213432312, - // "id": "593", - // "color": "orange", - // "color_tag": "4079", - // "color_coord": [ - // "8", - // "19" - // ] - // }, - // { - // "score": 1.7713876962661743, - // "id": "874", - // "color": "blue", - // "color_tag": "7029", - // "color_coord": [ - // "14", - // "8", - // "15" - // ] - // } - // ] - // - ``` - -- Filters entities that have exactly four elements. - -
- Python - Java - Node.js -
- - ```python - res = client.query( - collection_name="test_collection", - filter="ARRAY_LENGTH(color_coord) == 4", - output_fields=["id", "color", "color_tag", "color_coord"], - limit=3 - ) - - print(res) - - # Output: - # data: ["{'id': 2, 'color': 'green', 'color_tag': 3676, 'color_coord': [26, 37, 30, 10]}", "{'id': 3, 'color': 'yellow', 'color_tag': 2612, 'color_coord': [0, 12, 22, 7]}", "{'id': 4, 'color': 'green', 'color_tag': 6912, 'color_coord': [4, 5, 19, 28]}"] , extra_info: {'cost': 0} - ``` - - ```java - queryReq = QueryReq.builder() - .collectionName("test_collection") - .filter("ARRAY_LENGTH(color_coord) == 4") - .outputFields(Arrays.asList("id", "color", "color_tag", "color_coord")) - .limit(3) - .build(); - - queryResp = client.query(queryReq); - - System.out.println(JSONObject.toJSON(queryResp)); - - // Output: - // {"queryResults": [ - // {"entity": { - // "color": "green", - // "color_tag": 2984, - // "id": 2, - // "color_coord": [ - // 27, - // 31, - // 23, - // 29 - // ] - // }}, - // {"entity": { - // "color": "black", - // "color_tag": 6867, - // "id": 4, - // "color_coord": [ - // 37, - // 3, - // 30, - // 33 - // ] - // }}, - // {"entity": { - // "color": "brown", - // "color_tag": 3464, - // "id": 10, - // "color_coord": [ - // 31, - // 38, - // 21, - // 28 - // ] - // }} - // ]} - ``` - - ```javascript - res = await client.search({ - collection_name: "test_collection", - data: query_vectors, - filter: "ARRAY_LENGTH(color_coord) == 4", - output_fields: ["id", "color", "color_tag", "color_coord"], - limit: 3 - }) - - console.log(JSON.stringify(res.results, null, 4)) - - // Output - // - // [ - // { - // "score": 2.0404388904571533, - // "id": "439", - // "color": "orange", - // "color_tag": "7096", - // "color_coord": [ - // "27", - // "34", - // "26", - // "39" - // ] - // }, - // { - // "score": 1.9059759378433228, - // "id": "918", - // "color": "purple", - // "color_tag": "2903", - // "color_coord": [ - // "28", - // "19", - // "36", - // "35" - // ] - // }, - // { - // "score": 1.8385567665100098, - // "id": "92", - // "color": "yellow", - // "color_tag": "4693", - // "color_coord": [ - // "1", - // "23", - // "2", - // "3" - // ] - // } - // ] - // - ``` - -## Limits - -- Elements in an ARRAY field should be of the same data type, specified by `element_type`. Any valid data type available for scalar fields in Milvus can be used as `element_type`. For a list of supported data types, refer to [Supported data types](https://milvus.io/docs/schema.md#Supported-data-types). - -- The number of elements in an ARRAY field should be less than or equal to the maximum capacity of the array field, specified by `max_capacity`. - -## Reference on array filters - -When working with array fields, you can enclose a string value with either double quotation marks ("") or single quotation marks (''). It's important to note that Milvus stores string values in the array field as is without performing semantic escape or conversion. For instance, **'a"b'**, **"a'b"**, **'a\'b'**, and **"a\"b"** will be saved as is, while **'a'b'** and **"a"b"** will be treated as invalid values. - -Assume that two array fields `int_array` and `var_array` have been defined. The following table describes the supported boolean expressions that you can use in `expr` when searching with array fields. - -| Operator | Examples | Remarks | -|-------------------|-------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| < | 'int_array[0] < 3' | This expression evaluates to true if the value of int_array[0] is less than 3. | -| > | 'int_array[0] > 5' | This expression evaluates to true if the value of int_array[0] is greater than 5. | -| == | 'int_array[0] == 0' | This expression evaluates to true if the value of int_array[0] is equal to 0. | -| != | 'var_array[0] != "a"' | This expression evaluates to true if the value of var_array[0] is not equal to "a". | -| <= | 'int_array[0] <= 3' | This expression evaluates to true if the value of int_array[0] is smaller than or equal to 3. | -| >= | 'int_array[0] >= 10' | This expression evaluates to true if the value of int_array[0] is greater than or equal to 10. | -| in | 'var_array[0] in ["str1", "str2"]' | This expression evaluates to true if the value of var_array[0] is "str1" or "str2". | -| not in | 'int_array[0] not in [1, 2, 3]' | This expression evaluates to true if the value of int_array[0] is not 1, 2, or 3. | -| +, -, *, /, %, ** | 'int_array[0] + 100 > 200' | This expression evaluates to true if the value of int_array[0] + 100 is greater than 200. | -| like (LIKE) | 'var_array[0] like "prefix%"' | This expression evaluates to true if the value of var_array[0] is prefixed with "prefix". | -| and (&&) | 'var_array[0] like "prefix%" && int_array[0] <= 100' | This expression evaluates to true if the value of var_array[0] is prefixed with "prefix", and the value of int_array[0] is smaller than or equal to 100. | -| or (||) | 'var_array[0] like "prefix%" || int_array[0] <= 100' | This expression evaluates to true if the value of var_array[0] is prefixed with "prefix", or the value of int_array[0] is smaller than or equal to 100. | -| array_contains (ARRAY_CONTAINS) | 'array_contains(int_array, 100)' | This expression evaluates to true if int_array contains element 100. | -| array_contains_all (ARRAY_CONTAINS_ALL) | 'array_contains_all(int_array, [1, 2, 3])' | This expression evaluates to true if int_array contains all elements 1, 2, and 3. | -| array_contains_any (ARRAY_CONTAINS_ANY) | 'array_contains_any(var_array, ["a", "b", "c"])' | This expression evaluates to true if var_array contains any element of "a", "b", and "c". | -| array_length | 'array_length(int_array) == 10' | This expression evaluates to true if int_array contains exactly 10 elements. | diff --git a/v2.4.x/site/en/reference/boolean.md b/v2.4.x/site/en/reference/boolean.md deleted file mode 100644 index 48bab10a08..0000000000 --- a/v2.4.x/site/en/reference/boolean.md +++ /dev/null @@ -1,322 +0,0 @@ ---- -id: boolean.md -summary: Learn about boolean expression rules in Milvus. -title: Scalar Filtering Rules ---- - -# Scalar Filtering Rules - -## Overview - -A predicate expression outputs a boolean value. Milvus conducts scalar filtering by searching with predicates. A predicate expression, when evaluated, returns either TRUE or FALSE. View [Python SDK API Reference](/api-reference/pymilvus/v2.4.x/About.md) for instruction on using predicate expressions. - -[EBNF](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form) grammar rules describe boolean expressions rules: - -``` -Expr = LogicalExpr | NIL -LogicalExpr = LogicalExpr BinaryLogicalOp LogicalExpr - | UnaryLogicalOp LogicalExpr - | "(" LogicalExpr ")" - | SingleExpr; -BinaryLogicalOp = "&&" | "and" | "||" | "or"; -UnaryLogicalOp = "not"; -SingleExpr = TermExpr | CompareExpr; -TermExpr = IDENTIFIER "in" ConstantArray; -Constant = INTEGER | FLOAT -ConstantExpr = Constant - | ConstantExpr BinaryArithOp ConstantExpr - | UnaryArithOp ConstantExpr; - -ConstantArray = "[" ConstantExpr { "," ConstantExpr } "]"; -UnaryArithOp = "+" | "-" -BinaryArithOp = "+" | "-" | "*" | "/" | "%" | "**"; -CompareExpr = IDENTIFIER CmpOp IDENTIFIER - | IDENTIFIER CmpOp ConstantExpr - | ConstantExpr CmpOp IDENTIFIER - | ConstantExpr CmpOpRestricted IDENTIFIER CmpOpRestricted ConstantExpr; -CmpOpRestricted = "<" | "<="; -CmpOp = ">" | ">=" | "<" | "<=" | "=="| "!="; -MatchOp = "like" | "LIKE"; -JsonArrayOps = JsonDefs "(" IDENTIFIER "," JsonExpr | JsonArray ")"; -JsonArrayDefs = "json_contains" | "JSON_CONTAINS" - | "json_contains_all" | "JSON_CONTAINS_ALL" - | "json_contains_any" | "JSON_CONTAINS_ANY"; -JsonExpr = Constant | ConstantArray | STRING | BOOLEAN; -JsonArray = "[" JsonExpr { "," JsonExpr } "]"; -ArrayOps = ArrayDefs "(" IDENTIFIER "," ArrayExpr | Array ")"; -ArrayDefs = "array_contains" | "ARRAY_CONTAINS" - | "array_contains_all" | "ARRAY_CONTAINS_ALL" - | "array_contains_any" | "ARRAY_CONTAINS_ANY" - | "array_length" | "ARRAY_LENGTH"; -ArrayExpr = Constant | ConstantArray | STRING | BOOLEAN; -Array = "[" ArrayExpr { "," ArrayExpr } "]"; -``` - -The following table lists the description of each symbol mentioned in the above Boolean expression rules. - - -| Notation | Description | -| ----------- | ----------- | -| = | Definition. | -| , | Concatenation. | -| ; | Termination. | -| | | Alternation. | -| {...} | Repetition. | -| (...) | Grouping. | -| NIL | Empty. The expression can be an empty string. | -| INTEGER | Integers such as 1, 2, 3. | -| FLOAT | Float numbers such as 1.0, 2.0. | -| CONST | Integers or float numbers. | -| IDENTIFIER | Identifier. In Milvus, the IDENTIFIER represents the field name. | -| LogicalOp | A LogicalOp is a logical operator that supports combining more than one relational operation in one comparison. Returned value of a LogicalOp is either TRUE (1) or FALSE (0). There are two types of LogicalOps, including BinaryLogicalOps and UnaryLogicalOps. | -| UnaryLogicalOp | UnaryLogicalOp refers to the unary logical operator "not". | -| BinaryLogicalOp | Binary logical operators that perform actions on two operands. In a complex expression with two or more operands, the order of evaluation depends on precedence rules. | -| ArithmeticOp | An ArithmeticOp, namely an arithmetic operator, performs mathematical operations such as addition and subtraction on operands. | -| UnaryArithOp | A UnaryArithOp is an arithmetic operator that performs an operation on a single operand. The negative UnaryArithOp changes a positive expression into a negative one, or the other way round. | -| BinaryArithOp | A BinaryArithOp, namely a binary operator, performs operations on two operands. In a complex expression with two or more operands, the order of evaluation depends on precedence rules. | -| CmpOp | CmpOp is a relational operator that perform actions on two operands. | -| CmpOpRestricted | CmpOpRestricted is restricted to "Less than" and "Equal". | -| ConstantExpr | ConstantExpr can be a Constant or a BinaryArithOp on two ConstExprs or a UnaryArithOp on a single ConstantExpr. It is defined recursively. | -| ConstantArray | ConstantArray is wrapped by square brackets, and ConstantExpr can be repeated in the square brackets. ConstArray must include at least one ConstantExpr. | -| TermExpr | TermExpr is used to check whether the value of an IDENTIFIER appears in a ConstantArray. TermExpr is represented by "in". | -| CompareExpr | A CompareExpr, namely comparison expression can be relational operations on two IDENTIFIERs, or relational operations on one IDENTIFIER and one ConstantExpr, or ternary operation on two ConstantExprs and one IDENTIFIER. | -| SingleExpr | SingleExpr, namely single expression, can be either a TermExpr or a CompareExpr. | -| LogicalExpr | A LogicalExpr can be a BinaryLogicalOp on two LogicalExprs, or a UnaryLogicalOp on a single LogicalExpr, or a LogicalExpr grouped within parentheses, or a SingleExpr. The LogicalExpr is defined recursively. | -| Expr | Expr, an abbreviation meaning expression, can be LogicalExpr or NIL. | -| MatchOp | A MatchOp, namely a match operator, compares a string to a string constant or a string prefix, infix, or suffix constant. | -| JsonArrayOp | A JsonOp, namely a JSON operator, checks whether the specified identifier contains the specified elements. | -| ArrayOp | An ArrayOp, namely an array operator, checks whether the specified identifier contains the specified elements. | - -## Operators - -### Logical operators - -Logical operators perform a comparison between two expressions. - -| Symbol| Operation | Example | Description | -| ----------| ------------- | ----------- | ------------------------- | -| 'and' && | and | expr1 && expr2 | True if both expr1 and expr2 are true. | -| 'or' || | or | expr1 || expr2 | True if either expr1 or expr2 are true. | - -### Binary arithmetic operators - -Binary arithmetic operators contain two operands and can perform basic arithmetic operations and return the corresponding result. - -| Symbol| Operation | Example | Description | -| ----------| ------------- | ----------- | ------------------------- | -| + | Addition | a + b | Add the two operands. | -| - | Subtraction | a - b | Subtract the second operand from the first operand. | -| * | Multiplication| a * b | Multiply the two operands. | -| / | Division | a / b | Divide the first operand by the second operand. | -| ** | Power | a ** b | Raise the first operand to the power of the second operand. | -| % | Modulo | a % b | Divide the first operand by the second operand and yield the remainder portion. | - -### Relational operators - -Relational operators use symbols to check for equality, inequality, or relative order between two expressions. - -| Symbol| Operation | Example | Description | -| ----------| ------------- | ----------- | ------------------------- | -| < | Less than | a < b | True if a is less than b. | -| > | Greater than | a > b | True if a is greater than b. | -| == | Equal | a == b | True if a is equal to b. | -| != | Not equal | a != b | True if a is not equal to b. | -| <= | Less than or equal | a <= b | True if a is less than or equal to b. | -| >= | Greater than or equal | a >= b | True if a is greater than or equal to b. | - -## Operator precedence and associativity - -The following table lists the precedence and associativity of operators. Operators are listed top to bottom, in descending precedence. - -| Precedence | Operator | Description | Associativity | -|------------|---------------------------------------|---------------|---------------| -| 1 | + - | UnaryArithOp | Left-to-right | -| 2 | not | UnaryLogicOp | Right-to-left | -| 3 | ** | BinaryArithOp | Left-to-right | -| 4 | * / % | BinaryArithOp | Left-to-right | -| 5 | + - | BinaryArithOp | Left-to-right | -| 6 | < <= > >= | CmpOp | Left-to-right | -| 7 | == != | CmpOp | Left-to-right | -| 8 | like LIKE | MatchOp | Left-to-right | -| 9 | json_contains JSON_CONTAINS | JsonArrayOp | Left-to-right | -| 9 | array_contains ARRAY_CONTAINS | ArrayOp | Left-to-right | -| 10 | json_contains_all JSON_CONTAINS_ALL | JsonArrayOp | Left-to-right | -| 10 | array_contains_all ARRAY_CONTAINS_ALL | ArrayOp | Left-to-right | -| 11 | json_contains_any JSON_CONTAINS_ANY | JsonArrayOp | Left-to-right | -| 11 | array_contains_any ARRAY_CONTAINS_ANY | ArrayOp | Left-to-right | -| 12 | array_length ARRAY_LENGTH | ArrayOp | Left-to-right | -| 13 | && and | BinaryLogicOp | Left-to-right | -| 14 | || or | BinaryLogicOp | Left-to-right | - -Expressions are normally evaluated from left to right. Complex expressions are evaluated one at a time. The order in which the expressions are evaluated is determined by the precedence of the operators used. - -If an expression contains two or more operators with the same precedence, the operator to the left is evaluated first. - -
- -For example, 10 / 2 * 5 will be evaluated as (10 / 2) and the result multiplied by 5. - -
- -When a lower precedence operation should be processed first, it should be enclosed within parentheses. - -
- -For example, 30 / 2 + 8. This is normally evaluated as 30 divided by 2 then 8 added to the result. If you want to divide by 2 + 8, it should be written as 30 / (2 + 8). - -
- -Parentheses can be nested within expressions. Innermost parenthetical expressions are evaluated first. - -## Usage - -Samples of all available boolean expression usage in Milvus are listed as follows (`int64` represents the scalar field that contains data of INT64 type, `float` represents the scalar field that contains data of floating-point type, and `VARCHAR` represents the scalar field that contains data of VARCHAR type): - -1. CmpOp - -``` -"int64 > 0" -``` - -``` -"0 < int64 < 400" -``` - -``` -"500 <= int64 < 1000" -``` - -``` -VARCHAR > "str1" -``` - -2. BinaryLogicalOp and parentheses - -``` -"(int64 > 0 && int64 < 400) or (int64 > 500 && int64 < 1000)" -``` - -3. TermExpr and UnaryLogicOp - -``` -"int64 not in [1, 2, 3]" -``` - -``` -VARCHAR not in ["str1", "str2"] -``` - - -4. TermExpr, BinaryLogicalOp, and CmpOp (on different fields) - -``` -"int64 in [1, 2, 3] and float != 2" -``` - -5. BinaryLogicalOp and CmpOp - -``` -"int64 == 0 || int64 == 1 || int64 == 2" -``` - -6. CmpOp and UnaryArithOp or BinaryArithOp - -``` -"200+300 < int64 <= 500+500" -``` - -7. MatchOp - -``` -VARCHAR like "prefix%" -VARCHAR like "%suffix" -VARCHAR like "%middle%" -VARCHAR like "_suffix" -``` - -8. JsonArrayOp - -- `JSON_CONTAINS(identifier, JsonExpr)` - - If the JSON expression of a `JSON_CONTAINS` (the second argument) statement is a list, the identifier (the first argument) should be list of list. Otherwise, the statement always evaluates to False. - - ```python - # {"x": [1,2,3]} - json_contains(x, 1) # ==> true - json_contains(x, "a") # ==> false - - # {"x": [[1,2,3], [4,5,6], [7,8,9]]} - json_contains(x, [1,2,3]) # ==> true - json_contains(x, [3,2,1]) # ==> false - ``` - -- `JSON_CONTAINS_ALL(identifier, JsonExpr)` - - The JSON expression in a `JSON_CONTAINS_ALL` statement should always be a list. - - ```python - # {"x": [1,2,3,4,5,7,8]} - json_contains_all(x, [1,2,8]) # ==> true - json_contains_all(x, [4,5,6]) # ==> false 6 is not exists - ``` - -- `JSON_CONTAINS_ANY(identifier, JsonExpr)` - - The JSON expression in a `JSON_CONTAINS_ANY` statement should always be a list. Otherwise, it acts the same as `JSON_CONTAINS`. - - ```python - # {"x": [1,2,3,4,5,7,8]} - json_contains_any(x, [1,2,8]) # ==> true - json_contains_any(x, [4,5,6]) # ==> true - json_contains_any(x, [6,9]) # ==> false - ``` - -9. ArrayOp - -- `ARRAY_CONTAINS(identifier, ArrayExpr)` - - If the array expression of an `ARRAY_CONTAINS` (the second argument) statement is a list, the identifier (the first argument) should be list of list. Otherwise, the statement always evaluates to False. - - ```python - # 'int_array': [1,2,3] - array_contains(int_array, 1) # ==> true - array_contains(int_array, "a") # ==> false - ``` - -- `ARRAY_CONTAINS_ALL(identifier, ArrayExpr)` - - The array expression in an `ARRAY_CONTAINS_ALL` statement should always be a list. - - ```python - # "int_array": [1,2,3,4,5,7,8] - array_contains_all(int_array, [1,2,8]) # ==> true - array_contains_all(int_array, [4,5,6]) # ==> false 6 is not exists - ``` - -- `ARRAY_CONTAINS_ANY(identifier, ArrayExpr)` - - The array expression in an `ARRAY_CONTAINS_ANY` statement should always be a list. Otherwise, it acts the same as `ARRAY_CONTAINS`. - - ```python - # "int_array": [1,2,3,4,5,7,8] - array_contains_any(int_array, [1,2,8]) # ==> true - array_contains_any(int_array, [4,5,6]) # ==> true - array_contains_any(int_array, [6,9]) # ==> false - ``` - -- `ARRAY_LENGTH(identifier)` - - Check the number of elements in an array. - - ```python - # "int_array": [1,2,3,4,5,7,8] - array_length(int_array) # ==> 7 - ``` - -## What's next - -Now that you know how bitsets work in Milvus, you might also want to: - -- Learn how to conduct a [Hybrid Search](multi-vector-search.md). -- Learn how to [use strings to filter](https://milvus.io/blog/2022-08-08-How-to-use-string-data-to-empower-your-similarity-search-applications.md) your search results. -- Learn how to [use dynamic fields in building boolean expressions](enable-dynamic-field.md). diff --git a/v2.4.x/site/en/reference/consistency.md b/v2.4.x/site/en/reference/consistency.md deleted file mode 100644 index cdb2240e99..0000000000 --- a/v2.4.x/site/en/reference/consistency.md +++ /dev/null @@ -1,76 +0,0 @@ ---- -id: consistency.md -summary: Learn about the four levels of consistency in Milvus. -title: Consistency ---- -# Consistency - -This topic introduces the four levels of consistency in Milvus and their best-suited scenarios. The mechanism behind ensuring consistency in Milvus is also covered in this topic. - -## Overview - -Consistency in a distributed database specifically refers to the property that ensures every node or replica has the same view of data when writing or reading data at a given time. - -Milvus supports four consistency levels: strong, bounded staleness, session, and eventually. The default consistency level in Milvus is bounded staleness. You can easily tune the consistency level when conducting a [single-vector search](single-vector-search.md), [hybrid search](multi-vector-search.md) or [query](get-and-scalar-query.md) to make it best suit your application. - -## Consistency levels - -As defined by the [PACELC](https://en.wikipedia.org/wiki/PACELC_theorem) theorem, a distributed database has to trade off among consistency, availability, and latency. High consistency implies high accuracy but also high search latency, while low consistency leads to fast search speed but a certain loss of data visibility. Therefore, different levels of consistency suit different scenarios. - -The following explains the differences of the four consistency levels supported by Milvus and the scenarios they each suit. - -### Strong - -Strong is the highest and the most strict level of consistency. It ensures that users can read the latest version of data. - -![Strong consistency](../../../assets/Consistency_Strong.png "An illustration of strong consistency.") - -According to the PACELC theorem, if the consistency level is set to strong, the latency will increase. Therefore, we recommend choosing strong consistency during functional testings to ensure the accuracy of the test results. Strong consistency is also best suited for applications that have strict demand for data consistency at the cost of search speed. An example can be an online financial system dealing with order payments and billing. - -### Bounded staleness - -Bounded staleness, as its name suggests, allows data inconsistency during a certain period of time. However, generally, the data are always globally consistent out of that period of time. - -![Bounded staleness consistency](../../../assets/Consistency_Bounded.png "An illustration of bounded staleness consistency.") - -Bounded staleness is suitable for scenarios that need to control search latency and can accept sporadic data invisibility. For instance, in recommender systems like video recommendation engines, data invisibility sometimes has small impact on the overall recall rate, but can significantly boost the performance of the recommender system. - -### Session - -Session ensures that all data writes can be immediately perceived in reads during the same session. In other words, when you write data via one client, the newly inserted data instantaneously become searchable. - -![Session consistency](../../../assets/Consistency_Session.png "An illustration of session consistency.") - -We recommend choosing session as the consistency level for those scenarios where the demand for data consistency in the same session is high. An example can be deleting the data of a book entry from the library system, and after confirmation of the deletion and refreshing the page (a different session), the book should no longer be visible in the search results. - -### Eventually - -There is no guaranteed order of reads and writes, and replicas eventually converge to the same state given that no further write operations are done. Under the consistency of "eventually", replicas start working on read requests with the latest updated values. Eventually consistent is the weakest level among the four. - -![Eventual consistency](../../../assets/Consistency_Eventual.png "An illustration of eventually consistent.") - -However, according to the PACELC theorem, search latency can be tremendously shortened upon sacrificing consistency. Therefore, eventually consistent is best suited for scenarios that do not have a high demand for data consistency but require blazing-fast search performance. An example can be retrieving reviews and ratings of Amazon products with the level of eventually consistent. - -## Guarantee timestamp - -Milvus realizes different consistency levels by introducing the [Guarantee timestamp](https://github.com/milvus-io/milvus/blob/f3f46d3bb2dcae2de0bdb7bc0f7b20a72efceaab/docs/developer_guides/how-guarantee-ts-works.md) (GuaranteeTs). - -A GuaranteeTs serves to inform query nodes that a search or query request will not be performed until all the data before the GuaranteeTs can be seen by the query nodes. When you specify the consistency level, the consistency level will map to a specific GuaranteeTs value. Different GuaranteeTs values correspond to different consistency levels: - -- **Strong**: GuaranteeTs is set as identical to the newest system timestamp, and query nodes wait until all the data before the newest system timestamp can be seen, before processing the search or query request. - -- **Bounded staleness**: GuaranteeTs is set relatively smaller than the newest system timestamp, and query nodes search on a tolerable, less updated data view. - -- **Session**: The client uses the timestamp of the latest write operation as the GuaranteeTs, so that each client can at least retrieve the data inserted by the same client. - -- **Eventually**: GuaranteeTs is set to a very small value to skip the consistency check. Query nodes search immediately on the existing data view. - -See [How GuaranteeTs Works](https://github.com/milvus-io/milvus/blob/f3f46d3bb2dcae2de0bdb7bc0f7b20a72efceaab/docs/developer_guides/how-guarantee-ts-works.md) for more information about the mechanism behind ensuring different levels of consistency in Milvus. - -## What's next - -- Learn how to tune consistency level when: - - [conducting a single-vector search](single-vector-search.md) - - [conducting a hybrid search](multi-vector-search.md) - - [conducting a scalar query](get-and-scalar-query.md) - diff --git a/v2.4.x/site/en/reference/metric.md b/v2.4.x/site/en/reference/metric.md deleted file mode 100644 index 61932b733c..0000000000 --- a/v2.4.x/site/en/reference/metric.md +++ /dev/null @@ -1,196 +0,0 @@ ---- -id: metric.md -summary: Milvus supports a variety of similarity metrics, including Euclidean distance, inner product, Jaccard, etc. -title: Similarity Metrics ---- - -# Similarity Metrics - -In Milvus, similarity metrics are used to measure similarities among vectors. Choosing a good distance metric helps improve the classification and clustering performance significantly. - -The following table shows how these widely used similarity metrics fit with various input data forms and Milvus indexes. Currently, Milvus supports various types of data, including floating point embeddings (often known as floating point vectors or dense vectors), binary embeddings (also known as binary vectors), and sparse embeddings (also known as sparse vectors). - - -
- Floating point embeddings - Binary embeddings - Sparse embeddings -
- -
- - - - - - - - - - - - - - -
Metric TypesIndex Types
  • Euclidean distance (L2)
  • Inner product (IP)
  • Cosine similarity (COSINE)
  • FLAT
  • IVF_FLAT
  • IVF_SQ8
  • IVF_PQ
  • GPU_IVF_FLAT
  • GPU_IVF_PQ
  • HNSW
  • DISKANN
- -
- -
- - - - - - - - - - - - - - -
Metric TypesIndex Types
  • Jaccard
  • Hamming
  • BIN_FLAT
  • BIN_IVF_FLAT
- -
- -
- - - - - - - - - - - - - - -
Metric TypesIndex Types
IP
  • SPARSE_INVERTED_INDEX
  • SPARSE_WAND
- -
- - -### Euclidean distance (L2) - -Essentially, Euclidean distance measures the length of a segment that connects 2 points. - -The formula for Euclidean distance is as follows: - -![euclidean](../../../assets/euclidean_metric.png "Euclidean distance.") - -where **a** = (a0, a1,..., an-1) and **b** = (b0, b0,..., bn-1) are two points in n-dimensional Euclidean space - -It's the most commonly used distance metric and is very useful when the data are continuous. - -
-Milvus only caculates the value before applying square root when Euclidean distance is chosen as the distance metric. -
- -### Inner product (IP) - -The IP distance between two vector embeddings are defined as follows: - -![ip](../../../assets/IP_formula.png "Inner product.") - -IP is more useful if you need to compare non-normalized data or when you care about magnitude and angle. - -
- - If you apply the IP distance metric to normalized embeddings, the result will be equivalent to calculating the cosine similarity between the embeddings. - -
- -Suppose X' is normalized from embedding X: - -![normalize](../../../assets/normalize_formula.png "Normalize.") - -The correlation between the two embeddings is as follows: - -![normalization](../../../assets/normalization_formula.png "Normalization.") - -### Cosine Similarity - -Cosine similarity uses the cosine of the angle between two sets of vectors to measure how similar they are. You can think of the two sets of vectors as two line segments that start from the same origin ([0,0,...]) but point in different directions. - -To calculate the cosine similarity between two sets of vectors **A = (a0, a1,..., an-1)** and **B = (b0, b1,..., bn-1)**, use the following formula: - -![cosine_similarity](../../../assets/cosine_similarity.png "Cosine Similarity") - -The cosine similarity is always in the interval **[-1, 1]**. For example, two proportional vectors have a cosine similarity of **1**, two orthogonal vectors have a similarity of **0**, and two opposite vectors have a similarity of **-1**. The larger the cosine, the smaller the angle between two vectors, indicating that these two vectors are more similar to each other. - -By subtracting their cosine similarity from 1, you can get the cosine distance between two vectors. - -### Jaccard distance - -Jaccard similarity coefficient measures the similarity between two sample sets and is defined as the cardinality of the intersection of the defined sets divided by the cardinality of the union of them. It can only be applied to finite sample sets. - -![Jaccard similarity coefficient](../../../assets/jaccard_coeff.png "Jaccard similarity coefficient.") - -Jaccard distance measures the dissimilarity between data sets and is obtained by subtracting the Jaccard similarity coefficient from 1. For binary variables, Jaccard distance is equivalent to the Tanimoto coefficient. - -![Jaccard distance](../../../assets/jaccard_dist.png "Jaccard distance.") - -### Hamming distance - -Hamming distance measures binary data strings. The distance between two strings of equal length is the number of bit positions at which the bits are different. - -For example, suppose there are two strings, 1101 1001 and 1001 1101. - -11011001 ⊕ 10011101 = 01000100. Since, this contains two 1s, the Hamming distance, d (11011001, 10011101) = 2. - -### Structural Similarity - -When a chemical structure occurs as a part of a larger chemical structure, the former is called a substructure and the latter is called a superstructure. For example, ethanol is a substructure of acetic acid, and acetic acid is a superstructure of ethanol. - -Structural similarity is used to determine whether two chemical formulae are similar to each other in that one is the superstructure or substructure of the other. - -To determine whether A is a superstructure of B, use the following formula: - -![superstructure](../../../assets/superstructure.png "Superstructure") - -Where: - -- A is the binary representation of a chemical formula to be retrieved -- B is the binary representation of a chemical formula in the database - -Once it returns `0`, **A** is not a superstructure of **B**. Otherwise, the result is the other way around. - -To determine whether A is a substructure of B, use the following formula: - -![substructure](../../../assets/substructure.png "subsctructure") - -Where: - -- A is the binary representation of a chemical formula to be retrieved -- B is the binary representation of a chemical formula in the database - -Once it returns `0`, **A** is not a substructure of **B**. Otherwise, the result is the other way around. - -## FAQ - -
-Why is the top1 result of a vector search not the search vector itself, if the metric type is inner product? -This occurs if you have not normalized the vectors when using inner product as the distance metric. -
-
-What is normalization? Why is normalization needed? -

Normalization refers to the process of converting an embedding (vector) so that its norm equals 1. If you use Inner Product to calculate embeddings similarities, you must normalize your embeddings. After normalization, inner product equals cosine similarity. -

-

-See Wikipedia for more information. -

-
-
-Why do I get different results using Euclidean distance (L2) and inner product (IP) as the distance metric? -Check if the vectors are normalized. If not, you need to normalize the vectors first. Theoretically speaking, similarities worked out by L2 are different from similarities worked out by IP, if the vectors are not normalized. -
- - -## What's next - -- Learn more about the supported [index types](index.md) in Milvus. diff --git a/v2.4.x/site/en/reference/schema.md b/v2.4.x/site/en/reference/schema.md index 25a31fd1f1..6745403065 100644 --- a/v2.4.x/site/en/reference/schema.md +++ b/v2.4.x/site/en/reference/schema.md @@ -116,7 +116,7 @@ fields = [ - DOUBLE: numpy.double - VARCHAR: VARCHAR - JSON: [JSON](use-json-fields.md) - - Array: [Array](array_data_type.md) + - Array: [Array](array.md) JSON as a composite data type is available. A JSON field comprises key-value pairs. Each key is a string, and a value can be a number, string, boolean value, array, or list. For details, refer to [JSON: a new data type](use-json-fields.md). diff --git a/v2.4.x/site/en/reference/sparse_vector.md b/v2.4.x/site/en/reference/sparse_vector.md deleted file mode 100644 index 245992260c..0000000000 --- a/v2.4.x/site/en/reference/sparse_vector.md +++ /dev/null @@ -1,308 +0,0 @@ ---- -id: sparse_vector.md -summary: Learn how to use sparse vectors in Milvus. -title: Sparse Vector ---- - -# Sparse Vector - -Sparse vectors represent words or phrases using vector embeddings where most elements are zero, with only one non-zero element indicating the presence of a specific word. Sparse vector models, such as [SPLADEv2](https://arxiv.org/abs/2109.10086), outperform dense models in out-of-domain knowledge search, keyword-awareness, and interpretability. They are particularly useful in information retrieval, natural language processing, and recommendation systems, where combining sparse vectors for recall with a large model for ranking can significantly improve retrieval results. - -In Milvus, the use of sparse vectors follows a similar workflow to that of dense vectors. It involves creating a collection with a sparse vector column, inserting data, creating an index, and conducting similarity searches and scalar queries. - -In this tutorial, you will learn how to: - -- Prepare sparse vector embeddings; -- Create a collection with a sparse vector field; -- Insert entities with sparse vector embeddings; -- Index the collection and perform ANN search on sparse vectors. - -To see sparse vectors in action, refer to [hello_sparse.py](https://github.com/milvus-io/pymilvus/blob/master/examples/milvus_client/sparse.py). - -
-

notes

- Currently, the support for sparse vectors is a beta feature in 2.4.0, with plans to make it generally available in 3.0.0. -
- -## Prepare sparse vector embeddings - -To use sparse vectors in Milvus, prepare vector embeddings in one of the supported formats: - -- __Sparse Matrices__: Utilize the [scipy.sparse](https://docs.scipy.org/doc/scipy/reference/sparse.html#module-scipy.sparse) class family to represent your sparse embeddings. This method is efficient for handling large-scale, high-dimensional data. - -- __List of Dictionaries__: Represent each sparse embedding as a dictionary, structured as `{dimension_index: value, ...}`, where each key-value pair represents the dimension index and its corresponding value. - - Example: - - ```python - {2: 0.33, 98: 0.72, ...} - ``` - -- __List of Iterables of Tuples__: Similar to the list of dictionaries, but use an iterable of tuples, `[(dimension_index, value)]`, to specify only the non-zero dimensions and their values. - - Example: - - ```python - [(2, 0.33), (98, 0.72), ...] - ``` - -The following example prepares sparse embeddings by generating a random sparse matrix for 10,000 entities, each with 10,000 dimensions and a sparsity density of 0.005. - -```python -# Prepare entities with sparse vector representation -import numpy as np -import random - -rng = np.random.default_rng() - -num_entities, dim = 10000, 10000 - -# Generate random sparse rows with an average of 25 non-zero elements per row -entities = [ - { - "scalar_field": rng.random(), - # To represent a single sparse vector row, you can use: - # - Any of the scipy.sparse sparse matrices class family with shape[0] == 1 - # - Dict[int, float] - # - Iterable[Tuple[int, float]] - "sparse_vector": { - d: rng.random() for d in random.sample(range(dim), random.randint(20, 30)) - }, - } - for _ in range(num_entities) -] - -# print the first entity to check the representation -print(entities[0]) - -# Output: -# { -# 'scalar_field': 0.520821523849214, -# 'sparse_vector': { -# 5263: 0.2639375518635271, -# 3573: 0.34701499565746674, -# 9637: 0.30856525997853057, -# 4399: 0.19771651149001523, -# 6959: 0.31025067641541815, -# 1729: 0.8265339135915016, -# 1220: 0.15303302147479103, -# 7335: 0.9436728846033107, -# 6167: 0.19929870545596562, -# 5891: 0.8214617920371853, -# 2245: 0.7852255053773395, -# 2886: 0.8787982039149889, -# 8966: 0.9000606703940665, -# 4910: 0.3001170013981104, -# 17: 0.00875671667413136, -# 3279: 0.7003425473001098, -# 2622: 0.7571360018373428, -# 4962: 0.3901879090102064, -# 4698: 0.22589525720196246, -# 3290: 0.5510228492587324, -# 6185: 0.4508413201390492 -# } -# } -``` - -
- -

notes

- -

The vector dimensions must be of Python int or numpy.integer type, and the values must be of Python float or numpy.floating type.

- -
- -To generate embeddings, you can also use the `model` package built in the PyMilvus library, which offers a range of embedding functions. For details, refer to [Embeddings](embeddings.md). - -## Create a collection with a sparse vector field - -To create a collection with a sparse vector field, set the __datatype__ of the sparse vector field to __DataType.SPARSE_FLOAT_VECTOR__. Unlike dense vectors, there is no need to specify a dimension for sparse vectors. - -```python -from pymilvus import MilvusClient, DataType - -# Create a MilvusClient instance -client = MilvusClient(uri="http://localhost:19530") - -# Create a collection with a sparse vector field -schema = client.create_schema( - auto_id=True, - enable_dynamic_fields=True, -) - -schema.add_field(field_name="pk", datatype=DataType.VARCHAR, is_primary=True, max_length=100) -schema.add_field(field_name="scalar_field", datatype=DataType.DOUBLE) -# For sparse vector, no need to specify dimension -schema.add_field(field_name="sparse_vector", datatype=DataType.SPARSE_FLOAT_VECTOR) # set `datatype` to `SPARSE_FLOAT_VECTOR` - -client.create_collection(collection_name="test_sparse_vector", schema=schema) -``` - -For details on common collection parameters, refer to [create_collection() -](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Collections/create_collection.md). - -## Insert entities with sparse vector embeddings - -To insert entities with sparse vector embeddings, simply pass the list of entities to the [`insert()`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Vector/insert.md) method. - -```python -# Insert entities -client.insert(collection_name="test_sparse_vector", data=entities) -``` - -## Index the collection - -Before performing similarity searches, create an index for the collection. For more information on index types and parameters, refer to [add_index()](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Management/add_index.md) and [create_index()](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Management/create_index.md). - -```python -# Index the collection - -# Prepare index params -index_params = client.prepare_index_params() - -index_params.add_index( - field_name="sparse_vector", - index_name="sparse_inverted_index", - index_type="SPARSE_INVERTED_INDEX", # the type of index to be created. set to `SPARSE_INVERTED_INDEX` or `SPARSE_WAND`. - metric_type="IP", # the metric type to be used for the index. Currently, only `IP` (Inner Product) is supported. - params={"drop_ratio_build": 0.2}, # the ratio of small vector values to be dropped during indexing. -) - -# Create index -client.create_index(collection_name="test_sparse_vector", index_params=index_params) -``` - -For index building on sparse vectors, take note of the following: - -- `index_type`: The type of index to be built. Possible options for sparse vectors: - - - `SPARSE_INVERTED_INDEX`: An inverted index that maps each dimension to its non-zero vectors, facilitating direct access to relevant data during searches. Ideal for datasets with sparse but high-dimensional data. - - - `SPARSE_WAND`: Utilizes the Weak-AND (WAND) algorithm to quickly bypass unlikely candidates, focusing evaluation on those with higher ranking potential. Treats dimensions as terms and vectors as documents, speeding up searches in large, sparse datasets. - -- `metric_type`: Only `IP` (Inner Product) distance metric is supported for sparse vectors. - -- `params.drop_ratio_build`: The index parameter used specifically for sparse vectors. It controls the proportion of small vector values that are excluded during the indexing process. This parameter enables fine-tuning of the trade-off between efficiency and accuracy by disregarding small values when constructing the index. For instance, if `drop_ratio_build = 0.3`, during the index construction, all values from all sparse vectors are gathered and sorted. The smallest 30% of these values are not included in the index, thereby reducing the computational workload during search. - -For more information, refer to [In-memory Index](index.md). - -## Perform ANN search - -After the collection is indexed and loaded into memory, use the [`search()`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Vector/search.md) method to retrieve the relevant documents based on the query. - -```python -# Load the collection into memory -client.load_collection(collection_name="test_sparse_vector") - -# Perform ANN search on sparse vectors - -# for demo purpose we search for the last inserted vector -query_vector = entities[-1]["sparse_vector"] - -search_params = { - "metric_type": "IP", - "params": {"drop_ratio_search": 0.2}, # the ratio of small vector values to be dropped during search. -} - -search_res = client.search( - collection_name="test_sparse_vector", - data=[query_vector], - limit=3, - output_fields=["pk", "scalar_field"], - search_params=search_params, -) - -for hits in search_res: - for hit in hits: - print(f"hit: {hit}") - -# Output: -# hit: {'id': '448458373272710786', 'distance': 7.220192909240723, 'entity': {'pk': '448458373272710786', 'scalar_field': 0.46767865218233806}} -# hit: {'id': '448458373272708317', 'distance': 1.2287548780441284, 'entity': {'pk': '448458373272708317', 'scalar_field': 0.7315987515699472}} -# hit: {'id': '448458373272702005', 'distance': 0.9848432540893555, 'entity': {'pk': '448458373272702005', 'scalar_field': 0.9871869181562156}} -``` - -When configuring search parameters, take note of the following: - -- `params.drop_ratio_search`: The search parameter used specifically for sparse vectors. This option allows fine-tuning of the search process by specifying the ratio of the smallest values in the query vector to ignore. It helps balance search precision and performance. The smaller the value set for `drop_ratio_search`, the less these small values contribute to the final score. By ignoring some small values, search performance can be improved with minimal impact on accuracy. - -## Perform scalar queries - -In addition to ANN search, Milvus also supports scalar queries on sparse vectors. These queries allow you to retrieve documents based on a scalar value associated with the sparse vector. For more information on parameters, refer to [query()](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Vector/query.md). - -Filter entities with __scalar_field__ greater than 3: - -```python -# Perform a query by specifying filter expr -filter_query_res = client.query( - collection_name="test_sparse_vector", - filter="scalar_field > 0.999", -) - -print(filter_query_res[:2]) - -# Output: -# [{'pk': '448458373272701862', 'scalar_field': 0.9994093623822689, 'sparse_vector': {173: 0.35266244411468506, 400: 0.49995484948158264, 480: 0.8757831454277039, 661: 0.9931875467300415, 1040: 0.0965644046664238, 1728: 0.7478245496749878, 2365: 0.4351981580257416, 2923: 0.5505295395851135, 3181: 0.7396837472915649, 3848: 0.4428485333919525, 4701: 0.39119353890419006, 5199: 0.790219783782959, 5798: 0.9623121619224548, 6213: 0.453134149312973, 6341: 0.745091438293457, 6775: 0.27766478061676025, 6875: 0.017947908490896225, 8093: 0.11834774166345596, 8617: 0.2289179265499115, 8991: 0.36600416898727417, 9346: 0.5502803921699524}}, {'pk': '448458373272702421', 'scalar_field': 0.9990218525410719, 'sparse_vector': {448: 0.587817907333374, 1866: 0.0994109958410263, 2438: 0.8672442436218262, 2533: 0.8063794374465942, 2595: 0.02122959867119789, 2828: 0.33827054500579834, 2871: 0.1984412521123886, 2938: 0.09674275666475296, 3154: 0.21552987396717072, 3662: 0.5236313343048096, 3711: 0.6463911533355713, 4029: 0.4041993021965027, 7143: 0.7370485663414001, 7589: 0.37588241696357727, 7776: 0.436136394739151, 7962: 0.06377989053726196, 8385: 0.5808192491531372, 8592: 0.8865005970001221, 8648: 0.05727503448724747, 9071: 0.9450633525848389, 9161: 0.146037295460701, 9358: 0.1903032660484314, 9679: 0.3146636486053467, 9974: 0.8561339378356934, 9991: 0.15841573476791382}}] -``` - -Filter entities by primary key: - -```python -# primary keys of entities that satisfy the filter -pks = [ret["pk"] for ret in filter_query_res] - -# Perform a query by primary key -pk_query_res = client.query( - collection_name="test_sparse_vector", filter=f"pk == '{pks[0]}'" -) - -print(pk_query_res) - -# Output: -# [{'scalar_field': 0.9994093623822689, 'sparse_vector': {173: 0.35266244411468506, 400: 0.49995484948158264, 480: 0.8757831454277039, 661: 0.9931875467300415, 1040: 0.0965644046664238, 1728: 0.7478245496749878, 2365: 0.4351981580257416, 2923: 0.5505295395851135, 3181: 0.7396837472915649, 3848: 0.4428485333919525, 4701: 0.39119353890419006, 5199: 0.790219783782959, 5798: 0.9623121619224548, 6213: 0.453134149312973, 6341: 0.745091438293457, 6775: 0.27766478061676025, 6875: 0.017947908490896225, 8093: 0.11834774166345596, 8617: 0.2289179265499115, 8991: 0.36600416898727417, 9346: 0.5502803921699524}, 'pk': '448458373272701862'}] -``` - -## Limits - -When using sparse vectors in Milvus, consider the following limits: - -- Currently, only the __IP__ distance metric is supported for sparse vectors. - -- For sparse vector fields, only the __SPARSE_INVERTED_INDEX__ and __SPARSE_WAND__ index types are supported. - -- Currently, [range search](https://milvus.io/docs/single-vector-search.md#Range-search), [grouping search](https://milvus.io/docs/single-vector-search.md#Grouping-search), and [search iterator](https://milvus.io/docs/with-iterators.md#Search-with-iterator) are not supported for sparse vectors. - -## FAQ - -- __What distance metric is supported for sparse vectors?__ - - Sparse vectors only support the Inner Product (IP) distance metric due to the high dimensionality of sparse vectors, which makes L2 distance and cosine distance impractical. - -- __Can you explain the difference between SPARSE_INVERTED_INDEX and SPARSE_WAND, and how do I choose between them?__ - - __SPARSE_INVERTED_INDEX__ is a traditional inverted index, while __SPARSE_WAND__ uses the [Weak-AND](https://dl.acm.org/doi/10.1145/956863.956944) algorithm to reduce the number of full IP distance evaluations during search. __SPARSE_WAND__ is typically faster, but its performance can decline with increasing vector density. To choose between them, conduct experiments and benchmarks based on your specific dataset and use case. - -- __How should I choose the drop_ratio_build and drop_ratio_search parameters?__ - - The choice of __drop_ratio_build__ and __drop_ratio_search__ depends on the characteristics of your data and your requirements for search latency/throughput and accuracy. - -- __What data types are supported for sparse embeddings?__ - - The dimension part must be an unsigned 32-bit integer, and the value part can be a non-negative 32-bit floating-point number. - -- __Can the dimension of a sparse embedding be any discrete value within the uint32 space?__ - - Yes, with one exception. The dimension of a sparse embedding can be any value in the range of `[0, maximum of uint32)`. This means you cannot use the maximum value of uint32. - -- __Are searches on growing segments conducted through an index or by brute force?__ - - Searches on growing segments are conducted through an index of the same type as the sealed segment index. For new growing segments before the index is built, a brute force search is used. - -- __Is it possible to have both sparse and dense vectors in a single collection?__ - - Yes, with multiple vector type support, you can create collections with both sparse and dense vector columns and perform hybrid searches on them. - -- __What are the requirements for sparse embeddings to be inserted or searched?__ - - Sparse embeddings must have at least one non-zero value, and vector indices must be non-negative. - diff --git a/v2.4.x/site/en/userGuide/tune_consistency.md b/v2.4.x/site/en/reference/tune_consistency.md similarity index 100% rename from v2.4.x/site/en/userGuide/tune_consistency.md rename to v2.4.x/site/en/reference/tune_consistency.md diff --git a/v2.4.x/site/en/userGuide/collections/create-collection-instantly.md b/v2.4.x/site/en/userGuide/collections/create-collection-instantly.md new file mode 100644 index 0000000000..3ffd233964 --- /dev/null +++ b/v2.4.x/site/en/userGuide/collections/create-collection-instantly.md @@ -0,0 +1,335 @@ +--- +id: create-collection-instantly.md +title: Create Collection​ Instantly​ +--- + +# Create Collection Instantly​ + +You can create a collection instantly by setting its name and the vector field dimensionality. Milvus automatically indexes the vector field and loads the collection upon creation. This page demonstrates how to create a collection instantly with default settings.​ + +## Overview​ + +A collection is a two-dimensional table with fixed columns and variant rows. Each column represents a field, and each row represents an entity. A schema is required to implement such structural data management. Every entity to insert has to meet the constraints defined in the schema.​ + +AIGC applications usually use vector databases as a knowledge base to manage the data generated during the interaction between users and Large Language Models (LLMs). Such knowledge bases are almost similar. To accelerate the use of Milvus clusters in such scenarios, an instant method is available for you to create a collection with only two parameters, namely the collection name and the vector field dimensionality.​ + +When you create a collection instantly with default settings, the following settings apply:​ + +- The primary and vector fields are added to the schema (**id** and **vector**).​ + +- The primary field accepts integers and disables **AutoId**.​ + +- The vector field accepts floating vector embeddings.​ + +- **AUTOINDEX** is used to create an index on the vector field.​ + +- **COSINE** is used to measure similarities between vector embeddings.​ + +- The reserves dynamic field named **$meta** is enabled to save non-schema-defined fields and their values in key-value pairs.​ + +- The collection is automatically loaded upon creation.​ + +For details on the terminologies above, refer to [​Collection Explained](manage-collections.md). ​ + +It is worth noting that creating a collection instantly with default settings does not fit all scenarios. You are advised to familiarize yourself with the [common collection creation procedure](create-collection.md) so that you can gain a better understanding of Milvus's capabilities.​ + +## Quick Setup​ + +In this manner, you can create a collection instantly with only the collection name and the vector field dimensionality.​ + +
+ Python + Java + Node.js + Go + cURL +
+ +```python +from pymilvus import MilvusClient, DataType​ +​ +CLUSTER_ENDPOINT = "http://localhost:19530"​ +TOKEN = "root:Milvus"​ +​ +# 1. Set up a Milvus client​ +client = MilvusClient(​ + uri=CLUSTER_ENDPOINT,​ + token=TOKEN ​ +)​ +​ +# 2. Create a collection in quick setup mode​ +client.create_collection(​ + collection_name="quick_setup",​ + dimension=5​ +)​ +​ +res = client.get_load_state(​ + collection_name="quick_setup"​ +)​ +​ +print(res)​ +​ +# Output​ +#​ +# {​ +# "state": ""​ +# }​ + +``` + +```java +import io.milvus.v2.client.ConnectConfig;​ +import io.milvus.v2.client.MilvusClientV2;​ +import io.milvus.v2.service.collection.request.GetLoadStateReq;​ +import io.milvus.v2.service.collection.request.CreateCollectionReq;​ +​ +String CLUSTER_ENDPOINT = "http://localhost:19530";​ +String TOKEN = "root:Milvus";​ +​ +// 1. Connect to Milvus server​ +ConnectConfig connectConfig = ConnectConfig.builder()​ + .uri(CLUSTER_ENDPOINT)​ + .token(TOKEN)​ + .build();​ +​ +MilvusClientV2 client = new MilvusClientV2(connectConfig);​ +​ +// 2. Create a collection in quick setup mode​ +CreateCollectionReq quickSetupReq = CreateCollectionReq.builder()​ + .collectionName("quick_setup")​ + .dimension(5)​ + .build();​ +​ +client.createCollection(quickSetupReq);​ +​ +GetLoadStateReq quickSetupLoadStateReq = GetLoadStateReq.builder()​ + .collectionName("quick_setup")​ + .build();​ +​ +Boolean res = client.getLoadState(quickSetupLoadStateReq);​ +System.out.println(res);​ +​ +// Output:​ +// true​ + +``` + +```javascript +// 1. Set up a Milvus Client​ +import { MilvusClient, DataType } from "@zilliz/milvus2-sdk-node";​ +​ +const address = "http://localhost:19530";​ +const token = "root:Milvus";​ +const client = new MilvusClient({address, token});​ +​ +// 2. Create a collection in quick setup mode​ +let res = await client.createCollection({​ + collection_name: "quick_setup",​ + dimension: 5,​ +}); ​ +​ +console.log(res.error_code)​ +​ +// Output​ +// ​ +// Success​ +// ​ +​ +res = await client.getLoadState({​ + collection_name: "quick_setup"​ +})​ +​ +console.log(res.state)​ +​ +// Output​ +// ​ +// LoadStateLoaded​ +// ​ + +``` + +```go +// Go 缺失​ + +``` + +```curl +export CLUSTER_ENDPOINT="http://localhost:19530"​ +export TOKEN="root:Milvus"​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/collections/create" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "quick_setup",​ + "dimension": 5​ +}'​ +​ +# {​ +# "code": 0,​ +# "data": {}​ +# }​ + +``` + +## Quick Setup with Custom Fields​ + +If the default metric type, field names, and data types does not meet your need, you can tune these settings as follows.​ + +
+ Python + Java + Node.js + Go + cURL +
+ +```python +from pymilvus import MilvusClient, DataType​ +​ +CLUSTER_ENDPOINT = "http://localhost:19530"​ +TOKEN = "root:Milvus"​ +​ +# 1. Set up a Milvus client​ +client = MilvusClient(​ + uri=CLUSTER_ENDPOINT,​ + token=TOKEN ​ +)​ +​ +# 2. Create a collection in quick setup mode​ +client.create_collection(​ + collection_name="custom_quick_setup",​ + dimension=5,​ + primary_field_name="my_id",​ + id_type="string",​ + vector_field_name="my_vector",​ + metric_type="L2",​ + auto_id=True,​ + max_length=512​ +)​ +​ +res = client.get_load_state(​ + collection_name="custom_quick_setup"​ +)​ +​ +print(res)​ +​ +# Output​ +#​ +# {​ +# "state": ""​ +# }​ + +``` + +```java +import io.milvus.v2.client.ConnectConfig;​ +import io.milvus.v2.client.MilvusClientV2;​ +import io.milvus.v2.service.collection.request.GetLoadStateReq;​ +import io.milvus.v2.service.collection.request.CreateCollectionReq;​ +​ +String CLUSTER_ENDPOINT = "http://localhost:19530";​ +String TOKEN = "root:Milvus";​ +​ +// 1. Connect to Milvus server​ +ConnectConfig connectConfig = ConnectConfig.builder()​ + .uri(CLUSTER_ENDPOINT)​ + .token(TOKEN)​ + .build();​ +​ +MilvusClientV2 client = new MilvusClientV2(connectConfig);​ +​ +// 2. Create a collection in quick setup mode​ +CreateCollectionReq customQuickSetupReq = CreateCollectionReq.builder()​ + .collectionName("custom_quick_setup")​ + .dimension(5)​ + .primaryFieldName("my_id")​ + .idType(DataType.VarChar)​ + .maxLength(512)​ + .vectorFieldName("my_vector")​ + .metricType("L2")​ + .autoID(true)​ + .build();​ +​ +client.createCollection(customQuickSetupReq);​ +​ +GetLoadStateReq customQuickSetupLoadStateReq = GetLoadStateReq.builder()​ + .collectionName("custom_quick_setup")​ + .build();​ +​ +Boolean res = client.getLoadState(customQuickSetupLoadStateReq);​ +System.out.println(res);​ +​ +// Output:​ +// true​ + +``` + +```javascript +// 1. Set up a Milvus Client​ +const address = "http://localhost:19530";​ +const token = "root:Milvus";​ +const client = new MilvusClient({address, token});​ +​ +// 2. Create a collection in quick setup mode​ +let res = await client.createCollection({​ + collection_name: "custom_quick_setup",​ + dimension: 5,​ + primary_field_name: "my_id",​ + id_type: "Varchar",​ + max_length: 512,​ + vector_field_name: "my_vector",​ + metric_type: "L2",​ + auto_id: true​ +}); ​ +​ +console.log(res.error_code)​ +​ +// Output​ +// ​ +// Success​ +// ​ +​ +res = await client.getLoadState({​ + collection_name: "custom_quick_setup"​ +})​ +​ +console.log(res.state)​ +​ +// Output​ +// ​ +// LoadStateLoaded​ +// ​ + +``` + +```go +// Go 缺失​ + +``` + +```curl +export CLUSTER_ENDPOINT="http://localhost:19530"​ +export TOKEN="root:Milvus"​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/collections/create" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "custom_quick_setup",​ + "dimension": 5,​ + "primaryFieldName": "my_id",​ + "idType": "VarChar",​ + "vectorFieldName": "my_vector",​ + "metricType": "L2",​ + "autoId": true,​ + "params": {​ + "max_length": "512"​ + }​ +}'​ + +``` + +If the collections created using the above two manners still cannot meet your needs, consider following the procedure in [​Create Collection](create-collection.md).​ \ No newline at end of file diff --git a/v2.4.x/site/en/userGuide/collections/create-collection.md b/v2.4.x/site/en/userGuide/collections/create-collection.md new file mode 100644 index 0000000000..5ca1b8310c --- /dev/null +++ b/v2.4.x/site/en/userGuide/collections/create-collection.md @@ -0,0 +1,850 @@ +--- +id: create-collection.md +title: Create Collection​ +--- + +# Create Collection​ + +You can create a collection by defining its schema, index parameters, metric type, and whether to load it upon creation. This page introduces how to create a collection from scratch.​ + +## Overview​ + +A collection is a two-dimensional table with fixed columns and variant rows. Each column represents a field, and each row represents an entity. A schema is required to implement such structural data management. Every entity to insert has to meet the constraints defined in the schema.​ + +You can determine every aspect of a collection, including its schema, index parameters, metric type, and whether to load it upon creation to ensure that the collection fully meets your requirements.​ + +To create a collection, you need to​ + +- [Create schema](#create-schema)​ + +- [Set index parameters](#optional-set-index-parameters) (Optional)​ + +- [Create collection](#create-collection)​ + +## Create Schema​ + +A schema defines the data structure of a collection. When creating a collection, you need to design the schema based on your requirements. For details, refer to [​Schema Explained](manage-collections.md).​ + +The following code snippets create a schema with the enabled dynamic field and three mandatory fields named `my_id`, `my_vector`, and `my_varchar`.​ + +
+ +You can set default values for any scalar field and make it nullable. For details, refer to [​Nullable & Default](nullable-and-default.md).​ + +
+ +
+ Python + Java + Node.js + Go + cURL +
+ +```python +# 3. Create a collection in customized setup mode​ +from pymilvus import MilvusClient, DataType​ +​ +client = MilvusClient(​ + uri="http://localhost:19530",​ + token="root:Milvus"​ +)​ +​ +# 3.1. Create schema​ +schema = MilvusClient.create_schema(​ + auto_id=False,​ + enable_dynamic_field=True,​ +)​ +​ +# 3.2. Add fields to schema​ +schema.add_field(field_name="my_id", datatype=DataType.INT64, is_primary=True)​ +schema.add_field(field_name="my_vector", datatype=DataType.FLOAT_VECTOR, dim=5)​ +schema.add_field(field_name="my_varchar", datatype=DataType.VARCHAR, max_length=512)​ + +``` + +```java +import io.milvus.v2.common.DataType;​ +import io.milvus.v2.client.ConnectConfig;​ +import io.milvus.v2.client.MilvusClientV2;​ +import io.milvus.v2.service.collection.request.AddFieldReq;​ +import io.milvus.v2.service.collection.request.CreateCollectionReq;​ +​ +String CLUSTER_ENDPOINT = "http://localhost:19530";​ +String TOKEN = "root:Milvus";​ +​ +// 1. Connect to Milvus server​ +ConnectConfig connectConfig = ConnectConfig.builder()​ + .uri(CLUSTER_ENDPOINT)​ + .token(TOKEN)​ + .build();​ +​ +MilvusClientV2 client = new MilvusClientV2(connectConfig);​ +​ +// 3. Create a collection in customized setup mode​ +​ +// 3.1 Create schema​ +CreateCollectionReq.CollectionSchema schema = client.createSchema();​ +​ +// 3.2 Add fields to schema​ +schema.addField(AddFieldReq.builder()​ + .fieldName("my_id")​ + .dataType(DataType.Int64)​ + .isPrimaryKey(true)​ + .autoID(false)​ + .build());​ +​ +schema.addField(AddFieldReq.builder()​ + .fieldName("my_vector")​ + .dataType(DataType.FloatVector)​ + .dimension(5)​ + .build());​ +​ +schema.addField(AddFieldReq.builder()​ + .fieldName("my_varchar")​ + .dataType(DataType.VarChar)​ + .maxLength(512)​ + .build());​ + +``` + +```javascript +import { MilvusClient, DataType } from "@zilliz/milvus2-sdk-node";​ +​ +const address = "http://localhost:19530";​ +const token = "root:Milvus";​ +const client = new MilvusClient({address, token});​ +​ +// 3. Create a collection in customized setup mode​ +// 3.1 Define fields​ +const fields = [​ + {​ + name: "my_id",​ + data_type: DataType.Int64,​ + is_primary_key: true,​ + auto_id: false​ + },​ + {​ + name: "my_vector",​ + data_type: DataType.FloatVector,​ + dim: 5​ + },​ + {​ + name: "my_varchar",​ + data_type: DataType.VarChar,​ + max_length: 512​ + }​ +]​ + +``` + +```go +import "github.com/milvus-io/milvus/client/v2/entity"​ +​ +schema := entity.NewSchema().WithDynamicFieldEnabled(true).​ + WithField(entity.NewField().WithName("my_id").WithIsAutoID(true).WithDataType(entity.FieldTypeInt64).WithIsPrimaryKey(true)).​ + WithField(entity.NewField().WithName("my_vector").WithDataType(entity.FieldTypeFloatVector).WithDim(5)).​ + WithField(entity.NewField().WithName("my_varchar").WithDataType(entity.FieldTypeVarChar).WithMaxLength(512))thDim(5))​ + +``` + +```curl +export schema='{​ + "autoId": false,​ + "enabledDynamicField": false,​ + "fields": [​ + {​ + "fieldName": "my_id",​ + "dataType": "Int64",​ + "isPrimary": true​ + },​ + {​ + "fieldName": "my_vector",​ + "dataType": "FloatVector",​ + "elementTypeParams": {​ + "dim": "5"​ + }​ + },​ + {​ + "fieldName": "my_varchar",​ + "dataType": "VarChar",​ + "elementTypeParams": {​ + "max_length": 512​ + }​ + }​ + ]​ + }'​ + +``` + +## (Optional) Set Index Parameters​ + +Creating an index on a specific field accelerates the search against this field. An index records the order of entities within a collection. As shown in the following code snippets, you can use `metric_type` and `index_type` to select appropriate ways for Milvus to index a field and measure similarities between vector embeddings.​ + +In Milvus, you can use `AUTOINDEX` as the index type for all vector fields, and one of `COSINE`, `L2`, and `IP` as the metric type based on your needs.​ + +As demonstrated in the above code snippet, you need to set both the index type and metric type for vector fields and only the index type for the scalar fields. Indexes are mandatory for vector fields, and you are advised to create indexes on scalar fields frequently used in filtering conditions.​ + +For details, refer to [​Indexes](index-vector-fields.md).​ + +
+ Python + Java + Node.js + Go + cURL +
+ +```python +# 3.3. Prepare index parameters​ +index_params = client.prepare_index_params()​ +​ +# 3.4. Add indexes​ +index_params.add_index(​ + field_name="my_id",​ + index_type="STL_SORT"​ +)​ +​ +index_params.add_index(​ + field_name="my_vector", ​ + index_type="AUTOINDEX",​ + metric_type="COSINE"​ +)​ + +``` + +```java +import io.milvus.v2.common.IndexParam;​ +import java.util.*;​ +​ +// 3.3 Prepare index parameters​ +IndexParam indexParamForIdField = IndexParam.builder()​ + .fieldName("my_id")​ + .indexType(IndexParam.IndexType.STL_SORT)​ + .build();​ +​ +IndexParam indexParamForVectorField = IndexParam.builder()​ + .fieldName("my_vector")​ + .indexType(IndexParam.IndexType.AUTOINDEX)​ + .metricType(IndexParam.MetricType.COSINE)​ + .build();​ +​ +List indexParams = new ArrayList<>();​ +indexParams.add(indexParamForIdField);​ +indexParams.add(indexParamForVectorField);​ + +``` + +```javascript +// 3.2 Prepare index parameters​ +const index_params = [{​ + field_name: "my_id",​ + index_type: "STL_SORT"​ +},{​ + field_name: "my_vector",​ + index_type: "AUTOINDEX",​ + metric_type: "COSINE"​ +}]​ + +``` + +```go +import (​ + "github.com/milvus-io/milvus/client/v2"​ + "github.com/milvus-io/milvus/client/v2/entity"​ + "github.com/milvus-io/milvus/client/v2/index"​ +)​ +​ +indexOptions := []client.CreateIndexOption{​ + client.NewCreateIndexOption(collectionName, "my_vector", index.NewAutoIndex(entity.COSINE)).WithIndexName("my_vector"),​ + client.NewCreateIndexOption(collectionName, "my_id", index.NewSortedIndex()).WithIndexName("my_id"),​ +}​ + +``` + +```curl +export indexParams='[​ + {​ + "fieldName": "my_vector",​ + "metricType": "COSINE",​ + "indexName": "my_vector",​ + "indexType": "AUTOINDEX"​ + },​ + {​ + "fieldName": "my_id",​ + "indexName": "my_id",​ + "indexType": "STL_SORT"​ + }​ + ]'​ + +``` + +## Create Collection​ + +If you have created a collection with index parameters, Milvus automatically loads the collection upon its creation. In this case, all fields mentioned in the index parameters are indexed.​ + +The following code snippets demonstrate how to create the collection with index parameters and check its load status.​ + +
+ Python + Java + Node.js + Go + cURL +
+ +```python +# 3.5. Create a collection with the index loaded simultaneously​ +client.create_collection(​ + collection_name="customized_setup_1",​ + schema=schema,​ + index_params=index_params​ +)​ +​ +res = client.get_load_state(​ + collection_name="customized_setup_1"​ +)​ +​ +print(res)​ +​ +# Output​ +#​ +# {​ +# "state": ""​ +# }​ + +``` + +```java +import io.milvus.v2.service.collection.request.CreateCollectionReq;​ +import io.milvus.v2.service.collection.request.GetLoadStateReq;​ +​ +// 3.4 Create a collection with schema and index parameters​ +CreateCollectionReq customizedSetupReq1 = CreateCollectionReq.builder()​ + .collectionName("customized_setup_1")​ + .collectionSchema(schema)​ + .indexParams(indexParams)​ + .build();​ +​ +client.createCollection(customizedSetupReq1);​ +​ +// 3.5 Get load state of the collection​ +GetLoadStateReq customSetupLoadStateReq1 = GetLoadStateReq.builder()​ + .collectionName("customized_setup_1")​ + .build();​ +​ +Boolean loaded = client.getLoadState(customSetupLoadStateReq1);​ +System.out.println(loaded);​ +​ +// Output:​ +// true​ + +``` + +```javascript +// 3.3 Create a collection with fields and index parameters​ +res = await client.createCollection({​ + collection_name: "customized_setup_1",​ + fields: fields,​ + index_params: index_params,​ +})​ +​ +console.log(res.error_code) ​ +​ +// Output​ +// ​ +// Success​ +// ​ +​ +res = await client.getLoadState({​ + collection_name: "customized_setup_1"​ +})​ +​ +console.log(res.state)​ +​ +// Output​ +// ​ +// LoadStateLoaded​ +// ​ + +``` + +```go +import "github.com/milvus-io/milvus/client/v2"​ +​ +err := cli.CreateCollection(ctx, client.NewCreateCollectionOption("customized_setup_1", schema).​ + WithIndexOptions(indexOptions...),​ +)​ +if err != nil {​ + // handle error​ +}​ +fmt.Println("collection created")​ + +``` + +```curl +export CLUSTER_ENDPOINT="http://localhost:19530"​ +export TOKEN="root:Milvus"​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/collections/create" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d "{​ + \"collectionName\": \"customized_setup_1\",​ + \"schema\": $schema,​ + \"indexParams\": $indexParams​ +}"​ + +``` + +You can also create a collection without any index parameters and add them afterward. In this case, Milvus does not load the collection upon its creation. For details on how to create indexes for an existing collection, refer to [​Index Explained](index-vector-fields.md).​ + +The following code snippet demonstrates how to create a collection without a collection, and the load status of the collection remains unloaded upon creation.​ + +
+ Python + Java + Node.js + Go + cURL +
+ +```python +# 3.6. Create a collection and index it separately​ +client.create_collection(​ + collection_name="customized_setup_2",​ + schema=schema,​ +)​ +​ +res = client.get_load_state(​ + collection_name="customized_setup_2"​ +)​ +​ +print(res)​ +​ +# Output​ +#​ +# {​ +# "state": ""​ +# }​ + +``` + +```java +// 3.6 Create a collection and index it separately​ +CreateCollectionReq customizedSetupReq2 = CreateCollectionReq.builder()​ + .collectionName("customized_setup_2")​ + .collectionSchema(schema)​ + .build();​ +​ +client.createCollection(customizedSetupReq2);​ +​ +GetLoadStateReq customSetupLoadStateReq2 = GetLoadStateReq.builder()​ + .collectionName("customized_setup_2")​ + .build();​ + ​ +Boolean loaded = client.getLoadState(customSetupLoadStateReq2);​ +System.out.println(loaded);​ +​ +// Output:​ +// false​ + +``` + +```javascript +// 3.4 Create a collection and index it seperately​ +res = await client.createCollection({​ + collection_name: "customized_setup_2",​ + fields: fields,​ +})​ +​ +console.log(res.error_code)​ +​ +// Output​ +// ​ +// Success​ +// ​ +​ +res = await client.getLoadState({​ + collection_name: "customized_setup_2"​ +})​ +​ +console.log(res.state)​ +​ +// Output​ +// ​ +// LoadStateNotLoad​ +// ​ + +``` + +```go +import "github.com/milvus-io/milvus/client/v2"​ +​ +err := cli.CreateCollection(ctx, client.NewCreateCollectionOption("customized_setup_2", schema))​ +if err != nil {​ + // handle error​ +}​ +fmt.Println("collection created")​ + +``` + +```curl +export CLUSTER_ENDPOINT="http://localhost:19530"​ +export TOKEN="root:Milvus"​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/collections/create" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d "{​ + \"collectionName\": \"customized_setup_2\",​ + \"schema\": $schema​ +}"​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/collections/get_load_state" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d "{​ + \"collectionName\": \"customized_setup_2\"​ +}"​ + +``` + +Milvus also provides a way for you to create a collection instantly. For details, refer to [​Create Collection Instantly](create-collection-instantly.md).​ + +## Set Collection Properties​ + +You can set properties for the collection to create to make it fit into your service. The applicable properties are as follows.​ + +### Set Shard Number​ + +Shards are horizontal slices of a collection. Each shard corresponds to a data input channel. Every collection has a shard by default. You can set the appropriate number of shards when creating a collection based on the expected throughput and the volume of the data to insert into the collection.​ + +In common cases, consider increasing the shard number by one every time the expected throughput increases by 500 MB/s or the volume of data to insert increases by 100 GB. This suggestion does not prevent you from inserting data into the collection using the default shard number.​ + +The following code snippet demonstrates how to set the shard number when you create a collection.​ + +
+ Python + Java + Node.js + Go + cURL +
+ +```python +# With shard number​ +client.create_collection(​ + collection_name="customized_setup_3",​ + schema=schema,​ + # highlight-next-line​ + num_shards=1​ +)​ + +``` + +```java +// With shard number​ +CreateCollectionReq customizedSetupReq3 = CreateCollectionReq.builder()​ + .collectionName("customized_setup_3")​ + .collectionSchema(collectionSchema)​ + // highlight-next-line​ + .numShards(1)​ + .build();​ +client.createCollection(customizedSetupReq3);​ + +``` + +```javascript +const createCollectionReq = {​ + collection_name: "customized_setup_3",​ + schema: schema,​ + // highlight-next-line​ + shards_num: 1​ +}​ + +``` + +```go +import "github.com/milvus-io/milvus/client/v2"​ +​ +err := cli.CreateCollection(ctx, client.NewCreateCollectionOption("customized_setup_3", schema).WithShardNum(1))​ +if err != nil {​ + // handle error​ +}​ +fmt.Println("collection created")​ + +``` + +```curl +export params='{​ + "shardsNum": 1​ +}'​ +​ +export CLUSTER_ENDPOINT="http://localhost:19530"​ +export TOKEN="root:Milvus"​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/collections/create" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d "{​ + \"collectionName\": \"customized_setup_3\",​ + \"schema\": $schema,​ + \"params\": $params​ +}"​ + +``` + +### Enable mmap​ + +Milvus enables mmap on all collections by default, allowing Milvus to map raw field data into memory instead of fully loading them. This reduces memory footprints and increases collection capacity. For details on mmap, refer to [​Use mmap](mmap.md).​ + +
+ Python + Java + Node.js + Go + cURL +
+ +```python +# With mmap​ +client.create_collection(​ + collection_name="customized_setup_4",​ + schema=schema,​ + # highlight-next-line​ + enable_mmap=False​ +)​ + +``` + +```java +import io.milvus.param.Constant;​ +​ +// With MMap​ +CreateCollectionReq customizedSetupReq4 = CreateCollectionReq.builder()​ + .collectionName("customized_setup_4")​ + .collectionSchema(schema)​ + // highlight-next-line​ + .property(Constant.MMAP_ENABLED, "false")​ + .build();​ +client.createCollection(customizedSetupReq4);​ + +``` + +```javascript +client.create_collection({​ + collection_name: "customized_setup_4",​ + schema: schema,​ + properties: {​ + 'mmap.enabled': true,​ + },​ +})​ + +``` + +```go +import (​ + "github.com/milvus-io/milvus/client/v2"​ + "github.com/milvus-io/milvus/pkg/common"​ +)​ +​ +err := cli.CreateCollection(ctx, client.NewCreateCollectionOption("customized_setup_4", schema).WithProperty(common.MmapEnabledKey, true))​ +if err != nil {​ + // handle error​ +}​ +fmt.Println("collection created")​ + +``` + +```curl +# Currently not available for REST + +``` + +### Set Collection TTL​ + +If a collection needs to be dropped for a specific period, consider setting its Time-To-Live (TTL) in seconds. Once the TTL times out, Milvus deletes entities in the collection and drops the collection. The deletion is asynchronous, indicating that searches and queries are still possible before the deletion is complete.​ + +The following code snippet sets the TTL to one day (86400 seconds). You are advised to set the TTL to a couple of days at minimum.​ + +
+ Python + Java + Node.js + Go + cURL +
+ +```python +# With TTL​ +client.create_collection(​ + collection_name="customized_setup_5",​ + schema=schema,​ + # highlight-start​ + properties={​ + "collection.ttl.seconds": 86400​ + }​ + # highlight-end​ +)​ + +``` + +```java +import io.milvus.param.Constant;​ +​ +// With TTL​ +CreateCollectionReq customizedSetupReq5 = CreateCollectionReq.builder()​ + .collectionName("customized_setup_5")​ + .collectionSchema(schema)​ + // highlight-next-line​ + .property(Constant.TTL_SECONDS, "86400")​ + .build();​ +client.createCollection(customizedSetupReq5);​ + +``` + +```javascript +const createCollectionReq = {​ + collection_name: "customized_setup_5",​ + schema: schema,​ + // highlight-start​ + properties: {​ + "collection.ttl.seconds": 86400​ + }​ + // highlight-end​ +}​ + +``` + +```go +import (​ + "github.com/milvus-io/milvus/client/v2"​ + "github.com/milvus-io/milvus/pkg/common"​ +)​ +​ +err = cli.CreateCollection(ctx, client.NewCreateCollectionOption("customized_setup_5", schema).​ + WithProperty(common.CollectionTTLConfigKey, 86400)) // TTL in seconds​ +if err != nil {​ + // handle error​ +}​ +fmt.Println("collection created")​ + +``` + +```curl +export params='{​ + "ttlSeconds": 86400​ +}'​ +​ +export CLUSTER_ENDPOINT="http://localhost:19530"​ +export TOKEN="root:Milvus"​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/collections/create" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d "{​ + \"collectionName\": \"customized_setup_5\",​ + \"schema\": $schema,​ + \"params\": $params​ +}"​ + +``` + +### Set Consistency Level​ + +When creating a collection, you can set the consistency level for searches and queries in the collection. You can also change the consistency level of the collection during a specific search or query.​ + +
+ Python + Java + Node.js + Go + cURL +
+ +```python +# With consistency level​ +client.create_collection(​ + collection_name="customized_setup_6",​ + schema=schema,​ + # highlight-next​ + consistency_level="Bounded",​ +)​ + +``` + +```java +import io.milvus.v2.common.ConsistencyLevel;​ +​ +// With consistency level​ +CreateCollectionReq customizedSetupReq6 = CreateCollectionReq.builder()​ + .collectionName("customized_setup_6")​ + .collectionSchema(schema)​ + // highlight-next-line​ + .consistencyLevel(ConsistencyLevel.BOUNDED)​ + .build();​ +client.createCollection(customizedSetupReq6);​ + +``` + +```javascript +const createCollectionReq = {​ + collection_name: "customized_setup_6",​ + schema: schema,​ + // highlight-next​ + consistency_level: "Bounded",​ + // highlight-end​ +}​ +​ +client.createCollection(createCollectionReq);​ + +``` + +```go +import (​ + "github.com/milvus-io/milvus/client/v2"​ + "github.com/milvus-io/milvus/client/v2/entity"​ +)​ +​ +err := cli.CreateCollection(ctx, client.NewCreateCollectionOption("customized_setup_6", schema).​ + WithConsistencyLevel(entity.ClBounded))​ +if err != nil {​ + // handle error​ +}​ +fmt.Println("collection created")​ + +``` + +```curl +export params='{​ + "consistencyLevel": "Bounded"​ +}'​ +​ +export CLUSTER_ENDPOINT="http://localhost:19530"​ +export TOKEN="root:Milvus"​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/collections/create" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d "{​ + \"collectionName\": \"customized_setup_6\",​ + \"schema\": $schema,​ + \"params\": $params​ +}"​ + +``` + +For more on consistency levels, see [​Consistency Level](consistency.md).​ + +### Enable Dynamic Field​ + +The dynamic field in a collection is a reserved JavaScript Object Notation (JSON) field named **$meta**. Once you have enabled this field, Milvus saves all non-schema-defined fields carried in each entity and their values as key-value pairs in the reserved field.​ + +For details on how to use the dynamic field, refer to [​Dynamic Field](enable-dynamic-field.md).​ diff --git a/v2.4.x/site/en/userGuide/collections/drop-collection.md b/v2.4.x/site/en/userGuide/collections/drop-collection.md new file mode 100644 index 0000000000..cfecad78af --- /dev/null +++ b/v2.4.x/site/en/userGuide/collections/drop-collection.md @@ -0,0 +1,130 @@ +--- +id: drop-collection.md +title: Drop Collection​ +--- + +# Drop Collection​ + +You can drop a collection if it is no longer needed.​ + +## Examples + +The following code snippets assume that you have a collection named **customized_setup_2**.​ + +
+ Python + Java + Node.js + Go + cURL +
+ +```python +from pymilvus import MilvusClient​ +​ +client = MilvusClient(​ + uri="http://localhost:19530",​ + token="root:Milvus"​ +)​ +​ +client.drop_collection(​ + collection_name="customized_setup_2"​ +)​ + +``` + +```java +import io.milvus.v2.service.collection.request.DropCollectionReq;​ +import io.milvus.v2.client.ConnectConfig;​ +import io.milvus.v2.client.MilvusClientV2;​ +​ +String CLUSTER_ENDPOINT = "http://localhost:19530";​ +String TOKEN = "root:Milvus";​ +​ +// 1. Connect to Milvus server​ +ConnectConfig connectConfig = ConnectConfig.builder()​ + .uri(CLUSTER_ENDPOINT)​ + .token(TOKEN)​ + .build();​ +​ +MilvusClientV2 client = new MilvusClientV2(connectConfig);​ +​ +DropCollectionReq dropQuickSetupParam = DropCollectionReq.builder()​ + .collectionName("customized_setup_2")​ + .build();​ +​ +client.dropCollection(dropQuickSetupParam);​ + +``` + +```javascript +import { MilvusClient, DataType } from "@zilliz/milvus2-sdk-node";​ +​ +const address = "http://localhost:19530";​ +const token = "root:Milvus";​ +const client = new MilvusClient({address, token});​ +​ +// 10. Drop the collection​ +res = await client.dropCollection({​ + collection_name: "customized_setup_2"​ +})​ +​ +console.log(res.error_code)​ +​ +// Output​ +// ​ +// Success​ +// ​ + +``` + +```go +import (​ + "context"​ + "fmt"​ + "log"​ +​ + "github.com/milvus-io/milvus/client/v2"​ +)​ +​ +ctx, cancel := context.WithCancel(context.Background())​ +defer cancel()​ +​ +milvusAddr := "127.0.0.1:19530"​ +token := "root:Milvus"​ +​ +cli, err := client.New(ctx, &client.ClientConfig{​ + Address: milvusAddr,​ + APIKey: token,​ +})​ +if err != nil {​ + log.Fatal("failed to connect to milvus server: ", err.Error())​ +}​ +​ +defer cli.Close(ctx)​ +​ +err = cli.DropCollection(ctx, client.NewDropCollectionOption("customized_setup_2"))​ +if err != nil {​ + // handle error​ +}​ + +``` + +```curl +export CLUSTER_ENDPOINT="http://localhost:19530"​ +export TOKEN="root:Milvus"​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/collections/drop" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "customized_setup_2"​ +}'​ +​ +# {​ +# "code": 0,​ +# "data": {}​ +# }​ + +``` diff --git a/v2.4.x/site/en/userGuide/collections/load-and-release.md b/v2.4.x/site/en/userGuide/collections/load-and-release.md new file mode 100644 index 0000000000..13945e6bfe --- /dev/null +++ b/v2.4.x/site/en/userGuide/collections/load-and-release.md @@ -0,0 +1,417 @@ +--- +id: load-and-release.md +title: Load & Release +--- + +# Load & Release​ + +Loading a collection is the prerequisite to conducting similarity searches and queries in collections. This page focuses on the procedures for loading and releasing a collection.​ + +## Load Collection​ + +When you load a collection, Zilliz Cloud loads the index files and the raw data of all fields into memory for rapid response to searches and queries. Entities inserted after a collection load are automatically indexed and loaded.​ + +The following code snippets demonstrate how to load a collection.​ + +
+ Python + Java + Node.js + Go + cURL +
+ +```python +from pymilvus import MilvusClient​ +​ +client = MilvusClient(​ + uri="http://localhost:19530",​ + token="root:Milvus"​ +)​ +​ +# 7. Load the collection​ +client.load_collection(​ + collection_name="customized_setup_1"​ +)​ +​ +res = client.get_load_state(​ + collection_name="customized_setup_1"​ +)​ +​ +print(res)​ +​ +# Output​ +#​ +# {​ +# "state": ""​ +# }​ + +``` + +```java +import io.milvus.v2.service.collection.request.LoadCollectionReq;​ +import io.milvus.v2.service.collection.request.GetLoadStateReq;​ +import io.milvus.v2.client.ConnectConfig;​ +import io.milvus.v2.client.MilvusClientV2;​ +​ +String CLUSTER_ENDPOINT = "http://localhost:19530";​ +String TOKEN = "root:Milvus";​ +​ +// 1. Connect to Milvus server​ +ConnectConfig connectConfig = ConnectConfig.builder()​ + .uri(CLUSTER_ENDPOINT)​ + .token(TOKEN)​ + .build();​ +​ +MilvusClientV2 client = new MilvusClientV2(connectConfig);​ +​ +// 6. Load the collection​ +LoadCollectionReq loadCollectionReq = LoadCollectionReq.builder()​ + .collectionName("customized_setup_1")​ + .build();​ +​ +client.loadCollection(loadCollectionReq);​ +​ +// 7. Get load state of the collection​ +GetLoadStateReq loadStateReq = GetLoadStateReq.builder()​ + .collectionName("customized_setup_1")​ + .build();​ +​ +Boolean res = client.getLoadState(loadStateReq);​ +System.out.println(res);​ +​ +// Output:​ +// true​ + +``` + +```javascript +import { MilvusClient, DataType } from "@zilliz/milvus2-sdk-node";​ +​ +const address = "http://localhost:19530";​ +const token = "root:Milvus";​ +const client = new MilvusClient({address, token});​ +​ +// 7. Load the collection​ +res = await client.loadCollection({​ + collection_name: "customized_setup_1"​ +})​ +​ +console.log(res.error_code)​ +​ +// Output​ +// ​ +// Success​ +// ​ +​ +res = await client.getLoadState({​ + collection_name: "customized_setup_1"​ +})​ +​ +console.log(res.state)​ +​ +// Output​ +// ​ +// LoadStateLoaded​ +// ​ + +``` + +```go +import (​ + "context"​ + "fmt"​ + "log"​ +​ + "github.com/milvus-io/milvus/client/v2"​ +)​ +​ +defer cli.Close(ctx)​ +​ +loadTask, err := cli.LoadCollection(ctx, client.NewLoadCollectionOption("customized_setup_1"))​ +if err != nil {​ + // handle error​ +}​ +​ +// sync wait collection to be loaded​ +err = loadTask.Await(ctx)​ +if err != nil {​ + // handle error​ +}​ + +``` + +```curl +export CLUSTER_ENDPOINT="http://localhost:19530"​ +export TOKEN="root:Milvus"​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/collections/load" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "customized_setup_1"​ +}'​ +​ +# {​ +# "code": 0,​ +# "data": {}​ +# }​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/collections/get_load_state" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "customized_setup_1"​ +}'​ +​ +# {​ +# "code": 0,​ +# "data": {​ +# "loadProgress": 100,​ +# "loadState": "LoadStateLoaded",​ +# "message": ""​ +# }​ +# }​ + +``` + +## Load Specific Fields​ + +Zilliz Cloud can load only the fields involved in searches and queries, reducing memory usage and improving search performance.​ + +The following code snippet assumes that you have created a collection named **customized_setup_2**, and there are two fields named **my_id** and **my_vector** in the collection.​ + +
+ Python + Java + Node.js + Go + cURL +
+ +```python +client.load_collection(​ + collection_name="customized_setup_1",​ + # highlight-next-line​ + load_fields=["my_id", "my_vector"] # Load only the specified fields​ + skip_load_dynamic_field=True # Skip loading the dynamic field​ +)​ +​ +res = client.get_load_state(​ + collection_name="customized_setup_1"​ +)​ +​ +print(res)​ +​ +# Output​ +#​ +# {​ +# "state": ""​ +# }​ + +``` + +```java +// 6. Load the collection​ +LoadCollectionReq loadCollectionReq = LoadCollectionReq.builder()​ + .collectionName("customized_setup_1")​ + .loadFields(Arrays.asList("my_id", "my_vector"))​ + .build();​ +​ +client.loadCollection(loadCollectionReq);​ +​ +// 7. Get load state of the collection​ +GetLoadStateReq loadStateReq = GetLoadStateReq.builder()​ + .collectionName("customized_setup_1")​ + .build();​ +​ +Boolean res = client.getLoadState(loadStateReq);​ +System.out.println(res);​ + +``` + +```javascript +await client.load_collection({​ + collection_name: "customized_setup_1",​ + load_fields: ["my_id", "my_vector"], // Load only the specified fields​ + skip_load_dynamic_field: true //Skip loading the dynamic field​ +});​ +​ +const loadState = client.getCollectionLoadState({​ + collection_name: "customized_setup_1",​ +})​ +​ +console.log(loadState);​ + +``` + +```go +import (​ + "context"​ + "fmt"​ + "log"​ +​ + "github.com/milvus-io/milvus/client/v2"​ +)​ +​ +ctx, cancel := context.WithCancel(context.Background())​ +defer cancel()​ +​ +loadTask, err := cli.LoadCollection(ctx, client.NewLoadCollectionOption("customized_setup_1").​ + WithLoadFields("my_id", "my_vector"))​ +if err != nil {​ + // handle error​ +}​ +​ +// sync wait collection to be loaded​ +err = loadTask.Await(ctx)​ +if err != nil {​ + // handle error​ +}​ + +``` + +```curl +# REST 缺失​ + +``` + +If you choose to load specific fields, it is worth noting that only the fields included in `load_fields` can be used as filters and output fields in searches and queries. You should always include the names of the primary field and at least one vector field in `load_fields`.​ + +You can also use `skip_load_dynamic_field` to determine whether to load the dynamic field. The dynamic field is a reserved JSON field named **$meta** and saves all non-schema-defined fields and their values in key-value pairs. When loading the dynamic field, all keys in the fields are loaded and available for filtering and output. If all keys in the dynamic field are not involved in metadata filtering and output, set `skip_load_dynamic_field` to `True`.​ + +To load more fields after the collection load, you need to release the collection first to avoid possible errors prompted because of index changes.​ + +## Release Collection​ + +Searches and queries are memory-intensive operations. To save the cost, you are advised to release the collections that are currently not in use.​ + +The following code snippet demonstrates how to release a collection.​ + +
+ Python + Java + Node.js + Go + cURL +
+ +```python +# 8. Release the collection​ +client.release_collection(​ + collection_name="custom_quick_setup"​ +)​ +​ +res = client.get_load_state(​ + collection_name="custom_quick_setup"​ +)​ +​ +print(res)​ +​ +# Output​ +#​ +# {​ +# "state": ""​ +# }​ + +``` + +```java +import io.milvus.v2.service.collection.request.ReleaseCollectionReq;​ +​ +​ +// 8. Release the collection​ +ReleaseCollectionReq releaseCollectionReq = ReleaseCollectionReq.builder()​ + .collectionName("custom_quick_setup")​ + .build();​ +​ +client.releaseCollection(releaseCollectionReq);​ +​ +GetLoadStateReq loadStateReq = GetLoadStateReq.builder()​ + .collectionName("custom_quick_setup")​ + .build();​ +Boolean res = client.getLoadState(loadStateReq);​ +System.out.println(res);​ +​ +// Output:​ +// false​ + +``` + +```javascript +// 8. Release the collection​ +res = await client.releaseCollection({​ + collection_name: "custom_quick_setup"​ +})​ +​ +console.log(res.error_code)​ +​ +// Output​ +// ​ +// Success​ +// ​ +​ +res = await client.getLoadState({​ + collection_name: "custom_quick_setup"​ +})​ +​ +console.log(res.state)​ +​ +// Output​ +// ​ +// LoadStateNotLoad​ +// ​ + +``` + +```go +import (​ + "context"​ +​ + "github.com/milvus-io/milvus/client/v2"​ +)​ +​ +err := cli.ReleaseCollection(ctx, client.NewReleaseCollectionOption("custom_quick_setup"))​ +if err != nil {​ + // handle error​ +}​ + +``` + +```curl +export CLUSTER_ENDPOINT="http://localhost:19530"​ +export TOKEN="root:Milvus"​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/collections/release" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "custom_quick_setup"​ +}'​ +​ +# {​ +# "code": 0,​ +# "data": {}​ +# }​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/collections/get_load_state" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "custom_quick_setup"​ +}'​ +​ +# {​ +# "code": 0,​ +# "data": {​ +# "loadProgress": 0,​ +# "loadState": "LoadStateNotLoaded",​ +# "message": ""​ +# }​ +# }​ + +``` diff --git a/v2.4.x/site/en/userGuide/collections/manage-aliases.md b/v2.4.x/site/en/userGuide/collections/manage-aliases.md new file mode 100644 index 0000000000..19be351cb9 --- /dev/null +++ b/v2.4.x/site/en/userGuide/collections/manage-aliases.md @@ -0,0 +1,536 @@ +--- +id: manage-aliases.md +title: Manage Aliases​ +--- + +# Manage Aliases​ + +Milvus provides alias management capabilities. This page demonstrates the procedures to create, list, alter, and drop aliases.​ + +## Overview​ + +You can create aliases for your collections. A collection can have several aliases, but collections cannot share an alias. ​ + +Upon receiving a request against a collection, Milvus locates the collection based on the provided name. If the collection by the provided name does not exist, Milvus continues locating the provided name as an alias. You can use collection aliases to adapt your code to different scenarios.​ + +## Create Alias​ + +The following code snippet demonstrates how to create an alias for a collection.​ + +
+ Python + Java + Node.js + Go + cURL +
+ +```python +from pymilvus import MilvusClient​ +​ +client = MilvusClient(​ + uri="http://localhost:19530",​ + token="root:Milvus"​ +)​ +​ +# 9. Manage aliases​ +# 9.1. Create aliases​ +client.create_alias(​ + collection_name="customized_setup_2",​ + alias="bob"​ +)​ +​ +client.create_alias(​ + collection_name="customized_setup_2",​ + alias="alice"​ +)​ + +``` + +```java +import io.milvus.v2.service.utility.request.CreateAliasReq;​ +import io.milvus.v2.client.ConnectConfig;​ +import io.milvus.v2.client.MilvusClientV2;​ +​ +String CLUSTER_ENDPOINT = "http://localhost:19530";​ +String TOKEN = "root:Milvus";​ +​ +// 1. Connect to Milvus server​ +ConnectConfig connectConfig = ConnectConfig.builder()​ + .uri(CLUSTER_ENDPOINT)​ + .token(TOKEN)​ + .build();​ +​ +MilvusClientV2 client = new MilvusClientV2(connectConfig);​ +​ +// 9. Manage aliases​ +​ +// 9.1 Create alias​ +CreateAliasReq createAliasReq = CreateAliasReq.builder()​ + .collectionName("customized_setup_2")​ + .alias("bob")​ + .build();​ +​ +client.createAlias(createAliasReq);​ +​ +createAliasReq = CreateAliasReq.builder()​ + .collectionName("customized_setup_2")​ + .alias("alice")​ + .build();​ +​ +client.createAlias(createAliasReq);​ + +``` + +```javascript +import { MilvusClient, DataType } from "@zilliz/milvus2-sdk-node";​ +​ +const address = "http://localhost:19530";​ +const token = "root:Milvus";​ +const client = new MilvusClient({address, token});​ +​ +// 9. Manage aliases​ +// 9.1 Create aliases​ +res = await client.createAlias({​ + collection_name: "customized_setup_2",​ + alias: "bob"​ +})​ +​ +console.log(res.error_code)​ +​ +// Output​ +// ​ +// Success​ +// ​ +​ +res = await client.createAlias({​ + collection_name: "customized_setup_2",​ + alias: "alice"​ +})​ +​ +console.log(res.error_code)​ +​ +// Output​ +// ​ +// Success​ +// ​ + +``` + +```go +// Go 缺失​ + +``` + +```curl +export CLUSTER_ENDPOINT="http://localhost:19530"​ +export TOKEN="root:Milvus"​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/aliases/create" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "aliasName": "bob",​ + "collectionName": "customized_setup_2"​ +}'​ +​ +# {​ +# "code": 0,​ +# "data": {}​ +# }​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/aliases/create" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "aliasName": "alice",​ + "collectionName": "customized_setup_2"​ +}'​ +​ +# {​ +# "code": 0,​ +# "data": {}​ +# }​ + +``` + +## List Aliases​ + +The following code snippet demonstrates the procedure to list the aliases allocated to a specific collection.​ + +
+ Python + Java + Node.js + Go + cURL +
+ +```python +# 9.2. List aliases​ +res = client.list_aliases(​ + collection_name="customized_setup_2"​ +)​ +​ +print(res)​ +​ +# Output​ +#​ +# {​ +# "aliases": [​ +# "bob",​ +# "alice"​ +# ],​ +# "collection_name": "customized_setup_2",​ +# "db_name": "default"​ +# }​ + +``` + +```java +import io.milvus.v2.service.utility.request.ListAliasesReq;​ +import io.milvus.v2.service.utility.response.ListAliasResp;​ +​ +// 9.2 List alises​ +ListAliasesReq listAliasesReq = ListAliasesReq.builder()​ + .collectionName("customized_setup_2")​ + .build();​ +​ +ListAliasResp listAliasRes = client.listAliases(listAliasesReq);​ +​ +System.out.println(listAliasRes.getAlias());​ +​ +// Output:​ +// [bob, alice]​ + +``` + +```javascript +// 9.2 List aliases​ +res = await client.listAliases({​ + collection_name: "customized_setup_2"​ +})​ +​ +console.log(res.aliases)​ +​ +// Output​ +// ​ +// [ 'bob', 'alice' ]​ +// ​ + +``` + +```go +// Go 缺失​ + +``` + +```curl +export CLUSTER_ENDPOINT="http://localhost:19530"​ +export TOKEN="root:Milvus"​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/aliases/list" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{}'​ +​ +# {​ +# "code": 0,​ +# "data": [​ +# "bob",​ +# "alice"​ +# ]​ +# }​ + +``` + +## Describe Alias​ + +The following code snippet describes a specific alias in detail, including the name of the collection to which it has been allocated.​ + +
+ Python + Java + Node.js + Go + cURL +
+ +```python +# 9.3. Describe aliases​ +res = client.describe_alias(​ + alias="bob"​ +)​ +​ +print(res)​ +​ +# Output​ +#​ +# {​ +# "alias": "bob",​ +# "collection_name": "customized_setup_2",​ +# "db_name": "default"​ +# }​ + +``` + +```java +import io.milvus.v2.service.utility.request.DescribeAliasReq;​ +import io.milvus.v2.service.utility.response.DescribeAliasResp;​ +​ +// 9.3 Describe alias​ +DescribeAliasReq describeAliasReq = DescribeAliasReq.builder()​ + .alias("bob")​ + .build();​ +​ +DescribeAliasResp describeAliasRes = client.describeAlias(describeAliasReq);​ +​ +System.out.println(describeAliasRes);​ +​ +// Output:​ +// DescribeAliasResp(collectionName=customized_setup_2, alias=bob)​ + +``` + +```javascript +// 9.3 Describe aliases​ +res = await client.describeAlias({​ + collection_name: "customized_setup_2",​ + alias: "bob"​ +})​ +​ +console.log(res)​ +​ +// Output​ +// ​ +// {​ +// status: {​ +// extra_info: {},​ +// error_code: 'Success',​ +// reason: '',​ +// code: 0,​ +// retriable: false,​ +// detail: ''​ +// },​ +// db_name: 'default',​ +// alias: 'bob',​ +// collection: 'customized_setup_2'​ +// }​ +// ​ + +``` + +```go +// Go 缺失​ + +``` + +```curl +export CLUSTER_ENDPOINT="http://localhost:19530"​ +export TOKEN="root:Milvus"​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/aliases/describe" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "aliasName": "bob"​ +}'​ +​ +# {​ +# "code": 0,​ +# "data": {​ +# "aliasName": "bob",​ +# "collectionName": "customized_setup_2",​ +# "dbName": "default"​ +# }​ +# }​ + +``` + +## Alter Alias​ + +You can reallocate the alias already allocated to a specific collection to another.​ + +
+ Python + Java + Node.js + Go + cURL +
+ +```python +# 9.4 Reassign aliases to other collections​ +client.alter_alias(​ + collection_name="customized_setup_1",​ + alias="alice"​ +)​ +​ +res = client.list_aliases(​ + collection_name="customized_setup_1"​ +)​ +​ +print(res)​ +​ +# Output​ +#​ +# {​ +# "aliases": [​ +# "alice"​ +# ],​ +# "collection_name": "customized_setup_1",​ +# "db_name": "default"​ +# }​ +​ +res = client.list_aliases(​ + collection_name="customized_setup_2"​ +)​ +​ +print(res)​ +​ +# Output​ +#​ +# {​ +# "aliases": [​ +# "bob"​ +# ],​ +# "collection_name": "customized_setup_2",​ +# "db_name": "default"​ +# }​ + +``` + +```java +import io.milvus.v2.service.utility.request.AlterAliasReq;​ +​ +// 9.4 Reassign alias to other collections​ +AlterAliasReq alterAliasReq = AlterAliasReq.builder()​ + .collectionName("customized_setup_1")​ + .alias("alice")​ + .build();​ +​ +client.alterAlias(alterAliasReq);​ +​ +ListAliasesReq listAliasesReq = ListAliasesReq.builder()​ + .collectionName("customized_setup_1")​ + .build();​ +​ +ListAliasResp listAliasRes = client.listAliases(listAliasesReq);​ +​ +System.out.println(listAliasRes.getAlias());​ +​ +listAliasesReq = ListAliasesReq.builder()​ + .collectionName("customized_setup_2")​ + .build();​ +​ +listAliasRes = client.listAliases(listAliasesReq);​ +​ +System.out.println(listAliasRes.getAlias());​ +​ +// Output:​ +// [bob]​ + +``` + +```javascript +// 9.4 Reassign aliases to other collections​ +res = await client.alterAlias({​ + collection_name: "customized_setup_1",​ + alias: "alice"​ +})​ +​ +console.log(res.error_code)​ +​ +// Output​ +// ​ +// Success​ +// ​ +​ +res = await client.listAliases({​ + collection_name: "customized_setup_1"​ +})​ +​ +console.log(res.aliases)​ +​ +// Output​ +// ​ +// [ 'alice' ]​ +// ​ +​ +res = await client.listAliases({​ + collection_name: "customized_setup_2"​ +})​ +​ +console.log(res.aliases)​ +​ +// Output​ +// ​ +// [ 'bob' ]​ +// ​ +​ + +``` + +```go +// Go 缺失​ + +``` + +```curl +export CLUSTER_ENDPOINT="http://localhost:19530"​ +export TOKEN="root:Milvus"​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/aliases/alter" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "aliasName": "alice",​ + "collectionName": "customized_setup_1"​ +}'​ +​ +# {​ +# "code": 0,​ +# "data": {}​ +# }​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/aliases/describe" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "aliasName": "bob"​ +}'​ +​ +# {​ +# "code": 0,​ +# "data": {​ +# "aliasName": "bob",​ +# "collectionName": "customized_setup_2",​ +# "dbName": "default"​ +# }​ +# }​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/aliases/describe" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "aliasName": "alice"​ +}'​ +​ +# {​ +# "code": 0,​ +# "data": {​ +# "aliasName": "alice",​ +# "collectionName": "customized_setup_1",​ +# "dbName": "default"​ +# }​ +# }​ + +``` + diff --git a/v2.4.x/site/en/userGuide/collections/manage-collections.md b/v2.4.x/site/en/userGuide/collections/manage-collections.md new file mode 100644 index 0000000000..cd08b04f82 --- /dev/null +++ b/v2.4.x/site/en/userGuide/collections/manage-collections.md @@ -0,0 +1,131 @@ +--- +id: manage-collections.md +title: Collection Explained​ +--- + +# Collection Explained​ + +In Milvus, you can create multiple collections to manage your data, and insert your data as entities into the collections. Collection and entity are similar to tables and records in relational databases. This page helps you to learn about the collection and related concepts.​ + +## Collection​ + +A collection is a two-dimensional table with fixed columns and variant rows. Each column represents a field, and each row represents an entity. ​ + +The following chart shows a collection with eight columns and six entities.​ + +![Collection explained](../../../../assets/collection-explained.png) + +## Schema and Fields​ + +When describing an object, we usually mention its attributes, such as size, weight, and position. You can use these attributes as fields in a collection. Each field has various constraining properties, such as the data type and the dimensionality of a vector field. You can form a collection schema by creating the fields and defining their order. For possible applicable data types, refer to [​Schema Explained](schema.md).​ + +You should include all schema-defined fields in the entities to insert. To make some of them optional, consider​ + +- **Making them nullable or setting default values**​ + + For details on how to make a field nullable or set the default value, refer to [​Nullable & Default](nullable-and-default.md).​ + +- **Enabling dynamic field**​ + + For details on how to enable and use the dynamic field, refer to [​Dynamic Field](enable-dynamic-field.md).​ + +## Primary key and AutoId​ + +Similar to the primary field in a relational database, a collection has a primary field to distinguish an entity from others. Each value in the primary field is globally unique and corresponds to one specific entity. ​ + +As shown in the above chart, the field named **id** serves as the primary field, and the first ID **0** corresponds to an entity titled *The Mortality Rate of Coronavirus is Not Important*. There will not be any other entity that has the primary field of 0. ​ + +A primary field accepts only integers or strings. When inserting entities, you should include the primary field values by default. However, if you have enabled **AutoId** upon collection creation, Milvus will generate those values upon data insertion. In such a case, exclude the primary field values from the entities to be inserted.​ + +For more information, please refer to [​Primary Field & AutoID](primary-field.md).​ + +## Index​ + +Creating indexes on specific fields improves search efficiency. You are advised to create indexes for all the fields your service relies on, among which indexes on vector fields are mandatory. + +## Entity​ + +Entities are data records that share the same set of fields in a collection. The values in all fields of the same row comprise an entity.​ + +You can insert as many entities as you need into a collection. However, as the number of entities mounts, the memory size it takes also increases, affecting search performance.​ + +For more information, refer to [​Schema Explained](schema.md).​ + +## Load and Release​ + +Loading a collection is the prerequisite to conducting similarity searches and queries in collections. When you load a collection, Zilliz Cloud loads all index files and the raw data in each field into memory for fast response to searches and queries.​ + +Searches and queries are memory-intensive operations. To save the cost, you are advised to release the collections that are currently not in use.​ + +For more details, refer to [​Load & Release](load-and-release.md).​ + +## Search and Query​ + +Once you create indexes and load the collection, you can start a similarity search by feeding one or several query vectors. For example, when receiving the vector representation of your query carried in a search request, Zilliz Cloud uses the specified metric type to measure the similarity between the query vector and those in the target collection before returning those that are semantically similar to the query.​ + +You can also include metadata filtering within searches and queries to improve the relevancy of the results. Note that, metadata filtering conditions are mandatory in queries but optional in searches.​ + +For details on applicable metric types, refer to [​Metric Types](metric.md).​ + +For more information about searches and queries, refer to the articles in the [​Search & Rerank](single-vector-search.md) chapter, among which, basic features are:​ + +- [​Basic ANN Search](single-vector-search.md)​ + +- [​Filtered Search](filtered-search.md)​ + +- [​Range Search](range-search.md)​ + +- [​Grouping Search](grouping-search.md)​ + +- [​Hybrid Search](multi-vector-search.md)​ + +- [​Search Iterator](with-iterators.md)​ + +- [​Query](get-and-scalar-query.md)​ + +- [​Full-Text Search](full-text-search.md)​ + +- [Keyword Match](keyword-match.md)​ + +In addition, Zilliz Cloud also provides enhancements to improve search performance and efficiency. They are disabled by default, and you can enable and use them according to your service requirements. They are​ + +- [​Use Partition Key](use-partition-key.md)​ + +- [​Use mmap](mmap.md)​ + +- [​Clustering Compaction](clustering-compaction.md)​ + +## Partition​ + +Partitions are subsets of a collection, which share the same field set with its parent collection, each containing a subset of entities.​ + +By allocating entities into different partitions, you can create entity groups. You can conduct searches and queries in specific partitions to have Zilliz Cloud ignore entities in other partitions, and improve search efficiency.​ + +For details, refer to [​Manage Partitions](manage-partitions.md).​ + +## Shard​ + +Shards are horizontal slices of a collection. Each shard corresponds to a data input channel. Every collection has a shard by default. You can set the appropriate number of shards when creating a collection based on the expected throughput and the volume of the data to insert into the collection.​ + +For details on how to set the shard number, refer to [​Create Collection](create-collection.md).​ + +## Alias​ + +You can create aliases for your collections. A collection can have several aliases, but collections cannot share an alias. Upon receiving a request against a collection, Zilliz Cloud locates the collection based on the provided name. If the collection by the provided name does not exist, Zilliz Cloud continues locating the provided name as an alias. You can use collection aliases to adapt your code to different scenarios.​ + +For more details, refer to [​Manage Aliases](manage-aliases.md).​ + +## Function​ + +You can set functions for Zilliz Cloud to derive fields upon collection creation. For example, the full-text search function uses the user-defined function to derive a sparse vector field from a specific varchar field. For more information on full-text search, refer to [​Full-Text Search](full-text-search.md).​ + +## Consistency Level​ + +Distributed database systems usually use the consistency level to define the data sameness across data nodes and replicas. You can set separate consistency levels when you create a collection or conduct similarity searches within the collection. The applicable consistency levels are **Strong**, **Bounded Staleness**, **Session**, and **Eventually**.​ + + For details on these consistency levels, refer to [​Consistency Level](consistency.md).​ + +## Limits​ + +For limits and restrictions on collections, refer to [​Limits & Restrictions](limitations.md).​ + diff --git a/v2.4.x/site/en/userGuide/collections/manage-partitions.md b/v2.4.x/site/en/userGuide/collections/manage-partitions.md new file mode 100644 index 0000000000..72548e2d4e --- /dev/null +++ b/v2.4.x/site/en/userGuide/collections/manage-partitions.md @@ -0,0 +1,782 @@ +--- +id: manage-partitions.md +title: Manage Partitions +--- + +# Manage Partitions​ + +A partition is a subset of a collection. Each partition shares the same data structure with its parent collection but contains only a subset of the data in the collection. This page helps you understand how to manage partitions.​ + +## Overview​ + +When creating a collection, Zilliz Cloud also creates a partition named **_default** in the collection. If you are not going to add any other partitions, all entities inserted into the collection go into the default partition, and all searches and queries are also carried out within the default partition.​ + +You can add more partitions and insert entities into them based on certain criteria. Then you can restrict your searches and queries within certain partitions, improving search performance.​ + +A collection can have a maximum of 1,024 partitions.​ + +
+ +The **Partition Key** feature is a search optimization based on partitions and allows Zilliz Cloud to distribute entities into different partitions based on the values in a specific scalar field. This feature helps implement partition-oriented multi-tenancy and improves search performance.​ + +This feature will not be discussed on this page. To find more, refer to [​Use Partition Key](use-partition-key.md).​ + +
+ +## List Partitions​ + +When creating a collection, Zilliz Cloud also creates a partition named **_default** in the collection. You can list the partitions in a collection as follows.​ + +
+ Python + Java + Node.js + Go + cURL +
+ +```python +from pymilvus import MilvusClient​ +​ +client = MilvusClient(​ + uri="http://localhost:19530",​ + token="root:Milvus"​ +)​ +​ +res = client.list_partitions(​ + collection_name="quick_setup"​ +)​ +​ +print(res)​ +​ +# Output​ +#​ +# ["_default"]​ + +``` + +```java +import io.milvus.v2.service.partition.request.ListPartitionsReq;​ +import io.milvus.v2.client.ConnectConfig;​ +import io.milvus.v2.client.MilvusClientV2;​ +​ +import java.util.*;​ +​ +String CLUSTER_ENDPOINT = "http://localhost:19530";​ +String TOKEN = "root:Milvus";​ +​ +// 1. Connect to Milvus server​ +ConnectConfig connectConfig = ConnectConfig.builder()​ + .uri(CLUSTER_ENDPOINT)​ + .token(TOKEN)​ + .build();​ +​ +MilvusClientV2 client = new MilvusClientV2(connectConfig);​ +​ +ListPartitionsReq listPartitionsReq = ListPartitionsReq.builder()​ + .collectionName("quick_setup")​ + .build();​ +​ +List partitionNames = client.listPartitions(listPartitionsReq);​ +System.out.println(partitionNames);​ +​ +// Output:​ +// [_default]​ + +``` + +```javascript +import { MilvusClient, DataType } from "@zilliz/milvus2-sdk-node";​ +​ +const address = "http://localhost:19530";​ +const token = "root:Milvus";​ +const client = new MilvusClient({address, token});​ +​ +let res = await client.listPartitions({​ + collection_name: "quick_setup"​ +})​ +​ +console.log(res);​ +​ +// Output​ +// ["_default"]​ + +``` + +```go +import (​ + "context"​ + ​ + client "github.com/milvus-io/milvus/client/v2/milvucclient"​ +)​ +​ +ctx, cancel := context.WithCancel(context.Background())​ +defer cancel()​ +​ +milvusAddr := "127.0.0.1:19530"​ +token := "root:Milvus"​ +​ +cli, err := client.New(ctx, &client.ClientConfig{​ + Address: milvusAddr,​ + APIKey: token,​ +})​ +if err != nil {​ + // handle error​ +}​ +​ +defer cli.Close(ctx)​ +​ +partitionNames, err := cli.ListPartitions(ctx, client.NewListPartitionOption("quick_setup"))​ +if err != nil {​ + // handle error​ +}​ +​ +fmt.Println(partitionNames)​ + +``` + +```curl +export CLUSTER_ENDPOINT="http://localhost:19530"​ +export TOKEN="root:Milvus"​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/partitions/list" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "quick_setup"​ +}'​ +​ +# {​ +# "code": 0,​ +# "data": [​ +# "_default"​ +# ]​ +# }​ + +``` + +## Create Partition​ + +You can add more partitions to the collection and insert entities into these partitions based on certain criteria.​ + +
+ Python + Java + Node.js + Go + cURL +
+ +```python +client.create_partition(​ + collection_name="quick_setup",​ + partition_name="partitionA"​ +)​ +​ +res = client.list_partitions(​ + collection_name="quick_setup"​ +)​ +​ +print(res)​ +​ +# Output​ +#​ +# ["_default", "partitionA"]​ + +``` + +```java +import io.milvus.v2.service.partition.request.CreatePartitionReq;​ +​ +CreatePartitionReq createPartitionReq = CreatePartitionReq.builder()​ + .collectionName("quick_setup")​ + .partitionName("partitionA")​ + .build();​ +​ +client.createPartition(createPartitionReq);​ +​ +ListPartitionsReq listPartitionsReq = ListPartitionsReq.builder()​ + .collectionName("quick_setup")​ + .build();​ +​ +List partitionNames = client.listPartitions(listPartitionsReq);​ +System.out.println(partitionNames);​ +​ +// Output:​ +// [_default, partitionA]​ + +``` + +```javascript +await client.createPartition({​ + collection_name: "quick_setup",​ + partition_name: "partitionA"​ +})​ +​ +res = await client.listPartitions({​ + collection_name: "quick_setup"​ +})​ +​ +console.log(res)​ +​ +// Output​ +// ["_default", "partitionA"]​ + +``` + +```go +import (​ + "fmt"​ + ​ + client "github.com/milvus-io/milvus/client/v2/milvusclient"​ +)​ +​ +err = cli.CreatePartition(ctx, client.NewCreatePartitionOption("quick_setup", "partitionA"))​ +if err != nil {​ + // handle error​ +}​ +​ +partitionNames, err := cli.ListPartitions(ctx, client.NewListPartitionOption("quick_setup"))​ +if err != nil {​ + // handle error​ +}​ +​ +fmt.Println(partitionNames)​ +// Output​ +// ["_default", "partitionA"]​ + +``` + +```curl +export CLUSTER_ENDPOINT="http://localhost:19530"​ +export TOKEN="root:Milvus"​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/partitions/create" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "quick_setup",​ + "partitionName": "partitionA"​ +}'​ +​ +# {​ +# "code": 0,​ +# "data": {}​ +# }​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/partitions/list" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "quick_setup"​ +}'​ +​ +# {​ +# "code": 0,​ +# "data": [​ +# "_default",​ +# "partitionA"​ +# ]​ +# }​ + +``` + +## Check for a Specific Partition​ + +The following code snippets demonstrate how to check whether a partition exists in a specific collection.​ + +
+ Python + Java + Node.js + Go + cURL +
+ +```python +res = client.has_partition(​ + collection_name="quick_setup",​ + partition_name="partitionA"​ +)​ +​ +print(res)​ +​ +# Output​ +#​ +# True​ + +``` + +```java +import io.milvus.v2.service.partition.request.HasPartitionReq;​ +​ +HasPartitionReq hasPartitionReq = HasPartitionReq.builder()​ + .collectionName("quick_setup")​ + .partitionName("partitionA")​ + .build();​ +​ +Boolean hasPartitionRes = client.hasPartition(hasPartitionReq);​ +System.out.println(hasPartitionRes);​ +​ +// Output:​ +// true​ + +``` + +```javascript +res = await client.hasPartition({​ + collection_name: "quick_setup",​ + partition_name: "partitionA"​ +})​ +​ +console.log(res.value)​ +​ +// Output​ +// true​ + +``` + +```go +import (​ + "fmt"​ + ​ + "github.com/milvus-io/milvus/client/v2/milvusclient"​ +)​ +​ +result, err := cli.HasPartition(ctx, milvusclient.NewHasPartitionOption("quick_setup", "partitionA"))​ +if err != nil {​ + // handle error​ +}​ +​ +fmt.Println(result)​ +​ +// Output:​ +// true​ + +``` + +```curl +export CLUSTER_ENDPOINT="http://localhost:19530"​ +export TOKEN="root:Milvus"​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/partitions/has" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "quick_setup",​ + "partitionName": "partitionA"​ +}'​ +​ +# {​ +# "code": 0,​ +# "data": {​ +# "has": true​ +# }​ +# }​ + +``` + +## Load and Release Partitions​ + +You can separately load or release one or certain partitions.​ + +### Load Partitions​ + +You can separately load specific partitions in a collection. It is worth noting that the load status of a collection stays unloaded if there is an unloaded partition in the collection.​ + +
+ Python + Java + Node.js + Go + cURL +
+ +```python +client.load_partitions(​ + collection_name="quick_setup",​ + partition_names=["partitionA"]​ +)​ +​ +res = client.get_load_state(​ + collection_name="quick_setup",​ + partition_name="partitionA"​ +)​ +​ +print(res)​ +# Output​ +#​ +# {​ +# "state": ""​ +# }​ + +``` + +```java +import io.milvus.v2.service.partition.request.LoadPartitionsReq;​ +import io.milvus.v2.service.collection.request.GetLoadStateReq;​ +​ +​ +LoadPartitionsReq loadPartitionsReq = LoadPartitionsReq.builder()​ + .collectionName("quick_setup")​ + .partitionNames(Collections.singletonList("partitionA"))​ + .build();​ +​ +client.loadPartitions(loadPartitionsReq);​ +​ +GetLoadStateReq getLoadStateReq = GetLoadStateReq.builder()​ + .collectionName("quick_setup")​ + .partitionName("partitionA")​ + .build();​ +​ +Boolean getLoadStateRes = client.getLoadState(getLoadStateReq);​ +System.out.println(getLoadStateRes);​ +​ +// True​ + +``` + +```javascript +await client.loadPartitions({​ + collection_name: "quick_setup",​ + partition_names: ["partitionA"]​ +})​ +​ +res = await client.getLoadState({​ + collection_name: "quick_setup",​ + partition_name: "partitionA"​ +})​ +​ +console.log(res)​ +​ +// Output​ +// ​ +// LoadStateLoaded​ +// ​ + +``` + +```go +// Go 缺失​ + +``` + +```curl +export CLUSTER_ENDPOINT="http://localhost:19530"​ +export TOKEN="root:Milvus"​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/partitions/load" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "quick_setup",​ + "partitionNames": ["partitionA"]​ +}'​ +​ +# {​ +# "code": 0,​ +# "data": {}​ +# }​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/collections/get_load_state" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "quick_setup",​ + "partitionNames": ["partitionA"]​ +}'​ +​ +# {​ +# "code": 0,​ +# "data": {​ +# "loadProgress": 100,​ +# "loadState": "LoadStateLoaded",​ +# "message": ""​ +# }​ +# }​ + +``` + +### Release Partitions​ + +You can also release specific partitions.​ + +
+ Python + Java + Node.js + Go + cURL +
+ +```python +client.release_partitions(​ + collection_name="quick_setup",​ + partition_names=["partitionA"]​ +)​ +​ +res = client.get_load_state(​ + collection_name="quick_setup",​ + partition_name="partitionA"​ +)​ +​ +print(res)​ +​ +# Output​ +#​ +# {​ +# "state": ""​ +# }​ + +``` + +```java +import io.milvus.v2.service.partition.request.ReleasePartitionsReq;​ +​ +ReleasePartitionsReq releasePartitionsReq = ReleasePartitionsReq.builder()​ + .collectionName("quick_setup")​ + .partitionNames(Collections.singletonList("partitionA"))​ + .build();​ +​ +client.releasePartitions(releasePartitionsReq);​ +​ +GetLoadStateReq getLoadStateReq = GetLoadStateReq.builder()​ + .collectionName("quick_setup")​ + .partitionName("partitionA")​ + .build();​ +​ +Boolean getLoadStateRes = client.getLoadState(getLoadStateReq);​ +System.out.println(getLoadStateRes);​ +​ +// False​ + +``` + +```javascript +await client.releasePartitions({​ + collection_name: "quick_setup",​ + partition_names: ["partitionA"]​ +})​ +​ +res = await client.getLoadState({​ + collection_name: "quick_setup",​ + partition_name: "partitionA"​ +})​ +​ +console.log(res)​ +​ +// Output​ +// ​ +// LoadStateNotLoaded​ +// ​ + +``` + +```go +// Go 缺失​ + +``` + +```curl +export CLUSTER_ENDPOINT="http://localhost:19530"​ +export TOKEN="root:Milvus"​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/partitions/release" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "quick_setup",​ + "partitionNames": ["partitionA"]​ +}'​ +​ +# {​ +# "code": 0,​ +# "data": {}​ +# }​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/collections/get_load_state" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "quick_setup",​ + "partitionNames": ["partitionA"]​ +}'​ +​ +# {​ +# "code": 0,​ +# "data": {​ +# "loadProgress": 0,​ +# "loadState": "LoadStateNotLoaded",​ +# "message": ""​ +# }​ +# }​ + +``` + +## Data Operations Within Partitions​ + +### Insert and Delete Entities​ + +You can perform insert, upsert, and delete operations in specific operations. For details, refer to​ + +- Insert Entities into Partition​ + +- Upsert Entities into Partition​ + +- Delete Entities from Partition​ + +### Search and Query​ + +You can conduct searches and queries within specific partitions. For details, refer to ​ + +- Conduct ANN Searches within Partitions​ + +- Conduct Metadata Filtering within Partitions​ + +## Drop Partition​ + +You can drop partitions that are no longer needed. Before dropping a partition, ensure that the partition has been released.​ + +
+ Python + Java + Node.js + Go + cURL +
+ +```python +client.release_partitions(​ + collection_name="quick_setup",​ + partition_names=["partitionA"]​ +)​ +​ +client.drop_partition(​ + collection_name="quick_setup",​ + partition_name="partitionA"​ +)​ +​ +res = client.list_partitions(​ + collection_name="quick_setup"​ +)​ +​ +print(res)​ +​ +# ["_default"]​ + +``` + +```java +import io.milvus.v2.service.partition.request.DropPartitionReq;​ +import io.milvus.v2.service.partition.request.ReleasePartitionsReq;​ +import io.milvus.v2.service.partition.request.ListPartitionsReq;​ +​ +ReleasePartitionsReq releasePartitionsReq = ReleasePartitionsReq.builder()​ + .collectionName("quick_setup")​ + .partitionNames(Collections.singletonList("partitionA"))​ + .build();​ +​ +client.releasePartitions(releasePartitionsReq);​ +​ +DropPartitionReq dropPartitionReq = DropPartitionReq.builder()​ + .collectionName("quick_setup")​ + .partitionName("partitionA")​ + .build();​ +​ +client.dropPartition(dropPartitionReq);​ +​ +ListPartitionsReq listPartitionsReq = ListPartitionsReq.builder()​ + .collectionName("quick_setup")​ + .build();​ +​ +List partitionNames = client.listPartitions(listPartitionsReq);​ +System.out.println(partitionNames);​ +​ +// Output:​ +// [_default]​ + +``` + +```javascript +await client.releasePartitions({​ + collection_name: "quick_setup",​ + partition_names: ["partitionA"]​ +})​ +​ +await client.dropPartition({​ + collection_name: "quick_setup",​ + partition_name: "partitionA"​ +})​ +​ +res = await client.listPartitions({​ + collection_name: "quick_setup"​ +})​ +​ +console.log(res)​ +​ +// Output​ +// ["_default"]​ + +``` + +```go +// Go 缺失​ + +``` + +```curl +export CLUSTER_ENDPOINT="http://localhost:19530"​ +export TOKEN="root:Milvus"​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/partitions/release" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "quick_setup",​ + "partitionNames": ["partitionA"]​ +}'​ +​ +# {​ +# "code": 0,​ +# "data": {}​ +# }​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/partitions/drop" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "quick_setup",​ + "partitionName": "partitionA"​ +}'​ +​ +# {​ +# "code": 0,​ +# "data": {}​ +# }​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/partitions/list" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "quick_setup"​ +}'​ +​ +# {​ +# "code": 0,​ +# "data": [​ +# "_default"​ +# ]​ +# }​ + +``` diff --git a/v2.4.x/site/en/userGuide/collections/modify-collection.md b/v2.4.x/site/en/userGuide/collections/modify-collection.md new file mode 100644 index 0000000000..6843b0008e --- /dev/null +++ b/v2.4.x/site/en/userGuide/collections/modify-collection.md @@ -0,0 +1,206 @@ +--- +id: modify-collection.md +title: Modify Collection​ +--- + +# Modify Collection​ + +You can rename a collection or change its settings. This page focuses on how to modify a collection.​ + +## Rename Collection​ + +You can rename a collection as follows.​ + +
+ Python + Java + Node.js + Go + cURL +
+ +```python +from pymilvus import MilvusClient​ +​ +client = MilvusClient(​ + uri="http://localhost:19530",​ + token="root:Milvus"​ +)​ +​ +client.rename_collection(​ + old_name="my_collection",​ + new_name="my_new_collection"​ +)​ + +``` + +```java +import io.milvus.v2.service.collection.request.RenameCollectionReq;​ +import io.milvus.v2.client.ConnectConfig;​ +import io.milvus.v2.client.MilvusClientV2;​ +​ +String CLUSTER_ENDPOINT = "http://localhost:19530";​ +String TOKEN = "root:Milvus";​ +​ +// 1. Connect to Milvus server​ +ConnectConfig connectConfig = ConnectConfig.builder()​ + .uri(CLUSTER_ENDPOINT)​ + .token(TOKEN)​ + .build();​ + ​ +MilvusClientV2 client = new MilvusClientV2(connectConfig);​ +​ +RenameCollectionReq renameCollectionReq = RenameCollectionReq.builder()​ + .collectionName("my_collection")​ + .newCollectionName("my_new_collection")​ + .build();​ +​ +client.renameCollection(renameCollectionReq);​ + +``` + +```javascript +import { MilvusClient, DataType } from "@zilliz/milvus2-sdk-node";​ +​ +const address = "http://localhost:19530";​ +const token = "root:Milvus";​ +const client = new MilvusClient({address, token});​ +​ +const res = await client.renameCollection({​ + oldName: "my_collection",​ + newName: "my_new_collection"​ +});​ + +``` + +```go +import (​ + "context"​ + "fmt"​ + "log"​ +​ + "github.com/milvus-io/milvus/client/v2"​ +)​ +​ +ctx, cancel := context.WithCancel(context.Background())​ +defer cancel()​ +​ +milvusAddr := "127.0.0.1:19530"​ +token := "root:Milvus"​ +​ +cli, err := client.New(ctx, &client.ClientConfig{​ + Address: milvusAddr,​ + APIKey: token,​ +})​ +if err != nil {​ + log.Fatal("failed to connect to milvus server: ", err.Error())​ +}​ +​ +defer cli.Close(ctx)​ +​ +err = cli.RenameCollection(ctx, client.NewRenameCollectionOption("my_collection", "my_new_collection"))​ +if err != nil {​ + // handle error​ +}​ + +``` + +```curl +export CLUSTER_ENDPOINT="http://localhost:19530"​ +export TOKEN="root:Milvus"​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/collections/rename" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "my_collection",​ + "newCollectionName": "my_new_collection"​ +}'​ + +``` + +## Set Collection TTL​ + +If a collection needs to be dropped for a specific period, consider setting its Time-To-Live (TTL) in seconds. Once the TTL times out, Zilliz Cloud deletes entities in the collection and drops the collection. The deletion is asynchronous, indicating that searches and queries are still possible before the deletion is complete.​ + +The following code snippet demonstrates how to change the TTL of a collection.​ + +
+ Python + Java + Node.js + Go + cURL +
+ +```python +# Currently not available for Python + +``` + +```java +import io.milvus.v2.service.collection.request.AlterCollectionReq;​ +import java.util.HashMap;​ +import java.util.Map;​ +​ +Map properties = new HashMap<>();​ +properties.put("collection.ttl.seconds", "60");​ +​ +AlterCollectionReq alterCollectionReq = AlterCollectionReq.builder()​ + .collectionName("my_collection")​ + .properties(properties)​ + .build();​ +​ +client.alterCollection(alterCollectionReq);​ + +``` + +```javascript +res = await client.alterCollection({​ + collection_name: "my_collection",​ + properties: {​ + "collection.ttl.seconds": 60​ + }​ +})​ + +``` + +```go +import (​ + "context"​ + "fmt"​ + "log"​ +​ + "github.com/milvus-io/milvus/client/v2"​ + "github.com/milvus-io/milvus/pkg/common"​ +)​ +​ +​ +ctx, cancel := context.WithCancel(context.Background())​ +defer cancel()​ +​ +milvusAddr := "127.0.0.1:19530"​ +token := "root:Milvus"​ +​ +cli, err := client.New(ctx, &client.ClientConfig{​ + Address: milvusAddr,​ + APIKey: token,​ +})​ +if err != nil {​ + log.Fatal("failed to connect to milvus server: ", err.Error())​ +}​ +​ +defer cli.Close(ctx)​ +​ +err = cli.AlterCollection(ctx, client.NewAlterCollectionOption("my_collection").WithProperty(common.CollectionTTLConfigKey, 60))​ +if err != nil {​ + // handle error​ +}​ + +``` + +```curl +# Currently not available for REST + +``` diff --git a/v2.4.x/site/en/userGuide/collections/view-collections.md b/v2.4.x/site/en/userGuide/collections/view-collections.md new file mode 100644 index 0000000000..6203104aae --- /dev/null +++ b/v2.4.x/site/en/userGuide/collections/view-collections.md @@ -0,0 +1,201 @@ +--- +id: view-collections.md +title: View Collections​ +--- + +# View Collections​ + +You can obtain the name list of all the collections in the currently connected database, and check the details of a specific collection.​ + +## List Collections​ + +The following example demonstrates how to obtain the name list of all collections in the currently connected database.​ + +
+ Python + Java + Node.js + Go + cURL +
+ +```python +from pymilvus import MilvusClient, DataType​ +​ +client = MilvusClient(​ + uri="http://localhost:19530",​ + token="root:Milvus"​ +)​ +​ +res = client.list_collections()​ +​ +print(res)​ + +``` + +```java +import io.milvus.v2.client.ConnectConfig;​ +import io.milvus.v2.client.MilvusClientV2;​ +import io.milvus.v2.service.collection.response.ListCollectionsResp;​ +​ +ConnectConfig connectConfig = ConnectConfig.builder()​ + .uri("http://localhost:19530")​ + .token("root:Milvus")​ + .build();​ +​ +MilvusClientV2 client = new MilvusClientV2(connectConfig);​ +​ +ListCollectionsResp resp = client.listCollections();​ +System.out.println(resp.getCollectionNames());​ + +``` + +```javascript +import { MilvusClient } from '@zilliz/milvus2-sdk-node';​ +​ +const client = new MilvusClient({​ + address: 'localhost:19530',​ + token: 'root:Milvus'​ +});​ +​ +​ +const collections = await client.listCollections();​ +console.log(collections);​ + +``` + +```go +import (​ + "context"​ + "fmt"​ +​ + "github.com/milvus-io/milvus/client/v2"​ +)​ +​ +ctx, cancel := context.WithCancel(context.Background())​ +defer cancel()​ +​ +milvusAddr := "127.0.0.1:19530"​ +token := "root:Milvus"​ +​ +cli, err := client.New(ctx, &client.ClientConfig{​ + Address: milvusAddr,​ + APIKey: token,​ +})​ +if err != nil {​ + log.Fatal("failed to connect to milvus server: ", err.Error())​ +}​ +​ +defer cli.Close(ctx)​ +​ +collectionNames, err := cli.ListCollections(ctx, client.NewListCollectionOption())​ +if err != nil {​ + // handle error​ +}​ +​ +fmt.Println(collectionNames)​ + +``` + +```curl +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/collections/list" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{}​ +}'​ + +``` + +If you have already created a collection named `quick_setup`, the result of the above example should be similar to the following.​ + +```JSON +["quick_setup"]​ + +``` + +## Describe Collection​ + +You can also obtain the details of a specific collection. The following example assumes that you have already created a collection named quick_setup.​ + +
+ Python + Java + Node.js + Go + cURL +
+ +```python +res = client.describe_collection(​ + collection_name="quick_setup"​ +)​ +​ +print(res)​ + +``` + +```java +import io.milvus.v2.service.collection.request.DescribeCollectionReq;​ +import io.milvus.v2.service.collection.response.DescribeCollectionResp;​ +​ +DescribeCollectionReq request = DescribeCollectionReq.builder()​ + .collectionName("quick_setup")​ + .build();​ +DescribeCollectionResp resp = client.describeCollection(request);​ +System.out.println(resp);​ + +``` + +```javascript +const res = await client.describeCollection({​ + collection_name: "quick_setup"​ +});​ +​ +console.log(res);​ + +``` + +```go +import (​ + "context"​ + "fmt"​ +​ + "github.com/milvus-io/milvus/client/v2"​ +)​ +​ +ctx, cancel := context.WithCancel(context.Background())​ +defer cancel()​ +​ +milvusAddr := "127.0.0.1:19530"​ +token := "root:Milvus"​ +​ +cli, err := client.New(ctx, &client.ClientConfig{​ + Address: milvusAddr,​ + APIKey: token,​ +})​ +if err != nil {​ + log.Fatal("failed to connect to milvus server: ", err.Error())​ +}​ +​ +defer cli.Close(ctx)​ +​ +collection, err := cli.DescribeCollection(ctx, client.NewDescribeCollectionOption("quick_setup"))​ +if err != nil {​ + // handle error​ +}​ +​ +fmt.Println(collection)​ + +``` + +```curl +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/collections/describe" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "quick_setup"​ +}'​ + +``` diff --git a/v2.4.x/site/en/userGuide/enable-dynamic-field.md b/v2.4.x/site/en/userGuide/enable-dynamic-field.md deleted file mode 100644 index 6c01f3d39d..0000000000 --- a/v2.4.x/site/en/userGuide/enable-dynamic-field.md +++ /dev/null @@ -1,567 +0,0 @@ ---- -id: enable-dynamic-field.md -title: Enable Dynamic Field ---- - -# Enable Dynamic Field - -This page explains how to use the dynamic field in a collection for flexible data insertion and retrieval. - -## Overview - -Milvus allows you to define the schema of a collection by setting the name and the data type of each specific field so that you can create indexes in these fields for improved search performance. - -Once a field is defined, you need to include this field when you insert data. What if some fields are not always present in all your data entries? This is where the dynamic field comes in. - -The dynamic field in a collection is a reserved JSON field named $meta. It can hold non-schema-defined fields and their values as key-value pairs. Using the dynamic field, you can search and query both schema-defined fields and any non-schema-defined fields they may have. - -## Enable dynamic field - -When defining a schema for a collection, you can set `enable_dynamic_field` to `True` to enable the reserved dynamic field, indicating that any non-schema-defined fields and their values inserted later on will be saved as key-value pairs in the reserved dynamic field. - -The following snippet creates a collection with two schema-defined fields, namely id and vector, and enables the dynamic field. - -
- -For more information on parameters, refer to [`create_collection()`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Collections/create_collection.md) in the SDK reference. - -
- -
- -For more information on parameters, refer to [`createCollection()`](https://milvus.io/api-reference/java/v2.4.x/v2/Collections/createCollection.md) in the SDK reference. - -
- -
- -For more information on parameters, refer to [`createCollection()`](https://milvus.io/api-reference/node/v2.4.x/Collections/createCollection.md) in the SDK reference. - -
- -
- Python - Java - Node.js -
- -```python -import random, time -from pymilvus import connections, MilvusClient, DataType - -SERVER_ADDR = "http://localhost:19530" - -# 1. Set up a Milvus client -client = MilvusClient( - uri=SERVER_ADDR -) - -# 2. Create a collection -schema = MilvusClient.create_schema( - auto_id=False, - enable_dynamic_field=True, -) - -schema.add_field(field_name="id", datatype=DataType.INT64, is_primary=True) -schema.add_field(field_name="vector", datatype=DataType.FLOAT_VECTOR, dim=5) - -index_params = MilvusClient.prepare_index_params() - -index_params.add_index( - field_name="id", - index_type="STL_SORT" -) - -index_params.add_index( - field_name="vector", - index_type="IVF_FLAT", - metric_type="L2", - params={"nlist": 1024} -) - -client.create_collection( - collection_name="test_collection", - schema=schema, - index_params=index_params -) - -res = client.get_load_state( - collection_name="test_collection" -) - -print(res) - -# Output -# -# { -# "state": "" -# } -``` - -```java -import io.milvus.v2.client.ConnectConfig; -import io.milvus.v2.client.MilvusClientV2; -import io.milvus.v2.common.DataType; -import io.milvus.v2.common.IndexParam; -import io.milvus.v2.service.collection.request.AddFieldReq; -import io.milvus.v2.service.collection.request.CreateCollectionReq; -import io.milvus.v2.service.collection.request.GetLoadStateReq; - -String CLUSTER_ENDPOINT = "http://localhost:19530"; - -// 1. Connect to Milvus server -ConnectConfig connectConfig = ConnectConfig.builder() - .uri(CLUSTER_ENDPOINT) - .build(); - -MilvusClientV2 client = new MilvusClientV2(connectConfig); - -// 2. Create a collection in customized setup mode - -// 2.1 Create schema -CreateCollectionReq.CollectionSchema schema = client.createSchema(); - -// 2.2 Add fields to schema -schema.addField(AddFieldReq.builder().fieldName("id").dataType(DataType.Int64).isPrimaryKey(true).autoID(false).build()); -schema.addField(AddFieldReq.builder().fieldName("vector").dataType(DataType.FloatVector).dimension(5).build()); - -// 2.3 Prepare index parameters -IndexParam indexParamForIdField = IndexParam.builder() - .fieldName("id") - .indexType(IndexParam.IndexType.STL_SORT) - .build(); - -IndexParam indexParamForVectorField = IndexParam.builder() - .fieldName("vector") - .indexType(IndexParam.IndexType.IVF_FLAT) - .metricType(IndexParam.MetricType.IP) - .extraParams(Map.of("nlist", 1024)) - .build(); - -List indexParams = new ArrayList<>(); -indexParams.add(indexParamForIdField); -indexParams.add(indexParamForVectorField); - -// 2.4 Create a collection with schema and index parameters -CreateCollectionReq customizedSetupReq = CreateCollectionReq.builder() - .collectionName("customized_setup") - .collectionSchema(schema) - .indexParams(indexParams) - .enableDynamicField(true) - .build(); - -client.createCollection(customizedSetupReq); - -Thread.sleep(5000); - -// 2.5 Get load state of the collection -GetLoadStateReq customSetupLoadStateReq1 = GetLoadStateReq.builder() - .collectionName("customized_setup") - .build(); - -boolean res = client.getLoadState(customSetupLoadStateReq1); - -System.out.println(res); - -// Output: -// true -``` - -```javascript -const { MilvusClient, DataType, sleep } = require("@zilliz/milvus2-sdk-node") - -const address = "http://localhost:19530" - -async function main() { -// 1. Set up a Milvus Client -client = new MilvusClient({address}); - -// 2. Create a collection -// 2.1 Define fields -const fields = [ - { - name: "id", - data_type: DataType.Int64, - is_primary_key: true, - auto_id: false - }, - { - name: "vector", - data_type: DataType.FloatVector, - dim: 5 - }, -] - -// 2.2 Prepare index parameters -const index_params = [{ - field_name: "id", - index_type: "STL_SORT" -},{ - field_name: "vector", - index_type: "IVF_FLAT", - metric_type: "IP", - params: { nlist: 1024} -}] - -// 2.3 Create a collection with fields and index parameters -res = await client.createCollection({ - collection_name: "test_collection", - fields: fields, - index_params: index_params, - enable_dynamic_field: true -}) - -console.log(res.error_code) - -// Output -// -// Success -// - -res = await client.getLoadState({ - collection_name: "test_collection", -}) - -console.log(res.state) - -// Output -// -// LoadStateLoaded -// -``` - -## Insert dynamic data - -Once the collection is created, you can start inserting data, including the dynamic data into the collection. - -### Prepare data - -In this section, you need to prepare some randomly generated data for the insertion later on. - -
- Python - Java - Node.js -
- -```python -colors = ["green", "blue", "yellow", "red", "black", "white", "purple", "pink", "orange", "brown", "grey"] -data = [] - -for i in range(1000): - current_color = random.choice(colors) - current_tag = random.randint(1000, 9999) - data.append({ - "id": i, - "vector": [ random.uniform(-1, 1) for _ in range(5) ], - "color": current_color, - "tag": current_tag, - "color_tag": f"{current_color}_{str(current_tag)}" - }) - -print(data[0]) -``` - -```java -List colors = Arrays.asList("green", "blue", "yellow", "red", "black", "white", "purple", "pink", "orange", "brown", "grey"); -List data = new ArrayList<>(); - -for (int i=0; i<1000; i++) { - Random rand = new Random(); - String current_color = colors.get(rand.nextInt(colors.size()-1)); - int current_tag = rand.nextInt(8999) + 1000; - JSONObject row = new JSONObject(); - row.put("id", Long.valueOf(i)); - row.put("vector", Arrays.asList(rand.nextFloat(), rand.nextFloat(), rand.nextFloat(), rand.nextFloat(), rand.nextFloat())); - row.put("color", current_color); - row.put("tag", current_tag); - row.put("color_tag", current_color + "_" + String.valueOf(rand.nextInt(8999) + 1000)); - data.add(row); -} - -System.out.println(JSONObject.toJSON(data.get(0))); -``` - -```javascript -const colors = ["green", "blue", "yellow", "red", "black", "white", "purple", "pink", "orange", "brown", "grey"] -var data = [] - -for (let i = 0; i < 1000; i++) { - const current_color = colors[Math.floor(Math.random() * colors.length)] - const current_tag = Math.floor(Math.random() * 8999 + 1000) - data.push({ - id: i, - vector: [Math.random(), Math.random(), Math.random(), Math.random(), Math.random()], - color: current_color, - tag: current_tag, - color_tag: `${current_color}_${current_tag}` - }) -} - -console.log(data[0]) -``` - -You can view the structure of the generated data by checking its first entry. - -``` -{ - id: 0, - vector: [ - 0.1275656405044483, - 0.47417858592773277, - 0.13858264437643286, - 0.2390904907020377, - 0.8447862593689635 - ], - color: 'blue', - tag: 2064, - color_tag: 'blue_2064' -} -``` - -### Insert data - -Then you can safely insert the data into the collection. - -
- -For more information on parameters, refer to [`insert()`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Vector/insert.md) in the SDK reference. - -
- -
- -For more information on parameters, refer to [`insert()`](https://milvus.io/api-reference/java/v2.4.x/v2/Vector/insert.md) in the SDK reference. - -
- -
- -For more information on parameters, refer to [`insert()`](https://milvus.io/api-reference/node/v2.4.x/Vector/insert.md) in the SDK reference. - -
- -
- Python - Java - Node.js -
- -```python -res = client.insert( - collection_name="test_collection", - data=data, -) - -print(res) - -# Output -# -# { -# "insert_count": 1000, -# "ids": [ -# 0, -# 1, -# 2, -# 3, -# 4, -# 5, -# 6, -# 7, -# 8, -# 9, -# "(990 more items hidden)" -# ] -# } - -time.sleep(5) -``` - -```java -// 3.1 Insert data into the collection -InsertReq insertReq = InsertReq.builder() - .collectionName("customized_setup") - .data(data) - .build(); - -InsertResp insertResp = client.insert(insertReq); - -System.out.println(JSONObject.toJSON(insertResp)); - -// Output: -// {"insertCnt": 1000} - -Thread.sleep(5000); -``` - -```javascript -res = await client.insert({ - collection_name: "test_collection", - data: data, -}) - -console.log(res.insert_cnt) - -// Output -// -// 1000 -// - -await sleep(5000) -``` - -## Search with dynamic fields - -If you have created the collection with the dynamic field enabled and inserted non-schema-defined fields, you can use these fields in the filter expression of a search or a query as follows. - -
- -For more information on parameters, refer to [`search()`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Vector/search.md) in the SDK reference. - -
- -
- -For more information on parameters, refer to [`search()`](https://milvus.io/api-reference/java/v2.4.x/v2/Vector/search.md) in the SDK reference. - -
- -
- -For more information on parameters, refer to [`search()`](https://milvus.io/api-reference/node/v2.4.x/Vector/search.md) in the SDK reference. - -
- -
- Python - Java - Node.js -
- -```python -# 4. Search with dynamic fields -query_vectors = [[0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592]] - -res = client.search( - collection_name="test_collection", - data=query_vectors, - filter="color in [\"red\", \"green\"]", - search_params={"metric_type": "L2", "params": {"nprobe": 10}}, - limit=3 -) - -print(res) - -# Output -# -# [ -# [ -# { -# "id": 863, -# "distance": 0.188413605093956, -# "entity": { -# "id": 863, -# "color_tag": "red_2371" -# } -# }, -# { -# "id": 799, -# "distance": 0.29188022017478943, -# "entity": { -# "id": 799, -# "color_tag": "red_2235" -# } -# }, -# { -# "id": 564, -# "distance": 0.3492690920829773, -# "entity": { -# "id": 564, -# "color_tag": "red_9186" -# } -# } -# ] -# ] -``` - -```java -// 4. Search with non-schema-defined fields -List> queryVectors = Arrays.asList(Arrays.asList(0.3580376395471989f, -0.6023495712049978f, 0.18414012509913835f, -0.26286205330961354f, 0.9029438446296592f)); - -SearchReq searchReq = SearchReq.builder() - .collectionName("customized_setup") - .data(queryVectors) - .filter("$meta[\"color\"] in [\"red\", \"green\"]") - .outputFields(List.of("id", "color_tag")) - .topK(3) - .build(); - -SearchResp searchResp = client.search(searchReq); - -System.out.println(JSONObject.toJSON(searchResp)); - -// Output: -// {"searchResults": [[ -// { -// "distance": 1.3159835, -// "id": 979, -// "entity": { -// "color_tag": "red_7155", -// "id": 979 -// } -// }, -// { -// "distance": 1.0744804, -// "id": 44, -// "entity": { -// "color_tag": "green_8006", -// "id": 44 -// } -// }, -// { -// "distance": 1.0060014, -// "id": 617, -// "entity": { -// "color_tag": "red_4056", -// "id": 617 -// } -// } -// ]]} -``` - -```javascript -// 4. Search with non-schema-defined fields -const query_vectors = [[0.1, 0.2, 0.3, 0.4, 0.5]] - -res = await client.search({ - collection_name: "test_collection", - data: query_vectors, - filter: "color in [\"red\", \"green\"]", - output_fields: ["color_tag"], - limit: 3 -}) - -console.log(res.results) - -// Output -// -// [ -// { score: 1.2284551858901978, id: '301', color_tag: 'red_1270' }, -// { score: 1.2195171117782593, id: '205', color_tag: 'red_2780' }, -// { score: 1.2055039405822754, id: '487', color_tag: 'red_6653' } -// ] -// -``` - -## Recaps - -It is worth noting that __color__, __tag__, and __color_tag__ are not present when you define the collection schema, but you can use them as schema-defined fields when you conduct searches and queries. - -If the name of a non-schema-defined field contains characters other than digits, letters, and underscores, such as plus signs (+), asterisks (*), or dollar signs ($), you have to include the key within __$meta[]__ as shown in the following code snippet when using it in a boolean expression or including it in the output fields. - -```python -... -filter='$meta["$key"] in ["a", "b", "c"]', -output_fields='$meta["$key"]' -... -``` diff --git a/v2.4.x/site/en/userGuide/insert-and-delete/delete-entities.md b/v2.4.x/site/en/userGuide/insert-and-delete/delete-entities.md new file mode 100644 index 0000000000..efe813d3cd --- /dev/null +++ b/v2.4.x/site/en/userGuide/insert-and-delete/delete-entities.md @@ -0,0 +1,255 @@ +--- +id: delete-entities.md +title: Delete Entities +--- + +# Delete Entities​ + +You can delete the entities that are no longer needed by filtering conditions or their primary keys.​ + +## Delete Entities by Filtering Conditions​ + +When deleting multiple entities that share some attributes in a batch, you can use filter expressions. The example code below uses the **in** operator to bulk delete all Entities with thier **color** field set to the values of **red** and **green**. You can also use other operators to construct filter expressions that meet your requirements. For more information about filter expressions, please refer to [​Metadata Filtering](boolean.md).​ + +
+ Python + Java + Node.js + Go + cURL +
+ +```python +from pymilvus import MilvusClient​ +​ +client = MilvusClient(​ + uri="http://localhost:19530",​ + token="root:Milvus"​ +)​ +​ +res = client.delete(​ + collection_name="quick_setup",​ + # highlight-next-line​ + filter="color in ['red_3314', 'purple_7392']"​ +)​ +​ +print(res)​ +​ +# Output​ +# {'delete_count': 2}​ + +``` + +```java +import io.milvus.v2.client.ConnectConfig;​ +import io.milvus.v2.client.MilvusClientV2;​ +import io.milvus.v2.service.vector.request.DeleteReq;​ +import io.milvus.v2.service.vector.response.DeleteResp;​ +​ +ilvusClientV2 client = new MilvusClientV2(ConnectConfig.builder()​ + .uri("http://localhost:19530")​ + .token("root:Milvus")​ + .build());​ +​ +DeleteResp deleteResp = client.delete(DeleteReq.builder()​ + .collectionName("quick_setup")​ + .filter("color in ['red_3314', 'purple_7392']")​ + .build());​ +​ + +``` + +```javascript +const { MilvusClient, DataType } = require("@zilliz/milvus2-sdk-node")​ +​ +const address = "http://localhost:19530";​ +const token = "root:Milvus";​ +const client = new MilvusClient({address, token});​ +​ +// 7. Delete entities​ +res = await client.delete({​ + collection_name: "quick_setup",​ + // highlight-next-line​ + filter: "color in ['red', 'green']"​ +})​ +​ +console.log(res.delete_cnt)​ +​ +// Output​ +// ​ +// 3​ +// ​ + +``` + +```curl +export CLUSTER_ENDPOINT="http://localhost:19530"​ +export TOKEN="root:Milvus"​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/delete" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "quick_setup",​ + "filter": "color in [\"red_3314\", \"purple_7392\"]"​ +}'​ + +``` + + +## Delete Entities by Primary Keys​ + +In most cases, a primary key uniquely identifies an Entity. You can delete Entities by setting their primary keys in the delete request. The example code below demonstrates how to delete two entities with primary keys **18** and **19**.​ + +
+ Python + Java + Node.js + Go + cURL +
+ + +```python +res = client.delete(​ + collection_name="quick_setup",​ + # highlight-next-line​ + ids=[18, 19]​ +)​ +​ +print(res)​ +​ +# Output​ +# {'delete_count': 2}​ + +``` + +```java +import io.milvus.v2.service.vector.request.DeleteReq;​ +import io.milvus.v2.service.vector.response.DeleteResp;​ +​ +import java.util.Arrays;​ +​ +​ +DeleteResp deleteResp = client.delete(DeleteReq.builder()​ + .collectionName("quick_setup")​ + .ids(Arrays.asList(18, 19))​ + .build());​ + +``` + +```javascript +const { MilvusClient, DataType } = require("@zilliz/milvus2-sdk-node")​ +​ +res = await client.delete({​ + collection_name: "quick_setup",​ + ids: [18, 19]​ +})​ +​ +console.log(res.delete_cnt)​ +​ +// Output​ +// ​ +// 2​ +// ​ + +``` + +```curl +export CLUSTER_ENDPOINT="http://localhost:19530"​ +export TOKEN="root:Milvus"​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/delete" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "quick_setup",​ + "filter": "id in [18, 19]"​ +}'​ +## {"code":0,"cost":0,"data":{}}​ + +``` + + +## Delete Entities from Partitions​ + +You can also delete entities stored in specific partitions. The following code snippets assume that you have a partition named **PartitionA** in your collection. + +
+ Python + Java + Node.js + Go + cURL +
+ + +```python +res = client.delete(​ + collection_name="quick_setup",​ + ids=[18, 19],​ + partition_name="partitionA"​ +)​ +​ +print(res)​ +​ +# Output​ +# {'delete_count': 2}​ + +``` + +```java +import io.milvus.v2.service.vector.request.DeleteReq;​ +import io.milvus.v2.service.vector.response.DeleteResp;​ +​ +import java.util.Arrays;​ +​ +DeleteResp deleteResp = client.delete(DeleteReq.builder()​ + .collectionName("quick_setup")​ + .ids(Arrays.asList(18, 19))​ + .partitionName("partitionA")​ + .build());​ + +``` + +```javascript +const { MilvusClient, DataType } = require("@zilliz/milvus2-sdk-node")​ +​ +res = await client.delete({​ + collection_name: "quick_setup",​ + ids: [18, 19],​ + partition_name: "partitionA"​ +})​ +​ +console.log(res.delete_cnt)​ +​ +// Output​ +// ​ +// 2​ +// ​ + +``` + +```curl +export CLUSTER_ENDPOINT="http://localhost:19530"​ +export TOKEN="root:Milvus"​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/delete" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "quick_setup",​ + "partitionName": "partitionA",​ + "filter": "id in [18, 19]"​ +}'​ +​ +# {​ +# "code": 0,​ +# "cost": 0,​ +# "data": {}​ +# }​ + +``` diff --git a/v2.4.x/site/en/userGuide/insert-and-delete/insert-update-delete.md b/v2.4.x/site/en/userGuide/insert-and-delete/insert-update-delete.md new file mode 100644 index 0000000000..6f147b4b15 --- /dev/null +++ b/v2.4.x/site/en/userGuide/insert-and-delete/insert-update-delete.md @@ -0,0 +1,345 @@ +--- +id: insert-update-delete.md +summary: Entities in a collection are data records that share the same set of fields. Field values in every data record form an entity. This page introduces how to insert entities into a collection.​ +title: Insert Entities​ +--- + +# Insert Entities​ + +Entities in a collection are data records that share the same set of fields. Field values in every data record form an entity. This page introduces how to insert entities into a collection.​ + +## Overview​ + +In Milvus, an **Entity** refers to data records in a **Collection** that share the same **Schema**, with the data in each field of a row constituting an Entity. Therefore, the Entities within the same Collection have the same attributes (such as field names, data types, and other constraints).​ + +When inserting an Entity into a Collection, the Entity to be inserted can only be successfully added if it contains all the fields defined in the Schema. The inserted Entity will enter a Partition named **_default** in the order of insertion. Provided that a certain Partition exists, you can also insert Entities into that Partition by specifying the Partition name in the insertion request.​ + +Milvus also supports dynamic fields to maintain the scalability of the Collection. When the dynamic field is enabled, you can insert fields that are not defined in the Schema into the Collection. These fields and values will be stored as key-value pairs in a reserved field named **$meta**. For more information about dynamic fields, please refer to Dynamic Field.​ + +## Insert Entities into a Collection​ + +Before inserting data, you need to organize your data into a list of dictionaries according to the Schema, with each dictionary representing an Entity and containing all the fields defined in the Schema. If the Collection has the dynamic field enabled, each dictionary can also include fields that are not defined in the Schema.​ + +In this section, you will insert Entities into a Collection created [in the quick-setup manner](create-collection-instantly.md#Quick-Setup). A Collection created in this manner has only two fields, named **id** and **vector**. Additionally, this Collection has the dynamic field enabled, so the Entities in the example code include a field called **color **that is not defined in the Schema.​ + +
+ Python + Java + Node.js + Go + cURL +
+ +```python +from pymilvus import MilvusClient​ +​ +client = MilvusClient(​ + uri="http://localhost:19530",​ + token="root:Milvus"​ +)​ +​ +data=[​ + {"id": 0, "vector": [0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592], "color": "pink_8682"},​ + {"id": 1, "vector": [0.19886812562848388, 0.06023560599112088, 0.6976963061752597, 0.2614474506242501, 0.838729485096104], "color": "red_7025"},​ + {"id": 2, "vector": [0.43742130801983836, -0.5597502546264526, 0.6457887650909682, 0.7894058910881185, 0.20785793220625592], "color": "orange_6781"},​ + {"id": 3, "vector": [0.3172005263489739, 0.9719044792798428, -0.36981146090600725, -0.4860894583077995, 0.95791889146345], "color": "pink_9298"},​ + {"id": 4, "vector": [0.4452349528804562, -0.8757026943054742, 0.8220779437047674, 0.46406290649483184, 0.30337481143159106], "color": "red_4794"},​ + {"id": 5, "vector": [0.985825131989184, -0.8144651566660419, 0.6299267002202009, 0.1206906911183383, -0.1446277761879955], "color": "yellow_4222"},​ + {"id": 6, "vector": [0.8371977790571115, -0.015764369584852833, -0.31062937026679327, -0.562666951622192, -0.8984947637863987], "color": "red_9392"},​ + {"id": 7, "vector": [-0.33445148015177995, -0.2567135004164067, 0.8987539745369246, 0.9402995886420709, 0.5378064918413052], "color": "grey_8510"},​ + {"id": 8, "vector": [0.39524717779832685, 0.4000257286739164, -0.5890507376891594, -0.8650502298996872, -0.6140360785406336], "color": "white_9381"},​ + {"id": 9, "vector": [0.5718280481994695, 0.24070317428066512, -0.3737913482606834, -0.06726932177492717, -0.6980531615588608], "color": "purple_4976"}​ +]​ +​ +res = client.insert(​ + collection_name="quick_setup",​ + data=data​ +)​ +​ +print(res)​ +​ +# Output​ +# {'insert_count': 10, 'ids': [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]}​ + +``` + +```java +import com.google.gson.Gson;​ +import com.google.gson.JsonObject;​ +import io.milvus.v2.client.ConnectConfig;​ +import io.milvus.v2.client.MilvusClientV2;​ +import io.milvus.v2.service.vector.request.InsertReq;​ +import io.milvus.v2.service.vector.response.InsertResp;​ +​ +import java.util.*;​ +​ +MilvusClientV2 client = new MilvusClientV2(ConnectConfig.builder()​ + .uri("http://localhost:19530")​ + .token("root:Milvus")​ + .build());​ +​ +Gson gson = new Gson();​ +List data = Arrays.asList(​ + gson.fromJson("{\"id\": 0, \"vector\": [0.3580376395471989f, -0.6023495712049978f, 0.18414012509913835f, -0.26286205330961354f, 0.9029438446296592f], \"color\": \"pink_8682\"}", JsonObject.class),​ + gson.fromJson("{\"id\": 1, \"vector\": [0.19886812562848388f, 0.06023560599112088f, 0.6976963061752597f, 0.2614474506242501f, 0.838729485096104f], \"color\": \"red_7025\"}", JsonObject.class),​ + gson.fromJson("{\"id\": 2, \"vector\": [0.43742130801983836f, -0.5597502546264526f, 0.6457887650909682f, 0.7894058910881185f, 0.20785793220625592f], \"color\": \"orange_6781\"}", JsonObject.class),​ + gson.fromJson("{\"id\": 3, \"vector\": [0.3172005263489739f, 0.9719044792798428f, -0.36981146090600725f, -0.4860894583077995f, 0.95791889146345f], \"color\": \"pink_9298\"}", JsonObject.class),​ + gson.fromJson("{\"id\": 4, \"vector\": [0.4452349528804562f, -0.8757026943054742f, 0.8220779437047674f, 0.46406290649483184f, 0.30337481143159106f], \"color\": \"red_4794\"}", JsonObject.class),​ + gson.fromJson("{\"id\": 5, \"vector\": [0.985825131989184f, -0.8144651566660419f, 0.6299267002202009f, 0.1206906911183383f, -0.1446277761879955f], \"color\": \"yellow_4222\"}", JsonObject.class),​ + gson.fromJson("{\"id\": 6, \"vector\": [0.8371977790571115f, -0.015764369584852833f, -0.31062937026679327f, -0.562666951622192f, -0.8984947637863987f], \"color\": \"red_9392\"}", JsonObject.class),​ + gson.fromJson("{\"id\": 7, \"vector\": [-0.33445148015177995f, -0.2567135004164067f, 0.8987539745369246f, 0.9402995886420709f, 0.5378064918413052f], \"color\": \"grey_8510\"}", JsonObject.class),​ + gson.fromJson("{\"id\": 8, \"vector\": [0.39524717779832685f, 0.4000257286739164f, -0.5890507376891594f, -0.8650502298996872f, -0.6140360785406336f], \"color\": \"white_9381\"}", JsonObject.class),​ + gson.fromJson("{\"id\": 9, \"vector\": [0.5718280481994695f, 0.24070317428066512f, -0.3737913482606834f, -0.06726932177492717f, -0.6980531615588608f], \"color\": \"purple_4976\"}", JsonObject.class)​ +);​ +​ +InsertReq insertReq = InsertReq.builder()​ + .collectionName("quick_setup")​ + .data(data)​ + .build();​ +​ +InsertResp insertResp = client.insert(insertReq);​ +System.out.println(insertResp);​ +​ +// Output:​ +//​ +// InsertResp(InsertCnt=10, primaryKeys=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9])​ + +``` + +```javascript +const { MilvusClient, DataType } = require("@zilliz/milvus2-sdk-node")​ +​ +const address = "http://localhost:19530";​ +const token = "root:Milvus";​ +const client = new MilvusClient({address, token});​ +​ +// 3. Insert some data​ +​ +var data = [​ + {id: 0, vector: [0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592], color: "pink_8682"},​ + {id: 1, vector: [0.19886812562848388, 0.06023560599112088, 0.6976963061752597, 0.2614474506242501, 0.838729485096104], color: "red_7025"},​ + {id: 2, vector: [0.43742130801983836, -0.5597502546264526, 0.6457887650909682, 0.7894058910881185, 0.20785793220625592], color: "orange_6781"},​ + {id: 3, vector: [0.3172005263489739, 0.9719044792798428, -0.36981146090600725, -0.4860894583077995, 0.95791889146345], color: "pink_9298"},​ + {id: 4, vector: [0.4452349528804562, -0.8757026943054742, 0.8220779437047674, 0.46406290649483184, 0.30337481143159106], color: "red_4794"},​ + {id: 5, vector: [0.985825131989184, -0.8144651566660419, 0.6299267002202009, 0.1206906911183383, -0.1446277761879955], color: "yellow_4222"},​ + {id: 6, vector: [0.8371977790571115, -0.015764369584852833, -0.31062937026679327, -0.562666951622192, -0.8984947637863987], color: "red_9392"},​ + {id: 7, vector: [-0.33445148015177995, -0.2567135004164067, 0.8987539745369246, 0.9402995886420709, 0.5378064918413052], color: "grey_8510"},​ + {id: 8, vector: [0.39524717779832685, 0.4000257286739164, -0.5890507376891594, -0.8650502298996872, -0.6140360785406336], color: "white_9381"},​ + {id: 9, vector: [0.5718280481994695, 0.24070317428066512, -0.3737913482606834, -0.06726932177492717, -0.6980531615588608], color: "purple_4976"} ​ +]​ +​ +var res = await client.insert({​ + collection_name: "quick_setup",​ + data: data,​ +})​ +​ +console.log(res.insert_cnt)​ +​ +// Output​ +// ​ +// 10​ +// ​ + +``` + +```curl +export CLUSTER_ENDPOINT="http://localhost:19530"​ +export TOKEN="root:Milvus"​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/insert" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "data": [​ + {"id": 0, "vector": [0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592], "color": "pink_8682"},​ + {"id": 1, "vector": [0.19886812562848388, 0.06023560599112088, 0.6976963061752597, 0.2614474506242501, 0.838729485096104], "color": "red_7025"},​ + {"id": 2, "vector": [0.43742130801983836, -0.5597502546264526, 0.6457887650909682, 0.7894058910881185, 0.20785793220625592], "color": "orange_6781"},​ + {"id": 3, "vector": [0.3172005263489739, 0.9719044792798428, -0.36981146090600725, -0.4860894583077995, 0.95791889146345], "color": "pink_9298"},​ + {"id": 4, "vector": [0.4452349528804562, -0.8757026943054742, 0.8220779437047674, 0.46406290649483184, 0.30337481143159106], "color": "red_4794"},​ + {"id": 5, "vector": [0.985825131989184, -0.8144651566660419, 0.6299267002202009, 0.1206906911183383, -0.1446277761879955], "color": "yellow_4222"},​ + {"id": 6, "vector": [0.8371977790571115, -0.015764369584852833, -0.31062937026679327, -0.562666951622192, -0.8984947637863987], "color": "red_9392"},​ + {"id": 7, "vector": [-0.33445148015177995, -0.2567135004164067, 0.8987539745369246, 0.9402995886420709, 0.5378064918413052], "color": "grey_8510"},​ + {"id": 8, "vector": [0.39524717779832685, 0.4000257286739164, -0.5890507376891594, -0.8650502298996872, -0.6140360785406336], "color": "white_9381"},​ + {"id": 9, "vector": [0.5718280481994695, 0.24070317428066512, -0.3737913482606834, -0.06726932177492717, -0.6980531615588608], "color": "purple_4976"} ​ + ],​ + "collectionName": "quick_setup"​ +}'​ +​ +# {​ +# "code": 0,​ +# "data": {​ +# "insertCount": 10,​ +# "insertIds": [​ +# 0,​ +# 1,​ +# 2,​ +# 3,​ +# 4,​ +# 5,​ +# 6,​ +# 7,​ +# 8,​ +# 9​ +# ]​ +# }​ +# }​ + +``` + +## Insert Entities into a Partition​ + +You can also insert entities into a specified partition. The following code snippets assume that you have a partition named **PartitionA** in your collection.​ + +
+ Python + Java + Node.js + Go + cURL +
+ +```python +data=[​ + {"id": 10, "vector": [0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592], "color": "pink_8682"},​ + {"id": 11, "vector": [0.19886812562848388, 0.06023560599112088, 0.6976963061752597, 0.2614474506242501, 0.838729485096104], "color": "red_7025"},​ + {"id": 12, "vector": [0.43742130801983836, -0.5597502546264526, 0.6457887650909682, 0.7894058910881185, 0.20785793220625592], "color": "orange_6781"},​ + {"id": 13, "vector": [0.3172005263489739, 0.9719044792798428, -0.36981146090600725, -0.4860894583077995, 0.95791889146345], "color": "pink_9298"},​ + {"id": 14, "vector": [0.4452349528804562, -0.8757026943054742, 0.8220779437047674, 0.46406290649483184, 0.30337481143159106], "color": "red_4794"},​ + {"id": 15, "vector": [0.985825131989184, -0.8144651566660419, 0.6299267002202009, 0.1206906911183383, -0.1446277761879955], "color": "yellow_4222"},​ + {"id": 16, "vector": [0.8371977790571115, -0.015764369584852833, -0.31062937026679327, -0.562666951622192, -0.8984947637863987], "color": "red_9392"},​ + {"id": 17, "vector": [-0.33445148015177995, -0.2567135004164067, 0.8987539745369246, 0.9402995886420709, 0.5378064918413052], "color": "grey_8510"},​ + {"id": 18, "vector": [0.39524717779832685, 0.4000257286739164, -0.5890507376891594, -0.8650502298996872, -0.6140360785406336], "color": "white_9381"},​ + {"id": 19, "vector": [0.5718280481994695, 0.24070317428066512, -0.3737913482606834, -0.06726932177492717, -0.6980531615588608], "color": "purple_4976"}​ +]​ +​ +res = client.insert(​ + collection_name="quick_setup",​ + # highlight-next-line​ + partition_name="partitionA",​ + data=data​ +)​ +​ +print(res)​ +​ +# Output​ +# {'insert_count': 10, 'ids': [10, 11, 12, 13, 14, 15, 16, 17, 18, 19]}​ + +``` + +```java +import com.google.gson.Gson;​ +import com.google.gson.JsonObject;​ +import io.milvus.v2.service.vector.request.InsertReq;​ +import io.milvus.v2.service.vector.response.InsertResp;​ +​ +import java.util.*;​ +​ +Gson gson = new Gson();​ +List data = Arrays.asList(​ + gson.fromJson("{\"id\": 10, \"vector\": [0.3580376395471989f, -0.6023495712049978f, 0.18414012509913835f, -0.26286205330961354f, 0.9029438446296592f], \"color\": \"pink_8682\"}", JsonObject.class),​ + gson.fromJson("{\"id\": 11, \"vector\": [0.19886812562848388f, 0.06023560599112088f, 0.6976963061752597f, 0.2614474506242501f, 0.838729485096104f], \"color\": \"red_7025\"}", JsonObject.class),​ + gson.fromJson("{\"id\": 12, \"vector\": [0.43742130801983836f, -0.5597502546264526f, 0.6457887650909682f, 0.7894058910881185f, 0.20785793220625592f], \"color\": \"orange_6781\"}", JsonObject.class),​ + gson.fromJson("{\"id\": 13, \"vector\": [0.3172005263489739f, 0.9719044792798428f, -0.36981146090600725f, -0.4860894583077995f, 0.95791889146345f], \"color\": \"pink_9298\"}", JsonObject.class),​ + gson.fromJson("{\"id\": 14, \"vector\": [0.4452349528804562f, -0.8757026943054742f, 0.8220779437047674f, 0.46406290649483184f, 0.30337481143159106f], \"color\": \"red_4794\"}", JsonObject.class),​ + gson.fromJson("{\"id\": 15, \"vector\": [0.985825131989184f, -0.8144651566660419f, 0.6299267002202009f, 0.1206906911183383f, -0.1446277761879955f], \"color\": \"yellow_4222\"}", JsonObject.class),​ + gson.fromJson("{\"id\": 16, \"vector\": [0.8371977790571115f, -0.015764369584852833f, -0.31062937026679327f, -0.562666951622192f, -0.8984947637863987f], \"color\": \"red_9392\"}", JsonObject.class),​ + gson.fromJson("{\"id\": 17, \"vector\": [-0.33445148015177995f, -0.2567135004164067f, 0.8987539745369246f, 0.9402995886420709f, 0.5378064918413052f], \"color\": \"grey_8510\"}", JsonObject.class),​ + gson.fromJson("{\"id\": 18, \"vector\": [0.39524717779832685f, 0.4000257286739164f, -0.5890507376891594f, -0.8650502298996872f, -0.6140360785406336f], \"color\": \"white_9381\"}", JsonObject.class),​ + gson.fromJson("{\"id\": 19, \"vector\": [0.5718280481994695f, 0.24070317428066512f, -0.3737913482606834f, -0.06726932177492717f, -0.6980531615588608f], \"color\": \"purple_4976\"}", JsonObject.class)​ +);​ +​ +InsertReq insertReq = InsertReq.builder()​ + .collectionName("quick_setup")​ + .partitionName("partitionA")​ + .data(data)​ + .build();​ +​ +InsertResp insertResp = client.insert(insertReq);​ +System.out.println(insertResp);​ +​ +// Output:​ +//​ +// InsertResp(InsertCnt=10, primaryKeys=[10, 11, 12, 13, 14, 15, 16, 17, 18, 19])​ + +``` + +```javascript +const { MilvusClient, DataType } = require("@zilliz/milvus2-sdk-node")​ +​ +// 3. Insert some data​ +​ +var data = [​ + {id: 10, vector: [0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592], color: "pink_8682"},​ + {id: 11, vector: [0.19886812562848388, 0.06023560599112088, 0.6976963061752597, 0.2614474506242501, 0.838729485096104], color: "red_7025"},​ + {id: 12, vector: [0.43742130801983836, -0.5597502546264526, 0.6457887650909682, 0.7894058910881185, 0.20785793220625592], color: "orange_6781"},​ + {id: 13, vector: [0.3172005263489739, 0.9719044792798428, -0.36981146090600725, -0.4860894583077995, 0.95791889146345], color: "pink_9298"},​ + {id: 14, vector: [0.4452349528804562, -0.8757026943054742, 0.8220779437047674, 0.46406290649483184, 0.30337481143159106], color: "red_4794"},​ + {id: 15, vector: [0.985825131989184, -0.8144651566660419, 0.6299267002202009, 0.1206906911183383, -0.1446277761879955], color: "yellow_4222"},​ + {id: 16, vector: [0.8371977790571115, -0.015764369584852833, -0.31062937026679327, -0.562666951622192, -0.8984947637863987], color: "red_9392"},​ + {id: 17, vector: [-0.33445148015177995, -0.2567135004164067, 0.8987539745369246, 0.9402995886420709, 0.5378064918413052], color: "grey_8510"},​ + {id: 18, vector: [0.39524717779832685, 0.4000257286739164, -0.5890507376891594, -0.8650502298996872, -0.6140360785406336], color: "white_9381"},​ + {id: 19, vector: [0.5718280481994695, 0.24070317428066512, -0.3737913482606834, -0.06726932177492717, -0.6980531615588608], color: "purple_4976"} ​ +]​ +​ +var res = await client.insert({​ + collection_name: "quick_setup",​ + // highlight-next-line​ + partition_name: "partitionA",​ + data: data,​ +})​ +​ +console.log(res.insert_cnt)​ +​ +// Output​ +// ​ +// 10​ +// ​ + +``` + +```curl +export CLUSTER_ENDPOINT="http://localhost:19530"​ +export TOKEN="root:Milvus"​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/insert" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "data": [​ + {"id": 10, "vector": [0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592], "color": "pink_8682"},​ + {"id": 11, "vector": [0.19886812562848388, 0.06023560599112088, 0.6976963061752597, 0.2614474506242501, 0.838729485096104], "color": "red_7025"},​ + {"id": 12, "vector": [0.43742130801983836, -0.5597502546264526, 0.6457887650909682, 0.7894058910881185, 0.20785793220625592], "color": "orange_6781"},​ + {"id": 13, "vector": [0.3172005263489739, 0.9719044792798428, -0.36981146090600725, -0.4860894583077995, 0.95791889146345], "color": "pink_9298"},​ + {"id": 14, "vector": [0.4452349528804562, -0.8757026943054742, 0.8220779437047674, 0.46406290649483184, 0.30337481143159106], "color": "red_4794"},​ + {"id": 15, "vector": [0.985825131989184, -0.8144651566660419, 0.6299267002202009, 0.1206906911183383, -0.1446277761879955], "color": "yellow_4222"},​ + {"id": 16, "vector": [0.8371977790571115, -0.015764369584852833, -0.31062937026679327, -0.562666951622192, -0.8984947637863987], "color": "red_9392"},​ + {"id": 17, "vector": [-0.33445148015177995, -0.2567135004164067, 0.8987539745369246, 0.9402995886420709, 0.5378064918413052], "color": "grey_8510"},​ + {"id": 18, "vector": [0.39524717779832685, 0.4000257286739164, -0.5890507376891594, -0.8650502298996872, -0.6140360785406336], "color": "white_9381"},​ + {"id": 19, "vector": [0.5718280481994695, 0.24070317428066512, -0.3737913482606834, -0.06726932177492717, -0.6980531615588608], "color": "purple_4976"} ​ + ],​ + "collectionName": "quick_setup",​ + "partitionName": "partitionA"​ +}'​ +​ +# {​ +# "code": 0,​ +# "data": {​ +# "insertCount": 10,​ +# "insertIds": [​ +# 10,​ +# 11,​ +# 12,​ +# 13,​ +# 14,​ +# 15,​ +# 16,​ +# 17,​ +# 18,​ +# 19​ +# ]​ +# }​ +# }​ + +``` diff --git a/v2.4.x/site/en/userGuide/insert-and-delete/upsert-entities.md b/v2.4.x/site/en/userGuide/insert-and-delete/upsert-entities.md new file mode 100644 index 0000000000..1398d1b136 --- /dev/null +++ b/v2.4.x/site/en/userGuide/insert-and-delete/upsert-entities.md @@ -0,0 +1,346 @@ +--- +id: upsert-entities.md +title: Upsert Entities +--- + +# Upsert Entities​ + +The Upsert operation combines the actions of updating and inserting data. Milvus determines whether to perform an update or an insert operation by checking if the primary key exists. This section will introduce how to Upsert an Entity and the specific behaviors of the Upsert operation in different scenarios.​ + +## Overview​ + +When you need to update an Entity in a Collection or are not sure whether to update or insert, you can try using the Upsert operation. When using this operation, it is essential to ensure that the Entity included in the Upsert request contains the primary key; otherwise, an error will occur. Upon receiving an Upsert request, Milvus will execute the following process:​ + +1. Check whether the primary field of the Collection has AutoId enabled.​ + + a. If it is, Milvus will replace the primary key in the Entity with an automatically generated primary key and insert the data.​ + + b. If not, Milvus will use the primary key carried by the Entity to insert the data.​ + +2. Perform a delete operation based on the primary key value of the Entity included in the Upsert request.​ + +![Upsert Entities](../../../../assets/upsert-entities.png) + +## Upsert Entity in a Collection​ + +In this section, you will upsert Entities into a Collection created [in the quick-setup manner](create-collection-instantly.md#Quick-Setup). A Collection created in this manner has only two fields, named **id** and **vector**. Additionally, this Collection has the dynamic field enabled, so the Entities in the example code include a field called **color** that is not defined in the Schema.​ + +
+ Python + Java + Node.js + Go + cURL +
+ + +```python +from pymilvus import MilvusClient​ +​ +client = MilvusClient(​ + uri="http://localhost:19530",​ + token="root:Milvus"​ +)​ +​ +data=[​ + {"id": 0, "vector": [-0.619954382375778, 0.4479436794798608, -0.17493894838751745, -0.4248030059917294, -0.8648452746018911], "color": "black_9898"},​ + {"id": 1, "vector": [0.4762662251462588, -0.6942502138717026, -0.4490002642657902, -0.628696575798281, 0.9660395877041965], "color": "red_7319"},​ + {"id": 2, "vector": [-0.8864122635045097, 0.9260170474445351, 0.801326976181461, 0.6383943392381306, 0.7563037341572827], "color": "white_6465"},​ + {"id": 3, "vector": [0.14594326235891586, -0.3775407299900644, -0.3765479013078812, 0.20612075380355122, 0.4902678929632145], "color": "orange_7580"},​ + {"id": 4, "vector": [0.4548498669607359, -0.887610217681605, 0.5655081329910452, 0.19220509387904117, 0.016513983433433577], "color": "red_3314"},​ + {"id": 5, "vector": [0.11755001847051827, -0.7295149788999611, 0.2608115847524266, -0.1719167007897875, 0.7417611743754855], "color": "black_9955"},​ + {"id": 6, "vector": [0.9363032158314308, 0.030699901477745373, 0.8365910312319647, 0.7823840208444011, 0.2625222076909237], "color": "yellow_2461"},​ + {"id": 7, "vector": [0.0754823906014721, -0.6390658668265143, 0.5610517334334937, -0.8986261118798251, 0.9372056764266794], "color": "white_5015"},​ + {"id": 8, "vector": [-0.3038434006935904, 0.1279149203380523, 0.503958664270957, -0.2622661156746988, 0.7407627307791929], "color": "purple_6414"},​ + {"id": 9, "vector": [-0.7125086947677588, -0.8050968321012257, -0.32608864121785786, 0.3255654958645424, 0.26227968923834233], "color": "brown_7231"}​ +]​ +​ +res = client.upsert(​ + collection_name='quick_setup',​ + data=data​ +)​ +​ +print(res)​ +​ +# Output​ +# {'upsert_count': 10}​ + +``` + +```java +import com.google.gson.Gson;​ +import com.google.gson.JsonObject;​ +import io.milvus.v2.client.ConnectConfig;​ +import io.milvus.v2.client.MilvusClientV2;​ +import io.milvus.v2.service.vector.request.UpsertReq;​ +import io.milvus.v2.service.vector.response.UpsertResp;​ +​ +import java.util.*;​ +​ +MilvusClientV2 client = new MilvusClientV2(ConnectConfig.builder()​ + .uri("http://localhost:19530")​ + .token("root:Milvus")​ + .build());​ +​ +Gson gson = new Gson();​ +List data = Arrays.asList(​ + gson.fromJson("{\"id\": 0, \"vector\": [-0.619954382375778, 0.4479436794798608, -0.17493894838751745, -0.4248030059917294, -0.8648452746018911], \"color\": \"black_9898\"}", JsonObject.class),​ + gson.fromJson("{\"id\": 1, \"vector\": [0.4762662251462588, -0.6942502138717026, -0.4490002642657902, -0.628696575798281, 0.9660395877041965], \"color\": \"red_7319\"}", JsonObject.class),​ + gson.fromJson("{\"id\": 2, \"vector\": [-0.8864122635045097, 0.9260170474445351, 0.801326976181461, 0.6383943392381306, 0.7563037341572827], \"color\": \"white_6465\"}", JsonObject.class),​ + gson.fromJson("{\"id\": 3, \"vector\": [0.14594326235891586, -0.3775407299900644, -0.3765479013078812, 0.20612075380355122, 0.4902678929632145], \"color\": \"orange_7580\"}", JsonObject.class),​ + gson.fromJson("{\"id\": 4, \"vector\": [0.4548498669607359, -0.887610217681605, 0.5655081329910452, 0.19220509387904117, 0.016513983433433577], \"color\": \"red_3314\"}", JsonObject.class),​ + gson.fromJson("{\"id\": 5, \"vector\": [0.11755001847051827, -0.7295149788999611, 0.2608115847524266, -0.1719167007897875, 0.7417611743754855], \"color\": \"black_9955\"}", JsonObject.class),​ + gson.fromJson("{\"id\": 6, \"vector\": [0.9363032158314308, 0.030699901477745373, 0.8365910312319647, 0.7823840208444011, 0.2625222076909237], \"color\": \"yellow_2461\"}", JsonObject.class),​ + gson.fromJson("{\"id\": 7, \"vector\": [0.0754823906014721, -0.6390658668265143, 0.5610517334334937, -0.8986261118798251, 0.9372056764266794], \"color\": \"white_5015\"}", JsonObject.class),​ + gson.fromJson("{\"id\": 8, \"vector\": [-0.3038434006935904, 0.1279149203380523, 0.503958664270957, -0.2622661156746988, 0.7407627307791929], \"color\": \"purple_6414\"}", JsonObject.class),​ + gson.fromJson("{\"id\": 9, \"vector\": [-0.7125086947677588, -0.8050968321012257, -0.32608864121785786, 0.3255654958645424, 0.26227968923834233], \"color\": \"brown_7231\"}", JsonObject.class)​ +);​ +​ +UpsertReq upsertReq = UpsertReq.builder()​ + .collectionName("quick_setup")​ + .data(data)​ + .build();​ +​ +UpsertResp upsertResp = client.upsert(upsertReq);​ +System.out.println(upsertResp);​ +​ +// Output:​ +//​ +// UpsertResp(upsertCnt=10)​ + +``` + +```javascript +const { MilvusClient, DataType } = require("@zilliz/milvus2-sdk-node")​ +​ +const address = "http://localhost:19530";​ +const token = "root:Milvus";​ +const client = new MilvusClient({address, token});​ +​ +data = [​ + {id: 0, vector: [-0.619954382375778, 0.4479436794798608, -0.17493894838751745, -0.4248030059917294, -0.8648452746018911], color: "black_9898"},​ + {id: 1, vector: [0.4762662251462588, -0.6942502138717026, -0.4490002642657902, -0.628696575798281, 0.9660395877041965], color: "red_7319"},​ + {id: 2, vector: [-0.8864122635045097, 0.9260170474445351, 0.801326976181461, 0.6383943392381306, 0.7563037341572827], color: "white_6465"},​ + {id: 3, vector: [0.14594326235891586, -0.3775407299900644, -0.3765479013078812, 0.20612075380355122, 0.4902678929632145], color: "orange_7580"},​ + {id: 4, vector: [0.4548498669607359, -0.887610217681605, 0.5655081329910452, 0.19220509387904117, 0.016513983433433577], color: "red_3314"},​ + {id: 5, vector: [0.11755001847051827, -0.7295149788999611, 0.2608115847524266, -0.1719167007897875, 0.7417611743754855], color: "black_9955"},​ + {id: 6, vector: [0.9363032158314308, 0.030699901477745373, 0.8365910312319647, 0.7823840208444011, 0.2625222076909237], color: "yellow_2461"},​ + {id: 7, vector: [0.0754823906014721, -0.6390658668265143, 0.5610517334334937, -0.8986261118798251, 0.9372056764266794], color: "white_5015"},​ + {id: 8, vector: [-0.3038434006935904, 0.1279149203380523, 0.503958664270957, -0.2622661156746988, 0.7407627307791929], color: "purple_6414"},​ + {id: 9, vector: [-0.7125086947677588, -0.8050968321012257, -0.32608864121785786, 0.3255654958645424, 0.26227968923834233], color: "brown_7231"}​ +]​ +​ +res = await client.upsert({​ + collection_name: "quick_setup",​ + data: data,​ +})​ +​ +console.log(res.upsert_cnt)​ +​ +// Output​ +// ​ +// 10​ +// ​ + +``` + +```curl +export CLUSTER_ENDPOINT="http://localhost:19530"​ +export TOKEN="root:Milvus"​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/upsert" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "data": [​ + {"id": 0, "vector": [0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592], "color": "pink_8682"},​ + {"id": 1, "vector": [0.19886812562848388, 0.06023560599112088, 0.6976963061752597, 0.2614474506242501, 0.838729485096104], "color": "red_7025"},​ + {"id": 2, "vector": [0.43742130801983836, -0.5597502546264526, 0.6457887650909682, 0.7894058910881185, 0.20785793220625592], "color": "orange_6781"},​ + {"id": 3, "vector": [0.3172005263489739, 0.9719044792798428, -0.36981146090600725, -0.4860894583077995, 0.95791889146345], "color": "pink_9298"},​ + {"id": 4, "vector": [0.4452349528804562, -0.8757026943054742, 0.8220779437047674, 0.46406290649483184, 0.30337481143159106], "color": "red_4794"},​ + {"id": 5, "vector": [0.985825131989184, -0.8144651566660419, 0.6299267002202009, 0.1206906911183383, -0.1446277761879955], "color": "yellow_4222"},​ + {"id": 6, "vector": [0.8371977790571115, -0.015764369584852833, -0.31062937026679327, -0.562666951622192, -0.8984947637863987], "color": "red_9392"},​ + {"id": 7, "vector": [-0.33445148015177995, -0.2567135004164067, 0.8987539745369246, 0.9402995886420709, 0.5378064918413052], "color": "grey_8510"},​ + {"id": 8, "vector": [0.39524717779832685, 0.4000257286739164, -0.5890507376891594, -0.8650502298996872, -0.6140360785406336], "color": "white_9381"},​ + {"id": 9, "vector": [0.5718280481994695, 0.24070317428066512, -0.3737913482606834, -0.06726932177492717, -0.6980531615588608], "color": "purple_4976"} ​ + ],​ + "collectionName": "quick_setup"​ +}'​ +​ +# {​ +# "code": 0,​ +# "data": {​ +# "upsertCount": 10,​ +# "upsertIds": [​ +# 0,​ +# 1,​ +# 2,​ +# 3,​ +# 4,​ +# 5,​ +# 6,​ +# 7,​ +# 8,​ +# 9​ +# ]​ +# }​ +# }​ + +``` + + +## Upsert Entities in a Partition​ + +You can also insert entities into a specified partition. The following code snippets assume that you have a partition named **PartitionA** in your collection.​ + +
+ Python + Java + Node.js + Go + cURL +
+ + +```python +data=[​ + {"id": 10, "vector": [0.06998888224297328, 0.8582816610326578, -0.9657938677934292, 0.6527905683627726, -0.8668460657158576], "color": "black_3651"},​ + {"id": 11, "vector": [0.6060703043917468, -0.3765080534566074, -0.7710758854987239, 0.36993888322346136, 0.5507513364206531], "color": "grey_2049"},​ + {"id": 12, "vector": [-0.9041813104515337, -0.9610546012461163, 0.20033003106083358, 0.11842506351635174, 0.8327356724591011], "color": "blue_6168"},​ + {"id": 13, "vector": [0.3202914977909075, -0.7279137773695252, -0.04747830871620273, 0.8266053056909548, 0.8277957187455489], "color": "blue_1672"},​ + {"id": 14, "vector": [0.2975811497890859, 0.2946936202691086, 0.5399463833894609, 0.8385334966677529, -0.4450543984655133], "color": "pink_1601"},​ + {"id": 15, "vector": [-0.04697464305600074, -0.08509022265734134, 0.9067184632552001, -0.2281912685064822, -0.9747503428652762], "color": "yellow_9925"},​ + {"id": 16, "vector": [-0.9363075919673911, -0.8153981031085669, 0.7943039120490902, -0.2093886809842529, 0.0771191335807897], "color": "orange_9872"},​ + {"id": 17, "vector": [-0.050451522820639916, 0.18931572752321935, 0.7522886192190488, -0.9071793089474034, 0.6032647330692296], "color": "red_6450"},​ + {"id": 18, "vector": [-0.9181544231141592, 0.6700755998126806, -0.014174674636136642, 0.6325780463623432, -0.49662222164032976], "color": "purple_7392"},​ + {"id": 19, "vector": [0.11426945899602536, 0.6089190684002581, -0.5842735738352236, 0.057050610092692855, -0.035163433018196244], "color": "pink_4996"}​ +]​ +​ +res = client.upsert(​ + collection_name="quick_setup",​ + data=data,​ + partition_name="partitionA"​ +)​ +​ +print(res)​ +​ +# Output​ +# {'upsert_count': 10}​ + +``` + +```java +import io.milvus.v2.service.vector.request.UpsertReq;​ +import io.milvus.v2.service.vector.response.UpsertResp;​ +​ +​ +Gson gson = new Gson();​ +List data = Arrays.asList(​ + gson.fromJson("{\"id\": 10, \"vector\": [0.06998888224297328, 0.8582816610326578, -0.9657938677934292, 0.6527905683627726, -0.8668460657158576], \"color\": \"black_3651\"}", JsonObject.class),​ + gson.fromJson("{\"id\": 11, \"vector\": [0.6060703043917468, -0.3765080534566074, -0.7710758854987239, 0.36993888322346136, 0.5507513364206531], \"color\": \"grey_2049\"}", JsonObject.class),​ + gson.fromJson("{\"id\": 12, \"vector\": [-0.9041813104515337, -0.9610546012461163, 0.20033003106083358, 0.11842506351635174, 0.8327356724591011], \"color\": \"blue_6168\"}", JsonObject.class),​ + gson.fromJson("{\"id\": 13, \"vector\": [0.3202914977909075, -0.7279137773695252, -0.04747830871620273, 0.8266053056909548, 0.8277957187455489], \"color\": \"blue_1672\"}", JsonObject.class),​ + gson.fromJson("{\"id\": 14, \"vector\": [0.2975811497890859, 0.2946936202691086, 0.5399463833894609, 0.8385334966677529, -0.4450543984655133], \"color\": \"pink_1601\"}", JsonObject.class),​ + gson.fromJson("{\"id\": 15, \"vector\": [-0.04697464305600074, -0.08509022265734134, 0.9067184632552001, -0.2281912685064822, -0.9747503428652762], \"color\": \"yellow_9925\"}", JsonObject.class),​ + gson.fromJson("{\"id\": 16, \"vector\": [-0.9363075919673911, -0.8153981031085669, 0.7943039120490902, -0.2093886809842529, 0.0771191335807897], \"color\": \"orange_9872\"}", JsonObject.class),​ + gson.fromJson("{\"id\": 17, \"vector\": [-0.050451522820639916, 0.18931572752321935, 0.7522886192190488, -0.9071793089474034, 0.6032647330692296], \"color\": \"red_6450\"}", JsonObject.class),​ + gson.fromJson("{\"id\": 18, \"vector\": [-0.9181544231141592, 0.6700755998126806, -0.014174674636136642, 0.6325780463623432, -0.49662222164032976], \"color\": \"purple_7392\"}", JsonObject.class),​ + gson.fromJson("{\"id\": 19, \"vector\": [0.11426945899602536, 0.6089190684002581, -0.5842735738352236, 0.057050610092692855, -0.035163433018196244], \"color\": \"pink_4996\"}", JsonObject.class)​ +);​ +​ +UpsertReq upsertReq = UpsertReq.builder()​ + .collectionName("quick_setup")​ + .partitionName("partitionA")​ + .data(data)​ + .build();​ +​ +UpsertResp upsertResp = client.upsert(upsertReq);​ +System.out.println(upsertResp);​ +​ +// Output:​ +//​ +// UpsertResp(upsertCnt=10)​ + +``` + +```javascript +const { MilvusClient, DataType } = require("@zilliz/milvus2-sdk-node")​ +​ +// 6. Upsert data in partitions​ +data = [​ + {id: 10, vector: [0.06998888224297328, 0.8582816610326578, -0.9657938677934292, 0.6527905683627726, -0.8668460657158576], color: "black_3651"},​ + {id: 11, vector: [0.6060703043917468, -0.3765080534566074, -0.7710758854987239, 0.36993888322346136, 0.5507513364206531], color: "grey_2049"},​ + {id: 12, vector: [-0.9041813104515337, -0.9610546012461163, 0.20033003106083358, 0.11842506351635174, 0.8327356724591011], color: "blue_6168"},​ + {id: 13, vector: [0.3202914977909075, -0.7279137773695252, -0.04747830871620273, 0.8266053056909548, 0.8277957187455489], color: "blue_1672"},​ + {id: 14, vector: [0.2975811497890859, 0.2946936202691086, 0.5399463833894609, 0.8385334966677529, -0.4450543984655133], color: "pink_1601"},​ + {id: 15, vector: [-0.04697464305600074, -0.08509022265734134, 0.9067184632552001, -0.2281912685064822, -0.9747503428652762], color: "yellow_9925"},​ + {id: 16, vector: [-0.9363075919673911, -0.8153981031085669, 0.7943039120490902, -0.2093886809842529, 0.0771191335807897], color: "orange_9872"},​ + {id: 17, vector: [-0.050451522820639916, 0.18931572752321935, 0.7522886192190488, -0.9071793089474034, 0.6032647330692296], color: "red_6450"},​ + {id: 18, vector: [-0.9181544231141592, 0.6700755998126806, -0.014174674636136642, 0.6325780463623432, -0.49662222164032976], color: "purple_7392"},​ + {id: 19, vector: [0.11426945899602536, 0.6089190684002581, -0.5842735738352236, 0.057050610092692855, -0.035163433018196244], color: "pink_4996"}​ +]​ +​ +res = await client.upsert({​ + collection_name: "quick_setup",​ + data: data,​ + partition_name: "partitionA"​ +})​ +​ +console.log(res.upsert_cnt)​ +​ +// Output​ +// ​ +// 10​ +// ​ + +``` + +```curl +export CLUSTER_ENDPOINT="http://localhost:19530"​ +export TOKEN="root:Milvus"​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/upsert" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "data": [​ + {"id": 10, "vector": [0.06998888224297328, 0.8582816610326578, -0.9657938677934292, 0.6527905683627726, -0.8668460657158576], "color": "black_3651"},​ + {"id": 11, "vector": [0.6060703043917468, -0.3765080534566074, -0.7710758854987239, 0.36993888322346136, 0.5507513364206531], "color": "grey_2049"},​ + {"id": 12, "vector": [-0.9041813104515337, -0.9610546012461163, 0.20033003106083358, 0.11842506351635174, 0.8327356724591011], "color": "blue_6168"},​ + {"id": 13, "vector": [0.3202914977909075, -0.7279137773695252, -0.04747830871620273, 0.8266053056909548, 0.8277957187455489], "color": "blue_1672"},​ + {"id": 14, "vector": [0.2975811497890859, 0.2946936202691086, 0.5399463833894609, 0.8385334966677529, -0.4450543984655133], "color": "pink_1601"},​ + {"id": 15, "vector": [-0.04697464305600074, -0.08509022265734134, 0.9067184632552001, -0.2281912685064822, -0.9747503428652762], "color": "yellow_9925"},​ + {"id": 16, "vector": [-0.9363075919673911, -0.8153981031085669, 0.7943039120490902, -0.2093886809842529, 0.0771191335807897], "color": "orange_9872"},​ + {"id": 17, "vector": [-0.050451522820639916, 0.18931572752321935, 0.7522886192190488, -0.9071793089474034, 0.6032647330692296], "color": "red_6450"},​ + {"id": 18, "vector": [-0.9181544231141592, 0.6700755998126806, -0.014174674636136642, 0.6325780463623432, -0.49662222164032976], "color": "purple_7392"},​ + {"id": 19, "vector": [0.11426945899602536, 0.6089190684002581, -0.5842735738352236, 0.057050610092692855, -0.035163433018196244], "color": "pink_4996"}​ + ],​ + "collectionName": "quick_setup"​ +}'​ +​ +# {​ +# "code": 0,​ +# "data": {​ +# "upsertCount": 10,​ +# "upsertIds": [​ +# 0,​ +# 1,​ +# 2,​ +# 3,​ +# 4,​ +# 5,​ +# 6,​ +# 7,​ +# 8,​ +# 9​ +# ]​ +# }​ +# }​ + +``` + + +​ + diff --git a/v2.4.x/site/en/userGuide/insert-update-delete.md b/v2.4.x/site/en/userGuide/insert-update-delete.md deleted file mode 100644 index e3e5c038f8..0000000000 --- a/v2.4.x/site/en/userGuide/insert-update-delete.md +++ /dev/null @@ -1,807 +0,0 @@ ---- -id: insert-update-delete.md -summary: This guide walks you through the data manipulation operations within a collection, including insertion, upsertion, and deletion. -title: Insert, Upsert & Delete ---- - -# Insert, Upsert & Delete - -This guide walks you through the data manipulation operations within a collection, including insertion, upsertion, and deletion. - -## Before you start - -- You have installed the SDK of your choice. To install an SDK, refer to [Install SDKs](https://milvus.io/docs/install-pymilvus.md). - -- You have created a collection. To create a collection, refer to [Manage Collections](manage-collections.md). - -- To insert a large volume of data, you are advised to use [Data Import](https://milvus.io/api-reference/pymilvus/v2.4.x/DataImport/LocalBulkWriter/LocalBulkWriter.md). - -## Overview - -An entity, within the context of Milvus collections, is a singular, identifiable instance within a collection. It represents a distinct member of a particular class, be it a book in a library, a gene in a genome, or any other identifiable entity. - -Entities within a collection share a common set of attributes, termed schema, outlining the structure that each entity must adhere to, including field names, data types, and any other constraints. - -Successful insertion of entities into a collection requires that the provided data should contain all the schema-defined fields of the target collection. Additionally, you can also include non-schema-defined fields only if you have enabled the dynamic field. For details, refer to [Enable Dynamic Field](enable-dynamic-field.md). - -## Preparations - -The code snippet below repurposes the existing code to establish a connection to a Milvus cluster and quickly set up a collection. - -
- -For preparations, use [`MilvusClient`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Client/MilvusClient.md) to connect to the Milvus server and [`create_collection()`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Collections/create_collection.md) to create a collection in a quick-setup mode. - -
- -
- -For preparations, use [`MilvusClientV2`](https://milvus.io/api-reference/java/v2.4.x/v2/Client/MilvusClientV2.md) to connect to the Milvus server and [`createCollection()`](https://milvus.io/api-reference/java/v2.4.x/v2/Collections/createCollection.md) to create a collection in a quick-setup mode. - -
- -
- -For preparations, use [`MilvusClient`](https://milvus.io/api-reference/node/v2.4.x/Client/MilvusClient.md) to connect to the Milvus server and [`createCollection()`](https://milvus.io/api-reference/node/v2.4.x/Collections/createCollection.md) to create a collection in a quick-setup mode. - -
- -
- Python - Java - Node.js -
- -```python -from pymilvus import MilvusClient - -# 1. Set up a Milvus client -client = MilvusClient( - uri="http://localhost:19530" -) - -# 2. Create a collection -client.create_collection( - collection_name="quick_setup", - dimension=5, - metric_type="IP" -) -``` - -```java -import io.milvus.v2.client.ConnectConfig; -import io.milvus.v2.client.MilvusClientV2; -import io.milvus.v2.service.collection.request.CreateCollectionReq; - -String CLUSTER_ENDPOINT = "http://localhost:19530"; - -// 1. Connect to Milvus server -ConnectConfig connectConfig = ConnectConfig.builder() - .uri(CLUSTER_ENDPOINT) - .build(); - -MilvusClientV2 client = new MilvusClientV2(connectConfig); - -// 2. Create a collection in quick setup mode -CreateCollectionReq quickSetupReq = CreateCollectionReq.builder() - .collectionName("quick_setup") - .dimension(5) - .metricType("IP") - .build(); - -client.createCollection(quickSetupReq); -``` - -```javascript -const { MilvusClient, DataType, sleep } = require("@zilliz/milvus2-sdk-node") - -const address = "http://localhost:19530" - -// 1. Set up a Milvus Client -client = new MilvusClient({address}); - -// 2. Create a collection in quick setup mode -await client.createCollection({ - collection_name: "quick_setup", - dimension: 5, - metric_type: "IP" -}); -``` - -
- -

notes

- -

The collection generated in the above code contains only two fields: id (as the primary key) and vector (as the vector field), with auto_id and enable_dynamic_field settings enabled by default. When inserting data,

-
    -

  • You do not need to include id in the data to be inserted, because the primary field automatically increments as data is inserted.

    • -

  • Non-schema-defined fields will be saved as key-value pairs in a reserved JSON field named $meta.

    • -
- -
- -## Insert entities - -To insert entities, you need to organize the data into a list of dictionaries, where each dictionary represents an entity. Each dictionary contains the keys corresponding to both pre-defined and dynamic fields in the target collection. - -
- -To insert entities into a collection, use the [`insert()`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Vector/insert.md) method. - -
- -
- -To insert entities into a collection, use the [`insert()`](https://milvus.io/api-reference/java/v2.4.x/v2/Vector/insert.md) method. - -
- -
- - -To insert entities into a collection, use the [`insert()`](https://milvus.io/api-reference/node/v2.4.x/Vector/insert.md) method. - -
- -
- Python - Java - Node.js -
- -```python -# 3. Insert some data -data=[ - {"id": 0, "vector": [0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592], "color": "pink_8682"}, - {"id": 1, "vector": [0.19886812562848388, 0.06023560599112088, 0.6976963061752597, 0.2614474506242501, 0.838729485096104], "color": "red_7025"}, - {"id": 2, "vector": [0.43742130801983836, -0.5597502546264526, 0.6457887650909682, 0.7894058910881185, 0.20785793220625592], "color": "orange_6781"}, - {"id": 3, "vector": [0.3172005263489739, 0.9719044792798428, -0.36981146090600725, -0.4860894583077995, 0.95791889146345], "color": "pink_9298"}, - {"id": 4, "vector": [0.4452349528804562, -0.8757026943054742, 0.8220779437047674, 0.46406290649483184, 0.30337481143159106], "color": "red_4794"}, - {"id": 5, "vector": [0.985825131989184, -0.8144651566660419, 0.6299267002202009, 0.1206906911183383, -0.1446277761879955], "color": "yellow_4222"}, - {"id": 6, "vector": [0.8371977790571115, -0.015764369584852833, -0.31062937026679327, -0.562666951622192, -0.8984947637863987], "color": "red_9392"}, - {"id": 7, "vector": [-0.33445148015177995, -0.2567135004164067, 0.8987539745369246, 0.9402995886420709, 0.5378064918413052], "color": "grey_8510"}, - {"id": 8, "vector": [0.39524717779832685, 0.4000257286739164, -0.5890507376891594, -0.8650502298996872, -0.6140360785406336], "color": "white_9381"}, - {"id": 9, "vector": [0.5718280481994695, 0.24070317428066512, -0.3737913482606834, -0.06726932177492717, -0.6980531615588608], "color": "purple_4976"} -] - -res = client.insert( - collection_name="quick_setup", - data=data -) - -print(res) - -# Output -# -# { -# "insert_count": 10, -# "ids": [ -# 0, -# 1, -# 2, -# 3, -# 4, -# 5, -# 6, -# 7, -# 8, -# 9 -# ] -# } -``` - -```java -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import io.milvus.v2.service.vector.request.InsertReq; -import io.milvus.v2.service.vector.response.InsertResp; - -// 3. Insert some data -List data = Arrays.asList( - new JSONObject(Map.of("id", 0L, "vector", Arrays.asList(0.3580376395471989f, -0.6023495712049978f, 0.18414012509913835f, -0.26286205330961354f, 0.9029438446296592f), "color", "pink_8682")), - new JSONObject(Map.of("id", 1L, "vector", Arrays.asList(0.19886812562848388f, 0.06023560599112088f, 0.6976963061752597f, 0.2614474506242501f, 0.838729485096104f), "color", "red_7025")), - new JSONObject(Map.of("id", 2L, "vector", Arrays.asList(0.43742130801983836f, -0.5597502546264526f, 0.6457887650909682f, 0.7894058910881185f, 0.20785793220625592f), "color", "orange_6781")), - new JSONObject(Map.of("id", 3L, "vector", Arrays.asList(0.3172005263489739f, 0.9719044792798428f, -0.36981146090600725f, -0.4860894583077995f, 0.95791889146345f), "color", "pink_9298")), - new JSONObject(Map.of("id", 4L, "vector", Arrays.asList(0.4452349528804562f, -0.8757026943054742f, 0.8220779437047674f, 0.46406290649483184f, 0.30337481143159106f), "color", "red_4794")), - new JSONObject(Map.of("id", 5L, "vector", Arrays.asList(0.985825131989184f, -0.8144651566660419f, 0.6299267002202009f, 0.1206906911183383f, -0.1446277761879955f), "color", "yellow_4222")), - new JSONObject(Map.of("id", 6L, "vector", Arrays.asList(0.8371977790571115f, -0.015764369584852833f, -0.31062937026679327f, -0.562666951622192f, -0.8984947637863987f), "color", "red_9392")), - new JSONObject(Map.of("id", 7L, "vector", Arrays.asList(-0.33445148015177995f, -0.2567135004164067f, 0.8987539745369246f, 0.9402995886420709f, 0.5378064918413052f), "color", "grey_8510")), - new JSONObject(Map.of("id", 8L, "vector", Arrays.asList(0.39524717779832685f, 0.4000257286739164f, -0.5890507376891594f, -0.8650502298996872f, -0.6140360785406336f), "color", "white_9381")), - new JSONObject(Map.of("id", 9L, "vector", Arrays.asList(0.5718280481994695f, 0.24070317428066512f, -0.3737913482606834f, -0.06726932177492717f, -0.6980531615588608f), "color", "purple_4976")) -); - -InsertReq insertReq = InsertReq.builder() - .collectionName("quick_setup") - .data(data) - .build(); - -InsertResp insertResp = client.insert(insertReq); - -System.out.println(JSONObject.toJSON(insertResp)); - -// Output: -// {"insertCnt": 10} -``` - -```javascript -// 3. Insert some data - -var data = [ - {id: 0, vector: [0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592], color: "pink_8682"}, - {id: 1, vector: [0.19886812562848388, 0.06023560599112088, 0.6976963061752597, 0.2614474506242501, 0.838729485096104], color: "red_7025"}, - {id: 2, vector: [0.43742130801983836, -0.5597502546264526, 0.6457887650909682, 0.7894058910881185, 0.20785793220625592], color: "orange_6781"}, - {id: 3, vector: [0.3172005263489739, 0.9719044792798428, -0.36981146090600725, -0.4860894583077995, 0.95791889146345], color: "pink_9298"}, - {id: 4, vector: [0.4452349528804562, -0.8757026943054742, 0.8220779437047674, 0.46406290649483184, 0.30337481143159106], color: "red_4794"}, - {id: 5, vector: [0.985825131989184, -0.8144651566660419, 0.6299267002202009, 0.1206906911183383, -0.1446277761879955], color: "yellow_4222"}, - {id: 6, vector: [0.8371977790571115, -0.015764369584852833, -0.31062937026679327, -0.562666951622192, -0.8984947637863987], color: "red_9392"}, - {id: 7, vector: [-0.33445148015177995, -0.2567135004164067, 0.8987539745369246, 0.9402995886420709, 0.5378064918413052], color: "grey_8510"}, - {id: 8, vector: [0.39524717779832685, 0.4000257286739164, -0.5890507376891594, -0.8650502298996872, -0.6140360785406336], color: "white_9381"}, - {id: 9, vector: [0.5718280481994695, 0.24070317428066512, -0.3737913482606834, -0.06726932177492717, -0.6980531615588608], color: "purple_4976"} -] - -var res = await client.insert({ - collection_name: "quick_setup", - data: data, -}) - -console.log(res.insert_cnt) - -// Output -// -// 10 -// -``` - -### Insert into partitions - -To insert data into a specific partition, you can specify the name of the partition in the insert request as follows: - -
- Python - Java - Node.js -
- -```python -# 4. Insert some more data into a specific partition -data=[ - {"id": 10, "vector": [-0.5570353903748935, -0.8997887893201304, -0.7123782431855732, -0.6298990746450119, 0.6699215060604258], "color": "red_1202"}, - {"id": 11, "vector": [0.6319019033373907, 0.6821488267878275, 0.8552303045704168, 0.36929791364943054, -0.14152860714878068], "color": "blue_4150"}, - {"id": 12, "vector": [0.9483947484855766, -0.32294203351925344, 0.9759290319978025, 0.8262982148666174, -0.8351194181285713], "color": "orange_4590"}, - {"id": 13, "vector": [-0.5449109892498731, 0.043511240563786524, -0.25105249484790804, -0.012030655265886425, -0.0010987671273892108], "color": "pink_9619"}, - {"id": 14, "vector": [0.6603339372951424, -0.10866551787442225, -0.9435597754324891, 0.8230244263466688, -0.7986720938400362], "color": "orange_4863"}, - {"id": 15, "vector": [-0.8825129181091456, -0.9204557711667729, -0.935350065513425, 0.5484069690287079, 0.24448151140671204], "color": "orange_7984"}, - {"id": 16, "vector": [0.6285586391568163, 0.5389064528263487, -0.3163366239905099, 0.22036279378888013, 0.15077052220816167], "color": "blue_9010"}, - {"id": 17, "vector": [-0.20151825016059233, -0.905239387635804, 0.6749305353372479, -0.7324272081377843, -0.33007998971889263], "color": "blue_4521"}, - {"id": 18, "vector": [0.2432286610792349, 0.01785636564206139, -0.651356982731391, -0.35848148851027895, -0.7387383128324057], "color": "orange_2529"}, - {"id": 19, "vector": [0.055512329053363674, 0.7100266349039421, 0.4956956543575197, 0.24541352586717702, 0.4209030729923515], "color": "red_9437"} -] - -client.create_partition( - collection_name="quick_setup", - partition_name="partitionA" -) - -res = client.insert( - collection_name="quick_setup", - data=data, - partition_name="partitionA" -) - -print(res) - -# Output -# -# { -# "insert_count": 10, -# "ids": [ -# 10, -# 11, -# 12, -# 13, -# 14, -# 15, -# 16, -# 17, -# 18, -# 19 -# ] -# } -``` - -```java -// 4. Insert some more data into a specific partition -data = Arrays.asList( - new JSONObject(Map.of("id", 10L, "vector", Arrays.asList(-0.5570353903748935f, -0.8997887893201304f, -0.7123782431855732f, -0.6298990746450119f, 0.6699215060604258f), "color", "red_1202")), - new JSONObject(Map.of("id", 11L, "vector", Arrays.asList(0.6319019033373907f, 0.6821488267878275f, 0.8552303045704168f, 0.36929791364943054f, -0.14152860714878068f), "color", "blue_4150")), - new JSONObject(Map.of("id", 12L, "vector", Arrays.asList(0.9483947484855766f, -0.32294203351925344f, 0.9759290319978025f, 0.8262982148666174f, -0.8351194181285713f), "color", "orange_4590")), - new JSONObject(Map.of("id", 13L, "vector", Arrays.asList(-0.5449109892498731f, 0.043511240563786524f, -0.25105249484790804f, -0.012030655265886425f, -0.0010987671273892108f), "color", "pink_9619")), - new JSONObject(Map.of("id", 14L, "vector", Arrays.asList(0.6603339372951424f, -0.10866551787442225f, -0.9435597754324891f, 0.8230244263466688f, -0.7986720938400362f), "color", "orange_4863")), - new JSONObject(Map.of("id", 15L, "vector", Arrays.asList(-0.8825129181091456f, -0.9204557711667729f, -0.935350065513425f, 0.5484069690287079f, 0.24448151140671204f), "color", "orange_7984")), - new JSONObject(Map.of("id", 16L, "vector", Arrays.asList(0.6285586391568163f, 0.5389064528263487f, -0.3163366239905099f, 0.22036279378888013f, 0.15077052220816167f), "color", "blue_9010")), - new JSONObject(Map.of("id", 17L, "vector", Arrays.asList(-0.20151825016059233f, -0.905239387635804f, 0.6749305353372479f, -0.7324272081377843f, -0.33007998971889263f), "color", "blue_4521")), - new JSONObject(Map.of("id", 18L, "vector", Arrays.asList(0.2432286610792349f, 0.01785636564206139f, -0.651356982731391f, -0.35848148851027895f, -0.7387383128324057f), "color", "orange_2529")), - new JSONObject(Map.of("id", 19L, "vector", Arrays.asList(0.055512329053363674f, 0.7100266349039421f, 0.4956956543575197f, 0.24541352586717702f, 0.4209030729923515f), "color", "red_9437")) -); - -CreatePartitionReq createPartitionReq = CreatePartitionReq.builder() - .collectionName("quick_setup") - .partitionName("partitionA") - .build(); - -client.createPartition(createPartitionReq); - -insertReq = InsertReq.builder() - .collectionName("quick_setup") - .data(data) - .partitionName("partitionA") - .build(); - -insertResp = client.insert(insertReq); - -System.out.println(JSONObject.toJSON(insertResp)); - -// Output: -// {"insertCnt": 10} -``` - -```javascript -// 4. Insert some more data into a specific partition -data = [ - {id: 10, vector: [-0.5570353903748935, -0.8997887893201304, -0.7123782431855732, -0.6298990746450119, 0.6699215060604258], color: "red_1202"}, - {id: 11, vector: [0.6319019033373907, 0.6821488267878275, 0.8552303045704168, 0.36929791364943054, -0.14152860714878068], color: "blue_4150"}, - {id: 12, vector: [0.9483947484855766, -0.32294203351925344, 0.9759290319978025, 0.8262982148666174, -0.8351194181285713], color: "orange_4590"}, - {id: 13, vector: [-0.5449109892498731, 0.043511240563786524, -0.25105249484790804, -0.012030655265886425, -0.0010987671273892108], color: "pink_9619"}, - {id: 14, vector: [0.6603339372951424, -0.10866551787442225, -0.9435597754324891, 0.8230244263466688, -0.7986720938400362], color: "orange_4863"}, - {id: 15, vector: [-0.8825129181091456, -0.9204557711667729, -0.935350065513425, 0.5484069690287079, 0.24448151140671204], color: "orange_7984"}, - {id: 16, vector: [0.6285586391568163, 0.5389064528263487, -0.3163366239905099, 0.22036279378888013, 0.15077052220816167], color: "blue_9010"}, - {id: 17, vector: [-0.20151825016059233, -0.905239387635804, 0.6749305353372479, -0.7324272081377843, -0.33007998971889263], color: "blue_4521"}, - {id: 18, vector: [0.2432286610792349, 0.01785636564206139, -0.651356982731391, -0.35848148851027895, -0.7387383128324057], color: "orange_2529"}, - {id: 19, vector: [0.055512329053363674, 0.7100266349039421, 0.4956956543575197, 0.24541352586717702, 0.4209030729923515], color: "red_9437"} -] - -await client.createPartition({ - collection_name: "quick_setup", - partition_name: "partitionA" -}) - -res = await client.insert({ - collection_name: "quick_setup", - data: data, - partition_name: "partitionA" -}) - -console.log(res.insert_cnt) - -// Output -// -// 10 -// -``` - -The output is a dictionary containing the statistics on the affected entities. For details on partition operations, refer to [Manage Partitions](manage-partitions.md). - -## Upsert entities - -Upserting data is a combination of update and insert operations. In Milvus, an upsert operation performs a data-level action to either insert or update an entity based on whether its primary key already exists in a collection. Specifically: - -- If the primary key of the entity already exists in the collection, the existing entity will be overwritten. - -- If the primary key does not exist in the collection, a new entity will be inserted. - -
- -- Upsert operations will not update the primary keys. -- If you plan to use the upsert operation instead of insert for large-scale data ingestion (e.g. millions of vectors), be aware that this can lead to high memory consumption on Milvus data nodes. - -
- -
- -To upsert entities, use the [`upsert()`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Vector/upsert.md) method. - -
- -
- -To upsert entities, use the [`upsert()`](https://milvus.io/api-reference/java/v2.4.x/v2/Vector/insert.md) method. - -
- -
- -To upsert entities, use the [`upsert()`](https://milvus.io/api-reference/node/v2.4.x/Vector/upsert.md) method. - -
- -
- Python - Java - Node.js -
- -```python -# 5. Upsert some data -data=[ - {"id": 0, "vector": [-0.619954382375778, 0.4479436794798608, -0.17493894838751745, -0.4248030059917294, -0.8648452746018911], "color": "black_9898"}, - {"id": 1, "vector": [0.4762662251462588, -0.6942502138717026, -0.4490002642657902, -0.628696575798281, 0.9660395877041965], "color": "red_7319"}, - {"id": 2, "vector": [-0.8864122635045097, 0.9260170474445351, 0.801326976181461, 0.6383943392381306, 0.7563037341572827], "color": "white_6465"}, - {"id": 3, "vector": [0.14594326235891586, -0.3775407299900644, -0.3765479013078812, 0.20612075380355122, 0.4902678929632145], "color": "orange_7580"}, - {"id": 4, "vector": [0.4548498669607359, -0.887610217681605, 0.5655081329910452, 0.19220509387904117, 0.016513983433433577], "color": "red_3314"}, - {"id": 5, "vector": [0.11755001847051827, -0.7295149788999611, 0.2608115847524266, -0.1719167007897875, 0.7417611743754855], "color": "black_9955"}, - {"id": 6, "vector": [0.9363032158314308, 0.030699901477745373, 0.8365910312319647, 0.7823840208444011, 0.2625222076909237], "color": "yellow_2461"}, - {"id": 7, "vector": [0.0754823906014721, -0.6390658668265143, 0.5610517334334937, -0.8986261118798251, 0.9372056764266794], "color": "white_5015"}, - {"id": 8, "vector": [-0.3038434006935904, 0.1279149203380523, 0.503958664270957, -0.2622661156746988, 0.7407627307791929], "color": "purple_6414"}, - {"id": 9, "vector": [-0.7125086947677588, -0.8050968321012257, -0.32608864121785786, 0.3255654958645424, 0.26227968923834233], "color": "brown_7231"} -] - -res = client.upsert( - collection_name='quick_setup', - data=data -) - -print(res) - -# Output -# -# { -# "upsert_count": 10 -# } -``` - -```java -// 5. Upsert some data -data = Arrays.asList( - new JSONObject(Map.of("id", 0L, "vector", Arrays.asList(-0.619954382375778f, 0.4479436794798608f, -0.17493894838751745f, -0.4248030059917294f, -0.8648452746018911f), "color", "black_9898")), - new JSONObject(Map.of("id", 1L, "vector", Arrays.asList(0.4762662251462588f, -0.6942502138717026f, -0.4490002642657902f, -0.628696575798281f, 0.9660395877041965f), "color", "red_7319")), - new JSONObject(Map.of("id", 2L, "vector", Arrays.asList(-0.8864122635045097f, 0.9260170474445351f, 0.801326976181461f, 0.6383943392381306f, 0.7563037341572827f), "color", "white_6465")), - new JSONObject(Map.of("id", 3L, "vector", Arrays.asList(0.14594326235891586f, -0.3775407299900644f, -0.3765479013078812f, 0.20612075380355122f, 0.4902678929632145f), "color", "orange_7580")), - new JSONObject(Map.of("id", 4L, "vector", Arrays.asList(0.4548498669607359f, -0.887610217681605f, 0.5655081329910452f, 0.19220509387904117f, 0.016513983433433577f), "color", "red_3314")), - new JSONObject(Map.of("id", 5L, "vector", Arrays.asList(0.11755001847051827f, -0.7295149788999611f, 0.2608115847524266f, -0.1719167007897875f, 0.7417611743754855f), "color", "black_9955")), - new JSONObject(Map.of("id", 6L, "vector", Arrays.asList(0.9363032158314308f, 0.030699901477745373f, 0.8365910312319647f, 0.7823840208444011f, 0.2625222076909237f), "color", "yellow_2461")), - new JSONObject(Map.of("id", 7L, "vector", Arrays.asList(0.0754823906014721f, -0.6390658668265143f, 0.5610517334334937f, -0.8986261118798251f, 0.9372056764266794f), "color", "white_5015")), - new JSONObject(Map.of("id", 8L, "vector", Arrays.asList(-0.3038434006935904f, 0.1279149203380523f, 0.503958664270957f, -0.2622661156746988f, 0.7407627307791929f), "color", "purple_6414")), - new JSONObject(Map.of("id", 9L, "vector", Arrays.asList(-0.7125086947677588f, -0.8050968321012257f, -0.32608864121785786f, 0.3255654958645424f, 0.26227968923834233f), "color", "brown_7231")) -); - -UpsertReq upsertReq = UpsertReq.builder() - .collectionName("quick_setup") - .data(data) - .build(); - -UpsertResp upsertResp = client.upsert(upsertReq); - -System.out.println(JSONObject.toJSON(upsertResp)); - -// Output: -// {"upsertCnt": 10} -``` - -```javascript -// 5. Upsert some data -data = [ - {id: 0, vector: [-0.619954382375778, 0.4479436794798608, -0.17493894838751745, -0.4248030059917294, -0.8648452746018911], color: "black_9898"}, - {id: 1, vector: [0.4762662251462588, -0.6942502138717026, -0.4490002642657902, -0.628696575798281, 0.9660395877041965], color: "red_7319"}, - {id: 2, vector: [-0.8864122635045097, 0.9260170474445351, 0.801326976181461, 0.6383943392381306, 0.7563037341572827], color: "white_6465"}, - {id: 3, vector: [0.14594326235891586, -0.3775407299900644, -0.3765479013078812, 0.20612075380355122, 0.4902678929632145], color: "orange_7580"}, - {id: 4, vector: [0.4548498669607359, -0.887610217681605, 0.5655081329910452, 0.19220509387904117, 0.016513983433433577], color: "red_3314"}, - {id: 5, vector: [0.11755001847051827, -0.7295149788999611, 0.2608115847524266, -0.1719167007897875, 0.7417611743754855], color: "black_9955"}, - {id: 6, vector: [0.9363032158314308, 0.030699901477745373, 0.8365910312319647, 0.7823840208444011, 0.2625222076909237], color: "yellow_2461"}, - {id: 7, vector: [0.0754823906014721, -0.6390658668265143, 0.5610517334334937, -0.8986261118798251, 0.9372056764266794], color: "white_5015"}, - {id: 8, vector: [-0.3038434006935904, 0.1279149203380523, 0.503958664270957, -0.2622661156746988, 0.7407627307791929], color: "purple_6414"}, - {id: 9, vector: [-0.7125086947677588, -0.8050968321012257, -0.32608864121785786, 0.3255654958645424, 0.26227968923834233], color: "brown_7231"} -] - -res = await client.upsert({ - collection_name: "quick_setup", - data: data, -}) - -console.log(res.upsert_cnt) - -// Output -// -// 10 -// -``` - -### Upsert data in partitions - -To upsert data into a specific partition, you can specify the name of the partition in the insert request as follows: - -
- Python - Java - Node.js -
- -```python -# 6. Upsert data in partitions -data=[ - {"id": 10, "vector": [0.06998888224297328, 0.8582816610326578, -0.9657938677934292, 0.6527905683627726, -0.8668460657158576], "color": "black_3651"}, - {"id": 11, "vector": [0.6060703043917468, -0.3765080534566074, -0.7710758854987239, 0.36993888322346136, 0.5507513364206531], "color": "grey_2049"}, - {"id": 12, "vector": [-0.9041813104515337, -0.9610546012461163, 0.20033003106083358, 0.11842506351635174, 0.8327356724591011], "color": "blue_6168"}, - {"id": 13, "vector": [0.3202914977909075, -0.7279137773695252, -0.04747830871620273, 0.8266053056909548, 0.8277957187455489], "color": "blue_1672"}, - {"id": 14, "vector": [0.2975811497890859, 0.2946936202691086, 0.5399463833894609, 0.8385334966677529, -0.4450543984655133], "color": "pink_1601"}, - {"id": 15, "vector": [-0.04697464305600074, -0.08509022265734134, 0.9067184632552001, -0.2281912685064822, -0.9747503428652762], "color": "yellow_9925"}, - {"id": 16, "vector": [-0.9363075919673911, -0.8153981031085669, 0.7943039120490902, -0.2093886809842529, 0.0771191335807897], "color": "orange_9872"}, - {"id": 17, "vector": [-0.050451522820639916, 0.18931572752321935, 0.7522886192190488, -0.9071793089474034, 0.6032647330692296], "color": "red_6450"}, - {"id": 18, "vector": [-0.9181544231141592, 0.6700755998126806, -0.014174674636136642, 0.6325780463623432, -0.49662222164032976], "color": "purple_7392"}, - {"id": 19, "vector": [0.11426945899602536, 0.6089190684002581, -0.5842735738352236, 0.057050610092692855, -0.035163433018196244], "color": "pink_4996"} -] - -res = client.upsert( - collection_name="quick_setup", - data=data, - partition_name="partitionA" -) - -print(res) - -# Output -# -# { -# "upsert_count": 10 -# } -``` - -```java -import io.milvus.v2.service.vector.request.UpsertReq; -import io.milvus.v2.service.vector.response.UpsertResp; - -// 6. Upsert data in parition - -data = Arrays.asList( - new JSONObject(Map.of("id", 10L, "vector", Arrays.asList(0.06998888224297328f, 0.8582816610326578f, -0.9657938677934292f, 0.6527905683627726f, -0.8668460657158576f), "color", "black_3651")), - new JSONObject(Map.of("id", 11L, "vector", Arrays.asList(0.6060703043917468f, -0.3765080534566074f, -0.7710758854987239f, 0.36993888322346136f, 0.5507513364206531f), "color", "grey_2049")), - new JSONObject(Map.of("id", 12L, "vector", Arrays.asList(-0.9041813104515337f, -0.9610546012461163f, 0.20033003106083358f, 0.11842506351635174f, 0.8327356724591011f), "color", "blue_6168")), - new JSONObject(Map.of("id", 13L, "vector", Arrays.asList(0.3202914977909075f, -0.7279137773695252f, -0.04747830871620273f, 0.8266053056909548f, 0.8277957187455489f), "color", "blue_1672")), - new JSONObject(Map.of("id", 14L, "vector", Arrays.asList(0.2975811497890859f, 0.2946936202691086f, 0.5399463833894609f, 0.8385334966677529f, -0.4450543984655133f), "color", "pink_1601")), - new JSONObject(Map.of("id", 15L, "vector", Arrays.asList(-0.04697464305600074f, -0.08509022265734134f, 0.9067184632552001f, -0.2281912685064822f, -0.9747503428652762f), "color", "yellow_9925")), - new JSONObject(Map.of("id", 16L, "vector", Arrays.asList(-0.9363075919673911f, -0.8153981031085669f, 0.7943039120490902f, -0.2093886809842529f, 0.0771191335807897f), "color", "orange_9872")), - new JSONObject(Map.of("id", 17L, "vector", Arrays.asList(-0.050451522820639916f, 0.18931572752321935f, 0.7522886192190488f, -0.9071793089474034f, 0.6032647330692296f), "color", "red_6450")), - new JSONObject(Map.of("id", 18L, "vector", Arrays.asList(-0.9181544231141592f, 0.6700755998126806f, -0.014174674636136642f, 0.6325780463623432f, -0.49662222164032976f), "color", "purple_7392")), - new JSONObject(Map.of("id", 19L, "vector", Arrays.asList(0.11426945899602536f, 0.6089190684002581f, -0.5842735738352236f, 0.057050610092692855f, -0.035163433018196244f), "color", "pink_4996")) -); - -upsertReq = UpsertReq.builder() - .collectionName("quick_setup") - .data(data) - .partitionName("partitionA") - .build(); - -upsertResp = client.upsert(upsertReq); - -System.out.println(JSONObject.toJSON(upsertResp)); - -// Output: -// {"upsertCnt": 10} -``` - -```javascript -// 6. Upsert data in partitions -data = [ - {id: 10, vector: [0.06998888224297328, 0.8582816610326578, -0.9657938677934292, 0.6527905683627726, -0.8668460657158576], color: "black_3651"}, - {id: 11, vector: [0.6060703043917468, -0.3765080534566074, -0.7710758854987239, 0.36993888322346136, 0.5507513364206531], color: "grey_2049"}, - {id: 12, vector: [-0.9041813104515337, -0.9610546012461163, 0.20033003106083358, 0.11842506351635174, 0.8327356724591011], color: "blue_6168"}, - {id: 13, vector: [0.3202914977909075, -0.7279137773695252, -0.04747830871620273, 0.8266053056909548, 0.8277957187455489], color: "blue_1672"}, - {id: 14, vector: [0.2975811497890859, 0.2946936202691086, 0.5399463833894609, 0.8385334966677529, -0.4450543984655133], color: "pink_1601"}, - {id: 15, vector: [-0.04697464305600074, -0.08509022265734134, 0.9067184632552001, -0.2281912685064822, -0.9747503428652762], color: "yellow_9925"}, - {id: 16, vector: [-0.9363075919673911, -0.8153981031085669, 0.7943039120490902, -0.2093886809842529, 0.0771191335807897], color: "orange_9872"}, - {id: 17, vector: [-0.050451522820639916, 0.18931572752321935, 0.7522886192190488, -0.9071793089474034, 0.6032647330692296], color: "red_6450"}, - {id: 18, vector: [-0.9181544231141592, 0.6700755998126806, -0.014174674636136642, 0.6325780463623432, -0.49662222164032976], color: "purple_7392"}, - {id: 19, vector: [0.11426945899602536, 0.6089190684002581, -0.5842735738352236, 0.057050610092692855, -0.035163433018196244], color: "pink_4996"} -] - -res = await client.upsert({ - collection_name: "quick_setup", - data: data, - partition_name: "partitionA" -}) - -console.log(res.upsert_cnt) - -// Output -// -// 10 -// -``` - -The output is a dictionary containing the statistics on the affected entities. For details on partition operations, refer to [Manage Partitions](manage-partitions.md). - -## Delete entities - -
- -If an entity is no longer needed, you can delete it from the collection by using [`delete()`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Vector/delete.md). - -
- -
- -If an entity is no longer needed, you can delete it from the collection by using [`delete()`](https://milvus.io/api-reference/java/v2.4.x/v2/Vector/delete.md). - -
- -
- -If an entity is no longer needed, you can delete it from the collection by using [`delete()`](https://milvus.io/api-reference/node/v2.4.x/Vector/delete.md). - -
- -Milvus offers two ways for you to identify the entities to delete. - -- __Delete entities by filter.__ - -
- - When using filter expressions to delete entities, ensure the collection has been loaded. Otherwise, Milvus will return an error. - -
- -
- Python - Java - Node.js -
- - ```python - # 7. Delete entities - res = client.delete( - collection_name="quick_setup", - filter="id in [4,5,6]" - ) - - print(res) - - # Output - # - # { - # "delete_count": 3 - # } - ``` - - ```java - import io.milvus.v2.service.vector.request.DeleteReq; - import io.milvus.v2.service.vector.response.DeleteResp; - - - // 7. Delete entities - - DeleteReq deleteReq = DeleteReq.builder() - .collectionName("quick_setup") - .filter("id in [4, 5, 6]") - .build(); - - DeleteResp deleteResp = client.delete(deleteReq); - - System.out.println(JSONObject.toJSON(deleteResp)); - - // Output: - // {"deleteCnt": 3} - ``` - - ```javascript - // 7. Delete entities - res = await client.delete({ - collection_name: "quick_setup", - filter: "id in [4,5,6]" - }) - - console.log(res.delete_cnt) - - // Output - // - // 3 - // - ``` - -- __Delete entities by IDs.__ - - The following snippets demonstrate how to delete entities by IDs from a specific partition. It also works if you leave the partition name unspecified. - -
- Python - Java - Node.js -
- - ```python - res = client.delete( - collection_name="quick_setup", - ids=[18, 19], - partition_name="partitionA" - ) - - print(res) - - # Output - # - # { - # "delete_count": 2 - # } - ``` - - ```java - deleteReq = DeleteReq.builder() - .collectionName("quick_setup") - .ids(Arrays.asList(18L, 19L)) - .partitionName("partitionA") - .build(); - - deleteResp = client.delete(deleteReq); - - System.out.println(JSONObject.toJSON(deleteResp)); - - // Output: - // {"deleteCnt": 2} - ``` - - ```javascript - res = await client.delete({ - collection_name: "quick_setup", - ids: [18, 19], - partition_name: "partitionA" - }) - - console.log(res.delete_cnt) - - // Output - // - // 2 - // - ``` - - For details on how to use filter expressions, refer to [Get & Scalar Query](get-and-scalar-query.md). - -- __Delete entities by partition name__. - - If you want to delete entities from a specific partition, you can specify the partition name with the `partition_name` parameter in the `delete()` method. The following example deletes entities from `partitionA` that have a color starting with `blue`. - -
- Python - Java - Node.js -
- - ```python - res = client.delete( - collection_name='quick_setup', - partition_name='partitionA', - filter='color like "blue%"' - ) - - print("Entities deleted from partitionA: ", res['delete_count']) - - # Output: - # Entities deleted from partitionA: 3 - ``` - - ```java - deleteReq = DeleteReq.builder() - .collectionName("quick_setup") - .filter('color like "blue%"') - .partitionName("partitionA") - .build(); - - deleteResp = client.delete(deleteReq); - - System.out.println(JSONObject.toJSON(deleteResp)); - - // Output: - // {"deleteCnt": 3} - ``` - - ```javascript - res = await client.delete({ - collection_name: "quick_setup", - partition_name: "partitionA", - filter: 'color like "blue%"' - }) - - console.log("Entities deleted from partitionA: " + res.delete_cnt) - - // Output: - // Entities deleted from partitionA: 3 - ``` diff --git a/v2.4.x/site/en/userGuide/manage-collections.md b/v2.4.x/site/en/userGuide/manage-collections.md deleted file mode 100644 index 6278ffb438..0000000000 --- a/v2.4.x/site/en/userGuide/manage-collections.md +++ /dev/null @@ -1,2851 +0,0 @@ ---- -id: manage-collections.md -title: Manage Collections ---- - -# Manage Collections - -This guide walks you through creating and managing collections using the SDK of your choice. - -## Before you start - -- You have installed [Milvus standalone](https://milvus.io/docs/install_standalone-docker.md) or [Milvus cluster](https://milvus.io/docs/install_cluster-milvusoperator.md). - -- You have installed preferred SDKs. You can choose among various languages, including [Python](https://milvus.io/docs/install-pymilvus.md), [Java](https://milvus.io/docs/install-java.md), [Go](https://milvus.io/docs/install-go.md), and [Node.js](https://milvus.io/docs/install-node.md). - -## Overview - -In Milvus, you store your vector embeddings in collections. All vector embeddings within a collection share the same dimensionality and distance metric for measuring similarity. - -Milvus collections support dynamic fields (i.e., fields not pre-defined in the schema) and automatic incrementation of primary keys. - -To accommodate different preferences, Milvus offers two methods for creating a collection. One provides a quick setup, while the other allows for detailed customization of the collection schema and index parameters. - -Additionally, you can view, load, release, and drop a collection when necessary. - -## Create Collection - -You can create a collection in either of the following manners: - -- __Quick setup__ - - In this manner, you can create a collection by simply giving it a name and specifying the number of dimensions of the vector embeddings to be stored in this collection. For details, refer to [Quick setup](manage-collections.md). - -- __Customized setup__ - - Instead of letting In Milvus decide almost everything for your collection, you can determine the __schema__ and __index parameters__ of the collection on your own. For details, refer to [Customized setup](manage-collections.md). - -### Quick setup - -Against the backdrop of the great leap in the AI industry, most developers just need a simple yet dynamic collection to start with. Milvus allows a quick setup of such a collection with just three arguments: - -- Name of the collection to create, - -- Dimension of the vector embeddings to insert, and - -- Metric type used to measure similarities between vector embeddings. - -
- -For quick setup, use the [`create_collection()`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Collections/create_collection.md) method of the [`MilvusClient`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Client/MilvusClient.md) class to create a collection with the specified name and dimension. - -
- -
- -For quick setup, use the [`createCollection()`](https://milvus.io/api-reference/java/v2.4.x/v2/Collections/createCollection.md) method of the [`MilvusClientV2`](https://milvus.io/api-reference/java/v2.4.x/v2/Client/MilvusClientV2.md) class to create a collection with the specified name and dimension. - -
- -
- -For quick setup, use the [`createCollection()`](https://milvus.io/api-reference/node/v2.4.x/Collections/createCollection.md) method of the [`MilvusClient`](https://milvus.io/api-reference/node/v2.4.x/Client/MilvusClient.md) class to create a collection with the specified name and dimension. - -
- -
- -For quick setup, use the [`POST /v2/vectordb/collections/create`](https://milvus.io/api-reference/restful/v2.4.x/v2/Collection%20(v2)/Create.md) API endpoint to create a collection with the specified name and dimension. - -
- -
- Python - Java - Node.js - cURL -
- -```python -from pymilvus import MilvusClient, DataType - -# 1. Set up a Milvus client -client = MilvusClient( - uri="http://localhost:19530" -) - -# 2. Create a collection in quick setup mode -client.create_collection( - collection_name="quick_setup", - dimension=5 -) - -res = client.get_load_state( - collection_name="quick_setup" -) - -print(res) - -# Output -# -# { -# "state": "" -# } -``` - -```java -import io.milvus.v2.client.ConnectConfig; -import io.milvus.v2.client.MilvusClientV2; -import io.milvus.v2.service.collection.request.GetLoadStateReq; -import io.milvus.v2.service.collection.request.CreateCollectionReq; - -String CLUSTER_ENDPOINT = "http://localhost:19530"; - -// 1. Connect to Milvus server -ConnectConfig connectConfig = ConnectConfig.builder() - .uri(CLUSTER_ENDPOINT) - .build(); - -MilvusClientV2 client = new MilvusClientV2(connectConfig); - -// 2. Create a collection in quick setup mode -CreateCollectionReq quickSetupReq = CreateCollectionReq.builder() - .collectionName("quick_setup") - .dimension(5) - .build(); - -client.createCollection(quickSetupReq); - -// Thread.sleep(5000); - -GetLoadStateReq quickSetupLoadStateReq = GetLoadStateReq.builder() - .collectionName("quick_setup") - .build(); - -Boolean res = client.getLoadState(quickSetupLoadStateReq); - -System.out.println(res); - -// Output: -// true -``` - -```javascript -address = "http://localhost:19530" - -// 1. Set up a Milvus Client -client = new MilvusClient({address}); - -// 2. Create a collection in quick setup mode -let res = await client.createCollection({ - collection_name: "quick_setup", - dimension: 5, -}); - -console.log(res.error_code) - -// Output -// -// Success -// - -res = await client.getLoadState({ - collection_name: "quick_setup" -}) - -console.log(res.state) - -// Output -// -// LoadStateLoaded -// -``` - -```shell -$ export MILVUS_URI="localhost:19530" - -$ curl -X POST "http://${MILVUS_URI}/v2/vectordb/collections/create" \ --H "Content-Type: application/json" \ --d '{ - "collectionName": "quick_setup", - "dimension": 5 -}' - -# Output -# -# { -# "code": 0, -# "data": {}, -# } - -$ curl -X POST "http://${MILVUS_URI}/v2/vectordb/collections/get_load_state" \ --H "Content-Type: application/json" \ --d '{ - "collectionName": "quick_setup" -}' - -# { -# "code": 0, -# "data": { -# "loadProgress": 100, -# "loadState": "LoadStateLoaded" -# } -# } -``` - -The collection generated in the above code contains only two fields: `id` (as the primary key) and `vector` (as the vector field), with `auto_id` and `enable_dynamic_field` settings enabled by default. - -- `auto_id` - - Enabling this setting ensures that the primary key increments automatically. There's no need for manual provision of primary keys during data insertion. - -- `enable_dynamic_field` - - When enabled, all fields, excluding `id` and `vector` in the data to be inserted, are treated as dynamic fields. These additional fields are saved as key-value pairs within a special field named `$meta`. This feature allows the inclusion of extra fields during data insertion. - -The automatically indexed and loaded collection from the provided code is ready for immediate data insertions. - -### Customized setup - -Instead of letting Milvus decide almost everything for your collection, you can determine the __schema__ and __index parameters__ of the collection on your own. - -#### Step 1: Set up schema - -A schema defines the structure of a collection. Within the schema, you have the option to enable or disable `enable_dynamic_field`, add pre-defined fields, and set attributes for each field. For a detailed explanation of the concept and available data types, refer to [Schema Explained](schema.md). - -
- -To set up a schema, use [`create_schema()`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Collections/create_schema.md) to create a schema object and [`add_field()`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/CollectionSchema/add_field.md) to add fields to the schema. - -
- -
- -To set up a schema, use [`createSchema()`](https://milvus.io/api-reference/java/v2.4.x/v2/Collections/createSchema.md) to create a schema object and [`addField()`](https://milvus.io/api-reference/java/v2.4.x/v2/CollectionSchema/addField.md) to add fields to the schema. - -
- -
- -To set up a schema, use [`createCollection()`](https://milvus.io/api-reference/node/v2.4.x/Collections/createCollection.md). - -
- -
- -To set up a schema, you need to define a JSON object that follows the schema format as displayed on the [`POST /v2/vectordb/collections/create`](https://milvus.io/api-reference/restful/v2.4.x/v2/Collection%20(v2)/Create.md) API endpoint reference page. - -
- -
- Python - Java - Node.js - cURL -
- -```python -# 3. Create a collection in customized setup mode - -# 3.1. Create schema -schema = MilvusClient.create_schema( - auto_id=False, - enable_dynamic_field=True, -) - -# 3.2. Add fields to schema -schema.add_field(field_name="my_id", datatype=DataType.INT64, is_primary=True) -schema.add_field(field_name="my_vector", datatype=DataType.FLOAT_VECTOR, dim=5) -``` - -```java -import io.milvus.v2.common.DataType; -import io.milvus.v2.service.collection.request.CreateCollectionReq; - -// 3. Create a collection in customized setup mode - -// 3.1 Create schema -CreateCollectionReq.CollectionSchema schema = client.createSchema(); - -// 3.2 Add fields to schema -schema.addField(AddFieldReq.builder() - .fieldName("my_id") - .dataType(DataType.Int64) - .isPrimaryKey(true) - .autoID(false) - .build()); - -schema.addField(AddFieldReq.builder() - .fieldName("my_vector") - .dataType(DataType.FloatVector) - .dimension(5) - .build()); -``` - -```javascript -// 3. Create a collection in customized setup mode -// 3.1 Define fields -const fields = [ - { - name: "my_id", - data_type: DataType.Int64, - is_primary_key: true, - auto_id: false - }, - { - name: "my_vector", - data_type: DataType.FloatVector, - dim: 5 - }, -] -``` - -```shell -export fields='[{ \ - "fieldName": "my_id", \ - "dataType": "Int64", \ - "isPrimary": true \ -}, \ -{ \ - "fieldName": "my_vector", \ - "dataType": "FloatVector", \ - "elementTypeParams": { \ - "dim": 5 \ - } \ -}]' -``` - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
ParameterDescription
auto_idDetermines if the primary field automatically increments.
Setting this to True makes the primary field automatically increment. In this case, the primary field should not be included in the data to insert to avoid errors. The auto-generated IDs have a fixed length and cannot be altered.
enable_dynamic_fieldDetermines if Milvus saves the values of undefined fields in a dynamic field if the data being inserted into the target collection includes fields that are not defined in the collection's schema.
When you set this to True, Milvus will create a field called $meta to store any undefined fields and their values from the data that is inserted.
field_nameThe name of the field.
datatypeThe data type of the field. For a list of available data types, refer to DataType.
is_primaryWhether the current field is the primary field in a collection.
Each collection has only one primary field. A primary field should be of either the DataType.INT64 type or the DataType.VARCHAR type.
dimThe dimension of the vector embeddings.
This is mandatory for a field of the DataType.FLOAT_VECTOR, DataType.BINARY_VECTOR, DataType.FLOAT16_VECTOR, or DataType.BFLOAT16_VECTOR type. If you use DataType.SPARSE_FLOAT_VECTOR, omit this parameter.
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
ParameterDescription
fieldNameThe name of the field.
dataTypeThe data type of the field. For a list of available data types, refer to DataType.
isPrimaryKeyWhether the current field is the primary field in a collection.
Each collection has only one primary field. A primary field should be of either the DataType.Int64 type or the DataType.VarChar type.
autoIDWhether allows the primary field to automatically increment.
Setting this to true makes the primary field automatically increment. In this case, the primary field should not be included in the data to insert to avoid errors.
dimensionThe dimension of the vector embeddings.
This is mandatory for a field of the DataType.FloatVector, DataType.BinaryVector, DataType.Float16Vector, or DataType.BFloat16Vector type.
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
ParameterDescription
nameThe name of the field.
data_typeThe data type of the field. For an enumeration of all available data types, refer to DataType.
is_primary_keyWhether the current field is the primary field in a collection.
Each collection has only one primary field. A primary field should be of either the DataType.INT64 type or the DataType.VARCHAR type.
auto_idWhether the primary field automatically increments upon data insertions into this collection.
The value defaults to False. Setting this to True makes the primary field automatically increment. Skip this parameter if you need to set up a collection with a customized schema.
dimThe dimensionality of the collection field that holds vector embeddings.
The value should be an integer greater than 1 and is usually determined by the model you use to generate vector embeddings.
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
ParameterDescription
fieldNameThe name of the field to create in the target collection.
dataTypeThe data type of the field values.
isPrimaryWhether the current field is the primary field. Setting this to True makes the current field the primary field.
elementTypeParamsExtra field parameters.
dimAn optional parameter for FloatVector or BinaryVector fields that determines the vector dimension.
- -#### Step 2: Set up index parameters - -Index parameters dictate how Milvus organizes your data within a collection. You can tailor the indexing process for specific fields by adjusting their `metric_type` and `index_type`. For the vector field, you have the flexibility to select `COSINE`, `L2`, `IP`, `HAMMING`, or `JACCARD` as the `metric_type`, depending on the type of vectors you are working with. For more information, refer to [Similarity Metrics](metric.md). - -
- -To set up index parameters, use [`prepare_index_params()`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Management/prepare_index_params.md) to prepare index parameters and [`add_index()`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Management/add_index.md) to add the index. - -
- -
- -To set up index parameters, use [IndexParam](https://milvus.io/api-reference/java/v2.4.x/v2/Management/IndexParam.md). - -
- -
- -To set up index parameters, use [`createIndex()`](https://milvus.io/api-reference/node/v2.4.x/Management/createIndex.md). - -
- -
- -To set up index parameters, you need to define a JSON object that follows the index parameters format as displayed on the [`POST /v2/vectordb/collections/create`](https://milvus.io/api-reference/restful/v2.4.x/v2/Collection%20(v2)/Create.md) API endpoint reference page. - -
- -
- Python - Java - Node.js - cURL -
- -```python -# 3.3. Prepare index parameters -index_params = client.prepare_index_params() - -# 3.4. Add indexes -index_params.add_index( - field_name="my_id", - index_type="STL_SORT" -) - -index_params.add_index( - field_name="my_vector", - index_type="IVF_FLAT", - metric_type="IP", - params={ "nlist": 128 } -) -``` - -```java -import io.milvus.v2.common.IndexParam; - -// 3.3 Prepare index parameters -IndexParam indexParamForIdField = IndexParam.builder() - .fieldName("my_id") - .indexType(IndexParam.IndexType.STL_SORT) - .build(); - -IndexParam indexParamForVectorField = IndexParam.builder() - .fieldName("my_vector") - .indexType(IndexParam.IndexType.IVF_FLAT) - .metricType(IndexParam.MetricType.L2) - .extraParams(Map.of("nlist", 1024)) - .build(); - -List indexParams = new ArrayList<>(); -indexParams.add(indexParamForIdField); -indexParams.add(indexParamForVectorField); -``` - -```javascript -// 3.2 Prepare index parameters -const index_params = [{ - field_name: "my_id", - index_type: "STL_SORT" -},{ - field_name: "my_vector", - index_type: "IVF_FLAT", - metric_type: "IP", - params: { nlist: 1024} -}] -``` - -```shell -export indexParams='[{ \ - "fieldName": "my_id", \ - "indexName": "my_id", \ - "params": { \ - "index_type": "SLT_SORT" \ - } \ -}, { \ - "fieldName": "my_vector", \ - "metricType": "COSINE", \ - "indexName": "my_vector", \ - "params": { \ - "index_type": "IVF_FLAT", \ - "nlist": 1024 \ - } \ -}]' -``` - - - - - - - - - - - - - - - - - - - - - - - - - - -
ParameterDescription
field_nameThe name of the target file to apply this object applies.
index_typeThe name of the algorithm used to arrange data in the specific field. For applicable algorithms, refer to In-memory Index and On-disk Index.
metric_typeThe algorithm that is used to measure similarity between vectors. Possible values are IP, L2, COSINE, JACCARD, HAMMING. This is available only when the specified field is a vector field. For more information, refer to Indexes supported in Milvus.
paramsThe fine-tuning parameters for the specified index type. For details on possible keys and value ranges, refer to In-memory Index.
- - - - - - - - - - - - - - - - - - - - - - - - - - -
ParameterDescription
fieldNameThe name of the target field to apply this IndexParam object applies.
indexTypeThe name of the algorithm used to arrange data in the specific field. For applicable algorithms, refer to In-memory Index and On-disk Index.
metricTypeThe distance metric to use for the index. Possible values are IP, L2, COSINE, JACCARD, HAMMING.
extraParamsExtra index parameters. For details, refer to In-memory Index and On-disk Index.
- - - - - - - - - - - - - - - - - - - - - - - - - - -
ParameterDescription
field_nameThe name of the target field on which an index is to be created.
index_typeThe name of the algorithm used to arrange data in the specific field. For applicable algorithms, refer to In-memory Index and On-disk Index.
metric_typeThe algorithm that is used to measure similarity between vectors. Possible values are IP, L2, COSINE, JACCARD, HAMMING. This is available only when the specified field is a vector field. For more information, refer to Indexes supported in Milvus.
paramsThe fine-tuning parameters for the specified index type. For details on possible keys and value ranges, refer to In-memory Index.
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
ParameterDescription
fieldNameThe name of the target field on which an index is to be created.
indexNameThe name of the index to create. The value defaults to the target field name.
metricTypeThe algorithm that is used to measure similarity between vectors. Possible values are IP, L2, COSINE, JACCARD, HAMMING. This is available only when the specified field is a vector field. For more information, refer to Indexes supported in Milvus.
paramsThe index type and related settings. For details, refer to In-memory Index.
params.index_typeThe type of the index to create.
params.nlistThe number of cluster units. This applies to IVF-related index types.
- -The code snippet above demonstrates how to set up index parameters for the vector field and a scalar field, respectively. For the vector field, set both the metric type and the index type. For a scalar field, set only the index type. It is recommended to create an index for the vector field and any scalar fields that are frequently used for filtering. - -#### Step 3: Create the collection - -You have the option to create a collection and an index file separately or to create a collection with the index loaded simultaneously upon creation. - -
- -Use [create_collection()](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Collections/create_collection.md) to create a collection with the specified schema and index parameters and [get_load_state()](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Management/get_load_state.md) to check the load state of the collection. - -
- -
- -Use [createCollection()](https://milvus.io/api-reference/java/v2.4.x/v2/Collections/createCollection.md) to create a collection with the specified schema and index parameters and [getLoadState()](https://milvus.io/api-reference/java/v2.4.x/v2/Management/getLoadState.md) to check the load state of the collection. - -
- -
- -Use [createCollection()](https://milvus.io/api-reference/node/v2.4.x/Collections/createCollection.md) to create a collection with the specified schema and index parameters and [getLoadState()](https://milvus.io/api-reference/node/v2.4.x/Management/getLoadState.md) to check the load state of the collection. - -
- -- __Create a collection with the index loaded simultaneously upon creation.__ - -
- Python - Java - Node.js - cURL -
- - ```python - # 3.5. Create a collection with the index loaded simultaneously - client.create_collection( - collection_name="customized_setup_1", - schema=schema, - index_params=index_params - ) - - time.sleep(5) - - res = client.get_load_state( - collection_name="customized_setup_1" - ) - - print(res) - - # Output - # - # { - # "state": "" - # } - ``` - - ```java - import io.milvus.v2.service.collection.request.CreateCollectionReq; - import io.milvus.v2.service.collection.request.GetLoadStateReq; - - // 3.4 Create a collection with schema and index parameters - CreateCollectionReq customizedSetupReq1 = CreateCollectionReq.builder() - .collectionName("customized_setup_1") - .collectionSchema(schema) - .indexParams(indexParams) - .build(); - - client.createCollection(customizedSetupReq1); - - // Thread.sleep(5000); - - // 3.5 Get load state of the collection - GetLoadStateReq customSetupLoadStateReq1 = GetLoadStateReq.builder() - .collectionName("customized_setup_1") - .build(); - - res = client.getLoadState(customSetupLoadStateReq1); - - System.out.println(res); - - // Output: - // true - ``` - - ```javascript - // 3.3 Create a collection with fields and index parameters - res = await client.createCollection({ - collection_name: "customized_setup_1", - fields: fields, - index_params: index_params, - }) - - console.log(res.error_code) - - // Output - // - // Success - // - - res = await client.getLoadState({ - collection_name: "customized_setup_1" - }) - - console.log(res.state) - - // Output - // - // LoadStateLoaded - // - ``` - - ```shell - $ curl -X POST "http://${MILVUS_URI}/v2/vectordb/collections/create" \ - -H "Content-Type: application/json" \ - -d '{ - "collectionName": "customized_setup_1", - "schema": { - "autoId": false, - "enabledDynamicField": false, - "fields": [ - { - "fieldName": "my_id", - "dataType": "Int64", - "isPrimary": true - }, - { - "fieldName": "my_vector", - "dataType": "FloatVector", - "elementTypeParams": { - "dim": "5" - } - } - ] - }, - "indexParams": [ - { - "fieldName": "my_vector", - "metricType": "COSINE", - "indexName": "my_vector", - "params": { - "index_type": "IVF_FLAT", - "nlist": "1024" - } - }, - { - "fieldName": "my_id", - "indexName": "my_id", - "params": { - "index_type": "STL_SORT" - } - } - ] - }' - - # Output - # - # { - # "code": 0, - # "data": {}, - # } - - $ curl -X POST "http://${MILVUS_URI}/v2/vectordb/collections/get_load_state" \ - -H "Content-Type: application/json" \ - -d '{ - "collectionName": "customized_setup_1" - }' - - # { - # "code": 0, - # "data": { - # "loadProgress": 100, - # "loadState": "LoadStateLoaded" - # } - # } - ``` - - The collection created above is loaded automatically. To learn more about loading and releasing a collection, refer to [Load & Release Collection](manage-collections.md#Load--Release-Collection). - -- __Create a collection and an index file separately.__ - -
- Python - Java - Node.js - cURL -
- - ```python - # 3.6. Create a collection and index it separately - client.create_collection( - collection_name="customized_setup_2", - schema=schema, - ) - - res = client.get_load_state( - collection_name="customized_setup_2" - ) - - print(res) - - # Output - # - # { - # "state": "" - # } - ``` - - ```java - // 3.6 Create a collection and index it separately - CreateCollectionReq customizedSetupReq2 = CreateCollectionReq.builder() - .collectionName("customized_setup_2") - .collectionSchema(schema) - .build(); - - client.createCollection(customizedSetupReq2); - ``` - - ```javascript - // 3.4 Create a collection and index it seperately - res = await client.createCollection({ - collection_name: "customized_setup_2", - fields: fields, - }) - - console.log(res.error_code) - - // Output - // - // Success - // - - res = await client.getLoadState({ - collection_name: "customized_setup_2" - }) - - console.log(res.state) - - // Output - // - // LoadStateNotLoad - // - ``` - - ```shell - $ curl -X POST "http://${MILVUS_URI}/v2/vectordb/collections/create" \ - -H "Content-Type: application/json" \ - -d '{ - "collectionName": "customized_setup_2", - "schema": { - "autoId": false, - "enabledDynamicField": false, - "fields": [ - { - "fieldName": "my_id", - "dataType": "Int64", - "isPrimary": true - }, - { - "fieldName": "my_vector", - "dataType": "FloatVector", - "elementTypeParams": { - "dim": "5" - } - } - ] - - } - }' - - # Output - # - # { - # "code": 0, - # "data": {}, - # } - - $ curl -X POST "http://${MILVUS_URI}/v2/vectordb/collections/get_load_state" \ - -H "Content-Type: application/json" \ - -d '{ - "collectionName": "customized_setup_2" - }' - - # { - # "code": 0, - # "data": { - # "loadState": "LoadStateNotLoaded" - # } - # } - ``` - - The collection created above is not loaded automatically. You can create an index for the collection as follows. Creating an index for the collection in a separate manner does not automatically load the collection. For details, refer to [Load & Release Collection](manage-collections.md#Load--Release-Collection). - - - - - - - - - - - - - - - - - - - - - - -
ParameterDescription
collection_nameThe name of the collection.
schemaThe schema of this collection.
Setting this to None indicates this collection will be created with default settings.
To set up a collection with a customized schema, you need to create a CollectionSchema object and reference it here. In this case, Milvus ignores all other schema-related settings carried in the request.
index_paramsThe parameters for building the index on the vector field in this collection. To set up a collection with a customized schema and automatically load the collection to memory, you need to create an IndexParams object and reference it here.
You should at least add an index for the vector field in this collection. You can also skip this parameter if you prefer to set up the index parameters later on.
- - - - - - - - - - - - - - - - - - - - - - -
ParameterDescription
collectionNameThe name of the collection.
collectionSchemaThe schema of this collection.
Leaving it empty indicates this collection will be created with default settings. To set up a collection with a customized schema, you need to create a CollectionSchema object and reference it here.
indexParamsThe parameters for building the index on the vector field in this collection. To set up a collection with a customized schema and automatically load the collection to memory, create an IndexParams object with a list of IndexParam objects and reference it here.
- - - - - - - - - - - - - - - - - - - - - - -
ParameterDescription
collection_nameThe name of the collection.
fieldsThe fields in the collection.
index_paramsThe index parameters for the collection to create.
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
ParameterDescription
collectionNameThe name of the collection.
schemaThe schema is responsible for organizing data in the target collection. A valid schema should have multiple fields, which must include a primary key, a vector field, and several scalar fields.
schema.autoIDWhether allows the primary field to automatically increment. Setting this to True makes the primary field automatically increment. In this case, the primary field should not be included in the data to insert to avoid errors. Set this parameter in the field with is_primary set to True.
schema.enableDynamicFieldWhether allows to use the reserved $meta field to hold non-schema-defined fields in key-value pairs.
fieldsA list of field objects.
fields.fieldNameThe name of the field to create in the target collection.
fields.dataTypeThe data type of the field values.
fields.isPrimaryWhether the current field is the primary field. Setting this to True makes the current field the primary field.
fields.elementTypeParamsExtra field parameters.
fields.elementTypeParams.dimAn optional parameter for FloatVector or BinaryVector fields that determines the vector dimension.
- - The collection created above is not loaded automatically. You can create an index for the collection as follows. Creating an index for the collection in a separate manner does not automatically load the collection. For details, refer to [Load & Release Collection](manage-collections.md). - -
- Python - Java - Node.js - cURL -
- - ```python - # 3.6 Create index - client.create_index( - collection_name="customized_setup_2", - index_params=index_params - ) - - res = client.get_load_state( - collection_name="customized_setup_2" - ) - - print(res) - - # Output - # - # { - # "state": "" - # } - ``` - - ```java - CreateIndexReq createIndexReq = CreateIndexReq.builder() - .collectionName("customized_setup_2") - .indexParams(indexParams) - .build(); - - client.createIndex(createIndexReq); - - // Thread.sleep(1000); - - // 3.7 Get load state of the collection - GetLoadStateReq customSetupLoadStateReq2 = GetLoadStateReq.builder() - .collectionName("customized_setup_2") - .build(); - - res = client.getLoadState(customSetupLoadStateReq2); - - System.out.println(res); - - // Output: - // false - ``` - - ```javascript - // 3.5 Create index - res = await client.createIndex({ - collection_name: "customized_setup_2", - field_name: "my_vector", - index_type: "IVF_FLAT", - metric_type: "IP", - params: { nlist: 1024} - }) - - res = await client.getLoadState({ - collection_name: "customized_setup_2" - }) - - console.log(res.state) - - // Output - // - // LoadStateNotLoad - // - ``` - - ```shell - $ curl -X POST "http://${MILVUS_URI}/v2/vectordb/indexes/create" \ - -H "Content-Type: application/json" \ - -d '{ - "collectionName": "customized_setup_2", - "indexParams": [ - { - "metricType": "L2", - "fieldName": "my_vector", - "indexName": "my_vector", - "indexConfig": { - "index_type": "IVF_FLAT", - "nlist": "1024" - } - } - ] - }' - - # Output - # - # { - # "code": 0, - # "data": {}, - # } - - $ curl -X POST "http://${MILVUS_URI}/v2/vectordb/collections/get_load_state" \ - -H "Content-Type: application/json" \ - -d '{ - "collectionName": "customized_setup_2" - }' - - # { - # "code": 0, - # "data": { - # "loadState": "LoadStateNotLoaded" - # } - # } - ``` - - - - - - - - - - - - - - - - - - -
ParameterDescription
collection_nameThe name of the collection.
index_paramsAn IndexParams object containing a list of IndexParam objects.
- - - - - - - - - - - - - - - - - - -
ParameterDescription
collectionNameThe name of the collection.
indexParamsA list of IndexParam objects.
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
ParameterDescription
collection_nameThe name of the collection.
field_nameThe name of the field in which to create an index.
index_typeThe name of the algorithm used to arrange data in the specific field. For applicable algorithms, refer to In-memory Index and On-disk Index.
metric_typeThe algorithm that is used to measure similarity between vectors. Possible values are IP, L2, COSINE, JACCARD, HAMMING. This is available only when the specified field is a vector field. For more information, refer to Indexes supported in Milvus.
paramsThe fine-tuning parameters for the specified index type. For details on possible keys and value ranges, refer to In-memory Index.
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
ParameterDescription
collectionNameThe name of the collection.
indexParamsThe index parameters for the collection to create.
indexParams.metricTypeThe similarity metric type used to build the index. The value defaults to COSINE.
indexParams.fieldNameThe name of the target field on which an index is to be created.
indexParams.indexNameThe name of the index to create, the value defaults to the target field name.
indexParams.indexConfig.index_typeThe type of the index to create.
indexParams.indexConfig.nlistThe number of cluster units. This applies to IVF-related index types.
- -## View Collections - -
- -To check the details of an existing collection, use [describe_collection()](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Collections/describe_collection.md). - -
- -
- -To check the details of an existing collection, use [describeCollection()](https://milvus.io/api-reference/java/v2.4.x/v2/Collections/describeCollection.md). - -
- -
- -To check the details of an existing collection, use [describeCollection()](https://milvus.io/api-reference/node/v2.4.x/Collections/describeCollection.md). - -
- -
- -To view the definition of a collection, you can use the [`POST /v2/vectordb/collections/describe`](https://milvus.io/api-reference/restful/v2.4.x/v2/Collection%20(v2)/Describe.md) and the [`POST /v2/vectordb/collections/list`](https://milvus.io/api-reference/restful/v2.4.x/v2/Collection%20(v2)/List.md) API endpoints. - -
- -
- Python - Java - Node.js - cURL -
- -```python -# 5. View Collections -res = client.describe_collection( - collection_name="customized_setup_2" -) - -print(res) - -# Output -# -# { -# "collection_name": "customized_setup_2", -# "auto_id": false, -# "num_shards": 1, -# "description": "", -# "fields": [ -# { -# "field_id": 100, -# "name": "my_id", -# "description": "", -# "type": 5, -# "params": {}, -# "element_type": 0, -# "is_primary": true -# }, -# { -# "field_id": 101, -# "name": "my_vector", -# "description": "", -# "type": 101, -# "params": { -# "dim": 5 -# }, -# "element_type": 0 -# } -# ], -# "aliases": [], -# "collection_id": 448143479230158446, -# "consistency_level": 2, -# "properties": {}, -# "num_partitions": 1, -# "enable_dynamic_field": true -# } - -``` - -```java -import io.milvus.v2.service.collection.request.DescribeCollectionReq; -import io.milvus.v2.service.collection.response.DescribeCollectionResp; - -// 4. View collections -DescribeCollectionReq describeCollectionReq = DescribeCollectionReq.builder() - .collectionName("customized_setup_2") - .build(); - -DescribeCollectionResp describeCollectionRes = client.describeCollection(describeCollectionReq); - -System.out.println(JSONObject.toJSON(describeCollectionRes)); - -// Output: -// { -// "createTime": 449005822816026627, -// "collectionSchema": {"fieldSchemaList": [ -// { -// "autoID": false, -// "dataType": "Int64", -// "name": "my_id", -// "description": "", -// "isPrimaryKey": true, -// "maxLength": 65535, -// "isPartitionKey": false -// }, -// { -// "autoID": false, -// "dataType": "FloatVector", -// "name": "my_vector", -// "description": "", -// "isPrimaryKey": false, -// "dimension": 5, -// "maxLength": 65535, -// "isPartitionKey": false -// } -// ]}, -// "vectorFieldName": ["my_vector"], -// "autoID": false, -// "fieldNames": [ -// "my_id", -// "my_vector" -// ], -// "description": "", -// "numOfPartitions": 1, -// "primaryFieldName": "my_id", -// "enableDynamicField": true, -// "collectionName": "customized_setup_2" -// } -``` - -```javascript -// 5. View Collections -res = await client.describeCollection({ - collection_name: "customized_setup_2" -}) - -console.log(res) - -// Output -// -// { -// virtual_channel_names: [ 'by-dev-rootcoord-dml_13_449007919953017716v0' ], -// physical_channel_names: [ 'by-dev-rootcoord-dml_13' ], -// aliases: [], -// start_positions: [], -// properties: [], -// status: { -// extra_info: {}, -// error_code: 'Success', -// reason: '', -// code: 0, -// retriable: false, -// detail: '' -// }, -// schema: { -// fields: [ [Object], [Object] ], -// properties: [], -// name: 'customized_setup_2', -// description: '', -// autoID: false, -// enable_dynamic_field: false -// }, -// collectionID: '449007919953017716', -// created_timestamp: '449024569603784707', -// created_utc_timestamp: '1712892797866', -// shards_num: 1, -// consistency_level: 'Bounded', -// collection_name: 'customized_setup_2', -// db_name: 'default', -// num_partitions: '1' -// } -// -``` - -```shell -curl -X POST "http://${MILVUS_URI}/v2/vectordb/collections/describe" \ --H "Content-Type: application/json" \ --d '{ - "dbName": "default", - "collectionName": "test_collection" -}' - -# { -# "code": 0, -# "data": { -# "aliases": [], -# "autoId": false, -# "collectionID": 448707763883002014, -# "collectionName": "test_collection", -# "consistencyLevel": "Bounded", -# "description": "", -# "enableDynamicField": true, -# "fields": [ -# { -# "autoId": false, -# "description": "", -# "id": 100, -# "name": "id", -# "partitionKey": false, -# "primaryKey": true, -# "type": "Int64" -# }, -# { -# "autoId": false, -# "description": "", -# "id": 101, -# "name": "vector", -# "params": [ -# { -# "key": "dim", -# "value": "5" -# } -# ], -# "partitionKey": false, -# "primaryKey": false, -# "type": "FloatVector" -# } -# ], -# "indexes": [ -# { -# "fieldName": "vector", -# "indexName": "vector", -# "metricType": "COSINE" -# } -# ], -# "load": "LoadStateLoaded", -# "partitionsNum": 1, -# "properties": [], -# "shardsNum": 1 -# } -# } -``` - -To list all existing collections, you can do as follows: - -
- Python - Java - Node.js - cURL -
- -```python -# 6. List all collection names -res = client.list_collections() - -print(res) - -# Output -# -# [ -# "customized_setup_2", -# "quick_setup", -# "customized_setup_1" -# ] -``` - -```java -import io.milvus.v2.service.collection.response.ListCollectionsResp; - -// 5. List all collection names -ListCollectionsResp listCollectionsRes = client.listCollections(); - -System.out.println(listCollectionsRes.getCollectionNames()); - -// Output: -// [ -// "customized_setup_2", -// "quick_setup", -// "customized_setup_1" -// ] -``` - -```javascript -// 5. List all collection names -ListCollectionsResp listCollectionsRes = client.listCollections(); - -System.out.println(listCollectionsRes.getCollectionNames()); - -// Output: -// [ -// "customized_setup_1", -// "quick_setup", -// "customized_setup_2" -// ] -``` - -```shell -$ curl -X POST "http://${MILVUS_URI}/v2/vectordb/collections/list" \ --H "Content-Type: application/json" \ --d '{ - "dbName": "default" -}' - -# { -# "code": 0, -# "data": [ -# "quick_setup", -# "customized_setup_1", -# "customized_setup_2" -# ] -# } -``` - -## Load & Release Collection - -During the loading process of a collection, Milvus loads the collection's index file into memory. Conversely, when releasing a collection, Milvus unloads the index file from memory. Before conducting searches in a collection, ensure that the collection is loaded. - -### Load a collection - -
- -To load a collection, use the [`load_collection()`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Management/load_collection.md) method, specifying the collection name. You can also set `replica_number` to determine how many in-memory replicas of data segments to create on query nodes when the collection is loaded. - -- Milvus Standalone: The maximum allowed value for `replica_number` is 1. -- Milvus Cluster: The maximum value should not exceed the `queryNode.replicas` set in your Milvus configurations. For additional details, refer to [Query Node-related Configurations](https://milvus.io/docs/configure_querynode.md#Query-Node-related-Configurations). - -
- -
- -To load a collection, use the [`loadCollection()`](https://milvus.io/api-reference/java/v2.4.x/v2/Management/loadCollection.md) method, specifying the collection name. - -
- -
- -To load a collection, use the [`loadCollection()`](https://milvus.io/api-reference/node/v2.4.x/Management/loadCollection.md) method, specifying the collection name. - -
- -
- -To load a collection, you can use the [`POST /v2/vectordb/collections/load`](https://milvus.io/api-reference/restful/v2.4.x/v2/Collection%20(v2)/Load.md) and the [`POST /v2/vectordb/collections/get_load_state`](https://milvus.io/api-reference/restful/v2.4.x/v2/Collection%20(v2)/GetLoadState.md) API endpoints. - -
- -
- Python - Java - Node.js - cURL -
- -```python -# 7. Load the collection -client.load_collection( - collection_name="customized_setup_2", - replica_number=1 # Number of replicas to create on query nodes. Max value is 1 for Milvus Standalone, and no greater than `queryNode.replicas` for Milvus Cluster. -) - -res = client.get_load_state( - collection_name="customized_setup_2" -) - -print(res) - -# Output -# -# { -# "state": "" -# } -``` - -```java -import io.milvus.v2.service.collection.request.LoadCollectionReq; - -// 6. Load the collection -LoadCollectionReq loadCollectionReq = LoadCollectionReq.builder() - .collectionName("customized_setup_2") - .build(); - -client.loadCollection(loadCollectionReq); - -// Thread.sleep(5000); - -// 7. Get load state of the collection -GetLoadStateReq loadStateReq = GetLoadStateReq.builder() - .collectionName("customized_setup_2") - .build(); - -res = client.getLoadState(loadStateReq); - -System.out.println(res); - -// Output: -// true -``` - -```javascript -// 7. Load the collection -res = await client.loadCollection({ - collection_name: "customized_setup_2" -}) - -console.log(res.error_code) - -// Output -// -// Success -// - -await sleep(3000) - -res = await client.getLoadState({ - collection_name: "customized_setup_2" -}) - -console.log(res.state) - -// Output -// -// LoadStateLoaded -// -``` - -```shell -$ curl -X POST "http://${MILVUS_URI}/v2/vectordb/collections/load" \ --H "Content-Type: application/json" \ --d '{ - "collectionName": "customized_setup_2" -}' - -# Output -# -# { -# "code": 0, -# "data": {}, -# } - -$ curl -X POST "http://${MILVUS_URI}/v2/vectordb/collections/get_load_state" \ --H "Content-Type: application/json" \ --d '{ - "collectionName": "customized_setup_2" -}' - -# { -# "code": 0, -# "data": { -# "loadProgress": 100, -# "loadState": "LoadStateLoaded" -# } -# } -``` - -### Load a collection partially (Public Preview) - -
- -This feature is currently in public preview. The API and functionality may change in the future. - -
- -Upon receiving your load request, Milvus loads all vector field indexes and all scalar field data into memory. If some fields are not to be involved in searches and queries, you can exclude them from loading to reduce memory usage, improving search performance. - -
- -```python -# 7. Load the collection -client.load_collection( - collection_name="customized_setup_2", - load_fields=["my_id", "my_vector"], # Load only the specified fields - skip_load_dynamic_field=True # Skip loading the dynamic field -) - -res = client.get_load_state( - collection_name="customized_setup_2" -) - -print(res) - -# Output -# -# { -# "state": "" -# } -``` - -Note that only the fields listed in `load_fields` can be used as filtering conditions and output fields in searches and queries. You should always include the primary key in the list. The field names excluded from loading will not be available for filtering or output. - -You can use `skip_load_dynamic_field=True` to skip loading the dynamic field. Milvus treats the dynamic field as a single field, so all the keys in the dynamic field will be included or excluded together. - -
- -### Release a collection - -
- -To release a collection, use the [`release_collection()`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Management/release_collection.md) method, specifying the collection name. - -
- -
- -To release a collection, use the [`releaseCollection()`](https://milvus.io/api-reference/java/v2.4.x/v2/Management/releaseCollection.md) method, specifying the collection name. - -
- -
- -To release a collection, use the [`releaseCollection()`](https://milvus.io/api-reference/node/v2.4.x/Management/releaseCollection.md) method, specifying the collection name. - -
- -
- -To release a collection, you can use the [`POST /v2/vectordb/collections/release`](https://milvus.io/api-reference/restful/v2.4.x/v2/Collection%20(v2)/Release.md) and the [`POST /v2/vectordb/collections/get_load_state`](https://milvus.io/api-reference/restful/v2.4.x/v2/Collection%20(v2)/GetLoadState.md) API endpoints. - -
- -
- Python - Java - Node.js - cURL -
- -```python -# 8. Release the collection -client.release_collection( - collection_name="customized_setup_2" -) - -res = client.get_load_state( - collection_name="customized_setup_2" -) - -print(res) - -# Output -# -# { -# "state": "" -# } -``` - -```java -import io.milvus.v2.service.collection.request.ReleaseCollectionReq; - -// 8. Release the collection -ReleaseCollectionReq releaseCollectionReq = ReleaseCollectionReq.builder() - .collectionName("customized_setup_2") - .build(); - -client.releaseCollection(releaseCollectionReq); - -// Thread.sleep(1000); - -res = client.getLoadState(loadStateReq); - -System.out.println(res); - -// Output: -// false -``` - -```javascript -// 8. Release the collection -res = await client.releaseCollection({ - collection_name: "customized_setup_2" -}) - -console.log(res.error_code) - -// Output -// -// Success -// - -res = await client.getLoadState({ - collection_name: "customized_setup_2" -}) - -console.log(res.state) - -// Output -// -// LoadStateNotLoad -// -``` - -```shell -$ curl -X POST "http://${MILVUS_URI}/v2/vectordb/collections/release" \ --H "Content-Type: application/json" \ --d '{ - "collectionName": "customized_setup_2" -}' - -# Output -# -# { -# "code": 0, -# "data": {}, -# } - - -$ curl -X POST "http://${MILVUS_URI}/v2/vectordb/collections/get_load_state" \ --H "Content-Type: application/json" \ --d '{ - "collectionName": "customized_setup_2" -}' - - -# { -# "code": 0, -# "data": { -# "loadState": "LoadStateNotLoad" -# } -# } -``` - -## Set up aliases - -You can assign aliases for collections to make them more meaningful in a specific context. You can assign multiple aliases for a collection, but multiple collections cannot share an alias. - -### Create aliases - -
- -To create aliases, use the [`create_alias()`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Collections/create_alias.md) method, specifying the collection name and the alias. - -
- -
- -To create aliases, use the [`createAlias()`](https://milvus.io/api-reference/java/v2.4.x/v2/Collections/createAlias.md) method, specifying the collection name and the alias. - -
- -
- -To create aliases, use the [`createAlias()`](https://milvus.io/api-reference/node/v2.4.x/Collections/createAlias.md) method, specifying the collection name and the alias. - -
- -
- -To create aliases for a collection, you can use the [`POST /v2/vectordb/aliases/create`](https://milvus.io/api-reference/restful/v2.4.x/v2/Alias%20(v2)/Create.md) API endpoint. - -
- -
- Python - Java - Node.js - cURL -
- -```python -# 9.1. Create aliases -client.create_alias( - collection_name="customized_setup_2", - alias="bob" -) - -client.create_alias( - collection_name="customized_setup_2", - alias="alice" -) -``` - -```java -import io.milvus.v2.service.utility.request.CreateAliasReq; - -// 9. Manage aliases - -// 9.1 Create alias -CreateAliasReq createAliasReq = CreateAliasReq.builder() - .collectionName("customized_setup_2") - .alias("bob") - .build(); - -client.createAlias(createAliasReq); - -createAliasReq = CreateAliasReq.builder() - .collectionName("customized_setup_2") - .alias("alice") - .build(); - -client.createAlias(createAliasReq); -``` - -```javascript -// 9. Manage aliases -// 9.1 Create aliases -res = await client.createAlias({ - collection_name: "customized_setup_2", - alias: "bob" -}) - -console.log(res.error_code) - -// Output -// -// Success -// - -res = await client.createAlias({ - collection_name: "customized_setup_2", - alias: "alice" -}) - -console.log(res.error_code) - -// Output -// -// Success -// -``` - -```shell -$ curl -X POST "http://${MILVUS_URI}/v2/vectordb/aliases/create" \ --H "Content-Type: application/json" \ --d '{ - "collectionName": "customized_setup_2", - "aliasName": "bob" -}' - -# Output -# -# { -# "code": 0, -# "data": {} -# } - -$ curl -X POST "http://${MILVUS_URI}/v2/vectordb/aliases/create" \ --H "Content-Type: application/json" \ --d '{ - "collectionName": "customized_setup_2", - "aliasName": "alice" -}' - -# Output -# -# { -# "code": 0, -# "data": {} -# } -``` - - - - - - - - - - - - - - - - - - -
ParameterDescription
collection_nameThe name of the collection to create an alias for.
aliasThe alias of the collection. Before this operation, ensure that the alias does not already exist. If it does, exceptions will occur.
- - - - - - - - - - - - - - - - - - -
ParameterDescription
collectionNameThe name of the collection to create an alias for.
aliasThe alias of the collection. Before this operation, ensure that the alias does not already exist. If it does, exceptions will occur.
- - - - - - - - - - - - - - - - - - -
ParameterDescription
collection_nameThe name of the collection to create an alias for.
aliasThe alias of the collection. Before this operation, ensure that the alias does not already exist. If it does, exceptions will occur.
- - - - - - - - - - - - - - - - - - -
ParameterDescription
collectionNameThe name of the collection to create an alias for.
aliasNameThe alias of the collection. Before this operation, ensure that the alias does not already exist. If it does, exceptions will occur.
- -### List aliases - -
- -To list aliases, use the [`list_aliases()`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Collections/list_aliases.md) method, specifying the collection name. - -
- -
- -To list aliases, use the [`listAliases()`](https://milvus.io/api-reference/java/v2.4.x/v2/Collections/listAliases.md) method, specifying the collection name. - -
- -
- -To list aliases, use the [`listAliases()`](https://milvus.io/api-reference/node/v2.4.x/Collections/listAliases.md) method, specifying the collection name. - -
- -
- -To list aliases for a collection, you can use the [`POST /v2/vectordb/aliases/list`](https://milvus.io/api-reference/restful/v2.4.x/v2/Alias%20(v2)/List.md) API endpoint. - -
- -
- Python - Java - Node.js - cURL -
- -```python -# 9.2. List aliases -res = client.list_aliases( - collection_name="customized_setup_2" -) - -print(res) - -# Output -# -# { -# "aliases": [ -# "bob", -# "alice" -# ], -# "collection_name": "customized_setup_2", -# "db_name": "default" -# } -``` - -```java -import io.milvus.v2.service.utility.request.ListAliasesReq; -import io.milvus.v2.service.utility.response.ListAliasResp; - -// 9.2 List alises -ListAliasesReq listAliasesReq = ListAliasesReq.builder() - .collectionName("customized_setup_2") - .build(); - -ListAliasResp listAliasRes = client.listAliases(listAliasesReq); - -System.out.println(listAliasRes.getAlias()); - -// Output: -// [ -// "bob", -// "alice" -// ] -``` - -```javascript -// 9.2 List aliases -res = await client.listAliases({ - collection_name: "customized_setup_2" -}) - -console.log(res.aliases) - -// Output -// -// [ 'bob', 'alice' ] -// -``` - -```shell -$ curl -X POST "http://${MILVUS_URI}/v2/vectordb/aliases/list" \ --H "Content-Type: application/json" \ --d '{ - "collectionName": "customized_setup_2" -}' - -# { -# "code": 0, -# "data": [ -# "bob", -# "alice" -# ] -# } -``` - -### Describe aliases - -
- -To describe aliases, use the [`describe_alias()`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Collections/describe_alias.md) method, specifying the alias. - -
- -
- -To describe aliases, use the [`describeAlias()`](https://milvus.io/api-reference/java/v2.4.x/v2/Collections/describeAlias.md) method, specifying the alias. - -
- -
- -To describe aliases, use the [`describeAlias()`](https://milvus.io/api-reference/node/v2.4.x/Collections/describeAlias.md) method, specifying the alias. - -
- -
- -To describe aliases for a collection, you can use the [`POST /v2/vectordb/aliases/describe`](https://milvus.io/api-reference/restful/v2.4.x/v2/Alias%20(v2)/Describe.md) API endpoint. - -
- -
- Python - Java - Node.js - cURL -
- -```python -# 9.3. Describe aliases -res = client.describe_alias( - alias="bob" -) - -print(res) - -# Output -# -# { -# "alias": "bob", -# "collection_name": "customized_setup_2", -# "db_name": "default" -# } -``` - -```java -import io.milvus.v2.service.utility.request.DescribeAliasReq; -import io.milvus.v2.service.utility.response.DescribeAliasResp; - -// 9.3 Describe alias -DescribeAliasReq describeAliasReq = DescribeAliasReq.builder() - .alias("bob") - .build(); - -DescribeAliasResp describeAliasRes = client.describeAlias(describeAliasReq); - -System.out.println(JSONObject.toJSON(describeAliasRes)); - -// Output: -// { -// "alias": "bob", -// "collectionName": "customized_setup_2" -// } -``` - -```javascript -// 9.3 Describe aliases -res = await client.describeAlias({ - collection_name: "customized_setup_2", - alias: "bob" -}) - -console.log(res) - -// Output -// -// { -// status: { -// extra_info: {}, -// error_code: 'Success', -// reason: '', -// code: 0, -// retriable: false, -// detail: '' -// }, -// db_name: 'default', -// alias: 'bob', -// collection: 'customized_setup_2' -// } -// -``` - -```shell -$ curl -X POST "http://${MILVUS_URI}/v2/vectordb/aliases/describe" \ --H "Content-Type: application/json" \ --d '{ - "aliasName": "bob" -}' - -# { -# "code": 0, -# "data": { -# "aliasName": "bob", -# "collectionName": "quick_setup", -# "dbName": "default" -# } -# } -``` - -### Reassign aliases - -
- -To reassign aliases to other collections, use the [`alter_alias()`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Collections/alter_alias.md) method, specifying the collection name and the alias. - -
- -
- -To reassign aliases to other collections, use the [`alterAlias()`](https://milvus.io/api-reference/java/v2.4.x/v2/Collections/alterAlias.md) method, specifying the collection name and the alias. - -
- -
- -To reassign aliases to other collections, use the [`alterAlias()`](https://milvus.io/api-reference/node/v2.4.x/Collections/alterAlias.md) method, specifying the collection name and the alias. - -
- -
- -To reassign aliases to other collections, you can use the [`POST /v2/vectordb/aliases/alter`](https://milvus.io/api-reference/restful/v2.4.x/v2/Alias%20(v2)/Alter.md) API endpoint. - -
- -
- Python - Java - Node.js - cURL -
- -```python -# 9.4 Reassign aliases to other collections -client.alter_alias( - collection_name="customized_setup_1", - alias="alice" -) - -res = client.list_aliases( - collection_name="customized_setup_1" -) - -print(res) - -# Output -# -# { -# "aliases": [ -# "alice" -# ], -# "collection_name": "customized_setup_1", -# "db_name": "default" -# } - -res = client.list_aliases( - collection_name="customized_setup_2" -) - -print(res) - -# Output -# -# { -# "aliases": [ -# "bob" -# ], -# "collection_name": "customized_setup_2", -# "db_name": "default" -# } -``` - -```java -import io.milvus.v2.service.utility.request.AlterAliasReq; - -// 9.4 Reassign alias to other collections -AlterAliasReq alterAliasReq = AlterAliasReq.builder() - .collectionName("customized_setup_1") - .alias("alice") - .build(); - -client.alterAlias(alterAliasReq); - -listAliasesReq = ListAliasesReq.builder() - .collectionName("customized_setup_1") - .build(); - -listAliasRes = client.listAliases(listAliasesReq); - -System.out.println(listAliasRes.getAlias()); - -// Output: -// ["alice"] - -listAliasesReq = ListAliasesReq.builder() - .collectionName("customized_setup_2") - .build(); - -listAliasRes = client.listAliases(listAliasesReq); - -System.out.println(listAliasRes.getAlias()); - -// Output: -// ["bob"] -``` - -```javascript -// 9.4 Reassign aliases to other collections -res = await client.alterAlias({ - collection_name: "customized_setup_1", - alias: "alice" -}) - -console.log(res.error_code) - -// Output -// -// Success -// - -res = await client.listAliases({ - collection_name: "customized_setup_1" -}) - -console.log(res.aliases) - -// Output -// -// [ 'alice' ] -// - -res = await client.listAliases({ - collection_name: "customized_setup_2" -}) - -console.log(res.aliases) - -// Output -// -// [ 'bob' ] -// -``` - -```shell -$ curl -X POST "http://${MILVUS_URI}/v2/vectordb/aliases/alter" \ --H "Content-Type: application/json" \ --d '{ - "collectionName": "customized_setup_1", - "aliasName": "alice" -}' - -# { -# "code": 0, -# "data": {} -# } - - -$ curl -X POST "http://${MILVUS_URI}/v2/vectordb/aliases/list" \ --H "Content-Type: application/json" \ --d '{ - "collectionName": "customized_setup_1" -}' - - -# { -# "code": 0, -# "data": [ -# "alice" -# ] -# } - - -$ curl -X POST "http://${MILVUS_URI}/v2/vectordb/aliases/list" \ --H "Content-Type: application/json" \ --d '{ - "collectionName": "customized_setup_2" -}' - - -# { -# "code": 0, -# "data": [ -# "bob" -# ] -# } -``` - -### Drop aliases - -
- -To drop aliases, use the [`drop_alias()`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Collections/drop_alias.md) method, specifying the alias. - -
- -
- -To drop aliases, use the [`dropAlias()`](https://milvus.io/api-reference/java/v2.4.x/v2/Collections/dropAlias.md) method, specifying the alias. - -
- -
- -To drop aliases, use the [`dropAlias()`](https://milvus.io/api-reference/node/v2.4.x/Collections/dropAlias.md) method, specifying the alias. - -
- -
- -To drop aliases for a collection, you can use the [`POST /v2/vectordb/aliases/drop`](https://milvus.io/api-reference/restful/v2.4.x/v2/Alias%20(v2)/Drop.md) API endpoint. - -
- -
- Python - Java - Node.js -
- -```python -# 9.5 Drop aliases -client.drop_alias( - alias="bob" -) - -client.drop_alias( - alias="alice" -) -``` - -```java -import io.milvus.v2.service.utility.request.DropAliasReq; - -// 9.5 Drop alias -DropAliasReq dropAliasReq = DropAliasReq.builder() - .alias("bob") - .build(); - -client.dropAlias(dropAliasReq); - -dropAliasReq = DropAliasReq.builder() - .alias("alice") - .build(); - -client.dropAlias(dropAliasReq); -``` - -```javascript -// 9.5 Drop aliases -res = await client.dropAlias({ - alias: "bob" -}) - -console.log(res.error_code) - -// Output -// -// Success -// - -res = await client.dropAlias({ - alias: "alice" -}) - -console.log(res.error_code) - -// Output -// -// Success -// -``` -```shell -$ curl -X POST "http://${MILVUS_URI}/v2/vectordb/aliases/drop" \ --H "Content-Type: application/json" \ --d '{ - "aliasName": "bob" -}' - -# { -# "code": 0, -# "data": {} -# } - - -$ curl -X POST "http://${MILVUS_URI}/v2/vectordb/aliases/drop" \ --H "Content-Type: application/json" \ --d '{ - "aliasName": "alice" -}' - - -# { -# "code": 0, -# "data": {} -# } -``` - -## Set Properties - -You can set properties for a collection, such as `ttl.seconds` and `mmap.enabled`. For more information, refer to [set_properties()](https://milvus.io/api-reference/pymilvus/v2.4.x/ORM/Collection/set_properties.md). - -
- -The code snippets in this section use the PyMilvus ORM module to interact with Milvus. Code snippets with the new MilvusClient SDK will be available soon. - -
- -### Set TTL - -Set the Time-To-Live (TTL) for the data in the collection, which specifies how long the data should be retained before it is automatically deleted. - -```python -from pymilvus import Collection, connections - -# Connect to Milvus server -connections.connect(host="localhost", port="19530") # Change to your Milvus server IP and port - -# Get existing collection -collection = Collection("quick_setup") - -# Set the TTL for the data in the collection -collection.set_properties( - properties={ - "collection.ttl.seconds": 60 - } -) -``` - -### Set MMAP - -Configure the memory mapping (MMAP) property for the collection, which determines whether data is mapped into memory to improve query performance. For more information, refer to [Configure memory mapping -](https://milvus.io/docs/mmap.md#Configure-memory-mapping). - -
- -Before setting the MMAP property, release the collection first. Otherwise, an error will occur. - -
- -```python -from pymilvus import Collection, connections - -# Connect to Milvus server -connections.connect(host="localhost", port="19530") # Change to your Milvus server IP and port - -# Get existing collection -collection = Collection("quick_setup") - -# Before setting memory mapping property, we need to release the collection first. -collection.release() - -# Set memory mapping property to True or Flase -collection.set_properties( - properties={ - "mmap.enabled": True - } -) -``` - -## Drop a Collection - -If a collection is no longer needed, you can drop the collection. - -
- -To drop a collection, use the [`drop_collection()`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Collections/drop_collection.md) method, specifying the collection name. - -
- -
- -To drop a collection, use the [`dropCollection()`](https://milvus.io/api-reference/java/v2.4.x/v2/Collections/dropCollection.md) method, specifying the collection name. - -
- -
- -To drop a collection, use the [`dropCollection()`](https://milvus.io/api-reference/node/v2.4.x/Collections/dropCollection.md) method, specifying the collection name. - -
- -
- -To drop a collection, you can use the [`POST /v2/vectordb/collections/drop`](https://milvus.io/api-reference/restful/v2.4.x/v2/Collection%20(v2)/Drop.md) API endpoint. - -
- -
- Python - Java - Node.js - cURL -
- -```python -# 10. Drop the collections -client.drop_collection( - collection_name="quick_setup" -) - -client.drop_collection( - collection_name="customized_setup_1" -) - -client.drop_collection( - collection_name="customized_setup_2" -) -``` - -```java -import io.milvus.v2.service.collection.request.DropCollectionReq; - -// 10. Drop collections - -DropCollectionReq dropQuickSetupParam = DropCollectionReq.builder() - .collectionName("quick_setup") - .build(); - -client.dropCollection(dropQuickSetupParam); - -DropCollectionReq dropCustomizedSetupParam = DropCollectionReq.builder() - .collectionName("customized_setup_1") - .build(); - -client.dropCollection(dropCustomizedSetupParam); - -dropCustomizedSetupParam = DropCollectionReq.builder() - .collectionName("customized_setup_2") - .build(); - -client.dropCollection(dropCustomizedSetupParam); -``` - -```javascript -// 10. Drop the collection -res = await client.dropCollection({ - collection_name: "customized_setup_2" -}) - -console.log(res.error_code) - -// Output -// -// Success -// - -res = await client.dropCollection({ - collection_name: "customized_setup_1" -}) - -console.log(res.error_code) - -// Output -// -// Success -// - -res = await client.dropCollection({ - collection_name: "quick_setup" -}) - -console.log(res.error_code) - -// Output -// -// Success -// -``` - -```shell -$ curl -X POST "http://${MILVUS_URI}/v2/vectordb/collections/drop" \ --H "Content-Type: application/json" \ --d '{ - "collectionName": "quick_setup" -}' - -# { -# "code": 0, -# "data": {} -# } - - -$ curl -X POST "http://${MILVUS_URI}/v2/vectordb/collections/drop" \ --H "Content-Type: application/json" \ --d '{ - "collectionName": "customized_setup_1" -}' - - -# { -# "code": 0, -# "data": {} -# } - - -$ curl -X POST "http://${MILVUS_URI}/v2/vectordb/collections/drop" \ --H "Content-Type: application/json" \ --d '{ - "collectionName": "customized_setup_2" -}' - - -# { -# "code": 0, -# "data": {} -# } -``` - diff --git a/v2.4.x/site/en/userGuide/manage-indexes/bitmap.md b/v2.4.x/site/en/userGuide/manage-indexes/bitmap.md new file mode 100644 index 0000000000..37758a7937 --- /dev/null +++ b/v2.4.x/site/en/userGuide/manage-indexes/bitmap.md @@ -0,0 +1,130 @@ +--- +id: bitmap.md +title: BITMAP​ +related_key: bitmap +summary: Bitmap indexing is an efficient indexing technique designed to improve query performance on low-cardinality scalar fields. +--- + +# BITMAP​ + +Bitmap indexing is an efficient indexing technique designed to improve query performance on low-cardinality scalar fields. Cardinality refers to the number of distinct values in a field. Fields with fewer distinct elements are considered low-cardinality.​ + +This index type helps reduce the retrieval time of scalar queries by representing field values in a compact binary format and performing efficient bitwise operations on them. Compared to other types of indexes, bitmap indexes typically have higher space efficiency and faster query speeds when dealing with low-cardinality fields.​ + +## Overview + +The term Bitmap combines two words: **Bit** and **Map**. A bit represents the smallest unit of data in a computer, which can only hold a value of either **0** or **1**. A map, in this context, refers to the process of transforming and organizing data according to what value should be assigned to 0 and 1.​ + +A bitmap index consists of two main components: bitmaps and keys. Keys represent the unique values in the indexed field. For each unique value, there is a corresponding bitmap. The length of these bitmaps is equal to the number of records in the collection. Each bit in the bitmap corresponds to a record in the collection. If the value of the indexed field in a record matches the key, the corresponding bit is set to **1**; otherwise, it is set to **0**.​ + +Consider a collection of documents with fields **Category** and **Public**. We want to retrieve documents that fall into the **Tech** category and are open to the **Public**. In this case, the keys for our bitmap indexes are **Tech** and **Public**.​ + +![Bitmap indexing](../../../assets/bitmap.png) + +As shown in the figure, the bitmap indexes for **Category** and **Public** are:​ + +- **Tech**: [1, 0, 1, 0, 0], which shows that only the 1st and 3rd documents fall into the **Tech** category.​ + +- **Public**: [1, 0, 0, 1, 0], which shows that only the 1st and 4th documents are open to the **Public**.​ + +To find the documents that match both criteria, we perform a bitwise AND operation on these two bitmaps:​ + +- **Tech** AND **Public**: [1, 0, 0, 0, 0]​ + +The resulting bitmap [1, 0, 0, 0, 0] indicates that only the first document (**ID** **1**) satisfies both criteria. By using bitmap indexes and efficient bitwise operations, we can quickly narrow down the search scope, eliminating the need to scan the entire dataset.​ + +## Create a bitmap index + +To create a bitmap index in Milvus, use the `create_index()` method and set the `index_type` parameter to `"BITMAP"`.​ + +```python +from pymilvus import MilvusClient​ +​ +index_params = client.create_index_params() # Prepare an empty IndexParams object, without having to specify any index parameters​ +index_params.add_index(​ + field_name="category", # Name of the scalar field to be indexed​ + index_type="BITMAP", # Type of index to be created​ + index_name="category_bitmap_index" # Name of the index to be created​ +)​ +​ +client.create_index(​ + collection_name="my_collection", # Specify the collection name​ + index_params=index_params​ +)​ + +``` + +In this example, we create a bitmap index on the `category` field of the `my_collection` collection. The `add_index()` method is used to specify the field name, index type, and index name.​ + +Once the bitmap index is created, you can use the `filter` parameter in query operations to perform scalar filtering based on the indexed field. This allows you to efficiently narrow down the search results using the bitmap index. For more information, refer to Filtering.​ + +## Limits + +- Bitmap indexes are supported only for scalar fields that are not primary keys.​ + +- The data type of the field must be one of the following:​ + + - `BOOL`, `INT8`, `INT16`, `INT32`, `INT64`, `VARCHAR`​ + + - `ARRAY` (elements must be one of: `BOOL`, `INT8`, `INT16`, `INT32`, `INT64`, `VARCHAR`)​ + +- Bitmap indexes do not support the following data types:​ + + - `FLOAT`, `DOUBLE`: Floating-point types are not compatible with the binary nature of bitmap indexes.​ + + - `JSON`: JSON data types have a complex structure that cannot be efficiently represented using bitmap indexes.​ + +- Bitmap indexes are not suitable for fields with high cardinality (i.e., fields with a large number of distinct values).​ + + - As a general guideline, bitmap indexes are most effective when the cardinality of a field is less than 500.​ + + - When the cardinality increases beyond this threshold, the performance benefits of bitmap indexes diminish, and the storage overhead becomes significant.​ + + - For high-cardinality fields, consider using alternative indexing techniques such as inverted indexes, depending on your specific use case and query requirements.​ + +### Structural Similarity + +When a chemical structure occurs as a part of a larger chemical structure, the former is called a substructure and the latter is called a superstructure. For example, ethanol is a substructure of acetic acid, and acetic acid is a superstructure of ethanol. + +Structural similarity is used to determine whether two chemical formulae are similar to each other in that one is the superstructure or substructure of the other. + +To determine whether A is a superstructure of B, use the following formula: + +![superstructure](../../../assets/superstructure.png "Superstructure") + +Where: + +- A is the binary representation of a chemical formula to be retrieved +- B is the binary representation of a chemical formula in the database + +Once it returns `0`, **A** is not a superstructure of **B**. Otherwise, the result is the other way around. + +To determine whether A is a substructure of B, use the following formula: + +![substructure](../../../assets/substructure.png "subsctructure") + +Where: + +- A is the binary representation of a chemical formula to be retrieved +- B is the binary representation of a chemical formula in the database + +Once it returns `0`, **A** is not a substructure of **B**. Otherwise, the result is the other way around. + +## FAQ + +
+Why is the top1 result of a vector search not the search vector itself, if the metric type is inner product? +This occurs if you have not normalized the vectors when using inner product as the distance metric. +
+
+What is normalization? Why is normalization needed? +

Normalization refers to the process of converting an embedding (vector) so that its norm equals 1. If you use Inner Product to calculate embeddings similarities, you must normalize your embeddings. After normalization, inner product equals cosine similarity. +

+

+See Wikipedia for more information. +

+
+
+Why do I get different results using Euclidean distance (L2) and inner product (IP) as the distance metric? +Check if the vectors are normalized. If not, you need to normalize the vectors first. Theoretically speaking, similarities worked out by L2 are different from similarities worked out by IP, if the vectors are not normalized. +
\ No newline at end of file diff --git a/v2.4.x/site/en/userGuide/manage-partitions.md b/v2.4.x/site/en/userGuide/manage-partitions.md deleted file mode 100644 index 4ee2a2d2a2..0000000000 --- a/v2.4.x/site/en/userGuide/manage-partitions.md +++ /dev/null @@ -1,1054 +0,0 @@ ---- -id: manage-partitions.md -title: Manage Partitions ---- - -# Manage Partitions - -This guide walks you through how to create and manage partitions in a collection. - -## Overview - -A partition in Milvus represents a sub-division of a collection. This functionality allows the physical storage of a collection to be divided into multiple parts, contributing to improved query performance by narrowing down the focus to a smaller subset of data rather than the entire collection. - -Upon the creation of a collection, at least a default partition named ___default__ is automatically created. You can create a maximum of 1,024 partitions within a collection. - -
- -

notes

- -

Milvus introduces a feature called Partition Key, leveraging the underlying partitions to store entities based on the hashed values of a specific field. This feature facilitates the implementation of multi-tenancy, enhancing search performance. For details, read Use Partition Key.

-

If the Partition Key feature is on in a collection, Milvus takes care of managing all the partitions, relieving you of this responsibility.

- -
- -## Preparations - -The code snippet below repurposes the existing code to establish a connection to Milvus and create a collection in a quick-setup mode, indicating that the collection is loaded upon creation. - -
- -For preparations, use [`MilvusClient`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Client/MilvusClient.md) to connect to Milvus and [`create_collection()`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Collections/create_collection.md) to create a collection in a quick-setup mode. - -
- -
- -For preparations, use [`MilvusClientV2`](https://milvus.io/api-reference/java/v2.4.x/v2/Client/MilvusClientV2.md) to connect to Milvus and [`createCollection()`](https://milvus.io/api-reference/java/v2.4.x/v2/Collections/createCollection.md) to create a collection in a quick-setup mode. - -
- -
- -For preparations, use [`MilvusClient`](https://milvus.io/api-reference/node/v2.4.x/Client/MilvusClient.md) to connect to Milvus and [`createCollection()`](https://milvus.io/api-reference/node/v2.4.x/Collections/createCollection.md) to create a collection in a quick-setup mode. - -
- -
- Python - Java - Node.js -
- -```python -from pymilvus import MilvusClient, DataType - -# 1. Set up a Milvus client -client = MilvusClient( - uri="http://localhost:19530" -) - -# 2. Create a collection -client.create_collection( - collection_name="quick_setup", - dimension=5, -) - -``` - -```java -import io.milvus.v2.client.ConnectConfig; -import io.milvus.v2.client.MilvusClientV2; -import io.milvus.v2.service.collection.request.CreateCollectionReq; - -String CLUSTER_ENDPOINT = "http://localhost:19530"; - -// 1. Connect to Milvus server -ConnectConfig connectConfig = ConnectConfig.builder() - .uri(CLUSTER_ENDPOINT) - .build(); - -MilvusClientV2 client = new MilvusClientV2(connectConfig); - -// 2. Create a collection in quick setup mode -CreateCollectionReq quickSetupReq = CreateCollectionReq.builder() - .collectionName("quick_setup") - .dimension(5) - .build(); - -client.createCollection(quickSetupReq); -``` - -```javascript -const address = "http://localhost:19530" - -// 1. Set up a Milvus Client -client = new MilvusClient({address}); - -// 2. Create a collection in quick setup mode -await client.createCollection({ - collection_name: "quick_setup", - dimension: 5, -}); -``` - -
- -

notes

- -

In the above code snippet, the index of the collection has been created along with the collection, indicating that the collection is loaded upon creation.

- -
- -## List Partitions - -Once a collection is ready, you can list its partitions. - -
- -To list partitions, use [`list_partitions()`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Partitions/list_partitions.md). - -
- -
- -To list partitions, use [`listPartitions()`](https://milvus.io/api-reference/java/v2.4.x/v2/Partitions/listPartitions.md). - -
- -
- -To list partitions, use [`listPartitions()`](https://milvus.io/api-reference/node/v2.4.x/Partitions/listPartitions.md). - -
- -
- Python - Java - Node.js -
- -```python -# 3. List partitions -res = client.list_partitions(collection_name="quick_setup") -print(res) - -# Output -# -# ["_default"] -``` - -```java -import io.milvus.v2.service.partition.request.ListPartitionsReq; - -// 3. List all partitions in the collection -ListPartitionsReq listPartitionsReq = ListPartitionsReq.builder() - .collectionName("quick_setup") - .build(); - -List partitionNames = client.listPartitions(listPartitionsReq); - -System.out.println(partitionNames); - -// Output: -// ["_default"] -``` - -```javascript -// 3. List partitions -res = await client.listPartitions({ - collection_name: "quick_setup" -}) - -console.log(res.partition_names) - -// Output -// -// [ '_default' ] -// -``` - -The output of the above code snippet includes the names of the partitions within the specified collection. - -
- -

notes

- -

If you have set a field as the partition key in a collection, Milvus creates at least 64 partitions along with the collection. When listing the partitions, the results may differ from the output of the above code snippets.

-

For details, refer to Use Partition Key.

- -
- -## Create Partitions - -You can add more partitions to the collection. A collection can have up to 1,024 partitions. - -
- -To create partitions, use [`create_partition()`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Partitions/create_partition.md). - -
- -
- -To create partitions, use [`createPartition()`](https://milvus.io/api-reference/java/v2.4.x/v2/Partitions/createPartition.md). - -
- -
- -To create partitions, use [`createPartition()`](https://milvus.io/api-reference/node/v2.4.x/Partitions/createPartition.md). - -
- -
- Python - Java - Node.js -
- -```python -# 4. Create more partitions -client.create_partition( - collection_name="quick_setup", - partition_name="partitionA" -) - -client.create_partition( - collection_name="quick_setup", - partition_name="partitionB" -) - -res = client.list_partitions(collection_name="quick_setup") -print(res) - -# Output -# -# ["_default", "partitionA", "partitionB"] -``` - -```java -import io.milvus.v2.service.partition.request.CreatePartitionReq; - -// 4. Create more partitions -CreatePartitionReq createPartitionReq = CreatePartitionReq.builder() - .collectionName("quick_setup") - .partitionName("partitionA") - .build(); - -client.createPartition(createPartitionReq); - -createPartitionReq = CreatePartitionReq.builder() - .collectionName("quick_setup") - .partitionName("partitionB") - .build(); - -client.createPartition(createPartitionReq); - -listPartitionsReq = ListPartitionsReq.builder() - .collectionName("quick_setup") - .build(); - -partitionNames = client.listPartitions(listPartitionsReq); - -System.out.println(partitionNames); - -// Output: -// [ -// "_default", -// "partitionA", -// "partitionB" -// ] -``` - -```javascript -// 4. Create more partitions -await client.createPartition({ - collection_name: "quick_setup", - partition_name: "partitionA" -}) - -await client.createPartition({ - collection_name: "quick_setup", - partition_name: "partitionB" -}) - -res = await client.listPartitions({ - collection_name: "quick_setup" -}) - -console.log(res.partition_names) - -// Output -// -// [ '_default', 'partitionA', 'partitionB' ] -// -``` - -The code snippet above creates a partition in a collection and lists the partitions of the collection. - -
- -

notes

- -

If you have set a field as the partition key in a collection, Milvus takes care of managing the partitions in the collection. Therefore, you may encounter prompted errors when attempting to create partitions.

-

For details, refer to Use Partition Key.

- -
- -## Check for a Specific Partition - -You can also check the existence of a specific partition. - -
- -To check for a specific partition, use [`has_partition()`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Partitions/has_partition.md). - -
- -
- -To check for a specific partition, use [`hasPartition()`](https://milvus.io/api-reference/java/v2.4.x/v2/Partitions/hasPartition.md). - -
- -
- -To check for a specific partition, use [`hasPartition()`](https://milvus.io/api-reference/node/v2.4.x/Partitions/hasPartition.md). - -
- -
- Python - Java - Node.js -
- -```python -# 5. Check whether a partition exists -res = client.has_partition( - collection_name="quick_setup", - partition_name="partitionA" -) -print(res) - -# Output -# -# True - -res = client.has_partition( - collection_name="quick_setup", - partition_name="partitionC" -) -print(res) - -# Output -# -# False -``` - -```java -import io.milvus.v2.service.partition.request.HasPartitionReq; - -// 5. Check whether a partition exists -HasPartitionReq hasPartitionReq = HasPartitionReq.builder() - .collectionName("quick_setup") - .partitionName("partitionA") - .build(); - -boolean exists = client.hasPartition(hasPartitionReq); - -System.out.println(exists); - -// Output: -// true - -hasPartitionReq = HasPartitionReq.builder() - .collectionName("quick_setup") - .partitionName("partitionC") - .build(); - -exists = client.hasPartition(hasPartitionReq); - -System.out.println(exists); - -// Output: -// false -``` - -```javascript -// 5. Check whether a partition exists -res = await client.hasPartition({ - collection_name: "quick_setup", - partition_name: "partitionA" -}) - -console.log(res.value) - -// Output -// -// true -// - -res = await client.hasPartition({ - collection_name: "quick_setup", - partition_name: "partitionC" -}) - -console.log(res.value) - -// Output -// -// false -// -``` - -The code snippet above checks whether the collection has a partition named `partitionA` and `partitionC`. - -## Load & Release Partitions - -You can load and release specific partitions to make them available or unavailable for searches and queries. - -### Get Load Status - -
- -To check the load status of a collection and its partitions, use [`get_load_state()`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Management/get_load_state.md). - -
- -
- -To check the load status of a collection and its partitions, use [`getLoadState()`](https://milvus.io/api-reference/java/v2.4.x/v2/Management/getLoadState.md). - -
- -
- -To check the load status of a collection and its partitions, use [`getLoadState()`](https://milvus.io/api-reference/node/v2.4.x/Management/getLoadState.md). - -
- -
- Python - Java - Node.js -
- -```python -# Release the collection -client.release_collection(collection_name="quick_setup") - -# Check the load status -res = client.get_load_state(collection_name="quick_setup") -print(res) - -# Output -# -# { -# "state": "" -# } - -res = client.get_load_state( - collection_name="quick_setup", - partition_name="partitionA" -) - -print(res) - -# Output -# -# { -# "state": "" -# } - -res = client.get_load_state( - collection_name="quick_setup", - partition_name="partitionB" -) - -print(res) - -# Output -# -# { -# "state": "" -# } - -``` - -```java -import io.milvus.v2.service.collection.request.GetLoadStateReq; -import io.milvus.v2.service.collection.request.ReleaseCollectionReq; -import io.milvus.v2.service.partition.request.LoadPartitionsReq; -import io.milvus.v2.service.partition.request.ReleasePartitionsReq; - -// 6. Load a partition independantly -// 6.1 Release the collection -ReleaseCollectionReq releaseCollectionReq = ReleaseCollectionReq.builder() - .collectionName("quick_setup") - .build(); - -client.releaseCollection(releaseCollectionReq); - -// 6.2 Load partitionA -LoadPartitionsReq loadPartitionsReq = LoadPartitionsReq.builder() - .collectionName("quick_setup") - .partitionNames(List.of("partitionA")) - .build(); - -client.loadPartitions(loadPartitionsReq); - -Thread.sleep(3000); - -// 6.3 Check the load status of the collection and its partitions -GetLoadStateReq getLoadStateReq = GetLoadStateReq.builder() - .collectionName("quick_setup") - .build(); - -boolean state = client.getLoadState(getLoadStateReq); - -System.out.println(state); - -// Output: -// true - -getLoadStateReq = GetLoadStateReq.builder() - .collectionName("quick_setup") - .partitionName("partitionA") - .build(); - -state = client.getLoadState(getLoadStateReq); - -System.out.println(state); - -// Output: -// true - -getLoadStateReq = GetLoadStateReq.builder() - .collectionName("quick_setup") - .partitionName("partitionB") - .build(); - -state = client.getLoadState(getLoadStateReq); - -System.out.println(state); - -// Output: -// false -``` - -```javascript -// 6. Load a partition indenpendantly -await client.releaseCollection({ - collection_name: "quick_setup" -}) - -res = await client.getLoadState({ - collection_name: "quick_setup" -}) - -console.log(res.state) - -// Output -// -// LoadStateNotLoad -// - -await client.loadPartitions({ - collection_name: "quick_setup", - partition_names: ["partitionA"] -}) - -await sleep(3000) - -res = await client.getLoadState({ - collection_name: "quick_setup" -}) - -console.log(res.state) - -// Output -// -// LoadStateLoaded -// - -res = await client.getLoadState({ - collection_name: "quick_setup", - partition_name: "partitionA" -}) - -console.log(res.state) - -// Output -// -// LoadStateLoaded -// - -res = await client.getLoadState({ - collection_name: "quick_setup", - partition_name: "partitionB" -}) - -console.log(res.state) - -// Output -// -// LoadStateLoaded -// -``` - -Possible load status may be either of the following - -- __Loaded__ - - A collection is marked as `Loaded` if at least one of its partitions has been loaded. - -- __NotLoad__ - - A collection is marked as `NotLoad` if none of its partitions has been loaded. - -- __Loading__ - - A collection is marked as Loading if at least one of its partitions is in the loading process. - - -### Load Partitions - -
- -To load all partitions of a collection, you can just call [`load_collection()`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Management/load_collection.md). To load specific partitions of a collection, use [`load_partitions()`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Partitions/load_partitions.md). - -
- -
- -To load all partitions of a collection, you can just call [`loadCollection()`](https://milvus.io/api-reference/java/v2.4.x/v2/Management/loadCollection.md). To load specific partitions of a collection, use [`loadPartitions()`](https://milvus.io/api-reference/java/v2.4.x/v2/Partitions/loadPartitions.md). - -
- -
- -To load all partitions of a collection, you can just call [`loadCollection()`](https://milvus.io/api-reference/node/v2.4.x/Management/loadCollection.md). To load specific partitions of a collection, use [`loadPartitions()`](https://milvus.io/api-reference/node/v2.4.x/Partitions/loadPartitions.md). - -
- -
- Python - Java - Node.js -
- -```python -client.load_partitions( - collection_name="quick_setup", - partition_names=["partitionA"] -) - -res = client.get_load_state(collection_name="quick_setup") -print(res) - -# Output -# -# { -# "state": "" -# } -``` - -```java -LoadPartitionsReq loadPartitionsReq = LoadPartitionsReq.builder() - .collectionName("quick_setup") - .partitionNames(List.of("partitionA")) - .build(); - -client.loadPartitions(loadPartitionsReq); - -getLoadStateReq = GetLoadStateReq.builder() - .collectionName("quick_setup") - .partitionName("partitionA") - .build(); - -state = client.getLoadState(getLoadStateReq); - -System.out.println(state); - -// Output: -// true -``` - -```javascript -await client.loadPartitions({ - collection_name: "quick_setup", - partition_names: ["partitionA"] -}) - -res = await client.getLoadState({ - collection_name: "quick_setup", - partition_name: "partitionA" -}) - -console.log(res.state) - -// Output -// -// LoadStateLoaded -// -``` - -To load multiple partitions at a time, do as follows: - -
- Python - Java - Node.js -
- -```python -client.load_partitions( - collection_name="quick_setup", - partition_names=["partitionA", "partitionB"] -) - -res = client.get_load_state( - collection_name="quick_setup", - partition_name="partitionA" -) - -# Output -# -# { -# "state": "" -# } - -res = client.get_load_state( - collection_name="quick_setup", - partition_name="partitionB" -) - -# Output -# -# { -# "state": "" -# } -``` - -```java -LoadPartitionsReq loadPartitionsReq = LoadPartitionsReq.builder() - .collectionName("quick_setup") - .partitionNames(List.of("partitionA", "partitionB")) - .build(); - -client.loadPartitions(loadPartitionsReq); - -getLoadStateReq = GetLoadStateReq.builder() - .collectionName("quick_setup") - .partitionName("partitionA") - .build(); - -state = client.getLoadState(getLoadStateReq); - -System.out.println(state); - -// Output: -// true - -getLoadStateReq = GetLoadStateReq.builder() - .collectionName("quick_setup") - .partitionName("partitionB") - .build(); - -state = client.getLoadState(getLoadStateReq); - -System.out.println(state); - -// Output: -// true -``` - -```javascript -await client.loadPartitions({ - collection_name: "quick_setup", - partition_names: ["partitionA", "partitionB"] -}) - -res = await client.getLoadState({ - collection_name: "quick_setup", - partition_name: "partitionA" -}) - -console.log(res) - -// Output -// -// LoadStateLoaded -// - -res = await client.getLoadState({ - collection_name: "quick_setup", - partition_name: "partitionB" -}) - -console.log(res) - -// Output -// -// LoadStateLoaded -// -``` - -To load specified fields in one or more partitions, do as follows: - -```python -client.load_partitions( - collection_name="quick_setup", - partition_names=["partitionA"], - load_fields=["id", "vector"], - skip_load_dynamic_field=True -) -``` - -Note that only the fields listed in `load_fields` can be used as filtering conditions and output fields in searches and queries. You should always include the primary key in the list. The field names excluded from loading will not be available for filtering or output. - -You can use `skip_load_dynamic_field=True` to skip loading the dynamic field. Milvus treats the dynamic field as a single field, so all the keys in the dynamic field will be included or excluded together. - - -### Release Partitions - -
- -To release all partitions of a collection, you can just call [`release_collection()`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Management/release_collection.md). To release specific partitions of a collection, use [`release_partitions()`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Partitions/release_partitions.md). - -
- -
- -To release all partitions of a collection, you can just call [`releaseCollection()`](https://milvus.io/api-reference/java/v2.4.x/v2/Management/releaseCollection.md). To release specific partitions of a collection, use [`releasePartitions()`](https://milvus.io/api-reference/java/v2.4.x/v2/Partitions/releasePartitions.md). - -
- -
- -To release all partitions of a collection, you can just call [`releaseCollection()`](https://milvus.io/api-reference/node/v2.4.x/Management/releaseCollection.md). To release specific partitions of a collection, use [`releasePartitions()`](https://milvus.io/api-reference/node/v2.4.x/Partitions/releasePartitions.md). - -
- -
- Python - Java - Node.js -
- -```python -# 7. Release a partition -client.release_partitions( - collection_name="quick_setup", - partition_names=["partitionA"] -) - -res = client.get_load_state( - collection_name="quick_setup", - partition_name="partitionA" -) - -print(res) - -# Output -# -# { -# "state": "" -# } - -``` - -```java -import io.milvus.v2.service.partition.request.ReleasePartitionsReq; - -// 7. Release a partition -ReleasePartitionsReq releasePartitionsReq = ReleasePartitionsReq.builder() - .collectionName("quick_setup") - .partitionNames(List.of("partitionA")) - .build(); - -client.releasePartitions(releasePartitionsReq); - -getLoadStateReq = GetLoadStateReq.builder() - .collectionName("quick_setup") - .partitionName("partitionA") - .build(); - -state = client.getLoadState(getLoadStateReq); - -System.out.println(state); - -// Output: -// false -``` - -```javascript -// 7. Release a partition -await client.releasePartitions({ - collection_name: "quick_setup", - partition_names: ["partitionA"] -}) - -res = await client.getLoadState({ - collection_name: "quick_setup" -}) - -console.log(res.state) - -// Output -// -// LoadStateNotLoad -// -``` - -To release multiple partitions at a time, do as follows: - -```python -client.release_partitions( - collection_name="quick_setup", - partition_names=["_default", "partitionA", "partitionB"] -) - -res = client.get_load_state( - collection_name="quick_setup", -) - -# Output -# -# { -# "state": "" -# } -``` - -## Drop Partitions - -Once you release a partition, you can drop it if it is no longer needed. - -
- -To drop a partition, use [`drop_partition()`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Partitions/drop_partition.md). - -
- -
- -To drop a partition, use [`dropPartition()`](https://milvus.io/api-reference/java/v2.4.x/v2/Partitions/dropPartition.md). - -
- -
- -To drop a partition, use [`dropPartition()`](https://milvus.io/api-reference/node/v2.4.x/Partitions/dropPartition.md). - -
- -
- Python - Java - Node.js -
- -```python -# 8. Drop a partition -client.drop_partition( - collection_name="quick_setup", - partition_name="partitionB" -) - -res = client.list_partitions(collection_name="quick_setup") -print(res) - -# Output -# -# ["_default", "partitionA"] -``` - -```java -import io.milvus.v2.service.partition.request.ReleasePartitionsReq; - -ReleasePartitionsReq releasePartitionsReq = ReleasePartitionsReq.builder() - .collectionName("quick_setup") - .partitionNames(List.of("_default", "partitionA", "partitionB")) - .build(); - -client.releasePartitions(releasePartitionsReq); - -getLoadStateReq = GetLoadStateReq.builder() - .collectionName("quick_setup") - .build(); - -state = client.getLoadState(getLoadStateReq); - -System.out.println(state); - -// Output: -// false -``` - -```javascript - -await client.releasePartitions({ - collection_name: "quick_setup", - partition_names: ["_default", "partitionA", "partitionB"] -}) - -res = await client.getLoadState({ - collection_name: "quick_setup" -}) - -console.log(res) - -// Output -// -// { -// status: { -// error_code: 'Success', -// reason: '', -// code: 0, -// retriable: false, -// detail: '' -// }, -// state: 'LoadStateNotLoad' -// } -// -``` - -
- -

notes

- -

Before dropping a partition, you need to release it from memory.

- -
- -## FAQ - -- __How much data can be stored in a partition?__ - - It is recommended to store less than 1B of data in a partition. - -- __What is the maximum number of partitions that can be created?__ - - By default, Milvus allows a maximum of 1,024 partitions to be created. You can adjust the maximum number of partitions by configuring `rootCoord.maxPartitionNum`. For details, refer to [System Configurations](https://milvus.io/docs/configure_rootcoord.md#rootCoordmaxPartitionNum). - -- __How can I differentiate between partitions and partition keys?__ - - Partitions are physical storage units, whereas partition keys are logical concepts that automatically assign data to specific partitions based on a designated column. - - For instance, in Milvus, if you have a collection with a partition key defined as the `color` field, the system automatically assigns data to partitions based on the hashed values of the `color` field for each entity. This automated process relieves the user of the responsibility to manually specify the partition when inserting or searching data. - - On the other hand, when manually creating partitions, you need to assign data to each partition based on the criteria of the partition key. If you have a collection with a `color` field, you would manually assign entities with a `color` value of `red` to `partition A`, and entities with a `color` value of `blue` to `partition B`. This manual management requires more effort. - - In summary, both partitions and partition keys are utilized to optimize data computation and enhance query efficiency. It is essential to recognize that enabling a partition key means surrendering control over the manual management of partition data insertion and loading, as these processes are fully automated and handled by Milvus. diff --git a/v2.4.x/site/en/userGuide/schema/analyzer/analyzer-overview.md b/v2.4.x/site/en/userGuide/schema/analyzer/analyzer-overview.md new file mode 100644 index 0000000000..4f565f1076 --- /dev/null +++ b/v2.4.x/site/en/userGuide/schema/analyzer/analyzer-overview.md @@ -0,0 +1,239 @@ +--- +id: analyzer-overview.md +title: Analyzer Overview​ +summary: "In text processing, an analyzer is a crucial component that converts raw text into a structured, searchable format. Each analyzer typically consists of two core elements: tokenizer and filter. Together, they transform input text into tokens, refine these tokens, and prepare them for efficient indexing and retrieval.​" +--- + +# Analyzer Overview​ + +In text processing, an **analyzer** is a crucial component that converts raw text into a structured, searchable format. Each analyzer typically consists of two core elements: **tokenizer** and **filter**. Together, they transform input text into tokens, refine these tokens, and prepare them for efficient indexing and retrieval.​ + +In Milvus, analyzers are configured during collection creation when you add `VARCHAR` fields to the collection schema. Tokens produced by an analyzer can be used to build an index for keyword matching or converted into sparse embeddings for full text search. For more information, refer to [​Keyword Match](keyword-match.md) or [​Full Text Search](full-text-search.md).​ + +
+ +The use of analyzers may impact performance:​ + +- **Full text search:** For full text search, DataNode and **QueryNode** channels consume data more slowly because they must wait for tokenization to complete. As a result, newly ingested data takes longer to become available for search.​ + +- **Keyword match:** For keyword matching, index creation is also slower since tokenization needs to finish before an index can be built.​ + +
+ +## Anatomy of an analyzer​ + +An analyzer in Milvus consists of exactly one **tokenizer** and **zero or more** filters.​ + +- **Tokenizer**: The tokenizer breaks input text into discrete units called tokens. These tokens could be words or phrases, depending on the tokenizer type.​ + +- **Filters**: Filters can be applied to tokens to further refine them, for example, by making them lowercase or removing common words.​ + +The workflow below shows how an analyzer processes text.​ + +analyzer-overview + +## Analyzer types​ + +Milvus provides two types of analyzers to meet different text processing needs:​ + +- **Built-in analyzer**: These are predefined configurations that cover common text processing tasks with minimal setup. Built-in analyzers are ideal for general-purpose searches, as they require no complex configuration.​ + +- **Custom analyzer**: For more advanced requirements, custom analyzers allow you to define your own configuration by specifying both the tokenizer and zero or more filters. This level of customization is especially useful for specialized use cases where precise control over text processing is needed.​ + +
+ +If you omit analyzer configurations during collection creation, Milvus uses the `standard` analyzer for all text processing by default. For details, refer to [​Standard](standard-analyzer.md).​ + +
+ +### Built-in analyzer​ + +Built-in analyzers in Milvus are pre-configured with specific tokenizers and filters, allowing you to use them immediately without needing to define these components yourself. Each built-in analyzer serves as a template that includes a preset tokenizer and filters, with optional parameters for customization.​ + +For example, to use the `standard` built-in analyzer, simply specify its name `standard` as the `type` and optionally include extra configurations specific to this analyzer type, such as `stop_words`:​ + +```python +analyzer_params = {​ + "type": "standard", # Uses the standard built-in analyzer​ + "stop_words": ["a", "an", "for"] # Defines a list of common words (stop words) to exclude from tokenization​ +}​ + +``` + +The configuration of the `standard` built-in analyzer above is equivalent to setting up a custom analyzer with the following parameters, where `tokenizer` and `filter` options are explicitly defined to achieve the same functionality: + +```python +analyzer_params = {​ + "tokenizer": "standard",​ + "filter": [​ + "lowercase",​ + {​ + "type": "stop",​ + "stop_words": ["a", "an", "for"]​ + }​ + ]​ +}​ + +``` + +Milvus offers the following built-in analyzers, each of which can be used directly by specifying their name as the `type` parameter:​ + +- `standard`: Suitable for general-purpose text processing, applying standard tokenization and lowercase filtering.​ + +- `english`: Optimized for English-language text, with support for English stop words.​ + +- `chinese`: Specialized for processing Chinese text, including tokenization adapted for Chinese language structures.​ + +### Custom analyzer​ + +For more advanced text processing, custom analyzers in Milvus allow you to build a tailored text-handling pipeline by specifying both a **tokenizer** and filters. This setup is ideal for specialized use cases where precise control is required.​ + +#### Tokenizer​ + +The **tokenizer** is a **mandatory** component for a custom analyzer, which initiates the analyzer pipeline by breaking down input text into discrete units or **tokens**. Tokenization follows specific rules, such as splitting by whitespace or punctuation, depending on the tokenizer type. This process allows for more precise and independent handling of each word or phrase.​ + +For example, a tokenizer would convert text `"Vector Database Built for Scale"` into separate tokens:​ + +```Plain Text +["Vector", "Database", "Built", "for", "Scale"]​ +``` + +**Example of specifying a tokenizer**:​ + +```python +analyzer_params = {​ + "tokenizer": "whitespace",​ +}​ +``` + +#### Filter​ + +**Filters** are **optional** components working on the tokens produced by the tokenizer, transforming or refining them as needed. For example, after applying a `lowercase` filter to the tokenized terms `["Vector", "Database", "Built", "for", "Scale"]`, the result might be:​ + +```SQL +["vector", "database", "built", "for", "scale"]​ +``` + +Filters in a custom analyzer can be either **built-in** or **custom**, depending on configuration needs.​ + +- **Built-in filters**: Pre-configured by Milvus, requiring minimal setup. You can use these filters out-of-the-box by specifying their names. The filters below are built-in for direct use:​ + + - `lowercase`: Converts text to lowercase, ensuring case-insensitive matching. For details, refer to [​Lowercase](lowercase-filter.md).​ + + - `asciifolding`: Converts non-ASCII characters to ASCII equivalents, simplifying multilingual text handling. For details, refer to [​ASCII folding](ascii-folding-filter.md).​ + + - `alphanumonly`: Retains only alphanumeric characters by removing others. For details, refer to [​Alphanumonly](alphanumonly-filter.md).​ + + - `cnalphanumonly`: Removes tokens that contain any characters other than Chinese characters, English letters, or digits. For details, refer to [​Cnalphanumonly](cnalphanumonly-filter.md).​ + + - `cncharonly`: Removes tokens that contain any non-Chinese characters. For details, refer to [​Cncharonly](cncharonly-filter.md).​ + + **Example of using a built-in filter:**​ + + ```python + analyzer_params = {​ + "tokenizer": "standard", # Mandatory: Specifies tokenizer​ + "filter": ["lowercase"], # Optional: Built-in filter that converts text to lowercase​ + }​ + ``` + +- **Custom filters**: Custom filters allow for specialized configurations. You can define a custom filter by choosing a valid filter type (`filter.type`) and adding specific settings for each filter type. Examples of filter types that support customization:​ + + - `stop`: Removes specified common words by setting a list of stop words (e.g., `"stop_words": ["of", "to"]`). For details, refer to [​Stop](stop-filter.md).​ + + - `length`: Excludes tokens based on length criteria, such as setting a maximum token length. For details, refer to [​Length](length-filter.md).​ + + - `stemmer`: Reduces words to their root forms for more flexible matching. For details, refer to [​Stemmer](stemmer-filter.md).​ + + **Example of configuring a custom filter:**​ + + ```python + analyzer_params = {​ + "tokenizer": "standard", # Mandatory: Specifies tokenizer​ + "filter": [​ + {​ + "type": "stop", # Specifies 'stop' as the filter type​ + "stop_words": ["of", "to"], # Customizes stop words for this filter type​ + }​ + ]​ + }​ + + ``` + +## Example use​ + +In this example, we define a collection schema with a vector field for embeddings and two `VARCHAR` fields for text processing capabilities. Each `VARCHAR` field is configured with its own analyzer settings to handle different processing needs.​ + +```python +from pymilvus import MilvusClient, DataType​ +​ +# Set up a Milvus client​ +client = MilvusClient(​ + uri="http://localhost:19530"​ +)​ +​ +# Create schema​ +schema = client.create_schema(auto_id=True, enable_dynamic_field=False)​ +​ +# Add fields to schema​ +​ +# Use a built-in analyzer​ +analyzer_params_built_in = {​ + "type": "english"​ +}​ +​ +# Add VARCHAR field `title_en`​ +schema.add_field(​ + field_name='title_en', ​ + datatype=DataType.VARCHAR, ​ + max_length=1000, ​ + enable_analyzer=True，​ + analyzer_params=analyzer_params_built_in,​ + enable_match=True, ​ +)​ +​ +# Configure a custom analyzer​ +analyzer_params_custom = {​ + "tokenizer": "standard",​ + "filter": [​ + "lowercase", # Built-in filter​ + {​ + "type": "length", # Custom filter​ + "max": 40​ + },​ + {​ + "type": "stop", # Custom filter​ + "stop_words": ["of", "to"]​ + }​ + ]​ +}​ +​ +# Add VARCHAR field `title`​ +schema.add_field(​ + field_name='title', ​ + datatype=DataType.VARCHAR, ​ + max_length=1000, ​ + enable_analyzer=True，​ + analyzer_params=analyzer_params_custom,​ + enable_match=True, ​ +)​ +​ +# Add vector field​ +schema.add_field(field_name="embedding", datatype=DataType.FLOAT_VECTOR, dim=3)​ +# Add primary field​ +schema.add_field(field_name="id", datatype=DataType.INT64, is_primary=True)​ +​ +# Set up index params for vector field​ +index_params = client.prepare_index_params()​ +index_params.add_index(field_name="embedding", metric_type="COSINE", index_type="AUTOINDEX")​ +​ +# Create collection with defined schema​ +client.create_collection(​ + collection_name="YOUR_COLLECTION_NAME",​ + schema=schema,​ + index_params=index_params​ +)​ +``` + +​ + diff --git a/v2.4.x/site/en/userGuide/schema/analyzer/analyzer/chinese-analyzer.md b/v2.4.x/site/en/userGuide/schema/analyzer/analyzer/chinese-analyzer.md new file mode 100644 index 0000000000..2f40150132 --- /dev/null +++ b/v2.4.x/site/en/userGuide/schema/analyzer/analyzer/chinese-analyzer.md @@ -0,0 +1,59 @@ +--- +id: chinese-analyzer.md +title: Chinese Analyzer +related_key: chinese, analyzer +summary: The `chinese` analyzer is designed specifically to handle Chinese text, providing effective segmentation and tokenization.​ +--- + +# Chinese​ + +The `chinese` analyzer is designed specifically to handle Chinese text, providing effective segmentation and tokenization.​ + +### Definition​ + +The `chinese` analyzer consists of:​ + +- **Tokenizer**: Uses the `jieba` tokenizer to segment Chinese text into tokens based on vocabulary and context. For more information, refer to [​Jieba](jieba-tokenizer.md).​ + +- **Filter**: Uses the `cnalphanumonly` filter to remove tokens that contain any non-Chinese characters. For more information, refer to [​Cnalphanumonly](cnalphanumonly-filter.md).​ + +The functionality of the `chinese` analyzer is equivalent to the following custom analyzer configuration:​ + +```python +analyzer_params = {​ + "tokenizer": "jieba",​ + "filter": ["cnalphanumonly"]​ +}​ +``` + +### Configuration​ + +To apply the `chinese` analyzer to a field, simply set `type` to `chinese` in `analyzer_params`.​ + +```python +analyzer_params = {​ + "type": "chinese",​ +}​ +``` + +
+ +The `chinese` analyzer does not accept any optional parameters.​ + +
+ +### Example output​ + +Here’s how the `chinese` analyzer processes text.​ + +**Original text**:​ + +```python +"Milvus 是一个高性能、可扩展的向量数据库！"​ +``` + +**Expected output**:​ + +```python +["Milvus", "是", "一个", "高性", "性能", "高性能", "可", "扩展", "的", "向量", "数据", "据库", "数据库"]​ +``` diff --git a/v2.4.x/site/en/userGuide/schema/analyzer/analyzer/english-analyzer.md b/v2.4.x/site/en/userGuide/schema/analyzer/analyzer/english-analyzer.md new file mode 100644 index 0000000000..c4c89390ec --- /dev/null +++ b/v2.4.x/site/en/userGuide/schema/analyzer/analyzer/english-analyzer.md @@ -0,0 +1,91 @@ +--- +id: english-analyzer.md +title: English Analyzer +related_key: english, analyzer +summary: The `english` analyzer in Milvus is designed to process English text, applying language-specific rules for tokenization and filtering.​ +--- + +# English​ + +The `english` analyzer in Milvus is designed to process English text, applying language-specific rules for tokenization and filtering.​ + +### Definition​ + +The `english` analyzer uses the following components:​ + +- **Tokenizer**: Uses the [`standard tokenizer`](standard-tokenizer.md) to split text into discrete word units.​ + +- Filters: Includes multiple filters for comprehensive text processing:​ + + - [`lowercase`](lowercase-filter.md): Converts all tokens to lowercase, enabling case-insensitive searches.​ + + - [`stemmer`](stemmer-filter.md): Reduces words to their root form to support broader matching (e.g., "running" becomes "run").​ + + - [`stop_words`](stop-filter.md): Removes common English stop words to focus on key terms in text.​ + +The functionality of the `english` analyzer is equivalent to the following custom analyzer configuration:​ + +```python +analyzer_params = {​ + "tokenizer": "standard",​ + "filter": [​ + "lowercase",​ + {​ + "type": "stemmer",​ + "language": "english"​ + }，{​ + "type": "stop",​ + "stop_words": "_english_",​ + }​ + ]​ +}​ +``` + +### Configuration​ + +To apply the `english` analyzer to a field, simply set `type` to `english` in `analyzer_params`, and include optional parameters as needed.​ + +```python +analyzer_params = {​ + "type": "english",​ +}​ +``` + +The `english` analyzer accepts the following optional parameters: ​ + +

Parameter​

+ +

Description​

+ +

stop_words​

+ +

An array containing a list of stop words, which will be removed from tokenization. Defaults to _english_, a built-in set of common English stop words.​

+ +
+ +Example configuration with custom stop words:​ + +```python +analyzer_params = {​ + "type": "english",​ + "stop_words": ["a", "an", "the"]​ +}​ +``` + +After defining `analyzer_params`, you can apply them to a `VARCHAR` field when defining a collection schema. This allows Milvus to process the text in that field using the specified analyzer for efficient tokenization and filtering. For details, refer to [Example use](analyzer-overview.md#Example-use).​ + +### Example output​ + +Here’s how the `english` analyzer processes text.​ + +**Original text**:​ + +```python +"The Milvus vector database is built for scale!"​ +``` + +**Expected output**:​ + +```python +["milvus", "vector", "databas", "built", "scale"]​ +``` \ No newline at end of file diff --git a/v2.4.x/site/en/userGuide/schema/analyzer/analyzer/standard-analyzer.md b/v2.4.x/site/en/userGuide/schema/analyzer/analyzer/standard-analyzer.md new file mode 100644 index 0000000000..a35f471184 --- /dev/null +++ b/v2.4.x/site/en/userGuide/schema/analyzer/analyzer/standard-analyzer.md @@ -0,0 +1,76 @@ +--- +id: standard-analyzer.md +title: Standard​ Analyzer +related_key: standard, analyzer +summary: The `standard` analyzer is the default analyzer in Milvus, which is automatically applied to text fields if no analyzer is specified. It uses grammar-based tokenization, making it effective for most languages.​ +--- + +# Standard​ + +The `standard` analyzer is the default analyzer in Milvus, which is automatically applied to text fields if no analyzer is specified. It uses grammar-based tokenization, making it effective for most languages.​ + +## Definition​ + +The `standard` analyzer consists of:​ + +- **Tokenizer**: Uses the `standard` tokenizer to split text into discrete word units based on grammar rules. For more information, refer to [​Standard](standard-tokenizer.md).​ + +- **Filter**: Uses the `lowercase` filter to convert all tokens to lowercase, enabling case-insensitive searches. For more information, refer to ​[`lowercase filter`](lowercase-filter.md). + +The functionality of the `standard` analyzer is equivalent to the following custom analyzer configuration:​ + +```python +analyzer_params = {​ + "tokenizer": "standard",​ + "filter": ["lowercase"]​ +}​ +``` + +## Configuration​ + +To apply the `standard` analyzer to a field, simply set `type` to `standard` in `analyzer_params`, and include optional parameters as needed.​ + +```python +analyzer_params = {​ + "type": "standard", # Specifies the standard analyzer type​ +}​ +``` + +The `standard` analyzer accepts the following optional parameters: ​ + +

Parameter​

+ +

Description​

+ +

stop_words​

+ +

An array containing a list of stop words, which will be removed from tokenization. Defaults to _english_, a built-in set of common English stop words. The details of _english_ can be found here.​

+ +
+ +Example configuration of custom stop words:​ + +```python +analyzer_params = {​ + "type": "standard", # Specifies the standard analyzer type​ + "stop_words", ["of"] # Optional: List of words to exclude from tokenization​ +}​ +``` + +After defining `analyzer_params`, you can apply them to a `VARCHAR` field when defining a collection schema. This allows Milvus to process the text in that field using the specified analyzer for efficient tokenization and filtering. For more information, refer to [Example use](analyzer-overview.md#).​ + +## Example output​ + +Here’s how the `standard` analyzer processes text.​ + +**Original text**:​ + +```python +"The Milvus vector database is built for scale!"​ +``` + +**Expected output**:​ + +```python +["the", "milvus", "vector", "database", "is", "built", "for", "scale"]​ +``` \ No newline at end of file diff --git a/v2.4.x/site/en/userGuide/schema/analyzer/filter/alphanumonly-filter.md b/v2.4.x/site/en/userGuide/schema/analyzer/filter/alphanumonly-filter.md new file mode 100644 index 0000000000..f5413a5b59 --- /dev/null +++ b/v2.4.x/site/en/userGuide/schema/analyzer/filter/alphanumonly-filter.md @@ -0,0 +1,40 @@ +--- +id: alphanumonly-filter.md +title: Alphanumonly​ Filter +summary: The `alphanumonly` filter removes tokens that contain non-ASCII characters, keeping only alphanumeric terms. This filter is useful for processing text where only basic letters and numbers are relevant, excluding any special characters or symbols. +--- + +# Alphanumonly​ + +The `alphanumonly` filter removes tokens that contain non-ASCII characters, keeping only alphanumeric terms. This filter is useful for processing text where only basic letters and numbers are relevant, excluding any special characters or symbols.​ + +## Configuration​ + +The `alphanumonly` filter is built into Milvus. To use it, simply specify its name in the `filter` section within `analyzer_params`.​ + +```python +analyzer_params = {​ + "tokenizer": "standard",​ + "filter": ["alphanumonly"],​ +}​ +``` + +The `alphanumonly` filter operates on the terms generated by the tokenizer, so it must be used in combination with a tokenizer. + +After defining `analyzer_params`, you can apply them to a `VARCHAR` field when defining a collection schema. This allows Milvus to process the text in that field using the specified analyzer for efficient tokenization and filtering. For details, refer to [Example use](analyzer-overview.md#Example-use).​ + +## Example output​ + +Here’s an example of how the `alphanumonly` filter processes text:​ + +**Original text**:​ + +```python +"Milvus 2.0 @ Scale! #AI #Vector_Databasé"​ +``` + +**Expected output**:​ + +```python +["Milvus", "2", "0", "Scale", "AI", "Vector"]​ +``` \ No newline at end of file diff --git a/v2.4.x/site/en/userGuide/schema/analyzer/filter/ascii-folding-filter.md b/v2.4.x/site/en/userGuide/schema/analyzer/filter/ascii-folding-filter.md new file mode 100644 index 0000000000..d4e5822d30 --- /dev/null +++ b/v2.4.x/site/en/userGuide/schema/analyzer/filter/ascii-folding-filter.md @@ -0,0 +1,40 @@ +--- +id: ascii-folding-filter.md +title: ASCII folding​ +summary: The `asciifolding`** **filter converts characters outside the Basic Latin Unicode block (the first 127 ASCII characters) into their ASCII equivalents. +--- + +# ASCII folding​ + +The `asciifolding`** **filter converts characters outside the [Basic Latin Unicode block](https://en.wikipedia.org/wiki/Basic_Latin_(Unicode_block)) (the first 127 ASCII characters) into their ASCII equivalents. For instance, it transforms characters like `í` to `i`, making text processing simpler and more consistent, especially for multilingual content.​ + +## Configuration​ + +The `asciifolding` filter is built into Milvus. To use it, simply specify its name in the `filter` section within `analyzer_params`.​ + +```python +analyzer_params = {​ + "tokenizer": "standard",​ + "filter": ["asciifolding"],​ +}​ +``` + +The `asciifolding` filter operates on the terms generated by the tokenizer, so it must be used in combination with a tokenizer. + +After defining `analyzer_params`, you can apply them to a `VARCHAR` field when defining a collection schema. This allows Milvus to process the text in that field using the specified analyzer for efficient tokenization and filtering. For details, refer to [Example use](analyzer-overview.md#Example-use).​ + +## Example output​ + +Here’s an example of how the `asciifolding` filter processes text:​ + +**Original text**:​ + +```python +"Café Möller serves crème brûlée and piñatas."​ +``` + +**Expected output**:​ + +```python +["Cafe", "Moller", "serves", "creme", "brulee", "and", "pinatas"]​ +``` \ No newline at end of file diff --git a/v2.4.x/site/en/userGuide/schema/analyzer/filter/cnalphanumonly-filter.md b/v2.4.x/site/en/userGuide/schema/analyzer/filter/cnalphanumonly-filter.md new file mode 100644 index 0000000000..81968e51c5 --- /dev/null +++ b/v2.4.x/site/en/userGuide/schema/analyzer/filter/cnalphanumonly-filter.md @@ -0,0 +1,41 @@ +--- +id: cnalphanumonly-filter.md +title: Cnalphanumonly​ Filter +summary: The `cnalphanumonly` filter removes tokens that contain any characters other than Chinese characters, English letters, or digits.​ +--- + +# Cnalphanumonly​ + +The `cnalphanumonly` filter removes tokens that contain any characters other than Chinese characters, English letters, or digits.​ + +## Configuration​ + +The `cnalphanumonly` filter is built into Milvus. To use it, simply specify its name in the `filter` section within `analyzer_params`.​ + +```python +analyzer_params = {​ + "tokenizer": "standard",​ + "filter": ["cnalphanumonly"],​ +}​ + +``` + +The `cnalphanumonly` filter operates on the terms generated by the tokenizer, so it must be used in combination with a tokenizer. + +After defining `analyzer_params`, you can apply them to a `VARCHAR` field when defining a collection schema. This allows Milvus to process the text in that field using the specified analyzer for efficient tokenization and filtering. For details, refer to [Example use](analyzer-overview.md#Example-use).​ + +## Example output​ + +Here’s an example of how the `cnalphanumonly` filter processes text:​ + +**Original text**:​ + +```python +"Milvus 是 LF AI & Data Foundation 下的一个开源项目，以 Apache 2.0 许可发布。"​ +``` + +**Expected output**:​ + +```python +["Milvus", "是", "LF", "AI", "Data", "Foundation", "下", "的", "一个", "开源", "项目", "以", "Apache", "2.0", "许可", "发布"]​ +``` \ No newline at end of file diff --git a/v2.4.x/site/en/userGuide/schema/analyzer/filter/cncharonly-filter.md b/v2.4.x/site/en/userGuide/schema/analyzer/filter/cncharonly-filter.md new file mode 100644 index 0000000000..fdb20807c2 --- /dev/null +++ b/v2.4.x/site/en/userGuide/schema/analyzer/filter/cncharonly-filter.md @@ -0,0 +1,40 @@ +--- +id: cncharonly-filter.md +title: Cncharonly​ Filter +summary: The `cnalphanumonly` filter removes tokens that contain any characters other than Chinese characters, English letters, or digits.​ +--- + +# Cncharonly​ + +The `cncharonly` filter removes tokens that contain any non-Chinese characters. This filter is useful when you want to focus solely on Chinese text, filtering out any tokens that contain other scripts, numbers, or symbols.​ + +## Configuration​ + +The `cncharonly` filter is built into Milvus. To use it, simply specify its name in the `filter` section within `analyzer_params`.​ + +```python +analyzer_params = {​ + "tokenizer": "standard",​ + "filter": ["cncharonly"],​ +}​ +``` + +The `cncharonly` filter operates on the terms generated by the tokenizer, so it must be used in combination with a tokenizer. + +After defining `analyzer_params`, you can apply them to a `VARCHAR` field when defining a collection schema. This allows Milvus to process the text in that field using the specified analyzer for efficient tokenization and filtering. For details, refer to [Example use](analyzer-overview.md#Example-use).​ + +## Example output​ + +Here’s an example of how the `cncharonly` filter processes text:​ + +**Original text**:​ + +```python +"Milvus 是 LF AI & Data Foundation 下的一个开源项目，以 Apache 2.0 许可发布。"​ +``` + +**Expected output**:​ + +```python +["是", "下", "的", "一个", "开源", "项目", "以", "许可", "发布"]​ +``` \ No newline at end of file diff --git a/v2.4.x/site/en/userGuide/schema/analyzer/filter/decompounder-filter.md b/v2.4.x/site/en/userGuide/schema/analyzer/filter/decompounder-filter.md new file mode 100644 index 0000000000..20b0b1f933 --- /dev/null +++ b/v2.4.x/site/en/userGuide/schema/analyzer/filter/decompounder-filter.md @@ -0,0 +1,55 @@ +--- +id: decompounder-filter.md +title: Decompounder​ Filter +summary: The `decompounder` filter splits compound words into individual components based on a specified dictionary, making it easier to search for parts of compound terms. This filter is particularly useful for languages that frequently use compound words, such as German.​ +--- + +# Decompounder​ + +The `decompounder` filter splits compound words into individual components based on a specified dictionary, making it easier to search for parts of compound terms. This filter is particularly useful for languages that frequently use compound words, such as German.​ + +## Configuration​ + +The `decompounder` filter is a custom filter in Milvus. To use it, specify `"type": "decompounder"` in the filter configuration, along with a `word_list` parameter that provides the dictionary of word components to recognize.​ + +```python +analyzer_params = {​ + "tokenizer": "standard",​ + "filter":[{​ + "type": "decompounder", # Specifies the filter type as decompounder​ + "word_list": ["dampf", "schiff", "fahrt", "brot", "backen", "automat"],​ + }],​ +}​ +``` + +The `decompounder` filter accepts the following configurable parameters.​ + +

Parameter​

+ +

Description​

+ +

word_list​

+ +

A list of word components used to split compound terms. This dictionary determines how compound words are decomposed into individual terms.​

+ +
+ +The `decompounder` filter operates on the terms generated by the tokenizer, so it must be used in combination with a tokenizer. + +After defining `analyzer_params`, you can apply them to a `VARCHAR` field when defining a collection schema. This allows Milvus to process the text in that field using the specified analyzer for efficient tokenization and filtering. For details, refer to [Example use](analyzer-overview.md#Example-use).​ + +## Example output​ + +Here’s an example of how the `decompounder` filter processes text:​ + +**Original text**:​ + +```python +"dampfschifffahrt brotbackautomat"​ +``` + +**Expected output** (with `word_list: ["dampf", "schiff", "fahrt", "brot", "backen", "automat"]`):​ + +```python +["dampf", "schiff", "fahrt", "brotbackautomat"]​ +``` \ No newline at end of file diff --git a/v2.4.x/site/en/userGuide/schema/analyzer/filter/length-filter.md b/v2.4.x/site/en/userGuide/schema/analyzer/filter/length-filter.md new file mode 100644 index 0000000000..0cfb19163d --- /dev/null +++ b/v2.4.x/site/en/userGuide/schema/analyzer/filter/length-filter.md @@ -0,0 +1,55 @@ +--- +id: length-filter.md +title: Length​ Filter +summary: "The `length` filter removes tokens that do not meet specified length requirements, allowing you to control the length of tokens retained during text processing.​" +--- + +# Length​ + +The `length` filter removes tokens that do not meet specified length requirements, allowing you to control the length of tokens retained during text processing.​ + +## Configuration + +The `length` filter is a custom filter in Milvus, specified by setting `"type": "length"` in the filter configuration. You can configure it as a dictionary within the `analyzer_params` to define length limits.​ + +```python +analyzer_params = {​ + "tokenizer": "standard",​ + "filter":[{​ + "type": "length", # Specifies the filter type as length​ + "max": 10, # Sets the maximum token length to 10 characters​ + }],​ +}​ +``` + +The `length` filter accepts the following configurable parameters.​ + +

Parameter​

+ +

Description​

+ +

max​

+ +

Sets the maximum token length. Tokens longer than this length are removed.​

+ +
+ +The `length` filter operates on the terms generated by the tokenizer, so it must be used in combination with a tokenizer. + +After defining `analyzer_params`, you can apply them to a `VARCHAR` field when defining a collection schema. This allows Milvus to process the text in that field using the specified analyzer for efficient tokenization and filtering. For details, refer to [Example use](analyzer-overview.md#Example-use).​ + +## Example output + +Here’s an example of how the `length` filter processes text:​ + +**Example text**:​ + +```python +"The length filter allows control over token length requirements for text processing."​ +``` + +**Expected output** (with `max: 10`):​ + +```python +["length", "filter", "allows", "control", "over", "token", "length", "for", "text"]​ +``` \ No newline at end of file diff --git a/v2.4.x/site/en/userGuide/schema/analyzer/filter/lowercase-filter.md b/v2.4.x/site/en/userGuide/schema/analyzer/filter/lowercase-filter.md new file mode 100644 index 0000000000..ab62cd81b4 --- /dev/null +++ b/v2.4.x/site/en/userGuide/schema/analyzer/filter/lowercase-filter.md @@ -0,0 +1,40 @@ +--- +id: lowercase-filter.md +title: Lowercase​ Filter +summary: The `lowercase` filter converts terms generated by a tokenizer to lowercase, making searches case-insensitive. +--- + +# Lowercase​ + +The `lowercase` filter converts terms generated by a tokenizer to lowercase, making searches case-insensitive. For example, it can convert `["High", "Performance", "Vector", "Database"]` to `["high", "performance", "vector", "database"]`.​ + +## Configuration​ + +The `lowercase` filter is built into Milvus. To use it, simply specify its name in the `filter` section within `analyzer_params`.​ + +```python +analyzer_params = {​ + "tokenizer": "standard",​ + "filter": ["lowercase"],​ +}​ +``` + +The `lowercase` filter operates on the terms generated by the tokenizer, so it must be used in combination with a tokenizer. + +After defining `analyzer_params`, you can apply them to a `VARCHAR` field when defining a collection schema. This allows Milvus to process the text in that field using the specified analyzer for efficient tokenization and filtering. For details, refer to [Example use](analyzer-overview.md#Example-use).​ + +## Example output​ + +Here’s an example of how the `lowercase` filter processes text:​ + +**Original text**:​ + +```python +"The Lowercase Filter Ensures Uniformity In Text Processing."​ +``` + +**Expected output**:​ + +```python +["the", "lowercase", "filter", "ensures", "uniformity", "in", "text", "processing"]​ +``` diff --git a/v2.4.x/site/en/userGuide/schema/analyzer/filter/stemmer-filter.md b/v2.4.x/site/en/userGuide/schema/analyzer/filter/stemmer-filter.md new file mode 100644 index 0000000000..c18b933a7d --- /dev/null +++ b/v2.4.x/site/en/userGuide/schema/analyzer/filter/stemmer-filter.md @@ -0,0 +1,55 @@ +--- +id: stemmer-filter.md +title: Stemmer​ Filter +summary: The `stemmer` filter reduces words to their base or root form (known as stemming), making it easier to match words with similar meanings across different inflections. +--- + +# Stemmer​ + +The `stemmer` filter reduces words to their base or root form (known as stemming), making it easier to match words with similar meanings across different inflections. The `stemmer` filter supports multiple languages, allowing for effective search and indexing in various linguistic contexts.​ + +## Configuration​ + +The `stemmer` filter is a custom filter in Milvus. To use it, specify `"type": "stemmer"` in the filter configuration, along with a `language` parameter to select the desired language for stemming.​ + +```python +analyzer_params = {​ + "tokenizer": "standard",​ + "filter":[{​ + "type": "stemmer", # Specifies the filter type as stemmer​ + "language": "english", # Sets the language for stemming to English​ + }],​ +}​ +``` + +The `stemmer` filter accepts the following configurable parameters.​ + +

Parameter​

+ +

Description​

+ +

language​

+ +

Specifies the language for the stemming process. Supported languages include: "arabic", "danish", "dutch", "english", "finnish", "french", "german", "greek", "hungarian", "italian", "norwegian", "portuguese", "romanian", "russian", "spanish", "swedish", "tamil", "turkish"​

+ +
+ +The `stemmer` filter operates on the terms generated by the tokenizer, so it must be used in combination with a tokenizer. + +After defining `analyzer_params`, you can apply them to a `VARCHAR` field when defining a collection schema. This allows Milvus to process the text in that field using the specified analyzer for efficient tokenization and filtering. For details, refer to [Example use](analyzer-overview.md#Example-use).​ + +## Example output​ + +Here’s an example of how the `stemmer` filter processes text:​ + +**Original text**:​ + +```python +"running runs looked ran runner"​ +``` + +**Expected output** (with `language: "english"`):​ + +```python +["run", "run", "look", "ran", "runner"]​ +``` \ No newline at end of file diff --git a/v2.4.x/site/en/userGuide/schema/analyzer/filter/stop-filter.md b/v2.4.x/site/en/userGuide/schema/analyzer/filter/stop-filter.md new file mode 100644 index 0000000000..adbd921ab8 --- /dev/null +++ b/v2.4.x/site/en/userGuide/schema/analyzer/filter/stop-filter.md @@ -0,0 +1,55 @@ +--- +id: stop-filter.md +title: Stop​ Filter +summary: The `stop` filter removes specified stop words from tokenized text, helping to eliminate common, less meaningful words. You can configure the list of stop words using the `stop_words` parameter. +--- + +# Stop​ + +The `stop` filter removes specified stop words from tokenized text, helping to eliminate common, less meaningful words. You can configure the list of stop words using the `stop_words` parameter.​ + +## Configuration​ + +The `length` filter is a custom filter in Milvus. To use it, specify `"type": "stop"` in the filter configuration, along with a `stop_words` parameter that provides a list of stop words.​ + +```python +analyzer_params = {​ + "tokenizer": "standard",​ + "filter":[{​ + "type": "stop", # Specifies the filter type as stop​ + "stop_words": ["of", "to", "_english_"], # Defines custom stop words and includes the English stop word list​ + }],​ +}​ +``` + +The `stop` filter accepts the following configurable parameters.​ + +

Parameter​

+ +

Description​

+ +

stop_words​

+ +

A list of words to be removed from tokenization. By default, the predefined _english_ list, containing common English stop words, is used. The details of _english_ can be found here.​

+ +
+ +The `stop` filter operates on the terms generated by the tokenizer, so it must be used in combination with a tokenizer. + +After defining `analyzer_params`, you can apply them to a `VARCHAR` field when defining a collection schema. This allows Milvus to process the text in that field using the specified analyzer for efficient tokenization and filtering. For details, refer to [Example use](analyzer-overview.md#Example-use).​ + +## Example output​ + +Here’s an example of how the `stop` filter processes text:​ + +**Original text**:​ + +```python +"The stop filter allows control over common stop words for text processing."​ +``` + +**Expected output** (with `stop_words: ["the", "over", "_english_"]`):​ + +```python +["The", "stop", "filter", "allows", "control", "common", "stop", "words", "text", "processing"]​ +``` \ No newline at end of file diff --git a/v2.4.x/site/en/userGuide/schema/analyzer/tokenizer/jieba-tokenizer.md b/v2.4.x/site/en/userGuide/schema/analyzer/tokenizer/jieba-tokenizer.md new file mode 100644 index 0000000000..8a1a91fe41 --- /dev/null +++ b/v2.4.x/site/en/userGuide/schema/analyzer/tokenizer/jieba-tokenizer.md @@ -0,0 +1,37 @@ +--- +id: jieba-tokenizer.md +title: Jieba​ Tokenizer +summary: "The `jieba` tokenizer processes Chinese text by breaking it down into its component words.​" +--- + +# Jieba​ + +The `jieba` tokenizer processes Chinese text by breaking it down into its component words.​ + +## Configuration​ + +To configure an analyzer using the `jieba` tokenizer, set `tokenizer` to `jieba` in `analyzer_params`.​ + +```python +analyzer_params = {​ + "tokenizer": "jieba",​ +}​ +``` + +After defining `analyzer_params`, you can apply them to a `VARCHAR` field when defining a collection schema. This allows Milvus to process the text in that field using the specified analyzer for efficient tokenization and filtering. For details, refer to [Example use](analyzer-overview.md#Example-use).​ + +## Example output​ + +Here’s an example of how the `jieba` tokenizer processes text:​ + +**Original text**:​ + +```python +"Milvus 是一个高性能、可扩展的向量数据库！"​ +``` + +**Expected output**:​ + +```python +["Milvus", " ", "是", "一个", "高性", "性能", "高性能", "、", "可", "扩展", "的", "向量", "数据", "据库", "数据库", "！"]​ +``` \ No newline at end of file diff --git a/v2.4.x/site/en/userGuide/schema/analyzer/tokenizer/standard-tokenizer.md b/v2.4.x/site/en/userGuide/schema/analyzer/tokenizer/standard-tokenizer.md new file mode 100644 index 0000000000..8ab2e39d71 --- /dev/null +++ b/v2.4.x/site/en/userGuide/schema/analyzer/tokenizer/standard-tokenizer.md @@ -0,0 +1,54 @@ +--- +id: standard-tokenizer.md +title: Standard​ Tokenizer +summary: "The `standard` tokenizer in Milvus splits text based on spaces and punctuation marks, making it suitable for most languages.​" +--- + +# Standard​ + +The `standard` tokenizer in Milvus splits text based on spaces and punctuation marks, making it suitable for most languages.​ + +## Configuration​ + +To configure an analyzer using the `standard` tokenizer, set `tokenizer` to `standard` in `analyzer_params`.​ + +```python +analyzer_params = {​ + "tokenizer": "standard",​ +}​ + +``` + +The `standard` tokenizer can work in conjunction with one or more filters. For example, the following code defines an analyzer that uses the `standard` tokenizer and `lowercase` filter:​ + +```python +analyzer_params = {​ + "tokenizer": "standard",​ + "filter": ["lowercase"]​ +}​ + +``` + +
+ +For simpler setup, you may choose to use the [`standard analyzer`](standard-analyzer.md), which combines the `standard` tokenizer with the [`lowercase filter`](lowercase-filter.md).​ + +
+ +After defining `analyzer_params`, you can apply them to a `VARCHAR` field when defining a collection schema. This allows Milvus to process the text in that field using the specified analyzer for efficient tokenization and filtering. For details, refer to [Example use](analyzer-overview.md#Example-use).​ + +## Example output​ + +Here’s an example of how the `standard` tokenizer processes text:​ + +**Original text**:​ + +```python +"The Milvus vector database is built for scale!"​ +``` + +**Expected output**:​ + +```python +["The", "Milvus", "vector", "database", "is", "built", "for", "scale"]​ +``` \ No newline at end of file diff --git a/v2.4.x/site/en/userGuide/schema/analyzer/tokenizer/whitespace-tokenizer.md b/v2.4.x/site/en/userGuide/schema/analyzer/tokenizer/whitespace-tokenizer.md new file mode 100644 index 0000000000..ccacced100 --- /dev/null +++ b/v2.4.x/site/en/userGuide/schema/analyzer/tokenizer/whitespace-tokenizer.md @@ -0,0 +1,46 @@ +--- +id: whitespace-tokenizer.md +title: Whitespace​ Tokenizer +summary: "The `whitespace` tokenizer divides text into terms whenever there is a space between words.​" +--- + +# Whitespace​ + +The `whitespace` tokenizer divides text into terms whenever there is a space between words.​ + +## Configuration​ + +To configure an analyzer using the `whitespace` tokenizer, set `tokenizer` to `whitespace` in `analyzer_params`.​ + +```python +analyzer_params = {​ + "tokenizer": "whitespace",​ +}​ +``` + +The whitespace tokenizer can work in conjunction with one or more filters. For example, the following code defines an analyzer that uses the `whitespace` tokenizer and [`lowercase filter`](lowercase-filter.md):​ + +```python +analyzer_params = {​ + "tokenizer": "whitespace",​ + "filter": ["lowercase"]​ +}​ +``` + +After defining `analyzer_params`, you can apply them to a `VARCHAR` field when defining a collection schema. This allows Milvus to process the text in that field using the specified analyzer for efficient tokenization and filtering. For details, refer to [Example use](analyzer-overview.md#Example-use).​ + +## Example output​ + +Here’s an example of how the `whitespace` tokenizer processes text:​ + +**Original text**:​ + +```python +"The Milvus vector database is built for scale!"​ +``` + +**Expected output**:​ + +```python +["The", "Milvus", "vector", "database", "is", "built", "for", "scale!"]​ +``` \ No newline at end of file diff --git a/v2.4.x/site/en/userGuide/schema/array_data_type.md b/v2.4.x/site/en/userGuide/schema/array_data_type.md new file mode 100644 index 0000000000..e3bc688fa8 --- /dev/null +++ b/v2.4.x/site/en/userGuide/schema/array_data_type.md @@ -0,0 +1,664 @@ +--- +id: array.md +title: Array Field​​ +summary: The Array type is used to store fields containing multiple values of the same data type. It provides a flexible way to store attributes with multiple elements, making it especially useful in scenarios where a set of related data needs to be saved. In Milvus, you can store Array fields alongside vector data, enabling more complex query and filtering requirements.​ +--- + +# Array Field​ + +The Array type is used to store fields containing multiple values of the same data type. It provides a flexible way to store attributes with multiple elements, making it especially useful in scenarios where a set of related data needs to be saved. In Milvus, you can store Array fields alongside vector data, enabling more complex query and filtering requirements.​ + +For example, in a music recommendation system, an Array field can store a list of tags for a song; in user behavior analysis, it can store user ratings for songs. Below is an example of a typical Array field:​ + +```JSON +{​ + "tags": ["pop", "rock", "classic"],​ + "ratings": [5, 4, 3]​ +}​ + +``` + +In this example, `tags` and `ratings` are both Array fields. The `tags` field is a string array representing song genres like pop, rock, and classic, while the `ratings` field is an integer array representing user ratings for the song, ranging from 1 to 5. These Array fields provide a flexible way to store multi-value data, making it easier to perform detailed analysis during queries and filtering.​ + +## Add Array field​ + +To use Array fields in Milvus, define the relevant field type when creating the collection schema. This process includes:​ + +1. Setting `datatype` to the supported Array data type, `ARRAY`.​ + +2. Using the `element_type` parameter to specify the data type of elements in the array. This can be any scalar data type supported by Milvus, such as `VARCHAR` or `INT64`. All elements in the same Array must be of the same data type.​ + +3. Using the `max_capacity` parameter to define the maximum capacity of the array, i.e., the maximum number of elements it can contain.​ + +Here’s how to define a collection schema that includes Array fields:​ + +
+ Python + Java + Node.js + Go + cURL +
+ +```python +from pymilvus import MilvusClient, DataType​ +​ +client = MilvusClient(uri="http://localhost:19530")​ +​ +schema = client.create_schema(​ + auto_id=False,​ + enable_dynamic_fields=True,​ +)​ +​ +# Add an Array field with elements of type VARCHAR​ +schema.add_field(field_name="tags", datatype=DataType.ARRAY, element_type=DataType.VARCHAR, max_capacity=10)​ +# Add an Array field with elements of type INT64​ +schema.add_field(field_name="ratings", datatype=DataType.ARRAY, element_type=DataType.INT64, max_capacity=5)​ +​ +# Add primary field​ +schema.add_field(field_name="pk", datatype=DataType.INT64, is_primary=True)​ +​ +# Add vector field​ +schema.add_field(field_name="embedding", datatype=DataType.FLOAT_VECTOR, dim=3)​ + +``` + +```java +import io.milvus.v2.client.ConnectConfig;​ +import io.milvus.v2.client.MilvusClientV2;​ +import io.milvus.v2.common.DataType;​ +import io.milvus.v2.service.collection.request.AddFieldReq;​ +import io.milvus.v2.service.collection.request.CreateCollectionReq;​ +​ +MilvusClientV2 client = new MilvusClientV2(ConnectConfig.builder()​ + .uri("http://localhost:19530")​ + .build());​ + ​ +CreateCollectionReq.CollectionSchema schema = client.createSchema();​ +schema.setEnableDynamicField(true);​ +​ +schema.addField(AddFieldReq.builder()​ + .fieldName("tags")​ + .dataType(DataType.Array)​ + .elementType(DataType.VarChar)​ + .maxCapacity(10)​ + .build());​ +​ +schema.addField(AddFieldReq.builder()​ + .fieldName("ratings")​ + .dataType(DataType.Array)​ + .elementType(DataType.Int64)​ + .maxCapacity(5)​ + .build());​ +​ +schema.addField(AddFieldReq.builder()​ + .fieldName("pk")​ + .dataType(DataType.Int64)​ + .isPrimaryKey(true)​ + .build());​ +​ +schema.addField(AddFieldReq.builder()​ + .fieldName("embedding")​ + .dataType(DataType.FloatVector)​ + .dimension(3)​ + .build());​ + +``` + +```javascript +import { MilvusClient, DataType } from "@zilliz/milvus2-sdk-node";​ +const schema = [​ + {​ + name: "tags",​ + data_type: DataType.Array,​ + element_type: DataType.VarChar,​ + max_capacity: 10,​ + max_length: 65535​ + },​ + {​ + name: "rating",​ + data_type: DataType.Array,​ + element_type: DataType.Int64,​ + max_capacity: 5,​ + },​ + {​ + name: "pk",​ + data_type: DataType.Int64,​ + is_primary_key: true,​ + },​ + {​ + name: "embedding",​ + data_type: DataType.FloatVector,​ + dim: 3,​ + },​ +];​ + +``` + +```curl +export arrayField1='{​ + "fieldName": "tags",​ + "dataType": "Array",​ + "elementDataType": "VarChar",​ + "elementTypeParams": {​ + "max_capacity": 10,​ + "max_length": 100​ + }​ +}'​ +​ +export arrayField2='{​ + "fieldName": "ratings",​ + "dataType": "Array",​ + "elementDataType": "Int64",​ + "elementTypeParams": {​ + "max_capacity": 5​ + }​ +}'​ +​ +export pkField='{​ + "fieldName": "pk",​ + "dataType": "Int64",​ + "isPrimary": true​ +}'​ +​ +export vectorField='{​ + "fieldName": "embedding",​ + "dataType": "FloatVector",​ + "elementTypeParams": {​ + "dim": 3​ + }​ +}'​ +​ +export schema="{​ + \"autoID\": false,​ + \"fields\": [​ + $arrayField1,​ + $arrayField2,​ + $pkField,​ + $vectorField​ + ]​ +}"​ + +``` + +In this example:​ + +- `tags` is a string array with `element_type` set to `VARCHAR`, indicating that elements in the array must be strings. `max_capacity` is set to 10, meaning the array can contain up to 10 elements.​ + +- `ratings` is an integer array with `element_type` set to `INT64`, indicating that elements must be integers. `max_capacity` is set to 5, allowing up to 5 ratings.​ + +- We also add a primary key field `pk` and a vector field `embedding`.​ + +
+ +The primary field and vector field are mandatory when you create a collection. The primary field uniquely identifies each entity, while the vector field is crucial for similarity search. For more details, refer to [​Primary Field & AutoID](primary-field.md), [​Dense Vector](dense-vector.md), [​Binary Vector](binary-vector.md), or [​Sparse Vector](sparse_vector.md).​ + +
+ +## Set index params​ + +Setting index parameters for Array fields is optional but can significantly improve retrieval efficiency.​ + +In the following example, we create an `AUTOINDEX` for the `tags` field, which means Milvus will automatically create an appropriate scalar index based on the data type. + +
+ Python + Java + Node.js + Go + cURL +
+ +```python +# Prepare index parameters​ +index_params = client.prepare_index_params() # Prepare IndexParams object​ +​ +index_params.add_index(​ + field_name="tags", # Name of the Array field to index​ + index_type="AUTOINDEX", # Index type​ + index_name="inverted_index" # Index name​ +)​ + +``` + +```java +import io.milvus.v2.common.IndexParam;​ +import java.util.*;​ +​ +List indexes = new ArrayList<>();​ +indexes.add(IndexParam.builder()​ + .fieldName("tags")​ + .indexName("inverted_index")​ + .indexType(IndexParam.IndexType.AUTOINDEX)​ + .build());​ + +``` + +```javascript +const indexParams = [{​ + index_name: 'inverted_index',​ + field_name: 'tags',​ + index_type: IndexType.AUTOINDEX,​ +)];​ + +``` + +```curl +export indexParams='[​ + {​ + "fieldName": "tags",​ + "indexName": "inverted_index",​ + "indexType": "AUTOINDEX"​ + }​ + ]'​ + +``` + +In addition to `AUTOINDEX`, you can specify other scalar index types like `INVERTED` or `BITMAP`. For supported index types, refer to [​Scalar Indexes](index-scalar-fields.md).​ + +Moreover, you must create an index for the vector field before creating the collection. In this example, we use `AUTOINDEX` to simplify vector index setup.​ + +
+ Python + Java + Node.js + Go + cURL +
+ +```python +# Add vector index​ +index_params.add_index(​ + field_name="embedding",​ + index_type="AUTOINDEX", # Use automatic indexing to simplify complex index settings​ + metric_type="COSINE" # Specify similarity metric type, such as L2, COSINE, or IP​ +)​ + +``` + +```java +indexes.add(IndexParam.builder()​ + .fieldName("embedding")​ + .indexType(IndexParam.IndexType.AUTOINDEX)​ + .metricType(IndexParam.MetricType.COSINE)​ + .build());​ + +``` + +```javascript + indexParams.push({​ + index_name: 'embedding_index',​ + field_name: 'embedding',​ + index_type: IndexType.AUTOINDEX,​ +});​ + +``` + +```curl +export indexParams='[​ + {​ + "fieldName": "tags",​ + "indexName": "inverted_index",​ + "indexType": "AUTOINDEX"​ + },​ + {​ + "fieldName": "embedding",​ + "metricType": "COSINE",​ + "indexType": "AUTOINDEX"​ + }​ + ]'​ + +``` + +## Create collection​ + +Use the defined schema and index parameters to create a collection:​ + +
+ Python + Java + Node.js + Go + cURL +
+ +```python +client.create_collection(​ + collection_name="my_array_collection",​ + schema=schema,​ + index_params=index_params​ +)​ + +``` + +```java +CreateCollectionReq requestCreate = CreateCollectionReq.builder()​ + .collectionName("my_array_collection")​ + .collectionSchema(schema)​ + .indexParams(indexes)​ + .build();​ +client.createCollection(requestCreate);​ + +``` + +```javascript +client.create_collection({​ + collection_name: "my_array_collection",​ + schema: schema,​ + index_params: indexParams​ +})​ + +``` + +```curl +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/collections/create" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d "{​ + \"collectionName\": \"my_array_collection\",​ + \"schema\": $schema,​ + \"indexParams\": $indexParams​ +}"​ + +``` + +## Insert data​ + +After creating the collection, you can insert data that includes Array fields.​ + +
+ Python + Java + Node.js + Go + cURL +
+ +```python +data = [​ + {​ + "tags": ["pop", "rock", "classic"],​ + "ratings": [5, 4, 3],​ + "pk": 1,​ + "embedding": [0.12, 0.34, 0.56]​ + },​ + {​ + "tags": ["jazz", "blues"],​ + "ratings": [4, 5],​ + "pk": 2,​ + "embedding": [0.78, 0.91, 0.23]​ + },​ + {​ + "tags": ["electronic", "dance"],​ + "ratings": [3, 3, 4],​ + "pk": 3,​ + "embedding": [0.67, 0.45, 0.89]​ + }​ +]​ +​ +client.insert(​ + collection_name="my_array_collection",​ + data=data​ +)​ + +``` + +```java +import com.google.gson.Gson;​ +import com.google.gson.JsonObject;​ +​ +import io.milvus.v2.service.vector.request.InsertReq;​ +import io.milvus.v2.service.vector.response.InsertResp;​ +​ +List rows = new ArrayList<>();​ +Gson gson = new Gson();​ +rows.add(gson.fromJson("{\"tags\": [\"pop\", \"rock\", \"classic\"], \"ratings\": [5, 4, 3], \"pk\": 1, \"embedding\": [0.1, 0.2, 0.3]}", JsonObject.class));​ +rows.add(gson.fromJson("{\"tags\": [\"jazz\", \"blues\"], \"ratings\": [4, 5], \"pk\": 2, \"embedding\": [0.4, 0.5, 0.6]}", JsonObject.class));​ +rows.add(gson.fromJson("{\"tags\": [\"electronic\", \"dance\"], \"ratings\": [3, 3, 4], \"pk\": 3, \"embedding\": [0.7, 0.8, 0.9]}", JsonObject.class));​ +​ +InsertResp insertR = client.insert(InsertReq.builder()​ + .collectionName("my_array_collection")​ + .data(rows)​ + .build());​ + +``` + +```javascript +const data = [​ + {​ + "tags": ["pop", "rock", "classic"],​ + "ratings": [5, 4, 3],​ + "pk": 1,​ + "embedding": [0.12, 0.34, 0.56]​ + },​ + {​ + "tags": ["jazz", "blues"],​ + "ratings": [4, 5],​ + "pk": 2,​ + "embedding": [0.78, 0.91, 0.23]​ + },​ + {​ + "tags": ["electronic", "dance"],​ + "ratings": [3, 3, 4],​ + "pk": 3,​ + "embedding": [0.67, 0.45, 0.89]​ + }​ +];​ +​ +client.insert({​ + collection_name: "my_array_collection",​ + data: data,​ +});​ + +``` + +```curl +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/insert" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "data": [​ + {​ + "tags": ["pop", "rock", "classic"],​ + "ratings": [5, 4, 3],​ + "pk": 1,​ + "embedding": [0.12, 0.34, 0.56]​ + },​ + {​ + "tags": ["jazz", "blues"],​ + "ratings": [4, 5],​ + "pk": 2,​ + "embedding": [0.78, 0.91, 0.23]​ + },​ + {​ + "tags": ["electronic", "dance"],​ + "ratings": [3, 3, 4],​ + "pk": 3,​ + "embedding": [0.67, 0.45, 0.89]​ + } ​ + ],​ + "collectionName": "my_array_collection"​ +}'​ + +``` + +In this example:​ + +- Each data entry includes a primary field (`pk`), while `tags` and `ratings` are Array fields used to store tags and ratings.​ + +- `embedding` is a 3-dimensional vector field used for vector similarity searches.​ + +## Search and query​ + +Array fields enable scalar filtering during searches, enhancing Milvus's vector search capabilities. You can query based on the properties of Array fields alongside vector similarity searches.​ + +### Filter queries​ + +You can filter data based on properties of Array fields, such as accessing a specific element or checking if an array element meets a certain condition.​ + +
+ Python + Java + Node.js + Go + cURL +
+ +```python +filter = 'ratings[0] < 4'​ +​ +res = client.query(​ + collection_name="my_array_collection",​ + filter=filter,​ + output_fields=["tags", "ratings", "embedding"]​ +)​ +​ +print(res)​ +​ +# Output​ +# data: ["{'pk': 3, 'tags': ['electronic', 'dance'], 'ratings': [3, 3, 4], 'embedding': [np.float32(0.67), np.float32(0.45), np.float32(0.89)]}"] ​ + +``` + +```java +import io.milvus.v2.service.vector.request.QueryReq;​ +import io.milvus.v2.service.vector.response.QueryResp;​ +​ +String filter = "ratings[0] < 4";​ +QueryResp resp = client.query(QueryReq.builder()​ + .collectionName("my_array_collection")​ + .filter(filter)​ + .outputFields(Arrays.asList("tags", "ratings", "embedding"))​ + .build());​ +​ +System.out.println(resp.getQueryResults());​ +​ +// Output​ +//​ +// [QueryResp.QueryResult(entity={ratings=[3, 3, 4], pk=3, embedding=[0.7, 0.8, 0.9], tags=[electronic, dance]})]​ + +``` + +```javascript +client.query({​ + collection_name: 'my_array_collection',​ + filter: 'ratings[0] < 4',​ + output_fields: ['tags', 'ratings', 'embedding']​ +});​ + +``` + +```curl +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/query" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "my_array_collection",​ + "filter": "ratings[0] < 4",​ + "outputFields": ["tags", "ratings", "embedding"]​ +}'​ +# {"code":0,"cost":0,"data":[{"embedding":[0.67,0.45,0.89],"pk":3,"ratings":{"Data":{"LongData":{"data":[3,3,4]}}},"tags":{"Data":{"StringData":{"data":["electronic","dance"]}}}}]}​ + +``` + +In this query, Milvus filters out entities where the first element of the `ratings` array is less than 4, returning entities that match the condition.​ + +### Vector search with Array filtering​ + +By combining vector similarity with Array filtering, you can ensure that the retrieved data is not only similar in semantics but also meets specific conditions, making the search results more accurate and aligned with business needs.​ + +
+ Python + Java + Node.js + Go + cURL +
+ +```python +filter = 'tags[0] == "pop"'​ +​ +res = client.search(​ + collection_name="my_array_collection",​ + data=[[0.3, -0.6, 0.1]],​ + limit=5,​ + search_params={"params": {"nprobe": 10}},​ + output_fields=["tags", "ratings", "embedding"],​ + filter=filter​ +)​ +​ +print(res)​ +​ +# Output​ +# data: ["[{'id': 1, 'distance': 1.1276001930236816, 'entity': {'ratings': [5, 4, 3], 'embedding': [0.11999999731779099, 0.3400000035762787, 0.5600000023841858], 'tags': ['pop', 'rock', 'classic']}}]"]​ + +``` + +```java +import io.milvus.v2.service.vector.request.SearchReq;​ +import io.milvus.v2.service.vector.response.SearchResp;​ +​ +String filter = "tags[0] == \"pop\"";​ +SearchResp resp = client.search(SearchReq.builder()​ + .collectionName("my_array_collection")​ + .annsField("embedding")​ + .data(Collections.singletonList(new FloatVec(new float[]{0.3f, -0.6f, 0.1f})))​ + .topK(5)​ + .outputFields(Arrays.asList("tags", "ratings", "embedding"))​ + .filter(filter)​ + .build());​ +​ +System.out.println(resp.getSearchResults());​ +​ +// Output​ +//​ +// [[SearchResp.SearchResult(entity={ratings=[5, 4, 3], embedding=[0.1, 0.2, 0.3], tags=[pop, rock, classic]}, score=-0.2364331, id=1)]]​ + +``` + +```javascript +client.search({​ + collection_name: 'my_array_collection',​ + data: [0.3, -0.6, 0.1],​ + limit: 5,​ + output_fields: ['tags', 'ratings', 'embdding'],​ + filter: 'tags[0] == "pop"'​ +});​ + +``` + +```curl +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/search" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "my_array_collection",​ + "data": [​ + [0.3, -0.6, 0.1]​ + ],​ + "annsField": "embedding",​ + "limit": 5,​ + "filter": "tags[0] == \"pop\"",​ + "outputFields": ["tags", "ratings", "embedding"]​ +}'​ +​ +# {"code":0,"cost":0,"data":[{"distance":-0.24793813,"embedding":[0.12,0.34,0.56],"id":1,"ratings":{"Data":{"LongData":{"data":[5,4,3]}}},"tags":{"Data":{"StringData":{"data":["pop","rock","classic"]}}}}]}​ + +``` + +In this example, Milvus returns the top 5 entities most similar to the query vector, with the `tags` array's first element being `"pop"`.​ + +Additionally, Milvus supports advanced Array filtering operators like `ARRAY_CONTAINS`, `ARRAY_CONTAINS_ALL`, `ARRAY_CONTAINS_ANY`, and `ARRAY_LENGTH` to further enhance query capabilities. For more details, refer to [​Metadata Filtering](boolean.md).​ + +## Limits​ + +- **Data Type**: All elements in an Array field must have the same data type, as specified by the `element_type`.​ + +- **Array Capacity**: The number of elements in an Array field must be less than or equal to the maximum capacity defined when the Array was created, as specified by `max_capacity`.​ + +- **String Handling**: String values in Array fields are stored as-is, without semantic escaping or conversion. For example, `'a"b'`, `"a'b"`, `'a\'b'`, and `"a\"b"` are stored as entered, while `'a'b'` and `"a"b"` are considered invalid values.​ + diff --git a/v2.4.x/site/en/userGuide/schema/binary-vector.md b/v2.4.x/site/en/userGuide/schema/binary-vector.md new file mode 100644 index 0000000000..6fc7c84698 --- /dev/null +++ b/v2.4.x/site/en/userGuide/schema/binary-vector.md @@ -0,0 +1,500 @@ +--- +id: binary-vector.md +title: Binary Vector​ +summary: Binary vectors are a special form of data representation that convert traditional high-dimensional floating-point vectors into binary vectors containing only 0s and 1s. This transformation not only compresses the size of the vector but also reduces storage and computational costs while retaining semantic information. When precision for non-critical features is not essential, binary vectors can effectively maintain most of the integrity and utility of the original floating-point vectors.​ +--- + +# Binary Vector​ + +Binary vectors are a special form of data representation that convert traditional high-dimensional floating-point vectors into binary vectors containing only 0s and 1s. This transformation not only compresses the size of the vector but also reduces storage and computational costs while retaining semantic information. When precision for non-critical features is not essential, binary vectors can effectively maintain most of the integrity and utility of the original floating-point vectors.​ + +Binary vectors have a wide range of applications, particularly in situations where computational efficiency and storage optimization are crucial. In large-scale AI systems, such as search engines or recommendation systems, real-time processing of massive amounts of data is key. By reducing the size of the vectors, binary vectors help lower latency and computational costs without significantly sacrificing accuracy. Additionally, binary vectors are useful in resource-constrained environments, such as mobile devices and embedded systems, where memory and processing power are limited. Through the use of binary vectors, complex AI functions can be implemented in these restricted settings while maintaining high performance.​ + +## Overview​ + +Binary vectors are a method of encoding complex objects (like images, text, or audio) into fixed-length binary values. In Milvus, binary vectors are typically represented as bit arrays or byte arrays. For example, an 8-dimensional binary vector can be represented as `[1, 0, 1, 1, 0, 0, 1, 0]`.​ + +The diagram below shows how binary vectors represent the presence of keywords in text content. In this example, a 10-dimensional binary vector is used to represent two different texts (**Text 1** and **Text 2**), where each dimension corresponds to a word in the vocabulary: 1 indicates the presence of the word in the text, while 0 indicates its absence.​ + +![Binary vector representation of text content](../../../../assets/binary-vector.png) + +Binary vectors have the following characteristics:​ + +- **Efficient Storage:** Each dimension requires only 1 bit of storage, significantly reducing storage space.​ + +- **Fast Computation:** Similarity between vectors can be quickly calculated using bitwise operations like XOR.​ + +- **Fixed Length:** The length of the vector remains constant regardless of the original text length, making indexing and retrieval easier.​ + +- **Simple and Intuitive:** Directly reflects the presence of keywords, making it suitable for certain specialized retrieval tasks.​ + +Binary vectors can be generated through various methods. In text processing, predefined vocabularies can be used to set corresponding bits based on word presence. For image processing, perceptual hashing algorithms (like [pHash](https://en.wikipedia.org/wiki/Perceptual_hashing)) can generate binary features of images. In machine learning applications, model outputs can be binarized to obtain binary vector representations.​ + +After binary vectorization, the data can be stored in Milvus for management and vector retrieval. The diagram below shows the basic process.​ + +![Use binary vectors in Milvus](../../../../assets/use-binary-vector.png) + +
+ +Although binary vectors excel in specific scenarios, they have limitations in their expressive capability, making it difficult to capture complex semantic relationships. Therefore, in real-world scenarios, binary vectors are often used alongside other vector types to balance efficiency and expressiveness. For more information, refer to [​Dense Vector](dense-vector.md) and [​Sparse Vector](sparse_vector.md).​ + +
+ +## Use binary vectors in Milvus​ + +### Add vector field​ + +To use binary vectors in Milvus, first define a vector field for storing binary vectors when creating a collection. This process includes:​ + +1. Setting `datatype` to the supported binary vector data type, i.e., `BINARY_VECTOR`.​ + +2. Specifying the vector's dimensions using the `dim` parameter. Note that `dim` must be a multiple of 8 as binary vectors must be converted into a byte array when inserting. Every 8 boolean values (0 or 1) will be packed into 1 byte. For example, if `dim=128`, a 16-byte array is required for insertion.​ + +
+ Python + Java + Node.js + Go + cURL +
+ +```python +from pymilvus import MilvusClient, DataType​ +​ +client = MilvusClient(uri="http://localhost:19530")​ +​ +schema = client.create_schema(​ + auto_id=True,​ + enable_dynamic_fields=True,​ +)​ +​ +schema.add_field(field_name="pk", datatype=DataType.VARCHAR, is_primary=True, max_length=100)​ +schema.add_field(field_name="binary_vector", datatype=DataType.BINARY_VECTOR, dim=128)​ + +``` + +```java +import io.milvus.v2.client.ConnectConfig;​ +import io.milvus.v2.client.MilvusClientV2;​ +​ +import io.milvus.v2.common.DataType;​ +import io.milvus.v2.service.collection.request.AddFieldReq;​ +import io.milvus.v2.service.collection.request.CreateCollectionReq;​ +​ +MilvusClientV2 client = new MilvusClientV2(ConnectConfig.builder()​ + .uri("http://localhost:19530")​ + .build());​ + ​ +CreateCollectionReq.CollectionSchema schema = client.createSchema();​ +schema.setEnableDynamicField(true);​ +schema.addField(AddFieldReq.builder()​ + .fieldName("pk")​ + .dataType(DataType.VarChar)​ + .isPrimaryKey(true)​ + .autoID(true)​ + .maxLength(100)​ + .build());​ +​ +schema.addField(AddFieldReq.builder()​ + .fieldName("binary_vector")​ + .dataType(DataType.BinaryVector)​ + .dimension(128)​ + .build());​ + +``` + +```javascript +import { DataType } from "@zilliz/milvus2-sdk-node";​ +​ +schema.push({​ + name: "binary vector",​ + data_type: DataType.BinaryVector,​ + dim: 128,​ +});​ + +``` + +```curl +export primaryField='{​ + "fieldName": "pk",​ + "dataType": "VarChar",​ + "isPrimary": true,​ + "elementTypeParams": {​ + "max_length": 100​ + }​ +}'​ +​ +export vectorField='{​ + "fieldName": "binary_vector",​ + "dataType": "BinaryVector",​ + "elementTypeParams": {​ + "dim": 128​ + }​ +}'​ +​ +export schema="{​ + \"autoID\": true,​ + \"fields\": [​ + $primaryField,​ + $vectorField​ + ],​ + \"enableDynamicField\": true​ +}"​ +​ + +``` + +In this example, a vector field named `binary_vector` is added for storing binary vectors. The data type of this field is `BINARY_VECTOR`, with a dimension of 128.​ + +### Set index params for vector field​ + +To speed up searches, an index must be created for the binary vector field. Indexing can significantly enhance the retrieval efficiency of large-scale vector data.​ + +
+ Python + Java + Node.js + Go + cURL +
+ +```python +index_params = client.prepare_index_params()​ +​ +index_params.add_index(​ + field_name="binary_vector",​ + index_name="binary_vector_index",​ + index_type="BIN_IVF_FLAT",​ + metric_type="HAMMING",​ + params={"nlist": 128}​ +)​ + +``` + +```java +import io.milvus.v2.common.IndexParam;​ +import java.util.*;​ +​ +List indexParams = new ArrayList<>();​ +Map extraParams = new HashMap<>();​ +extraParams.put("nlist",128);​ +indexParams.add(IndexParam.builder()​ + .fieldName("binary_vector")​ + .indexType(IndexParam.IndexType.BIN_IVF_FLAT)​ + .metricType(IndexParam.MetricType.HAMMING)​ + .extraParams(extraParams)​ + .build());​ + +``` + +```javascript +import { MetricType, IndexType } from "@zilliz/milvus2-sdk-node";​ +​ +const indexParams = {​ + indexName: "binary_vector_index",​ + field_name: "binary_vector",​ + metric_type: MetricType.HAMMING,​ + index_type: IndexType.BIN_IVF_FLAT,​ + params: {​ + nlist: 128,​ + },​ +};​ + +``` + +```curl +export indexParams='[​ + {​ + "fieldName": "binary_vector",​ + "metricType": "HAMMING",​ + "indexName": "binary_vector_index",​ + "indexType": "BIN_IVF_FLAT",​ + "params":{"nlist": 128}​ + }​ + ]'​ + +``` + +In the example above, an index named `binary_vector_index` is created for the `binary_vector` field, using the `BIN_IVF_FLAT` index type. The `metric_type` is set to `HAMMING`, indicating that Hamming distance is used for similarity measurement.​ + +Besides `BIN_IVF_FLAT`, Milvus supports other index types for binary vectors. For more details, refer to [​Binary Vector Indexes](https://milvus.io/docs/index.md?tab=binary). Additionally, Milvus supports other similarity metrics for binary vectors. For more information, refer to [​Metric Types](metric.md).​ + +### Create collection​ + +Once the binary vector and index settings are complete, create a collection that contains binary vectors. The example below uses the `create_collection` method to create a collection named `my_binary_collection`.​ + +
+ Python + Java + Node.js + Go + cURL +
+ +```python +client.create_collection(​ + collection_name="my_binary_collection",​ + schema=schema,​ + index_params=index_params​ +)​ + +``` + +```java +import io.milvus.v2.client.ConnectConfig;​ +import io.milvus.v2.client.MilvusClientV2;​ +​ +MilvusClientV2 client = new MilvusClientV2(ConnectConfig.builder()​ + .uri("http://localhost:19530")​ + .build());​ +​ +CreateCollectionReq requestCreate = CreateCollectionReq.builder()​ + .collectionName("my_binary_collection")​ + .collectionSchema(schema)​ + .indexParams(indexParams)​ + .build();​ +client.createCollection(requestCreate);​ + +``` + +```javascript +import { MilvusClient } from "@zilliz/milvus2-sdk-node";​ +​ +const client = new MilvusClient({​ + address: 'http://localhost:19530'​ +});​ +​ +await client.createCollection({​ + collection_name: 'my_dense_collection',​ + schema: schema,​ + index_params: indexParams​ +});​ + +``` + +```curl +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/collections/create" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d "{​ + \"collectionName\": \"my_binary_collection\",​ + \"schema\": $schema,​ + \"indexParams\": $indexParams​ +}"​ + +``` + +### Insert data​ + +After creating the collection, use the `insert` method to add data containing binary vectors. Note that binary vectors should be provided in the form of a byte array, where each byte represents 8 boolean values.​ + +For example, for a 128-dimensional binary vector, a 16-byte array is required (since 128 bits ÷ 8 bits/byte = 16 bytes). Below is an example code for inserting data:​ + +
+ Python + Java + Node.js + Go + cURL +
+ +```python +def convert_bool_list_to_bytes(bool_list):​ + if len(bool_list) % 8 != 0:​ + raise ValueError("The length of a boolean list must be a multiple of 8")​ +​ + byte_array = bytearray(len(bool_list) // 8)​ + for i, bit in enumerate(bool_list):​ + if bit == 1:​ + index = i // 8​ + shift = i % 8​ + byte_array[index] |= (1 << shift)​ + return bytes(byte_array)​ +​ +​ +bool_vectors = [​ + [1, 0, 0, 1, 1, 0, 1, 1, 0, 1, 0, 1, 0, 1, 0, 0] + [0] * 112,​ + [0, 1, 0, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 1] + [0] * 112,​ +]​ +​ +data = [{"binary_vector": convert_bool_list_to_bytes(bool_vector) for bool_vector in bool_vectors}]​ +​ +client.insert(​ + collection_name="my_binary_collection",​ + data=data​ +)​ + +``` + +```java +import com.google.gson.Gson;​ +import com.google.gson.JsonObject;​ +import io.milvus.v2.service.vector.request.InsertReq;​ +import io.milvus.v2.service.vector.response.InsertResp;​ +​ +private static byte[] convertBoolArrayToBytes(boolean[] booleanArray) {​ + byte[] byteArray = new byte[booleanArray.length / Byte.SIZE];​ + for (int i = 0; i < booleanArray.length; i++) {​ + if (booleanArray[i]) {​ + int index = i / Byte.SIZE;​ + int shift = i % Byte.SIZE;​ + byteArray[index] |= (byte) (1 << shift);​ + }​ + }​ +​ + return byteArray;​ +}​ +​ +List rows = new ArrayList<>();​ +Gson gson = new Gson();​ +{​ + boolean[] boolArray = {true, false, false, true, true, false, true, true, false, true, false, false, true, true, false, true};​ + JsonObject row = new JsonObject();​ + row.add("binary_vector", gson.toJsonTree(convertBoolArrayToBytes(boolArray)));​ + rows.add(row);​ +}​ +{​ + boolean[] boolArray = {false, true, false, true, false, true, false, false, true, true, false, false, true, true, false, true};​ + JsonObject row = new JsonObject();​ + row.add("binary_vector", gson.toJsonTree(convertBoolArrayToBytes(boolArray)));​ + rows.add(row);​ +}​ +​ +InsertResp insertR = client.insert(InsertReq.builder()​ + .collectionName("my_binary_collection")​ + .data(rows)​ + .build());​ + +``` + +```javascript +const data = [​ + { binary_vector: [1, 0, 0, 1, 1, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 1] },​ + { binary_vector: [1, 0, 0, 1, 1, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 1] },​ +];​ +​ +client.insert({​ + collection_name: "my_binary_collection",​ + data: data,​ +});​ + +``` + +```curl +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/insert" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d "{​ + \"data\": $data,​ + \"collectionName\": \"my_binary_collection\"​ +}"​ + +``` + +### Perform similarity search​ + +Similarity search is one of the core features of Milvus, allowing you to quickly find data that is most similar to a query vector based on the distance between vectors. To perform a similarity search using binary vectors, prepare the query vector and search parameters, then call the `search` method.​ + +During search operations, binary vectors must also be provided in the form of a byte array. Ensure that the dimensionality of the query vector matches the dimension specified when defining `dim` and that every 8 boolean values are converted into 1 byte.​ + +
+ Python + Java + Node.js + Go + cURL +
+ +```python +search_params = {​ + "params": {"nprobe": 10}​ +}​ +​ +query_bool_list = [1, 0, 0, 1, 1, 0, 1, 1, 0, 1, 0, 1, 0, 1, 0, 0] + [0] * 112​ +query_vector = convert_bool_list_to_bytes(query_bool_list)​ +​ +res = client.search(​ + collection_name="my_binary_collection",​ + data=[query_vector],​ + anns_field="binary_vector",​ + search_params=search_params,​ + limit=5,​ + output_fields=["pk"]​ +)​ +​ +print(res)​ +​ +# Output​ +# data: ["[{'id': '453718927992172268', 'distance': 10.0, 'entity': {'pk': '453718927992172268'}}]"] ​ + +``` + +```java +import io.milvus.v2.service.vector.request.SearchReq;​ +import io.milvus.v2.service.vector.request.data.BinaryVec;​ +import io.milvus.v2.service.vector.response.SearchResp;​ +​ +Map searchParams = new HashMap<>();​ +searchParams.put("nprobe",10);​ +​ +boolean[] boolArray = {true, false, false, true, true, false, true, true, false, true, false, false, true, true, false, true};​ +BinaryVec queryVector = new BinaryVec(convertBoolArrayToBytes(boolArray));​ +​ +SearchResp searchR = client.search(SearchReq.builder()​ + .collectionName("my_binary_collection")​ + .data(Collections.singletonList(queryVector))​ + .annsField("binary_vector")​ + .searchParams(searchParams)​ + .topK(5)​ + .outputFields(Collections.singletonList("pk"))​ + .build());​ + ​ + System.out.println(searchR.getSearchResults());​ + ​ + // Output​ + //​ + // [[SearchResp.SearchResult(entity={pk=453444327741536775}, score=0.0, id=453444327741536775), SearchResp.SearchResult(entity={pk=453444327741536776}, score=7.0, id=453444327741536776)]]​ + +``` + +```javascript +query_vector = [1,0,1,0,1,1,1,1,1,1,1,1];​ +​ +client.search({​ + collection_name: 'my_binary_collection',​ + data: query_vector,​ + limit: 5,​ + output_fields: ['pk'],​ + params: {​ + nprobe: 10​ + }​ +});​ + +``` + +```curl +export searchParams='{​ + "params":{"nprobe":10}​ + }'​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/search" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d "{​ + \"collectionName\": \"my_binary_collection\",​ + \"data\": $data,​ + \"annsField\": \"binary_vector\",​ + \"limit\": 5,​ + \"searchParams\":$searchParams,​ + \"outputFields\": [\"pk\"]​ +}"​ + +``` + +For more information on similarity search parameters, refer to [​Basic ANN Search](single-vector-search.md).​ + +​ + diff --git a/v2.4.x/site/en/userGuide/schema/dense-vector.md b/v2.4.x/site/en/userGuide/schema/dense-vector.md new file mode 100644 index 0000000000..0d17f035b8 --- /dev/null +++ b/v2.4.x/site/en/userGuide/schema/dense-vector.md @@ -0,0 +1,481 @@ +--- +id: dense-vector.md +title: Dense Vector​ +summary: Dense vectors are numerical data representations widely used in machine learning and data analysis. They consist of arrays with real numbers, where most or all elements are non-zero. Compared to sparse vectors, dense vectors contain more information at the same dimensional level, as each dimension holds meaningful values. This representation can effectively capture complex patterns and relationships, making data easier to analyze and process in high-dimensional spaces. Dense vectors typically have a fixed number of dimensions, ranging from a few dozen to several hundred or even thousands, depending on the specific application and requirements.​​ +--- + +# Dense Vector​ + +Dense vectors are numerical data representations widely used in machine learning and data analysis. They consist of arrays with real numbers, where most or all elements are non-zero. Compared to sparse vectors, dense vectors contain more information at the same dimensional level, as each dimension holds meaningful values. This representation can effectively capture complex patterns and relationships, making data easier to analyze and process in high-dimensional spaces. Dense vectors typically have a fixed number of dimensions, ranging from a few dozen to several hundred or even thousands, depending on the specific application and requirements.​ + +Dense vectors are mainly used in scenarios that require understanding the semantics of data, such as semantic search and recommendation systems. In semantic search, dense vectors help capture the underlying connections between queries and documents, improving the relevance of search results. In recommendation systems, they aid in identifying similarities between users and items, offering more personalized suggestions.​ + +## Overview​ + +Dense vectors are typically represented as arrays of floating-point numbers with a fixed length, such as `[0.2, 0.7, 0.1, 0.8, 0.3, ..., 0.5]`. The dimensionality of these vectors usually ranges from hundreds to thousands, such as 128, 256, 768, or 1024. Each dimension captures specific semantic features of an object, making it applicable to various scenarios through similarity calculations.​ + +![Dense vectors in 2D space](../../../../assets/dense-vector.png) + +The image above illustrates the representation of dense vectors in a 2D space. Although dense vectors in real-world applications often have much higher dimensions, this 2D illustration effectively conveys several key concepts:​ + +- **Multidimensional Representation:** Each point represents a conceptual object (like **Milvus**, **vector database**, **retrieval system**, etc.), with its position determined by the values of its dimensions.​ + +- **Semantic Relationships:** The distances between points reflect the semantic similarity between concepts. Closer points indicate concepts that are more semantically related.​ + +- **Clustering Effect:** Related concepts (such as **Milvus**, **vector database**, and **retrieval system**) are positioned close to each other in space, forming a semantic cluster.​ + +Below is an example of a real dense vector representing the text `"Milvus is an efficient vector database"`:​ + +```JSON +[​ + -0.013052909,​ + 0.020387933,​ + -0.007869,​ + -0.11111383,​ + -0.030188112,​ + -0.0053388323,​ + 0.0010654867,​ + 0.072027855,​ + // ... more dimensions​ +]​ +​ + +``` + +Dense vectors can be generated using various [embedding](https://en.wikipedia.org/wiki/Embedding) models, such as CNN models (like [ResNet](https://pytorch.org/hub/pytorch_vision_resnet/), [VGG](https://pytorch.org/vision/stable/models/vgg.html)) for images and language models (like [BERT](https://en.wikipedia.org/wiki/BERT_(language_model)), [Word2Vec](https://en.wikipedia.org/wiki/Word2vec)) for text. These models transform raw data into points in high-dimensional space, capturing the semantic features of the data. Additionally, Milvus offers convenient methods to help users generate and process dense vectors, as detailed in Embeddings.​ + +Once data is vectorized, it can be stored in Milvus for management and vector retrieval. The diagram below shows the basic process.​ + +![Use dense vecctors in Milvus](../../../../assets/use-dense-vector.png) + +
+ +Besides dense vectors, Milvus also supports sparse vectors and binary vectors. Sparse vectors are suitable for precise matches based on specific terms, such as keyword search and term matching, while binary vectors are commonly used for efficiently handling binarized data, such as image pattern matching and certain hashing applications. For more information, refer to [​Binary Vector](binary-vector.md) and [​Sparse Vector](sparse_vector.md).​ + +
+ +## Use dense vectors in Milvus​ + +### Add vector field​ + +To use dense vectors in Milvus, first define a vector field for storing dense vectors when creating a collection. This process includes:​ + +1. Setting `datatype` to a supported dense vector data type. For supported dense vector data types, see Data Types.​ + +2. Specifying the dimensions of the dense vector using the `dim` parameter.​ + +In the example below, we add a vector field named `dense_vector` to store dense vectors. The field's data type is `FLOAT_VECTOR`, with a dimension of `4`.​ + +
+ Python + Java + Node.js + Go + cURL +
+ +```python +from pymilvus import MilvusClient, DataType​ +​ +client = MilvusClient(uri="http://localhost:19530")​ +​ +schema = client.create_schema(​ + auto_id=True,​ + enable_dynamic_fields=True,​ +)​ +​ +schema.add_field(field_name="pk", datatype=DataType.VARCHAR, is_primary=True, max_length=100)​ +schema.add_field(field_name="dense_vector", datatype=DataType.FLOAT_VECTOR, dim=4)​ + +``` + +```java +import io.milvus.v2.client.ConnectConfig;​ +import io.milvus.v2.client.MilvusClientV2;​ +​ +import io.milvus.v2.common.DataType;​ +import io.milvus.v2.service.collection.request.AddFieldReq;​ +import io.milvus.v2.service.collection.request.CreateCollectionReq;​ +​ +MilvusClientV2 client = new MilvusClientV2(ConnectConfig.builder()​ + .uri("http://localhost:19530")​ + .build());​ +​ +CreateCollectionReq.CollectionSchema schema = client.createSchema();​ +schema.setEnableDynamicField(true);​ +schema.addField(AddFieldReq.builder()​ + .fieldName("pk")​ + .dataType(DataType.VarChar)​ + .isPrimaryKey(true)​ + .autoID(true)​ + .maxLength(100)​ + .build());​ +​ +schema.addField(AddFieldReq.builder()​ + .fieldName("dense_vector")​ + .dataType(DataType.FloatVector)​ + .dimension(4)​ + .build());​ + +``` + +```javascript +import { DataType } from "@zilliz/milvus2-sdk-node";​ +​ +schema.push({​ + name: "dense_vector",​ + data_type: DataType.FloatVector,​ + dim: 128,​ +});​ +​ + +``` + +```curl +export primaryField='{​ + "fieldName": "pk",​ + "dataType": "VarChar",​ + "isPrimary": true,​ + "elementTypeParams": {​ + "max_length": 100​ + }​ +}'​ +​ +export vectorField='{​ + "fieldName": "dense_vector",​ + "dataType": "FloatVector",​ + "elementTypeParams": {​ + "dim": 4​ + }​ +}'​ +​ +export schema="{​ + \"autoID\": true,​ + \"fields\": [​ + $primaryField,​ + $vectorField​ + ]​ +}"​ + +``` + +**Supported data types for dense vector fields**: + +| **Type** | **Description** | +|--------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `FLOAT_VECTOR` | Stores 32-bit floating-point numbers, commonly used for representing real numbers in scientific computations and machine learning. Ideal for scenarios requiring high precision, such as distinguishing similar vectors. | +| `FLOAT16_VECTOR` | Stores 16-bit half-precision floating-point numbers, used for deep learning and GPU computations. It saves storage space in scenarios where precision is less critical, such as in the low-precision recall phase of recommendation systems. | +| `BFLOAT16_VECTOR` | Stores 16-bit Brain Floating Point (bfloat16) numbers, offering the same range of exponents as Float32 but with reduced precision. Suitable for scenarios that need to process large volumes of vectors quickly, such as large-scale image retrieval. | + +### Set index params for vector field​ + +To accelerate semantic searches, an index must be created for the vector field. Indexing can significantly improve the retrieval efficiency of large-scale vector data.​ + +
+ Python + Java + Node.js + Go + cURL +
+ +```python +index_params = client.prepare_index_params()​ +​ +index_params.add_index(​ + field_name="dense_vector",​ + index_name="dense_vector_index",​ + index_type="IVF_FLAT",​ + metric_type="IP",​ + params={"nlist": 128}​ +)​ + +``` + +```java +import io.milvus.v2.common.IndexParam;​ +import java.util.*;​ +​ +List indexes = new ArrayList<>();​ +Map extraParams = new HashMap<>();​ +extraParams.put("nlist",128);​ +indexes.add(IndexParam.builder()​ + .fieldName("dense_vector")​ + .indexType(IndexParam.IndexType.IVF_FLAT)​ + .metricType(IndexParam.MetricType.IP)​ + .extraParams(extraParams)​ + .build());​ + +``` + +```javascript +import { MetricType, IndexType } from "@zilliz/milvus2-sdk-node";​ +​ +const indexParams = {​ + index_name: 'dense_vector_index',​ + field_name: 'dense_vector',​ + metric_type: MetricType.IP,​ + index_type: IndexType.IVF_FLAT,​ + params: {​ + nlist: 128​ + },​ +};​ + +``` + +```curl +export indexParams='[​ + {​ + "fieldName": "dense_vector",​ + "metricType": "IP",​ + "indexName": "dense_vector_index",​ + "indexType": "IVF_FLAT",​ + "params":{"nlist": 128}​ + }​ + ]'​ + +``` + +In the example above, an index named `dense_vector_index` is created for the `dense_vector` field using the `IVF_FLAT` index type. The `metric_type` is set to `IP`, indicating that inner product will be used as the distance metric.​ + +Milvus supports other index types as well. For more details, refer to [​Floating Vector Indexes](https://milvus.io/docs/index.md?tab=floating). Additionally, Milvus supports other metric types. For more information, refer to [​Metric Types](metric.md).​ + +### Create collection​ + +Once the dense vector and index param settings are complete, you can create a collection containing dense vectors. The example below uses the `create_collection` method to create a collection named `my_dense_collection`.​ + +
+ Python + Java + Node.js + Go + cURL +
+ +```python +client.create_collection(​ + collection_name="my_dense_collection",​ + schema=schema,​ + index_params=index_params​ +)​ + +``` + +```java +import io.milvus.v2.client.ConnectConfig;​ +import io.milvus.v2.client.MilvusClientV2;​ +​ +MilvusClientV2 client = new MilvusClientV2(ConnectConfig.builder()​ + .uri("http://localhost:19530")​ + .build());​ +​ +CreateCollectionReq requestCreate = CreateCollectionReq.builder()​ + .collectionName("my_dense_collection")​ + .collectionSchema(schema)​ + .indexParams(indexes)​ + .build();​ +client.createCollection(requestCreate);​ + +``` + +```javascript +import { MilvusClient } from "@zilliz/milvus2-sdk-node";​ +​ +const client = new MilvusClient({​ + address: 'http://localhost:19530'​ +});​ +​ +await client.createCollection({​ + collection_name: 'my_dense_collection',​ + schema: schema,​ + index_params: indexParams​ +});​ +​ + +``` + +```curl +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/collections/create" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d "{​ + \"collectionName\": \"my_dense_collection\",​ + \"schema\": $schema,​ + \"indexParams\": $indexParams​ +}"​ + +``` + +### Insert data​ + +After creating the collection, use the `insert` method to add data containing dense vectors. Ensure that the dimensionality of the dense vectors being inserted matches the `dim` value defined when adding the dense vector field.​ + +
+ Python + Java + Node.js + Go + cURL +
+ +```python +data = [​ + {"dense_vector": [0.1, 0.2, 0.3, 0.7]},​ + {"dense_vector": [0.2, 0.3, 0.4, 0.8]},​ +]​ +​ +client.insert(​ + collection_name="my_dense_collection",​ + data=data​ +)​ + +``` + +```java +import com.google.gson.Gson;​ +import com.google.gson.JsonObject;​ +import io.milvus.v2.service.vector.request.InsertReq;​ +import io.milvus.v2.service.vector.response.InsertResp;​ +​ +List rows = new ArrayList<>();​ +Gson gson = new Gson();​ +rows.add(gson.fromJson("{\"dense_vector\": [0.1, 0.2, 0.3, 0.4]}", JsonObject.class));​ +rows.add(gson.fromJson("{\"dense_vector\": [0.2, 0.3, 0.4, 0.5]}", JsonObject.class));​ +​ +InsertResp insertR = client.insert(InsertReq.builder()​ + .collectionName("my_dense_collection")​ + .data(rows)​ + .build());​ + +``` + +```javascript +const data = [​ + { dense_vector: [0.1, 0.2, 0.3, 0.7] },​ + { dense_vector: [0.2, 0.3, 0.4, 0.8] },​ +];​ +​ +client.insert({​ + collection_name: "my_dense_collection",​ + data: data,​ +});​ + +``` + +```curl +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/insert" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "data": [​ + {"dense_vector": [0.1, 0.2, 0.3, 0.4]},​ + {"dense_vector": [0.2, 0.3, 0.4, 0.5]} ​ + ],​ + "collectionName": "my_dense_collection"​ +}'​ +​ +## {"code":0,"cost":0,"data":{"insertCount":2,"insertIds":["453577185629572531","453577185629572532"]}}​ + +``` + +### Perform similarity search​ + +Semantic search based on dense vectors is one of the core features of Milvus, allowing you to quickly find data that is most similar to a query vector based on the distance between vectors. To perform a similarity search, prepare the query vector and search parameters, then call the `search` method.​ + +
+ Python + Java + Node.js + Go + cURL +
+ +```python +search_params = {​ + "params": {"nprobe": 10}​ +}​ +​ +query_vector = [0.1, 0.2, 0.3, 0.7]​ +​ +res = client.search(​ + collection_name="my_dense_collection",​ + data=[query_vector],​ + anns_field="dense_vector",​ + search_params=search_params,​ + limit=5,​ + output_fields=["pk"]​ +)​ +​ +print(res)​ +​ +# Output​ +# data: ["[{'id': '453718927992172271', 'distance': 0.7599999904632568, 'entity': {'pk': '453718927992172271'}}, {'id': '453718927992172270', 'distance': 0.6299999952316284, 'entity': {'pk': '453718927992172270'}}]"]​ + +``` + +```java +import io.milvus.v2.service.vector.request.data.FloatVec;​ +​ +Map searchParams = new HashMap<>();​ +searchParams.put("nprobe",10);​ +​ +FloatVec queryVector = new FloatVec(new float[]{0.1f, 0.3f, 0.3f, 0.4f});​ +​ +SearchResp searchR = client.search(SearchReq.builder()​ + .collectionName("my_dense_collection")​ + .data(Collections.singletonList(queryVector))​ + .annsField("dense_vector")​ + .searchParams(searchParams)​ + .topK(5)​ + .outputFields(Collections.singletonList("pk"))​ + .build());​ + ​ +System.out.println(searchR.getSearchResults());​ +​ +// Output​ +//​ +// [[SearchResp.SearchResult(entity={pk=453444327741536779}, score=0.65, id=453444327741536779), SearchResp.SearchResult(entity={pk=453444327741536778}, score=0.65, id=453444327741536778)]]​ + +``` + +```javascript +query_vector = [0.1, 0.2, 0.3, 0.7];​ +​ +client.search({​ + collection_name: my_dense_collection,​ + data: query_vector,​ + limit: 5,​ + output_fields: ['pk'],​ + params: {​ + nprobe: 10​ + }​ +});​ + +``` + +```curl +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/search" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "my_dense_collection",​ + "data": [​ + [0.1, 0.2, 0.3, 0.7]​ + ],​ + "annsField": "dense_vector",​ + "limit": 5,​ + "searchParams":{​ + "params":{"nprobe":10}​ + },​ + "outputFields": ["pk"]​ +}'​ +​ +## {"code":0,"cost":0,"data":[{"distance":0.55,"id":"453577185629572532","pk":"453577185629572532"},{"distance":0.42,"id":"453577185629572531","pk":"453577185629572531"}]}​ + +``` + +For more information on similarity search parameters, refer to [​Basic ANN Search](single-vector-search.md).​ + diff --git a/v2.4.x/site/en/userGuide/schema/enable-dynamic-field.md b/v2.4.x/site/en/userGuide/schema/enable-dynamic-field.md new file mode 100644 index 0000000000..0e05608730 --- /dev/null +++ b/v2.4.x/site/en/userGuide/schema/enable-dynamic-field.md @@ -0,0 +1,389 @@ +--- +id: enable-dynamic-field.md +title: Enable Dynamic Field +summary: All fields defined in the schema of a collection must be included in the entities to be inserted. If you want some fields to be optional, consider enabling the dynamic field. This topic describes how to enable and use the dynamic field.​ +--- + +# Dynamic Field​ + +All fields defined in the schema of a collection must be included in the entities to be inserted. If you want some fields to be optional, consider enabling the dynamic field. This topic describes how to enable and use the dynamic field.​ + +## Overview​ + +In Milvus, you can create a collection schema by setting the names and data types for each field in the collection. When you add a field to the schema, make sure that this field is included in the entity you intend to insert. If you want some fields to be optional, enabling the dynamic field is one option.​ + +The dynamic field is a reserved field named `$meta`, which is of the JavaScript Object Notation (JSON) type. Any fields in the entities that are not defined in the schema will be stored in this reserved JSON field as key-value pairs.​ + +For a collection with the dynamic field enabled, you can use keys in the dynamic field for scalar filtering, just as you would with fields explicitly defined in the schema.​ + +## Enable dynamic field​ + +Collections created using the method described in [​Create Collection Instantly](create-collection-instantly.md) have the dynamic field enabled by default. You can also enable the dynamic field manually when creating a collection with custom settings.​ + +
+ Python + Java + Node.js + Go + cURL +
+ +```python +from pymilvus import MilvusClient​ +​ +client= MilvusClient(uri="http://localhost:19530")​ +​ +client.create_collection(​ + collection_name="my_dynamic_collection",​ + dimension=5,​ + # highlight-next-line​ + enable_dynamic_field=True​ +)​ + +``` + +```java +import io.milvus.v2.client.ConnectConfig;​ +import io.milvus.v2.client.MilvusClientV2;​ +import io.milvus.v2.service.collection.request.CreateCollectionReq;​ +​ +MilvusClientV2 client = new MilvusClientV2(ConnectConfig.builder()​ + .uri("http://localhost:19530")​ + .build());​ + ​ +CreateCollectionReq createCollectionReq = CreateCollectionReq.builder()​ + .collectionName("my_dynamic_collection")​ + .dimension(5)​ + // highlight-next-line​ + .enableDynamicField(true)​ + .build()​ +client.createCollection(createCollectionReq);​ + +``` + +```javascript +import { MilvusClient, DataType } from "@zilliz/milvus2-sdk-node";​ +​ +const client = new Client({​ + address: 'http://localhost:19530'​ +});​ +​ +await client.createCollection({​ + collection_name: "customized_setup_2",​ + schema: schema,​ + // highlight-next-line​ + enable_dynamic_field: true​ +});​ + +``` + +```curl +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/collections/create" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "my_dynamic_collection",​ + "dimension": 5,​ + "enableDynamicField": true​ +}'​ + +``` + +## Use dynamic field​ + +When the dynamic field is enabled in your collection, all fields and their values that are not defined in the schema will be stored as key-value pairs in the dynamic field.​ + +For example, suppose your collection schema defines only two fields, named `id` and `vector`, with the dynamic field enabled. Now, insert the following dataset into this collection.​ + +```JSON +[​ + {id: 0, vector: [0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592], color: "pink_8682"},​ + {id: 1, vector: [0.19886812562848388, 0.06023560599112088, 0.6976963061752597, 0.2614474506242501, 0.838729485096104], color: "red_7025"},​ + {id: 2, vector: [0.43742130801983836, -0.5597502546264526, 0.6457887650909682, 0.7894058910881185, 0.20785793220625592], color: "orange_6781"},​ + {id: 3, vector: [0.3172005263489739, 0.9719044792798428, -0.36981146090600725, -0.4860894583077995, 0.95791889146345], color: "pink_9298"},​ + {id: 4, vector: [0.4452349528804562, -0.8757026943054742, 0.8220779437047674, 0.46406290649483184, 0.30337481143159106], color: "red_4794"},​ + {id: 5, vector: [0.985825131989184, -0.8144651566660419, 0.6299267002202009, 0.1206906911183383, -0.1446277761879955], color: "yellow_4222"},​ + {id: 6, vector: [0.8371977790571115, -0.015764369584852833, -0.31062937026679327, -0.562666951622192, -0.8984947637863987], color: "red_9392"},​ + {id: 7, vector: [-0.33445148015177995, -0.2567135004164067, 0.8987539745369246, 0.9402995886420709, 0.5378064918413052], color: "grey_8510"},​ + {id: 8, vector: [0.39524717779832685, 0.4000257286739164, -0.5890507376891594, -0.8650502298996872, -0.6140360785406336], color: "white_9381"},​ + {id: 9, vector: [0.5718280481994695, 0.24070317428066512, -0.3737913482606834, -0.06726932177492717, -0.6980531615588608], color: "purple_4976"} ​ +]​ + +``` + +The dataset above contains 10 entities, each including the fields `id`, `vector`, and `color`. Here, the `color` field is not defined in the schema. Since the collection has the dynamic field enabled, the field `color` will be stored as a key-value pair within the dynamic field.​ + +### Insert data​ + +The following code demonstrates how to insert this dataset into the collection.​ + +
+ Python + Java + Node.js + Go + cURL +
+ +```python +data=[​ + {"id": 0, "vector": [0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592], "color": "pink_8682"},​ + {"id": 1, "vector": [0.19886812562848388, 0.06023560599112088, 0.6976963061752597, 0.2614474506242501, 0.838729485096104], "color": "red_7025"},​ + {"id": 2, "vector": [0.43742130801983836, -0.5597502546264526, 0.6457887650909682, 0.7894058910881185, 0.20785793220625592], "color": "orange_6781"},​ + {"id": 3, "vector": [0.3172005263489739, 0.9719044792798428, -0.36981146090600725, -0.4860894583077995, 0.95791889146345], "color": "pink_9298"},​ + {"id": 4, "vector": [0.4452349528804562, -0.8757026943054742, 0.8220779437047674, 0.46406290649483184, 0.30337481143159106], "color": "red_4794"},​ + {"id": 5, "vector": [0.985825131989184, -0.8144651566660419, 0.6299267002202009, 0.1206906911183383, -0.1446277761879955], "color": "yellow_4222"},​ + {"id": 6, "vector": [0.8371977790571115, -0.015764369584852833, -0.31062937026679327, -0.562666951622192, -0.8984947637863987], "color": "red_9392"},​ + {"id": 7, "vector": [-0.33445148015177995, -0.2567135004164067, 0.8987539745369246, 0.9402995886420709, 0.5378064918413052], "color": "grey_8510"},​ + {"id": 8, "vector": [0.39524717779832685, 0.4000257286739164, -0.5890507376891594, -0.8650502298996872, -0.6140360785406336], "color": "white_9381"},​ + {"id": 9, "vector": [0.5718280481994695, 0.24070317428066512, -0.3737913482606834, -0.06726932177492717, -0.6980531615588608], "color": "purple_4976"}​ +]​ +​ +res = client.insert(​ + collection_name="my_dynamic_collection",​ + data=data​ +)​ +​ +print(res)​ +​ +# Output​ +# {'insert_count': 10, 'ids': [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]}​ + +``` + +```java +import com.google.gson.Gson;​ +import com.google.gson.JsonObject;​ +​ +import io.milvus.v2.service.vector.request.InsertReq;​ +import io.milvus.v2.service.vector.response.InsertResp;​ + ​ +Gson gson = new Gson();​ +List data = Arrays.asList(​ + gson.fromJson("{\"id\": 0, \"vector\": [0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592], \"color\": \"pink_8682\"}", JsonObject.class),​ + gson.fromJson("{\"id\": 1, \"vector\": [0.19886812562848388, 0.06023560599112088, 0.6976963061752597, 0.2614474506242501, 0.838729485096104], \"color\": \"red_7025\"}", JsonObject.class),​ + gson.fromJson("{\"id\": 2, \"vector\": [0.43742130801983836, -0.5597502546264526, 0.6457887650909682, 0.7894058910881185, 0.20785793220625592], \"color\": \"orange_6781\"}", JsonObject.class),​ + gson.fromJson("{\"id\": 3, \"vector\": [0.3172005263489739, 0.9719044792798428, -0.36981146090600725, -0.4860894583077995, 0.95791889146345], \"color\": \"pink_9298\"}", JsonObject.class),​ + gson.fromJson("{\"id\": 4, \"vector\": [0.4452349528804562, -0.8757026943054742, 0.8220779437047674, 0.46406290649483184, 0.30337481143159106], \"color\": \"red_4794\"}", JsonObject.class),​ + gson.fromJson("{\"id\": 5, \"vector\": [0.985825131989184, -0.8144651566660419, 0.6299267002202009, 0.1206906911183383, -0.1446277761879955], \"color\": \"yellow_4222\"}", JsonObject.class),​ + gson.fromJson("{\"id\": 6, \"vector\": [0.8371977790571115, -0.015764369584852833, -0.31062937026679327, -0.562666951622192, -0.8984947637863987], \"color\": \"red_9392\"}", JsonObject.class),​ + gson.fromJson("{\"id\": 7, \"vector\": [-0.33445148015177995, -0.2567135004164067, 0.8987539745369246, 0.9402995886420709, 0.5378064918413052], \"color\": \"grey_8510\"}", JsonObject.class),​ + gson.fromJson("{\"id\": 8, \"vector\": [0.39524717779832685, 0.4000257286739164, -0.5890507376891594, -0.8650502298996872, -0.6140360785406336], \"color\": \"white_9381\"}", JsonObject.class),​ + gson.fromJson("{\"id\": 9, \"vector\": [0.5718280481994695, 0.24070317428066512, -0.3737913482606834, -0.06726932177492717, -0.6980531615588608], \"color\": \"purple_4976\"}", JsonObject.class)​ +);​ +​ +InsertReq insertReq = InsertReq.builder()​ + .collectionName("my_dynamic_collection")​ + .data(data)​ + .build();​ +​ +InsertResp insertResp = client.insert(insertReq);​ +System.out.println(insertResp);​ +​ +// Output:​ +//​ +// InsertResp(InsertCnt=10, primaryKeys=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9])​ + +``` + +```javascript +const { DataType } = require("@zilliz/milvus2-sdk-node")​ +​ +// 3. Insert some data​ +​ +var data = [​ + {id: 0, vector: [0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592], color: "pink_8682"},​ + {id: 1, vector: [0.19886812562848388, 0.06023560599112088, 0.6976963061752597, 0.2614474506242501, 0.838729485096104], color: "red_7025"},​ + {id: 2, vector: [0.43742130801983836, -0.5597502546264526, 0.6457887650909682, 0.7894058910881185, 0.20785793220625592], color: "orange_6781"},​ + {id: 3, vector: [0.3172005263489739, 0.9719044792798428, -0.36981146090600725, -0.4860894583077995, 0.95791889146345], color: "pink_9298"},​ + {id: 4, vector: [0.4452349528804562, -0.8757026943054742, 0.8220779437047674, 0.46406290649483184, 0.30337481143159106], color: "red_4794"},​ + {id: 5, vector: [0.985825131989184, -0.8144651566660419, 0.6299267002202009, 0.1206906911183383, -0.1446277761879955], color: "yellow_4222"},​ + {id: 6, vector: [0.8371977790571115, -0.015764369584852833, -0.31062937026679327, -0.562666951622192, -0.8984947637863987], color: "red_9392"},​ + {id: 7, vector: [-0.33445148015177995, -0.2567135004164067, 0.8987539745369246, 0.9402995886420709, 0.5378064918413052], color: "grey_8510"},​ + {id: 8, vector: [0.39524717779832685, 0.4000257286739164, -0.5890507376891594, -0.8650502298996872, -0.6140360785406336], color: "white_9381"},​ + {id: 9, vector: [0.5718280481994695, 0.24070317428066512, -0.3737913482606834, -0.06726932177492717, -0.6980531615588608], color: "purple_4976"} ​ +]​ +​ +var res = await client.insert({​ + collection_name: "quick_setup",​ + data: data,​ +})​ +​ +console.log(res.insert_cnt)​ +​ +// Output​ +// ​ +// 10​ +// ​ + +``` + +```curl +export CLUSTER_ENDPOINT="http://localhost:19530"​ +export TOKEN="root:Milvus"​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/insert" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "data": [​ + {"id": 0, "vector": [0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592], "color": "pink_8682"},​ + {"id": 1, "vector": [0.19886812562848388, 0.06023560599112088, 0.6976963061752597, 0.2614474506242501, 0.838729485096104], "color": "red_7025"},​ + {"id": 2, "vector": [0.43742130801983836, -0.5597502546264526, 0.6457887650909682, 0.7894058910881185, 0.20785793220625592], "color": "orange_6781"},​ + {"id": 3, "vector": [0.3172005263489739, 0.9719044792798428, -0.36981146090600725, -0.4860894583077995, 0.95791889146345], "color": "pink_9298"},​ + {"id": 4, "vector": [0.4452349528804562, -0.8757026943054742, 0.8220779437047674, 0.46406290649483184, 0.30337481143159106], "color": "red_4794"},​ + {"id": 5, "vector": [0.985825131989184, -0.8144651566660419, 0.6299267002202009, 0.1206906911183383, -0.1446277761879955], "color": "yellow_4222"},​ + {"id": 6, "vector": [0.8371977790571115, -0.015764369584852833, -0.31062937026679327, -0.562666951622192, -0.8984947637863987], "color": "red_9392"},​ + {"id": 7, "vector": [-0.33445148015177995, -0.2567135004164067, 0.8987539745369246, 0.9402995886420709, 0.5378064918413052], "color": "grey_8510"},​ + {"id": 8, "vector": [0.39524717779832685, 0.4000257286739164, -0.5890507376891594, -0.8650502298996872, -0.6140360785406336], "color": "white_9381"},​ + {"id": 9, "vector": [0.5718280481994695, 0.24070317428066512, -0.3737913482606834, -0.06726932177492717, -0.6980531615588608], "color": "purple_4976"} ​ + ],​ + "collectionName": "my_dynamic_collection"​ +}'​ +​ +# {​ +# "code": 0,​ +# "data": {​ +# "insertCount": 10,​ +# "insertIds": [​ +# 0,​ +# 1,​ +# 2,​ +# 3,​ +# 4,​ +# 5,​ +# 6,​ +# 7,​ +# 8,​ +# 9​ +# ]​ +# }​ +# }​ + +``` + +### Query and search with dynamic field​ + +Milvus supports the use of filter expressions during queries and searches, allowing you to specify which fields to include in the results. The following example demonstrates how to perform queries and searches using the `color` field, which is not defined in the schema, by using the dynamic field.​ + +
+ Python + Java + Node.js + Go + cURL +
+ +```python +query_vector = [0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592]​ +​ +res = client.search(​ + collection_name="my_dynamic_collection",​ + data=[query_vector],​ + limit=5,​ + # highlight-start​ + filter='color like "red%"',​ + output_fields=["color"]​ + # highlight-end​ +)​ +​ +print(res)​ +​ +# Output​ +# data: ["[{'id': 1, 'distance': 0.6290165185928345, 'entity': {'color': 'red_7025'}}, {'id': 4, 'distance': 0.5975797176361084, 'entity': {'color': 'red_4794'}}, {'id': 6, 'distance': -0.24996188282966614, 'entity': {'color': 'red_9392'}}]"] ​ +​ + +``` + +```java +import io.milvus.v2.service.vector.request.SearchReq​ +import io.milvus.v2.service.vector.request.data.FloatVec;​ +import io.milvus.v2.service.vector.response.SearchResp​ +​ +FloatVec queryVector = new FloatVec(new float[]{0.3580376395471989f, -0.6023495712049978f, 0.18414012509913835f, -0.26286205330961354f, 0.9029438446296592f});​ +SearchResp resp = client.search(SearchReq.builder()​ + .collectionName("my_dynamic_collection")​ + .annsField("vector")​ + .data(Collections.singletonList(queryVector))​ + .outputFields(Collections.singletonList("color"))​ + .filter("color like \"red%\"")​ + .topK(5)​ + .consistencyLevel(ConsistencyLevel.STRONG)​ + .build());​ +​ +System.out.println(resp.getSearchResults());​ +​ +// Output​ +//​ +// [[SearchResp.SearchResult(entity={color=red_7025}, score=0.6290165, id=1), SearchResp.SearchResult(entity={color=red_4794}, score=0.5975797, id=4), SearchResp.SearchResult(entity={color=red_9392}, score=-0.24996188, id=6)]]​ +​ + +``` + +```javascript +const query_vector = [0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592]​ +​ +res = await client.search({​ + collection_name: "quick_setup",​ + data: [query_vector],​ + limit: 5,​ + // highlight-start​ + filters: "color like \"red%\"",​ + output_fields: ["color"]​ + // highlight-end​ +})​ + +``` + +```curl +export CLUSTER_ENDPOINT="http://localhost:19530"​ +export TOKEN="root:Milvus"​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/search" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "my_dynamic_collection",​ + "data": [​ + [0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592]​ + ],​ + "annsField": "vector",​ + "filter": "color like \"red%\"",​ + "limit": 3,​ + "outputFields": ["color"]​ +}'​ +# {"code":0,"cost":0,"data":[{"color":"red_7025","distance":0.6290165,"id":1},{"color":"red_4794","distance":0.5975797,"id":4},{"color":"red_9392","distance":-0.24996185,"id":6}]}​ + +``` + +In the filter expression used in the code example above, `color like "red%" and likes > 50`, the conditions specify that the value of the `color` field must start with **"red"**. In the sample data, only two entities meet this condition. Thus, when `limit` (topK) is set to `3` or fewer, both of these entities will be returned.​ + +```JSON +[​ + {​ + "id": 4, ​ + "distance": 0.3345786594834839,​ + "entity": {​ + "vector": [0.4452349528804562, -0.8757026943054742, 0.8220779437047674, 0.46406290649483184, 0.30337481143159106], ​ + "color": "red_4794", ​ + "likes": 122​ + }​ + },​ + {​ + "id": 6, ​ + "distance": 0.6638239834383389，​ + "entity": {​ + "vector": [0.8371977790571115, -0.015764369584852833, -0.31062937026679327, -0.562666951622192, -0.8984947637863987], ​ + "color": "red_9392", ​ + "likes": 58​ + }​ + },​ +]​ + +``` + +​ + diff --git a/v2.4.x/site/en/userGuide/schema/nullable-and-default.md b/v2.4.x/site/en/userGuide/schema/nullable-and-default.md new file mode 100644 index 0000000000..f62d1ff8af --- /dev/null +++ b/v2.4.x/site/en/userGuide/schema/nullable-and-default.md @@ -0,0 +1,955 @@ +--- +id: nullable-and-default.md +title: Nullable & Default​ +related_key: nullable, default +summary: Milvus allows you to set the `nullable` attribute and default values for scalar fields, except the primary field. For fields marked as nullable=True, you can skip the field when inserting data, or set it directly to a null value, and the system will treat it as null without causing an error. +--- + +# Nullable & Default​ + +Milvus allows you to set the `nullable` attribute and default values for scalar fields, except the primary field. For fields marked as `nullable=True`, you can skip the field when inserting data, or set it directly to a null value, and the system will treat it as null without causing an error. When a field has a default value, the system will automatically apply this value if no data is specified for the field during insertion.​ + +The default value and nullable attributes streamline data migration from other database systems to Milvus by allowing handling of datasets with null values and preserving default value settings. When creating a collection, you can also enable nullable or set default values for fields where values might be uncertain.​ + +## Limits + +- Only scalar fields, excluding the primary field, support default values and the nullable attribute.​ + +- JSON and Array fields do not support default values.​ + +- Default values or the nullable attribute can only be configured during collection creation and cannot be modified afterward.​ + +- Scalar fields with the nullable attribute enabled cannot be used as `group_by_field` in Grouping Search. For more information about grouping search, refer to [​Grouping Search](grouping-search.md).​ + +- Fields marked as nullable cannot be used as partition keys. For more information about partition keys, refer to [​Use Partition Key](use-partition-key.md).​ + +- When creating an index on a scalar field with the nullable attribute enabled, null values will be excluded from the index.​ + +## Nullable attribute + +The `nullable` attribute allows you to store null values in a collection, providing flexibility when handling unknown data.​ + +### Set the nullable attribute​ + +When creating a collection, use `nullable=True` to define nullable fields (defaults to `False`). The following example creates a collection named `user_profiles_null` and sets the `age` field as nullable:​ + +
+ Python + Java + Node.js + cURL +
+ +```python +from pymilvus import MilvusClient, DataType​ +​ +client = MilvusClient(uri='http://localhost:19530')​ +​ +# Define collection schema​ +schema = client.create_schema(​ + auto_id=False,​ + enable_dynamic_schema=True,​ +)​ +​ +schema.add_field(field_name="id", datatype=DataType.INT64, is_primary=True)​ +schema.add_field(field_name="vector", datatype=DataType.FLOAT_VECTOR, dim=5)​ +schema.add_field(field_name="age", datatype=DataType.INT64, nullable=True) # Nullable field​ +​ +# Set index params​ +index_params = client.prepare_index_params()​ +index_params.add_index(field_name="vector", index_type="IVF_FLAT", metric_type="L2", params={ "nlist": 128 })​ +​ +# Create collection​ +client.create_collection(collection_name="user_profiles_null", schema=schema, index_params=index_params)​ + +``` + +```java +import io.milvus.v2.client.ConnectConfig;​ +import io.milvus.v2.client.MilvusClientV2;​ +import io.milvus.v2.common.DataType;​ +import io.milvus.v2.common.IndexParam;​ +import io.milvus.v2.service.collection.request.AddFieldReq;​ +import io.milvus.v2.service.collection.request.CreateCollectionReq;​ +​ +import java.util.*;​ +​ +MilvusClientV2 client = new MilvusClientV2(ConnectConfig.builder()​ + .uri("http://localhost:19530")​ + .build());​ + ​ +CreateCollectionReq.CollectionSchema schema = client.createSchema();​ +schema.setEnableDynamicField(true);​ +​ +schema.addField(AddFieldReq.builder()​ + .fieldName("id")​ + .dataType(DataType.Int64)​ + .isPrimaryKey(true)​ + .build());​ +​ +schema.addField(AddFieldReq.builder()​ + .fieldName("vector")​ + .dataType(DataType.FloatVector)​ + .dimension(5)​ + .build());​ +​ +schema.addField(AddFieldReq.builder()​ + .fieldName("age")​ + .dataType(DataType.Int64)​ + .isNullable(true)​ + .build());​ +​ +List indexes = new ArrayList<>();​ +Map extraParams = new HashMap<>();​ +extraParams.put("nlist", 128);​ +indexes.add(IndexParam.builder()​ + .fieldName("vector")​ + .indexType(IndexParam.IndexType.IVF_FLAT)​ + .metricType(IndexParam.MetricType.L2)​ + .extraParams(extraParams)​ + .build());​ +​ +CreateCollectionReq requestCreate = CreateCollectionReq.builder()​ + .collectionName("user_profiles_null")​ + .collectionSchema(schema)​ + .indexParams(indexes)​ + .build();​ +client.createCollection(requestCreate);​ + +``` + +```javascript +import { MilvusClient, DataType } from "@zilliz/milvus2-sdk-node";​ +​ +const client = new MilvusClient({​ + address: "http://localhost:19530",​ + token: "root:Milvus",​ +});​ +​ +await client.createCollection({​ + collection_name: "user_profiles_null",​ + schema: [​ + {​ + name: "id",​ + is_primary_key: true,​ + data_type: DataType.int64,​ + },​ + { name: "vector", data_type: DataType.Int64, dim: 5 },​ +​ + { name: "age", data_type: DataType.FloatVector, nullable: true },​ + ],​ +​ + index_params: [​ + {​ + index_name: "vector_inde",​ + field_name: "vector",​ + metric_type: MetricType.L2,​ + index_type: IndexType.AUTOINDEX,​ + },​ + ],​ +});​ +​ + +``` + +```curl +export pkField='{​ + "fieldName": "id",​ + "dataType": "Int64",​ + "isPrimary": true​ +}'​ +​ +export vectorField='{​ + "fieldName": "vector",​ + "dataType": "FloatVector",​ + "elementTypeParams": {​ + "dim": 5​ + }​ +}'​ +​ +export nullField='{​ + "fieldName": "age",​ + "dataType": "Int64",​ + "nullable": true​ +}'​ +​ +export schema="{​ + \"autoID\": false,​ + \"fields\": [​ + $pkField,​ + $vectorField,​ + $nullField​ + ]​ +}"​ +​ +export indexParams='[​ + {​ + "fieldName": "vector",​ + "metricType": "L2",​ + "indexType": "IVF_FLAT",​ + "params":{"nlist": 128}​ + }​ + ]'​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/collections/create" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d "{​ + \"collectionName\": \"user_profiles_null\",​ + \"schema\": $schema,​ + \"indexParams\": $indexParams​ +}"​ + +``` + +### Insert entities + +When you insert data into a nullable field, insert null or directly omit this field:​ + +
+ Python + Java + Node.js + cURL +
+ +```python +data = [​ + {"id": 1, "vector": [0.1, 0.2, 0.3, 0.4, 0.5], "age": 30},​ + {"id": 2, "vector": [0.2, 0.3, 0.4, 0.5, 0.6], "age": None},​ + {"id": 3, "vector": [0.3, 0.4, 0.5, 0.6, 0.7]}​ +]​ +​ +client.insert(collection_name="user_profiles_null", data=data)​ + +``` + +```java +import com.google.gson.Gson;​ +import com.google.gson.JsonObject;​ +​ +import io.milvus.v2.service.vector.request.InsertReq;​ +import io.milvus.v2.service.vector.response.InsertResp;​ +​ +List rows = new ArrayList<>();​ +Gson gson = new Gson();​ +rows.add(gson.fromJson("{\"id\": 1, \"vector\": [0.1, 0.2, 0.3, 0.4, 0.5], \"age\": 30}", JsonObject.class));​ +rows.add(gson.fromJson("{\"id\": 2, \"vector\": [0.2, 0.3, 0.4, 0.5, 0.6], \"age\": null}", JsonObject.class));​ +rows.add(gson.fromJson("{\"id\": 3, \"vector\": [0.3, 0.4, 0.5, 0.6, 0.7]}", JsonObject.class));​ +​ +InsertResp insertR = client.insert(InsertReq.builder()​ + .collectionName("user_profiles_null")​ + .data(rows)​ + .build());​ + +``` + +```javascript +const data = [​ + { id: 1, vector: [0.1, 0.2, 0.3, 0.4, 0.5], age: 30 },​ + { id: 2, vector: [0.2, 0.3, 0.4, 0.5, 0.6], age: null },​ + { id: 3, vector: [0.3, 0.4, 0.5, 0.6, 0.7] },​ +];​ +​ +client.insert({​ + collection_name: "user_profiles_null",​ + data: data,​ +});​ +​ + +``` + +```curl +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/insert" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "data": [​ + {"id": 1, "vector": [0.1, 0.2, 0.3, 0.4, 0.5], "age": 30},​ + {"id": 2, "vector": [0.2, 0.3, 0.4, 0.5, 0.6], "age": null}, ​ + {"id": 3, "vector": [0.3, 0.4, 0.5, 0.6, 0.7]} ​ + ],​ + "collectionName": "user_profiles_null"​ +}'​ + +``` + +### Search and query with null values​ + +When using the `search` method, if a field contains `null` values, the search result will return the field as null:​ + +
+ Python + Java + Node.js + cURL +
+ +```python +res = client.search(​ + collection_name="user_profiles_null",​ + data=[[0.1, 0.2, 0.4, 0.3, 0.128]],​ + limit=2,​ + search_params={"params": {"nprobe": 16}},​ + output_fields=["id", "age"]​ +)​ +​ +print(res)​ +​ +# Output​ +# data: ["[{'id': 1, 'distance': 0.15838398039340973, 'entity': {'age': 30, 'id': 1}}, {'id': 2, 'distance': 0.28278401494026184, 'entity': {'age': None, 'id': 2}}]"] ​ + +``` + +```java +import io.milvus.v2.service.vector.request.SearchReq;​ +import io.milvus.v2.service.vector.request.data.FloatVec;​ +import io.milvus.v2.service.vector.response.SearchResp;​ +​ +Map params = new HashMap<>();​ +params.put("nprobe", 16);​ +SearchResp resp = client.search(SearchReq.builder()​ + .collectionName("user_profiles_null")​ + .annsField("vector")​ + .data(Collections.singletonList(new FloatVec(new float[]{0.1f, 0.2f, 0.3f, 0.4f, 0.5f})))​ + .topK(2)​ + .searchParams(params)​ + .outputFields(Arrays.asList("id", "age"))​ + .build());​ +​ +System.out.println(resp.getSearchResults());​ +​ +// Output​ +//​ +// [[SearchResp.SearchResult(entity={id=1, age=30}, score=0.0, id=1), SearchResp.SearchResult(entity={id=2, age=null}, score=0.050000004, id=2)]]​ + +``` + +```javascript +client.search({​ + collection_name: 'user_profiles_null',​ + data: [0.3, -0.6, 0.1, 0.3, 0.5],​ + limit: 2,​ + output_fields: ['age', 'id'],​ + filter: '25 <= age <= 35',​ + params: {​ + nprobe: 16​ + }​ +});​ + +``` + +```curl +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/search" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "user_profiles_null",​ + "data": [​ + [0.1, -0.2, 0.3, 0.4, 0.5]​ + ],​ + "annsField": "vector",​ + "limit": 5,​ + "outputFields": ["id", "age"]​ +}'​ +​ +#{"code":0,"cost":0,"data":[{"age":30,"distance":0.16000001,"id":1},{"age":null,"distance":0.28999996,"id":2},{"age":null,"distance":0.52000004,"id":3}]}​ + +``` + +When you use the `query` method for scalar filtering, the filtering results for null values are all false, indicating that they will not be selected.​ + +
+ Python + Java + Node.js + cURL +
+ +```python +# Reviewing previously inserted data:​ +# {"id": 1, "vector": [0.1, 0.2, ..., 0.128], "age": 30}​ +# {"id": 2, "vector": [0.2, 0.3, ..., 0.129], "age": None}​ +# {"id": 3, "vector": [0.3, 0.4, ..., 0.130], "age": None} # Omitted age column is treated as None​ +​ +results = client.query(​ + collection_name="user_profiles_null",​ + filter="age >= 0",​ + output_fields=["id", "age"]​ +)​ +​ +# Example output:​ +# [​ +# {"id": 1, "age": 30}​ +# ]​ +# Note: Entities with `age` as `null` (id 2 and 3) will not appear in the result.​ + +``` + +```java +import io.milvus.v2.service.vector.request.QueryReq;​ +import io.milvus.v2.service.vector.response.QueryResp;​ +​ +QueryResp resp = client.query(QueryReq.builder()​ + .collectionName("user_profiles_null")​ + .filter("age >= 0")​ + .outputFields(Arrays.asList("id", "age"))​ + .build());​ +​ +System.out.println(resp.getQueryResults());​ +​ +// Output​ +//​ +// [QueryResp.QueryResult(entity={id=1, age=30})]​ + +``` + +```javascript +const results = await client.query(​ + collection_name: "user_profiles_null",​ + filter: "age >= 0",​ + output_fields: ["id", "age"]​ +);​ + +``` + +```curl +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/query" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "user_profiles_null",​ + "filter": "age >= 0",​ + "outputFields": ["id", "age"]​ +}'​ +​ +# {"code":0,"cost":0,"data":[{"age":30,"id":1}]}​ + +``` + +To query entities with `null` values, use an empty expression `""`:​ + +
+ Python + Java + Node.js + cURL +
+ +```python +null_results = client.query(​ + collection_name="user_profiles_null",​ + filter="",​ + output_fields=["id", "age"]​ +)​ +​ +# Example output:​ +# [{"id": 2, "age": None}, {"id": 3, "age": None}]​ + +``` + +```java +QueryResp resp = client.query(QueryReq.builder()​ + .collectionName("user_profiles_null")​ + .filter("")​ + .outputFields(Arrays.asList("id", "age"))​ + .limit(10)​ + .build());​ +​ +System.out.println(resp.getQueryResults());​ + +``` + +```javascript +const results = await client.query(​ + collection_name: "user_profiles_null",​ + filter: "",​ + output_fields: ["id", "age"]​ +);​ + +``` + +```curl +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/query" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "user_profiles_null",​ + "expr": "",​ + "outputFields": ["id", "age"]​ +}'​ +​ +# {"code":0,"cost":0,"data":[{"age":30,"id":1},{"age":null,"id":2},{"age":null,"id":3}]}​ + +``` + +## Default values​ + +Default values are preset values assigned to scalar fields. If you do not provide a value for a field with a default during insertion, the system automatically uses the default value.​ + +### Set default values + +When creating a collection, use the `default_value` parameter to define the default value for a field. The following example shows how to set the default value of `age` to `18` and `status` to `"active"`:​ + +
+ Python + Java + Node.js + cURL +
+ +```python +schema = client.create_schema(​ + auto_id=False,​ + enable_dynamic_schema=True,​ +)​ +​ +schema.add_field(field_name="id", datatype=DataType.INT64, is_primary=True)​ +schema.add_field(field_name="vector", datatype=DataType.FLOAT_VECTOR, dim=5)​ +schema.add_field(field_name="age", datatype=DataType.INT64, default_value=18)​ +schema.add_field(field_name="status", datatype=DataType.VARCHAR, default_value="active", max_length=10)​ +​ +index_params = client.prepare_index_params()​ +index_params.add_index(field_name="vector", index_type="IVF_FLAT", metric_type="L2", params={ "nlist": 128 })​ +​ +client.create_collection(collection_name="user_profiles_default", schema=schema, index_params=index_params)​ + +``` + +```java +import io.milvus.v2.common.DataType;​ +import io.milvus.v2.common.IndexParam;​ +import io.milvus.v2.service.collection.request.AddFieldReq;​ +import io.milvus.v2.service.collection.request.CreateCollectionReq;​ +​ +import java.util.*;​ +​ +CreateCollectionReq.CollectionSchema schema = client.createSchema();​ +schema.setEnableDynamicField(true);​ +​ +schema.addField(AddFieldReq.builder()​ + .fieldName("id")​ + .dataType(DataType.Int64)​ + .isPrimaryKey(true)​ + .build());​ +​ +schema.addField(AddFieldReq.builder()​ + .fieldName("vector")​ + .dataType(DataType.FloatVector)​ + .dimension(5)​ + .build());​ +​ +schema.addField(AddFieldReq.builder()​ + .fieldName("age")​ + .dataType(DataType.Int64)​ + .defaultValue(18L)​ + .build());​ +​ +schema.addField(AddFieldReq.builder()​ + .fieldName("status")​ + .dataType(DataType.VarChar)​ + .maxLength(10)​ + .defaultValue("active")​ + .build());​ +​ +List indexes = new ArrayList<>();​ +Map extraParams = new HashMap<>();​ +extraParams.put("nlist", 128);​ +indexes.add(IndexParam.builder()​ + .fieldName("vector")​ + .indexType(IndexParam.IndexType.IVF_FLAT)​ + .metricType(IndexParam.MetricType.L2)​ + .extraParams(extraParams)​ + .build());​ +​ +CreateCollectionReq requestCreate = CreateCollectionReq.builder()​ + .collectionName("user_profiles_default")​ + .collectionSchema(schema)​ + .indexParams(indexes)​ + .build();​ +client.createCollection(requestCreate);​ + +``` + +```javascript +import { MilvusClient, DataType } from "@zilliz/milvus2-sdk-node";​ +​ +const client = new MilvusClient({​ + address: "http://localhost:19530",​ + token: "root:Milvus",​ +});​ +​ +await client.createCollection({​ + collection_name: "user_profiles_default",​ + schema: [​ + {​ + name: "id",​ + is_primary_key: true,​ + data_type: DataType.int64,​ + },​ + { name: "vector", data_type: DataType.FloatVector, dim: 5 },​ + { name: "age", data_type: DataType.Int64, default_value: 18 },​ + { name: 'status', data_type: DataType.VarChar, max_length: 30, default_value: 'active'},​ + ],​ +​ + index_params: [​ + {​ + index_name: "vector_inde",​ + field_name: "vector",​ + metric_type: MetricType.L2,​ + index_type: IndexType.IVF_FLAT,​ + },​ + ],​ +});​ +​ + +``` + +```curl +export pkField='{​ + "fieldName": "id",​ + "dataType": "Int64",​ + "isPrimary": true​ +}'​ +​ +export vectorField='{​ + "fieldName": "vector",​ + "dataType": "FloatVector",​ + "elementTypeParams": {​ + "dim": 5​ + }​ +}'​ +​ +export defaultValueField1='{​ + "fieldName": "age",​ + "dataType": "Int64",​ + "defaultValue": 18​ +}'​ +​ +export defaultValueField2='{​ + "fieldName": "status",​ + "dataType": "VarChar",​ + "defaultValue": "active",​ + "elementTypeParams": {​ + "max_length": 10​ + }​ +}'​ +​ +export schema="{​ + \"autoID\": false,​ + \"fields\": [​ + $pkField,​ + $vectorField,​ + $defaultValueField1,​ + $defaultValueField2​ + ]​ +}"​ +​ +export indexParams='[​ + {​ + "fieldName": "vector",​ + "metricType": "L2",​ + "indexType": "IVF_FLAT",​ + "params":{"nlist": 128}​ + }​ + ]'​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/collections/create" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d "{​ + \"collectionName\": \"user_profiles_default\",​ + \"schema\": $schema,​ + \"indexParams\": $indexParams​ +}"​ + +``` + +### Insert entities + +When inserting data, if you omit fields with a default value or set their value to null, the system uses the default value:​ + +
+ Python + Java + Node.js + cURL +
+ +```python +data = [​ + {"id": 1, "vector": [0.1, 0.2, ..., 0.128], "age": 30, "status": "premium"},​ + {"id": 2, "vector": [0.2, 0.3, ..., 0.129]}, + {"id": 3, "vector": [0.3, 0.4, ..., 0.130], "age": 25, "status": None}, + {"id": 4, "vector": [0.4, 0.5, ..., 0.131], "age": None, "status": "inactive"} +]​ +​ +client.insert(collection_name="user_profiles_default", data=data)​ + +``` + +```java +import com.google.gson.Gson;​ +import com.google.gson.JsonObject;​ +​ +import io.milvus.v2.service.vector.request.InsertReq;​ +import io.milvus.v2.service.vector.response.InsertResp;​ +​ +List rows = new ArrayList<>();​ +Gson gson = new Gson();​ +rows.add(gson.fromJson("{\"id\": 1, \"vector\": [0.1, 0.2, 0.3, 0.4, 0.5], \"age\": 30, \"status\": \"premium\"}", JsonObject.class));​ +rows.add(gson.fromJson("{\"id\": 2, \"vector\": [0.2, 0.3, 0.4, 0.5, 0.6]}", JsonObject.class));​ +rows.add(gson.fromJson("{\"id\": 3, \"vector\": [0.3, 0.4, 0.5, 0.6, 0.7], \"age\": 25, \"status\": null}", JsonObject.class));​ +rows.add(gson.fromJson("{\"id\": 4, \"vector\": [0.4, 0.5, 0.6, 0.7, 0.8], \"age\": null, \"status\": \"inactive\"}", JsonObject.class));​ +​ +InsertResp insertR = client.insert(InsertReq.builder()​ + .collectionName("user_profiles_default")​ + .data(rows)​ + .build());​ + +``` + +```javascript +const data = [​ + {"id": 1, "vector": [0.1, 0.2, 0.3, 0.4, 0.5], "age": 30, "status": "premium"},​ + {"id": 2, "vector": [0.2, 0.3, 0.4, 0.5, 0.6]}, ​ + {"id": 3, "vector": [0.3, 0.4, 0.5, 0.6, 0.7], "age": 25, "status": null}, ​ + {"id": 4, "vector": [0.4, 0.5, 0.6, 0.7, 0.8], "age": null, "status": "inactive"} ​ +];​ +​ +client.insert({​ + collection_name: "user_profiles_default",​ + data: data,​ +});​ + +``` + +```curl +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/insert" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "data": [​ + {"id": 1, "vector": [0.1, 0.2, 0.3, 0.4, 0.5], "age": 30, "status": "premium"},​ + {"id": 2, "vector": [0.2, 0.3, 0.4, 0.5, 0.6]},​ + {"id": 3, "vector": [0.3, 0.4, 0.5, 0.6, 0.7], "age": 25, "status": null}, ​ + {"id": 4, "vector": [0.4, 0.5, 0.6, 0.7, 0.8], "age": null, "status": "inactive"} ​ + ],​ + "collectionName": "user_profiles_default"​ +}'​ + +``` + +
+ +For more information on how nullable and default value settings take effect, refer to [Applicable rules](#applicable-rules). + +
+ +### Search and query with default values + +Entities that contain default values are treated the same as any other entities during vector searches and scalar filtering. You can include default values as part of your `search` and `query` operations.​ + +For example, in a `search` operation, entities with `age` set to the default value of `18` will be included in the results:​ + +
+ Python + Java + Node.js + cURL +
+ +```python +res = client.search(​ + collection_name="user_profiles_default",​ + data=[[0.1, 0.2, 0.4, 0.3, 0.128]],​ + search_params={"params": {"nprobe": 16}},​ + filter="age == 18", # 18 is the default value of the `age` field​ + limit=10,​ + output_fields=["id", "age", "status"]​ +)​ +​ +print(res)​ +​ +# Output​ +# data: ["[{'id': 2, 'distance': 0.28278401494026184, 'entity': {'id': 2, 'age': 18, 'status': 'active'}}, {'id': 4, 'distance': 0.8315839767456055, 'entity': {'id': 4, 'age': 18, 'status': 'inactive'}}]"] ​ +​ + +``` + +```java +import io.milvus.v2.service.vector.request.SearchReq;​ +import io.milvus.v2.service.vector.request.data.FloatVec;​ +import io.milvus.v2.service.vector.response.SearchResp;​ +​ +Map params = new HashMap<>();​ +params.put("nprobe", 16);​ +SearchResp resp = client.search(SearchReq.builder()​ + .collectionName("user_profiles_default")​ + .annsField("vector")​ + .data(Collections.singletonList(new FloatVec(new float[]{0.1f, 0.2f, 0.3f, 0.4f, 0.5f})))​ + .searchParams(params)​ + .filter("age == 18")​ + .topK(10)​ + .outputFields(Arrays.asList("id", "age", "status"))​ + .build());​ +​ +System.out.println(resp.getSearchResults());​ +​ +// Output​ +//​ +// [[SearchResp.SearchResult(entity={id=2, age=18, status=active}, score=0.050000004, id=2), SearchResp.SearchResult(entity={id=4, age=18, status=inactive}, score=0.45000002, id=4)]]​ + +``` + +```javascript +client.search({​ + collection_name: 'user_profiles_default',​ + data: [0.3, -0.6, 0.1, 0.3, 0.5],​ + limit: 2,​ + output_fields: ['age', 'id', 'status'],​ + filter: 'age == 18',​ + params: {​ + nprobe: 16​ + }​ +});​ + +``` + +```curl +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/search" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "user_profiles_default",​ + "data": [​ + [0.1, 0.2, 0.3, 0.4, 0.5]​ + ],​ + "annsField": "vector",​ + "limit": 5,​ + "filter": "age == 18",​ + "outputFields": ["id", "age", "status"]​ +}'​ +​ +# {"code":0,"cost":0,"data":[{"age":18,"distance":0.050000004,"id":2,"status":"active"},{"age":18,"distance":0.45000002,"id":4,"status":"inactive"}]}​ + +``` + +In a `query` operation, you can match or filter by default values directly:​ + +
+ Python + Java + Node.js + cURL +
+ +```python +# Query all entities where `age` equals the default value (18)​ +default_age_results = client.query(​ + collection_name="user_profiles_default",​ + filter="age == 18",​ + output_fields=["id", "age", "status"]​ +)​ +​ +# Query all entities where `status` equals the default value ("active")​ +default_status_results = client.query(​ + collection_name="user_profiles_default",​ + filter='status == "active"',​ + output_fields=["id", "age", "status"]​ +)​ + +``` + +```java +import io.milvus.v2.service.vector.request.QueryReq;​ +import io.milvus.v2.service.vector.response.QueryResp;​ +​ +QueryResp ageResp = client.query(QueryReq.builder()​ + .collectionName("user_profiles_default")​ + .filter("age == 18")​ + .outputFields(Arrays.asList("id", "age", "status"))​ + .build());​ +​ +System.out.println(ageResp.getQueryResults());​ +​ +// Output​ +//​ +// [QueryResp.QueryResult(entity={id=2, age=18, status=active}), QueryResp.QueryResult(entity={id=4, age=18, status=inactive})]​ +​ +QueryResp statusResp = client.query(QueryReq.builder()​ + .collectionName("user_profiles_default")​ + .filter("status == \"active\"")​ + .outputFields(Arrays.asList("id", "age", "status"))​ + .build());​ +​ +System.out.println(statusResp.getQueryResults());​ +​ +// Output​ +//​ +// [QueryResp.QueryResult(entity={id=2, age=18, status=active}), QueryResp.QueryResult(entity={id=3, age=25, status=active})]​ + +``` + +```javascript +// Query all entities where `age` equals the default value (18)​ +const default_age_results = await client.query(​ + collection_name: "user_profiles_default",​ + filter: "age == 18",​ + output_fields: ["id", "age", "status"]​ +);​ +// Query all entities where `status` equals the default value ("active")​ +const default_status_results = await client.query(​ + collection_name: "user_profiles_default",​ + filter: 'status == "active"',​ + output_fields: ["id", "age", "status"]​ +)​ + +``` + +```curl +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/query" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "user_profiles_default",​ + "filter": "age == 18",​ + "outputFields": ["id", "age", "status"]​ +}'​ +​ +# {"code":0,"cost":0,"data":[{"age":18,"id":2,"status":"active"},{"age":18,"id":4,"status":"inactive"}]}​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/query" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "user_profiles_default",​ + "filter": "status == \"active\"",​ + "outputFields": ["id", "age", "status"]​ +}'​ +​ +# {"code":0,"cost":0,"data":[{"age":18,"id":2,"status":"active"},{"age":25,"id":3,"status":"active"}]}​ + +``` + +## Applicable rules + +The following table summarizes the behavior of nullable columns and default values under different configuration combinations. These rules determine how Milvus handles data when attempting to insert null values or if field values are not provided.​ +​ +| Nullable | Default Value | Default Value Type | User Input | Result | Example | +|----------|---------------|--------------------|------------|--------|---------| +| ✅ | ✅ | Non-null | None/null | Uses the default value |
  • Field: `age`
  • Default value: `18`
  • User input: null
  • Result: stored as `18`
| +| ✅ | ❌ | - | None/null | Stored as null |
  • Field: `middle_name`
  • Default value: -
  • User input: null
  • Result: stored as null | +| ❌ | ✅ | Non-null | None/null | Uses the default value |
    • Field: `status`
    • Default value: `"active"`
    • User input: null
    • Result: stored as `"active"` | +| ❌ | ❌ | - | None/null | Throws an error |
      • Field: `email`
      • Default value: -
      • User input: null
      • Result: Operation rejected, system throws an error | +| ❌ | ✅ | Null | None/null | Throws an error |
        • Field: `username`
        • Default value: null
        • User input: null
        • Result: Operation rejected, system throws an error | diff --git a/v2.4.x/site/en/userGuide/schema/number.md b/v2.4.x/site/en/userGuide/schema/number.md new file mode 100644 index 0000000000..200db8f9d9 --- /dev/null +++ b/v2.4.x/site/en/userGuide/schema/number.md @@ -0,0 +1,606 @@ +--- +id: number.md +title: Number Field​ +related_key: number, integer, float, double +summary: Number fields are used to store non-vector numerical data in Milvus. These fields are typically employed to describe additional information related to vector data, such as age, price, etc. By using this data, you can better describe vectors and improve the efficiency of data filtering and conditional queries.​ +--- + +# Number Field​ + +Number fields are used to store non-vector numerical data in Milvus. These fields are typically employed to describe additional information related to vector data, such as age, price, etc. By using this data, you can better describe vectors and improve the efficiency of data filtering and conditional queries.​ + +Number fields are particularly useful in many scenarios. For example, in e-commerce recommendations, a price field can be used for filtering; in user profile analysis, age ranges can help refine the results. Combined with vector data, number fields can help the system provide similarity searches while meeting personalized user needs more precisely.​ + +## Supported number field types​ + +Milvus supports various number field types to meet different data storage and query needs:​ + +

          Type​

          + +

          Description​

          + +

          BOOL​

          + +

          Boolean type for storing true or false, suitable for describing binary states.​

          + +

          INT8​

          + +

          8-bit integer, suitable for storing small-range integer data.​

          + +

          INT16​

          + +

          16-bit integer, for medium-range integer data.​

          + +

          INT32​

          + +

          32-bit integer, ideal for general integer data storage like product quantities or user IDs.​

          + +

          INT64​

          + +

          64-bit integer, suitable for storing large-range data like timestamps or identifiers.​

          + +

          FLOAT​

          + +

          32-bit floating-point number, for data requiring general precision, such as ratings or temperature.​

          + +

          DOUBLE​

          + +

          64-bit double-precision floating-point number, for high-precision data like financial information or scientific calculations.​

          + +
          + +## Add number field​ + +To use number fields in Milvus, define the relevant fields in the collection schema, setting the `datatype` to a supported type such as `BOOL` or `INT8`. For a complete list of supported number field types, refer to [Supported number field types](#Supported-number-field-types).​ + +The following example shows how to define a schema that includes number fields `age` and `price`:​ + +
          + Python + Java + Node.js + cURL +
          + +```python +from pymilvus import MilvusClient, DataType​ +​ +client = MilvusClient(uri="http://localhost:19530")​ +​ +schema = client.create_schema(​ + auto_id=False,​ + enable_dynamic_fields=True,​ +)​ +​ +schema.add_field(field_name="age", datatype=DataType.INT64)​ +schema.add_field(field_name="price", datatype=DataType.FLOAT)​ +schema.add_field(field_name="pk", datatype=DataType.INT64, is_primary=True)​ +schema.add_field(field_name="embedding", datatype=DataType.FLOAT_VECTOR, dim=3)​ + +``` + +```java +import io.milvus.v2.client.ConnectConfig;​ +import io.milvus.v2.client.MilvusClientV2;​ +​ +import io.milvus.v2.common.DataType;​ +import io.milvus.v2.service.collection.request.AddFieldReq;​ +import io.milvus.v2.service.collection.request.CreateCollectionReq;​ +​ +​ +MilvusClientV2 client = new MilvusClientV2(ConnectConfig.builder()​ + .uri("http://localhost:19530")​ + .build());​ + ​ +CreateCollectionReq.CollectionSchema schema = client.createSchema();​ +schema.setEnableDynamicField(true);​ +​ +schema.addField(AddFieldReq.builder()​ + .fieldName("age")​ + .dataType(DataType.Int64)​ + .build());​ +​ +schema.addField(AddFieldReq.builder()​ + .fieldName("price")​ + .dataType(DataType.Float)​ + .build());​ +​ +schema.addField(AddFieldReq.builder()​ + .fieldName("pk")​ + .dataType(DataType.Int64)​ + .isPrimaryKey(true)​ + .build());​ +​ +schema.addField(AddFieldReq.builder()​ + .fieldName("embedding")​ + .dataType(DataType.FloatVector)​ + .dimension(3)​ + .build());​ + +``` + +```javascript +import { MilvusClient, DataType } from "@zilliz/milvus2-sdk-node";​ +const schema = [​ + {​ + name: "age",​ + data_type: DataType.Int64,​ + },​ + {​ + name: "price",​ + data_type: DataType.Float,​ + },​ + {​ + name: "pk",​ + data_type: DataType.Int64,​ + is_primary_key: true,​ + },​ + {​ + name: "embedding",​ + data_type: DataType.FloatVector,​ + dim: 3,​ + },​ +];​ +​ + +``` + +```curl +export int64Field='{​ + "fieldName": "age",​ + "dataType": "Int64"​ +}'​ +​ +export floatField='{​ + "fieldName": "price",​ + "dataType": "Float"​ +}'​ +​ +export pkField='{​ + "fieldName": "pk",​ + "dataType": "Int64",​ + "isPrimary": true​ +}'​ +​ +export vectorField='{​ + "fieldName": "embedding",​ + "dataType": "FloatVector",​ + "elementTypeParams": {​ + "dim": 3​ + }​ +}'​ +​ +export schema="{​ + \"autoID\": false,​ + \"fields\": [​ + $int64Field,​ + $floatField,​ + $pkField,​ + $vectorField​ + ]​ +}"​ + +``` + +
          + +The primary field and vector field are mandatory when you create a collection. The primary field uniquely identifies each entity, while the vector field is crucial for similarity search. For more details, refer to [​Primary Field & AutoID](primary-field.md), [​Dense Vector](dense-vector.md), [​Binary Vector](binary-vector.md), or [​Sparse Vector](sparse_vector.md).​ + +
          + +## Set index params​ + +Setting index parameters for number fields is optional but can significantly improve retrieval efficiency.​ + +In the following example, we create an `AUTOINDEX` for the `age` number field, allowing Milvus to automatically create an appropriate index based on the data type. For more information, refer to [​AUTOINDEX](https://milvus.io/docs/glossary.md#Auto-Index).​ + +
          + Python + Java + Node.js + cURL +
          + +```python +index_params = client.prepare_index_params()​ +​ +index_params.add_index(​ + field_name="age",​ + index_type="AUTOINDEX",​ + index_name="inverted_index"​ +)​ + +``` + +```java +import io.milvus.v2.common.IndexParam;​ +import java.util.*;​ +​ +List indexes = new ArrayList<>();​ +indexes.add(IndexParam.builder()​ + .fieldName("age")​ + .indexType(IndexParam.IndexType.AUTOINDEX)​ + .build());​ +​ + +``` + +```javascript +const indexParams = {​ + index_name: 'inverted_index',​ + field_name: 'age',​ + index_type: IndexType.AUTOINDEX,​ +);​ + +``` + +```curl +export indexParams='[​ + {​ + "fieldName": "age",​ + "indexName": "inverted_index",​ + "indexType": "AUTOINDEX"​ + }​ + ]'​ + +``` + +In addition to `AUTOINDEX`, you can specify other number field index types. For supported index types, refer to [​Scalar Indexes](scalar_index.md).​ + +Moreover, before creating the collection, you must create an index for the vector field. In this example, we use `AUTOINDEX` to simplify vector index settings.​ + +
          + Python + Java + Node.js + cURL +
          + +```python +# Add vector index​ +index_params.add_index(​ + field_name="embedding",​ + index_type="AUTOINDEX", # Use automatic indexing to simplify complex index settings​ + metric_type="COSINE" # Specify similarity metric type, options include L2, COSINE, or IP​ +)​ + +``` + +```java +indexes.add(IndexParam.builder()​ + .fieldName("embedding")​ + .indexType(IndexParam.IndexType.AUTOINDEX)​ + .metricType(IndexParam.MetricType.COSINE)​ + .build());​ + +``` + +```javascript +import { IndexType } from "@zilliz/milvus2-sdk-node";​ +const indexParams = [​ + {​ + field_name: "age",​ + index_name: "inverted_index",​ + index_type: IndexType.AUTOINDEX,​ + },​ + {​ + field_name: "embedding",​ + metric_type: "COSINE",​ + index_type: IndexType.AUTOINDEX,​ + },​ +];​ +​ + +``` + +```curl +export indexParams='[​ + {​ + "fieldName": "age",​ + "indexName": "inverted_index",​ + "indexType": "AUTOINDEX"​ + },​ + {​ + "fieldName": "embedding",​ + "metricType": "COSINE",​ + "indexType": "AUTOINDEX"​ + }​ + ]'​ + +``` + +## Create collection​ + +Once the schema and indexes are defined, you can create a collection that includes number fields.​ + +
          + Python + Java + Node.js + cURL +
          + +```python +# Create Collection​ +client.create_collection(​ + collection_name="your_collection_name",​ + schema=schema,​ + index_params=index_params​ +)​ + +``` + +```java +CreateCollectionReq requestCreate = CreateCollectionReq.builder()​ + .collectionName("my_scalar_collection")​ + .collectionSchema(schema)​ + .indexParams(indexes)​ + .build();​ +client.createCollection(requestCreate);​ + +``` + +```javascript +client.create_collection({​ + collection_name: "my_scalar_collection",​ + schema: schema,​ + index_params: indexParams​ +})​ + +``` + +```curl +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/collections/create" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d "{​ + \"collectionName\": \"my_scalar_collection\",​ + \"schema\": $schema,​ + \"indexParams\": $indexParams​ +}"​ + +``` + +## Insert data​ + +After creating the collection, you can insert data that includes number fields.​ + +
          + Python + Java + Node.js + cURL +
          + +```python +data = [​ + {"age": 25, "price": 99.99, "pk": 1, "embedding": [0.1, 0.2, 0.3]},​ + {"age": 30, "price": 149.50, "pk": 2, "embedding": [0.4, 0.5, 0.6]},​ + {"age": 35, "price": 199.99, "pk": 3, "embedding": [0.7, 0.8, 0.9]},​ +]​ +​ +client.insert(​ + collection_name="my_scalar_collection",​ + data=data​ +)​ + +``` + +```java +import com.google.gson.Gson;​ +import com.google.gson.JsonObject;​ +​ +import io.milvus.v2.service.vector.request.InsertReq;​ +import io.milvus.v2.service.vector.response.InsertResp;​ +​ +List rows = new ArrayList<>();​ +Gson gson = new Gson();​ +rows.add(gson.fromJson("{\"age\": 25, \"price\": 99.99, \"pk\": 1, \"embedding\": [0.1, 0.2, 0.3]}", JsonObject.class));​ +rows.add(gson.fromJson("{\"age\": 30, \"price\": 149.50, \"pk\": 2, \"embedding\": [0.4, 0.5, 0.6]}", JsonObject.class));​ +rows.add(gson.fromJson("{\"age\": 35, \"price\": 199.99, \"pk\": 3, \"embedding\": [0.7, 0.8, 0.9]}", JsonObject.class));​ +​ +InsertResp insertR = client.insert(InsertReq.builder()​ + .collectionName("my_scalar_collection")​ + .data(rows)​ + .build());​ + +``` + +```javascript +const data = [​ + { age: 25, price: 99.99, pk: 1, embedding: [0.1, 0.2, 0.3] },​ + { age: 30, price: 149.5, pk: 2, embedding: [0.4, 0.5, 0.6] },​ + { age: 35, price: 199.99, pk: 3, embedding: [0.7, 0.8, 0.9] },​ +];​ +​ +client.insert({​ + collection_name: "my_scalar_collection",​ + data: data,​ +});​ +​ + +``` + +```curl +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/insert" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "data": [​ + {"age": 25, "price": 99.99, "pk": 1, "embedding": [0.1, 0.2, 0.3]},​ + {"age": 30, "price": 149.50, "pk": 2, "embedding": [0.4, 0.5, 0.6]},​ + {"age": 35, "price": 199.99, "pk": 3, "embedding": [0.7, 0.8, 0.9]} ​ + ],​ + "collectionName": "my_scalar_collection"​ +}'​ + +``` + +In this example, we insert data that includes `age`, `price`, `pk` (primary field), and vector representations (`embedding`). To ensure that the inserted data matches the fields defined in the schema, it's recommended to check data types in advance to avoid errors.​ + +If you set `enable_dynamic_fields=True` when defining the schema, Milvus allows you to insert number fields that were not defined in advance. However, keep in mind that this may increase the complexity of queries and management, potentially impacting performance. For more information, refer to [​Dynamic Field](enable_dynamic_field.md).​ + +## Search and query​ + +After adding number fields, you can use them for filtering in search and query operations to achieve more precise search results.​ + +### Filter queries​ + +After adding number fields, you can use them for filtering in queries. For example, you can query all entities where `age` is between 30 and 40:​ + +
          + Python + Java + Node.js + cURL +
          + +```python +filter = "30 <= age <= 40"​ +​ +res = client.query(​ + collection_name="my_scalar_collection",​ + filter=filter,​ + output_fields=["age","price"]​ +)​ +​ +print(res)​ +​ +# Output​ +# data: ["{'age': 30, 'price': np.float32(149.5), 'pk': 2}", "{'age': 35, 'price': np.float32(199.99), 'pk': 3}"] ​ + +``` + +```java +import io.milvus.v2.service.vector.request.QueryReq;​ +import io.milvus.v2.service.vector.response.QueryResp;​ +​ +String filter = "30 <= age <= 40";​ +​ +QueryResp resp = client.query(QueryReq.builder()​ + .collectionName("my_scalar_collection")​ + .filter(filter)​ + .outputFields(Arrays.asList("age", "price"))​ + .build());​ +System.out.println(resp.getQueryResults());​ +​ +// Output​ +//​ +// [QueryResp.QueryResult(entity={price=149.5, pk=2, age=30}), QueryResp.QueryResult(entity={price=199.99, pk=3, age=35})]​ + +``` + +```javascript +client.query({​ + collection_name: 'my_scalar_collection',​ + filter: '30 <= age <= 40',​ + output_fields: ['age', 'price']​ +});​ + +``` + +```curl +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/query" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "my_scalar_collection",​ + "filter": "30 <= age <= 40",​ + "outputFields": ["age","price"]​ +}'​ +​ +## {"code":0,"cost":0,"data":[{"age":30,"pk":2,"price":149.5},{"age":35,"pk":3,"price":199.99}]}​ + +``` + +This query expression returns all matching entities and outputs their `age` and `price` fields. For more information on filter queries, refer to [​Metadata Filtering](boolean.md).​ + +### Vector search with number filtering​ + +In addition to basic number field filtering, you can combine vector similarity searches with number field filters. For example, the following code shows how to add a number field filter to a vector search:​ + +
          + Python + Java + Node.js + cURL +
          + +```python +filter = "25 <= age <= 35"​ +​ +res = client.search(​ + collection_name="my_scalar_collection",​ + data=[[0.3, -0.6, 0.1]],​ + limit=5,​ + search_params={"params": {"nprobe": 10}},​ + output_fields=["age","price"],​ + filter=filter​ +)​ +​ +print(res)​ +​ +# Output​ +# data: ["[{'id': 1, 'distance': -0.06000000238418579, 'entity': {'age': 25, 'price': 99.98999786376953}}, {'id': 2, 'distance': -0.12000000476837158, 'entity': {'age': 30, 'price': 149.5}}, {'id': 3, 'distance': -0.18000000715255737, 'entity': {'age': 35, 'price': 199.99000549316406}}]"]​ + +``` + +```java +import io.milvus.v2.service.vector.request.SearchReq;​ +import io.milvus.v2.service.vector.request.data.FloatVec;​ +import io.milvus.v2.service.vector.response.SearchResp;​ +​ +String filter = "25 <= age <= 35";​ +​ +SearchResp resp = client.search(SearchReq.builder()​ + .collectionName("my_scalar_collection")​ + .annsField("embedding")​ + .data(Collections.singletonList(new FloatVec(new float[]{0.3f, -0.6f, 0.1f})))​ + .topK(5)​ + .outputFields(Arrays.asList("age", "price"))​ + .filter(filter)​ + .build());​ +​ +System.out.println(resp.getSearchResults());​ +​ +// Output​ +//​ +// [[SearchResp.SearchResult(entity={price=199.99, age=35}, score=-0.19054288, id=3), SearchResp.SearchResult(entity={price=149.5, age=30}, score=-0.20163085, id=2), SearchResp.SearchResult(entity={price=99.99, age=25}, score=-0.2364331, id=1)]]​ + +``` + +```javascript +client.search({​ + collection_name: 'my_scalar_collection',​ + data: [0.3, -0.6, 0.1],​ + limit: 5,​ + output_fields: ['age', 'price'],​ + filter: '25 <= age <= 35'​ +});​ + +``` + +```curl +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/search" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "my_scalar_collection",​ + "data": [​ + [0.3, -0.6, 0.1]​ + ],​ + "annsField": "embedding",​ + "limit": 5,​ + "outputFields": ["age", "price"]​ +}'​ +​ +## {"code":0,"cost":0,"data":[{"age":35,"distance":-0.19054288,"id":3,"price":199.99},{"age":30,"distance":-0.20163085,"id":2,"price":149.5},{"age":25,"distance":-0.2364331,"id":1,"price":99.99}]}​ + +``` + +In this example, we first define a query vector and add a filter condition `25 <= age <= 35` during the search. This ensures that the search results are not only similar to the query vector but also meet the specified age range. For more information, refer to [​Metadata Filtering](boolean.md).​ + diff --git a/v2.4.x/site/en/userGuide/schema/primary-field.md b/v2.4.x/site/en/userGuide/schema/primary-field.md new file mode 100644 index 0000000000..eb90f7b1cb --- /dev/null +++ b/v2.4.x/site/en/userGuide/schema/primary-field.md @@ -0,0 +1,177 @@ +--- +id: primary-field.md +title: "Primary Field & AutoID" +summary: The primary field uniquely identifies an entity. This page introduces how to add the primary field of two different data types and how to enable Milvus to automatically allocate primary field values.​ +--- + +# Primary Field & AutoID​ + +The primary field uniquely identifies an entity. This page introduces how to add the primary field of two different data types and how to enable Milvus to automatically allocate primary field values.​ + +## Overview​ + +In a collection, the primary key of each entity should be globally unique. When adding the primary field, you need to explicitly set its data type to **VARCHAR** or **INT64**. Setting its data type to **INT64** indicates that the primary keys should be an integer similar to `12345`; Setting its data type to **VARCHAR** indicates that the primary keys should be a string similar to `my_entity_1234`.​ + +You can also enable **AutoID** to make Milvus automatically allocate primary keys for incoming entities. Once you have enabled **AutoID** in your collection, do not include primary keys when inserting entities.​ + +The primary field in a collection does not have a default value and cannot be null.​ + +## Use Int64 Primary Keys​ + +To use primary keys of the Int64 type, you need to set `datatype` to `DataType.INT64` and set `is_primary` to `true`. If you also need Milvus to allocate the primary keys for the incoming entities, also set `auto_id` to `true`.​ + +
          + Python + Java + Node.js + cURL +
          + +```python +from pymilvus import MilvusClient, DataType​ +​ +schema = MilvusClient.create_schema()​ +​ +schema.add_field(​ + field_name="my_id",​ + datatype=DataType.INT64,​ + # highlight-start​ + is_primary=True,​ + auto_id=True,​ + # highlight-end​ +)​ + +``` + +```java +import io.milvus.v2.common.DataType;​ +import io.milvus.v2.service.collection.request.AddFieldReq; ​ +import io.milvus.v2.service.collection.request.CreateCollectionReq;​ +​ +CreateCollectionReq.CollectionSchema schema = client.createSchema();​ +​ +schema.addField(AddFieldReq.builder()​ + .fieldName("my_id")​ + .dataType(DataType.Int64)​ + // highlight-start​ + .isPrimaryKey(true)​ + .autoID(true)​ + // highlight-end​ + .build());​ +);​ + +``` + +```javascript +import { DataType } from "@zilliz/milvus2-sdk-node";​ +​ +const schema = [​ + {​ + name: "pk",​ + description: "ID field",​ + data_type: DataType.VARCHAR,​ + is_primary_key: true,​ + max_length: 100,​ + },​ +];​ + +``` + +```go +// Go 缺失​ + +``` + +```curl +export primaryField='{​ + "fieldName": "my_id",​ + "dataType": "Int64",​ + "isPrimary": true​ +}'​ +​ +export schema="{​ + \"autoID\": true,​ + \"fields\": [​ + $primaryField​ + ]​ +}"​ + +``` + +## Use VarChar Primary Keys​ + +To use VarChar primary keys, in addition to changing the value of the `data_type` parameter to `DataType.VARCHAR`, you also need to set the `max_length` parameter for the field. ​ + +
          + Python + Java + Node.js + cURL +
          + +```python +schema.add_field(​ + field_name="my_id",​ + datatype=DataType.VARCHAR,​ + # highlight-start​ + is_primary=True,​ + auto_id=True,​ + max_length=512,​ + # highlight-end​ +)​ + +``` + +```java +import io.milvus.v2.common.DataType;​ +import io.milvus.v2.service.collection.request.AddFieldReq; ​ +​ +schema.addField(AddFieldReq.builder()​ + .fieldName("my_id")​ + .dataType(DataType.VarChar)​ + // highlight-start​ + .isPrimaryKey(true)​ + .autoID(true)​ + .maxLength(512)​ + // highlight-end​ + .build());​ + +``` + +```javascript +schema.push({​ + name: "my_id",​ + data_type: DataType.VarChar,​ + // highlight-start​ + is_primary_key: true,​ + autoID: true,​ + maxLength: 512​ + // highlight-end​ +});​ + +``` + +```go +// Go 缺失​ + +``` + +```curl +export primaryField='{​ + "fieldName": "my_id",​ + "dataType": "VarChar",​ + "isPrimary": true​ +}'​ +​ +export schema="{​ + \"autoID\": true,​ + \"fields\": [​ + $primaryField​ + ],​ + \"params\": {​ + \"max_length\": 512​ + }​ +}"​ + +``` + diff --git a/v2.4.x/site/en/userGuide/schema/schema-hands-on.md b/v2.4.x/site/en/userGuide/schema/schema-hands-on.md new file mode 100644 index 0000000000..81ae8dc7a6 --- /dev/null +++ b/v2.4.x/site/en/userGuide/schema/schema-hands-on.md @@ -0,0 +1,208 @@ +--- +id: schema-hands-on.md +title: Schema Design Hands-On​ +summary: Milvus supports defining the data model through a collection schema. A collection organizes unstructured data like text and images, along with their vector representations, including dense and sparse vectors in various precision used for semantic search. Additionally, Milvus supports storing and filtering non-vector data types called "Scalar". Scalar types include BOOL, INT8/16/32/64, FLOAT/DOUBLE, VARCHAR, JSON, and Array.​ +--- + +# Schema Design Hands-On​ + +Information Retrieval (IR) systems, also known as search, are essential for various AI applications such as Retrieval-augmented generation (RAG), image search, and product recommendation. The first step in developing an IR system is designing the data model, which involves analyzing business requirements, determining how to organize information, and indexing data to make it semantically searchable.​ + +Milvus supports defining the data model through a collection schema. A collection organizes unstructured data like text and images, along with their vector representations, including dense and sparse vectors in various precision used for semantic search. Additionally, Milvus supports storing and filtering non-vector data types called "Scalar". Scalar types include BOOL, INT8/16/32/64, FLOAT/DOUBLE, VARCHAR, JSON, and Array.​ + +![Example data schema designed for searching news article](../../../../assets/schema-hands-on.png) + +The data model design of a search system involves analyzing business needs and abstracting information into a schema-expressed data model. For instance, to search a piece of text, it must be "indexed" by converting the literal string into a vector through "embedding", enabling vector search. Beyond this basic requirement, it may be necessary to store other properties such as publication timestamp and author. This metadata allows for semantic searches to be refined through filtering, returning only texts published after a specific date or by a particular author. They may also need to be retrieved together with the main text, for rendering the search result in the application. To organize these text pieces, each should be assigned a unique identifier, expressed as an integer or string. These elements are essential for achieving sophisticated search logic.​ + +A well-designed schema is important as it abstracts the data model and decides if the business objectives can be achieved through search. Furthermore, since every row of data inserted into the collection needs to follow the schema, it greatly helps to maintain data consistency and long-term quality. From a technical perspective, a well-defined schema leads to well-organized column data storage and a cleaner index structure, which can boost search performance.​ + +# An Example: News Search​ + +Let's say we want to build search for a news website and we have a corpus of news with text, thumbnail images, and other metadata. First, we need to analyze how we want to utilize the data to support the business requirement of search. Imagine the requirement is to retrieve the news based the thumbnail image and the summary of the content, and taking the metadata such as author info and publishing time as criteria to filter the search result. These requirements can be further broken down into:​ + +- To search images via text, we can embed images into vectors via multimodal embedding model that can map text and image data into the same latent space.​ + +- The summary text of an article is embedded into vectors via text embedding model.​ + +- To filter based on the publish time, the dates are stored as a scalar field and an index is needed for the scalar field for efficient filtering. Other more complex data structures such a JSON can be stored in a scalar and a filtered search performed on their contents (indexing JSON is an upcoming feature).​ + +- To retrieve the image thumbnail bytes and render it on the search result page, the image url is also stored. Similarly, for the summary text and title. (Alternatively, we could store the raw text and image file data as scalar fields if required.)​ + +- To improve the search result on the summary text, we design a hybrid search approach. For one retrieval path, we use regular embedding model to generate dense vector from the text, such as OpenAI's `text-embedding-3-large` or the open-source `bge-large-en-v1.5`. These models are good at representing the overall semantic of the text. The other path is to use sparse embedding models such as BM25 or SPLADE to generate a sparse vector, resembling the full-text search which is good at grasping the details and individual concepts in the text. Milvus supports using both in the same data collection thanks to its multi-vector feature. The search on multiple vectors can be done in a single `hybrid_search()` operation.​ + +- Finally, we also need an ID field to identify each individual news page, formally referred to as an "entity" in Milvus terminology. This field is used as the primary key (or "pk" for short).​ + +

          Field Name​

          + +

          article_id (Primary Key)​

          + +

          title​

          + +

          author_info​

          + +

          publish_ts​

          + +

          image_url​

          + +

          image_vector​

          + +

          summary​

          + +

          summary_dense_vector​

          + +

          summary_sparse_vector​

          + +

          Type​

          + +

          INT64​

          + +

          VARCHAR​

          + +

          JSON​

          + +

          INT32​

          + +

          VARCHAR​

          + +

          FLOAT_VECTOR​

          + +

          VARCHAR​

          + +

          FLOAT_VECTOR​

          + +

          SPARSE_FLOAT_VECTOR​

          + +

          Need Index​

          + +

          N​

          + +

          N​

          + +

          N (Support coming soon)​

          + +

          Y​

          + +

          N​

          + +

          Y​

          + +

          N​

          + +

          Y​

          + +

          Y​

          + +
          + +# How to Implement the Example Schema​ + +## Create Schema​ + +First, we create a Milvus client instance, which can be used to connect to the Milvus server and manage collections and data. ​ + +To set up a schema, we use [`create_schema()`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Collections/create_schema.md) to create a schema object and [`add_field()`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/CollectionSchema/add_field.md) to add fields to the schema.​ + +```python +from pymilvus import MilvusClient, DataType​ +​ +collection_name = "my_collection"​ +​ +# client = MilvusClient(uri="http://localhost:19530")​ +client = MilvusClient(uri="./milvus_demo.db")​ +​ +schema = MilvusClient.create_schema(​ + auto_id=False,​ +)​ +​ +schema.add_field(field_name="article_id", datatype=DataType.INT64, is_primary=True, description="article id")​ +schema.add_field(field_name="title", datatype=DataType.VARCHAR, max_length=200, description="article title")​ +schema.add_field(field_name="author_info", datatype=DataType.JSON, description="author information")​ +schema.add_field(field_name="publish_ts", datatype=DataType.INT32, description="publish timestamp")​ +schema.add_field(field_name="image_url", datatype=DataType.VARCHAR, max_length=500, description="image URL")​ +schema.add_field(field_name="image_vector", datatype=DataType.FLOAT_VECTOR, dim=768, description="image vector")​ +schema.add_field(field_name="summary", datatype=DataType.VARCHAR, max_length=1000, description="article summary")​ +schema.add_field(field_name="summary_dense_vector", datatype=DataType.FLOAT_VECTOR, dim=768, description="summary dense vector")​ +schema.add_field(field_name="summary_sparse_vector", datatype=DataType.SPARSE_FLOAT_VECTOR, description="summary sparse vector")​ + +``` + +You might notice the argument `uri` in `MilvusClient`, which is used to connect to the Milvus server. You can set the arguments as follows:​ + +- If you only need a local vector database for small scale data or prototypeing, setting the uri as a local file, e.g.``./milvus.db``, is the most convenient method, as it automatically utilizes [Milvus Lite](https://milvus.io/docs/milvus_lite.md) to store all data in this file.​ + +- If you have large scale of data, say more than a million vectors, you can set up a more performant Milvus server on [Docker or Kubernetes](https://milvus.io/docs/quickstart.md). In this setup, please use the server address and port as your uri, e.g.``http://localhost:19530``. If you enable the authentication feature on Milvus, use ":" as the token, otherwise don't set the token.​ + +- If you use [Zilliz Cloud](https://zilliz.com/cloud), the fully managed cloud service for Milvus, adjust the ``uri`` and ``token``, which correspond to the [Public Endpoint and API key](https://docs.zilliz.com/docs/on-zilliz-cloud-console#free-cluster-details) in Zilliz Cloud.​ + +As for the `auto_id` in `MilvusClient.create_schema`, AutoID is an attribute of the primary field that determines whether to enable auto increment for the primary field. Since we set the field`article_id` as the primary key and want to add article id manually, we set `auto_id` False to disable this feature.​ + +After adding all the fields to the schema object, our schema object agrees with the entries in the table above.​ + +## Define Index​ + +After defining the schema with various fields, including metadata and vector fields for image and summary data, the next step involves preparing the index parameters. Indexing is crucial for optimizing the search and retrieval of vectors, ensuring efficient query performance. In the following section, we will define the index parameters for the specified vector and scalar fields in the collection.​ + +```python +index_params = client.prepare_index_params()​ +​ +index_params.add_index(​ + field_name="image_vector",​ + index_type="AUTOINDEX",​ + metric_type="IP",​ +)​ +index_params.add_index(​ + field_name="summary_dense_vector",​ + index_type="AUTOINDEX",​ + metric_type="IP",​ +)​ +index_params.add_index(​ + field_name="summary_sparse_vector",​ + index_type="SPARSE_INVERTED_INDEX",​ + metric_type="IP",​ +)​ +index_params.add_index(​ + field_name="publish_ts",​ + index_type="INVERTED",​ +)​ + +``` + +Once the index parameters are set up and applied, Milvus is optimized for handling complex queries on vector and scalar data. This indexing enhances the performance and accuracy of similarity searches within the collection, allowing for efficient retrieval of articles based on image vectors and summary vectors. By leveraging the [`AUTOINDEX`](https://milvus.io/docs/glossary.md#Auto-Index) for dense vectors, the [`SPARSE_INVERTED_INDEX`](https://milvus.io/docs/sparse_vector.md#Index-the-collection) for sparse vectors and the [`INVERTED_INDEX`](https://milvus.io/docs/scalar_index.md#Inverted-indexing) for scalars, Milvus can quickly identify and return the most relevant results, significantly improving the overall user experience and effectiveness of the data retrieval process.​ + +There are many types of indices and metrics. For more information about them, you can refer to [Milvus index type](https://milvus.io/docs/overview.md#Index-types) and [Milvus metric type](https://milvus.io/docs/glossary.md#Metric-type).​ + +## Create Collection​ + +With the schema and indexes defined, we create a "collection" with these parameters. Collection to Milvus is like a table to a relational DB.​ + +```python +client.create_collection(​ + collection_name=collection_name,​ + schema=schema,​ + index_params=index_params,​ +)​ + +``` + +We can verify that the collection has been successfully created by describing the collection.​ + +```python +collection_desc = client.describe_collection(​ + collection_name=collection_name​ +)​ +print(collection_desc)​ + +``` + +# Other Considerations​ + +## Loading Index​ + +When creating a collection in Milvus, you can choose to load the index immediately or defer it until after bulk ingesting some data. Typically, you don't need to make an explicit choice about this, as the above examples show that the index is automatically built for any ingested data right after collection creation. This allows for immediate searchability of the ingested data. However, if you have a large bulk insert after collection creation and don't need to search for any data until a certain point, you can defer the index building by omitting index_params in the collection creation and build the index by calling load explicitly after ingesting all the data. This method is more efficient for building the index on a large collection, but no searches can be done until calling load().​ + +## How to Define Data Model For Multi-tenancy​ + +The concept of multiple tenants is commonly used in scenarios where a single software application or service needs to serve multiple independent users or organizations, each with their own isolated environment. This is frequently seen in cloud computing, SaaS (Software as a Service) applications, and database systems. For example, a cloud storage service may utilize multi-tenancy to allow different companies to store and manage their data separately while sharing the same underlying infrastructure. This approach maximizes resource utilization and efficiency while ensuring data security and privacy for each tenant.​ + +The easiest way to differentiate tenants is by isolating their data and resources from each other. Each tenant either has exclusive access to specific resources or shares resources with others to manage Milvus entities such as databases, collections, and partitions. There are specific methods aligned with these entities to implement Milvus multi-tenancy. You can refer to the [Milvus multi-tenancy page](https://milvus.io/docs/multi_tenancy.md#Multi-tenancy-strategies) for more information.​ + diff --git a/v2.4.x/site/en/userGuide/schema/schema.md b/v2.4.x/site/en/userGuide/schema/schema.md new file mode 100644 index 0000000000..9274291d2c --- /dev/null +++ b/v2.4.x/site/en/userGuide/schema/schema.md @@ -0,0 +1,511 @@ +--- +id: schema.md +title: Schema Explained +summary: A schema defines the data structure of a collection. Before creating a collection, you need to work out a design of its schema. This page helps you understand the collection schema and design an example schema on your own.​ +--- + +# Schema Explained​ + +A schema defines the data structure of a collection. Before creating a collection, you need to work out a design of its schema. This page helps you understand the collection schema and design an example schema on your own.​ + +## Overview​ + +On Zilliz Cloud, a collection schema assembles a table in a relational database, which defines how Zilliz Cloud organizes data in the collection. ​ + +A well-designed schema is essential as it abstracts the data model and decides if you can achieve the business objectives through a search. Furthermore, since every row of data inserted into the collection must follow the schema, it helps maintain data consistency and long-term quality. From a technical perspective, a well-defined schema leads to well-organized column data storage and a cleaner index structure, boosting search performance.​ + +A collection schema has a primary key, a maximum of four vector fields, and several scalar fields. The following diagram illustrates how to map an article to a list of schema fields.​ + +![Schema design](../../../../assets/schema-explained.PNG) + +The data model design of a search system involves analyzing business needs and abstracting information into a schema-expressed data model. For instance, searching a piece of text must be "indexed" by converting the literal string into a vector through "embedding" and enabling vector search. Beyond this essential requirement, storing other properties such as publication timestamp and author may be necessary. This metadata allows for semantic searches to be refined through filtering, returning only texts published after a specific date or by a particular author. You can also retrieve these scalars with the main text to render the search result in the application. Each should be assigned a unique identifier to organize these text pieces, expressed as an integer or string. These elements are essential for achieving sophisticated search logic.​ + +Refer to [Schema Design Hands-On](schema-hands-on.md) to figure out how to make a well-designed schema.​ + +## Create Schema​ + +The following code snippet demonstrates how to create a schema.​ + +
          + Python + Java + Node.js + cURL +
          + +```python +from pymilvus import MilvusClient, DataType​ +​ +schema = MilvusClient.create_schema()​ + +``` + +```java +import io.milvus.v2.service.collection.request.CreateCollectionReq;​ +​ +CreateCollectionReq.CollectionSchema schema = client.createSchema();​ + +``` + +```javascript +import { MilvusClient, DataType } from "@zilliz/milvus2-sdk-node";​ +​ +const schema = []​ + +``` + +```curl +export schema='{​ + "fields": []​ +}'​ + +``` + +## Add Primary Field​ + +The primary field in a collection uniquely identifies an entity. It only accepts **Int64** or **VarChar** values. The following code snippets demonstrate how to add the primary field.​ + +
          + Python + Java + Node.js + cURL +
          + +```python +schema.add_field(​ + field_name="my_id",​ + datatype=DataType.INT64,​ + # highlight-start​ + is_primary=True,​ + auto_id=False,​ + # highlight-end​ +)​ + +``` + +```java +import io.milvus.v2.common.DataType;​ +import io.milvus.v2.service.collection.request.AddFieldReq; ​ +​ +schema.addField(AddFieldReq.builder()​ + .fieldName("my_id")​ + .dataType(DataType.Int64)​ + // highlight-start​ + .isPrimaryKey(true)​ + .autoID(false)​ + // highlight-end​ + .build());​ + +``` + +```javascript +schema.push({​ + name: "my_id",​ + data_type: DataType.Int64,​ + // highlight-start​ + is_primary_key: true,​ + autoID: false​ + // highlight-end​ +});​ + +``` + +```curl +export primaryField='{​ + "fieldName": "my_id",​ + "dataType": "Int64",​ + "isPrimary": true​ +}'​ +​ +export schema='{​ + \"autoID\": false,​ + \"fields\": [​ + $primaryField​ + ]​ +}'​ + +``` + +When adding a field, you can explicitly clarify the field as the primary field by setting its `is_primary` property to `True`. A primary field accepts **Int64** values by default. In this case, the primary field value should be integers similar to `12345`. If you choose to use **VarChar** values in the primary field, the value should be strings similar to `my_entity_1234`.​ + +You can also set the `autoId` properties to `True` to make Zilliz Cloud automatically allocate primary field values upon data insertions.​ + +For details, refer to [​Primary Field & AutoID](primary-field.md).​ + +## Add Vector Fields​ + +Vector fields accept various sparse and dense vector embeddings. On Zilliz Cloud, you can add four vector fields to a collection. The following code snippets demonstrate how to add a vector field.​ + +
          + Python + Java + Node.js + cURL +
          + +```python +schema.add_field(​ + field_name="my_vector",​ + datatype=DataType.FLOAT_VECTOR,​ + # highlight-next-line​ + dim=5​ +)​ + +``` + +```java +schema.addField(AddFieldReq.builder()​ + .fieldName("my_vector")​ + .dataType(DataType.FloatVector)​ + // highlight-next-line​ + .dimension(5)​ + .build());​ + +``` + +```javascript +schema.push({​ + name: "my_vector",​ + data_type: DataType.FloatVector,​ + // highlight-next-line​ + dim: 5​ +});​ + +``` + +```curl +export vectorField='{​ + "fieldName": "my_vector",​ + "dataType": "FloatVector",​ + "elementTypeParams": {​ + "dim": 5​ + }​ +}'​ +​ +export schema="{​ + \"autoID\": false,​ + \"fields\": [​ + $primaryField,​ + $vectorField​ + ]​ +}"​ + +``` + +The `dim` paramter in the above code snippets indicates the dimensionality of the vector embeddings to be held in the vector field. The `FLOAT_VECTOR` value indicates that the vector field holds a list of 32-bit floating numbers, which are usually used to represent antilogarithms.In addition to that, Zilliz Cloud also supports the following types of vector embeddings:​ + +- `FLOAT16_VECTOR`​ + + A vector field of this type holds a list of 16-bit half-precision floating numbers and usually applies to memory- or bandwidth-restricted deep learning or GPU-based computing scenarios.​ + +- `BFLOAT16_VECTOR`​ + + A vector field of this type holds a list of 16-bit floating-point numbers that have reduced precision but the same exponent range as Float32. This type of data is commonly used in deep learning scenarios, as it reduces memory usage without significantly impacting accuracy.​ + +- `BINARY_VECTOR`​ + + A vector field of this type holds a list of 0s and 1s. They serve as compact features for representing data in image processing and information retrieval scenarios.​ + +- `SPARSE_FLOAT_VECTOR`​ + + A vector field of this type holds a list of non-zero numbers and their sequence numbers to represent sparse vector embeddings.​ + +## Add Scalar Fields​ + +In common cases, you can use scalar fields to store the metadata of the vector embeddings stored in Milvus, and conduct ANN searches with metadata filtering to improve the correctness of the search results. Zilliz Cloud supports multiple scalar field types, including **VarChar**, **Boolean**, **Int**, Float, **Double**, **Array**, and JSON.​ + +### Add String Fields​ + +In Milvus, you can use VarChar fields to store strings. For more on the VarChar field, refer to [​String Field](string.md).​ + +
          + Python + Java + Node.js + cURL +
          + +```python +schema.add_field(​ + field_name="my_varchar",​ + datatype=DataType.VARCHAR,​ + # highlight-next-line​ + max_length=512​ +)​ + +``` + +```java +schema.addField(AddFieldReq.builder()​ + .fieldName("my_varchar")​ + .dataType(DataType.VarChar)​ + // highlight-next-line​ + .maxLength(512)​ + .build());​ + +``` + +```javascript +schema.push({​ + name: "my_varchar",​ + data_type: DataType.VarChar,​ + // highlight-next-line​ + max_length: 512​ +});​ + +``` + +```curl +export varCharField='{​ + "fieldName": "my_varchar",​ + "dataType": "VarChar",​ + "elementTypeParams": {​ + "max_length": 256​ + }​ +}'​ +​ +export schema="{​ + \"autoID\": false,​ + \"fields\": [​ + $primaryField,​ + $vectorField,​ + $varCharField​ + ]​ +}"​ + +``` + +### Add Number Fields​ + +The types of numbers that Milvus supports are `Int8`, `Int16`, `Int32`, `Int64`, `Float`, and `Double`. For more on the number fields, refer to [​Number Field](number.md).​ + +
          + Python + Java + Node.js + cURL +
          + +```python +schema.add_field(​ + field_name="my_int64",​ + datatype=DataType.INT64,​ +)​ + +``` + +```java +schema.addField(AddFieldReq.builder()​ + .fieldName("my_int64")​ + .dataType(DataType.Int64)​ + .build());​ + +``` + +```javascript +schema.push({​ + name: "my_int64",​ + data_type: DataType.Int64,​ +});​ + +``` + +```curl +export int64Field='{​ + "fieldName": "my_int64",​ + "dataType": "Int64"​ +}'​ +​ +export schema="{​ + \"autoID\": false,​ + \"fields\": [​ + $primaryField,​ + $vectorField,​ + $varCharField,​ + $int64Field​ + ]​ +}"​ + +``` + +### Add Boolean Fields​ + +Milvus supports boolean fields. The following code snippets demonstrate how to add a boolean field.​ + +
          + Python + Java + Node.js + cURL +
          + +```python +schema.add_field(​ + field_name="my_bool",​ + datatype=DataType.BOOL,​ +)​ + +``` + +```java +schema.addField(AddFieldReq.builder()​ + .fieldName("my_bool")​ + .dataType(DataType.Bool)​ + .build());​ + +``` + +```javascript +schema.push({​ + name: "my_bool",​ + data_type: DataType.Boolean,​ +});​ + +``` + +```curl +export boolField='{​ + "fieldName": "my_bool",​ + "dataType": "Boolean"​ +}'​ +​ +export schema="{​ + \"autoID\": false,​ + \"fields\": [​ + $primaryField,​ + $vectorField,​ + $varCharField,​ + $int64Field,​ + $boolField​ + ]​ +}"​ + +``` + +### Add JSON fields​ + +A JSON field usually stores half-structured JSON data. For more on the JSON fields, refer to [​JSON Field](use-json-field.md).​ + +
          + Python + Java + Node.js + cURL +
          + +```python +schema.add_field(​ + field_name="my_json",​ + datatype=DataType.JSON,​ +)​ + +``` + +```java +schema.addField(AddFieldReq.builder()​ + .fieldName("my_json")​ + .dataType(DataType.JSON)​ + .build());​ + +``` + +```javascript +schema.push({​ + name: "my_json",​ + data_type: DataType.JSON,​ +});​ + +``` + +```curl +export jsonField='{​ + "fieldName": "my_json",​ + "dataType": "JSON"​ +}'​ +​ +export schema="{​ + \"autoID\": false,​ + \"fields\": [​ + $primaryField,​ + $vectorField,​ + $varCharField,​ + $int64Field,​ + $boolField,​ + $jsonField​ + ]​ +}"​ + +``` + +### Add Array Fields​ + +An array field stores a list of elements. The data types of all elements in an array field should be the same. For more on the array fields, refer to [​Array Field](array_data_field.md).​ + +
          + Python + Java + Node.js + cURL +
          + +```python +schema.add_field(​ + field_name="my_array",​ + datatype=DataType.ARRAY,​ + element_type=DataType.VARCHAR,​ + max_capacity=5,​ + max_length=512,​ +)​ + +``` + +```java +schema.addField(AddFieldReq.builder()​ + .fieldName("my_array")​ + .dataType(DataType.Array)​ + .elementType(DataType.VarChar)​ + .maxCapacity(5)​ + .maxLength(512)​ + .build());​ + +``` + +```javascript +schema.push({​ + name: "my_array",​ + data_type: DataType.Array,​ + element_type: DataType.VarChar,​ + max_capacity: 5,​ + max_length: 512​ +});​ + +``` + +```curl +export arrayField='{​ + "fieldName": "my_array",​ + "dataType": "Array",​ + "elementDataType": "VarChar",​ + "elementTypeParams": {​ + "max_length": 512​ + }​ +}'​ +​ +export schema="{​ + \"autoID\": false,​ + \"fields\": [​ + $primaryField,​ + $vectorField,​ + $varCharField,​ + $int64Field,​ + $boolField,​ + $jsonField,​ + $arrayField​ + ]​ +}"​ + +``` + +​ + diff --git a/v2.4.x/site/en/userGuide/schema/sparse_vector.md b/v2.4.x/site/en/userGuide/schema/sparse_vector.md new file mode 100644 index 0000000000..fee399fa24 --- /dev/null +++ b/v2.4.x/site/en/userGuide/schema/sparse_vector.md @@ -0,0 +1,561 @@ +--- +id: sparse_vector.md +title: Sparse Vector +summary: Sparse vectors are an important method of data representation in information retrieval and natural language processing. While dense vectors are popular for their excellent semantic understanding capabilities, sparse vectors often provide more accurate results when it comes to applications that require precise matching of keywords or phrases.​ +--- + +# Sparse Vector​ + +Sparse vectors are an important method of data representation in information retrieval and natural language processing. While dense vectors are popular for their excellent semantic understanding capabilities, sparse vectors often provide more accurate results when it comes to applications that require precise matching of keywords or phrases.​ + +## Overview​ + +A sparse vector is a special representation of high-dimensional vectors where most elements are zero, and only a few dimensions have non-zero values. This characteristic makes sparse vectors particularly effective in handling large-scale, high-dimensional, but sparse data. Common applications include:​ + +- **Text Analysis:** Representing documents as bag-of-words vectors, where each dimension corresponds to a word, and only words that appear in the document have non-zero values.​ + +- **Recommendation Systems:** User-item interaction matrices, where each dimension represents a user's rating for a particular item, with most users interacting with only a few items.​ + +- **Image Processing:** Local feature representation, focusing only on key points in the image, resulting in high-dimensional sparse vectors.​ + +As shown in the diagram below, dense vectors are typically represented as continuous arrays where each position has a value (e.g., `[0.3, 0.8, 0.2, 0.3, 0.1]`). In contrast, sparse vectors store only non-zero elements and their indices, often represented as key-value pairs (e.g., `[{2: 0.2}, ..., {9997: 0.5}, {9999: 0.7}]`). This representation significantly reduces storage space and increases computational efficiency, especially when dealing with extremely high-dimensional data (e.g., 10,000 dimensions).​ + +![Spare vector representation](../../../../assets/sparse-vector.png) + +Sparse vectors can be generated using various methods, such as [TF-IDF](https://en.wikipedia.org/wiki/Tf%E2%80%93idf) (Term Frequency-Inverse Document Frequency) and [BM25](https://en.wikipedia.org/wiki/Okapi_BM25) in text processing. Additionally, Milvus offers convenient methods to help generate and process sparse vectors. For details, refer to Embeddings.​ + +For text data, Milvus also provides full-text search capabilities, allowing you to perform vector searches directly on raw text data without using external embedding models to generate sparse vectors. For more information, refer to [​Full Text Search](full-text-search.md).​ + +After vectorization, the data can be stored in Milvus for management and vector retrieval. The diagram below illustrates the basic process.​ + +![Use sparse vector in Milvus](../../../../assets/use-sparse-vector.png) + +
          + +In addition to sparse vectors, Milvus also supports dense vectors and binary vectors. Dense vectors are ideal for capturing deep semantic relationships, while binary vectors excel in scenarios like quick similarity comparisons and content deduplication. For more information, refer to [​Dense Vector](dense-vector.md) and [​Binary Vector](binary-vector.md).​ + +
          + +## Use sparse vectors in Milvus​ + +Milvus supports representing sparse vectors in any of the following formats:​ + +- Sparse Matrix (using the `scipy.sparse` class) + +
          + Python + Java + Node.js + cURL +
          + + ```python + from scipy.sparse import csr_matrix​ + ​ + # Create a sparse matrix​ + row = [0, 0, 1, 2, 2, 2]​ + col = [0, 2, 2, 0, 1, 2]​ + data = [1, 2, 3, 4, 5, 6]​ + sparse_matrix = csr_matrix((data, (row, col)), shape=(3, 3))​ + ​ + # Represent sparse vector using the sparse matrix​ + sparse_vector = sparse_matrix.getrow(0)​ + + ``` + +- List of Dictionaries (formatted as `{dimension_index: value, ...}`) + +
          + Python + Java + Node.js + cURL +
          + + ```python + # Represent sparse vector using a dictionary​ + sparse_vector = [{1: 0.5, 100: 0.3, 500: 0.8, 1024: 0.2, 5000: 0.6}]​ + + ``` + + ```java + SortedMap sparseVector = new TreeMap<>();​ + sparseVector.put(1L, 0.5f);​ + sparseVector.put(100L, 0.3f);​ + sparseVector.put(500L, 0.8f);​ + sparseVector.put(1024L, 0.2f);​ + sparseVector.put(5000L, 0.6f);​ + + ``` + +- List of Tuple Iterators (formatted as `[(dimension_index, value)]`) + +
          + Python + Java + Node.js + cURL +
          + + ```python + # Represent sparse vector using a list of tuples​ + sparse_vector = [[(1, 0.5), (100, 0.3), (500, 0.8), (1024, 0.2), (5000, 0.6)]]​ + + ``` + +### Add vector field​ + +To use sparse vectors in Milvus, define a field for storing sparse vectors when creating a collection. This process includes:​ + +1. Setting `datatype` to the supported sparse vector data type, `SPARSE_FLOAT_VECTOR`.​ + +2. No need to specify the dimension.​ + +
          + Python + Java + Node.js + cURL +
          + +```python +from pymilvus import MilvusClient, DataType​ +​ +client = MilvusClient(uri="http://localhost:19530")​ +​ +client.drop_collection(collection_name="my_sparse_collection")​ +​ +schema = client.create_schema(​ + auto_id=True,​ + enable_dynamic_fields=True,​ +)​ +​ +schema.add_field(field_name="pk", datatype=DataType.VARCHAR, is_primary=True, max_length=100)​ +schema.add_field(field_name="sparse_vector", datatype=DataType.SPARSE_FLOAT_VECTOR)​ + +``` + +```java +import io.milvus.v2.client.ConnectConfig;​ +import io.milvus.v2.client.MilvusClientV2;​ +​ +import io.milvus.v2.common.DataType;​ +import io.milvus.v2.service.collection.request.AddFieldReq;​ +import io.milvus.v2.service.collection.request.CreateCollectionReq;​ +​ +MilvusClientV2 client = new MilvusClientV2(ConnectConfig.builder()​ + .uri("http://localhost:19530")​ + .build());​ + ​ +CreateCollectionReq.CollectionSchema schema = client.createSchema();​ +schema.setEnableDynamicField(true);​ +schema.addField(AddFieldReq.builder()​ + .fieldName("pk")​ + .dataType(DataType.VarChar)​ + .isPrimaryKey(true)​ + .autoID(true)​ + .maxLength(100)​ + .build());​ +​ +schema.addField(AddFieldReq.builder()​ + .fieldName("sparse_vector")​ + .dataType(DataType.SparseFloatVector)​ + .build());​ + +``` + +```javascript +import { DataType } from "@zilliz/milvus2-sdk-node";​ +​ +const schema = [​ + {​ + name: "metadata",​ + data_type: DataType.JSON,​ + },​ + {​ + name: "pk",​ + data_type: DataType.Int64,​ + is_primary_key: true,​ + },​ + {​ + name: "sparse_vector",​ + data_type: DataType.SparseFloatVector,​ + }​ +];​ +​ + +``` + +```curl +export primaryField='{​ + "fieldName": "pk",​ + "dataType": "VarChar",​ + "isPrimary": true,​ + "elementTypeParams": {​ + "max_length": 100​ + }​ +}'​ +​ +export vectorField='{​ + "fieldName": "sparse_vector",​ + "dataType": "SparseFloatVector"​ +}'​ +​ +export schema="{​ + \"autoID\": true,​ + \"fields\": [​ + $primaryField,​ + $vectorField​ + ]​ +}"​ + +``` + +In this example, a vector field named `sparse_vector` is added for storing sparse vectors. The data type of this field is `SPARSE_FLOAT_VECTOR`.​ + +### Set index params for vector field​ + +The process of creating an index for sparse vectors is similar to that for [dense vectors](dense-vector.md), but with differences in the specified index type (`index_type`), distance metric (`metric_type`), and index parameters (`params`).​ + +
          + Python + Java + Node.js + cURL +
          + +```python +index_params = client.prepare_index_params()​ +​ +index_params.add_index(​ + field_name="sparse_vector",​ + index_name="sparse_inverted_index",​ + index_type="SPARSE_INVERTED_INDEX",​ + metric_type="IP",​ + params={"drop_ratio_build": 0.2},​ +)​ + +``` + +```java +import io.milvus.v2.common.IndexParam;​ +import java.util.*;​ +​ +List indexes = new ArrayList<>();​ +Map extraParams = new HashMap<>();​ +extraParams.put("drop_ratio_build", 0.2);​ +indexes.add(IndexParam.builder()​ + .fieldName("sparse_vector")​ + .indexName("sparse_inverted_index")​ + .indexType(IndexParam.IndexType.SPARSE_INVERTED_INDEX)​ + .metricType(IndexParam.MetricType.IP)​ + .extraParams(extraParams)​ + .build());​ + +``` + +```javascript +const indexParams = await client.createIndex({​ + index_name: 'sparse_inverted_index',​ + field_name: 'sparse_vector',​ + metric_type: MetricType.IP,​ + index_type: IndexType.SPARSE_WAND,​ + params: {​ + drop_ratio_build: 0.2,​ + },​ +});​ + +``` + +```curl +export indexParams='[​ + {​ + "fieldName": "sparse_vector",​ + "metricType": "IP",​ + "indexName": "sparse_inverted_index",​ + "indexType": "SPARSE_INVERTED_INDEX",​ + "params":{"drop_ratio_build": 0.2}​ + }​ + ]'​ + +``` + +In the example above:​ + +- An index of type `SPARSE_INVERTED_INDEX` is created for the sparse vector. For sparse vectors, you can specify `SPARSE_INVERTED_INDEX` or `SPARSE_WAND`. For details, refer to [​Sparse Vector Indexes](https://milvus.io/docs/index.md?tab=sparse).​ + +- For sparse vectors, `metric_type` only supports `IP` (Inner Product), used to measure the similarity between two sparse vectors. For more information on similarity, refer to [​Metric Types](metric.md).​ + +- `drop_ratio_build` is an optional index parameter specifically for sparse vectors. It controls the proportion of small vector values excluded during index building. For example, with `{"drop_ratio_build": 0.2}`, the smallest 20% of vector values will be excluded during index creation, reducing computational effort during searches.​ + +### Create collection​ + +Once the sparse vector and index settings are complete, you can create a collection that contains sparse vectors. The example below uses the `create_collection` method to create a collection named `my_sparse_collection`.​ + +
          + Python + Java + Node.js + cURL +
          + +```python +client.create_collection(​ + collection_name="my_sparse_collection",​ + schema=schema,​ + index_params=index_params​ +)​ + +``` + +```java +import io.milvus.v2.client.ConnectConfig;​ +import io.milvus.v2.client.MilvusClientV2;​ +​ +MilvusClientV2 client = new MilvusClientV2(ConnectConfig.builder()​ + .uri("http://localhost:19530")​ + .build());​ + ​ +CreateCollectionReq requestCreate = CreateCollectionReq.builder()​ + .collectionName("my_sparse_collection")​ + .collectionSchema(schema)​ + .indexParams(indexes)​ + .build();​ +client.createCollection(requestCreate);​ + +``` + +```javascript +import { MilvusClient } from "@zilliz/milvus2-sdk-node";​ +​ +const client = new MilvusClient({​ + address: 'http://localhost:19530'​ +});​ +​ +await client.createCollection({​ + collection_name: 'my_sparse_collection',​ + schema: schema,​ + index_params: indexParams​ +});​ + +``` + +```curl +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/collections/create" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d "{​ + \"collectionName\": \"my_sparse_collection\",​ + \"schema\": $schema,​ + \"indexParams\": $indexParams​ +}"​ + +``` + +### Insert data​ + +After creating the collection, insert data containing sparse vectors.​ + +
          + Python + Java + Node.js + cURL +
          + +```python +sparse_vectors = [​ + {"sparse_vector": {1: 0.5, 100: 0.3, 500: 0.8}},​ + {"sparse_vector": {10: 0.1, 200: 0.7, 1000: 0.9}},​ +]​ +​ +client.insert(​ + collection_name="my_sparse_collection",​ + data=sparse_vectors​ +)​ + +``` + +```java +import com.google.gson.Gson;​ +import com.google.gson.JsonObject;​ +import io.milvus.v2.service.vector.request.InsertReq;​ +import io.milvus.v2.service.vector.response.InsertResp;​ +​ +List rows = new ArrayList<>();​ +Gson gson = new Gson();​ +{​ + JsonObject row = new JsonObject();​ + SortedMap sparse = new TreeMap<>();​ + sparse.put(1L, 0.5f);​ + sparse.put(100L, 0.3f);​ + sparse.put(500L, 0.8f);​ + row.add("sparse_vector", gson.toJsonTree(sparse));​ + rows.add(row);​ +}​ +{​ + JsonObject row = new JsonObject();​ + SortedMap sparse = new TreeMap<>();​ + sparse.put(10L, 0.1f);​ + sparse.put(200L, 0.7f);​ + sparse.put(1000L, 0.9f);​ + row.add("sparse_vector", gson.toJsonTree(sparse));​ + rows.add(row);​ +}​ +​ +InsertResp insertR = client.insert(InsertReq.builder()​ + .collectionName("my_sparse_collection")​ + .data(rows)​ + .build());​ + +``` + +```javascript +const data = [​ + { sparse_vector: { "1": 0.5, "100": 0.3, "500": 0.8 } },​ + { sparse_vector: { "10": 0.1, "200": 0.7, "1000": 0.9 } },​ +];​ +client.insert({​ + collection_name: "my_sparse_collection",​ + data: data,​ +});​ +​ + +``` + +```curl +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/insert" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "data": [​ + {"sparse_vector": {"1": 0.5, "100": 0.3, "500": 0.8}},​ + {"sparse_vector": {"10": 0.1, "200": 0.7, "1000": 0.9}} ​ + ],​ + "collectionName": "my_sparse_collection"​ +}'​ +​ +## {"code":0,"cost":0,"data":{"insertCount":2,"insertIds":["453577185629572534","453577185629572535"]}}​ + +``` + +### Perform similarity search​ + +To perform similarity search using sparse vectors, prepare the query vector and search parameters.​ + +
          + Python + Java + Node.js + cURL +
          + +```python +# Prepare search parameters​ +search_params = {​ + "params": {"drop_ratio_search": 0.2}, # Additional optional search parameters​ +}​ +​ +# Prepare the query vector​ +query_vector = [{1: 0.2, 50: 0.4, 1000: 0.7}]​ + +``` + +In this example, `drop_ratio_search` is an optional parameter specifically for sparse vectors, allowing fine-tuning of small values in the query vector during the search. For example, with `{"drop_ratio_search": 0.2}`, the smallest 20% of values in the query vector will be ignored during the search.​ + +Then, execute the similarity search using the `search` method:​ + +
          + Python + Java + Node.js + cURL +
          + +```python +res = client.search(​ + collection_name="my_sparse_collection",​ + data=query_vector,​ + limit=3,​ + output_fields=["pk"],​ + search_params=search_params,​ +)​ +​ +print(res)​ +​ +# Output​ +# data: ["[{'id': '453718927992172266', 'distance': 0.6299999952316284, 'entity': {'pk': '453718927992172266'}}, {'id': '453718927992172265', 'distance': 0.10000000149011612, 'entity': {'pk': '453718927992172265'}}]"]​ + +``` + +```java +import io.milvus.v2.service.vector.request.SearchReq;​ +import io.milvus.v2.service.vector.request.data.SparseFloatVec;​ +import io.milvus.v2.service.vector.response.SearchResp;​ +​ +Map searchParams = new HashMap<>();​ +searchParams.put("drop_ratio_search", 0.2);​ +​ +SortedMap sparse = new TreeMap<>();​ +sparse.put(10L, 0.1f);​ +sparse.put(200L, 0.7f);​ +sparse.put(1000L, 0.9f);​ +​ +SparseFloatVec queryVector = new SparseFloatVec(sparse);​ +​ +SearchResp searchR = client.search(SearchReq.builder()​ + .collectionName("my_sparse_collection")​ + .data(Collections.singletonList(queryVector))​ + .annsField("sparse_vector")​ + .searchParams(searchParams)​ + .topK(3)​ + .outputFields(Collections.singletonList("pk"))​ + .build());​ + ​ +System.out.println(searchR.getSearchResults());​ +​ +// Output​ +//​ +// [[SearchResp.SearchResult(entity={pk=453444327741536759}, score=1.31, id=453444327741536759), SearchResp.SearchResult(entity={pk=453444327741536756}, score=1.31, id=453444327741536756), SearchResp.SearchResult(entity={pk=453444327741536753}, score=1.31, id=453444327741536753)]]​ + +``` + +```javascript +client.search({​ + collection_name: 'my_sparse_collection',​ + data: {1: 0.2, 50: 0.4, 1000: 0.7},​ + limit: 3,​ + output_fields: ['pk'],​ + params: {​ + drop_ratio_search: 0.2​ + }​ +});​ + +``` + +```curl +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/search" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "my_sparse_collection",​ + "data": [​ + {"1": 0.2, "50": 0.4, "1000": 0.7}​ + ],​ + "annsField": "sparse_vector",​ + "limit": 3,​ + "searchParams":{​ + "params":{"drop_ratio_search": 0.2}​ + },​ + "outputFields": ["pk"]​ +}'​ +​ +## {"code":0,"cost":0,"data":[{"distance":0.63,"id":"453577185629572535","pk":"453577185629572535"},{"distance":0.1,"id":"453577185629572534","pk":"453577185629572534"}]}​ + +``` + +For more information on similarity search parameters, refer to [​Basic ANN Search](single-vector-search.md).​ + diff --git a/v2.4.x/site/en/userGuide/schema/string.md b/v2.4.x/site/en/userGuide/schema/string.md new file mode 100644 index 0000000000..592a85fde7 --- /dev/null +++ b/v2.4.x/site/en/userGuide/schema/string.md @@ -0,0 +1,590 @@ +--- +id: string.md +title: String Field +summary: "In Milvus, VARCHAR is the data type used for storing string-type data, suitable for storing variable-length strings. It can store strings with both single- and multi-byte characters, with a maximum length of up to 60,535 characters. When defining a VARCHAR field, you must also specify the maximum length parameter max_length. The VARCHAR string type offers an efficient and flexible way to store and manage text data, making it ideal for applications that handle strings of varying lengths.​​" +--- + +# String Field​ + +In Milvus, `VARCHAR` is the data type used for storing string-type data, suitable for storing variable-length strings. It can store strings with both single- and multi-byte characters, with a maximum length of up to 60,535 characters. When defining a `VARCHAR` field, you must also specify the maximum length parameter `max_length`. The `VARCHAR` string type offers an efficient and flexible way to store and manage text data, making it ideal for applications that handle strings of varying lengths.​ + +## Add VARCHAR field​ + +To use string data in Milvus, define a `VARCHAR` field when creating a collection. This process includes:​ + +1. Setting `datatype` to the supported string data type, i.e., `VARCHAR`.​ + +2. Specifying the maximum length of the string type using the `max_length` parameter, which cannot exceed 60,535 characters.​ + +
          + Python + Java + Node.js + cURL +
          + +```python +from pymilvus import MilvusClient, DataType​ +​ +client = MilvusClient(uri="http://localhost:19530")​ +​ +# define schema​ +schema = client.create_schema(​ + auto_id=False,​ + enable_dynamic_fields=True,​ +)​ +​ +schema.add_field(field_name="varchar_field1", datatype=DataType.VARCHAR, max_length=100)​ +schema.add_field(field_name="varchar_field2", datatype=DataType.VARCHAR, max_length=200)​ +schema.add_field(field_name="pk", datatype=DataType.INT64, is_primary=True)​ +schema.add_field(field_name="embedding", datatype=DataType.FLOAT_VECTOR, dim=3)​ + +``` + +```java +import io.milvus.v2.client.ConnectConfig;​ +import io.milvus.v2.client.MilvusClientV2;​ +​ +import io.milvus.v2.common.DataType;​ +import io.milvus.v2.service.collection.request.AddFieldReq;​ +import io.milvus.v2.service.collection.request.CreateCollectionReq;​ +​ +MilvusClientV2 client = new MilvusClientV2(ConnectConfig.builder()​ + .uri("http://localhost:19530")​ + .build());​ + ​ +CreateCollectionReq.CollectionSchema schema = client.createSchema();​ +schema.setEnableDynamicField(true);​ +​ +schema.addField(AddFieldReq.builder()​ + .fieldName("varchar_field1")​ + .dataType(DataType.VarChar)​ + .maxLength(100)​ + .build());​ +​ +schema.addField(AddFieldReq.builder()​ + .fieldName("varchar_field2")​ + .dataType(DataType.VarChar)​ + .maxLength(200)​ + .build());​ +​ +schema.addField(AddFieldReq.builder()​ + .fieldName("pk")​ + .dataType(DataType.Int64)​ + .isPrimaryKey(true)​ + .build());​ +​ +schema.addField(AddFieldReq.builder()​ + .fieldName("embedding")​ + .dataType(DataType.FloatVector)​ + .dimension(3)​ + .build());​ + +``` + +```javascript +import { MilvusClient, DataType } from "@zilliz/milvus2-sdk-node";​ +​ +const schema = [​ + {​ + name: "metadata",​ + data_type: DataType.JSON,​ + },​ + {​ + name: "pk",​ + data_type: DataType.Int64,​ + is_primary_key: true,​ + },​ + {​ + name: "varchar_field2",​ + data_type: DataType.VarChar,​ + max_length: 200,​ + },​ + {​ + name: "varchar_field1",​ + data_type: DataType.VarChar,​ + max_length: 100,​ + },​ +];​ + +``` + +```curl +export varcharField1='{​ + "fieldName": "varchar_field1",​ + "dataType": "VarChar",​ + "elementTypeParams": {​ + "max_length": 100​ + }​ +}'​ +​ +export varcharField2='{​ + "fieldName": "varchar_field2",​ + "dataType": "VarChar",​ + "elementTypeParams": {​ + "max_length": 200​ + }​ +}'​ +​ +export primaryField='{​ + "fieldName": "pk",​ + "dataType": "Int64",​ + "isPrimary": true​ +}'​ +​ +export vectorField='{​ + "fieldName": "embedding",​ + "dataType": "FloatVector",​ + "elementTypeParams": {​ + "dim": 3​ + }​ +}'​ +​ +export schema="{​ + \"autoID\": false,​ + \"fields\": [​ + $varcharField1,​ + $varcharField2,​ + $primaryField,​ + $vectorField​ + ]​ +}"​ + +``` + +In this example, we add two `VARCHAR` fields: `varchar_field1` and `varchar_field2`, with maximum lengths set to 100 and 200 characters, respectively. It is recommended to set `max_length` based on your data characteristics to ensure it accommodates the longest data while avoiding excessive space allocation. Additionally, we have added a primary field `pk` and a vector field `embedding`.​ + +
          + +The primary field and vector field are mandatory when you create a collection. The primary field uniquely identifies each entity, while the vector field is crucial for similarity search. For more details, refer to [​Primary Field & AutoID](primary-field.md), [​Dense Vector](dense-vector.md), [​Binary Vector](binary-vector.md), or [​Sparse Vector](sparse_vector.md).​ + +
          + +## Set index params​ + +Setting index parameters for `VARCHAR` fields is optional but can significantly improve retrieval efficiency.​ + +In the following example, we create an `AUTOINDEX` for `varchar_field1`, meaning Milvus will automatically create an appropriate index based on the data type. For more information, refer to [​AUTOINDEX](https://milvus.io/docs/glossary.md#Auto-Index).​ + +
          + Python + Java + Node.js + cURL +
          + +```python +index_params = client.prepare_index_params()​ +​ +index_params.add_index(​ + field_name="varchar_field1",​ + index_type="AUTOINDEX",​ + index_name="varchar_index"​ +)​ + +``` + +```java +​ +import io.milvus.v2.common.IndexParam;​ +import java.util.*;​ +​ +List indexes = new ArrayList<>();​ +indexes.add(IndexParam.builder()​ + .fieldName("varchar_field1")​ + .indexName("varchar_index")​ + .indexType(IndexParam.IndexType.AUTOINDEX)​ + .build());​ + +``` + +```javascript +const indexParams = [{​ + index_name: 'varchar_index',​ + field_name: 'varchar_field1',​ + index_type: IndexType.AUTOINDEX,​ +)];​ + +``` + +```curl +export indexParams='[​ + {​ + "fieldName": "varchar_field1",​ + "indexName": "varchar_index",​ + "indexType": "AUTOINDEX"​ + }​ + ]'​ + +``` + +In addition to `AUTOINDEX`, you can specify other scalar index types, such as `INVERTED` or `BITMAP`. For supported index types, refer to [​Scalar Indexes](https://milvus.io/docs/scalar_index.md).​ + +Moreover, before creating the collection, you must create an index for the vector field. In this example, we use `AUTOINDEX` to simplify vector index settings.​ + +
          + Python + Java + Node.js + cURL +
          + +```python +# Add vector index​ +index_params.add_index(​ + field_name="embedding",​ + index_type="AUTOINDEX", # Use automatic indexing to simplify complex index settings​ + metric_type="COSINE" # Specify similarity metric type, options include L2, COSINE, or IP​ +)​ + +``` + +```java +indexes.add(IndexParam.builder()​ + .fieldName("embedding")​ + .indexType(IndexParam.IndexType.AUTOINDEX)​ + .metricType(IndexParam.MetricType.COSINE)​ + .build());​ + +``` + +```javascript +indexParams.push({​ + index_name: 'embedding_index',​ + field_name: 'embedding',​ + metric_type: MetricType.COSINE,​ + index_type: IndexType.AUTOINDEX,​ +});​ + +``` + +```curl +export indexParams='[​ + {​ + "fieldName": "varchar_field1",​ + "indexName": "varchar_index",​ + "indexType": "AUTOINDEX"​ + },​ + {​ + "fieldName": "embedding",​ + "metricType": "COSINE",​ + "indexType": "AUTOINDEX"​ + }​ + ]'​ + +``` + +## Create collection​ + +Once the schema and index are defined, you can create a collection that includes string fields.​ + +
          + Python + Java + Node.js + cURL +
          + +```python +# Create Collection​ +client.create_collection(​ + collection_name="your_collection_name",​ + schema=schema,​ + index_params=index_params​ +)​ + +``` + +```java +CreateCollectionReq requestCreate = CreateCollectionReq.builder()​ + .collectionName("my_varchar_collection")​ + .collectionSchema(schema)​ + .indexParams(indexes)​ + .build();​ +client.createCollection(requestCreate);​ + +``` + +```javascript +client.create_collection({​ + collection_name: "my_varchar_collection",​ + schema: schema,​ + index_params: index_params​ +})​ + +``` + +```curl +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/collections/create" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d "{​ + \"collectionName\": \"my_varchar_collection\",​ + \"schema\": $schema,​ + \"indexParams\": $indexParams​ +}"​ +## {"code":0,"data":{}}​ + +``` + +## Insert data​ + +After creating the collection, you can insert data that includes string fields.​ + +
          + Python + Java + Node.js + cURL +
          + +```python +data = [​ + {"varchar_field1": "Product A", "varchar_field2": "High quality product", "pk": 1, "embedding": [0.1, 0.2, 0.3]},​ + {"varchar_field1": "Product B", "varchar_field2": "Affordable price", "pk": 2, "embedding": [0.4, 0.5, 0.6]},​ + {"varchar_field1": "Product C", "varchar_field2": "Best seller", "pk": 3, "embedding": [0.7, 0.8, 0.9]},​ +]​ +​ +client.insert(​ + collection_name="my_varchar_collection",​ + data=data​ +)​ + +``` + +```java +import com.google.gson.Gson;​ +import com.google.gson.JsonObject;​ +import io.milvus.v2.service.vector.request.InsertReq;​ +import io.milvus.v2.service.vector.response.InsertResp;​ +​ +List rows = new ArrayList<>();​ +Gson gson = new Gson();​ +rows.add(gson.fromJson("{\"varchar_field1\": \"Product A\", \"varchar_field2\": \"High quality product\", \"pk\": 1, \"embedding\": [0.1, 0.2, 0.3]}", JsonObject.class));​ +rows.add(gson.fromJson("{\"varchar_field1\": \"Product B\", \"varchar_field2\": \"Affordable price\", \"pk\": 2, \"embedding\": [0.4, 0.5, 0.6]}", JsonObject.class));​ +rows.add(gson.fromJson("{\"varchar_field1\": \"Product C\", \"varchar_field2\": \"Best seller\", \"pk\": 3, \"embedding\": [0.7, 0.8, 0.9]}", JsonObject.class));​ +​ +InsertResp insertR = client.insert(InsertReq.builder()​ + .collectionName("my_varchar_collection")​ + .data(rows)​ + .build());​ + +``` + +```javascript +const data = [​ + {​ + varchar_field1: "Product A",​ + varchar_field2: "High quality product",​ + pk: 1,​ + embedding: [0.1, 0.2, 0.3],​ + },​ + {​ + varchar_field1: "Product B",​ + varchar_field2: "Affordable price",​ + pk: 2,​ + embedding: [0.4, 0.5, 0.6],​ + },​ + {​ + varchar_field1: "Product C",​ + varchar_field2: "Best seller",​ + pk: 3,​ + embedding: [0.7, 0.8, 0.9],​ + },​ +];​ +client.insert({​ + collection_name: "my_sparse_collection",​ + data: data,​ +});​ +​ + +``` + +```curl +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/insert" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "data": [​ + {"varchar_field1": "Product A", "varchar_field2": "High quality product", "pk": 1, "embedding": [0.1, 0.2, 0.3]},​ + {"varchar_field1": "Product B", "varchar_field2": "Affordable price", "pk": 2, "embedding": [0.4, 0.5, 0.6]},​ + {"varchar_field1": "Product C", "varchar_field2": "Best seller", "pk": 3, "embedding": [0.7, 0.8, 0.9]} ​ + ],​ + "collectionName": "my_varchar_collection"​ +}'​ +​ +## {"code":0,"cost":0,"data":{"insertCount":3,"insertIds":[1,2,3]}}​ + +``` + +In this example, we insert data that includes `VARCHAR` fields (`varchar_field1` and `varchar_field2`), a primary field (`pk`), and vector representations (`embedding`). To ensure that the inserted data matches the fields defined in the schema, it is recommended to check data types in advance to avoid insertion errors.​ + +If you set `enable_dynamic_fields=True` when defining the schema, Milvus allows you to insert string fields that were not defined in advance. However, keep in mind that this may increase the complexity of queries and management, potentially impacting performance. For more information, refer to [​Dynamic Field](enable-dynammic-field.md).​ + +## Search and query​ + +After adding string fields, you can use them for filtering in search and query operations, achieving more precise search results.​ + +### Filter queries​ + +After adding string fields, you can filter results using these fields in queries. For example, you can query all entities where `varchar_field1` equals `"Product A"`:​ + +
          + Python + Java + Node.js + cURL +
          + +```python +filter = 'varchar_field1 == "Product A"'​ +​ +res = client.query(​ + collection_name="my_varchar_collection",​ + filter=filter,​ + output_fields=["varchar_field1", "varchar_field2"]​ +)​ +​ +print(res)​ +​ +# Output​ +# data: ["{'varchar_field1': 'Product A', 'varchar_field2': 'High quality product', 'pk': 1}"] ​ + +``` + +```java +import io.milvus.v2.service.vector.request.QueryReq;​ +import io.milvus.v2.service.vector.response.QueryResp;​ +​ +String filter = "varchar_field1 == \"Product A\"";​ +QueryResp resp = client.query(QueryReq.builder()​ + .collectionName("my_varchar_collection")​ + .filter(filter)​ + .outputFields(Arrays.asList("varchar_field1", "varchar_field2"))​ + .build());​ +​ +System.out.println(resp.getQueryResults());​ +​ +// Output​ +//​ +// [QueryResp.QueryResult(entity={varchar_field1=Product A, varchar_field2=High quality product, pk=1})]​ + +``` + +```javascript +client.query({​ + collection_name: 'my_varchar_collection',​ + filter: 'varchar_field1 == "Product A"',​ + output_fields: ['varchar_field1', 'varchar_field2']​ +});​ + +``` + +```curl +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/query" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "my_varchar_collection",​ + "filter": "varchar_field1 == \"Product A\"",​ + "outputFields": ["varchar_field1", "varchar_field2"]​ +}'​ +## {"code":0,"cost":0,"data":[{"pk":1,"varchar_field1":"Product A","varchar_field2":"High quality product"}]}​ + +``` + +This query expression returns all matching entities and outputs their `varchar_field1` and `varchar_field2` fields. For more information on filter queries, refer to [​Metadata Filtering](boolean.md).​ + +### Vector search with string filtering​ + +In addition to basic scalar field filtering, you can combine vector similarity searches with scalar field filters. For example, the following code shows how to add a scalar field filter to a vector search:​ + +
          + Python + Java + Node.js + cURL +
          + +```python +filter = 'varchar_field1 == "Product A"'​ +​ +res = client.search(​ + collection_name="my_varchar_collection",​ + data=[[0.3, -0.6, 0.1]],​ + limit=5,​ + search_params={"params": {"nprobe": 10}},​ + output_fields=["varchar_field1", "varchar_field2"],​ + filter=filter​ +)​ +​ +print(res)​ +​ +# Output​ +# data: ["[{'id': 1, 'distance': -0.06000000238418579, 'entity': {'varchar_field1': 'Product A', 'varchar_field2': 'High quality product'}}]"] ​ + +``` + +```java +import io.milvus.v2.service.vector.request.SearchReq;​ +import io.milvus.v2.service.vector.response.SearchResp;​ +​ +String filter = "varchar_field1 == \"Product A\"";​ +SearchResp resp = client.search(SearchReq.builder()​ + .collectionName("my_varchar_collection")​ + .annsField("embedding")​ + .data(Collections.singletonList(new FloatVec(new float[]{0.3f, -0.6f, 0.1f})))​ + .topK(5)​ + .outputFields(Arrays.asList("varchar_field1", "varchar_field2"))​ + .filter(filter)​ + .build());​ +​ +System.out.println(resp.getSearchResults());​ +​ +// Output​ +//​ +// [[SearchResp.SearchResult(entity={varchar_field1=Product A, varchar_field2=High quality product}, score=-0.2364331, id=1)]]​ + +``` + +```javascript +client.search({​ + collection_name: 'my_varchar_collection',​ + data: [0.3, -0.6, 0.1],​ + limit: 5,​ + output_fields: ['varchar_field1', 'varchar_field2'],​ + filter: 'varchar_field1 == "Product A"'​ + params: {​ + nprobe:10​ + }​ +});​ + +``` + +```curl +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/search" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "my_varchar_collection",​ + "data": [​ + [0.3, -0.6, 0.1]​ + ],​ + "limit": 5,​ + "searchParams":{​ + "params":{"nprobe":10}​ + },​ + "outputFields": ["varchar_field1", "varchar_field2"],​ + "filter": "varchar_field1 == \"Product A\""​ +}'​ +​ +## {"code":0,"cost":0,"data":[{"distance":-0.2364331,"id":1,"varchar_field1":"Product A","varchar_field2":"High quality product"}]}​ + +``` + +In this example, we first define a query vector and add a filter condition `varchar_field1 == "Product A"` during the search. This ensures that the search results are not only similar to the query vector but also match the specified string filter condition. For more information, refer to [​Metadata Filtering](boolean.md).​ + diff --git a/v2.4.x/site/en/userGuide/schema/use-json-fields.md b/v2.4.x/site/en/userGuide/schema/use-json-fields.md new file mode 100644 index 0000000000..dabd766bcd --- /dev/null +++ b/v2.4.x/site/en/userGuide/schema/use-json-fields.md @@ -0,0 +1,546 @@ +--- +id: use-json-fields.md +title: Use JSON Fields +summary: JSON (JavaScript Object Notation) is a lightweight data exchange format that provides a flexible way to store and query complex data structures. In Milvus, you can store additional structured information alongside vector data using JSON fields, enabling advanced searches and queries that combine vector similarity with structured filtering.​ +--- + +# JSON Field​ + +[JSON](https://en.wikipedia.org/wiki/JSON) (JavaScript Object Notation) is a lightweight data exchange format that provides a flexible way to store and query complex data structures. In Milvus, you can store additional structured information alongside vector data using JSON fields, enabling advanced searches and queries that combine vector similarity with structured filtering.​ + +JSON fields are ideal for applications that require metadata to optimize retrieval results. For example, in e-commerce, product vectors can be enhanced with attributes like category, price, and brand. In recommendation systems, user vectors can be combined with preferences and demographic information. Below is an example of a typical JSON field:​ + +```json +{​ + "category": "electronics",​ + "price": 99.99,​ + "brand": "BrandA"​ +}​ + +``` + +## Add JSON field​ +To use JSON fields in Milvus, define the relevant field type in the collection schema, setting the `datatype` to the supported JSON type, i.e., `JSON`.​ + +Here’s how to define a collection schema that includes a JSON field:​ + +
          + Python + Java + Node.js + cURL +
          + +```python +from pymilvus import MilvusClient, DataType​ +​ +client = MilvusClient(uri="http://localhost:19530")​ +​ +schema = client.create_schema(​ + auto_id=False,​ + enable_dynamic_fields=True,​ +)​ +​ +schema.add_field(field_name="metadata", datatype=DataType.JSON)​ +schema.add_field(field_name="pk", datatype=DataType.INT64, is_primary=True)​ +schema.add_field(field_name="embedding", datatype=DataType.FLOAT_VECTOR, dim=3)​ + +``` + +```java +import io.milvus.v2.client.ConnectConfig;​ +import io.milvus.v2.client.MilvusClientV2;​ +​ +import io.milvus.v2.common.DataType;​ +import io.milvus.v2.service.collection.request.AddFieldReq;​ +import io.milvus.v2.service.collection.request.CreateCollectionReq;​ +​ +MilvusClientV2 client = new MilvusClientV2(ConnectConfig.builder()​ + .uri("http://localhost:19530")​ + .build());​ + ​ +CreateCollectionReq.CollectionSchema schema = client.createSchema();​ +schema.setEnableDynamicField(true);​ +​ +schema.addField(AddFieldReq.builder()​ + .fieldName("metadata")​ + .dataType(DataType.JSON)​ + .build());​ +​ +schema.addField(AddFieldReq.builder()​ + .fieldName("pk")​ + .dataType(DataType.Int64)​ + .isPrimaryKey(true)​ + .build());​ +​ +schema.addField(AddFieldReq.builder()​ + .fieldName("embedding")​ + .dataType(DataType.FloatVector)​ + .dimension(3)​ + .build());​ + +``` + +```javascript +import { MilvusClient, DataType } from "@zilliz/milvus2-sdk-node";​ +const schema = [​ + {​ + name: "metadata",​ + data_type: DataType.JSON,​ + },​ + {​ + name: "pk",​ + data_type: DataType.Int64,​ + is_primary_key: true,​ + },​ + {​ + name: "embedding",​ + data_type: DataType.FloatVector,​ + dim: 3,​ + },​ +];​ + +``` + +```curl +export jsonField='{​ + "fieldName": "metadata",​ + "dataType": "JSON"​ +}'​ +​ +export pkField='{​ + "fieldName": "pk",​ + "dataType": "Int64",​ + "isPrimary": true​ +}'​ +​ +export vectorField='{​ + "fieldName": "embedding",​ + "dataType": "FloatVector",​ + "elementTypeParams": {​ + "dim": 3​ + }​ +}'​ +​ +export schema="{​ + \"autoID\": false,​ + \"fields\": [​ + $jsonField,​ + $pkField,​ + $vectorField​ + ]​ +}"​ + +``` + +In this example, we add a JSON field called `metadata` to store additional metadata related to vector data, such as product category, price, and brand information.​ + +
          + +The primary field and vector field are mandatory when you create a collection. The primary field uniquely identifies each entity, while the vector field is crucial for similarity search. For more details, refer to [​Primary Field & AutoID](primary-field.md), [​Dense Vector](dense-vector.md), [​Binary Vector](binary-vector.md), or [​Sparse Vector](sparse_vector.md).​ + +
          + +## Create collection​ + +When creating a collection, you must create an index for the vector field to ensure retrieval performance. In this example, we use `AUTOINDEX` to simplify index setup. For more details, refer to [​AUTOINDEX](https://milvus.io/docs/glossary.md#Auto-Index).​ + +
          + Python + Java + Node.js + cURL +
          + +```python +​ +index_params = client.prepare_index_params()​ +​ +index_params.add_index(​ + field_name="embedding",​ + index_type="AUTOINDEX",​ + metric_type="COSINE"​ +)​ + +``` + +```java +import io.milvus.v2.common.IndexParam;​ +import java.util.*;​ +​ +List indexes = new ArrayList<>();​ +indexes.add(IndexParam.builder()​ + .fieldName("embedding")​ + .indexType(IndexParam.IndexType.AUTOINDEX)​ + .metricType(IndexParam.MetricType.COSINE)​ + .build());​ + +``` + +```javascript +const indexParams = {​ + index_name: 'embedding_index',​ + field_name: 'embedding',​ + metricType: MetricType.CONSINE,​ + index_type: IndexType.AUTOINDEX,​ +);​ + +``` + +```curl +export indexParams='[​ + {​ + "fieldName": "embedding",​ + "metricType": "COSINE",​ + "indexType": "AUTOINDEX"​ + }​ + ]'​ + +``` + +Use the defined schema and index parameters to create a collection:​ + +
          + Python + Java + Node.js + cURL +
          + +```python +client.create_collection(​ + collection_name="my_json_collection",​ + schema=schema,​ + index_params=index_params​ +)​ + +``` + +```java +CreateCollectionReq requestCreate = CreateCollectionReq.builder()​ + .collectionName("my_json_collection")​ + .collectionSchema(schema)​ + .indexParams(indexes)​ + .build();​ +client.createCollection(requestCreate);​ + +``` + +```javascript +client.create_collection({​ + collection_name: "my_json_collection",​ + schema: schema,​ + index_params: indexParams​ +})​ + +``` + +```curl +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/collections/create" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d "{​ + \"collectionName\": \"my_json_collection\",​ + \"schema\": $schema,​ + \"indexParams\": $indexParams​ +}"​ + +``` + +## Insert data​ + +After creating the collection, you can insert data that includes JSON fields.​ + +
          + Python + Java + Node.js + cURL +
          + +```python +# Data to be inserted​ +data = [​ + {​ + "metadata": {"category": "electronics", "price": 99.99, "brand": "BrandA"},​ + "pk": 1,​ + "embedding": [0.12, 0.34, 0.56]​ + },​ + {​ + "metadata": {"category": "home_appliances", "price": 249.99, "brand": "BrandB"},​ + "pk": 2,​ + "embedding": [0.56, 0.78, 0.90]​ + },​ + {​ + "metadata": {"category": "furniture", "price": 399.99, "brand": "BrandC"},​ + "pk": 3,​ + "embedding": [0.91, 0.18, 0.23]​ + }​ +]​ +​ +# Insert data into the collection​ +client.insert(​ + collection_name="your_collection_name",​ + data=data​ +)​ + +``` + +```java +import com.google.gson.Gson;​ +import com.google.gson.JsonObject;​ +​ +import io.milvus.v2.service.vector.request.InsertReq;​ +import io.milvus.v2.service.vector.response.InsertResp;​ +​ +List rows = new ArrayList<>();​ +Gson gson = new Gson();​ +rows.add(gson.fromJson("{\"metadata\": {\"category\": \"electronics\", \"price\": 99.99, \"brand\": \"BrandA\"}, \"pk\": 1, \"embedding\": [0.1, 0.2, 0.3]}", JsonObject.class));​ +rows.add(gson.fromJson("{\"metadata\": {\"category\": \"home_appliances\", \"price\": 249.99, \"brand\": \"BrandB\"}, \"pk\": 2, \"embedding\": [0.4, 0.5, 0.6]}", JsonObject.class));​ +rows.add(gson.fromJson("{\"metadata\": {\"category\": \"furniture\", \"price\": 399.99, \"brand\": \"BrandC\"}, \"pk\": 3, \"embedding\": [0.7, 0.8, 0.9]}", JsonObject.class));​ +​ +InsertResp insertR = client.insert(InsertReq.builder()​ + .collectionName("my_json_collection")​ + .data(rows)​ + .build());​ + +``` + +```javascript +const data = [​ + {​ + "metadata": {"category": "electronics", "price": 99.99, "brand": "BrandA"},​ + "pk": 1,​ + "embedding": [0.12, 0.34, 0.56]​ + },​ + {​ + "metadata": {"category": "home_appliances", "price": 249.99, "brand": "BrandB"},​ + "pk": 2,​ + "embedding": [0.56, 0.78, 0.90]​ + },​ + {​ + "metadata": {"category": "furniture", "price": 399.99, "brand": "BrandC"},​ + "pk": 3,​ + "embedding": [0.91, 0.18, 0.23]​ + }​ +]​ +​ +client.insert({​ + collection_name: "my_json_collection",​ + data: data​ +});​ + +``` + +```curl +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/insert" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "data": [​ + {​ + "metadata": {"category": "electronics", "price": 99.99, "brand": "BrandA"},​ + "pk": 1,​ + "embedding": [0.12, 0.34, 0.56]​ + },​ + {​ + "metadata": {"category": "home_appliances", "price": 249.99, "brand": "BrandB"},​ + "pk": 2,​ + "embedding": [0.56, 0.78, 0.90]​ + },​ + {​ + "metadata": {"category": "furniture", "price": 399.99, "brand": "BrandC"},​ + "pk": 3,​ + "embedding": [0.91, 0.18, 0.23]​ + } ​ + ],​ + "collectionName": "my_json_collection"​ +}'​ + +``` + +In this example:​ + +- Each data entry includes a primary field (`pk`), `metadata` as a JSON field to store information such as product category, price, and brand.​ + +- `embedding` is a 3-dimensional vector field used for vector similarity search.​ + +## Search and query​ + +JSON fields allow scalar filtering during searches, enhancing Milvus's vector search capabilities. You can query based on JSON properties alongside vector similarity.​ + +### Filter queries​ + +You can filter data based on JSON properties, such as matching specific values or checking if a number falls within a certain range.​ + +
          + Python + Java + Node.js + cURL +
          + +```python +filter = 'metadata["category"] == "electronics" and metadata["price"] < 150'​ +​ +res = client.query(​ + collection_name="my_json_collection",​ + filter=filter,​ + output_fields=["metadata"]​ +)​ +​ +print(res)​ +​ +# Output​ +# data: ["{'metadata': {'category': 'electronics', 'price': 99.99, 'brand': 'BrandA'}, 'pk': 1}"] ​ + +``` + +```java +import io.milvus.v2.service.vector.request.QueryReq;​ +import io.milvus.v2.service.vector.response.QueryResp;​ +​ +String filter = "metadata[\"category\"] == \"electronics\" and metadata[\"price\"] < 150";​ +QueryResp resp = client.query(QueryReq.builder()​ + .collectionName("my_json_collection")​ + .filter(filter)​ + .outputFields(Collections.singletonList("metadata"))​ + .build());​ +​ +System.out.println(resp.getQueryResults());​ +​ +// Output​ +//​ +// [QueryResp.QueryResult(entity={metadata={"category":"electronics","price":99.99,"brand":"BrandA"}, pk=1})]​ + +``` + +```javascript +client.query({​ + collection_name: 'my_scalar_collection',​ + filter: 'metadata["category"] == "electronics" and metadata["price"] < 150',​ + output_fields: ['metadata']​ +});​ + +``` + +```curl +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/query" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "my_json_collection",​ + "filter": "metadata[\"category\"] == \"electronics\" and metadata[\"price\"] < 150",​ + "outputFields": ["metadata"]​ +}'​ +{"code":0,"cost":0,"data":[{"metadata":"{\"category\": \"electronics\", \"price\": 99.99, \"brand\": \"BrandA\"}","pk":1}]}​ + +``` + +In the above query, Milvus filters out entities where the `metadata` field has a category of `"electronics"` and a price below 150, returning entities that match these criteria.​ + +### Vector search with JSON filtering​ + +By combining vector similarity with JSON filtering, you can ensure that the retrieved data not only matches semantically but also meets specific business conditions, making the search results more precise and aligned with user needs.​ + +
          + Python + Java + Node.js + cURL +
          + +```python +filter = 'metadata["brand"] == "BrandA"'​ +​ +res = client.search(​ + collection_name="my_json_collection",​ + data=[[0.3, -0.6, 0.1]],​ + limit=5,​ + search_params={"params": {"nprobe": 10}},​ + output_fields=["metadata"],​ + filter=filter​ +)​ +​ +print(res)​ +​ +# Output​ +# data: ["[{'id': 1, 'distance': -0.2479381263256073, 'entity': {'metadata': {'category': 'electronics', 'price': 99.99, 'brand': 'BrandA'}}}]"] ​ + +``` + +```java +import io.milvus.v2.service.vector.request.SearchReq;​ +import io.milvus.v2.service.vector.response.SearchResp;​ +​ +String filter = "metadata[\"brand\"] == \"BrandA\"";​ +SearchResp resp = client.search(SearchReq.builder()​ + .collectionName("my_json_collection")​ + .annsField("embedding")​ + .data(Collections.singletonList(new FloatVec(new float[]{0.3f, -0.6f, 0.1f})))​ + .topK(5)​ + .outputFields(Collections.singletonList("metadata"))​ + .filter(filter)​ + .build());​ +​ +System.out.println(resp.getSearchResults());​ +​ +// Output​ +//​ +// [[SearchResp.SearchResult(entity={metadata={"category":"electronics","price":99.99,"brand":"BrandA"}}, score=-0.2364331, id=1)]]​ + +``` + +```javascript +client.search({​ + collection_name: 'my_json_collection',​ + data: [0.3, -0.6, 0.1],​ + limit: 5,​ + output_fields: ['metadata'],​ + filter: 'metadata["category"] == "electronics" and metadata["price"] < 150',​ +});​ + +``` + +```curl +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/search" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "my_json_collection",​ + "data": [​ + [0.3, -0.6, 0.1]​ + ],​ + "annsField": "embedding",​ + "limit": 5,​ + "searchParams":{​ + "params":{"nprobe":10}​ + },​ + "outputFields": ["metadata"],​ + "filter": "metadata[\"brand\"] == \"BrandA\""​ +}'​ +​ +## {"code":0,"cost":0,"data":[{"distance":-0.24793813,"id":1,"metadata":"{\"category\": \"electronics\", \"price\": 99.99, \"brand\": \"BrandA\"}"}]}​ + +``` + +In this example, Milvus returns the top 5 entities most similar to the query vector, with the `metadata` field containing a brand of `"BrandA"`.​ + +Additionally, Milvus supports advanced JSON filtering operators such as `JSON_CONTAINS`, `JSON_CONTAINS_ALL`, and `JSON_CONTAINS_ANY`, which can further enhance query capabilities. For more details, refer to [​Metadata Filtering](boolean.md).​ + +## Limits​ + +- **Indexing Limitations**: Due to the complexity of data structures, indexing JSON fields is not supported.​ + +- **Data Type Matching**: If a JSON field's key value is an integer or floating point, it can only be compared with another integer or float key or `INT32/64` or `FLOAT32/64` fields. If the key value is a string (`VARCHAR`), it can only be compared with another string key.​ + +- **Naming Restrictions**: When naming JSON keys, it is recommended to use only letters, numeric characters, and underscores, as other characters may cause issues during filtering or searching.​ + +- **Handling String Values**: For string values (`VARCHAR`), Milvus stores JSON field strings as-is without semantic conversion. For example: `'a"b'`, `"a'b"`, `'a\\'b'`, and `"a\\"b"` are stored as entered; however, `'a'b'` and `"a"b"` are considered invalid.​ + +- **Handling Nested Dictionaries**: Any nested dictionaries within JSON field values are treated as strings.​ + diff --git a/v2.4.x/site/en/userGuide/search-query-get/boolean.md b/v2.4.x/site/en/userGuide/search-query-get/boolean.md new file mode 100644 index 0000000000..3a8f58d7d9 --- /dev/null +++ b/v2.4.x/site/en/userGuide/search-query-get/boolean.md @@ -0,0 +1,2317 @@ +--- +id: boolean.md +summary: A filter expression can be used to filter a specific scalar field during a search or query to obtain precisely matched results. This guide will introduce how to use filter expressions in Zilliz through an example dataset. For demonstration purposes, this guide will only provide examples of query operations.​ +title: Metadata Filtering +--- + +# Metadata Filtering​ + +A filter expression can be used to filter a specific scalar field during a search or query to obtain precisely matched results. This guide will introduce how to use filter expressions in Zilliz through an example dataset. For demonstration purposes, this guide will only provide examples of query operations.​ + +## Example Dataset + +Suppose the example dataset is stored in a collection named "my_collection" and includes 10 entities of electronic products. The following is the example dataset.​ + +

          id​

          + +

          color​

          + +

          vector​

          + +

          price​

          + +

          inventory​

          + +

          sales_volume​

          + +

          description​

          + +

          pink_8682​

          + +

          [0.3580376395471989, -0.6023495712049978, ...]​

          + +

          593​

          + +

          {"brand": "Sony", "quantity": 310}​

          + +

          [161, 81, 51]​

          + +

          Sony Xperia 1 VI is a flagship Android smartphone released in 2024 with a 6.5-inch LTPO OLED display​

          + +

          2​

          + +

          red_7025​

          + +

          [0.43742130801983836, -0.5597502546264526, ....]​

          + +

          196​

          + +

          {"brand": "Samsung", "quantity": 274}​

          + +

          [126, 126, 125, 96, 155]​

          + +

          Galaxy S24 Ultra, Samsung’s latest flagship smartphone.​

          + +

          3​

          + +

          orange_6781​

          + +

          [0.19886812562848388, 0.06023560599112088, ...]​

          + +

          862​

          + +

          {"brand": "Samsung", "quantity": 103}​

          + +

          [124, 117, 90, 188]​

          + +

          Galaxy Fold features the world’s first 7.3-inch Infinity Flex Display.​

          + +

          4​

          + +

          pink_9298​

          + +

          [0.3172005263489739, 0.9719044792798428, ...]​

          + +

          991​

          + +

          {"brand": "Microsoft", "quantity": 175,}​

          + +

          [133, 92, 181, 61, 193]​

          + +

          Surface Duo 2, now with lightning-fast 5G(Footnote1) and dynamic triple lens camera.​

          + +

          5​

          + +

          red_4794​

          + +

          [0.4452349528804562, -0.8757026943054742, ...]​

          + +

          327​

          + +

          {"brand": "Apple", "quantity": 193}​

          + +

          [155, 161, 106, 86, 99]​

          + +

          iPhone 15 Pro, A new chip designed for better gaming and other 'pro' features.​

          + +

          6​

          + +

          yellow_4222​

          + +

          [0.985825131989184, -0.8144651566660419, ...]​

          + +

          996​

          + +

          {"brand": "Microsoft", "quantity": 376}​

          + +

          [173, 151, 148]​

          + +

          The Microsoft Surface Duo seems at first like the perfect little device for this new work-from-home world.​

          + +

          7​

          + +

          red_9392​

          + +

          [0.8371977790571115, -0.015764369584852833, ...]​

          + +

          848​

          + +

          {"brand": "Apple", "quantity": 61}​

          + +

          [59, 156, 126, 60, 177]​

          + +

          The iPhone 14 is a smartphone from Apple Inc. that comes in various colors and sizes.​

          + +

          8​

          + +

          grey_8510​

          + +

          [-0.33445148015177995, -0.2567135004164067, ...]​

          + +

          241​

          + +

          {"brand": "Dell", "quantity": 248}​

          + +

          [105, 126, 114, 132]​

          + +

          The Dell Inspiron 15 3000 laptop is equipped with a powerful Intel Core i5-1135G7 Quad-Core Processor, 12GB RAM and 256GB SSD storage.​

          + +

          9​

          + +

          white_9381​

          + +

          [0.39524717779832685, 0.4000257286739164, ...]​

          + +

          597​

          + +

          {"brand": "Apple", "quantity": 351}​

          + +

          [150, 150, 73]​

          + +

          The iPhone 16 features a 6.1-inch OLED display, is powered by Apple's A18 processor, and has dual cameras at the back.​

          + +

          10​

          + +

          purple_4976​

          + +

          [0.5718280481994695, 0.24070317428066512, ...]​

          + +

          450​

          + +

          {"brand": "Apple", "quantity": 268}​

          + +

          [190, 149, 85, 79, 80]​

          + +

          The iPad is a brand of iOS- and iPadOS-based tablet computers that are developed and marketed by Apple.​

          + +
          + +- `id`: The ID of the product. The data type of this field is INT64.​ + +- `vector`: The embedding vector of the product image that can represent different features of the product (such as product size, style, pattern, etc.). For convenience, this field is omitted in the demonstration. ​ + +- `color`: The color of the product. The data type of this field is VARCHAR. The numeric value in this field indicates the hue, which helps differentiate various shades of colors.​ + +- `price`: The price of the product. The data type of this field is INT64.​ + +- `inventory`: The inventory of the product. The data type of this field is JSON and contains two keys: the key `brand` represents the brand of the product and the key `quantity` represents the number of items in stock.​ + +- `sales_volume`: The sales volume of products in different countries. The data type of this field is Array. The values in this array contain 3 to 5 integers.​ + +- `description`: The description of the product. The data type of this field is VARCHAR. It offers a summary of the product features, functionality, and intended users.​ + +## Single-condition filtering + +The following types of operators can be used in filters with single condition:​ + +- [Comparison operators](#Comparison-operators)​ + +- [Term operators](#Term-operators)​ + +- [Match operators](#Match-operators)​ + +- [Arithmetic operators](#Arithmetic-operators)​ + +- [Advanced JSON operators](#Advanced-json-operators)​ + +- [Advanced Array operators](#Advanced-array-operators)​ + +### Comparison operators + +Comparison operators include:​ + +- `>`: Greater than​ + +- `<`: Less than​ + +- `==`: Equal​ + +- `<=`: Less than or equal​ + +- `>=`: Greater than or equal​ + +- `!=`: Not equal​ + +#### Example 1: Apply filter on scalar field​ + +The following example demonstrates how to filter products with prices ranging from 500 to 900:​ + +
          + Python + Java + Node.js + cURL +
          + +```python +from pymilvus import MilvusClient​ +​ +client = MilvusClient(​ + uri="http://localhost:19530",​ + token="root:Milvus"​ +)​ +​ +results = client.query(​ + collection_name="my_collection",​ + filter="500 < price < 900",​ + output_fields=["id", "color", "price"]​ +)​ +​ +# Output​ +# {'id': 1, 'color': 'pink_8682', 'price': np.float32(593.0)}​ +# {'id': 3, 'color': 'orange_6781', 'price': np.float32(862.0)}​ +# {'id': 7, 'color': 'red_9392', 'price': np.float32(848.0)}​ +# {'id': 9, 'color': 'white_9381', 'price': np.float32(597.0)}​ + +``` + +```java +import io.milvus.v2.client.ConnectConfig;​ +import io.milvus.v2.client.MilvusClientV2;​ +import io.milvus.v2.service.vector.request.QueryReq​ +import io.milvus.v2.service.vector.request.QueryResp​ +​ +MilvusClientV2 client = new MilvusClientV2(ConnectConfig.builder()​ + .uri("http://localhost:19530")​ + .token("root:Milvus")​ + .build());​ +​ +QueryReq queryReq = QueryReq.builder()​ + .collectionName("my_collection")​ + .filter("500 < price < 900")​ + .outputFields(Arrays.asList("id", "color", "price"))​ + .build();​ +​ +QueryResp getResp = client.query(queryReq);​ +​ +// Output​ +// {color=pink_8682, price=593.0, id=1}​ +// {color=orange_6781, price=862.0, id=3}​ +// {color=red_9392, price=848.0, id=7}​ +// {color=white_9381, price=597.0, id=9}​ + +``` + +```javascript +import { MilvusClient, DataType } from "@zilliz/milvus2-sdk-node";​ +​ +const address = "http://localhost:19530";​ +const token = "root:Milvus";​ +const client = new MilvusClient({address, token});​ +​ +var res = client.query({​ + collection_name="my_collection",​ + filter="500 < price < 900",​ + output_fields=["id", "color", "price"]​ +})​ + +``` + +```curl +export CLUSTER_ENDPOINT="http://localhost:19530"​ +export TOKEN="root:Milvus"​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/query" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "my_collection",​ + "filter": "500 < price < 900",​ + "outputFields": ["id", "color", "price"]​ +}'​ + +``` + +The filtered results are as follows:​ + +```json +[​ + {"id": 1, "color": "pink_8682" "price":593},​ + {"id": 3, "color": "orange_6781" "price":862},​ + {"id": 7, "color": "red_9392" "price":848},​ + {"id": 9, "color": "white_9381" "price":597}​ +]​ + +``` + +#### Example 2: Apply filter on JSON field + +The following example demonstrates how to filter products with an inventory quantity of 250 or more.​ + +
          + Python + Java + Node.js + cURL +
          + +```python +results = client.query(​ + collection_name="my_collection",​ + filter='inventory["quantity"] >= 250',​ + output_fields=["id", "color", "price", "inventory"]​ +)​ +​ +# Output​ +# {'id': 1, 'color': 'pink_8682', 'price': np.float32(593.0), 'inventory': {'brand': 'Sony', 'quantity': 310}}​ +# {'id': 2, 'color': 'red_7025', 'price': np.float32(196.0), 'inventory': {'brand': 'Samsung', 'quantity': 274}}​ +# {'id': 6, 'color': 'yellow_4222', 'price': np.float32(996.0), 'inventory': {'brand': 'Microsoft', 'quantity': 376}}​ +# {'id': 9, 'color': 'white_9381', 'price': np.float32(597.0), 'inventory': {'brand': 'Apple', 'quantity': 351}}​ +# {'id': 10, 'color': 'purple_4976', 'price': np.float32(450.0), 'inventory': {'brand': 'Apple', 'quantity': 268}}​ + +``` + +```java +import io.milvus.v2.service.vector.request.QueryReq​ +import io.milvus.v2.service.vector.request.QueryResp​ +​ +QueryReq queryReq = QueryReq.builder()​ + .collectionName("my_collection")​ + .filter("inventory[\"quantity\"] >= 250")​ + .outputFields(Arrays.asList("id", "color", "price", "inventory"))​ + .build();​ +​ +QueryResp getResp = client.query(queryReq);​ +​ +// Output​ +// {color=pink_8682, price=593.0, id=1, inventory={"brand":"Sony","quantity":310}}​ +// {color=red_7025, price=196.0, id=2, inventory={"brand":"Samsung","quantity":274}}​ +// {color=yellow_4222, price=996.0, id=6, inventory={"brand":"Microsoft","quantity":376}}​ +// {color=white_9381, price=597.0, id=9, inventory={"brand":"Apple","quantity":351}}​ +// {color=purple_4976, price=450.0, id=10, inventory={"brand":"Apple","quantity":268}}​ + +``` + +```javascript +import { MilvusClient, DataType } from "@zilliz/milvus2-sdk-node";​ +​ +var res = client.query({​ + collection_name="my_collection",​ + filter='inventory["quantity"] >= 250',​ + output_fields=["id", "color", "price", "inventory"]​ +})​ + +``` + +```curl +export CLUSTER_ENDPOINT="http://localhost:19530"​ +export TOKEN="root:Milvus"​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/query" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "my_collection",​ + "filter": "inventory[\"quantity\"] >=250",​ + "outputFields": ["id", "color", "price", "inventory"]​ +}'​ + +``` + +The filtered results are as follows:​ + +```JSON +[​ + {​ + "id": 1,​ + "color": "pink_8682",​ + "price": 593,​ + "inventory": {​ + "brand": "Sony",​ + "quantity": 310​ + },​ + "sales_volume": [​ + 161,​ + 81,​ + 51​ + ]​ + },​ + {​ + "id": 2,​ + "color": "red_7025",​ + "price": 196,​ + "inventory": {​ + "brand": "Samsung",​ + "quantity": 274​ + },​ + "sales_volume": [​ + 126,​ + 126,​ + 125,​ + 96,​ + 155​ + ]​ + },​ + {​ + "id": 6,​ + "color": "yellow_4222",​ + "price": 996,​ + "inventory": {​ + "brand": "Microsoft",​ + "quantity": 376​ + },​ + "sales_volume": [​ + 173,​ + 151,​ + 148​ + ]​ + },​ + {​ + "id": 9,​ + "color": "white_9381",​ + "price": 597,​ + "inventory": {​ + "brand": "Apple",​ + "quantity": 351​ + },​ + "sales_volume": [​ + 150,​ + 150,​ + 73​ + ]​ + },​ + {​ + "id": 10,​ + "color": "purple_4976",​ + "price": 450,​ + "inventory": {​ + "brand": "Apple",​ + "quantity": 268​ + },​ + "sales_volume": [​ + 190,​ + 149,​ + 85,​ + 79,​ + 80​ + ]​ + }​ +]​ + +``` + +#### Example 3: Apply filter on Array field​ + +The following example demonstrates how to filter products whose sales volume in the first country is 150 or more.​ + +
          + Python + Java + Node.js + cURL +
          + +```python +results = client.query(​ + collection_name="my_collection",​ + filter="sales_volume[0] >= 150",​ + output_fields=["id", "color", "price", "sales_volume"]​ +)​ +​ +# Output​ +# {'id': 1, 'color': 'pink_8682', 'price': np.float32(593.0), 'sales_volume': [161, 81, 51]}​ +# {'id': 5, 'color': 'red_4794', 'price': np.float32(327.0), 'sales_volume': [155, 161, 106, 86, 99]}​ +# {'id': 6, 'color': 'yellow_4222', 'price': np.float32(996.0), 'sales_volume': [173, 151, 148]}​ +# {'id': 9, 'color': 'white_9381', 'price': np.float32(597.0), 'sales_volume': [150, 150, 73]}​ +# {'id': 10, 'color': 'purple_4976', 'price': np.float32(450.0), 'sales_volume': [190, 149, 85, 79, 80]}​ + +``` + +```java +import io.milvus.v2.service.vector.request.QueryReq​ +import io.milvus.v2.service.vector.request.QueryResp​ +​ +QueryReq queryReq = QueryReq.builder()​ + .collectionName("my_collection")​ + .filter("sales_volume[0] >= 150")​ + .outputFields(Arrays.asList("id", "color", "price", "sales_volume"))​ + .build();​ +​ +QueryResp getResp = client.query(queryReq);​ +​ +// Output​ +// {color=pink_8682, price=593.0, id=1, sales_volume=[161, 81, 51]}​ +// {color=red_4794, price=327.0, id=5, sales_volume=[155, 161, 106, 86, 99]}​ +// {color=yellow_4222, price=996.0, id=6, sales_volume=[173, 151, 148]}​ +// {color=white_9381, price=597.0, id=9, sales_volume=[150, 150, 73]}​ +// {color=purple_4976, price=450.0, id=10, sales_volume=[190, 149, 85, 79, 80]}​ + +``` + +```javascript +import { MilvusClient, DataType } from "@zilliz/milvus2-sdk-node";​ +​ +var res = client.query({​ + collection_name="my_collection",​ + filter="sales_volume[0] >= 150",​ + output_fields=["id", "color","price", "sales_volume"]​ +})​ + +``` + +```curl +export CLUSTER_ENDPOINT="http://localhost:19530"​ +export TOKEN="root:Milvus"​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/query" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "my_collection",​ + "filter": "sales_volume[0] >= 150",​ + "outputFields": ["id", "color", "price", "sales_volume"]​ +}'​ + +``` + +The filtered results are as follows:​ + +```JSON +[​ + {​ + "id": 1,​ + "color": "pink_8682",​ + "price": 593,​ + "sales_volume": [​ + 161,​ + 81,​ + 51​ + ]​ + },​ + {​ + "id": 5,​ + "color": "red_4794",​ + "price": 327,​ + "sales_volume": [​ + 155,​ + 161,​ + 106,​ + 86,​ + 99​ + ]​ + },​ + {​ + "id": 6,​ + "color": "yellow_4222",​ + "price": 996,​ + "sales_volume": [​ + 173,​ + 151,​ + 148​ + ]​ + },​ + {​ + "id": 9,​ + "color": "white_9381",​ + "price": 597,​ + "sales_volume": [​ + 150,​ + 150,​ + 73​ + ]​ + },​ + {​ + "id": 10,​ + "color": "purple_4976",​ + "price": 450,​ + "sales_volume": [​ + 190,​ + 149,​ + 85,​ + 79,​ + 80​ + ]​ + }​ +]​ + +``` + +### Term operators​ + +Term operators include:​ + +- `in`: Filter results that match the condition​ + +- `not in`: Filter results that do not match the condition​ + +#### Example 1: Apply filter on scalar field​ + +The following example demonstrates how to filter products whose color is not red.​ + +
          + Python + Java + Node.js + cURL +
          + +```python +results = client.query(​ + collection_name="my_collection",​ + filter='color not in ["red_7025","red_4794","red_9392"]',​ + output_fields=["id", "color", "price"]​ +)​ +​ +# Output​ +# {'id': 1, 'color': 'pink_8682', 'price': np.float32(593.0)}​ +# {'id': 3, 'color': 'orange_6781', 'price': np.float32(862.0)}​ +# {'id': 4, 'color': 'pink_9298', 'price': np.float32(991.0)}​ +# {'id': 6, 'color': 'yellow_4222', 'price': np.float32(996.0)}​ +# {'id': 8, 'color': 'grey_8510', 'price': np.float32(241.0)}​ +# {'id': 9, 'color': 'white_9381', 'price': np.float32(597.0)}​ +# {'id': 10, 'color': 'purple_4976', 'price': np.float32(450.0)}​ + +``` + +```java +import io.milvus.v2.service.vector.request.QueryReq​ +import io.milvus.v2.service.vector.request.QueryResp​ +​ +QueryReq queryReq = QueryReq.builder()​ + .collectionName("my_collection")​ + .filter("color not in [\"red_7025\",\"red_4794\",\"red_9392\"]")​ + .outputFields(Arrays.asList("id", "color", "price"))​ + .build();​ +​ +QueryResp getResp = client.query(queryReq);​ +​ +// Output​ +// {color=pink_8682, price=593.0, id=1}​ +// {color=orange_6781, price=862.0, id=3}​ +// {color=pink_9298, price=991.0, id=4}​ +// {color=yellow_4222, price=996.0, id=6}​ +// {color=grey_8510, price=241.0, id=8}​ +// {color=white_9381, price=597.0, id=9}​ +// {color=purple_4976, price=450.0, id=10}​ + +``` + +```javascript +import { MilvusClient, DataType } from "@zilliz/milvus2-sdk-node";​ +​ +var res = client.query({​ + collection_name="my_collection",​ + filter='color not in ["red_7025","red_4794","red_9392"]',​ + output_fields=["id", "color", "price"]​ +})​ + +``` + +```curl +export CLUSTER_ENDPOINT="http://localhost:19530"​ +export TOKEN="root:Milvus"​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/query" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "my_collection",​ + "filter": "color not in [\"red_7025\",\"red_4794\",\"red_9392\"]",​ + "outputFields": ["id", "color", "price"]​ +}'​ + +``` + +The filtered results are as follows:​ + +```json +[​ + {"id": 1, "color": "pink_8682", "price":593},​ + {"id": 3, "color": "orange_6781", "price":863},​ + {"id": 4, "color": "pink_9298" "price":991},​ + {"id": 6, "color": "yellow_4222" "price":996},​ + {"id": 8, "color": "grey_8510" "price":241},​ + {"id": 9, "color": "white_9381" "price":597},​ + {"id": 10, "color": "purple_4976" "price":450}​ +]​ + +``` + +#### Example 2: Apply filter on JSON field​ + +The following example demonstrates how to filter products whose brand is Apple.​ + +
          + Python + Java + Node.js + cURL +
          + +```python +results = client.query(​ + collection_name="my_collection",​ + filter='inventory["brand"] in ["Apple"]',​ + output_fields=["id", "color", "price", "inventory"]​ +)​ +​ +# Output​ +# {'id': 5, 'color': 'red_4794', 'price': np.float32(327.0), 'inventory': {'brand': 'Apple', 'quantity': 193}}​ +# {'id': 7, 'color': 'red_9392', 'price': np.float32(848.0), 'inventory': {'brand': 'Apple', 'quantity': 61}}​ +# {'id': 9, 'color': 'white_9381', 'price': np.float32(597.0), 'inventory': {'brand': 'Apple', 'quantity': 351}}​ +# {'id': 10, 'color': 'purple_4976', 'price': np.float32(450.0), 'inventory': {'brand': 'Apple', 'quantity': 268}}} ​ + +``` + +```java +import io.milvus.v2.service.vector.request.QueryReq​ +import io.milvus.v2.service.vector.request.QueryResp​ +​ +QueryReq queryReq = QueryReq.builder()​ + .collectionName("my_collection")​ + .filter("inventory[\"brand\"] in [\"Apple\"]")​ + .outputFields(Arrays.asList("id", "color", "price", "inventory"))​ + .build();​ +​ +QueryResp getResp = client.query(queryReq);​ +​ +// Output​ +// {'id': 5, 'color': 'red_4794', 'price': np.float32(327.0), 'inventory': {'brand': 'Apple', 'quantity': 193}}​ +// {'id': 7, 'color': 'red_9392', 'price': np.float32(848.0), 'inventory': {'brand': 'Apple', 'quantity': 61}}​ +// {'id': 9, 'color': 'white_9381', 'price': np.float32(597.0), 'inventory': {'brand': 'Apple', 'quantity': 351}}​ +// {'id': 10, 'color': 'purple_4976', 'price': np.float32(450.0), 'inventory': {'brand': 'Apple', 'quantity': 268}}​ + +``` + +```javascript +import { MilvusClient, DataType } from "@zilliz/milvus2-sdk-node";​ +​ +var res = client.query({​ + collection_name="my_collection",​ + filter='inventory["brand"] in ["Apple"]',​ + output_fields=["id", "color", "price", "inventory"]​ +})​ + +``` + +```curl +export CLUSTER_ENDPOINT="http://localhost:19530"​ +export TOKEN="root:Milvus"​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/query" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "my_collection",​ + "filter": "inventory[\"brand\"] in [\"Apple\"]",​ + "outputFields": ["id", "color","price", "inventory"]​ +}'​ + +``` + +The filtered results are as follows:​ + +```JSON +[​ + {​ + "id": 5,​ + "color": "red_4794",​ + "price": 327,​ + "inventory": {​ + "brand": "Apple",​ + "quantity": 193​ + }​ + },​ + {​ + "id": 7,​ + "color": "red_9392",​ + "price": 848,​ + "inventory": {​ + "brand": "Apple",​ + "quantity": 61​ + }​ + },​ + {​ + "id": 9,​ + "color": "white_9381",​ + "price": 597,​ + "inventory": {​ + "brand": "Apple",​ + "quantity": 351​ + }​ + },​ + {​ + "id": 10,​ + "color": "purple_4976",​ + "price": 450,​ + "inventory": {​ + "brand": "Apple",​ + "quantity": 268​ + }​ + }​ +]​ + +``` + +### Match operators​ + +Match operators include:​ + +- `like`: Match constants or prefixes (prefix%), infixes (%infix%), and suffixes (%suffix) within constants. It relies on a brute-force search mechanism using wildcards and does not involve text tokenization. While it can achieve exact matches, its query efficiency is relatively low, making it suitable for simple matching tasks or queries on smaller datasets.​ + +- `TEXT_MATCH`: Match specific terms or keywords on VARCHAR fields, using tokenization and inverted index to enable efficient text search. Compared to `like`, `TEXT_MATCH` offers more advanced text tokenization and filtering capabilities. It is suited for large-scale datasets where higher query performance is required for complex text search scenarios.​ +
          + + To use the `TEXT_MATCH` filter expression, you must enable text matching for the target `VARCHAR` field when creating the collection. For details, refer to [​Keyword Match](keyword-match.md).​ + +
          + +#### Example 1: Apply filter on scalar field​ + +The following example demonstrates how to filter products whose color is red. In this case, you can quickly filter all red products by matching the prefix 'red%'. Similarly, you can use the expression color in ['red_7025', 'red_4794', 'red_9392'] to filter all red products. However, when the data is more complex, we recommend using the like operator for more efficient filtering.​ + +
          + Python + Java + Node.js + cURL +
          + +```python +results = client.query(​ + collection_name="my_collection",​ + filter='color like "red%"',​ + output_fields=["id", "color", "price"]​ +)​ +​ +# Output​ +# {'id': 2, 'color': 'red_7025', 'price': np.float32(196.0)}​ +# {'id': 5, 'color': 'red_4794', 'price': np.float32(327.0)}​ +# {'id': 7, 'color': 'red_9392', 'price': np.float32(848.0)}​ + +``` + +```java +import io.milvus.v2.service.vector.request.QueryReq​ +import io.milvus.v2.service.vector.request.QueryResp​ +​ +QueryReq queryReq = QueryReq.builder()​ + .collectionName("my_collection")​ + .filter("color like \"red%\"")​ + .outputFields(Arrays.asList("id", "color", "price"))​ + .build();​ +​ +QueryResp getResp = client.query(queryReq);​ +​ +// Output​ +// {color=red_7025, price=196.0, id=2}​ +// {color=red_4794, price=327.0, id=5}​ +// {color=red_9392, price=848.0, id=7}​ + +``` + +```javascript +import { MilvusClient, DataType } from "@zilliz/milvus2-sdk-node";​ +​ +var res = client.query({​ + collection_name="my_collection",​ + filter='color like "red%"',​ + output_fields=["id", "color", "price"]​ +})​ + +``` + +```curl +export CLUSTER_ENDPOINT="http://localhost:19530"​ +export TOKEN="root:Milvus"​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/query" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "my_collection",​ + "filter": "color like \"red%\"",​ + "outputFields": ["id", "color", "price"]​ +}'​ + +``` + +The filtered results are as follows:​ + +```json +[​ + {"id": 2, "color": "red_7025", "price":196},​ + {"id": 5, "color": "red_4794" "price":327},​ + {"id": 7, "color": "red_9392" "price":848}​ +]​ + +``` + +#### Example 2: Apply filter on JSON field​ + +The following example demonstrates how to filter products whose brand name starts with the letter 'S'.​ + +```python +results = client.query(​ + collection_name="my_collection",​ + filter='inventory["brand"] like "S%"',​ + output_fields=["id", "color", "price", "inventory"]​ +)​ +​ +# Output​ +# {'id': 1, 'color': 'pink_8682', 'price': np.float32(593.0), 'inventory': {'brand': 'Sony', 'quantity': 310}}​ +# {'id': 2, 'color': 'red_7025', 'price': np.float32(196.0), 'inventory': {'brand': 'Samsung', 'quantity': 274}}​ +# {'id': 3, 'color': 'orange_6781', 'price': np.float32(862.0), 'inventory': {'brand': 'Samsung', 'quantity': 103}}​ + +``` + +```java +import io.milvus.v2.service.vector.request.QueryReq​ +import io.milvus.v2.service.vector.request.QueryResp​ +​ +QueryReq queryReq = QueryReq.builder()​ + .collectionName("my_collection")​ + .filter("inventory[\"brand\"] like \"S%\"")​ + .outputFields(Arrays.asList("id", "color", "price", "inventory"))​ + .build();​ +​ +QueryResp getResp = client.query(queryReq);​ +​ +// Output​ +// {color=pink_8682, price=593.0, id=1, inventory={"brand":"Sony","quantity":310}}​ +// {color=red_7025, price=196.0, id=2, inventory={"brand":"Samsung","quantity":274}}​ +// {color=orange_6781, price=862.0, id=3, inventory={"brand":"Samsung","quantity":103}}​ + +``` + +```javascript +import { MilvusClient, DataType } from "@zilliz/milvus2-sdk-node";​ +​ +var res = client.query({​ + collection_name="my_collection",​ + filter='inventory["brand"] like "S%"',​ + output_fields=["id", "color", "price", "inventory"]​ +})​ + +``` + +```curl +export CLUSTER_ENDPOINT="http://localhost:19530"​ +export TOKEN="root:Milvus"​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/query" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "my_collection",​ + "filter": "inventory[\"brand\"] like \"S%\"",​ + "outputFields": ["id", "color", "price", "inventory"]​ +}'​ + +``` + +The filtered results are as follows:​ + +```json +[​ + {​ + "id": 1,​ + "color": "pink_8682",​ + "price": 593,​ + "inventory": {​ + "brand": "Sony",​ + "quantity": 310​ + }​ + },​ + {​ + "id": 2,​ + "color": "red_7025",​ + "price": 196,​ + "inventory": {​ + "brand": "Samsung",​ + "quantity": 274​ + }​ + },​ + {​ + "id": 3,​ + "color": "orange_6781",​ + "price": 862,​ + "inventory": {​ + "brand": "Samsung",​ + "quantity": 103​ + }​ + }​ +]​ + +``` + +#### Example 3: Keyword match on VARCHAR fields​ + +The `TEXT_MATCH` expression is used for keyword match on `VARCHAR` fields. By default, it applies an **OR** logic, but you can combine it with other logical operators to create more complex query conditions. For details, refer to [​Keyword Match](keyword-match.md).​ + +The following example demonstrates how to use the `TEXT_MATCH` expression to filter products where the `description` field contains either the keyword `"Apple"` or `"iPhone"`:​ + +
          + Python + Java + Node.js + cURL +
          + +```python +results = client.query(​ + collection_name="my_collection",​ + filter='TEXT_MATCH(description, "Apple iPhone")',​ + output_fields=["id", "description"],​ +)​ + +``` + +```java +QueryReq queryReq = QueryReq.builder()​ + .collectionName("my_collection")​ + .filter("TEXT_MATCH(description, \"Apple iPhone\")")​ + .outputFields(Arrays.asList("id", "description"))​ + .build();​ +​ +QueryResp getResp = client.query(queryReq);​ + +``` + +```javascript +import { MilvusClient, DataType } from "@zilliz/milvus2-sdk-node";​ +​ +const results = client.query({​ + collection_name: "my_collection",​ + filter: "TEXT_MATCH(description, 'Apple iPhone')",​ + output_fields: ["id", "description"]​ +});​ + +``` + +The filtered results are as follows:​ + +```JSON +[​ + {'id': 5, 'description': "iPhone 15 Pro, A new chip designed for better gaming and other 'pro' features."}​ + {'id': 7, 'description': "The iPhone 14 is a smartphone from Apple Inc. that comes in various colors and sizes."}​ + {'id': 9, 'description': "The iPhone 16 features a 6.1-inch OLED display, is powered by Apple's A18 processor, and has dual cameras at the back."}​ + {'id': 10, 'description': "The iPad is a brand of iOS- and iPadOS-based tablet computers that are developed and marketed by Apple."}​ +]​ + +``` + +To filter for descriptions containing multiple keywords simultaneously, you can use the `and` operator. The following example demonstrates how to filter products where the `description` field contains both `"chip"` and `"iPhone"`:​ + +```python +results = client.query(​ + collection_name="my_collection",​ + filter='TEXT_MATCH(description, "chip") and TEXT_MATCH(description, "iPhone")',​ + output_fields=["id", "description"],​ +)​ + +``` + +```java +QueryReq queryReq = QueryReq.builder()​ + .collectionName("my_collection")​ + .filter("TEXT_MATCH(description, \"chip\") and TEXT_MATCH(description, \"iPhone\")")​ + .outputFields(Arrays.asList("id", "description"))​ + .build();​ +​ +QueryResp getResp = client.query(queryReq);​ + +``` + +```javascript +import { MilvusClient, DataType } from "@zilliz/milvus2-sdk-node";​ +​ +const results = client.query({​ + collection_name: "my_collection",​ + filter: "TEXT_MATCH(description, 'chip') and TEXT_MATCH(description, 'iPhone')",​ + output_fields: ["id", "description"]​ +});​ + +``` + +The filtered results are as follows:​ + +```python +[​ + {'id': 5, 'description': "iPhone 15 Pro, A new chip designed for better gaming and other 'pro' features."}​ +]​ + +``` + +### Arithmetic operators​ + +Arithmetic operators include:​ + +- `+`: Addition​ + +- `-`: Subtraction​ + +- `*`: Multiplication​ + +- `/`: Division​ + +- `**`: Power​ + +- `%`: Modulo​ + +#### Example 1: Apply filter on scalar field​ + +The following example demonstrates how to filter products whose price, after a 50% discount, is between 200 and 300 (both inclusive).​ + +
          + Python + Java + Node.js + cURL +
          + +```python +results = client.query(​ + collection_name="my_collection",​ + filter="200 <= price*0.5 and price*0.5 <= 300",​ + output_fields=["id", "price"]​ +)​ +​ +# Output​ +# {'id': 1, 'price': np.float32(593.0)}​ +# {'id': 9, 'price': np.float32(597.0)}​ +# {'id': 10, 'price': np.float32(450.0)}​ + +``` + +```java +import io.milvus.v2.service.vector.request.QueryReq​ +import io.milvus.v2.service.vector.request.QueryResp​ +​ +ueryReq queryReq = QueryReq.builder()​ + .collectionName("my_collection")​ + .filter("200 <= price*0.5 and price*0.5 <= 300")​ + .outputFields(Arrays.asList("id", "price"))​ + .build();​ +​ +QueryResp getResp = client.query(queryReq);​ +​ +// Output​ +// {price=593.0, id=1}​ +// {price=597.0, id=9}​ +// {price=450.0, id=10}​ + +``` + +```javascript +import { MilvusClient, DataType } from "@zilliz/milvus2-sdk-node";​ +​ +var res = client.query({​ + collection_name="my_collection",​ + filter="200 <= price*0.5 <= 300",​ + output_fields=["id", "price"]​ +})​ + +``` + +```curl +export CLUSTER_ENDPOINT="http://localhost:19530"​ +export TOKEN="root:Milvus"​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/query" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "my_collection",​ + "filter": "200 <= price*0.5 <= 300",​ + "outputFields": ["id", "price"]​ +}'​ + +``` + +The filtered results are as follows:​ + +```json +[​ + {"id": 1, "color": "pink_8682", "price":593},​ + {"id": 9, "color": "white_9381", "price":597},​ + {"id": 10, "color": "purple_4976" "price":450}​ +]​ + +``` + +#### Example 2: Apply filter on JSON field​ + +The following example demonstrates how to filter products whose inventory, when doubled, exceeds 600 items.​ + +
          + Python + Java + Node.js + cURL +
          + +```python +results = client.query(​ + collection_name="my_collection",​ + filter='inventory["quantity"] * 2 > 600',​ + output_fields=["id", "color", "price", "inventory"]​ +)​ +​ +# Output​ +# {'id': 1, 'color': 'pink_8682', 'price': np.float32(593.0), 'inventory': {'brand': 'Sony', 'quantity': 310}}​ +# {'id': 6, 'color': 'yellow_4222', 'price': np.float32(996.0), 'inventory': {'brand': 'Microsoft', 'quantity': 376}}​ +# {'id': 9, 'color': 'white_9381', 'price': np.float32(597.0), 'inventory': {'brand': 'Apple', 'quantity': 351}}​ + +``` + +```java +import io.milvus.v2.service.vector.request.QueryReq​ +import io.milvus.v2.service.vector.request.QueryResp​ +​ +QueryReq queryReq = QueryReq.builder()​ + .collectionName("my_collection")​ + .filter("inventory[\"quantity\"] * 2 > 600")​ + .outputFields(Arrays.asList("id", "color", "price", "inventory"))​ + .build();​ +​ +QueryResp getResp = client.query(queryReq);​ +​ +// Output​ +// {color=pink_8682, price=593.0, id=1, inventory={"brand":"Sony","quantity":310}}​ +// {color=yellow_4222, price=996.0, id=6, inventory={"brand":"Microsoft","quantity":376}}​ +// {color=white_9381, price=597.0, id=9, inventory={"brand":"Apple","quantity":351}}​ + +``` + +```javascript +import { MilvusClient, DataType } from "@zilliz/milvus2-sdk-node";​ +​ +var res = client.query({​ + collection_name="my_collection",​ + filter='inventory["quantity"] * 2 > 600',​ + output_fields=["id", "color", "price", "inventory"]​ +})​ + +``` + +```curl +export CLUSTER_ENDPOINT="http://localhost:19530"​ +export TOKEN="root:Milvus"​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/query" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "my_collection",​ + "filter": "inventory[\"quantity\"] * 2 > 600",​ + "outputFields": ["id", "color", "price", "inventory"]​ +}'​ + +``` + +The filtered results are as follows:​ + +```JSON +[​ + {​ + "id": 1,​ + "color": "pink_8682",​ + "price": 593,​ + "inventory": {​ + "brand": "Sony",​ + "quantity": 310​ + }​ + },​ + {​ + "id": 6,​ + "color": "yellow_4222",​ + "price": 996,​ + "inventory": {​ + "brand": "Microsoft",​ + "quantity": 376​ + }​ + },​ + {​ + "id": 9,​ + "color": "white_9381",​ + "price": 597,​ + "inventory": {​ + "brand": "Apple",​ + "quantity": 351​ + }​ + }​ +]​ + +``` + +#### Example 3: Apply filter on Array field​ + +The following example demonstrates how to filter products whose combined sales in the first and second countries exceed 300.​ + +
          + Python + Java + Node.js + cURL +
          + +```python +results = client.query(​ + collection_name="my_collection",​ + filter="sales_volume[0]*2 > 300",​ + output_fields=["id", "color", "price", "sales_volume"]​ +)​ +​ +# Output​ +# {'color': 'pink_8682', 'price': np.float32(593.0), 'sales_volume': [161, 81, 51], 'id': 1}​ +# {'color': 'red_4794', 'price': np.float32(327.0), 'sales_volume': [155, 161, 106, 86, 99], 'id': 5}​ +# {'color': 'yellow_4222', 'price': np.float32(996.0), 'sales_volume': [173, 151, 148], 'id': 6}​ +# {'color': 'purple_4976', 'price': np.float32(450.0), 'sales_volume': [190, 149, 85, 79, 80], 'id': 10}​ + +``` + +```java +import io.milvus.v2.service.vector.request.QueryReq​ +import io.milvus.v2.service.vector.request.QueryResp​ +​ +QueryReq queryReq = QueryReq.builder()​ + .collectionName("my_collection")​ + .filter("sales_volume[0]*2 > 300")​ + .outputFields(Arrays.asList("id", "color", "price", "sales_volume"))​ + .build();​ +​ +QueryResp getResp = client.query(queryReq);​ +​ +// Output​ +// {color=pink_8682, price=593.0, id=1, sales_volume=[161, 81, 51]}​ +// {color=red_4794, price=327.0, id=5, sales_volume=[155, 161, 106, 86, 99]}​ +// {color=yellow_4222, price=996.0, id=6, sales_volume=[173, 151, 148]}​ +// {color=purple_4976, price=450.0, id=10, sales_volume=[190, 149, 85, 79, 80]}​ + +``` + +```javascript +import { MilvusClient, DataType } from "@zilliz/milvus2-sdk-node";​ +​ +var res = client.query({​ + collection_name="my_collection",​ + filter="sales_volume[0]*2 > 300",​ + output_fields=["id", "color","price", "sales_volume"]​ +})​ + +``` + +```curl +export CLUSTER_ENDPOINT="http://localhost:19530"​ +export TOKEN="root:Milvus"​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/query" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "my_collection",​ + "filter": "sales_volume[0]*2 > 300",​ + "outputFields": ["id", "color", "price", "sales_volume"]​ +}'​ + +``` + +The filtered results are as follows:​ + +```JSON +[​ + {​ + "id": 5,​ + "color": "red_4794",​ + "price": 327,​ + "sales_volume": [​ + 155,​ + 161,​ + 106,​ + 86,​ + 99​ + ]​ + },​ + {​ + "id": 6,​ + "color": "yellow_4222",​ + "price": 996,​ + "sales_volume": [​ + 173,​ + 151,​ + 148​ + ]​ + },​ + {​ + "id": 10,​ + "color": "purple_4976",​ + "price": 450,​ + "sales_volume": [​ + 190,​ + 149,​ + 85,​ + 79,​ + 80​ + ]​ + }​ +]​ + +``` + +### Advanced JSON operators​ + +JSON operators include: ​ + +- `JSON_CONTAINS`: Filter entities whose JSON field contains elements from a specific list.​ + +- `JSON_CONTAINS_ALL`: Filter entities whose JSON field contains all elements from a specific list in the same order.​ + +- `JSON_CONTAINS_ANY`: Filter all entities whose JSON field contains any one element from a specific list.​ + +
          + +When using JSON opertors, the JSON field must contain at least one key whose value is a list.​ + +
          + +To demonstrate how to use advanced filtering operators on JSON fields, we make a slight adjustment to the example dataset in this section. A new key named 'previous_sales' has been added to the JSON 'inventory' field, which represents the previous sales of the product in three countries. The value of this key is a list of numbers. Below is the modified new example dataset:​ + +#### Example 1: `JSON_CONTAINS`​ + +`JSON_CONTAINS(identifier, JsonExpr)`：`identifier` is the key name in the JSON field and `JsonExpr` is the list of filtering conditions.​ + +The following example demonstrates how to filter products that previously had sales of 232 items in a specific country.​ + +
          + Python + Java + Node.js + cURL +
          + +```python +results = client.query(​ + collection_name="my_collection",​ + filter='JSON_CONTAINS(inventory[\"previous_sales\"], 232)',​ + output_fields=["id", "color", "price", "inventory"]​ +)​ +​ +# Output​ +# {'inventory': {'brand': 'Samsung', 'quantity': 103, 'previous_sales': [232, 254, 275]}, 'id': 3, 'color': 'orange_6781', 'price': np.float32(862.0)}​ +# {'inventory': {'brand': 'Microsoft', 'quantity': 376, 'previous_sales': [254, 275, 232]}, 'id': 6, 'color': 'yellow_4222', 'price': np.float32(996.0)}​ + +``` + +```java +import io.milvus.v2.service.vector.request.QueryReq​ +import io.milvus.v2.service.vector.request.QueryResp​ +​ +QueryReq queryReq = QueryReq.builder()​ + .collectionName("my_collection")​ + .filter("JSON_CONTAINS(inventory[\"previous_sales\"], 232)")​ + .outputFields(Arrays.asList("id", "color", "price", "inventory"))​ + .build();​ +​ +QueryResp getResp = client.query(queryReq);​ +​ +// Output​ +// {color=orange_6781, price=862.0, id=3, inventory={"brand":"Samsung","quantity":103,"previous_sales":[232,254,275]}}​ +// {color=yellow_4222, price=996.0, id=6, inventory={"brand":"Microsoft","quantity":376,"previous_sales":[254,275,232]}}​ + +``` + +```javascript +import { MilvusClient, DataType } from "@zilliz/milvus2-sdk-node";​ +​ +var res = client.query({​ + collection_name="my_collection",​ + filter='JSON_CONTAINS(inventory[\"previous_sales\"], 232)',​ + output_fields=["id", "color","price", "inventory"]​ +})​ + +``` + +```curl +export CLUSTER_ENDPOINT="http://localhost:19530"​ +export TOKEN="root:Milvus"​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/query" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "my_collection",​ + "filter": "JSON_CONTAINS(inventory[\"previous_sales\"], 232)",​ + "outputFields": ["id", "color", "price", "inventory"]​ +}'​ + +``` + +The filtered results are as follows:​ + +```JSON +[​ + {​ + "id": 3,​ + "color": "orange_6781",​ + "price": 862,​ + "inventory": {​ + "brand": "Samsung",​ + "quantity": 103,​ + "previous_sales": [​ + 232,​ + 254,​ + 275​ + ]​ + }​ + }​ +]​ + +``` + +#### Example 2: `JSON_CONTAINS_ALL`​ + +`JSON_CONTAINS_ALL(identifier, JsonExpr)`：`identifier` is the key name in the JSON field and `JsonExpr` is the list of filtering conditions.​ + +The following example demonstrates how to filter products that had previous sales of 232, 254, and 275 items in three different countries.​ + +
          + Python + Java + Node.js + cURL +
          + +```python +results = client.query(​ + collection_name="my_collection",​ + filter='JSON_CONTAINS_ALL(inventory["previous_sales"], [232, 254, 275])',​ + output_fields=["id", "color", "price", "inventory"]​ +)​ +​ +# Output​ +# {'id': 3, 'color': 'orange_6781', 'price': np.float32(862.0), 'inventory': {'brand': 'Samsung', 'quantity': 103, 'previous_sales': [232, 254, 275]}}​ +# {'id': 6, 'color': 'yellow_4222', 'price': np.float32(996.0), 'inventory': {'brand': 'Microsoft', 'quantity': 376, 'previous_sales': [254, 275, 232]}}​ + +``` + +```java +import io.milvus.v2.service.vector.request.QueryReq​ +import io.milvus.v2.service.vector.request.QueryResp​ +​ +QueryReq queryReq = QueryReq.builder()​ + .collectionName("my_collection")​ + .filter("JSON_CONTAINS_ALL(inventory[\"previous_sales\"], [232, 254, 275])")​ + .outputFields(Arrays.asList("id", "color", "price", "inventory"))​ + .build();​ +​ +QueryResp getResp = client.query(queryReq);​ +​ +// Output​ +// {color=orange_6781, price=862.0, id=3, inventory={"brand":"Samsung","quantity":103,"previous_sales":[232,254,275]}}​ +// {color=yellow_4222, price=996.0, id=6, inventory={"brand":"Microsoft","quantity":376,"previous_sales":[254,275,232]}}​ + +``` + +```javascript +const { MilvusClient, DataType } = require("@zilliz/milvus2-sdk-node")​ +​ +var res = client.query({​ + collection_name="my_collection",​ + filter='JSON_CONTAINS_ALL(inventory["previous_sales"], [232, 254, 275])',​ + output_fields=["id", "color","price", "inventory"]​ +})​ + +``` + +```curl +export CLUSTER_ENDPOINT="http://localhost:19530"​ +export TOKEN="root:Milvus"​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/query" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "my_collection",​ + "filter": "JSON_CONTAINS_ALL(inventory[\"previous_sales\"], [232, 254, 275])",​ + "outputFields": ["id", "color", "price", "inventory"]​ +}'​ + +``` + +The filtered results are as follows:​ + +```JSON +[​ + {​ + "id": 3,​ + "color": "orange_6781",​ + "price": 862,​ + "inventory": {​ + "brand": "Samsung",​ + "quantity": 103,​ + "previous_sales": [​ + 232,​ + 254,​ + 275​ + ]​ + }​ + },​ + {​ + "id": 6,​ + "color": "yellow_4222",​ + "price": 996,​ + "inventory": {​ + "brand": "Microsoft",​ + "quantity": 376,​ + "previous_sales": [​ + 254,​ + 275,​ + 232​ + ]​ + }​ + }​ +]​ + +``` + +#### Example 3: `JSON_CONTAINS_ANY`​ + +`JSON_CONTAINS_ANY(identifier, JsonExpr)`：`identifier` is the key name in the JSON field and `JsonExpr` is the list of filtering conditions.​ + +The following example demonstrates how to filter products that had previous sales of either 232, 254, or 275 items in any one of the three countries.​ + +
          + Python + Java + Node.js + cURL +
          + +```python +results = client.query(​ + collection_name="my_collection",​ + filter='JSON_CONTAINS_ANY(inventory["previous_sales"], [232, 254, 275])',​ + output_fields=["id", "color", "price", "inventory"]​ +)​ +​ +# Output​ +# {'id': 3, 'color': 'orange_6781', 'price': np.float32(862.0), 'inventory': {'brand': 'Samsung', 'quantity': 103, 'previous_sales': [232, 254, 275]}}​ +# {'id': 6, 'color': 'yellow_4222', 'price': np.float32(996.0), 'inventory': {'brand': 'Microsoft', 'quantity': 376, 'previous_sales': [254, 275, 232]}}​ +# {'id': 7, 'color': 'red_9392', 'price': np.float32(848.0), 'inventory': {'brand': 'Apple', 'quantity': 61, 'previous_sales': [312, 254, 367]}}​ + +``` + +```java +import io.milvus.v2.service.vector.request.QueryReq​ +import io.milvus.v2.service.vector.request.QueryResp​ +​ +QueryReq queryReq = QueryReq.builder()​ + .collectionName("my_collection")​ + .filter("JSON_CONTAINS_ANY(inventory[\"previous_sales\"], [232, 254, 275])")​ + .outputFields(Arrays.asList("id", "color", "price", "inventory"))​ + .build();​ +​ +QueryResp getResp = client.query(queryReq);​ +​ +// Output​ +// {color=orange_6781, price=862.0, id=3, inventory={"brand":"Samsung","quantity":103,"previous_sales":[232,254,275]}}​ +// {color=yellow_4222, price=996.0, id=6, inventory={"brand":"Microsoft","quantity":376,"previous_sales":[254,275,232]}}​ +// {color=red_9392, price=848.0, id=7, inventory={"brand":"Apple","quantity":61,"previous_sales":[312,254,367]}}​ + +``` + +```javascript +const { MilvusClient, DataType } = require("@zilliz/milvus2-sdk-node")​ +​ +var res = client.query({​ + collection_name="my_collection",​ + filter='JSON_CONTAINS_ANY(inventory["previous_sales"], [232, 254, 275])',​ + output_fields=["id", "color","price", "inventory"]​ +})​ + +``` + +```curl +export CLUSTER_ENDPOINT="http://localhost:19530"​ +export TOKEN="root:Milvus"​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/query" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "my_collection",​ + "filter": "JSON_CONTAINS_ANY(inventory[\"previous_sales\"], [232, 254, 275])",​ + "outputFields": ["id", "color", "price", "inventory"]​ +}'​ + +``` + +The filtered results are as follows:​ + +```JSON +[​ + {​ + "id": 3,​ + "color": "orange_6781",​ + "price": 862,​ + "inventory": {​ + "brand": "Samsung",​ + "quantity": 103,​ + "previous_sales": [​ + 232,​ + 254,​ + 275​ + ]​ + }​ + },​ + {​ + "id": 6,​ + "color": "yellow_4222",​ + "price": 996,​ + "inventory": {​ + "brand": "Microsoft",​ + "quantity": 376,​ + "previous_sales": [​ + 254,​ + 275,​ + 232​ + ]​ + }​ + },​ + {​ + "id": 7,​ + "color": "red_9392",​ + "price": 848,​ + "inventory": {​ + "brand": "Apple",​ + "quantity": 61,​ + "previous_sales": [​ + 312,​ + 254,​ + 367​ + ]​ + }​ + }​ +]​ + +``` + +### Advanced Array operators​ + +Array operators include:​ + +- `ARRAY_CONTAINS`: Filter all entities whose Array field contains a specific element.​ + +- `ARRAY_CONTAINS_ALL`: Filter all entities whose Array field contains all specified elements.​ + +- `ARRAY_CONTAINS_ANY`: Filter all entities whose Array field contains any one of the specified elements.​ + +- `ARRAY_LENGTH`: Check the number of elements in the list.​ + +#### Example 1: `ARRAY_CONTAINS` ​ + +`ARRAY_CONTAINS(identifier, ArrayExpr)`：`identifier` is the name of the Array field, and `ArrayExpr` is the array of filtering conditions.​ + +The following example demonstrates how to filter products with current sales of 161 items in a specific country.​ + +
          + Python + Java + Node.js + cURL +
          + +```python +results = client.query(​ + collection_name="my_collection",​ + filter='ARRAY_CONTAINS(sales_volume, 161)',​ + output_fields=["id", "color", "price", "sales_volume"]​ +)​ +​ +# Output​ +# {'id': 1, 'color': 'pink_8682', 'price': np.float32(593.0), 'sales_volume': [161, 81, 51]}​ +# {'id': 5, 'color': 'red_4794', 'price': np.float32(327.0), 'sales_volume': [155, 161, 106, 86, 99]}​ + +``` + +```java +import io.milvus.v2.service.vector.request.QueryReq​ +import io.milvus.v2.service.vector.request.QueryResp​ +​ +QueryReq queryReq = QueryReq.builder()​ + .collectionName("my_collection")​ + .filter("ARRAY_CONTAINS(sales_volume, 161)")​ + .outputFields(Arrays.asList("id", "color", "price", "sales_volume"))​ + .build();​ +​ +QueryResp getResp = client.query(queryReq);​ +​ +// Output​ +// {color=pink_8682, price=593.0, id=1, sales_volume=[161, 81, 51]}​ +// {color=red_4794, price=327.0, id=5, sales_volume=[155, 161, 106, 86, 99]}​ + +``` + +```javascript +import { MilvusClient, DataType } from "@zilliz/milvus2-sdk-node";​ +​ +var res = client.query({​ + collection_name="my_collection",​ + filter="ARRAY_CONTAINS(sales_volume, 161)",​ + output_fields=["id", "color","price", "sales_volume"]​ +})​ + +``` + +```curl +export CLUSTER_ENDPOINT="http://localhost:19530"​ +export TOKEN="root:Milvus"​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/query" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "my_collection",​ + "filter": "ARRAY_CONTAINS(sales_volume, 161)",​ + "outputFields": ["id", "color", "price", "sales_volume"]​ +}'​ + +``` + +The filtered results are as follows:​ + +```JSON +[​ + {​ + "id": 1,​ + "color": "pink_8682",​ + "price": 593,​ + "sales_volume": [​ + 161,​ + 81,​ + 51​ + ]​ + },​ + {​ + "id": 5,​ + "color": "red_4794",​ + "price": 327,​ + "sales_volume": [​ + 155,​ + 161,​ + 106,​ + 86,​ + 99​ + ]​ + }​ +]​ + +``` + +#### Example 2: `ARRAY_CONTAINS_ALL`​ + +`ARRAY_CONTAINS_ALL(identifier, ArrayExpr)`：`identifier` is the name of the Array field, and`ArrayExpr` is the array of filtering conditions. ​ + +The following example demonstrates how to filter products with current sales of 150 items in both the first and second countries.​ + +
          + Python + Java + Node.js + cURL +
          + +```python +results = client.query(​ + collection_name="my_collection",​ + filter='ARRAY_CONTAINS_ALL(sales_volume, [150, 150])',​ + output_fields=["id", "color", "price", "sales_volume"]​ +)​ +​ +# Output​ +# {'price': np.float32(597.0), 'sales_volume': [150, 150, 73], 'id': 9, 'color': 'white_9381'}​ + +``` + +```java +import io.milvus.v2.service.vector.request.QueryReq​ +import io.milvus.v2.service.vector.request.QueryResp​ +​ +QueryReq queryReq = QueryReq.builder()​ + .collectionName("my_collection")​ + .filter("ARRAY_CONTAINS_ALL(sales_volume, [150, 150])")​ + .outputFields(Arrays.asList("id", "color", "price", "sales_volume"))​ + .build();​ +​ +QueryResp getResp = client.query(queryReq);​ +​ +// Output​ +// {color=white_9381, price=597.0, id=9, sales_volume=[150, 150, 73]}​ + +``` + +```javascript +import { MilvusClient, DataType } from "@zilliz/milvus2-sdk-node";​ +​ +var res = client.query({​ + collection_name="my_collection",​ + filter="ARRAY_CONTAINS_ALL(sales_volume, [150, 150])",​ + output_fields=["id", "color","price", "sales_volume"]​ +})​ + +``` + +```curl +export CLUSTER_ENDPOINT="http://localhost:19530"​ +export TOKEN="root:Milvus"​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/query" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "my_collection",​ + "filter": "ARRAY_CONTAINS_ALL(sales_volume, [150, 150])",​ + "outputFields": ["id", "color", "price", "sales_volume"]​ +}'​ + +``` + +The filtered results are as follows:​ + +```JSON +[​ + {​ + "id": 9,​ + "color": "white_9381",​ + "price": 597,​ + "sales_volume": [​ + 150,​ + 150,​ + 73​ + ]​ + }​ +]​ + +``` + +#### Example 3: `ARRAY_CONTAINS_ANY`​ + +`ARRAY_CONTAINS_ANY(identifier, ArrayExpr)`：`identifier` is the name of the Array field, and `ArrayExpr` is the array of filtering conditions. ​ + +The following example demonstrates how to filter products with current sales of either 150, 190, or 90 items in any country.​ + +
          + Python + Java + Node.js + cURL +
          + +```python +results = client.query(​ + collection_name="my_collection",​ + filter='ARRAY_CONTAINS_ANY(sales_volume, [150, 190, 90])',​ + output_fields=["id", "color", "price", "sales_volume"]​ +)​ +​ +# Output​ +# {'id': 3, 'color': 'orange_6781', 'price': np.float32(862.0), 'sales_volume': [124, 117, 90, 188]}​ +# {'id': 9, 'color': 'white_9381', 'price': np.float32(597.0), 'sales_volume': [150, 150, 73]}​ +# {'id': 10, 'color': 'purple_4976', 'price': np.float32(450.0), 'sales_volume': [190, 149, 85, 79, 80]}​ + +``` + +```java +import io.milvus.v2.service.vector.request.QueryReq​ +import io.milvus.v2.service.vector.request.QueryResp​ +​ +QueryReq queryReq = QueryReq.builder()​ + .collectionName("my_collection")​ + .filter("ARRAY_CONTAINS_ANY(sales_volume, [150, 190, 90])")​ + .outputFields(Arrays.asList("id", "color", "price", "sales_volume"))​ + .build();​ +​ +QueryResp getResp = client.query(queryReq);​ +​ +// Output​ +// {color=orange_6781, price=862.0, id=3, sales_volume=[124, 117, 90, 188]}​ +// {color=white_9381, price=597.0, id=9, sales_volume=[150, 150, 73]}​ +// {color=purple_4976, price=450.0, id=10, sales_volume=[190, 149, 85, 79, 80]}​ + +``` + +```javascript +import { MilvusClient, DataType } from "@zilliz/milvus2-sdk-node";​ +​ +var res = client.query({​ + collection_name="my_collection",​ + filter="ARRAY_CONTAINS_ANY(sales_volume, [150, 190, 90])",​ + output_fields=["id", "color","price", "sales_volume"]​ +})​ + +``` + +```curl +export CLUSTER_ENDPOINT="http://localhost:19530"​ +export TOKEN="root:Milvus"​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/query" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "my_collection",​ + "filter": "ARRAY_CONTAINS_ANY(sales_volume, [150, 190, 90])",​ + "outputFields": ["id", "color", "price", "sales_volume"]​ +}'​ + +``` + +The filtered results are as follows:​ + +```JSON +[​ + {​ + "id": 3,​ + "color": "orange_6781",​ + "price": 862,​ + "sales_volume": [​ + 124,​ + 117,​ + 90,​ + 188​ + ]​ + },​ + {​ + "id": 9,​ + "color": "white_9381",​ + "price": 597,​ + "sales_volume": [​ + 150,​ + 150,​ + 73​ + ]​ + },​ + {​ + "id": 10,​ + "color": "purple_4976",​ + "price": 450,​ + "sales_volume": [​ + 190,​ + 149,​ + 85,​ + 79,​ + 80​ + ]​ + }​ +]​ + +``` + +#### Example 4: `ARRAY_LENGTH`​ + +The following example demonstrates how to filter products that are sold in only three countries.​ + +
          + Python + Java + Node.js + cURL +
          + +```python +results = client.query(​ + collection_name="my_collection",​ + filter='ARRAY_LENGTH(sales_volume) == 3',​ + output_fields=["id", "color", "price", "sales_volume"]​ +)​ +​ +# Output​ +# {'id': 1, 'color': 'pink_8682', 'price': np.float32(593.0), 'sales_volume': [161, 81, 51]}​ +# {'id': 6, 'color': 'yellow_4222', 'price': np.float32(996.0), 'sales_volume': [173, 151, 148]}​ +# {'id': 9, 'color': 'white_9381', 'price': np.float32(597.0), 'sales_volume': [150, 150, 73]}​ + +``` + +```java +import io.milvus.v2.service.vector.request.QueryReq​ +import io.milvus.v2.service.vector.request.QueryResp​ +​ +QueryReq queryReq = QueryReq.builder()​ + .collectionName("my_collection")​ + .filter("ARRAY_LENGTH(sales_volume) == 3")​ + .outputFields(Arrays.asList("id", "color", "price", "sales_volume"))​ + .build();​ +​ +QueryResp getResp = client.query(queryReq);​ +​ +// Output​ +// {color=pink_8682, price=593.0, id=1, sales_volume=[161, 81, 51]}​ +// {color=yellow_4222, price=996.0, id=6, sales_volume=[173, 151, 148]}​ +// {color=white_9381, price=597.0, id=9, sales_volume=[150, 150, 73]}​ + +``` + +```javascript +import { MilvusClient, DataType } from "@zilliz/milvus2-sdk-node";​ +​ +var res = client.query({​ + collection_name="my_collection",​ + filter="ARRAY_LENGTH(sales_volume) == 3",​ + output_fields=["id", "color","price", "sales_volume"]​ +})​ + +``` + +```curl +export CLUSTER_ENDPOINT="http://localhost:19530"​ +export TOKEN="root:Milvus"​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/query" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "my_collection",​ + "filter": "ARRAY_LENGTH(sales_volume) == 3",​ + "outputFields": ["id", "color", "price", "sales_volume"]​ +}'​ + +``` + +The filtered results are as follows:​ + +```JSON +[​ + {​ + "id": 1,​ + "color": "pink_8682",​ + "price": 593,​ + "sales_volume": [​ + 161,​ + 81,​ + 51​ + ]​ + },​ + {​ + "id": 6,​ + "color": "yellow_4222",​ + "price": 996,​ + "sales_volume": [​ + 173,​ + 151,​ + 148​ + ]​ + },​ + {​ + "id": 9,​ + "color": "white_9381",​ + "price": 597,​ + "sales_volume": [​ + 150,​ + 150,​ + 73​ + ]​ + }​ +]​ + +``` + +## Multi-condition filtering​ + +The logical operators that can be used to combine multiple filtering conditions include:​ + +- `and` or `&&`: Both conditions should be satisfied.​ + +- `or` or `||`: Only one of the two conditions should be satisfied.​ + +
          + +When a lower precedence operation should be processed first, it should be enclosed within parentheses. Innermost parenthetical expressions are evaluated first.​ + +
          + +#### Example​ + +The following example demonstrates how to filter products that are red in color, priced below 500, branded as Apple, and have sales over 100 items in the first country​ + +
          + Python + Java + Node.js + cURL +
          + +```python +results = client.query(​ + collection_name="my_collection",​ + filter='color like "red%" and price < 500 and inventory["brand"] in ["Apple"] and sales_volume[0] > 100',​ + output_fields=["id", "color", "price", "inventory", "sales_volume"]​ +)​ +​ +# Output​ +# {'id': 5, 'color': 'red_4794', 'price': np.float32(327.0), 'inventory': {'brand': 'Apple', 'quantity': 193, 'previous_sales': [225, 286, 202]}, 'sales_volume': [155, 161, 106, 86, 99]}​ + +``` + +```java +import io.milvus.v2.service.vector.request.QueryReq​ +import io.milvus.v2.service.vector.request.QueryResp​ +​ +QueryReq queryReq = QueryReq.builder()​ + .collectionName("my_collection")​ + .filter("color like \"red%\" and price < 500 and inventory[\"brand\"] in [\"Apple\"] and sales_volume[0] > 100")​ + .outputFields(Arrays.asList("id", "color", "price", "inventory", "sales_volume"))​ + .build();​ +​ +QueryResp getResp = client.query(queryReq);​ +​ +// Output​ +// {color=red_4794, price=327.0, id=5, inventory={"brand":"Apple","quantity":193,"previous_sales":[225,286,202]}, sales_volume=[155, 161, 106, 86, 99]}​ + +``` + +```javascript +import { MilvusClient, DataType } from "@zilliz/milvus2-sdk-node";​ +​ +var res = client.query({​ + collection_name="my_collection",​ + filter='color like "red%" and price < 500 and inventory["brand"] in ["Apple"] and sales_volume[0] > 100',​ + output_fields=["id", "color", "price", "inventory", "sales_volume"]​ +})​ + +``` + +```curl +export CLUSTER_ENDPOINT="http://localhost:19530"​ +export TOKEN="root:Milvus"​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/query" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "my_collection",​ + "filter": "color like \"red%\" and price < 500 and inventory[\"brand\"] in [\"Apple\"] and sales_volume[0] > 100",​ + "outputFields": ["id", "color", "price", "inventory", "sales_volume"]​ +}'​ + +``` + +The filtered results are as follows:​ + +```JSON +[​ + {​ + "id": 5,​ + "color": "red_4794",​ + "price": 327,​ + "inventory": {​ + "brand": "Apple",​ + "quantity": 193​ + },​ + "sales_volume": [​ + 155,​ + 161,​ + 106,​ + 86,​ + 99​ + ]​ + }​ +]​ + +``` + +## Operator precedence​ + +The following table lists the precedence of operators. Operators are listed top to bottom, in descending precedence.​ + +Expressions are normally evaluated from left to right. Complex expressions are evaluated one at a time. The order in which the expressions are evaluated is determined by the precedence of the operators used.​ + +If an expression contains two or more operators with the same precedence, the operator to the left is evaluated first. When a lower precedence operation should be processed first, it should be enclosed within parentheses.​ diff --git a/v2.4.x/site/en/userGuide/clustering-compaction.md b/v2.4.x/site/en/userGuide/search-query-get/clustering-compaction.md similarity index 100% rename from v2.4.x/site/en/userGuide/clustering-compaction.md rename to v2.4.x/site/en/userGuide/search-query-get/clustering-compaction.md diff --git a/v2.4.x/site/en/userGuide/search-query-get/consistency.md b/v2.4.x/site/en/userGuide/search-query-get/consistency.md new file mode 100644 index 0000000000..1b1029d3ad --- /dev/null +++ b/v2.4.x/site/en/userGuide/search-query-get/consistency.md @@ -0,0 +1,221 @@ +--- +id: consistency.md +summary: Learn about the four levels of consistency in Milvus. +title: Consistency +--- + +# Consistency Level​ + +As a distributed vector database, Milvus offers multiple levels of consistency to ensure that each node or replica can access the same data during read and write operations. Currently, the supported levels of consistency include **Strong**, **Bounded**, **Eventually**, and **Session**, with **Bounded** being the default level of consistency used.​ + +## Overview​{#overview​} + +Milvus is a system that separates storage and computation. In this system, **DataNodes** are responsible for the persistence of data and ultimately store it in distributed object storage such as MinIO/S3. **QueryNodes** handle computational tasks like Search. These tasks involve processing both **batch data** and **streaming data**. Simply put, batch data can be understood as data that has already been stored in object storage while streaming data refers to data that has not yet been stored in object storage. Due to network latency, QueryNodes often do not hold the most recent streaming data. Without additional safeguards, performing Search directly on streaming data may result in the loss of many uncommitted data points, affecting the accuracy of search results.​ + +Milvus Commercial Edition is a system that separates storage and computation. In this system, DataNodes are responsible for the persistence of data and ultimately store it in distributed object storage such as MinIO/S3. QueryNodes handle computational tasks like Search. These tasks involve processing both batch data and streaming data. Simply put, batch data can be understood as data that has already been stored in object storage, while streaming data refers to data that has not yet been stored in object storage. Due to network latency, QueryNodes often do not hold the most recent streaming data. Without additional safeguards, performing Search directly on streaming data may result in the loss of many uncommitted data points, affecting the accuracy of search results.​ + +![Batch data and streaming data](../../../../assets/batch-data-and-streaming-data.png) + +As shown in the figure above, QueryNodes can receive both streaming data and batch data simultaneously after receiving a Search request. However, due to network latency, the streaming data obtained by QueryNodes may be incomplete.​ + +To address this issue, Milvus timestamps each record in the data queue and continuously inserts synchronization timestamps into the data queue. Whenever a synchronization timestamp (syncTs) is received, QueryNodes sets it as the ServiceTime, meaning that QueryNodes can see all data prior to that Service Time. Based on the ServiceTime, Milvus can provide guarantee timestamps (GuaranteeTs) to meet different user requirements for consistency and availability. Users can inform QueryNodes of the need to include data prior to a specified point in time in the search scope by specifying GuaranteeTs in their Search requests.​ + +![ServiceTime and GuaranteeTs](../../../../assets/service-time-and-guarantee-time.png) + +As shown in the figure above, if GuaranteeTs is less than ServiceTime, it means that all data before the specified time point has been fully written to disk, allowing QueryNodes to immediately perform the Search operation. When GuaranteeTs is greater than ServiceTime, QueryNodes must wait until ServiceTime exceeds GuaranteeTs before they can execute the Search operation.​ + +Users need to make a trade-off between query accuracy and query latency. If users have high consistency requirements and are not sensitive to query latency, they can set GuaranteeTs to a value as large as possible; if users wish to receive search results quickly and are more tolerant of query accuracy, then GuaranteeTs can be set to a smaller value.​ + +![Consistency Levels Illustrated](../../../../assets/consistency-level-illustrated.png) + +Milvus provides four types of consistency levels with different GuaranteeTs.​ + +- **Strong**​ + + The latest timestamp is used as the GuaranteeTs, and QueryNodes have to wait until the ServiceTime meets the GuaranteeTs before executing Search requests.​ + +- **Eventual**​ + + The GuaranteeTs is set to an extremely small value, such as 1, to avoid consistency checks so that QueryNodes can immediately execute Search requests upon all batch data.​ + +- **Bounded Staleness**​ + + The GuranteeTs is set to a time point earlier than the latest timestamp to make QueryNodes to perform searches with a tolerance of certain data loss.​ + +- **Session**​ + + The latest time point at which the client inserts data is used as the GuaranteeTs so that QueryNodes can perform searches upon all the data inserted by the client.​ + +Milvus uses Bounded Staleness as the default consistency level. If the GuaranteeTs is left unspecified, the latest ServiceTime is used as the GuaranteeTs.​ + +## Set Consistency Level​{#set-consistency-level​} + +You can set different consistency levels when you create a collection as well as perform searches and queries.​ + +### Set Consistency Level upon Creating Collection​{#-set-consistency-level-upon-creating-collection​} + +When creating a collection, you can set the consistency level for the searches and queries within the collection. The following code example sets the consistency level to **Strong**.​ + +
          + python + java + cURL +
          + +```python +client.create_collection(​ + collection_name="my_collection",​ + schema=schema,​ + # highlight-next​ + consistency_level="Strong",​ +)​ + +``` + +```java +CreateCollectionReq createCollectionReq = CreateCollectionReq.builder()​ + .collectionName("my_collection")​ + .collectionSchema(schema)​ + // highlight-next​ + .consistencyLevel(ConsistencyLevel.STRONG)​ + .build();​ +client.createCollection(createCollectionReq);​ + +``` + +```bash +export schema='{​ + "autoId": true,​ + "enabledDynamicField": false,​ + "fields": [​ + {​ + "fieldName": "my_id",​ + "dataType": "Int64",​ + "isPrimary": true​ + },​ + {​ + "fieldName": "my_vector",​ + "dataType": "FloatVector",​ + "elementTypeParams": {​ + "dim": "5"​ + }​ + },​ + {​ + "fieldName": "my_varchar",​ + "dataType": "VarChar",​ + "isClusteringKey": true,​ + "elementTypeParams": {​ + "max_length": 512​ + }​ + }​ + ]​ + }'​ +​ +export params='{​ + "consistencyLevel": "Strong"​ +}'​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/collections/create" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d "{​ + \"collectionName\": \"my_collection\",​ + \"schema\": $schema,​ + \"params\": $params​ +}"​ + +``` + +Possible values for the `consistency_level` parameter are `Strong`, `Bounded`, `Eventually`, and `Session`.​ + +### Set Consistency Level in Search​{#set-consistency-level-in-search​} + +You can always change the consistency level for a specific search. The following code example sets the consistency level back to the Bounded. The change applies only to the current search request.​ + +
          + python + java + cURL +
          + +```python +res = client.search(​ + collection_name="my_collection",​ + data=[query_vector],​ + limit=3,​ + search_params={"metric_type": "IP"}，​ + # highlight-start​ + consistency_level="Bounded",​ + # highlight-next​ +)​ + +``` + +```java +SearchReq searchReq = SearchReq.builder()​ + .collectionName("my_collection")​ + .data(Collections.singletonList(queryVector))​ + .topK(3)​ + .searchParams(params)​ + .consistencyLevel(ConsistencyLevel.BOUNDED)​ + .build();​ +​ +SearchResp searchResp = client.search(searchReq);​ + +``` + +```bash +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/search" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "my_collection",​ + "data": [​ + [0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592]​ + ],​ + "limit": 3,​ + "consistencyLevel": "Bounded"​ +}'​ + +``` + +This parameter is also available in hybrid searches and the search iterator. Possible values for the `consistency_level` parameter are `Strong`, `Bounded`, `Eventually`, and `Session`.​ + +### Set Consistency Level in Query​{#set-consistency-level-in-query​} + +You can always change the consistency level for a specific search. The following code example sets the consistency level to the **Eventually**. The setting applies only to the current query request.​ + +
          + python + java +
          + +```python +res = client.query(​ + collection_name="my_collection",​ + filter="color like \"red%\"",​ + output_fields=["vector", "color"],​ + limit=3，​ + # highlight-start​ + consistency_level="Eventually",​ + # highlight-next​ +)​ + +``` + +```java +QueryReq queryReq = QueryReq.builder()​ + .collectionName("my_collection")​ + .filter("color like \"red%\"")​ + .outputFields(Arrays.asList("vector", "color"))​ + .limit(3)​ + .consistencyLevel(ConsistencyLevel.EVENTUALLY)​ + .build();​ + ​ + QueryResp getResp = client.query(queryReq);​ + +``` + +This parameter is also available in the query iterator. Possible values for the `consistency_level` parameter are `Strong`, `Bounded`, `Eventually`, and `Session`.​ \ No newline at end of file diff --git a/v2.4.x/site/en/userGuide/search-query-get/filtered-search.md b/v2.4.x/site/en/userGuide/search-query-get/filtered-search.md new file mode 100644 index 0000000000..9f2935971c --- /dev/null +++ b/v2.4.x/site/en/userGuide/search-query-get/filtered-search.md @@ -0,0 +1,238 @@ +--- +id: filtered-search.md +title: Filtered Search​ +related_key: ann search, filtered search +summary: An ANN search finds vector embeddings most similar to specified vector embeddings. However, the search results may not always be correct. You can include filtering conditions in a search request so that Zilliz Cloud conducts metadata filtering before conducting ANN searches, reducing the search scope from the whole collection to only the entities matching the specified filtering conditions.​ +--- + +# Filtered Search​ + +An ANN search finds vector embeddings most similar to specified vector embeddings. However, the search results may not always be correct. You can include filtering conditions in a search request so that Zilliz Cloud conducts metadata filtering before conducting ANN searches, reducing the search scope from the whole collection to only the entities matching the specified filtering conditions.​ + +## Overview + +If a collection contains both vector embeddings and their metadata, you can filter metadata before ANN search to improve the relevancy of the search result. Once Zilliz Cloud receives a search request carrying a filtering condition, it restricts the search scope within the entities matching the specified filtering condition.​ + +![Filtered search](../../../../assets/filtered-search.png) + +As shown in the above diagram, the search request carries `chunk like % red %` as the filtering condition, indicating that Zilliz Cloud should conduct the ANN search within all the entities that have the word `red` in the `chunk` field. Specifically, Zilliz Cloud does the following:​ + +- Filter entities that match the filtering conditions carried in the search request.​ + +- Conduct the ANN search within the filtered entities.​ + +- Returns top-K entities.​ + +## Examples + +This section demonstrates how to conduct a filtered search. Code snippets in this section assume you already have the following entities in your collection. Each entity has four fields, namely **id**, **vector**, **color**, and **likes**.​ + +```JSON +[​ + {"id": 0, "vector": [0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592], "color": "pink_8682", "likes": 165},​ + {"id": 1, "vector": [0.19886812562848388, 0.06023560599112088, 0.6976963061752597, 0.2614474506242501, 0.838729485096104], "color": "red_7025", "likes": 25},​ + {"id": 2, "vector": [0.43742130801983836, -0.5597502546264526, 0.6457887650909682, 0.7894058910881185, 0.20785793220625592], "color": "orange_6781", "likes": 764},​ + {"id": 3, "vector": [0.3172005263489739, 0.9719044792798428, -0.36981146090600725, -0.4860894583077995, 0.95791889146345], "color": "pink_9298", "likes": 234},​ + {"id": 4, "vector": [0.4452349528804562, -0.8757026943054742, 0.8220779437047674, 0.46406290649483184, 0.30337481143159106], "color": "red_4794", "likes": 122},​ + {"id": 5, "vector": [0.985825131989184, -0.8144651566660419, 0.6299267002202009, 0.1206906911183383, -0.1446277761879955], "color": "yellow_4222", "likes": 12},​ + {"id": 6, "vector": [0.8371977790571115, -0.015764369584852833, -0.31062937026679327, -0.562666951622192, -0.8984947637863987], "color": "red_9392", "likes": 58},​ + {"id": 7, "vector": [-0.33445148015177995, -0.2567135004164067, 0.8987539745369246, 0.9402995886420709, 0.5378064918413052], "color": "grey_8510", "likes": 775},​ + {"id": 8, "vector": [0.39524717779832685, 0.4000257286739164, -0.5890507376891594, -0.8650502298996872, -0.6140360785406336], "color": "white_9381", "likes": 876},​ + {"id": 9, "vector": [0.5718280481994695, 0.24070317428066512, -0.3737913482606834, -0.06726932177492717, -0.6980531615588608], "color": "purple_4976", "likes": 765}​ +]​ + +``` + +The search request in the following code snippet carries a filtering condition and several output fields.​ + +
          + Python + Java + Node.js + cURL +
          + +```python +from pymilvus import MilvusClient​ +​ +client = MilvusClient(​ + uri="http://localhost:19530",​ + token="root:Milvus"​ +)​ +​ +query_vector = [0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592]​ +​ +res = client.search(​ + collection_name="my_collection",​ + data=[query_vector],​ + limit=5,​ + # highlight-start​ + filter='color like "red%" and likes > 50',​ + output_fields=["color", "likes"]​ + # highlight-end​ +)​ +​ +for hits in res:​ + print("TopK results:")​ + for hit in hits:​ + print(hit)​ + +``` + +```java +import io.milvus.v2.client.ConnectConfig;​ +import io.milvus.v2.client.MilvusClientV2;​ +import io.milvus.v2.service.vector.request.SearchReq​ +import io.milvus.v2.service.vector.request.data.FloatVec;​ +import io.milvus.v2.service.vector.response.SearchResp​ +​ +MilvusClientV2 client = new MilvusClientV2(ConnectConfig.builder()​ + .uri("http://localhost:19530")​ + .token("root:Milvus")​ + .build());​ +​ +FloatVec queryVector = new FloatVec(new float[]{0.3580376395471989f, -0.6023495712049978f, 0.18414012509913835f, -0.26286205330961354f, 0.9029438446296592f});​ +SearchReq searchReq = SearchReq.builder()​ + .collectionName("filtered_search_collection")​ + .data(Collections.singletonList(queryVector))​ + .topK(5)​ + .filter("color like \"red%\" and likes > 50")​ + .outputFields(Arrays.asList("color", "likes"))​ + .build();​ +​ +SearchResp searchResp = client.search(searchReq);​ +​ +List> searchResults = searchResp.getSearchResults();​ +for (List results : searchResults) {​ + System.out.println("TopK results:");​ + for (SearchResp.SearchResult result : results) {​ + System.out.println(result);​ + }​ +}​ +​ +// Output​ +// TopK results:​ +// SearchResp.SearchResult(entity={color=red_4794, likes=122}, score=0.5975797, id=4)​ +// SearchResp.SearchResult(entity={color=red_9392, likes=58}, score=-0.24996188, id=6)​ + +``` + +```go +import (​ + "context"​ + "log"​ +​ + "github.com/milvus-io/milvus/client/v2"​ + "github.com/milvus-io/milvus/client/v2/entity"​ +)​ +​ +func ExampleClient_Search_filter() {​ + ctx, cancel := context.WithCancel(context.Background())​ + defer cancel()​ +​ + milvusAddr := "127.0.0.1:19530"​ + token := "root:Milvus"​ +​ + cli, err := client.New(ctx, &client.ClientConfig{​ + Address: milvusAddr,​ + APIKey: token,​ + })​ + if err != nil {​ + log.Fatal("failed to connect to milvus server: ", err.Error())​ + }​ +​ + defer cli.Close(ctx)​ +​ + queryVector := []float32{0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592}​ +​ + resultSets, err := cli.Search(ctx, client.NewSearchOption(​ + "filtered_search_collection", // collectionName​ + 3, // limit​ + []entity.Vector{entity.FloatVector(queryVector)},​ + ).WithFilter(`color like "red%" and likes > 50`).WithOutputFields("color", "likes"))​ + if err != nil {​ + log.Fatal("failed to perform basic ANN search collection: ", err.Error())​ + }​ +​ + for _, resultSet := range resultSets {​ + log.Println("IDs: ", resultSet.IDs)​ + log.Println("Scores: ", resultSet.Scores)​ + }​ + // Output:​ + // IDs:​ + // Scores:​ +}​ +​ + +``` + +```javascript +import { MilvusClient, DataType } from "@zilliz/milvus2-sdk-node";​ +​ +const address = "http://localhost:19530";​ +const token = "root:Milvus";​ +const client = new MilvusClient({address, token});​ +​ +const query_vector = [0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592]​ +​ +const res = await client.search({​ + collection_name: "filtered_search_collection",​ + data: [query_vector],​ + limit: 5,​ + // highlight-start​ + filters: 'color like "red%" and likes > 50',​ + output_fields: ["color", "likes"]​ + // highlight-end​ +})​ + +``` + +```curl +export CLUSTER_ENDPOINT="http://localhost:19530"​ +export TOKEN="root:Milvus"​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/search" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "quick_setup",​ + "data": [​ + [0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592]​ + ],​ + "annsField": "vector",​ + "filter": "color like \"red%\" and likes > 50",​ + "limit": 3,​ + "outputFields": ["color", "likes"]​ +}'​ +# {"code":0,"cost":0,"data":[]}​ + +``` + +The filtering condition carried in the search request reads `color like "red%" and likes > 50`. It uses the and operator to include two conditions: the first one asks for entities that have a value starting with `red` in the `color` field, and the other asks for entities with a value greater than `50` in the `likes` field. There are only two entities meeting these requirements. With the top-K set to `3`, Zilliz Cloud will calculate the distance between these two entities to the query vector and return them as the search results.​ + +```JSON +[​ + {​ + "id": 4, ​ + "distance": 0.3345786594834839,​ + "entity": {​ + "vector": [0.4452349528804562, -0.8757026943054742, 0.8220779437047674, 0.46406290649483184, 0.30337481143159106], ​ + "color": "red_4794", ​ + "likes": 122​ + }​ + },​ + {​ + "id": 6, ​ + "distance": 0.6638239834383389，​ + "entity": {​ + "vector": [0.8371977790571115, -0.015764369584852833, -0.31062937026679327, -0.562666951622192, -0.8984947637863987], ​ + "color": "red_9392", ​ + "likes": 58​ + }​ + },​ +]​ + +``` + +For more information on the operators that you can use in metadata filtering, refer to [​Metadata Filtering](boolean.md).​ \ No newline at end of file diff --git a/v2.4.x/site/en/userGuide/search-query-get/full-text-search.md b/v2.4.x/site/en/userGuide/search-query-get/full-text-search.md new file mode 100644 index 0000000000..edd33a07f1 --- /dev/null +++ b/v2.4.x/site/en/userGuide/search-query-get/full-text-search.md @@ -0,0 +1,226 @@ +--- +id: full-text-search.md +title: Full Text Search +related_key: full, text, search +summary: Full text search is a feature that retrieves documents containing specific terms or phrases in text datasets, then ranking the results based on relevance. +--- + +# Full Text Search​ + +Full text search is a feature that retrieves documents containing specific terms or phrases in text datasets, then ranking the results based on relevance. This feature overcomes semantic search limitations, which might overlook precise terms, ensuring you receive the most accurate and contextually relevant results. Additionally, it simplifies vector searches by accepting raw text input, automatically converting your text data into sparse embeddings without the need to manually generate vector embeddings.​ + +Using the BM25 algorithm for relevance scoring, this feature is particularly valuable in retrieval-augmented generation (RAG) scenarios, where it prioritizes documents that closely match specific search terms.​ + +
          + +By integrating full text search with semantic-based dense vector search, you can enhance the accuracy and relevance of search results. For more information, refer to [​Hybrid Search](multi-vector-search.md).​ + +
          + +## Overview​ + +Full text search simplifies the process of text-based searching by eliminating the need for manual embedding. This feature operates through the following workflow:​ + +1. **Text input**: You insert raw text documents or provide query text without any need for manual embedding.​ + +2. **Text analysis**: Milvus uses an analyzer to tokenize input text into individual, searchable terms.​ + +3. **Function processing**: The built-in function receives tokenized terms and converts them into sparse vector representations.​ + +4. **Collection store**: Milvus stores these sparse embeddings in a collection for efficient retrieval.​ + +5. **BM25 scoring**: During a search, Milvus applies the BM25 algorithm to calculate scores for the stored documents and ranks matched results based on relevance to the query text.​ + +![Full text search](../../../assets/full-text-search.png) + +To use full text search, follow these main steps:​ + +1. [Create a collection](#Create-a-collection-for-full-text-search): Set up a collection with necessary fields and define a function to convert raw text into sparse embeddings.​ + +2. [Insert data](#Insert-text-data): Ingest your raw text documents to the collection.​ + +3. [Perform searches](#Perform-full-text-search): Use query texts to search through your collection and retrieve relevant results.​ + +## Create a collection for full text search​ + +To enable full text search, create a collection with a specific schema. This schema must include three necessary fields:​ + +- The primary field that uniquely identifies each entity in a collection.​ + +- A `VARCHAR` field that stores raw text documents, with the `enable_analyzer` attribute set to `True`. This allows Milvus to tokenize text into specific terms for function processing.​ + +- A `SPARSE_FLOAT_VECTOR` field reserved to store sparse embeddings that Milvus will automatically generate for the `VARCHAR` field.​ + +### Define the collection schema + +First, create the schema and add the necessary fields:​ + +```python +from pymilvus import MilvusClient, DataType, Function, FunctionType​ +​ +schema = MilvusClient.create_schema()​ +​ +schema.add_field(field_name="id", datatype=DataType.INT64, is_primary=True, auto_id=True)​ +schema.add_field(field_name="text", datatype=DataType.VARCHAR, max_length=1000, enable_analyzer=True)​ +schema.add_field(field_name="sparse", datatype=DataType.SPARSE_FLOAT_VECTOR)​ + +``` + +In this configuration,​ + +- `id`: serves as the primary key and is automatically generated with `auto_id=True`.​ + +- `text`: stores your raw text data for full text search operations. The data type must be `VARCHAR`, as `VARCHAR` is Milvus' string data type for text storage. Set `enable_analyzer=True` to allow Milvus to tokenize the text. By default, Milvus uses the [standard analyzer](standard-analyzer.md) for text analysis. To configure a different analyzer, refer to [​Overview](analyzer-overview.md).​ + +- `sparse`: a vector field reserved to store internally generated sparse embeddings for full text search operations. The data type must be `SPARSE_FLOAT_VECTOR`.​ + +Now, define a function that will convert your text into sparse vector representations and then add it to the schema:​ + +```python +bm25_function = Function(​ + name="text_bm25_emb", # Function name​ + input_field_names=["text"], # Name of the VARCHAR field containing raw text data​ + output_field_names=["sparse"], # Name of the SPARSE_FLOAT_VECTOR field reserved to store generated embeddings​ + function_type=FunctionType.BM25,​ +)​ +​ +schema.add_function(bm25_function)​ + +``` + +

          Parameter​

          + +

          Description​

          + +

          name​

          + +

          ​

          + +

          The name of the function. This function converts your raw text from the text field into searchable vectors that will be stored in the sparse field.​

          + +

          input_field_names​

          + +

          The name of the VARCHAR field requiring text-to-sparse-vector conversion. For FunctionType.BM25, this parameter accepts only one field name.​

          + +

          output_field_names​

          + +

          The name of the field where the internally generated sparse vectors will be stored. For FunctionType.BM25, this parameter accepts only one field name.​

          + +

          function_type​

          + +

          The type of the function to use. Set the value to FunctionType.BM25.​

          + +
          + +
          + +For collections with multiple `VARCHAR` fields requiring text-to-sparse-vector conversion, add separate functions to the collection schema, ensuring each function has a unique name and `output_field_names` value.​ + +
          + +### Configure the index + +After defining the schema with necessary fields and the built-in function, set up the index for your collection. To simplify this process, use `AUTOINDEX` as the `index_type`, an option that allows Milvus to choose and configure the most suitable index type based on the structure of your data.​ + +```python +index_params = MilvusClient.prepare_index_params()​ +​ +index_params.add_index(​ + field_name="sparse",​ + index_type="AUTOINDEX", ​ + metric_type="BM25"​ +)​ + +``` + +

          Parameter​

          + +

          Description​

          + +

          field_name​

          + +

          The name of the vector field to index. For full text search, this should be the field that stores the generated sparse vectors. In this example, set the value to sparse.​

          + +

          index_type​

          + +

          The type of the index to create. AUTOINDEX allows Milvus to automatically optimize index settings. If you need more control over your index settings, you can choose from various index types available for sparse vectors in Milvus. For more information, refer to Indexes supported in Milvus.​

          + +

          metric_type​

          + +

          The value for this parameter must be set to BM25 specifically for full text search functionality.​

          + +
          + +### Create the collection​ + +Now create the collection using the schema and index parameters defined.​ + +```python +MilvusClient.create_collection(​ + collection_name='demo', ​ + schema=schema, ​ + index_params=index_params​ +)​ + +``` + +## Insert text data + +After setting up your collection and index, you're ready to insert text data. In this process, you need only to provide the raw text. The built-in function we defined earlier automatically generates the corresponding sparse vector for each text entry.​ + +```python +MilvusClient.insert('demo', [​ + {'text': 'Artificial intelligence was founded as an academic discipline in 1956.'},​ + {'text': 'Alan Turing was the first person to conduct substantial research in AI.'},​ + {'text': 'Born in Maida Vale, London, Turing was raised in southern England.'},​ +])​ + +``` + +## Perform full text search + +Once you've inserted data into your collection, you can perform full text searches using raw text queries. Milvus automatically converts your query into a sparse vector and ranks the matched search results using the BM25 algorithm, and then returns the topK (`limit`) results.​ + +```python +search_params = {​ + 'params': {'drop_ratio_search': 0.6},​ +}​ +​ +MilvusClient.search(​ + collection_name='demo', ​ + data=['Who started AI research?'],​ + anns_field='sparse',​ + limit=3,​ + search_params=search_params​ +)​ + +``` + +

          Parameter​

          + +

          Description​

          + +

          search_params​

          + +

          A dictionary containing search parameters.​

          + +

          params.drop_ratio_search​

          + +

          Proportion of low-frequency terms to ignore during search. For details, refer to Sparse Vector.​

          + +

          data​

          + +

          The raw query text.​

          + +

          anns_field​

          + +

          The name of the field that contains internally generated sparse vectors.​

          + +

          limit​

          + +

          Maximum number of top matches to return.​

          + +
          + +​ + diff --git a/v2.4.x/site/en/userGuide/search-query-get/get-and-scalar-query.md b/v2.4.x/site/en/userGuide/search-query-get/get-and-scalar-query.md index b92484d2eb..65c8d45a4e 100644 --- a/v2.4.x/site/en/userGuide/search-query-get/get-and-scalar-query.md +++ b/v2.4.x/site/en/userGuide/search-query-get/get-and-scalar-query.md @@ -1,1591 +1,608 @@ --- id: get-and-scalar-query.md -order: 3 -summary: This guide demonstrates how to get entities by ID and conduct scalar filtering. -title: Get & Scalar Query +summary: In addition to ANN searches, Milvus also supports metadata filtering through queries. This page introduces how to use Query, Get, and QueryIterators to perform metadata filtering.​ +title: Query --- -# Get & Scalar Query +# Query​ -This guide demonstrates how to get entities by ID and conduct scalar filtering. A scalar filtering retrieves entities that match the specified filtering conditions. +In addition to ANN searches, Milvus also supports metadata filtering through queries. This page introduces how to use Query, Get, and QueryIterators to perform metadata filtering.​ -## Overview +## Overview​ -A scalar query filters entities in a collection based on a defined condition using boolean expressions. The query result is a set of entities that match the defined condition. Unlike a vector search, which identifies the closest vector to a given vector in a collection, queries filter entities based on specific criteria. +A Collection can store various types of scalar fields. You can have Milvus filter Entities based on one or more scalar fields. Milvus offers three types of queries: Query, Get, and QueryIterator. The table below compares these three query types.​ -In Milvus, __a filter is always a string compising field names joined by operators__. In this guide, you will find various filter examples. To learn more about the operator details, go to the [Reference](https://milvus.io/docs/get-and-scalar-query.md#Reference-on-scalar-filters) section. +

          ​

          -## Preparations +

          Get​

          -The following steps repurpose the code to connect to Milvus, quickly set up a collection, and insert over 1,000 randomly generated entities into the collection. +

          Query​

          -### Step 1: Create a collection +

          QueryIterator​

          -
          +

          Applicable scenarios​

          -Use [`MilvusClient`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Client/MilvusClient.md) to connect to the Milvus server and [`create_collection()`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Collections/create_collection.md) to create a collection. +

          To find entities that hold the specified primary keys.​

          - +

          ​

          -
          +

          To find all or a specified number of entities that meet the custom filtering conditions​

          -Use [`MilvusClientV2`](https://milvus.io/api-reference/java/v2.4.x/v2/Client/MilvusClientV2.md) to connect to the Milvus server and [`createCollection()`](https://milvus.io/api-reference/java/v2.4.x/v2/Collections/createCollection.md) to create a collection. +

          To find all entities that meet the custom filtering conditions in paginated queries.​

          - +

          Filtering method​

          -
          +

          By primary keys​

          -Use [`MilvusClient`](https://milvus.io/api-reference/node/v2.4.x/Client/MilvusClient.md) to connect to the Milvus server and [`createCollection()`](https://milvus.io/api-reference/node/v2.4.x/Collections/createCollection.md) to create a collection. +

          By filtering expressions.​

          - +

          By filtering expressions.​

          -
          - Python - Java - Node.js -
          +

          Mandatory parameters​

          -```python -from pymilvus import MilvusClient - -# 1. Set up a Milvus client -client = MilvusClient( - uri="http://localhost:19530" -) - -# 2. Create a collection -client.create_collection( - collection_name="quick_setup", - dimension=5, -) -``` +

          • Collection name​

            • -```java -String CLUSTER_ENDPOINT = "http://localhost:19530"; +

          • Primary keys​

          -// 1. Connect to Milvus server -ConnectConfig connectConfig = ConnectConfig.builder() - .uri(CLUSTER_ENDPOINT) - .build(); +

          ​

          -MilvusClientV2 client = new MilvusClientV2(connectConfig); +

          • Collection name​

            • -// 2. Create a collection in quick setup mode -CreateCollectionReq quickSetupReq = CreateCollectionReq.builder() - .collectionName("quick_setup") - .dimension(5) - .metricType("IP") - .build(); +

          • Filtering expressions​

          -client.createCollection(quickSetupReq); -``` +

          • Collection name​

            • -```javascript -const { MilvusClient, DataType, sleep } = require("@zilliz/milvus2-sdk-node") +

          • Filtering expressions​

            • -const address = "http://localhost:19530" +

          • Number of entities to return per query​

          -// 1. Set up a Milvus Client -client = new MilvusClient({address}); +

          Optional parameters​

          -// 2. Create a collection in quick setup mode -await client.createCollection({ - collection_name: "quick_setup", - dimension: 5, -}); -``` +

          • Partition name​

            • -### Step 2: Insert randomly generated entities +

          • Output fields​

          -
          +

          • Partition name​

            • -Use [`insert()`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Vector/insert.md) to insert entities into the collection. +

          • Number of entities to return​

            • - +

          • Output fields​

          -
          +

          • Partition name​

            • -Use [`insert()`](https://milvus.io/api-reference/java/v2.4.x/v2/Vector/insert.md) to insert entities into the collection. +

          • Number of entities to return in total​

            • - +

          • Output fields​

          -
          +

          Returns​

          -Use [`insert()`](https://milvus.io/api-reference/node/v2.4.x/Vector/insert.md) to insert entities into the collection. +

          Returns entities that hold the specified primary keys in the specified collection or partition.​

          - +

          Returns all or a specified number of entities that meet the custom filtering conditions in the specified collection or partition.​

          + +

          Returns all entities that meet the custom filtering conditions in the specified collection or partition through paginated queries.​

          + +
          + +For more on metadata filtering, refer to [​Metadata Filtering](boolean.md).​ + +## Use Get​ + +When you need to find entities by their primary keys, you can use the **Get** method. The following code examples assume that there are three fields named `id`, `vector`, and `color` in your collection and return the entities with primary keys `1`, `2`, and `3`.​ + +```json +[​ + {"id": 0, "vector": [0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592], "color": "pink_8682"},​ + {"id": 1, "vector": [0.19886812562848388, 0.06023560599112088, 0.6976963061752597, 0.2614474506242501, 0.838729485096104], "color": "red_7025"},​ + {"id": 2, "vector": [0.43742130801983836, -0.5597502546264526, 0.6457887650909682, 0.7894058910881185, 0.20785793220625592], "color": "orange_6781"},​ + {"id": 3, "vector": [0.3172005263489739, 0.9719044792798428, -0.36981146090600725, -0.4860894583077995, 0.95791889146345], "color": "pink_9298"},​ + {"id": 4, "vector": [0.4452349528804562, -0.8757026943054742, 0.8220779437047674, 0.46406290649483184, 0.30337481143159106], "color": "red_4794"},​ + {"id": 5, "vector": [0.985825131989184, -0.8144651566660419, 0.6299267002202009, 0.1206906911183383, -0.1446277761879955], "color": "yellow_4222"},​ + {"id": 6, "vector": [0.8371977790571115, -0.015764369584852833, -0.31062937026679327, -0.562666951622192, -0.8984947637863987], "color": "red_9392"},​ + {"id": 7, "vector": [-0.33445148015177995, -0.2567135004164067, 0.8987539745369246, 0.9402995886420709, 0.5378064918413052], "color": "grey_8510"},​ + {"id": 8, "vector": [0.39524717779832685, 0.4000257286739164, -0.5890507376891594, -0.8650502298996872, -0.6140360785406336], "color": "white_9381"},​ + {"id": 9, "vector": [0.5718280481994695, 0.24070317428066512, -0.3737913482606834, -0.06726932177492717, -0.6980531615588608], "color": "purple_4976"},​ +]​ + +```
          Python Java Node.js + cURL
          ```python -# 3. Insert randomly generated vectors -colors = ["green", "blue", "yellow", "red", "black", "white", "purple", "pink", "orange", "brown", "grey"] -data = [] - -for i in range(1000): - current_color = random.choice(colors) - current_tag = random.randint(1000, 9999) - data.append({ - "id": i, - "vector": [ random.uniform(-1, 1) for _ in range(5) ], - "color": current_color, - "tag": current_tag, - "color_tag": f"{current_color}_{str(current_tag)}" - }) - -print(data[0]) - -# Output -# -# { -# "id": 0, -# "vector": [ -# 0.7371107800002366, -# -0.7290389773227746, -# 0.38367002049157417, -# 0.36996000494220627, -# -0.3641898951462792 -# ], -# "color": "yellow", -# "tag": 6781, -# "color_tag": "yellow_6781" -# } - -res = client.insert( - collection_name="quick_setup", - data=data -) - -print(res) - -# Output -# -# { -# "insert_count": 1000, -# "ids": [ -# 0, -# 1, -# 2, -# 3, -# 4, -# 5, -# 6, -# 7, -# 8, -# 9, -# "(990 more items hidden)" -# ] -# } +from pymilvus import MilvusClient​ +​ +client = MilvusClient(​ + uri="http://localhost:19530",​ + token="root:Milvus"​ +)​ +​ +res = client.get(​ + collection_name="query_collection",​ + ids=[0, 1, 2],​ + output_fields=["vector", "color"]​ +)​ +​ +print(res)​ + ``` ```java -// 3. Insert randomly generated vectors into the collection -List colors = Arrays.asList("green", "blue", "yellow", "red", "black", "white", "purple", "pink", "orange", "brown", "grey"); -List data = new ArrayList<>(); - -for (int i=0; i<1000; i++) { - Random rand = new Random(); - String current_color = colors.get(rand.nextInt(colors.size()-1)); - int current_tag = rand.nextInt(8999) + 1000; - JSONObject row = new JSONObject(); - row.put("id", Long.valueOf(i)); - row.put("vector", Arrays.asList(rand.nextFloat(), rand.nextFloat(), rand.nextFloat(), rand.nextFloat(), rand.nextFloat())); - row.put("color", current_color); - row.put("tag", current_tag); - row.put("color_tag", current_color + '_' + String.valueOf(rand.nextInt(8999) + 1000)); - data.add(row); -} - -InsertReq insertReq = InsertReq.builder() - .collectionName("quick_setup") - .data(data) - .build(); - -InsertResp insertResp = client.insert(insertReq); - -System.out.println(JSONObject.toJSON(insertResp)); - -// Output: -// {"insertCnt": 1000} -``` +import io.milvus.v2.client.ConnectConfig;​ +import io.milvus.v2.client.MilvusClientV2;​ +import io.milvus.v2.service.vector.request.GetReq​ +import io.milvus.v2.service.vector.request.GetResp​ +import java.util.*;​ +​ +MilvusClientV2 client = new MilvusClientV2(ConnectConfig.builder()​ + .uri("http://localhost:19530")​ + .token("root:Milvus")​ + .build());​ + ​ +GetReq getReq = GetReq.builder()​ + .collectionName("query_collection")​ + .ids(Arrays.asList(0, 1, 2))​ + .outputFields(Arrays.asList("vector", "color"))​ + .build();​ +​ +GetResp getResp = client.get(getReq);​ +​ +List results = getResp.getGetResults();​ +for (QueryResp.QueryResult result : results) {​ + System.out.println(result.getEntity());​ +}​ +​ +// Output​ +// {color=pink_8682, vector=[0.35803765, -0.6023496, 0.18414013, -0.26286206, 0.90294385], id=0}​ +// {color=red_7025, vector=[0.19886813, 0.060235605, 0.6976963, 0.26144746, 0.8387295], id=1}​ +// {color=orange_6781, vector=[0.43742132, -0.55975026, 0.6457888, 0.7894059, 0.20785794], id=2}​ -```javascript -// 3. Insert randomly generated vectors -const colors = ["green", "blue", "yellow", "red", "black", "white", "purple", "pink", "orange", "brown", "grey"] -var data = [] - -for (let i = 0; i < 1000; i++) { - current_color = colors[Math.floor(Math.random() * colors.length)] - current_tag = Math.floor(Math.random() * 8999 + 1000) - data.push({ - "id": i, - "vector": [Math.random(), Math.random(), Math.random(), Math.random(), Math.random()], - "color": current_color, - "tag": current_tag, - "color_tag": `${current_color}_${current_tag}` - }) -} - -console.log(data[0]) - -// Output -// -// { -// id: 0, -// vector: [ -// 0.16022394821966035, -// 0.6514875214491056, -// 0.18294484964044666, -// 0.30227694168725394, -// 0.47553087493572255 -// ], -// color: 'blue', -// tag: 8907, -// color_tag: 'blue_8907' -// } -// - -res = await client.insert({ - collection_name: "quick_setup", - data: data -}) - -console.log(res.insert_cnt) - -// Output -// -// 1000 -// ``` -### Step 3: Create partitions and insert more entities - -
          - -Use [`create_partition()`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Partitions/create_partition.md) to create partitions and [`insert()`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Vector/insert.md) to insert more entities into the collection. - -
          - -
          +```javascript +import { MilvusClient, DataType } from "@zilliz/milvus2-sdk-node";​ +​ +const address = "http://localhost:19530";​ +const token = "root:Milvus";​ +const client = new MilvusClient({address, token});​ +​ +const res = client.get({​ + collection_name="query_collection",​ + ids=[0,1,2],​ + output_fields=["vector", "color"]​ +})​ -Use [`createPartition()`](https://milvus.io/api-reference/java/v2.4.x/v2/Partitions/createPartition.md) to create partitions and [`insert()`](https://milvus.io/api-reference/java/v2.4.x/v2/Vector/insert.md) to insert more entities into the collection. +``` -
          +```curl +export CLUSTER_ENDPOINT="http://localhost:19530"​ +export TOKEN="root:Milvus"​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/get" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "quick_setup",​ + "id": [0, 1, 2],​ + "outputFields": ["vector", "color"]​ +}'​ +​ +# {"code":0,"cost":0,"data":[{"color":"pink_8682","id":0,"vector":[0.35803765,-0.6023496,0.18414013,-0.26286206,0.90294385]},{"color":"red_7025","id":1,"vector":[0.19886813,0.060235605,0.6976963,0.26144746,0.8387295]},{"color":"orange_6781","id":2,"vector":[0.43742132,-0.55975026,0.6457888,0.7894059,0.20785794]}]}​ -
          +``` -Use [`createPartition()`](https://milvus.io/api-reference/node/v2.4.x/Partitions/createPartition.md) to create partitions and [`insert()`](https://milvus.io/api-reference/node/v2.4.x/Vector/insert.md) to insert more entities into the collection. +## Use Query​ -
          +When you need to find entities by custom filtering conditions, use the Query method. The following code examples assume there are three fields named `id`, `vector`, and `color` and return the specified number of entities that hold a `color` value starting with `red`.​
          Python Java Node.js + cURL
          ```python -# 4. Create partitions and insert more entities -client.create_partition( - collection_name="quick_setup", - partition_name="partitionA" -) - -client.create_partition( - collection_name="quick_setup", - partition_name="partitionB" -) - -data = [] - -for i in range(1000, 1500): - current_color = random.choice(colors) - data.append({ - "id": i, - "vector": [ random.uniform(-1, 1) for _ in range(5) ], - "color": current_color, - "tag": current_tag, - "color_tag": f"{current_color}_{str(current_tag)}" - }) - -res = client.insert( - collection_name="quick_setup", - data=data, - partition_name="partitionA" -) - -print(res) - -# Output -# -# { -# "insert_count": 500, -# "ids": [ -# 1000, -# 1001, -# 1002, -# 1003, -# 1004, -# 1005, -# 1006, -# 1007, -# 1008, -# 1009, -# "(490 more items hidden)" -# ] -# } - -data = [] - -for i in range(1500, 2000): - current_color = random.choice(colors) - data.append({ - "id": i, - "vector": [ random.uniform(-1, 1) for _ in range(5) ], - "color": current_color, - "tag": current_tag, - "color_tag": f"{current_color}_{str(current_tag)}" - }) - -res = client.insert( - collection_name="quick_setup", - data=data, - partition_name="partitionB" -) - -print(res) - -# Output -# -# { -# "insert_count": 500, -# "ids": [ -# 1500, -# 1501, -# 1502, -# 1503, -# 1504, -# 1505, -# 1506, -# 1507, -# 1508, -# 1509, -# "(490 more items hidden)" -# ] -# } +from pymilvus import MilvusClient​ +​ +client = MilvusClient(​ + uri="http://localhost:19530",​ + token="root:Milvus"​ +)​ +​ +res = client.query(​ + collection_name="query_collection",​ + filter="color like \"red%\"",​ + output_fields=["vector", "color"],​ + limit=3​ +)​ + ``` ```java -// 4. Create partitions and insert some more data -CreatePartitionReq createPartitionReq = CreatePartitionReq.builder() - .collectionName("quick_setup") - .partitionName("partitionA") - .build(); - -client.createPartition(createPartitionReq); - -createPartitionReq = CreatePartitionReq.builder() - .collectionName("quick_setup") - .partitionName("partitionB") - .build(); - -client.createPartition(createPartitionReq); - -data.clear(); - -for (int i=1000; i<1500; i++) { - Random rand = new Random(); - String current_color = colors.get(rand.nextInt(colors.size()-1)); - int current_tag = rand.nextInt(8999) + 1000; - JSONObject row = new JSONObject(); - row.put("id", Long.valueOf(i)); - row.put("vector", Arrays.asList(rand.nextFloat(), rand.nextFloat(), rand.nextFloat(), rand.nextFloat(), rand.nextFloat())); - row.put("color", current_color); - row.put("tag", current_tag); - data.add(row); -} - -insertReq = InsertReq.builder() - .collectionName("quick_setup") - .data(data) - .partitionName("partitionA") - .build(); - -insertResp = client.insert(insertReq); - -System.out.println(JSONObject.toJSON(insertResp)); - -// Output: -// {"insertCnt": 500} - -data.clear(); - -for (int i=1500; i<2000; i++) { - Random rand = new Random(); - String current_color = colors.get(rand.nextInt(colors.size()-1)); - int current_tag = rand.nextInt(8999) + 1000; - JSONObject row = new JSONObject(); - row.put("id", Long.valueOf(i)); - row.put("vector", Arrays.asList(rand.nextFloat(), rand.nextFloat(), rand.nextFloat(), rand.nextFloat(), rand.nextFloat())); - row.put("color", current_color); - row.put("tag", current_tag); - data.add(row); -} - -insertReq = InsertReq.builder() - .collectionName("quick_setup") - .data(data) - .partitionName("partitionB") - .build(); - -insertResp = client.insert(insertReq); - -System.out.println(JSONObject.toJSON(insertResp)); - -// Output: -// {"insertCnt": 500} -``` +​ +import io.milvus.v2.service.vector.request.QueryReq​ +import io.milvus.v2.service.vector.request.QueryResp​ +​ +​ +QueryReq queryReq = QueryReq.builder()​ + .collectionName("query_collection")​ + .filter("color like \"red%\"")​ + .outputFields(Arrays.asList("vector", "color"))​ + .limit(3)​ + .build();​ +​ +QueryResp getResp = client.query(queryReq);​ +​ +List results = getResp.getQueryResults();​ +for (QueryResp.QueryResult result : results) {​ + System.out.println(result.getEntity());​ +}​ +​ +// Output​ +// {color=red_7025, vector=[0.19886813, 0.060235605, 0.6976963, 0.26144746, 0.8387295], id=1}​ +// {color=red_4794, vector=[0.44523495, -0.8757027, 0.82207793, 0.4640629, 0.3033748], id=4}​ +// {color=red_9392, vector=[0.8371978, -0.015764369, -0.31062937, -0.56266695, -0.8984948], id=6}​ -```javascript -// 4. Create partitions and insert more entities -await client.createPartition({ - collection_name: "quick_setup", - partition_name: "partitionA" -}) - -await client.createPartition({ - collection_name: "quick_setup", - partition_name: "partitionB" -}) - -data = [] - -for (let i = 1000; i < 1500; i++) { - current_color = colors[Math.floor(Math.random() * colors.length)] - current_tag = Math.floor(Math.random() * 8999 + 1000) - data.push({ - "id": i, - "vector": [Math.random(), Math.random(), Math.random(), Math.random(), Math.random()], - "color": current_color, - "tag": current_tag, - "color_tag": `${current_color}_${current_tag}` - }) -} - -res = await client.insert({ - collection_name: "quick_setup", - data: data, - partition_name: "partitionA" -}) - -console.log(res.insert_cnt) - -// Output -// -// 500 -// - -await sleep(5000) - -data = [] - -for (let i = 1500; i < 2000; i++) { - current_color = colors[Math.floor(Math.random() * colors.length)] - current_tag = Math.floor(Math.random() * 8999 + 1000) - data.push({ - "id": i, - "vector": [Math.random(), Math.random(), Math.random(), Math.random(), Math.random()], - "color": current_color, - "tag": current_tag, - "color_tag": `${current_color}_${current_tag}` - }) -} - -res = await client.insert({ - collection_name: "quick_setup", - data: data, - partition_name: "partitionB" -}) - -console.log(res.insert_cnt) - -// Output -// -// 500 -// ``` -## Get Entities by ID - -
          +```go +import (​ + "context"​ + "fmt"​ + "log"​ +​ + "github.com/milvus-io/milvus/client/v2"​ +)​ +​ +func ExampleClient_Query_basic() {​ + ctx, cancel := context.WithCancel(context.Background())​ + defer cancel()​ +​ + milvusAddr := "127.0.0.1:19530"​ + token := "root:Milvus"​ +​ + cli, err := client.New(ctx, &client.ClientConfig{​ + Address: milvusAddr,​ + APIKey: token,​ + })​ + if err != nil {​ + log.Fatal("failed to connect to milvus server: ", err.Error())​ + }​ +​ + defer cli.Close(ctx)​ +​ + resultSet, err := cli.Query(ctx, client.NewQueryOption("query_collection").​ + WithFilter(`color like "red%"`).​ + WithOutputFields("vector", "color").​ + WithLimit(3))​ +​ + fmt.Println(resultSet.GetColumn("color"))​ +}​ +​ -If you know the IDs of the entities of your interests, you can use the [`get()`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Vector/get.md) method. +``` -
          +```javascript +import { MilvusClient, DataType } from "@zilliz/milvus2-sdk-node";​ +​ +const address = "http://localhost:19530";​ +const token = "root:Milvus";​ +const client = new MilvusClient({address, token});​ +​ +const res = client.query({​ + collection_name="quick_setup",​ + filter='color like "red%"',​ + output_fields=["vector", "color"],​ + limit(3)​ +})​ -
          +``` -If you know the IDs of the entities of your interests, you can use the [`get()`](https://milvus.io/api-reference/java/v2.4.x/v2/Vector/get.md) method. +```curl +export CLUSTER_ENDPOINT="http://localhost:19530"​ +export TOKEN="root:Milvus"​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/query" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "quick_setup",​ + "filter": "color like \"red%\"",​ + "limit": 3,​ + "outputFields": ["vector", "color"]​ +}'​ +#{"code":0,"cost":0,"data":[{"color":"red_7025","id":1,"vector":[0.19886813,0.060235605,0.6976963,0.26144746,0.8387295]},{"color":"red_4794","id":4,"vector":[0.44523495,-0.8757027,0.82207793,0.4640629,0.3033748]},{"color":"red_9392","id":6,"vector":[0.8371978,-0.015764369,-0.31062937,-0.56266695,-0.8984948]}]}​ -
          - -
          +``` -If you know the IDs of the entities of your interests, you can use the [`get()`](https://milvus.io/api-reference/node/v2.4.x/Vector/get.md) method. +## Use QueryIterator​ -
          +When you need to find entities by custom filtering conditions through paginated queries, create a **QueryIterator** and use its **next()** method to iterate over all entities to find those meeting the filtering conditions. The following code examples assume that there are three fields named `id`, `vector`, and `color` and return all entities that hold a `color` value starting with `red`.​
          Python Java Node.js + cURL
          ```python -# 4. Get entities by ID -res = client.get( - collection_name="quick_setup", - ids=[0, 1, 2] -) - -print(res) - -# Output -# -# [ -# { -# "id": 0, -# "vector": [ -# 0.68824464, -# 0.6552274, -# 0.33593303, -# -0.7099536, -# -0.07070546 -# ], -# "color_tag": "green_2006", -# "color": "green" -# }, -# { -# "id": 1, -# "vector": [ -# -0.98531723, -# 0.33456197, -# 0.2844234, -# 0.42886782, -# 0.32753858 -# ], -# "color_tag": "white_9298", -# "color": "white" -# }, -# { -# "id": 2, -# "vector": [ -# -0.9886812, -# -0.44129863, -# -0.29859528, -# 0.06059075, -# -0.43817034 -# ], -# "color_tag": "grey_5312", -# "color": "grey" -# } -# ] +from pymilvus import connections, Collection​ +​ +connections.connect(​ + uri="http://localhost:19530",​ + token="root:Milvus"​ +)​ +​ +collection = Collection("query_collection")​ +​ +iterator = collection.query_iterator(​ + batch_size=10,​ + expr="color like \"red%\"",​ + output_fields=["color"]​ +)​ +​ +results = []​ +​ +while True:​ + result = iterator.next()​ + if not result:​ + iterator.close()​ + break​ +​ + print(result)​ + results += result​ + ``` ```java -// 5. Get entities by ID -GetReq getReq = GetReq.builder() - .collectionName("quick_setup") - .ids(Arrays.asList(0L, 1L, 2L)) - .build(); - -GetResp entities = client.get(getReq); - -System.out.println(JSONObject.toJSON(entities)); - -// Output: -// {"getResults": [ -// {"entity": { -// "color": "white", -// "color_tag": "white_4597", -// "vector": [ -// 0.09665024, -// 0.1163497, -// 0.0701347, -// 0.32577968, -// 0.40943468 -// ], -// "tag": 8946, -// "id": 0 -// }}, -// {"entity": { -// "color": "green", -// "color_tag": "green_3039", -// "vector": [ -// 0.90689456, -// 0.4377399, -// 0.75387514, -// 0.36454988, -// 0.8702918 -// ], -// "tag": 2341, -// "id": 1 -// }}, -// {"entity": { -// "color": "white", -// "color_tag": "white_8708", -// "vector": [ -// 0.9757728, -// 0.13974023, -// 0.8023141, -// 0.61947155, -// 0.8290197 -// ], -// "tag": 9913, -// "id": 2 -// }} -// ]} +import io.milvus.orm.iterator.QueryIterator;​ +import io.milvus.response.QueryResultsWrapper;​ +import io.milvus.v2.common.ConsistencyLevel;​ +import io.milvus.v2.service.vector.request.QueryIteratorReq;​ +​ +​ +QueryIteratorReq req = QueryIteratorReq.builder()​ + .collectionName("query_collection")​ + .expr("color like \"red%\"")​ + .batchSize(50L)​ + .outputFields(Collections.singletonList("color"))​ + .consistencyLevel(ConsistencyLevel.BOUNDED)​ + .build();​ +QueryIterator queryIterator = client.queryIterator(req);​ +​ +while (true) {​ + List res = queryIterator.next();​ + if (res.isEmpty()) {​ + queryIterator.close();​ + break;​ + }​ +​ + for (QueryResultsWrapper.RowRecord record : res) {​ + System.out.println(record);​ + }​ +}​ +​ +// Output​ +// [color:red_7025, id:1]​ +// [color:red_4794, id:4]​ +// [color:red_9392, id:6]​ + ``` ```javascript -// 5. Get entities by id -res = await client.get({ - collection_name: "quick_setup", - ids: [0, 1, 2], - output_fields: ["vector", "color_tag"] -}) - -console.log(res.data) - -// Output -// -// [ -// { -// vector: [ -// 0.16022394597530365, -// 0.6514875292778015, -// 0.18294484913349152, -// 0.30227693915367126, -// 0.47553086280822754 -// ], -// '$meta': { color: 'blue', tag: 8907, color_tag: 'blue_8907' }, -// id: '0' -// }, -// { -// vector: [ -// 0.2459285855293274, -// 0.4974019527435303, -// 0.2154673933982849, -// 0.03719571232795715, -// 0.8348019123077393 -// ], -// '$meta': { color: 'grey', tag: 3710, color_tag: 'grey_3710' }, -// id: '1' -// }, -// { -// vector: [ -// 0.9404329061508179, -// 0.49662265181541443, -// 0.8088793158531189, -// 0.9337621331214905, -// 0.8269071578979492 -// ], -// '$meta': { color: 'blue', tag: 2993, color_tag: 'blue_2993' }, -// id: '2' -// } -// ] -// +import { MilvusClient, DataType } from "@zilliz/milvus2-sdk-node";​ +​ +const iterator = await milvusClient.queryIterator({​ + collection_name: 'query_collection',​ + batchSize: 10,​ + expr: 'color like "red%"',​ + output_fields: ['color'],​ +});​ +​ +const results = [];​ +for await (const value of iterator) {​ + results.push(...value);​ + page += 1;​ +}​ + +``` + +```curl +# Currently not available + ``` -### Get entities from partitions +## Queries in Partitions​ -You can also get entities from specific partitions. +You can also perform queries within one or multiple partitions by including the partition names in the Get, Query, or QueryIterator request. The following code examples assume that there is a partition named **PartitionA** in the collection.​
          Python Java Node.js + cURL
          ```python -# 5. Get entities from partitions -res = client.get( - collection_name="quick_setup", - ids=[1000, 1001, 1002], - partition_names=["partitionA"] -) - -print(res) - -# Output -# -# [ -# { -# "color": "green", -# "tag": 1995, -# "color_tag": "green_1995", -# "id": 1000, -# "vector": [ -# 0.7807706, -# 0.8083741, -# 0.17276904, -# -0.8580777, -# 0.024156934 -# ] -# }, -# { -# "color": "red", -# "tag": 1995, -# "color_tag": "red_1995", -# "id": 1001, -# "vector": [ -# 0.065074645, -# -0.44882354, -# -0.29479212, -# -0.19798489, -# -0.77542555 -# ] -# }, -# { -# "color": "green", -# "tag": 1995, -# "color_tag": "green_1995", -# "id": 1002, -# "vector": [ -# 0.027934508, -# -0.44199976, -# -0.40262738, -# -0.041511405, -# 0.024782438 -# ] -# } -# ] +from pymilvus import MilvusClient​ +client = MilvusClient(​ + uri="http://localhost:19530",​ + token="root:Milvus"​ +)​ +​ +res = client.get(​ + collection_name="query_collection",​ + # highlight-next-line​ + partitionNames=["partitionA"],​ + ids=[0, 1, 2],​ + output_fields=["vector", "color"]​ +)​ +​ +from pymilvus import MilvusClient​ +​ +client = MilvusClient(​ + uri="http://localhost:19530",​ + token="root:Milvus"​ +)​ +​ +res = client.query(​ + collection_name="query_collection",​ + # highlight-next-line​ + partitionNames=["partitionA"],​ + filter="color like \"red%\"",​ + output_fields=["vector", "color"],​ + limit=3​ +)​ +​ +# 使用 QueryIterator​ +from pymilvus import connections, Collection​ +​ +connections.connect(​ + uri="http://localhost:19530",​ + token="root:Milvus"​ +)​ +​ +collection = Collection("query_collection")​ +​ +iterator = collection.query_iterator(​ + # highlight-next-line​ + partition_names=["partitionA"],​ + batch_size=10,​ + expr="color like \"red%\"",​ + output_fields=["color"]​ +)​ +​ +results = []​ +​ +while True:​ + result = iterator.next()​ + if not result:​ + iterator.close()​ + break​ +​ + print(result)​ + results += result​ + ``` ```java -// 5. Get entities by ID in a partition -getReq = GetReq.builder() - .collectionName("quick_setup") - .ids(Arrays.asList(1001L, 1002L, 1003L)) - .partitionName("partitionA") - .build(); - -entities = client.get(getReq); - -System.out.println(JSONObject.toJSON(entities)); - -// Output: -// {"getResults": [ -// {"entity": { -// "color": "yellow", -// "vector": [ -// 0.4300114, -// 0.599917, -// 0.799163, -// 0.75395125, -// 0.89947814 -// ], -// "id": 1001, -// "tag": 5803 -// }}, -// {"entity": { -// "color": "blue", -// "vector": [ -// 0.009218454, -// 0.64637834, -// 0.19815737, -// 0.30519038, -// 0.8218663 -// ], -// "id": 1002, -// "tag": 7212 -// }}, -// {"entity": { -// "color": "black", -// "vector": [ -// 0.76521933, -// 0.7818409, -// 0.16976339, -// 0.8719652, -// 0.1434964 -// ], -// "id": 1003, -// "tag": 1710 -// }} -// ]} -``` +GetReq getReq = GetReq.builder()​ + .collectionName("query_collection")​ + .partitionName("partitionA")​ + .ids(Arrays.asList(10, 11, 12))​ + .outputFields(Collections.singletonList("color"))​ + .build();​ +​ +GetResp getResp = client.get(getReq);​ +​ +​ +QueryReq queryReq = QueryReq.builder()​ + .collectionName("query_collection")​ + .partitionNames(Collections.singletonList("partitionA"))​ + .filter("color like \"red%\"")​ + .outputFields(Collections.singletonList("color"))​ + .limit(3)​ + .build();​ +​ +QueryResp getResp = client.query(queryReq);​ +​ +​ +QueryIteratorReq req = QueryIteratorReq.builder()​ + .collectionName("query_collection")​ + .partitionNames(Collections.singletonList("partitionA"))​ + .expr("color like \"red%\"")​ + .batchSize(50L)​ + .outputFields(Collections.singletonList("color"))​ + .consistencyLevel(ConsistencyLevel.BOUNDED)​ + .build();​ +QueryIterator queryIterator = client.queryIterator(req);​ -```javascript -// 5.1 Get entities by id in a partition -res = await client.get({ - collection_name: "quick_setup", - ids: [1000, 1001, 1002], - partition_names: ["partitionA"], - output_fields: ["vector", "color_tag"] -}) - -console.log(res.data) - -// Output -// -// [ -// { -// id: '1000', -// vector: [ -// 0.014254206791520119, -// 0.5817716121673584, -// 0.19793470203876495, -// 0.8064294457435608, -// 0.7745839357376099 -// ], -// '$meta': { color: 'white', tag: 5996, color_tag: 'white_5996' } -// }, -// { -// id: '1001', -// vector: [ -// 0.6073881983757019, -// 0.05214758217334747, -// 0.730999231338501, -// 0.20900958776474, -// 0.03665429726243019 -// ], -// '$meta': { color: 'grey', tag: 2834, color_tag: 'grey_2834' } -// }, -// { -// id: '1002', -// vector: [ -// 0.48877206444740295, -// 0.34028753638267517, -// 0.6527213454246521, -// 0.9763909578323364, -// 0.8031482100486755 -// ], -// '$meta': { color: 'pink', tag: 9107, color_tag: 'pink_9107' } -// } -// ] -// ``` +```javascript +import { MilvusClient, DataType } from "@zilliz/milvus2-sdk-node";​ +​ +const address = "http://localhost:19530";​ +const token = "root:Milvus";​ +const client = new MilvusClient({address, token});​ +​ +// 使用 Get 方法​ +var res = client.query({​ + collection_name="query_collection",​ + // highlight-next-line​ + partition_names=["partitionA"],​ + filter='color like "red%"',​ + output_fields=["vector", "color"],​ + limit(3)​ +})​ +​ +// 使用 Query 方法​ +res = client.query({​ + collection_name="query_collection",​ + // highlight-next-line​ + partition_names=["partitionA"],​ + filter="color like \"red%\"",​ + output_fields=["vector", "color"],​ + limit(3)​ +})​ +​ +// 暂不支持使用 QueryIterator​ +const iterator = await milvusClient.queryIterator({​ + collection_name: 'query_collection',​ + partition_names: ['partitionA'],​ + batchSize: 10,​ + expr: 'color like "red%"',​ + output_fields: ['vector', 'color'],​ +});​ +​ +const results = [];​ +for await (const value of iterator) {​ + results.push(...value);​ + page += 1;​ +}​ -## Use Basic Operators - -In this section, you will find examples of how to use basic operators in scalar filtering. You can apply these filters to [vector searches](https://milvus.io/docs/single-vector-search.md#Filtered-search) and [data deletions](https://milvus.io/docs/insert-update-delete.md#Delete-entities) too. - -
          - -For more information, refer to [`query()`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Vector/query.md) in the SDK reference. - -
          - -
          - -For more information, refer to [`query()`](https://milvus.io/api-reference/java/v2.4.x/v2/Vector/query.md) in the SDK reference. - -
          - -
          - -For more information, refer to [`query()`](https://milvus.io/api-reference/node/v2.4.x/Vector/query.md) in the SDK reference. +``` -
          +```curl +export CLUSTER_ENDPOINT="http://localhost:19530"​ +export TOKEN="root:Milvus"​ +​ +# 使用 Get 方法​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/get" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "query_collection",​ + "partitionNames": ["partitionA"],​ + "id": [0, 1, 2],​ + "outputFields": ["vector", "color"]​ +}'​ +​ +# 使用 Query 方法​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/get" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "query_collection",​ + "partitionNames": ["partitionA"],​ + "filter": "color like \"red%\"",​ + "limit": 3,​ + "outputFields": ["vector", "color"],​ + "id": [0, 1, 2]​ +}'​ -- Filter entities with their tag values falling between 1,000 to 1,500. - -
          - Python - Java - Node.js -
          - - ```python - # 6. Use basic operators - - res = client.query( - collection_name="quick_setup", - filter="1000 < tag < 1500", - output_fields=["color_tag"], - limit=3 - ) - - print(res) - - # Output - # - # [ - # { - # "id": 1, - # "color_tag": "pink_1023" - # }, - # { - # "id": 41, - # "color_tag": "red_1483" - # }, - # { - # "id": 44, - # "color_tag": "grey_1146" - # } - # ] - ``` - - ```java - // 6. Use basic operators - - QueryReq queryReq = QueryReq.builder() - .collectionName("quick_setup") - .filter("1000 < tag < 1500") - .outputFields(Arrays.asList("color_tag")) - .limit(3) - .build(); - - QueryResp queryResp = client.query(queryReq); - - System.out.println(JSONObject.toJSON(queryResp)); - - // Output: - // {"queryResults": [ - // {"entity": { - // "color_tag": "white_7588", - // "id": 34 - // }}, - // {"entity": { - // "color_tag": "orange_4989", - // "id": 64 - // }}, - // {"entity": { - // "color_tag": "white_3415", - // "id": 73 - // }} - // ]} - ``` - - ```javascript - // 6. Use basic operators - res = await client.query({ - collection_name: "quick_setup", - filter: "1000 < tag < 1500", - output_fields: ["color_tag"], - limit: 3 - }) - - console.log(res.data) - - // Output - // - // [ - // { - // '$meta': { color: 'pink', tag: 1050, color_tag: 'pink_1050' }, - // id: '6' - // }, - // { - // '$meta': { color: 'purple', tag: 1174, color_tag: 'purple_1174' }, - // id: '24' - // }, - // { - // '$meta': { color: 'orange', tag: 1023, color_tag: 'orange_1023' }, - // id: '40' - // } - // ] - // - ``` - -- Filter entities with their __color__ values set to __brown__. - -
          - Python - Java - Node.js -
          - - ```python - res = client.query( - collection_name="quick_setup", - filter='color == "brown"', - output_fields=["color_tag"], - limit=3 - ) - - print(res) - - # Output - # - # [ - # { - # "color_tag": "brown_5343", - # "id": 15 - # }, - # { - # "color_tag": "brown_3167", - # "id": 27 - # }, - # { - # "color_tag": "brown_3100", - # "id": 30 - # } - # ] - ``` - - ```java - queryReq = QueryReq.builder() - .collectionName("quick_setup") - .filter("color == \"brown\"") - .outputFields(Arrays.asList("color_tag")) - .limit(3) - .build(); - - queryResp = client.query(queryReq); - - System.out.println(JSONObject.toJSON(queryResp)); - - // Output: - // {"queryResults": [ - // {"entity": { - // "color_tag": "brown_7792", - // "id": 3 - // }}, - // {"entity": { - // "color_tag": "brown_9695", - // "id": 7 - // }}, - // {"entity": { - // "color_tag": "brown_2551", - // "id": 15 - // }} - // ]} - ``` - - ```javascript - res = await client.query({ - collection_name: "quick_setup", - filter: 'color == "brown"', - output_fields: ["color_tag"], - limit: 3 - }) - - console.log(res.data) - - // Output - // - // [ - // { - // '$meta': { color: 'brown', tag: 6839, color_tag: 'brown_6839' }, - // id: '22' - // }, - // { - // '$meta': { color: 'brown', tag: 7849, color_tag: 'brown_7849' }, - // id: '32' - // }, - // { - // '$meta': { color: 'brown', tag: 7855, color_tag: 'brown_7855' }, - // id: '33' - // } - // ] - // - ``` - -- Filter entities with their __color__ values not set to __green__ and __purple__. - -
          - Python - Java - Node.js -
          - - ```python - res = client.query( - collection_name="quick_setup", - filter='color not in ["green", "purple"]', - output_fields=["color_tag"], - limit=3 - ) - - print(res) - - # Output - # - # [ - # { - # "color_tag": "yellow_6781", - # "id": 0 - # }, - # { - # "color_tag": "pink_1023", - # "id": 1 - # }, - # { - # "color_tag": "blue_3972", - # "id": 2 - # } - # ] - ``` - - ```java - queryReq = QueryReq.builder() - .collectionName("quick_setup") - .filter("color not in [\"green\", \"purple\"]") - .outputFields(Arrays.asList("color_tag")) - .limit(3) - .build(); - - queryResp = client.query(queryReq); - - System.out.println(JSONObject.toJSON(queryResp)); - - // Output: - // {"queryResults": [ - // {"entity": { - // "color_tag": "white_4597", - // "id": 0 - // }}, - // {"entity": { - // "color_tag": "white_8708", - // "id": 2 - // }}, - // {"entity": { - // "color_tag": "brown_7792", - // "id": 3 - // }} - // ]} - ``` - - ```javascript - res = await client.query({ - collection_name: "quick_setup", - filter: 'color not in ["green", "purple"]', - output_fields: ["color_tag"], - limit: 3 - }) - - console.log(res.data) - - // Output - // - // [ - // { - // '$meta': { color: 'blue', tag: 8907, color_tag: 'blue_8907' }, - // id: '0' - // }, - // { - // '$meta': { color: 'grey', tag: 3710, color_tag: 'grey_3710' }, - // id: '1' - // }, - // { - // '$meta': { color: 'blue', tag: 2993, color_tag: 'blue_2993' }, - // id: '2' - // } - // ] - // - ``` - -- Filter articles whose color tags start with __red__. - -
          - Python - Java - Node.js -
          - - ```python - res = client.query( - collection_name="quick_setup", - filter='color_tag like "red%"', - output_fields=["color_tag"], - limit=3 - ) - - print(res) - - # Output - # - # [ - # { - # "color_tag": "red_6443", - # "id": 17 - # }, - # { - # "color_tag": "red_1483", - # "id": 41 - # }, - # { - # "color_tag": "red_4348", - # "id": 47 - # } - # ] - ``` - - ```java - queryReq = QueryReq.builder() - .collectionName("quick_setup") - .filter("color_tag like \"red%\"") - .outputFields(Arrays.asList("color_tag")) - .limit(3) - .build(); - - queryResp = client.query(queryReq); - - System.out.println(JSONObject.toJSON(queryResp)); - - // Output: - // {"queryResults": [ - // {"entity": { - // "color_tag": "red_4929", - // "id": 9 - // }}, - // {"entity": { - // "color_tag": "red_8284", - // "id": 13 - // }}, - // {"entity": { - // "color_tag": "red_3021", - // "id": 44 - // }} - // ]} - ``` - - ```javascript - res = await client.query({ - collection_name: "quick_setup", - filter: 'color_tag like "red%"', - output_fields: ["color_tag"], - limit: 3 - }) - - console.log(res.data) - - // Output - // - // [ - // { - // '$meta': { color: 'red', tag: 8773, color_tag: 'red_8773' }, - // id: '17' - // }, - // { - // '$meta': { color: 'red', tag: 9197, color_tag: 'red_9197' }, - // id: '34' - // }, - // { - // '$meta': { color: 'red', tag: 7914, color_tag: 'red_7914' }, - // id: '46' - // } - // ] - // - ``` - -- Filter entities with their colors set to red and tag values within the range from 1,000 to 1,500. - -
          - Python - Java - Node.js -
          - - ```python - res = client.query( - collection_name="quick_setup", - filter='(color == "red") and (1000 < tag < 1500)', - output_fields=["color_tag"], - limit=3 - ) - - print(res) - - # Output - # - # [ - # { - # "color_tag": "red_1483", - # "id": 41 - # }, - # { - # "color_tag": "red_1100", - # "id": 94 - # }, - # { - # "color_tag": "red_1343", - # "id": 526 - # } - # ] - ``` - - ```java - queryReq = QueryReq.builder() - .collectionName("quick_setup") - .filter("(color == \"red\") and (1000 < tag < 1500)") - .outputFields(Arrays.asList("color_tag")) - .limit(3) - .build(); - - queryResp = client.query(queryReq); - - System.out.println(JSONObject.toJSON(queryResp)); - - // Output: - // {"queryResults": [ - // {"entity": { - // "color_tag": "red_8124", - // "id": 83 - // }}, - // {"entity": { - // "color_tag": "red_5358", - // "id": 501 - // }}, - // {"entity": { - // "color_tag": "red_3564", - // "id": 638 - // }} - // ]} - ``` - - ```javascript - res = await client.query({ - collection_name: "quick_setup", - filter: '(color == "red") and (1000 < tag < 1500)', - output_fields: ["color_tag"], - limit: 3 - }) - - console.log(res.data) - - // Output - // - // [ - // { - // '$meta': { color: 'red', tag: 1436, color_tag: 'red_1436' }, - // id: '67' - // }, - // { - // '$meta': { color: 'red', tag: 1463, color_tag: 'red_1463' }, - // id: '160' - // }, - // { - // '$meta': { color: 'red', tag: 1073, color_tag: 'red_1073' }, - // id: '291' - // } - // ] - // - ``` - -## Use Advanced Operators - -In this section, you will find examples of how to use advanced operators in scalar filtering. You can apply these filters to [vector searches](https://milvus.io/docs/single-vector-search.md#Filtered-search) and [data deletions](https://milvus.io/docs/insert-update-delete.md#Delete-entities) too. - -### Count entities - -- Counts the total number of entities in a collection. - -
          - Python - Java - Node.js -
          - - ```python - # 7. Use advanced operators - - # Count the total number of entities in a collection - res = client.query( - collection_name="quick_setup", - output_fields=["count(*)"] - ) - - print(res) - - # Output - # - # [ - # { - # "count(*)": 2000 - # } - # ] - ``` - - ```java - // 7. Use advanced operators - // Count the total number of entities in the collection - queryReq = QueryReq.builder() - .collectionName("quick_setup") - .filter("") - .outputFields(Arrays.asList("count(*)")) - .build(); - - queryResp = client.query(queryReq); - - System.out.println(JSONObject.toJSON(queryResp)); - - // Output: - // {"queryResults": [{"entity": {"count(*)": 2000}}]} - ``` - - ```javascript - // 7. Use advanced operators - // Count the total number of entities in a collection - res = await client.query({ - collection_name: "quick_setup", - output_fields: ["count(*)"] - }) - - console.log(res.data) - - // Output - // - // [ { 'count(*)': '2000' } ] - // - ``` - -- Counts the total number of entities in specific partitions. - -
          - Python - Java - Node.js -
          - - ```python - # Count the number of entities in a partition - res = client.query( - collection_name="quick_setup", - output_fields=["count(*)"], - partition_names=["partitionA"] - ) - - print(res) - - # Output - # - # [ - # { - # "count(*)": 500 - # } - # ] - ``` - - ```java - // Count the number of entities in a partition - queryReq = QueryReq.builder() - .collectionName("quick_setup") - .partitionNames(Arrays.asList("partitionA")) - .filter("") - .outputFields(Arrays.asList("count(*)")) - .build(); - - queryResp = client.query(queryReq); - - System.out.println(JSONObject.toJSON(queryResp)); - - // Output: - // {"queryResults": [{"entity": {"count(*)": 500}}]} - ``` - - ```javascript - // Count the number of entities in a partition - res = await client.query({ - collection_name: "quick_setup", - output_fields: ["count(*)"], - partition_names: ["partitionA"] - }) - - console.log(res.data) - - // Output - // - // [ { 'count(*)': '500' } ] - // - ``` - -- Counts the number of entities that match a filtering condition - -
          - Python - Java - Node.js -
          - - ```python - # Count the number of entities that match a specific filter - res = client.query( - collection_name="quick_setup", - filter='(color == "red") and (1000 < tag < 1500)', - output_fields=["count(*)"], - ) - - print(res) - - # Output - # - # [ - # { - # "count(*)": 3 - # } - # ] - ``` - - ```java - // Count the number of entities that match a specific filter - queryReq = QueryReq.builder() - .collectionName("quick_setup") - .filter("(color == \"red\") and (1000 < tag < 1500)") - .outputFields(Arrays.asList("count(*)")) - .build(); - - queryResp = client.query(queryReq); - - System.out.println(JSONObject.toJSON(queryResp)); - - // Output: - // {"queryResults": [{"entity": {"count(*)": 7}}]} - ``` - - ```javascript - // Count the number of entities that match a specific filter - res = await client.query({ - collection_name: "quick_setup", - filter: '(color == "red") and (1000 < tag < 1500)', - output_fields: ["count(*)"] - }) - - console.log(res.data) - - // Output - // - // [ { 'count(*)': '10' } ] - // - ``` - -## Reference on scalar filters - -### Basic Operators - -A __boolean expression__ is always __a string comprising field names joined by operators__. In this section, you will learn more about basic operators. - -| __Operator__ | __Description__ | -| --------------- | ------------------------------------------------------------------------------------------------------------------------------------------- | -| __and (&&)__ | True if both operands are true | -| __or (||)__ | True if either operand is true | -| __+, -, *, /__ | Addition, subtraction, multiplication, and division | -| __**__ | Exponent | -| __%__ | Modulus | -| __<, >__ | Less than, greater than | -| __==, !=__ | Equal to, not equal to | -| __<=, >=__ | Less than or equal to, greater than or equal to | -| __not__ | Reverses the result of a given condition. | -| __like__ | Compares a value to similar values using wildcard operators.
          For example, like "prefix%" matches strings that begin with "prefix". | -| __in__ | Tests if an expression matches any value in a list of values. | - -### Advanced operators - -- `count(*)` - - Counts the exact number of entities in the collection. Use this as an output field to get the exact number of entities in a collection or partition. - -
          - -

          notes

          - -

          This applies to loaded collections. You should use it as the only output field.

          +``` -
          +​ diff --git a/v2.4.x/site/en/userGuide/search-query-get/grouping-search.md b/v2.4.x/site/en/userGuide/search-query-get/grouping-search.md new file mode 100644 index 0000000000..31d43e610a --- /dev/null +++ b/v2.4.x/site/en/userGuide/search-query-get/grouping-search.md @@ -0,0 +1,337 @@ +--- +id: grouping-search.md +summary: A grouping search allows Milvus to group the search results by the values in a specified field to aggregate data at a higher level. For example, you can use a basic ANN search to find books similar to the one at hand, but you can use a grouping search to find the book categories that may involve the topics discussed in that book. This topic describes how to use Grouping Search along with key considerations.​ +title: Grouping Search +--- + +# Grouping Search​ + +A grouping search allows Milvus to group the search results by the values in a specified field to aggregate data at a higher level. For example, you can use a basic ANN search to find books similar to the one at hand, but you can use a grouping search to find the book categories that may involve the topics discussed in that book. This topic describes how to use Grouping Search along with key considerations.​ + +## Overview​ + +When entities in the search results share the same value in a scalar field, this indicates that they are similar in a particular attribute, which may negatively impact the search results.​ + +Assume a collection stores multiple documents (denoted by **docId**). To retain as much semantic information as possible when converting documents into vectors, each document is split into smaller, manageable paragraphs (or **chunks**) and stored as separate entities. Even though the document is divided into smaller sections, users are often still interested in identifying which documents are most relevant to their needs.​ + +![ANN Search](../../../../assets/ann-search.png) + +When performing an Approximate Nearest Neighbor (ANN) search on such a collection, the search results may include several paragraphs from the same document, potentially causing other documents to be overlooked, which may not align with the intended use case.​ + +![Grouping Search](../../../../assets/grouping-search.png) + +To improve the diversity of search results, you can add the `group_by_field` parameter in the search request to enable Grouping Search. As shown in the diagram, you can set `group_by_field` to `docId`. Upon receiving this request, Milvus will:​ + +- Perform an ANN search based on the provided query vector to find all entities most similar to the query.​ + +- Group the search results by the specified `group_by_field`, such as `docId`.​ + +- Return the top results for each group, as defined by the `limit` parameter, with the most similar entity from each group.​ + +
          + +By default, Grouping Search returns only one entity per group. If you want to increase the number of results to return per group, you can control this with the `group_size` and `strict_group_size` parameters.​ + +
          + +## Perform Grouping Search​ + +This section provides example code to demonstrate the use of Grouping Search. The following example assumes the collection includes fields for `id`, `vector`, `chunk`, and `docId`.​ + +```json +[​ + {"id": 0, "vector": [0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592], "chunk": "pink_8682", "docId": 1},​ + {"id": 1, "vector": [0.19886812562848388, 0.06023560599112088, 0.6976963061752597, 0.2614474506242501, 0.838729485096104], "chunk": "red_7025", "docId": 5},​ + {"id": 2, "vector": [0.43742130801983836, -0.5597502546264526, 0.6457887650909682, 0.7894058910881185, 0.20785793220625592], "chunk": "orange_6781", "docId": 2},​ + {"id": 3, "vector": [0.3172005263489739, 0.9719044792798428, -0.36981146090600725, -0.4860894583077995, 0.95791889146345], "chunk": "pink_9298", "docId": 3},​ + {"id": 4, "vector": [0.4452349528804562, -0.8757026943054742, 0.8220779437047674, 0.46406290649483184, 0.30337481143159106], "chunk": "red_4794", "docId": 3},​ + {"id": 5, "vector": [0.985825131989184, -0.8144651566660419, 0.6299267002202009, 0.1206906911183383, -0.1446277761879955], "chunk": "yellow_4222", "docId": 4},​ + {"id": 6, "vector": [0.8371977790571115, -0.015764369584852833, -0.31062937026679327, -0.562666951622192, -0.8984947637863987], "chunk": "red_9392", "docId": 1},​ + {"id": 7, "vector": [-0.33445148015177995, -0.2567135004164067, 0.8987539745369246, 0.9402995886420709, 0.5378064918413052], "chunk": "grey_8510", "docId": 2},​ + {"id": 8, "vector": [0.39524717779832685, 0.4000257286739164, -0.5890507376891594, -0.8650502298996872, -0.6140360785406336], "chunk": "white_9381", "docId": 5},​ + {"id": 9, "vector": [0.5718280481994695, 0.24070317428066512, -0.3737913482606834, -0.06726932177492717, -0.6980531615588608], "chunk": "purple_4976", "docId": 3},​ +]​ +​ + +``` + +In the search request, set both `group_by_field` and `output_fields` to `docId`. Milvus will group the results by the specified field and return the most similar entity from each group, including the value of `docId` for each returned entity.​ + +
          + Python + Java + Node.js + cURL +
          + +```python +from pymilvus import MilvusClient​ +​ +client = MilvusClient(​ + uri="http://localhost:19530",​ + token="root:Milvus"​ +)​ +​ +query_vectors = [​ + [0.14529211512077012, 0.9147257273453546, 0.7965055218724449, 0.7009258593102812, 0.5605206522382088]]​ +​ +# Group search results​ +res = client.search(​ + collection_name="group_search_collection",​ + data=query_vectors,​ + limit=3,​ + group_by_field="docId",​ + output_fields=["docId"]​ +)​ +​ +# Retrieve the values in the `docId` column​ +doc_ids = [result['entity']['docId'] for result in res[0]]​ + +``` + +```java +import io.milvus.v2.client.ConnectConfig;​ +import io.milvus.v2.client.MilvusClientV2;​ +import io.milvus.v2.service.vector.request.SearchReq​ +import io.milvus.v2.service.vector.request.data.FloatVec;​ +import io.milvus.v2.service.vector.response.SearchResp​ +​ +MilvusClientV2 client = new MilvusClientV2(ConnectConfig.builder()​ + .uri("http://localhost:19530")​ + .token("root:Milvus")​ + .build());​ +​ +FloatVec queryVector = new FloatVec(new float[]{0.14529211512077012f, 0.9147257273453546f, 0.7965055218724449f, 0.7009258593102812f, 0.5605206522382088f});​ +SearchReq searchReq = SearchReq.builder()​ + .collectionName("group_search_collection")​ + .data(Collections.singletonList(queryVector))​ + .topK(3)​ + .groupByFieldName("docId")​ + .outputFields(Collections.singletonList("docId"))​ + .build();​ +​ +SearchResp searchResp = client.search(searchReq);​ +​ +List> searchResults = searchResp.getSearchResults();​ +for (List results : searchResults) {​ + System.out.println("TopK results:");​ + for (SearchResp.SearchResult result : results) {​ + System.out.println(result);​ + }​ +}​ +​ +// Output​ +// TopK results:​ +// SearchResp.SearchResult(entity={docId=5}, score=0.74767184, id=1)​ +// SearchResp.SearchResult(entity={docId=2}, score=0.6254269, id=7)​ +// SearchResp.SearchResult(entity={docId=3}, score=0.3611898, id=3)​ + +``` + +```go +// nolint​ +func ExampleClient_Search_grouping() {​ + ctx, cancel := context.WithCancel(context.Background())​ + defer cancel()​ +​ + milvusAddr := "127.0.0.1:19530"​ + token := "root:Milvus"​ +​ + cli, err := client.New(ctx, &client.ClientConfig{​ + Address: milvusAddr,​ + APIKey: token,​ + })​ + if err != nil {​ + log.Fatal("failed to connect to milvus server: ", err.Error())​ + }​ +​ + defer cli.Close(ctx)​ +​ + queryVector := []float32{0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592}​ +​ + resultSets, err := cli.Search(ctx, client.NewSearchOption(​ + "my_collection", // collectionName​ + 3, // limit​ + []entity.Vector{entity.FloatVector(queryVector)},​ + ).WithGroupByField("docId"))​ + if err != nil {​ + log.Fatal("failed to perform basic ANN search collection: ", err.Error())​ + }​ +​ + for _, resultSet := range resultSets {​ + log.Println("IDs: ", resultSet.IDs)​ + log.Println("Scores: ", resultSet.Scores)​ + }​ + // Output:​ + // IDs:​ + // Scores:​ +}​ + +``` + +```javascript +import { MilvusClient, DataType } from "@zilliz/milvus2-sdk-node";​ +​ +const address = "http://localhost:19530";​ +const token = "root:Milvus";​ +const client = new MilvusClient({address, token});​ +​ +var query_vector = [0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592]​ +​ +res = await client.search({​ + collection_name: "my_collection",​ + data: [query_vector],​ + limit: 3,​ + // highlight-start​ + group_by_field: "docId"​ + // highlight-end​ +})​ +​ +// Retrieve the values in the `docId` column​ +var docIds = res.results.map(result => result.entity.docId)​ + +``` + +```curl +export CLUSTER_ENDPOINT="http://localhost:19530"​ +export TOKEN="root:Milvus"​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/search" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "group_search_collection",​ + "data": [​ + [0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592]​ + ],​ + "annsField": "vector",​ + "limit": 3,​ + "groupingField": "docId",​ + "outputFields": ["docId"]​ +}'​ + +``` + +In the request above, `limit=3` indicates that the system will return search results from three groups, with each group containing the single most similar entity to the query vector.​ + +## Configure group size​ + +By default, Grouping Search returns only one entity per group. If you want multiple results per group, adjust the `group_size` and `strict_group_size` parameters.​ + +
          + Python + Java + Node.js + cURL +
          + +```python +# Group search results​ +​ +res = client.search(​ + collection_name="group_search_collection", ​ + data=query_vectors, # Query vector + limit=5, # Top K results​ to return​ + group_by_field="docId", # Group by docId​ + group_size=2, # Return 2 entities per group​ + strict_group_size=True, # Ensure each group has 2 entities​ + output_fields=["docId"]​ +)​ + +``` + +```java +FloatVec queryVector = new FloatVec(new float[]{0.14529211512077012f, 0.9147257273453546f, 0.7965055218724449f, 0.7009258593102812f, 0.5605206522382088f});​ +SearchReq searchReq = SearchReq.builder()​ + .collectionName("group_search_collection")​ + .data(Collections.singletonList(queryVector))​ + .topK(5)​ + .groupByFieldName("docId")​ + .groupSize(2)​ + .strictGroupSize(true)​ + .outputFields(Collections.singletonList("docId"))​ + .build();​ +​ +SearchResp searchResp = client.search(searchReq);​ +​ +List> searchResults = searchResp.getSearchResults();​ +for (List results : searchResults) {​ + System.out.println("TopK results:");​ + for (SearchResp.SearchResult result : results) {​ + System.out.println(result);​ + }​ +}​ +​ +// Output​ +// TopK results:​ +// SearchResp.SearchResult(entity={docId=5}, score=0.74767184, id=1)​ +// SearchResp.SearchResult(entity={docId=5}, score=-0.49148706, id=8)​ +// SearchResp.SearchResult(entity={docId=2}, score=0.6254269, id=7)​ +// SearchResp.SearchResult(entity={docId=2}, score=0.38515577, id=2)​ +// SearchResp.SearchResult(entity={docId=3}, score=0.3611898, id=3)​ +// SearchResp.SearchResult(entity={docId=3}, score=0.19556211, id=4)​ + +``` + +```javascript +import { MilvusClient, DataType } from "@zilliz/milvus2-sdk-node";​ +​ +const address = "http://localhost:19530";​ +const token = "root:Milvus";​ +const client = new MilvusClient({address, token});​ +​ +var query_vector = [0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592]​ +​ +res = await client.search({​ + collection_name: "my_collection",​ + data: [query_vector],​ + limit: 3,​ + group_by_field: "docId",​ + // highlight-start​ + group_size: 2,​ + strict_group_size: true​ + // highlight-end​ +})​ +​ +// Retrieve the values in the `docId` column​ +var docIds = res.results.map(result => result.entity.docId)​ + +``` + +```curl +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/search" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "group_search_collection",​ + "data": [​ + [0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592]​ + ],​ + "annsField": "vector",​ + "limit": 5,​ + "groupingField": "docId",​ + "groupSize":2,​ + "strictGroupSize":true,​ + "outputFields": ["docId"]​ +}'​ + +``` + +In the example above:​ + +- `group_size`: Specifies the desired number of entities to return per group. For instance, setting `group_size=2` means each group (or each `docId`) should ideally return two of the most similar paragraphs (or **chunks**). If `group_size` is not set, the system defaults to returning one result per group.​ + +- `strict_group_size`: This boolean parameter controls whether the system should strictly enforce the count set by `group_size`. When `strict_group_size=True`, the system will attempt to include the exact number of entities specified by `group_size` in each group (e.g., two paragraphs), unless there isn’t enough data in that group. By default (`strict_group_size=False`), the system prioritizes meeting the number of groups specified by the `limit` parameter, rather than ensuring each group contains `group_size` entities. This approach is generally more efficient in cases where data distribution is uneven.​ + +For additional parameter details, refer to [​search()](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Vector/search.md).​ + +## Considerations​ + +- **Number of groups**: The `limit` parameter controls the number of groups from which search results are returned, rather than the specific number of entities within each group. Setting an appropriate `limit` helps control search diversity and query performance. Reducing `limit` can reduce computation costs if data is densely distributed or performance is a concern.​ + +- **Entities per group**: The `group_size` parameter controls the number of entities returned per group. Adjusting `group_size` based on your use case can increase the richness of search results. However, if data is unevenly distributed, some groups may return fewer entities than specified by `group_size`, particularly in limited data scenarios.​ + +- **Strict group size**: When `strict_group_size=True`, the system will attempt to return the specified number of entities (`group_size`) for each group, unless there isn’t enough data in that group. This setting ensures consistent entity counts per group but may lead to performance degradation with uneven data distribution or limited resources. If strict entity counts aren’t required, setting `strict_group_size=False` can improve query speed.​ + diff --git a/v2.4.x/site/en/userGuide/search-query-get/keyword-match.md b/v2.4.x/site/en/userGuide/search-query-get/keyword-match.md new file mode 100644 index 0000000000..6c660eaff3 --- /dev/null +++ b/v2.4.x/site/en/userGuide/search-query-get/keyword-match.md @@ -0,0 +1,154 @@ +--- +id: keyword-match.md +summary: "Keyword match in Milvus enables precise document retrieval based on specific terms. This feature is primarily used for filtered search to satisfy specific conditions and can incorporate scalar filtering to refine query results, allowing similarity searches within vectors that meet scalar criteria.​" +title: Keyword Match​ +--- + +# Keyword Match​ + +Keyword match in Milvus enables precise document retrieval based on specific terms. This feature is primarily used for filtered search to satisfy specific conditions and can incorporate scalar filtering to refine query results, allowing similarity searches within vectors that meet scalar criteria.​ + +
          + +Keyword match focuses on finding exact occurrences of the query terms, without scoring the relevance of the matched documents. If you want to retrieve the most relevant documents based on the semantic meaning and importance of the query terms, we recommend you use [​Full Text Search](full-text-search.md).​ + +
          + +## Overview + +Milvus integrates [Tantivy](https://github.com/quickwit-oss/tantivy) to power its underlying inverted index and keyword search. For each text entry, Milvus indexes it following the procedure:​ + +1. [Analyzer](analyzer-overview.md): The analyzer processes input text by tokenizing it into individual words, or tokens, and then applying filters as needed. This allows Milvus to build an index based on these tokens.​ + +2. [Indexing](index-scalar-fields.md): After text analysis, Milvus creates an inverted index that maps each unique token to the documents containing it.​ + +When a user performs a keyword match, the inverted index is used to quickly retrieve all documents containing the keywords. This is much faster than scanning through each document individually.​ + +![Keyword Match](../../../assets/keyword-match.png) + +## Enable keyword match + +Keyword match works on the `VARCHAR` field type, which is essentially the string data type in Milvus. To enable keyword match, set both `enable_analyzer` and `enable_match` to `True` and then optionally configure an analyzer for text analysis when defining your collection schema.​ + +### Set `enable_analyzer` and `enable_match`​ + +To enable keyword match for a specific `VARCHAR` field, set both the `enable_analyzer` and `enable_match` parameters to `True` when defining the field schema. This instructs Milvus to tokenize text and create an inverted index for the specified field, allowing fast and efficient keyword matches.​ + +```python +from pymilvus import MilvusClient, DataType​ +​ +schema = MilvusClient.create_schema(auto_id=True, enable_dynamic_field=False)​ +​ +schema.add_field(​ + field_name='text', ​ + datatype=DataType.VARCHAR, ​ + max_length=1000, ​ + enable_analyzer=True, # Whether to enable text analysis for this field​ + enable_match=True # Whether to enable text match​ +)​ + +``` + +### Optional: Configure an analyzer​ + +The performance and accuracy of keyword matching depend on the selected analyzer. Different analyzers are tailored to various languages and text structures, so choosing the right one can significantly impact search results for your specific use case.​ + +By default, Milvus uses the `standard` analyzer, which tokenizes text based on whitespace and punctuation, removes tokens longer than 40 characters, and converts text to lowercase. No additional parameters are needed to apply this default setting. For more information, refer to [​Standard](standard-analyzer.md).​ + +In cases where a different analyzer is required, you can configure one using the `analyzer_params` parameter. For example, to apply the `english` analyzer for processing English text:​ + +```python +analyzer_params={​ + "type": "english"​ +}​ +​ +schema.add_field(​ + field_name='text', ​ + datatype=DataType.VARCHAR, ​ + max_length=200, ​ + enable_analyzer=True，​ + analyzer_params=analyzer_params,​ + enable_match=True, ​ +)​ + +``` + +Milvus also provides various other analyzers suited to different languages and scenarios. For more details, refer to [​Overview](analyzer-overview.md).​ + +## Use keyword match + +Once you have enabled keyword match for a VARCHAR field in your collection schema, you can perform keyword matches using the `TEXT_MATCH` expression.​ + +### TEXT_MATCH expression syntax​ + +The `TEXT_MATCH` expression is used to specify the field and the keywords to search for. Its syntax is as follows:​ + +```python +TEXT_MATCH(field_name, text)​ + +``` + +- `field_name`: The name of the VARCHAR field to search for.​ + +- `text`: The keywords to search for. Multiple keywords can be separated by spaces or other appropriate delimiters based on the language and configured analyzer.​ + +By default, `TEXT_MATCH` uses the **OR** matching logic, meaning it will return documents that contain any of the specified keywords. For example, to search for documents containing the keywords `machine` or `deep` in the `text` field, use the following expression:​ + +```python +filter = "TEXT_MATCH(text, 'machine deep')"​ + +``` + +You can also combine multiple `TEXT_MATCH` expressions using logical operators to perform **AND** matching. For example, to search for documents containing both `machine` and `deep` in the `text` field, use the following expression:​ + +```python +filter = "TEXT_MATCH(text, 'machine') and TEXT_MATCH(text, 'deep')"​ + +``` + +### Search with keyword match​ + +Keyword match can be used in combination with vector similarity search to narrow the search scope and improve search performance. By filtering the collection using keyword match before vector similarity search, you can reduce the number of documents that need to be searched, resulting in faster query times.​ + +In this example, the `filter` expression filters the search results to only include documents that match the specified keywords `keyword1` or `keyword2`. The vector similarity search is then performed on this filtered subset of documents.​ + +```python +# Match entities with `keyword1` or `keyword2`​ +filter = "TEXT_MATCH(text, 'keyword1 keyword2')"​ +​ +# Assuming 'embeddings' is the vector field and 'text' is the VARCHAR field​ +result = MilvusClient.search(​ + collection_name="YOUR_COLLECTION_NAME", # Your collection name​ + anns_field="embeddings", # Vector field name​ + data=[query_vector], # Query vector​ + filter=filter,​ + search_params={"params": {"nprobe": 10}},​ + limit=10, # Max. number of results to return​ + output_fields=["id", "text"] # Fields to return​ +)​ + +``` + +### Query with keyword match​ + +Keyword match can also be used for scalar filtering in query operations. By specifying a `TEXT_MATCH` expression in the `expr` parameter of the `query()` method, you can retrieve documents that match the given keywords.​ + +The example below retrieves documents where the `text` field contains both keywords `keyword1` and `keyword2`.​ + +```python +# Match entities with both `keyword1` and `keyword2`​ +filter = "TEXT_MATCH(text, 'keyword1') and TEXT_MATCH(text, 'keyword2')"​ +​ +result = MilvusClient.query(​ + collection_name="YOUR_COLLECTION_NAME",​ + filter=filter, ​ + output_fields=["id", "text"]​ +)​ + +``` + +## Considerations + +- Enabling keyword matching for a field triggers the creation of an inverted index, which consumes storage resources. Consider storage impact when deciding to enable this feature, as it varies based on text size, unique tokens, and the analyzer used.​ + +- Once you've defined an analyzer in your schema, its settings become permanent for that collection. If you decide that a different analyzer would better suit your needs, you may consider dropping the existing collection and creating a new one with the desired analyzer configuration.​ \ No newline at end of file diff --git a/v2.4.x/site/en/userGuide/search-query-get/metric.md b/v2.4.x/site/en/userGuide/search-query-get/metric.md new file mode 100644 index 0000000000..9e585a11e2 --- /dev/null +++ b/v2.4.x/site/en/userGuide/search-query-get/metric.md @@ -0,0 +1,232 @@ +--- +id: metric.md +summary: Similarity metrics are used to measure similarities among vectors. Choosing an appropriate distance metric helps improve classification and clustering performance significantly.​ +title: Metric Types +--- + +# Metric Types​ + +Similarity metrics are used to measure similarities among vectors. Choosing an appropriate distance metric helps improve classification and clustering performance significantly.​ + +Currently, Zilliz Cloud supports these types of similarity Metrics: Euclidean distance (`L2`), Inner Product (`IP`), Cosine Similarity (`COSINE`), `JACCARD`, `HAMMING`, and `BM25` (specifically designed for full text search on sparse vectors).​ + +The table below summarizes the mapping between different field types and their corresponding metric types.​ + +

          Field Type​

          + +

          Dimension Range​

          + +

          Supported Metric Types​

          + +

          Default Metric Type​

          + +

          FLOAT_VECTOR​

          + +

          2-32,768​

          + +

          COSINE, L2, IP​

          + +

          COSINE​

          + +

          FLOAT16_VECTOR​

          + +

          2-32,768​

          + +

          COSINE, L2, IP​

          + +

          ​

          + +

          COSINE​

          + +

          BFLOAT16_VECTOR​

          + +

          2-32,768​

          + +

          COSINE, L2, IP​

          + +

          COSINE​

          + +

          SPARSE_FLOAT_VECTOR​

          + +

          No need to specify the dimension.​

          + +

          IP, BM25 (used only for full text search)​

          + +

          IP​

          + +

          BINARY_VECTOR​

          + +

          8-32,768*8​

          + +

          ​

          + +

          HAMMING, JACCARD​

          + +

          HAMMING​

          + +
          + +
          + +- For vector fields of the `SPARSE_FLOAT_VECTOR` type, use the `BM25` metric type only when performing full text search. For more information, refer to [​Full Text Search](full-text-search.md).​ + +- For vector fields of the `BINARY_VECTOR` type, the dimension value (`dim`) must be a multiple of 8. ​ + +
          + +The table below summarizes the characteristics of the similarity distance values of all supported metric types and their value range.​ + +

          Metric Type​

          + +

          Characteristics of the Similarity Distance Values​

          + +

          Similarity Distance Value Range​

          + +

          L2​

          + +

          A smaller value indicates a greater similarity.​

          + +

          [0, ∞)​

          + +

          IP​

          + +

          A greater value indicates a greater similarity.​

          + +

          [-1, 1]​

          + +

          COSINE​

          + +

          A greater value indicates a greater similarity.​

          + +

          [-1, 1]​

          + +

          JACCARD ​

          + +

          A smaller value indicates a greater similarity.​

          + +

          [0, 1]​

          + +

          HAMMING ​

          + +

          A smaller value indicates a greater similarity.​

          + +

          [0, dim(vector)]​

          + +

          BM25​

          + +

          Score the relevance based on the term frequency, inverted document frequency, and document normalization.​

          + +

          [0, ∞)​

          + +

          ​

          + +
          + +## Euclidean distance (L2)​ + +Essentially, Euclidean distance measures the length of a segment that connects 2 points.​ + +The formula for Euclidean distance is as follows:​ + +![Euclidean distance formula](../../../../assets/euclidean_metric.png) + +where **a = (a0, a1,..., an-1)** and **b = (b0, b1,..., bn-1)** are two points in n-dimensional Euclidean space.​ + +It's the most commonly used distance metric and is very useful when the data are continuous.​ + +
          + +Zilliz Cloud only calculates the value before applying the square root when Euclidean distance is chosen as the distance metric.​ + +
          + +## Inner product (IP)​ + +The IP distance between two embeddings is defined as follows:​ + +![Inner product formula](../../../../assets/IP_formula.png) + +IP is more useful if you need to compare non-normalized data or when you care about magnitude and angle.​ + +
          + +If you use IP to calculate similarities between embeddings, you must normalize your embeddings. After normalization, the inner product equals cosine similarity.​ + +
          + +Suppose X' is normalized from embedding X:​ + +![Normalized inner product formula](../../../../assets/normalize_formula.png) + +The correlation between the two embeddings is as follows:​ + +![Correlation between embeddings](../../../../assets/normalization_formula.png) + +## Cosine similarity ​ + +Cosine similarity uses the cosine of the angle between two sets of vectors to measure how similar they are. You can think of the two sets of vectors as line segments starting from the same point, such as [0,0,...], but pointing in different directions.​ + +To calculate the cosine similarity between two sets of vectors **A = (a0, a1,..., an-1)** and **B = (b0, b1,..., bn-1)**, use the following formula:​ + +![Cosine similarity formula](../../../../assets/cosine_similarity.png) + +The cosine similarity is always in the interval **[-1, 1]**. For example, two proportional vectors have a cosine similarity of **1**, two orthogonal vectors have a similarity of **0**, and two opposite vectors have a similarity of **-1**. The larger the cosine, the smaller the angle between the two vectors, indicating that these two vectors are more similar to each other.​ + +By subtracting their cosine similarity from 1, you can get the cosine distance between two vectors.​ + +## JACCARD distance​ + +JACCARD similarity coefficient measures the similarity between two sample sets and is defined as the cardinality of the intersection of the defined sets divided by the cardinality of the union of them. It can only be applied to finite sample sets.​ + +![JACCARD similarity coefficient formula](../../../../assets/jaccard_coeff.png) + +JACCARD distance measures the dissimilarity between data sets and is obtained by subtracting the JACCARD similarity coefficient from 1. For binary variables, JACCARD distance is equivalent to the Tanimoto coefficient.​ + +![JACCARD distance formula](../../../../assets/jaccard_dist.png) + +## HAMMING distance​ + +HAMMING distance measures binary data strings. The distance between two strings of equal length is the number of bit positions at which the bits are different.​ + +For example, suppose there are two strings, 1101 1001 and 1001 1101.​ + +11011001 ⊕ 10011101 = 01000100. Since, this contains two 1s, the HAMMING distance, d (11011001, 10011101) = 2.​ + +## BM25 similarity​ + +BM25 is a widely used text relevance measurement method, specifically designed for [full text search](full-text-search.md). It combines the following three key factors:​ + +- **Term Frequency (TF):** Measures how frequently a term appears in a document. While higher frequencies often indicate greater importance, BM25 uses the saturation parameter ​k1​ to prevent overly frequent terms from dominating the relevance score.​ + +- **Inverse Document Frequency (IDF):** Reflects the importance of a term across the entire corpus. Terms appearing in fewer documents receive a higher IDF value, indicating greater contribution to relevance.​ + +- **Document Length **Normalization**:** Longer documents tend to score higher due to containing more terms. BM25 mitigates this bias by normalizing document lengths, with parameter ​b controlling the strength of this normalization.​ + +The BM25 scoring is calculated as follows:​ + +![BM25 similarity formula](../../../../assets/bm25.png) + +Parameter description:​ + +- `​Q`: The query text provided by the user.​ + +- `​D`: The document being evaluated.​ + +- `​TF(qi​,D)`: Term frequency, representing how often term ​qi​appears in document ​D.​ + +- `​IDF(qi​)`: Inverse document frequency, calculated as:​ + + ![IDF formula](../../../../assets/idf.png) + + where `​N` is the total number of documents in the corpus, and `​n(qi​)` is the number of documents containing term ​qi​.​ + +- `​∣D∣`: Length of document `​D` (total number of terms).​ + +- `​avgdl`: Average length of all documents in the corpus.​ + +- `​k1​`: Controls the influence of term frequency on the score. Higher values increase the importance of term frequency. The typical range is [1.2, 2.0], while Milvus allows a range of [0, 3].​ + +- `​b`: Controls the degree of length normalization, ranging from 0 to 1. When the value is 0, no normalization is applied; when the value is 1, full normalization is applied.​ + +​ + diff --git a/v2.4.x/site/en/reference/mmap.md b/v2.4.x/site/en/userGuide/search-query-get/mmap.md similarity index 100% rename from v2.4.x/site/en/reference/mmap.md rename to v2.4.x/site/en/userGuide/search-query-get/mmap.md diff --git a/v2.4.x/site/en/userGuide/search-query-get/multi-vector-search.md b/v2.4.x/site/en/userGuide/search-query-get/multi-vector-search.md index bbaa82f846..1aa8cbde15 100644 --- a/v2.4.x/site/en/userGuide/search-query-get/multi-vector-search.md +++ b/v2.4.x/site/en/userGuide/search-query-get/multi-vector-search.md @@ -5,243 +5,788 @@ summary: This guide demonstrates how to perform hybrid search in Milvus and unde title: Hybrid Search --- -# Hybrid Search +# Hybrid Search​ -Since Milvus 2.4, we introduced multi-vector support and a hybrid search framework, which means users can bring in several vector fields (up to 10) into a single collection. These vectors in different columns represent diverse facets of data, originating from different embedding models or undergoing distinct processing methods. The results of hybrid searches are integrated using reranking strategies, such as Reciprocal Rank Fusion (RRF) and Weighted Scoring. To learn more about reranking strategies, refer to [Reranking](reranking.md). +Hybrid Search refers to a search method that conducts multiple ANN searches simultaneously, reranks multiple sets of results from these ANN searches, and ultimately returns a single set of results. Using Hybrid Search can enhance the search accuracy. Zilliz supports conducting Hybrid Search on a collection with multiple vector fields. ​ -This feature is particularly useful in comprehensive search scenarios, such as identifying the most similar person in a vector library based on various attributes like pictures, voice, fingerprints, etc. +Hybrid Search is most commonly used in scenarios including sparse-dense vector searches and multimodal searches. This guide will demonstrate how to conduct a Hybrid Search in Zilliz with a specific example.​ -In this tutorial, you will learn how to: +## Scenarios​ -- Create multiple `AnnSearchRequest` instances for similarity searches on different vector fields; +Hybrid Search is suitable for the following two scenarios:​ -- Configure a reranking strategy to combine and rerank search results from multiple `AnnSearchRequest` instances; +### Sparse-Dense Vector Search​ -- Use the [`hybrid_search()`](https://milvus.io/api-reference/pymilvus/v2.4.x/ORM/Collection/hybrid_search.md) method to perform a hybrid search. +Different types of vectors can represent different information, and using various embedding models can more comprehensively represent different features and aspects of the data. For example, using different embedding models for the same sentence can generate a dense vector to represent the semantic meaning and a sparse vector to represent the word frequency in the sentence.​ -
          +- **Sparse vectors:** Sparse vectors are characterized by their high vector dimensionality and the presence of few non-zero values. This structure makes them particularly well-suited for traditional information retrieval applications. In most cases, the number of dimensions used in sparse vectors correspond to different tokens across one or more languages. Each dimension is assigned a value that indicates the relative importance of that token within the document. This layout proves advantageous for tasks that involve keyword matching.​ -The code snippets on this page use the [PyMilvus ORM module](https://milvus.io/api-reference/pymilvus/v2.4.x/ORM/Connections/connect.md) to interact with Milvus. Code snippets with the new [MilvusClient SDK](https://milvus.io/api-reference/pymilvus/v2.4.x/About.md) will be available soon. +- **Dense vectors:** Dense vectors are embeddings derived from neural networks. When arranged in an ordered array, these vectors capture the semantic essence of the input text. Note that dense vectors are not limited to text processing; they are also extensively used in computer vision to represent the semantics of visual data. These dense vectors, usually generated by text embedding models, are characterized by most or all elements being non-zero. Thus, dense vectors are particularly effective for semantic search applications, as they can return the most similar results based on vector distance even in the absence of exact keyword matches. This capability allows for more nuanced and context-aware search results, often capturing relationships between concepts that might be missed by keyword-based approaches.​ -
          +For more details, refer to [​Sparse Vector](sparse_vector.md) and [​Dense Vector](dense-vector.md).​ + +### Multimodal Search​ + +Multimodal search refers to the similarity search of unstructured data across multiple modalities (such as images, videos, audio, text, etc). For instance, a person can be represented using various modalities of data such as fingerprints, voiceprints, and facial features. Hybrid Search supports multiple searches simultaneously. For example searching a person with both similar fingerprints and voiceprints.​ + +## Workflow​ + +The main workflow for conducting a Hybrid Search is as follows:​ + +1. Generate dense vectors through embedding models like [BERT](https://zilliz.com/learn/explore-colbert-token-level-embedding-and-ranking-model-for-similarity-search#A-Quick-Recap-of-BERT) and [Transformers](https://zilliz.com/learn/NLP-essentials-understanding-transformers-in-AI).​ + +2. Generate sparse vectors through embedding models like [BM25](https://zilliz.com/learn/mastering-bm25-a-deep-dive-into-the-algorithm-and-application-in-milvus), [BGE-M3](https://zilliz.com/learn/bge-m3-and-splade-two-machine-learning-models-for-generating-sparse-embeddings#BGE-M3), [SPLADE](https://zilliz.com/learn/bge-m3-and-splade-two-machine-learning-models-for-generating-sparse-embeddings#SPLADE), etc.​ + +3. Create a collection in Zilliz and define the collection schema which includes both dense and sparse vector fields.​ + +4. Insert sparse-dense vectors into the collection just created in the previous step.​ + +5. Conduct a Hybrid Search: ANN Search on dense vectors will return a set of top-K most similar results, and text match on sparse vectors will also return a set of top-K results.​ + +6. Normalization: Normalize the scores of the two sets of top-K results, converting the scores to a range between [0,1].​ + +7. Choose an appropriate reranking strategy to merge and rerank the two sets of top-K results and ultimately return a final set of top-K results.​ + +![Hybrid Search Workflow](../../../../assets/hybrid-search.png) -## Preparations +## Examples​ -Before starting a hybrid search, ensure you have a collection with multiple vector fields. Currently, Milvus introduces a default of four vector fields per collection, which can be extended to a maximum of ten by modifying the [proxy.maxVectorFieldNum](https://milvus.io/docs/configure_proxy.md#proxymaxVectorFieldNum) configuration. +This section will use a specific example to illustrate how to conduct a Hybrid Search on sparse-dense vectors to enhance the accuracy of text searches.​ -Below is an example of creating a collection named `test_collection` with two vector fields, `filmVector` and `posterVector`, and inserting random entities into it. +### Create a collection with multiple vector fields​ + +The process of creating a collection includes three parts: defining the collection schema, configuring the index parameters, and creating the collection.​ + +#### Define schema​ + +In this example, multiple vector fields need to be defined within the collection schema. Currently, each collection can include up to 4 vector fields by default. But you can also modify the value of [`proxy.maxVectorFieldNum`](https://milvus.io/docs/configure_proxy.md#proxymaxVectorFieldNum) to include up to 10 vector fields in a collection as needed.​ + +The following example defines a collection schema, where `dense` and `sparse` are the two vector fields:​ + +- `id`: This field serves as the primary key for storing text IDs. The data type of this field is INT64.​ + +- `text`: This field is used for storing textual content. The data type of this field is VARCHAR, with a maximum length of 1000 characters.​ + +- `dense`: This field is used to store the dense vectors of the texts. The data type of this field is FLOAT_VECTOR, with a vector dimension of 768.​ + +- `sparse`: This field is used to store the sparse vectors of the texts. The data type of this field is SPARSE_FLOAT_VECTOR.​ + +
          + Python + Java + Node.js + cURL +
          ```python -from pymilvus import connections, Collection, FieldSchema, CollectionSchema, DataType -import random - -# Connect to Milvus -connections.connect( - host="127.0.0.1", # Replace with your Milvus server IP - port="19530" -) - -# Create schema -fields = [ - FieldSchema(name="film_id", dtype=DataType.INT64, is_primary=True), - FieldSchema(name="filmVector", dtype=DataType.FLOAT_VECTOR, dim=5), # Vector field for film vectors - FieldSchema(name="posterVector", dtype=DataType.FLOAT_VECTOR, dim=5)] # Vector field for poster vectors - -schema = CollectionSchema(fields=fields,enable_dynamic_field=False) - -# Create collection -collection = Collection(name="test_collection", schema=schema) - -# Create index for each vector field -index_params = { - "metric_type": "L2", - "index_type": "IVF_FLAT", - "params": {"nlist": 128}, -} - -collection.create_index("filmVector", index_params) -collection.create_index("posterVector", index_params) - -# Generate random entities to insert -entities = [] - -for _ in range(1000): - # generate random values for each field in the schema - film_id = random.randint(1, 1000) - film_vector = [ random.random() for _ in range(5) ] - poster_vector = [ random.random() for _ in range(5) ] - - # create a dictionary for each entity - entity = { - "film_id": film_id, - "filmVector": film_vector, - "posterVector": poster_vector - } - - # add the entity to the list - entities.append(entity) - -collection.insert(entities) +# Create a collection in customized setup mode​ +from pymilvus import (​ + MilvusClient, DataType​ +)​ +​ +client = MilvusClient(​ + uri="http://localhost:19530",​ + token="root:Milvus"​ +)​ +​ +# Create schema​ +schema = MilvusClient.create_schema(​ + auto_id=False,​ + enable_dynamic_field=True,​ +)​ +# Add fields to schema​ +schema.add_field(field_name="id", datatype=DataType.INT64, is_primary=True)​ +schema.add_field(field_name="text", datatype=DataType.VARCHAR, max_length=1000)​ +schema.add_field(field_name="sparse", datatype=DataType.SPARSE_FLOAT_VECTOR)​ +schema.add_field(field_name="dense", datatype=DataType.FLOAT_VECTOR, dim=5)​ + ``` -## Step 1: Create Multiple AnnSearchRequest Instances +```java +import io.milvus.v2.client.ConnectConfig;​ +import io.milvus.v2.client.MilvusClientV2;​ +import io.milvus.v2.common.DataType;​ +import io.milvus.v2.service.collection.request.AddFieldReq;​ +import io.milvus.v2.service.collection.request.CreateCollectionReq;​ +​ +MilvusClientV2 client = new MilvusClientV2(ConnectConfig.builder()​ + .uri("http://localhost:19530")​ + .token("root:Milvus")​ + .build());​ +​ +CreateCollectionReq.CollectionSchema schema = client.createSchema();​ +schema.addField(AddFieldReq.builder()​ + .fieldName("id")​ + .dataType(DataType.Int64)​ + .isPrimaryKey(true)​ + .autoID(false)​ + .build());​ +​ +schema.addField(AddFieldReq.builder()​ + .fieldName("text")​ + .dataType(DataType.VarChar)​ + .maxLength(1000)​ + .build());​ +​ +schema.addField(AddFieldReq.builder()​ + .fieldName("dense")​ + .dataType(DataType.FloatVector)​ + .dimension(768)​ + .build());​ +​ +schema.addField(AddFieldReq.builder()​ + .fieldName("sparse")​ + .dataType(DataType.SparseFloatVector)​ + .build());​ -A hybrid search uses the `hybrid_search()` API to perform multiple ANN search requests in a single call. Each `AnnSearchRequest` represents a single search request on a specific vector field. +``` -The following example creates two `AnnSearchRequest` instances to perform individual similarity searches on two vector fields. +```go +// WIP​ -```python -from pymilvus import AnnSearchRequest +``` -# Create ANN search request 1 for filmVector -query_filmVector = [[0.8896863042430693, 0.370613100114602, 0.23779315077113428, 0.38227915951132996, 0.5997064603128835]] +```javascript +import { MilvusClient, DataType } from "@zilliz/milvus2-sdk-node";​ +​ +const address = "http://localhost:19530";​ +const token = "root:Milvus";​ +const client = new MilvusClient({address, token});​ +​ +// Create a collection in customized setup mode​ +// Define fields​ +const fields = [​ + {​ + name: "id",​ + data_type: DataType.Int64,​ + is_primary_key: true,​ + auto_id: false​ + },​ + {​ + name: "text",​ + data_type: DataType.VarChar,​ + max_length: 1000​ + },​ + {​ + name: "sparse",​ + data_type: DataType.SPARSE_FLOAT_VECTOR​ + },​ + {​ + name: "dense",​ + data_type: DataType.FloatVector,​ + dim: 768​ + }​ +]​ -search_param_1 = { - "data": query_filmVector, # Query vector - "anns_field": "filmVector", # Vector field name - "param": { - "metric_type": "L2", # This parameter value must be identical to the one used in the collection schema - "params": {"nprobe": 10} - }, - "limit": 2 # Number of search results to return in this AnnSearchRequest -} -request_1 = AnnSearchRequest(**search_param_1) +``` -# Create ANN search request 2 for posterVector -query_posterVector = [[0.02550758562349764, 0.006085637357292062, 0.5325251250159071, 0.7676432650114147, 0.5521074424751443]] -search_param_2 = { - "data": query_posterVector, # Query vector - "anns_field": "posterVector", # Vector field name - "param": { - "metric_type": "L2", # This parameter value must be identical to the one used in the collection schema - "params": {"nprobe": 10} - }, - "limit": 2 # Number of search results to return in this AnnSearchRequest -} -request_2 = AnnSearchRequest(**search_param_2) +```curl +export schema='{​ + "autoId": false,​ + "enabledDynamicField": true,​ + "fields": [​ + {​ + "fieldName": "id",​ + "dataType": "Int64",​ + "isPrimary": true​ + },​ + {​ + "fieldName": "text",​ + "dataType": "VarChar",​ + "elementTypeParams": {​ + "max_length": 1000​ + }​ + },​ + {​ + "fieldName": "sparse",​ + "dataType": "SparseFloatVector"​ + },​ + {​ + "fieldName": "dense",​ + "dataType": "FloatVector",​ + "elementTypeParams": {​ + "dim": "768"​ + }​ + }​ + ]​ + }'​ -# Store these two requests as a list in `reqs` -reqs = [request_1, request_2] ``` -Parameters: +During sparse vector searches, you can simplify the process of generating sparse embedding vectors by leveraging Full Text Search capabilities. For more details, see [​Full Text Search](full-text-search.md).​ -- `AnnSearchRequest` (_object_) +#### Create index​ - A class representing an ANN search request. Each hybrid search can contain 1 to 1,024 `ANNSearchRequest` objects at a time. +After defining the collection schema, it is necessary to set up the vector indexes and the similarity metrics. In this example, an IVF_FLAT index is created for the dense vector field `dense`, and a SPARSE_INVERTED_INDEX is created for the sparse vector field `sparse`. To learn about the types of indexes supported, see [​Index Explained](https://milvus.io/docs/index.md?tab=floating).​ -- `data` (_list_) +
          + Python + Java + Node.js + cURL +
          - The query vector to search in a single `AnnSearchRequest`. Currently, this parameter accepts a list containing only a single query vector, for example, `[[0.5791814851218929, 0.5792985702614121, 0.8480776460143558, 0.16098005945243, 0.2842979317256803]]`. In the future, this parameter will be expanded to accept multiple query vectors. +```python +from pymilvus import MilvusClient​ +​ +# Prepare index parameters​ +index_params = client.prepare_index_params()​ +​ +# Add indexes​ +index_params.add_index(​ + field_name="dense",​ + index_name="dense_index",​ + index_type="IVF_FLAT",​ + metric_type="IP",​ + params={"nlist": 128},​ +)​ +​ +index_params.add_index(​ + field_name="sparse",​ + index_name="sparse_index",​ + index_type="SPARSE_INVERTED_INDEX", # Index type for sparse vectors​ + metric_type="IP", # Currently, only IP (Inner Product) is supported for sparse vectors​ + params={"drop_ratio_build": 0.2}, # The ratio of small vector values to be dropped during indexing​ +)​ -- `anns_field` (_string_) +``` - The name of the vector field to use in a single `AnnSearchRequest`. +```java +import io.milvus.v2.common.IndexParam;​ +import java.util.*;​ +​ +Map denseParams = new HashMap<>();​ +denseParams.put("nlist", 128);​ +IndexParam indexParamForDenseField = IndexParam.builder()​ + .fieldName("dense")​ + .indexName("dense_index")​ + .indexType(IndexParam.IndexType.IVF_FLAT)​ + .metricType(IndexParam.MetricType.IP)​ + .extraParams(denseParams)​ + .build();​ +​ +Map sparseParams = new HashMap<>();​ +sparseParams.put("drop_ratio_build", 0.2);​ +IndexParam indexParamForSparseField = IndexParam.builder()​ + .fieldName("sparse")​ + .indexName("sparse_index")​ + .indexType(IndexParam.IndexType.SPARSE_INVERTED_INDEX)​ + .metricType(IndexParam.MetricType.IP)​ + .extraParams(sparseParams)​ + .build();​ +​ +List indexParams = new ArrayList<>();​ +indexParams.add(indexParamForDenseField);​ +indexParams.add(indexParamForSparseField);​ -- `param` (_dict_) +``` - A dictionary of search parameters for a single `AnnSearchRequest`. These search parameters are identical to those for a single-vector search. For more information, refer to [Search parameters](https://milvus.io/docs/single-vector-search.md#Search-parameters). +```javascript +const index_params = [{​ + field_name: "dense",​ + index_type: "IVF_FLAT",​ + metric_type: "IP"​ +},{​ + field_name: "sparse",​ + index_type: "SPARSE_INVERTED_INDEX",​ + metric_type: "IP"​ +}]​ -- `limit` (_int_) +``` - The maximum number of search results to include in a single `ANNSearchRequest`. +```curl +export indexParams='[​ + {​ + "fieldName": "dense",​ + "metricType": "IP",​ + "indexName": "dense_index",​ + "indexType":"IVF_FLAT",​ + "params":{"nlist":128}​ + },​ + {​ + "fieldName": "sparse",​ + "metricType": "IP",​ + "indexName": "sparse_index",​ + "indexType": "SPARSE_INVERTED_INDEX"​ + }​ + ]'​ - This parameter only affects the number of search results to return within an individual `ANNSearchRequest`, and it does not decide the final results to return for a `hybrid_search` call. In a hybrid search, the final results are determined by combining and reranking the results from multiple `ANNSearchRequest` instances. +``` -## Step 2: Configure a Reranking Strategy +#### Create collection​ -After creating `AnnSearchRequest` instances, configure a reranking strategy to combine and rerank the results. Currently, there are two options: `WeightedRanker` and `RRFRanker`. For more information about reranking strategies, refer to [Reranking](reranking.md). +Create a collection named `demo` with the collection schema and indexes configured in the previous two steps.​ -- Use weighted scoring +
          + Python + Java + Node.js + cURL +
          - The `WeightedRanker` is used to assign importance to the results from each vector field search with specified weights. If you prioritize some vector fields over others, `WeightedRanker(value1, value2, ..., valueN)` can reflect this in the combined search results. +```python +from pymilvus import MilvusClient​ +​ +client.create_collection(​ + collection_name="hybrid_search_collection",​ + schema=schema,​ + index_params=index_params​ +)​ - ```python - from pymilvus import WeightedRanker - # Use WeightedRanker to combine results with specified weights - # Assign weights of 0.8 to text search and 0.2 to image search - rerank = WeightedRanker(0.8, 0.2) - ``` +``` - When using `WeightedRanker`, note that: +```java +CreateCollectionReq createCollectionReq = CreateCollectionReq.builder()​ + .collectionName("hybrid_search_collection")​ + .collectionSchema(schema)​ + .indexParams(indexParams)​ + .build();​ +client.createCollection(createCollectionReq);​ - - Each weight value ranges from 0 (least important) to 1 (most important), influencing the final aggregated score. - - The total number of weight values provided in `WeightedRanker` should equal the number of `AnnSearchRequest` instances you have created. +``` -- Use Reciprocal Rank Fusion (RFF) +```javascript +res = await client.createCollection({​ + collection_name: "hybrid_search_collection",​ + fields: fields,​ + index_params: index_params,​ +})​ - ```python - # Alternatively, use RRFRanker for reciprocal rank fusion reranking - from pymilvus import RRFRanker - - rerank = RRFRanker() - ``` +``` + +```curl +export CLUSTER_ENDPOINT="http://localhost:19530"​ +export TOKEN="root:Milvus"​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/collections/create" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d "{​ + \"collectionName\": \"hybrid_search_collection\",​ + \"schema\": $schema,​ + \"indexParams\": $indexParams​ +}"​ + +``` -## Step 3: Perform a Hybrid Search +### Insert data​ -With the `AnnSearchRequest` instances and reranking strategy set, use the `hybrid_search()` method to perform the hybrid search. +Insert the sparse-dense vectors into the the collection `demo`.​ + +
          + Python + Java + Node.js + cURL +
          ```python -# Before conducting hybrid search, load the collection into memory. -collection.load() +from pymilvus import MilvusClient​ +​ +data=[​ + {"id": 0, "text": "Artificial intelligence was founded as an academic discipline in 1956.", "sparse":{9637: 0.30856525997853057, 4399: 0.19771651149001523, ...}, "dense": [0.3580376395471989, -0.6023495712049978, 0.18414012509913835, ...]},​ + {"id": 1, "text": "Alan Turing was the first person to conduct substantial research in AI.", "sparse":{6959: 0.31025067641541815, 1729: 0.8265339135915016, ...}, "dense": [0.19886812562848388, 0.06023560599112088, 0.6976963061752597, ...]},​ + {"id": 2, "text": "Born in Maida Vale, London, Turing was raised in southern England.", "sparse":{1220: 0.15303302147479103, 7335: 0.9436728846033107, ...}, "dense": [0.43742130801983836, -0.5597502546264526, 0.6457887650909682, ...]}​ +​ +res = client.insert(​ + collection_name="hybrid_search_collection",​ + data=data​ +)​ +​ + +``` + +```java +import com.google.gson.Gson;​ +import com.google.gson.JsonObject;​ +import io.milvus.v2.service.vector.request.InsertReq;​ +​ +Gson gson = new Gson();​ +JsonObject row1 = new JsonObject();​ +row1.addProperty("id", 1);​ +row1.addProperty("text", "Artificial intelligence was founded as an academic discipline in 1956.");​ +row1.add("dense", gson.toJsonTree(dense1));​ +row1.add("sparse", gson.toJsonTree(sparse1));​ +​ +JsonObject row2 = new JsonObject();​ +row2.addProperty("id", 2);​ +row2.addProperty("text", "Alan Turing was the first person to conduct substantial research in AI.");​ +row2.add("dense", gson.toJsonTree(dense2));​ +row2.add("sparse", gson.toJsonTree(sparse2));​ +​ +JsonObject row3 = new JsonObject();​ +row3.addProperty("id", 3);​ +row3.addProperty("text", "Born in Maida Vale, London, Turing was raised in southern England.");​ +row3.add("dense", gson.toJsonTree(dense3));​ +row3.add("sparse", gson.toJsonTree(sparse3));​ +​ +List data = Arrays.asList(row1, row2, row3);​ +InsertReq insertReq = InsertReq.builder()​ + .collectionName("hybrid_search_collection")​ + .data(data)​ + .build();​ +​ +InsertResp insertResp = client.insert(insertReq);​ + +``` -res = collection.hybrid_search( - reqs, # List of AnnSearchRequests created in step 1 - rerank, # Reranking strategy specified in step 2 - limit=2 # Number of final search results to return -) +```javascript +const { MilvusClient, DataType } = require("@zilliz/milvus2-sdk-node")​ +​ +var data = [​ + {id: 0, text: "Artificial intelligence was founded as an academic discipline in 1956.", sparse:[9637: 0.30856525997853057, 4399: 0.19771651149001523, ...] , dense: [0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592]},​ + {id: 1, text: "Alan Turing was the first person to conduct substantial research in AI.", sparse:[6959: 0.31025067641541815, 1729: 0.8265339135915016, ...] , dense: [0.19886812562848388, 0.06023560599112088, 0.6976963061752597, 0.2614474506242501, 0.838729485096104]},​ + {id: 2, text: "Born in Maida Vale, London, Turing was raised in southern England." , sparse:[1220: 0.15303302147479103, 7335: 0.9436728846033107, ...] , dense: [0.43742130801983836, -0.5597502546264526, 0.6457887650909682, 0.7894058910881185, 0.20785793220625592]} ​ +]​ +​ +var res = await client.insert({​ + collection_name: "hybrid_search_collection",​ + data: data,​ +})​ -print(res) ``` -Parameters: +```curl +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/insert" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "data": [​ + {"id": 0, "text": "Artificial intelligence was founded as an academic discipline in 1956.", "sparse":{"9637": 0.30856525997853057, "4399": 0.19771651149001523}, "dense": [0.3580376395471989, -0.6023495712049978, 0.18414012509913835, ...]},​ + {"id": 1, "text": "Alan Turing was the first person to conduct substantial research in AI.", "sparse":{"6959": 0.31025067641541815, "1729": 0.8265339135915016}, "dense": [0.19886812562848388, 0.06023560599112088, 0.6976963061752597, ...]},​ + {"id": 2, "text": "Born in Maida Vale, London, Turing was raised in southern England.", "sparse":{"1220": 0.15303302147479103, "7335": 0.9436728846033107}, "dense": [0.43742130801983836, -0.5597502546264526, 0.6457887650909682, ...]}​ + ],​ + "collectionName": "hybrid_search_collection"​ +}'​ -- `reqs` (_list_) +``` - A list of search requests, where each request is an `ANNSearchRequest` object. Each request can correspond to a different vector field and a different set of search parameters. +### Create multiple AnnSearchRequest instances​ -- `rerank` (_object_) +Hybrid Search is implemented by creating multiple `AnnSearchRequest` in the `hybrid_search()` function, where each `AnnSearchRequest` represents a basic ANN search request for a specific vector field. Therefore, before conducting a Hybrid Search, it is necessary to create an `AnnSearchRequest` for each vector field.​ - The reranking strategy to use for hybrid search. Possible values: `WeightedRanker(value1, value2, ..., valueN)` and `RRFRanker()`. +
          - For more information about reranking strategies, refer to [Reranking](reranking.md). +In Hybrid Search, each `AnnSearchRequest` supports only one query vector.​ -- `limit` (_int_) +
          - The maximum number of final results to return in the hybrid search. +Suppose the query text "Who started AI research?" has already been converted into sparse and dense vectors. Based on this, two `AnnSearchRequest` search requests are created for the `sparse` and `dense` vector fields respectively, as shown in the following example.​ -The output is similar to the following: +
          + Python + Java + Node.js + cURL +
          ```python -["['id: 844, distance: 0.006047376897186041, entity: {}', 'id: 876, distance: 0.006422005593776703, entity: {}']"] +from pymilvus import AnnSearchRequest​ +​ +query_dense_vector = [0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592]​ +​ +search_param_1 = {​ + "data": [query_dense_vector],​ + "anns_field": "dense",​ + "param": {​ + "metric_type": "IP",​ + "params": {"nprobe": 10}​ + },​ + "limit": 2​ +}​ +request_1 = AnnSearchRequest(**search_param_1)​ +​ +query_sparse_vector = {3573: 0.34701499565746674}, {5263: 0.2639375518635271}​ +search_param_2 = {​ + "data": [query_sparse_vector],​ + "anns_field": "sparse",​ + "param": {​ + "metric_type": "IP",​ + "params": {"drop_ratio_build": 0.2}​ + },​ + "limit": 2​ +}​ +request_2 = AnnSearchRequest(**search_param_2)​ +​ +reqs = [request_1, request_2]​ +​ + +``` + +```java +import io.milvus.v2.service.vector.request.AnnSearchReq;​ +import io.milvus.v2.service.vector.request.data.BaseVector;​ +import io.milvus.v2.service.vector.request.data.FloatVec;​ +import io.milvus.v2.service.vector.request.data.SparseFloatVec;​ +​ +float[] dense = new float[]{-0.0475336798f, 0.0521207601f, 0.0904406682f, ...};​ +SortedMap sparse = new TreeMap() {{​ + put(3573L, 0.34701499f);​ + put(5263L, 0.263937551f);​ + ...​ +}};​ +​ +​ +List queryDenseVectors = Collections.singletonList(new FloatVec(dense));​ +List querySparseVectors = Collections.singletonList(new SparseFloatVec(sparse));​ +​ +List searchRequests = new ArrayList<>();​ +searchRequests.add(AnnSearchReq.builder()​ + .vectorFieldName("dense")​ + .vectors(queryDenseVectors)​ + .metricType(IndexParam.MetricType.IP)​ + .params("{\"nprobe\": 10}")​ + .topK(2)​ + .build());​ +searchRequests.add(AnnSearchReq.builder()​ + .vectorFieldName("sparse")​ + .vectors(querySparseVectors)​ + .metricType(IndexParam.MetricType.IP)​ + .params("{\"drop_ratio_build\": 0.2}")​ + .topK(2)​ + .build());​ + +``` + +```javascript +const search_param_1 = {​ + "data": query_vector, ​ + "anns_field": "dense", ​ + "param": {​ + "metric_type": "IP", // 参数值需要与 Collection Schema 中定义的保持一致​ + "params": {"nprobe": 10}​ + },​ + "limit": 2 // AnnSearchRequest 返还的搜索结果数量​ +}​ +​ +const search_param_2 = {​ + "data": query_sparse_vector, ​ + "anns_field": "sparse", ​ + "param": {​ + "metric_type": "IP", // 参数值需要与 Collection Schema 中定义的保持一致​ + "params": {"drop_ratio_build": 0.2}​ + },​ + "limit": 2 // AnnSearchRequest 返还的搜索结果数量​ +}​ + +``` + +```curl +export req='[​ + {​ + "data": [[0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592,....]],​ + "annsField": "dense",​ + "params": {​ + "params": {​ + "nprobe": 10​ + }​ + },​ + "limit": 2​ + },​ + {​ + "data": [{"3573": 0.34701499565746674}, {"5263": 0.2639375518635271}],​ + "annsField": "sparse",​ + "params": {​ + "params": {​ + "drop_ratio_build": 0.2​ + }​ + },​ + "limit": 2​ + }​ + ]'​ + ``` -## Limits +Since the parameter `limit` is set to 2, each `AnnSearchRequest` returns 2 search results. In this example, 2 `AnnSearchRequest` are created, therefore a total of 4 search results will be returned.​ + +### Configure a reranking strategy​ + +To merge and rerank the two sets of ANN search results, it is necessary to select an appropriate reranking strategy. Zilliz supports two types of reranking strategy: **WeightedRanker** and **RRFRanker**. When choosing a reranking strategy, one thing to consider is whether to there is any emphasis for one or more basic ANN search on the vector fields.​ + +- **WeightedRanker**: This strategy is recommended if you require the results to emphasize a particular vector field. The WeightedRanker allows you to assign higher weights to certain vector fields, emphasizing them more. For instance, in multimodal searches, textual descriptions of an image might be considered more important than the colors in this image.​ + +- **RRFRanker (Reciprocal Rank Fusion Ranker)**: This strategy is recommended when there is no specific emphasis. The RRF can effectively balance the importance of each vector field.​ + +For more details about the mechanisms of these two reranking strategies, refer to [​Reranking](reranking.md).​ + +The following two examples demonstrate how to use the WeightedRanker and RRFRanker reranking strategies:​ -- Typically, each collection has a default allowance of up to 4 vector fields. However, you have the option to adjust the `proxy.maxVectorFieldNum` configuration to expand the maximum number of vector fields in a collection, with a maximum limit of 10 vector fields per collection. See [Proxy-related Configurations](https://milvus.io/docs/configure_proxy.md#Proxy-related-Configurations) for more. +1. **Example 1: Using WeightedRanker**​ -- Partially indexed or loaded vector fields in a collection will result in an error. + When using the WeightedRanker strategy, you need to input weight values into the `WeightedRanker` function. The number of basic ANN searches in a Hybrid Search corresponds to the number of values that need to be inputted. The input values should be in the range [0,1], with values closer to 1 indicating greater importance.​ -- Currently, each `AnnSearchRequest` in a hybrid search can carry one query vector only. +
          + Python + Java + Node.js + cURL +
          + + ```python + from pymilvus import WeightedRanker​ + ​ + rerank= WeightedRanker(0.8, 0.3) ​ -## FAQ + ``` + + ```java + import io.milvus.v2.service.vector.request.ranker.BaseRanker;​ + import io.milvus.v2.service.vector.request.ranker.WeightedRanker;​ + ​ + BaseRanker reranker = new WeightedRanker(Arrays.asList(0.8f, 0.3f));​ -- **In which scenario is hybrid search recommended?** + ``` + + ```javascript + import { MilvusClient, DataType } from "@zilliz/milvus2-sdk-node";​ + ​ + const rerank = WeightedRanker(0.8, 0.3);​ - Hybrid search is ideal for complex situations demanding high accuracy, especially when an entity can be represented by multiple, diverse vectors. This applies to cases where the same data, such as a sentence, is processed through different embedding models or when multimodal information (like images, fingerprints, and voiceprints of an individual) is converted into various vector formats. By assigning weights to these vectors, their combined influence can significantly enrich recall and improve the effectiveness of search results. + ``` + + ```curl + export rerank='{​ + "strategy": "ws",​ + "params": {"weights": [0.8,0.3]}​ + }'​ -- **How does a weighted ranker normalize distances between different vector fields?** + ``` - A weighted ranker normalizes the distances between vector fields using assigned weights to each field. It calculates the importance of each vector field according to its weight, prioritizing those with higher weights. It's advised to use the same metric type across ANN search requests to ensure consistency. This method ensures that vectors deemed more significant have a greater influence on the overall ranking. +2. **Example 2: Using RRFRanker**​ -- **Is it possible to use alternative rankers like Cohere Ranker or BGE Ranker?** + When using the RRFRanker strategy, you need to input the parameter value `k` into the RRFRanker. The default value of `k` is 60. This parameter helps to determine how the ranks are combined from different ANN searches, aiming to balance and blend the importance across all searches.​ - Currently, only the provided rankers are supported. Plans to include additional rankers are underway for future updates. +
          + Python + Java + Node.js + cURL +
          -- **Is it possible to conduct multiple hybrid search operations at the same time?** + ```python + from pymilvus import RRFRanker​ + ​ + ranker = RRFRanker(100)​ - Yes, simultaneous execution of multiple hybrid search operations is supported. + ``` + + ```java + import io.milvus.v2.service.vector.request.ranker.BaseRanker;​ + import io.milvus.v2.service.vector.request.ranker.RRFRanker;​ + ​ + BaseRanker reranker = new RRFRanker(100);​ + + ``` + + ```javascript + import { MilvusClient, DataType } from "@zilliz/milvus2-sdk-node";​ + ​ + const rerank = RRFRanker("100");​ + + ``` + + ```curl + export rerank='{​ + "strategy": "rrf",​ + "params": { "k": 100}​ + }'​ + + ``` + +### Perform a Hybrid Search​ + +Before conducting a Hybrid Search, it is necessary to load the collection into memory. If any vector fields in the collection do not have an index or are not loaded, an error will occur when calling the Hybrid Search method. ​ + +
          + Python + Java + Node.js + cURL +
          + +```python +from pymilvus import MilvusClient​ +​ +res = client.hybrid_search(​ + collection_name="hybrid_search_collection",​ + reqs=reqs,​ + ranker=ranker,​ + limit=2​ +)​ +for hits in res:​ + print("TopK results:")​ + for hit in hits:​ + print(hit)​ + +``` + +```java +import io.milvus.v2.common.ConsistencyLevel;​ +import io.milvus.v2.service.vector.request.HybridSearchReq;​ +import io.milvus.v2.service.vector.response.SearchResp;​ +​ +HybridSearchReq hybridSearchReq = HybridSearchReq.builder()​ + .collectionName("hybrid_search_collection")​ + .searchRequests(searchRequests)​ + .ranker(reranker)​ + .topK(2)​ + .consistencyLevel(ConsistencyLevel.BOUNDED)​ + .build();​ +​ +SearchResp searchResp = client.hybridSearch(hybridSearchReq);​ + +``` + +```javascript +const { MilvusClient, DataType } = require("@zilliz/milvus2-sdk-node")​ +​ +res = await client.loadCollection({​ + collection_name: "hybrid_search_collection"​ +})​ +​ +import { MilvusClient, RRFRanker, WeightedRanker } from '@zilliz/milvus2-sdk-node';​ +​ +const search = await client.search({​ + collection_name: "hybrid_search_collection",​ + data: [search_param_1, search_param_2],​ + limit: 2,​ + rerank: RRFRanker(100)​ +});​ + +``` + +```curl +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/advanced_search" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d "{​ + \"collectionName\": \"hybrid_search_collection\",​ + \"search\": ${req},​ + \"rerank\": {​ + \"strategy\":\"rrf\",​ + \"params\": {​ + \"k\": 10​ + }​ + },​ + \"limit\": 3,​ + \"outputFields\": [​ + \"user_id\",​ + \"word_count\",​ + \"book_describe\"​ + ]​ +}"​ + +``` + +The following is the output:​ + +```json +["['id: 844, distance: 0.006047376897186041, entity: {}', 'id: 876, distance: 0.006422005593776703, entity: {}']"]​ + +``` -- **Can I use the same vector field in multiple AnnSearchRequest objects to perform hybrid searches?** +Since `limit=2` is specified in the Hybrid Search, Zilliz will rerank the four search results from step 3 and ultimately return only the top 2 most similar search results. ​ - Technically, it is possible to use the same vector field in multiple AnnSearchRequest objects for hybrid searches. It is not necessary to have multiple vector fields for a hybrid search. diff --git a/v2.4.x/site/en/userGuide/search-query-get/range-search.md b/v2.4.x/site/en/userGuide/search-query-get/range-search.md new file mode 100644 index 0000000000..fc2164b198 --- /dev/null +++ b/v2.4.x/site/en/userGuide/search-query-get/range-search.md @@ -0,0 +1,217 @@ +--- +id: range-search.md +summary: A range search improves search result relevancy by restricting the distance or score of the returned entities within a specific range. This page helps you understand what range search is and the procedures to conduct a range search.​ +title: Range Search +--- + +# Range Search​ + +A range search improves search result relevancy by restricting the distance or score of the returned entities within a specific range. This page helps you understand what range search is and the procedures to conduct a range search.​ + +## Overview​ + +When executing a Range Search request, Milvus uses the most similar vectors to the query vector from the ANN Search results as the center, with the radius specified in the Search request as the outer circle's radius, and the **range_filter** as the inner circle's radius to draw two concentric circles. All vectors with similarity scores that fall within the annular region formed by these two concentric circles will be returned. Here, the **range_filter** can be set to **0**, indicating that all entities within the specified similarity score (radius) will be returned.​ + +![Range search](../../../../assets/range-search.png) + +The above diagram shows that a range search request carries two parameters: radius and **range_filter**. Upon receiving a range search request, Milvus does the following:​ + +- Use the specified metric type (**COSINE**) to find all vector embeddings most similar to the query vector.​ + +- Filter the vector embeddings whose **distances** or **scores** to the query vector fall within the range specified by the **radius** and **range_filter** parameters.​ + +- Return the **top-K** entities from the filtered ones.​ + +The way to set radius and **range_filter** varies with the metric type of the search. The following table lists the requirements for setting these two parameters with different metric types.​ + +

          Metric Type​

          + +

          Denotations​

          + +

          Requirements for Setting radius and range_filter​

          + +

          L2​

          + +

          A smaller L2 distance indicates a higher similarity.​

          + +

          To ignore the most similar vector embeddings, ensure that​

          + +

          range_filter <= distance < radius​

          + +

          IP​

          + +

          A greater IP distance indicates a higher similarity.​

          + +

          To ignore the most similar vector embeddings, ensure that​

          + +

          radius < distance <= range_filter​

          + +

          COSINE​

          + +

          A greater COSINE distance indicates a higher similarity.​

          + +

          To ignore the most similar vector embeddings, ensure that​

          + +

          radius < distance <= range_filter​

          + +

          JACCARD ​

          + +

          A smaller Jaccard distance indicates a higher similarity.​

          + +

          ​

          + +

          To ignore the most similar vector embeddings, ensure that​

          + +

          range_filter <= distance < radius​

          + +

          HAMMING ​

          + +

          A smaller Hamming distance indicates a higher similarity.​

          + +

          To ignore the most similar vector embeddings, ensure that​

          + +

          range_filter <= distance < radius​

          + +
          + +## Examples​ + +This section demonstrates how to conduct a range search. The search requests in the following code snippets do not carry a metric type, indicating the default metric type **COSINE** applies. In this case, ensure that the radius value is smaller than the **range_filter** value.​ + +In the following code snippets, set `radius` to `0.4` and `range_filter` to `0.6` so that Milvus returns all entities whose distances or scores to the query vector fall within **0.4** to **0.6**.​ + +
          + Python + Java + Node.js + cURL +
          + +```python +from pymilvus import MilvusClient​ +​ +client = MilvusClient(​ + uri="http://localhost:19530",​ + token="root:Milvus"​ +)​ +​ +query_vector = [0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592]​ +​ +res = client.search(​ + collection_name="my_collection",​ + data=[query_vector],​ + limit=3,​ + search_params={​ + # highlight-start​ + "params": {​ + "radius": 0.4,​ + "range_filter": 0.6​ + }​ + # highlight-end​ + }​ +)​ +​ +for hits in res:​ + print("TopK results:")​ + for hit in hits:​ + print(hit)​ + +``` + +```java +import io.milvus.v2.client.ConnectConfig;​ +import io.milvus.v2.client.MilvusClientV2;​ + io.milvus.v2.service.vector.request.SearchReq​ +import io.milvus.v2.service.vector.request.data.FloatVec;​ +import io.milvus.v2.service.vector.response.SearchResp​ +​ +​ +MilvusClientV2 client = new MilvusClientV2(ConnectConfig.builder()​ + .uri("http://localhost:19530")​ + .token("root:Milvus")​ + .build());​ +​ +FloatVec queryVector = new FloatVec(new float[]{0.3580376395471989f, -0.6023495712049978f, 0.18414012509913835f, -0.26286205330961354f, 0.9029438446296592f});​ +Map extraParams = new HashMap<>();​ +extraParams.put("radius", 0.4);​ +extraParams.put("range_filter", 0.6);​ +SearchReq searchReq = SearchReq.builder()​ + .collectionName("range_search_collection")​ + .data(Collections.singletonList(queryVector))​ + .topK(5)​ + .searchParams(extraParams)​ + .build();​ +​ +SearchResp searchResp = client.search(searchReq);​ +​ +List> searchResults = searchResp.getSearchResults();​ +for (List results : searchResults) {​ + System.out.println("TopK results:");​ + for (SearchResp.SearchResult result : results) {​ + System.out.println(result);​ + }​ +}​ +​ +// Output​ +// TopK results:​ +// SearchResp.SearchResult(entity={}, score=0.5975797, id=4)​ +// SearchResp.SearchResult(entity={}, score=0.46704385, id=5)​ + +``` + +```go +// TODO ​ + +``` + +```javascript +import { MilvusClient, DataType } from "@zilliz/milvus2-sdk-node";​ +​ +const address = "http://localhost:19530";​ +const token = "root:Milvus";​ +const client = new MilvusClient({address, token});​ +​ +var query_vector = [0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592]​ +​ +res = await client.search({​ + collection_name: "range_search_collection",​ + data: [query_vector],​ + limit: 5,​ + // highlight-start​ + params: {​ + "radius": 0.4,​ + "range_filter": 0.6​ + }​ + // highlight-end​ +})​ + +``` + +```curl +export CLUSTER_ENDPOINT="http://localhost:19530"​ +export TOKEN="root:Milvus"​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/search" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "quick_setup",​ + "data": [​ + [0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592]​ + ],​ + "annsField": "vector",​ + "filter": "color like \"red%\" and likes > 50",​ + "limit": 3,​ + "searchParams": {​ + "params": {​ + "radius": 0.4,​ + "range_filter": 0.6​ + }​ + }​ +}'​ +# {"code":0,"cost":0,"data":[]}​ + +``` + + \ No newline at end of file diff --git a/v2.4.x/site/en/reference/reranking.md b/v2.4.x/site/en/userGuide/search-query-get/reranking.md similarity index 100% rename from v2.4.x/site/en/reference/reranking.md rename to v2.4.x/site/en/userGuide/search-query-get/reranking.md diff --git a/v2.4.x/site/en/userGuide/search-query-get/single-vector-search.md b/v2.4.x/site/en/userGuide/search-query-get/single-vector-search.md index 62f26c292f..1506afadb3 100644 --- a/v2.4.x/site/en/userGuide/search-query-get/single-vector-search.md +++ b/v2.4.x/site/en/userGuide/search-query-get/single-vector-search.md @@ -2,1991 +2,971 @@ id: single-vector-search.md order: 1 summary: This article describes how to search for vectors in a Milvus collection using a single query vector. -title: Single-Vector Search +title: Ba​sic ANN Search --- -# Single-Vector Search +# Ba​sic ANN Search -Once you have inserted your data, the next step is to perform similarity searches on your collection in Milvus. +Based on an index file recording the sorted order of vector embeddings, the Approximate Nearest Neighbor (ANN) search locates a subset of vector embeddings based on the query vector carried in a received search request, compares the query vector with those in the subgroup, and returns the most similar results. With ANN search, Zilliz Cloud provides an efficient search experience. This page helps you to learn how to conduct basic ANN searches.​ -Milvus allows you to conduct two types of searches, depending on the number of vector fields in your collection: +## Overview​ -- **Single-vector search**: If your collection has only one vector field, use the [`search()`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Vector/search.md) method to find the most similar entities. This method compares your query vector with the existing vectors in your collection and returns the IDs of the closest matches along with the distances between them. Optionally, it can also return the vector values and metadata of the results. -- **Hybrid search**: For collections with two or more vector fields, use the [`hybrid_search()`](https://milvus.io/api-reference/pymilvus/v2.4.x/ORM/Collection/hybrid_search.md) method. This method performs multiple Approximate Nearest Neighbor (ANN) search requests and combines the results to return the most relevant matches after reranking. +The ANN and the k-Nearest Neighbors (kNN) search are the usual methods in vector similarity searches. In a kNN search, you must compare all vectors in a vector space with the query vector carried in the search request before figuring out the most similar ones, which is time-consuming and resource-intensive.​ -This guide focuses on how to perform a single-vector search in Milvus. For details on hybrid search, refer to [Hybrid search](https://milvus.io/docs/multi-vector-search.md). +Unlike kNN searches, an ANN search algorithm asks for an **index** file that records the sorted order of vector embeddings. When a search request comes in, you can use the index file as a reference to quickly locate a subgroup probably containing vector embeddings most similar to the query vector. Then, you can use the specified **metric type** to measure the similarity between the query vector and those in the subgroup, sort the group members based on similarity to the query vector, and figure out the **top-K** group members.​ -## Overview +ANN searches depend on pre-built indexes, and the search throughput, memory usage, and search correctness may vary with the index types you choose. You need to balance search performance and correctness. ​ -There are a variety of search types to meet different requirements: +To reduce the learning curve, Zilliz Cloud provides **AUTOINDEX**. With **AUTOINDEX**, Zilliz Cloud can analyze the data distribution within your collection while building the index and sets the most optimized index parameters based on the analysis to strike a balance between search performance and correctness. ​ -- [Basic search](https://milvus.io/docs/single-vector-search.md#Basic-search): Includes single-vector search, bulk-vector search, partition search, and search with specified output fields. +For details on AUTOINDEX and applicable metric types, refer to [​AUTOINDEX](https://milvus.io/docs/glossary.md#Auto-Index) and [​Metric Types](metric.md). In this section, you will find detailed information about the following topics:​ -- [Filtered search](https://milvus.io/docs/single-vector-search.md#Filtered-search): Applies filtering criteria based on scalar fields to refine search results. +- [Single-vector search](#Single-Vector-Search)​ -- [Range search](https://milvus.io/docs/single-vector-search.md#Range-search): Finds vectors within a specific distance range from the query vector. +- [Bulk-vector search](#Bulk-Vector-Search)​ -- [Grouping search](https://milvus.io/docs/single-vector-search.md#Grouping-search): Groups search results based on a specific field to ensure diversity in the results. +- [ANN search in partition](#ANN-Search-in-Partition)​ -## Preparations +- [Use output fields](#Use-Output-Fields)​ -The code snippet below repurposes the existing code to establish a connection to Milvus and quickly set up a collection. +- [Use limit and offset](#Use-Limit-and-Offset)​ -
          - Python - Java - Node.js -
          - -```python -from pymilvus import MilvusClient -import random - -# 1. Set up a Milvus client -client = MilvusClient( - uri="http://localhost:19530" -) - -# 2. Create a collection -client.create_collection( - collection_name="quick_setup", - dimension=5, - metric_type="IP" -) - -# 3. Insert randomly generated vectors -colors = ["green", "blue", "yellow", "red", "black", "white", "purple", "pink", "orange", "brown", "grey"] -data = [] - -for i in range(1000): - current_color = random.choice(colors) - data.append({ - "id": i, - "vector": [ random.uniform(-1, 1) for _ in range(5) ], - "color": current_color, - "color_tag": f"{current_color}_{str(random.randint(1000, 9999))}" - }) - -res = client.insert( - collection_name="quick_setup", - data=data -) - -print(res) - -# Output -# -# { -# "insert_count": 1000, -# "ids": [ -# 0, -# 1, -# 2, -# 3, -# 4, -# 5, -# 6, -# 7, -# 8, -# 9, -# "(990 more items hidden)" -# ] -# } - -# 6.1 Create partitions -client.create_partition( - collection_name="quick_setup", - partition_name="red" -) - -client.create_partition( - collection_name="quick_setup", - partition_name="blue" -) - -# 6.1 Insert data into partitions -red_data = [ {"id": i, "vector": [ random.uniform(-1, 1) for _ in range(5) ], "color": "red", "color_tag": f"red_{str(random.randint(1000, 9999))}" } for i in range(500) ] -blue_data = [ {"id": i, "vector": [ random.uniform(-1, 1) for _ in range(5) ], "color": "blue", "color_tag": f"blue_{str(random.randint(1000, 9999))}" } for i in range(500) ] - -res = client.insert( - collection_name="quick_setup", - data=red_data, - partition_name="red" -) - -print(res) - -# Output -# -# { -# "insert_count": 500, -# "ids": [ -# 0, -# 1, -# 2, -# 3, -# 4, -# 5, -# 6, -# 7, -# 8, -# 9, -# "(490 more items hidden)" -# ] -# } - -res = client.insert( - collection_name="quick_setup", - data=blue_data, - partition_name="blue" -) - -print(res) - -# Output -# -# { -# "insert_count": 500, -# "ids": [ -# 0, -# 1, -# 2, -# 3, -# 4, -# 5, -# 6, -# 7, -# 8, -# 9, -# "(490 more items hidden)" -# ] -# } -``` - -```java -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Random; - -import com.alibaba.fastjson.JSONObject; - -import io.milvus.v2.client.ConnectConfig; -import io.milvus.v2.client.MilvusClientV2; -import io.milvus.v2.service.collection.request.CreateCollectionReq; -import io.milvus.v2.service.collection.request.GetLoadStateReq; -import io.milvus.v2.service.vector.request.InsertReq; -import io.milvus.v2.service.vector.response.InsertResp; - -String CLUSTER_ENDPOINT = "http://localhost:19530"; - -// 1. Connect to Milvus server -ConnectConfig connectConfig = ConnectConfig.builder() - .uri(CLUSTER_ENDPOINT) - .build(); - -MilvusClientV2 client = new MilvusClientV2(connectConfig); - -// 2. Create a collection in quick setup mode -CreateCollectionReq quickSetupReq = CreateCollectionReq.builder() - .collectionName("quick_setup") - .dimension(5) - .metricType("IP") - .build(); - -client.createCollection(quickSetupReq); - -GetLoadStateReq loadStateReq = GetLoadStateReq.builder() - .collectionName("quick_setup") - .build(); - -boolean state = client.getLoadState(loadStateReq); - -System.out.println(state); - -// Output: -// true - -// 3. Insert randomly generated vectors into the collection -List colors = Arrays.asList("green", "blue", "yellow", "red", "black", "white", "purple", "pink", "orange", "brown", "grey"); -List data = new ArrayList<>(); - -for (int i=0; i<1000; i++) { - Random rand = new Random(); - String current_color = colors.get(rand.nextInt(colors.size()-1)); - JSONObject row = new JSONObject(); - row.put("id", Long.valueOf(i)); - row.put("vector", Arrays.asList(rand.nextFloat(), rand.nextFloat(), rand.nextFloat(), rand.nextFloat(), rand.nextFloat())); - row.put("color_tag", current_color + "_" + String.valueOf(rand.nextInt(8999) + 1000)); - data.add(row); -} - -InsertReq insertReq = InsertReq.builder() - .collectionName("quick_setup") - .data(data) - .build(); - -InsertResp insertResp = client.insert(insertReq); - -System.out.println(JSONObject.toJSON(insertResp)); - -// Output: -// {"insertCnt": 1000} - -// 6.1. Create a partition -CreatePartitionReq partitionReq = CreatePartitionReq.builder() - .collectionName("quick_setup") - .partitionName("red") - .build(); - -client.createPartition(partitionReq); - -partitionReq = CreatePartitionReq.builder() - .collectionName("quick_setup") - .partitionName("blue") - .build(); - -client.createPartition(partitionReq); - -// 6.2 Insert data into the partition -data = new ArrayList<>(); - -for (int i=1000; i<1500; i++) { - Random rand = new Random(); - String current_color = "red"; - JSONObject row = new JSONObject(); - row.put("id", Long.valueOf(i)); - row.put("vector", Arrays.asList(rand.nextFloat(), rand.nextFloat(), rand.nextFloat(), rand.nextFloat(), rand.nextFloat())); - row.put("color", current_color); - row.put("color_tag", current_color + "_" + String.valueOf(rand.nextInt(8999) + 1000)); - data.add(row); -} - -insertReq = InsertReq.builder() - .collectionName("quick_setup") - .data(data) - .partitionName("red") - .build(); - -insertResp = client.insert(insertReq); - -System.out.println(JSONObject.toJSON(insertResp)); - -// Output: -// {"insertCnt": 500} - -data = new ArrayList<>(); - -for (int i=1500; i<2000; i++) { - Random rand = new Random(); - String current_color = "blue"; - JSONObject row = new JSONObject(); - row.put("id", Long.valueOf(i)); - row.put("vector", Arrays.asList(rand.nextFloat(), rand.nextFloat(), rand.nextFloat(), rand.nextFloat(), rand.nextFloat())); - row.put("color", current_color); - row.put("color_tag", current_color + "_" + String.valueOf(rand.nextInt(8999) + 1000)); - data.add(row); -} - -insertReq = InsertReq.builder() - .collectionName("quick_setup") - .data(data) - .partitionName("blue") - .build(); - -insertResp = client.insert(insertReq); - -System.out.println(JSONObject.toJSON(insertResp)); - -// Output: -// {"insertCnt": 500} -``` - -```javascript -const { MilvusClient, DataType, sleep } = require("@zilliz/milvus2-sdk-node") - -const address = "http://localhost:19530" - -// 1. Set up a Milvus Client -client = new MilvusClient({address}); - -// 2. Create a collection in quick setup mode -await client.createCollection({ - collection_name: "quick_setup", - dimension: 5, - metric_type: "IP" -}); - -// 3. Insert randomly generated vectors -const colors = ["green", "blue", "yellow", "red", "black", "white", "purple", "pink", "orange", "brown", "grey"] -data = [] - -for (let i = 0; i < 1000; i++) { - current_color = colors[Math.floor(Math.random() * colors.length)] - data.push({ - id: i, - vector: [Math.random(), Math.random(), Math.random(), Math.random(), Math.random()], - color: current_color, - color_tag: `${current_color}_${Math.floor(Math.random() * 8999) + 1000}` - }) -} - -var res = await client.insert({ - collection_name: "quick_setup", - data: data -}) - -console.log(res.insert_cnt) - -// Output -// -// 1000 -// - -await client.createPartition({ - collection_name: "quick_setup", - partition_name: "red" -}) - -await client.createPartition({ - collection_name: "quick_setup", - partition_name: "blue" -}) - -// 6.1 Insert data into partitions -var red_data = [] -var blue_data = [] - -for (let i = 1000; i < 1500; i++) { - red_data.push({ - id: i, - vector: [Math.random(), Math.random(), Math.random(), Math.random(), Math.random()], - color: "red", - color_tag: `red_${Math.floor(Math.random() * 8999) + 1000}` - }) -} - -for (let i = 1500; i < 2000; i++) { - blue_data.push({ - id: i, - vector: [Math.random(), Math.random(), Math.random(), Math.random(), Math.random()], - color: "blue", - color_tag: `blue_${Math.floor(Math.random() * 8999) + 1000}` - }) -} - -res = await client.insert({ - collection_name: "quick_setup", - data: red_data, - partition_name: "red" -}) - -console.log(res.insert_cnt) - -// Output -// -// 500 -// - -res = await client.insert({ - collection_name: "quick_setup", - data: blue_data, - partition_name: "blue" -}) - -console.log(res.insert_cnt) - -// Output -// -// 500 -// -``` - -## Basic search - -When sending a `search` request, you can provide one or more vector values representing your query embeddings and a `limit` value indicating the number of results to return. - -Depending on your data and your query vector, you may get fewer than `limit` results. This happens when `limit` is larger than the number of possible matching vectors for your query. +- [Enhancing ANN search](#Enhance-ANN-Search)​ -### Single-vector search +## Single-Vector Search​ -Single-vector search is the simplest form of `search` operations in Milvus, designed to find the most similar vectors to a given query vector. +In ANN searches, a single-vector search refers to a search that involves only one query vector. Based on the pre-built index and the metric type carried in the search request, Zilliz Cloud will find the top-K vectors most similar to the query vector.​ -To perform a single-vector search, specify the target collection name, the query vector, and the desired number of results (`limit`). This operation returns a result set comprising the most similar vectors, their IDs, and distances from the query vector. - -Here is an example of searching for the top 5 entities that are most similar to the query vector: +In this section, you will learn how to conduct a single-vector search. The code snippet assumes you have created a collection in a [quick-setup](create-collection-instantly#Quick-Setup) manner. The search request carries a single query vector and asks Zilliz Cloud to use Inner Product (IP) to calculate the similarity between query vectors and vectors in the collection and returns the three most similar ones.​
          - Python - Java - Node.js + Python + Java + Node.js + Go + cURL
          ```python -# Single vector search -res = client.search( - collection_name="quick_setup", # Replace with the actual name of your collection - # Replace with your query vector - data=[[0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592]], - limit=5, # Max. number of search results to return - search_params={"metric_type": "IP", "params": {}} # Search parameters -) - -# Convert the output to a formatted JSON string -result = json.dumps(res, indent=4) -print(result) +from pymilvus import MilvusClient​ +​ +client = MilvusClient(​ + uri="http://localhost:19530",​ + token="root:Milvus"​ +)​ +​ +# 4. Single vector search​ +query_vector = [0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592]​ +res = client.search(​ + collection_name="my_collection",​ + anns_field="vector",​ + data=[query_vector],​ + limit=3,​ + search_params={"metric_type": "IP"}​ +)​ +​ +for hits in res:​ + for hit in hits:​ + print(hit)​ +​ +# [​ +# [​ +# {​ +# "id": 551,​ +# "distance": 0.08821295201778412,​ +# "entity": {}​ +# },​ +# {​ +# "id": 296,​ +# "distance": 0.0800950899720192,​ +# "entity": {}​ +# },​ +# {​ +# "id": 43,​ +# "distance": 0.07794742286205292,​ +# "entity": {}​ +# }​ +# ]​ +# ]​ + ``` ```java -// 4. Single vector search -List> query_vectors = Arrays.asList(Arrays.asList(0.3580376395471989f, -0.6023495712049978f, 0.18414012509913835f, -0.26286205330961354f, 0.9029438446296592f)); - -SearchReq searchReq = SearchReq.builder() - .collectionName("quick_setup") - .data(query_vectors) - .topK(3) // The number of results to return - .build(); +import io.milvus.v2.client.ConnectConfig;​ +import io.milvus.v2.client.MilvusClientV2;​ +import io.milvus.v2.service.vector.request.SearchReq;​ +import io.milvus.v2.service.vector.request.data.FloatVec;​ +import io.milvus.v2.service.vector.response.SearchResp;​ +​ +import java.util.*;​ +​ +MilvusClientV2 client = new MilvusClientV2(ConnectConfig.builder()​ + .uri("http://localhost:19530")​ + .token("root:Milvus")​ + .build());​ + ​ +FloatVec queryVector = new FloatVec(new float[]{0.3580376395471989f, -0.6023495712049978f, 0.18414012509913835f, -0.26286205330961354f, 0.9029438446296592f});​ +SearchReq searchReq = SearchReq.builder()​ + .collectionName("my_collection")​ + .data(Collections.singletonList(queryVector))​ + .topK(3)​ + .build();​ +​ +SearchResp searchResp = client.search(searchReq);​ +​ +List> searchResults = searchResp.getSearchResults();​ +for (List results : searchResults) {​ + System.out.println("TopK results:");​ + for (SearchResp.SearchResult result : results) {​ + System.out.println(result);​ + }​ +}​ +​ +// Output​ +// TopK results:​ +// SearchResp.SearchResult(entity={}, score=0.95944905, id=5)​ +// SearchResp.SearchResult(entity={}, score=0.8689616, id=1)​ +// SearchResp.SearchResult(entity={}, score=0.866088, id=7)​ + +``` + +```go +import (​ + "context"​ + "fmt"​ + "log"​ +​ + "github.com/milvus-io/milvus/client/v2"​ + "github.com/milvus-io/milvus/client/v2/entity"​ +)​ +​ +func ExampleClient_Search_basic() {​ + ctx, cancel := context.WithCancel(context.Background())​ + defer cancel()​ +​ + milvusAddr := "127.0.0.1:19530"​ + token := "root:Milvus"​ +​ + cli, err := client.New(ctx, &client.ClientConfig{​ + Address: milvusAddr,​ + APIKey: token,​ + })​ + if err != nil {​ + log.Fatal("failed to connect to milvus server: ", err.Error())​ + }​ +​ + defer cli.Close(ctx)​ +​ + queryVector := []float32{0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592}​ +​ + resultSets, err := cli.Search(ctx, client.NewSearchOption(​ + "my_collection", // collectionName​ + 3, // limit​ + []entity.Vector{entity.FloatVector(queryVector)},​ + ))​ + if err != nil {​ + log.Fatal("failed to perform basic ANN search collection: ", err.Error())​ + }​ +​ + for _, resultSet := range resultSets {​ + log.Println("IDs: ", resultSet.IDs)​ + log.Println("Scores: ", resultSet.Scores)​ + }​ + // Output:​ + // IDs:​ + // Scores:​ +}​ -SearchResp searchResp = client.search(searchReq); - -System.out.println(JSONObject.toJSON(searchResp)); ``` ```javascript -// 4. Single vector search -var query_vector = [0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592], - -res = await client.search({ - collection_name: "quick_setup", - data: [query_vector], - limit: 3, // The number of results to return -}) +import { MilvusClient, DataType } from "@zilliz/milvus2-sdk-node";​ +​ +const address = "http://localhost:19530";​ +const token = "root:Milvus";​ +const client = new MilvusClient({address, token});​ +​ +// 4. Single vector search​ +var query_vector = [0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592],​ +​ +res = await client.search({​ + collection_name: "my_collection",​ + data: query_vector,​ + limit: 3, // The number of results to return​ +})​ +​ +console.log(res.results)​ +​ +// [​ +// { score: 0.08821295201778412, id: '551' },​ +// { score: 0.0800950899720192, id: '296' },​ +// { score: 0.07794742286205292, id: '43' }​ +// ]​ -console.log(res.results) ``` - - - - - - - - - - - - - - - - - - - - - - - - - -
          ParameterDescription
          collection_nameThe name of an existing collection.
          dataA list of vector embeddings.
          Milvus searches for the most similar vector embeddings to the specified ones.
          limitThe total number of entities to return.
          You can use this parameter in combination with offset in param to enable pagination.
          The sum of this value and offset in param should be less than 16,384.
          search_paramsThe parameter settings specific to this operation.
          • metric_type: The metric type applied to this operation. This should be the same as the one used when you index the vector field specified above. Possible values are L2, IP, COSINE, JACCARD, HAMMING.
          • params: Additional parameters. For details, refer to search().
          - - - - - - - - - - - - - - - - - - - - - - -
          ParameterDescription
          collectionNameThe name of an existing collection.
          dataA list of vector embeddings.
          Milvus searches for the most similar vector embeddings to the specified ones.
          topKThe number of records to return in the search result. This parameter uses the same syntax as the limit parameter, so you should only set one of them.
          You can use this parameter in combination with offset in param to enable pagination.
          The sum of this value and offset in param should be less than 16,384.
          - - - - - - - - - - - - - - - - - - - - - - -
          ParameterDescription
          collection_nameThe name of an existing collection.
          dataA list of vector embeddings.
          Milvus searches for the most similar vector embeddings to the specified ones.
          limitThe total number of entities to return.
          You can use this parameter in combination with offset in param to enable pagination.
          The sum of this value and offset in param should be less than 16,384.
          - -The output is similar to the following: - -
          - Python - Java - Node.js -
          +```curl +export CLUSTER_ENDPOINT="http://localhost:19530"​ +export TOKEN="root:Milvus"​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/search" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "quick_setup",​ + "data": [​ + [0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592]​ + ],​ + "annsField": "vector",​ + "limit": 3​ +}'​ +​ +# {​ +# "code": 0,​ +# "data": [​ +# {​ +# "distance": 0.08821295201778412,​ +# "id": 551​ +# },​ +# {​ +# "distance": 0.0800950899720192,​ +# "id": 296​ +# },​ +# {​ +# "distance": 0.07794742286205292,​ +# "id": 43​ +# }​ +# ]​ +# }​ -```python -[ - [ - { - "id": 0, - "distance": 1.4093276262283325, - "entity": {} - }, - { - "id": 4, - "distance": 0.9902134537696838, - "entity": {} - }, - { - "id": 1, - "distance": 0.8519943356513977, - "entity": {} - }, - { - "id": 5, - "distance": 0.7972343564033508, - "entity": {} - }, - { - "id": 2, - "distance": 0.5928734540939331, - "entity": {} - } - ] -] ``` -```java -{"searchResults": [[ - { - "score": 1.263043, - "fields": { - "vector": [ - 0.9533119, - 0.02538395, - 0.76714665, - 0.35481733, - 0.9845762 - ], - "id": 740 - } - }, - { - "score": 1.2377806, - "fields": { - "vector": [ - 0.7411156, - 0.08687937, - 0.8254139, - 0.08370924, - 0.99095553 - ], - "id": 640 - } - }, - { - "score": 1.1869997, - "fields": { - "vector": [ - 0.87928146, - 0.05324632, - 0.6312755, - 0.28005534, - 0.9542448 - ], - "id": 455 - } - } -]]} -``` +Milvus ranks the search results by their similarity scores to the query vector in descending order. The similarity score is also termed the distance to the query vector, and its value ranges vary with the metric types in use.​ -```javascript -[ - { score: 1.7463608980178833, id: '854' }, - { score: 1.744946002960205, id: '425' }, - { score: 1.7258622646331787, id: '718' } -] -``` +The following table lists the applicable metric types and the corresponding distance ranges.​ -The output showcases the top 5 neighbors nearest to your query vector, including their unique IDs and the calculated distances. +

          Metric Type​

          -### Bulk-vector search +

          Characteristics​

          -A bulk-vector search extends the [single-vector search](https://milvus.io/docs/single-vector-search.md#Single-Vector-Search) concept by allowing multiple query vectors to be searched in a single request. This type of search is ideal for scenarios where you need to find similar vectors for a set of query vectors, significantly reducing the time and computational resources required. +

          Distance Range​

          -In a bulk-vector search, you can include several query vectors in the `data` field. The system processes these vectors in parallel, returning a separate result set for each query vector, each set containing the closest matches found within the collection. +

          L2​

          -Here is an example of searching for two distinct sets of the most similar entities from two query vectors: +

          A smaller value indicates a higher similarity.​

          -
          - Python - Java - Node.js -
          +

          [0, ∞)​

          -```python -# Bulk-vector search -res = client.search( - collection_name="quick_setup", # Replace with the actual name of your collection - data=[ - [0.19886812562848388, 0.06023560599112088, 0.6976963061752597, 0.2614474506242501, 0.838729485096104], - [0.3172005263489739, 0.9719044792798428, -0.36981146090600725, -0.4860894583077995, 0.95791889146345] - ], # Replace with your query vectors - limit=2, # Max. number of search results to return - search_params={"metric_type": "IP", "params": {}} # Search parameters -) - -result = json.dumps(res, indent=4) -print(result) -``` +

          IP​

          -```java -// 5. Batch vector search -query_vectors = Arrays.asList( - Arrays.asList(0.3580376395471989f, -0.6023495712049978f, 0.18414012509913835f, -0.26286205330961354f, 0.9029438446296592f), - Arrays.asList(0.19886812562848388f, 0.06023560599112088f, 0.6976963061752597f, 0.2614474506242501f, 0.838729485096104f) -); +

          A greater value indicates a higher similarity.​

          -searchReq = SearchReq.builder() - .collectionName("quick_setup") - .data(query_vectors) - .topK(2) - .build(); +

          [-1, 1]​

          -searchResp = client.search(searchReq); +

          COSINE​

          -System.out.println(JSONObject.toJSON(searchResp)); -``` +

          A greater value indicates a higher similarity.​

          -```javascript -// 5. Batch vector search -var query_vectors = [ - [0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592], - [0.19886812562848388, 0.06023560599112088, 0.6976963061752597, 0.2614474506242501, 0.838729485096104] -] - -res = await client.search({ - collection_name: "quick_setup", - data: query_vectors, - limit: 2, -}) - -console.log(res.results) -``` +

          [-1, 1]​

          -The output is similar to the following: +

          JACCARD ​

          -
          - Python - Java - Node.js -
          +

          A smaller value indicates a higher similarity.​

          -```python -[ - [ - { - "id": 1, - "distance": 1.3017789125442505, - "entity": {} - }, - { - "id": 7, - "distance": 1.2419954538345337, - "entity": {} - } - ], # Result set 1 - [ - { - "id": 3, - "distance": 2.3358664512634277, - "entity": {} - }, - { - "id": 8, - "distance": 0.5642921924591064, - "entity": {} - } - ] # Result set 2 -] -``` - -```java -// Two sets of vectors are returned as expected - -{"searchResults": [ - [ - { - "score": 1.263043, - "fields": { - "vector": [ - 0.9533119, - 0.02538395, - 0.76714665, - 0.35481733, - 0.9845762 - ], - "id": 740 - } - }, - { - "score": 1.2377806, - "fields": { - "vector": [ - 0.7411156, - 0.08687937, - 0.8254139, - 0.08370924, - 0.99095553 - ], - "id": 640 - } - } - ], - [ - { - "score": 1.8654699, - "fields": { - "vector": [ - 0.4671427, - 0.8378432, - 0.98844475, - 0.82763994, - 0.9729997 - ], - "id": 638 - } - }, - { - "score": 1.8581753, - "fields": { - "vector": [ - 0.735541, - 0.60140246, - 0.86730254, - 0.93152493, - 0.98603314 - ], - "id": 855 - } - } - ] -]} -``` +

          [0, 1]​

          -```javascript -[ - [ - { score: 2.3590476512908936, id: '854' }, - { score: 2.2896690368652344, id: '59' } - [ - { score: 2.664059638977051, id: '59' }, - { score: 2.59483003616333, id: '854' } - ] -] -``` +

          HAMMING ​

          -The results include two sets of nearest neighbors, one for each query vector, showcasing the efficiency of bulk-vector searches in handling multiple query vectors at once. +

          A smaller value indicates a higher similarity.​

          -### Partition search +

          [0, dim(vector)]​

          -Partition search narrows the scope of your search to a specific subset or partition of your collection. This is particularly useful for organized datasets where data is segmented into logical or categorical divisions, allowing for faster search operations by reducing the volume of data to scan. +
          -To conduct a partition search, simply include the name of the target partition in `partition_names` of your search request. This specifies that the `search` operation only considers vectors within the specified partition. +## Bulk-Vector Search​ -Here is an example of searching for entities in `red`: +Similarly, you can include multiple query vectors in a search request. Zilliz Cloud will conduct ANN searches for the query vectors in parallel and return two sets of results.​
          - Python - Java - Node.js + Python + Java + Node.js + Go + cURL
          ```python -# 6.2 Search within a partition -query_vector = [0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592] - -res = client.search( - collection_name="quick_setup", - data=[query_vector], - limit=5, - search_params={"metric_type": "IP", "params": {"level": 1}}, - partition_names=["red"] -) - -print(res) +# 7. Search with multiple vectors​ +# 7.1. Prepare query vectors​ +query_vectors = [​ + [0.041732933, 0.013779674, -0.027564144, -0.013061441, 0.009748648],​ + [0.0039737443, 0.003020432, -0.0006188639, 0.03913546, -0.00089768134]​ +]​ +​ +# 7.2. Start search​ +res = client.search(​ + collection_name="my_collection",​ + data=query_vectors,​ + limit=3,​ +)​ +​ +for hits in res:​ + print("TopK results:")​ + for hit in hits:​ + print(hit)​ +​ +# Output​ +#​ +# [​ +# [​ +# {​ +# "id": 551,​ +# "distance": 0.08821295201778412,​ +# "entity": {}​ +# },​ +# {​ +# "id": 296,​ +# "distance": 0.0800950899720192,​ +# "entity": {}​ +# },​ +# {​ +# "id": 43,​ +# "distance": 0.07794742286205292,​ +# "entity": {}​ +# }​ +# ],​ +# [​ +# {​ +# "id": 730,​ +# "distance": 0.04431751370429993,​ +# "entity": {}​ +# },​ +# {​ +# "id": 333,​ +# "distance": 0.04231833666563034,​ +# "entity": {}​ +# },​ +# {​ +# "id": 232,​ +# "distance": 0.04221535101532936,​ +# "entity": {}​ +# }​ +# ]​ +# ]​ +​ + ``` ```java -// 6.3 Search within partitions -query_vectors = Arrays.asList(Arrays.asList(0.3580376395471989f, -0.6023495712049978f, 0.18414012509913835f, -0.26286205330961354f, 0.9029438446296592f)); - -searchReq = SearchReq.builder() - .collectionName("quick_setup") - .data(query_vectors) - .partitionNames(Arrays.asList("red")) - .topK(5) - .build(); +import io.milvus.v2.service.vector.request.SearchReq​ +import io.milvus.v2.service.vector.request.data.BaseVector;​ +import io.milvus.v2.service.vector.request.data.FloatVec;​ +import io.milvus.v2.service.vector.response.SearchResp​ +​ +List queryVectors = Arrays.asList(​ + new FloatVec(new float[]{0.041732933f, 0.013779674f, -0.027564144f, -0.013061441f, 0.009748648f}),​ + new FloatVec(new float[]{0.0039737443f, 0.003020432f, -0.0006188639f, 0.03913546f, -0.00089768134f})​ +);​ +SearchReq searchReq = SearchReq.builder()​ + .collectionName("quick_setup")​ + .data(queryVectors)​ + .topK(3)​ + .build();​ +​ +SearchResp searchResp = client.search(searchReq);​ +​ +List> searchResults = searchResp.getSearchResults();​ +for (List results : searchResults) {​ + System.out.println("TopK results:");​ + for (SearchResp.SearchResult result : results) {​ + System.out.println(result);​ + }​ +}​ +​ +// Output​ +// TopK results:​ +// SearchResp.SearchResult(entity={}, score=0.49548206, id=1)​ +// SearchResp.SearchResult(entity={}, score=0.320147, id=3)​ +// SearchResp.SearchResult(entity={}, score=0.107413776, id=6)​ +// TopK results:​ +// SearchResp.SearchResult(entity={}, score=0.5678123, id=6)​ +// SearchResp.SearchResult(entity={}, score=0.32368967, id=2)​ +// SearchResp.SearchResult(entity={}, score=0.24108477, id=3)​ -searchResp = client.search(searchReq); - -System.out.println(JSONObject.toJSON(searchResp)); ``` ```javascript -// 6.2 Search within partitions -query_vector = [0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592] - -res = await client.search({ - collection_name: "quick_setup", - data: [query_vector], - partition_names: ["red"], - limit: 5, -}) - -console.log(res.results) -``` - -The output is similar to the following: +// 7. Search with multiple vectors​ +const query_vectors = [​ + [0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592], ​ + [0.19886812562848388, 0.06023560599112088, 0.6976963061752597, 0.2614474506242501, 0.838729485096104]​ +]​ +​ +res = await client.search({​ + collection_name: "quick_setup",​ + vectors: query_vectors,​ + limit: 5,​ +})​ +​ +console.log(res.results)​ +​ +// Output​ +// ​ +// [​ +// [​ +// { score: 0.08821295201778412, id: '551' },​ +// { score: 0.0800950899720192, id: '296' },​ +// { score: 0.07794742286205292, id: '43' }​ +// ],​ +// [​ +// { score: 0.04431751370429993, id: '730' },​ +// { score: 0.04231833666563034, id: '333' },​ +// { score: 0.04221535101532936, id: '232' },​ +// ]​ +// ]​ + +``` + +```curl +export CLUSTER_ENDPOINT="http://localhost:19530"​ +export TOKEN="root:Milvus"​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/search" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "quick_setup",​ + "data": [​ + [0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592],​ + [0.19886812562848388, 0.06023560599112088, 0.6976963061752597, 0.2614474506242501, 0.838729485096104]​ + ],​ + "annsField": "vector",​ + "limit": 3​ +}'​ +​ +# {​ +# "code": 0,​ +# "data": [​ +# [​ +# {​ +# "distance": 0.08821295201778412,​ +# "id": 551​ +# },​ +# {​ +# "distance": 0.0800950899720192,​ +# "id": 296​ +# },​ +# {​ +# "distance": 0.07794742286205292,​ +# "id": 43​ +# }​ +# ],​ +# [​ +# {​ +# "distance": 0.04431751370429993,​ +# "id": 730​ +# },​ +# {​ +# "distance": 0.04231833666563034,​ +# "id": 333​ +# },​ +# {​ +# "distance": 0.04221535101532936,​ +# "id": 232​ +# }​ +# ]​ +# ]​ +# }​ + +``` + +## ANN Search in Partition​ + +Suppose you have created multiple partitions in a collection, and you can narrow the search scope to a specific number of partitions. In that case, you can include the target partition names in the search request to restrict the search scope within the specified partitions. Reducing the number of partitions involved in the search improves search performance.​ + +The following code snippet assumes a partition named **PartitionA** in your collection.​
          - Python - Java - Node.js + Python + Java + Node.js + Go + cURL
          ```python -[ - [ - { - "id": 16, - "distance": 0.9200337529182434, - "entity": {} - }, - { - "id": 14, - "distance": 0.4505271911621094, - "entity": {} - }, - { - "id": 15, - "distance": 0.19924677908420563, - "entity": {} - }, - { - "id": 17, - "distance": 0.0075093843042850494, - "entity": {} - }, - { - "id": 13, - "distance": -0.14609718322753906, - "entity": {} - } - ] -] -``` +# 4. Single vector search​ +query_vector = [0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592]​ +res = client.search(​ + collection_name="my_collection",​ + # highlight-next-line​ + partition_names=["partitionA"],​ + data=[query_vector],​ + limit=3,​ +)​ +​ +for hits in res:​ + print("TopK results:")​ + for hit in hits:​ + print(hit)​ +​ +# [​ +# [​ +# {​ +# "id": 551,​ +# "distance": 0.08821295201778412,​ +# "entity": {}​ +# },​ +# {​ +# "id": 296,​ +# "distance": 0.0800950899720192,​ +# "entity": {}​ +# },​ +# {​ +# "id": 43,​ +# "distance": 0.07794742286205292,​ +# "entity": {}​ +# }​ +# ]​ +# ]​ -```java -{"searchResults": [ - [ - { - "score": 1.1677284, - "fields": { - "vector": [ - 0.9986977, - 0.17964739, - 0.49086612, - 0.23155272, - 0.98438674 - ], - "id": 1435 - } - }, - { - "score": 1.1476475, - "fields": { - "vector": [ - 0.6952647, - 0.13417172, - 0.91045254, - 0.119336545, - 0.9338931 - ], - "id": 1291 - } - }, - { - "score": 1.0969629, - "fields": { - "vector": [ - 0.3363194, - 0.028906643, - 0.6675426, - 0.030419827, - 0.9735209 - ], - "id": 1168 - } - }, - { - "score": 1.0741848, - "fields": { - "vector": [ - 0.9980543, - 0.36063594, - 0.66427994, - 0.17359233, - 0.94954175 - ], - "id": 1164 - } - }, - { - "score": 1.0584627, - "fields": { - "vector": [ - 0.7187005, - 0.12674773, - 0.987718, - 0.3110777, - 0.86093885 - ], - "id": 1085 - } - } - ], - [ - { - "score": 1.8030131, - "fields": { - "vector": [ - 0.59726167, - 0.7054632, - 0.9573117, - 0.94529945, - 0.8664103 - ], - "id": 1203 - } - }, - { - "score": 1.7728865, - "fields": { - "vector": [ - 0.6672442, - 0.60448086, - 0.9325822, - 0.80272985, - 0.8861626 - ], - "id": 1448 - } - }, - { - "score": 1.7536311, - "fields": { - "vector": [ - 0.59663296, - 0.77831805, - 0.8578314, - 0.88818026, - 0.9030075 - ], - "id": 1010 - } - }, - { - "score": 1.7520742, - "fields": { - "vector": [ - 0.854198, - 0.72294194, - 0.9245805, - 0.86126596, - 0.7969224 - ], - "id": 1219 - } - }, - { - "score": 1.7452049, - "fields": { - "vector": [ - 0.96419, - 0.943535, - 0.87611496, - 0.8268136, - 0.79786557 - ], - "id": 1149 - } - } - ] -]} -``` - -```javascript -[ - { score: 3.0258803367614746, id: '1201' }, - { score: 3.004319190979004, id: '1458' }, - { score: 2.880324363708496, id: '1187' }, - { score: 2.8246407508850098, id: '1347' }, - { score: 2.797295093536377, id: '1406' } -] -``` - -Then, search for entities in `blue`: - -
          - Python - Java - Node.js -
          - -```python -res = client.search( - collection_name="quick_setup", - data=[query_vector], - limit=5, - search_params={"metric_type": "IP", "params": {"level": 1}}, - partition_names=["blue"] -) - -print(res) ``` ```java -searchReq = SearchReq.builder() - .collectionName("quick_setup") - .data(query_vectors) - .partitionNames(Arrays.asList("blue")) - .topK(5) - .build(); +import io.milvus.v2.service.vector.request.SearchReq​ +import io.milvus.v2.service.vector.request.data.FloatVec;​ +import io.milvus.v2.service.vector.response.SearchResp​ +​ +FloatVec queryVector = new FloatVec(new float[]{0.3580376395471989f, -0.6023495712049978f, 0.18414012509913835f, -0.26286205330961354f, 0.9029438446296592f});​ +SearchReq searchReq = SearchReq.builder()​ + .collectionName("quick_setup")​ + .partitionNames(Collections.singletonList("partitionA"))​ + .data(Collections.singletonList(queryVector))​ + .topK(3)​ + .build();​ +​ +SearchResp searchResp = client.search(searchReq);​ +​ +List> searchResults = searchResp.getSearchResults();​ +for (List results : searchResults) {​ + System.out.println("TopK results:");​ + for (SearchResp.SearchResult result : results) {​ + System.out.println(result);​ + }​ +}​ +​ +// Output​ +// TopK results:​ +// SearchResp.SearchResult(entity={}, score=0.6395302, id=13)​ +// SearchResp.SearchResult(entity={}, score=0.5408028, id=12)​ +// SearchResp.SearchResult(entity={}, score=0.49696884, id=17)​ -searchResp = client.search(searchReq); - -System.out.println(JSONObject.toJSON(searchResp)); ``` ```javascript -res = await client.search({ - collection_name: "quick_setup", - data: [query_vector], - partition_names: ["blue"], - limit: 5, -}) - -console.log(res.results) -``` - -The output is similar to the following: +// 4. Single vector search​ +var query_vector = [0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592],​ +​ +res = await client.search({​ + collection_name: "quick_setup",​ + // highlight-next-line​ + partition_names: ["partitionA"],​ + data: query_vector,​ + limit: 3, // The number of results to return​ +})​ +​ +console.log(res.results)​ +​ +// [​ +// { score: 0.08821295201778412, id: '551' },​ +// { score: 0.0800950899720192, id: '296' },​ +// { score: 0.07794742286205292, id: '43' }​ +// ]​ + +``` + +```curl +export CLUSTER_ENDPOINT="http://localhost:19530"​ +export TOKEN="root:Milvus"​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/search" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "quick_setup",​ + "partitionNames": ["partitionA"],​ + "data": [​ + [0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592]​ + ],​ + "annsField": "vector",​ + "limit": 3​ +}'​ +​ +# {​ +# "code": 0,​ +# "data": [​ +# {​ +# "distance": 0.08821295201778412,​ +# "id": 551​ +# },​ +# {​ +# "distance": 0.0800950899720192,​ +# "id": 296​ +# },​ +# {​ +# "distance": 0.07794742286205292,​ +# "id": 43​ +# }​ +# ]​ +# }​ + +``` + +## Use Output Fields​ + +In a search result, Zilliz Cloud includes the primary field values and similarity distances/scores of the entities that contain the top-K vector embeddings by default. You can include the target field names in a search request as the output fields to make the search results carry the values from other fields in these entities.​
          - Python - Java - Node.js + Python + Java + Node.js + Go + cURL
          ```python -[ - [ - { - "id": 20, - "distance": 2.363696813583374, - "entity": {} - }, - { - "id": 26, - "distance": 1.0665391683578491, - "entity": {} - }, - { - "id": 23, - "distance": 1.066049575805664, - "entity": {} - }, - { - "id": 29, - "distance": 0.8353596925735474, - "entity": {} - }, - { - "id": 28, - "distance": 0.7484277486801147, - "entity": {} - } - ] -] -``` - -```java -{"searchResults": [ - [ - { - "score": 1.1628494, - "fields": { - "vector": [ - 0.7442872, - 0.046407282, - 0.71031404, - 0.3544345, - 0.9819991 - ], - "id": 1992 - } - }, - { - "score": 1.1470042, - "fields": { - "vector": [ - 0.5505825, - 0.04367262, - 0.9985836, - 0.18922359, - 0.93255126 - ], - "id": 1977 - } - }, - { - "score": 1.1450152, - "fields": { - "vector": [ - 0.89994013, - 0.052991092, - 0.8645576, - 0.6406729, - 0.95679337 - ], - "id": 1573 - } - }, - { - "score": 1.1439825, - "fields": { - "vector": [ - 0.9253267, - 0.15890503, - 0.7999555, - 0.19126713, - 0.898583 - ], - "id": 1552 - } - }, - { - "score": 1.1029172, - "fields": { - "vector": [ - 0.95661926, - 0.18777144, - 0.38115507, - 0.14323527, - 0.93137646 - ], - "id": 1823 - } - } - ], - [ - { - "score": 1.8005109, - "fields": { - "vector": [ - 0.5953582, - 0.7794224, - 0.9388869, - 0.79825854, - 0.9197286 - ], - "id": 1888 - } - }, - { - "score": 1.7714822, - "fields": { - "vector": [ - 0.56805456, - 0.89422905, - 0.88187534, - 0.914824, - 0.8944365 - ], - "id": 1648 - } - }, - { - "score": 1.7561421, - "fields": { - "vector": [ - 0.83421993, - 0.39865613, - 0.92319834, - 0.42695504, - 0.96633124 - ], - "id": 1688 - } - }, - { - "score": 1.7553532, - "fields": { - "vector": [ - 0.89994013, - 0.052991092, - 0.8645576, - 0.6406729, - 0.95679337 - ], - "id": 1573 - } - }, - { - "score": 1.7543385, - "fields": { - "vector": [ - 0.16542226, - 0.38248396, - 0.9888778, - 0.80913955, - 0.9501492 - ], - "id": 1544 - } - } - ] -]} -``` - -```javascript -[ - { score: 2.8421106338500977, id: '1745' }, - { score: 2.838560104370117, id: '1782' }, - { score: 2.8134000301361084, id: '1511' }, - { score: 2.718268871307373, id: '1679' }, - { score: 2.7014894485473633, id: '1597' } -] -``` - -The data in `red` differs from that in `blue`. Therefore, the search results will be constrained to the specified partition, reflecting the unique characteristics and data distribution of that subset. - -### Search with output fields - -Search with output fields allows you to specify which attributes or fields of the matched vectors should be included in the search results. - -You can specify `output_fields` in a request to return results with specific fields. +# 4. Single vector search​ +query_vector = [0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592],​ +​ +res = client.search(​ + collection_name="quick_setup",​ + data=[query_vector],​ + limit=3, # The number of results to return​ + search_params={"metric_type": "IP"}，​ + # highlight-next-line​ + output_fields=["color"]​ +)​ +​ +print(res)​ +​ +# [​ +# [​ +# {​ +# "id": 551,​ +# "distance": 0.08821295201778412,​ +# "entity": {​ +# "color": "orange_6781"​ +# }​ +# },​ +# {​ +# "id": 296,​ +# "distance": 0.0800950899720192,​ +# "entity": {​ +# "color": "red_4794"​ +# }​ +# },​ +# {​ +# "id": 43,​ +# "distance": 0.07794742286205292,​ +# "entity": {​ +# "color": "grey_8510"​ +# }​ +# }​ +# ]​ +# ]​ -Here is an example of returning results with `color` attribute values: - -
          - Python - Java - Node.js -
          - -```python -# Search with output fields -res = client.search( - collection_name="quick_setup", # Replace with the actual name of your collection - data=[[0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592]], - limit=5, # Max. number of search results to return - search_params={"metric_type": "IP", "params": {}}, # Search parameters - output_fields=["color"] # Output fields to return -) - -result = json.dumps(res, indent=4) -print(result) ``` ```java -// 7. Search with output fields -query_vectors = Arrays.asList(Arrays.asList(0.3580376395471989f, -0.6023495712049978f, 0.18414012509913835f, -0.26286205330961354f, 0.9029438446296592f)); +import io.milvus.v2.service.vector.request.SearchReq​ +import io.milvus.v2.service.vector.request.data.FloatVec;​ +import io.milvus.v2.service.vector.response.SearchResp​ +​ +FloatVec queryVector = new FloatVec(new float[]{0.3580376395471989f, -0.6023495712049978f, 0.18414012509913835f, -0.26286205330961354f, 0.9029438446296592f});​ +SearchReq searchReq = SearchReq.builder()​ + .collectionName("quick_setup")​ + .data(Collections.singletonList(queryVector))​ + .topK(3)​ + .outputFields(Collections.singletonList("color"))​ + .build();​ +​ +SearchResp searchResp = client.search(searchReq);​ +​ +List> searchResults = searchResp.getSearchResults();​ +for (List results : searchResults) {​ + System.out.println("TopK results:");​ + for (SearchResp.SearchResult result : results) {​ + System.out.println(result);​ + }​ +}​ +​ +// Output​ +// TopK results:​ +// SearchResp.SearchResult(entity={color=black_9955}, score=0.95944905, id=5)​ +// SearchResp.SearchResult(entity={color=red_7319}, score=0.8689616, id=1)​ +// SearchResp.SearchResult(entity={color=white_5015}, score=0.866088, id=7)​ -searchReq = SearchReq.builder() - .collectionName("quick_setup") - .data(query_vectors) - .outputFields(Arrays.asList("color")) - .topK(5) - .build(); - -searchResp = client.search(searchReq); - -System.out.println(JSONObject.toJSON(searchResp)); ``` ```javascript -// 7. Search with output fields -query_vector = [0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592] - -res = await client.search({ - collection_name: "quick_setup", - data: [query_vector], - limit: 5, - output_fields: ["color"], -}) +// 4. Single vector search​ +var query_vector = [0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592],​ +​ +res = await client.search({​ + collection_name: "quick_setup",​ + data: query_vector,​ + limit: 3, // The number of results to return​ + // highlight-next-line​ + output_fields: ["color"]​ +})​ +​ +console.log(res.results)​ +​ +// [​ +// { score: 0.08821295201778412, id: '551', entity: {"color": "orange_6781"}},​ +// { score: 0.0800950899720192, id: '296' entity: {"color": "red_4794"}},​ +// { score: 0.07794742286205292, id: '43' entity: {"color": "grey_8510"}}​ +// ]​ -console.log(res.results) ``` -The output is similar to the following: - -
          - Python - Java - Node.js -
          - -```python -[ - [ - { - "id": 0, - "distance": 1.4093276262283325, - "entity": { - "color": "pink_8682" - } - }, - { - "id": 16, - "distance": 1.0159327983856201, - "entity": { - "color": "yellow_1496" - } - }, - { - "id": 4, - "distance": 0.9902134537696838, - "entity": { - "color": "red_4794" - } - }, - { - "id": 14, - "distance": 0.9803846478462219, - "entity": { - "color": "green_2899" - } - }, - { - "id": 1, - "distance": 0.8519943356513977, - "entity": { - "color": "red_7025" - } - } - ] -] -``` +```curl +export CLUSTER_ENDPOINT="http://localhost:19530"​ +export TOKEN="root:Milvus"​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/search" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "quick_setup",​ + "data": [​ + [0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592]​ + ],​ + "annsField": "vector",​ + "limit": 3,​ + "outputFields": ["color"]​ +}'​ +​ +# {​ +# "code": 0,​ +# "data": [​ +# {​ +# "distance": 0.08821295201778412,​ +# "id": 551,​ +# "color": "orange_6781"​ +# },​ +# {​ +# "distance": 0.0800950899720192,​ +# "id": 296,​ +# "color": "red_4794"​ +# },​ +# {​ +# "distance": 0.07794742286205292,​ +# "id": 43​ +# "color": "grey_8510"​ +# }​ +# ]​ +# }​ -```java -{"searchResults": [ - [ - { - "score": 1.263043, - "fields": {} - }, - { - "score": 1.2377806, - "fields": {} - }, - { - "score": 1.1869997, - "fields": {} - }, - { - "score": 1.1748955, - "fields": {} - }, - { - "score": 1.1720343, - "fields": {} - } - ] -]} ``` -```javascript +## Use Limit and Offset​ -[ - { score: 3.036271572113037, id: '59', color: 'orange' }, - { score: 3.0267879962921143, id: '1745', color: 'blue' }, - { score: 3.0069446563720703, id: '854', color: 'black' }, - { score: 2.984386682510376, id: '718', color: 'black' }, - { score: 2.916019916534424, id: '425', color: 'purple' } -] -``` +You may notice that the parameter `limit` carried in the search requests determines the number of entities to include in the search results. This parameter specifies the maximum number of entities to return in a single search, and it is usually termed **top-K**.​ -Alongside the nearest neighbors, the search results will include the specified field `color`, providing a richer set of information for each matching vector. +If you wish to perform paginated queries, you can use a loop to send multiple Search requests, with the **Limit** and **Offset** parameters carried in each query request. Specifically, you can set the **Limit** parameter to the number of Entities you want to include in the current query results, and set the **Offset** to the total number of Entities that have already been returned.​ -## Filtered search +The table below outlines how to set the **Limit** and **Offset** parameters for paginated queries when returning 100 Entities at a time.​ -Filtered search applies scalar filters to vector searches, allowing you to refine the search results based on specific criteria. You can find more about filter expressions in [Boolean Expression Rules](https://milvus.io/docs/boolean.md) and examples in [Get & Scalar Query](https://milvus.io/docs/get-and-scalar-query.md). +

          Queries​

          -### Use the `like` operator +

          Entities to return per query​

          -The `like` operator enhances string searches by evaluating patterns including prefixes, infixes, and suffixes: +

          Entities already been returned in total​

          -- __Prefix matching__: To find values starting with a specific prefix, use the syntax `'like "prefix%"'`. -- __Infix matching__: To find values containing a specific sequence of characters anywhere within the string, use the syntax `'like "%infix%"'`. -- __Suffix matching__: To find values ending with a specific suffix, use the syntax `'like "%suffix"'`. +

          The **1st** query​

          -For single-character matching, underscore (`_`) acts as a wildcard for one character, e.g., `'like "y_llow"'`. +

          100​

          -### Special characters in search strings +

          0​

          -If you want to search for a string that contains special characters like underscores (`_`) or percent signs (`%`), which are normally used as wildcards in search patterns (`_` for any single character and `%` for any sequence of characters), you must escape these characters to treat them as literal characters. Use a backslash (`\`) to escape special characters, and remember to escape the backslash itself. For instance: +

          The **2nd** query​

          -- To search for a literal underscore, use `\\_`. -- To search for a literal percent sign, use `\\%`. +

          100​

          -So, if you need to search for the text `"_version_"`, your query should be formatted as `'like "\\_version\\_"'` to ensure the underscores are treated as part of the search term and not as wildcards. +

          100​

          -Filter results whose __color__ is prefixed with __red__: +

          The **3rd** query​

          -
          - Python - Java - Node.js -
          - -```python -# Search with filter -res = client.search( - collection_name="quick_setup", # Replace with the actual name of your collection - data=[[0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592]], - limit=5, # Max. number of search results to return - search_params={"metric_type": "IP", "params": {}}, # Search parameters - output_fields=["color"], # Output fields to return - filter='color like "red%"' -) - -result = json.dumps(res, indent=4) -print(result) -``` - -```java -// 8. Filtered search -query_vectors = Arrays.asList(Arrays.asList(0.3580376395471989f, -0.6023495712049978f, 0.18414012509913835f, -0.26286205330961354f, 0.9029438446296592f)); - -searchReq = SearchReq.builder() - .collectionName("quick_setup") - .data(query_vectors) - .outputFields(Arrays.asList("color_tag")) - .filter("color_tag like \"red%\"") - .topK(5) - .build(); - -searchResp = client.search(searchReq); - -System.out.println(JSONObject.toJSON(searchResp)); -``` - -```javascript -// 8. Filtered search -// 8.1 Filter with "like" operator and prefix wildcard -query_vector = [0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592] - -res = await client.search({ - collection_name: "quick_setup", - data: [query_vector], - limit: 5, - filters: "color_tag like \"red%\"", - output_fields: ["color_tag"] -}) - -console.log(res.results) -``` - -The output is similar to the following: - -
          - Python - Java - Node.js -
          - -```python -[ - [ - { - "id": 4, - "distance": 0.9902134537696838, - "entity": { - "color": "red_4794" - } - }, - { - "id": 1, - "distance": 0.8519943356513977, - "entity": { - "color": "red_7025" - } - }, - { - "id": 6, - "distance": -0.4113418459892273, - "entity": { - "color": "red_9392" - } - } - ] -] -``` - -```java -{"searchResults": [ - [ - { - "score": 1.1869997, - "fields": {"color_tag": "red_3026"} - }, - { - "score": 1.1677284, - "fields": {"color_tag": "red_9030"} - }, - { - "score": 1.1476475, - "fields": {"color_tag": "red_3744"} - }, - { - "score": 1.0969629, - "fields": {"color_tag": "red_4168"} - }, - { - "score": 1.0741848, - "fields": {"color_tag": "red_9678"} - } - ] -]} -``` - -```javascript -[ - { score: 2.5080761909484863, id: '1201', color_tag: 'red_8904' }, - { score: 2.491129159927368, id: '425', color_tag: 'purple_8212' }, - { score: 2.4889798164367676, id: '1458', color_tag: 'red_6891' }, - { score: 2.42964243888855, id: '724', color_tag: 'black_9885' }, - { score: 2.4004223346710205, id: '854', color_tag: 'black_5990' } -] -``` - -Filter results whose __color__ contains the letters __ll__ anywhere within the string: - -
          - Python - Java - Node.js -
          - -```python -# Infix match on color field -res = client.search( - collection_name="quick_setup", # Replace with the actual name of your collection - data=[[0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592]], - limit=5, # Max. number of search results to return - search_params={"metric_type": "IP", "params": {}}, # Search parameters - output_fields=["color"], # Output fields to return - filter='color like "%ll%"' # Filter on color field, infix match on "ll" -) - -result = json.dumps(res, indent=4) -print(result) -``` +

          100​

          -```java -// 8. Filtered search -query_vectors = Arrays.asList(Arrays.asList(0.3580376395471989f, -0.6023495712049978f, 0.18414012509913835f, -0.26286205330961354f, 0.9029438446296592f)); +

          200​

          -searchReq = SearchReq.builder() - .collectionName("quick_setup") - .data(query_vectors) - .outputFields(Arrays.asList("color_tag")) - .filter("color like \"%ll%\"") - .topK(5) - .build(); +

          The **nth** query ​

          -searchResp = client.search(searchReq); +

          100​

          -System.out.println(JSONObject.toJSON(searchResp)); -``` +

          100 x (n-1)​

          -```javascript -// 8. Filtered search -// 8.1 Filter with "like" operator and prefix wildcard -query_vector = [0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592] - -res = await client.search({ - collection_name: "quick_setup", - data: [query_vector], - limit: 5, - filters: "color_tag like \"%ll%\"", - output_fields: ["color_tag"] -}) - -console.log(res.results) -``` +
          -The output is similar to the following: +Note that, the sum of `limit` and `offset` in a single ANN search should be less than 16,384.​
          - Python - Java - Node.js + Python + Java + Node.js + Go + cURL
          ```python -[ - [ - { - "id": 5, - "distance": 0.7972343564033508, - "entity": { - "color": "yellow_4222" - } - } - ] -] -``` +# 4. Single vector search​ +query_vector = [0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592],​ +​ +res = client.search(​ + collection_name="quick_setup",​ + data=[query_vector],​ + limit=3, # The number of results to return​ + search_params={​ + "metric_type": "IP", ​ + # highlight-next-line​ + "offset": 10 # The records to skip​ + }​ +)​ -```java -{"searchResults": [ - [ - { - "score": 1.1869997, - "fields": {"color_tag": "yellow_4222"} - } - ] -]} -``` - -```javascript -[ - { score: 2.5080761909484863, id: '1201', color_tag: 'yellow_4222' } -] -``` - -## Range search - -Range search allows you to find vectors that lie within a specified distance range from your query vector. - -By setting `radius` and optionally `range_filter`, you can adjust the breadth of your search to include vectors that are somewhat similar to the query vector, providing a more comprehensive view of potential matches. - -- `radius`: Defines the outer boundary of your search space. Only vectors that are within this distance from the query vector are considered potential matches. - -- `range_filter`: While `radius` sets the outer limit of the search, `range_filter` can be optionally used to define an inner boundary, creating a distance range within which vectors must fall to be considered matches. - -
          - Python - Java - Node.js -
          - -```python -# Conduct a range search -search_params = { - "metric_type": "IP", - "params": { - "radius": 0.8, # Radius of the search circle - "range_filter": 1.0 # Range filter to filter out vectors that are not within the search circle - } -} - -res = client.search( - collection_name="quick_setup", # Replace with the actual name of your collection - data=[[0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592]], - limit=3, # Max. number of search results to return - search_params=search_params, # Search parameters - output_fields=["color"], # Output fields to return -) - -result = json.dumps(res, indent=4) -print(result) ``` ```java -// 9. Range search -query_vectors = Arrays.asList(Arrays.asList(0.3580376395471989f, -0.6023495712049978f, 0.18414012509913835f, -0.26286205330961354f, 0.9029438446296592f)); - -searchReq = SearchReq.builder() - .collectionName("quick_setup") - .data(query_vectors) - .outputFields(Arrays.asList("color_tag")) - .searchParams(Map.of("radius", 0.1, "range", 1.0)) - .topK(5) - .build(); - -searchResp = client.search(searchReq); +import io.milvus.v2.service.vector.request.SearchReq​ +import io.milvus.v2.service.vector.request.data.FloatVec;​ +import io.milvus.v2.service.vector.response.SearchResp​ +​ +FloatVec queryVector = new FloatVec(new float[]{0.3580376395471989f, -0.6023495712049978f, 0.18414012509913835f, -0.26286205330961354f, 0.9029438446296592f});​ +SearchReq searchReq = SearchReq.builder()​ + .collectionName("quick_setup")​ + .data(Collections.singletonList(queryVector))​ + .topK(3)​ + .offset(10)​ + .build();​ +​ +SearchResp searchResp = client.search(searchReq);​ +​ +List> searchResults = searchResp.getSearchResults();​ +for (List results : searchResults) {​ + System.out.println("TopK results:");​ + for (SearchResp.SearchResult result : results) {​ + System.out.println(result);​ + }​ +}​ +​ +// Output​ +// TopK results:​ +// SearchResp.SearchResult(entity={}, score=0.24120237, id=16)​ +// SearchResp.SearchResult(entity={}, score=0.22559784, id=9)​ +// SearchResp.SearchResult(entity={}, score=-0.09906838, id=2)​ -System.out.println(JSONObject.toJSON(searchResp)); ``` ```javascript -// 9. Range search -query_vector = [0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592] - -res = await client.search({ - collection_name: "quick_setup", - data: [query_vector], - limit: 5, - params: { - radius: 0.1, - range: 1.0 - }, - output_fields: ["color_tag"] -}) - -console.log(res.results) -``` - -The output is similar to the following: - -
          - Python - Java - Node.js -
          +// 4. Single vector search​ +var query_vector = [0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592],​ +​ +res = await client.search({​ + collection_name: "quick_setup",​ + data: query_vector,​ + limit: 3, // The number of results to return,​ + // highlight-next-line​ + offset: 10 // The record to skip.​ +})​ -```python -[ - [ - { - "id": 4, - "distance": 0.9902134537696838, - "entity": { - "color": "red_4794" - } - }, - { - "id": 14, - "distance": 0.9803846478462219, - "entity": { - "color": "green_2899" - } - }, - { - "id": 1, - "distance": 0.8519943356513977, - "entity": { - "color": "red_7025" - } - } - ] -] ``` -```java -{"searchResults": [ - [ - { - "score": 1.263043, - "fields": {"color_tag": "green_2052"} - }, - { - "score": 1.2377806, - "fields": {"color_tag": "purple_3709"} - }, - { - "score": 1.1869997, - "fields": {"color_tag": "red_3026"} - }, - { - "score": 1.1748955, - "fields": {"color_tag": "black_1646"} - }, - { - "score": 1.1720343, - "fields": {"color_tag": "green_4853"} - } - ] -]} -``` +```curl +export CLUSTER_ENDPOINT="http://localhost:19530"​ +export TOKEN="root:Milvus"​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/entities/search" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d '{​ + "collectionName": "quick_setup",​ + "data": [​ + [0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592]​ + ],​ + "annsField": "vector",​ + "limit": 3,​ + "offset": 10​ +}'​ -```javascript -[ - { score: 2.3387961387634277, id: '718', color_tag: 'black_7154' }, - { score: 2.3352415561676025, id: '1745', color_tag: 'blue_8741' }, - { score: 2.290485382080078, id: '1408', color_tag: 'red_2324' }, - { score: 2.285870313644409, id: '854', color_tag: 'black_5990' }, - { score: 2.2593345642089844, id: '1309', color_tag: 'red_8458' } -] ``` -You will observe that all the entities returned have a distance that falls within the range of 0.8 to 1.0 from the query vector. - -The parameter settings for `radius` and `range_filter` vary with the metric type in use. - -| __Metric Type__ | __Charactericstics__ | __Range Search Settings__ | -| ---------------- | ------------------------------------------------- | -------------------------------------------------------------------------------------------------------- | -| `L2` | Smaller L2 distances indicate higher similarity. | To exclude the closest vectors from results, ensure that:
          `range_filter` <= distance < `radius` | -| `IP` | Larger IP distances indicate higher similarity. | To exclude the closest vectors from results, ensure that:
          `radius` < distance <= `range_filter` | -| `COSINE` | Larger cosine value indicates higher similarity. | To exclude the closest vectors from results, ensure that:
          `radius` < distance <= `range_filter` | -| `JACCARD` | Smaller Jaccard distances indicate higher similarity. | To exclude the closest vectors from results, ensure that:
          `range_filter` <= distance < `radius` | -| `HAMMING` | Smaller Hamming distances indicate higher similarity. | To exclude the closest vectors from results, ensure that:
          `range_filter` <= distance < `radius` | +## Enhancing ANN Search​ -To learn more about distance metric types, refer to [Similarity Metrics](metric.md). +AUTOINDEX considerably flattens the learning curve of ANN searches. However, the search results may not always be correct as the top-K increases. By reducing the search scope, improving search result relevancy, and diversifying the search results, Zilliz Cloud works out the following search enhancements.​ -## Grouping search +- Filtered Search​ -In Milvus, grouping search is designed to improve comprehensiveness and accuracy for search results. + You can include filtering conditions in a search request so that Zilliz Cloud conducts metadata filtering before conducting ANN searches, reducing the search scope from the whole collection to only the entities matching the specified filtering conditions.​ -Consider a scenario in RAG, where loads of documents are split into various passages, and each passage is represented by one vector embedding. Users want to find the most relevant passages to prompt the LLMs accurately. The ordinary Milvus search function can meet this requirement, but it may result in highly skewed and biased results: most of the passages come from only a few documents, and the comprehensiveness of the search results is very poor. This can seriously impair the accuracy or even correctness of the results given by the LLM and influence the LLM users' experience negatively. + For more about metadata filtering and filtering conditions, refer to [​Filtered Search](filtered-search.md) and [​Metadata Filtering](boolean.md).​ -Grouping search can effectively solve this problem. By passing a group_by_field and group_size, Milvus users can bucket the search results into several groups and ensure that the number of entities from each group does not exceed a specific group_size. This feature can significantly enhance the comprehensiveness and fairness of search results, noticeably improving the quality of LLM output. +- Range Search​ -Here is the example code to group search results by field: + You can improve search result relevancy by restricting the distance or score of the returned entities within a specific range. In Zilliz Cloud, a range search involves drawing two concentric circles with the vector embedding most similar to the query vector as the center. The search request specifies the radius of both circles, and Zilliz Cloud returns all vector embeddings that fall within the outer circle but not the inner circle.​ -```python -# Connect to Milvus -client = MilvusClient(uri='http://localhost:19530') # Milvus server address - -# Load data into collection -client.load_collection("group_search") # Collection name - -# Group search results -res = client.search( - collection_name="group_search", # Collection name - data=[[0.14529211512077012, 0.9147257273453546, 0.7965055218724449, 0.7009258593102812, 0.5605206522382088]], # Query vector - search_params={ - "metric_type": "L2", - "params": {"nprobe": 10}, - }, # Search parameters - limit=5, # Max. number of groups to return - group_by_field="doc_id", # Group results by document ID - group_size=2, # returned at most 2 passages per document, the default value is 1 - group_strict_size=True, # ensure every group contains exactly 3 passages - output_fields=["doc_id", "passage_id"] -) - -# Retrieve the values in the `doc_id` column -doc_ids = [result['entity']['doc_id'] for result in res[0]] -passage_ids = [result['entity']['passage_id'] for result in res[0]] - -print(doc_ids) -print(passage_ids) -``` + For more about range search, refer to [​Range Search](range-search.md).​ -The output is similar to the following: +- Grouping Search​ -```python -["doc_11", "doc_11", "doc_7", "doc_7", "doc_3", "doc_3", "doc_2", "doc_2", "doc_8", "doc_8"] -[5, 10, 11, 10, 9, 6, 5, 4, 9, 2] -``` + If the returned entities hold the same value in a specific field, the search results may not represent the distribution of all vector embeddings in the vector space. To diversify the search results, consider using the grouping search.​ -In the given output, it can be observed that for each document, exactly two passages are retrieved and a total of 5 documents collectively make up the results. + For more about grouping search, refer to [​Grouping Search](grouping-search.md),​ -For comparison, let's comment out the group-related parameters and conduct a regular search: +- Hybrid Search​ -```python -# Connect to Milvus -client = MilvusClient(uri='http://localhost:19530') # Milvus server address - -# Load data into collection -client.load_collection("group_search") # Collection name - -# Search without `group_by_field` -res = client.search( - collection_name="group_search", # Collection name - data=query_passage_vector, # Replace with your query vector - search_params={ - "metric_type": "L2", - "params": {"nprobe": 10}, - }, # Search parameters - limit=5, # Max. number of search results to return - # group_by_field="doc_id", # Group results by document ID - # group_size=2, - # group_strict_size=True, - output_fields=["doc_id", "passage_id"] -) - -# Retrieve the values in the `doc_id` column -doc_ids = [result['entity']['doc_id'] for result in res[0]] -passage_ids = [result['entity']['passage_id'] for result in res[0]] - -print(doc_ids) -print(passage_ids) -``` + A collection can include up to four vector fields to save the vector embeddings generated using different embedding models. By doing so, you can use a hybrid search to rerank the search results from these vector fields, improving the recall rate.​ -The output is similar to the following: + For more about hybrid search, refer to [​Hybrid Search](multi-vector-search.md).​ -```python -["doc_11", "doc_11", "doc_11", "doc_11", "doc_11"] -[1, 10, 3, 12, 9] -``` +- Search Iterator​ -In the given output, it can be observed that "doc_11" completely dominated the search results, overshadowing the high-quality paragraphs from other documents, which can be a poor prompt to LLM. + A single ANN search returns a maximum of 16,384 entities. Consider using search iterators if you need more entities to return in a single search.​ -One more point to note: by default, grouping_search will return results instantly when it has enough groups, which may lead to the number of results in each group not being sufficient to meet the group_size. If you care about the number of results for each group, set group_strict_size=True as shown in the code above. This will make Milvus strive to obtain enough results for each group, at a slight cost to performance. + For details on search iterators, refer to [​Search Iterator](with-iterators.md).​ -__Limitations__ +- Full-Text Search​ -- __Indexing__: This grouping feature works only for collections that are indexed with the __HNSW__, __IVF_FLAT__, or __FLAT__ type. For more information, refer to [In-memory Index](https://milvus.io/docs/index.md#HNSW). + Full text search is a feature that retrieves documents containing specific terms or phrases in text datasets, then ranking the results based on relevance. This feature overcomes semantic search limitations, which might overlook precise terms, ensuring you receive the most accurate and contextually relevant results. Additionally, it simplifies vector searches by accepting raw text input, automatically converting your text data into sparse embeddings without the need to manually generate vector embeddings.​ -- __Vector__: Currently, grouping search does not support a vector field of the __BINARY_VECTOR__ type. For more information on data types, refer to [Supported data types](https://milvus.io/docs/schema.md#Supported-data-types). + For details on full-text search, refer to [​Full Text Search](full-text-search.md).​ -- __Field__: Currently, grouping search allows only for a single column. You cannot specify multiple field names in the `group_by_field` config. Additionally, grouping search is incompatible with data types of JSON, FLOAT, DOUBLE, ARRAY, or vector fields. +- Keyword Match​ -- __Performance Impact__: Be mindful that performance degrades with increasing query vector counts. Using a cluster with 2 CPU cores and 8 GB of memory as an example, the execution time for grouping search increases proportionally with the number of input query vectors. + Keyword match in Milvus enables precise document retrieval based on specific terms. This feature is primarily used for filtered search to satisfy specific conditions and can incorporate scalar filtering to refine query results, allowing similarity searches within vectors that meet scalar criteria.​ -- __Functionality__: Currently, grouping search is not supported by [range search](https://milvus.io/docs/single-vector-search.md#Range-search), [search iterators](https://milvus.io/docs/with-iterators.md#Search-with-iterator) + For details on keyword match, refer to [​Keyword Match](keyword-match.md).​ -## Search parameters +- Use Partition Key​ -In the above searches except the range search, the default search parameters apply. In normal cases, you do not need to manually set search parameters. + Involving multiple scalar fields in metadata filtering and using a rather complicated filtering condition may affect search efficiency. Once you set a scalar field as the partition key and use a filtering condition involving the partition key in the search request, it can help restrict the search scope within the partitions corresponding to the specified partition key values. ​ -```python -# In normal cases, you do not need to set search parameters manually -# Except for range searches. -search_parameters = { - 'metric_type': 'L2', - 'params': { - 'nprobe': 10, - 'level': 1， - 'radius': 1.0 - 'range_filter': 0.8 - } -} -``` + For details on the partition key, refer to [​Use Partition Key](use-partition-key.md).​ -The following table lists all possible settings in the search parameters. +- Use mmap​ -| __Parameter Name__ | __Parameter Description__ | -| ---------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `metric_type` | How to measure similarity between vector embeddings.
          Possible values are `IP`, `L2`, `COSINE`, `JACCARD`, and `HAMMING`, and defaults to that of the loaded index file. | -| `params.nprobe` | Number of units to query during the search.
          The value falls in the range [1, nlist[1]]. | -| `params.level` | Search precision level.
          Possible values are `1`, `2`, and `3`, and defaults to `1`. Higher values yield more accurate results but slower performance. | -| `params.radius` | Defines the outer boundary of your search space. Only vectors that are within this distance from the query vector are considered potential matches.
          The value range is determined by the `metric_type` parameter. For instance, if `metric_type` is set to `L2`, the valid value range is `[0, ∞]`. If `metric_type` is set to `COSINE`, the valid value range is `[-1, 1]`. For more information, refer to [Similarity Metrics](metric.md). | -| `params.range_filter` | While `radius` sets the outer limit of the search, `range_filter` can be optionally used to define an inner boundary, creating a distance range within which vectors must fall to be considered matches.
          The value range is determined by the `metric_type` parameter. For instance, if `metric_type` is set to `L2`, the valid value range is `[0, ∞]`. If `metric_type` is set to `COSINE`, the valid value range is `[-1, 1]`. For more information, refer to [Similarity Metrics](metric.md). | + In Milvus, memory-mapped files allow for direct mapping of file contents into memory. This feature enhances memory efficiency, particularly in situations where available memory is scarce but complete data loading is infeasible. This optimization mechanism can increase data capacity while ensuring performance up to a certain limit; however, when the amount of data exceeds memory by too much, search and query performance may suffer serious degradation, so please choose to turn this feature on or off as appropriate. -
          + For details on mmap-settings, refer to [​Use mmap](mmap.md).​ -

          notes

          +- Clustering Compaction​ -

          [1] Number of cluster units after indexing. When indexing a collection, Milvus sub-divides the vector data into multiple cluster units, the number of which varies with the actual index settings.

          -

          [2] Number of entities to return in a search.

          + Clustering compaction is designed to improve search performance and reduce costs in large collections. This guide will help you understand clustering compaction and how this feature can improve search performance. -
          + For details on clustering compactions, refer to [​Clustering Compaction](clustering-compaction.md).​ diff --git a/v2.4.x/site/en/userGuide/search-query-get/use-partition-key.md b/v2.4.x/site/en/userGuide/search-query-get/use-partition-key.md new file mode 100644 index 0000000000..f6a2f45253 --- /dev/null +++ b/v2.4.x/site/en/userGuide/search-query-get/use-partition-key.md @@ -0,0 +1,261 @@ +--- +id: use-partition-key.md +title: Use Partition Key +--- + +# Use Partition Key​ + +The Partition Key is a search optimization solution based on partitions. By designating a specific scalar field as the Partition Key and specifying filtering conditions based on the Partition Key during the search, the search scope can be narrowed down to several partitions, thereby improving search efficiency. This article will introduce how to use the Partition Key and related considerations.​ + +## Overview​ + +In Milvus, you can use partitions to implement data segregation and improve search performance by restricting the search scope to specific partitions. If you choose to manage partitions manually, you can create a maximum of 1,024 partitions in a collection, and insert entities into these partitions based on a specific rule so that you can narrow the search scope by restricting searches within a specific number of partitions.​ + +Milvus introduces the Partition Key for you to reuse partitions in data segregation to overcome the limit on the number of partitions you can create in a collection. When creating a collection, you can use a scalar field as the Partition Key. Once the collection is ready, Milvus creates the specified number of partitions inside the collection with each partition corresponding to a range of the values in the Partition Key. Upon receiving inserted entities, Milvus stores them into different partitions based on their Partition Key values.​ + +![Partition v.s. Partition Key](../../../../assets/partition-vs-partition-key.png) + +The following figure illustrates how Milvus processes the search requests in a collection with or without the Partition Key feature enabled. ​ + +- If the Partition Key is disabled, Milvus searches for entities that are the most similar to the query vector within the collection. You can narrow the search scope if you know which partition contains the most relevant results. ​ + +- If the Partition Key is enabled, Milvus determines the search scope based on the Partition Key value specified in a search filter and scans only the entities within the partitions that match. ​ + +![With or Without Partition Key](../../../../assets/with-and-without-partition-key.png) + +## Use Partition Key​ + +To use the Partition Key, you need to​ + +- Set the Partition Key,​ + +- Set the number of partitions to create (Optional), and ​ + +- Create a filtering condition based on the Partition Key.​ + +### Set Partition Key​ + +To designate a scalar field as the Partition Key, you need to set its `is_partition_key` attribute to `true` when you add the scalar field.​ + +
          + Python + Java + Node.js + Go + cURL +
          + +```python +from pymilvus import (​ + MilvusClient, DataType​ +)​ +​ +client = MilvusClient(​ + uri="http://localhost:19530",​ + token="root:Milvus"​ +)​ +​ +schema = client.create_schema()​ +​ +# Add the partition key​ +schema.add_field(​ + field_name="my_varchar", ​ + datatype=DataType.VARCHAR, ​ + max_length=512,​ + # highlight-next-line​ + is_partition_key=True,​ +)​ + +``` + +```java +import io.milvus.v2.client.ConnectConfig;​ +import io.milvus.v2.client.MilvusClientV2;​ +import io.milvus.v2.common.DataType;​ +import io.milvus.v2.service.collection.request.AddFieldReq;​ +import io.milvus.v2.service.collection.request.CreateCollectionReq;​ +​ +MilvusClientV2 client = new MilvusClientV2(ConnectConfig.builder()​ + .uri("http://localhost:19530")​ + .token("root:Milvus")​ + .build());​ +​ +// Create schema​ +CreateCollectionReq.CollectionSchema schema = client.createSchema();​ +​ +// Add the partition key​ +schema.addField(AddFieldReq.builder()​ + .fieldName("my_varchar")​ + .dataType(DataType.VarChar)​ + .maxLength(512)​ + // highlight-next-line​ + .isPartitionKey(true)​ + .build());​ + +``` + +```javascript +import { MilvusClient, DataType } from "@zilliz/milvus2-sdk-node";​ +​ +const address = "http://localhost:19530";​ +const token = "root:Milvus";​ +const client = new MilvusClient({address, token});​ +​ +// 3. Create a collection in customized setup mode​ +// 3.1 Define fields​ +const fields = [​ + {​ + name: "my_varchar",​ + data_type: DataType.VarChar,​ + max_length: 512,​ + // highlight-next-line​ + is_partition_key: true​ + }​ +]​ + +``` + +```curl +export schema='{​ + "autoId": true,​ + "enabledDynamicField": false,​ + "fields": [​ + {​ + "fieldName": "my_id",​ + "dataType": "Int64",​ + "isPrimary": true​ + },​ + {​ + "fieldName": "my_vector",​ + "dataType": "FloatVector",​ + "elementTypeParams": {​ + "dim": "5"​ + }​ + },​ + {​ + "fieldName": "my_varchar",​ + "dataType": "VarChar",​ + "isPartitionKey": true,​ + "elementTypeParams": {​ + "max_length": 512​ + }​ + }​ + ]​ + }'​ + +``` + +### Set Partition Numbers​ + +When you designate a scalar field in a collection as the Partition Key, Milvus automatically creates 16 partitions in the collection. Upon receiving an entity, Milvus chooses a partition based on the Partition Key value of this entity and stores the entity in the partition, resulting in some or all partitions holding entities with different Partition Key values. ​ + +You can also determine the number of partitions to create along with the collection. This is valid only if you have a scalar field designated as the Partition Key.​ + +
          + Python + Java + Node.js + Go + cURL +
          + +```python +client.create_collection(​ + collection_name="my_collection",​ + schema=schema,​ + # highlight-next-line​ + num_partitions=1024​ +)​ + +``` + +```java +import io.milvus.v2.service.collection.request.CreateCollectionReq;​ +​ +CreateCollectionReq createCollectionReq = CreateCollectionReq.builder()​ + .collectionName("my_collection")​ + .collectionSchema(schema)​ + .numPartitions(1024)​ + .build();​ + client.createCollection(createCollectionReq);​ + +``` + +```javascript +await client.create_collection({​ + collection_name: "my_collection",​ + schema: schema,​ + num_partitions: 1024​ +})​ + +``` + +```curl +export params='{​ + "partitionsNum": 1024​ +}'​ +​ +export CLUSTER_ENDPOINT="http://localhost:19530"​ +export TOKEN="root:Milvus"​ +​ +curl --request POST \​ +--url "${CLUSTER_ENDPOINT}/v2/vectordb/collections/create" \​ +--header "Authorization: Bearer ${TOKEN}" \​ +--header "Content-Type: application/json" \​ +-d "{​ + \"collectionName\": \"myCollection\",​ + \"schema\": $schema,​ + \"params\": $params​ +}"​ + +``` + +### Create Filtering Condition​ + +When conducting ANN searches in a collection with the Partition Key feature enabled, you need to include a filtering expression involving the Partition Key in the search request. In the filtering expression, you can restrict the Partition Key value within a specific range so that Milvus restricts the search scope within the corresponding partitions.​ + +The following examples demonstrate Partition-Key-based filtering based on a specific Partition Key value and a set of Partition Key values.​ + +
          + Python + Java + Node.js + Go + cURL +
          + +```python +# Filter based on a single partition key value, or​ +filter='partition_key == "x" && '​ +​ +# Filter based on multiple partition key values​ +filter='partition_key in ["x", "y", "z"] && '​ + +``` + +```java +// Filter based on a single partition key value, or​ +String filter = "partition_key == 'x' && ";​ +​ +// Filter based on multiple partition key values​ +String filter = "partition_key in ['x', 'y', 'z'] && ";​ + +``` + +```javascript +// Filter based on a single partition key value, or​ +const filter = 'partition_key == "x" && '​ +​ +// Filter based on multiple partition key values​ +const filter = 'partition_key in ["x", "y", "z"] && '​ + +``` + +```curl +# Filter based on a single partition key value, or​ +export filter='partition_key == "x" && '​ +​ +# Filter based on multiple partition key values​ +export filter='partition_key in ["x", "y", "z"] && '​ + +``` diff --git a/v2.4.x/site/en/userGuide/search-query-get/with-iterators.md b/v2.4.x/site/en/userGuide/search-query-get/with-iterators.md index 1b0c974cde..18769457e1 100644 --- a/v2.4.x/site/en/userGuide/search-query-get/with-iterators.md +++ b/v2.4.x/site/en/userGuide/search-query-get/with-iterators.md @@ -5,593 +5,127 @@ summary: Milvus provides search and query iterators for iterating results with a title: With Iterators --- -# With Iterators +# Search Iterator​ -Milvus provides search and query iterators for iterating through a large volume of entities. Since Milvus limits TopK to 16384, users can use iterators to return large numbers or even whole entities in a collection in batch mode. +The ANN Search has a maximum limit on the number of entities that can be recalled in a single query, and simply using basic ANN Search may not meet the demands of large-scale retrieval. For ANN Search requests where topK exceeds 16,384, it is advisable to consider using the SearchIterator. This section will introduce how to use the SearchIterator and related considerations.​ -## Overview +## Overview​ -Iterators are efficient tool for scanning a whole collection or iterating through a large volume of entities by specifying primary key values or a filter expression. Compared to a search or query call with __offset__ and __limit__ parameters, using iterators is more efficient and scalable. +A Search request returns search results, while a SearchIterator returns an iterator. You can call the **next()** method of this iterator to get the search results.​ -### Benefits of using iterators +Specifically, you can use the SearchIterators as follows:​ -- __Simplicity__: Eliminates the complex __offset__ and __limit__ settings. +1. Create a SearchIterator and set **the number of entities to return per search request** and **the total number of entities to return**.​ -- __Efficiency__: Provides scalable data retrieval by fetching only the data in need. +2. Call the **next()** method of the SearchIterator in a loop to get the search result in a paginated manner.​ -- __Consistency__: Ensures a consistent dataset size with boolean filters. +3. Call the **close()** method of the iterator to end the loop if the **next()** method returns an empty result.​ -
          +## Create SearchIterator​ -

          notes

          - -
            - -
          • This feature is available for Milvus 2.3.x or later.
            • - -
          - -
          - -## Preparations - -The following preparation step connects to Milvus and inserts randomly generated entities into a collection. - -### Step 1: Create a collection - -
          - -Use [`MilvusClient`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Client/MilvusClient.md) to connect to the Milvus server and [`create_collection()`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Collections/create_collection.md) to create a collection. - -
          - -
          - -Use [`MilvusClientV2`](https://milvus.io/api-reference/java/v2.4.x/v2/Client/MilvusClientV2.md) to connect to the Milvus server and [`createCollection()`](https://milvus.io/api-reference/java/v2.4.x/v2/Collections/createCollection.md) to create a collection. - -
          - -
          - Python - Java -
          - -```python -from pymilvus import MilvusClient - -# 1. Set up a Milvus client -client = MilvusClient( - uri="http://localhost:19530" -) - -# 2. Create a collection -client.create_collection( - collection_name="quick_setup", - dimension=5, -) -``` - -```java -import io.milvus.client.MilvusServiceClient; -import io.milvus.param.ConnectParam; -import io.milvus.param.highlevel.collection.CreateSimpleCollectionParam; - -String CLUSTER_ENDPOINT = "http://localhost:19530"; - -// 1. Connect to Milvus server -ConnectParam connectParam = ConnectParam.newBuilder() - .withUri(CLUSTER_ENDPOINT) - .build(); - -MilvusServiceClient client = new MilvusServiceClient(connectParam); - -// 2. Create a collection -CreateSimpleCollectionParam createCollectionParam = CreateSimpleCollectionParam.newBuilder() - .withCollectionName("quick_setup") - .withDimension(5) - .build(); - -client.createCollection(createCollectionParam); -``` - -### Step 2: Insert randomly generated entities - -
          - -Use [`insert()`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Vector/insert.md) to insert entities into the collection. - -
          - -
          - -Use [`insert()`](https://milvus.io/api-reference/java/v2.4.x/v2/Vector/insert.md) to insert entities into the collection. - -
          - -
          - Python - Java -
          - -```python -# 3. Insert randomly generated vectors -colors = ["green", "blue", "yellow", "red", "black", "white", "purple", "pink", "orange", "brown", "grey"] -data = [] - -for i in range(10000): - current_color = random.choice(colors) - current_tag = random.randint(1000, 9999) - data.append({ - "id": i, - "vector": [ random.uniform(-1, 1) for _ in range(5) ], - "color": current_color, - "tag": current_tag, - "color_tag": f"{current_color}_{str(current_tag)}" - }) - -print(data[0]) - -# Output -# -# { -# "id": 0, -# "vector": [ -# -0.5705990742218152, -# 0.39844925120642083, -# -0.8791287928610869, -# 0.024163154953680932, -# 0.6837669917169638 -# ], -# "color": "purple", -# "tag": 7774, -# "color_tag": "purple_7774" -# } - -res = client.insert( - collection_name="quick_setup", - data=data, -) - -print(res) - -# Output -# -# { -# "insert_count": 10000, -# "ids": [ -# 0, -# 1, -# 2, -# 3, -# 4, -# 5, -# 6, -# 7, -# 8, -# 9, -# "(9990 more items hidden)" -# ] -# } -``` - -```java -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Random; - -import com.alibaba.fastjson.JSONObject; - -import io.milvus.param.R; -import io.milvus.param.dml.InsertParam; -import io.milvus.response.MutationResultWrapper; -import io.milvus.grpc.MutationResult; - - -// 3. Insert randomly generated vectors into the collection -List colors = Arrays.asList("green", "blue", "yellow", "red", "black", "white", "purple", "pink", "orange", "brown", "grey"); -List data = new ArrayList<>(); - -for (int i=0; i<10000; i++) { - Random rand = new Random(); - String current_color = colors.get(rand.nextInt(colors.size()-1)); - JSONObject row = new JSONObject(); - row.put("id", Long.valueOf(i)); - row.put("vector", Arrays.asList(rand.nextFloat(), rand.nextFloat(), rand.nextFloat(), rand.nextFloat(), rand.nextFloat())); - row.put("color_tag", current_color + "_" + String.valueOf(rand.nextInt(8999) + 1000)); - data.add(row); -} - -InsertParam insertParam = InsertParam.newBuilder() - .withCollectionName("quick_setup") - .withRows(data) - .build(); - -R insertRes = client.insert(insertParam); - -if (insertRes.getStatus() != R.Status.Success.getCode()) { - System.err.println(insertRes.getMessage()); -} - -MutationResultWrapper wrapper = new MutationResultWrapper(insertRes.getData()); -System.out.println(wrapper.getInsertCount()); -``` - -## Search with iterator - -Iterators make similarity searches more scalable. - -
          - -To search with an iterator, call the [search_iterator()](https://milvus.io/api-reference/pymilvus/v2.4.x/ORM/Collection/search_iterator.md) method: - -
          - -
          - -To search with an iterator, call the [searchIterator()](https://milvus.io/api-reference/java/v2.4.x/v1/QuerySearch/searchIterator.md) method: - -
          - -1. Initialize the search iterator to define the search parameters and output fields. - -1. Use the __next()__ method within a loop to paginate through the search results. - - - If the method returns an empty array, the loop ends, and no more pages are available. - - - All results carry the specified output fields. - -1. Manually call the __close()__ method to close the iterator once all data has been retrieved. +The following code snippet demonstrates how to create a SearchIterator.​
          - Python - Java + Python + Java
          ```python -from pymilvus import Collection,connections - -# 4. Search with iterator -connections.connect(host="127.0.0.1", port=19530) -collection = Collection("quick_setup") - -query_vectors = [[0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592]] -search_params = { - "metric_type": "IP", - "params": {"nprobe": 10} -} - -iterator = collection.search_iterator( - data=query_vectors, - anns_field="vector", - batch_size=10, - param=search_params, - output_fields=["color_tag"], - limit=300 -) -# search 300 entities totally with 10 entities per page - -results = [] +from pymilvus import connections, Collection​ +​ +connections.connect(​ + uri="http://localhost:19530",​ + token="root:Milvus"​ +)​ +​ +# create iterator​ +query_vectors = [​ + [0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592]]​ +​ +collection = Collection("iterator_collection")​ +​ +iterator = collection.search_iterator(​ + data=query_vectors,​ + anns_field="vector",​ + param={"metric_type": "L2", "params": {"nprobe": 16}},​ + # highlight-next-line​ + batch_size=50,​ + output_fields=["color"],​ + # highlight-next-line​ + limit=20000​ +)​ -while True: - result = iterator.next() - if not result: - iterator.close() - break - - results.extend(result) - - for hit in result: - results.append(hit.to_dict()) - -print(results) - -# Output -# -# [ -# { -# "id": 1756, -# "distance": 2.0642056465148926, -# "entity": { -# "color_tag": "black_9109" -# } -# }, -# { -# "id": 6488, -# "distance": 1.9437453746795654, -# "entity": { -# "color_tag": "purple_8164" -# } -# }, -# { -# "id": 3338, -# "distance": 1.9107104539871216, -# "entity": { -# "color_tag": "brown_8121" -# } -# } -# ] ``` ```java -import io.milvus.param.dml.QueryIteratorParam; -import io.milvus.param.dml.SearchIteratorParam; -import io.milvus.response.QueryResultsWrapper; -import io.milvus.orm.iterator.SearchIterator; - -// 4. Search with iterators -SearchIteratorParam iteratorParam = SearchIteratorParam.newBuilder() - .withCollectionName("quick_setup") - .withVectorFieldName("vector") - // Use withFloatVectors() in clusters compatible with Milvus 2.4.x - .withVectors(Arrays.asList(0.3580376395471989f, -0.6023495712049978f, 0.18414012509913835f, -0.26286205330961354f, 0.9029438446296592f)) - .withBatchSize(10L) - .withParams("{\"metric_type\": \"COSINE\", \"params\": {\"level\": 1}}") - .build(); - - -R searchIteratorRes = client.searchIterator(iteratorParam); - -if (searchIteratorRes.getStatus() != R.Status.Success.getCode()) { - System.err.println(searchIteratorRes.getMessage()); -} - -SearchIterator searchIterator = searchIteratorRes.getData(); -List results = new ArrayList<>(); - -while (true) { - List batchResults = searchIterator.next(); - if (batchResults.isEmpty()) { - searchIterator.close(); - break; - } - for (QueryResultsWrapper.RowRecord rowRecord : batchResults) { - results.add(rowRecord); - } -} +import io.milvus.v2.client.ConnectConfig;​ +import io.milvus.v2.client.MilvusClientV2;​ +import io.milvus.orm.iterator.SearchIterator;​ +import io.milvus.v2.common.IndexParam.MetricType;​ +import io.milvus.v2.service.vector.request.data.FloatVec;​ +​ +import java.util.*;​ +​ +MilvusClientV2 client = new MilvusClientV2(ConnectConfig.builder()​ + .uri("http://localhost:19530")​ + .token("root:Milvus")​ + .build());​ +​ +FloatVec queryVector = new FloatVec(new float[]{0.3580376395471989f, -0.6023495712049978f, 0.18414012509913835f, -0.26286205330961354f, 0.9029438446296592f});​ +SearchIterator searchIterator = client.searchIterator(SearchIteratorReq.builder()​ + .collectionName("iterator_collection")​ + .vectors(Collections.singletonList(queryVector))​ + .vectorFieldName("vector")​ + .batchSize(500L)​ + .outputFields(Lists.newArrayList("color"))​ + .topK(20000)​ + .metricType(IndexParam.MetricType.COSINE)​ + .build());​ -System.out.println(results.size()); ``` - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
          ParameterDescription
          dataA list of vector embeddings.
          Milvus searches for the most similar vector embeddings to the specified ones.
          anns_fieldThe name of the vector field in the current collection.
          batch_sizeThe number of entities to return each time you call next() on the current iterator.
          The value defaults to 1000. Set it to a proper value to control the number of entities to return per iteration.
          paramThe parameter settings specific to this operation.
          • metric_type: The metric type applied to this operation. This should be the same as the one used when you index the vector field specified above. Possible values are L2, IP, COSINE, JACCARD, HAMMING.
          • params: Additional parameters. For details, refer to search_iterator().
          output_fieldsA list of field names to include in each entity in return.
          The value defaults to None. If left unspecified, only the primary field is included.
          limitThe total number of entities to return.
          The value defaults to -1, indicating all matching entities will be in return.
          - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
          ParameterDescription
          withCollectionNameSet the collection name. Collection name cannot be empty or null.
          withVectorFieldNameSet target vector field by name. Field name cannot be empty or null.
          withVectorsSet the target vectors. Up to 16384 vectors allowed.
          withBatchSizeThe number of entities to return each time you call next() on the current iterator.
          The value defaults to 1000. Set it to a proper value to control the number of entities to return per iteration.
          withParamsSpecifies the parameters of search in JSON format. For more information, refer to searchIterator().
          - -## Query with an iterator - -
          - -To query with an iterator, call the [query_iterator()](https://milvus.io/api-reference/pymilvus/v2.4.x/ORM/Collection/query_iterator.md) method: - -
          - -
          +In the above examples, you have set the number of entities to return per search (**batch_size**/**batchSize**) to 50, and the total number of entities to return (**topK**) to 20,000.​ -To search with an iterator, call the [queryIterator()](https://milvus.io/api-reference/java/v2.4.x/v1/QuerySearch/queryIterator.md) method: +## Use SearchIterator​ -
          +Once the SearchIterator is ready, you can call its next() method to get the search results in a paginated manner.​
          - Python - Java + Python + Java
          ```python -# 6. Query with iterator -iterator = collection.query_iterator( - batch_size=10, # Controls the size of the return each time you call next() - expr="color_tag like \"brown_8\"", - output_fields=["color_tag"] -) - -results = [] - -while True: - result = iterator.next() - if not result: - iterator.close() - break - - results.extend(result) - -# 8. Check the search results -print(len(results)) - -print(results[:3]) +results = []​ +​ +while True:​ + # highlight-next-line​ + result = iterator.next()​ + if not result:​ + # highlight-next-line​ + iterator.close()​ + break​ + ​ + for hit in result:​ + results.append(hit.to_dict())​ -# Output -# -# [ -# { -# "color_tag": "brown_8785", -# "id": 94 -# }, -# { -# "color_tag": "brown_8568", -# "id": 176 -# }, -# { -# "color_tag": "brown_8721", -# "id": 289 -# } -# ] ``` ```java -import io.milvus.param.dml.QueryIteratorParam; -import io.milvus.orm.iterator.QueryIterator; - -// 5. Query with iterators - -try { - Files.write(Path.of("results.json"), JSON.toJSONString(new ArrayList<>()).getBytes(), StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); -} catch (Exception e) { - // TODO: handle exception - e.printStackTrace(); -} - -QueryIteratorParam queryIteratorParam = QueryIteratorParam.newBuilder() - .withCollectionName("quick_setup") - .withExpr("color_tag like \"brown_8%\"") - .withBatchSize(50L) - .addOutField("vector") - .addOutField("color_tag") - .build(); +import io.milvus.response.QueryResultsWrapper;​ +​ +while (true) {​ + List res = searchIterator.next();​ + if (res.isEmpty()) {​ + searchIterator.close();​ + break;​ + }​ +​ + for (QueryResultsWrapper.RowRecord record : res) {​ + System.out.println(record);​ + }​ +}​ -R queryIteratRes = client.queryIterator(queryIteratorParam); - -if (queryIteratRes.getStatus() != R.Status.Success.getCode()) { - System.err.println(queryIteratRes.getMessage()); -} - -QueryIterator queryIterator = queryIteratRes.getData(); - -while (true) { - List batchResults = queryIterator.next(); - if (batchResults.isEmpty()) { - queryIterator.close(); - break; - } - - String jsonString = ""; - List jsonObject = new ArrayList<>(); - try { - jsonString = Files.readString(Path.of("results.json")); - jsonObject = JSON.parseArray(jsonString).toJavaList(null); - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - - for (QueryResultsWrapper.RowRecord queryResult : batchResults) { - JSONObject row = new JSONObject(); - row.put("id", queryResult.get("id")); - row.put("vector", queryResult.get("vector")); - row.put("color_tag", queryResult.get("color_tag")); - jsonObject.add(row); - } - - try { - Files.write(Path.of("results.json"), JSON.toJSONString(jsonObject).getBytes(), StandardOpenOption.WRITE); - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } -} ``` - - - - - - - - - - - - - - - - - - - - - - - - - -
          ParameterDescription
          batch_sizeThe number of entities to return each time you call next() on the current iterator.
          The value defaults to 1000. Set it to a proper value to control the number of entities to return per iteration.
          exprA scalar filtering condition to filter matching entities.
          The value defaults to None, indicating that scalar filtering is ignored. To build a scalar filtering condition, refer to Boolean Expression Rules.
          output_fieldsA list of field names to include in each entity in return.
          The value defaults to None. If left unspecified, only the primary field is included.
          limitThe total number of entities to return.
          The value defaults to -1, indicating all matching entities will be in return.
          - - - - - - - - - - - - - - - - - - - - - - - - - - -
          ParameterDescription
          withCollectionNameSet the collection name. Collection name cannot be empty or null.
          withExprSet the expression to query entities. To build a scalar filtering condition, refer to Boolean Expression Rules.
          withBatchSizeThe number of entities to return each time you call next() on the current iterator.
          The value defaults to 1000. Set it to a proper value to control the number of entities to return per iteration.
          addOutFieldSpecifies an output scalar field (Optional).
          +In the above code examples, you have created an infinite loop and called the **next()** method in the loop to store the search results in a variable and closed the iterator when the **next()** returns nothing.​ \ No newline at end of file diff --git a/v2.4.x/site/en/userGuide/use-json-fields.md b/v2.4.x/site/en/userGuide/use-json-fields.md deleted file mode 100644 index b05fb88528..0000000000 --- a/v2.4.x/site/en/userGuide/use-json-fields.md +++ /dev/null @@ -1,1796 +0,0 @@ ---- -id: use-json-fields.md -title: Use JSON Fields ---- - -# Use JSON Fields - -This guide explains how to use the JSON fields, such as inserting JSON values as well as searching and querying in JSON fields with basic and advanced operators. - -## Overview - -JSON stands for Javascript Object Notation, which is a lightweight and simple text-based data format. Data in JSON is structured in key-value pairs, where every key is a string that maps to a value of a number, string, boolean, list, or array. With Milvus clusters, it's possible to store dictionaries as a field value in collections. - -For example, the following code randomly generates key-value pairs, each containing a JSON field with the key __color__. - -
          - Python - Java - Node.js -
          - -```python -# 3. Insert randomly generated vectors -colors = ["green", "blue", "yellow", "red", "black", "white", "purple", "pink", "orange", "brown", "grey"] -data = [] - -for i in range(1000): - current_color = random.choice(colors) - current_tag = random.randint(1000, 9999) - current_coord = [ random.randint(0, 40) for _ in range(3) ] - current_ref = [ [ random.choice(colors) for _ in range(3) ] for _ in range(3) ] - data.append({ - "id": i, - "vector": [ random.uniform(-1, 1) for _ in range(5) ], - "color": { - "label": current_color, - "tag": current_tag, - "coord": current_coord, - "ref": current_ref - } - }) - -print(data[0]) -``` - -```java -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Random; - -import com.alibaba.fastjson.JSONObject; - -// 3. Insert randomly generated vectors and JSON data into the collection -List colors = Arrays.asList("green", "blue", "yellow", "red", "black", "white", "purple", "pink", "orange", "brown", "grey"); -List data = new ArrayList<>(); - -for (int i=0; i<1000; i++) { - Random rand = new Random(); - String current_color = colors.get(rand.nextInt(colors.size()-1)); - Integer current_tag = rand.nextInt(8999) + 1000; - List current_coord = Arrays.asList(rand.nextInt(40), rand.nextInt(40), rand.nextInt(40)); - List> current_ref = Arrays.asList( - Arrays.asList(colors.get(rand.nextInt(colors.size()-1)), colors.get(rand.nextInt(colors.size()-1)), colors.get(rand.nextInt(colors.size()-1))), - Arrays.asList(colors.get(rand.nextInt(colors.size()-1)), colors.get(rand.nextInt(colors.size()-1)), colors.get(rand.nextInt(colors.size()-1))), - Arrays.asList(colors.get(rand.nextInt(colors.size()-1)), colors.get(rand.nextInt(colors.size()-1)), colors.get(rand.nextInt(colors.size()-1))) - ); - JSONObject row = new JSONObject(); - row.put("id", Long.valueOf(i)); - row.put("vector", Arrays.asList(rand.nextFloat(), rand.nextFloat(), rand.nextFloat(), rand.nextFloat(), rand.nextFloat())); - JSONObject color = new JSONObject(); - color.put("label", current_color); - color.put("tag", current_tag); - color.put("coord", current_coord); - color.put("ref", current_ref); - row.put("color", color); - data.add(row); -} - -System.out.println(JSONObject.toJSON(data.get(0))); -``` - -```javascript -// 3. Insert randomly generated vectors -const colors = ["green", "blue", "yellow", "red", "black", "white", "purple", "pink", "orange", "brown", "grey"] -var data = [] - -for (let i = 0; i < 1000; i++) { - const current_color = colors[Math.floor(Math.random() * colors.length)] - const current_tag = Math.floor(Math.random() * 8999 + 1000) - const current_coord = Array(3).fill(0).map(() => Math.floor(Math.random() * 40)) - const current_ref = [ Array(3).fill(0).map(() => colors[Math.floor(Math.random() * colors.length)]) ] - - data.push({ - id: i, - vector: [Math.random(), Math.random(), Math.random(), Math.random(), Math.random()], - color: { - label: current_color, - tag: current_tag, - coord: current_coord, - ref: current_ref - } - }) -} - -console.log(data[0]) -``` - -You can view the structure of the generated data by checking its first entry. - -``` -{ - "id": 0, - "vector": [ - -0.8017921296923975, - 0.550046715206634, - 0.764922589768134, - 0.6371433836123146, - 0.2705233937454232 - ], - "color": { - "label": "blue", - "tag": 9927, - "coord": [ - 22, - 36, - 6 - ], - "ref": [ - [ - "blue", - "green", - "white" - ], - [ - "black", - "green", - "pink" - ], - [ - "grey", - "black", - "brown" - ] - ] - } -} -``` - -
          - -

          notes

          - -
            -

          • Ensure that all values in a list or array are of the same data type.

            • -

          • Any nested dictionaries in a JSON field value will be considered strings.

            • -

          • Use only alphanumeric characters and underscores to name JSON keys, as other characters may cause problems with filtering or searching.

            • -
          • Currently, indexing JSON fields is not available, which can make filtering time-consuming. However, this limitation will be addressed in upcoming releases.
            • -
          - -
          - -## Define JSON field - -To define a JSON field, simply follow the same procedure as defining fields of other types. - -
          - -For more information on parameters, refer to [`MilvusClient`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Client/MilvusClient.md), [`create_schema()`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Collections/create_schema.md), [`add_field()`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/CollectionSchema/add_field.md), [`add_index()`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Management/add_index.md), [`create_collection()`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Collections/create_collection.md), and [`get_load_state()`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Management/get_load_state.md) in the SDK reference. - -
          - -
          - -For more information on parameters, refer to [`MilvusClientV2`](https://milvus.io/api-reference/java/v2.4.x/v2/Client/MilvusClientV2.md), [`createSchema()`](https://milvus.io/api-reference/java/v2.4.x/v2/Collections/createSchema.md), [`addField()`](https://milvus.io/api-reference/java/v2.4.x/v2/CollectionSchema/addField.md), [`IndexParam`](https://milvus.io/api-reference/java/v2.4.x/v2/Management/IndexParam.md), [`createCollection()`](https://milvus.io/api-reference/java/v2.4.x/v2/Collections/createCollection.md), and [`getLoadState()`](https://milvus.io/api-reference/java/v2.4.x/v2/Management/getLoadState.md) in the SDK reference. - -
          - -
          - -For more information on parameters, refer to [`MilvusClient`](https://milvus.io/api-reference/node/v2.4.x/Client/MilvusClient.md) and [`createCollection()`](https://milvus.io/api-reference/node/v2.4.x/Collections/createCollection.md) and [`createCollection()`](https://milvus.io/api-reference/node/v2.4.x/Collections/createCollection.md) in the SDK reference. - -
          - -
          - Python - Java - Node.js -
          - -```python -import random, time -from pymilvus import connections, MilvusClient, DataType - -CLUSTER_ENDPOINT = "http://localhost:19530" - -# 1. Set up a Milvus client -client = MilvusClient( - uri=CLUSTER_ENDPOINT -) - -# 2. Create a collection -schema = MilvusClient.create_schema( - auto_id=False, - enable_dynamic_field=False, -) - -schema.add_field(field_name="id", datatype=DataType.INT64, is_primary=True) -schema.add_field(field_name="vector", datatype=DataType.FLOAT_VECTOR, dim=5) -schema.add_field(field_name="color", datatype=DataType.JSON) - -index_params = MilvusClient.prepare_index_params() - -index_params.add_index( - field_name="id", - index_type="STL_SORT" -) - -index_params.add_index( - field_name="vector", - index_type="IVF_FLAT", - metric_type="L2", - params={"nlist": 1024} -) - -client.create_collection( - collection_name="test_collection", - schema=schema, - index_params=index_params -) - -res = client.get_load_state( - collection_name="test_collection" -) - -print(res) - -# Output -# -# { -# "state": "" -# } -``` - -```java -String CLUSTER_ENDPOINT = "http://localhost:19530"; - -// 1. Connect to Milvus server -ConnectConfig connectConfig = ConnectConfig.builder() - .uri(CLUSTER_ENDPOINT) - .build(); - -MilvusClientV2 client = new MilvusClientV2(connectConfig); - -// 2. Create a collection in customized setup mode - -// 2.1 Create schema -CreateCollectionReq.CollectionSchema schema = client.createSchema(); - -// 2.2 Add fields to schema -schema.addField(AddFieldReq.builder() - .fieldName("id") - .dataType(DataType.Int64) - .isPrimaryKey(true) - .autoID(false) - .build()); - -schema.addField(AddFieldReq.builder() - .fieldName("vector") - .dataType(DataType.FloatVector) - .dimension(5) - .build()); - -schema.addField(AddFieldReq.builder() - .fieldName("color") - .dataType(DataType.JSON) - .build()); - -// 2.3 Prepare index parameters -IndexParam indexParamForIdField = IndexParam.builder() - .fieldName("id") - .indexType(IndexParam.IndexType.STL_SORT) - .build(); - -IndexParam indexParamForVectorField = IndexParam.builder() - .fieldName("vector") - .indexType(IndexParam.IndexType.IVF_FLAT) - .metricType(IndexParam.MetricType.IP) - .extraParams(Map.of("nlist", 1024)) - .build(); - -List indexParams = new ArrayList<>(); -indexParams.add(indexParamForIdField); -indexParams.add(indexParamForVectorField); - -// 2.4 Create a collection with schema and index parameters -CreateCollectionReq customizedSetupReq = CreateCollectionReq.builder() - .collectionName("test_collection") - .collectionSchema(schema) - .indexParams(indexParams) - .build(); - -client.createCollection(customizedSetupReq); - -// 2.5 Check if the collection is loaded -GetLoadStateReq getLoadStateReq = GetLoadStateReq.builder() - .collectionName("test_collection") - .build(); - -Boolean isLoaded = client.getLoadState(getLoadStateReq); - -System.out.println(isLoaded); - -// Output: -// true -``` - -```javascript -const { MilvusClient, DataType, sleep } = require("@zilliz/milvus2-sdk-node") - -const address = "http://localhost:19530" - -async function main() { -// 1. Set up a Milvus Client -client = new MilvusClient({address}); - -// 2. Create a collection -// 2.1 Define fields -const fields = [ - { - name: "id", - data_type: DataType.Int64, - is_primary_key: true, - auto_id: false - }, - { - name: "vector", - data_type: DataType.FloatVector, - dim: 5 - }, - { - name: "color", - data_type: DataType.JSON, - } -] - -// 2.2 Prepare index parameters -const index_params = [{ - field_name: "vector", - index_type: "IVF_FLAT", - metric_type: "IP", - params: { nlist: 1024} -}] - -// 2.3 Create a collection with fields and index parameters -res = await client.createCollection({ - collection_name: "test_collection", - fields: fields, - index_params: index_params -}) - -console.log(res.error_code) - -// Output -// -// Success -// - -res = await client.getLoadState({ - collection_name: "test_collection", -}) - -console.log(res.state) - -// Output -// -// LoadStateLoaded -// -``` - -
          - -For more information on parameters, refer to [`MilvusClient`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Client/MilvusClient.md), [`create_schema()`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Collections/create_schema.md), [`add_field()`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/CollectionSchema/add_field.md), [`add_index()`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Management/add_index.md), [`create_collection()`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Collections/create_collection.md), and [`get_load_state()`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Management/get_load_state.md) in the SDK reference. - -
          - -
          - -For more information on parameters, refer to [`MilvusClientV2`](https://milvus.io/api-reference/java/v2.4.x/v2/Client/MilvusClientV2.md), [`createSchema()`](https://milvus.io/api-reference/java/v2.4.x/v2/Collections/createSchema.md), [`addField()`](https://milvus.io/api-reference/java/v2.4.x/v2/CollectionSchema/addField.md), [`IndexParam`](https://milvus.io/api-reference/java/v2.4.x/v2/Management/IndexParam.md), [`createCollection()`](https://milvus.io/api-reference/java/v2.4.x/v2/Collections/createCollection.md), and [`getLoadState()`](https://milvus.io/api-reference/java/v2.4.x/v2/Management/getLoadState.md) in the SDK reference. - -
          - -
          - -For more information on parameters, refer to [`MilvusClient`](https://milvus.io/api-reference/node/v2.4.x/Client/MilvusClient.md), [`createCollection()`](https://milvus.io/api-reference/node/v2.4.x/Collections/createCollection.md), and [`getLoadState()`](https://milvus.io/api-reference/node/v2.4.x/Management/getLoadState.md) in the SDK reference. - -
          - -## Insert field values - -After creating a collection from the `CollectionSchema` object, dictionaries such as the one above can be inserted into it. - -
          - -Use the [`insert()`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Vector/insert.md) method to insert the data into the collection. - -
          - -
          - -Use the [`insert()`](https://milvus.io/api-reference/java/v2.4.x/v2/Vector/insert.md) method to insert the data into the collection. - -
          - -
          - -Use the [`insert()`](https://milvus.io/api-reference/node/v2.4.x/Vector/insert.md) method to insert the data into the collection. - -
          - -
          - Python - Java - Node.js -
          - -```python -res = client.insert( - collection_name="test_collection", - data=data -) - -print(res) - -# Output -# -# { -# "insert_count": 1000, -# "ids": [ -# 0, -# 1, -# 2, -# 3, -# 4, -# 5, -# 6, -# 7, -# 8, -# 9, -# "(990 more items hidden)" -# ] -# } -``` - -```java -// 3.1 Insert data into the collection -InsertReq insertReq = InsertReq.builder() - .collectionName("test_collection") - .data(data) - .build(); - -InsertResp insertResp = client.insert(insertReq); - -System.out.println(JSONObject.toJSON(insertResp)); - -// Output: -// {"insertCnt": 1000} -``` - -```javascript -// 3. Insert randomly generated vectors -const colors = ["green", "blue", "yellow", "red", "black", "white", "purple", "pink", "orange", "brown", "grey"] -var data = [] - -for (let i = 0; i < 1000; i++) { - const current_color = colors[Math.floor(Math.random() * colors.length)] - const current_tag = Math.floor(Math.random() * 8999 + 1000) - const current_coord = Array(3).fill(0).map(() => Math.floor(Math.random() * 40)) - const current_ref = [ Array(3).fill(0).map(() => colors[Math.floor(Math.random() * colors.length)]) ] - - data.push({ - id: i, - vector: [Math.random(), Math.random(), Math.random(), Math.random(), Math.random()], - color: { - label: current_color, - tag: current_tag, - coord: current_coord, - ref: current_ref - } - }) -} - -console.log(data[0]) - -// Output -// -// { -// id: 0, -// vector: [ -// 0.11455530974226114, -// 0.21704086958595314, -// 0.9430119822312437, -// 0.7802712923612023, -// 0.9106927960926137 -// ], -// color: { label: 'grey', tag: 7393, coord: [ 22, 1, 22 ], ref: [ [Array] ] } -// } -// - -res = await client.insert({ - collection_name: "test_collection", - data: data, -}) - -console.log(res.insert_cnt) - -// Output -// -// 1000 -// -``` - -## Basic scalar filtering - -Once all of your data has been added, you can conduct searches and queries using the keys in the JSON field in the same manner as you would with a standard scalar field. - -
          - -For more information on parameters, refer to [`search()`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Vector/search.md) in the SDK reference. - -
          - -
          - -For more information on parameters, refer to [`search()`](https://milvus.io/api-reference/java/v2.4.x/v2/Vector/search.md) in the SDK reference. - -
          - -
          - -For more information on parameters, refer to [`search()`](https://milvus.io/api-reference/node/v2.4.x/Vector/search.md) in the SDK reference. - -
          - -
          - Python - Java - Node.js -
          - -```python -# 4. Basic search with a JSON field -query_vectors = [ [ random.uniform(-1, 1) for _ in range(5) ]] - -res = client.search( - collection_name="test_collection", - data=query_vectors, - filter='color["label"] in ["red"]', - search_params={ - "metric_type": "L2", - "params": {"nprobe": 16} - }, - output_fields=["id", "color"], - limit=3 -) - -print(res) - -# Output -# -# [ -# [ -# { -# "id": 460, -# "distance": 0.4016231596469879, -# "entity": { -# "id": 460, -# "color": { -# "label": "red", -# "tag": 5030, -# "coord": [14, 32, 40], -# "ref": [ -# [ "pink", "green", "brown" ], -# [ "red", "grey", "black"], -# [ "red", "yellow", "orange"] -# ] -# } -# } -# }, -# { -# "id": 785, -# "distance": 0.451080858707428, -# "entity": { -# "id": 785, -# "color": { -# "label": "red", -# "tag": 5290, -# "coord": [31, 13, 23], -# "ref": [ -# ["yellow", "pink", "pink"], -# ["purple", "grey", "orange"], -# ["grey", "purple", "pink"] -# ] -# } -# } -# }, -# { -# "id": 355, -# "distance": 0.5839247703552246, -# "entity": { -# "id": 355, -# "color": { -# "label": "red", -# "tag": 8725, -# "coord": [5, 10, 22], -# "ref": [ -# ["white", "purple", "yellow"], -# ["white", "purple", "white"], -# ["orange", "white", "pink"] -# ] -# } -# } -# } -# ] -# ] -``` - -```java -// 4. Basic search with a JSON field -List> query_vectors = Arrays.asList(Arrays.asList(0.3580376395471989f, -0.6023495712049978f, 0.18414012509913835f, -0.26286205330961354f, 0.9029438446296592f)); - -SearchReq searchReq = SearchReq.builder() - .collectionName("test_collection") - .data(query_vectors) - .filter("color[\"label\"] in [\"red\"]") - .outputFields(Arrays.asList("id", "color")) - .topK(3) - .build(); - -SearchResp searchResp = client.search(searchReq); - -System.out.println(JSONObject.toJSON(searchResp)); - -// Output: -// {"searchResults": [[ -// { -// "distance": 1.2636482, -// "id": 290, -// "entity": { -// "color": { -// "coord": [32,37,32], -// "ref": [ -// ["green", "blue", "yellow"], -// ["yellow", "pink", "pink"], -// ["purple", "red", "brown"] -// ], -// "label": "red", -// "tag": 8949 -// }, -// "id": 290 -// } -// }, -// { -// "distance": 1.002122, -// "id": 629, -// "entity": { -// "color": { -// "coord": [23,5,35], -// "ref": [ -// ["black", ""yellow", "black"], -// ["black", "purple", "white"], -// ["black", "brown", "orange"] -// ], -// "label": "red", -// "tag": 5072 -// }, -// "id": 629 -// } -// }, -// { -// "distance": 0.9542817, -// "id": 279, -// "entity": { -// "color": { -// "coord": [20,33,33], -// "ref": [ -// ["yellow", "white", "brown"], -// ["black", "white", "purple"], -// ["green", "brown", "blue"] -// ], -// "label": "red", -// "tag": 4704 -// }, -// "id": 279 -// } -// } -// ]]} -``` - -```javascript -// 4. Basic search with a JSON field -query_vectors = [[0.6765405125697714, 0.759217474274025, 0.4122471841491111, 0.3346805565394215, 0.09679748345514638]] - -res = await client.search({ - collection_name: "test_collection", - data: query_vectors, - filter: 'color["label"] in ["red"]', - output_fields: ["color", "id"], - limit: 3 -}) - -console.log(JSON.stringify(res.results, null, 4)) - -// Output -// -// [ -// { -// "score": 1.777988076210022, -// "id": "595", -// "color": { -// "label": "red", -// "tag": 7393, -// "coord": [31,34,18], -// "ref": [ -// ["grey", "white", "orange"] -// ] -// } -// }, -// { -// "score": 1.7542595863342285, -// "id": "82", -// "color": { -// "label": "red", -// "tag": 8636, -// "coord": [4,37,29], -// "ref": [ -// ["brown", "brown", "pink"] -// ] -// } -// }, -// { -// "score": 1.7537562847137451, -// "id": "748", -// "color": { -// "label": "red", -// "tag": 1626, -// "coord": [31,4,25 -// ], -// "ref": [ -// ["grey", "green", "blue"] -// ] -// } -// } -// ] -// -``` - -## Advanced scalar filtering - -Milvus provides a set of advanced filters for scalar filtering in JSON fields. These filters are `JSON_CONTAINS`, `JSON_CONTAINS_ALL`, and `JSON_CONTAINS_ANY`. - -- Filters all entities that have `["blue", "brown", "grey"]` as the reference color set. - -
          - Python - Java - Node.js -
          - - ```python - # 5. Advanced search within a JSON field - - res = client.query( - collection_name="test_collection", - data=query_vectors, - filter='JSON_CONTAINS(color["ref"], ["blue", "brown", "grey"])', - output_fields=["id", "color"], - limit=3 - ) - - print(res) - - # Output - # - # [ - # { - # "id": 79, - # "color": { - # "label": "orange", - # "tag": 8857, - # "coord": [ - # 10, - # 14, - # 5 - # ], - # "ref": [ - # [ - # "yellow", - # "white", - # "green" - # ], - # [ - # "blue", - # "purple", - # "purple" - # ], - # [ - # "blue", - # "brown", - # "grey" - # ] - # ] - # } - # }, - # { - # "id": 371, - # "color": { - # "label": "black", - # "tag": 1324, - # "coord": [ - # 2, - # 18, - # 32 - # ], - # "ref": [ - # [ - # "purple", - # "orange", - # "brown" - # ], - # [ - # "blue", - # "brown", - # "grey" - # ], - # [ - # "purple", - # "blue", - # "blue" - # ] - # ] - # } - # }, - # { - # "id": 590, - # "color": { - # "label": "red", - # "tag": 3340, - # "coord": [ - # 13, - # 21, - # 13 - # ], - # "ref": [ - # [ - # "yellow", - # "yellow", - # "red" - # ], - # [ - # "blue", - # "brown", - # "grey" - # ], - # [ - # "pink", - # "yellow", - # "purple" - # ] - # ] - # } - # } - # ] - ``` - - ```java - // 5. Advanced search within a JSON field - searchReq = SearchReq.builder() - .collectionName("test_collection") - .data(query_vectors) - .filter("JSON_CONTAINS(color[\"ref\"], [\"purple\", \"pink\", \"orange\"])") - .outputFields(Arrays.asList("id", "color")) - .topK(3) - .build(); - - searchResp = client.search(searchReq); - - System.out.println(JSONObject.toJSON(searchResp)); - - // Output: - // {"searchResults": [[ - // { - // "distance": 1.1811467, - // "id": 180, - // "entity": { - // "color": { - // "coord": [ - // 17, - // 26, - // 14 - // ], - // "ref": [ - // [ - // "white", - // "black", - // "brown" - // ], - // [ - // "purple", - // "pink", - // "orange" - // ], - // [ - // "black", - // "pink", - // "red" - // ] - // ], - // "label": "green", - // "tag": 2470 - // }, - // "id": 180 - // } - // }, - // { - // "distance": 0.6487204, - // "id": 331, - // "entity": { - // "color": { - // "coord": [ - // 16, - // 32, - // 23 - // ], - // "ref": [ - // [ - // "purple", - // "pink", - // "orange" - // ], - // [ - // "brown", - // "red", - // "orange" - // ], - // [ - // "red", - // "yellow", - // "brown" - // ] - // ], - // "label": "white", - // "tag": 1236 - // }, - // "id": 331 - // } - // }, - // { - // "distance": 0.59387654, - // "id": 483, - // "entity": { - // "color": { - // "coord": [ - // 8, - // 33, - // 2 - // ], - // "ref": [ - // [ - // "red", - // "orange", - // "brown" - // ], - // [ - // "purple", - // "pink", - // "orange" - // ], - // [ - // "brown", - // "blue", - // "green" - // ] - // ], - // "label": "pink", - // "tag": 5686 - // }, - // "id": 483 - // } - // } - // ]]} - ``` - - ```javascript - // 5. Advanced search within a JSON field - res = await client.search({ - collection_name: "test_collection", - data: query_vectors, - filter: 'JSON_CONTAINS(color["ref"], ["blue", "brown", "grey"])', - output_fields: ["color", "id"], - limit: 3 - }) - - console.log(JSON.stringify(res.results, null, 4)) - - // Output - // - // [ - // { - // "id": 79, - // "color": { - // "label": "orange", - // "tag": 8857, - // "coord": [ - // 10, - // 14, - // 5 - // ], - // "ref": [ - // [ - // "yellow", - // "white", - // "green" - // ], - // [ - // "blue", - // "purple", - // "purple" - // ], - // [ - // "blue", - // "brown", - // "grey" - // ] - // ] - // } - // }, - // { - // "id": 371, - // "color": { - // "label": "black", - // "tag": 1324, - // "coord": [ - // 2, - // 18, - // 32 - // ], - // "ref": [ - // [ - // "purple", - // "orange", - // "brown" - // ], - // [ - // "blue", - // "brown", - // "grey" - // ], - // [ - // "purple", - // "blue", - // "blue" - // ] - // ] - // } - // }, - // { - // "id": 590, - // "color": { - // "label": "red", - // "tag": 3340, - // "coord": [ - // 13, - // 21, - // 13 - // ], - // "ref": [ - // [ - // "yellow", - // "yellow", - // "red" - // ], - // [ - // "blue", - // "brown", - // "grey" - // ], - // [ - // "pink", - // "yellow", - // "purple" - // ] - // ] - // } - // } - // ] - // - ``` - -- Filters entities that have the coordinator of `[4, 5]`. - -
          - Python - Java - Node.js -
          - - ```python - res = client.query( - collection_name="test_collection", - data=query_vectors, - filter='JSON_CONTAINS_ALL(color["coord"], [4, 5])', - output_fields=["id", "color"], - limit=3 - ) - - print(res) - - # Output - # - # [ - # { - # "id": 281, - # "color": { - # "label": "red", - # "tag": 3645, - # "coord": [ - # 5, - # 33, - # 4 - # ], - # "ref": [ - # [ - # "orange", - # "blue", - # "pink" - # ], - # [ - # "purple", - # "blue", - # "purple" - # ], - # [ - # "black", - # "brown", - # "yellow" - # ] - # ] - # } - # }, - # { - # "id": 464, - # "color": { - # "label": "brown", - # "tag": 6261, - # "coord": [ - # 5, - # 9, - # 4 - # ], - # "ref": [ - # [ - # "purple", - # "purple", - # "brown" - # ], - # [ - # "black", - # "pink", - # "white" - # ], - # [ - # "brown", - # "grey", - # "brown" - # ] - # ] - # } - # }, - # { - # "id": 567, - # "color": { - # "label": "green", - # "tag": 4589, - # "coord": [ - # 5, - # 39, - # 4 - # ], - # "ref": [ - # [ - # "purple", - # "yellow", - # "white" - # ], - # [ - # "yellow", - # "yellow", - # "brown" - # ], - # [ - # "blue", - # "red", - # "yellow" - # ] - # ] - # } - # } - # ] - ``` - - ```java - searchReq = SearchReq.builder() - .collectionName("test_collection") - .data(query_vectors) - .filter("JSON_CONTAINS_ALL(color[\"coord\"], [4, 5])") - .outputFields(Arrays.asList("id", "color")) - .topK(3) - .build(); - - searchResp = client.search(searchReq); - - System.out.println(JSONObject.toJSON(searchResp)); - - // Output: - // {"searchResults": [[ - // { - // "distance": 0.77485126, - // "id": 304, - // "entity": { - // "color": { - // "coord": [ - // 4, - // 5, - // 13 - // ], - // "ref": [ - // [ - // "purple", - // "pink", - // "brown" - // ], - // [ - // "orange", - // "red", - // "blue" - // ], - // [ - // "yellow", - // "blue", - // "purple" - // ] - // ], - // "label": "blue", - // "tag": 7228 - // }, - // "id": 304 - // } - // }, - // { - // "distance": 0.68138736, - // "id": 253, - // "entity": { - // "color": { - // "coord": [ - // 5, - // 38, - // 4 - // ], - // "ref": [ - // [ - // "black", - // "pink", - // "blue" - // ], - // [ - // "pink", - // "brown", - // "pink" - // ], - // [ - // "red", - // "pink", - // "orange" - // ] - // ], - // "label": "blue", - // "tag": 6935 - // }, - // "id": 253 - // } - // }, - // { - // "distance": 0.56997097, - // "id": 944, - // "entity": { - // "color": { - // "coord": [ - // 5, - // 6, - // 4 - // ], - // "ref": [ - // [ - // "blue", - // "yellow", - // "orange" - // ], - // [ - // "orange", - // "white", - // "orange" - // ], - // [ - // "pink", - // "brown", - // "white" - // ] - // ], - // "label": "pink", - // "tag": 3325 - // }, - // "id": 944 - // } - // } - // ]]} - ``` - - ```javascript - res = await client.search({ - collection_name: "test_collection", - data: query_vectors, - filter: 'JSON_CONTAINS_ALL(color["coord"], [4, 5])', - output_fields: ["color", "id"], - limit: 3 - }) - - console.log(JSON.stringify(res.results, null, 4)) - - // Output - // - // [ - // { - // "score": 1.8944344520568848, - // "id": "792", - // "color": { - // "label": "purple", - // "tag": 8161, - // "coord": [ - // 4, - // 38, - // 5 - // ], - // "ref": [ - // [ - // "red", - // "white", - // "grey" - // ] - // ] - // } - // }, - // { - // "score": 1.2801706790924072, - // "id": "489", - // "color": { - // "label": "red", - // "tag": 4358, - // "coord": [ - // 5, - // 4, - // 1 - // ], - // "ref": [ - // [ - // "blue", - // "orange", - // "orange" - // ] - // ] - // } - // }, - // { - // "score": 1.2097992897033691, - // "id": "656", - // "color": { - // "label": "red", - // "tag": 7856, - // "coord": [ - // 5, - // 20, - // 4 - // ], - // "ref": [ - // [ - // "black", - // "orange", - // "white" - // ] - // ] - // } - // } - // ] - // - ``` - -- Filters entities that have the coordinator containing either `4` or `5`. - -
          - Python - Java - Node.js -
          - - ```python - res = client.query( - collection_name="test_collection", - data=query_vectors, - filter='JSON_CONTAINS_ANY(color["coord"], [4, 5])', - output_fields=["id", "color"], - limit=3 - ) - - print(res) - - # Output - # - # [ - # { - # "id": 0, - # "color": { - # "label": "yellow", - # "tag": 6340, - # "coord": [ - # 40, - # 4, - # 40 - # ], - # "ref": [ - # [ - # "purple", - # "yellow", - # "orange" - # ], - # [ - # "green", - # "grey", - # "purple" - # ], - # [ - # "black", - # "white", - # "yellow" - # ] - # ] - # } - # }, - # { - # "id": 2, - # "color": { - # "label": "brown", - # "tag": 9359, - # "coord": [ - # 38, - # 21, - # 5 - # ], - # "ref": [ - # [ - # "red", - # "brown", - # "white" - # ], - # [ - # "purple", - # "red", - # "brown" - # ], - # [ - # "pink", - # "grey", - # "black" - # ] - # ] - # } - # }, - # { - # "id": 7, - # "color": { - # "label": "green", - # "tag": 3560, - # "coord": [ - # 5, - # 9, - # 5 - # ], - # "ref": [ - # [ - # "blue", - # "orange", - # "green" - # ], - # [ - # "blue", - # "blue", - # "black" - # ], - # [ - # "green", - # "purple", - # "green" - # ] - # ] - # } - # } - # ] - ``` - - ```java - searchReq = SearchReq.builder() - .collectionName("test_collection") - .data(query_vectors) - .filter("JSON_CONTAINS_ANY(color[\"coord\"], [4, 5])") - .outputFields(Arrays.asList("id", "color")) - .topK(3) - .build(); - - searchResp = client.search(searchReq); - - System.out.println(JSONObject.toJSON(searchResp)); - - // Output: - // {"searchResults": [[ - // { - // "distance": 1.002122, - // "id": 629, - // "entity": { - // "color": { - // "coord": [ - // 23, - // 5, - // 35 - // ], - // "ref": [ - // [ - // "black", - // "yellow", - // "black" - // ], - // [ - // "black", - // "purple", - // "white" - // ], - // [ - // "black", - // "brown", - // "orange" - // ] - // ], - // "label": "red", - // "tag": 5072 - // }, - // "id": 629 - // } - // }, - // { - // "distance": 0.85788506, - // "id": 108, - // "entity": { - // "color": { - // "coord": [ - // 25, - // 5, - // 38 - // ], - // "ref": [ - // [ - // "green", - // "brown", - // "pink" - // ], - // [ - // "purple", - // "green", - // "green" - // ], - // [ - // "green", - // "pink", - // "black" - // ] - // ], - // "label": "orange", - // "tag": 8982 - // }, - // "id": 108 - // } - // }, - // { - // "distance": 0.80550396, - // "id": 120, - // "entity": { - // "color": { - // "coord": [ - // 25, - // 16, - // 4 - // ], - // "ref": [ - // [ - // "red", - // "green", - // "orange" - // ], - // [ - // "blue", - // "pink", - // "blue" - // ], - // [ - // "brown", - // "black", - // "green" - // ] - // ], - // "label": "purple", - // "tag": 6711 - // }, - // "id": 120 - // } - // } - // ]]} - ``` - - ```javascript - res = await client.search({ - collection_name: "test_collection", - data: query_vectors, - filter: 'JSON_CONTAINS_ANY(color["coord"], [4, 5])', - output_fields: ["color", "id"], - limit: 3 - }) - - console.log(JSON.stringify(res.results, null, 4)) - - // Output - // - // [ - // { - // "score": 1.9083369970321655, - // "id": "453", - // "color": { - // "label": "brown", - // "tag": 8788, - // "coord": [ - // 21, - // 18, - // 5 - // ], - // "ref": [ - // [ - // "pink", - // "black", - // "brown" - // ] - // ] - // } - // }, - // { - // "score": 1.8944344520568848, - // "id": "792", - // "color": { - // "label": "purple", - // "tag": 8161, - // "coord": [ - // 4, - // 38, - // 5 - // ], - // "ref": [ - // [ - // "red", - // "white", - // "grey" - // ] - // ] - // } - // }, - // { - // "score": 1.8615753650665283, - // "id": "272", - // "color": { - // "label": "grey", - // "tag": 3400, - // "coord": [ - // 5, - // 1, - // 32 - // ], - // "ref": [ - // [ - // "purple", - // "green", - // "white" - // ] - // ] - // } - // } - // ] - // - ``` - -## Reference on JSON filters - -When working with JSON fields, you can either use the JSON fields as filters or some of its specific keys. - -
          - -

          notes

          - -
            -
          • Milvus stores string values in the JSON field as is without performing semantic escape or conversion.
            • -
          -

          For instance, 'a"b', "a'b", 'a\\\\'b', and "a\\\\"b" will be saved as is, while 'a'b' and "a"b" will be treated as invalid values.

          -
            -

          • To build filter expressions using a JSON field, you can utilize the keys within the field.

            • -

          • If a key's value is an integer or a float, you can compare it with another integer or float key or an INT32/64 or FLOAT32/64 field.

            • -

          • If a key's value is a string, you can compare it only with another string key or a VARCHAR field.

            • -
          - -
          - -### Basic Operators in JSON Fields - -The following table assumes that the value of a JSON field named `json_key` has a key named `A`. Use it as a reference when constructing boolean expressions using JSON field keys. - -| __Operator__ | __Examples__ | __Remarks__ | -| -------------- | ----------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| __<__ | `'json_field["A"] < 3'` | This expression evaluates to true if the value of `json_field["A"]` is less than `3`. | -| __>__ | `'json_field["A"] > 1'` | This expression evaluates to true if the value of `json_field["A"]` is greater than `1`. | -| __==__ | `'json_field["A"] == 1'` | This expression evaluates to true if the value of `json_field["A"]` is equal to `1`. | -| __!=__ | `'json_field["A"][0]' != "abc"'` | This expression evaluates to true if
          - `json_field` does not have a key named `A`.
          - `json_field` has a key named `A` but `json_field["A"]` is not an array.
          - `json_field["A"]` is an empty array.
          - `json_field["A"]` is an array but the first element is not `abc`.
          | -| __<=__ | `'json_field["A"] <= 5'` | This expression evaluates to true if the value of `json_field["A"]` is less than or equal to `5`. | -| __>=__ | `'json_field["A"] >= 1'` | This expression evaluates to true if the value of `json_field["A"]` is greater than or equal to `1`. | -| __not__ | `'not json_field["A"] == 1'` | This expression evaluates to true if
          - `json_field` does not have a key named `A`.
          - `json_field["A"]` is not equal to `1`.
          | -| __in__ | `'json_field["A"] in [1, 2, 3]'` | This expression evaluates to true if the value of `json_field["A"]` is `1`, `2`, or `3`. | -| __and (&&)__ | `'json_field["A"] > 1 && json_field["A"] < 3'` | This expression evaluates to true if the value of `json_field["A"]` is greater than 1 and less than `3`. | -| __or (||)__ | 'json_field["A"] > 1 || json_field["A"] < 3' | This expression evaluates to true if the value of `json_field["A"]` is greater than `1` or less than `3`. | -| __exists__ | `'exists json_field["A"]'` | This expression evaluates to true if `json_field` has a key named `A`. | - -### Advanced Operators - -The following operators are specific to JSON fields: - -- `json_contains(identifier, jsonExpr)` - - This operator filters entities whose identifier contains the specified JSON expression. - - - Example 1: `{"x": [1,2,3]}` - - ```python - json_contains(x, 1) # => True (x contains 1.) - json_contains(x, "a") # => False (x does not contain a member "a".) - ``` - - - Example 2: `{"x", [[1,2,3], [4,5,6], [7,8,9]]}` - - ```python - json_contains(x, [1,2,3]) # => True (x contains [1,2,3].) - json_contains(x, [3,2,1]) # => False (x does contain a member [3,2,1].) - ``` - -- `json_contains_all(identifier, jsonExpr)` - - This operator filters entities whose identifier contains all the members of the JSON expression. - - Example: `{"x": [1,2,3,4,5,7,8]}` - - ```python - json_contains_all(x, [1,2,8]) # => True (x contains 1, 2, and 8.) - json_contains_all(x, [4,5,6]) # => False (x does not has a member 6.) - ``` - -- `json_contains_any(identifier, jsonExpr)` - - This operator filters entities whose identifier contains any members of the JSON expression. - - Example: `{"x": [1,2,3,4,5,7,8]}` - - ```python - json_contains_any(x, [1,2,8]) # => True (x contains 1, 2, and 8.) - json_contains_any(x, [4,5,6]) # => True (x contains 4 and 5.) - json_contains_any(x, [6,9]) # => False (x contains none of 6 and 9.) - ``` diff --git a/v2.4.x/site/en/userGuide/use-partition-key.md b/v2.4.x/site/en/userGuide/use-partition-key.md deleted file mode 100644 index d119d455d5..0000000000 --- a/v2.4.x/site/en/userGuide/use-partition-key.md +++ /dev/null @@ -1,576 +0,0 @@ ---- -id: use-partition-key.md -title: Use Partition Key ---- - -# Use Partition Key - -This guide walks you through using the partition key to accelerate data retrieval from your collection. - -## Overview - -You can set a particular field in a collection as the partition key so that Milvus distributes incoming entities into different partitions according to their respective partition values in this field. This allows entities with the same key value to be grouped in a partition, accelerating search performance by avoiding the need to scan irrelevant partitions when filtering by the key field. When compared to traditional filtering methods, the partition key can greatly enhance query performance. - -You can use the partition key to implement multi-tenancy. For details on multi-tenancy, read [Multi-tenancy](https://milvus.io/docs/multi_tenancy.md) for more. - -## Enable partition key - -To set a field as the partition key, specify `partition_key_field` when creating a collection schema. - -In the example code below, `num_partitions` determines the number of partitions that will be created. By default, it is set to `64`. We recommend you retain the default value. - -
          - -For more information on parameters, refer to [`MilvusClient`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Client/MilvusClient.md), [`create_schema()`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Collections/create_schema.md), and [`add_field()`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/CollectionSchema/add_field.md) in the SDK reference. - -
          - -
          - -For more information on parameters, refer to [`MilvusClientV2`](https://milvus.io/api-reference/java/v2.4.x/v2/Client/MilvusClientV2.md), [`createSchema()`](https://milvus.io/api-reference/java/v2.4.x/v2/Collections/createSchema.md), and [`addField()`](https://milvus.io/api-reference/java/v2.4.x/v2/CollectionSchema/addField.md) in the SDK reference. - -
          - -
          - -For more information on parameters, refer to [`MilvusClient`](https://milvus.io/api-reference/node/v2.4.x/Client/MilvusClient.md) and [`createCollection()`](https://milvus.io/api-reference/node/v2.4.x/Collections/createCollection.md) in the SDK reference. - -
          - -
          - Python - Java - Node.js -
          - -```python -import random, time -from pymilvus import connections, MilvusClient, DataType - -SERVER_ADDR = "http://localhost:19530" - -# 1. Set up a Milvus client -client = MilvusClient( - uri=SERVER_ADDR -) - -# 2. Create a collection -schema = MilvusClient.create_schema( - auto_id=False, - enable_dynamic_field=True, - partition_key_field="color", - num_partitions=64 # Number of partitions. Defaults to 64. -) - -schema.add_field(field_name="id", datatype=DataType.INT64, is_primary=True) -schema.add_field(field_name="vector", datatype=DataType.FLOAT_VECTOR, dim=5) -schema.add_field(field_name="color", datatype=DataType.VARCHAR, max_length=512) -``` - -```java -import io.milvus.v2.client.ConnectConfig; -import io.milvus.v2.client.MilvusClientV2; -import io.milvus.v2.common.DataType; -import io.milvus.v2.common.IndexParam; -import io.milvus.v2.service.collection.request.AddFieldReq; -import io.milvus.v2.service.collection.request.CreateCollectionReq; - -String CLUSTER_ENDPOINT = "http://localhost:19530"; - -// 1. Connect to Milvus server -ConnectConfig connectConfig = ConnectConfig.builder() - .uri(CLUSTER_ENDPOINT) - .build(); - -MilvusClientV2 client = new MilvusClientV2(connectConfig); - -// 2. Create a collection in customized setup mode - -// 2.1 Create schema -CreateCollectionReq.CollectionSchema schema = client.createSchema(); - -// 2.2 Add fields to schema -schema.addField(AddFieldReq.builder() - .fieldName("id") - .dataType(DataType.Int64) - .isPrimaryKey(true) - .autoID(false) - .build()); - -schema.addField(AddFieldReq.builder() - .fieldName("vector") - .dataType(DataType.FloatVector) - .dimension(5) - .build()); - -schema.addField(AddFieldReq.builder() - .fieldName("color") - .dataType(DataType.VarChar) - .maxLength(512) - .isPartitionKey(true) - .build()); -``` - -```javascript -const { MilvusClient, DataType, sleep } = require("@zilliz/milvus2-sdk-node") - -const address = "http://localhost:19530" - -async function main() { -// 1. Set up a Milvus Client -client = new MilvusClient({address}); - -// 2. Create a collection -// 2.1 Define fields -const fields = [ - { - name: "id", - data_type: DataType.Int64, - is_primary_key: true, - auto_id: false - }, - { - name: "vector", - data_type: DataType.FloatVector, - dim: 5 - }, - { - name: "color", - data_type: DataType.VarChar, - max_length: 512, - is_partition_key: true - } -] -``` - -After you have defined the fields, set up the index parameters. - -
          - Python - Java - Node.js -
          - -```python -index_params = MilvusClient.prepare_index_params() - -index_params.add_index( - field_name="id", - index_type="STL_SORT" -) - -index_params.add_index( - field_name="color", - index_type="Trie" -) - -index_params.add_index( - field_name="vector", - index_type="IVF_FLAT", - metric_type="L2", - params={"nlist": 1024} -) -``` - -```java -// 2.3 Prepare index parameters -IndexParam indexParamForVectorField = IndexParam.builder() - .fieldName("vector") - .indexType(IndexParam.IndexType.IVF_FLAT) - .metricType(IndexParam.MetricType.IP) - .extraParams(Map.of("nlist", 1024)) - .build(); - -List indexParams = new ArrayList<>(); -indexParams.add(indexParamForVectorField); -``` - -```javascript -// 2.2 Prepare index parameters -const index_params = [{ - field_name: "color", - index_type: "Trie" -},{ - field_name: "id", - index_type: "STL_SORT" -},{ - field_name: "vector", - index_type: "IVF_FLAT", - metric_type: "IP", - params: { nlist: 1024} -}] -``` - -Finally, you can create a collection. - -
          - Python - Java - Node.js -
          - -```python -client.create_collection( - collection_name="test_collection", - schema=schema, - index_params=index_params -) -``` - -```java -// 2.4 Create a collection with schema and index parameters -CreateCollectionReq customizedSetupReq = CreateCollectionReq.builder() - .collectionName("test_collection") - .collectionSchema(schema) - .indexParams(indexParams) - .build(); - -client.createCollection(customizedSetupReq); -``` - -```javascript -// 2.3 Create a collection with fields and index parameters -res = await client.createCollection({ - collection_name: "test_collection", - fields: fields, - index_params: index_params, -}) - -console.log(res.error_code) - -// Output -// -// Success -// -``` - -## List partitions - -Once a field of a collection is used as the partition key, Milvus creates the specified number of partitions and manages them on your behalf. Therefore, you cannot manipulate the partitions in this collection anymore. - -The following snippet demonstrates that 64 partitions in a collection once one of its fields is used as the partition key. - -## Insert data - -Once the collection is ready, start inserting data as follows: - -### Prepare data - -
          - Python - Java - Node.js -
          - -```python -# 3. Insert randomly generated vectors -colors = ["green", "blue", "yellow", "red", "black", "white", "purple", "pink", "orange", "brown", "grey"] -data = [] - -for i in range(1000): - current_color = random.choice(colors) - current_tag = random.randint(1000, 9999) - data.append({ - "id": i, - "vector": [ random.uniform(-1, 1) for _ in range(5) ], - "color": current_color, - "tag": current_tag, - "color_tag": f"{current_color}_{str(current_tag)}" - }) - -print(data[0]) -``` - -```java -// 3. Insert randomly generated vectors -List colors = Arrays.asList("green", "blue", "yellow", "red", "black", "white", "purple", "pink", "orange", "brown", "grey"); -List data = new ArrayList<>(); - -for (int i=0; i<1000; i++) { - Random rand = new Random(); - String current_color = colors.get(rand.nextInt(colors.size()-1)); - int current_tag = rand.nextInt(8999) + 1000; - JSONObject row = new JSONObject(); - row.put("id", Long.valueOf(i)); - row.put("vector", Arrays.asList(rand.nextFloat(), rand.nextFloat(), rand.nextFloat(), rand.nextFloat(), rand.nextFloat())); - row.put("color", current_color); - row.put("tag", current_tag); - row.put("color_tag", current_color + "_" + String.valueOf(rand.nextInt(8999) + 1000)); - data.add(row); -} - -System.out.println(JSONObject.toJSON(data.get(0))); -``` - -```javascript -// 3. Insert randomly generated vectors -const colors = ["green", "blue", "yellow", "red", "black", "white", "purple", "pink", "orange", "brown", "grey"] -var data = [] - -for (let i = 0; i < 1000; i++) { - const current_color = colors[Math.floor(Math.random() * colors.length)] - const current_tag = Math.floor(Math.random() * 8999 + 1000) - data.push({ - id: i, - vector: [Math.random(), Math.random(), Math.random(), Math.random(), Math.random()], - color: current_color, - tag: current_tag, - color_tag: `${current_color}_${current_tag}` - }) -} - -console.log(data[0]) -``` - -You can view the structure of the generated data by checking its first entry. - -``` -{ - id: 0, - vector: [ - 0.1275656405044483, - 0.47417858592773277, - 0.13858264437643286, - 0.2390904907020377, - 0.8447862593689635 - ], - color: 'blue', - tag: 2064, - color_tag: 'blue_2064' -} -``` - -### Insert data - -
          - -Use the [`insert()`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Vector/insert.md) method to insert the data into the collection. - -
          - -
          - -Use the [`insert()`](https://milvus.io/api-reference/java/v2.4.x/v2/Vector/insert.md) method to insert the data into the collection. - -
          - -
          - -Use the [`insert()`](https://milvus.io/api-reference/node/v2.4.x/Vector/insert.md) method to insert the data into the collection. - -
          - -
          - Python - Java - Node.js -
          - -```python -res = client.insert( - collection_name="test_collection", - data=data -) - -print(res) - -# Output -# -# { -# "insert_count": 1000, -# "ids": [ -# 0, -# 1, -# 2, -# 3, -# 4, -# 5, -# 6, -# 7, -# 8, -# 9, -# "(990 more items hidden)" -# ] -# } -``` - -```java -// 3.1 Insert data into the collection -InsertReq insertReq = InsertReq.builder() - .collectionName("test_collection") - .data(data) - .build(); - -InsertResp insertResp = client.insert(insertReq); - -System.out.println(JSONObject.toJSON(insertResp)); - -// Output: -// {"insertCnt": 1000} -``` - -```javascript -res = await client.insert({ - collection_name: "test_collection", - data: data, -}) - -console.log(res.insert_cnt) - -// Output -// -// 1000 -// -``` - -## Use partition key - -Once you have indexed and loaded the collection as well as inserted data, you can conduct a similarity search using the partition key. - -
          - -For more information on parameters, refer to [`search()`](https://milvus.io/api-reference/pymilvus/v2.4.x/MilvusClient/Vector/search.md) in the SDK reference. - -
          - -
          - -For more information on parameters, refer to [`search()`](https://milvus.io/api-reference/java/v2.4.x/v2/Vector/search.md) in the SDK reference. - -
          - -
          - -For more information on parameters, refer to [`search()`](https://milvus.io/api-reference/node/v2.4.x/Vector/search.md) in the SDK reference. - -
          - -
          - -

          notes

          - -

          To conduct a similarity search using the partition key, you should include either of the following in the boolean expression of the search request:

          -
            -

          • expr='<partition_key>=="xxxx"'

            • -

          • expr='<partition_key> in ["xxx", "xxx"]'

            • -
          -

          Do replace <partition_key> with the name of the field that is designated as the partition key.

          - -
          - -
          - Python - Java - Node.js -
          - -```python -# 4. Search with partition key -query_vectors = [[0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592]] - -res = client.search( - collection_name="test_collection", - data=query_vectors, - filter="color == 'green'", - search_params={"metric_type": "L2", "params": {"nprobe": 10}}, - output_fields=["id", "color_tag"], - limit=3 -) - -print(res) - -# Output -# -# [ -# [ -# { -# "id": 970, -# "distance": 0.5770174264907837, -# "entity": { -# "id": 970, -# "color_tag": "green_9828" -# } -# }, -# { -# "id": 115, -# "distance": 0.6898155808448792, -# "entity": { -# "id": 115, -# "color_tag": "green_4073" -# } -# }, -# { -# "id": 899, -# "distance": 0.7028976678848267, -# "entity": { -# "id": 899, -# "color_tag": "green_9897" -# } -# } -# ] -# ] -``` - -```java -// 4. Search with partition key -List> query_vectors = Arrays.asList(Arrays.asList(0.3580376395471989f, -0.6023495712049978f, 0.18414012509913835f, -0.26286205330961354f, 0.9029438446296592f)); - -SearchReq searchReq = SearchReq.builder() - .collectionName("test_collection") - .data(query_vectors) - .filter("color == \"green\"") - .topK(3) - .build(); - -SearchResp searchResp = client.search(searchReq); - -System.out.println(JSONObject.toJSON(searchResp)); - -// Output: -// {"searchResults": [[ -// { -// "distance": 1.0586997, -// "id": 414, -// "entity": {} -// }, -// { -// "distance": 0.981384, -// "id": 293, -// "entity": {} -// }, -// { -// "distance": 0.9548756, -// "id": 325, -// "entity": {} -// } -// ]]} -``` - -```javascript -// 4. Search with partition key -const query_vectors = [0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592] - -res = await client.search({ - collection_name: "test_collection", - data: query_vectors, - filter: "color == 'green'", - output_fields: ["color_tag"], - limit: 3 -}) - -console.log(res.results) - -// Output -// -// [ -// { score: 2.402090549468994, id: '135', color_tag: 'green_2694' }, -// { score: 2.3938629627227783, id: '326', color_tag: 'green_7104' }, -// { score: 2.3235254287719727, id: '801', color_tag: 'green_3162' } -// ] -// -``` - -## Typical use cases - -You can utilize the partition key feature to achieve better search performance and enable multi-tenancy. This can be done by assigning a tenant-specific value as the partition key field for each entity. When searching or querying the collection, you can filter entities by the tenant-specific value by including the partition key field in the boolean expression. This approach ensures data isolation by tenants and avoids scanning unnecessary partitions. - diff --git a/v2.4.x/yarn.lock b/v2.4.x/yarn.lock new file mode 100644 index 0000000000..a1453c8fa7 --- /dev/null +++ b/v2.4.x/yarn.lock @@ -0,0 +1,35 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +argparse@^1.0.10: + version "1.0.10" + resolved "https://registry.yarnpkg.com/argparse/-/argparse-1.0.10.tgz#bcd6791ea5ae09725e17e5ad988134cd40b3d911" + integrity sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg== + dependencies: + sprintf-js "~1.0.2" + +autolinker@^3.11.0: + version "3.16.2" + resolved "https://registry.yarnpkg.com/autolinker/-/autolinker-3.16.2.tgz#6bb4f32432fc111b65659336863e653973bfbcc9" + integrity sha512-JiYl7j2Z19F9NdTmirENSUUIIL/9MytEWtmzhfmsKPCp9E+G35Y0UNCMoM9tFigxT59qSc8Ml2dlZXOCVTYwuA== + dependencies: + tslib "^2.3.0" + +remarkable@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/remarkable/-/remarkable-2.0.1.tgz#280ae6627384dfb13d98ee3995627ca550a12f31" + integrity sha512-YJyMcOH5lrR+kZdmB0aJJ4+93bEojRZ1HGDn9Eagu6ibg7aVZhc3OWbbShRid+Q5eAfsEqWxpe+g5W5nYNfNiA== + dependencies: + argparse "^1.0.10" + autolinker "^3.11.0" + +sprintf-js@~1.0.2: + version "1.0.3" + resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" + integrity sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g== + +tslib@^2.3.0: + version "2.6.3" + resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.3.tgz#0438f810ad7a9edcde7a241c3d80db693c8cbfe0" + integrity sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ==