diff --git a/Cargo.lock b/Cargo.lock index 1c71cdd705f..4ea0a23f554 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4919,9 +4919,9 @@ dependencies = [ [[package]] name = "lance-namespace-reqwest-client" -version = "0.0.18" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ea349999bcda4eea53fc05d334b3775ec314761e6a706555c777d7a29b18d19" +checksum = "b748e89a3a0e5d9fb1b51e4382f783f8aa6b620d755012d4856180968014e619" dependencies = [ "reqwest", "serde", diff --git a/Cargo.toml b/Cargo.toml index c5cd509f311..faa2f6d5f8f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -63,7 +63,7 @@ lance-io = { version = "=1.1.0-beta.2", path = "./rust/lance-io", default-featur lance-linalg = { version = "=1.1.0-beta.2", path = "./rust/lance-linalg" } lance-namespace = { version = "=1.1.0-beta.2", path = "./rust/lance-namespace" } lance-namespace-impls = { version = "=1.1.0-beta.2", path = "./rust/lance-namespace-impls" } -lance-namespace-reqwest-client = "0.0.18" +lance-namespace-reqwest-client = "0.3.1" lance-table = { version = "=1.1.0-beta.2", path = "./rust/lance-table" } lance-test-macros = { version = "=1.1.0-beta.2", path = "./rust/lance-test-macros" } lance-testing = { version = "=1.1.0-beta.2", path = "./rust/lance-testing" } diff --git a/java/lance-jni/Cargo.lock b/java/lance-jni/Cargo.lock index 06eb7e045f9..0fb00f0a54d 100644 --- a/java/lance-jni/Cargo.lock +++ b/java/lance-jni/Cargo.lock @@ -3417,6 +3417,7 @@ dependencies = [ "arrow-buffer", "arrow-cast", "arrow-data", + "arrow-ord", "arrow-schema", "arrow-select", "bytes", @@ -3451,6 +3452,7 @@ dependencies = [ "datafusion-sql", "deepsize", "futures", + "itertools 0.13.0", "lance-arrow", "libc", "log", @@ -3793,9 +3795,9 @@ dependencies = [ [[package]] name = "lance-namespace-reqwest-client" -version = "0.0.18" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ea349999bcda4eea53fc05d334b3775ec314761e6a706555c777d7a29b18d19" +checksum = "b748e89a3a0e5d9fb1b51e4382f783f8aa6b620d755012d4856180968014e619" dependencies = [ "reqwest", "serde", diff --git a/java/pom.xml b/java/pom.xml index b09d7576a97..d68a6ca3ee2 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -108,12 +108,12 @@ org.lance lance-namespace-core - 0.2.1 + 0.3.1 org.lance lance-namespace-apache-client - 0.2.1 + 0.3.1 com.fasterxml.jackson.core diff --git a/python/Cargo.lock b/python/Cargo.lock index 8a0fa929db8..8d226fd11a1 100644 --- a/python/Cargo.lock +++ b/python/Cargo.lock @@ -4277,9 +4277,9 @@ dependencies = [ [[package]] name = "lance-namespace-reqwest-client" -version = "0.0.18" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ea349999bcda4eea53fc05d334b3775ec314761e6a706555c777d7a29b18d19" +checksum = "b748e89a3a0e5d9fb1b51e4382f783f8aa6b620d755012d4856180968014e619" dependencies = [ "reqwest", "serde", diff --git a/python/pyproject.toml b/python/pyproject.toml index 7aa500668df..bffb76c33d7 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "pylance" dynamic = ["version"] -dependencies = ["pyarrow>=14", "numpy>=1.22", "lance-namespace>=0.2.1"] +dependencies = ["pyarrow>=14", "numpy>=1.22", "lance-namespace>=0.3.1"] description = "python wrapper for Lance columnar format" authors = [{ name = "Lance Devs", email = "dev@lance.org" }] license = { file = "LICENSE" } diff --git a/rust/lance-io/src/object_store/storage_options.rs b/rust/lance-io/src/object_store/storage_options.rs index 9405f95d70c..f809df8d1d3 100644 --- a/rust/lance-io/src/object_store/storage_options.rs +++ b/rust/lance-io/src/object_store/storage_options.rs @@ -114,6 +114,7 @@ impl StorageOptionsProvider for LanceNamespaceStorageOptionsProvider { let request = DescribeTableRequest { id: Some(self.table_id.clone()), version: None, + with_table_uri: None, }; let response = self diff --git a/rust/lance-namespace-impls/src/dir.rs b/rust/lance-namespace-impls/src/dir.rs index fd5a63a0848..fdb4370f6ab 100644 --- a/rust/lance-namespace-impls/src/dir.rs +++ b/rust/lance-namespace-impls/src/dir.rs @@ -777,11 +777,20 @@ impl LanceNamespace for DirectoryNamespace { let arrow_schema: arrow_schema::Schema = lance_schema.into(); let json_schema = arrow_schema_to_json(&arrow_schema)?; Ok(DescribeTableResponse { + table: Some(table_name), + namespace: request.id.as_ref().map(|id| { + if id.len() > 1 { + id[..id.len() - 1].to_vec() + } else { + vec![] + } + }), version: Some(version as i64), - location: Some(table_uri), + location: Some(table_uri.clone()), + table_uri: Some(table_uri), schema: Some(Box::new(json_schema)), - properties: None, storage_options: self.storage_options.clone(), + stats: None, }) } Err(err) => { @@ -793,11 +802,20 @@ impl LanceNamespace for DirectoryNamespace { .unwrap_or(false) { Ok(DescribeTableResponse { + table: Some(table_name), + namespace: request.id.as_ref().map(|id| { + if id.len() > 1 { + id[..id.len() - 1].to_vec() + } else { + vec![] + } + }), version: None, - location: Some(table_uri), + location: Some(table_uri.clone()), + table_uri: Some(table_uri), schema: None, - properties: None, storage_options: self.storage_options.clone(), + stats: None, }) } else { Err(Error::Namespace { @@ -886,21 +904,6 @@ impl LanceNamespace for DirectoryNamespace { }); } - // Validate location if provided - if let Some(location) = &request.location { - let location = location.trim_end_matches('/'); - if location != table_uri { - return Err(Error::Namespace { - source: format!( - "Cannot create table {} at location {}, must be at location {}", - table_name, location, table_uri - ) - .into(), - location: snafu::location!(), - }); - } - } - // Parse the Arrow IPC stream from request_data let cursor = Cursor::new(request_data.to_vec()); let stream_reader = StreamReader::try_new(cursor, None).map_err(|e| Error::Namespace { @@ -948,9 +951,9 @@ impl LanceNamespace for DirectoryNamespace { })?; Ok(CreateTableResponse { + transaction_id: None, version: Some(1), location: Some(table_uri), - properties: None, storage_options: self.storage_options.clone(), }) } @@ -1007,6 +1010,7 @@ impl LanceNamespace for DirectoryNamespace { })?; Ok(CreateEmptyTableResponse { + transaction_id: None, location: Some(table_uri), properties: None, storage_options: self.storage_options.clone(), @@ -1188,28 +1192,6 @@ mod tests { ); } - #[tokio::test] - async fn test_create_table_with_wrong_location() { - let (namespace, _temp_dir) = create_test_namespace().await; - - // Create test IPC data - let schema = create_test_schema(); - let ipc_data = create_test_ipc_data(&schema); - - let mut request = CreateTableRequest::new(); - request.id = Some(vec!["test_table".to_string()]); - request.location = Some("/wrong/path/table.lance".to_string()); - - let result = namespace - .create_table(request, bytes::Bytes::from(ipc_data)) - .await; - assert!(result.is_err()); - assert!(result - .unwrap_err() - .to_string() - .contains("must be at location")); - } - #[tokio::test] async fn test_list_tables() { let (namespace, _temp_dir) = create_test_namespace().await; @@ -2360,7 +2342,7 @@ mod tests { register_req.id = Some(vec!["registered_table".to_string()]); let response = namespace.register_table(register_req).await.unwrap(); - assert_eq!(response.location, "external_table.lance"); + assert_eq!(response.location, Some("external_table.lance".to_string())); // Verify table exists in namespace let mut exists_req = TableExistsRequest::new(); diff --git a/rust/lance-namespace-impls/src/dir/manifest.rs b/rust/lance-namespace-impls/src/dir/manifest.rs index ddc934d79ee..bc45c084307 100644 --- a/rust/lance-namespace-impls/src/dir/manifest.rs +++ b/rust/lance-namespace-impls/src/dir/manifest.rs @@ -1069,6 +1069,14 @@ impl LanceNamespace for ManifestNamespace { let object_id = Self::str_object_id(table_id); let table_info = self.query_manifest_for_table(&object_id).await?; + // Extract table name and namespace from table_id + let table_name = table_id.last().cloned().unwrap_or_default(); + let namespace_id: Vec = if table_id.len() > 1 { + table_id[..table_id.len() - 1].to_vec() + } else { + vec![] + }; + match table_info { Some(info) => { // Construct full URI from relative location @@ -1088,21 +1096,27 @@ impl LanceNamespace for ManifestNamespace { let json_schema = arrow_schema_to_json(&arrow_schema)?; Ok(DescribeTableResponse { + table: Some(table_name.clone()), + namespace: Some(namespace_id.clone()), version: Some(version as i64), - location: Some(table_uri), + location: Some(table_uri.clone()), + table_uri: Some(table_uri), schema: Some(Box::new(json_schema)), - properties: None, storage_options: self.storage_options.clone(), + stats: None, }) } Err(_) => { // If dataset can't be opened (e.g., empty table), return minimal info Ok(DescribeTableResponse { + table: Some(table_name), + namespace: Some(namespace_id), version: None, - location: Some(table_uri), + location: Some(table_uri.clone()), + table_uri: Some(table_uri), schema: None, - properties: None, storage_options: self.storage_options.clone(), + stats: None, }) } } @@ -1188,21 +1202,6 @@ impl LanceNamespace for ManifestNamespace { }); } - // Validate location if provided - if let Some(location) = &request.location { - let location = location.trim_end_matches('/'); - if location != table_uri { - return Err(Error::Namespace { - source: format!( - "Cannot create table {} at location {}, must be at location {}", - table_name, location, table_uri - ) - .into(), - location: location!(), - }); - } - } - // Write the data using Lance Dataset let cursor = Cursor::new(data.to_vec()); let stream_reader = StreamReader::try_new(cursor, None) @@ -1241,9 +1240,9 @@ impl LanceNamespace for ManifestNamespace { .await?; Ok(CreateTableResponse { + transaction_id: None, version: Some(1), location: Some(table_uri), - properties: None, storage_options: self.storage_options.clone(), }) } @@ -1431,6 +1430,7 @@ impl LanceNamespace for ManifestNamespace { .await?; Ok(CreateNamespaceResponse { + transaction_id: None, properties: request.properties, }) } @@ -1613,6 +1613,7 @@ impl LanceNamespace for ManifestNamespace { ); Ok(CreateEmptyTableResponse { + transaction_id: None, location: Some(table_uri), properties: None, storage_options: self.storage_options.clone(), @@ -1688,7 +1689,8 @@ impl LanceNamespace for ManifestNamespace { .await?; Ok(RegisterTableResponse { - location, + transaction_id: None, + location: Some(location), properties: None, }) } @@ -1732,6 +1734,7 @@ impl LanceNamespace for ManifestNamespace { }; Ok(DeregisterTableResponse { + transaction_id: None, id: request.id.clone(), location: Some(table_uri), properties: None, diff --git a/rust/lance-namespace-impls/src/rest.rs b/rust/lance-namespace-impls/src/rest.rs index 1f7ee341d26..3b5d0650659 100644 --- a/rust/lance-namespace-impls/src/rest.rs +++ b/rust/lance-namespace-impls/src/rest.rs @@ -9,22 +9,32 @@ use async_trait::async_trait; use bytes::Bytes; use lance_namespace::apis::{ - configuration::Configuration, namespace_api, table_api, transaction_api, + configuration::Configuration, namespace_api, table_api, tag_api, transaction_api, }; use lance_namespace::models::{ - AlterTransactionRequest, AlterTransactionResponse, CountTableRowsRequest, - CreateEmptyTableRequest, CreateEmptyTableResponse, CreateNamespaceRequest, - CreateNamespaceResponse, CreateTableIndexRequest, CreateTableIndexResponse, CreateTableRequest, - CreateTableResponse, DeleteFromTableRequest, DeleteFromTableResponse, DeregisterTableRequest, - DeregisterTableResponse, DescribeNamespaceRequest, DescribeNamespaceResponse, - DescribeTableIndexStatsRequest, DescribeTableIndexStatsResponse, DescribeTableRequest, - DescribeTableResponse, DescribeTransactionRequest, DescribeTransactionResponse, - DropNamespaceRequest, DropNamespaceResponse, DropTableRequest, DropTableResponse, - InsertIntoTableRequest, InsertIntoTableResponse, ListNamespacesRequest, ListNamespacesResponse, - ListTableIndicesRequest, ListTableIndicesResponse, ListTablesRequest, ListTablesResponse, + AlterTableAddColumnsRequest, AlterTableAddColumnsResponse, AlterTableAlterColumnsRequest, + AlterTableAlterColumnsResponse, AlterTableDropColumnsRequest, AlterTableDropColumnsResponse, + AlterTransactionRequest, AlterTransactionResponse, AnalyzeTableQueryPlanRequest, + CountTableRowsRequest, CreateEmptyTableRequest, CreateEmptyTableResponse, + CreateNamespaceRequest, CreateNamespaceResponse, CreateTableIndexRequest, + CreateTableIndexResponse, CreateTableRequest, CreateTableResponse, + CreateTableScalarIndexResponse, CreateTableTagRequest, CreateTableTagResponse, + DeleteFromTableRequest, DeleteFromTableResponse, DeleteTableTagRequest, DeleteTableTagResponse, + DeregisterTableRequest, DeregisterTableResponse, DescribeNamespaceRequest, + DescribeNamespaceResponse, DescribeTableIndexStatsRequest, DescribeTableIndexStatsResponse, + DescribeTableRequest, DescribeTableResponse, DescribeTransactionRequest, + DescribeTransactionResponse, DropNamespaceRequest, DropNamespaceResponse, + DropTableIndexRequest, DropTableIndexResponse, DropTableRequest, DropTableResponse, + ExplainTableQueryPlanRequest, GetTableStatsRequest, GetTableStatsResponse, + GetTableTagVersionRequest, GetTableTagVersionResponse, InsertIntoTableRequest, + InsertIntoTableResponse, ListNamespacesRequest, ListNamespacesResponse, + ListTableIndicesRequest, ListTableIndicesResponse, ListTableTagsRequest, ListTableTagsResponse, + ListTableVersionsRequest, ListTableVersionsResponse, ListTablesRequest, ListTablesResponse, MergeInsertIntoTableRequest, MergeInsertIntoTableResponse, NamespaceExistsRequest, - QueryTableRequest, RegisterTableRequest, RegisterTableResponse, TableExistsRequest, - UpdateTableRequest, UpdateTableResponse, + QueryTableRequest, RegisterTableRequest, RegisterTableResponse, RenameTableRequest, + RenameTableResponse, RestoreTableRequest, RestoreTableResponse, TableExistsRequest, + UpdateTableRequest, UpdateTableResponse, UpdateTableSchemaMetadataRequest, + UpdateTableSchemaMetadataResponse, UpdateTableTagRequest, UpdateTableTagResponse, }; use lance_core::{box_error, Error, Result}; @@ -456,9 +466,15 @@ impl LanceNamespace for RestNamespace { async fn describe_table(&self, request: DescribeTableRequest) -> Result { let id = object_id_str(&request.id, &self.delimiter)?; - table_api::describe_table(&self.reqwest_config, &id, request, Some(&self.delimiter)) - .await - .map_err(convert_api_error) + table_api::describe_table( + &self.reqwest_config, + &id, + request.clone(), + Some(&self.delimiter), + request.with_table_uri, + ) + .await + .map_err(convert_api_error) } async fn register_table(&self, request: RegisterTableRequest) -> Result { @@ -480,7 +496,7 @@ impl LanceNamespace for RestNamespace { async fn drop_table(&self, request: DropTableRequest) -> Result { let id = object_id_str(&request.id, &self.delimiter)?; - table_api::drop_table(&self.reqwest_config, &id, request, Some(&self.delimiter)) + table_api::drop_table(&self.reqwest_config, &id, Some(&self.delimiter)) .await .map_err(convert_api_error) } @@ -511,26 +527,12 @@ impl LanceNamespace for RestNamespace { ) -> Result { let id = object_id_str(&request.id, &self.delimiter)?; - let properties_json = request - .properties - .as_ref() - .map(|props| serde_json::to_string(props).unwrap_or_else(|_| "{}".to_string())); - - use lance_namespace::models::create_table_request::Mode; - let mode = request.mode.as_ref().map(|m| match m { - Mode::Create => "create", - Mode::ExistOk => "exist_ok", - Mode::Overwrite => "overwrite", - }); - table_api::create_table( &self.reqwest_config, &id, request_data.to_vec(), Some(&self.delimiter), - mode, - request.location.as_deref(), - properties_json.as_deref(), + request.mode.as_deref(), ) .await .map_err(convert_api_error) @@ -554,18 +556,12 @@ impl LanceNamespace for RestNamespace { ) -> Result { let id = object_id_str(&request.id, &self.delimiter)?; - use lance_namespace::models::insert_into_table_request::Mode; - let mode = request.mode.as_ref().map(|m| match m { - Mode::Append => "append", - Mode::Overwrite => "overwrite", - }); - table_api::insert_into_table( &self.reqwest_config, &id, request_data.to_vec(), Some(&self.delimiter), - mode, + request.mode.as_deref(), ) .await .map_err(convert_api_error) @@ -594,6 +590,8 @@ impl LanceNamespace for RestNamespace { request.when_not_matched_insert_all, request.when_not_matched_by_source_delete, request.when_not_matched_by_source_delete_filt.as_deref(), + request.timeout.as_deref(), + request.use_index, ) .await .map_err(convert_api_error) @@ -710,6 +708,254 @@ impl LanceNamespace for RestNamespace { .map_err(convert_api_error) } + async fn create_table_scalar_index( + &self, + request: CreateTableIndexRequest, + ) -> Result { + let id = object_id_str(&request.id, &self.delimiter)?; + + table_api::create_table_scalar_index( + &self.reqwest_config, + &id, + request, + Some(&self.delimiter), + ) + .await + .map_err(convert_api_error) + } + + async fn drop_table_index( + &self, + request: DropTableIndexRequest, + ) -> Result { + let id = object_id_str(&request.id, &self.delimiter)?; + + let index_name = request.index_name.as_deref().unwrap_or(""); + + table_api::drop_table_index(&self.reqwest_config, &id, index_name, Some(&self.delimiter)) + .await + .map_err(convert_api_error) + } + + async fn list_all_tables(&self, request: ListTablesRequest) -> Result { + table_api::list_all_tables( + &self.reqwest_config, + Some(&self.delimiter), + request.page_token.as_deref(), + request.limit, + ) + .await + .map_err(convert_api_error) + } + + async fn restore_table(&self, request: RestoreTableRequest) -> Result { + let id = object_id_str(&request.id, &self.delimiter)?; + + table_api::restore_table(&self.reqwest_config, &id, request, Some(&self.delimiter)) + .await + .map_err(convert_api_error) + } + + async fn rename_table(&self, request: RenameTableRequest) -> Result { + let id = object_id_str(&request.id, &self.delimiter)?; + + table_api::rename_table(&self.reqwest_config, &id, request, Some(&self.delimiter)) + .await + .map_err(convert_api_error) + } + + async fn list_table_versions( + &self, + request: ListTableVersionsRequest, + ) -> Result { + let id = object_id_str(&request.id, &self.delimiter)?; + + table_api::list_table_versions( + &self.reqwest_config, + &id, + Some(&self.delimiter), + request.page_token.as_deref(), + request.limit, + ) + .await + .map_err(convert_api_error) + } + + async fn update_table_schema_metadata( + &self, + request: UpdateTableSchemaMetadataRequest, + ) -> Result { + let id = object_id_str(&request.id, &self.delimiter)?; + + let metadata = request.metadata.unwrap_or_default(); + + let result = table_api::update_table_schema_metadata( + &self.reqwest_config, + &id, + metadata, + Some(&self.delimiter), + ) + .await + .map_err(convert_api_error)?; + + Ok(UpdateTableSchemaMetadataResponse { + metadata: Some(result), + ..Default::default() + }) + } + + async fn get_table_stats( + &self, + request: GetTableStatsRequest, + ) -> Result { + let id = object_id_str(&request.id, &self.delimiter)?; + + table_api::get_table_stats(&self.reqwest_config, &id, request, Some(&self.delimiter)) + .await + .map_err(convert_api_error) + } + + async fn explain_table_query_plan( + &self, + request: ExplainTableQueryPlanRequest, + ) -> Result { + let id = object_id_str(&request.id, &self.delimiter)?; + + table_api::explain_table_query_plan( + &self.reqwest_config, + &id, + request, + Some(&self.delimiter), + ) + .await + .map_err(convert_api_error) + } + + async fn analyze_table_query_plan( + &self, + request: AnalyzeTableQueryPlanRequest, + ) -> Result { + let id = object_id_str(&request.id, &self.delimiter)?; + + table_api::analyze_table_query_plan( + &self.reqwest_config, + &id, + request, + Some(&self.delimiter), + ) + .await + .map_err(convert_api_error) + } + + async fn alter_table_add_columns( + &self, + request: AlterTableAddColumnsRequest, + ) -> Result { + let id = object_id_str(&request.id, &self.delimiter)?; + + table_api::alter_table_add_columns( + &self.reqwest_config, + &id, + request, + Some(&self.delimiter), + ) + .await + .map_err(convert_api_error) + } + + async fn alter_table_alter_columns( + &self, + request: AlterTableAlterColumnsRequest, + ) -> Result { + let id = object_id_str(&request.id, &self.delimiter)?; + + table_api::alter_table_alter_columns( + &self.reqwest_config, + &id, + request, + Some(&self.delimiter), + ) + .await + .map_err(convert_api_error) + } + + async fn alter_table_drop_columns( + &self, + request: AlterTableDropColumnsRequest, + ) -> Result { + let id = object_id_str(&request.id, &self.delimiter)?; + + table_api::alter_table_drop_columns( + &self.reqwest_config, + &id, + request, + Some(&self.delimiter), + ) + .await + .map_err(convert_api_error) + } + + async fn list_table_tags( + &self, + request: ListTableTagsRequest, + ) -> Result { + let id = object_id_str(&request.id, &self.delimiter)?; + + tag_api::list_table_tags( + &self.reqwest_config, + &id, + Some(&self.delimiter), + request.page_token.as_deref(), + request.limit, + ) + .await + .map_err(convert_api_error) + } + + async fn get_table_tag_version( + &self, + request: GetTableTagVersionRequest, + ) -> Result { + let id = object_id_str(&request.id, &self.delimiter)?; + + tag_api::get_table_tag_version(&self.reqwest_config, &id, request, Some(&self.delimiter)) + .await + .map_err(convert_api_error) + } + + async fn create_table_tag( + &self, + request: CreateTableTagRequest, + ) -> Result { + let id = object_id_str(&request.id, &self.delimiter)?; + + tag_api::create_table_tag(&self.reqwest_config, &id, request, Some(&self.delimiter)) + .await + .map_err(convert_api_error) + } + + async fn delete_table_tag( + &self, + request: DeleteTableTagRequest, + ) -> Result { + let id = object_id_str(&request.id, &self.delimiter)?; + + tag_api::delete_table_tag(&self.reqwest_config, &id, request, Some(&self.delimiter)) + .await + .map_err(convert_api_error) + } + + async fn update_table_tag( + &self, + request: UpdateTableTagRequest, + ) -> Result { + let id = object_id_str(&request.id, &self.delimiter)?; + + tag_api::update_table_tag(&self.reqwest_config, &id, request, Some(&self.delimiter)) + .await + .map_err(convert_api_error) + } + fn namespace_id(&self) -> String { format!( "RestNamespace {{ endpoint: {:?}, delimiter: {:?} }}", @@ -722,7 +968,6 @@ impl LanceNamespace for RestNamespace { mod tests { use super::*; use bytes::Bytes; - use lance_namespace::models::{create_table_request, insert_into_table_request}; use wiremock::matchers::{method, path}; use wiremock::{Mock, MockServer, ResponseTemplate}; @@ -1023,9 +1268,7 @@ mod tests { "namespace".to_string(), "table".to_string(), ]), - location: None, - mode: Some(create_table_request::Mode::Create), - properties: None, + mode: Some("Create".to_string()), }; let data = Bytes::from("arrow data here"); @@ -1045,7 +1288,7 @@ mod tests { Mock::given(method("POST")) .and(path(path_str.as_str())) .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ - "version": 2 + "transaction_id": "txn-123" }))) .mount(&mock_server) .await; @@ -1062,7 +1305,7 @@ mod tests { "namespace".to_string(), "table".to_string(), ]), - mode: Some(insert_into_table_request::Mode::Append), + mode: Some("Append".to_string()), }; let data = Bytes::from("arrow data here"); @@ -1071,6 +1314,6 @@ mod tests { // Should succeed with mock server assert!(result.is_ok()); let response = result.unwrap(); - assert_eq!(response.version, Some(2)); + assert_eq!(response.transaction_id, Some("txn-123".to_string())); } } diff --git a/rust/lance-namespace-impls/src/rest_adapter.rs b/rust/lance-namespace-impls/src/rest_adapter.rs index eae81d73ba6..dd94b15e7c4 100644 --- a/rust/lance-namespace-impls/src/rest_adapter.rs +++ b/rust/lance-namespace-impls/src/rest_adapter.rs @@ -65,14 +65,65 @@ impl RestAdapter { .route("/v1/namespace/:id/drop", post(drop_namespace)) .route("/v1/namespace/:id/exists", post(namespace_exists)) .route("/v1/namespace/:id/table/list", get(list_tables)) - // Table operations + // Table metadata operations .route("/v1/table/:id/register", post(register_table)) .route("/v1/table/:id/describe", post(describe_table)) .route("/v1/table/:id/exists", post(table_exists)) .route("/v1/table/:id/drop", post(drop_table)) .route("/v1/table/:id/deregister", post(deregister_table)) + .route("/v1/table/:id/rename", post(rename_table)) + .route("/v1/table/:id/restore", post(restore_table)) + .route("/v1/table/:id/version/list", get(list_table_versions)) + .route("/v1/table/:id/stats", get(get_table_stats)) + // Table data operations .route("/v1/table/:id/create", post(create_table)) .route("/v1/table/:id/create-empty", post(create_empty_table)) + .route("/v1/table/:id/insert", post(insert_into_table)) + .route("/v1/table/:id/merge_insert", post(merge_insert_into_table)) + .route("/v1/table/:id/update", post(update_table)) + .route("/v1/table/:id/delete", post(delete_from_table)) + .route("/v1/table/:id/query", post(query_table)) + .route("/v1/table/:id/count_rows", get(count_table_rows)) + // Index operations + .route("/v1/table/:id/create_index", post(create_table_index)) + .route( + "/v1/table/:id/create_scalar_index", + post(create_table_scalar_index), + ) + .route("/v1/table/:id/index/list", get(list_table_indices)) + .route( + "/v1/table/:id/index/:index_name/stats", + get(describe_table_index_stats), + ) + .route( + "/v1/table/:id/index/:index_name/drop", + post(drop_table_index), + ) + // Schema operations + .route("/v1/table/:id/add_columns", post(alter_table_add_columns)) + .route( + "/v1/table/:id/alter_columns", + post(alter_table_alter_columns), + ) + .route("/v1/table/:id/drop_columns", post(alter_table_drop_columns)) + .route( + "/v1/table/:id/schema_metadata/update", + post(update_table_schema_metadata), + ) + // Tag operations + .route("/v1/table/:id/tags/list", get(list_table_tags)) + .route("/v1/table/:id/tags/version", post(get_table_tag_version)) + .route("/v1/table/:id/tags/create", post(create_table_tag)) + .route("/v1/table/:id/tags/delete", post(delete_table_tag)) + .route("/v1/table/:id/tags/update", post(update_table_tag)) + // Query plan operations + .route("/v1/table/:id/explain_plan", post(explain_table_query_plan)) + .route("/v1/table/:id/analyze_plan", post(analyze_table_query_plan)) + // Transaction operations + .route("/v1/transaction/:id/describe", post(describe_transaction)) + .route("/v1/transaction/:id/alter", post(alter_transaction)) + // Global table operations + .route("/v1/table", get(list_all_tables)) .layer(TraceLayer::new_for_http()) .with_state(self.backend.clone()) } @@ -395,9 +446,10 @@ async fn drop_table( State(backend): State>, Path(id): Path, Query(params): Query, - Json(mut request): Json, ) -> Response { - request.id = Some(parse_id(&id, params.delimiter.as_deref())); + let request = DropTableRequest { + id: Some(parse_id(&id, params.delimiter.as_deref())), + }; match backend.drop_table(request).await { Ok(response) => (StatusCode::OK, Json(response)).into_response(), @@ -427,8 +479,6 @@ async fn deregister_table( struct CreateTableQuery { delimiter: Option, mode: Option, - location: Option, - properties: Option, } async fn create_table( @@ -437,25 +487,9 @@ async fn create_table( Query(params): Query, body: Bytes, ) -> Response { - use lance_namespace::models::create_table_request::Mode; - - let mode = params.mode.as_deref().and_then(|m| match m { - "create" => Some(Mode::Create), - "exist_ok" => Some(Mode::ExistOk), - "overwrite" => Some(Mode::Overwrite), - _ => None, - }); - - let properties = params - .properties - .as_ref() - .and_then(|p| serde_json::from_str(p).ok()); - let request = CreateTableRequest { id: Some(parse_id(&id, params.delimiter.as_deref())), - location: params.location, - mode, - properties, + mode: params.mode.clone(), }; match backend.create_table(request, body).await { @@ -478,6 +512,509 @@ async fn create_empty_table( } } +#[derive(Debug, Deserialize)] +struct InsertQuery { + delimiter: Option, + mode: Option, +} + +async fn insert_into_table( + State(backend): State>, + Path(id): Path, + Query(params): Query, + body: Bytes, +) -> Response { + let request = InsertIntoTableRequest { + id: Some(parse_id(&id, params.delimiter.as_deref())), + mode: params.mode.clone(), + }; + + match backend.insert_into_table(request, body).await { + Ok(response) => (StatusCode::OK, Json(response)).into_response(), + Err(e) => error_to_response(e), + } +} + +#[derive(Debug, Deserialize)] +struct MergeInsertQuery { + delimiter: Option, + on: Option, + when_matched_update_all: Option, + when_matched_update_all_filt: Option, + when_not_matched_insert_all: Option, + when_not_matched_by_source_delete: Option, + when_not_matched_by_source_delete_filt: Option, + timeout: Option, + use_index: Option, +} + +async fn merge_insert_into_table( + State(backend): State>, + Path(id): Path, + Query(params): Query, + body: Bytes, +) -> Response { + let request = MergeInsertIntoTableRequest { + id: Some(parse_id(&id, params.delimiter.as_deref())), + on: params.on, + when_matched_update_all: params.when_matched_update_all, + when_matched_update_all_filt: params.when_matched_update_all_filt, + when_not_matched_insert_all: params.when_not_matched_insert_all, + when_not_matched_by_source_delete: params.when_not_matched_by_source_delete, + when_not_matched_by_source_delete_filt: params.when_not_matched_by_source_delete_filt, + timeout: params.timeout, + use_index: params.use_index, + }; + + match backend.merge_insert_into_table(request, body).await { + Ok(response) => (StatusCode::OK, Json(response)).into_response(), + Err(e) => error_to_response(e), + } +} + +async fn update_table( + State(backend): State>, + Path(id): Path, + Query(params): Query, + Json(mut request): Json, +) -> Response { + request.id = Some(parse_id(&id, params.delimiter.as_deref())); + + match backend.update_table(request).await { + Ok(response) => (StatusCode::OK, Json(response)).into_response(), + Err(e) => error_to_response(e), + } +} + +async fn delete_from_table( + State(backend): State>, + Path(id): Path, + Query(params): Query, + Json(mut request): Json, +) -> Response { + request.id = Some(parse_id(&id, params.delimiter.as_deref())); + + match backend.delete_from_table(request).await { + Ok(response) => (StatusCode::OK, Json(response)).into_response(), + Err(e) => error_to_response(e), + } +} + +async fn query_table( + State(backend): State>, + Path(id): Path, + Query(params): Query, + Json(mut request): Json, +) -> Response { + request.id = Some(parse_id(&id, params.delimiter.as_deref())); + + match backend.query_table(request).await { + Ok(bytes) => (StatusCode::OK, bytes).into_response(), + Err(e) => error_to_response(e), + } +} + +async fn count_table_rows( + State(backend): State>, + Path(id): Path, + Query(params): Query, +) -> Response { + let request = CountTableRowsRequest { + id: Some(parse_id(&id, params.delimiter.as_deref())), + version: None, + predicate: None, + }; + + match backend.count_table_rows(request).await { + Ok(count) => (StatusCode::OK, Json(serde_json::json!({ "count": count }))).into_response(), + Err(e) => error_to_response(e), + } +} + +// ============================================================================ +// Table Management Operation Handlers +// ============================================================================ + +async fn rename_table( + State(backend): State>, + Path(id): Path, + Query(params): Query, + Json(mut request): Json, +) -> Response { + request.id = Some(parse_id(&id, params.delimiter.as_deref())); + + match backend.rename_table(request).await { + Ok(response) => (StatusCode::OK, Json(response)).into_response(), + Err(e) => error_to_response(e), + } +} + +async fn restore_table( + State(backend): State>, + Path(id): Path, + Query(params): Query, + Json(mut request): Json, +) -> Response { + request.id = Some(parse_id(&id, params.delimiter.as_deref())); + + match backend.restore_table(request).await { + Ok(response) => (StatusCode::OK, Json(response)).into_response(), + Err(e) => error_to_response(e), + } +} + +async fn list_table_versions( + State(backend): State>, + Path(id): Path, + Query(params): Query, +) -> Response { + let request = ListTableVersionsRequest { + id: Some(parse_id(&id, params.delimiter.as_deref())), + page_token: params.page_token, + limit: params.limit, + }; + + match backend.list_table_versions(request).await { + Ok(response) => (StatusCode::OK, Json(response)).into_response(), + Err(e) => error_to_response(e), + } +} + +async fn get_table_stats( + State(backend): State>, + Path(id): Path, + Query(params): Query, +) -> Response { + let request = GetTableStatsRequest { + id: Some(parse_id(&id, params.delimiter.as_deref())), + }; + + match backend.get_table_stats(request).await { + Ok(response) => (StatusCode::OK, Json(response)).into_response(), + Err(e) => error_to_response(e), + } +} + +async fn list_all_tables( + State(backend): State>, + Query(params): Query, +) -> Response { + let request = ListTablesRequest { + id: None, + page_token: params.page_token, + limit: params.limit, + }; + + match backend.list_all_tables(request).await { + Ok(response) => (StatusCode::OK, Json(response)).into_response(), + Err(e) => error_to_response(e), + } +} + +// ============================================================================ +// Index Operation Handlers +// ============================================================================ + +async fn create_table_index( + State(backend): State>, + Path(id): Path, + Query(params): Query, + Json(mut request): Json, +) -> Response { + request.id = Some(parse_id(&id, params.delimiter.as_deref())); + + match backend.create_table_index(request).await { + Ok(response) => (StatusCode::CREATED, Json(response)).into_response(), + Err(e) => error_to_response(e), + } +} + +async fn create_table_scalar_index( + State(backend): State>, + Path(id): Path, + Query(params): Query, + Json(mut request): Json, +) -> Response { + request.id = Some(parse_id(&id, params.delimiter.as_deref())); + + match backend.create_table_scalar_index(request).await { + Ok(response) => (StatusCode::CREATED, Json(response)).into_response(), + Err(e) => error_to_response(e), + } +} + +async fn list_table_indices( + State(backend): State>, + Path(id): Path, + Query(params): Query, +) -> Response { + let request = ListTableIndicesRequest { + id: Some(parse_id(&id, params.delimiter.as_deref())), + version: None, + page_token: None, + limit: None, + }; + + match backend.list_table_indices(request).await { + Ok(response) => (StatusCode::OK, Json(response)).into_response(), + Err(e) => error_to_response(e), + } +} + +#[derive(Debug, Deserialize)] +struct IndexPathParams { + id: String, + index_name: String, +} + +async fn describe_table_index_stats( + State(backend): State>, + Path(params): Path, + Query(query): Query, +) -> Response { + let request = DescribeTableIndexStatsRequest { + id: Some(parse_id(¶ms.id, query.delimiter.as_deref())), + version: None, + index_name: Some(params.index_name), + }; + + match backend.describe_table_index_stats(request).await { + Ok(response) => (StatusCode::OK, Json(response)).into_response(), + Err(e) => error_to_response(e), + } +} + +async fn drop_table_index( + State(backend): State>, + Path(params): Path, + Query(query): Query, +) -> Response { + let request = DropTableIndexRequest { + id: Some(parse_id(¶ms.id, query.delimiter.as_deref())), + index_name: Some(params.index_name), + }; + + match backend.drop_table_index(request).await { + Ok(response) => (StatusCode::OK, Json(response)).into_response(), + Err(e) => error_to_response(e), + } +} + +// ============================================================================ +// Schema Operation Handlers +// ============================================================================ + +async fn alter_table_add_columns( + State(backend): State>, + Path(id): Path, + Query(params): Query, + Json(mut request): Json, +) -> Response { + request.id = Some(parse_id(&id, params.delimiter.as_deref())); + + match backend.alter_table_add_columns(request).await { + Ok(response) => (StatusCode::OK, Json(response)).into_response(), + Err(e) => error_to_response(e), + } +} + +async fn alter_table_alter_columns( + State(backend): State>, + Path(id): Path, + Query(params): Query, + Json(mut request): Json, +) -> Response { + request.id = Some(parse_id(&id, params.delimiter.as_deref())); + + match backend.alter_table_alter_columns(request).await { + Ok(response) => (StatusCode::OK, Json(response)).into_response(), + Err(e) => error_to_response(e), + } +} + +async fn alter_table_drop_columns( + State(backend): State>, + Path(id): Path, + Query(params): Query, + Json(mut request): Json, +) -> Response { + request.id = Some(parse_id(&id, params.delimiter.as_deref())); + + match backend.alter_table_drop_columns(request).await { + Ok(response) => (StatusCode::OK, Json(response)).into_response(), + Err(e) => error_to_response(e), + } +} + +async fn update_table_schema_metadata( + State(backend): State>, + Path(id): Path, + Query(params): Query, + Json(mut request): Json, +) -> Response { + request.id = Some(parse_id(&id, params.delimiter.as_deref())); + + match backend.update_table_schema_metadata(request).await { + Ok(response) => (StatusCode::OK, Json(response)).into_response(), + Err(e) => error_to_response(e), + } +} + +// ============================================================================ +// Tag Operation Handlers +// ============================================================================ + +async fn list_table_tags( + State(backend): State>, + Path(id): Path, + Query(params): Query, +) -> Response { + let request = ListTableTagsRequest { + id: Some(parse_id(&id, params.delimiter.as_deref())), + page_token: params.page_token, + limit: params.limit, + }; + + match backend.list_table_tags(request).await { + Ok(response) => (StatusCode::OK, Json(response)).into_response(), + Err(e) => error_to_response(e), + } +} + +async fn get_table_tag_version( + State(backend): State>, + Path(id): Path, + Query(params): Query, + Json(mut request): Json, +) -> Response { + request.id = Some(parse_id(&id, params.delimiter.as_deref())); + + match backend.get_table_tag_version(request).await { + Ok(response) => (StatusCode::OK, Json(response)).into_response(), + Err(e) => error_to_response(e), + } +} + +async fn create_table_tag( + State(backend): State>, + Path(id): Path, + Query(params): Query, + Json(mut request): Json, +) -> Response { + request.id = Some(parse_id(&id, params.delimiter.as_deref())); + + match backend.create_table_tag(request).await { + Ok(response) => (StatusCode::CREATED, Json(response)).into_response(), + Err(e) => error_to_response(e), + } +} + +async fn delete_table_tag( + State(backend): State>, + Path(id): Path, + Query(params): Query, + Json(mut request): Json, +) -> Response { + request.id = Some(parse_id(&id, params.delimiter.as_deref())); + + match backend.delete_table_tag(request).await { + Ok(response) => (StatusCode::OK, Json(response)).into_response(), + Err(e) => error_to_response(e), + } +} + +async fn update_table_tag( + State(backend): State>, + Path(id): Path, + Query(params): Query, + Json(mut request): Json, +) -> Response { + request.id = Some(parse_id(&id, params.delimiter.as_deref())); + + match backend.update_table_tag(request).await { + Ok(response) => (StatusCode::OK, Json(response)).into_response(), + Err(e) => error_to_response(e), + } +} + +// ============================================================================ +// Query Plan Operation Handlers +// ============================================================================ + +async fn explain_table_query_plan( + State(backend): State>, + Path(id): Path, + Query(params): Query, + Json(mut request): Json, +) -> Response { + request.id = Some(parse_id(&id, params.delimiter.as_deref())); + + match backend.explain_table_query_plan(request).await { + Ok(plan) => (StatusCode::OK, plan).into_response(), + Err(e) => error_to_response(e), + } +} + +async fn analyze_table_query_plan( + State(backend): State>, + Path(id): Path, + Query(params): Query, + Json(mut request): Json, +) -> Response { + request.id = Some(parse_id(&id, params.delimiter.as_deref())); + + match backend.analyze_table_query_plan(request).await { + Ok(plan) => (StatusCode::OK, plan).into_response(), + Err(e) => error_to_response(e), + } +} + +// ============================================================================ +// Transaction Operation Handlers +// ============================================================================ + +async fn describe_transaction( + State(backend): State>, + Path(id): Path, + Query(_params): Query, + Json(mut request): Json, +) -> Response { + // The path id is the transaction identifier + // The request.id in body is the table ID (namespace path) + // For the trait, we set request.id to include both table ID and transaction ID + // by appending the transaction ID to the table ID path + if let Some(ref mut table_id) = request.id { + table_id.push(id); + } else { + request.id = Some(vec![id]); + } + + match backend.describe_transaction(request).await { + Ok(response) => (StatusCode::OK, Json(response)).into_response(), + Err(e) => error_to_response(e), + } +} + +async fn alter_transaction( + State(backend): State>, + Path(id): Path, + Query(_params): Query, + Json(mut request): Json, +) -> Response { + // The path id is the transaction identifier + // Append it to the table ID path in the request + if let Some(ref mut table_id) = request.id { + table_id.push(id); + } else { + request.id = Some(vec![id]); + } + + match backend.alter_transaction(request).await { + Ok(response) => (StatusCode::OK, Json(response)).into_response(), + Err(e) => error_to_response(e), + } +} + // ============================================================================ // Helper Functions // ============================================================================ @@ -734,9 +1271,7 @@ mod tests { // Create table in child namespace let create_table_req = CreateTableRequest { id: Some(vec!["test_namespace".to_string(), "test_table".to_string()]), - location: None, - mode: Some(create_table_request::Mode::Create), - properties: None, + mode: Some("Create".to_string()), }; let result = fixture @@ -788,9 +1323,7 @@ mod tests { for i in 1..=3 { let create_table_req = CreateTableRequest { id: Some(vec!["test_namespace".to_string(), format!("table{}", i)]), - location: None, - mode: Some(create_table_request::Mode::Create), - properties: None, + mode: Some("Create".to_string()), }; fixture .namespace @@ -834,9 +1367,7 @@ mod tests { // Create table let create_table_req = CreateTableRequest { id: Some(vec!["test_namespace".to_string(), "test_table".to_string()]), - location: None, - mode: Some(create_table_request::Mode::Create), - properties: None, + mode: Some("Create".to_string()), }; fixture .namespace @@ -906,9 +1437,7 @@ mod tests { // Create table let create_table_req = CreateTableRequest { id: Some(vec!["test_namespace".to_string(), "test_table".to_string()]), - location: None, - mode: Some(create_table_request::Mode::Create), - properties: None, + mode: Some("Create".to_string()), }; fixture .namespace @@ -996,9 +1525,7 @@ mod tests { // Create table let create_table_req = CreateTableRequest { id: Some(vec!["test_namespace".to_string(), "test_table".to_string()]), - location: None, - mode: Some(create_table_request::Mode::Create), - properties: None, + mode: Some("Create".to_string()), }; fixture .namespace @@ -1304,9 +1831,7 @@ mod tests { "level3".to_string(), "deep_table".to_string(), ]), - location: None, - mode: Some(create_table_request::Mode::Create), - properties: None, + mode: Some("Create".to_string()), }; let result = fixture @@ -1365,9 +1890,7 @@ mod tests { // Create table with same name in both namespaces let create_table_req = CreateTableRequest { id: Some(vec!["namespace1".to_string(), "shared_table".to_string()]), - location: None, - mode: Some(create_table_request::Mode::Create), - properties: None, + mode: Some("Create".to_string()), }; fixture .namespace @@ -1377,9 +1900,7 @@ mod tests { let create_table_req = CreateTableRequest { id: Some(vec!["namespace2".to_string(), "shared_table".to_string()]), - location: None, - mode: Some(create_table_request::Mode::Create), - properties: None, + mode: Some("Create".to_string()), }; fixture .namespace @@ -1429,9 +1950,7 @@ mod tests { // Create table in namespace let create_table_req = CreateTableRequest { id: Some(vec!["test_namespace".to_string(), "test_table".to_string()]), - location: None, - mode: Some(create_table_request::Mode::Create), - properties: None, + mode: Some("Create".to_string()), }; fixture .namespace @@ -1583,9 +2102,7 @@ mod tests { "test_namespace".to_string(), "physical_table".to_string(), ]), - location: None, - mode: Some(create_table_request::Mode::Create), - properties: None, + mode: Some("Create".to_string()), }; fixture .namespace @@ -1612,7 +2129,10 @@ mod tests { ); let response = result.unwrap(); - assert_eq!(response.location, "test_namespace$physical_table.lance"); + assert_eq!( + response.location, + Some("test_namespace$physical_table.lance".to_string()) + ); // Verify registered table exists let mut exists_req = TableExistsRequest::new(); @@ -1712,9 +2232,7 @@ mod tests { // Create a table let create_table_req = CreateTableRequest { id: Some(vec!["test_namespace".to_string(), "test_table".to_string()]), - location: None, - mode: Some(create_table_request::Mode::Create), - properties: None, + mode: Some("Create".to_string()), }; fixture .namespace @@ -1792,9 +2310,7 @@ mod tests { "test_namespace".to_string(), "original_table".to_string(), ]), - location: None, - mode: Some(create_table_request::Mode::Create), - properties: None, + mode: Some("Create".to_string()), }; let create_response = fixture .namespace @@ -1846,7 +2362,7 @@ mod tests { .expect("Failed to re-register table with new name"); // Should return the exact location we registered - assert_eq!(register_response.location, relative_location); + assert_eq!(register_response.location, Some(relative_location.clone())); // Verify new table exists let mut exists_req = TableExistsRequest::new(); diff --git a/rust/lance-namespace/src/namespace.rs b/rust/lance-namespace/src/namespace.rs index ac2d0c8e176..60c206530f4 100644 --- a/rust/lance-namespace/src/namespace.rs +++ b/rust/lance-namespace/src/namespace.rs @@ -9,19 +9,29 @@ use lance_core::{Error, Result}; use snafu::Location; use lance_namespace_reqwest_client::models::{ - AlterTransactionRequest, AlterTransactionResponse, CountTableRowsRequest, - CreateEmptyTableRequest, CreateEmptyTableResponse, CreateNamespaceRequest, - CreateNamespaceResponse, CreateTableIndexRequest, CreateTableIndexResponse, CreateTableRequest, - CreateTableResponse, DeleteFromTableRequest, DeleteFromTableResponse, DeregisterTableRequest, - DeregisterTableResponse, DescribeNamespaceRequest, DescribeNamespaceResponse, - DescribeTableIndexStatsRequest, DescribeTableIndexStatsResponse, DescribeTableRequest, - DescribeTableResponse, DescribeTransactionRequest, DescribeTransactionResponse, - DropNamespaceRequest, DropNamespaceResponse, DropTableRequest, DropTableResponse, - InsertIntoTableRequest, InsertIntoTableResponse, ListNamespacesRequest, ListNamespacesResponse, - ListTableIndicesRequest, ListTableIndicesResponse, ListTablesRequest, ListTablesResponse, + AlterTableAddColumnsRequest, AlterTableAddColumnsResponse, AlterTableAlterColumnsRequest, + AlterTableAlterColumnsResponse, AlterTableDropColumnsRequest, AlterTableDropColumnsResponse, + AlterTransactionRequest, AlterTransactionResponse, AnalyzeTableQueryPlanRequest, + CountTableRowsRequest, CreateEmptyTableRequest, CreateEmptyTableResponse, + CreateNamespaceRequest, CreateNamespaceResponse, CreateTableIndexRequest, + CreateTableIndexResponse, CreateTableRequest, CreateTableResponse, + CreateTableScalarIndexResponse, CreateTableTagRequest, CreateTableTagResponse, + DeleteFromTableRequest, DeleteFromTableResponse, DeleteTableTagRequest, DeleteTableTagResponse, + DeregisterTableRequest, DeregisterTableResponse, DescribeNamespaceRequest, + DescribeNamespaceResponse, DescribeTableIndexStatsRequest, DescribeTableIndexStatsResponse, + DescribeTableRequest, DescribeTableResponse, DescribeTransactionRequest, + DescribeTransactionResponse, DropNamespaceRequest, DropNamespaceResponse, + DropTableIndexRequest, DropTableIndexResponse, DropTableRequest, DropTableResponse, + ExplainTableQueryPlanRequest, GetTableStatsRequest, GetTableStatsResponse, + GetTableTagVersionRequest, GetTableTagVersionResponse, InsertIntoTableRequest, + InsertIntoTableResponse, ListNamespacesRequest, ListNamespacesResponse, + ListTableIndicesRequest, ListTableIndicesResponse, ListTableTagsRequest, ListTableTagsResponse, + ListTableVersionsRequest, ListTableVersionsResponse, ListTablesRequest, ListTablesResponse, MergeInsertIntoTableRequest, MergeInsertIntoTableResponse, NamespaceExistsRequest, - QueryTableRequest, RegisterTableRequest, RegisterTableResponse, TableExistsRequest, - UpdateTableRequest, UpdateTableResponse, + QueryTableRequest, RegisterTableRequest, RegisterTableResponse, RenameTableRequest, + RenameTableResponse, RestoreTableRequest, RestoreTableResponse, TableExistsRequest, + UpdateTableRequest, UpdateTableResponse, UpdateTableSchemaMetadataRequest, + UpdateTableSchemaMetadataResponse, UpdateTableTagRequest, UpdateTableTagResponse, }; /// Base trait for Lance Namespace implementations. @@ -277,6 +287,195 @@ pub trait LanceNamespace: Send + Sync + std::fmt::Debug { }) } + /// Create a scalar index on a table. + async fn create_table_scalar_index( + &self, + _request: CreateTableIndexRequest, + ) -> Result { + Err(Error::NotSupported { + source: "create_table_scalar_index not implemented".into(), + location: Location::new(file!(), line!(), column!()), + }) + } + + /// Drop a table index. + async fn drop_table_index( + &self, + _request: DropTableIndexRequest, + ) -> Result { + Err(Error::NotSupported { + source: "drop_table_index not implemented".into(), + location: Location::new(file!(), line!(), column!()), + }) + } + + /// List all tables across all namespaces. + async fn list_all_tables(&self, _request: ListTablesRequest) -> Result { + Err(Error::NotSupported { + source: "list_all_tables not implemented".into(), + location: Location::new(file!(), line!(), column!()), + }) + } + + /// Restore a table to a specific version. + async fn restore_table(&self, _request: RestoreTableRequest) -> Result { + Err(Error::NotSupported { + source: "restore_table not implemented".into(), + location: Location::new(file!(), line!(), column!()), + }) + } + + /// Rename a table. + async fn rename_table(&self, _request: RenameTableRequest) -> Result { + Err(Error::NotSupported { + source: "rename_table not implemented".into(), + location: Location::new(file!(), line!(), column!()), + }) + } + + /// List all versions of a table. + async fn list_table_versions( + &self, + _request: ListTableVersionsRequest, + ) -> Result { + Err(Error::NotSupported { + source: "list_table_versions not implemented".into(), + location: Location::new(file!(), line!(), column!()), + }) + } + + /// Update table schema metadata. + async fn update_table_schema_metadata( + &self, + _request: UpdateTableSchemaMetadataRequest, + ) -> Result { + Err(Error::NotSupported { + source: "update_table_schema_metadata not implemented".into(), + location: Location::new(file!(), line!(), column!()), + }) + } + + /// Get table statistics. + async fn get_table_stats( + &self, + _request: GetTableStatsRequest, + ) -> Result { + Err(Error::NotSupported { + source: "get_table_stats not implemented".into(), + location: Location::new(file!(), line!(), column!()), + }) + } + + /// Explain a table query plan. + async fn explain_table_query_plan( + &self, + _request: ExplainTableQueryPlanRequest, + ) -> Result { + Err(Error::NotSupported { + source: "explain_table_query_plan not implemented".into(), + location: Location::new(file!(), line!(), column!()), + }) + } + + /// Analyze a table query plan. + async fn analyze_table_query_plan( + &self, + _request: AnalyzeTableQueryPlanRequest, + ) -> Result { + Err(Error::NotSupported { + source: "analyze_table_query_plan not implemented".into(), + location: Location::new(file!(), line!(), column!()), + }) + } + + /// Add columns to a table. + async fn alter_table_add_columns( + &self, + _request: AlterTableAddColumnsRequest, + ) -> Result { + Err(Error::NotSupported { + source: "alter_table_add_columns not implemented".into(), + location: Location::new(file!(), line!(), column!()), + }) + } + + /// Alter columns in a table. + async fn alter_table_alter_columns( + &self, + _request: AlterTableAlterColumnsRequest, + ) -> Result { + Err(Error::NotSupported { + source: "alter_table_alter_columns not implemented".into(), + location: Location::new(file!(), line!(), column!()), + }) + } + + /// Drop columns from a table. + async fn alter_table_drop_columns( + &self, + _request: AlterTableDropColumnsRequest, + ) -> Result { + Err(Error::NotSupported { + source: "alter_table_drop_columns not implemented".into(), + location: Location::new(file!(), line!(), column!()), + }) + } + + /// List all tags for a table. + async fn list_table_tags( + &self, + _request: ListTableTagsRequest, + ) -> Result { + Err(Error::NotSupported { + source: "list_table_tags not implemented".into(), + location: Location::new(file!(), line!(), column!()), + }) + } + + /// Get the version for a specific tag. + async fn get_table_tag_version( + &self, + _request: GetTableTagVersionRequest, + ) -> Result { + Err(Error::NotSupported { + source: "get_table_tag_version not implemented".into(), + location: Location::new(file!(), line!(), column!()), + }) + } + + /// Create a tag for a table. + async fn create_table_tag( + &self, + _request: CreateTableTagRequest, + ) -> Result { + Err(Error::NotSupported { + source: "create_table_tag not implemented".into(), + location: Location::new(file!(), line!(), column!()), + }) + } + + /// Delete a tag from a table. + async fn delete_table_tag( + &self, + _request: DeleteTableTagRequest, + ) -> Result { + Err(Error::NotSupported { + source: "delete_table_tag not implemented".into(), + location: Location::new(file!(), line!(), column!()), + }) + } + + /// Update a tag for a table. + async fn update_table_tag( + &self, + _request: UpdateTableTagRequest, + ) -> Result { + Err(Error::NotSupported { + source: "update_table_tag not implemented".into(), + location: Location::new(file!(), line!(), column!()), + }) + } + /// Return a human-readable unique identifier for this namespace instance. /// /// This is used for equality comparison and hashing when the namespace is diff --git a/rust/lance/src/dataset.rs b/rust/lance/src/dataset.rs index 722ba7c97e1..1079a72d600 100644 --- a/rust/lance/src/dataset.rs +++ b/rust/lance/src/dataset.rs @@ -876,6 +876,7 @@ impl Dataset { let request = DescribeTableRequest { id: Some(table_id.clone()), version: None, + with_table_uri: None, }; let response = namespace diff --git a/rust/lance/src/dataset/builder.rs b/rust/lance/src/dataset/builder.rs index 16326630d23..332ba504cf9 100644 --- a/rust/lance/src/dataset/builder.rs +++ b/rust/lance/src/dataset/builder.rs @@ -137,6 +137,7 @@ impl DatasetBuilder { let request = DescribeTableRequest { id: Some(table_id.clone()), version: None, + with_table_uri: None, }; let response = namespace