Add support for getting/setting array metadata via REST.

- Serialization C API added
- Internal REST plumbing added

test/src/unit-capi-rest-dense_array.cc

@@ -2409,5 +2409,123 @@ TEST_CASE_METHOD(
   tiledb_array_free(&array);
   tiledb_query_free(&query);
 
+  remove_temp_dir(FILE_URI_PREFIX + FILE_TEMP_DIR);
+}
+
+TEST_CASE_METHOD(
+    DenseArrayRESTFx,
+    "C API: REST Test dense array, array metadata",
+    "[capi][dense][rest][metadata]") {
+  std::string array_name =
+      TILEDB_URI_PREFIX + FILE_URI_PREFIX + FILE_TEMP_DIR + "metadata_array";
+  create_temp_dir(FILE_URI_PREFIX + FILE_TEMP_DIR);
+
+  int64_t dim_domain[] = {1, 10};
+  int64_t tile_extents[] = {2};
+  tiledb_dimension_t* d1;
+  REQUIRE(
+      tiledb_dimension_alloc(
+          ctx_, "d1", TILEDB_INT64, &dim_domain[0], &tile_extents[0], &d1) ==
+      TILEDB_OK);
+
+  tiledb_domain_t* domain;
+  REQUIRE(tiledb_domain_alloc(ctx_, &domain) == TILEDB_OK);
+  REQUIRE(tiledb_domain_add_dimension(ctx_, domain, d1) == TILEDB_OK);
+
+  tiledb_attribute_t* a1;
+  REQUIRE(tiledb_attribute_alloc(ctx_, "a1", TILEDB_INT64, &a1) == TILEDB_OK);
+
+  tiledb_array_schema_t* array_schema;
+  REQUIRE(
+      tiledb_array_schema_alloc(ctx_, TILEDB_DENSE, &array_schema) ==
+      TILEDB_OK);
+  REQUIRE(
+      tiledb_array_schema_set_domain(ctx_, array_schema, domain) == TILEDB_OK);
+  REQUIRE(
+      tiledb_array_schema_add_attribute(ctx_, array_schema, a1) == TILEDB_OK);
+  REQUIRE(tiledb_array_schema_check(ctx_, array_schema) == TILEDB_OK);
+
+  // Create array
+  REQUIRE(
+      tiledb_array_create(ctx_, array_name.c_str(), array_schema) == TILEDB_OK);
+  to_deregister_.insert(array_name);
+
+  // Clean up
+  tiledb_attribute_free(&a1);
+  tiledb_dimension_free(&d1);
+  tiledb_domain_free(&domain);
+  tiledb_array_schema_free(&array_schema);
+
+  // Write some metadata values
+  tiledb_array_t* array;
+  REQUIRE(tiledb_array_alloc(ctx_, array_name.c_str(), &array) == TILEDB_OK);
+  REQUIRE(tiledb_array_open(ctx_, array, TILEDB_WRITE) == TILEDB_OK);
+  int32_t v = 5;
+  float f[] = {1.1f, 1.2f};
+  REQUIRE(
+      tiledb_array_put_metadata(ctx_, array, "aaa", TILEDB_INT32, 1, &v) ==
+      TILEDB_OK);
+  REQUIRE(
+      tiledb_array_put_metadata(ctx_, array, "bb", TILEDB_FLOAT32, 2, f) ==
+      TILEDB_OK);
+  REQUIRE(tiledb_array_close(ctx_, array) == TILEDB_OK);
+  tiledb_array_free(&array);
+
+  // Read metadata and check values.
+  REQUIRE(tiledb_array_alloc(ctx_, array_name.c_str(), &array) == TILEDB_OK);
+  REQUIRE(tiledb_array_open(ctx_, array, TILEDB_READ) == TILEDB_OK);
+  uint64_t num_metadata = 0;
+  REQUIRE(
+      tiledb_array_get_metadata_num(ctx_, array, &num_metadata) == TILEDB_OK);
+  REQUIRE(num_metadata == 2);
+  tiledb_datatype_t datatype = TILEDB_UINT8;
+  uint32_t value_num = 0;
+  const void* value = nullptr;
+  REQUIRE(
+      tiledb_array_get_metadata(
+          ctx_, array, "aaa", &datatype, &value_num, &value) == TILEDB_OK);
+  REQUIRE(datatype == TILEDB_INT32);
+  REQUIRE(value_num == 1);
+  REQUIRE(*static_cast<const int32_t*>(value) == 5);
+  REQUIRE(
+      tiledb_array_get_metadata(
+          ctx_, array, "bb", &datatype, &value_num, &value) == TILEDB_OK);
+  REQUIRE(datatype == TILEDB_FLOAT32);
+  REQUIRE(value_num == 2);
+  REQUIRE(static_cast<const float*>(value)[0] == 1.1f);
+  REQUIRE(static_cast<const float*>(value)[1] == 1.2f);
+  REQUIRE(tiledb_array_close(ctx_, array) == TILEDB_OK);
+  tiledb_array_free(&array);
+
+  // Prevent array metadata filename/timestamp conflicts
+  std::this_thread::sleep_for(std::chrono::milliseconds(100));
+
+  // Open for writing and delete a key.
+  REQUIRE(tiledb_array_alloc(ctx_, array_name.c_str(), &array) == TILEDB_OK);
+  REQUIRE(tiledb_array_open(ctx_, array, TILEDB_WRITE) == TILEDB_OK);
+  REQUIRE(tiledb_array_delete_metadata(ctx_, array, "aaa") == TILEDB_OK);
+  REQUIRE(tiledb_array_close(ctx_, array) == TILEDB_OK);
+  tiledb_array_free(&array);
+
+  // Read metadata and check values again.
+  REQUIRE(tiledb_array_alloc(ctx_, array_name.c_str(), &array) == TILEDB_OK);
+  REQUIRE(tiledb_array_open(ctx_, array, TILEDB_READ) == TILEDB_OK);
+  REQUIRE(
+      tiledb_array_get_metadata_num(ctx_, array, &num_metadata) == TILEDB_OK);
+  REQUIRE(num_metadata == 1);
+  REQUIRE(
+      tiledb_array_get_metadata(
+          ctx_, array, "aaa", &datatype, &value_num, &value) == TILEDB_OK);
+  REQUIRE(value == nullptr);
+  REQUIRE(
+      tiledb_array_get_metadata(
+          ctx_, array, "bb", &datatype, &value_num, &value) == TILEDB_OK);
+  REQUIRE(datatype == TILEDB_FLOAT32);
+  REQUIRE(value_num == 2);
+  REQUIRE(static_cast<const float*>(value)[0] == 1.1f);
+  REQUIRE(static_cast<const float*>(value)[1] == 1.2f);
+  REQUIRE(tiledb_array_close(ctx_, array) == TILEDB_OK);
+  tiledb_array_free(&array);
+
   remove_temp_dir(FILE_URI_PREFIX + FILE_TEMP_DIR);
 }

tiledb/sm/array/array.cc

@@ -159,7 +159,8 @@ Status Array::open(
           "Cannot open array; remote array with no REST client."));
     RETURN_NOT_OK(
         rest_client->get_array_schema_from_rest(array_uri_, &array_schema_));
-    // TODO: get metadata from REST
+    RETURN_NOT_OK(rest_client->get_array_metadata_from_rest(
+        array_uri_, timestamp_, this));
   } else {
     // Open the array.
     RETURN_NOT_OK(storage_manager_->array_open_for_reads(
@@ -212,7 +213,8 @@ Status Array::open(
           "Cannot open array; remote array with no REST client."));
     RETURN_NOT_OK(
         rest_client->get_array_schema_from_rest(array_uri_, &array_schema_));
-    // TODO: get metadata from REST
+    RETURN_NOT_OK(rest_client->get_array_metadata_from_rest(
+        array_uri_, timestamp_, this));
   } else {
     // Open the array.
     RETURN_NOT_OK(storage_manager_->array_open_for_reads(
@@ -257,7 +259,8 @@ Status Array::open(
           "Cannot open array; remote array with no REST client."));
     RETURN_NOT_OK(
         rest_client->get_array_schema_from_rest(array_uri_, &array_schema_));
-    // TODO: get metadata from REST
+    RETURN_NOT_OK(rest_client->get_array_metadata_from_rest(
+        array_uri_, timestamp_, this));
   } else if (query_type == QueryType::READ) {
     RETURN_NOT_OK(storage_manager_->array_open_for_reads(
         array_uri_,
@@ -289,6 +292,15 @@ Status Array::close() {
   fragment_metadata_.clear();
 
   if (remote_) {
+    // Update array metadata for write queries.
+    if (query_type_ == QueryType::WRITE) {
+      auto rest_client = storage_manager_->rest_client();
+      if (rest_client == nullptr)
+        return LOG_STATUS(Status::ArrayError(
+            "Error closing array; remote array with no REST client."));
+      RETURN_NOT_OK(rest_client->post_array_metadata_to_rest(array_uri_, this));
+    }
+
     // Storage manager does not own the array schema for remote arrays.
     delete array_schema_;
     array_schema_ = nullptr;

tiledb/sm/c_api/tiledb.cc

@@ -4297,6 +4297,58 @@ int32_t tiledb_serialize_array_max_buffer_sizes(
   return TILEDB_OK;
 }
 
+int32_t tiledb_serialize_array_metadata(
+    tiledb_ctx_t* ctx,
+    const tiledb_array_t* array,
+    tiledb_serialization_type_t serialize_type,
+    tiledb_buffer_t** buffer) {
+  // Sanity check
+  if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, array) == TILEDB_ERR)
+    return TILEDB_ERR;
+
+  // Allocate buffer
+  if (tiledb_buffer_alloc(ctx, buffer) != TILEDB_OK ||
+      sanity_check(ctx, *buffer) == TILEDB_ERR)
+    return TILEDB_ERR;
+
+  // Serialize
+  if (SAVE_ERROR_CATCH(
+          ctx,
+          tiledb::sm::serialization::array_metadata_serialize(
+              array->array_,
+              (tiledb::sm::SerializationType)serialize_type,
+              (*buffer)->buffer_))) {
+    tiledb_buffer_free(buffer);
+    return TILEDB_ERR;
+  }
+
+  return TILEDB_OK;
+}
+
+int32_t tiledb_deserialize_array_metadata(
+    tiledb_ctx_t* ctx,
+    tiledb_array_t* array,
+    tiledb_serialization_type_t serialize_type,
+    const tiledb_buffer_t* buffer) {
+  // Sanity check
+  if (sanity_check(ctx) == TILEDB_ERR ||
+      sanity_check(ctx, array) == TILEDB_ERR ||
+      sanity_check(ctx, buffer) == TILEDB_ERR)
+    return TILEDB_ERR;
+
+  // Deserialize
+  if (SAVE_ERROR_CATCH(
+          ctx,
+          tiledb::sm::serialization::array_metadata_deserialize(
+              array->array_,
+              (tiledb::sm::SerializationType)serialize_type,
+              *(buffer->buffer_)))) {
+    return TILEDB_ERR;
+  }
+
+  return TILEDB_OK;
+}
+
 /* ****************************** */
 /*            C++ API             */
 /* ****************************** */

tiledb/sm/c_api/tiledb_serialization.h

@@ -224,6 +224,40 @@ TILEDB_EXPORT int32_t tiledb_serialize_array_max_buffer_sizes(
     tiledb_serialization_type_t serialize_type,
     tiledb_buffer_t** buffer);
 
+/**
+ * Serializes the array metadata into the given buffer.
+ *
+ * @note The caller must free the returned `tiledb_buffer_t`.
+ *
+ * @param ctx The TileDB context.
+ * @param array Array whose metadata to serialize.
+ * @param serialization_type Type of serialization to use
+ * @param buffer Will be set to a newly allocated buffer containing the
+ *      serialized array metadata.
+ * @return `TILEDB_OK` for success and `TILEDB_ERR` for error.
+ */
+TILEDB_EXPORT int32_t tiledb_serialize_array_metadata(
+    tiledb_ctx_t* ctx,
+    const tiledb_array_t* array,
+    tiledb_serialization_type_t serialization_type,
+    tiledb_buffer_t** buffer);
+
+/**
+ * Sets the array metadata on the given array instance by deserializing the
+ * array metadata from the given buffer.
+ *
+ * @param ctx The TileDB context.
+ * @param array Array whose metadata to set.
+ * @param serialization_type Type of serialization to use
+ * @param buffer Buffer containing serialized array metadata.
+ * @return `TILEDB_OK` for success and `TILEDB_ERR` for error.
+ */
+TILEDB_EXPORT int32_t tiledb_deserialize_array_metadata(
+    tiledb_ctx_t* ctx,
+    tiledb_array_t* array,
+    tiledb_serialization_type_t serialization_type,
+    const tiledb_buffer_t* buffer);
+
 #ifdef __cplusplus
 }
 #endif

tiledb/sm/rest/rest_client.cc

@@ -222,6 +222,58 @@ Status RestClient::get_array_max_buffer_sizes(
       schema, returned_data, serialization_type_, buffer_sizes);
 }
 
+Status RestClient::get_array_metadata_from_rest(
+    const URI& uri, uint64_t timestamp, Array* array) {
+  if (array == nullptr)
+    return LOG_STATUS(Status::RestError(
+        "Error getting array metadata from REST; array is null."));
+
+  // Init curl and form the URL
+  Curl curlc;
+  RETURN_NOT_OK(curlc.init(config_, extra_headers_));
+  std::string array_ns, array_uri;
+  RETURN_NOT_OK(uri.get_rest_components(&array_ns, &array_uri));
+  std::string url = rest_server_ + "/v1/arrays/" + array_ns + "/" +
+                    curlc.url_escape(array_uri) +
+                    "/array_metadata?timestamp=" + std::to_string(timestamp);
+
+  // Get the data
+  Buffer returned_data;
+  RETURN_NOT_OK(curlc.get_data(url, serialization_type_, &returned_data));
+  if (returned_data.data() == nullptr || returned_data.size() == 0)
+    return LOG_STATUS(Status::RestError(
+        "Error getting array metadata from REST; server returned no data."));
+
+  return serialization::array_metadata_deserialize(
+      array, serialization_type_, returned_data);
+}
+
+Status RestClient::post_array_metadata_to_rest(
+    const URI& uri, const Array* array) {
+  if (array == nullptr)
+    return LOG_STATUS(Status::RestError(
+        "Error posting array metadata to REST; array is null."));
+
+  Buffer buff;
+  RETURN_NOT_OK(serialization::array_metadata_serialize(
+      array, serialization_type_, &buff));
+  // Wrap in a list
+  BufferList serialized;
+  RETURN_NOT_OK(serialized.add_buffer(std::move(buff)));
+
+  // Init curl and form the URL
+  Curl curlc;
+  RETURN_NOT_OK(curlc.init(config_, extra_headers_));
+  std::string array_ns, array_uri;
+  RETURN_NOT_OK(uri.get_rest_components(&array_ns, &array_uri));
+  std::string url = rest_server_ + "/v1/arrays/" + array_ns + "/" +
+                    curlc.url_escape(array_uri) + "/array_metadata";
+
+  // Put the data
+  Buffer returned_data;
+  return curlc.post_data(url, serialization_type_, &serialized, &returned_data);
+}
+
 Status RestClient::submit_query_to_rest(const URI& uri, Query* query) {
   STATS_FUNC_IN(rest_query_submit);
 
@@ -674,6 +726,16 @@ Status RestClient::get_array_max_buffer_sizes(
       Status::RestError("Cannot use rest client; serialization not enabled."));
 }
 
+Status RestClient::get_array_metadata_from_rest(const URI&, uint64_t, Array*) {
+  return LOG_STATUS(
+      Status::RestError("Cannot use rest client; serialization not enabled."));
+}
+
+Status RestClient::post_array_metadata_to_rest(const URI&, const Array*) {
+  return LOG_STATUS(
+      Status::RestError("Cannot use rest client; serialization not enabled."));
+}
+
 Status RestClient::submit_query_to_rest(const URI&, Query*) {
   return LOG_STATUS(
       Status::RestError("Cannot use rest client; serialization not enabled."));

tiledb/sm/rest/rest_client.h

@@ -107,6 +107,27 @@ class RestClient {
       std::unordered_map<std::string, std::pair<uint64_t, uint64_t>>*
           buffer_sizes);
 
+  /**
+   * Gets the array's metadata from the REST server (and updates the in-memory
+   * Metadata of the array to match the returned values).
+   *
+   * @param uri Array URI
+   * @param timestamp Timestamp at which to open array
+   * @param array Array to fetch metadata for
+   * @return Status
+   */
+  Status get_array_metadata_from_rest(
+      const URI& uri, uint64_t timestamp, Array* array);
+
+  /**
+   * Posts the array's metadata to the REST server.
+   *
+   * @param uri Array URI
+   * @param array Array to update/post metadata for.
+   * @return Status
+   */
+  Status post_array_metadata_to_rest(const URI& uri, const Array* array);
+
   /**
    * Post a data query to rest server
    *