From 39d1036e5b6aa6d65efec08ef18b05ef9f1ce31a Mon Sep 17 00:00:00 2001 From: SimbaGithub <48035983+SimbaGithub@users.noreply.github.com> Date: Thu, 17 Oct 2024 02:26:08 -0700 Subject: [PATCH 1/8] SNOW-692945: multiple statements support (#727) Co-authored-by: sfc-gh-ext-simba-hx --- CMakeLists.txt | 4 - cpp/lib/ResultSet.cpp | 9 +- cpp/lib/ResultSet.hpp | 26 +- cpp/lib/ResultSetArrow.cpp | 37 ++- cpp/lib/ResultSetArrow.hpp | 14 + cpp/lib/ResultSetJson.cpp | 7 +- cpp/lib/result_set.cpp | 353 +++++++------------- cpp/lib/result_set_arrow.cpp | 480 --------------------------- cpp/lib/result_set_json.cpp | 260 --------------- include/snowflake/client.h | 33 +- include/snowflake/version.h | 3 +- lib/client.c | 547 +++++++++++++++++++++---------- lib/client_int.h | 1 + lib/connection.c | 27 +- lib/connection.h | 7 +- lib/result_set.h | 103 ++---- lib/result_set_arrow.h | 268 --------------- lib/result_set_json.h | 259 --------------- tests/CMakeLists.txt | 1 + tests/test_multiple_statements.c | 307 +++++++++++++++++ 20 files changed, 960 insertions(+), 1786 deletions(-) delete mode 100644 cpp/lib/result_set_arrow.cpp delete mode 100644 cpp/lib/result_set_json.cpp delete mode 100644 lib/result_set_arrow.h delete mode 100644 lib/result_set_json.h create mode 100644 tests/test_multiple_statements.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 529f322af2..b71591988f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -225,8 +225,6 @@ set(SOURCE_FILES_CPP_WRAPPER cpp/lib/ClientQueryContextCache.cpp cpp/lib/ClientQueryContextCache.hpp cpp/lib/result_set.cpp - cpp/lib/result_set_arrow.cpp - cpp/lib/result_set_json.cpp cpp/lib/ResultSet.cpp cpp/lib/ResultSet.hpp cpp/lib/ResultSetArrow.cpp @@ -241,8 +239,6 @@ set(SOURCE_FILES_CPP_WRAPPER cpp/util/CurlDesc.cpp cpp/util/CurlDescPool.cpp lib/result_set.h - lib/result_set_arrow.h - lib/result_set_json.h lib/query_context_cache.h lib/curl_desc_pool.h lib/authenticator.h) diff --git a/cpp/lib/ResultSet.cpp b/cpp/lib/ResultSet.cpp index f304164c3d..bd8484a20f 100644 --- a/cpp/lib/ResultSet.cpp +++ b/cpp/lib/ResultSet.cpp @@ -17,7 +17,7 @@ namespace Snowflake namespace Client { -ResultSet::ResultSet() : +ResultSet::ResultSet(QueryResultFormat format) : m_binaryOutputFormat("HEX"), m_dateOutputFormat("YYYY-MM-DD"), m_timeOutputFormat("HH24:MI:SS"), @@ -28,14 +28,16 @@ ResultSet::ResultSet() : m_currChunkIdx(0), m_currChunkRowIdx(0), m_currColumnIdx(0), - m_currRowIdx(0) + m_currRowIdx(0), + m_queryResultFormat(format) { ; } ResultSet::ResultSet( SF_COLUMN_DESC * metadata, - std::string tzString + std::string tzString, + QueryResultFormat format ) : m_currChunkIdx(0), m_currChunkRowIdx(0), @@ -44,6 +46,7 @@ ResultSet::ResultSet( m_totalChunkCount(0), m_totalColumnCount(0), m_metadata(metadata), + m_queryResultFormat(format), m_isFirstChunk(true), m_tzString(tzString), m_error(SF_STATUS_SUCCESS) diff --git a/cpp/lib/ResultSet.hpp b/cpp/lib/ResultSet.hpp index 8f3a46f059..157bd9cc99 100644 --- a/cpp/lib/ResultSet.hpp +++ b/cpp/lib/ResultSet.hpp @@ -12,21 +12,13 @@ #include "cJSON.h" #include "snowflake/basic_types.h" #include "snowflake/client.h" +#include "result_set.h" namespace Snowflake { namespace Client { -/** - * Enumeration over valid query result formats. - */ -enum QueryResultFormat -{ - ARROW, - JSON -}; - /** * The implementation of a base result set. * @@ -40,7 +32,7 @@ class ResultSet /** * Default constructor. */ - ResultSet(); + ResultSet(QueryResultFormat format); /** * Parameterized constructor. @@ -48,7 +40,7 @@ class ResultSet * @param metadata The metadata of the result set. * @param tzString The time zone. */ - ResultSet(SF_COLUMN_DESC * metadata, std::string tzString); + ResultSet(SF_COLUMN_DESC * metadata, std::string tzString, QueryResultFormat format); /** * Destructor. @@ -194,6 +186,13 @@ class ResultSet */ virtual SF_STATUS STDCALL isCellNull(size_t idx, sf_bool * out_data) = 0; + /** + * Gets the total number of rows in the current chunk being processed. + * + * @return the number of rows in the current chunk. + */ + virtual size_t getRowCountInChunk() = 0; + // Other member getters ======================================================================== SF_STATUS getError() @@ -215,6 +214,11 @@ class ResultSet } } + QueryResultFormat getResultFormat() + { + return m_queryResultFormat; + } + protected: // Protected members =========================================================================== diff --git a/cpp/lib/ResultSetArrow.cpp b/cpp/lib/ResultSetArrow.cpp index 78779bc80a..ef20ded1fe 100644 --- a/cpp/lib/ResultSetArrow.cpp +++ b/cpp/lib/ResultSetArrow.cpp @@ -10,7 +10,6 @@ #include "../logger/SFLogger.hpp" #include "ResultSetArrow.hpp" -#include "result_set_arrow.h" #include "DataConversion.hpp" #include "results.h" @@ -22,9 +21,9 @@ namespace Client ResultSetArrow::ResultSetArrow() : - Snowflake::Client::ResultSet() + Snowflake::Client::ResultSet(SF_ARROW_FORMAT) { - m_queryResultFormat = QueryResultFormat::ARROW; + ; // Do nothing } ResultSetArrow::ResultSetArrow( @@ -32,10 +31,8 @@ ResultSetArrow::ResultSetArrow( SF_COLUMN_DESC * metadata, std::string tzString ) : - ResultSet(metadata, tzString) + ResultSet(metadata, tzString, SF_ARROW_FORMAT) { - m_queryResultFormat = QueryResultFormat::ARROW; - this->appendChunk(initialChunk); // Reset row indices so that they can be re-used by public API. @@ -44,6 +41,34 @@ ResultSetArrow::ResultSetArrow( m_currRowIdx = 0; } +ResultSetArrow::ResultSetArrow( + cJSON * jsonRowset64, + SF_COLUMN_DESC * metadata, + std::string tzString +) : + ResultSet(metadata, tzString, SF_ARROW_FORMAT) +{ + arrow::BufferBuilder* bufferBuilder = NULL; + if (jsonRowset64) + { + const char* base64RowsetStr = snowflake_cJSON_GetStringValue(jsonRowset64); + if (base64RowsetStr && strlen(base64RowsetStr) > 0) + { + // Decode Base64-encoded Arrow-format rowset of the chunk and build a buffer builder from it. + std::string decodedRowsetStr = arrow::util::base64_decode(std::string(base64RowsetStr)); + bufferBuilder = new arrow::BufferBuilder(); + (void)bufferBuilder->Append((void*)decodedRowsetStr.c_str(), decodedRowsetStr.length()); + } + } + + this->appendChunk(bufferBuilder); + + // Reset row indices so that they can be re-used by public API. + m_currChunkIdx = 0; + m_currChunkRowIdx = 0; + m_currRowIdx = 0; +} + ResultSetArrow::~ResultSetArrow() { ; // Do nothing. diff --git a/cpp/lib/ResultSetArrow.hpp b/cpp/lib/ResultSetArrow.hpp index 4c85c27501..1805094b12 100644 --- a/cpp/lib/ResultSetArrow.hpp +++ b/cpp/lib/ResultSetArrow.hpp @@ -64,6 +64,20 @@ class ResultSetArrow : public Snowflake::Client::ResultSet */ ResultSetArrow(arrow::BufferBuilder * initialChunk, SF_COLUMN_DESC * metadata, std::string tzString); + + /** + * Parameterized constructor. + * + * This constructor will initialize m_records with the (partial) results + * contained in the initial chunk. It will also initialize m_metadata with + * the metadata in "metadata". + * + * @param jsonRowset64 A pointer to the rowset64 data in json result set. + * @param metadata An array of metadata objects for each column. + * @param tzString The time zone. + */ + ResultSetArrow(cJSON* jsonRowset64, SF_COLUMN_DESC* metadata, std::string tzString); + /** * Destructor. */ diff --git a/cpp/lib/ResultSetJson.cpp b/cpp/lib/ResultSetJson.cpp index 9b27947495..a9bfc42490 100644 --- a/cpp/lib/ResultSetJson.cpp +++ b/cpp/lib/ResultSetJson.cpp @@ -18,9 +18,9 @@ namespace Client ResultSetJson::ResultSetJson() : - ResultSet() + ResultSet(SF_JSON_FORMAT) { - m_queryResultFormat = QueryResultFormat::JSON; + ; // Do nothing } ResultSetJson::ResultSetJson( @@ -28,9 +28,8 @@ ResultSetJson::ResultSetJson( SF_COLUMN_DESC * metadata, std::string tzString ) : - ResultSet(metadata, tzString) + ResultSet(metadata, tzString, SF_JSON_FORMAT) { - m_queryResultFormat = QueryResultFormat::JSON; m_chunk = nullptr; appendChunk(rowset); } diff --git a/cpp/lib/result_set.cpp b/cpp/lib/result_set.cpp index 5cfcb1238a..8923fb77aa 100644 --- a/cpp/lib/result_set.cpp +++ b/cpp/lib/result_set.cpp @@ -4,382 +4,283 @@ #include "result_set.h" #include "ResultSet.hpp" +#include "ResultSetArrow.hpp" +#include "ResultSetJson.hpp" #ifdef __cplusplus extern "C" { #endif - void * rs_create_with_json_result( + result_set_ptr rs_create_with_json_result( cJSON * json_rowset, SF_COLUMN_DESC * metadata, - QueryResultFormat_t * query_result_format, + QueryResultFormat query_result_format, const char * tz_string ) { - switch (*query_result_format) + switch (query_result_format) { - case ARROW_FORMAT: - return rs_arrow_create_with_json_result(json_rowset, metadata, tz_string); - case JSON_FORMAT: - return rs_json_create(json_rowset, metadata, tz_string); +#ifndef SF_WIN32 + case SF_ARROW_FORMAT: + return new Snowflake::Client::ResultSetArrow(json_rowset, metadata, std::string(tz_string)); +#endif + case SF_JSON_FORMAT: + return new Snowflake::Client::ResultSetJson(json_rowset, metadata, std::string(tz_string)); default: return nullptr; } } - void * rs_create_with_chunk( + result_set_ptr rs_create_with_chunk( void * initial_chunk, SF_COLUMN_DESC * metadata, - QueryResultFormat_t * query_result_format, + QueryResultFormat query_result_format, const char * tz_string ) { - switch (*query_result_format) + switch (query_result_format) { - case ARROW_FORMAT: - return rs_arrow_create_with_chunk((NON_JSON_RESP*)initial_chunk, metadata, tz_string); - case JSON_FORMAT: - return rs_json_create((cJSON*)initial_chunk, metadata, tz_string); +#ifndef SF_WIN32 + case SF_ARROW_FORMAT: + return new Snowflake::Client::ResultSetArrow((arrow::BufferBuilder*)(((NON_JSON_RESP*)initial_chunk)->buffer), metadata, std::string(tz_string)); +#endif + case SF_JSON_FORMAT: + return new Snowflake::Client::ResultSetJson((cJSON*)initial_chunk, metadata, std::string(tz_string)); default: return nullptr; } } - void rs_destroy(void * rs, QueryResultFormat_t * query_result_format) + void rs_destroy(result_set_ptr rs) { - switch (*query_result_format){ - case ARROW_FORMAT: - rs_arrow_destroy((rs_arrow_t *) rs); + if (!rs) + { + return; + } + QueryResultFormat query_result_format = + static_cast(rs)->getResultFormat(); + switch (query_result_format){ +#ifndef SF_WIN32 + case SF_ARROW_FORMAT: + delete static_cast(rs); break; - case JSON_FORMAT: - rs_json_destroy((rs_json_t *) rs); +#endif + case SF_JSON_FORMAT: + delete static_cast(rs); break; default: break; } } +#define ERROR_IF_NULL(ptr) \ +{ \ + if (ptr == NULL) { return SF_STATUS_ERROR_NULL_POINTER; } \ +} + SF_STATUS STDCALL - rs_append_chunk(void * rs, QueryResultFormat_t * query_result_format, void * chunk) + rs_append_chunk(result_set_ptr rs, void * chunk) { - switch (*query_result_format) + ERROR_IF_NULL(rs); + QueryResultFormat query_result_format = + static_cast(rs)->getResultFormat(); + switch (query_result_format) { - case ARROW_FORMAT: - return rs_arrow_append_chunk((rs_arrow_t *) rs, (NON_JSON_RESP*)chunk); - case JSON_FORMAT: - return rs_json_append_chunk((rs_json_t *) rs, (cJSON*)chunk); +#ifndef SF_WIN32 + case SF_ARROW_FORMAT: + return static_cast(rs)->appendChunk( + (arrow::BufferBuilder*)(((NON_JSON_RESP*)chunk)->buffer)); +#endif + case SF_JSON_FORMAT: + return static_cast(rs)->appendChunk( + (cJSON*)chunk); default: return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; } } - SF_STATUS STDCALL rs_next(void * rs, QueryResultFormat_t * query_result_format) + SF_STATUS STDCALL rs_next(result_set_ptr rs) { - switch (*query_result_format) - { - case ARROW_FORMAT: - return rs_arrow_next((rs_arrow_t *) rs); - case JSON_FORMAT: - return rs_json_next((rs_json_t *) rs); - default: - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; - } + ERROR_IF_NULL(rs); + return static_cast(rs)->next(); } SF_STATUS STDCALL rs_get_cell_as_bool( - void * rs, - QueryResultFormat_t * query_result_format, + result_set_ptr rs, size_t idx, sf_bool * out_data ) { - switch (*query_result_format) - { - case ARROW_FORMAT: - return rs_arrow_get_cell_as_bool((rs_arrow_t *) rs, idx, out_data); - case JSON_FORMAT: - return rs_json_get_cell_as_bool((rs_json_t *) rs, idx, out_data); - default: - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; - } + ERROR_IF_NULL(rs); + return static_cast(rs)->getCellAsBool(idx, out_data); } SF_STATUS STDCALL rs_get_cell_as_int8( - void * rs, - QueryResultFormat_t * query_result_format, + result_set_ptr rs, size_t idx, int8 * out_data ) { - switch (*query_result_format) - { - case ARROW_FORMAT: - return rs_arrow_get_cell_as_int8((rs_arrow_t *) rs, idx, out_data); - case JSON_FORMAT: - return rs_json_get_cell_as_int8((rs_json_t *) rs, idx, out_data); - default: - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; - } + ERROR_IF_NULL(rs); + return static_cast(rs)->getCellAsInt8(idx, out_data); } SF_STATUS STDCALL rs_get_cell_as_int32( - void * rs, - QueryResultFormat_t * query_result_format, + result_set_ptr rs, size_t idx, int32 * out_data ) { - switch (*query_result_format) - { - case ARROW_FORMAT: - return rs_arrow_get_cell_as_int32((rs_arrow_t *) rs, idx, out_data); - case JSON_FORMAT: - return rs_json_get_cell_as_int32((rs_json_t *) rs, idx, out_data); - default: - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; - } + ERROR_IF_NULL(rs); + return static_cast(rs)->getCellAsInt32(idx, out_data); } SF_STATUS STDCALL rs_get_cell_as_int64( - void * rs, - QueryResultFormat_t * query_result_format, + result_set_ptr rs, size_t idx, int64 * out_data ) { - switch (*query_result_format) - { - case ARROW_FORMAT: - return rs_arrow_get_cell_as_int64((rs_arrow_t *) rs, idx, out_data); - case JSON_FORMAT: - return rs_json_get_cell_as_int64((rs_json_t *) rs, idx, out_data); - default: - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; - } + ERROR_IF_NULL(rs); + return static_cast(rs)->getCellAsInt64(idx, out_data); } SF_STATUS STDCALL rs_get_cell_as_uint8( - void * rs, - QueryResultFormat_t * query_result_format, + result_set_ptr rs, size_t idx, uint8 * out_data ) { - switch (*query_result_format) - { - case ARROW_FORMAT: - return rs_arrow_get_cell_as_uint8((rs_arrow_t *) rs, idx, out_data); - case JSON_FORMAT: - return rs_json_get_cell_as_uint8((rs_json_t *) rs, idx, out_data); - default: - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; - } + ERROR_IF_NULL(rs); + return static_cast(rs)->getCellAsUint8(idx, out_data); } SF_STATUS STDCALL rs_get_cell_as_uint32( - void * rs, - QueryResultFormat_t * query_result_format, + result_set_ptr rs, size_t idx, uint32 * out_data ) { - switch (*query_result_format) - { - case ARROW_FORMAT: - return rs_arrow_get_cell_as_uint32((rs_arrow_t *) rs, idx, out_data); - case JSON_FORMAT: - return rs_json_get_cell_as_uint32((rs_json_t *) rs, idx, out_data); - default: - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; - } + ERROR_IF_NULL(rs); + return static_cast(rs)->getCellAsUint32(idx, out_data); } SF_STATUS STDCALL rs_get_cell_as_uint64( - void * rs, - QueryResultFormat_t * query_result_format, + result_set_ptr rs, size_t idx, uint64 * out_data ) { - switch (*query_result_format) - { - case ARROW_FORMAT: - return rs_arrow_get_cell_as_uint64((rs_arrow_t *) rs, idx, out_data); - case JSON_FORMAT: - return rs_json_get_cell_as_uint64((rs_json_t *) rs, idx, out_data); - default: - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; - } + ERROR_IF_NULL(rs); + return static_cast(rs)->getCellAsUint64(idx, out_data); } SF_STATUS STDCALL rs_get_cell_as_float32( - void * rs, - QueryResultFormat_t * query_result_format, + result_set_ptr rs, size_t idx, float32 * out_data ) { - switch (*query_result_format) - { - case ARROW_FORMAT: - return rs_arrow_get_cell_as_float32((rs_arrow_t *) rs, idx, out_data); - case JSON_FORMAT: - return rs_json_get_cell_as_float32((rs_json_t *) rs, idx, out_data); - default: - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; - } + ERROR_IF_NULL(rs); + return static_cast(rs)->getCellAsFloat32(idx, out_data); } SF_STATUS STDCALL rs_get_cell_as_float64( - void * rs, - QueryResultFormat_t * query_result_format, + result_set_ptr rs, size_t idx, float64 * out_data ) { - switch (*query_result_format) - { - case ARROW_FORMAT: - return rs_arrow_get_cell_as_float64((rs_arrow_t *) rs, idx, out_data); - case JSON_FORMAT: - return rs_json_get_cell_as_float64((rs_json_t *) rs, idx, out_data); - default: - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; - } + ERROR_IF_NULL(rs); + return static_cast(rs)->getCellAsFloat64(idx, out_data); } SF_STATUS STDCALL rs_get_cell_as_const_string( - void * rs, - QueryResultFormat_t * query_result_format, + result_set_ptr rs, size_t idx, const char ** out_data ) { - switch (*query_result_format) - { - case ARROW_FORMAT: - return rs_arrow_get_cell_as_const_string((rs_arrow_t *) rs, idx, out_data); - case JSON_FORMAT: - return rs_json_get_cell_as_const_string((rs_json_t *) rs, idx, out_data); - default: - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; - } + ERROR_IF_NULL(rs); + return static_cast(rs)->getCellAsConstString(idx, out_data); } SF_STATUS STDCALL rs_get_cell_as_timestamp( - void * rs, - QueryResultFormat_t * query_result_format, + result_set_ptr rs, size_t idx, SF_TIMESTAMP * out_data ) { - switch (*query_result_format) - { - case ARROW_FORMAT: - return rs_arrow_get_cell_as_timestamp((rs_arrow_t *) rs, idx, out_data); - case JSON_FORMAT: - return rs_json_get_cell_as_timestamp((rs_json_t *) rs, idx, out_data); - default: - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; - } + ERROR_IF_NULL(rs); + return static_cast(rs)->getCellAsTimestamp(idx, out_data); } SF_STATUS STDCALL rs_get_cell_strlen( - void * rs, - QueryResultFormat_t * query_result_format, + result_set_ptr rs, size_t idx, size_t * out_data ) { - switch (*query_result_format) - { - case ARROW_FORMAT: - return rs_arrow_get_cell_strlen((rs_arrow_t *) rs, idx, out_data); - case JSON_FORMAT: - return rs_json_get_cell_strlen((rs_json_t *) rs, idx, out_data); - default: - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; - } + ERROR_IF_NULL(rs); + return static_cast(rs)->getCellStrlen(idx, out_data); } - size_t rs_get_row_count_in_chunk(void * rs, QueryResultFormat_t * query_result_format) + size_t rs_get_row_count_in_chunk(result_set_ptr rs) { - switch (*query_result_format) - { - case ARROW_FORMAT: - return rs_arrow_get_row_count_in_chunk((rs_arrow_t *) rs); - case JSON_FORMAT: - return rs_json_get_row_count_in_chunk((rs_json_t *) rs); - default: - return 0; - } + ERROR_IF_NULL(rs); + return static_cast(rs)->getRowCountInChunk(); } SF_STATUS STDCALL rs_is_cell_null( - void * rs, - QueryResultFormat_t * query_result_format, + result_set_ptr rs, size_t idx, sf_bool * out_data ) { - switch (*query_result_format) - { - case ARROW_FORMAT: - return rs_arrow_is_cell_null((rs_arrow_t *) rs, idx, out_data); - case JSON_FORMAT: - return rs_json_is_cell_null((rs_json_t *) rs, idx, out_data); - default: - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; - } + ERROR_IF_NULL(rs); + return static_cast(rs)->isCellNull(idx, out_data); } - SF_STATUS STDCALL rs_get_error( - void * rs, - QueryResultFormat_t * query_result_format - ) + SF_STATUS STDCALL rs_get_error(result_set_ptr rs) { - if (!rs || !query_result_format) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - Snowflake::Client::ResultSet * rs_obj = NULL; - switch (*query_result_format) - { - case ARROW_FORMAT: - rs_obj = static_cast(((rs_arrow_t *)rs)->rs_object); - case JSON_FORMAT: - rs_obj = static_cast(((rs_json_t *)rs)->rs_object); - default: - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; - } - - return rs_obj->getError(); + ERROR_IF_NULL(rs); + return static_cast(rs)->getError(); } - const char* rs_get_error_message( - void * rs, - QueryResultFormat_t * query_result_format - ) + const char* rs_get_error_message(result_set_ptr rs) { - if (!rs || !query_result_format) + if (!rs) { return ""; } + return static_cast(rs)->getErrorMessage(); + } - Snowflake::Client::ResultSet * rs_obj = NULL; - switch (*query_result_format) - { - case ARROW_FORMAT: - rs_obj = static_cast(((rs_arrow_t *)rs)->rs_object); - case JSON_FORMAT: - rs_obj = static_cast(((rs_json_t *)rs)->rs_object); - default: - return ""; - } +#ifndef SF_WIN32 + size_t arrow_write_callback(char *ptr, size_t size, size_t nmemb, void *userdata) + { + size_t data_size = size * nmemb; + arrow::BufferBuilder * arrowBufBuilder = (arrow::BufferBuilder*)(userdata); + + log_debug("Curl response for arrow chunk size: %zu", data_size); + (void) arrowBufBuilder->Append(ptr, data_size); + return data_size; + } - return rs_obj->getErrorMessage(); + NON_JSON_RESP* callback_create_arrow_resp(void) + { + NON_JSON_RESP* arrow_resp = new NON_JSON_RESP; + arrow_resp->buffer = new arrow::BufferBuilder(); + arrow_resp->write_callback = arrow_write_callback; + return arrow_resp; } +#else + NON_JSON_RESP* callback_create_arrow_resp(void) + { + log_error("Query results were fetched using Arrow"); + return NULL; + } +#endif #ifdef __cplusplus } // extern "C" diff --git a/cpp/lib/result_set_arrow.cpp b/cpp/lib/result_set_arrow.cpp deleted file mode 100644 index 090cbe1788..0000000000 --- a/cpp/lib/result_set_arrow.cpp +++ /dev/null @@ -1,480 +0,0 @@ -/* - * Copyright (c) 2021 Snowflake Computing, Inc. All rights reserved. - */ - -#include "arrowheaders.hpp" - -#include - -#include "memory.h" -#include "result_set_arrow.h" -#include "ResultSetArrow.hpp" - -#ifdef __cplusplus -extern "C" { -#endif - -#ifndef SF_WIN32 - rs_arrow_t * rs_arrow_create_with_json_result( - cJSON * json_rowset64, - SF_COLUMN_DESC * metadata, - const char * tz_string - ) - { - arrow::BufferBuilder* bufferBuilder = NULL; - if (json_rowset64) - { - const char * base64RowsetStr = snowflake_cJSON_GetStringValue(json_rowset64); - if (base64RowsetStr && strlen(base64RowsetStr) > 0) - { - // Decode Base64-encoded Arrow-format rowset of the chunk and build a buffer builder from it. - std::string decodedRowsetStr = arrow::util::base64_decode(std::string(base64RowsetStr)); - bufferBuilder = new arrow::BufferBuilder(); - (void) bufferBuilder->Append((void *)decodedRowsetStr.c_str(), decodedRowsetStr.length()); - } - } - - rs_arrow_t * rs_struct = (rs_arrow_t *) SF_MALLOC(sizeof(rs_arrow_t)); - Snowflake::Client::ResultSetArrow * rs_obj = - new Snowflake::Client::ResultSetArrow(bufferBuilder, metadata, std::string(tz_string)); - rs_struct->rs_object = rs_obj; - - return rs_struct; - } - - rs_arrow_t * rs_arrow_create_with_chunk( - NON_JSON_RESP * initial_chunk, - SF_COLUMN_DESC * metadata, - const char * tz_string - ) - { - rs_arrow_t * rs_struct = (rs_arrow_t *)SF_MALLOC(sizeof(rs_arrow_t)); - Snowflake::Client::ResultSetArrow * rs_obj = - new Snowflake::Client::ResultSetArrow((arrow::BufferBuilder*)(initial_chunk->buffer), metadata, std::string(tz_string)); - rs_struct->rs_object = rs_obj; - - // the buffer is passed in to result set so the response is no longer needed - delete initial_chunk; - - return rs_struct; - } - - void rs_arrow_destroy(rs_arrow_t * rs) - { - if (rs == NULL) - { - return; - } - - delete static_cast(rs->rs_object); - SF_FREE(rs); - } - - SF_STATUS STDCALL rs_arrow_append_chunk(rs_arrow_t * rs, NON_JSON_RESP * chunk) - { - Snowflake::Client::ResultSetArrow * rs_obj; - - if (rs == NULL) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - rs_obj = static_cast (rs->rs_object); - - arrow::BufferBuilder * buffer = (arrow::BufferBuilder*)(chunk->buffer); - delete chunk; - return rs_obj->appendChunk(buffer); - } - - SF_STATUS STDCALL rs_arrow_next(rs_arrow_t * rs) - { - Snowflake::Client::ResultSetArrow * rs_obj; - - if (rs == NULL) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->next(); - } - - SF_STATUS STDCALL rs_arrow_get_cell_as_bool(rs_arrow_t * rs, size_t idx, sf_bool * out_data) - { - Snowflake::Client::ResultSetArrow * rs_obj; - - if (rs == NULL) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->getCellAsBool(idx, out_data); - } - - SF_STATUS STDCALL rs_arrow_get_cell_as_int8(rs_arrow_t * rs, size_t idx, int8 * out_data) - { - Snowflake::Client::ResultSetArrow * rs_obj; - - if (rs == NULL) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->getCellAsInt8(idx, out_data); - } - - SF_STATUS STDCALL rs_arrow_get_cell_as_int32(rs_arrow_t * rs, size_t idx, int32 * out_data) - { - Snowflake::Client::ResultSetArrow * rs_obj; - - if (rs == NULL) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->getCellAsInt32(idx, out_data); - } - - SF_STATUS STDCALL rs_arrow_get_cell_as_int64(rs_arrow_t * rs, size_t idx, int64 * out_data) - { - Snowflake::Client::ResultSetArrow * rs_obj; - - if (rs == NULL) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->getCellAsInt64(idx, out_data); - } - - SF_STATUS STDCALL rs_arrow_get_cell_as_uint8(rs_arrow_t * rs, size_t idx, uint8 * out_data) - { - Snowflake::Client::ResultSetArrow * rs_obj; - - if (rs == NULL) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->getCellAsUint8(idx, out_data); - } - - SF_STATUS STDCALL rs_arrow_get_cell_as_uint32(rs_arrow_t * rs, size_t idx, uint32 * out_data) - { - Snowflake::Client::ResultSetArrow * rs_obj; - - if (rs == NULL) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->getCellAsUint32(idx, out_data); - } - - SF_STATUS STDCALL rs_arrow_get_cell_as_uint64(rs_arrow_t * rs, size_t idx, uint64 * out_data) - { - Snowflake::Client::ResultSetArrow * rs_obj; - - if (rs == NULL) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->getCellAsUint64(idx, out_data); - } - - SF_STATUS STDCALL rs_arrow_get_cell_as_float32(rs_arrow_t * rs, size_t idx, float32 * out_data) - { - Snowflake::Client::ResultSetArrow * rs_obj; - - if (rs == NULL) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->getCellAsFloat32(idx, out_data); - } - - SF_STATUS STDCALL rs_arrow_get_cell_as_float64(rs_arrow_t * rs, size_t idx, float64 * out_data) - { - Snowflake::Client::ResultSetArrow * rs_obj; - - if (rs == NULL) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->getCellAsFloat64(idx, out_data); - } - - SF_STATUS STDCALL rs_arrow_get_cell_as_const_string( - rs_arrow_t * rs, - size_t idx, - const char ** out_data - ) - { - Snowflake::Client::ResultSetArrow * rs_obj; - - if (rs == NULL) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->getCellAsConstString(idx, out_data); - } - - SF_STATUS STDCALL rs_arrow_get_cell_as_timestamp( - rs_arrow_t * rs, - size_t idx, - SF_TIMESTAMP * out_data - ) - { - Snowflake::Client::ResultSetArrow * rs_obj; - - if (rs == NULL) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->getCellAsTimestamp(idx, out_data); - } - - SF_STATUS STDCALL rs_arrow_get_cell_strlen(rs_arrow_t * rs, size_t idx, size_t * out_data) - { - Snowflake::Client::ResultSetArrow * rs_obj; - - if (rs == NULL) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->getCellStrlen(idx, out_data); - } - - size_t rs_arrow_get_row_count_in_chunk(rs_arrow_t * rs) - { - Snowflake::Client::ResultSetArrow * rs_obj; - - if (rs == NULL) - { - return 0; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->getRowCountInChunk(); - } - - SF_STATUS STDCALL rs_arrow_is_cell_null(rs_arrow_t * rs, size_t idx, sf_bool * out_data) - { - Snowflake::Client::ResultSetArrow * rs_obj; - - if (rs == NULL) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->isCellNull(idx, out_data); - } - - size_t arrow_write_callback(char *ptr, size_t size, size_t nmemb, void *userdata) - { - size_t data_size = size * nmemb; - arrow::BufferBuilder * arrowBufBuilder = (arrow::BufferBuilder*)(userdata); - - log_debug("Curl response for arrow chunk size: %zu", data_size); - (void) arrowBufBuilder->Append(ptr, data_size); - return data_size; - } - - NON_JSON_RESP* callback_create_arrow_resp(void) - { - NON_JSON_RESP* arrow_resp = new NON_JSON_RESP; - arrow_resp->buffer = new arrow::BufferBuilder(); - arrow_resp->write_callback = arrow_write_callback; - return arrow_resp; - } - -#else // SF_WIN32 -rs_arrow_t * rs_arrow_create_with_json_result( - cJSON * json_rowset64, - SF_COLUMN_DESC * metadata, - const char * tz_string -) -{ - log_error("Query results were fetched using Arrow"); - return NULL; -} - -rs_arrow_t * rs_arrow_create_with_chunk( - NON_JSON_RESP * initial_chunk, - SF_COLUMN_DESC * metadata, - const char * tz_string -) -{ - log_error("Query results were fetched using Arrow"); - return NULL; -} - -void rs_arrow_destroy(rs_arrow_t * rs) -{ - log_error("Query results were fetched using Arrow"); -} - -SF_STATUS STDCALL rs_arrow_append_chunk(rs_arrow_t * rs, NON_JSON_RESP * chunk) -{ - log_error("Query results were fetched using Arrow"); - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; -} - -SF_STATUS STDCALL rs_arrow_finish_result_set(rs_arrow_t * rs) -{ - log_error("Query results were fetched using Arrow"); - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; -} - -SF_STATUS STDCALL rs_arrow_next(rs_arrow_t * rs) -{ - log_error("Query results were fetched using Arrow"); - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; -} - -SF_STATUS STDCALL rs_arrow_get_cell_as_bool(rs_arrow_t * rs, size_t idx, sf_bool * out_data) -{ - log_error("Query results were fetched using Arrow"); - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; -} - -SF_STATUS STDCALL rs_arrow_get_cell_as_int8(rs_arrow_t * rs, size_t idx, int8 * out_data) -{ - log_error("Query results were fetched using Arrow"); - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; -} - -SF_STATUS STDCALL rs_arrow_get_cell_as_int32(rs_arrow_t * rs, size_t idx, int32 * out_data) -{ - log_error("Query results were fetched using Arrow"); - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; -} - -SF_STATUS STDCALL rs_arrow_get_cell_as_int64(rs_arrow_t * rs, size_t idx, int64 * out_data) -{ - log_error("Query results were fetched using Arrow"); - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; -} - -SF_STATUS STDCALL rs_arrow_get_cell_as_uint8(rs_arrow_t * rs, size_t idx, uint8 * out_data) -{ - log_error("Query results were fetched using Arrow"); - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; -} - -SF_STATUS STDCALL rs_arrow_get_cell_as_uint32(rs_arrow_t * rs, size_t idx, uint32 * out_data) -{ - log_error("Query results were fetched using Arrow"); - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; -} - -SF_STATUS STDCALL rs_arrow_get_cell_as_uint64(rs_arrow_t * rs, size_t idx, uint64 * out_data) -{ - log_error("Query results were fetched using Arrow"); - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; -} - -SF_STATUS STDCALL rs_arrow_get_cell_as_float32(rs_arrow_t * rs, size_t idx, float32 * out_data) -{ - log_error("Query results were fetched using Arrow"); - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; -} - -SF_STATUS STDCALL rs_arrow_get_cell_as_float64(rs_arrow_t * rs, size_t idx, float64 * out_data) -{ - log_error("Query results were fetched using Arrow"); - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; -} - -SF_STATUS STDCALL rs_arrow_get_cell_as_const_string( - rs_arrow_t * rs, - size_t idx, - const char ** out_data -) -{ - log_error("Query results were fetched using Arrow"); - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; -} - -SF_STATUS STDCALL rs_arrow_get_cell_as_string( - rs_arrow_t * rs, - size_t idx, - char ** out_data, - size_t * io_len, - size_t * io_capacity -) -{ - log_error("Query results were fetched using Arrow"); - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; -} - -SF_STATUS STDCALL rs_arrow_get_cell_as_timestamp( - rs_arrow_t * rs, - size_t idx, - SF_TIMESTAMP * out_data -) -{ - log_error("Query results were fetched using Arrow"); - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; -} - -SF_STATUS STDCALL rs_arrow_get_cell_strlen(rs_arrow_t * rs, size_t idx, size_t * out_data) -{ - log_error("Query results were fetched using Arrow"); - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; -} - -size_t rs_arrow_get_row_count_in_chunk(rs_arrow_t * rs) -{ - log_error("Query results were fetched using Arrow"); - return 0; -} - -size_t rs_arrow_get_total_row_count(rs_arrow_t * rs) -{ - log_error("Query results were fetched using Arrow"); - return 0; -} - -SF_STATUS STDCALL rs_arrow_is_cell_null(rs_arrow_t * rs, size_t idx, sf_bool * out_data) -{ - log_error("Query results were fetched using Arrow"); - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; -} - -size_t arrow_write_callback(char *ptr, size_t size, size_t nmemb, void *userdata) -{ - log_error("Query results were fetched using Arrow"); - return 0; -} - -NON_JSON_RESP* callback_create_arrow_resp(void) -{ - log_error("Query results were fetched using Arrow"); - return NULL; -} - -#endif // SF_WIN32 - -#ifdef __cplusplus -} // extern "C" -#endif diff --git a/cpp/lib/result_set_json.cpp b/cpp/lib/result_set_json.cpp deleted file mode 100644 index b276b865f5..0000000000 --- a/cpp/lib/result_set_json.cpp +++ /dev/null @@ -1,260 +0,0 @@ -/* - * Copyright (c) 2021 Snowflake Computing, Inc. All rights reserved. - */ - -#include - -#include "memory.h" -#include "result_set_json.h" -#include "ResultSetJson.hpp" - - -#ifdef __cplusplus -extern "C" { -#endif - - rs_json_t * rs_json_create( - cJSON * rowset, - SF_COLUMN_DESC * metadata, - const char * tz_string - ) - { - rs_json_t * rs_struct = (rs_json_t *) SF_MALLOC(sizeof(rs_json_t)); - Snowflake::Client::ResultSetJson * rs_obj = new Snowflake::Client::ResultSetJson( - rowset, metadata, std::string(tz_string)); - rs_struct->rs_object = rs_obj; - - return rs_struct; - } - - void rs_json_destroy(rs_json_t * rs) - { - if (rs == NULL) - { - return; - } - - delete static_cast(rs->rs_object); - SF_FREE(rs); - } - - SF_STATUS STDCALL rs_json_append_chunk(rs_json_t * rs, cJSON * chunk) - { - Snowflake::Client::ResultSetJson * rs_obj; - - if (rs == NULL) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->appendChunk(chunk); - } - - SF_STATUS STDCALL rs_json_next(rs_json_t * rs) - { - Snowflake::Client::ResultSetJson * rs_obj; - - if (rs == NULL) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->next(); - } - - SF_STATUS STDCALL rs_json_get_cell_as_bool(rs_json_t * rs, size_t idx, sf_bool * out_data) - { - Snowflake::Client::ResultSetJson * rs_obj; - - if (rs == NULL) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->getCellAsBool(idx, out_data); - } - - SF_STATUS STDCALL rs_json_get_cell_as_int8(rs_json_t * rs, size_t idx, int8 * out_data) - { - Snowflake::Client::ResultSetJson * rs_obj; - - if (rs == NULL) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->getCellAsInt8(idx, out_data); - } - - SF_STATUS STDCALL rs_json_get_cell_as_int32(rs_json_t * rs, size_t idx, int32 * out_data) - { - Snowflake::Client::ResultSetJson * rs_obj; - - if (rs == NULL) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->getCellAsInt32(idx, out_data); - } - - SF_STATUS STDCALL rs_json_get_cell_as_int64(rs_json_t * rs, size_t idx, int64 * out_data) - { - Snowflake::Client::ResultSetJson * rs_obj; - - if (rs == NULL) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->getCellAsInt64(idx, out_data); - } - - SF_STATUS STDCALL rs_json_get_cell_as_uint8(rs_json_t * rs, size_t idx, uint8 * out_data) - { - Snowflake::Client::ResultSetJson * rs_obj; - - if (rs == NULL) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->getCellAsUint8(idx, out_data); - } - - SF_STATUS STDCALL rs_json_get_cell_as_uint32(rs_json_t * rs, size_t idx, uint32 * out_data) - { - Snowflake::Client::ResultSetJson * rs_obj; - - if (rs == NULL) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->getCellAsUint32(idx, out_data); - } - - SF_STATUS STDCALL rs_json_get_cell_as_uint64(rs_json_t * rs, size_t idx, uint64 * out_data) - { - Snowflake::Client::ResultSetJson * rs_obj; - - if (rs == NULL) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->getCellAsUint64(idx, out_data); - } - - SF_STATUS STDCALL rs_json_get_cell_as_float32(rs_json_t * rs, size_t idx, float32 * out_data) - { - Snowflake::Client::ResultSetJson * rs_obj; - - if (rs == NULL) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->getCellAsFloat32(idx, out_data); - } - - SF_STATUS STDCALL rs_json_get_cell_as_float64(rs_json_t * rs, size_t idx, float64 * out_data) - { - Snowflake::Client::ResultSetJson * rs_obj; - - if (rs == NULL) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->getCellAsFloat64(idx, out_data); - } - - SF_STATUS STDCALL rs_json_get_cell_as_const_string( - rs_json_t * rs, - size_t idx, - const char ** out_data - ) - { - Snowflake::Client::ResultSetJson * rs_obj; - - if (rs == NULL) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->getCellAsConstString(idx, out_data); - } - - SF_STATUS STDCALL rs_json_get_cell_as_timestamp( - rs_json_t * rs, - size_t idx, - SF_TIMESTAMP * out_data - ) - { - Snowflake::Client::ResultSetJson * rs_obj; - - if (rs == NULL) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->getCellAsTimestamp(idx, out_data); - } - - - SF_STATUS STDCALL rs_json_get_cell_strlen(rs_json_t * rs, size_t idx, size_t * out_data) - { - Snowflake::Client::ResultSetJson * rs_obj; - - if (rs == NULL) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->getCellStrlen(idx, out_data); - } - - size_t rs_json_get_row_count_in_chunk(rs_json_t * rs) - { - Snowflake::Client::ResultSetJson * rs_obj; - - if (rs == NULL) - { - return 0; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->getRowCountInChunk(); - } - - SF_STATUS STDCALL rs_json_is_cell_null(rs_json_t * rs, size_t idx, sf_bool * out_data) - { - Snowflake::Client::ResultSetJson * rs_obj; - - if (rs == NULL) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->isCellNull(idx, out_data); - } - -#ifdef __cplusplus -} // extern "C" -#endif diff --git a/include/snowflake/client.h b/include/snowflake/client.h index abd332f5be..0ba8bb0568 100644 --- a/include/snowflake/client.h +++ b/include/snowflake/client.h @@ -18,7 +18,7 @@ extern "C" { /** * API Name */ -#define SF_API_NAME "C API" +#define SF_API_NAME "ODBC" /** * SQLState code length @@ -284,8 +284,11 @@ typedef enum SF_GLOBAL_ATTRIBUTE { * Attributes for Snowflake statement context. */ typedef enum SF_STMT_ATTRIBUTE { - SF_STMT_USER_REALLOC_FUNC + SF_STMT_USER_REALLOC_FUNC, + SF_STMT_MULTI_STMT_COUNT } SF_STMT_ATTRIBUTE; +#define SF_MULTI_STMT_COUNT_UNSET (-1) +#define SF_MULTI_STMT_COUNT_UNLIMITED 0 /** * Snowflake Error @@ -442,6 +445,16 @@ typedef struct SF_CHUNK_DOWNLOADER SF_CHUNK_DOWNLOADER; */ typedef struct SF_PUT_GET_RESPONSE SF_PUT_GET_RESPONSE; +typedef void* result_set_ptr; + +/** + * An enumeration over all supported query result formats. + */ +typedef enum QueryResultFormat_e +{ + SF_ARROW_FORMAT, SF_JSON_FORMAT, SF_FORMAT_UNKNOWN +} QueryResultFormat; + /** * Statement context */ @@ -451,9 +464,9 @@ typedef struct SF_STMT { char request_id[SF_UUID4_LEN]; SF_ERROR_STRUCT error; SF_CONNECT *connection; - void *qrf; + QueryResultFormat qrf; char *sql_text; - void *result_set; + result_set_ptr result_set; int64 chunk_rowcount; int64 total_rowcount; int64 total_fieldcount; @@ -465,6 +478,9 @@ typedef struct SF_STMT { SF_STATS *stats; void *stmt_attrs; sf_bool is_dml; + sf_bool is_multi_stmt; + void* multi_stmt_result_ids; + int64 multi_stmt_count; /** * User realloc function used in snowflake_fetch @@ -782,6 +798,15 @@ SF_STATUS STDCALL snowflake_execute_with_capture(SF_STMT *sfstmt, SF_STATUS STDCALL snowflake_describe_with_capture(SF_STMT *sfstmt, SF_QUERY_RESULT_CAPTURE *result_capture); +/** + * Determines whether more results are available and, if so, + * initializes processing for the next one. + * @param sfstmt SNOWFLAKE_STMT context. + * + * @return 0 if success, otherwise an errno is returned. + */ +SF_STATUS STDCALL snowflake_next_result(SF_STMT* sfstmt); + /** * Fetches the next row for the statement and stores on the bound buffer * if any. Noop if no buffer is bound. diff --git a/include/snowflake/version.h b/include/snowflake/version.h index b21a93dd34..83ddc59f55 100644 --- a/include/snowflake/version.h +++ b/include/snowflake/version.h @@ -5,6 +5,7 @@ #ifndef SNOWFLAKE_CLIENT_VERSION_H #define SNOWFLAKE_CLIENT_VERSION_H -#define SF_API_VERSION "1.0.14" +// TODO: temporary change for testing, will remove +#define SF_API_VERSION "3.0.1" #endif /* SNOWFLAKE_CLIENT_VERSION_H */ diff --git a/lib/client.c b/lib/client.c index 90eb215469..505a582991 100644 --- a/lib/client.c +++ b/lib/client.c @@ -66,6 +66,7 @@ sf_bool validate_application(const char *application); #define _SF_STMT_TYPE_DELETE (_SF_STMT_TYPE_DML + 0x300) #define _SF_STMT_TYPE_MERGE (_SF_STMT_TYPE_DML + 0x400) #define _SF_STMT_TYPE_MULTI_TABLE_INSERT (_SF_STMT_TYPE_DML + 0x500) +#define _SF_STMT_TYPE_MULTI_STMT 0xA000 /** * Detects statement type is DML @@ -1319,6 +1320,315 @@ static void STDCALL _snowflake_stmt_row_metadata_reset(SF_STMT *sfstmt) { sfstmt->stats = NULL; } +/** + * Setup result set from json response. + * could be result of a regular query or one of the results + * in multiple statements + */ +static sf_bool setup_result_with_json_resp(SF_STMT* sfstmt, cJSON* data) +{ + // Set Database info + _mutex_lock(&sfstmt->connection->mutex_parameters); + /* Set other parameters. Ignore the status */ + _set_current_objects(sfstmt, data); + _set_parameters_session_info(sfstmt->connection, data); + qcc_deserialize(sfstmt->connection, snowflake_cJSON_GetObjectItem(data, SF_QCC_RSP_KEY)); + _mutex_unlock(&sfstmt->connection->mutex_parameters); + + // clean up from preivous result + sfstmt->chunk_rowcount = -1; + sfstmt->total_rowcount = -1; + sfstmt->total_fieldcount = -1; + sfstmt->total_row_index = -1; + + // Destroy chunk downloader + chunk_downloader_term(sfstmt->chunk_downloader); + sfstmt->chunk_downloader = NULL; + + int64 stmt_type_id; + if (json_copy_int(&stmt_type_id, data, "statementTypeId")) { + /* failed to get statement type id */ + sfstmt->is_dml = SF_BOOLEAN_FALSE; + } else { + sfstmt->is_dml = detect_stmt_type(stmt_type_id); + } + cJSON* rowtype = snowflake_cJSON_GetObjectItem(data, "rowtype"); + if (snowflake_cJSON_IsArray(rowtype)) { + _snowflake_stmt_desc_reset(sfstmt); + sfstmt->total_fieldcount = snowflake_cJSON_GetArraySize( + rowtype); + sfstmt->desc = set_description(rowtype); + } + cJSON* stats = snowflake_cJSON_GetObjectItem(data, "stats"); + if (snowflake_cJSON_IsObject(stats)) { + _snowflake_stmt_row_metadata_reset(sfstmt); + sfstmt->stats = set_stats(stats); + } else { + sfstmt->stats = NULL; + } + + // Determine query result format and detach rowset object from data. + cJSON * qrf = snowflake_cJSON_GetObjectItem(data, "queryResultFormat"); + char * qrf_str = snowflake_cJSON_GetStringValue(qrf); + cJSON * rowset = NULL; + + if (strcmp(qrf_str, "arrow") == 0 || strcmp(qrf_str, "arrow_force") == 0) { +#ifdef SF_WIN32 + SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT, + "Query results were fetched using Arrow, " + "but the client library does not yet support decoding Arrow results", "", + sfstmt->sfqid); + + return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; +#endif + sfstmt->qrf = SF_ARROW_FORMAT; + rowset = snowflake_cJSON_DetachItemFromObject(data, "rowsetBase64"); + if (!rowset) + { + log_error("No valid rowset found in response"); + SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, + SF_STATUS_ERROR_BAD_JSON, + "Missing rowset from response. No results found.", + SF_SQLSTATE_APP_REJECT_CONNECTION, + sfstmt->sfqid); + return SF_BOOLEAN_FALSE; + } + } + else if (strcmp(qrf_str, "json") == 0) { + sfstmt->qrf = SF_JSON_FORMAT; + if (json_detach_array_from_object((cJSON **)(&rowset), data, "rowset")) + { + log_error("No valid rowset found in response"); + SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, + SF_STATUS_ERROR_BAD_JSON, + "Missing rowset from response. No results found.", + SF_SQLSTATE_APP_REJECT_CONNECTION, + sfstmt->sfqid); + return SF_BOOLEAN_FALSE; + } + } + else { + log_error("Unsupported query result format: %s", qrf_str); + return SF_BOOLEAN_FALSE; + } + // Index starts at 0 and incremented each fetch + sfstmt->total_row_index = 0; + cJSON* chunks = NULL; + cJSON* chunk_headers = NULL; + char* qrmk = NULL; + // When the result set is sufficient large, the server response will contain + // an empty "rowset" object. Instead, it will have a "chunks" object that contains, + // among other fields, a URL from which the result set can be downloaded in chunks. + // In this case, we initialize the chunk downloader, which will download in the + // background as calls to snowflake_fetch() are made. + if ((chunks = snowflake_cJSON_GetObjectItem(data, "chunks")) != NULL) { + // We don't care if there is no qrmk, so ignore return code + json_copy_string(&qrmk, data, "qrmk"); + chunk_headers = snowflake_cJSON_GetObjectItem(data, "chunkHeaders"); + NON_JSON_RESP* (*callback_create_resp)(void) = NULL; + if (SF_ARROW_FORMAT == sfstmt->qrf) { + callback_create_resp = callback_create_arrow_resp; + } + sfstmt->chunk_downloader = chunk_downloader_init( + qrmk, + chunk_headers, + chunks, + 2, // thread count + 4, // fetch slot + &sfstmt->error, + sfstmt->connection->insecure_mode, + callback_create_resp, + sfstmt->connection->proxy, + sfstmt->connection->no_proxy, + get_retry_timeout(sfstmt->connection), + sfstmt->connection->retry_count); + SF_FREE(qrmk); + if (!sfstmt->chunk_downloader) { + // Unable to create chunk downloader. + // Error is set in chunk_downloader_init function. + return SF_BOOLEAN_FALSE; + } + // Even when the result set is split into chunks, JSON format will still + // response with the first chunk in "rowset", so be sure to include it. + sfstmt->result_set = rs_create_with_json_result( + rowset, + sfstmt->desc, + sfstmt->qrf, + sfstmt->connection->timezone); + // Update chunk row count. Controls the chunk downloader. + sfstmt->chunk_rowcount = rs_get_row_count_in_chunk( + sfstmt->result_set); + // Update total row count. Used in snowflake_num_rows(). + if (json_copy_int(&sfstmt->total_rowcount, data, "total")) { + log_warn( + "No total count found in response. Reverting to using array size of results"); + sfstmt->total_rowcount = sfstmt->chunk_rowcount; + } + } else { + // Create a result set object and update the total rowcount. + sfstmt->result_set = rs_create_with_json_result( + rowset, + sfstmt->desc, + sfstmt->qrf, + sfstmt->connection->timezone); + // Update chunk row count. Controls the chunk downloader. + sfstmt->chunk_rowcount = rs_get_row_count_in_chunk( + sfstmt->result_set); + // Update total row count. Used in snowflake_num_rows(). + if (json_copy_int(&sfstmt->total_rowcount, data, "total")) { + log_warn( + "No total count found in response. Reverting to using array size of results"); + sfstmt->total_rowcount = sfstmt->chunk_rowcount; + } + } + + return SF_BOOLEAN_TRUE; +} + +/** + * Setup result set from json response. + * could be result of a regular query or one of the results + * in multiple statements + */ +static sf_bool setup_multi_stmt_result(SF_STMT* sfstmt, cJSON* data) +{ + // Set Database info + _mutex_lock(&sfstmt->connection->mutex_parameters); + /* Set other parameters. Ignore the status */ + _set_current_objects(sfstmt, data); + _set_parameters_session_info(sfstmt->connection, data); + qcc_deserialize(sfstmt->connection, snowflake_cJSON_GetObjectItem(data, SF_QCC_RSP_KEY)); + _mutex_unlock(&sfstmt->connection->mutex_parameters); + + if (sfstmt->multi_stmt_result_ids) + { + snowflake_cJSON_Delete(sfstmt->multi_stmt_result_ids); + sfstmt->multi_stmt_result_ids = NULL; + } + char* result_ids = NULL; + if (json_copy_string(&result_ids, data, "resultIds")) + { + log_error("No valid resultIds found in response"); + SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, + SF_STATUS_ERROR_BAD_RESPONSE, + "No valid resultIds found in multiple statements response.", + SF_SQLSTATE_GENERAL_ERROR, + sfstmt->sfqid); + return SF_BOOLEAN_FALSE; + } + + // split result ids with comma(,) + cJSON* result_ids_json = snowflake_cJSON_CreateArray(); + char* start = result_ids; + char* end = NULL; + size_t len = strlen(result_ids); + while ((start - result_ids) < len) + { + end = strchr(start, ','); + if (!end) + { + // last part, set to end of the entire string + end = result_ids + len; + } + *end = '\0'; + snowflake_cJSON_AddItemToArray(result_ids_json, snowflake_cJSON_CreateString(start)); + start = end + 1; + } + + sfstmt->multi_stmt_result_ids = result_ids_json; + + return SF_STATUS_SUCCESS == snowflake_next_result(sfstmt); +} + +SF_STATUS STDCALL snowflake_next_result(SF_STMT* sfstmt) +{ + cJSON* result_id_json = NULL; + if (!sfstmt || !sfstmt->is_multi_stmt || !sfstmt->multi_stmt_result_ids || + !snowflake_cJSON_IsArray(sfstmt->multi_stmt_result_ids) || + !(result_id_json = snowflake_cJSON_DetachItemFromArray(sfstmt->multi_stmt_result_ids, 0))) + { + // no more results available. + return SF_STATUS_EOF; + } + + char* result_id = snowflake_cJSON_GetStringValue(result_id_json); + if (!result_id || (strlen(result_id) == 0)) + { + log_error("Empty result id found for multiple statements."); + SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, + SF_STATUS_ERROR_BAD_RESPONSE, + "Empty result id found in multiple statements response.", + SF_SQLSTATE_GENERAL_ERROR, + sfstmt->sfqid); + snowflake_cJSON_Delete(result_id_json); + return SF_STATUS_ERROR_BAD_RESPONSE; + } + + char* result_url = NULL; + size_t url_size = strlen(QUERY_RESULT_URL_FORMAT) - 2 + strlen(result_id) + 1; + result_url = (char*)SF_CALLOC(1, url_size); + if (!result_url) + { + SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, + SF_STATUS_ERROR_OUT_OF_MEMORY, + "Run out of memory trying to create result url.", + SF_SQLSTATE_MEMORY_ALLOCATION_ERROR, + sfstmt->sfqid); + snowflake_cJSON_Delete(result_id_json); + return SF_STATUS_ERROR_OUT_OF_MEMORY; + } + sf_sprintf(result_url, url_size, QUERY_RESULT_URL_FORMAT, result_id); + snowflake_cJSON_Delete(result_id_json); + + cJSON* result = NULL; + if (!request(sfstmt->connection, &result, result_url, NULL, 0, NULL, NULL, + GET_REQUEST_TYPE, &sfstmt->error, SF_BOOLEAN_FALSE, + 0, sfstmt->connection->retry_count, get_retry_timeout(sfstmt->connection), + NULL, NULL, NULL, SF_BOOLEAN_FALSE)) + { + SF_FREE(result_url); + return sfstmt->error.error_code; + } + SF_FREE(result_url); + + cJSON* data = snowflake_cJSON_GetObjectItem(result, "data"); + + sf_bool success = SF_BOOLEAN_FALSE; + if ((json_copy_bool(&success, result, "success") != SF_JSON_ERROR_NONE) || !success) + { + cJSON *messageJson = NULL; + char *message = NULL; + cJSON *codeJson = NULL; + int64 code = -1; + if (json_copy_string_no_alloc(sfstmt->error.sqlstate, data, + "sqlState", SF_SQLSTATE_LEN)) { + log_debug("No valid sqlstate found in response"); + } + messageJson = snowflake_cJSON_GetObjectItem(result, "message"); + if (messageJson) { + message = messageJson->valuestring; + } + codeJson = snowflake_cJSON_GetObjectItem(result, "code"); + if (codeJson) { + code = (int64) strtol(codeJson->valuestring, NULL, 10); + } else { + log_debug("no code element."); + } + SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, code, + message ? message + : "Query was not successful", + NULL, sfstmt->sfqid); + + snowflake_cJSON_Delete(result); + return sfstmt->error.error_code; + } + + setup_result_with_json_resp(sfstmt, data); + snowflake_cJSON_Delete(result); + + return SF_STATUS_SUCCESS; +} + /** * Returns what kind of params are being bound - Named / Positional * based on the first bind input entry. Should be used only if @@ -1443,14 +1753,18 @@ static void STDCALL _snowflake_stmt_reset(SF_STMT *sfstmt) { sfstmt->sql_text = NULL; if (sfstmt->result_set) { - rs_destroy(sfstmt->result_set, (QueryResultFormat_t *) sfstmt->qrf); + rs_destroy(sfstmt->result_set); } sfstmt->result_set = NULL; - if (sfstmt->qrf) { - SF_FREE(sfstmt->qrf); + sfstmt->qrf = SF_FORMAT_UNKNOWN; + sfstmt->is_dml = SF_BOOLEAN_FALSE; + sfstmt->is_multi_stmt = SF_BOOLEAN_FALSE; + if (sfstmt->multi_stmt_result_ids) + { + snowflake_cJSON_Delete(sfstmt->multi_stmt_result_ids); } - sfstmt->qrf = NULL; + sfstmt->multi_stmt_result_ids = NULL; if (_snowflake_get_current_param_style(sfstmt) == NAMED) { @@ -1541,7 +1855,7 @@ SF_STMT *STDCALL snowflake_stmt(SF_CONNECT *sf) { if (sfstmt) { _snowflake_stmt_reset(sfstmt); sfstmt->connection = sf; - + sfstmt->multi_stmt_count = SF_MULTI_STMT_COUNT_UNSET; } return sfstmt; } @@ -1771,12 +2085,11 @@ SF_STATUS STDCALL snowflake_fetch(SF_STMT *sfstmt) { sfstmt->result_set = rs_create_with_chunk( sfstmt->chunk_downloader->queue[index].chunk, sfstmt->desc, - (QueryResultFormat_t *) sfstmt->qrf, + sfstmt->qrf, sfstmt->connection->timezone); } else { rs_append_chunk( sfstmt->result_set, - (QueryResultFormat_t *) sfstmt->qrf, sfstmt->chunk_downloader->queue[index].chunk); } @@ -1948,12 +2261,7 @@ SF_STATUS STDCALL _snowflake_execute_ex(SF_STMT *sfstmt, const char *error_msg; cJSON *body = NULL; cJSON *data = NULL; - cJSON *rowtype = NULL; - cJSON *stats = NULL; cJSON *resp = NULL; - cJSON *chunks = NULL; - cJSON *chunk_headers = NULL; - char *qrmk = NULL; char *s_body = NULL; char *s_resp = NULL; sf_bool success = SF_BOOLEAN_FALSE; @@ -2042,7 +2350,8 @@ SF_STATUS STDCALL _snowflake_execute_ex(SF_STMT *sfstmt, // Create Body body = create_query_json_body(sfstmt->sql_text, sfstmt->sequence_counter, is_string_empty(sfstmt->connection->directURL) ? - NULL : sfstmt->request_id, is_describe_only); + NULL : sfstmt->request_id, is_describe_only, + sfstmt->multi_stmt_count); if (bindings != NULL) { /* binding parameters if exists */ snowflake_cJSON_AddItemToObject(body, "bindings", bindings); @@ -2175,155 +2484,29 @@ SF_STATUS STDCALL _snowflake_execute_ex(SF_STMT *sfstmt, "localLocation"); } else { - // Set Database info - _mutex_lock(&sfstmt->connection->mutex_parameters); - /* Set other parameters. Ignore the status */ - _set_current_objects(sfstmt, data); - _set_parameters_session_info(sfstmt->connection, data); - qcc_deserialize(sfstmt->connection, snowflake_cJSON_GetObjectItem(data, SF_QCC_RSP_KEY)); - _mutex_unlock(&sfstmt->connection->mutex_parameters); int64 stmt_type_id; if (json_copy_int(&stmt_type_id, data, "statementTypeId")) { - /* failed to get statement type id */ - sfstmt->is_dml = SF_BOOLEAN_FALSE; - } else { - sfstmt->is_dml = detect_stmt_type(stmt_type_id); - } - rowtype = snowflake_cJSON_GetObjectItem(data, "rowtype"); - if (snowflake_cJSON_IsArray(rowtype)) { - sfstmt->total_fieldcount = snowflake_cJSON_GetArraySize( - rowtype); - _snowflake_stmt_desc_reset(sfstmt); - sfstmt->desc = set_description(rowtype); + /* failed to get statement type id */ + sfstmt->is_multi_stmt = SF_BOOLEAN_FALSE; } - stats = snowflake_cJSON_GetObjectItem(data, "stats"); - if (snowflake_cJSON_IsObject(stats)) { - _snowflake_stmt_row_metadata_reset(sfstmt); - sfstmt->stats = set_stats(stats); - } else { - sfstmt->stats = NULL; + else { + sfstmt->is_multi_stmt = (_SF_STMT_TYPE_MULTI_STMT == stmt_type_id); } - // Determine query result format and detach rowset object from data. - cJSON * qrf = snowflake_cJSON_GetObjectItem(data, "queryResultFormat"); - char * qrf_str = snowflake_cJSON_GetStringValue(qrf); - sfstmt->qrf = SF_CALLOC(1, sizeof(QueryResultFormat_t)); - cJSON * rowset = NULL; - - if (strcmp(qrf_str, "arrow") == 0 || strcmp(qrf_str, "arrow_force") == 0) { -#ifdef SF_WIN32 - SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT, - "Query results were fetched using Arrow, " - "but the client library does not yet support decoding Arrow results", "", - sfstmt->sfqid); - - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; -#endif - *((QueryResultFormat_t *) sfstmt->qrf) = ARROW_FORMAT; - rowset = snowflake_cJSON_DetachItemFromObject(data, "rowsetBase64"); - if (!rowset) + if (sfstmt->is_multi_stmt) + { + if (!setup_multi_stmt_result(sfstmt, data)) { - log_error("No valid rowset found in response"); - SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, - SF_STATUS_ERROR_BAD_JSON, - "Missing rowset from response. No results found.", - SF_SQLSTATE_APP_REJECT_CONNECTION, - sfstmt->sfqid); goto cleanup; } } - else if (strcmp(qrf_str, "json") == 0) { - *((QueryResultFormat_t *) sfstmt->qrf) = JSON_FORMAT; - if (json_detach_array_from_object((cJSON **)(&rowset), data, "rowset")) + else + { + if (!setup_result_with_json_resp(sfstmt, data)) { - log_error("No valid rowset found in response"); - SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, - SF_STATUS_ERROR_BAD_JSON, - "Missing rowset from response. No results found.", - SF_SQLSTATE_APP_REJECT_CONNECTION, - sfstmt->sfqid); goto cleanup; } } - else { - log_error("Unsupported query result format: %s", qrf_str); - } - - // Index starts at 0 and incremented each fetch - sfstmt->total_row_index = 0; - - // When the result set is sufficient large, the server response will contain - // an empty "rowset" object. Instead, it will have a "chunks" object that contains, - // among other fields, a URL from which the result set can be downloaded in chunks. - // In this case, we initialize the chunk downloader, which will download in the - // background as calls to snowflake_fetch() are made. - if ((chunks = snowflake_cJSON_GetObjectItem(data, "chunks")) != NULL) { - // We don't care if there is no qrmk, so ignore return code - json_copy_string(&qrmk, data, "qrmk"); - chunk_headers = snowflake_cJSON_GetObjectItem(data, "chunkHeaders"); - NON_JSON_RESP* (*callback_create_resp)(void) = NULL; - if (ARROW_FORMAT == *((QueryResultFormat_t *)sfstmt->qrf)) { - callback_create_resp = callback_create_arrow_resp; - } - - sfstmt->chunk_downloader = chunk_downloader_init( - qrmk, - chunk_headers, - chunks, - 2, // thread count - 4, // fetch slot - &sfstmt->error, - sfstmt->connection->insecure_mode, - callback_create_resp, - sfstmt->connection->proxy, - sfstmt->connection->no_proxy, - get_retry_timeout(sfstmt->connection), - sfstmt->connection->retry_count); - if (!sfstmt->chunk_downloader) { - // Unable to create chunk downloader. - // Error is set in chunk_downloader_init function. - goto cleanup; - } - - // Even when the result set is split into chunks, JSON format will still - // response with the first chunk in "rowset", so be sure to include it. - sfstmt->result_set = rs_create_with_json_result( - rowset, - sfstmt->desc, - (QueryResultFormat_t *)sfstmt->qrf, - sfstmt->connection->timezone); - - // Update chunk row count. Controls the chunk downloader. - sfstmt->chunk_rowcount = rs_get_row_count_in_chunk( - sfstmt->result_set, - (QueryResultFormat_t *) sfstmt->qrf); - - // Update total row count. Used in snowflake_num_rows(). - if (json_copy_int(&sfstmt->total_rowcount, data, "total")) { - log_warn( - "No total count found in response. Reverting to using array size of results"); - sfstmt->total_rowcount = sfstmt->chunk_rowcount; - } - } else { - // Create a result set object and update the total rowcount. - sfstmt->result_set = rs_create_with_json_result( - rowset, - sfstmt->desc, - (QueryResultFormat_t *) sfstmt->qrf, - sfstmt->connection->timezone); - - // Update chunk row count. Controls the chunk downloader. - sfstmt->chunk_rowcount = rs_get_row_count_in_chunk( - sfstmt->result_set, - (QueryResultFormat_t *) sfstmt->qrf); - - // Update total row count. Used in snowflake_num_rows(). - if (json_copy_int(&sfstmt->total_rowcount, data, "total")) { - log_warn( - "No total count found in response. Reverting to using array size of results"); - sfstmt->total_rowcount = sfstmt->chunk_rowcount; - } - } } } else if (json_error != SF_JSON_ERROR_NONE) { JSON_ERROR_MSG(json_error, error_msg, "Success code"); @@ -2371,7 +2554,6 @@ SF_STATUS STDCALL _snowflake_execute_ex(SF_STMT *sfstmt, snowflake_cJSON_Delete(body); snowflake_cJSON_Delete(resp); SF_FREE(s_body); - SF_FREE(qrmk); if (result_capture == NULL) { // If no result capture, we always free s_resp SF_FREE(s_resp); @@ -2451,6 +2633,9 @@ SF_STATUS STDCALL snowflake_stmt_get_attr( case SF_STMT_USER_REALLOC_FUNC: *value = sfstmt->user_realloc_func; break; + case SF_STMT_MULTI_STMT_COUNT: + *value = &sfstmt->multi_stmt_count; + break; default: SET_SNOWFLAKE_ERROR( &sfstmt->error, SF_STATUS_ERROR_BAD_ATTRIBUTE_TYPE, @@ -2471,6 +2656,9 @@ SF_STATUS STDCALL snowflake_stmt_set_attr( case SF_STMT_USER_REALLOC_FUNC: sfstmt->user_realloc_func = (void*(*)(void*, size_t))value; break; + case SF_STMT_MULTI_STMT_COUNT: + sfstmt->multi_stmt_count = value ? *((int64*)value) : SF_MULTI_STMT_COUNT_UNSET; + break; default: SET_SNOWFLAKE_ERROR( &sfstmt->error, SF_STATUS_ERROR_BAD_ATTRIBUTE_TYPE, @@ -2525,7 +2713,7 @@ SF_STATUS STDCALL _snowflake_column_null_checks(SF_STMT *sfstmt, void *value_ptr } SF_STATUS STDCALL _snowflake_next(SF_STMT *sfstmt) { - return rs_next(sfstmt->result_set, (QueryResultFormat_t *) sfstmt->qrf); + return rs_next(sfstmt->result_set); } SF_STATUS STDCALL snowflake_column_as_boolean(SF_STMT *sfstmt, int idx, sf_bool *value_ptr) { @@ -2536,9 +2724,9 @@ SF_STATUS STDCALL snowflake_column_as_boolean(SF_STMT *sfstmt, int idx, sf_bool } if ((status = rs_get_cell_as_bool( - sfstmt->result_set, sfstmt->qrf, idx, value_ptr)) != SF_STATUS_SUCCESS) { + sfstmt->result_set, idx, value_ptr)) != SF_STATUS_SUCCESS) { SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, status, - rs_get_error_message(sfstmt->result_set, sfstmt->qrf), "", sfstmt->sfqid); + rs_get_error_message(sfstmt->result_set), "", sfstmt->sfqid); } return status; } @@ -2551,9 +2739,9 @@ SF_STATUS STDCALL snowflake_column_as_uint8(SF_STMT *sfstmt, int idx, uint8 *val } if ((status = rs_get_cell_as_uint8( - sfstmt->result_set, sfstmt->qrf, idx, value_ptr)) != SF_STATUS_SUCCESS) { + sfstmt->result_set, idx, value_ptr)) != SF_STATUS_SUCCESS) { SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, status, - rs_get_error_message(sfstmt->result_set, sfstmt->qrf), "", sfstmt->sfqid); + rs_get_error_message(sfstmt->result_set), "", sfstmt->sfqid); } return status; } @@ -2566,9 +2754,9 @@ SF_STATUS STDCALL snowflake_column_as_uint32(SF_STMT *sfstmt, int idx, uint32 *v } if ((status = rs_get_cell_as_uint32( - sfstmt->result_set, sfstmt->qrf, idx, value_ptr)) != SF_STATUS_SUCCESS) { + sfstmt->result_set, idx, value_ptr)) != SF_STATUS_SUCCESS) { SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, status, - rs_get_error_message(sfstmt->result_set, sfstmt->qrf), "", sfstmt->sfqid); + rs_get_error_message(sfstmt->result_set), "", sfstmt->sfqid); } return status; } @@ -2581,9 +2769,9 @@ SF_STATUS STDCALL snowflake_column_as_uint64(SF_STMT *sfstmt, int idx, uint64 *v } if ((status = rs_get_cell_as_uint64( - sfstmt->result_set, sfstmt->qrf, idx, value_ptr)) != SF_STATUS_SUCCESS) { + sfstmt->result_set, idx, value_ptr)) != SF_STATUS_SUCCESS) { SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, status, - rs_get_error_message(sfstmt->result_set, sfstmt->qrf), "", sfstmt->sfqid); + rs_get_error_message(sfstmt->result_set), "", sfstmt->sfqid); } return status; } @@ -2596,9 +2784,9 @@ SF_STATUS STDCALL snowflake_column_as_int8(SF_STMT *sfstmt, int idx, int8 *value } if ((status = rs_get_cell_as_int8( - sfstmt->result_set, sfstmt->qrf, idx, value_ptr)) != SF_STATUS_SUCCESS) { + sfstmt->result_set, idx, value_ptr)) != SF_STATUS_SUCCESS) { SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, status, - rs_get_error_message(sfstmt->result_set, sfstmt->qrf), "", sfstmt->sfqid); + rs_get_error_message(sfstmt->result_set), "", sfstmt->sfqid); } return status; } @@ -2611,9 +2799,9 @@ SF_STATUS STDCALL snowflake_column_as_int32(SF_STMT *sfstmt, int idx, int32 *val } if ((status = rs_get_cell_as_int32( - sfstmt->result_set, sfstmt->qrf, idx, value_ptr)) != SF_STATUS_SUCCESS) { + sfstmt->result_set, idx, value_ptr)) != SF_STATUS_SUCCESS) { SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, status, - rs_get_error_message(sfstmt->result_set, sfstmt->qrf), "", sfstmt->sfqid); + rs_get_error_message(sfstmt->result_set), "", sfstmt->sfqid); } return status; } @@ -2626,9 +2814,9 @@ SF_STATUS STDCALL snowflake_column_as_int64(SF_STMT *sfstmt, int idx, int64 *val } if ((status = rs_get_cell_as_int64( - sfstmt->result_set, sfstmt->qrf, idx, value_ptr)) != SF_STATUS_SUCCESS) { + sfstmt->result_set, idx, value_ptr)) != SF_STATUS_SUCCESS) { SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, status, - rs_get_error_message(sfstmt->result_set, sfstmt->qrf), "", sfstmt->sfqid); + rs_get_error_message(sfstmt->result_set), "", sfstmt->sfqid); } return status; } @@ -2641,9 +2829,9 @@ SF_STATUS STDCALL snowflake_column_as_float32(SF_STMT *sfstmt, int idx, float32 } if ((status = rs_get_cell_as_float32( - sfstmt->result_set, sfstmt->qrf, idx, value_ptr)) != SF_STATUS_SUCCESS) { + sfstmt->result_set, idx, value_ptr)) != SF_STATUS_SUCCESS) { SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, status, - rs_get_error_message(sfstmt->result_set, sfstmt->qrf), "", sfstmt->sfqid); + rs_get_error_message(sfstmt->result_set), "", sfstmt->sfqid); } return status; } @@ -2656,9 +2844,9 @@ SF_STATUS STDCALL snowflake_column_as_float64(SF_STMT *sfstmt, int idx, float64 } if ((status = rs_get_cell_as_float64( - sfstmt->result_set, sfstmt->qrf, idx, value_ptr)) != SF_STATUS_SUCCESS) { + sfstmt->result_set, idx, value_ptr)) != SF_STATUS_SUCCESS) { SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, status, - rs_get_error_message(sfstmt->result_set, sfstmt->qrf), "", sfstmt->sfqid); + rs_get_error_message(sfstmt->result_set), "", sfstmt->sfqid); } return status; } @@ -2671,9 +2859,9 @@ SF_STATUS STDCALL snowflake_column_as_timestamp(SF_STMT *sfstmt, int idx, SF_TIM } if ((status = rs_get_cell_as_timestamp( - sfstmt->result_set, sfstmt->qrf, idx, value_ptr)) != SF_STATUS_SUCCESS) { + sfstmt->result_set, idx, value_ptr)) != SF_STATUS_SUCCESS) { SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, status, - rs_get_error_message(sfstmt->result_set, sfstmt->qrf), "", sfstmt->sfqid); + rs_get_error_message(sfstmt->result_set), "", sfstmt->sfqid); } return status; } @@ -2686,9 +2874,9 @@ SF_STATUS STDCALL snowflake_column_as_const_str(SF_STMT *sfstmt, int idx, const } if ((status = rs_get_cell_as_const_string( - sfstmt->result_set, sfstmt->qrf, idx, value_ptr)) != SF_STATUS_SUCCESS) { + sfstmt->result_set, idx, value_ptr)) != SF_STATUS_SUCCESS) { SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, status, - rs_get_error_message(sfstmt->result_set, sfstmt->qrf), "", sfstmt->sfqid); + rs_get_error_message(sfstmt->result_set), "", sfstmt->sfqid); } return status; } @@ -2909,16 +3097,15 @@ SF_STATUS STDCALL snowflake_column_as_str(SF_STMT *sfstmt, int idx, char **value const char* str_val = NULL; if ((status = rs_get_cell_as_const_string( sfstmt->result_set, - sfstmt->qrf, idx, &str_val)) != SF_STATUS_SUCCESS) { SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, status, - rs_get_error_message(sfstmt->result_set, sfstmt->qrf), "", sfstmt->sfqid); + rs_get_error_message(sfstmt->result_set), "", sfstmt->sfqid); return status; } - if (ARROW_FORMAT == *((QueryResultFormat_t *)sfstmt->qrf)) + if (SF_ARROW_FORMAT == sfstmt->qrf) { // For Arrow the const string is formatted already return snowflake_raw_value_to_str_rep(sfstmt, str_val, @@ -2949,9 +3136,9 @@ SF_STATUS STDCALL snowflake_column_strlen(SF_STMT *sfstmt, int idx, size_t *valu } if ((status = rs_get_cell_strlen( - sfstmt->result_set, sfstmt->qrf, idx, value_ptr)) != SF_STATUS_SUCCESS) { + sfstmt->result_set, idx, value_ptr)) != SF_STATUS_SUCCESS) { SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, status, - rs_get_error_message(sfstmt->result_set, sfstmt->qrf), "", sfstmt->sfqid); + rs_get_error_message(sfstmt->result_set), "", sfstmt->sfqid); } return status; } @@ -2964,9 +3151,9 @@ SF_STATUS STDCALL snowflake_column_is_null(SF_STMT *sfstmt, int idx, sf_bool *va } if ((status = rs_is_cell_null( - sfstmt->result_set, sfstmt->qrf, idx, value_ptr)) != SF_STATUS_SUCCESS) { + sfstmt->result_set, idx, value_ptr)) != SF_STATUS_SUCCESS) { SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, status, - rs_get_error_message(sfstmt->result_set, sfstmt->qrf), "", sfstmt->sfqid); + rs_get_error_message(sfstmt->result_set), "", sfstmt->sfqid); } return status; } diff --git a/lib/client_int.h b/lib/client_int.h index ed6b46c5a2..e8b111cd68 100644 --- a/lib/client_int.h +++ b/lib/client_int.h @@ -29,6 +29,7 @@ #define QUERY_URL "/queries/v1/query-request" #define RENEW_SESSION_URL "/session/token-request" #define DELETE_SESSION_URL "/session" +#define QUERY_RESULT_URL_FORMAT "/queries/%s/result" // not used for now but add for URL checking on connection requests #define AUTHENTICATOR_URL "/session/authenticator-request" diff --git a/lib/connection.c b/lib/connection.c index 64f9578c83..3c9d5f2570 100644 --- a/lib/connection.c +++ b/lib/connection.c @@ -201,7 +201,12 @@ cJSON *STDCALL create_auth_json_body(SF_CONNECT *sf, return body; } -cJSON *STDCALL create_query_json_body(const char *sql_text, int64 sequence_id, const char *request_id, sf_bool is_describe_only) { +cJSON *STDCALL create_query_json_body(const char *sql_text, + int64 sequence_id, + const char *request_id, + sf_bool is_describe_only, + int64 multi_stmt_count) +{ cJSON *body; double submission_time; // Create body @@ -221,11 +226,27 @@ cJSON *STDCALL create_query_json_body(const char *sql_text, int64 sequence_id, c snowflake_cJSON_AddStringToObject(body, "requestId", request_id); } + cJSON* parameters = NULL; + if (multi_stmt_count >= 0) + { + parameters = snowflake_cJSON_CreateObject(); + snowflake_cJSON_AddNumberToObject(parameters, "MULTI_STATEMENT_COUNT", (double)multi_stmt_count); + } + #ifdef SF_WIN32 - cJSON * parameters = snowflake_cJSON_CreateObject(); + if (!parameters) + { + parameters = snowflake_cJSON_CreateObject(); + } snowflake_cJSON_AddStringToObject(parameters, "C_API_QUERY_RESULT_FORMAT", "JSON"); - snowflake_cJSON_AddItemToObject(body, "parameters", parameters); + + // temporary code to fake as ODBC to have multiple statements enabled + snowflake_cJSON_AddStringToObject(parameters, "ODBC_QUERY_RESULT_FORMAT", "JSON"); #endif + if (parameters) + { + snowflake_cJSON_AddItemToObject(body, "parameters", parameters); + } return body; } diff --git a/lib/connection.h b/lib/connection.h index af56504750..7dfcfb4688 100644 --- a/lib/connection.h +++ b/lib/connection.h @@ -201,9 +201,14 @@ cJSON *STDCALL create_auth_json_body(SF_CONNECT *sf, const char *application, co * @param sequence_id Sequence ID from the Snowflake Connection object. * @param request_id requestId to be passed as a part of body instead of header. * @param is_describe_only is the query describe only. + * @param multi_stmt_count The value of MULTI_STATEMENT_COUNT set with the query. No setting if < 0. * @return Query cJSON Body. */ -cJSON *STDCALL create_query_json_body(const char *sql_text, int64 sequence_id, const char *request_id, sf_bool is_describe_only); +cJSON *STDCALL create_query_json_body(const char *sql_text, + int64 sequence_id, + const char *request_id, + sf_bool is_describe_only, + int64 multi_stmt_count); /** * Creates a cJSON blob that is used to renew a session with Snowflake. cJSON blob must be freed by the caller using diff --git a/lib/result_set.h b/lib/result_set.h index 865d8eba69..c5faac33ff 100644 --- a/lib/result_set.h +++ b/lib/result_set.h @@ -6,26 +6,12 @@ #define SNOWFLAKE_RESULTSET_H #include "snowflake/client.h" -#include "result_set_arrow.h" -#include "result_set_json.h" +#include "connection.h" #ifdef __cplusplus extern "C" { #endif - // Utility ===================================================================================== - - /** - * An enumeration over all supported query result formats. - * This is used to help deciding which result set to create. - * - * NOTE: Keep the order consistent with rowset_key_map! - */ - typedef enum QueryResultFormat - { - ARROW_FORMAT, JSON_FORMAT, FORMAT_MAX - } QueryResultFormat_t; - // Result Set API ============================================================================== /** @@ -39,10 +25,10 @@ extern "C" { * * @return the created result set. */ - void * rs_create_with_json_result( + result_set_ptr rs_create_with_json_result( cJSON * json_rowset, SF_COLUMN_DESC * metadata, - QueryResultFormat_t * query_result_format, + QueryResultFormat query_result_format, const char * tz_string); /** @@ -56,55 +42,50 @@ extern "C" { * * @return the created result set. */ - void * rs_create_with_chunk( + result_set_ptr rs_create_with_chunk( void * initial_chunk, SF_COLUMN_DESC * metadata, - QueryResultFormat_t * query_result_format, + QueryResultFormat query_result_format, const char * tz_string); /** * Destructor. * * @param rs The ResultSet object. - * @param query_result_format The query result format. */ - void rs_destroy(void * rs, QueryResultFormat_t * query_result_format); + void rs_destroy(result_set_ptr rs); /** * Appends the given chunk to the internal result set. * * @param rs The ResultSet object. - * @param query_result_format The query result format. * @param chunk The chunk to append. * * @return 0 if successful, otherwise an error is returned. */ SF_STATUS STDCALL - rs_append_chunk(void * rs, QueryResultFormat_t * query_result_format, void * chunk); + rs_append_chunk(result_set_ptr rs, void * chunk); /** * Advances to the next row. * * @param rs The ResultSet object. - * @param query_result_format The query result format. * * @return 0 if successful, otherwise an error is returned. */ - SF_STATUS STDCALL rs_next(void * rs, QueryResultFormat_t * query_result_format); + SF_STATUS STDCALL rs_next(result_set_ptr rs); /** * Writes the value of the current cell as a boolean to the provided buffer. * * @param rs The ResultSet object. - * @param query_result_format The query result format. * @param idx The index of the column or row to retrieve. * @param out_data The buffer to write to. * * @return 0 if successful, otherwise an error is returned. */ SF_STATUS STDCALL rs_get_cell_as_bool( - void * rs, - QueryResultFormat_t * query_result_format, + result_set_ptr rs, size_t idx, sf_bool * out_data); @@ -112,15 +93,13 @@ extern "C" { * Writes the value of the current cell as an int8 to the provided buffer. * * @param rs The ResultSet object. - * @param query_result_format The query result format. * @param idx The index of the column or row to retrieve. * @param out_data The buffer to write to. * * @return 0 if successful, otherwise an error is returned. */ SF_STATUS STDCALL rs_get_cell_as_int8( - void * rs, - QueryResultFormat_t * query_result_format, + result_set_ptr rs, size_t idx, int8 * out_data); @@ -128,15 +107,13 @@ extern "C" { * Writes the value of the current cell as an int32 to the provided buffer. * * @param rs The ResultSet object. - * @param query_result_format The query result format. * @param idx The index of the column or row to retrieve. * @param out_data The buffer to write to. * * @return 0 if successful, otherwise an error is returned. */ SF_STATUS STDCALL rs_get_cell_as_int32( - void * rs, - QueryResultFormat_t * query_result_format, + result_set_ptr rs, size_t idx, int32 * out_data); @@ -144,15 +121,13 @@ extern "C" { * Writes the value of the current cell as an int64 to the provided buffer. * * @param rs The ResultSet object. - * @param query_result_format The query result format. * @param idx The index of the column or row to retrieve. * @param out_data The buffer to write to. * * @return 0 if successful, otherwise an error is returned. */ SF_STATUS STDCALL rs_get_cell_as_int64( - void * rs, - QueryResultFormat_t * query_result_format, + result_set_ptr rs, size_t idx, int64 * out_data); @@ -160,15 +135,13 @@ extern "C" { * Writes the value of the current cell as a uint8 to the provided buffer. * * @param rs The ResultSet object. - * @param query_result_format The query result format. * @param idx The index of the column or row to retrieve. * @param out_data The buffer to write to. * * @return 0 if successful, otherwise an error is returned. */ SF_STATUS STDCALL rs_get_cell_as_uint8( - void * rs, - QueryResultFormat_t * query_result_format, + result_set_ptr rs, size_t idx, uint8 * out_data); @@ -176,15 +149,13 @@ extern "C" { * Writes the value of the current cell as a uint32 to the provided buffer. * * @param rs The ResultSet object. - * @param query_result_format The query result format. * @param idx The index of the column or row to retrieve. * @param out_data The buffer to write to. * * @return 0 if successful, otherwise an error is returned. */ SF_STATUS STDCALL rs_get_cell_as_uint32( - void * rs, - QueryResultFormat_t * query_result_format, + result_set_ptr rs, size_t idx, uint32 * out_data); @@ -192,15 +163,13 @@ extern "C" { * Writes the value of the current cell as a uint64 to the provided buffer. * * @param rs The ResultSet object. - * @param query_result_format The query result format. * @param idx The index of the column or row to retrieve. * @param out_data The buffer to write to. * * @return 0 if successful, otherwise an error is returned. */ SF_STATUS STDCALL rs_get_cell_as_uint64( - void * rs, - QueryResultFormat_t * query_result_format, + result_set_ptr rs, size_t idx, uint64 * out_data); @@ -208,15 +177,13 @@ extern "C" { * Writes the value of the current cell as a float32 to the provided buffer. * * @param rs The ResultSet object. - * @param query_result_format The query result format. * @param idx The index of the column or row to retrieve. * @param out_data The buffer to write to. * * @return 0 if successful, otherwise an error is returned. */ SF_STATUS STDCALL rs_get_cell_as_float32( - void * rs, - QueryResultFormat_t * query_result_format, + result_set_ptr rs, size_t idx, float32 * out_data); @@ -224,15 +191,13 @@ extern "C" { * Writes the value of the current cell as a float64 to the provided buffer. * * @param rs The ResultSet object. - * @param query_result_format The query result format. * @param idx The index of the column or row to retrieve. * @param out_data The buffer to write to. * * @return 0 if successful, otherwise an error is returned. */ SF_STATUS STDCALL rs_get_cell_as_float64( - void * rs, - QueryResultFormat_t * query_result_format, + result_set_ptr rs, size_t idx, float64 * out_data); @@ -240,15 +205,13 @@ extern "C" { * Writes the value of the current cell as a constant C-string to the provided buffer. * * @param rs The ResultSet object. - * @param query_result_format The query result format. * @param idx The index of the column or row to retrieve. * @param out_data The buffer to write to. * * @return 0 if successful, otherwise an error is returned. */ SF_STATUS STDCALL rs_get_cell_as_const_string( - void * rs, - QueryResultFormat_t * query_result_format, + result_set_ptr rs, size_t idx, const char ** out_data); @@ -256,15 +219,13 @@ extern "C" { * Writes the value of the current cell as a timestamp to the provided buffer. * * @param rs The ResultSet object. - * @param query_result_format The query result format. * @param idx The index of the column or row to retrieve. * @param out_data The buffer to write to. * * @return 0 if successful, otherwise an error is returned. */ SF_STATUS STDCALL rs_get_cell_as_timestamp( - void * rs, - QueryResultFormat_t * query_result_format, + result_set_ptr rs, size_t idx, SF_TIMESTAMP * out_data); @@ -272,15 +233,13 @@ extern "C" { * Writes the length of the current cell to the provided buffer. * * @param rs The ResultSet object. - * @param query_result_format The query result format. * @param idx The index of the column or row to retrieve. * @param out_data The buffer to write to. * * @return 0 if successful, otherwise an error is returned. */ SF_STATUS STDCALL rs_get_cell_strlen( - void * rs, - QueryResultFormat_t * query_result_format, + result_set_ptr rs, size_t idx, size_t * out_data); @@ -288,25 +247,22 @@ extern "C" { * Gets the number of rows in the current chunk being processed. * * @param rs The ResultSet object. - * @param query_result_format The query result format. * * @return the number of rows in the current chunk. */ - size_t rs_get_row_count_in_chunk(void * rs, QueryResultFormat_t * query_result_format); + size_t rs_get_row_count_in_chunk(result_set_ptr rs); /** * Indiciates whether the current cell is null or not. * * @param rs The ResultSet object. - * @param query_result_format The query result format. * @param idx The index of the column or row to retrieve. * @param out_data The buffer to write to. * * @return 0 if successful, otherwise an error is returned. */ SF_STATUS STDCALL rs_is_cell_null( - void * rs, - QueryResultFormat_t * query_result_format, + result_set_ptr rs, size_t idx, sf_bool * out_data); @@ -314,27 +270,22 @@ extern "C" { * Get the latest error code. * * @param rs The ResultSet object. - * @param query_result_format The query result format. * * @return the latest error code. 0 if no error. */ - SF_STATUS STDCALL rs_get_error( - void * rs, - QueryResultFormat_t * query_result_format - ); + SF_STATUS STDCALL rs_get_error(result_set_ptr rs); /** * Get the latest error code. * * @param rs The ResultSet object. - * @param query_result_format The query result format. * * @return the latest error message. empty string if no error. */ - const char* rs_get_error_message( - void * rs, - QueryResultFormat_t * query_result_format - ); + const char* rs_get_error_message(result_set_ptr rs); + + // return callback struct for arrow chunk downloading + NON_JSON_RESP* callback_create_arrow_resp(void); #ifdef __cplusplus } // extern "C" diff --git a/lib/result_set_arrow.h b/lib/result_set_arrow.h deleted file mode 100644 index 3250aad46f..0000000000 --- a/lib/result_set_arrow.h +++ /dev/null @@ -1,268 +0,0 @@ -/* - * Copyright (c) 2021 Snowflake Computing, Inc. All rights reserved. - */ - -#ifndef SNOWFLAKE_RESULTSETARROW_H -#define SNOWFLAKE_RESULTSETARROW_H - -#include "cJSON.h" -#include "snowflake/basic_types.h" -#include "snowflake/client.h" -#include "connection.h" - -#ifdef __cplusplus -extern "C" { -#endif - - /** - * A result set interface for Arrow result format. - * - * @see cpp/lib/ResultSet.hpp - * @see cpp/lib/ResultSetArrow.hpp - */ - typedef struct rs_arrow { - void * rs_object; - } rs_arrow_t; - - /** - * Parameterized constructor. - * Initializes the result set with rowset64 data in json result set. - * - * @param json_rowset64 A pointer to the rowset64 data in json result set. - * @param metadata A pointer to the metadata for the result set. - * @param tz_string The time zone. - */ - rs_arrow_t * rs_arrow_create_with_json_result( - cJSON * json_rowset64, - SF_COLUMN_DESC * metadata, - const char * tz_string); - - /** - * Parameterized constructor. - * Initializes the result set with the first arrow chunk of the result set. - * - * @param initial_chunk A pointer to the first chunk of the result set. - * @param metadata A pointer to the metadata for the result set. - * @param tz_string The time zone. - */ - rs_arrow_t * rs_arrow_create_with_chunk( - NON_JSON_RESP * initial_chunk, - SF_COLUMN_DESC * metadata, - const char * tz_string); - - /** - * Destructor. - */ - void rs_arrow_destroy(rs_arrow_t * rs); - - /** - * Appends the given chunk to the internal result set. - * - * @param rs The ResultSetArrow object. - * @param chunk The chunk to append. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL rs_arrow_append_chunk(rs_arrow_t * rs, NON_JSON_RESP * chunk); - - /** - * Advances to the next row. - * - * @param rs The ResultSetArrow object. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL rs_arrow_next(rs_arrow_t * rs); - - /** - * Writes the value of the current cell as a boolean to the provided buffer. - * - * @param rs The ResultSetArrow object. - * @param idx The index of the row to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL rs_arrow_get_cell_as_bool( - rs_arrow_t * rs, - size_t idx, - sf_bool * out_data); - - /** - * Writes the value of the current cell as an int8 to the provided buffer. - * - * @param rs The ResultSetArrow object. - * @param idx The index of the row to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL rs_arrow_get_cell_as_int8( - rs_arrow_t * rs, - size_t idx, - int8 * out_data); - - /** - * Writes the value of the current cell as an int32 to the provided buffer. - * - * @param rs The ResultSetArrow object. - * @param idx The index of the row to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL rs_arrow_get_cell_as_int32( - rs_arrow_t * rs, - size_t idx, - int32 * out_data); - - /** - * Writes the value of the current cell as an int64 to the provided buffer. - * - * @param rs The ResultSetArrow object. - * @param idx The index of the row to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL rs_arrow_get_cell_as_int64( - rs_arrow_t * rs, - size_t idx, - int64 * out_data); - - /** - * Writes the value of the current cell as a uint8 to the provided buffer. - * - * @param rs The ResultSetArrow object. - * @param idx The index of the row to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL rs_arrow_get_cell_as_uint8( - rs_arrow_t * rs, - size_t idx, - uint8 * out_data); - - /** - * Writes the value of the current cell as a uint32 to the provided buffer. - * - * @param rs The ResultSetArrow object. - * @param idx The index of the row to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL rs_arrow_get_cell_as_uint32( - rs_arrow_t * rs, - size_t idx, - uint32 * out_data); - - /** - * Writes the value of the current cell as a uint64 to the provided buffer. - * - * @param rs The ResultSetArrow object. - * @param idx The index of the row to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL rs_arrow_get_cell_as_uint64( - rs_arrow_t * rs, - size_t idx, - uint64 * out_data); - - /** - * Writes the value of the current cell as a float32 to the provided buffer. - * - * @param rs The ResultSetArrow object. - * @param idx The index of the row to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL rs_arrow_get_cell_as_float32( - rs_arrow_t * rs, - size_t idx, - float32 * out_data); - - /** - * Writes the value of the current cell as a float64 to the provided buffer. - * - * @param rs The ResultSetArrow object. - * @param idx The index of the row to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL rs_arrow_get_cell_as_float64( - rs_arrow_t * rs, - size_t idx, - float64 * out_data); - - /** - * Writes the value of the current cell as a constant C-string to the provided buffer. - * - * @param rs The ResultSetArrow object. - * @param idx The index of the row to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL rs_arrow_get_cell_as_const_string( - rs_arrow_t * rs, - size_t idx, - const char ** out_data); - - /** - * Writes the value of the current cell as a timestamp to the provided buffer. - * - * @param rs The ResultSetArrow object. - * @param idx The index of the row to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL rs_arrow_get_cell_as_timestamp( - rs_arrow_t * rs, - size_t idx, - SF_TIMESTAMP * out_data); - - /** - * Writes the length of the current cell to the provided buffer. - * - * @param rs The ResultSetArrow object. - * @param idx The index of the row to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL rs_arrow_get_cell_strlen(rs_arrow_t * rs, size_t idx, size_t * out_data); - - /** - * Gets the number of rows in the current chunk being processed. - * - * @param rs The ResultSetArrow object. - * - * @return the number of rows in the current chunk. - */ - size_t rs_arrow_get_row_count_in_chunk(rs_arrow_t * rs); - - /** - * Indiciates whether the current cell is null or not. - * - * @param rs The ResultSetArrow object. - * @param idx The index of the row to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL rs_arrow_is_cell_null(rs_arrow_t * rs, size_t idx, sf_bool * out_data); - - NON_JSON_RESP* callback_create_arrow_resp(void); - -#ifdef __cplusplus -} // extern "C" -#endif - -#endif // SNOWFLAKE_RESULTSETARROW_H diff --git a/lib/result_set_json.h b/lib/result_set_json.h deleted file mode 100644 index c2a1d28d20..0000000000 --- a/lib/result_set_json.h +++ /dev/null @@ -1,259 +0,0 @@ -/* - * Copyright (c) 2021 Snowflake Computing, Inc. All rights reserved. - */ - -#ifndef SNOWFLAKE_RESULTSETJSON_H -#define SNOWFLAKE_RESULTSETJSON_H - -#include "cJSON.h" -#include "snowflake/basic_types.h" -#include "snowflake/client.h" - -#ifdef __cplusplus -extern "C" { -#endif - - /** - * A result set interface for JSON result format. - * - * @see cpp/lib/ResultSet.hpp - * @see cpp/lib/ResultSetJson.hpp - */ - typedef struct rs_json { - void * rs_object; - } rs_json_t; - - /** - * Parameterized constructor. - * Initializes the result set with required information as well as data. - * - * @param rowset A pointer to the result set data. - * @param metadata A pointer to the metadata for the result set. - * @param tz_string The time zone. - */ - rs_json_t * rs_json_create( - cJSON * rowset, - SF_COLUMN_DESC * metadata, - const char * tz_string); - - /** - * Destructor. - */ - void rs_json_destroy(rs_json_t * rs); - - /** - * Appends the given chunk to the internal result set. - * - * @param rs The ResultSetJson object. - * @param chunk The chunk to append. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL rs_json_append_chunk(rs_json_t * rs, cJSON * chunk); - - /** - * Advances to next row. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL rs_json_next(rs_json_t * rs); - - /** - * Writes the value of the current cell as a boolean to the provided buffer. - * - * @param rs The ResultSetJson object. - * @param idx The index of the column to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL rs_json_get_cell_as_bool( - rs_json_t * rs, - size_t idx, - sf_bool * out_data); - - /** - * Writes the value of the current cell as an int8 to the provided buffer. - * - * @param rs The ResultSetJson object. - * @param idx The index of the column to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL rs_json_get_cell_as_int8( - rs_json_t * rs, - size_t idx, - int8 * out_data); - - /** - * Writes the value of the current cell as an int32 to the provided buffer. - * - * @param rs The ResultSetJson object. - * @param idx The index of the column to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL rs_json_get_cell_as_int32( - rs_json_t * rs, - size_t idx, - int32 * out_data); - - /** - * Writes the value of the current cell as an int64 to the provided buffer. - * - * @param rs The ResultSetJson object. - * @param idx The index of the column to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL rs_json_get_cell_as_int64( - rs_json_t * rs, - size_t idx, - int64 * out_data); - - /** - * Writes the value of the current cell as a uint8 to the provided buffer. - * - * @param rs The ResultSetJson object. - * @param idx The index of the column to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL rs_json_get_cell_as_uint8( - rs_json_t * rs, - size_t idx, - uint8 * out_data); - - /** - * Writes the value of the current cell as a uint32 to the provided buffer. - * - * @param rs The ResultSetJson object. - * @param idx The index of the column to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL rs_json_get_cell_as_uint32( - rs_json_t * rs, - size_t idx, - uint32 * out_data); - - /** - * Writes the value of the current cell as a uint64 to the provided buffer. - * - * @param rs The ResultSetJson object. - * @param idx The index of the column to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL rs_json_get_cell_as_uint64( - rs_json_t * rs, - size_t idx, - uint64 * out_data); - - /** - * Writes the value of the current cell as a float32 to the provided buffer. - * - * @param rs The ResultSet object. - * @param idx The index of the column to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL rs_json_get_cell_as_float32( - rs_json_t * rs, - size_t idx, - float32 * out_data); - - /** - * Writes the value of the current cell as a float64 to the provided buffer. - * - * @param rs The ResultSetJson object. - * @param idx The index of the column to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL rs_json_get_cell_as_float64( - rs_json_t * rs, - size_t idx, - float64 * out_data); - - /** - * Writes the value of the current cell as a constant C-string to the provided buffer. - * - * @param rs The ResultSetJson object. - * @param idx The index of the column to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL rs_json_get_cell_as_const_string( - rs_json_t * rs, - size_t idx, - const char ** out_data); - - /** - * Writes the value of the current cell as a timestamp to the provided buffer. - * - * @param rs The ResultSetJson object. - * @param idx The index of the column to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL rs_json_get_cell_as_timestamp( - rs_json_t * rs, - size_t idx, - SF_TIMESTAMP * out_data); - - /** - * Writes the length of the current cell to the provided buffer. - * - * @param rs The ResultSetJson object. - * @param idx The index of the column to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL rs_json_get_cell_strlen(rs_json_t * rs, size_t idx, size_t * out_data); - - /** - * Gets the number of rows in the current chunk being processed. - * - * @param rs The ResultSetJson object. - * - * @return the number of rows in the current chunk. - */ - size_t rs_json_get_row_count_in_chunk(rs_json_t * rs); - - /** - * Gets the total number of rows in the entire result set. - * - * @param rs The ResultSetJson object. - * - * @return the number of rows in the result set. - */ - size_t rs_json_get_total_row_count(rs_json_t * rs); - - /** - * Indiciates whether the current cell is null or not. - * - * @param rs The ResultSetJson object. - * @param idx The index of the column to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL rs_json_is_cell_null(rs_json_t * rs, size_t idx, sf_bool * out_data); - -#ifdef __cplusplus -} // extern "C" -#endif - -#endif // SNOWFLAKE_RESULTSETJSON_H diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 20b09d62eb..76e811808b 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -42,6 +42,7 @@ SET(TESTS_C test_get_describe_only_query_result test_stmt_functions test_unit_mfa_auth + test_multiple_statements # FEATURE_INCREASED_MAX_LOB_SIZE_IN_MEMORY is internal switch # will enable lob test when the change on server side will be published # test_lob diff --git a/tests/test_multiple_statements.c b/tests/test_multiple_statements.c new file mode 100644 index 0000000000..f4a2cf37e3 --- /dev/null +++ b/tests/test_multiple_statements.c @@ -0,0 +1,307 @@ +/* + * Copyright (c) 2024 Snowflake Computing, Inc. All rights reserved. + */ +#include +#include "utils/test_setup.h" + +void test_multi_stmt_transaction(void **unused) +{ + SF_CONNECT *sf = setup_snowflake_connection(); + SF_STATUS status = snowflake_connect(sf); + if (status != SF_STATUS_SUCCESS) { + dump_error(&(sf->error)); + } + assert_int_equal(status, SF_STATUS_SUCCESS); + + /* query */ + SF_STMT *sfstmt = snowflake_stmt(sf); + status = snowflake_query(sfstmt, "create or replace temporary table test_multi_txn(c1 number, c2 string) as select 10, 'z'", 0); + assert_int_equal(status, SF_STATUS_SUCCESS); + + int64 multi_stmt_count = 5; + status = snowflake_stmt_set_attr(sfstmt, SF_STMT_MULTI_STMT_COUNT, &multi_stmt_count); + assert_int_equal(status, SF_STATUS_SUCCESS); + + status = snowflake_query(sfstmt, + "begin;\n" + "delete from test_multi_txn;\n" + "insert into test_multi_txn values (1, 'a'), (2, 'b');\n" + "commit;\n" + "select count(*) from test_multi_txn", + 0); + if (status != SF_STATUS_SUCCESS) { + dump_error(&(sfstmt->error)); + } + assert_int_equal(status, SF_STATUS_SUCCESS); + + // first statement (begin) + assert_int_equal(snowflake_num_rows(sfstmt), 1); + assert_int_equal(snowflake_affected_rows(sfstmt), 1); + + // second statement (delete) + assert_int_equal(snowflake_next_result(sfstmt), SF_STATUS_SUCCESS); + assert_int_equal(snowflake_num_rows(sfstmt), 1); + assert_int_equal(snowflake_affected_rows(sfstmt), 1); + + // third statement (insert) + assert_int_equal(snowflake_next_result(sfstmt), SF_STATUS_SUCCESS); + assert_int_equal(snowflake_num_rows(sfstmt), 1); + assert_int_equal(snowflake_affected_rows(sfstmt), 2); + + // fourth statement (commit) + assert_int_equal(snowflake_next_result(sfstmt), SF_STATUS_SUCCESS); + assert_int_equal(snowflake_num_rows(sfstmt), 1); + assert_int_equal(snowflake_affected_rows(sfstmt), 1); + + // fifth statement (select) + assert_int_equal(snowflake_next_result(sfstmt), SF_STATUS_SUCCESS); + assert_int_equal(snowflake_num_rows(sfstmt), 1); + + int counter = 0; + int64 out; + while ((status = snowflake_fetch(sfstmt)) == SF_STATUS_SUCCESS) { + snowflake_column_as_int64(sfstmt, 1, &out); + assert_int_equal(out, 2); + ++counter; + } + assert_int_equal(status, SF_STATUS_EOF); + assert_int_equal(counter, 1); + + // no more result + assert_int_equal(snowflake_next_result(sfstmt), SF_STATUS_EOF); + + snowflake_stmt_term(sfstmt); + snowflake_term(sf); +} + +void test_multi_stmt_transaction_rollback(void **unused) +{ + SF_CONNECT *sf = setup_snowflake_connection(); + SF_STATUS status = snowflake_connect(sf); + if (status != SF_STATUS_SUCCESS) { + dump_error(&(sf->error)); + } + assert_int_equal(status, SF_STATUS_SUCCESS); + + /* query */ + SF_STMT *sfstmt = snowflake_stmt(sf); + status = snowflake_query(sfstmt, "create or replace temporary table test_multi_txn(c1 number, c2 string) as select 10, 'z'", 0); + assert_int_equal(status, SF_STATUS_SUCCESS); + + int64 multi_stmt_count = 5; + status = snowflake_stmt_set_attr(sfstmt, SF_STMT_MULTI_STMT_COUNT, &multi_stmt_count); + assert_int_equal(status, SF_STATUS_SUCCESS); + + status = snowflake_query(sfstmt, + "begin;\n" + "delete from test_multi_txn;\n" + "insert into test_multi_txn values (1, 'a'), (2, 'b');\n" + "rollback;\n" + "select count(*) from test_multi_txn", + 0); + if (status != SF_STATUS_SUCCESS) { + dump_error(&(sfstmt->error)); + } + assert_int_equal(status, SF_STATUS_SUCCESS); + + // first statement (begin) + assert_int_equal(snowflake_num_rows(sfstmt), 1); + assert_int_equal(snowflake_affected_rows(sfstmt), 1); + + // second statement (delete) + assert_int_equal(snowflake_next_result(sfstmt), SF_STATUS_SUCCESS); + assert_int_equal(snowflake_num_rows(sfstmt), 1); + assert_int_equal(snowflake_affected_rows(sfstmt), 1); + + // third statement (insert) + assert_int_equal(snowflake_next_result(sfstmt), SF_STATUS_SUCCESS); + assert_int_equal(snowflake_num_rows(sfstmt), 1); + assert_int_equal(snowflake_affected_rows(sfstmt), 2); + + // fourth statement (rollback) + assert_int_equal(snowflake_next_result(sfstmt), SF_STATUS_SUCCESS); + assert_int_equal(snowflake_num_rows(sfstmt), 1); + assert_int_equal(snowflake_affected_rows(sfstmt), 1); + + // fifth statement (select) + assert_int_equal(snowflake_next_result(sfstmt), SF_STATUS_SUCCESS); + assert_int_equal(snowflake_num_rows(sfstmt), 1); + + int counter = 0; + int64 out; + while ((status = snowflake_fetch(sfstmt)) == SF_STATUS_SUCCESS) { + snowflake_column_as_int64(sfstmt, 1, &out); + assert_int_equal(out, 1); + ++counter; + } + assert_int_equal(status, SF_STATUS_EOF); + assert_int_equal(counter, 1); + + // no more result + assert_int_equal(snowflake_next_result(sfstmt), SF_STATUS_EOF); + + snowflake_stmt_term(sfstmt); + snowflake_term(sf); +} + +void test_multi_stmt_with_large_result(void **unused) +{ + const int rownum = 100000; + SF_CONNECT *sf = setup_snowflake_connection(); + + // TODO SNOW-1526335 + // Sometime we can't get OCSP response from cache server or responder + // Usually happen on GCP and should be ignored by FAIL_OPEN + // Unfortunately libsnowflakeclient doesn't support FAIL_OPEN for now + // so we have to disable OCSP validation to around it. + // Will remove this code when adding support for FAIL_OPEN (which is + // the default behavior for all other drivers) + char *cenv = getenv("CLOUD_PROVIDER"); + if (cenv && !strncmp(cenv, "GCP", 4)) { + sf_bool insecure_mode = SF_BOOLEAN_TRUE; + snowflake_set_attribute(sf, SF_CON_INSECURE_MODE, &insecure_mode); + } + + SF_STATUS status = snowflake_connect(sf); + if (status != SF_STATUS_SUCCESS) { + dump_error(&(sf->error)); + } + assert_int_equal(status, SF_STATUS_SUCCESS); + + /* query */ + SF_STMT *sfstmt = snowflake_stmt(sf); + int64 multi_stmt_count = 3; + status = snowflake_stmt_set_attr(sfstmt, SF_STMT_MULTI_STMT_COUNT, &multi_stmt_count); + assert_int_equal(status, SF_STATUS_SUCCESS); + + status = snowflake_query(sfstmt, + "create or replace temporary table test_multi_large(c1 number, c2 number);\n" + "insert into test_multi_large select seq4(), TO_VARCHAR(seq4()) from table(generator(rowcount => 100000));\n" + "select * from test_multi_large", + 0); + if (status != SF_STATUS_SUCCESS) { + dump_error(&(sfstmt->error)); + } + assert_int_equal(status, SF_STATUS_SUCCESS); + + // first statement (begin) + assert_int_equal(snowflake_num_rows(sfstmt), 1); + assert_int_equal(snowflake_affected_rows(sfstmt), 1); + + // second statement (insert) + assert_int_equal(snowflake_next_result(sfstmt), SF_STATUS_SUCCESS); + assert_int_equal(snowflake_num_rows(sfstmt), 1); + assert_int_equal(snowflake_affected_rows(sfstmt), rownum); + + // third statement (select) + assert_int_equal(snowflake_next_result(sfstmt), SF_STATUS_SUCCESS); + assert_int_equal(snowflake_num_rows(sfstmt), rownum); + + int counter = 0; + int64 intout; + const char* strout; + char strexp[64]; + while ((status = snowflake_fetch(sfstmt)) == SF_STATUS_SUCCESS) { + snowflake_column_as_int64(sfstmt, 1, &intout); + assert_int_equal(intout, counter); + snowflake_column_as_const_str(sfstmt, 2, &strout); + sprintf(strexp, "%d", counter); + assert_string_equal(strout, strexp); + ++counter; + } + assert_int_equal(status, SF_STATUS_EOF); + assert_int_equal(counter, rownum); + + // no more result + assert_int_equal(snowflake_next_result(sfstmt), SF_STATUS_EOF); + + snowflake_stmt_term(sfstmt); + snowflake_term(sf); +} + +/* helper function for testing multi_stmt_count, running a query with + * multiple statements of 3. + * @param use_session_param Whethter to set MULTI_STATEMENT_COUNT through + * session parameter or statement attribute. + * @param count The count number to be set + * @return True if the query succeeded, otherwise false. + */ +sf_bool test_multi_stmt_core(sf_bool use_session_param, int count) +{ + SF_CONNECT *sf = setup_snowflake_connection(); + SF_STATUS status = snowflake_connect(sf); + if (status != SF_STATUS_SUCCESS) { + dump_error(&(sf->error)); + } + assert_int_equal(status, SF_STATUS_SUCCESS); + + SF_STMT* sfstmt = snowflake_stmt(sf); + + char query[1024]; + int64 multi_stmt_count = count; + if (SF_BOOLEAN_TRUE == use_session_param) + { + sprintf(query, "alter session set MULTI_STATEMENT_COUNT=%d", count); + status = snowflake_query(sfstmt, query, 0); + } + else + { + status = snowflake_stmt_set_attr(sfstmt, SF_STMT_MULTI_STMT_COUNT, &multi_stmt_count); + } + if (status != SF_STATUS_SUCCESS) { + dump_error(&(sfstmt->error)); + } + assert_int_equal(status, SF_STATUS_SUCCESS); + + sprintf(query, "%s", "select 1; select 2; select 3"); + status = snowflake_query(sfstmt, query, 0); + + snowflake_stmt_term(sfstmt); + snowflake_term(sf); + + return (status == SF_STATUS_SUCCESS) ? SF_BOOLEAN_TRUE : SF_BOOLEAN_FALSE; +} + +void test_multi_stmt_count_session_param_off(void** unused) +{ + // disable multiple statements by setting session parameter to 1 + // the query is expected to fail + assert_int_equal(test_multi_stmt_core(SF_BOOLEAN_TRUE, 1), SF_BOOLEAN_FALSE); +} + +void test_multi_stmt_count_session_param_on(void** unused) +{ + // enable multiple statements by setting session parameter to 0 + // the query should work + assert_int_equal(test_multi_stmt_core(SF_BOOLEAN_TRUE, 0), SF_BOOLEAN_TRUE); +} + +void test_multi_stmt_count_stmt_attr_match(void** unused) +{ + // set statement attribute with match number + // the query should work + assert_int_equal(test_multi_stmt_core(SF_BOOLEAN_FALSE, 3), SF_BOOLEAN_TRUE); +} + +void test_multi_stmt_count_stmt_attr_mismatch(void** unused) +{ + // set statement attribute with mismatch number + // the query is expected to fail + assert_int_equal(test_multi_stmt_core(SF_BOOLEAN_FALSE, 2), SF_BOOLEAN_FALSE); +} + +int main(void) { + initialize_test(SF_BOOLEAN_FALSE); + const struct CMUnitTest tests[] = { + cmocka_unit_test(test_multi_stmt_transaction), + cmocka_unit_test(test_multi_stmt_transaction_rollback), + cmocka_unit_test(test_multi_stmt_with_large_result), + cmocka_unit_test(test_multi_stmt_count_session_param_off), + cmocka_unit_test(test_multi_stmt_count_session_param_on), + cmocka_unit_test(test_multi_stmt_count_stmt_attr_match), + cmocka_unit_test(test_multi_stmt_count_stmt_attr_mismatch), + }; + int ret = cmocka_run_group_tests(tests, NULL, NULL); + snowflake_global_term(); + return ret; +} From 6bc1b928311c56794c045d50edba168bd8f62c39 Mon Sep 17 00:00:00 2001 From: SimbaGithub <48035983+SimbaGithub@users.noreply.github.com> Date: Fri, 18 Oct 2024 16:21:26 -0700 Subject: [PATCH 2/8] SNOW-692945: refactoring for resultset C wrapper (#724) --- cpp/lib/ResultSet.cpp | 40 +++++++++++++++++++++++++- cpp/lib/ResultSet.hpp | 26 ++++++++++++++++- cpp/lib/ResultSetArrow.cpp | 11 +++++--- cpp/lib/ResultSetArrow.hpp | 22 ++++++++++++--- cpp/lib/ResultSetJson.cpp | 5 ++-- cpp/lib/ResultSetJson.hpp | 4 +-- cpp/lib/result_set.cpp | 57 ++++---------------------------------- 7 files changed, 100 insertions(+), 65 deletions(-) diff --git a/cpp/lib/ResultSet.cpp b/cpp/lib/ResultSet.cpp index bd8484a20f..e72769c8f9 100644 --- a/cpp/lib/ResultSet.cpp +++ b/cpp/lib/ResultSet.cpp @@ -11,6 +11,8 @@ #include "snowflake/SF_CRTFunctionSafe.h" #include "DataConversion.hpp" #include "ResultSet.hpp" +#include "ResultSetJson.hpp" +#include "ResultSetArrow.hpp" namespace Snowflake { @@ -36,7 +38,7 @@ ResultSet::ResultSet(QueryResultFormat format) : ResultSet::ResultSet( SF_COLUMN_DESC * metadata, - std::string tzString, + const std::string& tzString, QueryResultFormat format ) : m_currChunkIdx(0), @@ -54,5 +56,41 @@ ResultSet::ResultSet( ; } +ResultSet* ResultSet::CreateResultFromJson(cJSON* json_rowset, + SF_COLUMN_DESC* metadata, + QueryResultFormat query_result_format, + const std::string& tz_string) +{ + switch (query_result_format) + { +#ifndef SF_WIN32 + case SF_ARROW_FORMAT: + return new Snowflake::Client::ResultSetArrow(json_rowset, metadata, tz_string); +#endif + case SF_JSON_FORMAT: + return new Snowflake::Client::ResultSetJson(json_rowset, metadata, tz_string); + default: + return nullptr; + } +} + +ResultSet* ResultSet::CreateResultFromChunk(void* initial_chunk, + SF_COLUMN_DESC* metadata, + QueryResultFormat query_result_format, + const std::string& tz_string) +{ + switch (query_result_format) + { +#ifndef SF_WIN32 + case SF_ARROW_FORMAT: + return new Snowflake::Client::ResultSetArrow((arrow::BufferBuilder*)(((NON_JSON_RESP*)initial_chunk)->buffer), metadata, tz_string); +#endif + case SF_JSON_FORMAT: + return new Snowflake::Client::ResultSetJson((cJSON*)initial_chunk, metadata, tz_string); + default: + return nullptr; + } +} + } // namespace Client } // namespace Snowflake diff --git a/cpp/lib/ResultSet.hpp b/cpp/lib/ResultSet.hpp index 157bd9cc99..bd5ee8d0e4 100644 --- a/cpp/lib/ResultSet.hpp +++ b/cpp/lib/ResultSet.hpp @@ -40,7 +40,17 @@ class ResultSet * @param metadata The metadata of the result set. * @param tzString The time zone. */ - ResultSet(SF_COLUMN_DESC * metadata, std::string tzString, QueryResultFormat format); + ResultSet(SF_COLUMN_DESC * metadata, const std::string& tzString, QueryResultFormat format); + + static ResultSet* CreateResultFromJson(cJSON* json_rowset, + SF_COLUMN_DESC* metadata, + QueryResultFormat query_result_format, + const std::string& tz_string); + + static ResultSet* CreateResultFromChunk(void* initial_chunk, + SF_COLUMN_DESC* metadata, + QueryResultFormat query_result_format, + const std::string& tz_string); /** * Destructor. @@ -49,6 +59,20 @@ class ResultSet // API methods ================================================================================= + /** + * Frees the previously held chunk in m_chunk and sets the current chunk to the given chunk. + * Each result type should override this function to handle chunks, + * while some result might not apply (won't be called), return error by default. + * + * @param chunkPtr The chunk to append. + * + * @return 0 if successful, otherwise an error is returned. + */ + virtual SF_STATUS STDCALL appendChunk(void* chunkPtr) + { + return SF_STATUS_ERROR_GENERAL; + } + /** * Advances to the next row. * diff --git a/cpp/lib/ResultSetArrow.cpp b/cpp/lib/ResultSetArrow.cpp index ef20ded1fe..6b20a6b022 100644 --- a/cpp/lib/ResultSetArrow.cpp +++ b/cpp/lib/ResultSetArrow.cpp @@ -29,7 +29,7 @@ ResultSetArrow::ResultSetArrow() : ResultSetArrow::ResultSetArrow( arrow::BufferBuilder* initialChunk, SF_COLUMN_DESC * metadata, - std::string tzString + const std::string& tzString ) : ResultSet(metadata, tzString, SF_ARROW_FORMAT) { @@ -44,10 +44,11 @@ ResultSetArrow::ResultSetArrow( ResultSetArrow::ResultSetArrow( cJSON * jsonRowset64, SF_COLUMN_DESC * metadata, - std::string tzString + const std::string& tzString ) : ResultSet(metadata, tzString, SF_ARROW_FORMAT) { + NON_JSON_RESP resp; arrow::BufferBuilder* bufferBuilder = NULL; if (jsonRowset64) { @@ -60,8 +61,9 @@ ResultSetArrow::ResultSetArrow( (void)bufferBuilder->Append((void*)decodedRowsetStr.c_str(), decodedRowsetStr.length()); } } + resp.buffer = bufferBuilder; - this->appendChunk(bufferBuilder); + this->appendChunk(&resp); // Reset row indices so that they can be re-used by public API. m_currChunkIdx = 0; @@ -76,8 +78,9 @@ ResultSetArrow::~ResultSetArrow() // Public methods ================================================================================== -SF_STATUS STDCALL ResultSetArrow::appendChunk(arrow::BufferBuilder * chunk) +SF_STATUS STDCALL ResultSetArrow::appendChunk(void* chunkPtr) { + arrow::BufferBuilder * chunk = (arrow::BufferBuilder*)(((NON_JSON_RESP*)chunkPtr)->buffer); if (chunk == nullptr) { if (m_isFirstChunk) diff --git a/cpp/lib/ResultSetArrow.hpp b/cpp/lib/ResultSetArrow.hpp index 1805094b12..3ad409dd12 100644 --- a/cpp/lib/ResultSetArrow.hpp +++ b/cpp/lib/ResultSetArrow.hpp @@ -62,7 +62,7 @@ class ResultSetArrow : public Snowflake::Client::ResultSet * @param metadata An array of metadata objects for each column. * @param tzString The time zone. */ - ResultSetArrow(arrow::BufferBuilder * initialChunk, SF_COLUMN_DESC * metadata, std::string tzString); + ResultSetArrow(arrow::BufferBuilder * initialChunk, SF_COLUMN_DESC * metadata, const std::string& tzString); /** @@ -76,7 +76,21 @@ class ResultSetArrow : public Snowflake::Client::ResultSet * @param metadata An array of metadata objects for each column. * @param tzString The time zone. */ - ResultSetArrow(cJSON* jsonRowset64, SF_COLUMN_DESC* metadata, std::string tzString); + ResultSetArrow(cJSON* jsonRowset64, SF_COLUMN_DESC* metadata, const std::string& tzString); + + + /** + * Parameterized constructor. + * + * This constructor will initialize m_records with the (partial) results + * contained in the initial chunk. It will also initialize m_metadata with + * the metadata in "metadata". + * + * @param jsonRowset64 A pointer to the rowset64 data in json result set. + * @param metadata An array of metadata objects for each column. + * @param tzString The time zone. + */ + ResultSetArrow(cJSON* jsonRowset64, SF_COLUMN_DESC* metadata, std::string& tzString); /** * Destructor. @@ -88,11 +102,11 @@ class ResultSetArrow : public Snowflake::Client::ResultSet /** * Appends the given chunk to the internal result set. * - * @param chunk The chunk to append. + * @param chunkPtr The chunk to append. * * @return 0 if successful, otherwise an error is returned. */ - SF_STATUS STDCALL appendChunk(arrow::BufferBuilder * chunk); + SF_STATUS STDCALL appendChunk(void* chunkPtr); /** * Advances the internal iterator to the next row. diff --git a/cpp/lib/ResultSetJson.cpp b/cpp/lib/ResultSetJson.cpp index a9bfc42490..a3e5510990 100644 --- a/cpp/lib/ResultSetJson.cpp +++ b/cpp/lib/ResultSetJson.cpp @@ -26,7 +26,7 @@ ResultSetJson::ResultSetJson() : ResultSetJson::ResultSetJson( cJSON * rowset, SF_COLUMN_DESC * metadata, - std::string tzString + const std::string& tzString ) : ResultSet(metadata, tzString, SF_JSON_FORMAT) { @@ -42,8 +42,9 @@ ResultSetJson::~ResultSetJson() // Public methods ================================================================================== -SF_STATUS STDCALL ResultSetJson::appendChunk(cJSON * chunk) +SF_STATUS STDCALL ResultSetJson::appendChunk(void* chunkPtr) { + cJSON * chunk = (cJSON *)chunkPtr; if (chunk == nullptr) { CXX_LOG_ERROR("appendChunk -- Received a null chunk to append."); diff --git a/cpp/lib/ResultSetJson.hpp b/cpp/lib/ResultSetJson.hpp index 0c853e7ee2..5dc292dcbd 100644 --- a/cpp/lib/ResultSetJson.hpp +++ b/cpp/lib/ResultSetJson.hpp @@ -52,7 +52,7 @@ class ResultSetJson : public Snowflake::Client::ResultSet ResultSetJson( cJSON * rowset, SF_COLUMN_DESC * metadata, - std::string tzString); + const std::string& tzString); /** * Destructor. @@ -66,7 +66,7 @@ class ResultSetJson : public Snowflake::Client::ResultSet * * @return 0 if successful, otherwise an error is returned. */ - SF_STATUS STDCALL appendChunk(cJSON * chunk); + SF_STATUS STDCALL appendChunk(void * chunkPtr); /** * Advances the internal iterator to the next row. If there are no more rows to consume, diff --git a/cpp/lib/result_set.cpp b/cpp/lib/result_set.cpp index 8923fb77aa..1c5e83c9ee 100644 --- a/cpp/lib/result_set.cpp +++ b/cpp/lib/result_set.cpp @@ -18,17 +18,8 @@ extern "C" { const char * tz_string ) { - switch (query_result_format) - { -#ifndef SF_WIN32 - case SF_ARROW_FORMAT: - return new Snowflake::Client::ResultSetArrow(json_rowset, metadata, std::string(tz_string)); -#endif - case SF_JSON_FORMAT: - return new Snowflake::Client::ResultSetJson(json_rowset, metadata, std::string(tz_string)); - default: - return nullptr; - } + return Snowflake::Client::ResultSet::CreateResultFromJson( + json_rowset, metadata, query_result_format, std::string(tz_string)); } result_set_ptr rs_create_with_chunk( @@ -38,17 +29,8 @@ extern "C" { const char * tz_string ) { - switch (query_result_format) - { -#ifndef SF_WIN32 - case SF_ARROW_FORMAT: - return new Snowflake::Client::ResultSetArrow((arrow::BufferBuilder*)(((NON_JSON_RESP*)initial_chunk)->buffer), metadata, std::string(tz_string)); -#endif - case SF_JSON_FORMAT: - return new Snowflake::Client::ResultSetJson((cJSON*)initial_chunk, metadata, std::string(tz_string)); - default: - return nullptr; - } + return Snowflake::Client::ResultSet::CreateResultFromChunk( + initial_chunk, metadata, query_result_format, std::string(tz_string)); } void rs_destroy(result_set_ptr rs) @@ -57,20 +39,7 @@ extern "C" { { return; } - QueryResultFormat query_result_format = - static_cast(rs)->getResultFormat(); - switch (query_result_format){ -#ifndef SF_WIN32 - case SF_ARROW_FORMAT: - delete static_cast(rs); - break; -#endif - case SF_JSON_FORMAT: - delete static_cast(rs); - break; - default: - break; - } + delete static_cast(rs); } #define ERROR_IF_NULL(ptr) \ @@ -82,21 +51,7 @@ extern "C" { rs_append_chunk(result_set_ptr rs, void * chunk) { ERROR_IF_NULL(rs); - QueryResultFormat query_result_format = - static_cast(rs)->getResultFormat(); - switch (query_result_format) - { -#ifndef SF_WIN32 - case SF_ARROW_FORMAT: - return static_cast(rs)->appendChunk( - (arrow::BufferBuilder*)(((NON_JSON_RESP*)chunk)->buffer)); -#endif - case SF_JSON_FORMAT: - return static_cast(rs)->appendChunk( - (cJSON*)chunk); - default: - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; - } + return static_cast(rs)->appendChunk(chunk); } SF_STATUS STDCALL rs_next(result_set_ptr rs) From e3e1b88d52f2df47a1ce4a417e2baf0ae64a7b79 Mon Sep 17 00:00:00 2001 From: Harry Xi Date: Tue, 22 Oct 2024 09:31:10 -0700 Subject: [PATCH 3/8] SNOW-1524269: support put/get for GCP (#738) --- cpp/StatementPutGet.cpp | 153 ++++++++++++++++++++++++++++++++++++ cpp/StatementPutGet.hpp | 33 ++++++++ include/snowflake/client.h | 5 ++ include/snowflake/version.h | 6 +- lib/chunk_downloader.c | 4 +- lib/client.c | 3 + lib/client_int.h | 1 + lib/connection.c | 19 +++-- lib/connection.h | 32 ++++++-- lib/http_perform.c | 76 +++++++++++++++++- lib/mock_http_perform.h | 2 +- tests/test_simple_put.cpp | 32 ++++---- 12 files changed, 322 insertions(+), 44 deletions(-) diff --git a/cpp/StatementPutGet.cpp b/cpp/StatementPutGet.cpp index 989affbf89..883a129a98 100755 --- a/cpp/StatementPutGet.cpp +++ b/cpp/StatementPutGet.cpp @@ -3,11 +3,35 @@ */ #include +#include "connection.h" #include "snowflake/PutGetParseResponse.hpp" #include "StatementPutGet.hpp" +#include "curl_desc_pool.h" using namespace Snowflake::Client; +static size_t file_get_write_callback(char* ptr, size_t size, size_t nmemb, void* userdata) +{ + size_t data_size = size * nmemb; + std::basic_iostream* recvStream = (std::basic_iostream*)(userdata); + if (recvStream) + { + recvStream->write(static_cast(ptr), data_size); + } + + return data_size; +} + +static size_t file_put_read_callback(void* ptr, size_t size, size_t nmemb, void* userdata) +{ + std::basic_iostream* payload = (std::basic_iostream*)(userdata); + size_t data_size = size * nmemb; + + payload->read(static_cast(ptr), data_size); + size_t ret = payload->gcount(); + return payload->gcount(); +} + StatementPutGet::StatementPutGet(SF_STMT *stmt) : m_stmt(stmt), m_useProxy(false) { @@ -104,6 +128,14 @@ bool StatementPutGet::parsePutGetCommand(std::string *sql, }; putGetParseResponse->stageInfo.endPoint = response->stage_info->endPoint; + } + else if (sf_strncasecmp(response->stage_info->location_type, "gcs", 3) == 0) + { + putGetParseResponse->stageInfo.stageType = StageType::GCS; + putGetParseResponse->stageInfo.credentials = { + {"GCS_ACCESS_TOKEN", response->stage_info->stage_cred->gcs_access_token} + }; + } else if (sf_strncasecmp(response->stage_info->location_type, "local_fs", 8) == 0) { @@ -123,3 +155,124 @@ Util::Proxy* StatementPutGet::get_proxy() return &m_proxy; } } + +bool StatementPutGet::http_put(std::string const& url, + std::vector const& headers, + std::basic_iostream& payload, + size_t payloadLen, + std::string& responseHeaders) +{ + if (!m_stmt || !m_stmt->connection) + { + return false; + } + SF_CONNECT* sf = m_stmt->connection; + void* curl_desc = get_curl_desc_from_pool(url.c_str(), sf->proxy, sf->no_proxy); + CURL* curl = get_curl_from_desc(curl_desc); + if (!curl) + { + return false; + } + + char* urlbuf = (char*)SF_CALLOC(1, url.length() + 1); + sf_strcpy(urlbuf, url.length() + 1, url.c_str()); + + SF_HEADER reqHeaders; + reqHeaders.header = NULL; + for (auto itr = headers.begin(); itr != headers.end(); itr++) + { + reqHeaders.header = curl_slist_append(reqHeaders.header, itr->c_str()); + } + + PUT_PAYLOAD putPayload; + putPayload.buffer = &payload; + putPayload.length = payloadLen; + putPayload.read_callback = file_put_read_callback; + + char* respHeaders = NULL; + sf_bool success = SF_BOOLEAN_FALSE; + + success = http_perform(curl, PUT_REQUEST_TYPE, urlbuf, &reqHeaders, NULL, &putPayload, NULL, + NULL, &respHeaders, get_retry_timeout(sf), + SF_BOOLEAN_FALSE, &m_stmt->error, sf->insecure_mode,sf->ocsp_fail_open, + sf->retry_on_curle_couldnt_connect_count, + 0, sf->retry_count, NULL, NULL, NULL, SF_BOOLEAN_FALSE, + sf->proxy, sf->no_proxy, SF_BOOLEAN_FALSE, SF_BOOLEAN_FALSE); + + free_curl_desc(curl_desc); + SF_FREE(urlbuf); + curl_slist_free_all(reqHeaders.header); + if (respHeaders) + { + responseHeaders = std::string(respHeaders); + SF_FREE(respHeaders); + } + + return success; +} + +bool StatementPutGet::http_get(std::string const& url, + std::vector const& headers, + std::basic_iostream* payload, + std::string& responseHeaders, + bool headerOnly) +{ + SF_REQUEST_TYPE reqType = GET_REQUEST_TYPE; + if (headerOnly) + { + reqType = HEAD_REQUEST_TYPE; + } + + if (!m_stmt || !m_stmt->connection) + { + return false; + } + SF_CONNECT* sf = m_stmt->connection; + + void* curl_desc = get_curl_desc_from_pool(url.c_str(), sf->proxy, sf->no_proxy); + CURL* curl = get_curl_from_desc(curl_desc); + if (!curl) + { + return false; + } + + char* urlbuf = (char*)SF_CALLOC(1, url.length() + 1); + sf_strcpy(urlbuf, url.length() + 1, url.c_str()); + + SF_HEADER reqHeaders; + reqHeaders.header = NULL; + for (auto itr = headers.begin(); itr != headers.end(); itr++) + { + reqHeaders.header = curl_slist_append(reqHeaders.header, itr->c_str()); + } + + NON_JSON_RESP resp; + resp.buffer = payload; + resp.write_callback = file_get_write_callback; + + char* respHeaders = NULL; + sf_bool success = SF_BOOLEAN_FALSE; + + success = http_perform(curl, reqType, urlbuf, &reqHeaders, NULL, NULL, NULL, + &resp, &respHeaders, get_retry_timeout(sf), + SF_BOOLEAN_FALSE, &m_stmt->error, sf->insecure_mode, sf->ocsp_fail_open, + sf->retry_on_curle_couldnt_connect_count, + 0, sf->retry_count, NULL, NULL, NULL, SF_BOOLEAN_FALSE, + sf->proxy, sf->no_proxy, SF_BOOLEAN_FALSE, SF_BOOLEAN_FALSE); + + free_curl_desc(curl_desc); + SF_FREE(urlbuf); + curl_slist_free_all(reqHeaders.header); + if (respHeaders) + { + responseHeaders = respHeaders; + SF_FREE(respHeaders); + } + + if (payload) + { + payload->flush(); + } + + return success; +} diff --git a/cpp/StatementPutGet.hpp b/cpp/StatementPutGet.hpp index 94321fac73..19c00df0fd 100644 --- a/cpp/StatementPutGet.hpp +++ b/cpp/StatementPutGet.hpp @@ -28,6 +28,39 @@ class StatementPutGet : public Snowflake::Client::IStatementPutGet virtual Util::Proxy* get_proxy(); + /** + * PUT/GET on GCS use this interface to perform put request. + * Not implemented by default. + * @param url The url of the request. + * @param headers The headers of the request. + * @param payload The upload data. + * @param responseHeaders The headers of the response. + * + * return true if succeed otherwise false + */ + virtual bool http_put(std::string const& url, + std::vector const& headers, + std::basic_iostream& payload, + size_t payloadLen, + std::string& responseHeaders); + + /** + * PUT/GET on GCS use this interface to perform put request. + * Not implemented by default. + * @param url The url of the request. + * @param headers The headers of the request. + * @param payload The upload data. + * @param responseHeaders The headers of the response. + * @param headerOnly True if get response header only without payload body. + * + * return true if succeed otherwise false + */ + virtual bool http_get(std::string const& url, + std::vector const& headers, + std::basic_iostream* payload, + std::string& responseHeaders, + bool headerOnly); + private: SF_STMT *m_stmt; Util::Proxy m_proxy; diff --git a/include/snowflake/client.h b/include/snowflake/client.h index e97ef7e373..756a20a892 100644 --- a/include/snowflake/client.h +++ b/include/snowflake/client.h @@ -18,6 +18,11 @@ extern "C" { /** * API Name */ +/* TODO: Temporarily change to ODBC for now to pass the test before + * features (PUT for GCP, multiple statements etc.) unblocked + * on server side. + * Need to revert to C_API when merging to master. + */ #define SF_API_NAME "ODBC" /** diff --git a/include/snowflake/version.h b/include/snowflake/version.h index 83ddc59f55..437ac8e5fb 100644 --- a/include/snowflake/version.h +++ b/include/snowflake/version.h @@ -5,7 +5,11 @@ #ifndef SNOWFLAKE_CLIENT_VERSION_H #define SNOWFLAKE_CLIENT_VERSION_H -// TODO: temporary change for testing, will remove +/* TODO: Temporarily change to ODBC version for now to pass the test before + * features (PUT for GCP, multiple statements etc.) unblocked + * on server side. + * Need to revert to libsfclient version when merging to master. + */ #define SF_API_VERSION "3.0.1" #endif /* SNOWFLAKE_CLIENT_VERSION_H */ diff --git a/lib/chunk_downloader.c b/lib/chunk_downloader.c index 6f6e923cb2..5657cb4465 100644 --- a/lib/chunk_downloader.c +++ b/lib/chunk_downloader.c @@ -217,8 +217,8 @@ sf_bool STDCALL download_chunk(char *url, SF_HEADER *headers, CURL *curl = get_curl_from_desc(curl_desc); if (!curl || - !http_perform(curl, GET_REQUEST_TYPE, url, headers, NULL, chunk, - non_json_resp, network_timeout, + !http_perform(curl, GET_REQUEST_TYPE, url, headers, NULL, NULL, chunk, + non_json_resp, NULL, network_timeout, SF_BOOLEAN_TRUE, error, insecure_mode, fail_open, 0, 0, retry_max_count, NULL, NULL, NULL, SF_BOOLEAN_FALSE, proxy, no_proxy, SF_BOOLEAN_FALSE, SF_BOOLEAN_FALSE)) { diff --git a/lib/client.c b/lib/client.c index 83607e06db..9880113bab 100644 --- a/lib/client.c +++ b/lib/client.c @@ -2488,6 +2488,9 @@ SF_STATUS STDCALL _snowflake_execute_ex(SF_STMT *sfstmt, json_copy_string( &sfstmt->put_get_response->stage_info->stage_cred->azure_sas_token, stage_cred, "AZURE_SAS_TOKEN"); + json_copy_string( + &sfstmt->put_get_response->stage_info->stage_cred->gcs_access_token, + stage_cred, "GCS_ACCESS_TOKEN"); json_copy_string( &sfstmt->put_get_response->localLocation, data, "localLocation"); diff --git a/lib/client_int.h b/lib/client_int.h index e8b111cd68..632856f031 100644 --- a/lib/client_int.h +++ b/lib/client_int.h @@ -86,6 +86,7 @@ typedef struct SF_STAGE_CRED { char *aws_secret_key; char *aws_token; char *azure_sas_token; + char* gcs_access_token; } SF_STAGE_CRED; typedef struct SF_STAGE_INFO { diff --git a/lib/connection.c b/lib/connection.c index 4ca5a12ca4..cf1916cbdf 100644 --- a/lib/connection.c +++ b/lib/connection.c @@ -239,7 +239,6 @@ cJSON *STDCALL create_query_json_body(const char *sql_text, parameters = snowflake_cJSON_CreateObject(); } snowflake_cJSON_AddStringToObject(parameters, "C_API_QUERY_RESULT_FORMAT", "JSON"); - // temporary code to fake as ODBC to have multiple statements enabled snowflake_cJSON_AddStringToObject(parameters, "ODBC_QUERY_RESULT_FORMAT", "JSON"); #endif @@ -376,7 +375,7 @@ sf_bool STDCALL curl_post_call(SF_CONNECT *sf, } do { - if (!http_perform(curl, POST_REQUEST_TYPE, url, header, body, json, NULL, + if (!http_perform(curl, POST_REQUEST_TYPE, url, header, body, NULL, json, NULL, NULL, retry_timeout, SF_BOOLEAN_FALSE, error, sf->insecure_mode, sf->ocsp_fail_open, sf->retry_on_curle_couldnt_connect_count, @@ -503,7 +502,7 @@ sf_bool STDCALL curl_get_call(SF_CONNECT *sf, memset(query_code, 0, QUERYCODE_LEN); do { - if (!http_perform(curl, GET_REQUEST_TYPE, url, header, NULL, json, NULL, + if (!http_perform(curl, GET_REQUEST_TYPE, url, header, NULL, NULL, json, NULL, NULL, get_retry_timeout(sf), SF_BOOLEAN_FALSE, error, sf->insecure_mode, sf->ocsp_fail_open, sf->retry_on_curle_couldnt_connect_count, @@ -906,16 +905,16 @@ ARRAY_LIST *json_get_object_keys(const cJSON *item) { } size_t -json_resp_cb(char *data, size_t size, size_t nmemb, RAW_JSON_BUFFER *raw_json) { +char_resp_cb(char *data, size_t size, size_t nmemb, RAW_CHAR_BUFFER *raw_buf) { size_t data_size = size * nmemb; log_debug("Curl response size: %zu", data_size); - raw_json->buffer = (char *) SF_REALLOC(raw_json->buffer, - raw_json->size + data_size + 1); + raw_buf->buffer = (char *) SF_REALLOC(raw_buf->buffer, + raw_buf->size + data_size + 1); // Start copying where last null terminator existed - sf_memcpy(&raw_json->buffer[raw_json->size], data_size, data, data_size); - raw_json->size += data_size; - // Set null terminator - raw_json->buffer[raw_json->size] = '\0'; + sf_memcpy(&raw_buf->buffer[raw_buf->size], data_size, data, data_size); + raw_buf->size += data_size; + // Set null raw_buf + raw_buf->buffer[raw_buf->size] = '\0'; return data_size; } diff --git a/lib/connection.h b/lib/connection.h index c536aa9b45..04ef521ee8 100644 --- a/lib/connection.h +++ b/lib/connection.h @@ -56,6 +56,9 @@ typedef enum SF_REQUEST_TYPE { /** we are doing a http delete */ DELETE_REQUEST_TYPE, + + /** we are doing a http head */ + HEAD_REQUEST_TYPE, } SF_REQUEST_TYPE; /** @@ -81,12 +84,12 @@ typedef enum SF_JSON_ERROR { /** * Dynamically growing char buffer to hold retrieved in cURL call. */ -typedef struct RAW_JSON_BUFFER { +typedef struct RAW_CHAR_BUFFER { // Char buffer char *buffer; // Number of characters in char buffer size_t size; -} RAW_JSON_BUFFER; +} RAW_CHAR_BUFFER; /** * URL Parameter struct used to construct an encoded URL. @@ -167,6 +170,15 @@ typedef struct non_json_response { void * buffer; } NON_JSON_RESP; +/** +* payload struct for put request +*/ +typedef struct put_payload { + size_t (*read_callback)(void* ptr, size_t size, size_t nmemb, void* userdata); + void * buffer; + size_t length; +} PUT_PAYLOAD; + /** * Macro to get a custom error message to pass to the Snowflake Error object. */ @@ -398,16 +410,16 @@ SF_JSON_ERROR STDCALL json_detach_object_from_array(cJSON **dest, cJSON *data, i ARRAY_LIST *json_get_object_keys(const cJSON *item); /** - * A write callback function to use to write the response text received from the cURL response. The raw JSON buffer + * A write callback function to use to write the response text received from the cURL response. The raw CHAR buffer * will grow in size until * * @param data The data to copy in the buffer. * @param size The size (in bytes) of each data member. * @param nmemb The number of data members. - * @param raw_json The Raw JSON Buffer object that grows in size to copy multiple writes for a single cURL call. + * @param raw_buf The Raw CHAR Buffer object that grows in size to copy multiple writes for a single cURL call. * @return The number of bytes copied into the buffer. */ -size_t json_resp_cb(char *data, size_t size, size_t nmemb, RAW_JSON_BUFFER *raw_json); +size_t char_resp_cb(char *data, size_t size, size_t nmemb, RAW_CHAR_BUFFER *raw_buf); /** * Performs an HTTP request with retry. @@ -416,10 +428,13 @@ size_t json_resp_cb(char *data, size_t size, size_t nmemb, RAW_JSON_BUFFER *raw_ * @param request_type The type of HTTP request. * @param url The fully qualified URL to use for the HTTP request. * @param header The header to use for the HTTP request. - * @param body The body to send over the HTTP request. If running GET request, set this to NULL. + * @param body The body to send over the HTTP request. If running GET/PUT request, set this to NULL. + * @param put_payload The payload to send over the PUT HTTP request. If not running PUT request, set this to NULL. * @param json A reference to a cJSON pointer where we should store a successful request. * @param non_json_resp A reference to a non-json response to retrieve response in non-json format. * Used only when json is set to NULL. + * @param resp_headers A reference to retrieve response headers. Needs to be freed with SF_FREE. + * Set to NULL if it's not needed. * @param network_timeout The network request timeout to use for each request try. * @param chunk_downloader A boolean value determining whether or not we are running this request from the chunk * downloader. Each chunk that we download from AWS is invalid JSON so we need to add an @@ -450,8 +465,9 @@ size_t json_resp_cb(char *data, size_t size, size_t nmemb, RAW_JSON_BUFFER *raw_ * @return Success/failure status of http request call. 1 = Success; 0 = Failure/renew timeout */ sf_bool STDCALL http_perform(CURL *curl, SF_REQUEST_TYPE request_type, char *url, SF_HEADER *header, - char *body, cJSON **json, NON_JSON_RESP* non_json_resp, int64 network_timeout, sf_bool chunk_downloader, - SF_ERROR_STRUCT *error, sf_bool insecure_mode, sf_bool fail_open, + char *body, PUT_PAYLOAD* put_payload, cJSON **json, NON_JSON_RESP* non_json_resp, + char** resp_headers, int64 network_timeout, sf_bool chunk_downloader, + SF_ERROR_STRUCT* error, sf_bool insecure_mode, sf_bool fail_open, int8 retry_on_curle_couldnt_connect_count, int64 renew_timeout, int8 retry_max_count, int64 *elapsed_time, int8 *retried_count, diff --git a/lib/http_perform.c b/lib/http_perform.c index b7291e2034..56d093967d 100644 --- a/lib/http_perform.c +++ b/lib/http_perform.c @@ -143,8 +143,10 @@ sf_bool STDCALL http_perform(CURL *curl, char *url, SF_HEADER *header, char *body, + PUT_PAYLOAD* put_payload, cJSON **json, NON_JSON_RESP *non_json_resp, + char** resp_headers, int64 network_timeout, sf_bool chunk_downloader, SF_ERROR_STRUCT *error, @@ -190,7 +192,8 @@ sf_bool STDCALL http_perform(CURL *curl, sf_get_current_time_millis() // start time }; time_t elapsedRetryTime = time(NULL); - RAW_JSON_BUFFER buffer = {NULL, 0}; + RAW_CHAR_BUFFER buffer = {NULL, 0}; + RAW_CHAR_BUFFER headerBuffer = { NULL, 0 }; struct data config; config.trace_ascii = 1; @@ -204,6 +207,8 @@ sf_bool STDCALL http_perform(CURL *curl, // Reset buffer since this may not be our first rodeo SF_FREE(buffer.buffer); buffer.size = 0; + SF_FREE(headerBuffer.buffer); + headerBuffer.size = 0; // Generate new request guid, if request guid exists in url if (SF_BOOLEAN_TRUE != retry_ctx_update_url(&curl_retry_ctx, url, include_retry_reason)) { @@ -269,6 +274,46 @@ sf_bool STDCALL http_perform(CURL *curl, break; } } + else if (request_type == HEAD_REQUEST_TYPE) + { + /** we want response header only */ + curl_easy_setopt(curl, CURLOPT_NOBODY, 1L); + } + else if (request_type == PUT_REQUEST_TYPE) + { + // we need to upload the data + curl_easy_setopt(curl, CURLOPT_UPLOAD, 1L); + + if (!put_payload) + { + log_error("Invalid payload for put request"); + break; + } + /** set read callback function */ + res = curl_easy_setopt(curl, CURLOPT_READFUNCTION, put_payload->read_callback); + if (res != CURLE_OK) { + log_error("Failed to set read function [%s]", curl_easy_strerror(res)); + break; + } + + /** set data object to pass to callback function */ + res = curl_easy_setopt(curl, CURLOPT_READDATA, put_payload->buffer); + if (res != CURLE_OK) { + log_error("Failed to set read data [%s]", curl_easy_strerror(res)); + break; + } + + /** set size of put */ + if (put_payload->length <= SF_INT32_MAX) + { + res = curl_easy_setopt(curl, CURLOPT_INFILESIZE, (long)put_payload->length); + } + else + { + res = curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, (curl_off_t)put_payload->length); + } + } + if (!json && non_json_resp) { @@ -276,7 +321,7 @@ sf_bool STDCALL http_perform(CURL *curl, } else { - res = curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, (void*)&json_resp_cb); + res = curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, (void*)&char_resp_cb); } if (res != CURLE_OK) { log_error("Failed to set writer [%s]", curl_easy_strerror(res)); @@ -292,8 +337,20 @@ sf_bool STDCALL http_perform(CURL *curl, res = curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&buffer); } if (res != CURLE_OK) { - log_error("Failed to set write data [%s]", curl_easy_strerror(res)); - break; + log_error("Failed to set write data [%s]", curl_easy_strerror(res)); + break; + } + + res = curl_easy_setopt(curl, CURLOPT_HEADERDATA, (void*)&headerBuffer); + if (res != CURLE_OK) { + log_error("Failed to set header data [%s]", curl_easy_strerror(res)); + break; + } + + res = curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, (void*)&char_resp_cb); + if (res != CURLE_OK) { + log_error("Failed to set header function [%s]", curl_easy_strerror(res)); + break; } if (DISABLE_VERIFY_PEER) { @@ -501,6 +558,15 @@ sf_bool STDCALL http_perform(CURL *curl, SF_FREE(buffer.buffer); + if (resp_headers) + { + *resp_headers = headerBuffer.buffer; + } + else + { + SF_FREE(headerBuffer.buffer); + } + return ret; } @@ -511,8 +577,10 @@ sf_bool STDCALL __wrap_http_perform(CURL *curl, char *url, SF_HEADER *header, char *body, + PUT_PAYLOAD* put_payload, cJSON **json, NON_JSON_RESP *non_json_resp, + char** resp_headers, int64 network_timeout, sf_bool chunk_downloader, SF_ERROR_STRUCT *error, diff --git a/lib/mock_http_perform.h b/lib/mock_http_perform.h index ce6cad20c3..71ea0c8b79 100644 --- a/lib/mock_http_perform.h +++ b/lib/mock_http_perform.h @@ -17,7 +17,7 @@ extern "C" { // The parameters for this are identical to http_perform located in connection.h // This is just the mock interface sf_bool STDCALL __wrap_http_perform(CURL *curl, SF_REQUEST_TYPE request_type, char *url, SF_HEADER *header, - char *body, cJSON **json, NON_JSON_RESP *non_json_resp, int64 network_timeout, sf_bool chunk_downloader, + char *body, PUT_PAYLOAD* put_payload, cJSON **json, NON_JSON_RESP *non_json_resp, char** resp_headers, int64 network_timeout, sf_bool chunk_downloader, SF_ERROR_STRUCT *error, sf_bool insecure_mode, sf_bool fail_open); #endif diff --git a/tests/test_simple_put.cpp b/tests/test_simple_put.cpp index e592371e46..c16a3f98c2 100755 --- a/tests/test_simple_put.cpp +++ b/tests/test_simple_put.cpp @@ -856,6 +856,19 @@ static int gr_setup(void **unused) { initialize_test(SF_BOOLEAN_FALSE); + // TODO SNOW-1526335 + // Sometime we can't get OCSP response from cache server or responder + // Usually happen on GCP and should be ignored by FAIL_OPEN + // Unfortunately libsnowflakeclient doesn't support FAIL_OPEN for now + // so we have to disable OCSP validation to around it. + // Will remove this code when adding support for FAIL_OPEN (which is + // the default behavior for all other drivers) + char *cenv = getenv("CLOUD_PROVIDER"); + if (cenv && !strncmp(cenv, "GCP", 4)) { + sf_bool value = SF_BOOLEAN_FALSE; + snowflake_global_set_attribute(SF_GLOBAL_OCSP_CHECK, &value); + } + if(!setup_random_database()) { std::cout << "Failed to setup random database, fallback to use regular one." << std::endl; } @@ -1231,12 +1244,6 @@ void test_2GBlarge_put(void **unused) return; } } - // put/get for GCP is not supported in libsnowflakeclient - // will test that in odbc. - if (cenv && !strncmp(cenv, "GCP", 4)) { - errno = 0; - return; - } // Jenkins node on Mac has issue with large file. #ifdef __APPLE__ @@ -1270,12 +1277,6 @@ void test_2GBlarge_get(void **unused) return; } } - // put/get for GCP is not supported in libsnowflakeclient - // will test that in odbc. - if (cenv && !strncmp(cenv, "GCP", 4)) { - errno = 0; - return; - } // Jenkins node on Mac has issue with large file. #ifdef __APPLE__ @@ -1637,7 +1638,7 @@ int main(void) { }); if(testAccount.find("GCP") != std::string::npos) { - setenv("CLOUD_PROVIDER", "GCP", 1); + setenv("CLOUD_PROVIDER", "GCP", 1); } else if(testAccount.find("AZURE") != std::string::npos) { @@ -1651,11 +1652,6 @@ int main(void) { char *cp = getenv("CLOUD_PROVIDER"); std::cout << "Cloud provider is " << cp << std::endl; #endif - const char *cloud_provider = std::getenv("CLOUD_PROVIDER"); - if(cloud_provider && ( strcmp(cloud_provider, "GCP") == 0 ) ) { - std::cout << "GCP put/get feature is not available in libsnowflakeclient." << std::endl; - return 0; - } const struct CMUnitTest tests[] = { cmocka_unit_test_teardown(test_simple_put_auto_compress, teardown), From 4b2cc1642b6758df053e0660f0c7fee54fd9c6dc Mon Sep 17 00:00:00 2001 From: Harry Xi Date: Wed, 23 Oct 2024 11:12:30 -0700 Subject: [PATCH 4/8] SNOW-1524269 native put get support (#745) --- CMakeLists.txt | 2 + ci/build_linux.sh | 4 +- cpp/FileTransferAgent.cpp | 76 ++++- cpp/StatementPutGet.cpp | 2 +- cpp/lib/ResultSet.hpp | 8 + cpp/lib/ResultSetArrow.hpp | 15 - cpp/lib/ResultSetPutGet.cpp | 336 +++++++++++++++++++ cpp/lib/ResultSetPutGet.hpp | 205 ++++++++++++ cpp/lib/result_set.cpp | 1 + include/snowflake/client.h | 24 +- lib/client.c | 122 ++++++- lib/client_int.h | 42 ++- tests/CMakeLists.txt | 8 +- tests/test_simple_put.cpp | 635 +++++++++++++++++++++++++++--------- 14 files changed, 1304 insertions(+), 176 deletions(-) create mode 100644 cpp/lib/ResultSetPutGet.cpp create mode 100644 cpp/lib/ResultSetPutGet.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index b71591988f..82ccd84aa5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -190,6 +190,8 @@ set (SOURCE_FILES_PUT_GET cpp/logger/SFLogger.hpp cpp/logger/SecretDetector.cpp cpp/logger/SecretDetector.hpp + cpp/lib/ResultSetPutGet.cpp + cpp/lib/ResultSetPutGet.hpp include/snowflake/IFileTransferAgent.hpp include/snowflake/ISFLogger.hpp include/snowflake/IStatementPutGet.hpp diff --git a/ci/build_linux.sh b/ci/build_linux.sh index d42ec7c84b..6e58b1a987 100755 --- a/ci/build_linux.sh +++ b/ci/build_linux.sh @@ -51,5 +51,7 @@ docker run \ #remove image to save disk space on github if [[ -n "$GITHUB_ACTIONS" ]]; then docker rm -vf $(docker ps -aq --filter ancestor=${BUILD_IMAGE_NAME}) - docker rmi -f "${BUILD_IMAGE_NAME}" + if [[ $CLIENT_CODE_COVERAGE -ne 1 ]] && [[ "$BUILD_TYPE" != "Debug" ]]; then + docker rmi -f "${BUILD_IMAGE_NAME}" + fi fi diff --git a/cpp/FileTransferAgent.cpp b/cpp/FileTransferAgent.cpp index 80066899a1..cc0fbaee90 100755 --- a/cpp/FileTransferAgent.cpp +++ b/cpp/FileTransferAgent.cpp @@ -9,8 +9,9 @@ #include "FileTransferAgent.hpp" #include "snowflake/SnowflakeTransferException.hpp" #include "snowflake/IStatementPutGet.hpp" +#include "StatementPutGet.hpp" +#include "lib/ResultSetPutGet.hpp" #include "util/Base64.hpp" -#include "SnowflakeS3Client.hpp" #include "StorageClientFactory.hpp" #include "crypto/CipherStreamBuf.hpp" #include "crypto/Cryptor.hpp" @@ -18,6 +19,7 @@ #include "util/ThreadPool.hpp" #include "EncryptionProvider.hpp" #include "logger/SFLogger.hpp" +#include "error.h" #include "snowflake/platform.h" #include "snowflake/SF_CRTFunctionSafe.h" #include @@ -961,3 +963,75 @@ std::string Snowflake::Client::FileTransferAgent::getLocalFilePathFromCommand( return localFilePath; } + +using namespace Snowflake::Client; +extern "C" { + SF_STATUS STDCALL _snowflake_execute_put_get_native( + SF_STMT* sfstmt, + struct SF_QUERY_RESULT_CAPTURE* result_capture) + { + if (!sfstmt) + { + return SF_STATUS_ERROR_STATEMENT_NOT_EXIST; + } + SF_CONNECT* sfconn = sfstmt->connection; + if (!sfconn) + { + return SF_STATUS_ERROR_CONNECTION_NOT_EXIST; + } + StatementPutGet stmtPutGet(sfstmt); + TransferConfig transConfig; + transConfig.caBundleFile = NULL; // use the one from global settings + transConfig.compressLevel = sfconn->put_compress_level; + transConfig.getSizeThreshold = sfconn->get_threshold; + transConfig.proxy = NULL; // use the one from statement + transConfig.tempDir = sfconn->put_temp_dir; + transConfig.useS3regionalUrl = sfconn->use_s3_regional_url; + string command(sfstmt->sql_text); + + FileTransferAgent agent(&stmtPutGet, &transConfig); + agent.setPutFastFail(sfconn->put_fastfail); + agent.setPutMaxRetries(sfconn->put_maxretries); + agent.setGetFastFail(sfconn->get_fastfail); + agent.setGetMaxRetries(sfconn->get_maxretries); + agent.setRandomDeviceAsUrand(sfconn->put_use_urand_dev); + + ITransferResult* result; + try + { + result = agent.execute(&command); + } + catch (std::exception& e) + { + std::string errmsg("File transfer failed: "); + errmsg += e.what(); + SET_SNOWFLAKE_ERROR(&sfstmt->error, SF_STATUS_ERROR_FILE_TRANSFER, + errmsg.c_str(), SF_SQLSTATE_GENERAL_ERROR); + return SF_STATUS_ERROR_FILE_TRANSFER; + } + catch (...) + { + std::string errmsg("File transfer failed with unknown exception."); + SET_SNOWFLAKE_ERROR(&sfstmt->error, SF_STATUS_ERROR_FILE_TRANSFER, + errmsg.c_str(), SF_SQLSTATE_GENERAL_ERROR); + return SF_STATUS_ERROR_FILE_TRANSFER; + } + + ResultSetPutGet * resultset = new Snowflake::Client::ResultSetPutGet(result); + if (!resultset) + { + std::string errmsg("Failed to allocate put get result set."); + SET_SNOWFLAKE_ERROR(&sfstmt->error, SF_STATUS_ERROR_OUT_OF_MEMORY, + errmsg.c_str(), SF_SQLSTATE_MEMORY_ALLOCATION_ERROR); + return SF_STATUS_ERROR_OUT_OF_MEMORY; + } + + sfstmt->qrf = SF_PUTGET_FORMAT; + sfstmt->total_row_index = 0; + sfstmt->result_set = resultset; + sfstmt->chunk_rowcount = sfstmt->total_rowcount = result->getResultSize(); + sfstmt->total_fieldcount = resultset->setup_column_desc(&sfstmt->desc); + + return SF_STATUS_SUCCESS; + } +} diff --git a/cpp/StatementPutGet.cpp b/cpp/StatementPutGet.cpp index 883a129a98..e5077854a9 100755 --- a/cpp/StatementPutGet.cpp +++ b/cpp/StatementPutGet.cpp @@ -49,7 +49,7 @@ StatementPutGet::StatementPutGet(SF_STMT *stmt) : bool StatementPutGet::parsePutGetCommand(std::string *sql, PutGetParseResponse *putGetParseResponse) { - if (snowflake_query(m_stmt, sql->c_str(), 0) != SF_STATUS_SUCCESS) + if (_snowflake_query_put_get_legacy(m_stmt, sql->c_str(), 0) != SF_STATUS_SUCCESS) { return false; } diff --git a/cpp/lib/ResultSet.hpp b/cpp/lib/ResultSet.hpp index bd5ee8d0e4..5e583c7848 100644 --- a/cpp/lib/ResultSet.hpp +++ b/cpp/lib/ResultSet.hpp @@ -14,6 +14,14 @@ #include "snowflake/client.h" #include "result_set.h" +#define VERIFY_COLUMN_INDEX(index, total) \ + if (index < 1 || index > total) \ + { \ + setError(SF_STATUS_ERROR_OUT_OF_BOUNDS, \ + "Column index must be between 1 and snowflake_num_fields()"); \ + return SF_STATUS_ERROR_OUT_OF_BOUNDS; \ + } + namespace Snowflake { namespace Client diff --git a/cpp/lib/ResultSetArrow.hpp b/cpp/lib/ResultSetArrow.hpp index 3ad409dd12..68aa43456f 100644 --- a/cpp/lib/ResultSetArrow.hpp +++ b/cpp/lib/ResultSetArrow.hpp @@ -64,7 +64,6 @@ class ResultSetArrow : public Snowflake::Client::ResultSet */ ResultSetArrow(arrow::BufferBuilder * initialChunk, SF_COLUMN_DESC * metadata, const std::string& tzString); - /** * Parameterized constructor. * @@ -78,20 +77,6 @@ class ResultSetArrow : public Snowflake::Client::ResultSet */ ResultSetArrow(cJSON* jsonRowset64, SF_COLUMN_DESC* metadata, const std::string& tzString); - - /** - * Parameterized constructor. - * - * This constructor will initialize m_records with the (partial) results - * contained in the initial chunk. It will also initialize m_metadata with - * the metadata in "metadata". - * - * @param jsonRowset64 A pointer to the rowset64 data in json result set. - * @param metadata An array of metadata objects for each column. - * @param tzString The time zone. - */ - ResultSetArrow(cJSON* jsonRowset64, SF_COLUMN_DESC* metadata, std::string& tzString); - /** * Destructor. */ diff --git a/cpp/lib/ResultSetPutGet.cpp b/cpp/lib/ResultSetPutGet.cpp new file mode 100644 index 0000000000..173ea18a70 --- /dev/null +++ b/cpp/lib/ResultSetPutGet.cpp @@ -0,0 +1,336 @@ +/* + * Copyright (c) 2024 Snowflake Computing, Inc. All rights reserved. + */ + +#include "ResultSetPutGet.hpp" +#include "../logger/SFLogger.hpp" +#include "memory.h" +#include "client_int.h" + +// helper functions +namespace +{ + void setup_string_column_desc(const std::string& name, SF_COLUMN_DESC& col_desc, size_t idx) + { + col_desc.name = (char*)SF_CALLOC(1, name.length() + 1); + sf_strncpy(col_desc.name, name.length() + 1, name.c_str(), name.length() + 1); + col_desc.byte_size = SF_DEFAULT_MAX_OBJECT_SIZE; + col_desc.c_type = SF_C_TYPE_STRING; + col_desc.internal_size = SF_DEFAULT_MAX_OBJECT_SIZE; + col_desc.null_ok = SF_BOOLEAN_TRUE; + col_desc.precision = 0; + col_desc.scale = 0; + col_desc.type = SF_DB_TYPE_TEXT; + col_desc.idx = idx; + } + + void setup_integer_column_desc(const std::string& name, SF_COLUMN_DESC& col_desc, size_t idx) + { + col_desc.name = (char*)SF_CALLOC(1, name.length() + 1); + sf_strncpy(col_desc.name, name.length() + 1, name.c_str(), name.length() + 1); + col_desc.byte_size = 8; + col_desc.c_type = SF_C_TYPE_UINT64; + col_desc.internal_size = 8; + col_desc.null_ok = SF_BOOLEAN_TRUE; + col_desc.precision = 38; + col_desc.scale = 0; + col_desc.type = SF_DB_TYPE_FIXED; + col_desc.idx = idx; + } + + struct putget_column + { + std::string name; + bool isInteger; + }; + + std::vector PUTGET_COLUMNS[2] = + { + // UPLOAD + { + {"source", false}, + {"target", false}, + {"source_size", true}, + {"target_size", true}, + {"source_compression", false}, + {"target_compression", false}, + {"status", false}, + {"encryption", false}, + {"message", false}, + }, + // DOWNLOAD + { + {"file", false}, + {"size", true}, + {"status", false}, + {"encryption", false}, + {"message", false}, + }, + }; +} + +namespace Snowflake +{ +namespace Client +{ +ResultSetPutGet::ResultSetPutGet(ITransferResult *result) : + ResultSet(SF_PUTGET_FORMAT), + m_cmdType(result->getCommandType()) +{ + m_values.reserve(result->getResultSize()); + while (result->next()) + { + std::vector row; + row.reserve(result->getColumnSize()); + for (unsigned int i = 0; i < result->getColumnSize(); i++) + { + row.emplace_back(); + result->getColumnAsString(i, row.back()); + } + m_values.push_back(row); + } +} + +ResultSetPutGet::~ResultSetPutGet() +{ + ; // Do nothing +} + +// Public methods ================================================================================== + +size_t ResultSetPutGet::setup_column_desc(SF_COLUMN_DESC** desc) +{ + if ((m_cmdType != CommandType::UPLOAD) && (m_cmdType != CommandType::DOWNLOAD)) + { + // impossible + return 0; + } + + SF_COLUMN_DESC * col_desc = NULL; + m_totalColumnCount = PUTGET_COLUMNS[m_cmdType].size(); + col_desc = (SF_COLUMN_DESC*)SF_CALLOC(m_totalColumnCount, sizeof(SF_COLUMN_DESC)); + for (size_t i = 0; i < m_totalColumnCount; i++) + { + if (PUTGET_COLUMNS[m_cmdType][i].isInteger) + { + setup_integer_column_desc(PUTGET_COLUMNS[m_cmdType][i].name, col_desc[i], i + 1); + } + else + { + setup_string_column_desc(PUTGET_COLUMNS[m_cmdType][i].name, col_desc[i], i + 1); + } + } + + *desc = col_desc; + return m_totalColumnCount; +} + +SF_STATUS STDCALL ResultSetPutGet::next() +{ + if (m_currRowIdx < m_values.size()) + { + m_currRowIdx++; + return SF_STATUS_SUCCESS; + } + + return SF_STATUS_ERROR_OUT_OF_BOUNDS; +} + +SF_STATUS STDCALL ResultSetPutGet::getCellAsBool(size_t idx, sf_bool * out_data) +{ + setError(SF_STATUS_ERROR_CONVERSION_FAILURE, + "Value cannot be converted to boolean."); + return SF_STATUS_ERROR_CONVERSION_FAILURE; +} + +SF_STATUS STDCALL ResultSetPutGet::getCellAsInt8(size_t idx, int8 * out_data) +{ + VERIFY_COLUMN_INDEX(idx, m_totalColumnCount); + size_t row_idx = m_currRowIdx > 0 ? m_currRowIdx - 1 : 0; + *out_data = static_cast(m_values[row_idx][idx - 1][0]); + return SF_STATUS_SUCCESS; +} + +SF_STATUS STDCALL ResultSetPutGet::getCellAsInt32(size_t idx, int32 * out_data) +{ + *out_data = 0; + + uint64 value = 0; + SF_STATUS ret = getCellAsUint64(idx, &value); + + if (SF_STATUS_SUCCESS != ret) + { + return ret; + } + + if (value > SF_INT32_MAX) + { + CXX_LOG_ERROR("Value out of range for int32."); + setError(SF_STATUS_ERROR_OUT_OF_RANGE, + "Value out of range for int32."); + return SF_STATUS_ERROR_OUT_OF_RANGE; + } + + *out_data = static_cast(value); + return SF_STATUS_SUCCESS; +} + +SF_STATUS STDCALL ResultSetPutGet::getCellAsInt64(size_t idx, int64 * out_data) +{ + *out_data = 0; + + uint64 value = 0; + SF_STATUS ret = getCellAsUint64(idx, &value); + + if (SF_STATUS_SUCCESS != ret) + { + return ret; + } + + if (value > SF_INT64_MAX) + { + CXX_LOG_ERROR("Value out of range for int64."); + setError(SF_STATUS_ERROR_OUT_OF_RANGE, + "Value out of range for int64."); + return SF_STATUS_ERROR_OUT_OF_RANGE; + } + + *out_data = static_cast(value); + return SF_STATUS_SUCCESS; +} + +SF_STATUS STDCALL ResultSetPutGet::getCellAsUint8(size_t idx, uint8 * out_data) +{ + return getCellAsInt8(idx, (int8*)out_data); +} + +SF_STATUS STDCALL ResultSetPutGet::getCellAsUint32(size_t idx, uint32 * out_data) +{ + *out_data = 0; + + uint64 value = 0; + SF_STATUS ret = getCellAsUint64(idx, &value); + + if (SF_STATUS_SUCCESS != ret) + { + return ret; + } + + if (value > SF_UINT32_MAX) + { + CXX_LOG_ERROR("Value out of range for uint32."); + setError(SF_STATUS_ERROR_OUT_OF_RANGE, + "Value out of range for uint32."); + return SF_STATUS_ERROR_OUT_OF_RANGE; + } + + *out_data = static_cast(value); + return SF_STATUS_SUCCESS; +} + +SF_STATUS STDCALL ResultSetPutGet::getCellAsUint64(size_t idx, uint64 * out_data) +{ + VERIFY_COLUMN_INDEX(idx, m_totalColumnCount); + + *out_data = 0; + + if (!PUTGET_COLUMNS[m_cmdType][idx - 1].isInteger) + { + setError(SF_STATUS_ERROR_CONVERSION_FAILURE, + "Invalid conversion from string column to integer."); + return SF_STATUS_ERROR_CONVERSION_FAILURE; + } + + uint64 value = 0; + size_t row_idx = m_currRowIdx > 0 ? m_currRowIdx - 1 : 0; + + try + { + value = std::stoull(m_values[row_idx][idx - 1], NULL, 10); + } + catch (...) + { + CXX_LOG_ERROR("Cannot convert value to uint64."); + setError(SF_STATUS_ERROR_CONVERSION_FAILURE, + "Cannot convert value to uint64."); + return SF_STATUS_ERROR_CONVERSION_FAILURE; + } + + *out_data = value; + return SF_STATUS_SUCCESS; +} + +SF_STATUS STDCALL ResultSetPutGet::getCellAsFloat32(size_t idx, float32 * out_data) +{ + *out_data = 0; + + uint64 value = 0; + SF_STATUS ret = getCellAsUint64(idx, &value); + + if (SF_STATUS_SUCCESS != ret) + { + return ret; + } + + *out_data = (float32)value; + return SF_STATUS_SUCCESS; +} + +SF_STATUS STDCALL ResultSetPutGet::getCellAsFloat64(size_t idx, float64 * out_data) +{ + *out_data = 0; + + uint64 value = 0; + SF_STATUS ret = getCellAsUint64(idx, &value); + + if (SF_STATUS_SUCCESS != ret) + { + return ret; + } + + *out_data = (float64)value; + return SF_STATUS_SUCCESS; +} + +SF_STATUS STDCALL ResultSetPutGet::getCellAsConstString(size_t idx, const char ** out_data) +{ + VERIFY_COLUMN_INDEX(idx, m_totalColumnCount); + + size_t row_idx = m_currRowIdx > 0 ? m_currRowIdx - 1 : 0; + + *out_data = m_values[row_idx][idx - 1].c_str(); + return SF_STATUS_SUCCESS; +} + +SF_STATUS STDCALL ResultSetPutGet::getCellAsTimestamp(size_t idx, SF_TIMESTAMP * out_data) +{ + setError(SF_STATUS_ERROR_CONVERSION_FAILURE, + "Value cannot be converted to timestamp."); + return SF_STATUS_ERROR_CONVERSION_FAILURE; +} + +SF_STATUS STDCALL ResultSetPutGet::getCellStrlen(size_t idx, size_t * out_data) +{ + VERIFY_COLUMN_INDEX(idx, m_totalColumnCount); + + size_t row_idx = m_currRowIdx > 0 ? m_currRowIdx - 1 : 0; + + *out_data = m_values[row_idx][idx - 1].length(); + return SF_STATUS_SUCCESS; +} + +size_t ResultSetPutGet::getRowCountInChunk() +{ + return m_values.size(); +} + +SF_STATUS STDCALL ResultSetPutGet::isCellNull(size_t idx, sf_bool * out_data) +{ + VERIFY_COLUMN_INDEX(idx, m_totalColumnCount); + *out_data = SF_BOOLEAN_FALSE; + + return SF_STATUS_SUCCESS; +} + +} // namespace Client +} // namespace Snowflake diff --git a/cpp/lib/ResultSetPutGet.hpp b/cpp/lib/ResultSetPutGet.hpp new file mode 100644 index 0000000000..7e8e8c8681 --- /dev/null +++ b/cpp/lib/ResultSetPutGet.hpp @@ -0,0 +1,205 @@ +/* + * Copyright (c) 2024 Snowflake Computing, Inc. All rights reserved. + */ + +#ifndef SNOWFLAKECLIENT_RESULTSETPUTGET_HPP +#define SNOWFLAKECLIENT_RESULTSETPUTGET_HPP + +#include "ResultSet.hpp" +#include "snowflake/ITransferResult.hpp" + +namespace Snowflake +{ +namespace Client +{ + +/** + * Represents a result set retrieved from PUT/GET execution. + */ +class ResultSetPutGet : public Snowflake::Client::ResultSet +{ +public: + + /** + * Parameterized constructor. + * + * @param result A pointer to the transfer result. + */ + ResultSetPutGet(ITransferResult *result); + + /** + * Destructor. + */ + ~ResultSetPutGet(); + + /** + * Setup column description of the transfer result. + * + * param desc The output parameter to return the pointer to the buffer allocated + * for the column description. + * Needs to be freed by the caller using SF_FREE. + * + * @return The number of columns. + */ + size_t setup_column_desc(SF_COLUMN_DESC** desc); + + /** + * Advances the internal iterator to the next row. If there are no more rows to consume, + * + * @return 0 if successful, otherwise an error is returned. + */ + SF_STATUS STDCALL next(); + + /** + * Writes the value of the given cell as a boolean to the provided buffer. + * + * @param idx The index of the column to retrieve. + * @param out_data The buffer to write to. + * + * @return 0 if successful, otherwise an error is returned. + */ + SF_STATUS STDCALL getCellAsBool(size_t idx, sf_bool * out_data); + + /** + * Writes the value of the given cell as an int8 to the provided buffer. + * + * @param idx The index of the column to retrieve. + * @param out_data The buffer to write to. + * + * @return 0 if successful, otherwise an error is returned. + */ + SF_STATUS STDCALL getCellAsInt8(size_t idx, int8 * out_data); + + /** + * Writes the value of the given cell as an int32 to the provided buffer. + * + * @param idx The index of the column to retrieve. + * @param out_data The buffer to write to. + * + * @return 0 if successful, otherwise an error is returned. + */ + SF_STATUS STDCALL getCellAsInt32(size_t idx, int32 * out_data); + + /** + * Writes the value of the given cell as an int64 to the provided buffer. + * + * @param idx The index of the column to retrieve. + * @param out_data The buffer to write to. + * + * @return 0 if successful, otherwise an error is returned. + */ + SF_STATUS STDCALL getCellAsInt64(size_t idx, int64 * out_data); + + /** + * Writes the value of the given cell as a uint8 to the provided buffer. + * + * @param idx The index of the column to retrieve. + * @param out_data The buffer to write to. + * + * @return 0 if successful, otherwise an error is returned. + */ + SF_STATUS STDCALL getCellAsUint8(size_t idx, uint8 * out_data); + + /** + * Writes the value of the given cell as a uint32 to the provided buffer. + * + * @param idx The index of the column to retrieve. + * @param out_data The buffer to write to. + * + * @return 0 if successful, otherwise an error is returned. + */ + SF_STATUS STDCALL getCellAsUint32(size_t idx, uint32 * out_data); + + /** + * Writes the value of the given cell as a uint64 to the provided buffer. + * + * @param idx The index of the column to retrieve. + * @param out_data The buffer to write to. + * + * @return 0 if successful, otherwise an error is returned. + */ + SF_STATUS STDCALL getCellAsUint64(size_t idx, uint64 * out_data); + + /** + * Writes the value of the given cell as a float32 to the provided buffer. + * + * @param idx The index of the column to retrieve. + * @param out_data The buffer to write to. + * + * @return 0 if successful, otherwise an error is returned. + */ + SF_STATUS STDCALL getCellAsFloat32(size_t idx, float32 * out_data); + + /** + * Writes the value of the given cell as a float64 to the provided buffer. + * + * @param idx The index of the column to retrieve. + * @param out_data The buffer to write to. + * + * @return 0 if successful, otherwise an error is returned. + */ + SF_STATUS STDCALL getCellAsFloat64(size_t idx, float64 * out_data); + + /** + * Writes the value of the given cell as a constant C-string to the provided buffer. + * + * @param idx The index of the column to retrieve. + * @param out_data The buffer to write to. + * + * @return 0 if successful, otherwise an error is returned. + */ + SF_STATUS STDCALL getCellAsConstString(size_t idx, const char ** out_data); + + /** + * Writes the value of the given cell as a timestamp to the provided buffer. + * + * @param idx The index of the column to retrieve. + * @param out_data The buffer to write to. + * + * @return 0 if successful, otherwise an error is returned. + */ + SF_STATUS STDCALL getCellAsTimestamp(size_t idx, SF_TIMESTAMP * out_data); + + /** + * Writes the length of the given cell to the provided buffer. + * + * @param idx The index of the column to retrieve. + * @param out_data The buffer to write to. + * + * @return 0 if successful, otherwise an error is returned. + */ + SF_STATUS STDCALL getCellStrlen(size_t idx, size_t * out_data); + + /** + * Gets the total number of rows in the current chunk being processed. + * + * @return the number of rows in the current chunk. + */ + size_t getRowCountInChunk(); + + /** + * Indicates whether the given cell is null. + * + * @param idx The index of the column to check is null. + * @param out_data The buffer to write to. + * + * @return 0 if successful, otherwise an error is returned. + */ + SF_STATUS STDCALL isCellNull(size_t idx, sf_bool * out_data); + +private: + + // Hidden default constructor. + ResultSetPutGet(); + + // the result values from transfer result + std::vector> m_values; + + // the command type of UPLOAD(PUT) or DOWNLOAD(GET) + CommandType m_cmdType; +}; + +} // namespace Client +} // namespace Snowflake + +#endif // SNOWFLAKECLIENT_RESULTSETPUTGET_HPP diff --git a/cpp/lib/result_set.cpp b/cpp/lib/result_set.cpp index 1c5e83c9ee..3fca6fe496 100644 --- a/cpp/lib/result_set.cpp +++ b/cpp/lib/result_set.cpp @@ -6,6 +6,7 @@ #include "ResultSet.hpp" #include "ResultSetArrow.hpp" #include "ResultSetJson.hpp" +#include "ResultSetPutGet.hpp" #ifdef __cplusplus extern "C" { diff --git a/include/snowflake/client.h b/include/snowflake/client.h index 756a20a892..0a08109f92 100644 --- a/include/snowflake/client.h +++ b/include/snowflake/client.h @@ -164,7 +164,8 @@ typedef enum SF_STATUS { SF_STATUS_ERROR_NULL_POINTER = 240022, SF_STATUS_ERROR_BUFFER_TOO_SMALL = 240023, SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT = 240024, - SF_STATUS_ERROR_OTHER = 240025 + SF_STATUS_ERROR_OTHER = 240025, + SF_STATUS_ERROR_FILE_TRANSFER = 240026 } SF_STATUS; /** @@ -268,6 +269,14 @@ typedef enum SF_ATTRIBUTE { SF_CON_MAX_BINARY_SIZE, SF_CON_MAX_VARIANT_SIZE, SF_CON_OCSP_FAIL_OPEN, + SF_CON_PUT_TEMPDIR, + SF_CON_PUT_COMPRESSLV, + SF_CON_PUT_USE_URANDOM_DEV, + SF_CON_PUT_FASTFAIL, + SF_CON_PUT_MAXRETRIES, + SF_CON_GET_FASTFAIL, + SF_CON_GET_MAXRETRIES, + SF_CON_GET_THRESHOLD, SF_DIR_QUERY_URL, SF_DIR_QUERY_URL_PARAM, SF_DIR_QUERY_TOKEN, @@ -403,6 +412,17 @@ typedef struct SF_CONNECT { uint64 max_varchar_size; uint64 max_binary_size; uint64 max_variant_size; + + // put get configurations + sf_bool use_s3_regional_url; + sf_bool put_use_urand_dev; + int8 put_compress_level; + char* put_temp_dir; + sf_bool put_fastfail; + int8 put_maxretries; + sf_bool get_fastfail; + int8 get_maxretries; + int64 get_threshold; } SF_CONNECT; /** @@ -459,7 +479,7 @@ typedef void* result_set_ptr; */ typedef enum QueryResultFormat_e { - SF_ARROW_FORMAT, SF_JSON_FORMAT, SF_FORMAT_UNKNOWN + SF_ARROW_FORMAT, SF_JSON_FORMAT, SF_PUTGET_FORMAT, SF_FORMAT_UNKNOWN } QueryResultFormat; /** diff --git a/lib/client.c b/lib/client.c index 9880113bab..729e2f4328 100644 --- a/lib/client.c +++ b/lib/client.c @@ -183,6 +183,9 @@ static SF_STATUS STDCALL _reset_connection_parameters( else if (strcmp(name->valuestring, "VARIANT_MAX_SIZE_IN_RESULT") == 0) { sf->max_variant_size = snowflake_cJSON_GetUint64Value(value); } + else if (strcmp(name->valuestring, "ENABLE_STAGE_S3_PRIVATELINK_FOR_US_EAST_1") == 0) { + sf->use_s3_regional_url = snowflake_cJSON_IsTrue(value) ? SF_BOOLEAN_TRUE : SF_BOOLEAN_FALSE; + } } } SF_STATUS ret = SF_STATUS_ERROR_GENERAL; @@ -534,6 +537,15 @@ _snowflake_check_connection_parameters(SF_CONNECT *sf) { log_debug("retry_count: %d", sf->retry_count); log_debug("qcc_disable: %s", sf->qcc_disable ? "true" : "false"); log_debug("include_retry_reason: %s", sf->include_retry_reason ? "true" : "false"); + log_debug("use_s3_regional_url: %s", sf->use_s3_regional_url ? "true" : "false"); + log_debug("put_use_urand_dev: %s", sf->put_use_urand_dev ? "true" : "false"); + log_debug("put_compress_level: %d", sf->put_compress_level); + log_debug("put_temp_dir: %s", sf->put_temp_dir ? sf->put_temp_dir : ""); + log_debug("put_fastfail: %s", sf->put_fastfail ? "true" : "false"); + log_debug("put_maxretries: %d", sf->put_maxretries); + log_debug("get_fastfail: %s", sf->get_fastfail ? "true" : "false"); + log_debug("get_maxretries: %d", sf->get_maxretries); + log_debug("get_threshold: %d", sf->get_threshold); return SF_STATUS_SUCCESS; } @@ -726,6 +738,16 @@ SF_CONNECT *STDCALL snowflake_init() { sf->max_varchar_size = SF_DEFAULT_MAX_OBJECT_SIZE; sf->max_binary_size = SF_DEFAULT_MAX_OBJECT_SIZE / 2; sf->max_variant_size = SF_DEFAULT_MAX_OBJECT_SIZE; + + sf->use_s3_regional_url = SF_BOOLEAN_FALSE; + sf->put_use_urand_dev = SF_BOOLEAN_FALSE; + sf->put_compress_level = SF_DEFAULT_PUT_COMPRESS_LEVEL; + sf->put_temp_dir = NULL; + sf->put_fastfail = SF_BOOLEAN_FALSE; + sf->put_maxretries = SF_DEFAULT_PUT_MAX_RETRIES; + sf->get_fastfail = SF_BOOLEAN_FALSE; + sf->get_maxretries = SF_DEFAULT_GET_MAX_RETRIES; + sf->get_threshold = SF_DEFAULT_GET_THRESHOLD; } return sf; @@ -1146,6 +1168,45 @@ SF_STATUS STDCALL snowflake_set_attribute( case SF_CON_INCLUDE_RETRY_REASON: sf->include_retry_reason = value ? *((sf_bool *)value) : SF_BOOLEAN_TRUE; break; + case SF_CON_PUT_TEMPDIR: + alloc_buffer_and_copy(&sf->put_temp_dir, value); + break; + case SF_CON_PUT_COMPRESSLV: + sf->put_compress_level = value ? *((int8 *)value) : SF_DEFAULT_PUT_COMPRESS_LEVEL; + if ((sf->put_compress_level > SF_MAX_PUT_COMPRESS_LEVEL) || + (sf->put_compress_level < 0)) + { + sf->put_compress_level = SF_DEFAULT_PUT_COMPRESS_LEVEL; + } + break; + case SF_CON_PUT_USE_URANDOM_DEV: + sf->put_use_urand_dev = value ? *((sf_bool *)value) : SF_BOOLEAN_FALSE; + break; + case SF_CON_PUT_FASTFAIL: + sf->put_fastfail = value ? *((sf_bool *)value) : SF_BOOLEAN_FALSE; + break; + case SF_CON_PUT_MAXRETRIES: + sf->put_maxretries = value ? *((int8 *)value) : SF_DEFAULT_PUT_MAX_RETRIES; + if ((sf->put_maxretries > SF_MAX_PUT_MAX_RETRIES) || + (sf->put_maxretries < 0)) + { + sf->put_maxretries = SF_DEFAULT_PUT_MAX_RETRIES; + } + break; + case SF_CON_GET_FASTFAIL: + sf->get_fastfail = value ? *((sf_bool *)value) : SF_BOOLEAN_FALSE; + break; + case SF_CON_GET_MAXRETRIES: + sf->get_maxretries = value ? *((int8 *)value) : SF_DEFAULT_GET_MAX_RETRIES; + if ((sf->get_maxretries > SF_MAX_GET_MAX_RETRIES) || + (sf->get_maxretries < 0)) + { + sf->get_maxretries = SF_DEFAULT_GET_MAX_RETRIES; + } + break; + case SF_CON_GET_THRESHOLD: + sf->get_threshold = value ? *((int64 *)value) : SF_DEFAULT_GET_THRESHOLD; + break; default: SET_SNOWFLAKE_ERROR(&sf->error, SF_STATUS_ERROR_BAD_ATTRIBUTE_TYPE, "Invalid attribute type", @@ -1292,6 +1353,30 @@ SF_STATUS STDCALL snowflake_get_attribute( case SF_CON_MAX_VARIANT_SIZE: *value = &sf->max_variant_size; break; + case SF_CON_PUT_TEMPDIR: + *value = sf->put_temp_dir; + break; + case SF_CON_PUT_COMPRESSLV: + *value = &sf->put_compress_level; + break; + case SF_CON_PUT_USE_URANDOM_DEV: + *value = &sf->put_use_urand_dev; + break; + case SF_CON_PUT_FASTFAIL: + *value = &sf->put_fastfail; + break; + case SF_CON_PUT_MAXRETRIES: + *value = &sf->put_maxretries; + break; + case SF_CON_GET_FASTFAIL: + *value = &sf->get_fastfail; + break; + case SF_CON_GET_MAXRETRIES: + *value = &sf->get_maxretries; + break; + case SF_CON_GET_THRESHOLD: + *value = &sf->get_threshold; + break; default: SET_SNOWFLAKE_ERROR(&sf->error, SF_STATUS_ERROR_BAD_ATTRIBUTE_TYPE, "Invalid attribute type", @@ -2039,6 +2124,29 @@ SF_STATUS STDCALL snowflake_query( return SF_STATUS_SUCCESS; } +SF_STATUS STDCALL _snowflake_query_put_get_legacy( + SF_STMT *sfstmt, const char *command, size_t command_size) { + if (!sfstmt) { + return SF_STATUS_ERROR_STATEMENT_NOT_EXIST; + } + clear_snowflake_error(&sfstmt->error); + SF_STATUS ret = snowflake_prepare(sfstmt, command, command_size); + if (ret != SF_STATUS_SUCCESS) { + return ret; + } + if (!_is_put_get_command(sfstmt->sql_text)) + { + // this should never happen as this function should only be + // called internally for put/get command. + SET_SNOWFLAKE_ERROR(&sfstmt->error, SF_STATUS_ERROR_GENERAL, + "Invalid query type, can be used for put get only", + SF_SQLSTATE_GENERAL_ERROR); + return SF_STATUS_ERROR_GENERAL; + } + + return _snowflake_execute_ex(sfstmt, SF_BOOLEAN_TRUE, SF_BOOLEAN_FALSE, NULL, SF_BOOLEAN_FALSE); +} + SF_STATUS STDCALL snowflake_fetch(SF_STMT *sfstmt) { if (!sfstmt) { return SF_STATUS_ERROR_STATEMENT_NOT_EXIST; @@ -2246,24 +2354,32 @@ snowflake_prepare(SF_STMT *sfstmt, const char *command, size_t command_size) { SF_STATUS STDCALL snowflake_describe_with_capture(SF_STMT *sfstmt, SF_QUERY_RESULT_CAPTURE *result_capture) { - return _snowflake_execute_ex(sfstmt, _is_put_get_command(sfstmt->sql_text), result_capture, SF_BOOLEAN_TRUE); + return _snowflake_execute_ex(sfstmt, _is_put_get_command(sfstmt->sql_text), SF_BOOLEAN_FALSE, result_capture, SF_BOOLEAN_TRUE); } SF_STATUS STDCALL snowflake_execute(SF_STMT *sfstmt) { - return _snowflake_execute_ex(sfstmt, _is_put_get_command(sfstmt->sql_text), NULL, SF_BOOLEAN_FALSE); + return _snowflake_execute_ex(sfstmt, _is_put_get_command(sfstmt->sql_text), SF_BOOLEAN_TRUE, NULL, SF_BOOLEAN_FALSE); } SF_STATUS STDCALL snowflake_execute_with_capture(SF_STMT *sfstmt, SF_QUERY_RESULT_CAPTURE *result_capture) { - return _snowflake_execute_ex(sfstmt, _is_put_get_command(sfstmt->sql_text), result_capture, SF_BOOLEAN_FALSE); + return _snowflake_execute_ex(sfstmt, _is_put_get_command(sfstmt->sql_text), SF_BOOLEAN_TRUE, result_capture, SF_BOOLEAN_FALSE); } SF_STATUS STDCALL _snowflake_execute_ex(SF_STMT *sfstmt, sf_bool is_put_get_command, + sf_bool is_native_put_get, SF_QUERY_RESULT_CAPTURE* result_capture, sf_bool is_describe_only) { if (!sfstmt) { return SF_STATUS_ERROR_STATEMENT_NOT_EXIST; } + + if (is_put_get_command && is_native_put_get && !is_describe_only) + { + _snowflake_stmt_desc_reset(sfstmt); + return _snowflake_execute_put_get_native(sfstmt, result_capture); + } + clear_snowflake_error(&sfstmt->error); SF_STATUS ret = SF_STATUS_ERROR_GENERAL; SF_JSON_ERROR json_error; diff --git a/lib/client_int.h b/lib/client_int.h index 632856f031..47db68c595 100644 --- a/lib/client_int.h +++ b/lib/client_int.h @@ -25,6 +25,15 @@ #define SF_DEFAULT_MAX_OBJECT_SIZE 16777216 +// defaults for put get configurations +#define SF_DEFAULT_PUT_COMPRESS_LEVEL (-1) +#define SF_MAX_PUT_COMPRESS_LEVEL 9 +#define SF_DEFAULT_PUT_MAX_RETRIES 5 +#define SF_MAX_PUT_MAX_RETRIES 100 +#define SF_DEFAULT_GET_MAX_RETRIES 5 +#define SF_MAX_GET_MAX_RETRIES 100 +#define SF_DEFAULT_GET_THRESHOLD 5 + #define SESSION_URL "/session/v1/login-request" #define QUERY_URL "/queries/v1/query-request" #define RENEW_SESSION_URL "/session/token-request" @@ -141,7 +150,7 @@ SF_PUT_GET_RESPONSE *STDCALL sf_put_get_response_allocate(); /** * Executes a statement. * @param sfstmt SNOWFLAKE_STMT context. - * @param sf_use_application_json_accept type true if this is a put/get command + * @param is_put_get_command type true if this is a put/get command * @param raw_response_buffer optional pointer to an SF_QUERY_RESULT_CAPTURE, * @param is_describe_only should the statement be executed in describe only mode * if the query response is to be captured. @@ -149,7 +158,8 @@ SF_PUT_GET_RESPONSE *STDCALL sf_put_get_response_allocate(); * @return 0 if success, otherwise an errno is returned. */ SF_STATUS STDCALL _snowflake_execute_ex(SF_STMT *sfstmt, - sf_bool use_application_json_accept_type, + sf_bool is_put_get_command, + sf_bool is_native_put_get, struct SF_QUERY_RESULT_CAPTURE* result_capture, sf_bool is_describe_only); @@ -163,4 +173,32 @@ sf_bool STDCALL _is_put_get_command(char* sql_text); */ PARAM_TYPE STDCALL _snowflake_get_param_style(const SF_BIND_INPUT *input); +#ifdef __cplusplus +extern "C" { +#endif +/** + * Legacy approach of Executing a query, not to execute put/get natively. + * Should only be called internally for put/get queries. + * + * @param sf SNOWFLAKE_STMT context. + * @param command a query or command that returns results. + * @return 0 if success, otherwise an errno is returned. + */ +SF_STATUS STDCALL +_snowflake_query_put_get_legacy(SF_STMT* sfstmt, const char* command, size_t command_size); + +/** + * Executes put get command natively. + * @param sfstmt SNOWFLAKE_STMT context. + * @param raw_response_buffer optional pointer to an SF_QUERY_RESULT_CAPTURE, + * + * @return 0 if success, otherwise an errno is returned. + */ +SF_STATUS STDCALL _snowflake_execute_put_get_native( + SF_STMT *sfstmt, + struct SF_QUERY_RESULT_CAPTURE* result_capture); +#ifdef __cplusplus +} // extern "C" +#endif + #endif //SNOWFLAKE_CLIENT_INT_H diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 2e1254ff77..6e55a0d063 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -195,7 +195,7 @@ if (LINUX) endif () message("valgrind suppression file is located at " ${VALGRIND_SUPPRESSION}) if (LINUX) - set(TESTLIB_OPTS_C snowflakeclient ${CMOCKA_LIB} ${ARROW_ALL_LIBS} ${AZURE_STORAGE_LITE_LIB} -Wl,--whole-archive telemetry curl ssl crypto uuid + set(TESTLIB_OPTS_C snowflakeclient ${CMOCKA_LIB} ${ARROW_ALL_LIBS} ${AZURE_STORAGE_LITE_LIB} ${AWS_ALL_LIBS} -Wl,--whole-archive telemetry curl ssl crypto uuid -Wl,--no-whole-archive pthread -Wl,--as-needed -static-libgcc -static-libstdc++) set(TESTLIB_OPTS_CXX snowflakeclient ${CMOCKA_LIB} ${ARROW_ALL_LIBS} ${AZURE_STORAGE_LITE_LIB} ${AWS_ALL_LIBS} -Wl,--whole-archive telemetry curl ssl crypto uuid pthread -Wl,--no-whole-archive -Wl,--as-needed -static-libgcc -static-libstdc++) @@ -209,12 +209,14 @@ endif() if (WIN32) if (WIN32_DEBUG) if(CMAKE_SIZEOF_VOID_P EQUAL 4) - set(TESTLIB_OPTS_C snowflakeclient ${CMOCKA_LIB} ${ARROW_ALL_LIBS} ${OOB_LIB} ${CURL_LIB} ${SSL_LIB} ${CRYPTO_LIB} ${ZLIB_LIB} ucrtd.lib) + set(TESTLIB_OPTS_C snowflakeclient ${CMOCKA_LIB} ${ARROW_ALL_LIBS} ${AWS_ALL_LIBS} ${OOB_LIB} ${CURL_LIB} ${SSL_LIB} ${CRYPTO_LIB} ${ZLIB_LIB} ${AZURE_STORAGE_LITE_LIB} + Version.lib Userenv.lib Bcrypt.lib ucrtd.lib Secur32.lib Ncrypt.lib Shlwapi.lib) set(TESTLIB_OPTS_CXX snowflakeclient ${CMOCKA_LIB} ${ARROW_ALL_LIBS} ${AWS_ALL_LIBS} ${OOB_LIB} ${CURL_LIB} ${SSL_LIB} ${CRYPTO_LIB} ${ZLIB_LIB} ${AZURE_STORAGE_LITE_LIB} Version.lib Userenv.lib Bcrypt.lib ucrtd.lib Secur32.lib Ncrypt.lib Shlwapi.lib) endif() else() - set(TESTLIB_OPTS_C snowflakeclient ${CMOCKA_LIB} ${ARROW_ALL_LIBS} ${OOB_LIB} ${CURL_LIB} ${SSL_LIB} ${CRYPTO_LIB} ${ZLIB_LIB}) + set(TESTLIB_OPTS_C snowflakeclient ${CMOCKA_LIB} ${AWS_ALL_LIBS} ${AZURE_STORAGE_LITE_LIB} ${ARROW_ALL_LIBS} ${OOB_LIB} ${CURL_LIB} ${SSL_LIB} ${CRYPTO_LIB} ${ZLIB_LIB} + Version.lib Userenv.lib Bcrypt.lib Secur32.lib Ncrypt.lib Shlwapi.lib) set(TESTLIB_OPTS_CXX snowflakeclient ${CMOCKA_LIB} ${AWS_ALL_LIBS} ${AZURE_STORAGE_LITE_LIB} ${ARROW_ALL_LIBS} ${CURL_LIB} ${OOB_LIB} ${SSL_LIB} ${CRYPTO_LIB} ${ZLIB_LIB} Version.lib Userenv.lib Bcrypt.lib Secur32.lib Ncrypt.lib Shlwapi.lib) endif() diff --git a/tests/test_simple_put.cpp b/tests/test_simple_put.cpp index c16a3f98c2..4096263221 100755 --- a/tests/test_simple_put.cpp +++ b/tests/test_simple_put.cpp @@ -118,7 +118,8 @@ void test_simple_put_core(const char * fileName, int compressLevel = -1, bool overwrite = false, SF_CONNECT * connection = nullptr, - bool testUnicode = false) + bool testUnicode = false, + bool native = false) { /* init */ SF_STATUS status; @@ -194,71 +195,140 @@ void test_simple_put_core(const char * fileName, putCommand += " overwrite=true"; } std::unique_ptr stmtPutGet; - if (testUnicode) - { - stmtPutGet = std::unique_ptr - (new Snowflake::Client::StatementPutGetUnicode(sfstmt)); - } - else + if (!native) { - stmtPutGet = std::unique_ptr - (new Snowflake::Client::StatementPutGet(sfstmt)); + if (testUnicode) + { + stmtPutGet = std::unique_ptr + (new Snowflake::Client::StatementPutGetUnicode(sfstmt)); + } + else + { + stmtPutGet = std::unique_ptr + (new Snowflake::Client::StatementPutGet(sfstmt)); + } } TransferConfig transConfig; TransferConfig * transConfigPtr = nullptr; if (tmpDir) { + if (native) + { + snowflake_set_attribute(sf, SF_CON_PUT_TEMPDIR, tmpDir); + } + else + { transConfig.tempDir = tmpDir; transConfigPtr = &transConfig; + } } if(useS3regionalUrl) { - transConfig.useS3regionalUrl = true; - transConfigPtr = &transConfig; + if (native) + { + std::string cmd = "alter session set ENABLE_STAGE_S3_PRIVATELINK_FOR_US_EAST_1=true"; + snowflake_query(sfstmt, cmd.c_str(), cmd.size()); + } + else + { + transConfig.useS3regionalUrl = true; + transConfigPtr = &transConfig; + } } if(compressLevel > 0) { - transConfig.compressLevel = compressLevel; - transConfigPtr = &transConfig; + if (native) + { + int8 lv = (int8)compressLevel; + snowflake_set_attribute(sf, SF_CON_PUT_COMPRESSLV, &lv); + } + else + { + transConfig.compressLevel = compressLevel; + transConfigPtr = &transConfig; + } } Snowflake::Client::FileTransferAgent agent(stmtPutGet.get(), transConfigPtr); if(useDevUrand){ - agent.setRandomDeviceAsUrand(true); + if (native) + { + sf_bool use_urand = SF_BOOLEAN_TRUE; + snowflake_set_attribute(sf, SF_CON_PUT_USE_URANDOM_DEV, &use_urand); + } + else + { + agent.setRandomDeviceAsUrand(true); + } } - ITransferResult * results = agent.execute(&putCommand); - assert_int_equal(1, results->getResultSize()); + std::string expectedSrc = sf_filename_from_path(fileName); + std::string expectedTarget = (autoCompress && !strstr(expectedSrc.c_str(), ".gz")) ? + expectedSrc + ".gz" : expectedSrc; + std::string expectedSourceCompression = !strstr(fileName, ".gz") ? "none" : "gzip"; + std::string expectedTargetCompression = (!autoCompress && !strstr(fileName, ".gz")) ? "none" : "gzip"; - while(results->next()) + if (native) { - std::string value; - results->getColumnAsString(0, value); // source - assert_string_equal( sf_filename_from_path(fileName), value.c_str()); - - std::string expectedTarget = (autoCompress && !strstr(fileName, ".gz")) ? - std::string(fileName) + ".gz" : - std::string(fileName); - results->getColumnAsString(1, value); // get target - assert_string_equal(sf_filename_from_path(expectedTarget.c_str()), value.c_str()); - - std::string expectedSourceCompression = !strstr(fileName, ".gz") ? - "none" : "gzip"; - results->getColumnAsString(4, value); // get source_compression - assert_string_equal(expectedSourceCompression.c_str(), value.c_str()); - - std::string expectedTargetCompression = (!autoCompress && - !strstr(fileName, ".gz")) ? "none" : "gzip"; - results->getColumnAsString(5, value); // get target_compression - assert_string_equal(expectedTargetCompression.c_str(), value.c_str()); - - results->getColumnAsString(6, value); // get encryption - assert_string_equal("UPLOADED", value.c_str()); - - results->getColumnAsString(7, value); // get encryption - assert_string_equal("ENCRYPTED", value.c_str()); + ret = snowflake_query(sfstmt, putCommand.c_str(), putCommand.size()); + assert_int_equal(SF_STATUS_SUCCESS, ret); + + assert_int_equal(snowflake_num_rows(sfstmt), 1); + + ret = snowflake_fetch(sfstmt); + assert_int_equal(SF_STATUS_SUCCESS, ret); + + const char *out; + // source + snowflake_column_as_const_str(sfstmt, 1, &out); + assert_string_equal(expectedSrc.c_str(), out); + // target + snowflake_column_as_const_str(sfstmt, 2, &out); + assert_string_equal(expectedTarget.c_str(), out); + // source comparession + snowflake_column_as_const_str(sfstmt, 5, &out); + assert_string_equal(expectedSourceCompression.c_str(), out); + // target compression + snowflake_column_as_const_str(sfstmt, 6, &out); + assert_string_equal(expectedTargetCompression.c_str(), out); + snowflake_column_as_const_str(sfstmt, 7, &out); + // status + assert_string_equal("UPLOADED", out); + // encryption + snowflake_column_as_const_str(sfstmt, 8, &out); + assert_string_equal("ENCRYPTED", out); + + ret = snowflake_fetch(sfstmt); + assert_int_equal(SF_STATUS_EOF, ret); + } + else + { + ITransferResult * results = agent.execute(&putCommand); + assert_int_equal(1, results->getResultSize()); + + while(results->next()) + { + std::string value; + results->getColumnAsString(0, value); // source + assert_string_equal(expectedSrc.c_str(), value.c_str()); + + results->getColumnAsString(1, value); // get target + assert_string_equal(expectedTarget.c_str(), value.c_str()); + + results->getColumnAsString(4, value); // get source_compression + assert_string_equal(expectedSourceCompression.c_str(), value.c_str()); + + results->getColumnAsString(5, value); // get target_compression + assert_string_equal(expectedTargetCompression.c_str(), value.c_str()); + + results->getColumnAsString(6, value); // get encryption + assert_string_equal("UPLOADED", value.c_str()); + + results->getColumnAsString(7, value); // get encryption + assert_string_equal("ENCRYPTED", value.c_str()); + } } if (copyUploadFile) @@ -352,7 +422,8 @@ static int teardown(void **unused) } void test_simple_get_data(const char *getCommand, const char *size, - long getThreshold = 0, bool testUnicode = false) + long getThreshold = 0, bool testUnicode = false, + bool native = false) { /* init */ SF_STATUS status; @@ -367,39 +438,72 @@ void test_simple_get_data(const char *getCommand, const char *size, sfstmt = snowflake_stmt(sf); std::unique_ptr stmtPutGet; - if (testUnicode) + if (!native) { - stmtPutGet = std::unique_ptr - (new Snowflake::Client::StatementPutGetUnicode(sfstmt)); - } - else - { - stmtPutGet = std::unique_ptr - (new Snowflake::Client::StatementPutGet(sfstmt)); + if (testUnicode) + { + stmtPutGet = std::unique_ptr + (new Snowflake::Client::StatementPutGetUnicode(sfstmt)); + } + else + { + stmtPutGet = std::unique_ptr + (new Snowflake::Client::StatementPutGet(sfstmt)); + } } TransferConfig transConfig; TransferConfig * transConfigPtr = nullptr; if (getThreshold > 0) { + if (native) + { + int64 threshold = getThreshold; + status = snowflake_set_attribute(sf, SF_CON_GET_THRESHOLD, &threshold); + assert_int_equal(SF_STATUS_SUCCESS, status); + } + else + { transConfig.getSizeThreshold = getThreshold; transConfigPtr = &transConfig; + } } - Snowflake::Client::FileTransferAgent agent(stmtPutGet.get(), transConfigPtr); - - // load first time should return uploaded - std::string get_status; - std::string getcmd(getCommand); - ITransferResult * results = agent.execute(&getcmd); - while(results && results->next()) + if (native) { - results->getColumnAsString(1, get_status); + status = snowflake_query(sfstmt, getCommand, strlen(getCommand)); + assert_int_equal(SF_STATUS_SUCCESS, status); + + while ((status = snowflake_fetch(sfstmt)) == SF_STATUS_SUCCESS) + { + const char *out; + // source + snowflake_column_as_const_str(sfstmt, 2, &out); //Compressed File sizes vary on Windows/Linux, So not verifying size. - results->getColumnAsString(2, get_status); - assert_string_equal("DOWNLOADED", get_status.c_str()); - results->getColumnAsString(3, get_status); - assert_string_equal("DECRYPTED", get_status.c_str()); + snowflake_column_as_const_str(sfstmt, 3, &out); + assert_string_equal("DOWNLOADED", out); + snowflake_column_as_const_str(sfstmt, 4, &out); + assert_string_equal("DECRYPTED", out); + } + assert_int_equal(SF_STATUS_EOF, status); + } + else + { + Snowflake::Client::FileTransferAgent agent(stmtPutGet.get(), transConfigPtr); + + // load first time should return uploaded + std::string get_status; + std::string getcmd(getCommand); + ITransferResult * results = agent.execute(&getcmd); + while(results && results->next()) + { + results->getColumnAsString(1, get_status); + //Compressed File sizes vary on Windows/Linux, So not verifying size. + results->getColumnAsString(2, get_status); + assert_string_equal("DOWNLOADED", get_status.c_str()); + results->getColumnAsString(3, get_status); + assert_string_equal("DECRYPTED", get_status.c_str()); + } } snowflake_stmt_term(sfstmt); @@ -409,7 +513,7 @@ void test_simple_get_data(const char *getCommand, const char *size, } -void test_large_put_auto_compress(void **unused) +void test_large_put_auto_compress_core(bool native) { char *cenv = getenv("CLOUD_PROVIDER"); if ( cenv && !strncmp(cenv, "AWS", 4) ) { @@ -423,10 +527,32 @@ void test_large_put_auto_compress(void **unused) false, // auto compress true, // Load data into table false, // Run select * on loaded table (Not good for large data set) - true // copy data from Table to Staging. + true, // copy data from Table to Staging. + false, // createDupTable + false, // setCustomThreshold + 64*1024*1024, // customThreshold + false, // useDevUrand + false, // createSubfolder + nullptr, // tmpDir + false, // useS3regionalUrl + -1, //compressLevel + false, // overwrite + nullptr, // connection + false, // testUnicode + native ); } +void test_large_put_auto_compress(void **unused) +{ + test_large_put_auto_compress_core(false); +} + +void test_large_put_auto_compress_native(void **unused) +{ + test_large_put_auto_compress_core(true); +} + void test_large_put_threshold(void **unused) { char *cenv = getenv("CLOUD_PROVIDER"); @@ -448,10 +574,10 @@ void test_large_put_threshold(void **unused) ); } -void test_large_reupload(void **unused) +void test_large_reupload_core(bool native) { - char *cenv = getenv("CLOUD_PROVIDER"); - if (cenv && !strncmp(cenv, "AWS", 4)) { + char *cenv = getenv("CLOUD_PROVIDER"); + if (cenv && !strncmp(cenv, "AWS", 4)) { errno = 0; return; } @@ -476,7 +602,7 @@ void test_large_reupload(void **unused) char tempFile[MAX_BUF_SIZE] ={0}; sf_get_tmp_dir(tempDir); #ifdef _WIN32 - getLongTempPath(tempDir); + getLongTempPath(tempDir); #endif for(const std::string s : fileList) { tempFile[0] = 0; @@ -487,10 +613,30 @@ void test_large_reupload(void **unused) true, // Load data into table false, // Run select * on loaded table (Not good for large data set) false, // copy data from Table to Staging. - true //Creates a dup table to compare uploaded data. + true, //Creates a dup table to compare uploaded data. + false, // setCustomThreshold + 64*1024*1024, // customThreshold + false, // useDevUrand + false, // createSubfolder + nullptr, // tmpDir + false, // useS3regionalUrl + -1, //compressLevel + false, // overwrite + nullptr, // connection + false, // testUnicode + native ); } +} +void test_large_reupload(void** unused) +{ + test_large_reupload_core(false); +} + +void test_large_reupload_native(void** unused) +{ + test_large_reupload_core(true); } /* @@ -538,7 +684,7 @@ void test_verify_upload(void **unused) } -void test_simple_put_use_dev_urandom(void **unused) +void test_simple_put_use_dev_urandom_core(bool native) { std::string dataDir = TestSetup::getDataDir(); std::string file = dataDir + "medium_file.csv"; @@ -567,32 +713,67 @@ void test_simple_put_use_dev_urandom(void **unused) std::string putCommand = "put file://" + file + " @%test_small_put auto_compress=true overwrite=true"; - std::unique_ptr stmtPutGet = std::unique_ptr - (new Snowflake::Client::StatementPutGet(sfstmt)); - - Snowflake::Client::FileTransferAgent agent(stmtPutGet.get()); - agent.setRandomDeviceAsUrand(true); + if (native) + { + sf_bool useUrand = SF_BOOLEAN_TRUE; + ret = snowflake_set_attribute(sf, SF_CON_PUT_USE_URANDOM_DEV, &useUrand); + assert_int_equal(SF_STATUS_SUCCESS, ret); + ret = snowflake_query(sfstmt, putCommand.c_str(), putCommand.size()); + assert_int_equal(SF_STATUS_SUCCESS, ret); + assert_int_equal(snowflake_num_rows(sfstmt), 1); - ITransferResult * results = agent.execute(&putCommand); - assert_int_equal(1, results->getResultSize()); + ret = snowflake_fetch(sfstmt); + assert_int_equal(SF_STATUS_SUCCESS, ret); - while (results->next()) + const char* out = NULL; + // source + snowflake_column_as_const_str(sfstmt, 1, &out); + assert_string_equal(sf_filename_from_path(file.c_str()), out); + // target + snowflake_column_as_const_str(sfstmt, 2, &out); + assert_string_equal("medium_file.csv.gz", out); + // source compression + snowflake_column_as_const_str(sfstmt, 5, &out); + assert_string_equal("none", out); + // status + snowflake_column_as_const_str(sfstmt, 7, &out); + assert_string_equal("UPLOADED", out); + // encryption + snowflake_column_as_const_str(sfstmt, 8, &out); + assert_string_equal("ENCRYPTED", out); + + ret = snowflake_fetch(sfstmt); + assert_int_equal(SF_STATUS_EOF, ret); + } + else { - std::string value; - results->getColumnAsString(0, value); // source - assert_string_equal(sf_filename_from_path(file.c_str()), value.c_str()); + std::unique_ptr stmtPutGet = std::unique_ptr + (new Snowflake::Client::StatementPutGet(sfstmt)); - results->getColumnAsString(1, value); // get target - assert_string_equal("medium_file.csv.gz", value.c_str()); + Snowflake::Client::FileTransferAgent agent(stmtPutGet.get()); + agent.setRandomDeviceAsUrand(true); + + ITransferResult * results = agent.execute(&putCommand); + assert_int_equal(1, results->getResultSize()); + + while (results->next()) + { + std::string value; + results->getColumnAsString(0, value); // source + assert_string_equal(sf_filename_from_path(file.c_str()), value.c_str()); - results->getColumnAsString(4, value); // get source_compression - assert_string_equal("none", value.c_str()); + results->getColumnAsString(1, value); // get target + assert_string_equal("medium_file.csv.gz", value.c_str()); - results->getColumnAsString(6, value); // get encryption - assert_string_equal("UPLOADED", value.c_str()); + results->getColumnAsString(4, value); // get source_compression + assert_string_equal("none", value.c_str()); - results->getColumnAsString(7, value); // get encryption - assert_string_equal("ENCRYPTED", value.c_str()); + results->getColumnAsString(6, value); // get encryption + assert_string_equal("UPLOADED", value.c_str()); + + results->getColumnAsString(7, value); // get encryption + assert_string_equal("ENCRYPTED", value.c_str()); + } } std::string copyCommand = "copy into test_small_put from @%test_small_put"; @@ -610,30 +791,56 @@ void test_simple_put_use_dev_urandom(void **unused) for(int i = 0; i < 200000; ++i) { - ret = snowflake_fetch(sfstmt); - assert_int_equal(SF_STATUS_SUCCESS, ret); - snowflake_column_as_const_str(sfstmt, 1, &out_c1); - snowflake_column_as_const_str(sfstmt, 2, &out_c2); - snowflake_column_as_const_str(sfstmt, 3, &out_c3); - std::string c1 = std::to_string(i); - std::string c2 = std::to_string(i + 1); - assert_string_equal(out_c1, c1.c_str()); - assert_string_equal(out_c2, c2.c_str()); - assert_string_equal(out_c3, "ABCDEFGHIJKLMNOPQRSTUVWXYZ"); - out_c1 = NULL; - out_c2 = NULL; - out_c3 = NULL; + ret = snowflake_fetch(sfstmt); + assert_int_equal(SF_STATUS_SUCCESS, ret); + snowflake_column_as_const_str(sfstmt, 1, &out_c1); + snowflake_column_as_const_str(sfstmt, 2, &out_c2); + snowflake_column_as_const_str(sfstmt, 3, &out_c3); + std::string c1 = std::to_string(i); + std::string c2 = std::to_string(i + 1); + assert_string_equal(out_c1, c1.c_str()); + assert_string_equal(out_c2, c2.c_str()); + assert_string_equal(out_c3, "ABCDEFGHIJKLMNOPQRSTUVWXYZ"); + out_c1 = NULL; + out_c2 = NULL; + out_c3 = NULL; } ret = snowflake_fetch(sfstmt); assert_int_equal(SF_STATUS_EOF, ret); } -void test_simple_put_auto_compress(void **unused) +void test_simple_put_use_dev_urandom(void **unused) +{ + test_simple_put_use_dev_urandom_core(false); +} + +void test_simple_put_use_dev_urandom_native(void **unused) +{ + test_simple_put_use_dev_urandom_core(true); +} + + +void test_simple_put_auto_compress_core(bool native) { test_simple_put_core("small_file.csv", // filename "auto", //source compression - true // auto compress + true, // auto compress + true, // copyUploadFile + true, // verifyCopyUploadFile + false, // copyTableToStaging + false, // createDupTable + false, // setCustomThreshold + 64*1024*1024, // customThreshold + false, // useDevUrand + false, // createSubfolder + nullptr, // tmpDir + false, // useS3regionalUrl + -1, //compressLevel + false, // overwrite + nullptr, // connection + false, // testUnicode + native ); test_simple_put_core("small_file.csv", // filename @@ -650,7 +857,11 @@ void test_simple_put_auto_compress(void **unused) nullptr, false, 1, - true); + true, // overwrite + nullptr, // connection + false, // testUnicode + native + ); test_simple_put_core("small_file.csv", // filename "auto", //source compression @@ -666,10 +877,24 @@ void test_simple_put_auto_compress(void **unused) nullptr, false, 9, - true); + true, // overwrite + nullptr, // connection + false, // testUnicode + native + ); } -void test_simple_put_config_temp_dir(void **unused) +void test_simple_put_auto_compress(void **unused) +{ + test_simple_put_auto_compress_core(false); +} + +void test_simple_put_auto_compress_native(void **unused) +{ + test_simple_put_auto_compress_core(true); +} + +void test_simple_put_config_temp_dir_core(bool native) { char tmpDir[MAX_PATH] = {0}; char tmpDirInjection[MAX_PATH] = {0}; @@ -696,7 +921,13 @@ void test_simple_put_config_temp_dir(void **unused) 64*1024*1024, // customThreshold false, // useDevUrand false, // createSubfolder - tmpDir + tmpDir, + false, // useS3regionalUrl + -1, //compressLevel + false, // overwrite + nullptr, // connection + false, // testUnicode + native ); assert_true(sf_is_directory_exist(tmpDir)); @@ -721,31 +952,51 @@ void test_simple_put_config_temp_dir(void **unused) sf_delete_directory_if_exists(tmpDir); } - // try injection the command for folder deletion like - // rm -rf xxx ; mkdir /injection ; xxx - sprintf(tmpDirInjection, "xxx %s mkdir %s %s xxx", - CMD_SEPARATOR, tmpDir, CMD_SEPARATOR); - try + // native execution doesn't throw exception + if (!native) { - test_simple_put_core("small_file.csv", // filename - "auto", //source compression - true, // auto compress - true, // copyUploadFile - true, // verifyCopyUploadFile - false, // copyTableToStaging - false, // createDupTable - false, // setCustomThreshold - 64*1024*1024, // customThreshold - false, // useDevUrand - false, // createSubfolder - tmpDirInjection - ); - } - catch (...) - { - //ignore exception as the failure is expected. - } - assert_false(sf_is_directory_exist(tmpDir)); + // try injection the command for folder deletion like + // rm -rf xxx ; mkdir /injection ; xxx + sprintf(tmpDirInjection, "xxx %s mkdir %s %s xxx", + CMD_SEPARATOR, tmpDir, CMD_SEPARATOR); + try + { + test_simple_put_core("small_file.csv", // filename + "auto", //source compression + true, // auto compress + true, // copyUploadFile + true, // verifyCopyUploadFile + false, // copyTableToStaging + false, // createDupTable + false, // setCustomThreshold + 64*1024*1024, // customThreshold + false, // useDevUrand + false, // createSubfolder + tmpDirInjection, + false, // useS3regionalUrl + -1, //compressLevel + false, // overwrite + nullptr, // connection + false, // testUnicode + native + ); + } + catch (...) + { + //ignore exception as the failure is expected. + } + assert_false(sf_is_directory_exist(tmpDir)); + } +} + +void test_simple_put_config_temp_dir(void **unused) +{ + test_simple_put_config_temp_dir_core(false); +} + +void test_simple_put_config_temp_dir_native(void **unused) +{ + test_simple_put_config_temp_dir_core(true); } void test_simple_put_auto_detect_gzip(void ** unused) @@ -789,7 +1040,7 @@ void test_simple_put_create_subfolder(void **unused) test_simple_put_core("small_file.csv.gz", "gzip", false, false, false, false, false, false, 100*1024*1024, false, true); } -void test_simple_put_use_s3_regionalURL(void **unused) +void test_simple_put_use_s3_regionalURL_core(bool native) { test_simple_put_core("small_file.csv.gz", "gzip", false,false, false, @@ -800,30 +1051,76 @@ void test_simple_put_use_s3_regionalURL(void **unused) false, false, nullptr, - true); + true, // useS3regionalUrl + -1, //compressLevel + false, // overwrite + nullptr, // connection + false, // testUnicode + native + ); } -void test_simple_get(void **unused) +void test_simple_put_use_s3_regionalURL(void **unused) +{ + test_simple_put_use_s3_regionalURL_core(false); +} + +void test_simple_put_use_s3_regionalURL_native(void **unused) +{ + test_simple_put_use_s3_regionalURL_core(true); +} + +void test_simple_get_core(bool native) { test_simple_put_core("small_file.csv", // filename "auto", //source compression - true // auto compress + true, // auto compress + true, // copyUploadFile + true, // verifyCopyUploadFile + false, // copyTableToStaging + false, // createDupTable + false, // setCustomThreshold + 64*1024*1024, // customThreshold + false, // useDevUrand + false, // createSubfolder + nullptr, // tmpDir + false, // useS3regionalUrl + -1, //compressLevel + false, // overwrite + nullptr, // connection + false, // testUnicode + native ); char tempDir[MAX_BUF_SIZE] = { 0 }; - char tempPath[MAX_BUF_SIZE] = "get @%test_small_put/small_file.csv.gz file://"; + char command[MAX_BUF_SIZE] = "get @%test_small_put/small_file.csv.gz file://"; sf_get_tmp_dir(tempDir); #ifdef _WIN32 getLongTempPath(tempDir); #endif - strcat(tempPath, tempDir); - test_simple_get_data(tempPath, "48"); + strcat(command, tempDir); + test_simple_get_data(command, // getCommand + "48", // size + 0, // getThreshold + false, // testUnicode + native + ); } -void test_large_get(void **unused) +void test_simple_get(void **unused) +{ + test_simple_get_core(false); +} + +void test_simple_get_native(void **unused) +{ + test_simple_get_core(true); +} + +void test_large_get_core(bool native) { char tempDir[MAX_BUF_SIZE] = { 0 }; - char tempPath[MAX_BUF_SIZE] = "get @%test_small_put/bigFile.csv.gz file://"; + char command[MAX_BUF_SIZE] = "get @%test_small_put/bigFile.csv.gz file://"; if ( ! strncmp(getenv("CLOUD_PROVIDER"), "AWS", 6) ) { errno = 0; return; @@ -832,14 +1129,29 @@ void test_large_get(void **unused) #ifdef _WIN32 getLongTempPath(tempDir); #endif - strcat(tempPath, tempDir); - test_simple_get_data(tempPath, "5166848"); + strcat(command, tempDir); + test_simple_get_data(command, // getCommand + "5166848", // size + 0, // getThreshold + false, // testUnicode + native + ); } -void test_large_get_threshold(void **unused) +void test_large_get(void **unused) +{ + test_large_get_core(false); +} + +void test_large_get_native(void **unused) +{ + test_large_get_core(true); +} + +void test_large_get_threshold_core(bool native) { char tempDir[MAX_BUF_SIZE] = { 0 }; - char tempPath[MAX_BUF_SIZE] = "get @%test_small_put/bigFile.csv.gz file://"; + char command[MAX_BUF_SIZE] = "get @%test_small_put/bigFile.csv.gz file://"; if ( ! strncmp(getenv("CLOUD_PROVIDER"), "AWS", 6) ) { errno = 0; return; @@ -848,8 +1160,23 @@ void test_large_get_threshold(void **unused) #ifdef _WIN32 getLongTempPath(tempDir); #endif - strcat(tempPath, tempDir); - test_simple_get_data(tempPath, "5166848", 1000000); + strcat(command, tempDir); + test_simple_get_data(command, // getCommand + "5166848", // size + 1000000, // getThreshold + false, // testUnicode + native + ); +} + +void test_large_get_threshold(void **unused) +{ + test_large_get_threshold_core(false); +} + +void test_large_get_threshold_native(void **unused) +{ + test_large_get_threshold_core(true); } static int gr_setup(void **unused) @@ -1654,6 +1981,7 @@ int main(void) { #endif const struct CMUnitTest tests[] = { +/* cmocka_unit_test_teardown(test_simple_put_auto_compress, teardown), cmocka_unit_test_teardown(test_simple_put_config_temp_dir, teardown), cmocka_unit_test_teardown(test_simple_put_auto_detect_gzip, teardown), @@ -1685,6 +2013,17 @@ int main(void) { cmocka_unit_test_teardown(test_simple_put_with_noproxy_fromenv, teardown), cmocka_unit_test_teardown(test_upload_file_to_stage_using_stream, donothing), cmocka_unit_test_teardown(test_put_get_with_unicode, teardown), +*/ + cmocka_unit_test_teardown(test_simple_put_auto_compress_native, teardown), + cmocka_unit_test_teardown(test_simple_put_config_temp_dir_native, teardown), + cmocka_unit_test_teardown(test_simple_get_native, teardown), + cmocka_unit_test_teardown(test_large_put_auto_compress_native, donothing), + cmocka_unit_test_teardown(test_large_get_native, donothing), + cmocka_unit_test_teardown(test_large_get_threshold_native, donothing), + cmocka_unit_test_teardown(test_large_reupload_native, donothing), + cmocka_unit_test_teardown(test_verify_upload, teardown), + cmocka_unit_test_teardown(test_simple_put_use_dev_urandom_native, teardown), + cmocka_unit_test_teardown(test_simple_put_use_s3_regionalURL_native, teardown), }; int ret = cmocka_run_group_tests(tests, gr_setup, gr_teardown); return ret; From 17fc0d5d27535e3e9fe3b4ac484b983911c2d79c Mon Sep 17 00:00:00 2001 From: Harry Xi Date: Tue, 3 Dec 2024 15:53:10 -0800 Subject: [PATCH 5/8] SNOW-710476: bulk update essential (#776) --- CMakeLists.txt | 4 + cpp/FileTransferAgent.cpp | 34 +-- cpp/lib/BindUploader.cpp | 419 +++++++++++++++++++++++++++++ cpp/lib/ClientBindUploader.cpp | 184 +++++++++++++ cpp/lib/ClientBindUploader.hpp | 72 +++++ cpp/lib/Exceptions.cpp | 124 ++++++++- cpp/util/SnowflakeCommon.cpp | 26 ++ cpp/util/SnowflakeCommon.hpp | 24 +- include/snowflake/BindUploader.hpp | 222 +++++++++++++++ include/snowflake/Exceptions.hpp | 136 +++++++++- include/snowflake/basic_types.h | 2 +- include/snowflake/client.h | 39 ++- lib/client.c | 246 +++++++++++++---- lib/client_int.h | 48 ++++ tests/CMakeLists.txt | 5 + tests/test_bind_params.c | 189 +++++++++++++ 16 files changed, 1668 insertions(+), 106 deletions(-) create mode 100644 cpp/lib/BindUploader.cpp create mode 100644 cpp/lib/ClientBindUploader.cpp create mode 100644 cpp/lib/ClientBindUploader.hpp create mode 100644 include/snowflake/BindUploader.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index b9f8361cf2..e1cffacdb6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -215,6 +215,7 @@ set(SOURCE_FILES_CPP_WRAPPER include/snowflake/SFURL.hpp include/snowflake/CurlDesc.hpp include/snowflake/CurlDescPool.hpp + include/snowflake/BindUploader.hpp cpp/lib/Exceptions.cpp cpp/lib/Connection.cpp cpp/lib/Statement.cpp @@ -235,6 +236,9 @@ set(SOURCE_FILES_CPP_WRAPPER cpp/lib/ResultSetJson.hpp cpp/lib/Authenticator.cpp cpp/lib/Authenticator.hpp + cpp/lib/BindUploader.cpp + cpp/lib/ClientBindUploader.hpp + cpp/lib/ClientBindUploader.cpp cpp/jwt/jwtWrapper.cpp cpp/util/SnowflakeCommon.cpp cpp/util/SFURL.cpp diff --git a/cpp/FileTransferAgent.cpp b/cpp/FileTransferAgent.cpp index cc0fbaee90..fc2ce86613 100755 --- a/cpp/FileTransferAgent.cpp +++ b/cpp/FileTransferAgent.cpp @@ -17,6 +17,7 @@ #include "crypto/Cryptor.hpp" #include "util/CompressionUtil.hpp" #include "util/ThreadPool.hpp" +#include "util/SnowflakeCommon.hpp" #include "EncryptionProvider.hpp" #include "logger/SFLogger.hpp" #include "error.h" @@ -31,35 +32,11 @@ using ::std::string; using ::std::vector; using ::Snowflake::Client::RemoteStorageRequestOutcome; +using namespace Snowflake::Client::Util; namespace { const std::string FILE_PROTOCOL = "file://"; - - void replaceStrAll(std::string& stringToReplace, - std::string const& oldValue, - std::string const& newValue) - { - size_t oldValueLen = oldValue.length(); - size_t newValueLen = newValue.length(); - if (0 == oldValueLen) - { - return; - } - - size_t index = 0; - while (true) { - /* Locate the substring to replace. */ - index = stringToReplace.find(oldValue, index); - if (index == std::string::npos) break; - - /* Make the replacement. */ - stringToReplace.replace(index, oldValueLen, newValue); - - /* Advance index forward so the next iteration doesn't pick it up as well. */ - index += newValueLen; - } - } } Snowflake::Client::FileTransferAgent::FileTransferAgent( @@ -968,6 +945,8 @@ using namespace Snowflake::Client; extern "C" { SF_STATUS STDCALL _snowflake_execute_put_get_native( SF_STMT* sfstmt, + void* upload_stream, + size_t stream_size, struct SF_QUERY_RESULT_CAPTURE* result_capture) { if (!sfstmt) @@ -996,6 +975,11 @@ extern "C" { agent.setGetMaxRetries(sfconn->get_maxretries); agent.setRandomDeviceAsUrand(sfconn->put_use_urand_dev); + if (upload_stream) + { + agent.setUploadStream((std::basic_iostream*)upload_stream, stream_size); + } + ITransferResult* result; try { diff --git a/cpp/lib/BindUploader.cpp b/cpp/lib/BindUploader.cpp new file mode 100644 index 0000000000..e990c0e749 --- /dev/null +++ b/cpp/lib/BindUploader.cpp @@ -0,0 +1,419 @@ +/* + * Copyright (c) 2024 Snowflake Computing, Inc. All rights reserved. + */ + + +#include +#include +#include "zlib.h" + +#include "snowflake/BindUploader.hpp" +#include "../logger/SFLogger.hpp" +#include "snowflake/basic_types.h" +#include "snowflake/SF_CRTFunctionSafe.h" +#include "../util/SnowflakeCommon.hpp" + +#ifdef _WIN32 +# include +# include +# define SET_BINARY_MODE(file) _setmode(_fileno(file), O_BINARY) +#else +# define SET_BINARY_MODE(file) +#endif + +#define CHUNK 16384 +#define WINDOW_BIT 15 +#define GZIP_ENCODING 16 + +using namespace Snowflake::Client::Util; + +namespace +{ + static const std::string STAGE_NAME("SYSTEM$BIND"); + + static const std::string CREATE_STAGE_STMT( + "CREATE TEMPORARY STAGE " + + STAGE_NAME + + " file_format=(" + + " type=csv" + + " field_optionally_enclosed_by='\"'" + + ")"); + + static const std::string PUT_STMT( + "PUT" + " file://%s" // argument 1: fake file name + " '%s'" // argument 2: stage path + " overwrite=true" // skip file existence check + " auto_compress=false" // we compress already + " source_compression=gzip" // (with gzip) + ); + + static const unsigned int PUT_RETRY_COUNT = 3; +} + +namespace Snowflake +{ +namespace Client +{ +BindUploader::BindUploader(const std::string& stageDir, + unsigned int numParams, unsigned int numParamSets, + unsigned int maxFileSize, + int compressLevel) : + m_stagePath("@" + STAGE_NAME + "/" + stageDir + "/"), + m_fileNo(0), + m_retryCount(PUT_RETRY_COUNT), + m_maxFileSize(maxFileSize), + m_numParams(numParams), + m_numParamSets(numParamSets), + m_curParamIndex(0), + m_curParamSetIndex(0), + m_dataSize(0), + m_startTime(std::chrono::steady_clock::now()), + m_serializeStartTime(std::chrono::steady_clock::now()), + m_compressTime(0), + m_serializeTime(0), + m_putTime(0), + m_hasBindingUploaded(false), + m_compressLevel(compressLevel) +{ + CXX_LOG_TRACE("Constructing BindUploader: stageDir:%s, numParams: %d, numParamSets: %d, " + "maxFileSize: %d, compressLevel: %d", + stageDir.c_str(), numParams, numParamSets, + maxFileSize, compressLevel); +} + +void BindUploader::putBinds() +{ + // count serialize time since this function is called when serialization for + // one chunk is done + m_serializeTime += std::chrono::duration_cast(std::chrono::steady_clock::now() - m_serializeStartTime).count(); + m_serializeStartTime = std::chrono::steady_clock::now(); + + createStageIfNeeded(); + auto compressStartTime = std::chrono::steady_clock::now(); + size_t compressedSize = compressWithGzip(); + m_compressTime += std::chrono::duration_cast(std::chrono::steady_clock::now() - compressStartTime).count(); + + auto putStartTime = std::chrono::steady_clock::now(); + std::string filename = std::to_string(m_fileNo++); + while (m_retryCount > 0) + { + std::string putStmt = getPutStmt(filename); + try + { + executeUploading(putStmt, m_compressStream, compressedSize); + m_hasBindingUploaded = true; + break; + } + catch (...) + { + CXX_LOG_WARN("BindUploader::putBinds: Failed to upload array binds, retry"); + m_retryCount--; + if (0 == m_retryCount) + { + CXX_LOG_ERROR("BindUploader::putBinds: Failed to upload array binds with all retry"); + throw; + } + } + } + m_putTime += std::chrono::duration_cast(std::chrono::steady_clock::now() - putStartTime).count(); + + m_csvStream = std::stringstream(); + m_dataSize = 0; + if (m_curParamSetIndex >= m_numParamSets) + { + auto totalTime = std::chrono::duration_cast(std::chrono::steady_clock::now() - m_startTime).count(); + CXX_LOG_INFO("BindUploader::putBinds: total time: %ld, serialize time: %d, compress time: %ld, put time %ld", + totalTime, m_serializeTime, m_compressTime, m_putTime); + } +} + +size_t BindUploader::compressWithGzip() +{ + int ret, flush; + unsigned have; + z_stream strm; + unsigned char in[CHUNK]; + unsigned char out[CHUNK]; + + m_compressStream = std::stringstream(); + m_csvStream.seekg(0); + + /* allocate deflate state */ + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + ret = deflateInit2(&strm, m_compressLevel, Z_DEFLATED, + WINDOW_BIT | GZIP_ENCODING, 8, Z_DEFAULT_STRATEGY); + if (ret != Z_OK) + { + CXX_LOG_TRACE("BindUploader: Compression initial failed with error code %d", ret); + throw; + } + + /* compress until end of source data */ + do + { + m_csvStream.read((char*)in, CHUNK); + strm.next_in = in; + strm.avail_in = m_csvStream.gcount(); + flush = m_csvStream.eof() ? Z_FINISH : Z_NO_FLUSH; + + /* run deflate() on input until output buffer not full, finish + compression if all of source has been read in */ + do + { + strm.avail_out = CHUNK; + strm.next_out = out; + ret = deflate(&strm, flush); /* no bad return value */ + have = CHUNK - strm.avail_out; + m_compressStream.write((char*)out, have); + } while (strm.avail_out == 0); + + /* done when last data in file processed */ + } while (flush != Z_FINISH); + + size_t destSize = strm.total_out; + + /* clean up and return */ + (void)deflateEnd(&strm); + return destSize; +} + +std::string BindUploader::getPutStmt(const std::string& srcFilePath) +{ + char strBuf[MAX_PATH * 2]; // *2 to make sure there is enough space + sf_sprintf(strBuf, sizeof(strBuf), PUT_STMT.c_str(), + srcFilePath.c_str(), getStagePath().c_str()); + + return std::string(strBuf); +} + +std::string BindUploader::getCreateStageStmt() +{ + return CREATE_STAGE_STMT; +} + +void BindUploader::addStringValue(const std::string& val, SF_DB_TYPE type) +{ + if (m_curParamIndex != 0) + { + m_csvStream << ","; + m_dataSize++; + } + else if (m_dataSize == 0) + { + m_serializeStartTime = std::chrono::steady_clock::now(); + } + + if (val.empty()) + { + m_csvStream << "\"\""; // an empty string => an empty string with quotes + m_dataSize += sizeof("\"\""); + } + else + { + switch (type) + { + case SF_DB_TYPE_TIME: + { + std::string timeStr = convertTimeFormat(val); + m_csvStream << timeStr; + m_dataSize += timeStr.length(); + break; + } + + case SF_DB_TYPE_DATE: + { + std::string dateStr = convertDateFormat(val); + m_csvStream << dateStr; + m_dataSize += dateStr.length(); + break; + } + + case SF_DB_TYPE_TIMESTAMP_LTZ: + case SF_DB_TYPE_TIMESTAMP_NTZ: + case SF_DB_TYPE_TIMESTAMP_TZ: + { + std::string timestampStr = convertTimestampFormat(val, type); + m_csvStream << timestampStr; + m_dataSize += timestampStr.length(); + break; + } + + default: + { + if (std::string::npos == val.find_first_of("\"\n,\\")) + { + m_csvStream << val; + m_dataSize += val.length(); + } + else + { + std::string escapeStr(val); + replaceStrAll(escapeStr, "\"", "\"\""); + escapeStr = "\"" + escapeStr + "\""; + + m_csvStream << escapeStr; + m_dataSize += escapeStr.length(); + } + break; + } + } + } + + // The last column in the current row, add new line + // Also upload the data as needed. + if (++m_curParamIndex >= m_numParams) + { + m_csvStream << "\n"; + m_dataSize++; + m_curParamIndex = 0; + m_curParamSetIndex++; + + // Upload data when exceed file size limit or all rows are added + if ((m_dataSize >= m_maxFileSize) || + (m_curParamSetIndex >= m_numParamSets)) + { + putBinds(); + } + } +} + +void BindUploader::addNullValue() +{ + if (m_curParamIndex != 0) + { + m_csvStream << ","; + m_dataSize++; + } + + // The last column in the current row, add new line + // Also upload the data as needed. + if (++m_curParamIndex >= m_numParams) + { + m_csvStream << "\n"; + m_dataSize++; + m_curParamIndex = 0; + m_curParamSetIndex++; + + // Upload data when exceed file size limit or all rows are added + if ((m_dataSize >= m_maxFileSize) || + (m_curParamSetIndex >= m_numParamSets)) + { + putBinds(); + } + } +} + +bool BindUploader::csvGetNextField(std::string& fieldValue, + bool& isNull, bool& isEndOfRow) +{ + char c; + + // the flag indecate if currently in a quoted value + bool inQuote = false; + // the flag indecate if the value has been quoted, quoted empty string is + // empty value (like ,"",) while unquoted empty string is null (like ,,) + bool quoted = false; + // the flag indecate a value is found to end the loop + bool done = false; + // the flag indicate the next char already fetched by checking double quote escape ("") + bool nextCharFetched = false; + + fieldValue.clear(); + + if (!m_csvStream.get(c)) + { + return false; + } + + while (!done) + { + switch (c) + { + case ',': + { + if (!inQuote) + { + done = true; + } + else + { + fieldValue.push_back(c); + } + break; + } + + case '\n': + { + if (!inQuote) + { + done = true; + isEndOfRow = true; + } + else + { + fieldValue.push_back(c); + } + break; + } + + case '\"': + { + if (!inQuote) + { + quoted = true; + inQuote = true; + } + else + { + if (!m_csvStream.get(c)) + { + isEndOfRow = true; + done = true; + } + else + { + if (c == '\"') + { + // escape double qoute in quoted string + fieldValue.push_back(c); + } + else + { + inQuote = false; + nextCharFetched = true; + } + } + } + + break; + } + + default: + { + fieldValue.push_back(c); + } + } + + if ((!done) && (!nextCharFetched)) + { + if (!m_csvStream.get(c)) + { + isEndOfRow = true; + break; + } + } + else + { + nextCharFetched = false; + } + } + + isNull = (fieldValue.empty() && !quoted); + return true; +} + +} // namespace Client +} // namespace Snowflake diff --git a/cpp/lib/ClientBindUploader.cpp b/cpp/lib/ClientBindUploader.cpp new file mode 100644 index 0000000000..9b6a864ddb --- /dev/null +++ b/cpp/lib/ClientBindUploader.cpp @@ -0,0 +1,184 @@ +/* + * Copyright (c) 2024 Snowflake Computing, Inc. All rights reserved. + */ + + +#include +#include + +#include "ClientBindUploader.hpp" +#include "../logger/SFLogger.hpp" +#include "snowflake/basic_types.h" +#include "snowflake/SF_CRTFunctionSafe.h" +#include "../util/SnowflakeCommon.hpp" +#include "snowflake/Exceptions.hpp" +#include "client_int.h" +#include "results.h" +#include "error.h" + +namespace Snowflake +{ +namespace Client +{ +ClientBindUploader::ClientBindUploader(SF_STMT *sfstmt, + const std::string& stageDir, + unsigned int numParams, unsigned int numParamSets, + unsigned int maxFileSize, + int compressLevel) : + BindUploader(stageDir, numParams, numParamSets, maxFileSize, compressLevel) +{ + if (!sfstmt || !sfstmt->connection) + { + SNOWFLAKE_THROW("BindUploader:: Invalid statement"); + } + SF_STATUS ret; + m_stmt = snowflake_stmt(sfstmt->connection); + if (sfstmt == NULL) { + SET_SNOWFLAKE_ERROR( + &sfstmt->error, + SF_STATUS_ERROR_OUT_OF_MEMORY, + "Out of memory in creating SF_STMT. ", + SF_SQLSTATE_UNABLE_TO_CONNECT); + + SNOWFLAKE_THROW_S(&sfstmt->error); + } +} + +ClientBindUploader::~ClientBindUploader() +{ + if (m_stmt) + { + snowflake_stmt_term(m_stmt); + } +} + +void ClientBindUploader::createStageIfNeeded() +{ + SF_CONNECT* conn = m_stmt->connection; + // Check the flag without locking to get better performance. + if (conn->binding_stage_created) + { + return; + } + + _mutex_lock(&conn->mutex_stage_bind); + if (conn->binding_stage_created) + { + _mutex_unlock(&conn->mutex_stage_bind); + return; + } + + std::string command = getCreateStageStmt(); + SF_STATUS ret = snowflake_query(m_stmt, command.c_str(), 0); + if (ret != SF_STATUS_SUCCESS) + { + _mutex_unlock(&conn->mutex_stage_bind); + SNOWFLAKE_THROW_S(&m_stmt->error); + } + + conn->binding_stage_created = SF_BOOLEAN_TRUE; + _mutex_unlock(&conn->mutex_stage_bind); +} + +void ClientBindUploader::executeUploading(const std::string &sql, + std::basic_iostream& uploadStream, + size_t dataSize) +{ + snowflake_prepare(m_stmt, sql.c_str(), 0); + SF_STATUS ret = _snowflake_execute_put_get_native(m_stmt, &uploadStream, dataSize, NULL); + if (ret != SF_STATUS_SUCCESS) + { + SNOWFLAKE_THROW_S(&m_stmt->error); + } +} + +} // namespace Client +} // namespace Snowflake + +extern "C" { + +using namespace Snowflake::Client; + +char* STDCALL +_snowflake_stage_bind_upload(SF_STMT* sfstmt) +{ + std::string bindStage; + try + { + ClientBindUploader uploader(sfstmt, sfstmt->request_id, + sfstmt->params_len, sfstmt->paramset_size, + SF_DEFAULT_STAGE_BINDING_MAX_FILESIZE, 0); + + const char* type; + char name_buf[SF_PARAM_NAME_BUF_LEN]; + char* name = NULL; + char* value = NULL; + struct bind_info { + SF_BIND_INPUT* input; + void* val_ptr; + int step; + }; + std::vector bindInfo; + for (unsigned int i = 0; i < sfstmt->params_len; i++) + { + SF_BIND_INPUT* input = _snowflake_get_binding_by_index(sfstmt, i, &name, + name_buf, SF_PARAM_NAME_BUF_LEN); + if (input == NULL) + { + log_error("_snowflake_execute_ex: No parameter by this name %s", name); + return NULL; + } + bindInfo.emplace_back(); + bindInfo.back().input = input; + bindInfo.back().val_ptr = input->value; + bindInfo.back().step = _snowflake_get_binding_value_size(input); + } + for (int64 i = 0; i < sfstmt->paramset_size; i++) + { + for (unsigned int j = 0; j < sfstmt->params_len; j++) + { + SF_BIND_INPUT* input = bindInfo[j].input; + void* val_ptr = bindInfo[j].val_ptr; + int val_len = input->len; + if (input->len_ind) + { + val_len = input->len_ind[i]; + } + + if (SF_BIND_LEN_NULL == val_len) + { + uploader.addNullValue(); + } + + if ((SF_C_TYPE_STRING == input->c_type) && + (SF_BIND_LEN_NTS == val_len)) + { + val_len = strlen((char*)val_ptr); + } + + value = value_to_string(val_ptr, val_len, input->c_type); + if (value) { + uploader.addStringValue(value, input->type); + SF_FREE(value); + } + bindInfo[j].val_ptr = (char*)bindInfo[j].val_ptr + bindInfo[j].step; + } + } + bindStage = uploader.getStagePath(); + } + catch (SnowflakeGeneralException& e) + { + return NULL; + } + + if (!bindStage.empty()) + { + char* bind_stage = (char*) SF_CALLOC(1, bindStage.size() + 1); + sf_strncpy(bind_stage, bindStage.size() + 1, bindStage.c_str(), bindStage.size()); + return bind_stage; + } + + return NULL; +} + +} // extern "C" diff --git a/cpp/lib/ClientBindUploader.hpp b/cpp/lib/ClientBindUploader.hpp new file mode 100644 index 0000000000..c81134fdfe --- /dev/null +++ b/cpp/lib/ClientBindUploader.hpp @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2024 Snowflake Computing, Inc. All rights reserved. + */ + +#pragma once +#ifndef SNOWFLAKECLIENT_CLIENTBINDUPLOADER_HPP +#define SNOWFLAKECLIENT_CLIENTBINDUPLOADER_HPP + +#include +#include "snowflake/client.h" +#include "snowflake/BindUploader.hpp" + +namespace Snowflake +{ +namespace Client +{ + +class ClientBindUploader : public BindUploader +{ +public: + /** + * constructor + * + * @param sfstmt The SNOWFLAKE_STMT context. + * @param stageDir The unique stage path for bindings uploading, could be a GUID. + * @param numParams Number of parameters. + * @param numParamSets Number of parameter sets. + * @param maxFileSize The max size of single file for bindings uploading. + * Separate into multiple files when exceed. + * @param compressLevel The compress level, between -1(default) to 9. + */ + explicit ClientBindUploader(SF_STMT *sfstmt, + const std::string& stageDir, + unsigned int numParams, + unsigned int numParamSets, + unsigned int maxFileSize, + int compressLevel); + + ~ClientBindUploader(); + +protected: + /** + * Check whether the session's temporary stage has been created, and create it + * if not. + * + * @throws Exception if creating the stage fails + */ + virtual void createStageIfNeeded() override; + + /** + * Execute uploading for single data file. + * + * @param sql PUT command for single data file uploading + * @param uploadStream stream for data file to be uploaded + * @param dataSize Size of the data to be uploaded. + * + * @throws Exception if uploading fails + */ + virtual void executeUploading(const std::string &sql, + std::basic_iostream& uploadStream, + size_t dataSize) override; + +private: + // SNOWFLAKE_STMT context + SF_STMT * m_stmt; + +}; + +} // namespace Client +} // namespace Snowflake + +#endif // SNOWFLAKECLIENT_CLIENTBINDUPLOADER_HPP diff --git a/cpp/lib/Exceptions.cpp b/cpp/lib/Exceptions.cpp index 28f834e885..7fdf7ffbe9 100644 --- a/cpp/lib/Exceptions.cpp +++ b/cpp/lib/Exceptions.cpp @@ -1,4 +1,126 @@ /* - * Copyright (c) 2018-2019 Snowflake Computing, Inc. All rights reserved. + * Copyright (c) 2018-2024 Snowflake Computing, Inc. All rights reserved. */ +#include "snowflake/Exceptions.hpp" +#include "../logger/SFLogger.hpp" + +// helper functions +namespace +{ + std::string setupErrorMessage(const std::string& message, + const std::string& file, + int line, + const std::string& queryId, + const std::string& sqlState, + int code) + { + std::string errmsg = "Snowflake exception: "; + if (!file.empty()) + { + errmsg += file + ":" + std::to_string(line) + ", "; + } + if (!queryId.empty()) + { + errmsg += std::string("query ID: ") + queryId + ", "; + } + if (!sqlState.empty()) + { + errmsg += std::string("SQLState: ") + sqlState + ", "; + } + errmsg += std::string("error code :") + std::to_string(code) + ", "; + + errmsg += std::string("error message: ") + message; + + return errmsg; + } +} + +namespace Snowflake +{ +namespace Client +{ + +void SnowflakeException::setErrorMessage(const std::string& errmsg) +{ + m_errmsg = SFLogger::getMaskedMsg("%s", errmsg.c_str()); +} + +void SnowflakeException::setErrorMessage(const char* fmt, va_list args) +{ + m_errmsg = SFLogger::getMaskedMsgVA(fmt, args); +} + +SnowflakeGeneralException::SnowflakeGeneralException(SF_ERROR_STRUCT *error) : + m_message(error->msg ? error->msg : ""), + m_file(error->file ? error->file : ""), + m_line(error->line), + m_queryId(error->sfqid), + m_sqlState(error->sqlstate), + m_code((int)error->error_code) +{ + std::string errmsg = setupErrorMessage(m_message, m_file, m_line, m_queryId, m_sqlState, m_code); + setErrorMessage(errmsg); +} + +SnowflakeGeneralException::SnowflakeGeneralException(const std::string& message, + const char* file, int line, + int code, + const std::string queryId, + const std::string sqlState) : + m_message(message), + m_file(file ? file : ""), + m_line(line), + m_queryId(queryId), + m_sqlState(sqlState), + m_code(code) +{ + std::string errmsg = setupErrorMessage(m_message, m_file, m_line, m_queryId, m_sqlState, m_code); + setErrorMessage(errmsg); +} + +SnowflakeGeneralException::SnowflakeGeneralException(const char* file, int line, + int code, + const std::string queryId, + const std::string sqlState, + const char* fmt, ...) +{ + va_list args; + va_start(args, fmt); + m_message = SFLogger::getMaskedMsgVA(fmt, args); + va_end(args); + m_errmsg = setupErrorMessage(m_message, m_file, m_line, m_queryId, m_sqlState, m_code); +} + +int SnowflakeGeneralException::code() +{ + return m_code; +} + +const char* SnowflakeGeneralException::sqlstate() +{ + return m_sqlState.c_str(); +} + +const char* SnowflakeGeneralException::msg() +{ + return m_message.c_str(); +} + +const char* SnowflakeGeneralException::sfqid() +{ + return m_queryId.c_str(); +} + +const char* SnowflakeGeneralException::file() +{ + return m_file.c_str(); +} + +int SnowflakeGeneralException::line() +{ + return m_line; +} + +} // namespace Client +} // namespace Snowflake diff --git a/cpp/util/SnowflakeCommon.cpp b/cpp/util/SnowflakeCommon.cpp index b12642b6c4..b66f3c129c 100644 --- a/cpp/util/SnowflakeCommon.cpp +++ b/cpp/util/SnowflakeCommon.cpp @@ -11,6 +11,7 @@ #include "snowflake/Proxy.hpp" #include "../logger/SFLogger.hpp" #include +#include "SnowflakeCommon.hpp" using namespace Snowflake; using namespace Snowflake::Client; @@ -144,3 +145,28 @@ uint64 sf_get_current_time_millis() } +void Snowflake::Client::Util::replaceStrAll(std::string& stringToReplace, + std::string const& oldValue, + std::string const& newValue) +{ + size_t oldValueLen = oldValue.length(); + size_t newValueLen = newValue.length(); + if (0 == oldValueLen) + { + return; + } + + size_t index = 0; + while (true) { + /* Locate the substring to replace. */ + index = stringToReplace.find(oldValue, index); + if (index == std::string::npos) break; + + /* Make the replacement. */ + stringToReplace.replace(index, oldValueLen, newValue); + + /* Advance index forward so the next iteration doesn't pick it up as well. */ + index += newValueLen; + } +} + diff --git a/cpp/util/SnowflakeCommon.hpp b/cpp/util/SnowflakeCommon.hpp index f151d1c691..4557ebb430 100644 --- a/cpp/util/SnowflakeCommon.hpp +++ b/cpp/util/SnowflakeCommon.hpp @@ -1,20 +1,28 @@ /* - * Copyright (c) 2018-2019 Snowflake Computing, Inc. All rights reserved. + * Copyright (c) 2024 Snowflake Computing, Inc. All rights reserved. */ #ifndef SNOWFLAKECLIENT_SNOWFLAKECOMMON_HPP #define SNOWFLAKECLIENT_SNOWFLAKECOMMON_HPP -#include +#include #include #include -// unsigned integer types -typedef uint8_t ub1; -typedef uint16_t ub2; -typedef uint32_t ub4; -typedef uint64_t ub8; -typedef uint128_t ub16; +/* CPP only utilities */ +namespace Snowflake +{ +namespace Client +{ +namespace Util +{ +void replaceStrAll(std::string& stringToReplace, + std::string const& oldValue, + std::string const& newValue); + +} // namespace Util +} // namespace Client +} // namespace Snowflake #endif //SNOWFLAKECLIENT_SNOWFLAKECOMMON_HPP diff --git a/include/snowflake/BindUploader.hpp b/include/snowflake/BindUploader.hpp new file mode 100644 index 0000000000..e32668f569 --- /dev/null +++ b/include/snowflake/BindUploader.hpp @@ -0,0 +1,222 @@ +/* + * Copyright (c) 2024 Snowflake Computing, Inc. All rights reserved. + */ + +/* + * Notice: For Snowflake internal use only. + * External application should not use this class. + */ + +#pragma once +#ifndef SNOWFLAKECLIENT_BINDUPLOADER_HPP +#define SNOWFLAKECLIENT_BINDUPLOADER_HPP + +#include +#include +#include "client.h" + +namespace Snowflake +{ +namespace Client +{ + +class BindUploader +{ +public: + /** + * constructor + * + * @param stageDir The unique stage path for bindings uploading, could be a GUID. + * @param numParams Number of parameters. + * @param numParamSets Number of parameter sets. + * @param maxFileSize The max size of single file for bindings uploading. + * Separate into multiple files when exceed. + * @param compressLevel The compress level, between -1(default) to 9. + */ + explicit BindUploader(const std::string& stageDir, + unsigned int numParams, + unsigned int numParamSets, + unsigned int maxFileSize, + int compressLevel); + + void addStringValue(const std::string& value, SF_DB_TYPE type); + + void addNullValue(); + + inline std::string getStagePath() + { + return m_stagePath; + } + + inline bool hasBindingUploaded() + { + return m_hasBindingUploaded; + } + +protected: + /** + * @return The statement for creating temporary stage for bind uploading. + */ + std::string getCreateStageStmt(); + + /** + * Check whether the session's temporary stage has been created, and create it + * if not. + * + * @throws Exception if creating the stage fails + */ + virtual void createStageIfNeeded() = 0; + + /** + * Execute uploading for single data file. + * + * @param sql PUT command for single data file uploading + * @param uploadStream stream for data file to be uploaded + * @param dataSize Size of the data to be uploaded. + * + * @throws Exception if uploading fails + */ + virtual void executeUploading(const std::string &sql, + std::basic_iostream& uploadStream, + size_t dataSize) = 0; + + /* date/time format conversions to be overridden by drivers (such as ODBC) + * that need native date/time type support. + * Will be called to converting binding format between regular binding and + * bulk binding. + * No conversion by default, in such case application/driver should bind + * data/time data as string. + */ + + /** + * Convert time data format from nanoseconds to HH:MM:SS.F9 + * @param timeInNano The time data string in nanoseconds. + */ + virtual std::string convertTimeFormat(const std::string& timeInNano) + { + return timeInNano; + } + + /** + * Convert date data format from days to YYYY-MM-DD + * @param milliseconds since Epoch + */ + virtual std::string convertDateFormat(const std::string& millisecondSinceEpoch) + { + return millisecondSinceEpoch; + } + + /** + * Convert timestamp data format from nanoseconds to YYYY_MM_DD HH:MM:SS.F9 + * @param timestampInNano The timestamp data string in nanoseconds. + * @param type Either TIMESTAMP_LTZ or NTZ depends on CLIENT_TIMESTAMP_TYPE_MAPPING + */ + virtual std::string convertTimestampFormat(const std::string& timestampInNano, + SF_DB_TYPE type) + { + return timestampInNano; + } + + /** + * Revert time data format from HH:MM:SS.F9 to nanoseconds + * @param formatedTime The time data string in HH:MM:SS.F9. + */ + virtual std::string revertTimeFormat(const std::string& formatedTime) + { + return formatedTime; + } + + /** + * Convert date data format from YYYY-MM-DD to milliseconds since Epoch + * @param formatedDate the date string in YYYY-MM-DD + */ + virtual std::string revertDateFormat(const std::string& formatedDate) + { + return formatedDate; + } + + /** + * Convert timestamp data format from YYYY_MM_DD HH:MM:SS.F9 to nanoseconds + * @param Formatedtimestamp The timestamp data string in YYYY_MM_DD HH:MM:SS.F9. + * @param type Either TIMESTAMP_LTZ or NTZ depends on CLIENT_TIMESTAMP_TYPE_MAPPING + */ + virtual std::string revertTimestampFormat(const std::string& Formatedtimestamp, + SF_DB_TYPE type) + { + return Formatedtimestamp; + } + +private: + /** + * Upload serialized binds in CSV stream to stage + * + * @throws BindException if uploading the binds fails + */ + void putBinds(); + + /** + * Compress data from csv stream to compress stream with gzip + * @return The data size of compress stream if compress succeeded. + * @throw when compress failed. + */ + size_t compressWithGzip(); + + /** + * Build PUT statement string. Handle filesystem differences and escaping backslashes. + * @param srcFilePath The faked source file path to upload. + */ + std::string getPutStmt(const std::string& srcFilePath); + + /** + * csv parsing function called by convertBindingFromCsvToJson(), get value of + * next field. + * @param fieldValue The output of the field value. + * @param isNull The output of the flag whether the filed is null. + * @param isEndofRow The output of the flag wether the end of row is reached. + * @return true if a field value is retrieved successfully, false if end of data + * is reached and no field value available. + */ + bool csvGetNextField(std::string& fieldValue, bool& isNull, bool& isEndofRow); + + std::stringstream m_csvStream; + + std::stringstream m_compressStream; + + std::string m_stagePath; + + unsigned int m_fileNo; + + unsigned int m_retryCount; + + unsigned int m_maxFileSize; + + unsigned int m_numParams; + + unsigned int m_numParamSets; + + unsigned int m_curParamIndex; + + unsigned int m_curParamSetIndex; + + size_t m_dataSize; + + std::chrono::steady_clock::time_point m_startTime; + + std::chrono::steady_clock::time_point m_serializeStartTime; + + long long m_compressTime; + + long long m_serializeTime; + + long long m_putTime; + + bool m_hasBindingUploaded; + + int m_compressLevel; + +}; + +} // namespace Client +} // namespace Snowflake + +#endif // SNOWFLAKECLIENT_BINDUPLOADER_HPP diff --git a/include/snowflake/Exceptions.hpp b/include/snowflake/Exceptions.hpp index ea32f0a6b0..e8c0282623 100644 --- a/include/snowflake/Exceptions.hpp +++ b/include/snowflake/Exceptions.hpp @@ -1,38 +1,146 @@ /* - * Copyright (c) 2018-2019 Snowflake Computing, Inc. All rights reserved. + * Copyright (c) 2018-2024 Snowflake Computing, Inc. All rights reserved. */ #ifndef SNOWFLAKECLIENT_EXCEPTIONS_HPP #define SNOWFLAKECLIENT_EXCEPTIONS_HPP #include +#include #include "client.h" -class SnowflakeException: public std::exception { -public: - SnowflakeException(SF_ERROR_STRUCT *error); +namespace Snowflake +{ +namespace Client +{ - const char * what() const throw(); +class SnowflakeException: public std::exception +{ +public: + // Return error message combine all information + // sub-classes need to setup m_errmsg. + virtual const char* what() const noexcept override + { + return m_errmsg.c_str(); + } - SF_STATUS code(); + // optional properties sub-classes could choose what to override + // with information available + virtual int code() + { + return 0; + } - const char *sqlstate(); + virtual const char* sqlstate() + { + return ""; + } - const char *msg(); + // Return the original error message without other information (sqlstate etc.). + virtual const char* msg() + { + return ""; + } - const char *sfqid(); + virtual const char* sfqid() + { + return ""; + } - const char *file(); + virtual const char* file() + { + return ""; + } - int line(); + virtual int line() + { + return 0; + } protected: - SF_ERROR_STRUCT *error; + // update error message + void setErrorMessage(const std::string& errmsg); + + // update error message with formatted arguments + void setErrorMessage(const char* fmt, va_list args); + + std::string m_errmsg; }; -class GeneralException: public SnowflakeException { +class SnowflakeGeneralException: public SnowflakeException +{ public: - GeneralException(SF_ERROR_STRUCT *error) : SnowflakeException(error) {}; + SnowflakeGeneralException(SF_ERROR_STRUCT *error); + SnowflakeGeneralException(const std::string& message, + const char* file, int line, + int code = 0, + const std::string queryId = "", + const std::string sqlState = ""); + SnowflakeGeneralException(const char* file, int line, + int code, + const std::string queryId, + const std::string sqlState, + const char* fmt, ...); + + virtual int code() override; + + virtual const char* sqlstate() override; + + virtual const char* msg() override; + + virtual const char* sfqid() override; + + virtual const char* file() override; + + virtual int line() override; + +protected: + std::string m_message; + std::string m_file; + int m_line; + std::string m_queryId; + std::string m_sqlState; + int m_code; }; +// macro for throw general exception with SF_ERROR_STRUCT +#define SNOWFLAKE_THROW_S(error) \ +{ \ + throw SnowflakeGeneralException(error); \ +} + +// macro for throw general exception with error message +#define SNOWFLAKE_THROW(errmsg) \ +{ \ + throw SnowflakeGeneralException(errmsg, \ + __FILE__, __LINE__); \ +} + +// macro for throw general exception with more detail information +#define SNOWFLAKE_THROW_DETAIL(errmsg, code, qid, state) \ +{ \ + throw SnowflakeGeneralException(errmsg, \ + __FILE__, __LINE__, \ + code, qid, state); \ +} + +// macro for throw general exception with formatted arguments +#define SNOWFLAKE_THROW_FORMATTED(fmt, ...) \ +{ \ + throw SnowflakeGeneralException(__FILE__, __LINE__, \ + 0, "", "", \ + fmt, __VA_ARGS__); \ +} + +// macro for throw general exception with formatted arguments and detail information. +#define SNOWFLAKE_THROW_FORMATTED_DETAIL(code, qid, state, fmt, ...) \ +{ \ + throw SnowflakeGeneralException(__FILE__, __LINE__, \ + code, qid, state, \ + fmt, __VA_ARGS__); \ +} + +} // namespace Client +} // namespace Snowflake + #endif //SNOWFLAKECLIENT_EXCEPTIONS_HPP diff --git a/include/snowflake/basic_types.h b/include/snowflake/basic_types.h index 45f3703c13..72d1bdd500 100644 --- a/include/snowflake/basic_types.h +++ b/include/snowflake/basic_types.h @@ -17,7 +17,7 @@ extern "C" { /** * Supported data types */ -typedef char int8; +typedef signed char int8; typedef unsigned char uint8; typedef unsigned int uint32; typedef int int32; diff --git a/include/snowflake/client.h b/include/snowflake/client.h index c045795f6a..80a9bcae7c 100644 --- a/include/snowflake/client.h +++ b/include/snowflake/client.h @@ -282,6 +282,7 @@ typedef enum SF_ATTRIBUTE { SF_CON_GET_FASTFAIL, SF_CON_GET_MAXRETRIES, SF_CON_GET_THRESHOLD, + SF_CON_STAGE_BIND_THRESHOLD, SF_DIR_QUERY_URL, SF_DIR_QUERY_URL_PARAM, SF_DIR_QUERY_TOKEN, @@ -306,7 +307,8 @@ typedef enum SF_GLOBAL_ATTRIBUTE { */ typedef enum SF_STMT_ATTRIBUTE { SF_STMT_USER_REALLOC_FUNC, - SF_STMT_MULTI_STMT_COUNT + SF_STMT_MULTI_STMT_COUNT, + SF_STMT_PARAMSET_SIZE } SF_STMT_ATTRIBUTE; #define SF_MULTI_STMT_COUNT_UNSET (-1) #define SF_MULTI_STMT_COUNT_UNLIMITED 0 @@ -432,6 +434,16 @@ typedef struct SF_CONNECT { sf_bool get_fastfail; int8 get_maxretries; int64 get_threshold; + + // stage binding + /* used when updating stage binding options */ + SF_MUTEX_HANDLE mutex_stage_bind; + sf_bool binding_stage_created; + uint64 stage_binding_threshold; + // the flag indecates the threshold from session parameter is overridden + // by the setting from connection attribute + sf_bool binding_threshold_overridden; + sf_bool stage_binding_disabled; } SF_CONNECT; /** @@ -517,6 +529,7 @@ typedef struct SF_STMT { sf_bool is_multi_stmt; void* multi_stmt_result_ids; int64 multi_stmt_count; + int64 paramset_size; /** * User realloc function used in snowflake_fetch @@ -529,14 +542,32 @@ typedef struct SF_STMT { /** * Bind input parameter context - */ + * Array binding (usually for insert/update multiple rows with one query) supported. + * To do that, value should be set to the array having multiple values, + * statement attribute SF_STMT_PARAMSET_SIZE set to the number of elements of the array + * in each binding. + * for SF_C_TYPE_STRING len should be set to the buffer length of each string value, + * NOT the entire length of the array. It would be used to find the start of each value. + * len_ind should be set to an array of length,indicating the actual length of each value. + * each length could be set to + * SF_BIND_LEN_NULL to indicate NULL data + * SF_BIND_LEN_NTS to indicate NULL terminated string (for SF_C_TYPE_STRING only). + * >= 0 for actual data length (for SF_C_TYPE_STRING only). + * len_ind could be omitted (set to NULL) as well if no NULL data, + * and for for SF_C_TYPE_STRING, all string values are null terminated. + */ + +#define SF_BIND_LEN_NULL -1 +#define SF_BIND_LEN_NTS -3 + typedef struct { size_t idx; /* One based index of the columns, 0 if Named */ char * name; /* Named Parameter name, NULL if positional */ SF_C_TYPE c_type; /* input data type in C */ - void *value; /* input value */ - size_t len; /* input value length. valid only for SF_C_TYPE_STRING */ + void *value; /* input value, could be array of multiple values */ + size_t len; /* The length of each input value. valid only for SF_C_TYPE_STRING */ SF_DB_TYPE type; /* (optional) target Snowflake data type */ + int* len_ind; /* (optional) The array of length indicator to support array binding*/ } SF_BIND_INPUT; /** diff --git a/lib/client.c b/lib/client.c index f6bdc93cdf..99aef499a9 100644 --- a/lib/client.c +++ b/lib/client.c @@ -186,6 +186,9 @@ static SF_STATUS STDCALL _reset_connection_parameters( else if (strcmp(name->valuestring, "ENABLE_STAGE_S3_PRIVATELINK_FOR_US_EAST_1") == 0) { sf->use_s3_regional_url = snowflake_cJSON_IsTrue(value) ? SF_BOOLEAN_TRUE : SF_BOOLEAN_FALSE; } + else if (strcmp(name->valuestring, "CLIENT_STAGE_ARRAY_BINDING_THRESHOLD") == 0) { + sf->stage_binding_threshold = snowflake_cJSON_GetUint64Value(value); + } } } SF_STATUS ret = SF_STATUS_ERROR_GENERAL; @@ -764,6 +767,10 @@ SF_CONNECT *STDCALL snowflake_init() { sf->get_fastfail = SF_BOOLEAN_FALSE; sf->get_maxretries = SF_DEFAULT_GET_MAX_RETRIES; sf->get_threshold = SF_DEFAULT_GET_THRESHOLD; + + _mutex_init(&sf->mutex_stage_bind); + sf->binding_stage_created = SF_BOOLEAN_FALSE; + sf->stage_binding_threshold = SF_DEFAULT_STAGE_BINDING_THRESHOLD; } return sf; @@ -803,6 +810,7 @@ SF_STATUS STDCALL snowflake_term(SF_CONNECT *sf) { _mutex_term(&sf->mutex_sequence_counter); _mutex_term(&sf->mutex_parameters); + _mutex_term(&sf->mutex_stage_bind); SF_FREE(sf->host); SF_FREE(sf->port); SF_FREE(sf->user); @@ -1852,6 +1860,170 @@ static void STDCALL _snowflake_deallocate_named_param_list(void * name_list) SF_FREE(name_list); } +#define SF_PARAM_NAME_BUF_LEN 20 +/** + * Get parameter binding by index for both POSITIONAL and NAMED cases. + * @param sfstmt SNOWFLAKE_STMT context. + * @param index The 0 based index of parameter binding to get. + * @param name Output the name of binding. + * @param name_buf The buffer to store name. + Used for POSITIONAL and name will point to this buffer in such case. + * @param name_buf_len The size of name_buf. + * @return parameter binding with specified index. + */ +SF_BIND_INPUT* STDCALL _snowflake_get_binding_by_index(SF_STMT* sfstmt, + size_t index, + char** name, + char* name_buf, + size_t name_buf_len) +{ + SF_BIND_INPUT* input = NULL; + if (_snowflake_get_current_param_style(sfstmt) == POSITIONAL) + { + input = (SF_BIND_INPUT*)sf_param_store_get(sfstmt->params, + index + 1, NULL); + sf_sprintf(name_buf, name_buf_len, "%lu", (unsigned long)(index + 1)); + *name = name_buf; + } + else if (_snowflake_get_current_param_style(sfstmt) == NAMED) + { + *name = (char*)(((NamedParams*)sfstmt->name_list)->name_list[index]); + input = (SF_BIND_INPUT*)sf_param_store_get(sfstmt->params, 0, *name); + } + + return input; +} + +/* + * @return size of single binding value per data type. + */ +size_t STDCALL _snowflake_get_binding_value_size(SF_BIND_INPUT* bind) +{ + switch (bind->c_type) + { + case SF_C_TYPE_INT8: + return sizeof (int8); + case SF_C_TYPE_UINT8: + return sizeof(uint8); + case SF_C_TYPE_INT64: + return sizeof(int64); + case SF_C_TYPE_UINT64: + return sizeof(uint64); + case SF_C_TYPE_FLOAT64: + return sizeof(float64); + case SF_C_TYPE_BOOLEAN: + return sizeof(sf_bool); + case SF_C_TYPE_BINARY: + case SF_C_TYPE_STRING: + return bind->len; + case SF_C_TYPE_TIMESTAMP: + // TODO Add timestamp case + case SF_C_TYPE_NULL: + default: + return 0; + } +} + +/** + * @param sfstmt SNOWFLAKE_STMT context. + * @return parameter bindings in cJSON. + */ +cJSON* STDCALL _snowflake_get_binding_json(SF_STMT* sfstmt) +{ + size_t i; + SF_BIND_INPUT* input; + const char* type; + char name_buf[SF_PARAM_NAME_BUF_LEN]; + char* name = NULL; + char* value = NULL; + cJSON* bindings = NULL; + + if (_snowflake_get_current_param_style(sfstmt) == INVALID_PARAM_TYPE) + { + return NULL; + } + bindings = snowflake_cJSON_CreateObject(); + for (i = 0; i < sfstmt->params_len; i++) + { + cJSON* binding; + input = _snowflake_get_binding_by_index(sfstmt, i, &name, + name_buf, SF_PARAM_NAME_BUF_LEN); + if (input == NULL) + { + log_error("_snowflake_execute_ex: No parameter by this name %s", name); + continue; + } + binding = snowflake_cJSON_CreateObject(); + type = snowflake_type_to_string( + c_type_to_snowflake(input->c_type, SF_DB_TYPE_TIMESTAMP_NTZ)); + if (sfstmt->paramset_size > 1) + { + cJSON* val_array = snowflake_cJSON_CreateArray(); + size_t step = _snowflake_get_binding_value_size(input); + void* val_ptr = input->value; + int64 val_len; + cJSON* single_val = NULL; + for (int64 j = 0; j < sfstmt->paramset_size; j++, val_ptr = (char*)val_ptr + step) + { + val_len = input->len; + if (input->len_ind) + { + val_len = input->len_ind[j]; + } + + if (SF_BIND_LEN_NULL == val_len) + { + single_val = snowflake_cJSON_CreateNull(); + snowflake_cJSON_AddItemToArray(val_array, single_val); + continue; + } + + if ((SF_C_TYPE_STRING == input->c_type) && + (SF_BIND_LEN_NTS == val_len)) + { + val_len = strlen((char*)val_ptr); + } + + value = value_to_string(val_ptr, val_len, input->c_type); + single_val = snowflake_cJSON_CreateString(value); + snowflake_cJSON_AddItemToArray(val_array, single_val); + if (value) { + SF_FREE(value); + } + } + snowflake_cJSON_AddItemToObject(binding, "value", val_array); + } + else // paramset_size = 1, single value binding + { + value = value_to_string(input->value, input->len, input->c_type); + snowflake_cJSON_AddStringToObject(binding, "value", value); + if (value) { + SF_FREE(value); + } + } + snowflake_cJSON_AddStringToObject(binding, "type", type); + snowflake_cJSON_AddItemToObject(bindings, name, binding); + } + + return bindings; +} + +sf_bool STDCALL _snowflake_needs_stage_binding(SF_STMT* sfstmt) +{ + if (!sfstmt || !sfstmt->connection || + (_snowflake_get_current_param_style(sfstmt) == INVALID_PARAM_TYPE) || + sfstmt->connection->stage_binding_disabled || + sfstmt->paramset_size <= 1) + { + return SF_BOOLEAN_FALSE; + } + + if (sfstmt->paramset_size * sfstmt->params_len >= sfstmt->connection->stage_binding_threshold) + { + return SF_BOOLEAN_TRUE; + } + return SF_BOOLEAN_FALSE; +} /** * Resets SNOWFLAKE_STMT parameters. * @@ -1973,6 +2145,8 @@ SF_STMT *STDCALL snowflake_stmt(SF_CONNECT *sf) { _snowflake_stmt_reset(sfstmt); sfstmt->connection = sf; sfstmt->multi_stmt_count = SF_MULTI_STMT_COUNT_UNSET; + // single value binding by default + sfstmt->paramset_size = 1; } return sfstmt; } @@ -2010,6 +2184,7 @@ void STDCALL snowflake_bind_input_init(SF_BIND_INPUT * input) input->idx = 0; input->name = NULL; input->value = NULL; + input->len_ind = NULL; } /** @@ -2400,7 +2575,7 @@ SF_STATUS STDCALL _snowflake_execute_ex(SF_STMT *sfstmt, if (is_put_get_command && is_native_put_get && !is_describe_only) { _snowflake_stmt_desc_reset(sfstmt); - return _snowflake_execute_put_get_native(sfstmt, result_capture); + return _snowflake_execute_put_get_native(sfstmt, NULL, 0, result_capture); } clear_snowflake_error(&sfstmt->error); @@ -2419,6 +2594,7 @@ SF_STATUS STDCALL _snowflake_execute_ex(SF_STMT *sfstmt, }; size_t i; cJSON *bindings = NULL; + char* bind_stage = NULL; SF_BIND_INPUT *input; const char *type; char *value; @@ -2427,60 +2603,13 @@ SF_STATUS STDCALL _snowflake_execute_ex(SF_STMT *sfstmt, sfstmt->sequence_counter = ++sfstmt->connection->sequence_counter; _mutex_unlock(&sfstmt->connection->mutex_sequence_counter); - if (_snowflake_get_current_param_style(sfstmt) == POSITIONAL) + if (_snowflake_needs_stage_binding(sfstmt)) { - bindings = snowflake_cJSON_CreateObject(); - for (i = 0; i < sfstmt->params_len; i++) - { - cJSON *binding; - input = (SF_BIND_INPUT *) sf_param_store_get(sfstmt->params, - i+1,NULL); - if (input == NULL) { - continue; - } - // TODO check if input is null and either set error or write msg to log - type = snowflake_type_to_string( - c_type_to_snowflake(input->c_type, SF_DB_TYPE_TIMESTAMP_NTZ)); - value = value_to_string(input->value, input->len, input->c_type); - binding = snowflake_cJSON_CreateObject(); - char idxbuf[20]; - sf_sprintf(idxbuf, sizeof(idxbuf), "%lu", (unsigned long) (i + 1)); - snowflake_cJSON_AddStringToObject(binding, "type", type); - snowflake_cJSON_AddStringToObject(binding, "value", value); - snowflake_cJSON_AddItemToObject(bindings, idxbuf, binding); - if (value) { - SF_FREE(value); - } - } + bind_stage = _snowflake_stage_bind_upload(sfstmt); } - else if (_snowflake_get_current_param_style(sfstmt) == NAMED) + if (!bind_stage) { - bindings = snowflake_cJSON_CreateObject(); - char *named_param = NULL; - for(i = 0; i < sfstmt->params_len; i++) - { - cJSON *binding; - named_param = (char *)(((NamedParams *)sfstmt->name_list)->name_list[i]); - input = (SF_BIND_INPUT *) sf_param_store_get(sfstmt->params, - 0,named_param); - if (input == NULL) - { - log_error("_snowflake_execute_ex: No parameter by this name %s",named_param); - continue; - } - type = snowflake_type_to_string( - c_type_to_snowflake(input->c_type, SF_DB_TYPE_TIMESTAMP_NTZ)); - value = value_to_string(input->value, input->len, input->c_type); - binding = snowflake_cJSON_CreateObject(); - - snowflake_cJSON_AddStringToObject(binding, "type", type); - snowflake_cJSON_AddStringToObject(binding, "value", value); - snowflake_cJSON_AddItemToObject(bindings, named_param, binding); - if (value) - { - SF_FREE(value); - } - } + bindings = _snowflake_get_binding_json(sfstmt); } if (is_string_empty(sfstmt->connection->directURL) && @@ -2500,7 +2629,12 @@ SF_STATUS STDCALL _snowflake_execute_ex(SF_STMT *sfstmt, is_string_empty(sfstmt->connection->directURL) ? NULL : sfstmt->request_id, is_describe_only, sfstmt->multi_stmt_count); - if (bindings != NULL) { + if (bind_stage) + { + snowflake_cJSON_AddStringToObject(body, "bindStage", bind_stage); + SF_FREE(bind_stage); + } + else if (bindings != NULL) { /* binding parameters if exists */ snowflake_cJSON_AddItemToObject(body, "bindings", bindings); } @@ -2787,6 +2921,9 @@ SF_STATUS STDCALL snowflake_stmt_get_attr( case SF_STMT_MULTI_STMT_COUNT: *value = &sfstmt->multi_stmt_count; break; + case SF_STMT_PARAMSET_SIZE: + *value = &sfstmt->paramset_size; + break; default: SET_SNOWFLAKE_ERROR( &sfstmt->error, SF_STATUS_ERROR_BAD_ATTRIBUTE_TYPE, @@ -2810,6 +2947,9 @@ SF_STATUS STDCALL snowflake_stmt_set_attr( case SF_STMT_MULTI_STMT_COUNT: sfstmt->multi_stmt_count = value ? *((int64*)value) : SF_MULTI_STMT_COUNT_UNSET; break; + case SF_STMT_PARAMSET_SIZE: + sfstmt->paramset_size = value ? *((int64*)value) : 1; + break; default: SET_SNOWFLAKE_ERROR( &sfstmt->error, SF_STATUS_ERROR_BAD_ATTRIBUTE_TYPE, diff --git a/lib/client_int.h b/lib/client_int.h index 47db68c595..c6e97ca986 100644 --- a/lib/client_int.h +++ b/lib/client_int.h @@ -25,6 +25,9 @@ #define SF_DEFAULT_MAX_OBJECT_SIZE 16777216 +#define SF_DEFAULT_STAGE_BINDING_THRESHOLD 65280 +#define SF_DEFAULT_STAGE_BINDING_MAX_FILESIZE 100 * 1024 * 1024 + // defaults for put get configurations #define SF_DEFAULT_PUT_COMPRESS_LEVEL (-1) #define SF_MAX_PUT_COMPRESS_LEVEL 9 @@ -173,6 +176,12 @@ sf_bool STDCALL _is_put_get_command(char* sql_text); */ PARAM_TYPE STDCALL _snowflake_get_param_style(const SF_BIND_INPUT *input); +/** + * @param sfstmt SNOWFLAKE_STMT context. + * @return parameter bindings in cJSON. + */ +cJSON* STDCALL _snowflake_get_binding_json(SF_STMT *sfstmt); + #ifdef __cplusplus extern "C" { #endif @@ -190,13 +199,52 @@ _snowflake_query_put_get_legacy(SF_STMT* sfstmt, const char* command, size_t com /** * Executes put get command natively. * @param sfstmt SNOWFLAKE_STMT context. + * @param upload_stream Internal support for bind uploading, pointer to std::basic_iostream. + * @param stream_size The data size of upload_stream. * @param raw_response_buffer optional pointer to an SF_QUERY_RESULT_CAPTURE, * * @return 0 if success, otherwise an errno is returned. */ SF_STATUS STDCALL _snowflake_execute_put_get_native( SF_STMT *sfstmt, + void* upload_stream, + size_t stream_size, struct SF_QUERY_RESULT_CAPTURE* result_capture); + +/* + * @return size of single binding value per data type. + */ +size_t STDCALL _snowflake_get_binding_value_size(SF_BIND_INPUT* bind); + +#define SF_PARAM_NAME_BUF_LEN 20 +/** + * Get parameter binding by index for both POSITIONAL and NAMED cases. + * @param sfstmt SNOWFLAKE_STMT context. + * @param index The 0 based index of parameter binding to get. + * @param name Output the name of binding. + * @param name_buf The buffer to store name. + Used for POSITIONAL and name will point to this buffer in such case. + * @param name_buf_len The size of name_buf. + * @return parameter binding with specified index. + */ +SF_BIND_INPUT* STDCALL _snowflake_get_binding_by_index(SF_STMT* sfstmt, + size_t index, + char** name, + char* name_buf, + size_t name_buf_len); + +sf_bool STDCALL _snowflake_needs_stage_binding(SF_STMT* sfstmt); + +/** + * Upload parameter bindings through internal stage. + * @param sfstmt SNOWFLAKE_STMT context. + * + * @return Stage path for uploaded bindings if success, + * otherwise NULL is returned and error is put in sfstmt->error. + */ +char* STDCALL +_snowflake_stage_bind_upload(SF_STMT* sfstmt); + #ifdef __cplusplus } // extern "C" #endif diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 35d42ed177..a56a33f439 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -150,6 +150,11 @@ message("CLOUD_PROVIDER is set to " $ENV{CLOUD_PROVIDER}) if (APPLE OR WIN32 OR CLIENT_CODE_COVERAGE) set(TESTS_C ${TESTS_C}) set(TESTS_CXX ${TESTS_CXX} ${TESTS_PUTGET}) +elseif(DEFINED ENV{GITHUB_ACTIONS}) + if ("$ENV{BUILD_TYPE}" STREQUAL "Debug") + message("Skip CXX test on github Linux Debug for disk space issue") + set (TESTS_CXX "") + endif () endif () if (UNIX) diff --git a/tests/test_bind_params.c b/tests/test_bind_params.c index 255cc0b857..0f65f57a3a 100644 --- a/tests/test_bind_params.c +++ b/tests/test_bind_params.c @@ -4,6 +4,7 @@ #include #include "utils/test_setup.h" +#include "memory.h" #define INPUT_ARRAY_SIZE 3 @@ -111,10 +112,198 @@ void test_bind_parameters(void **unused) { snowflake_term(sf); } +void test_array_binding_core(unsigned int array_size) { + /* init */ + SF_STATUS status; + int8* int8_array = NULL; + int8 int8_value = -12; + char int8_expected_result[] = "-12"; + uint8* uint8_array = NULL; + uint8 uint8_value = 12; + char uint8_expected_result[] = "12"; + int64* int64_array = NULL; + int64 int64_value = 12345; + char int64_expected_result[] = "12345"; + uint64* uint64_array = NULL; + uint64 uint64_value = 12345; + char uint64_expected_result[] = "12345"; + float64* float_array = NULL; + float64 float_value = 1.23; + char float_expected_result[] = "1.23"; + char* string_array = NULL; + char string_value[] = "str"; + char string_expected_result[] = "str"; + unsigned char* binary_array = NULL; + unsigned char binary_value[] = {0x12, 0x34, 0x56, 0x78}; + char binary_expected_result[] = "12345678"; + sf_bool* bool_array = NULL; + sf_bool bool_value = SF_BOOLEAN_TRUE; + char bool_expected_result[] = "1"; + SF_BIND_INPUT int8_input; + SF_BIND_INPUT uint8_input; + SF_BIND_INPUT int64_input; + SF_BIND_INPUT uint64_input; + SF_BIND_INPUT float_input; + SF_BIND_INPUT string_input; + SF_BIND_INPUT binary_input; + SF_BIND_INPUT bool_input; + + SF_BIND_INPUT input_array[8]; + char* expected_results[8]; + unsigned int i = 0, j = 0; + + // initialize bindings with argument + int8_array = SF_CALLOC(array_size, sizeof(int8_value)); + uint8_array = SF_CALLOC(array_size, sizeof(uint8_value)); + int64_array = SF_CALLOC(array_size, sizeof(int64_value)); + uint64_array = SF_CALLOC(array_size, sizeof(uint64_value)); + float_array = SF_CALLOC(array_size, sizeof(float_value)); + string_array = SF_CALLOC(array_size, sizeof(string_value)); + binary_array = SF_CALLOC(array_size, sizeof(binary_value)); + bool_array = SF_CALLOC(array_size, sizeof(bool_value)); + + for (i = 0; i < array_size; i++) + { + int8_array[i] = int8_value; + uint8_array[i] = uint8_value; + int64_array[i] = int64_value; + uint64_array[i] = uint64_value; + float_array[i] = float_value; + memcpy(string_array + sizeof(string_value) * i, string_value, sizeof(string_value)); + memcpy(binary_array + sizeof(binary_value) * i, binary_value, sizeof(binary_value)); + bool_array[i] = bool_value; + } + + snowflake_bind_input_init(&int8_input); + snowflake_bind_input_init(&uint8_input); + snowflake_bind_input_init(&int64_input); + snowflake_bind_input_init(&uint64_input); + snowflake_bind_input_init(&float_input); + snowflake_bind_input_init(&string_input); + snowflake_bind_input_init(&binary_input); + snowflake_bind_input_init(&bool_input); + + int8_input.idx = 1; + int8_input.c_type = SF_C_TYPE_INT8; + int8_input.value = int8_array; + + uint8_input.idx = 2; + uint8_input.c_type = SF_C_TYPE_UINT8; + uint8_input.value = uint8_array; + + int64_input.idx = 3; + int64_input.c_type = SF_C_TYPE_INT64; + int64_input.value = int64_array; + + uint64_input.idx = 4; + uint64_input.c_type = SF_C_TYPE_UINT64; + uint64_input.value = uint64_array; + + float_input.idx = 5; + float_input.c_type = SF_C_TYPE_FLOAT64; + float_input.value = float_array; + + string_input.idx = 6; + string_input.c_type = SF_C_TYPE_STRING; + string_input.value = string_array; + string_input.len = sizeof(string_value); + + binary_input.idx = 7; + binary_input.c_type = SF_C_TYPE_BINARY; + binary_input.value = binary_array; + binary_input.len = sizeof(binary_value); + + bool_input.idx = 8; + bool_input.c_type = SF_C_TYPE_BOOLEAN; + bool_input.value = bool_array; + + input_array[0] = int8_input; + input_array[1] = uint8_input; + input_array[2] = int64_input; + input_array[3] = uint64_input; + input_array[4] = float_input; + input_array[5] = string_input; + input_array[6] = binary_input; + input_array[7] = bool_input; + + expected_results[0] = int8_expected_result; + expected_results[1] = uint8_expected_result; + expected_results[2] = int64_expected_result; + expected_results[3] = uint64_expected_result; + expected_results[4] = float_expected_result; + expected_results[5] = string_expected_result; + expected_results[6] = binary_expected_result; + expected_results[7] = bool_expected_result; + + /* Connect with all parameters set */ + SF_CONNECT* sf = setup_snowflake_connection(); + // turn on FAIL_OPEN to around certificate issue with GCP + sf_bool value = SF_BOOLEAN_TRUE; + snowflake_set_attribute(sf, SF_CON_OCSP_FAIL_OPEN, &value); + status = snowflake_connect(sf); + assert_int_equal(status, SF_STATUS_SUCCESS); + + /* Create a statement once and reused */ + SF_STMT* stmt = snowflake_stmt(sf); + status = snowflake_query( + stmt, + "create or replace temporary table t (c1 number, c2 number, c3 number, c4 number, c5 float, c6 string, c7 binary, c8 boolean)", + 0 + ); + assert_int_equal(status, SF_STATUS_SUCCESS); + + int64 paramset_size = array_size; + status = snowflake_stmt_set_attr(stmt, SF_STMT_PARAMSET_SIZE, ¶mset_size); + status = snowflake_prepare( + stmt, + "insert into t values(?, ?, ?, ?, ?, ?, ?, ?)", + 0 + ); + assert_int_equal(status, SF_STATUS_SUCCESS); + + status = snowflake_bind_param_array(stmt, input_array, sizeof(input_array) / sizeof(SF_BIND_INPUT)); + assert_int_equal(status, SF_STATUS_SUCCESS); + + status = snowflake_execute(stmt); + assert_int_equal(status, SF_STATUS_SUCCESS); + assert_int_equal(snowflake_affected_rows(stmt), array_size); + + status = snowflake_query(stmt, "select * from t", 0); + assert_int_equal(status, SF_STATUS_SUCCESS); + assert_int_equal(snowflake_num_rows(stmt), array_size); + + for (i = 0; i < array_size; i++) + { + status = snowflake_fetch(stmt); + if (status != SF_STATUS_SUCCESS) { + dump_error(&(stmt->error)); + } + assert_int_equal(status, SF_STATUS_SUCCESS); + const char* result = NULL; + for (j = 0; j < 8; j++) + { + snowflake_column_as_const_str(stmt, j + 1, &result); + assert_string_equal(result, expected_results[j]); + } + } + snowflake_stmt_term(stmt); + snowflake_term(sf); +} + +void test_array_binding_normal(void** unused) { + test_array_binding_core(1000); +} + +void test_array_binding_stage(void** unused) { + test_array_binding_core(100000); +} + int main(void) { initialize_test(SF_BOOLEAN_FALSE); const struct CMUnitTest tests[] = { cmocka_unit_test(test_bind_parameters), + cmocka_unit_test(test_array_binding_normal), + cmocka_unit_test(test_array_binding_stage), }; int ret = cmocka_run_group_tests(tests, NULL, NULL); snowflake_global_term(); From 0f9f288da49a5f1acd0d225b972b4a99cc12ce0e Mon Sep 17 00:00:00 2001 From: Harry Xi Date: Wed, 4 Dec 2024 09:58:38 -0800 Subject: [PATCH 6/8] SNOW-1821508: OCSP deprecation plan steps (#789) --- deps/curl/lib/vtls/sf_ocsp.c | 20 +++++--------------- lib/client.c | 4 ++-- scripts/build_curl.bat | 2 +- scripts/build_curl.sh | 2 +- tests/test_ocsp_fail_open.c | 2 +- 5 files changed, 10 insertions(+), 20 deletions(-) diff --git a/deps/curl/lib/vtls/sf_ocsp.c b/deps/curl/lib/vtls/sf_ocsp.c index 2df7a8d0b4..f559ec69c6 100644 --- a/deps/curl/lib/vtls/sf_ocsp.c +++ b/deps/curl/lib/vtls/sf_ocsp.c @@ -100,7 +100,7 @@ typedef pthread_mutex_t SF_MUTEX_HANDLE; #define CA_OCSP_RESPONDER_MAX_RETRY_FO 1 // Max number of connection retry attempts for OCSP Responder in Fail Close Mode -#define CA_OCSP_RESPONDER_MAX_RETRY_FC 3 +#define CA_OCSP_RESPONDER_MAX_RETRY_FC 2 // Max number of connection retry attempts for OCSP Cache Server #define OCSP_CACHE_SERVER_MAX_RETRY 1 @@ -1667,7 +1667,6 @@ OCSP_RESPONSE * getOCSPResponse(X509 *cert, X509 *issuer, char *ocsp_url = sk_OPENSSL_STRING_value(ocsp_list, i); if (ocsp_url == NULL) { - failf(data, "OCSP Validation URL is not present"); /* * Try the next OCSP Server in ocsp_list, if present. */ @@ -1698,8 +1697,9 @@ OCSP_RESPONSE * getOCSPResponse(X509 *cert, X509 *issuer, break; /* good if any OCSP server works */ } - if(ocsp_url_missing || ocsp_url_invalid) + if((ocsp_url_missing || ocsp_url_invalid) && (ocsp_fail_open == DISABLED)) { + failf(data, "OCSP Validation URL is not present or invalid."); sf_otd_set_event_sub_type(OCSP_URL_MISSING_OR_INVALID, ocsp_log_data); } @@ -1720,18 +1720,8 @@ static void printOCSPFailOpenWarning(SF_OTD *ocsp_log, struct Curl_easy *data, b { char *ocsp_log_data = NULL; ocsp_log_data = generateOCSPTelemetryData(ocsp_log); - infof(data, "WARNING!!! Using fail-open to connect. Driver is connecting to an " - "HTTPS endpoint without OCSP based Certificate Revocation checking " - "as it could not obtain a valid OCSP Response to use from the CA OCSP " - "responder. Details:%s",ocsp_log_data); - if (ocsp_log_data) - { - if (oob_enable) - { - sendOOBevent(ocsp_log_data); - } - if(ocsp_log_data) sf_curl_cJSON_free(ocsp_log_data); - } + infof(data, "OCSP responder didn't respond correctly. Assuming certificate is not revoked." + " Details:%s", ocsp_log_data); } static char * generateOCSPTelemetryData(SF_OTD *ocsp_log) diff --git a/lib/client.c b/lib/client.c index 99aef499a9..0cadc7763d 100644 --- a/lib/client.c +++ b/lib/client.c @@ -708,7 +708,7 @@ SF_CONNECT *STDCALL snowflake_init() { sf->passcode = NULL; sf->passcode_in_password = SF_BOOLEAN_FALSE; sf->insecure_mode = SF_BOOLEAN_FALSE; - sf->ocsp_fail_open = SF_BOOLEAN_FALSE; + sf->ocsp_fail_open = SF_BOOLEAN_TRUE; sf->autocommit = SF_BOOLEAN_TRUE; sf->qcc_disable = SF_BOOLEAN_FALSE; sf->include_retry_reason = SF_BOOLEAN_TRUE; @@ -1129,7 +1129,7 @@ SF_STATUS STDCALL snowflake_set_attribute( sf->insecure_mode = value ? *((sf_bool *) value) : SF_BOOLEAN_FALSE; break; case SF_CON_OCSP_FAIL_OPEN: - sf->ocsp_fail_open = value ? *((sf_bool*)value) : SF_BOOLEAN_FALSE; + sf->ocsp_fail_open = value ? *((sf_bool*)value) : SF_BOOLEAN_TRUE; break; case SF_CON_LOGIN_TIMEOUT: sf->login_timeout = value ? *((int64 *) value) : SF_LOGIN_TIMEOUT; diff --git a/scripts/build_curl.bat b/scripts/build_curl.bat index b770c6ffaa..1004b64d71 100644 --- a/scripts/build_curl.bat +++ b/scripts/build_curl.bat @@ -11,7 +11,7 @@ @echo off set CURL_SRC_VERSION=8.10.1 -set CURL_BUILD_VERSION=2 +set CURL_BUILD_VERSION=4 set CURL_VERSION=%CURL_SRC_VERSION%.%CURL_BUILD_VERSION% call %* goto :EOF diff --git a/scripts/build_curl.sh b/scripts/build_curl.sh index 8b2462dd82..63df1259c8 100755 --- a/scripts/build_curl.sh +++ b/scripts/build_curl.sh @@ -13,7 +13,7 @@ function usage() { set -o pipefail CURL_SRC_VERSION=8.10.1 -CURL_BUILD_VERSION=3 +CURL_BUILD_VERSION=4 CURL_VERSION=${CURL_SRC_VERSION}.${CURL_BUILD_VERSION} DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" diff --git a/tests/test_ocsp_fail_open.c b/tests/test_ocsp_fail_open.c index 58bf0ddc4b..451e493a67 100644 --- a/tests/test_ocsp_fail_open.c +++ b/tests/test_ocsp_fail_open.c @@ -4,7 +4,7 @@ #include "utils/test_setup.h" -#define FAIL_OPEN_DEFAULT SF_BOOLEAN_FALSE +#define FAIL_OPEN_DEFAULT SF_BOOLEAN_TRUE void setCacheFile(char *cache_file) { From 8ff0b3817cd507b521c491b4854a9927635f42f2 Mon Sep 17 00:00:00 2001 From: Maxim Mishchenko Date: Fri, 13 Dec 2024 12:09:10 +0100 Subject: [PATCH 7/8] SNOW-1852178 Remove the deprecated part of API (#794) --- include/snowflake/SF_CRTFunctionSafe.h | 34 ----------------------- include/snowflake/Simba_CRTFunctionSafe.h | 12 -------- include/snowflake/client.h | 5 ---- include/snowflake/platform.h | 12 ++++++++ 4 files changed, 12 insertions(+), 51 deletions(-) delete mode 100644 include/snowflake/Simba_CRTFunctionSafe.h diff --git a/include/snowflake/SF_CRTFunctionSafe.h b/include/snowflake/SF_CRTFunctionSafe.h index 636323487e..f160f4a2f9 100644 --- a/include/snowflake/SF_CRTFunctionSafe.h +++ b/include/snowflake/SF_CRTFunctionSafe.h @@ -26,40 +26,6 @@ extern "C" { #endif #endif -// Defined for #pragma warning messages. -#define MACRO_TO_STRING2(x) #x -#define MACRO_TO_STRING(x) MACRO_TO_STRING2(x) - -#if defined(_MSC_VER) -#define SF_MACRO_DEPRECATED_WARNING(MSG) __pragma(message ( __FILE__ "(" MACRO_TO_STRING(__LINE__) ") : warning C4996: " MSG)) -#else -#define SF_MACRO_DEPRECATED_WARNING(MSG) _Pragma(MACRO_TO_STRING(GCC warning MSG)) -#endif - -// sf_getenv and sf_strerror is unsafe and deprecated. -// Please change to use sf_getenv_s and sf_strerror_s. -#define sf_getenv SF_MACRO_DEPRECATED_WARNING("sf_getenv is deprecated, please use sf_getenv_s instead.") getenv -#define sf_strerror SF_MACRO_DEPRECATED_WARNING("sf_strerror is deprecated, please use sf_strerror_s instead.") strerror -// sf_free_s is deprecated. It does nothing and should not be called anymore. -#define sf_free_s SF_MACRO_DEPRECATED_WARNING("sf_free_s is deprecated and it does nothing.") - -// all sb_ functions are deprecated and should use sf_ functions instead. -#define sb_memcpy SF_MACRO_DEPRECATED_WARNING("sb_memcpy is deprecated, please use sf_memcpy instead.") sf_memcpy -#define sb_copy SF_MACRO_DEPRECATED_WARNING("sb_copy is deprecated, please use sf_copy instead.") sf_copy -#define sb_cat SF_MACRO_DEPRECATED_WARNING("sb_cat is deprecated, please use sf_cat instead.") sf_cat -#define sb_strcpy SF_MACRO_DEPRECATED_WARNING("sb_strcpy is deprecated, please use sf_strcpy instead.") sf_strcpy -#define sb_strncpy SF_MACRO_DEPRECATED_WARNING("sb_strncpy is deprecated, please use sf_strncpy instead.") sf_strncpy -#define sb_strcat SF_MACRO_DEPRECATED_WARNING("sb_strcat is deprecated, please use sf_strcat instead.") sf_strcat -#define sb_strncat SF_MACRO_DEPRECATED_WARNING("sb_strncat is deprecated, please use sf_strncat instead.") sf_strncat -#define sb_vsnprintf SF_MACRO_DEPRECATED_WARNING("sb_vsnprintf is deprecated, please use sf_vsnprintf instead.") sf_vsnprintf -#define sb_sprintf SF_MACRO_DEPRECATED_WARNING("sb_sprintf is deprecated, please use sf_sprintf instead.") sf_sprintf -#define sb_snprintf SF_MACRO_DEPRECATED_WARNING("sb_snprintf is deprecated, please use sf_snprintf instead.") sf_snprintf -#define sb_sscanf SF_MACRO_DEPRECATED_WARNING("sb_sscanf is deprecated, please use sf_sscanf instead.") sf_sscanf -#define sb_vfprintf SF_MACRO_DEPRECATED_WARNING("sb_vfprintf is deprecated, please use sf_vfprintf instead.") sf_vfprintf -#define sb_fprintf SF_MACRO_DEPRECATED_WARNING("sb_fprintf is deprecated, please use sf_fprintf instead.") sf_fprintf -#define sb_printf SF_MACRO_DEPRECATED_WARNING("sb_printf is deprecated, please use sf_printf instead.") sf_printf -#define sb_fopen SF_MACRO_DEPRECATED_WARNING("sb_fopen is deprecated, please use sf_fopen instead.") sf_fopen - /// @brief Copy bytes between buffers. /// /// @param out_dest Destination buffer. (NOT OWN) diff --git a/include/snowflake/Simba_CRTFunctionSafe.h b/include/snowflake/Simba_CRTFunctionSafe.h deleted file mode 100644 index 60f4a61263..0000000000 --- a/include/snowflake/Simba_CRTFunctionSafe.h +++ /dev/null @@ -1,12 +0,0 @@ -/* - * Copyright (c) 2019-2024 Snowflake Computing, Inc. All rights reserved. - */ - -#ifndef _SIMBA_CRTFUNCTIONSAFE_H_ -#define _SIMBA_CRTFUNCTIONSAFE_H_ - -#include "SF_CRTFunctionSafe.h" - -SF_MACRO_DEPRECATED_WARNING("Simba_CRTFunctionSafe.h is deprecated. Please include SF_CRTFunctionSafe.h instead.") - -#endif diff --git a/include/snowflake/client.h b/include/snowflake/client.h index 80a9bcae7c..74eedf68fc 100644 --- a/include/snowflake/client.h +++ b/include/snowflake/client.h @@ -66,11 +66,6 @@ extern "C" { */ #define SF_COMMAND_LEN 10 -/** - * The maximum object size - */ -#define SF_MAX_OBJECT_SIZE SF_MACRO_DEPRECATED_WARNING("SF_MAX_OBJECT_SIZE is deprecated, please use snowflake_get_attribute() instead to retrieve the max LOB size.") 16777216 - /** * Login timeout in seconds */ diff --git a/include/snowflake/platform.h b/include/snowflake/platform.h index 807d6904f0..caaba53a63 100755 --- a/include/snowflake/platform.h +++ b/include/snowflake/platform.h @@ -46,6 +46,18 @@ typedef pthread_mutex_t SF_MUTEX_HANDLE; #endif +// Defined for #pragma warning messages. +#define MACRO_TO_STRING2(x) #x +#define MACRO_TO_STRING(x) MACRO_TO_STRING2(x) + +// For marking functions and variables deprecated, i.e. like following: +// #define FOO SF_MACRO_DEPRECATED_WARNING("FOO is deprecated, use BAR instead") BAR +#if defined(_MSC_VER) +#define SF_MACRO_DEPRECATED_WARNING(MSG) __pragma(message ( __FILE__ "(" MACRO_TO_STRING(__LINE__) ") : warning C4996: " MSG)) +#else +#define SF_MACRO_DEPRECATED_WARNING(MSG) _Pragma(MACRO_TO_STRING(GCC warning MSG)) +#endif + #include "SF_CRTFunctionSafe.h" struct tm *STDCALL sf_gmtime(const time_t *timep, struct tm *result); From e107644f8255ade63b5c4fdf2ff7fa6dc440008f Mon Sep 17 00:00:00 2001 From: Dominik Przybysz <132913826+sfc-gh-dprzybysz@users.noreply.github.com> Date: Fri, 20 Dec 2024 12:20:27 +0100 Subject: [PATCH 8/8] SNOW-1732752: Set version to 2.0.0 (#804) --- include/snowflake/client.h | 7 +------ include/snowflake/version.h | 7 +------ 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/include/snowflake/client.h b/include/snowflake/client.h index 74eedf68fc..4adbabbaab 100644 --- a/include/snowflake/client.h +++ b/include/snowflake/client.h @@ -18,12 +18,7 @@ extern "C" { /** * API Name */ -/* TODO: Temporarily change to ODBC for now to pass the test before - * features (PUT for GCP, multiple statements etc.) unblocked - * on server side. - * Need to revert to C_API when merging to master. - */ -#define SF_API_NAME "ODBC" +#define SF_API_NAME "C API" /** * SQLState code length diff --git a/include/snowflake/version.h b/include/snowflake/version.h index 33abb0338e..9c70eb1707 100644 --- a/include/snowflake/version.h +++ b/include/snowflake/version.h @@ -5,12 +5,7 @@ #ifndef SNOWFLAKE_CLIENT_VERSION_H #define SNOWFLAKE_CLIENT_VERSION_H -/* TODO: Temporarily change to ODBC version for now to pass the test before - * features (PUT for GCP, multiple statements etc.) unblocked - * on server side. - * Need to revert to libsfclient version when merging to master. - */ -#define SF_API_VERSION "3.0.1" +#define SF_API_VERSION "2.0.0" #endif /* SNOWFLAKE_CLIENT_VERSION_H */