From 9a478edb06684cf663e2fda4b96a1a92aa9e8fb4 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Mon, 20 May 2024 19:15:19 -0600 Subject: [PATCH] Fix duplicate definition when using aws-sdk-cpp. re: Issue https://github.com/Unidata/netcdf-c/issues/2927 The NC_s3sdkinitialize NC_s3sdkfinalize functions were misplaced. They should have been moved from ds3util.c to ncs3sdk_h5.c. When using ncs3sdl_aws.cpp, this resulted in a duplicate definition. Also, found and fixed a memory leak in the NCZarr S3 code. --- include/ncs3sdk.h | 4 ++-- libdispatch/ds3util.c | 48 +++++++++++-------------------------- libdispatch/ncs3sdk_aws.cpp | 9 +++---- libdispatch/ncs3sdk_h5.c | 31 ++++++++++++++++++++++++ libnczarr/zmap_s3sdk.c | 11 +++++---- nczarr_test/run_corrupt.sh | 5 +++- 6 files changed, 62 insertions(+), 46 deletions(-) diff --git a/include/ncs3sdk.h b/include/ncs3sdk.h index 7be09da337..50f2ad2708 100644 --- a/include/ncs3sdk.h +++ b/include/ncs3sdk.h @@ -45,6 +45,7 @@ struct NCglobalstate; extern "C" { #endif +/* API for ncs3sdk_XXX.[c|cpp] */ EXTERNL int NC_s3sdkinitialize(void); EXTERNL int NC_s3sdkfinalize(void); EXTERNL void* NC_s3sdkcreateclient(NCS3INFO* context); @@ -60,8 +61,7 @@ EXTERNL int NC_s3sdksearch(void* s3client0, const char* bucket, const char* pref EXTERNL int NC_s3sdkdeletekey(void* client0, const char* bucket, const char* pathkey, char** errmsgp); /* From ds3util.c */ -EXTERNL int NC_s3sdkinitialize(void); -EXTERNL int NC_s3sdkfinalize(void); +EXTERNL void NC_s3sdkenvironment(void); EXTERNL int NC_getdefaults3region(NCURI* uri, const char** regionp); EXTERNL int NC_s3urlprocess(NCURI* url, NCS3INFO* s3, NCURI** newurlp); diff --git a/libdispatch/ds3util.c b/libdispatch/ds3util.c index 8db08d2e82..aab8fe987b 100644 --- a/libdispatch/ds3util.c +++ b/libdispatch/ds3util.c @@ -43,9 +43,6 @@ enum URLFORMAT {UF_NONE=0, UF_VIRTUAL=1, UF_PATH=2, UF_S3=3, UF_OTHER=4}; static const char* awsconfigfiles[] = {".aws/config",".aws/credentials",NULL}; #define NCONFIGFILES (sizeof(awsconfigfiles)/sizeof(char*)) -static int ncs3_initialized = 0; -static int ncs3_finalized = 0; - /**************************************************/ /* Forward */ @@ -56,38 +53,21 @@ static int awsparse(const char* text, NClist* profiles); /**************************************************/ /* Capture environmental Info */ -EXTERNL int -NC_s3sdkinitialize(void) -{ - if(!ncs3_initialized) { - ncs3_initialized = 1; - ncs3_finalized = 0; - } - { - /* Get various environment variables as defined by the AWS sdk */ - NCglobalstate* gs = NC_getglobalstate(); - if(getenv("AWS_REGION")!=NULL) - gs->aws.default_region = nulldup(getenv("AWS_REGION")); - else if(getenv("AWS_DEFAULT_REGION")!=NULL) - gs->aws.default_region = nulldup(getenv("AWS_DEFAULT_REGION")); - else if(gs->aws.default_region == NULL) - gs->aws.default_region = nulldup(AWS_GLOBAL_DEFAULT_REGION); - gs->aws.access_key_id = nulldup(getenv("AWS_ACCESS_KEY_ID")); - gs->aws.config_file = nulldup(getenv("AWS_CONFIG_FILE")); - gs->aws.profile = nulldup(getenv("AWS_PROFILE")); - gs->aws.secret_access_key = nulldup(getenv("AWS_SECRET_ACCESS_KEY")); - } - return NC_NOERR; -} - -EXTERNL int -NC_s3sdkfinalize(void) +EXTERNL void +NC_s3sdkenvironment(void) { - if(!ncs3_finalized) { - ncs3_initialized = 0; - ncs3_finalized = 1; - } - return NC_NOERR; + /* Get various environment variables as defined by the AWS sdk */ + NCglobalstate* gs = NC_getglobalstate(); + if(getenv("AWS_REGION")!=NULL) + gs->aws.default_region = nulldup(getenv("AWS_REGION")); + else if(getenv("AWS_DEFAULT_REGION")!=NULL) + gs->aws.default_region = nulldup(getenv("AWS_DEFAULT_REGION")); + else if(gs->aws.default_region == NULL) + gs->aws.default_region = nulldup(AWS_GLOBAL_DEFAULT_REGION); + gs->aws.access_key_id = nulldup(getenv("AWS_ACCESS_KEY_ID")); + gs->aws.config_file = nulldup(getenv("AWS_CONFIG_FILE")); + gs->aws.profile = nulldup(getenv("AWS_PROFILE")); + gs->aws.secret_access_key = nulldup(getenv("AWS_SECRET_ACCESS_KEY")); } /**************************************************/ diff --git a/libdispatch/ncs3sdk_aws.cpp b/libdispatch/ncs3sdk_aws.cpp index 806edae8a8..e1bceb7835 100644 --- a/libdispatch/ncs3sdk_aws.cpp +++ b/libdispatch/ncs3sdk_aws.cpp @@ -133,10 +133,9 @@ NC_s3sdkinitialize(void) if(!ncs3_initialized) { ncs3_initialized = 1; ncs3_finalized = 0; - #ifdef DEBUG - //ncs3options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Debug; + //ncs3options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Debug; ncs3options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Trace; ncs3options.httpOptions.installSigPipeHandler = true; ncs3options.loggingOptions.logger_create_fn = [] { return std::make_shared(Aws::Utils::Logging::LogLevel::Trace); }; @@ -144,6 +143,9 @@ NC_s3sdkinitialize(void) #endif Aws::InitAPI(ncs3options); + /* Get environment information */ + NC_s3sdkenvironment(); + } return NCUNTRACE(NC_NOERR); } @@ -500,7 +502,6 @@ NC_s3sdkwriteobject(void* s3client0, const char* bucket, const char* pathkey, s int stat = NC_NOERR; const char* key = NULL; - const char* mcontent = (char*)content; NCTRACE(11,"bucket=%s pathkey=%s count=%lld content=%p",bucket,pathkey,count,content); AWSS3CLIENT s3client = (AWSS3CLIENT)s3client0; @@ -535,7 +536,7 @@ NC_s3sdkwriteobject(void* s3client0, const char* bucket, const char* pathkey, s put_request.SetContentLength((long long)count); std::shared_ptr data = std::shared_ptr(new Aws::StringStream()); - data->rdbuf()->pubsetbuf((char*)content,count); + data->rdbuf()->pubsetbuf((char*)content,(std::streamsize)count); put_request.SetBody(data); auto put_result = AWSS3GET(s3client)->PutObject(put_request); if(!put_result.IsSuccess()) { diff --git a/libdispatch/ncs3sdk_h5.c b/libdispatch/ncs3sdk_h5.c index 5f7f223db9..f8263293b7 100644 --- a/libdispatch/ncs3sdk_h5.c +++ b/libdispatch/ncs3sdk_h5.c @@ -108,6 +108,37 @@ static int queryinsert(NClist* list, char* ekey, char* evalue); #define NT(x) ((x)==NULL?"null":x) +/**************************************************/ + +static int ncs3_initialized = 0; +static int ncs3_finalized = 0; + +EXTERNL int +NC_s3sdkinitialize(void) +{ + if(!ncs3_initialized) { + ncs3_initialized = 1; + ncs3_finalized = 0; + } + + /* Get environment information */ + NC_s3sdkenvironment(void); + + return NC_NOERR; +} + +EXTERNL int +NC_s3sdkfinalize(void) +{ + if(!ncs3_finalized) { + ncs3_initialized = 0; + ncs3_finalized = 1; + } + return NC_NOERR; +} + +/**************************************************/ + #if 0 static void dumps3info(NCS3INFO* s3info, const char* tag) diff --git a/libnczarr/zmap_s3sdk.c b/libnczarr/zmap_s3sdk.c index 0acdaf8e99..552a73473d 100644 --- a/libnczarr/zmap_s3sdk.c +++ b/libnczarr/zmap_s3sdk.c @@ -499,20 +499,21 @@ s3clear(void* s3client, const char* bucket, const char* rootkey) { int stat = NC_NOERR; char** list = NULL; - char** p; size_t nkeys = 0; if(s3client && bucket && rootkey) { if((stat = NC_s3sdksearch(s3client, bucket, rootkey, &nkeys, &list, NULL))) goto done; if(list != NULL) { - for(p=list;*p;p++) { + size_t i; + for(i=0;i