From 9a4badaf163164f844b0636f8a0c80b61463c3c1 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Wed, 31 Mar 2021 21:56:36 -0600 Subject: [PATCH 01/16] ckp --- ncdap_test/Makefile.am | 3 ++- ncdap_test/t_auth.c | 41 ++++++++++++++++------------------------- ncdap_test/testauth.sh | 13 +++++++------ 3 files changed, 25 insertions(+), 32 deletions(-) diff --git a/ncdap_test/Makefile.am b/ncdap_test/Makefile.am index b38999acca..998e3ddfa7 100644 --- a/ncdap_test/Makefile.am +++ b/ncdap_test/Makefile.am @@ -74,7 +74,8 @@ check_PROGRAMS += t_misc check_PROGRAMS += t_ncf330 if ENABLE_DAP_AUTH_TESTS -TESTS += testbasicauth.sh +check_PROGRAMS += t_auth +TESTS += testauth.sh TESTS += testcontainerauth.sh endif diff --git a/ncdap_test/t_auth.c b/ncdap_test/t_auth.c index 2156154afe..a8c247885f 100644 --- a/ncdap_test/t_auth.c +++ b/ncdap_test/t_auth.c @@ -28,15 +28,17 @@ See \ref copyright file for more info. #define KEEPRC -#define RC ".ocrc" -#define SPECRC "./ocrc" +#define AUTHTESTSERVER "thredds-dev.unidata.ucar.edu" -#define USERPWD "tiggeUser:tigge" +#define RC ".daprc" +#define SPECRC "./daprc" + +#define USERPWD "authuser:auth" #define COOKIEFILE "./cookies" -#define URL1 "https://%s@%s/dodsC/restrict/testData.nc" -#define URL2 "https://%s/dodsC/restrict/testData.nc" -#define URL3 "https://%s@thredds-test.ucar.edu/thredds/dodsC/restrict/testData.nc" +#define URL1 "https://%s@%s/dodsC/test2/testData.nc" +#define URL2 "https://%s/dodsC/test2/testData.nc" +#define URL3 "https://%s@" AUTHTESTSERVER "/thredds/dodsC/test2/testData.nc" /* Embedded user:pwd */ static char url1[1024]; @@ -45,36 +47,25 @@ static char url1[1024]; static char url2[1024]; /* Test redirect from different machine*/ +#ifndef NOREDIR static char url3[1024]; +#endif static int testrc(const char* prefix, const char* url); static void fillrc(const char* path); static void killrc(); -#ifdef DEBUG -static void -CHECK(int e, const char* msg) -{ - if(e == NC_NOERR) return; - if(msg == NULL) msg = "Error"; - fprintf(stderr,"%s: %s\n", msg, nc_strerror(e)); - exit(1); -} -#endif - int main(int argc, char** argv) { int ncid,retval,pass; - FILE* rc; const char* dfaltsvc; - char buffer[8192]; const char* home; fprintf(stderr,"Testing: Authorization\n"); - dfaltsvc = nc_findtestserver("thredds",REMOTETESTSERVERS); - if(svc == NULL) { + dfaltsvc = nc_findtestserver("thredds",AUTHTESTSERVER); + if(dfaltsvc == NULL) { fprintf(stderr,"WARNING: Cannot locate test server\n"); exit(0); } @@ -110,7 +101,7 @@ fflush(stderr); #ifndef NOLOCAL { /* Test 1: RC in ./ */ - fprintf(stderr,"Testing: user:pwd in %s/%s: %s\n",".",RC); + fprintf(stderr,"Testing: user:pwd in %s/%s\n",".",RC); if(!testrc(".",url2)) { fprintf(stderr,"user:pwd in %s/%s failed\n",".",RC); exit(1); @@ -122,7 +113,7 @@ fflush(stderr); { /* Test 1: RC in HOME */ home = getenv("HOME"); - fprintf(stderr,"user:pwd in %s/%s: %s\n",home,RC); + fprintf(stderr,"user:pwd in %s/%s\n",home,RC); if(!testrc(home,url2)) { fprintf(stderr,"user:pwd in %s/%s failed\n",home,RC); exit(1); @@ -205,10 +196,10 @@ static void killrc() { const char* home; - char path[1024]; #ifdef KEEPRC fprintf(stderr,"kill: ./%s\n",RC); -#else +#else + char path[1024]; snprintf(path,sizeof(path),"%s/%s",".",RC); unlink(path); /* delete the file */ #endif diff --git a/ncdap_test/testauth.sh b/ncdap_test/testauth.sh index 02dd6769ca..29ea27e26f 100755 --- a/ncdap_test/testauth.sh +++ b/ncdap_test/testauth.sh @@ -4,7 +4,7 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh # Enable if using localhost -LOCAL=1 +#LOCAL=1 RCEMBED=1 RCLOCAL=1 @@ -41,12 +41,13 @@ fi # Major parameters -BASICCOMBO="tiggeUser:tigge" -BADCOMBO="tiggeUser:xxxxx" -URLPATH="thredds/dodsC/testRestrictedDataset/testData2.nc" -PROTO=http +AUTHSERVER="thredds-dev.unidata.ucar.edu" +BASICCOMBO="authuser:auth" +BADCOMBO="authuser:xxxxx" +URLPATH="thredds/dodsC/test2/testData.nc" +PROTO=https if test "x$LOCAL" = x ; then -URLSERVER="remotetest.unidata.ucar.edu" +URLSERVER=${AUTHSERVER} else URLSERVER="localhost:8081" fi From 2e1438c4449bd35afafaf8903355906a30bf9cab Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Tue, 6 Apr 2021 21:43:42 -0600 Subject: [PATCH 02/16] ckp --- libdap2/ncd2dispatch.c | 5 ++-- libdap4/d4file.c | 5 ++++ ncdap_test/t_auth.c | 10 +++---- ncdap_test/testauth.sh | 68 +++++++++++++----------------------------- 4 files changed, 32 insertions(+), 56 deletions(-) diff --git a/libdap2/ncd2dispatch.c b/libdap2/ncd2dispatch.c index 1f34b1b682..10dac4138c 100644 --- a/libdap2/ncd2dispatch.c +++ b/libdap2/ncd2dispatch.c @@ -403,9 +403,8 @@ fprintf(stderr,"ce=%s\n",dumpconstraint(dapcomm->oc.dapconstraint)); ocstat = oc_open(dapcomm->oc.urltext,&dapcomm->oc.conn); if(ocstat != OC_NOERR) {THROWCHK(ocstat); goto done;} -#ifdef DEBUG1 - (void)oc_trace_curl(dapcomm->oc.conn); -#endif + if(getenv("CURLOPT_VERBOSE") != NULL) + (void)oc_trace_curl(dapcomm->oc.conn); nullfree(dapcomm->oc.urltext); /* clean up */ dapcomm->oc.urltext = NULL; diff --git a/libdap4/d4file.c b/libdap4/d4file.c index 47a460562b..cb24c0b9f1 100644 --- a/libdap4/d4file.c +++ b/libdap4/d4file.c @@ -117,6 +117,11 @@ NCD4_open(const char * path, int mode, ncsetlogging(1); } + /* Check env values */ + if(getenv("CURLOPT_VERBOSE") != NULL) + d4info->auth->curlflags.verbose = 1; + + /* Setup a curl connection */ { CURL* curl = NULL; /* curl handle*/ diff --git a/ncdap_test/t_auth.c b/ncdap_test/t_auth.c index a8c247885f..e70a068e87 100644 --- a/ncdap_test/t_auth.c +++ b/ncdap_test/t_auth.c @@ -28,17 +28,17 @@ See \ref copyright file for more info. #define KEEPRC -#define AUTHTESTSERVER "thredds-dev.unidata.ucar.edu" +#define AUTHTESTSERVER "thredds.ucar.edu" #define RC ".daprc" #define SPECRC "./daprc" -#define USERPWD "authuser:auth" +#define USERPWD "authtester:auth" #define COOKIEFILE "./cookies" -#define URL1 "https://%s@%s/dodsC/test2/testData.nc" -#define URL2 "https://%s/dodsC/test2/testData.nc" -#define URL3 "https://%s@" AUTHTESTSERVER "/thredds/dodsC/test2/testData.nc" +#define URL1 "https://%s@%s/thredds/dodsC/test3/testData.nc" +#define URL2 "https://thredds/%s/dodsC/test3/testData.nc" +#define URL3 "https://%s@" AUTHTESTSERVER "/thredds/dodsC/test3/testData.nc" /* Embedded user:pwd */ static char url1[1024]; diff --git a/ncdap_test/testauth.sh b/ncdap_test/testauth.sh index 29ea27e26f..600a56429f 100755 --- a/ncdap_test/testauth.sh +++ b/ncdap_test/testauth.sh @@ -3,14 +3,11 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh -# Enable if using localhost -#LOCAL=1 - RCEMBED=1 -RCLOCAL=1 -RCHOME=1 -RCENV=1 -RCPREC=1 +#RCLOCAL=1 +#RCHOME=1 +#RCENV=1 +#RCPREC=1 # Not currently testable in netcdf #RCSPEC=1 @@ -34,34 +31,18 @@ COOKIES="${WD}/test_auth_cookies" RC=.daprc -OCLOGFILE=stderr if test "x$DBG" = x1 ; then SHOW=1 fi # Major parameters -AUTHSERVER="thredds-dev.unidata.ucar.edu" -BASICCOMBO="authuser:auth" -BADCOMBO="authuser:xxxxx" -URLPATH="thredds/dodsC/test2/testData.nc" +AUTHSERVER="thredds.ucar.edu" +BASICCOMBO="authtester:auth" +BADCOMBO="authtester:xxxxx" +URLPATH="thredds/dodsC/test3/testData.nc" PROTO=https -if test "x$LOCAL" = x ; then URLSERVER=${AUTHSERVER} -else -URLSERVER="localhost:8081" -fi - -# See if we need to override -if test "x$URS" != "x" ; then -#https://54.86.135.31/opendap/data/nc/fnoc1.nc.dds -URLSERVER="54.86.135.31" -URLPATH="opendap/data/nc/fnoc1.nc" -BASICCOMBO="$URS" -RCEMBED=0 -NETRC=$NETRCFILE -PROTO=https -fi if test "x$DBG" = x1 ; then URLPATH="${URLPATH}#log&show=fetch" @@ -71,8 +52,6 @@ fi BASICUSER=`echo $BASICCOMBO | cut -d: -f1` BASICPWD=`echo $BASICCOMBO | cut -d: -f2` -OUTPUT="./.output" - if test "x$TEMP" = x ; then TEMP="/tmp" fi @@ -84,6 +63,10 @@ HOMERC=`echo "$HOMERC" | sed -e "s|//|/|g"` SPECRC="$TEMP/temprc" ENVRC="$WD/envrc" +show() { +echo "==================================================" +} + createrc() { RCP="$1" ; shift unset NOPWD @@ -157,8 +140,8 @@ createnetrc() { } reset() { - for f in ./$RC $HOME/$RC $SPECRC $ENVRC $COOKIES $NETRC $OUTPUT ; do - rm -f ${f} + for f in ./$RC $HOME/$RC $SPECRC $ENVRC $COOKIES $NETRC ; do +echo rm -f ${f} done unset DAPRCFILE } @@ -186,11 +169,6 @@ save() { done } -show() { - if test "x$SHOW" = x1 ; then cat $OUTPUT; fi - if test "x$OUTPUT" != "x"; then rm -f $OUTPUT; fi -} - # Assemble the ncdump command if test "x$DBG" = x1; then NCDUMP="$NCDUMP -D1" @@ -212,8 +190,7 @@ if test "x$RCEMBED" = x1 ; then URL="${PROTO}://${BASICCOMBO}@${URLSERVER}/$URLPATH" unset NETRC # Invoke ncdump to extract a file the URL - echo "command: ${NCDUMP} -h ${URL} > $OUTPUT" - ${NCDUMP} -h "$URL" > $OUTPUT + ${NCDUMP} -h "$URL" show fi @@ -229,8 +206,7 @@ if test "x$RCLOCAL" = x1 ; then createrc $LOCALRC # Invoke ncdump to extract a file using the URL - echo "command: ${NCDUMP} -h ${URL} > $OUTPUT" - ${NCDUMP} -h "$URL" > $OUTPUT + ${NCDUMP} -h "$URL" show fi @@ -242,8 +218,7 @@ if test "x$RCHOME" = x1 ; then createrc $HOMERC # Invoke ncdump to extract a file the URL - echo "command: ${NCDUMP} -h ${URL} > $OUTPUT" - ${NCDUMP} -h "$URL" > $OUTPUT + ${NCDUMP} -h "$URL" show fi @@ -255,8 +230,7 @@ if test "x$RCSPEC" == x1 ; then createrc $SPECRC # Invoke ncdump to extract a file the URL - echo "command: ${NCDUMP} -h ${URL} > $OUTPUT" - ${NCDUMP} -h "$URL" > $OUTPUT + ${NCDUMP} -h "$URL" show fi @@ -270,8 +244,7 @@ if test "x$RCENV" = x1 ; then createrc $DAPRCFILE # Invoke ncdump to extract a file the URL - echo "command: ${NCDUMP} -h ${URL} > $OUTPUT" - ${NCDUMP} -h "$URL" > $OUTPUT + ${NCDUMP} -h "$URL" show export DAPRCFILE= fi @@ -287,8 +260,7 @@ if test "x$RCPREC" = x1 ; then createrc $LOCALRC # Invoke ncdump to extract a file using the URL - echo "command: ${NCDUMP} -h ${URL} > $OUTPUT" - ${NCDUMP} -h "$URL" > $OUTPUT + ${NCDUMP} -h "$URL" ${NCDUMP} -h "$URL" show fi From 52b71b3d30e32a1ea52cf7f1ea11aaddc0c19aed Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Fri, 9 Apr 2021 15:45:06 -0600 Subject: [PATCH 03/16] ckp --- libdap2/cdf.c | 2 +- libdap2/dapattr.c | 6 +- ncdap_test/Makefile.am | 2 - ncdap_test/testauth.sh | 275 +++++++++++++---------------------------- 4 files changed, 92 insertions(+), 193 deletions(-) diff --git a/libdap2/cdf.c b/libdap2/cdf.c index cc5f522b18..4349dbe687 100644 --- a/libdap2/cdf.c +++ b/libdap2/cdf.c @@ -1089,6 +1089,7 @@ free1cdfnode(CDFnode* node) nullfree(node->ocname); nullfree(node->ncbasename); nullfree(node->ncfullname); + nullfree(node->dodsspecial.dimname); if(node->attributes != NULL) { for(j=0;jattributes);j++) { NCattribute* att = (NCattribute*)nclistget(node->attributes,j); @@ -1099,7 +1100,6 @@ free1cdfnode(CDFnode* node) nullfree(att); } } - nullfree(node->dodsspecial.dimname); nclistfree(node->subnodes); nclistfree(node->attributes); nclistfree(node->array.dimsetplus); diff --git a/libdap2/dapattr.c b/libdap2/dapattr.c index 03a31e9391..566994e3bf 100644 --- a/libdap2/dapattr.c +++ b/libdap2/dapattr.c @@ -79,12 +79,14 @@ fprintf(stderr,"%s.maxstrlen=%d\n",node->ocname,(int)node->dodsspecial.maxstrlen #endif } else if(strcmp(ocname,"DODS.dimName")==0 || strcmp(ocname,"DODS_EXTRA.dimName")==0) { + nullfree(node->dodsspecial.dimname); /* in case repeated */ + node->dodsspecial.dimname = NULL; if(values != NULL) { - node->dodsspecial.dimname = nulldup(values[0]); + node->dodsspecial.dimname = nulldup(values[0]); #ifdef DEBUG fprintf(stderr,"%s.dimname=%s\n",node->ocname,node->dodsspecial.dimname); #endif - } else node->dodsspecial.dimname = NULL; + } } else if(strcmp(ocname,"DODS.Unlimited_Dimension")==0 || strcmp(ocname,"DODS_EXTRA.Unlimited_Dimension")==0) { char* val0 = NULL; diff --git a/ncdap_test/Makefile.am b/ncdap_test/Makefile.am index 998e3ddfa7..4a03956efb 100644 --- a/ncdap_test/Makefile.am +++ b/ncdap_test/Makefile.am @@ -74,9 +74,7 @@ check_PROGRAMS += t_misc check_PROGRAMS += t_ncf330 if ENABLE_DAP_AUTH_TESTS -check_PROGRAMS += t_auth TESTS += testauth.sh -TESTS += testcontainerauth.sh endif endif #ENABLE_DAP_REMOTE_TESTS diff --git a/ncdap_test/testauth.sh b/ncdap_test/testauth.sh index 600a56429f..a385b3b7e7 100755 --- a/ncdap_test/testauth.sh +++ b/ncdap_test/testauth.sh @@ -3,33 +3,25 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh -RCEMBED=1 -#RCLOCAL=1 -#RCHOME=1 -#RCENV=1 -#RCPREC=1 - -# Not currently testable in netcdf -#RCSPEC=1 - #SHOW=1 #DBG=1 -# Choose at most 1 -#GDB=1 -#VG=1 - -NFL=1 +if test -d "/home/dmh" ; then + RCHOME=1 +fi WD=`pwd` -NETRCFILE=$WD/test_auth_netrc -# This is the control variable; set when needed -unset NETRC - -COOKIES="${WD}/test_auth_cookies" +COOKIES="${WD}/.cookies_test" RC=.daprc +NETRC=.netrc_test + +NETRCFILE=$WD/$NETRC +DAPRCFILE=$WD/$RC + +HOMENETRCFILE=$HOME/$NETRC +HOMEDAPRCFILE=$HOME/$RC if test "x$DBG" = x1 ; then SHOW=1 @@ -39,7 +31,6 @@ fi AUTHSERVER="thredds.ucar.edu" BASICCOMBO="authtester:auth" -BADCOMBO="authtester:xxxxx" URLPATH="thredds/dodsC/test3/testData.nc" PROTO=https URLSERVER=${AUTHSERVER} @@ -57,216 +48,124 @@ if test "x$TEMP" = x ; then fi TEMP=`echo "$TEMP" | sed -e "s|/$||"` -LOCALRC=./$RC +LOCALRC=${WD}/$RC +LOCALNETRC=${WD}/$NETRC HOMERC=${HOME}/$RC HOMERC=`echo "$HOMERC" | sed -e "s|//|/|g"` -SPECRC="$TEMP/temprc" -ENVRC="$WD/envrc" - -show() { -echo "==================================================" -} +HOMENETRC=${HOME}/$NETRC +HOMENETRC=`echo "$HOMENETRC" | sed -e "s|//|/|g"` createrc() { - RCP="$1" ; shift - unset NOPWD - unset BADPWD - while [[ $# > 0 ]] ; do - case "$1" in - nopwd) NOPWD=1 ;; - badpwd) BADPWD=1 ;; - *) ;; - esac - shift - done - if test "x$RCP" != x ; then - rm -f $RCP - echo "Creating rc file $RCP" - else + local RCP + local NETRC + RCP="$1" + if test "x$RCP" = x ; then echo "createrc: no rc specified" exit 1 fi - if test "x${DBG}" != x ; then + shift + NETRC="$1" + echo "Creating rc file $RCP" + if test "x${DBG}" = x1 ; then echo "HTTP.VERBOSE=1" >>$RCP fi - echo "HTTP.COOKIEJAR=${COOKIES}" >>$RCP - if test "x${URS}" = x ; then - if test "x${NOPWD}" = x ; then - if test "x${BADPWD}" = x ; then - echo "HTTP.CREDENTIALS.USERPASSWORD=${BASICCOMBO}" >>$RCP - else - echo "HTTP.CREDENTIALS.USERPASSWORD=${BADCOMBO}" >>$RCP - fi - fi - fi - if test "x${NETRC}" != x && test "x$NFL" = x ; then + if test "x$NETRC" = x ; then + echo "HTTP.CREDENTIALS.USERPASSWORD=${BASICCOMBO}" >>$RCP + elif test "x$NETRC" != ximplicit ; then echo "HTTP.NETRC=${NETRC}" >>$RCP fi + echo "HTTP.COOKIEJAR=${COOKIES}" >>$RCP } createnetrc() { - NCP="$1" ; shift - unset NOPWD - unset BADPWD - while [[ $# > 0 ]] ; do - case "$1" in - nopwd) NOPWD=1 ;; - badpwd) BADPWD=1 ;; - *) ;; - esac - shift - done - if test "x$NCP" != x ; then - rm -f $NCP - echo "Creating netrc file $NCP" - else - echo "createnetrc: no rc specified" - exit 1 - fi - if test "x$URS" != x ; then - echo "machine uat.urs.earthdata.nasa.gov login $BASICUSER password $BASICPWD" >>$NCP - #echo "machine 54.86.135.31 login $BASICUSER password $BASICPWD" >>$1 - else - echo -n "${PROTO}://$URLSERVER/$URLPATH" >>$NCP - if test "x$NOPWD" = x ; then - if test "x$BADPWD" = x ; then - echo -n " login $BASICUSER password $BASICPWD" >>$NCP - else - echo -n " login $BASICUSER password xxxxxx" >>$NCP - fi - fi - echo "" >>$NCP - fi + local NETRC + NETRC="$1"; # netrc file path + if test "x$NETRC" = x ; then return; fi + echo "Creating netrc file $NETRC" + echo -n "${PROTO}://$URLSERVER/$URLPATH" >>$NETRC + echo -n " login $BASICUSER password $BASICPWD" >>$NETRC + echo "" >>$NETRC + chmod go-rwx $NETRC } -reset() { - for f in ./$RC $HOME/$RC $SPECRC $ENVRC $COOKIES $NETRC ; do -echo rm -f ${f} - done - unset DAPRCFILE -} +# Test cases -restore() { +# Case: !daprc !netrc embedded usr:pwd +rcembed() { + echo "***Testing with embedded user:pwd" reset - for f in ./$RC $HOME/$RC $SPECRC $ENVRC $COOKIES $NETRC ; do - if test -f ${f}.save ; then - echo "restoring old ${f}" - cp ${f}.save ${f} - fi - done -} - -save() { - for f in ./$RC $HOME/$RC $SPECRC $ENVRC $COOKIES $NETRC ; do - if test -f $f ; then - if test -f ${f}.save ; then - ignore=1 - else - echo "saving $f" - cp ${f} ${f}.save - fi - fi - done -} - -# Assemble the ncdump command -if test "x$DBG" = x1; then -NCDUMP="$NCDUMP -D1" -fi - -if test "x$GDB" = x1 ; then - NCDUMP="gdb --args $NCDUMP" -fi -if test "x$VG" = x1 ; then -NCDUMP="valgrind --leak-check=full $NCDUMP" -fi - -# Initialize -save -reset - -if test "x$RCEMBED" = x1 ; then - echo "***Testing rc file with embedded user:pwd" URL="${PROTO}://${BASICCOMBO}@${URLSERVER}/$URLPATH" - unset NETRC # Invoke ncdump to extract a file the URL - ${NCDUMP} -h "$URL" - show -fi - -# Rest of tests assume these defaults -URL="${PROTO}://${URLSERVER}/$URLPATH" -NETRC=$NETRCFILE + ${NCDUMP} -h "$URL" > testauthoutput +} -if test "x$RCLOCAL" = x1 ; then +# Case: local daprc no netrc no embed +rclocal() { echo "***Testing rc file in local directory" - # Create the rc file and (optional) netrc fil in ./ reset - createnetrc $NETRC - createrc $LOCALRC + # Create the rc file and (optional) netrc fil in ./ + createrc $LOCALRC $LOCALNETRC # Invoke ncdump to extract a file using the URL - ${NCDUMP} -h "$URL" - show -fi + ${NCDUMP} -h "$URL" > testauthoutput +} -if test "x$RCHOME" = x1 ; then - echo "***Testing rc file in home directory" - # Create the rc file and (optional) netrc file in ./ +# Case: local daprc local netrc no embed +rclocalnetrc() { + echo "***Testing rc file in local directory" reset - createnetrc $NETRC - createrc $HOMERC + # Create the rc file and (optional) netrc fil in ./ + createnetrc $LOCALNETRC + createrc $LOCALRC $LOCALNETRC - # Invoke ncdump to extract a file the URL - ${NCDUMP} -h "$URL" - show -fi + # Invoke ncdump to extract a file using the URL + ${NCDUMP} -h "$URL" > testauthoutput +} -if test "x$RCSPEC" == x1 ; then - echo "*** Testing rc file in specified directory" - # Create the rc file and (optional) netrc file +rchome() { + echo "***Testing rc file and netrc in home directory" reset - createnetrc $NETRC - createrc $SPECRC + # Create the rc file and (optional) netrc file in ./ + createnetrc $HOMENETRC + createrc $HOMERC $HOMENETRC # Invoke ncdump to extract a file the URL - ${NCDUMP} -h "$URL" - show -fi + ${NCDUMP} -h "$URL" > testauthoutput +} -if test "x$RCENV" = x1 ; then - echo "*** Testing rc file using env variable" - # Create the rc file and (optional) netrc file +rchome2() { + echo "***Testing local rc file and .netrc implicit in home directory" reset - createnetrc $NETRC - echo "ENV: export DAPRCFILE=$ENVRC" - export DAPRCFILE=$ENVRC - createrc $DAPRCFILE + # Create the rc file and (optional) netrc file in ./ + createnetrc $HOME/.netrc + createrc $HOMERC implicit # Invoke ncdump to extract a file the URL - ${NCDUMP} -h "$URL" - show - export DAPRCFILE= -fi + ${NCDUMP} -h "$URL" > testauthoutput +} + +reset() { +if test "x$RCHOME" = x1 ; then rm -f $HOMENETRCFILE $HOMEDAPRCFILE; fi +rm -f $NETRCFILE $DAPRCFILE $LOCALRC $LOCALNETRC +unset DAPRCFILE +rm -f testauthoutput +} + +rcembed -# Test that .daprc overrides netcrc for password +# Next set tests assume these defaults URL="${PROTO}://${URLSERVER}/$URLPATH" -NETRC=$NETRCFILE -if test "x$RCPREC" = x1 ; then - echo "***Testing rc vs netrc file precedence" - # Create the rc file and (optional) netrc file in ./ - reset - createnetrc $NETRC badpwd - createrc $LOCALRC - # Invoke ncdump to extract a file using the URL - ${NCDUMP} -h "$URL" - ${NCDUMP} -h "$URL" - show +rclocal +rclocalnetrc + +# Do not do this unless you know what you are doing +if test "x$RCHOME" = x1 ; then + rchome + rchome2 fi reset -restore exit From 304b5f7e72103c1de7276b806358b7ec52f9efe8 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Fri, 9 Apr 2021 15:45:51 -0600 Subject: [PATCH 04/16] actions --- .github/workflows/run_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index d361773220..91434c896a 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -4,7 +4,7 @@ name: Run netCDF Tests -on: [pull_request] +on: [pull_request,push] jobs: From f720c5edbe2545335f55c95ba82831c040329048 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Wed, 14 Apr 2021 15:14:14 -0600 Subject: [PATCH 05/16] try18 --- libdap4/d4curlfunctions.c | 6 +- ncdap_test/testauth.sh | 114 +++++++++++++++++++++++++------------- oc2/occurlfunctions.c | 6 +- 3 files changed, 84 insertions(+), 42 deletions(-) diff --git a/libdap4/d4curlfunctions.c b/libdap4/d4curlfunctions.c index 1fdde93f15..691ec28205 100644 --- a/libdap4/d4curlfunctions.c +++ b/libdap4/d4curlfunctions.c @@ -71,8 +71,10 @@ set_curlflag(NCD4INFO* state, int flag) break; case CURLOPT_NETRC: case CURLOPT_NETRC_FILE: if(state->auth->curlflags.netrc) { - SETCURLOPT(state, CURLOPT_NETRC, (OPTARG)CURL_NETRC_REQUIRED); - SETCURLOPT(state, CURLOPT_NETRC_FILE, state->auth->curlflags.netrc); + SETCURLOPT(state, CURLOPT_NETRC, (OPTARG)CURL_NETRC_OPTIONAL); + /* IF HTTP.NETRC is set with "", then assume the default .netrc file (which is apparently CWD) */ + if(strlen(state->auth->curlflags.netrc)>0) + SETCURLOPT(state, CURLOPT_NETRC_FILE, state->auth->curlflags.netrc); } break; case CURLOPT_VERBOSE: diff --git a/ncdap_test/testauth.sh b/ncdap_test/testauth.sh index a385b3b7e7..b5c3ddbdb1 100755 --- a/ncdap_test/testauth.sh +++ b/ncdap_test/testauth.sh @@ -6,16 +6,27 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi #SHOW=1 #DBG=1 -if test -d "/home/dmh" ; then +if test "x$NCAUTH_HOMETEST" != x ; then RCHOME=1 fi +RCHOME=1 WD=`pwd` COOKIES="${WD}/.cookies_test" RC=.daprc + +if test "x$FPISMSVC" = x ; then NETRC=.netrc_test +NETRCIMP=.netrc +else +NETRC=_netrc_test +NETRCIMP=_netrc +fi + +LOCALRCFILES="$WD/.dodsrc $WD/.daprc $WD/.ncrc $WD/$NETRC $WD/$NETRCIMP" +HOMERCFILES="$HOME/.dodsrc $HOME/.daprc $HOME/.ncrc $HOME/$NETRC $HOME/$NETRCIMP" NETRCFILE=$WD/$NETRC DAPRCFILE=$WD/$RC @@ -57,35 +68,38 @@ HOMENETRC=`echo "$HOMENETRC" | sed -e "s|//|/|g"` createrc() { local RCP - local NETRC + local NETRCFILE RCP="$1" if test "x$RCP" = x ; then echo "createrc: no rc specified" exit 1 fi shift - NETRC="$1" + NETRCPATH="$1" echo "Creating rc file $RCP" if test "x${DBG}" = x1 ; then echo "HTTP.VERBOSE=1" >>$RCP fi - if test "x$NETRC" = x ; then + if test "x$NETRCPATH" = x ; then echo "HTTP.CREDENTIALS.USERPASSWORD=${BASICCOMBO}" >>$RCP - elif test "x$NETRC" != ximplicit ; then - echo "HTTP.NETRC=${NETRC}" >>$RCP + elif test "x$NETRCPATH" != ximplicit ; then + echo "HTTP.NETRC=${NETRCPATH}" >>$RCP + elif test "x$NETRCPATH" = ximplicit ; then + echo "HTTP.NETRC=" >>$RCP fi echo "HTTP.COOKIEJAR=${COOKIES}" >>$RCP } createnetrc() { - local NETRC - NETRC="$1"; # netrc file path - if test "x$NETRC" = x ; then return; fi - echo "Creating netrc file $NETRC" - echo -n "${PROTO}://$URLSERVER/$URLPATH" >>$NETRC - echo -n " login $BASICUSER password $BASICPWD" >>$NETRC - echo "" >>$NETRC - chmod go-rwx $NETRC + local NETRCPATH + NETRCPATH="$1"; # netrc file path + if test "x$NETRCPATH" = x ; then return; fi + echo "Creating netrc file $NETRCPATH" +# echo -n "${PROTO}://$URLSERVER/$URLPATH" >>$NETRCPATH + echo -n "machine $URLSERVER" >>$NETRCPATH + echo -n " login $BASICUSER password $BASICPWD" >>$NETRCPATH + echo "" >>$NETRCPATH + chmod go-rwx $NETRCPATH } # Test cases @@ -100,72 +114,96 @@ rcembed() { } # Case: local daprc no netrc no embed -rclocal() { +rclocal1() { echo "***Testing rc file in local directory" reset - # Create the rc file and (optional) netrc fil in ./ - createrc $LOCALRC $LOCALNETRC - + # Create the rc file in ./ + createrc $LOCALRC # Invoke ncdump to extract a file using the URL ${NCDUMP} -h "$URL" > testauthoutput } # Case: local daprc local netrc no embed -rclocalnetrc() { +rclocal2() { echo "***Testing rc file in local directory" reset # Create the rc file and (optional) netrc fil in ./ createnetrc $LOCALNETRC createrc $LOCALRC $LOCALNETRC + # Invoke ncdump to extract a file using the URL + ${NCDUMP} -h "$URL" > testauthoutput +} +# Case: home rc no netrc no embed +rchome1() { + echo "***Testing home rc file no netrc in home directory" + reset + # Create the rc file file in ./ + createrc $HOMERC + # Invoke ncdump to extract a file the URL + ${NCDUMP} -h "$URL" > testauthoutput +} + +# Case: home daprc implicit home netrc no embed +rchome2() { + echo "***Testing .netrc file in home directory" + reset + createnetrc $HOME/$NETRCIMP + createrc $HOMERC implicit # Invoke ncdump to extract a file using the URL ${NCDUMP} -h "$URL" > testauthoutput } -rchome() { - echo "***Testing rc file and netrc in home directory" +# Case: local rc explicit netrc no embed +rchome3() { + echo "***Testing local rc file and .netrc explicit in home directory" reset # Create the rc file and (optional) netrc file in ./ - createnetrc $HOMENETRC - createrc $HOMERC $HOMENETRC - + createnetrc $HOME/$NETRC + createrc $LOCALRC $HOME/$NETRC # Invoke ncdump to extract a file the URL ${NCDUMP} -h "$URL" > testauthoutput } -rchome2() { +# Case: local rc implicit netrc no embed +rchome4() { echo "***Testing local rc file and .netrc implicit in home directory" reset # Create the rc file and (optional) netrc file in ./ - createnetrc $HOME/.netrc - createrc $HOMERC implicit - + createnetrc $HOME/$NETRCIMP + createrc $LOCALRC implicit # Invoke ncdump to extract a file the URL ${NCDUMP} -h "$URL" > testauthoutput } reset() { -if test "x$RCHOME" = x1 ; then rm -f $HOMENETRCFILE $HOMEDAPRCFILE; fi -rm -f $NETRCFILE $DAPRCFILE $LOCALRC $LOCALNETRC -unset DAPRCFILE -rm -f testauthoutput + if test "x$RCHOME" = x1 ; then + rm -f $HOMERCFILES + fi + rm -f $LOCALRCFILES + unset DAPRCFILE + rm -f ./testauthoutput } -rcembed +set -x + +#rcembed # Next set tests assume these defaults URL="${PROTO}://${URLSERVER}/$URLPATH" -rclocal -rclocalnetrc +#rclocal1 +#rclocal2 # Do not do this unless you know what you are doing if test "x$RCHOME" = x1 ; then - rchome - rchome2 +# rchome1 +# rchome2 +# rchome3 + rchome4 fi -reset +#reset exit diff --git a/oc2/occurlfunctions.c b/oc2/occurlfunctions.c index b6d7d3e47c..076ecfb625 100644 --- a/oc2/occurlfunctions.c +++ b/oc2/occurlfunctions.c @@ -66,8 +66,10 @@ ocset_curlflag(OCstate* state, int flag) case CURLOPT_NETRC: case CURLOPT_NETRC_FILE: if(state->auth->curlflags.netrc) { - SETCURLOPT(state, CURLOPT_NETRC, (OPTARG)CURL_NETRC_REQUIRED); - SETCURLOPT(state, CURLOPT_NETRC_FILE, state->auth->curlflags.netrc); + SETCURLOPT(state, CURLOPT_NETRC, (OPTARG)CURL_NETRC_OPTIONAL); + /* IF HTTP.NETRC is set with "", then assume the default .netrc file (which is apparently CWD) */ + if(strlen(state->auth->curlflags.netrc)>0) + SETCURLOPT(state, CURLOPT_NETRC_FILE, state->auth->curlflags.netrc); } break; From 74793520b20ba23c06f10d51f382c3eba67c2f67 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Fri, 28 May 2021 14:27:22 -0600 Subject: [PATCH 06/16] szip --- nc_test4/tst_szip.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nc_test4/tst_szip.sh b/nc_test4/tst_szip.sh index dfffdb0cab..f2d48d05f1 100755 --- a/nc_test4/tst_szip.sh +++ b/nc_test4/tst_szip.sh @@ -8,7 +8,7 @@ set -e rm -f testnc.h5 testszip.nc szip_dump.cdl echo "*** Test read of known szip file" -${NCDUMP} ${srcdir}/ref_szip.h5 >szip_dump.cdl +${NCDUMP} -n ref_szip ${srcdir}/ref_szip.h5 >szip_dump.cdl diff -w ${srcdir}/ref_szip.cdl ./szip_dump.cdl echo "*** Testing tst_szip " From 42a9360f9a6abc0fb64fbf380fc8f9cc35a6aa72 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Fri, 28 May 2021 21:53:12 -0600 Subject: [PATCH 07/16] charvlen --- nc_test4/Makefile.am | 4 ++-- nc_test4/tst_misc.sh | 2 ++ ncdap_test/testurl.sh | 6 +++--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/nc_test4/Makefile.am b/nc_test4/Makefile.am index 8075b7373b..31c34a9321 100644 --- a/nc_test4/Makefile.am +++ b/nc_test4/Makefile.am @@ -35,7 +35,7 @@ tst_h_scalar tst_rename tst_rename2 tst_rename3 tst_h5_endians \ tst_atts_string_rewrite tst_hdf5_file_compat tst_fill_attr_vanish \ tst_rehash tst_filterparser tst_bug324 tst_types tst_atts3 \ tst_put_vars tst_elatefill tst_udf tst_put_vars_two_unlim_dim \ -tst_bug1442 tst_charvlenbug +tst_bug1442 # Temporary I hoped, but hoped in vain. if !ISCYGWIN @@ -43,7 +43,7 @@ NC4_TESTS += tst_h_strbug tst_h_refs endif # Build test programs plus programs used in test scripts. -check_PROGRAMS = $(NC4_TESTS) tst_empty_vlen_unlim +check_PROGRAMS = $(NC4_TESTS) tst_empty_vlen_unlim tst_charvlenbug TESTS = $(NC4_TESTS) run_empty_vlen_test.sh # Add these if large file tests are turned on. diff --git a/nc_test4/tst_misc.sh b/nc_test4/tst_misc.sh index 1e594d3fdf..767c4f7a17 100755 --- a/nc_test4/tst_misc.sh +++ b/nc_test4/tst_misc.sh @@ -16,6 +16,7 @@ echo "*** Fail: phony dimension creation" ECODE=1 fi +if "x$NC_VLEN_NOTEST" = x1 ; then echo "*** Testing char(*) type printout error in ncdump" rm -f ./tst_charvlenbug.nc ./tmp ${execdir}/tst_charvlenbug @@ -25,6 +26,7 @@ else echo "*** Fail: char(*) ncdump printout" ECODE=1 fi +fi rm -f tmp diff --git a/ncdap_test/testurl.sh b/ncdap_test/testurl.sh index 17ae9231e9..92c08cb2fc 100755 --- a/ncdap_test/testurl.sh +++ b/ncdap_test/testurl.sh @@ -1,8 +1,8 @@ #!/bin/sh -#NOP=1 -#NOS=1 -#NOB=1 +NOP=1 +NOS=1 +NOB=1 #SHOW=1 #DBG=1 From c361c1056d880cc08d905091a3e1c081308bf407 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Sat, 29 May 2021 16:40:38 -0600 Subject: [PATCH 08/16] update file permission --- ncdap_test/testpathcvt.sh | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100755 ncdap_test/testpathcvt.sh diff --git a/ncdap_test/testpathcvt.sh b/ncdap_test/testpathcvt.sh new file mode 100755 index 0000000000..71043b95fd --- /dev/null +++ b/ncdap_test/testpathcvt.sh @@ -0,0 +1,33 @@ +#!/bin/sh + +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + +set -e + +testcase1() { +T="$1" +P="$2" +echo -n ' ' +${execdir}/pathcvt $T "$P" +echo "" +} + +testcase() { +echo "path: $1:" + testcase1 "-u" "$1" + testcase1 "-c" "$1" + testcase1 "-m" "$1" + testcase1 "-w" "$1" +} + +testcase "/xxx/a/b" +testcase "d:/x/y" +testcase "d:\\x\\y" +testcase "/cygdrive/d/x/y" +testcase "/d/x/y" +testcase "/cygdrive/d" +testcase "/d" +testcase "/cygdrive/d/git/netcdf-c/dap4_test/test_anon_dim.2.syn" + +exit 0 From 2cb7f4768b69517147338d132795272c2d83a58f Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Sat, 29 May 2021 16:52:41 -0600 Subject: [PATCH 09/16] pathcvt --- ncdap_test/CMakeLists.txt | 5 ++++- ncdap_test/Makefile.am | 10 ++++++---- ncdap_test/ref_pathcvt.txt | 40 ++++++++++++++++++++++++++++++++++++++ ncdap_test/testauth.sh | 7 ++++++- ncdap_test/testpathcvt.sh | 20 +++++++++++-------- 5 files changed, 68 insertions(+), 14 deletions(-) create mode 100644 ncdap_test/ref_pathcvt.txt diff --git a/ncdap_test/CMakeLists.txt b/ncdap_test/CMakeLists.txt index 7298a2d9d9..bedbc5e7c5 100644 --- a/ncdap_test/CMakeLists.txt +++ b/ncdap_test/CMakeLists.txt @@ -26,7 +26,10 @@ IF(ENABLE_TESTS) # keep it from being too bad. # Binary Test Macro - add_sh_test(ncdap tst_ncdap3) + IF(BUILD_UTILITIES) + add_sh_test(ncdap tst_ncdap3) + add_sh_test(ncdap testpathcvt) + ENDIF() IF(NOT MSVC) add_bin_env_test(ncdap t_dap3a) diff --git a/ncdap_test/Makefile.am b/ncdap_test/Makefile.am index 4a03956efb..3b3aae0bd8 100644 --- a/ncdap_test/Makefile.am +++ b/ncdap_test/Makefile.am @@ -32,15 +32,17 @@ check_PROGRAMS += t_dap3a test_cvt3 test_vara TESTS += t_dap3a test_cvt3 test_vara if BUILD_UTILITIES TESTS += tst_ncdap3.sh +TESTS += testpathcvt.sh endif # remote tests are optional # because the server may be down or inaccessible if ENABLE_DAP_REMOTE_TESTS -noinst_PROGRAMS = findtestserver pingurl +noinst_PROGRAMS = findtestserver pingurl pathcvt findtestserver_SOURCES = findtestserver.c pingurl_SOURCES = pingurl.c +pathcvt_SOURCES = pathcvt.c endif if ENABLE_DAP_REMOTE_TESTS @@ -90,10 +92,10 @@ EXTRA_DIST = tst_ncdap3.sh \ tst_zero_len_var.sh \ tst_filelists.sh tst_urls.sh tst_utils.sh \ t_dap.c CMakeLists.txt tst_formatx.sh testauth.sh testurl.sh \ - t_ncf330.c tst_ber.sh tst_fillmismatch.sh tst_encode.sh \ - findtestserver.c.in + t_ncf330.c tst_ber.sh tst_fillmismatch.sh tst_encode.sh testpathcvt.sh \ + findtestserver.c.in ref_pathcvt.txt -CLEANFILES = test_varm3 test_cvt3 file_results/* remote_results/* datadds* t_dap3a test_nstride_cached *.exe +CLEANFILES = test_varm3 test_cvt3 file_results/* remote_results/* datadds* t_dap3a test_nstride_cached *.exe tmp*.txt # This should only be left behind if using parallel io CLEANFILES += tmp_* diff --git a/ncdap_test/ref_pathcvt.txt b/ncdap_test/ref_pathcvt.txt new file mode 100644 index 0000000000..6397eb5b37 --- /dev/null +++ b/ncdap_test/ref_pathcvt.txt @@ -0,0 +1,40 @@ +path: /xxx/a/b: + /xxx/a/b + /cygdrive/c/xxx/a/b + /c/xxx/a/b + c:\xxx\a\b +path: d:/x/y: + /d/x/y + /cygdrive/d/x/y + /d/x/y + d:\x\y +path: d:\x\y: + /d/x/y + /cygdrive/d/x/y + /d/x/y + d:\x\y +path: /cygdrive/d/x/y: + /d/x/y + /cygdrive/d/x/y + /d/x/y + d:\x\y +path: /d/x/y: + /d/x/y + /cygdrive/d/x/y + /d/x/y + d:\x\y +path: /cygdrive/d: + /d + /cygdrive/d + /d + d: +path: /d: + /d + /cygdrive/d + /d + d: +path: /cygdrive/d/git/netcdf-c/dap4_test/test_anon_dim.2.syn: + /d/git/netcdf-c/dap4_test/test_anon_dim.2.syn + /cygdrive/d/git/netcdf-c/dap4_test/test_anon_dim.2.syn + /d/git/netcdf-c/dap4_test/test_anon_dim.2.syn + d:\git\netcdf-c\dap4_test\test_anon_dim.2.syn diff --git a/ncdap_test/testauth.sh b/ncdap_test/testauth.sh index e8ed03b907..3bd00b8158 100755 --- a/ncdap_test/testauth.sh +++ b/ncdap_test/testauth.sh @@ -39,9 +39,14 @@ fi LOCALRCFILES="$WD/.dodsrc $WD/.daprc $WD/.ncrc $WD/$NETRC $WD/$NETRCIMP" HOMERCFILES="$HOME/.dodsrc $HOME/.daprc $HOME/.ncrc $HOME/$NETRC $HOME/$NETRCIMP" - NETRCFILE=$WD/$NETRC DAPRCFILE=$WD/$RC +if test "x$FPISMSVC" = x1 ; then + LOCALRCFILES=`${execdir}/pathcvt -w "$LOCALRCFILES"` + HOMERCFILES=`${execdir}/pathcvt -w "$HOMERCFILES"` + NETRCFILE=`${execdir}/pathcvt -w "$NETRCFILE"` + DAPRCFILE=`${execdir}/pathcvt -w "$DAPRCFILE"` +fi HOMENETRCFILE=$HOME/$NETRC HOMEDAPRCFILE=$HOME/$RC diff --git a/ncdap_test/testpathcvt.sh b/ncdap_test/testpathcvt.sh index 71043b95fd..a91146a350 100755 --- a/ncdap_test/testpathcvt.sh +++ b/ncdap_test/testpathcvt.sh @@ -21,13 +21,17 @@ echo "path: $1:" testcase1 "-w" "$1" } -testcase "/xxx/a/b" -testcase "d:/x/y" -testcase "d:\\x\\y" -testcase "/cygdrive/d/x/y" -testcase "/d/x/y" -testcase "/cygdrive/d" -testcase "/d" -testcase "/cygdrive/d/git/netcdf-c/dap4_test/test_anon_dim.2.syn" +rm -f tmp_pathcvt.txt + +testcase "/xxx/a/b" >> tmp_pathcvt.txt +testcase "d:/x/y" >> tmp_pathcvt.txt +testcase "d:\\x\\y" >> tmp_pathcvt.txt +testcase "/cygdrive/d/x/y" >> tmp_pathcvt.txt +testcase "/d/x/y" >> tmp_pathcvt.txt +testcase "/cygdrive/d" >> tmp_pathcvt.txt +testcase "/d" >> tmp_pathcvt.txt +testcase "/cygdrive/d/git/netcdf-c/dap4_test/test_anon_dim.2.syn" >> tmp_pathcvt.txt + +diff -w ${srcdir}/ref_pathcvt.txt ./tmp_pathcvt.txt exit 0 From b95d2302b5c59f1c5f95cb457b400665ad7d36fe Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Sat, 29 May 2021 17:14:16 -0600 Subject: [PATCH 10/16] pathcvtadd --- ncdap_test/pathcvt.c | 96 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100755 ncdap_test/pathcvt.c diff --git a/ncdap_test/pathcvt.c b/ncdap_test/pathcvt.c new file mode 100755 index 0000000000..a8547687c5 --- /dev/null +++ b/ncdap_test/pathcvt.c @@ -0,0 +1,96 @@ +/* + * Copyright 2018, University Corporation for Atmospheric Research + * See netcdf/COPYRIGHT file for copying and redistribution conditions. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include + +#ifdef HAVE_UNISTD_H +#include +#endif +#ifdef HAVE_GETOPT_H +#include +#endif + +#if defined(_WIN32) && !defined(__MINGW32__) +#include "XGetopt.h" +#endif + +#include "ncpathmgr.h" + +/* +Synopsis + +pathcvt [-u|-w|-m|-c] PATH + +Options + +Output type options: + -u (default) convert to Unix form of path + -w convert to Windows form of path + -m convert to MSYS form of path + -c convert to Cygwin form of path +*/ + +#define DEBUG + +struct Options { + int target; + int debug; +} cvtoptions; + +static void +usage(const char* msg) +{ + if(msg != NULL) fprintf(stderr,"%s\n",msg); + fprintf(stderr,"pathcvt [-u|-w|-m|-c] PATH\n"); + if(msg == NULL) exit(0); else exit(1); +} + +int +main(int argc, char** argv) +{ + int c; + char* cvtpath = NULL; + char* inpath; + + memset((void*)&cvtoptions,0,sizeof(cvtoptions)); + + while ((c = getopt(argc, argv, "cD:hmuw")) != EOF) { + switch(c) { + case 'c': cvtoptions.target = NCPD_CYGWIN; break; + case 'h': usage(NULL); break; + case 'm': cvtoptions.target = NCPD_MSYS; break; + case 'u': cvtoptions.target = NCPD_NIX; break; + case 'w': cvtoptions.target = NCPD_WIN; break; + case 'D': + sscanf(optarg,"%d",&cvtoptions.debug); + break; + case '?': + usage("unknown option"); + break; + } + } + if(cvtoptions.target == NCPD_UNKNOWN) usage("No conversion type specified"); + + argc -= optind; + argv += optind; + + /* If no file arguments left or more than one, print usage message. */ + if (argc == 0) + usage("no path specified"); + if (argc > 1) + usage("more than one path specified"); + inpath = argv[0]; + + cvtpath = NCpathcvt_test(inpath,cvtoptions.target,'c'); + printf("%s",cvtpath); + if(cvtpath) free(cvtpath); + return 0; +} From 67df42a536766ca0f36c7fd4f6249298c316e501 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Sat, 29 May 2021 19:50:41 -0600 Subject: [PATCH 11/16] converted --- CMakeLists.txt | 4 +++- ncdap_test/CMakeLists.txt | 1 + ncdap_test/testauth.sh | 5 +++-- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6d6ce07a04..68d6d76164 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1008,7 +1008,9 @@ IF(MSVC) FILE(COPY ${netCDF_SOURCE_DIR}/libsrc/XGetopt.c DESTINATION ${netCDF_BINARY_DIR}/ncdump/) FILE(COPY ${netCDF_SOURCE_DIR}/libsrc/XGetopt.c - DESTINATION ${netCDF_BINARY_DIR}/nczarr_test/) + DESTINATION ${netCDF_BINARY_DIR}/nczarr_test/) + FILE(COPY ${netCDF_SOURCE_DIR}/libsrc/XGetopt.c + DESTINATION ${netCDF_BINARY_DIR}/ncdap_test/) ENDIF() ENDIF() diff --git a/ncdap_test/CMakeLists.txt b/ncdap_test/CMakeLists.txt index bedbc5e7c5..5c31d07b85 100644 --- a/ncdap_test/CMakeLists.txt +++ b/ncdap_test/CMakeLists.txt @@ -17,6 +17,7 @@ FILE(COPY ${COPY_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/ FILE_PERMISSION IF(ENABLE_DAP_REMOTE_TESTS) BUILD_BIN_TEST(findtestserver) BUILD_BIN_TEST(pingurl) + BUILD_BIN_TEST(pathcvt XGetopt.c) ENDIF() IF(ENABLE_TESTS) diff --git a/ncdap_test/testauth.sh b/ncdap_test/testauth.sh index 3bd00b8158..6179ab1931 100755 --- a/ncdap_test/testauth.sh +++ b/ncdap_test/testauth.sh @@ -142,9 +142,9 @@ rclocal1() { # Case: local daprc local netrc no embed rclocal2() { - echo "***Testing rc file in local directory" + echo "***Testing rc file + .netrc in local directory" reset - # Create the rc file and (optional) netrc fil in ./ + # Create the rc file and (optional) netrc file in ./ createnetrc $LOCALNETRC createrc $LOCALRC $LOCALNETRC # Invoke ncdump to extract a file using the URL @@ -208,6 +208,7 @@ rcembed URL="${PROTO}://${URLSERVER}/$URLPATH" rclocal1 +exit rclocal2 # Do not do this unless you know what you are doing From fe16d0884f8f9a734e0b5a4669fe086f728a162f Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Sat, 29 May 2021 20:51:09 -0600 Subject: [PATCH 12/16] final --- ncdap_test/pathcvt.c | 11 ++++++++--- ncdap_test/testauth.sh | 11 +++++------ 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/ncdap_test/pathcvt.c b/ncdap_test/pathcvt.c index a8547687c5..14cbbfe54c 100755 --- a/ncdap_test/pathcvt.c +++ b/ncdap_test/pathcvt.c @@ -32,10 +32,13 @@ pathcvt [-u|-w|-m|-c] PATH Options Output type options: - -u (default) convert to Unix form of path + -u convert to Unix form of path -w convert to Windows form of path -m convert to MSYS form of path -c convert to Cygwin form of path + +Default is to convert to the format used by the platform. + */ #define DEBUG @@ -77,7 +80,6 @@ main(int argc, char** argv) break; } } - if(cvtoptions.target == NCPD_UNKNOWN) usage("No conversion type specified"); argc -= optind; argv += optind; @@ -89,7 +91,10 @@ main(int argc, char** argv) usage("more than one path specified"); inpath = argv[0]; - cvtpath = NCpathcvt_test(inpath,cvtoptions.target,'c'); + if(cvtoptions.target == NCPD_UNKNOWN) + cvtpath = NCpathcvt(inpath); + else + cvtpath = NCpathcvt_test(inpath,cvtoptions.target,'c'); printf("%s",cvtpath); if(cvtpath) free(cvtpath); return 0; diff --git a/ncdap_test/testauth.sh b/ncdap_test/testauth.sh index 6179ab1931..c35dd81530 100755 --- a/ncdap_test/testauth.sh +++ b/ncdap_test/testauth.sh @@ -41,11 +41,11 @@ LOCALRCFILES="$WD/.dodsrc $WD/.daprc $WD/.ncrc $WD/$NETRC $WD/$NETRCIMP" HOMERCFILES="$HOME/.dodsrc $HOME/.daprc $HOME/.ncrc $HOME/$NETRC $HOME/$NETRCIMP" NETRCFILE=$WD/$NETRC DAPRCFILE=$WD/$RC -if test "x$FPISMSVC" = x1 ; then - LOCALRCFILES=`${execdir}/pathcvt -w "$LOCALRCFILES"` - HOMERCFILES=`${execdir}/pathcvt -w "$HOMERCFILES"` - NETRCFILE=`${execdir}/pathcvt -w "$NETRCFILE"` - DAPRCFILE=`${execdir}/pathcvt -w "$DAPRCFILE"` +if test "x$FP_ISMSVC" = x1 ; then + LOCALRCFILES=`${execdir}/pathcvt "$LOCALRCFILES"` + HOMERCFILES=`${execdir}/pathcvt "$HOMERCFILES"` + NETRCFILE=`${execdir}/pathcvt "$NETRCFILE"` + DAPRCFILE=`${execdir}/pathcvt "$DAPRCFILE"` fi HOMENETRCFILE=$HOME/$NETRC @@ -208,7 +208,6 @@ rcembed URL="${PROTO}://${URLSERVER}/$URLPATH" rclocal1 -exit rclocal2 # Do not do this unless you know what you are doing From 3c5740dde5eebc046b8944efdcecda9bac637aa8 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Sat, 29 May 2021 21:17:52 -0600 Subject: [PATCH 13/16] sigh --- ncdap_test/CMakeLists.txt | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/ncdap_test/CMakeLists.txt b/ncdap_test/CMakeLists.txt index 5c31d07b85..50f5db5368 100644 --- a/ncdap_test/CMakeLists.txt +++ b/ncdap_test/CMakeLists.txt @@ -14,13 +14,18 @@ FILE(GLOB COPY_FILES ${CMAKE_CURRENT_SOURCE_DIR}/*.sh) FILE(COPY ${COPY_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/ FILE_PERMISSIONS OWNER_WRITE OWNER_READ OWNER_EXECUTE) -IF(ENABLE_DAP_REMOTE_TESTS) +IF(ENABLE_TESTS) + + IF(ENABLE_DAP_REMOTE_TESTS) BUILD_BIN_TEST(findtestserver) BUILD_BIN_TEST(pingurl) - BUILD_BIN_TEST(pathcvt XGetopt.c) -ENDIF() + ENDIF() + + IF(USE_X_GETOPT) + SET(XGSRC XGetopt.c) + ENDIF() + BUILD_BIN_TEST(pathcvt ${XGSRC}) -IF(ENABLE_TESTS) # Base tests # The tests are set up as a combination of shell scripts and executables that # must be run in a particular order. It is painful but will use macros to help From e632d02041369537aaebf7f91835d7002583de1a Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Sat, 29 May 2021 21:30:33 -0600 Subject: [PATCH 14/16] Re-enable DAP2 authorization tests The thredds-test server now has some password protected datasets that can be used to test DAP2 authorization support. The general location is ```` https://thredds.ucar.edu/thredds/tdscapabilities/authTest.html ```` and specifically: ```` https://thredds.ucar.edu/thredds/dodsC/test3/testData.nc.html ```` This PR replaces old testcases with ncdap_test/testauth.sh. This testcase allows us to test use of the .dodsrc file and .netrc file and embedded user+pwd. As part of this, I had to create a program (ncdap_test/pathcvt.c) that is essentially the equivalent to cygpath. Given a path in windows, unix, msys or cygwin format, it converts it to the equivalent format in one of those four cases. So it can be used to convert a cygwin path to a windows path, for example. This is needed in testpathcvt and testauth to make sure that the paths in .daprc (e.g. the reference to .netrc) are of the proper format. Misc. Other Changes: 1. Fix some memory leaks in libdap2 2. Setting the env variable CURLOPT_VERBOSE allows tracking of curl operations. 3. Make tst_charvlenbug be conditional on NC_VLEN_NOTEST. --- .github/workflows/run_tests.yml | 2 +- CMakeLists.txt | 4 +- libdap2/cdf.c | 2 +- libdap2/dapattr.c | 2 + libdap2/ncd2dispatch.c | 5 +- libdap4/d4curlfunctions.c | 6 +- libdap4/d4file.c | 5 ++ nc_test4/Makefile.am | 4 +- nc_test4/tst_misc.sh | 2 + nc_test4/tst_szip.sh | 2 +- ncdap_test/CMakeLists.txt | 17 ++++-- ncdap_test/Makefile.am | 10 ++-- ncdap_test/pathcvt.c | 101 ++++++++++++++++++++++++++++++++ ncdap_test/ref_pathcvt.txt | 40 +++++++++++++ ncdap_test/t_auth.c | 41 +++++-------- ncdap_test/testauth.sh | 12 ++-- ncdap_test/testpathcvt.sh | 37 ++++++++++++ ncdap_test/testurl.sh | 6 +- oc2/occurlfunctions.c | 6 +- 19 files changed, 251 insertions(+), 53 deletions(-) create mode 100755 ncdap_test/pathcvt.c create mode 100644 ncdap_test/ref_pathcvt.txt create mode 100755 ncdap_test/testpathcvt.sh diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index ad7ccb61ce..b575d1bce4 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -4,7 +4,7 @@ name: Run netCDF Tests -on: [pull_request] +on: [pull_request,push] jobs: diff --git a/CMakeLists.txt b/CMakeLists.txt index 6d6ce07a04..68d6d76164 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1008,7 +1008,9 @@ IF(MSVC) FILE(COPY ${netCDF_SOURCE_DIR}/libsrc/XGetopt.c DESTINATION ${netCDF_BINARY_DIR}/ncdump/) FILE(COPY ${netCDF_SOURCE_DIR}/libsrc/XGetopt.c - DESTINATION ${netCDF_BINARY_DIR}/nczarr_test/) + DESTINATION ${netCDF_BINARY_DIR}/nczarr_test/) + FILE(COPY ${netCDF_SOURCE_DIR}/libsrc/XGetopt.c + DESTINATION ${netCDF_BINARY_DIR}/ncdap_test/) ENDIF() ENDIF() diff --git a/libdap2/cdf.c b/libdap2/cdf.c index cc5f522b18..4349dbe687 100644 --- a/libdap2/cdf.c +++ b/libdap2/cdf.c @@ -1089,6 +1089,7 @@ free1cdfnode(CDFnode* node) nullfree(node->ocname); nullfree(node->ncbasename); nullfree(node->ncfullname); + nullfree(node->dodsspecial.dimname); if(node->attributes != NULL) { for(j=0;jattributes);j++) { NCattribute* att = (NCattribute*)nclistget(node->attributes,j); @@ -1099,7 +1100,6 @@ free1cdfnode(CDFnode* node) nullfree(att); } } - nullfree(node->dodsspecial.dimname); nclistfree(node->subnodes); nclistfree(node->attributes); nclistfree(node->array.dimsetplus); diff --git a/libdap2/dapattr.c b/libdap2/dapattr.c index f0dedfb8e4..c0e37aa9a7 100644 --- a/libdap2/dapattr.c +++ b/libdap2/dapattr.c @@ -79,6 +79,8 @@ fprintf(stderr,"%s.maxstrlen=%d\n",node->ocname,(int)node->dodsspecial.maxstrlen #endif } else if(strcmp(ocname,"DODS.dimName")==0 || strcmp(ocname,"DODS_EXTRA.dimName")==0) { + nullfree(node->dodsspecial.dimname); /* in case repeated */ + node->dodsspecial.dimname = NULL; if(values != NULL) { nullfree(node->dodsspecial.dimname); node->dodsspecial.dimname = nulldup(values[0]); diff --git a/libdap2/ncd2dispatch.c b/libdap2/ncd2dispatch.c index f9edcfe265..fd2d07d929 100644 --- a/libdap2/ncd2dispatch.c +++ b/libdap2/ncd2dispatch.c @@ -403,9 +403,8 @@ fprintf(stderr,"ce=%s\n",dumpconstraint(dapcomm->oc.dapconstraint)); ocstat = oc_open(dapcomm->oc.urltext,&dapcomm->oc.conn); if(ocstat != OC_NOERR) {THROWCHK(ocstat); goto done;} -#ifdef DEBUG1 - (void)oc_trace_curl(dapcomm->oc.conn); -#endif + if(getenv("CURLOPT_VERBOSE") != NULL) + (void)oc_trace_curl(dapcomm->oc.conn); nullfree(dapcomm->oc.urltext); /* clean up */ dapcomm->oc.urltext = NULL; diff --git a/libdap4/d4curlfunctions.c b/libdap4/d4curlfunctions.c index 1fdde93f15..691ec28205 100644 --- a/libdap4/d4curlfunctions.c +++ b/libdap4/d4curlfunctions.c @@ -71,8 +71,10 @@ set_curlflag(NCD4INFO* state, int flag) break; case CURLOPT_NETRC: case CURLOPT_NETRC_FILE: if(state->auth->curlflags.netrc) { - SETCURLOPT(state, CURLOPT_NETRC, (OPTARG)CURL_NETRC_REQUIRED); - SETCURLOPT(state, CURLOPT_NETRC_FILE, state->auth->curlflags.netrc); + SETCURLOPT(state, CURLOPT_NETRC, (OPTARG)CURL_NETRC_OPTIONAL); + /* IF HTTP.NETRC is set with "", then assume the default .netrc file (which is apparently CWD) */ + if(strlen(state->auth->curlflags.netrc)>0) + SETCURLOPT(state, CURLOPT_NETRC_FILE, state->auth->curlflags.netrc); } break; case CURLOPT_VERBOSE: diff --git a/libdap4/d4file.c b/libdap4/d4file.c index 47a460562b..cb24c0b9f1 100644 --- a/libdap4/d4file.c +++ b/libdap4/d4file.c @@ -117,6 +117,11 @@ NCD4_open(const char * path, int mode, ncsetlogging(1); } + /* Check env values */ + if(getenv("CURLOPT_VERBOSE") != NULL) + d4info->auth->curlflags.verbose = 1; + + /* Setup a curl connection */ { CURL* curl = NULL; /* curl handle*/ diff --git a/nc_test4/Makefile.am b/nc_test4/Makefile.am index 8075b7373b..31c34a9321 100644 --- a/nc_test4/Makefile.am +++ b/nc_test4/Makefile.am @@ -35,7 +35,7 @@ tst_h_scalar tst_rename tst_rename2 tst_rename3 tst_h5_endians \ tst_atts_string_rewrite tst_hdf5_file_compat tst_fill_attr_vanish \ tst_rehash tst_filterparser tst_bug324 tst_types tst_atts3 \ tst_put_vars tst_elatefill tst_udf tst_put_vars_two_unlim_dim \ -tst_bug1442 tst_charvlenbug +tst_bug1442 # Temporary I hoped, but hoped in vain. if !ISCYGWIN @@ -43,7 +43,7 @@ NC4_TESTS += tst_h_strbug tst_h_refs endif # Build test programs plus programs used in test scripts. -check_PROGRAMS = $(NC4_TESTS) tst_empty_vlen_unlim +check_PROGRAMS = $(NC4_TESTS) tst_empty_vlen_unlim tst_charvlenbug TESTS = $(NC4_TESTS) run_empty_vlen_test.sh # Add these if large file tests are turned on. diff --git a/nc_test4/tst_misc.sh b/nc_test4/tst_misc.sh index 1e594d3fdf..767c4f7a17 100755 --- a/nc_test4/tst_misc.sh +++ b/nc_test4/tst_misc.sh @@ -16,6 +16,7 @@ echo "*** Fail: phony dimension creation" ECODE=1 fi +if "x$NC_VLEN_NOTEST" = x1 ; then echo "*** Testing char(*) type printout error in ncdump" rm -f ./tst_charvlenbug.nc ./tmp ${execdir}/tst_charvlenbug @@ -25,6 +26,7 @@ else echo "*** Fail: char(*) ncdump printout" ECODE=1 fi +fi rm -f tmp diff --git a/nc_test4/tst_szip.sh b/nc_test4/tst_szip.sh index dfffdb0cab..f2d48d05f1 100755 --- a/nc_test4/tst_szip.sh +++ b/nc_test4/tst_szip.sh @@ -8,7 +8,7 @@ set -e rm -f testnc.h5 testszip.nc szip_dump.cdl echo "*** Test read of known szip file" -${NCDUMP} ${srcdir}/ref_szip.h5 >szip_dump.cdl +${NCDUMP} -n ref_szip ${srcdir}/ref_szip.h5 >szip_dump.cdl diff -w ${srcdir}/ref_szip.cdl ./szip_dump.cdl echo "*** Testing tst_szip " diff --git a/ncdap_test/CMakeLists.txt b/ncdap_test/CMakeLists.txt index 7298a2d9d9..50f5db5368 100644 --- a/ncdap_test/CMakeLists.txt +++ b/ncdap_test/CMakeLists.txt @@ -14,19 +14,28 @@ FILE(GLOB COPY_FILES ${CMAKE_CURRENT_SOURCE_DIR}/*.sh) FILE(COPY ${COPY_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/ FILE_PERMISSIONS OWNER_WRITE OWNER_READ OWNER_EXECUTE) -IF(ENABLE_DAP_REMOTE_TESTS) +IF(ENABLE_TESTS) + + IF(ENABLE_DAP_REMOTE_TESTS) BUILD_BIN_TEST(findtestserver) BUILD_BIN_TEST(pingurl) -ENDIF() + ENDIF() + + IF(USE_X_GETOPT) + SET(XGSRC XGetopt.c) + ENDIF() + BUILD_BIN_TEST(pathcvt ${XGSRC}) -IF(ENABLE_TESTS) # Base tests # The tests are set up as a combination of shell scripts and executables that # must be run in a particular order. It is painful but will use macros to help # keep it from being too bad. # Binary Test Macro - add_sh_test(ncdap tst_ncdap3) + IF(BUILD_UTILITIES) + add_sh_test(ncdap tst_ncdap3) + add_sh_test(ncdap testpathcvt) + ENDIF() IF(NOT MSVC) add_bin_env_test(ncdap t_dap3a) diff --git a/ncdap_test/Makefile.am b/ncdap_test/Makefile.am index 4a03956efb..3b3aae0bd8 100644 --- a/ncdap_test/Makefile.am +++ b/ncdap_test/Makefile.am @@ -32,15 +32,17 @@ check_PROGRAMS += t_dap3a test_cvt3 test_vara TESTS += t_dap3a test_cvt3 test_vara if BUILD_UTILITIES TESTS += tst_ncdap3.sh +TESTS += testpathcvt.sh endif # remote tests are optional # because the server may be down or inaccessible if ENABLE_DAP_REMOTE_TESTS -noinst_PROGRAMS = findtestserver pingurl +noinst_PROGRAMS = findtestserver pingurl pathcvt findtestserver_SOURCES = findtestserver.c pingurl_SOURCES = pingurl.c +pathcvt_SOURCES = pathcvt.c endif if ENABLE_DAP_REMOTE_TESTS @@ -90,10 +92,10 @@ EXTRA_DIST = tst_ncdap3.sh \ tst_zero_len_var.sh \ tst_filelists.sh tst_urls.sh tst_utils.sh \ t_dap.c CMakeLists.txt tst_formatx.sh testauth.sh testurl.sh \ - t_ncf330.c tst_ber.sh tst_fillmismatch.sh tst_encode.sh \ - findtestserver.c.in + t_ncf330.c tst_ber.sh tst_fillmismatch.sh tst_encode.sh testpathcvt.sh \ + findtestserver.c.in ref_pathcvt.txt -CLEANFILES = test_varm3 test_cvt3 file_results/* remote_results/* datadds* t_dap3a test_nstride_cached *.exe +CLEANFILES = test_varm3 test_cvt3 file_results/* remote_results/* datadds* t_dap3a test_nstride_cached *.exe tmp*.txt # This should only be left behind if using parallel io CLEANFILES += tmp_* diff --git a/ncdap_test/pathcvt.c b/ncdap_test/pathcvt.c new file mode 100755 index 0000000000..14cbbfe54c --- /dev/null +++ b/ncdap_test/pathcvt.c @@ -0,0 +1,101 @@ +/* + * Copyright 2018, University Corporation for Atmospheric Research + * See netcdf/COPYRIGHT file for copying and redistribution conditions. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include + +#ifdef HAVE_UNISTD_H +#include +#endif +#ifdef HAVE_GETOPT_H +#include +#endif + +#if defined(_WIN32) && !defined(__MINGW32__) +#include "XGetopt.h" +#endif + +#include "ncpathmgr.h" + +/* +Synopsis + +pathcvt [-u|-w|-m|-c] PATH + +Options + +Output type options: + -u convert to Unix form of path + -w convert to Windows form of path + -m convert to MSYS form of path + -c convert to Cygwin form of path + +Default is to convert to the format used by the platform. + +*/ + +#define DEBUG + +struct Options { + int target; + int debug; +} cvtoptions; + +static void +usage(const char* msg) +{ + if(msg != NULL) fprintf(stderr,"%s\n",msg); + fprintf(stderr,"pathcvt [-u|-w|-m|-c] PATH\n"); + if(msg == NULL) exit(0); else exit(1); +} + +int +main(int argc, char** argv) +{ + int c; + char* cvtpath = NULL; + char* inpath; + + memset((void*)&cvtoptions,0,sizeof(cvtoptions)); + + while ((c = getopt(argc, argv, "cD:hmuw")) != EOF) { + switch(c) { + case 'c': cvtoptions.target = NCPD_CYGWIN; break; + case 'h': usage(NULL); break; + case 'm': cvtoptions.target = NCPD_MSYS; break; + case 'u': cvtoptions.target = NCPD_NIX; break; + case 'w': cvtoptions.target = NCPD_WIN; break; + case 'D': + sscanf(optarg,"%d",&cvtoptions.debug); + break; + case '?': + usage("unknown option"); + break; + } + } + + argc -= optind; + argv += optind; + + /* If no file arguments left or more than one, print usage message. */ + if (argc == 0) + usage("no path specified"); + if (argc > 1) + usage("more than one path specified"); + inpath = argv[0]; + + if(cvtoptions.target == NCPD_UNKNOWN) + cvtpath = NCpathcvt(inpath); + else + cvtpath = NCpathcvt_test(inpath,cvtoptions.target,'c'); + printf("%s",cvtpath); + if(cvtpath) free(cvtpath); + return 0; +} diff --git a/ncdap_test/ref_pathcvt.txt b/ncdap_test/ref_pathcvt.txt new file mode 100644 index 0000000000..6397eb5b37 --- /dev/null +++ b/ncdap_test/ref_pathcvt.txt @@ -0,0 +1,40 @@ +path: /xxx/a/b: + /xxx/a/b + /cygdrive/c/xxx/a/b + /c/xxx/a/b + c:\xxx\a\b +path: d:/x/y: + /d/x/y + /cygdrive/d/x/y + /d/x/y + d:\x\y +path: d:\x\y: + /d/x/y + /cygdrive/d/x/y + /d/x/y + d:\x\y +path: /cygdrive/d/x/y: + /d/x/y + /cygdrive/d/x/y + /d/x/y + d:\x\y +path: /d/x/y: + /d/x/y + /cygdrive/d/x/y + /d/x/y + d:\x\y +path: /cygdrive/d: + /d + /cygdrive/d + /d + d: +path: /d: + /d + /cygdrive/d + /d + d: +path: /cygdrive/d/git/netcdf-c/dap4_test/test_anon_dim.2.syn: + /d/git/netcdf-c/dap4_test/test_anon_dim.2.syn + /cygdrive/d/git/netcdf-c/dap4_test/test_anon_dim.2.syn + /d/git/netcdf-c/dap4_test/test_anon_dim.2.syn + d:\git\netcdf-c\dap4_test\test_anon_dim.2.syn diff --git a/ncdap_test/t_auth.c b/ncdap_test/t_auth.c index 2156154afe..e70a068e87 100644 --- a/ncdap_test/t_auth.c +++ b/ncdap_test/t_auth.c @@ -28,15 +28,17 @@ See \ref copyright file for more info. #define KEEPRC -#define RC ".ocrc" -#define SPECRC "./ocrc" +#define AUTHTESTSERVER "thredds.ucar.edu" -#define USERPWD "tiggeUser:tigge" +#define RC ".daprc" +#define SPECRC "./daprc" + +#define USERPWD "authtester:auth" #define COOKIEFILE "./cookies" -#define URL1 "https://%s@%s/dodsC/restrict/testData.nc" -#define URL2 "https://%s/dodsC/restrict/testData.nc" -#define URL3 "https://%s@thredds-test.ucar.edu/thredds/dodsC/restrict/testData.nc" +#define URL1 "https://%s@%s/thredds/dodsC/test3/testData.nc" +#define URL2 "https://thredds/%s/dodsC/test3/testData.nc" +#define URL3 "https://%s@" AUTHTESTSERVER "/thredds/dodsC/test3/testData.nc" /* Embedded user:pwd */ static char url1[1024]; @@ -45,36 +47,25 @@ static char url1[1024]; static char url2[1024]; /* Test redirect from different machine*/ +#ifndef NOREDIR static char url3[1024]; +#endif static int testrc(const char* prefix, const char* url); static void fillrc(const char* path); static void killrc(); -#ifdef DEBUG -static void -CHECK(int e, const char* msg) -{ - if(e == NC_NOERR) return; - if(msg == NULL) msg = "Error"; - fprintf(stderr,"%s: %s\n", msg, nc_strerror(e)); - exit(1); -} -#endif - int main(int argc, char** argv) { int ncid,retval,pass; - FILE* rc; const char* dfaltsvc; - char buffer[8192]; const char* home; fprintf(stderr,"Testing: Authorization\n"); - dfaltsvc = nc_findtestserver("thredds",REMOTETESTSERVERS); - if(svc == NULL) { + dfaltsvc = nc_findtestserver("thredds",AUTHTESTSERVER); + if(dfaltsvc == NULL) { fprintf(stderr,"WARNING: Cannot locate test server\n"); exit(0); } @@ -110,7 +101,7 @@ fflush(stderr); #ifndef NOLOCAL { /* Test 1: RC in ./ */ - fprintf(stderr,"Testing: user:pwd in %s/%s: %s\n",".",RC); + fprintf(stderr,"Testing: user:pwd in %s/%s\n",".",RC); if(!testrc(".",url2)) { fprintf(stderr,"user:pwd in %s/%s failed\n",".",RC); exit(1); @@ -122,7 +113,7 @@ fflush(stderr); { /* Test 1: RC in HOME */ home = getenv("HOME"); - fprintf(stderr,"user:pwd in %s/%s: %s\n",home,RC); + fprintf(stderr,"user:pwd in %s/%s\n",home,RC); if(!testrc(home,url2)) { fprintf(stderr,"user:pwd in %s/%s failed\n",home,RC); exit(1); @@ -205,10 +196,10 @@ static void killrc() { const char* home; - char path[1024]; #ifdef KEEPRC fprintf(stderr,"kill: ./%s\n",RC); -#else +#else + char path[1024]; snprintf(path,sizeof(path),"%s/%s",".",RC); unlink(path); /* delete the file */ #endif diff --git a/ncdap_test/testauth.sh b/ncdap_test/testauth.sh index 8a16661fc9..c35dd81530 100755 --- a/ncdap_test/testauth.sh +++ b/ncdap_test/testauth.sh @@ -39,9 +39,14 @@ fi LOCALRCFILES="$WD/.dodsrc $WD/.daprc $WD/.ncrc $WD/$NETRC $WD/$NETRCIMP" HOMERCFILES="$HOME/.dodsrc $HOME/.daprc $HOME/.ncrc $HOME/$NETRC $HOME/$NETRCIMP" - NETRCFILE=$WD/$NETRC DAPRCFILE=$WD/$RC +if test "x$FP_ISMSVC" = x1 ; then + LOCALRCFILES=`${execdir}/pathcvt "$LOCALRCFILES"` + HOMERCFILES=`${execdir}/pathcvt "$HOMERCFILES"` + NETRCFILE=`${execdir}/pathcvt "$NETRCFILE"` + DAPRCFILE=`${execdir}/pathcvt "$DAPRCFILE"` +fi HOMENETRCFILE=$HOME/$NETRC HOMEDAPRCFILE=$HOME/$RC @@ -137,9 +142,9 @@ rclocal1() { # Case: local daprc local netrc no embed rclocal2() { - echo "***Testing rc file in local directory" + echo "***Testing rc file + .netrc in local directory" reset - # Create the rc file and (optional) netrc fil in ./ + # Create the rc file and (optional) netrc file in ./ createnetrc $LOCALNETRC createrc $LOCALRC $LOCALNETRC # Invoke ncdump to extract a file using the URL @@ -214,6 +219,5 @@ if test "x$RCHOME" = x1 ; then fi reset - exit diff --git a/ncdap_test/testpathcvt.sh b/ncdap_test/testpathcvt.sh new file mode 100755 index 0000000000..a91146a350 --- /dev/null +++ b/ncdap_test/testpathcvt.sh @@ -0,0 +1,37 @@ +#!/bin/sh + +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + +set -e + +testcase1() { +T="$1" +P="$2" +echo -n ' ' +${execdir}/pathcvt $T "$P" +echo "" +} + +testcase() { +echo "path: $1:" + testcase1 "-u" "$1" + testcase1 "-c" "$1" + testcase1 "-m" "$1" + testcase1 "-w" "$1" +} + +rm -f tmp_pathcvt.txt + +testcase "/xxx/a/b" >> tmp_pathcvt.txt +testcase "d:/x/y" >> tmp_pathcvt.txt +testcase "d:\\x\\y" >> tmp_pathcvt.txt +testcase "/cygdrive/d/x/y" >> tmp_pathcvt.txt +testcase "/d/x/y" >> tmp_pathcvt.txt +testcase "/cygdrive/d" >> tmp_pathcvt.txt +testcase "/d" >> tmp_pathcvt.txt +testcase "/cygdrive/d/git/netcdf-c/dap4_test/test_anon_dim.2.syn" >> tmp_pathcvt.txt + +diff -w ${srcdir}/ref_pathcvt.txt ./tmp_pathcvt.txt + +exit 0 diff --git a/ncdap_test/testurl.sh b/ncdap_test/testurl.sh index 17ae9231e9..92c08cb2fc 100755 --- a/ncdap_test/testurl.sh +++ b/ncdap_test/testurl.sh @@ -1,8 +1,8 @@ #!/bin/sh -#NOP=1 -#NOS=1 -#NOB=1 +NOP=1 +NOS=1 +NOB=1 #SHOW=1 #DBG=1 diff --git a/oc2/occurlfunctions.c b/oc2/occurlfunctions.c index b6d7d3e47c..076ecfb625 100644 --- a/oc2/occurlfunctions.c +++ b/oc2/occurlfunctions.c @@ -66,8 +66,10 @@ ocset_curlflag(OCstate* state, int flag) case CURLOPT_NETRC: case CURLOPT_NETRC_FILE: if(state->auth->curlflags.netrc) { - SETCURLOPT(state, CURLOPT_NETRC, (OPTARG)CURL_NETRC_REQUIRED); - SETCURLOPT(state, CURLOPT_NETRC_FILE, state->auth->curlflags.netrc); + SETCURLOPT(state, CURLOPT_NETRC, (OPTARG)CURL_NETRC_OPTIONAL); + /* IF HTTP.NETRC is set with "", then assume the default .netrc file (which is apparently CWD) */ + if(strlen(state->auth->curlflags.netrc)>0) + SETCURLOPT(state, CURLOPT_NETRC_FILE, state->auth->curlflags.netrc); } break; From 1bb482dbaef9bce58c9b83684357374aa27693fa Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Sat, 29 May 2021 21:57:06 -0600 Subject: [PATCH 15/16] Update RELEASE NOTES --- RELEASE_NOTES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 4afb742f47..549b431fc9 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -7,7 +7,7 @@ This file contains a high-level description of this package's evolution. Release ## 4.8.1 - TBD - +* [Bug Fix] Re-enable DAP2 authorization testing. See [Github #2011](https://github.com/Unidata/netcdf-c/issues/2011). * [Bug Fix] Fix ncdump bug when printing VLENs with basetype char. See [Github #1986](https://github.com/Unidata/netcdf-c/issues/1986). * [Bug Fixes] The netcdf-c library was incorrectly determining the scope of types referred to by nc_inq_type_equal. See [Github #1959](https://github.com/Unidata/netcdf-c/pull/1959) for more information. * [Bug Fix] Fix bug in use of XGetopt when building under Mingw. See [Github #2009](https://github.com/Unidata/netcdf-c/issues/2009). From a99afdefc64f7f78e3148c984556f550c1cfff06 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Sun, 30 May 2021 16:12:40 -0600 Subject: [PATCH 16/16] Add backslash escape option to pathcvt and test --- ncdap_test/pathcvt.c | 60 ++++++++++++++++++++++++++++++-------- ncdap_test/ref_pathcvt.txt | 39 ++++++++++++++----------- ncdap_test/testpathcvt.sh | 14 +++++---- 3 files changed, 78 insertions(+), 35 deletions(-) diff --git a/ncdap_test/pathcvt.c b/ncdap_test/pathcvt.c index 14cbbfe54c..d85e0564f6 100755 --- a/ncdap_test/pathcvt.c +++ b/ncdap_test/pathcvt.c @@ -27,16 +27,16 @@ /* Synopsis -pathcvt [-u|-w|-m|-c] PATH +pathcvt [-u|-w|-m|-c] [-e] PATH Options - + -e add backslash escapes to '\' and ' ' Output type options: -u convert to Unix form of path -w convert to Windows form of path -m convert to MSYS form of path -c convert to Cygwin form of path - + Default is to convert to the format used by the platform. */ @@ -45,16 +45,12 @@ Default is to convert to the format used by the platform. struct Options { int target; + int escape; int debug; } cvtoptions; -static void -usage(const char* msg) -{ - if(msg != NULL) fprintf(stderr,"%s\n",msg); - fprintf(stderr,"pathcvt [-u|-w|-m|-c] PATH\n"); - if(msg == NULL) exit(0); else exit(1); -} +static char* escape(const char* path); +static void usage(const char* msg); int main(int argc, char** argv) @@ -65,9 +61,10 @@ main(int argc, char** argv) memset((void*)&cvtoptions,0,sizeof(cvtoptions)); - while ((c = getopt(argc, argv, "cD:hmuw")) != EOF) { + while ((c = getopt(argc, argv, "cD:ehmuw")) != EOF) { switch(c) { case 'c': cvtoptions.target = NCPD_CYGWIN; break; + case 'e': cvtoptions.escape = 1; break; case 'h': usage(NULL); break; case 'm': cvtoptions.target = NCPD_MSYS; break; case 'u': cvtoptions.target = NCPD_NIX; break; @@ -90,12 +87,51 @@ main(int argc, char** argv) if (argc > 1) usage("more than one path specified"); inpath = argv[0]; - if(cvtoptions.target == NCPD_UNKNOWN) cvtpath = NCpathcvt(inpath); else cvtpath = NCpathcvt_test(inpath,cvtoptions.target,'c'); + if(cvtpath && cvtoptions.escape) { + char* path = cvtpath; cvtpath = NULL; + cvtpath = escape(path); + free(path); + } printf("%s",cvtpath); if(cvtpath) free(cvtpath); return 0; } + +static void +usage(const char* msg) +{ + if(msg != NULL) fprintf(stderr,"%s\n",msg); + fprintf(stderr,"pathcvt [-u|-w|-m|-c] PATH\n"); + if(msg == NULL) exit(0); else exit(1); +} + +static char* +escape(const char* path) +{ + size_t slen = strlen(path); + const char* p; + char* q; + char* epath = NULL; + + epath = (char*)malloc((2*slen) + 1); + if(epath == NULL) usage("out of memtory"); + p = path; + q = epath; + for(;*p;p++) { + switch (*p) { + case '\\': case ' ': + *q++ = '\\'; + /* fall thru */ + default: + *q++ = *p; + break; + } + } + *q = '\0'; + return epath; +} + diff --git a/ncdap_test/ref_pathcvt.txt b/ncdap_test/ref_pathcvt.txt index 6397eb5b37..1398eb14f9 100644 --- a/ncdap_test/ref_pathcvt.txt +++ b/ncdap_test/ref_pathcvt.txt @@ -1,40 +1,45 @@ -path: /xxx/a/b: +path: /xxx/a/b /xxx/a/b /cygdrive/c/xxx/a/b /c/xxx/a/b - c:\xxx\a\b -path: d:/x/y: + c:\\xxx\\a\\b +path: d:/x/y /d/x/y /cygdrive/d/x/y /d/x/y - d:\x\y -path: d:\x\y: + d:\\x\\y +path: /cygdrive/d/x/y /d/x/y /cygdrive/d/x/y /d/x/y - d:\x\y -path: /cygdrive/d/x/y: + d:\\x\\y +path: /d/x/y /d/x/y /cygdrive/d/x/y /d/x/y - d:\x\y -path: /d/x/y: - /d/x/y - /cygdrive/d/x/y - /d/x/y - d:\x\y -path: /cygdrive/d: + d:\\x\\y +path: /cygdrive/d /d /cygdrive/d /d d: -path: /d: +path: /d /d /cygdrive/d /d d: -path: /cygdrive/d/git/netcdf-c/dap4_test/test_anon_dim.2.syn: +path: /cygdrive/d/git/netcdf-c/dap4_test/test_anon_dim.2.syn /d/git/netcdf-c/dap4_test/test_anon_dim.2.syn /cygdrive/d/git/netcdf-c/dap4_test/test_anon_dim.2.syn /d/git/netcdf-c/dap4_test/test_anon_dim.2.syn - d:\git\netcdf-c\dap4_test\test_anon_dim.2.syn + d:\\git\\netcdf-c\\dap4_test\\test_anon_dim.2.syn +path: d:\x\y + /d/x/y + /cygdrive/d/x/y + /d/x/y + d:\\x\\y +path: d:\x\y w\z + /d/x/y w/z + /cygdrive/d/x/y w/z + /d/x/y w/z + d:\\x\\y\\ w\\z diff --git a/ncdap_test/testpathcvt.sh b/ncdap_test/testpathcvt.sh index a91146a350..e4588b66a4 100755 --- a/ncdap_test/testpathcvt.sh +++ b/ncdap_test/testpathcvt.sh @@ -8,29 +8,31 @@ set -e testcase1() { T="$1" P="$2" -echo -n ' ' -${execdir}/pathcvt $T "$P" -echo "" +C=`${execdir}/pathcvt $T "$P"` +echo " $C" } testcase() { -echo "path: $1:" +#X=`echo -n "$1" | sed -e 's/\\\/\\\\\\\/g'` +X="$1" +echo "path:" "$X" testcase1 "-u" "$1" testcase1 "-c" "$1" testcase1 "-m" "$1" - testcase1 "-w" "$1" + testcase1 "-ew" "$1" | sed -e 's/\\/\\\\/g' } rm -f tmp_pathcvt.txt testcase "/xxx/a/b" >> tmp_pathcvt.txt testcase "d:/x/y" >> tmp_pathcvt.txt -testcase "d:\\x\\y" >> tmp_pathcvt.txt testcase "/cygdrive/d/x/y" >> tmp_pathcvt.txt testcase "/d/x/y" >> tmp_pathcvt.txt testcase "/cygdrive/d" >> tmp_pathcvt.txt testcase "/d" >> tmp_pathcvt.txt testcase "/cygdrive/d/git/netcdf-c/dap4_test/test_anon_dim.2.syn" >> tmp_pathcvt.txt +testcase "d:\\x\\y" >> tmp_pathcvt.txt +testcase "d:\\x\\y w\\z" >> tmp_pathcvt.txt diff -w ${srcdir}/ref_pathcvt.txt ./tmp_pathcvt.txt