Codify cross-platform file paths

CMakeLists.txt

@@ -662,7 +662,6 @@ IF(USE_HDF5)
     IF(HDF5_VERSION_STRING AND NOT HDF5_VERSION)
       SET(HDF5_VERSION ${HDF5_VERSION_STRING})
     ENDIF()
-
     IF("${HDF5_VERSION}" STREQUAL "")
       MESSAGE(STATUS "Unable to determine hdf5 version.  NetCDF requires at least version ${HDF5_VERSION_REQUIRED}")
     ELSE()
@@ -803,6 +802,15 @@ IF(USE_HDF5)
     SET(HAS_PAR_FILTERS no CACHE STRING "")
   ENDIF()
 
+  # Check to see if HDF5 library is 1.10.6 or greater.
+  # Used to control path name conversion
+  IF(${HDF5_VERSION} VERSION_LESS "1.10.6")
+    SET(HDF5_UTF8_PATHS FALSE)
+  ELSE()
+    SET(HDF5_UTF8_PATHS TRUE)
+  ENDIF()
+  MESSAGE("-- Checking for HDF5 version 1.10.6 or later: ${HDF5_UTF8_PATHS}")
+
   SET(H5_USE_16_API 1)
   OPTION(NC_ENABLE_HDF_16_API "Enable HDF5 1.6.x Compatibility(Required)" ON)
   IF(NOT NC_ENABLE_HDF_16_API)
@@ -2204,6 +2212,7 @@ configure_file(
 SET(EXTRA_DIST ${EXTRA_DIST} ${CMAKE_CURRENT_SOURCE_DIR}/test_common.in)
 SET(TOPSRCDIR "${CMAKE_CURRENT_SOURCE_DIR}")
 SET(TOPBUILDDIR "${CMAKE_CURRENT_BINARY_DIR}")
+SET(ISMSVC "${MSVC}")
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test_common.in ${CMAKE_CURRENT_BINARY_DIR}/test_common.sh @ONLY NEWLINE_STYLE LF)
 
 

RELEASE_NOTES.md

@@ -7,6 +7,7 @@ This file contains a high-level description of this package's evolution. Release
 
 ## 4.8.0 - TBD
 
+* [Enhancement] Provide better cross platform path name management. This converts paths for various platforms (e.g. Windows, MSYS, etc.) so that they are in the proper format for the executing platform. See [Github #1958](https://github.com/Unidata/netcdf-c/pull/1958) for more information.
 * [Bug Fixes] The nccopy program was treating -d0 as turning deflation on rather than interpreting it as "turn off deflation". See [Github #1944](https://github.com/Unidata/netcdf-c/pull/1944) for more information.
 * [Enhancement] Add support for storing NCZarr data in zip files. See [Github #1942](https://github.com/Unidata/netcdf-c/pull/1942) for more information.
 * [Bug Fixes] Make fillmismatch the default for DAP2 and DAP4; too many servers ignore this requirement.

configure.ac

@@ -79,9 +79,9 @@ AC_MSG_NOTICE([checking supported formats])
 # An explicit disable of netcdf-4 | netcdf4 is treated as if it was disable-hdf5
 AC_MSG_CHECKING([whether we should build with netcdf4 (alias for HDF5)])
 AC_ARG_ENABLE([netcdf4], [AS_HELP_STRING([--disable-netcdf4],
-              [(synonym for --enable-hdf5)])])
+              [(deprecated synonym for --enable-hdf5)])])
 test "x$enable_netcdf4" = xno || enable_netcdf4=yes
-AC_MSG_RESULT([$enable_netcdf4])
+AC_MSG_RESULT([$enable_netcdf4 (deprecated; Please use with --disable-hdf5)])
 AC_MSG_CHECKING([whether we should build with netcdf-4 (alias for HDF5)])
 AC_ARG_ENABLE([netcdf-4], [AS_HELP_STRING([--disable-netcdf-4],
               [(synonym for --disable-netcdf4)])])

debug/cf.cmake

@@ -15,7 +15,7 @@ S3TEST=1
 #CDF5=1
 #HDF4=1
 
-#TR=--trace
+H518=1
 
 for arg in "$@" ; do
 case "$arg" in

include/Makefile.am

@@ -19,7 +19,7 @@ ncbytes.h nchashmap.h ceconstraints.h rnd.h nclog.h ncconfigure.h	\
 nc4internal.h nctime.h nc3internal.h onstack.h ncrc.h ncauth.h		\
 ncoffsets.h nctestserver.h nc4dispatch.h nc3dispatch.h ncexternl.h	\
 ncpathmgr.h ncindex.h hdf4dispatch.h hdf5internal.h nc_provenance.h	\
-hdf5dispatch.h ncmodel.h isnan.h nccrc.h ncexhash.h ncxcache.h
+hdf5dispatch.h ncmodel.h isnan.h nccrc.h ncexhash.h ncxcache.h ncfilter.h
 
 if USE_DAP
 noinst_HEADERS += ncdap.h

include/hdf5internal.h

@@ -201,23 +201,9 @@ extern int NC4_hdf5get_libversion(unsigned*,unsigned*,unsigned*);/*libsrc4/nc4hd
 extern int NC4_hdf5get_superblock(struct NC_FILE_INFO*, int*);/*libsrc4/nc4hdf.c*/
 extern int NC4_isnetcdf4(struct NC_FILE_INFO*); /*libsrc4/nc4hdf.c*/
 
-#ifdef _WIN32
+extern int nc4_find_default_chunksizes2(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var);
 
-/* Maxinum length of a typical path in UTF-8.
- * When converting from ANSI to UTF-8, the length will be up to 3 times,
- * so round up 260*3 to 1024. (260=MAX_PATH) */
-#define MAX_PATHBUF_SIZE 1024
-
-/* Struct for converting ANSI to UTF-8. */
-typedef struct pathbuf
-{
-    void *ptr;
-    char buffer[MAX_PATHBUF_SIZE];
-} pathbuf_t;
-
-const char *nc4_ndf5_ansi_to_utf8(pathbuf_t *pb, const char *path);
-void nc4_hdf5_free_pathbuf(pathbuf_t *pb);
-
-#endif /* _WIN32 */
+EXTERNL hid_t nc4_H5Fopen(const char *filename, unsigned flags, hid_t fapl_id);
+EXTERNL hid_t nc4_H5Fcreate(const char *filename, unsigned flags, hid_t fcpl_id, hid_t fapl_id);
 
 #endif /* _HDF5INTERNAL_ */

include/nchttp.h

@@ -20,6 +20,7 @@ typedef struct NC_HTTP_STATE {
     const char** headset; /* which headers to capture */
     NClist* headers;
     NCbytes* buf;
+    char errbuf[1024]; /* assert(CURL_ERROR_SIZE <= 1024) */
 } NC_HTTP_STATE;
 
 extern int nc_http_open(const char* objecturl, NC_HTTP_STATE** state, size64_t* lenp);

include/ncpathmgr.h

@@ -2,8 +2,8 @@
  *	Copyright 2018, University Corporation for Atmospheric Research
  *      See netcdf/COPYRIGHT file for copying and redistribution conditions.
  */
-#ifndef _NCWINIO_H_
-#define _NCWINIO_H_
+#ifndef _NCPATHMGR_H_
+#define _NCPATHMGR_H_
 
 #include "config.h"
 #include <stdlib.h>
@@ -28,6 +28,7 @@
 #endif
 #endif
 
+/* Define wrapper constants for use with NCaccess */
 /* Define wrapper constants for use with NCaccess */
 #ifdef _WIN32
 #define ACCESS_MODE_EXISTS 0
@@ -49,16 +50,63 @@
 #define ACCESS_MODE_RW (R_OK|W_OK)
 #endif
 
-/* Path Converter */
+#ifdef _WIN32
+#ifndef S_IFDIR
+#define S_IFDIR _S_IFDIR
+#define S_IFREG _S_IFREG
+#endif
+#ifndef S_ISDIR
+#define S_ISDIR(mode) ((mode) & _S_IFDIR)
+#define S_ISREG(mode) ((mode) & _S_IFREG)
+#endif
+#endif /*_WIN32*/
+
+/*
+WARNING: you should never need to explictly call this function;
+rather it is invoked as part of the wrappers for e.g. NCfopen, etc.
+
+This function attempts to take an arbitrary path and convert
+it to a canonical form.
+Assumptions about Input path:
+1. It is a relative or absolute path
+2. It is not a URL
+3. It conforms to the format expected by one of the following:
+       Linux (/x/y/...), Cygwin (/cygdrive/D/...),
+       Windows (D:/...), or MSYS (/D/...), or relative (x/y...)
+4. It is encoded in the local platform character set.
+   Note that for most systems, this is utf-8. But for Windows,
+   the encoding is most likely some form of ANSI code page, probably
+   the windows 1252 encoding.
+   Note that in any case, the path must be representable in the
+   local Code Page.
+
+On output it produces a re-written path that has the following
+properties:
+1. The path is normalized to match the platform on which the code
+   is running (e.g. cygwin, windows, msys, linux). So for example
+   using a cygwin path under visual studio will convert e.g.
+   /cygdrive/d/x/y to d:\x\y. See ../unit_test/test_pathcvt.c
+   for example conversions.
+It returns the converted path.
+
+Note that this function is intended to be Idempotent: f(f(x) == f(x).
+This means it is ok to call it repeatedly with no harm.
+*/
 EXTERNL char* NCpathcvt(const char* path);
 
-/* path -> URL Path converter */
-EXTERNL char* NCurlpath(const char* path);
+/* Canonicalize and make absolute by prefixing the current working directory */
+EXTERNL char* NCpathabsolute(const char* name);
 
-/* Fix path in case it was escaped by shell */
-EXTERNL char* NCdeescape(const char* name);
+/* Convert from the local coding (e.g. ANSI) to utf-8;
+   note that this can produce unexpected results for Windows
+   because it first converts to wide character and then to utf8. */
+EXTERNL int NCpath2utf8(const char* path, char** u8p);
 
-#ifdef WINPATH
+/* Wrap various stdio and unistd IO functions.
+It is especially important to use for windows so that
+NCpathcvt (above) is invoked on the path.
+*/
+#if defined(WINPATH)
 /* path converter wrappers*/
 EXTERNL FILE* NCfopen(const char* path, const char* flags);
 EXTERNL int NCopen3(const char* path, int flags, int mode);
@@ -68,6 +116,10 @@ EXTERNL int NCremove(const char* path);
 EXTERNL int NCmkdir(const char* path, int mode);
 EXTERNL int NCrmdir(const char* path);
 EXTERNL char* NCcwd(char* cwdbuf, size_t len);
+EXTERNL char* NCcwd(char* cwdbuf, size_t len);
+#ifdef HAVE_SYS_STAT_H
+EXTERNL int NCstat(char* path, struct stat* buf);
+#endif
 #ifdef HAVE_DIRENT_H
 EXTERNL DIR* NCopendir(const char* path);
 EXTERNL int NCclosedir(DIR* ent);
@@ -78,8 +130,12 @@ EXTERNL int NCclosedir(DIR* ent);
 #define NCopen2(path,flags) open((path),(flags))
 #define NCremove(path) remove(path)
 #define NCaccess(path,mode) access(path,mode)
+#define NCmkdir(path,mode) mkdir(path,mode)
+#define NCgetcwd(buf,len) getcwd(buf,len)
+#ifdef HAVE_SYS_STAT_H
+#define NCstat(path,buf) stat(path,buf)
+#endif
 #define NCcwd(buf, len) getcwd(buf,len)
-#define NCmkdir(path, mode) mkdir(path,mode)
 #define NCrmdir(path) rmdir(path)
 #ifdef HAVE_DIRENT_H
 #define NCopendir(path) opendir(path)
@@ -89,7 +145,26 @@ EXTERNL int NCclosedir(DIR* ent);
 
 /* Platform independent */
 #define NCclose(fd) close(fd)
+#define NCfstat(fd,buf) fstat(fd,buf)
+
+/**************************************************/
+/* Following definitions are for testing only */
+
+/* Possible Kinds Of Output */
+#define NCPD_UNKNOWN 0
+#define NCPD_NIX 1
+#define NCPD_MSYS 2
+#define NCPD_CYGWIN 3 
+#define NCPD_WIN 4
+#define NCPD_REL 5 /* actual kind is unknown */
+
+EXTERNL char* NCpathcvt_test(const char* path, int ukind, int udrive);
+
+EXTERNL void printutf8hex(const char* s, char* sx);
 
-EXTERNL int NChasdriveletter(const char* path);
+/**************************************************/
+/* From dutil.c */
+EXTERNL char* NC_backslashEscape(const char* s);
+EXTERNL char* NC_backslashUnescape(const char* esc);
 
-#endif /* _NCWINIO_H_ */
+#endif /* _NCPATHMGR_H_ */

include/ncrc.h

@@ -55,8 +55,6 @@ extern NCTriple* NC_rcfile_ith(NCRCinfo*,size_t);
 /* From dutil.c (Might later move to e.g. nc.h */
 extern int NC__testurl(const char* path, char** basenamep);
 extern int NC_isLittleEndian(void);
-extern char* NC_backslashEscape(const char* s);
-extern char* NC_backslashUnescape(const char* esc);
 extern char* NC_entityescape(const char* s);
 extern int NC_readfile(const char* filename, NCbytes* content);
 extern int NC_writefile(const char* filename, size_t size, void* content);

include/ncuri.h

@@ -67,6 +67,9 @@ EXTERNL void ncurifree(NCURI* ncuri);
 /* Replace the protocol */
 EXTERNL int ncurisetprotocol(NCURI*,const char* newprotocol);
 
+/* Replace the path */
+EXTERNL int ncurisetpath(NCURI*,const char* newpath);
+
 /* Replace the constraints */
 EXTERNL int ncurisetquery(NCURI*,const char* query);
 

lib_flags.am

@@ -13,7 +13,7 @@ if USE_DAP
 AM_CPPFLAGS += -I${top_srcdir}/oc2
 endif
 
-if HAVE_AWS
+if ENABLE_S3_SDK
 AM_LDFLAGS += -lstdc++
 endif
 

libdispatch/ddispatch.c

@@ -10,6 +10,7 @@ See LICENSE.txt for license information.
 #include "ncbytes.h"
 #include "ncrc.h"
 #include "ncoffsets.h"
+#include "ncpathmgr.h"
 
 /* Required for getcwd, other functions. */
 #ifdef HAVE_UNISTD_H
@@ -19,7 +20,6 @@ See LICENSE.txt for license information.
 /* Required for getcwd, other functions. */
 #ifdef _WIN32
 #include <direct.h>
-#define getcwd _getcwd
 #endif
 
 #if defined(ENABLE_BYTERANGE) || defined(ENABLE_DAP) || defined(ENABLE_DAP4)
@@ -57,61 +57,28 @@ NCDISPATCH_initialize(void)
 
     /* Capture temp dir*/
     {
-	char* tempdir;
-	char* p;
-	char* q;
-	char cwd[4096];
-#ifdef _WIN32
+	char* tempdir = NULL;
+#if defined _WIN32 || defined __MSYS__
         tempdir = getenv("TEMP");
 #else
 	tempdir = "/tmp";
 #endif
         if(tempdir == NULL) {
 	    fprintf(stderr,"Cannot find a temp dir; using ./\n");
-	    tempdir = getcwd(cwd,sizeof(cwd));
-	    if(tempdir == NULL || *tempdir == '\0') tempdir = ".";
-	}
-        globalstate->tempdir= (char*)malloc(strlen(tempdir) + 1);
-	for(p=tempdir,q=globalstate->tempdir;*p;p++,q++) {
-	    if((*p == '/' && *(p+1) == '/')
-	       || (*p == '\\' && *(p+1) == '\\')) {p++;}
-	    *q = *p;
-	}
-	*q = '\0';
-#ifdef _WIN32
-#else
-        /* Canonicalize */
-	for(p=globalstate->tempdir;*p;p++) {
-	    if(*p == '\\') {*p = '/'; };
+	    tempdir = ".";
 	}
-	*q = '\0';
-#endif
+	globalstate->tempdir= strdup(tempdir);
     }
 
     /* Capture $HOME */
     {
-	char* p;
-	char* q;
         char* home = getenv("HOME");
 
         if(home == NULL) {
 	    /* use tempdir */
 	    home = globalstate->tempdir;
 	}
-        globalstate->home = (char*)malloc(strlen(home) + 1);
-	for(p=home,q=globalstate->home;*p;p++,q++) {
-	    if((*p == '/' && *(p+1) == '/')
-	       || (*p == '\\' && *(p+1) == '\\')) {p++;}
-	    *q = *p;
-	}
-	*q = '\0';
-#ifdef _WIN32
-#else
-        /* Canonicalize */
-	for(p=home;*p;p++) {
-	    if(*p == '\\') {*p = '/'; };
-	}
-#endif
+        globalstate->home = strdup(home);
     }
 
     /* Now load RC File */

libdispatch/dfile.c

@@ -1853,12 +1853,7 @@ NC_create(const char *path0, int cmode, size_t initialsz,
         /* Skip past any leading whitespace in path */
         const unsigned char* p;
         for(p=(const unsigned char*)path0;*p;p++) {if(*p > ' ') break;}
-#ifdef WINPATH
-        /* Need to do path conversion */
-        path = NCpathcvt((const char*)p);
-#else
         path = nulldup((const char*)p);
-#endif
     }
 
     memset(&model,0,sizeof(model));
@@ -2009,12 +2004,7 @@ NC_open(const char *path0, int omode, int basepe, size_t *chunksizehintp,
         /* Skip past any leading whitespace in path */
         const char* p;
         for(p=(const char*)path0;*p;p++) {if(*p < 0 || *p > ' ') break;}
-#ifdef WINPATH
-        /* Need to do path conversion */
-        path = NCpathcvt(p);
-#else
         path = nulldup(p);
-#endif
     }
 
     memset(&model,0,sizeof(model));