ENH: Add netCDF 64-bit offset option to ocli (#506, #519)

- Add 64bit_offset option to ocli
- Pass large_file only when it is True allowing older ESMF versions to work
- Add flag documentation

doc/sphinx_examples/chunked_rwg_help.sh

@@ -91,4 +91,13 @@ Options:
                                   and weight factor variables. WITHAUX adds
                                   auxiliary variables and additional file
                                   metadata to the output weight file.
+  --64bit_offset / --no_64bit_offset
+                                  If provided, create the weight file in
+                                  NetCDF using the 64-bit offset format to
+                                  allow variables larger than 2GB. Note the
+                                  64-bit offset format is not supported in the
+                                  NetCDF version earlier than 3.6.0.  An error
+                                  message will be generated if this flag is
+                                  specified while the application is linked
+                                  with a NetCDF library earlier than 3.6.0.
   --help                          Show this message and exit.

src/ocgis/messages.py

@@ -1,4 +1,5 @@
 M1 = 'No dimensioned variables found. This typically means no target variables in the dataset have space and/or time dimensions associated with them. Consider using a dimension map if file metadata is non-standard. Overloading "variable" is also an option to avoid dimension checking.'
 M3 = 'Output path exists "{0}" and must be removed before proceeding. Set "overwrite" argument or env.OVERWRITE to True to overwrite.'
 M4 = """A level subset was requested but the target dataset does not have a level dimension. The dataset's alias is: {0}"""
-M5 = "The ESMF FileMode constant value. BASIC (the default) only writes the factor index list and weight factor variables. WITHAUX adds auxiliary variables and additional file metadata to the output weight file."
+M5 = "The ESMF FileMode constant value. BASIC (the default) only writes the factor index list and weight factor variables. WITHAUX adds auxiliary variables and additional file metadata to the output weight file."
+M6 = "If provided, create the weight file in NetCDF using the 64-bit offset format to allow variables larger than 2GB. Note the 64-bit offset format is not supported in the NetCDF version earlier than 3.6.0.  An error message will be generated if this flag is specified while the application is linked with a NetCDF library earlier than 3.6.0."

src/ocgis/ocli.py

@@ -14,7 +14,7 @@
 from ocgis.base import grid_abstraction_scope, raise_if_empty
 from ocgis.constants import DriverKey, Topology, GridChunkerConstants, DecompositionType
 from ocgis.driver.nc_ugrid import DriverNetcdfUGRID
-from ocgis.messages import M5
+from ocgis.messages import M5, M6
 from ocgis.spatial.grid_chunker import GridChunker
 from ocgis.spatial.spatial_subset import SpatialSubsetOperation
 from ocgis.util.logging_ocgis import ocgis_lh
@@ -94,9 +94,11 @@ def ocli():
 @click.option('--loglvl', default="INFO", help='Verbosity level for standard out logging. Default is '
               '"INFO". See Python logging level docs for additional values: https://docs.python.org/3/howto/logging.html')
 @click.option('--weightfilemode', default="BASIC", help=M5)
+@click.option('--64bit_offset/--no_64bit_offset', 'large_file', default=False, help=M6)
 def chunked_rwg(source, destination, weight, nchunks_dst, merge, esmf_src_type, esmf_dst_type, genweights,
                 esmf_regrid_method, spatial_subset, src_resolution, dst_resolution, buffer_distance, wd, persist,
-                eager, ignore_degenerate, data_variables, spatial_subset_path, verbose, loglvl, weightfilemode):
+                eager, ignore_degenerate, data_variables, spatial_subset_path, verbose, loglvl, weightfilemode,
+                large_file):
 
     # Used for creating the history string.
     the_locals = locals()
@@ -172,7 +174,10 @@ def chunked_rwg(source, destination, weight, nchunks_dst, merge, esmf_src_type,
 
     # Arguments to ESMF regridding.
     esmf_kwargs = {'regrid_method': esmf_regrid_method,
-                   'ignore_degenerate': ignore_degenerate}
+                   'ignore_degenerate': ignore_degenerate,}
+    # Allow older versions of ESMF to work with a default large file flag.
+    if large_file:
+        esmf_kwargs['large_file'] = large_file
 
     # Create the chunked regridding object. This is used for both chunked regridding and a regrid with a spatial subset.
     gs = GridChunker(rd_src, rd_dst, nchunks_dst=nchunks_dst, src_grid_resolution=src_resolution, paths=paths,