Merge pull request #229 from JesseMckinzie/python_ram_limit

Add ram limit parameter to python API

src/nyx/environment.cpp

@@ -620,19 +620,14 @@ bool Environment::parse_cmdline(int argc, char** argv)
 			return false;
 		}
 
-		// Megabytes to bytes
-		size_t requestedCeiling = (size_t)value * 1048576;
+		auto success = set_ram_limit(value);
 
 		// Check if it over the actual limit
 		unsigned long long actualRam = Nyxus::getAvailPhysMemory();
-		if (requestedCeiling > actualRam)
+		if (!success)
 		{
-			std::cerr << "Error: RAM limit " << value << " megabytes (=" << requestedCeiling << " bytes) exceeds the actual amount of available RAM " << actualRam << " bytes\n";
 			return false;
 		}
-
-		// Set the member variable
-		ramLimit = requestedCeiling;
 	}
 
 	//==== Parse the temp directory
@@ -846,6 +841,24 @@ void Environment::set_coarse_gray_depth(unsigned int new_depth)
 	coarse_grayscale_depth = new_depth;
 }
 
+bool Environment::set_ram_limit(size_t megabytes) {
+
+	// Megabytes to bytes
+	size_t requestedCeiling = megabytes * 1048576;
+
+	// Check if it over the actual limit
+	unsigned long long actualRam = Nyxus::getAvailPhysMemory();
+	if (requestedCeiling > actualRam)
+	{
+		std::cerr << "Error: RAM limit " << megabytes << " megabytes (=" << requestedCeiling << " bytes) exceeds the actual amount of available RAM " << actualRam << " bytes\n";
+		return false;
+	}
+
+	// Set the member variable
+	ramLimit = requestedCeiling;
+	return true;
+}
+
 bool Environment::gpu_is_available() {
 #ifdef USE_GPU
 	return get_gpu_properties().size() > 0 ? true : false;
@@ -942,6 +955,7 @@ bool Environment::arrow_is_enabled()
 #endif
 }
 
+
 #ifdef USE_GPU
 
 void Environment::set_gpu_device_id (int choice) 

src/nyx/environment.h

@@ -182,6 +182,8 @@ class Environment: public BasicEnvironment
 	void clear_roi_blacklist ();
 	void get_roi_blacklist_summary(std::string& response);
 
+	bool set_ram_limit(size_t bytes);
+
 	// implementation of Gabor feature options
 	bool parse_gabor_options_raw_inputs (std::string& error_message);
 	GaborOptions gaborOptions;

src/nyx/python/new_bindings_py.cpp

@@ -79,7 +79,8 @@ void initialize_environment(
     float dynamic_range,
     float min_intensity,
     float max_intensity,
-    bool is_imq)
+    bool is_imq,
+    int ram_limit_mb)
 {
     theEnvironment.set_imq(is_imq);
     theEnvironment.set_dim(n_dim);
@@ -101,6 +102,8 @@ void initialize_environment(
     theEnvironment.fpimageOptions.set_min_intensity(min_intensity);
     theEnvironment.fpimageOptions.set_max_intensity(max_intensity);
 
+    if (ram_limit_mb >= 0) theEnvironment.set_ram_limit(ram_limit_mb);
+
     #ifdef USE_GPU
         if(using_gpu == -1) {
             theEnvironment.set_use_gpu(false);
@@ -130,7 +133,8 @@ void set_environment_params_imp (
     int verb_level = 0,
     float dynamic_range = -1,
     float min_intensity = -1,
-    float max_intensity = -1
+    float max_intensity = -1,
+    int ram_limit_mb = -1
 ) {
     if (features.size() > 0) {
         theEnvironment.recognizedFeatureNames = features;
@@ -168,10 +172,15 @@ void set_environment_params_imp (
         theEnvironment.fpimageOptions.set_max_intensity(max_intensity);
     }
 
-    if (verb_level >= 0)
+    if (verb_level >= 0) {
         theEnvironment.set_verbosity_level (verb_level);
-    else
+    } else {
         std::cerr << "Error: verbosity (" + std::to_string(verb_level) + ") should be a non-negative value" << std::endl;
+    }
+
+    if (ram_limit_mb >= 0) {
+        auto success = theEnvironment.set_ram_limit(ram_limit_mb);
+    }
 }
 
 py::tuple featurize_directory_imp (
@@ -732,6 +741,7 @@ std::map<std::string, ParameterTypes> get_params_imp(const std::vector<std::stri
     params["dynamic_range"] = theEnvironment.fpimageOptions.target_dyn_range();
     params["min_intensity"] = theEnvironment.fpimageOptions.min_intensity();
     params["max_intensity"] = theEnvironment.fpimageOptions.max_intensity();
+    params["ram_limit"] = (int)(theEnvironment.get_ram_limit()/1048576); // convert from bytes to megabytes
 
     if (vars.size() == 0) 
         return params;

src/nyx/python/nyxus/nyxus.py

@@ -115,6 +115,8 @@ class Nyxus:
         Minimum intensity of voxels of a floating point TIFF image.
     max_intensity: (optional, default 1.0)
         Maximum intensity of voxels of a floating point TIFF image.
+    ram_limit: int (optional)
+        Maximum amount of ram to be used by Nyxus in megabytes
     """
 
     def __init__(
@@ -128,7 +130,7 @@ def __init__(
             'gabor_kersize', 'gabor_gamma', 'gabor_sig2lam', 'gabor_f0',
             'gabor_thold', 'gabor_thetas', 'gabor_freqs', 'channel_signature', 
             'parent_channel', 'child_channel', 'aggregate', 'dynamic_range', 'min_intensity',
-            'max_intensity'
+            'max_intensity', 'ram_limit'
         }
 
         # Check for unexpected keyword arguments
@@ -155,6 +157,7 @@ def __init__(
         dynamic_range = kwargs.get('dynamic_range', 10000)
         min_intensity = kwargs.get('min_intensity', 0.0)
         max_intensity = kwargs.get('max_intensity', 1.0)
+        ram_limit = kwargs.get('ram_limit', -1)
 
         if neighbor_distance <= 0:
             raise ValueError("Neighbor distance must be greater than zero.")
@@ -193,7 +196,8 @@ def __init__(
             dynamic_range,
             min_intensity,
             max_intensity,
-            False)
+            False,
+            ram_limit)
 
         self.set_gabor_feature_params(
             kersize = gabor_kersize,
@@ -628,6 +632,8 @@ def set_environment_params (self, **params):
             thread for feature calculation. 
         * verbose: int (optional, non-negative, default 0)
             Level of diagnostic output shown in the console. Set '0' to disable any diagnostic output.
+        * ram_limit: int (optional)
+            Maximum amount of ram to be used by Nyxus in megabytes
         """
 
         valid_params = [
@@ -641,7 +647,8 @@ def set_environment_params (self, **params):
             'verbose',
             'dynamic_range',
             'min_intensity',
-            'max_intensity'
+            'max_intensity',
+            'ram_limit',
         ]
 
         for key in params:
@@ -659,6 +666,7 @@ def set_environment_params (self, **params):
         dynamic_range = params.get('dynamic_range', -1)
         min_intensity = params.get('min_intensity', -1)
         max_intensity = params.get('max_intensity', -1)
+        ram_limit = params.get('ram_limit', -1)
 
         set_environment_params_imp(features, 
                                    neighbor_distance, 
@@ -670,7 +678,8 @@ def set_environment_params (self, **params):
                                    verbosity_lvl,
                                    dynamic_range,
                                    min_intensity,
-                                   max_intensity)
+                                   max_intensity,
+                                   ram_limit,)
 
     def set_params(self, **params):
         """Sets parameters of the Nyxus class
@@ -709,14 +718,14 @@ def set_params(self, **params):
             "ibsi",
             "dynamic_range",
             "min_intensity",
-            "max_intensity"
+            "max_intensity",
+            "ram_limit",
         ]
 
         environment_params = {}
 
         gabor_params = {}
 
-
         for key, value in params.items():
             if key.startswith("gabor_"):
                 gabor_params[key[len("gabor_"):]] = value
@@ -892,7 +901,7 @@ def __init__(
             'neighbor_distance', 'pixels_per_micron', 'coarse_gray_depth',
             'n_feature_calc_threads', 'using_gpu', 'ibsi', 'channel_signature', 
             'parent_channel', 'child_channel', 'aggregate', 
-            'dynamic_range', 'min_intensity', 'max_intensity'
+            'dynamic_range', 'min_intensity', 'max_intensity', 'ram_limit'
         }
 
         # Check for unexpected keyword arguments
@@ -932,6 +941,8 @@ def __init__(
             print("No gpu available.")
             using_gpu = -1
 
+        ram_limit = kwargs.get('ram_limit', -1)
+
         initialize_environment(
             3, # 3D
             features,
@@ -945,7 +956,8 @@ def __init__(
             dynamic_range,
             min_intensity,
             max_intensity,
-            False)
+            False,
+            ram_limit)
 
         # list of valid outputs that are used throughout featurize functions
         self._valid_output_types = ['pandas', 'arrowipc', 'parquet']
@@ -1328,6 +1340,7 @@ def __init__(
             print("No gpu available.")
             using_gpu = -1
 
+        ram_limit = -1
 
         initialize_environment(
             2, # 2D
@@ -1342,7 +1355,8 @@ def __init__(
             dynamic_range,
             min_intensity,
             max_intensity,
-            True)
+            True,
+            ram_limit)
 
         # list of valid outputs that are used throughout featurize functions
         self._valid_output_types = ['pandas', 'arrowipc', 'parquet']

tests/python/test_nyxus.py

@@ -169,7 +169,7 @@ def test_get_default_params(self):
                 'max_intensity': 1.0
                 }
 
-            for key in a:
+            for key in e:
 
                 if (isinstance(a[key], float)):
                     assert a[key] == pytest.approx(e[key])
@@ -226,7 +226,7 @@ def test_set_params(self):
 
             params = nyx.get_params()
 
-            for key in params:
+            for key in new_values:
 
                 if (isinstance(params[key], float)):
                     assert params[key] == pytest.approx(new_values[key])
@@ -313,13 +313,12 @@ def test_constructor_with_gabor(self):
                       'min_intensity': 0.0,
                       'max_intensity': 1.0}
 
-            for key in params:
+            for key in result:
 
                 if (isinstance(params[key], float)):
                     assert params[key] == pytest.approx(result[key])
                 else:
                     assert params[key] == pytest.approx(result[key])    
-
 
         def test_in_memory_2d(self):
 
@@ -696,4 +695,12 @@ def test_image_quality_montage_single_roi(self):
             directory_features = nyx.featurize(tissuenet_int, intensity_names=['p0_y1_r1_c0.ome.tif', 'p0_y1_r1_c1.ome.tif'], label_names=['p0_y1_r1_c0.ome.tif', 'p0_y1_r1_c1.ome.tif'])    
 
             assert directory_features.shape[1] > 3
+
+        def test_set_ram_limit_param(self):
+            nyx = nyxus.Nyxus (["*ALL*"])
+            assert nyx is not None
+            nyx.set_params (ram_limit = 1)
+            actual = nyx.get_params()
+            expected = {'ram_limit': 1}
+            assert actual['ram_limit'] == expected['ram_limit']