From fb9437aabc6ef6b1abc5d1ef0a100a9e477b225a Mon Sep 17 00:00:00 2001 From: Joris Vankerschaver Date: Tue, 31 Oct 2017 13:30:21 +0000 Subject: [PATCH] MAINT: Upcast to int_ instead of int64 to avoid bincount issue. --- chaco/tests/test_image_plot.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/chaco/tests/test_image_plot.py b/chaco/tests/test_image_plot.py index 1df5de21e..80cfb384d 100644 --- a/chaco/tests/test_image_plot.py +++ b/chaco/tests/test_image_plot.py @@ -81,11 +81,12 @@ def calculate_rms(image_result, expected_image): """ # calculate the per-pixel errors, then compute the root mean square error num_values = np.prod(expected_image.shape) - # Cast to int64 to reduce likelihood of over-/under-flow. - abs_diff_image = abs(np.int64(expected_image) - np.int64(image_result)) + # Images may be e.g. 8-bit unsigned integer; upcast to default integer size + # (32 or 64 bit) to reduce likelihood of over-/under-flow. + abs_diff_image = abs(np.int_(expected_image) - np.int_(image_result)) histogram = np.bincount(abs_diff_image.ravel(), minlength=256) - sum_of_squares = np.sum(histogram * np.arange(len(histogram))**2) + sum_of_squares = np.sum(np.int64(histogram) * np.arange(len(histogram))**2) rms = np.sqrt(float(sum_of_squares) / num_values) return rms