From 4eba782e8d32f4a639a98b008822b8243578d0ab Mon Sep 17 00:00:00 2001 From: zyfncg Date: Tue, 20 Jul 2021 07:17:20 +0000 Subject: [PATCH 1/2] Support getitem by ellipsis index in dynamic mode --- paddle/fluid/pybind/imperative.cc | 37 +++++++++---- .../fluid/tests/unittests/test_var_base.py | 54 +++++++++++++++++++ 2 files changed, 81 insertions(+), 10 deletions(-) diff --git a/paddle/fluid/pybind/imperative.cc b/paddle/fluid/pybind/imperative.cc index 619301e3b45d3..5a0e17822e85d 100644 --- a/paddle/fluid/pybind/imperative.cc +++ b/paddle/fluid/pybind/imperative.cc @@ -432,19 +432,31 @@ static void ParseIndexingSlice(framework::LoDTensor *tensor, PyObject *_index, const auto &shape = tensor->dims(); const int rank = shape.size(); const int size = PyTuple_GET_SIZE(index); + + // specified_dims is the number of dimensions which indexed by Interger, + // Slices. + int specified_dims = 0; + for (int dim = 0; dim < size; ++dim) { + PyObject *slice_item = PyTuple_GetItem(index, dim); + if (PyCheckInteger(slice_item) || PySlice_Check(slice_item)) { + specified_dims++; + } + } + PADDLE_ENFORCE_EQ( size <= rank, true, platform::errors::InvalidArgument( "too many indices (%d) for tensor of dimension %d", size, rank)); - for (int dim = 0; dim < size; ++dim) { - PyObject *slice_item = PyTuple_GetItem(index, dim); - PADDLE_ENFORCE_EQ(PyCheckInteger(slice_item) || PySlice_Check(slice_item), - true, - platform::errors::InvalidArgument( - "Currently, VarBase.__getitem__() only allows " - "indexing by Integers, Slices, and tuples of " - "these types, but received %s in %dth slice item", - std::string(Py_TYPE(slice_item)->tp_name), dim + 1)); + for (int i = 0, dim = 0; i < size; ++i) { + PyObject *slice_item = PyTuple_GetItem(index, i); + PADDLE_ENFORCE_EQ( + PyCheckInteger(slice_item) || PySlice_Check(slice_item) || + slice_item == Py_Ellipsis, + true, platform::errors::InvalidArgument( + "Currently, VarBase.__getitem__() only allows " + "indexing by Integers, Slices, Ellipsis, and tuples of " + "these types, but received %s in %dth slice item", + std::string(Py_TYPE(slice_item)->tp_name), i + 1)); infer_flags->push_back(1); int dim_len = shape[dim]; if (PyCheckInteger(slice_item)) { @@ -467,7 +479,8 @@ static void ParseIndexingSlice(framework::LoDTensor *tensor, PyObject *_index, slice_ends->push_back(start + 1); slice_strides->push_back(1); decrease_axis->push_back(dim); - } else { + dim++; + } else if (PySlice_Check(slice_item)) { // slice item Py_ssize_t start, end, step; PySliceObject *p = reinterpret_cast(slice_item); @@ -475,12 +488,16 @@ static void ParseIndexingSlice(framework::LoDTensor *tensor, PyObject *_index, // :: or : or 0:dim_len:1 if (start == 0 && end == dim_len && step == 1) { + dim++; continue; } slice_axes->push_back(dim); slice_starts->push_back(start); slice_ends->push_back(end); slice_strides->push_back(step); + dim++; + } else if (slice_item == Py_Ellipsis) { + dim += rank - specified_dims; } } if (!PyTuple_Check(_index)) Py_DecRef(index); diff --git a/python/paddle/fluid/tests/unittests/test_var_base.py b/python/paddle/fluid/tests/unittests/test_var_base.py index 644e46f108158..e20ed21383082 100644 --- a/python/paddle/fluid/tests/unittests/test_var_base.py +++ b/python/paddle/fluid/tests/unittests/test_var_base.py @@ -652,6 +652,59 @@ def _test_slice_for_tensor_attr(self): np.array_equal(local_out[15], tensor_array[::-1, ::-1, ::-1])) self.assertTrue(np.array_equal(local_out[16], tensor_array[-4:4])) + def _test_for_getitem_ellipsis_index(self): + shape = (64, 3, 5, 256) + np_fp32_value = np.random.random(shape).astype('float32') + np_int_value = np.random.randint(1, 100, shape) + + var_fp32 = paddle.to_tensor(np_fp32_value) + var_int = paddle.to_tensor(np_int_value) + + # test for float32 + var = [ + var_fp32[..., 0].numpy(), + var_fp32[..., 1, 0].numpy(), + var_fp32[0, ..., 1, 0].numpy(), + var_fp32[1, ..., 1].numpy(), + var_fp32[2, ...].numpy(), + var_fp32[2, 0, ...].numpy(), + var_fp32[2, 0, 1, ...].numpy(), + var_fp32[...].numpy(), + var_fp32[:, ..., 100].numpy(), + ] + + self.assertTrue(np.array_equal(var[0], np_fp32_value[..., 0])) + self.assertTrue(np.array_equal(var[1], np_fp32_value[..., 1, 0])) + self.assertTrue(np.array_equal(var[2], np_fp32_value[0, ..., 1, 0])) + self.assertTrue(np.array_equal(var[3], np_fp32_value[1, ..., 1])) + self.assertTrue(np.array_equal(var[4], np_fp32_value[2, ...])) + self.assertTrue(np.array_equal(var[5], np_fp32_value[2, 0, ...])) + self.assertTrue(np.array_equal(var[6], np_fp32_value[2, 0, 1, ...])) + self.assertTrue(np.array_equal(var[7], np_fp32_value[...])) + self.assertTrue(np.array_equal(var[8], np_fp32_value[:, ..., 100])) + + # test for int type + var = [ + var_int[..., 0].numpy(), + var_int[..., 1, 0].numpy(), + var_int[0, ..., 1, 0].numpy(), + var_int[1, ..., 1].numpy(), + var_int[2, ...].numpy(), + var_int[2, 0, ...].numpy(), + var_int[2, 0, 1, ...].numpy(), + var_int[...].numpy(), + var_int[:, ..., 100].numpy(), + ] + self.assertTrue(np.array_equal(var[0], np_int_value[..., 0])) + self.assertTrue(np.array_equal(var[1], np_int_value[..., 1, 0])) + self.assertTrue(np.array_equal(var[2], np_int_value[0, ..., 1, 0])) + self.assertTrue(np.array_equal(var[3], np_int_value[1, ..., 1])) + self.assertTrue(np.array_equal(var[4], np_int_value[2, ...])) + self.assertTrue(np.array_equal(var[5], np_int_value[2, 0, ...])) + self.assertTrue(np.array_equal(var[6], np_int_value[2, 0, 1, ...])) + self.assertTrue(np.array_equal(var[7], np_int_value[...])) + self.assertTrue(np.array_equal(var[8], np_int_value[:, ..., 100])) + def _test_for_var(self): np_value = np.random.random((30, 100, 100)).astype('float32') w = fluid.dygraph.to_variable(np_value) @@ -664,6 +717,7 @@ def test_slice(self): self._test_slice() self._test_slice_for_tensor_attr() self._test_for_var() + self._test_for_getitem_ellipsis_index() var = fluid.dygraph.to_variable(self.array) self.assertTrue(np.array_equal(var[1, :].numpy(), self.array[1, :])) From db60dc29f6c2430a03fb079a4ef780890a5f916f Mon Sep 17 00:00:00 2001 From: zyfncg Date: Wed, 21 Jul 2021 11:34:54 +0000 Subject: [PATCH 2/2] change some code style --- paddle/fluid/pybind/imperative.cc | 15 ++-- .../fluid/tests/unittests/test_var_base.py | 70 +++++++------------ 2 files changed, 34 insertions(+), 51 deletions(-) diff --git a/paddle/fluid/pybind/imperative.cc b/paddle/fluid/pybind/imperative.cc index 5a0e17822e85d..7b99c7df188f3 100644 --- a/paddle/fluid/pybind/imperative.cc +++ b/paddle/fluid/pybind/imperative.cc @@ -449,14 +449,7 @@ static void ParseIndexingSlice(framework::LoDTensor *tensor, PyObject *_index, "too many indices (%d) for tensor of dimension %d", size, rank)); for (int i = 0, dim = 0; i < size; ++i) { PyObject *slice_item = PyTuple_GetItem(index, i); - PADDLE_ENFORCE_EQ( - PyCheckInteger(slice_item) || PySlice_Check(slice_item) || - slice_item == Py_Ellipsis, - true, platform::errors::InvalidArgument( - "Currently, VarBase.__getitem__() only allows " - "indexing by Integers, Slices, Ellipsis, and tuples of " - "these types, but received %s in %dth slice item", - std::string(Py_TYPE(slice_item)->tp_name), i + 1)); + infer_flags->push_back(1); int dim_len = shape[dim]; if (PyCheckInteger(slice_item)) { @@ -498,6 +491,12 @@ static void ParseIndexingSlice(framework::LoDTensor *tensor, PyObject *_index, dim++; } else if (slice_item == Py_Ellipsis) { dim += rank - specified_dims; + } else { + PADDLE_THROW(platform::errors::InvalidArgument( + "Currently, VarBase.__getitem__() only allows " + "indexing by Integers, Slices, Ellipsis, and tuples of " + "these types, but received %s in %dth slice item", + std::string(Py_TYPE(slice_item)->tp_name), i + 1)); } } if (!PyTuple_Check(_index)) Py_DecRef(index); diff --git a/python/paddle/fluid/tests/unittests/test_var_base.py b/python/paddle/fluid/tests/unittests/test_var_base.py index e20ed21383082..87594f0f2d0be 100644 --- a/python/paddle/fluid/tests/unittests/test_var_base.py +++ b/python/paddle/fluid/tests/unittests/test_var_base.py @@ -660,50 +660,34 @@ def _test_for_getitem_ellipsis_index(self): var_fp32 = paddle.to_tensor(np_fp32_value) var_int = paddle.to_tensor(np_int_value) - # test for float32 - var = [ - var_fp32[..., 0].numpy(), - var_fp32[..., 1, 0].numpy(), - var_fp32[0, ..., 1, 0].numpy(), - var_fp32[1, ..., 1].numpy(), - var_fp32[2, ...].numpy(), - var_fp32[2, 0, ...].numpy(), - var_fp32[2, 0, 1, ...].numpy(), - var_fp32[...].numpy(), - var_fp32[:, ..., 100].numpy(), - ] + def assert_getitem_ellipsis_index(var_tensor, var_np): + var = [ + var_tensor[..., 0].numpy(), + var_tensor[..., 1, 0].numpy(), + var_tensor[0, ..., 1, 0].numpy(), + var_tensor[1, ..., 1].numpy(), + var_tensor[2, ...].numpy(), + var_tensor[2, 0, ...].numpy(), + var_tensor[2, 0, 1, ...].numpy(), + var_tensor[...].numpy(), + var_tensor[:, ..., 100].numpy(), + ] + + self.assertTrue(np.array_equal(var[0], var_np[..., 0])) + self.assertTrue(np.array_equal(var[1], var_np[..., 1, 0])) + self.assertTrue(np.array_equal(var[2], var_np[0, ..., 1, 0])) + self.assertTrue(np.array_equal(var[3], var_np[1, ..., 1])) + self.assertTrue(np.array_equal(var[4], var_np[2, ...])) + self.assertTrue(np.array_equal(var[5], var_np[2, 0, ...])) + self.assertTrue(np.array_equal(var[6], var_np[2, 0, 1, ...])) + self.assertTrue(np.array_equal(var[7], var_np[...])) + self.assertTrue(np.array_equal(var[8], var_np[:, ..., 100])) - self.assertTrue(np.array_equal(var[0], np_fp32_value[..., 0])) - self.assertTrue(np.array_equal(var[1], np_fp32_value[..., 1, 0])) - self.assertTrue(np.array_equal(var[2], np_fp32_value[0, ..., 1, 0])) - self.assertTrue(np.array_equal(var[3], np_fp32_value[1, ..., 1])) - self.assertTrue(np.array_equal(var[4], np_fp32_value[2, ...])) - self.assertTrue(np.array_equal(var[5], np_fp32_value[2, 0, ...])) - self.assertTrue(np.array_equal(var[6], np_fp32_value[2, 0, 1, ...])) - self.assertTrue(np.array_equal(var[7], np_fp32_value[...])) - self.assertTrue(np.array_equal(var[8], np_fp32_value[:, ..., 100])) - - # test for int type - var = [ - var_int[..., 0].numpy(), - var_int[..., 1, 0].numpy(), - var_int[0, ..., 1, 0].numpy(), - var_int[1, ..., 1].numpy(), - var_int[2, ...].numpy(), - var_int[2, 0, ...].numpy(), - var_int[2, 0, 1, ...].numpy(), - var_int[...].numpy(), - var_int[:, ..., 100].numpy(), - ] - self.assertTrue(np.array_equal(var[0], np_int_value[..., 0])) - self.assertTrue(np.array_equal(var[1], np_int_value[..., 1, 0])) - self.assertTrue(np.array_equal(var[2], np_int_value[0, ..., 1, 0])) - self.assertTrue(np.array_equal(var[3], np_int_value[1, ..., 1])) - self.assertTrue(np.array_equal(var[4], np_int_value[2, ...])) - self.assertTrue(np.array_equal(var[5], np_int_value[2, 0, ...])) - self.assertTrue(np.array_equal(var[6], np_int_value[2, 0, 1, ...])) - self.assertTrue(np.array_equal(var[7], np_int_value[...])) - self.assertTrue(np.array_equal(var[8], np_int_value[:, ..., 100])) + var_fp32 = paddle.to_tensor(np_fp32_value) + var_int = paddle.to_tensor(np_int_value) + + assert_getitem_ellipsis_index(var_fp32, np_fp32_value) + assert_getitem_ellipsis_index(var_int, np_int_value) def _test_for_var(self): np_value = np.random.random((30, 100, 100)).astype('float32')