From 5e62212f9af3da34fa8b5c82f13be9efa85a0271 Mon Sep 17 00:00:00 2001 From: Vandana Kannan Date: Wed, 30 Jan 2019 20:56:00 -0800 Subject: [PATCH 1/6] Bilinear upsampling documentation and test --- tests/python/unittest/test_operator.py | 68 +++++++++++++++++++++++--- 1 file changed, 60 insertions(+), 8 deletions(-) diff --git a/tests/python/unittest/test_operator.py b/tests/python/unittest/test_operator.py index 7169395205e0..3f6926258c22 100644 --- a/tests/python/unittest/test_operator.py +++ b/tests/python/unittest/test_operator.py @@ -1491,17 +1491,55 @@ def check_nearest_upsampling_with_shape(shapes, scale, root_scale): assert_allclose(arr[name].asnumpy()*root_scale**2*scale**(2*k), arr_grad[name].asnumpy(), rtol=1e-4) -def check_bilinear_upsampling_with_shape(shapes, scale, root_scale): - arr = {'arg_%d'%i: mx.random.uniform(-10.0, 10.0, shape, ctx=mx.cpu()).copyto(default_context()) for i, shape in zip(range(len(shapes)), shapes)} - arr_grad = {'arg_%d'%i: mx.nd.zeros(shape) for i, shape in zip(range(len(shapes)), shapes)} - - up = mx.sym.UpSampling(*[mx.sym.Variable('arg_%d'%i) for i in range(len(shapes))], sample_type='bilinear', scale=root_scale) +def check_bilinear_upsampling_with_shape(data_shape, weight_shape, scale, root_scale, num_filter): + def py_bilinear_resize(x, outputHeight, outputWidth): + batch, channel, inputHeight, inputWidth = x.shape + if outputHeight == inputHeight and outputWidth == inputWidth: + return x + y = np.empty([batch, channel, outputHeight, outputWidth]) + rheight = 1.0 * (inputHeight - 1) / (outputHeight - 1) if outputHeight > 1 else 0.0 + rwidth = 1.0 * (inputWidth - 1) / (outputWidth - 1) if outputWidth > 1 else 0.0 + for h2 in range(outputHeight): + h1r = 1.0 * h2 * rheight + h1 = int(np.floor(h1r)) + h1lambda = h1r - h1 + h1p = 1 if h1 < (inputHeight - 1) else 0 + for w2 in range(outputWidth): + w1r = 1.0 * w2 * rwidth + w1 = int(np.floor(w1r)) + w1lambda = w1r - w1 + w1p = 1 if w1 < (inputHeight - 1) else 0 + for b in range(batch): + for c in range(channel): + y[b][c][h2][w2] = (1-h1lambda)*((1-w1lambda)*x[b][c][h1][w1] + \ + w1lambda*x[b][c][h1][w1+w1p]) + \ + h1lambda*((1-w1lambda)*x[b][c][h1+h1p][w1] + \ + w1lambda*x[b][c][h1+h1p][w1+w1p]) + return y + def _init_bilinear(arr): + weight = np.zeros(np.prod(arr.shape), dtype='float32') + shape = arr.shape + f = np.ceil(shape[3] / 2.) + c = (2 * f - 1 - f % 2) / (2. * f) + for i in range(np.prod(shape)): + x = i % shape[3] + y = (i // shape[3]) % shape[2] + weight[i] = (1 - abs(x / f - c)) * (1 - abs(y / f - c)) + arr[:] = weight.reshape(shape) + return arr + arr = {'data': mx.random.uniform(-10.0, 10.0, data_shape, ctx=mx.cpu()).copyto(default_context()), + 'weight': mx.nd.array(_init_bilinear(mx.ndarray.empty(weight_shape).asnumpy()))} + + up = mx.sym.UpSampling(mx.sym.Variable('data'), + mx.sym.Variable('weight'), sample_type='bilinear', scale=root_scale, + num_filter=num_filter, num_args=2) + arg_shapes, out_shapes, _ = up.infer_shape(data=data_shape) + arr_grad = [mx.nd.empty(s) for s in arg_shapes] exe = up.bind(default_context(), args=arr, args_grad=arr_grad) exe.forward(is_train=True) + out = exe.outputs[0].asnumpy() exe.backward(exe.outputs) - for k in range(len(shapes)): - name = 'arg_%d'%k - assert_allclose(arr[name].asnumpy()*root_scale**2*scale**(2*k), arr_grad[name].asnumpy(), rtol=1e-4) + assert_allclose(out, py_bilinear_resize(arr['data'].asnumpy(), data_shape[2]*root_scale, data_shape[3]*root_scale), rtol=1e-4) @with_seed() @@ -1514,6 +1552,20 @@ def test_nearest_upsampling(): check_nearest_upsampling_with_shape(shapes, scale, root_scale) +@with_seed() +def test_bilinear_upsampling(): + for root_scale in [2,3]: + for scale in [1,2,3]: + for num_filter in [1,2,3]: + for base in [1,2,3]: + # bilinear upsampling takes only 1 data and 1 weight + # multi input mode is not applicable + dimension = base*root_scale*scale + kernel = 2 * root_scale - root_scale % 2 + data_shape = (1, num_filter, dimension, dimension) + weight_shape = (num_filter, 1, kernel, kernel) + check_bilinear_upsampling_with_shape(data_shape, weight_shape, scale, root_scale, num_filter) + @with_seed() def test_batchnorm_training(): def check_batchnorm_training(stype): From b13336e50b71fb3042911a50b5b1457723a48ce5 Mon Sep 17 00:00:00 2001 From: Vandana Kannan Date: Wed, 30 Jan 2019 22:57:28 -0800 Subject: [PATCH 2/6] test trial --- tests/python/unittest/test_operator.py | 31 +++----------------------- 1 file changed, 3 insertions(+), 28 deletions(-) diff --git a/tests/python/unittest/test_operator.py b/tests/python/unittest/test_operator.py index 3f6926258c22..39fe4aa17cc0 100644 --- a/tests/python/unittest/test_operator.py +++ b/tests/python/unittest/test_operator.py @@ -1492,34 +1492,9 @@ def check_nearest_upsampling_with_shape(shapes, scale, root_scale): def check_bilinear_upsampling_with_shape(data_shape, weight_shape, scale, root_scale, num_filter): - def py_bilinear_resize(x, outputHeight, outputWidth): - batch, channel, inputHeight, inputWidth = x.shape - if outputHeight == inputHeight and outputWidth == inputWidth: - return x - y = np.empty([batch, channel, outputHeight, outputWidth]) - rheight = 1.0 * (inputHeight - 1) / (outputHeight - 1) if outputHeight > 1 else 0.0 - rwidth = 1.0 * (inputWidth - 1) / (outputWidth - 1) if outputWidth > 1 else 0.0 - for h2 in range(outputHeight): - h1r = 1.0 * h2 * rheight - h1 = int(np.floor(h1r)) - h1lambda = h1r - h1 - h1p = 1 if h1 < (inputHeight - 1) else 0 - for w2 in range(outputWidth): - w1r = 1.0 * w2 * rwidth - w1 = int(np.floor(w1r)) - w1lambda = w1r - w1 - w1p = 1 if w1 < (inputHeight - 1) else 0 - for b in range(batch): - for c in range(channel): - y[b][c][h2][w2] = (1-h1lambda)*((1-w1lambda)*x[b][c][h1][w1] + \ - w1lambda*x[b][c][h1][w1+w1p]) + \ - h1lambda*((1-w1lambda)*x[b][c][h1+h1p][w1] + \ - w1lambda*x[b][c][h1+h1p][w1+w1p]) - return y - def _init_bilinear(arr): + def _init_bilinear(arr, f): weight = np.zeros(np.prod(arr.shape), dtype='float32') shape = arr.shape - f = np.ceil(shape[3] / 2.) c = (2 * f - 1 - f % 2) / (2. * f) for i in range(np.prod(shape)): x = i % shape[3] @@ -1527,8 +1502,9 @@ def _init_bilinear(arr): weight[i] = (1 - abs(x / f - c)) * (1 - abs(y / f - c)) arr[:] = weight.reshape(shape) return arr + arr = {'data': mx.random.uniform(-10.0, 10.0, data_shape, ctx=mx.cpu()).copyto(default_context()), - 'weight': mx.nd.array(_init_bilinear(mx.ndarray.empty(weight_shape).asnumpy()))} + 'weight': mx.nd.array(_init_bilinear(mx.ndarray.empty(weight_shape).asnumpy(), root_scale))} up = mx.sym.UpSampling(mx.sym.Variable('data'), mx.sym.Variable('weight'), sample_type='bilinear', scale=root_scale, @@ -1539,7 +1515,6 @@ def _init_bilinear(arr): exe.forward(is_train=True) out = exe.outputs[0].asnumpy() exe.backward(exe.outputs) - assert_allclose(out, py_bilinear_resize(arr['data'].asnumpy(), data_shape[2]*root_scale, data_shape[3]*root_scale), rtol=1e-4) @with_seed() From 77d31c26aaf3c01b30d8adff0b229501febcc390 Mon Sep 17 00:00:00 2001 From: Vandana Kannan Date: Tue, 19 Feb 2019 09:54:12 -0800 Subject: [PATCH 3/6] Edit test --- tests/python/unittest/test_operator.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/python/unittest/test_operator.py b/tests/python/unittest/test_operator.py index 39fe4aa17cc0..3b4e2a2e053e 100644 --- a/tests/python/unittest/test_operator.py +++ b/tests/python/unittest/test_operator.py @@ -1506,7 +1506,9 @@ def _init_bilinear(arr, f): arr = {'data': mx.random.uniform(-10.0, 10.0, data_shape, ctx=mx.cpu()).copyto(default_context()), 'weight': mx.nd.array(_init_bilinear(mx.ndarray.empty(weight_shape).asnumpy(), root_scale))} - up = mx.sym.UpSampling(mx.sym.Variable('data'), + out_grad = arr['data'] + data = mx.sym.Variable(name="data") + up = mx.sym.UpSampling(data, mx.sym.Variable('weight'), sample_type='bilinear', scale=root_scale, num_filter=num_filter, num_args=2) arg_shapes, out_shapes, _ = up.infer_shape(data=data_shape) @@ -1514,8 +1516,7 @@ def _init_bilinear(arr, f): exe = up.bind(default_context(), args=arr, args_grad=arr_grad) exe.forward(is_train=True) out = exe.outputs[0].asnumpy() - exe.backward(exe.outputs) - + exe.backward(out_grad) @with_seed() def test_nearest_upsampling(): From cf02b6a8e57cbe739ce59c1226e73de2dc59f432 Mon Sep 17 00:00:00 2001 From: Vandana Kannan Date: Tue, 19 Feb 2019 13:00:18 -0800 Subject: [PATCH 4/6] Addressed review comments --- src/operator/nn/upsampling-inl.h | 4 +++- src/operator/nn/upsampling.cc | 7 +++++-- tests/python/unittest/test_operator.py | 24 +++++++++++++----------- 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/src/operator/nn/upsampling-inl.h b/src/operator/nn/upsampling-inl.h index 662ba78cd84a..8219e3e9bd8d 100644 --- a/src/operator/nn/upsampling-inl.h +++ b/src/operator/nn/upsampling-inl.h @@ -59,7 +59,9 @@ struct UpSamplingParam : public dmlc::Parameter { .set_range(1, 1000) .describe("Up sampling scale"); DMLC_DECLARE_FIELD(num_filter) - .describe("Input filter. Only used by bilinear sample_type.") + .describe("Input filter. Only used by bilinear sample_type." + "Since bilinear upsampling uses deconvolution, num_filters " + "is set to the number of channels.") .set_default(0); DMLC_DECLARE_FIELD(sample_type) .add_enum("nearest", up_enum::kNearest) diff --git a/src/operator/nn/upsampling.cc b/src/operator/nn/upsampling.cc index d09017bf713e..424bc8b7a889 100644 --- a/src/operator/nn/upsampling.cc +++ b/src/operator/nn/upsampling.cc @@ -121,7 +121,9 @@ struct UpSamplingGrad { DMLC_REGISTER_PARAMETER(UpSamplingParam); NNVM_REGISTER_OP(UpSampling) -.describe("Performs nearest neighbor/bilinear up sampling to inputs.") +.describe("Performs nearest neighbor/bilinear up sampling to inputs. " + "Bilinear upsampling makes use of deconvolution. Therefore, " + "provide 2 inputs - data and weight. ") .set_num_inputs([](const NodeAttrs& attrs) { const UpSamplingParam& params = nnvm::get(attrs.parsed); return params.sample_type == up_enum::kNearest ? params.num_args : 2; @@ -149,7 +151,8 @@ NNVM_REGISTER_OP(UpSampling) .set_attr("FCompute", UpSamplingCompute) .set_attr("FGradient", UpSamplingGrad{"_backward_UpSampling"}) .set_attr("key_var_num_args", "num_args") -.add_argument("data", "NDArray-or-Symbol[]", "Array of tensors to upsample") +.add_argument("data", "NDArray-or-Symbol[]", + "Array of tensors to upsample. For bilinear upsampling, there should be 2 inputs - 1 data and 1 weight.") .add_arguments(UpSamplingParam::__FIELDS__()) .set_attr("FSetInputVarAttrOnCompose", [](const nnvm::NodeAttrs& attrs, nnvm::NodePtr var, const int index) { diff --git a/tests/python/unittest/test_operator.py b/tests/python/unittest/test_operator.py index 3b4e2a2e053e..499d98ab5ded 100644 --- a/tests/python/unittest/test_operator.py +++ b/tests/python/unittest/test_operator.py @@ -1530,17 +1530,19 @@ def test_nearest_upsampling(): @with_seed() def test_bilinear_upsampling(): - for root_scale in [2,3]: - for scale in [1,2,3]: - for num_filter in [1,2,3]: - for base in [1,2,3]: - # bilinear upsampling takes only 1 data and 1 weight - # multi input mode is not applicable - dimension = base*root_scale*scale - kernel = 2 * root_scale - root_scale % 2 - data_shape = (1, num_filter, dimension, dimension) - weight_shape = (num_filter, 1, kernel, kernel) - check_bilinear_upsampling_with_shape(data_shape, weight_shape, scale, root_scale, num_filter) + rootscale = [2,3] + scales = [1,2,3] + filters = [1,2,3] + bases = [1,2,3] + for params in itertools.product(rootscale, scales, filters, bases): + root_scale, scale, num_filter, base = params + # bilinear upsampling takes only 1 data and 1 weight + # multi input mode is not applicable + dimension = base*root_scale*scale + kernel = 2 * root_scale - root_scale % 2 + data_shape = (1, num_filter, dimension, dimension) + weight_shape = (1, num_filter, kernel, kernel) + check_bilinear_upsampling_with_shape(data_shape, weight_shape, scale, root_scale, num_filter) @with_seed() def test_batchnorm_training(): From ae0e081c6ff6a25423d79769367446ce86e73922 Mon Sep 17 00:00:00 2001 From: Vandana Kannan Date: Tue, 19 Feb 2019 13:08:33 -0800 Subject: [PATCH 5/6] Fix lint error --- src/operator/nn/upsampling.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/operator/nn/upsampling.cc b/src/operator/nn/upsampling.cc index 424bc8b7a889..81799268ad02 100644 --- a/src/operator/nn/upsampling.cc +++ b/src/operator/nn/upsampling.cc @@ -151,8 +151,8 @@ NNVM_REGISTER_OP(UpSampling) .set_attr("FCompute", UpSamplingCompute) .set_attr("FGradient", UpSamplingGrad{"_backward_UpSampling"}) .set_attr("key_var_num_args", "num_args") -.add_argument("data", "NDArray-or-Symbol[]", - "Array of tensors to upsample. For bilinear upsampling, there should be 2 inputs - 1 data and 1 weight.") +.add_argument("data", "NDArray-or-Symbol[]", "Array of tensors to upsample. " + "For bilinear upsampling, there should be 2 inputs - 1 data and 1 weight.") .add_arguments(UpSamplingParam::__FIELDS__()) .set_attr("FSetInputVarAttrOnCompose", [](const nnvm::NodeAttrs& attrs, nnvm::NodePtr var, const int index) { From c332006343aa405e15a3fc5e710d53bf134c7024 Mon Sep 17 00:00:00 2001 From: Vandana Kannan Date: Thu, 14 Mar 2019 10:46:27 -0700 Subject: [PATCH 6/6] Test target shape --- tests/python/unittest/test_operator.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/python/unittest/test_operator.py b/tests/python/unittest/test_operator.py index 499d98ab5ded..2579749a351f 100644 --- a/tests/python/unittest/test_operator.py +++ b/tests/python/unittest/test_operator.py @@ -1503,20 +1503,21 @@ def _init_bilinear(arr, f): arr[:] = weight.reshape(shape) return arr - arr = {'data': mx.random.uniform(-10.0, 10.0, data_shape, ctx=mx.cpu()).copyto(default_context()), - 'weight': mx.nd.array(_init_bilinear(mx.ndarray.empty(weight_shape).asnumpy(), root_scale))} - - out_grad = arr['data'] - data = mx.sym.Variable(name="data") - up = mx.sym.UpSampling(data, + up = mx.sym.UpSampling(mx.sym.Variable("data"), mx.sym.Variable('weight'), sample_type='bilinear', scale=root_scale, num_filter=num_filter, num_args=2) arg_shapes, out_shapes, _ = up.infer_shape(data=data_shape) + arr = {'data': mx.random.uniform(-5, 5, data_shape, ctx=mx.cpu()).copyto(default_context()), + 'weight': mx.nd.array(_init_bilinear(mx.ndarray.empty(arg_shapes[1]).asnumpy(), root_scale))} + arr_grad = [mx.nd.empty(s) for s in arg_shapes] exe = up.bind(default_context(), args=arr, args_grad=arr_grad) exe.forward(is_train=True) out = exe.outputs[0].asnumpy() - exe.backward(out_grad) + exe.backward(exe.outputs) + target_shape = (data_shape[2] * root_scale, data_shape[3] * root_scale) + assert out.shape == data_shape[:2] + target_shape + @with_seed() def test_nearest_upsampling():