From 90925e00130610e05d74a5555441c009f1baa488 Mon Sep 17 00:00:00 2001 From: Carlo Lucibello Date: Wed, 30 Dec 2020 10:07:14 +0100 Subject: [PATCH 1/7] add pixel_shuffle --- src/NNlib.jl | 4 ++++ src/misc.jl | 20 ++++++++++++++++++++ test/misc.jl | 10 ++++++++++ test/runtests.jl | 4 ++++ 4 files changed, 38 insertions(+) create mode 100644 src/misc.jl create mode 100644 test/misc.jl diff --git a/src/NNlib.jl b/src/NNlib.jl index 64b13d309..785dc31cd 100644 --- a/src/NNlib.jl +++ b/src/NNlib.jl @@ -26,6 +26,10 @@ end include("activations.jl") include("softmax.jl") + +include("misc.jl") +export pixel_shuffle + include("batched/batchedmul.jl") include("gemm.jl") include("conv.jl") diff --git a/src/misc.jl b/src/misc.jl new file mode 100644 index 000000000..2806d4576 --- /dev/null +++ b/src/misc.jl @@ -0,0 +1,20 @@ +""" + pixel_shuffle(x, r) + +Pixel shuffling operation. `r` is the scale factor for shuffling. +The operation converts an input of size [W,H,r²C,N] to size [rW,rH,C,N] +Used extensively in super-resolution networks to upsample +towards high resolution features. + +Reference : https://arxiv.org/pdf/1609.05158.pdf +""" +function pixel_shuffle(x::AbstractArray{T, 4}, r::Integer) where T <:Number + w, h, c, n = size(x) + @assert c % r^2 == 0 + c_out = c ÷ r^2 + w_out = w * r + h_out = h * r + x = reshape(x, (w, h, r, r, c_out, n)) + x = permutedims(x, (3, 1, 4, 2, 5, 6)) + return reshape(x, (w_out, h_out, c_out, n)) +end \ No newline at end of file diff --git a/test/misc.jl b/test/misc.jl new file mode 100644 index 000000000..780812f54 --- /dev/null +++ b/test/misc.jl @@ -0,0 +1,10 @@ +@testset "pixel_shuffle" begin + x = reshape(1:16, (2,2,4,1)) + y = pixel_shuffle(x, 2) + @test size(x) == (4, 4, 1, 1) + + x = reshape(1:4*3*27*2, (4,3,27,2)) + y = pixel_shuffle(x, 3) + @test size(x) == (12, 9, 3, 2) +end + diff --git a/test/runtests.jl b/test/runtests.jl index 480a88d96..0ef0676fc 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -35,3 +35,7 @@ end @testset "Softmax" begin include("softmax.jl") end + +@testset "Misc Stuff" begin + include("misc.jl") +end From 99ce7db0ad6a452ef10d0c74c34115f1cadc6515 Mon Sep 17 00:00:00 2001 From: Carlo Lucibello Date: Wed, 30 Dec 2020 22:58:35 +0100 Subject: [PATCH 2/7] fix tests --- src/NNlib.jl | 3 --- src/misc.jl | 2 ++ test/misc.jl | 32 ++++++++++++++++++++++++++++---- 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/src/NNlib.jl b/src/NNlib.jl index 785dc31cd..5f4851fd7 100644 --- a/src/NNlib.jl +++ b/src/NNlib.jl @@ -26,10 +26,7 @@ end include("activations.jl") include("softmax.jl") - include("misc.jl") -export pixel_shuffle - include("batched/batchedmul.jl") include("gemm.jl") include("conv.jl") diff --git a/src/misc.jl b/src/misc.jl index 2806d4576..1cc148327 100644 --- a/src/misc.jl +++ b/src/misc.jl @@ -1,3 +1,5 @@ +export pixel_shuffle + """ pixel_shuffle(x, r) diff --git a/test/misc.jl b/test/misc.jl index 780812f54..c2f21ac91 100644 --- a/test/misc.jl +++ b/test/misc.jl @@ -1,10 +1,34 @@ @testset "pixel_shuffle" begin - x = reshape(1:16, (2,2,4,1)) + x = reshape(1:16, (2, 2, 4, 1)) + # [:, :, 1, 1] = + # 1 3 + # 2 4 + # [:, :, 2, 1] = + # 5 7 + # 6 8 + # [:, :, 3, 1] = + # 9 11 + # 10 12 + # [:, :, 4, 1] = + # 13 15 + # 14 16 + + y_true = [1 9 3 11 + 5 13 7 15 + 2 10 4 12 + 6 14 8 16][:,:,:,:] + y = pixel_shuffle(x, 2) - @test size(x) == (4, 4, 1, 1) + @test size(y) == (4, 4, 1, 1) + @test y_true == y x = reshape(1:4*3*27*2, (4,3,27,2)) y = pixel_shuffle(x, 3) - @test size(x) == (12, 9, 3, 2) + @test size(y) == (12, 9, 3, 2) + x1 = x[:,:,:,[1]] + x2 = x[:,:,:,[2]] + y1 = pixel_shuffle(x1, 3) + y2 = pixel_shuffle(x2, 3) + @test cat(y1, y2, dims=4) == y + gradtest(x -> pixel_shuffle(x, 2), rand(3,3,8,2)) end - From 9070bd4d9a3db3460a037f92a1b825ce3b1e09f2 Mon Sep 17 00:00:00 2001 From: Carlo Lucibello Date: Wed, 30 Dec 2020 23:00:35 +0100 Subject: [PATCH 3/7] cleanup --- src/misc.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/misc.jl b/src/misc.jl index 1cc148327..8ae211468 100644 --- a/src/misc.jl +++ b/src/misc.jl @@ -19,4 +19,4 @@ function pixel_shuffle(x::AbstractArray{T, 4}, r::Integer) where T <:Number x = reshape(x, (w, h, r, r, c_out, n)) x = permutedims(x, (3, 1, 4, 2, 5, 6)) return reshape(x, (w_out, h_out, c_out, n)) -end \ No newline at end of file +end From fe63ea3fbe983bdb53904a573baf7e698866a99f Mon Sep 17 00:00:00 2001 From: Carlo Lucibello Date: Wed, 30 Dec 2020 23:19:22 +0100 Subject: [PATCH 4/7] error on nightly --- .github/workflows/ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e446f7c9a..a4ba07f0c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,13 +48,13 @@ jobs: steps: - uses: actions/checkout@v2 - uses: julia-actions/setup-julia@v1 - # `allow-failure` not available yet https://github.com/actions/toolkit/issues/399 + ## `allow-failure` not available yet https://github.com/actions/toolkit/issues/399 continue-on-error: ${{ matrix.julia-version == 'nightly' }} with: version: ${{ matrix.version }} arch: ${{ matrix.arch }} - uses: actions/cache@v1 - continue-on-error: ${{ matrix.julia-version == 'nightly' }} + # continue-on-error: ${{ matrix.julia-version == 'nightly' }} env: cache-name: cache-artifacts with: @@ -65,11 +65,11 @@ jobs: ${{ runner.os }}-test- ${{ runner.os }}- - uses: julia-actions/julia-buildpkg@v1 - continue-on-error: ${{ matrix.julia-version == 'nightly' }} + # continue-on-error: ${{ matrix.julia-version == 'nightly' }} - uses: julia-actions/julia-runtest@v1 - continue-on-error: ${{ matrix.julia-version == 'nightly' }} + # continue-on-error: ${{ matrix.julia-version == 'nightly' }} - uses: codecov/codecov-action@v1 - continue-on-error: ${{ matrix.version == 'nightly' }} + # continue-on-error: ${{ matrix.version == 'nightly' }} with: file: lcov.info From e0541f8ea6d2d88e2fdb32be622f02738c85bf9f Mon Sep 17 00:00:00 2001 From: Carlo Lucibello Date: Thu, 31 Dec 2020 13:38:59 +0100 Subject: [PATCH 5/7] fix --- src/misc.jl | 4 ++-- test/misc.jl | 22 ++++++++++++++++++++-- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/src/misc.jl b/src/misc.jl index 8ae211468..157010703 100644 --- a/src/misc.jl +++ b/src/misc.jl @@ -3,14 +3,14 @@ export pixel_shuffle """ pixel_shuffle(x, r) -Pixel shuffling operation. `r` is the scale factor for shuffling. +Pixel shuffling operation. `r` is the upscale factor for shuffling. The operation converts an input of size [W,H,r²C,N] to size [rW,rH,C,N] Used extensively in super-resolution networks to upsample towards high resolution features. Reference : https://arxiv.org/pdf/1609.05158.pdf """ -function pixel_shuffle(x::AbstractArray{T, 4}, r::Integer) where T <:Number +function pixel_shuffle(x::AbstractArray{T, 4}, r::Integer) where T w, h, c, n = size(x) @assert c % r^2 == 0 c_out = c ÷ r^2 diff --git a/test/misc.jl b/test/misc.jl index c2f21ac91..1ba1f63e5 100644 --- a/test/misc.jl +++ b/test/misc.jl @@ -19,16 +19,34 @@ 6 14 8 16][:,:,:,:] y = pixel_shuffle(x, 2) - @test size(y) == (4, 4, 1, 1) + @test size(y) == size(y_true) + @test y_true == y + + x = reshape(1:32, (2, 2, 8, 1)) + y_true = zeros(Int, 4, 4, 2, 1) + y_true[:,:,1,1] = [ 1 9 3 11 + 5 13 7 15 + 2 10 4 12 + 6 14 8 16 ] + + y_true[:,:,2,1] = [ 17 25 19 27 + 21 29 23 31 + 18 26 20 28 + 22 30 24 32] + + y = pixel_shuffle(x, 2) + @test size(y) == size(y_true) @test y_true == y x = reshape(1:4*3*27*2, (4,3,27,2)) y = pixel_shuffle(x, 3) @test size(y) == (12, 9, 3, 2) + # batch dimension is preserved x1 = x[:,:,:,[1]] x2 = x[:,:,:,[2]] y1 = pixel_shuffle(x1, 3) y2 = pixel_shuffle(x2, 3) @test cat(y1, y2, dims=4) == y - gradtest(x -> pixel_shuffle(x, 2), rand(3,3,8,2)) + + gradtest(x -> pixel_shuffle(x, 2), rand(3, 3, 8, 2)) end From d68039df9f91bc2520de0f7568eefa7f101c4007 Mon Sep 17 00:00:00 2001 From: Carlo Lucibello Date: Thu, 31 Dec 2020 13:41:33 +0100 Subject: [PATCH 6/7] fix --- test/misc.jl | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/test/misc.jl b/test/misc.jl index 1ba1f63e5..7fb420cce 100644 --- a/test/misc.jl +++ b/test/misc.jl @@ -24,15 +24,15 @@ x = reshape(1:32, (2, 2, 8, 1)) y_true = zeros(Int, 4, 4, 2, 1) - y_true[:,:,1,1] = [ 1 9 3 11 - 5 13 7 15 - 2 10 4 12 - 6 14 8 16 ] + y_true[:,:,1,1] .= [ 1 9 3 11 + 5 13 7 15 + 2 10 4 12 + 6 14 8 16 ] - y_true[:,:,2,1] = [ 17 25 19 27 - 21 29 23 31 - 18 26 20 28 - 22 30 24 32] + y_true[:,:,2,1] .= [ 17 25 19 27 + 21 29 23 31 + 18 26 20 28 + 22 30 24 32] y = pixel_shuffle(x, 2) @test size(y) == size(y_true) From dcb015fc62edc2b63fe46d90b7bbdc5dcc99fe69 Mon Sep 17 00:00:00 2001 From: Carlo Lucibello Date: Thu, 31 Dec 2020 17:00:38 +0100 Subject: [PATCH 7/7] arbitrary dimension --- src/misc.jl | 21 ++++++++++++--------- test/misc.jl | 13 ++++++++++++- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/src/misc.jl b/src/misc.jl index 157010703..02c9fa53d 100644 --- a/src/misc.jl +++ b/src/misc.jl @@ -10,13 +10,16 @@ towards high resolution features. Reference : https://arxiv.org/pdf/1609.05158.pdf """ -function pixel_shuffle(x::AbstractArray{T, 4}, r::Integer) where T - w, h, c, n = size(x) - @assert c % r^2 == 0 - c_out = c ÷ r^2 - w_out = w * r - h_out = h * r - x = reshape(x, (w, h, r, r, c_out, n)) - x = permutedims(x, (3, 1, 4, 2, 5, 6)) - return reshape(x, (w_out, h_out, c_out, n)) +function pixel_shuffle(x::AbstractArray, r::Integer) + @assert ndims(x) > 2 + d = ndims(x) - 2 + sizein = size(x)[1:d] + cin, n = size(x, d+1), size(x, d+2) + @assert cin % r^d == 0 + cout = cin ÷ r^d + # x = reshape(x, sizein..., fill(r, d)..., cout, n) # bug https://github.com/FluxML/Zygote.jl/issues/866 + x = reshape(x, sizein..., ntuple(i->r, d)..., cout, n) + perm = [d+1:2d 1:d]' |> vec # = [d+1, 1, d+2, 2, ..., 2d, d] + x = permutedims(x, (perm..., 2d+1, 2d+2)) + return reshape(x, ((r .* sizein)..., cout, n)) end diff --git a/test/misc.jl b/test/misc.jl index 7fb420cce..2f2d724da 100644 --- a/test/misc.jl +++ b/test/misc.jl @@ -48,5 +48,16 @@ y2 = pixel_shuffle(x2, 3) @test cat(y1, y2, dims=4) == y - gradtest(x -> pixel_shuffle(x, 2), rand(3, 3, 8, 2)) + for d in [1, 2, 3] + r = rand(1:5) + n = rand(1:5) + c = rand(1:5) + insize = rand(1:5, d) + x = rand(insize..., r^d*c, n) + + y = pixel_shuffle(x, r) + @test size(y) == ((r .* insize)..., c, n) + + gradtest(x -> pixel_shuffle(x, r), x) + end end