Add/organize reduce- and stats-related methods

Add: -`Base.mean(X::NullableArray)` -`Base.mean(X::NullableArray, w::WeightVec)` -`Base.varm(X::NullableArray, m)` -`Base.varzm(X::NullableArray)` -`Base.var(X::NullableArray) -`Base.stdm{T}(X::NullableArray, m::Union{T, Nullable{T}}) -`Base.std(X::NullableArray) Organization: -move above stats-related methods to `src/statistics.jl` Also includes some minor bug fixes in `src/mapreduce.jl`
JuliaStats · Jul 13, 2015 · 76309a0 · 76309a0
1 parent 0512336
commit 76309a0
Show file tree

Hide file tree

Showing 5 changed files with 441 additions and 28 deletions.
diff --git a/perf/mapreduce.jl b/perf/mapreduce.jl
@@ -15,30 +15,213 @@ f{T<:Number}(x::Nullable{T}) = Nullable(5 * x.value, x.isnull)
 
 #-----------------------------------------------------------------------------#
 
-function profile_mapreduce(A, X, D)
+function profile_reduce_methods()
+ A = rand(5_000_000)
+ B = rand(Bool, 5_000_000)
+ X = NullableArray(A)
+ Y = NullableArray(A, B)
+ D = DataArray(A)
+ E = DataArray(A, B)
+
+ profile_mapreduce(A, X, Y, D, E)
+ println()
+ profile_reduce(A, X, Y, D, E)
+ println()
+
+ for method in (
+ sum,
+ prod,
+ minimum,
+ maximum,
+ )
+ (method)(A)
+ (method)(X)
+ (method)(D)
+ println("Method: $method(A) (0 missing entries)")
+ print(" for Array{Float64}: ")
+ @time((method)(A))
+ print(" for NullableArray{Float64}: ")
+ @time((method)(X))
+ print(" for DataArray{Float64}: ")
+ @time((method)(D))
+
+ (method)(f, A)
+ (method)(f, X)
+ (method)(f, D)
+ println("Method: $method(f, A) (0 missing entries)")
+ print(" for Array{Float64}: ")
+ @time((method)(f, A))
+ print(" for NullableArray{Float64}: ")
+ @time((method)(f, X))
+ print(" for DataArray{Float64}: ")
+ @time((method)(f, D))
+ end
+ println()
+
+ for method in (
+ sum,
+ prod,
+ minimum,
+ maximum,
+ )
+ (method)(Y)
+ println("Method: $method(A) (~half missing entries, skip=false)")
+ print(" for NullableArray{Float64}: ")
+ @time((method)(Y))
+ (method)(E)
+ print(" for DataArray{Float64}: ")
+ @time((method)(E))
+
+ (method)(f, Y)
+ println("Method: $method(f, A) (~half missing entries, skip=false)")
+ print(" for NullableArray{Float64}: ")
+ @time((method)(f, Y))
+ if in(method, (sum, prod))
+ (method)(f, E)
+ print(" for DataArray{Float64}: ")
+ @time((method)(f, E))
+ else
+ println(" $method(f, A::DataArray) currently incurs error")
+ end
+ end
+ println()
+
+ for method in (
+ sum,
+ prod,
+ minimum,
+ maximum,
+ )
+ (method)(Y, skipnull=true)
+ println("Method: $method(A) (~half missing entries, skip=true)")
+ print(" for NullableArray{Float64}: ")
+ @time((method)(Y, skipnull=true))
+ (method)(E, skipna=true)
+ print(" for DataArray{Float64}: ")
+ @time((method)(E, skipna=true))
+
+ (method)(f, Y, skipnull=true)
+ println("Method: $method(f, A) (~half missing entries, skip=true)")
+ print(" for NullableArray{Float64}: ")
+ @time((method)(f, Y, skipnull=true))
+ (method)(f, E, skipna=true)
+ print(" for DataArray{Float64}: ")
+ @time((method)(f, E, skipna=true))
+ end
+ println()
+
+ for method in (
+ sumabs,
+ sumabs2
+ )
+ (method)(A)
+ (method)(X)
+ (method)(D)
+ println("Method: $method(A) (0 missing entries)")
+ print(" for Array{Float64}: ")
+ @time((method)(A))
+ print(" for NullableArray{Float64}: ")
+ @time((method)(X))
+ print(" for DataArray{Float64}: ")
+ @time((method)(D))
+
+ (method)(f, A)
+ (method)(f, X)
+ (method)(f, D)
+ println("Method: $method(f, A) (0 missing entries)")
+ print(" for Array{Float64}: ")
+ @time((method)(f, A))
+ print(" for NullableArray{Float64}: ")
+ @time((method)(f, X))
+ print(" for DataArray{Float64}: ")
+ @time((method)(f, D))
+ end
+
+ for method in (
+ mean,
+ var,
+ )
+ (method)(A)
+ (method)(X)
+ (method)(D)
+ println("Method: $method(A) (0 missing entries)")
+ print(" for Array{Float64}: ")
+ @time((method)(A))
+ print(" for NullableArray{Float64}: ")
+ @time((method)(X))
+ print(" for DataArray{Float64}: ")
+ @time((method)(D))
+
+ (method)(f, A)
+ (method)(f, X)
+ (method)(f, D)
+ println("Method: $method(f, A) (0 missing entries)")
+ print(" for Array{Float64}: ")
+ @time((method)(f, A))
+ print(" for NullableArray{Float64}: ")
+ @time((method)(f, X))
+ print(" for DataArray{Float64}: ")
+ @time((method)(f, D))
+ end
+end
+
+
+function profile_mapreduce(A, X, Y, D, E)
+ println("Method: mapreduce(f, op, A) (0 missing entries)")
  mapreduce(f, Base.(:+), A)
- mapreduce(f, Base.(:+), X)
- mapreduce(f, Base.(:+), D)
- println("Method: mapreduce(f, op, A)")
  print(" for Array{Float64}: ")
  @time(mapreduce(f, Base.(:+), A))
+ mapreduce(f, Base.(:+), X)
  print(" for NullableArray{Float64}: ")
  @time(mapreduce(f, Base.(:+), X))
+ mapreduce(f, Base.(:+), D)
  print(" for DataArray{Float64}: ")
  @time(mapreduce(f, Base.(:+), D))
+
+ println("Method: mapreduce(f, op, A) (~half missing entries, skip=false)")
+ mapreduce(f, Base.(:+), Y)
+ print(" for NullableArray{Float64}: ")
+ @time(mapreduce(f, Base.(:+), Y))
+ mapreduce(f, Base.(:+), E)
+ print(" for DataArray{Float64}: ")
+ @time(mapreduce(f, Base.(:+), E))
+
+ println("Method: mapreduce(f, op, A) (~half missing entries, skip=true)")
+ mapreduce(f, Base.(:+), Y, skipnull=true)
+ print(" for NullableArray{Float64}: ")
+ @time(mapreduce(f, Base.(:+), Y, skipnull=true))
+ mapreduce(f, Base.(:+), E, skipna=true)
+ print(" for DataArray{Float64}: ")
+ @time(mapreduce(f, Base.(:+), E, skipna=true))
 end
 
-function profile_reduce(A, X, D)
+function profile_reduce(A, X, Y, D, E)
+ println("Method: reduce(f, op, A) (0 missing entries)")
  reduce(Base.(:+), A)
- reduce(Base.(:+), X)
- reduce(Base.(:+), D)
- println("Method: reduce(op, A)")
  print(" for Array{Float64}: ")
  @time(reduce(Base.(:+), A))
+ reduce(Base.(:+), X)
  print(" for NullableArray{Float64}: ")
  @time(reduce(Base.(:+), X))
+ reduce(Base.(:+), D)
  print(" for DataArray{Float64}: ")
  @time(reduce(Base.(:+), D))
+
+ println("Method: reduce(f, op, A) (~half missing entries, skip=false)")
+ reduce(Base.(:+), Y)
+ print(" for NullableArray{Float64}: ")
+ @time(reduce(Base.(:+), Y))
+ reduce(Base.(:+), E)
+ print(" for DataArray{Float64}: ")
+ @time(reduce(Base.(:+), E))
+
+ println("Method: reduce(f, op, A) (~half missing entries, skip=true)")
+ reduce(Base.(:+), Y, skipnull=true)
+ print(" for NullableArray{Float64}: ")
+ @time(reduce(Base.(:+), Y, skipnull=true))
+ reduce(Base.(:+), E, skipna=true)
+ print(" for DataArray{Float64}: ")
+ @time(reduce(Base.(:+), E, skipna=true))
 end
 
 function profile_sum1(A, X, D)

diff --git a/perf/statistics.jl b/perf/statistics.jl
@@ -0,0 +1,119 @@
+using NullableArrays
+using DataArrays
+using StatsBase
+
+srand(1)
+N = 5_000_000
+
+function profile_stats_methods()
+ A = rand(N)
+ B = rand(Bool, N)
+ X = NullableArray(A)
+ Y = NullableArray(A, B)
+ D = DataArray(A)
+ E = DataArray(A, B)
+
+ profile_mean(A, X, D, Y, E)
+ profile_var(A, X, D, Y, E)
+ nothing
+end
+
+function profile_mean(A, X, D, Y, E)
+ W = WeightVec(rand(N))
+
+ mean(A)
+ println("Method: mean(A) (0 missing entries)")
+ print(" for Array{Float64}: ")
+ @time(mean(A))
+ mean(X)
+ print(" for NullableArray{Float64}: ")
+ @time(mean(X))
+ mean(D)
+ print(" for DataArray{Float64}: ")
+ @time(mean(D))
+ println()
+
+ mean(Y, skipnull=false)
+ println("Method: mean(A) (~half missing entries, skip=false)")
+ print(" for NullableArray{Float64}: ")
+ @time(mean(Y, skipnull=false))
+ mean(E, skipna=false)
+ print(" for DataArray{Float64}: ")
+ @time(mean(E, skipna=false))
+ println()
+
+ mean(Y, skipnull=true)
+ println("Method: mean(A) (~half missing entries, skip=true)")
+ print(" for NullableArray{Float64}: ")
+ @time(mean(Y, skipnull=true))
+ mean(E, skipna=true)
+ print(" for DataArray{Float64}: ")
+ @time(mean(E, skipna=true))
+ println()
+
+ mean(A, W)
+ println("Method: mean(A, w::WeightVec{W, V}) (0 missing entries, V<:Array)")
+ print(" for Array{Float64}: ")
+ @time(mean(A, W))
+ mean(X, W)
+ print(" for NullableArray{Float64}: ")
+ @time(mean(X, W))
+ mean(D, W)
+ print(" for DataArray{Float64}: ")
+ @time(mean(D, W))
+ println()
+
+ println("Method: mean(A, W::WeightVec) (~half missing entries, skip=false)")
+ mean(Y, W, skipnull=false)
+ print(" for NullableArray{Float64}: ")
+ @time(mean(Y, W, skipnull=false))
+ mean(E, W, skipna=false)
+ print(" for DataArray{Float64}: ")
+ @time(mean(E, W, skipna=false))
+ println()
+
+ println("Method: mean(A, W::WeightVec) (~half missing entries, skip=true)")
+ mean(Y, W, skipnull=true)
+ print(" for NullableArray{Float64}: ")
+ @time(mean(Y, W, skipnull=true))
+ mean(E, W, skipna=true)
+ print(" for DataArray{Float64}: ")
+ @time(mean(E, W, skipna=true))
+ println()
+end
+
+function profile_var(A, X, D, Y, E)
+ mu = mean(A)
+ mu2 = mean(X, skipnull=true)
+
+ varm(A, mu)
+ println("Method: varm(A, mu) (0 missing entries)")
+ print(" for Array{Float64}: ")
+ @time(varm(A, mu))
+ println(" ", varm(A, mu))
+ varm(X, mu)
+ print(" for NullableArray{Float64}: ")
+ @time(varm(X, mu))
+ varm(D, mu)
+ print(" for DataArray{Float64}: ")
+ @time(varm(D, mu))
+ println()
+
+ varm(Y, mu; skipnull=false)
+ println("Method: varm(A, mu) (~half missing entries, skip=false)")
+ print(" for NullableArray{Float64}: ")
+ @time(varm(Y, mu; skipnull=false))
+ varm(E, mu; skipna=false)
+ print(" for DataArray{Float64}: ")
+ @time(varm(E, mu; skipna=false))
+ println()
+
+ varm(Y, mu; skipnull=true)
+ println("Method: varm(A, mu) (~half missing entries, skip=true)")
+ print(" for NullableArray{Float64}: ")
+ @time(varm(Y, mu; skipnull=true))
+ varm(E, mu; skipna=true)
+ print(" for DataArray{Float64}: ")
+ @time(varm(E, mu; skipna=true))
+ println()
+end
diff --git a/src/NullableArrays.jl b/src/NullableArrays.jl
@@ -27,4 +27,5 @@ module NullableArrays
  include("operators.jl")
  include("broadcast.jl")
  include("mapreduce.jl")
+ include("statistics.jl")
 end