From 961faeefb3d223c350d911614b30d0ade83474ef Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Tue, 16 Jan 2024 17:12:22 +1100 Subject: [PATCH 1/6] add prediction type check for Explicit strategy --- src/strategies/explicit.jl | 18 +++++++++++++++--- test/strategies/explicit.jl | 15 +++++++++++++++ test/tuned_models.jl | 2 +- 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/src/strategies/explicit.jl b/src/strategies/explicit.jl index 65a218b..445eae3 100644 --- a/src/strategies/explicit.jl +++ b/src/strategies/explicit.jl @@ -1,15 +1,21 @@ +const ERR_INCONSISTENT_PREDICTION_TYPE = ArgumentError( + "Not all models to be evaluated have the same prediction type. Inspect "* + "these with `prediction_type(model)`. " +) + mutable struct Explicit <: TuningStrategy end struct ExplicitState{R, N} range::R # a model-generating iterator next::N # to hold output of `iterate(range)` + prediction_type::Symbol end ExplictState(r::R, n::N) where {R,N} = ExplicitState{R, Union{Nothing, N}}(r, n) function MLJTuning.setup(tuning::Explicit, model, range, n, verbosity) next = iterate(range) - return ExplicitState(range, next) + return ExplicitState(range, next, MLJBase.prediction_type(model)) end # models! returns as many models as possible but no more than `n_remaining`: @@ -20,11 +26,15 @@ function MLJTuning.models(tuning::Explicit, n_remaining, verbosity) - range, next = state.range, state.next + range, next, prediction_type = state.range, state.next, state.prediction_type + + check = ==(prediction_type) next === nothing && return nothing, state m, s = next + check(MLJBase.prediction_type(m)) || throw(ERR_INCONSISTENT_PREDICTION_TYPE) + models = Any[m, ] # types not known until run-time next = iterate(range, s) @@ -33,12 +43,14 @@ function MLJTuning.models(tuning::Explicit, while i < n_remaining next === nothing && break m, s = next + check(MLJBase.prediction_type(m)) || + throw(ERR_INCONSISTENT_PREDICTION_TYPE) push!(models, m) i += 1 next = iterate(range, s) end - new_state = ExplicitState(range, next) + new_state = ExplicitState(range, next, prediction_type) return models, new_state diff --git a/test/strategies/explicit.jl b/test/strategies/explicit.jl index c2e8dc7..aa09715 100644 --- a/test/strategies/explicit.jl +++ b/test/strategies/explicit.jl @@ -1,6 +1,7 @@ good = KNNClassifier(K=2) bad = KNNClassifier(K=10) ugly = ConstantClassifier() +evil = DeterministicConstantClassifier() r = [good, bad, ugly] @@ -44,4 +45,18 @@ X, y = make_blobs(rng=rng) @test_throws ArgumentError TunedModel(; models=[dcc, dcc]) end +r = [good, bad, evil, ugly] + +@testset "inconsistent prediction types" begin + tmodel = TunedModel( + models=r, + resampling = Holdout(fraction_train=0.6), + measure=LogLoss(), + ) + @test_throws( + MLJTuning.ERR_INCONSISTENT_PREDICTION_TYPE, + MLJBase.fit(tmodel, 0, X, y), + ) +end + true diff --git a/test/tuned_models.jl b/test/tuned_models.jl index 381efef..3780ea9 100644 --- a/test/tuned_models.jl +++ b/test/tuned_models.jl @@ -79,7 +79,7 @@ results = [(evaluate(model, X, y, tm = TunedModel( models=r, resampling=CV(nfolds=2), - measures=cross_entropy + measures=cross_entropy, ) @test_logs((:error, r"Problem"), (:info, r""), From a90975ba032599831b5ecac35796cb32d9cdd858 Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Fri, 19 Jan 2024 09:27:06 +1300 Subject: [PATCH 2/6] fix how measures are displayed on plot axes --- src/plotrecipes.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plotrecipes.jl b/src/plotrecipes.jl index fd9b7ed..e7e5e30 100644 --- a/src/plotrecipes.jl +++ b/src/plotrecipes.jl @@ -1,6 +1,6 @@ @recipe function f(mach::MLJBase.Machine{<:EitherTunedModel}) rep = report(mach) - measurement = string(typeof(rep.best_history_entry.measure[1])) + measurement = repr(rep.best_history_entry.measure[1]) r = rep.plotting z = r.measurements X = r.parameter_values From f475d80fa43b8923a080535035bcf73bb90ed2f3 Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Fri, 19 Jan 2024 09:58:10 +1300 Subject: [PATCH 3/6] add codecov config --- .github/codecov.yml | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .github/codecov.yml diff --git a/.github/codecov.yml b/.github/codecov.yml new file mode 100644 index 0000000..ed9d9f1 --- /dev/null +++ b/.github/codecov.yml @@ -0,0 +1,8 @@ +coverage: + status: + project: + default: + threshold: 0.5% + patch: + default: + target: 80% \ No newline at end of file From accc6b368392e0e73f2612639e9ed39b25b90997 Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Wed, 24 Jan 2024 08:21:53 +1300 Subject: [PATCH 4/6] all different prediction types but issue warning --- src/strategies/explicit.jl | 34 ++++++++++++++++++++-------------- src/tuned_models.jl | 4 +++- test/strategies/explicit.jl | 34 ++++++++++++++++++++++++++++++---- 3 files changed, 53 insertions(+), 19 deletions(-) diff --git a/src/strategies/explicit.jl b/src/strategies/explicit.jl index 445eae3..1cd7dab 100644 --- a/src/strategies/explicit.jl +++ b/src/strategies/explicit.jl @@ -1,21 +1,22 @@ -const ERR_INCONSISTENT_PREDICTION_TYPE = ArgumentError( - "Not all models to be evaluated have the same prediction type. Inspect "* - "these with `prediction_type(model)`. " -) +const WARN_INCONSISTENT_PREDICTION_TYPE = + "Not all models to be evaluated have the same prediction type, and this may "* + "cause problems for some measures. For example, a probabilistic metric "* + "like `log_loss` cannot be applied to a model making point (deterministic) "* + "predictions. Inspect the prediction type with "* + "`prediction_type(model)`. " mutable struct Explicit <: TuningStrategy end struct ExplicitState{R, N} range::R # a model-generating iterator - next::N # to hold output of `iterate(range)` + next::N # to hold output of `iterate(range)` prediction_type::Symbol + user_warned::Bool end -ExplictState(r::R, n::N) where {R,N} = ExplicitState{R, Union{Nothing, N}}(r, n) - function MLJTuning.setup(tuning::Explicit, model, range, n, verbosity) next = iterate(range) - return ExplicitState(range, next, MLJBase.prediction_type(model)) + return ExplicitState(range, next, MLJBase.prediction_type(model), false) end # models! returns as many models as possible but no more than `n_remaining`: @@ -26,14 +27,20 @@ function MLJTuning.models(tuning::Explicit, n_remaining, verbosity) - range, next, prediction_type = state.range, state.next, state.prediction_type + range, next, prediction_type, user_warned = + state.range, state.next, state.prediction_type, state.user_warned - check = ==(prediction_type) + function check(m) + if !user_warned && verbosity > -1 && MLJBase.prediction_type(m) != prediction_type + @warn WARN_INCONSISTENT_PREDICTION_TYPE + user_warned = true + end + end next === nothing && return nothing, state m, s = next - check(MLJBase.prediction_type(m)) || throw(ERR_INCONSISTENT_PREDICTION_TYPE) + check(m) models = Any[m, ] # types not known until run-time @@ -43,14 +50,13 @@ function MLJTuning.models(tuning::Explicit, while i < n_remaining next === nothing && break m, s = next - check(MLJBase.prediction_type(m)) || - throw(ERR_INCONSISTENT_PREDICTION_TYPE) + check(m) push!(models, m) i += 1 next = iterate(range, s) end - new_state = ExplicitState(range, next, prediction_type) + new_state = ExplicitState(range, next, prediction_type, user_warned) return models, new_state diff --git a/src/tuned_models.jl b/src/tuned_models.jl index 514a279..cd8328d 100644 --- a/src/tuned_models.jl +++ b/src/tuned_models.jl @@ -431,9 +431,11 @@ function event!(metamodel, state) model = _first(metamodel) metadata = _last(metamodel) + force = typeof(resampling_machine.model.model) != + typeof(model) resampling_machine.model.model = model verb = (verbosity >= 2 ? verbosity - 3 : verbosity - 1) - fit!(resampling_machine, verbosity=verb) + fit!(resampling_machine; verbosity=verb, force) E = evaluate(resampling_machine) entry0 = (model = model, measure = E.measure, diff --git a/test/strategies/explicit.jl b/test/strategies/explicit.jl index aa09715..b504642 100644 --- a/test/strategies/explicit.jl +++ b/test/strategies/explicit.jl @@ -46,16 +46,42 @@ X, y = make_blobs(rng=rng) end r = [good, bad, evil, ugly] +r = [good, bad, ugly] +r= [evil,] @testset "inconsistent prediction types" begin + # case where different predictions types is actually okay (but still + # a warning is issued): tmodel = TunedModel( models=r, - resampling = Holdout(fraction_train=0.6), - measure=LogLoss(), + resampling = Holdout(), + measure=accuracy, ) - @test_throws( - MLJTuning.ERR_INCONSISTENT_PREDICTION_TYPE, + @test_logs( + (:warn, MLJTuning.WARN_INCONSISTENT_PREDICTION_TYPE), MLJBase.fit(tmodel, 0, X, y), + ); + + # verbosity = -1 suppresses the warning: + @test_logs( + MLJBase.fit(tmodel, -1, X, y), + ); + + # case where there really is a problem with different prediction types: + tmodel = TunedModel( + models=r, + resampling = Holdout(), + measure=log_loss, + ) + @test_logs( + (:warn, MLJTuning.WARN_INCONSISTENT_PREDICTION_TYPE), + (:error,), + (:info,), + (:info,), + @test_throws( + ArgumentError, # indicates the problem is with incompatible measure + MLJBase.fit(tmodel, 0, X, y), + ) ) end From b509a71a7ef40966b0e2e3da0b43286bd88a46bc Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Wed, 24 Jan 2024 08:22:26 +1300 Subject: [PATCH 5/6] bump 0.8.1 --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 0540983..d2b54c2 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "MLJTuning" uuid = "03970b2e-30c4-11ea-3135-d1576263f10f" authors = ["Anthony D. Blaom "] -version = "0.8.0" +version = "0.8.1" [deps] ComputationalResources = "ed09eef8-17a6-5b46-8889-db040fac31e3" From 53700b3faabdf698a6ed23d51b759946d0403ef4 Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Wed, 24 Jan 2024 08:41:35 +1300 Subject: [PATCH 6/6] rm extraneous debugging code --- test/strategies/explicit.jl | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/strategies/explicit.jl b/test/strategies/explicit.jl index b504642..13b5c97 100644 --- a/test/strategies/explicit.jl +++ b/test/strategies/explicit.jl @@ -46,8 +46,6 @@ X, y = make_blobs(rng=rng) end r = [good, bad, evil, ugly] -r = [good, bad, ugly] -r= [evil,] @testset "inconsistent prediction types" begin # case where different predictions types is actually okay (but still