diff --git a/DESCRIPTION b/DESCRIPTION index 3b4d6877d..c1a2087f1 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: mlr3proba Title: Probabilistic Supervised Learning for 'mlr3' -Version: 0.5.0 +Version: 0.5.1 Authors@R: c(person(given = "Raphael", family = "Sonabend", diff --git a/NAMESPACE b/NAMESPACE index 090967098..7898d16bb 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -43,6 +43,7 @@ export(LearnerSurvKaplan) export(LearnerSurvRpart) export(MeasureDens) export(MeasureDensLogloss) +export(MeasureRegrLogloss) export(MeasureSurv) export(MeasureSurvAUC) export(MeasureSurvCalibrationAlpha) diff --git a/NEWS.md b/NEWS.md index b82559d34..8010c5fe4 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,7 @@ +# mlr3proba 0.5.1 + +* Add `regr.logloss` + # mlr3proba 0.5.0 * Possibly small breaking change, renamed `PipeOpProbregrCompositor` to `PipeOpProbregr` and default distribution now `"Uniform"`. diff --git a/R/MeasureRegrLogloss.R b/R/MeasureRegrLogloss.R index e37d505ca..6c95f96b2 100644 --- a/R/MeasureRegrLogloss.R +++ b/R/MeasureRegrLogloss.R @@ -1,21 +1,56 @@ -#' MeasureRegrLogloss = R6::R6Class("MeasureRegrLogloss", -#' inherit = MeasureRegr, -#' public = list( -#' #' @description Creates a new instance of this [R6][R6::R6Class] class. -#' initialize = function() { -#' super$initialize( -#' id = "regr.logloss", -#' range = c(0, Inf), -#' minimize = TRUE, -#' predict_type = "distr" -#' # task_properties = "twoclass", -#' # packages = "Metrics" -#' ) -#' }, +#' @template regr_measure +#' @templateVar title Log loss +#' @templateVar inherit [MeasureRegr] +#' @templateVar fullname MeasureRegrLogloss +#' @templateVar pars eps = 1e-15 +#' @templateVar eps_par TRUE #' -#' .score = function(prediction, ...) { -#' return(mean(-log(as.numeric(do.call(prediction$prob$pdf, -#' as.list(prediction$truth)))))) -#' } -#' ) -#' ) +#' @template param_eps +#' +#' @description +#' Calculates the cross-entropy, or logarithmic (log), loss. +#' +#' The logloss, in the context of probabilistic predictions, is defined as the negative log +#' probability density function, \eqn{f}, evaluated at the observed value, \eqn{y}, +#' \deqn{L(f, y) = -\log(f(y))}{L(f, y) = -log(f(y))} +#' +#' @export +MeasureRegrLogloss = R6::R6Class("MeasureRegrLogloss", + inherit = MeasureRegr, + public = list( + #' @description + #' Creates a new instance of this [R6][R6::R6Class] class. + initialize = function() { + ps = ps( + eps = p_dbl(0, 1, default = 1e-15) + ) + ps$values$eps = 1e-15 + + super$initialize( + id = "regr.logloss", + range = c(0, Inf), + minimize = TRUE, + predict_type = "distr", + man = "mlr3proba::mlr_measures_regr.logloss", + label = "Log Loss", + param_set = ps + ) + } + ), + + private = list( + .score = function(prediction, ...) { + distr = prediction$distr + truth = prediction$truth + + if (inherits(distr, "Matdist")) { + pdf = diag(distr$pdf(truth)) + } else { + pdf = as.numeric(distr$pdf(data = matrix(truth, nrow = 1))) + } + + pdf[pdf == 0] = self$param_set$values$eps + mean(-log(pdf)) + } + ) +) diff --git a/R/zzz.R b/R/zzz.R index 292694d6d..db2283598 100644 --- a/R/zzz.R +++ b/R/zzz.R @@ -104,7 +104,7 @@ register_mlr3 = function() { x$add("dens.logloss", MeasureDensLogloss) - # x$add("regr.logloss", MeasureRegrLogloss) + x$add("regr.logloss", MeasureRegrLogloss) x$add("surv.graf", MeasureSurvGraf) x$add("surv.brier", MeasureSurvGraf) diff --git a/man-roxygen/regr_measure.R b/man-roxygen/regr_measure.R new file mode 100644 index 000000000..03d51281c --- /dev/null +++ b/man-roxygen/regr_measure.R @@ -0,0 +1,15 @@ +#' <% meas = get(fullname)$new() %> +#' <% shortname = meas$id %> +#' +#' @include MeasureRegr.R +#' @title <%=title%> Regression Measure +#' @name <%= paste("mlr_measures", shortname, sep = "_")%> +#' +#' @section Meta Information: +#' * Type: `"regr"` +#' * Range: <%= format_range(meas$range) %> +#' * Minimize: `<%=meas$minimize%>` +#' * Required prediction: `<%=meas$predict_type%>` +#' +#' @family regression measures +#' @template seealso_measure diff --git a/man/mlr_measures_regr.logloss.Rd b/man/mlr_measures_regr.logloss.Rd new file mode 100644 index 000000000..024228ae4 --- /dev/null +++ b/man/mlr_measures_regr.logloss.Rd @@ -0,0 +1,73 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/MeasureRegrLogloss.R +\name{mlr_measures_regr.logloss} +\alias{mlr_measures_regr.logloss} +\alias{MeasureRegrLogloss} +\title{Log loss Regression Measure} +\description{ +Calculates the cross-entropy, or logarithmic (log), loss. + +The logloss, in the context of probabilistic predictions, is defined as the negative log +probability density function, \eqn{f}, evaluated at the observed value, \eqn{y}, +\deqn{L(f, y) = -\log(f(y))}{L(f, y) = -log(f(y))} +} +\section{Meta Information}{ + +\itemize{ +\item Type: \code{"regr"} +\item Range: \eqn{[0, \infty)}{[0, Inf)} +\item Minimize: \code{TRUE} +\item Required prediction: \code{distr} +} +} + +\concept{regression measures} +\section{Super classes}{ +\code{\link[mlr3:Measure]{mlr3::Measure}} -> \code{\link[mlr3:MeasureRegr]{mlr3::MeasureRegr}} -> \code{MeasureRegrLogloss} +} +\section{Methods}{ +\subsection{Public methods}{ +\itemize{ +\item \href{#method-MeasureRegrLogloss-new}{\code{MeasureRegrLogloss$new()}} +\item \href{#method-MeasureRegrLogloss-clone}{\code{MeasureRegrLogloss$clone()}} +} +} +\if{html}{\out{ +
Inherited methods + +
+}} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-MeasureRegrLogloss-new}{}}} +\subsection{Method \code{new()}}{ +Creates a new instance of this \link[R6:R6Class]{R6} class. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{MeasureRegrLogloss$new()}\if{html}{\out{
}} +} + +} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-MeasureRegrLogloss-clone}{}}} +\subsection{Method \code{clone()}}{ +The objects of this class are cloneable with this method. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{MeasureRegrLogloss$clone(deep = FALSE)}\if{html}{\out{
}} +} + +\subsection{Arguments}{ +\if{html}{\out{
}} +\describe{ +\item{\code{deep}}{Whether to make a deep clone.} +} +\if{html}{\out{
}} +} +} +} diff --git a/tests/testthat/test_mlr_measures_regr.R b/tests/testthat/test_mlr_measures_regr.R new file mode 100644 index 000000000..fb3971227 --- /dev/null +++ b/tests/testthat/test_mlr_measures_regr.R @@ -0,0 +1,7 @@ +library(mlr3pipelines) +l = as_learner(ppl("probregr", learner = lrn("regr.featureless"))) +task = tsk("mtcars") +p = l$train(task)$predict(task) + +expect_measure(msr("regr.logloss")) +expect_numeric(p$score(msr("regr.logloss")))