From babf866366ca2a52472c8b940869d499182dcda4 Mon Sep 17 00:00:00 2001
From: RaphaelS1 <raphaelsonabend@gmail.com>
Date: Fri, 10 Feb 2023 11:37:30 +0000
Subject: [PATCH] add regr.logloss
---
DESCRIPTION | 2 +-
NAMESPACE | 1 +
NEWS.md | 4 ++
R/MeasureRegrLogloss.R | 75 ++++++++++++++++++-------
R/zzz.R | 2 +-
man-roxygen/regr_measure.R | 15 +++++
man/mlr_measures_regr.logloss.Rd | 73 ++++++++++++++++++++++++
tests/testthat/test_mlr_measures_regr.R | 7 +++
8 files changed, 157 insertions(+), 22 deletions(-)
create mode 100644 man-roxygen/regr_measure.R
create mode 100644 man/mlr_measures_regr.logloss.Rd
create mode 100644 tests/testthat/test_mlr_measures_regr.R
diff --git a/DESCRIPTION b/DESCRIPTION
index 3b4d6877d..c1a2087f1 100644
--- a/DESCRIPTION
+++ b/DESCRIPTION
@@ -1,6 +1,6 @@
Package: mlr3proba
Title: Probabilistic Supervised Learning for 'mlr3'
-Version: 0.5.0
+Version: 0.5.1
Authors@R:
c(person(given = "Raphael",
family = "Sonabend",
diff --git a/NAMESPACE b/NAMESPACE
index 090967098..7898d16bb 100644
--- a/NAMESPACE
+++ b/NAMESPACE
@@ -43,6 +43,7 @@ export(LearnerSurvKaplan)
export(LearnerSurvRpart)
export(MeasureDens)
export(MeasureDensLogloss)
+export(MeasureRegrLogloss)
export(MeasureSurv)
export(MeasureSurvAUC)
export(MeasureSurvCalibrationAlpha)
diff --git a/NEWS.md b/NEWS.md
index b82559d34..8010c5fe4 100644
--- a/NEWS.md
+++ b/NEWS.md
@@ -1,3 +1,7 @@
+# mlr3proba 0.5.1
+
+* Add `regr.logloss`
+
# mlr3proba 0.5.0
* Possibly small breaking change, renamed `PipeOpProbregrCompositor` to `PipeOpProbregr` and default distribution now `"Uniform"`.
diff --git a/R/MeasureRegrLogloss.R b/R/MeasureRegrLogloss.R
index e37d505ca..6c95f96b2 100644
--- a/R/MeasureRegrLogloss.R
+++ b/R/MeasureRegrLogloss.R
@@ -1,21 +1,56 @@
-#' MeasureRegrLogloss = R6::R6Class("MeasureRegrLogloss",
-#' inherit = MeasureRegr,
-#' public = list(
-#' #' @description Creates a new instance of this [R6][R6::R6Class] class.
-#' initialize = function() {
-#' super$initialize(
-#' id = "regr.logloss",
-#' range = c(0, Inf),
-#' minimize = TRUE,
-#' predict_type = "distr"
-#' # task_properties = "twoclass",
-#' # packages = "Metrics"
-#' )
-#' },
+#' @template regr_measure
+#' @templateVar title Log loss
+#' @templateVar inherit [MeasureRegr]
+#' @templateVar fullname MeasureRegrLogloss
+#' @templateVar pars eps = 1e-15
+#' @templateVar eps_par TRUE
#'
-#' .score = function(prediction, ...) {
-#' return(mean(-log(as.numeric(do.call(prediction$prob$pdf,
-#' as.list(prediction$truth))))))
-#' }
-#' )
-#' )
+#' @template param_eps
+#'
+#' @description
+#' Calculates the cross-entropy, or logarithmic (log), loss.
+#'
+#' The logloss, in the context of probabilistic predictions, is defined as the negative log
+#' probability density function, \eqn{f}, evaluated at the observed value, \eqn{y},
+#' \deqn{L(f, y) = -\log(f(y))}{L(f, y) = -log(f(y))}
+#'
+#' @export
+MeasureRegrLogloss = R6::R6Class("MeasureRegrLogloss",
+ inherit = MeasureRegr,
+ public = list(
+ #' @description
+ #' Creates a new instance of this [R6][R6::R6Class] class.
+ initialize = function() {
+ ps = ps(
+ eps = p_dbl(0, 1, default = 1e-15)
+ )
+ ps$values$eps = 1e-15
+
+ super$initialize(
+ id = "regr.logloss",
+ range = c(0, Inf),
+ minimize = TRUE,
+ predict_type = "distr",
+ man = "mlr3proba::mlr_measures_regr.logloss",
+ label = "Log Loss",
+ param_set = ps
+ )
+ }
+ ),
+
+ private = list(
+ .score = function(prediction, ...) {
+ distr = prediction$distr
+ truth = prediction$truth
+
+ if (inherits(distr, "Matdist")) {
+ pdf = diag(distr$pdf(truth))
+ } else {
+ pdf = as.numeric(distr$pdf(data = matrix(truth, nrow = 1)))
+ }
+
+ pdf[pdf == 0] = self$param_set$values$eps
+ mean(-log(pdf))
+ }
+ )
+)
diff --git a/R/zzz.R b/R/zzz.R
index 292694d6d..db2283598 100644
--- a/R/zzz.R
+++ b/R/zzz.R
@@ -104,7 +104,7 @@ register_mlr3 = function() {
x$add("dens.logloss", MeasureDensLogloss)
- # x$add("regr.logloss", MeasureRegrLogloss)
+ x$add("regr.logloss", MeasureRegrLogloss)
x$add("surv.graf", MeasureSurvGraf)
x$add("surv.brier", MeasureSurvGraf)
diff --git a/man-roxygen/regr_measure.R b/man-roxygen/regr_measure.R
new file mode 100644
index 000000000..03d51281c
--- /dev/null
+++ b/man-roxygen/regr_measure.R
@@ -0,0 +1,15 @@
+#' <% meas = get(fullname)$new() %>
+#' <% shortname = meas$id %>
+#'
+#' @include MeasureRegr.R
+#' @title <%=title%> Regression Measure
+#' @name <%= paste("mlr_measures", shortname, sep = "_")%>
+#'
+#' @section Meta Information:
+#' * Type: `"regr"`
+#' * Range: <%= format_range(meas$range) %>
+#' * Minimize: `<%=meas$minimize%>`
+#' * Required prediction: `<%=meas$predict_type%>`
+#'
+#' @family regression measures
+#' @template seealso_measure
diff --git a/man/mlr_measures_regr.logloss.Rd b/man/mlr_measures_regr.logloss.Rd
new file mode 100644
index 000000000..024228ae4
--- /dev/null
+++ b/man/mlr_measures_regr.logloss.Rd
@@ -0,0 +1,73 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/MeasureRegrLogloss.R
+\name{mlr_measures_regr.logloss}
+\alias{mlr_measures_regr.logloss}
+\alias{MeasureRegrLogloss}
+\title{Log loss Regression Measure}
+\description{
+Calculates the cross-entropy, or logarithmic (log), loss.
+
+The logloss, in the context of probabilistic predictions, is defined as the negative log
+probability density function, \eqn{f}, evaluated at the observed value, \eqn{y},
+\deqn{L(f, y) = -\log(f(y))}{L(f, y) = -log(f(y))}
+}
+\section{Meta Information}{
+
+\itemize{
+\item Type: \code{"regr"}
+\item Range: \eqn{[0, \infty)}{[0, Inf)}
+\item Minimize: \code{TRUE}
+\item Required prediction: \code{distr}
+}
+}
+
+\concept{regression measures}
+\section{Super classes}{
+\code{\link[mlr3:Measure]{mlr3::Measure}} -> \code{\link[mlr3:MeasureRegr]{mlr3::MeasureRegr}} -> \code{MeasureRegrLogloss}
+}
+\section{Methods}{
+\subsection{Public methods}{
+\itemize{
+\item \href{#method-MeasureRegrLogloss-new}{\code{MeasureRegrLogloss$new()}}
+\item \href{#method-MeasureRegrLogloss-clone}{\code{MeasureRegrLogloss$clone()}}
+}
+}
+\if{html}{\out{
+<details open><summary>Inherited methods</summary>
+<ul>
+<li><span class="pkg-link" data-pkg="mlr3" data-topic="Measure" data-id="aggregate"><a href='../../mlr3/html/Measure.html#method-Measure-aggregate'><code>mlr3::Measure$aggregate()</code></a></span></li>
+<li><span class="pkg-link" data-pkg="mlr3" data-topic="Measure" data-id="format"><a href='../../mlr3/html/Measure.html#method-Measure-format'><code>mlr3::Measure$format()</code></a></span></li>
+<li><span class="pkg-link" data-pkg="mlr3" data-topic="Measure" data-id="help"><a href='../../mlr3/html/Measure.html#method-Measure-help'><code>mlr3::Measure$help()</code></a></span></li>
+<li><span class="pkg-link" data-pkg="mlr3" data-topic="Measure" data-id="print"><a href='../../mlr3/html/Measure.html#method-Measure-print'><code>mlr3::Measure$print()</code></a></span></li>
+<li><span class="pkg-link" data-pkg="mlr3" data-topic="Measure" data-id="score"><a href='../../mlr3/html/Measure.html#method-Measure-score'><code>mlr3::Measure$score()</code></a></span></li>
+</ul>
+</details>
+}}
+\if{html}{\out{<hr>}}
+\if{html}{\out{<a id="method-MeasureRegrLogloss-new"></a>}}
+\if{latex}{\out{\hypertarget{method-MeasureRegrLogloss-new}{}}}
+\subsection{Method \code{new()}}{
+Creates a new instance of this \link[R6:R6Class]{R6} class.
+\subsection{Usage}{
+\if{html}{\out{<div class="r">}}\preformatted{MeasureRegrLogloss$new()}\if{html}{\out{</div>}}
+}
+
+}
+\if{html}{\out{<hr>}}
+\if{html}{\out{<a id="method-MeasureRegrLogloss-clone"></a>}}
+\if{latex}{\out{\hypertarget{method-MeasureRegrLogloss-clone}{}}}
+\subsection{Method \code{clone()}}{
+The objects of this class are cloneable with this method.
+\subsection{Usage}{
+\if{html}{\out{<div class="r">}}\preformatted{MeasureRegrLogloss$clone(deep = FALSE)}\if{html}{\out{</div>}}
+}
+
+\subsection{Arguments}{
+\if{html}{\out{<div class="arguments">}}
+\describe{
+\item{\code{deep}}{Whether to make a deep clone.}
+}
+\if{html}{\out{</div>}}
+}
+}
+}
diff --git a/tests/testthat/test_mlr_measures_regr.R b/tests/testthat/test_mlr_measures_regr.R
new file mode 100644
index 000000000..fb3971227
--- /dev/null
+++ b/tests/testthat/test_mlr_measures_regr.R
@@ -0,0 +1,7 @@
+library(mlr3pipelines)
+l = as_learner(ppl("probregr", learner = lrn("regr.featureless")))
+task = tsk("mtcars")
+p = l$train(task)$predict(task)
+
+expect_measure(msr("regr.logloss"))
+expect_numeric(p$score(msr("regr.logloss")))