mlr3learners_lightgbm_tuning.Rmd

---
title: "mlr3learners.lightgbm: Hyperparameter Tuning Example"
date: "`r Sys.Date()`"
output:
  rmarkdown::html_vignette:
    keep_md: true
vignette: >
  %\VignetteIndexEntry{mlr3learners_lightgbm_tuning}
  %\VignetteEncoding{UTF-8}
  %\VignetteEngine{knitr::rmarkdown}
editor_options: 
  chunk_output_type: console
---

```{r include = FALSE}
knitr::opts_chunk$set(
  collapse = TRUE,
  comment = "#>",
  eval = FALSE
)
```

```{r setup}
library(mlr3)
library(mlr3learners.lightgbm)
library(paradox)
library(mlbench)
```

# Load the dataset 

```{r}
data("PimaIndiansDiabetes2")
dataset = data.table::as.data.table(PimaIndiansDiabetes2)
target_col = "diabetes"

vec = setdiff(colnames(dataset), target_col)

dataset = cbind(
  dataset[, c(target_col), with = F],
  lightgbm::lgb.convert_with_rules(dataset[, vec, with = F])[[1]]
)

task = mlr3::TaskClassif$new(
  id = "pima",
  backend = dataset,
  target = target_col,
  positive = "pos"
)
```

```{r}
set.seed(17)
split = list(
  train_index = sample(seq_len(task$nrow), size = 0.7 * task$nrow)
)
split$test_index = setdiff(seq_len(task$nrow), split$train_index)
```

# Instantiate the lightgbm learner 

Initially, the `classif.lgbpy` class needs to be instantiated: 

```{r}
learner = mlr3::lrn("classif.lightgbm", objective = "binary")
```

# Configure the learner 

We will here switch off the parallelization of the lightgbm learner by setting the parameter `num_threads = 1L`. Instead, we will later parallelize the resampling using the `future` package, as recommended by the [mlr3 team](https://mlr3book.mlr-org.com/parallelization.html#parallelization).  

```{r}
learner$param_set$values = mlr3misc::insert_named(
  learner$param_set$values,
    list(
      "learning_rate" = 0.1,
      "bagging_freq" = 5L,
      "seed" = 17L,
      "metric" = "auc",
      "num_threads" = 1
  )
)

tune_ps = ParamSet$new(list(
  ParamDbl$new("bagging_fraction", lower = 0.4, upper = 1),
  ParamInt$new("min_data_in_leaf", lower = 5, upper = 30)
))

# design_points
design = paradox::generate_design_grid(
  tune_ps,
  param_resolutions = c(
    bagging_fraction = 2,
    min_data_in_leaf = 5
  ))

# shuffle order of design
set.seed(17)
shuffle = sample(seq_len(nrow(design$data)), size = nrow(design$data))
design$data = design$data[shuffle, ]
```

# Create the resampling strategy and the measure 

```{r}
resampling = mlr3::rsmp("cv", folds = 5)
measure = mlr3::msr("classif.auc")
```


# Create the tuner 

```{r}
# grid_resolution = 2
# tuner = mlr3tuning::tnr("grid_search", resolution = grid_resolution, batch_size = 1)

tuner = mlr3tuning::tnr("design_points", design = design$data, batch_size = 1)
```

# Create the terminator 

```{r}
# using a specific number of iterations
# n_iterations = (grid_resolution ^ tune_ps$length)

n_iterations = nrow(design$data)

n_iterations
terminator = mlr3tuning::term("evals", n_evals = n_iterations)
```

# Instantiate the AutoTuner instance 

```{r}
at = mlr3tuning::AutoTuner$new(
  learner = learner,
  resampling = resampling,
  measures = measure,
  tune_ps = tune_ps,
  terminator = terminator,
  tuner = tuner
)
at
```

# Train the tuner 

```{r}
future::plan("multisession")
set.seed(17)
at$train(task, row_ids = split$train_index)
future::plan("sequential")
```

# Evaluate the best model 

```{r}
at$tuning_result
best = at$tuning_instance$best()
best$score(mlr3::msr("classif.auc"))
```

```{r}
mlr3viz::autoplot(at$tuning_instance$best(), type = "roc")
```

```{r}
at$tuning_instance$archive(unnest = "params")[, c("bagging_fraction", "min_data_in_leaf", "classif.auc")]
```

# Best parameters 

```{r}
at$tuning_instance$result$params
```

# Importance 

```{r}
importance = at$learner$importance()
```

# Predict test data with best model

```{r}
predictions = at$predict(task, row_ids = split$test_index)
head(predictions$response)
```

```{r}
predictions$confusion
```

```{r}
predictions$score(mlr3::msr("classif.logloss"))
predictions$score(mlr3::msr("classif.auc"))
```

```{r}
mlr3viz::autoplot(predictions)
```

### ROC 

```{r}
mlr3viz::autoplot(predictions, type = "roc")
```