From d049430e4b7533b69313b1da3676e9b57df6ead7 Mon Sep 17 00:00:00 2001 From: mayer79 Date: Fri, 12 Feb 2021 08:34:39 +0100 Subject: [PATCH 01/82] added intro vignette --- R-package/DESCRIPTION | 4 + R-package/vignettes/.gitignore | 3 + R-package/vignettes/basic_walkthrough.Rmd | 119 ++++++++++++++++++++++ R-package/vignettes/biblio.bib | 18 ++++ 4 files changed, 144 insertions(+) create mode 100644 R-package/vignettes/.gitignore create mode 100644 R-package/vignettes/basic_walkthrough.Rmd create mode 100644 R-package/vignettes/biblio.bib diff --git a/R-package/DESCRIPTION b/R-package/DESCRIPTION index 6e61f459c99b..3fa5dcc435ae 100755 --- a/R-package/DESCRIPTION +++ b/R-package/DESCRIPTION @@ -42,7 +42,11 @@ URL: https://github.com/Microsoft/LightGBM BugReports: https://github.com/Microsoft/LightGBM/issues NeedsCompilation: yes Biarch: true +VignetteBuilder: + knitr Suggests: + knitr, + rmarkdown, processx, testthat Depends: diff --git a/R-package/vignettes/.gitignore b/R-package/vignettes/.gitignore new file mode 100644 index 000000000000..a9d285d9d065 --- /dev/null +++ b/R-package/vignettes/.gitignore @@ -0,0 +1,3 @@ +*.html +*.R +*.model diff --git a/R-package/vignettes/basic_walkthrough.Rmd b/R-package/vignettes/basic_walkthrough.Rmd new file mode 100644 index 000000000000..577e99aaa448 --- /dev/null +++ b/R-package/vignettes/basic_walkthrough.Rmd @@ -0,0 +1,119 @@ +--- +title: + "Basic Walkthrough" +description: > + This vignette illustrates the basics of LightGBM. +bibliography: "biblio.bib" +link-citations: true +output: rmarkdown::html_vignette +vignette: > + %\VignetteIndexEntry{Basic Walkthrough} + %\VignetteEngine{knitr::rmarkdown} + %\VignetteEncoding{UTF-8} +--- + + +```{r, include = FALSE} +knitr::opts_chunk$set( + collapse = TRUE, + comment = "#>", + warning = FALSE, + message = FALSE +) +``` + +## Introduction + +Welcome to the world of [LightGBM](https://lightgbm.readthedocs.io/en/latest/), a hightly efficient gradient boosting implementation [@ke2017]. + +```{r setup} +library(lightgbm) +``` + +This vignette will guide you through its basic usage. For illustration, we will use a subset of the `bank` dataset [@moro2014] to predict whether a client has subscribed a term deposit. As input features, we will use two numeric predictors, "age" and "balance". + +## The dataset + +Let's have a look at the dataset first. + +```{r} +data(bank, package = "lightgbm") + +bank[1L:5L, c("y", "age", "balance")] + +# Distribution of the response +table(bank$y) +``` + +## Training the model + +Building a strong LightGBM model typically involves a little bit of hyperparameter tuning. To keep things as simple as possible, we skip this step here and directly jump into fitting a first model. + +The R package of LightGBM offers two ways to do so: + +- `lgb.train`: This is the main training logic. It offers full flexibility but requires data created by the `lgb.Dataset` function. +- `lightgbm`: Simpler, but less flexible. Data can be passed without having to bother with `lgb.Dataset`. + +Let's illustrate both logics: + +### Using the `lightgbm` function + +In a first step, we will make sure that all data is numeric. Then, we will fit the model by the `lightgbm` function and check how the distribution of the predictions looks like. + +```{r} +# Numeric response and feature matrix +y <- as.numeric(bank$y == "yes") +X <- data.matrix(bank[, c("age", "balance")]) + +# Train +fit <- lightgbm( + data = X + , label = y + , num_leaves = 4L + , learning_rate = 1.0 + , nrounds = 10L + , objective = "binary" + , verbose = -1L +) + +# Result +summary(predict(fit, X)) +``` + +It seems to have worked! And we can see that the predictions are indeed probabilities between 0 and 1. + +### Using the `lgb.train` function + +Alternatively, we can go for the more flexible interface `lgb.train`. Here, as an additional step, we need to prepare `y` and `X` by the data API `lgb.Dataset` of LightGBM. Parameters are passed to `lgb.train` as a named list. + +```{r} +# Data interface +dtrain <- lgb.Dataset(X, label = y) + +# Parameters +params <- list( + objective = "binary" + , num_leaves = 4L + , learning_rate = 1.0 +) + +# Train +fit <- lgb.train( + params + , data = dtrain + , nrounds = 10L + , verbose = -1L +) +``` + +Try it out! If you get stuck, visit LightGBM's [online docu](https://lightgbm.readthedocs.io/en/latest/) for more details. + +```{r, echo = FALSE, results = "hide"} +# Cleanup +if (file.exists("lightgbm.model")) { + file.remove("lightgbm.model") +} +``` + +## References + diff --git a/R-package/vignettes/biblio.bib b/R-package/vignettes/biblio.bib new file mode 100644 index 000000000000..7508fffb632c --- /dev/null +++ b/R-package/vignettes/biblio.bib @@ -0,0 +1,18 @@ +% Encoding: UTF-8 + +@InProceedings{ke2017, + author = {Ke, Guolin and Meng, Qi and Finely, Thomas and Wang, Taifeng and Chen, Wei and Ma, Weidong and Ye, Qiwei and Liu, Tie-Yan}, + title = {LightGBM: A Highly Efficient Gradient Boosting Decision Tree}, + booktitle = {Advances in Neural Information Processing Systems 30 (NIP 2017)}, + year = {2017} +} + +@article{moro2014, + title = {A data-driven approach to predict the success of bank telemarketing}, + journal = {Decision Support Systems}, + volume = {62}, + pages = {22-31}, + year = {2014}, + issn = {0167-9236}, + author = {Sérgio Moro and Paulo Cortez and Paulo Rita}, +} From ce27eabeae3fed401e40502aa73780f9dd536305 Mon Sep 17 00:00:00 2001 From: mayer79 Date: Fri, 12 Feb 2021 11:37:48 +0100 Subject: [PATCH 02/82] adding knitr, rmarkdown to github/workflows packages --- .github/workflows/r_package.yml | 2 +- .github/workflows/r_valgrind.yml | 2 +- .github/workflows/static_analysis.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/r_package.yml b/.github/workflows/r_package.yml index 54b2798fa40a..d743376cae35 100644 --- a/.github/workflows/r_package.yml +++ b/.github/workflows/r_package.yml @@ -165,7 +165,7 @@ jobs: - name: Install packages shell: bash run: | - Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'testthat'), repos = 'https://cran.r-project.org')" + Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'testthat', 'knitr', 'rmarkdown'), repos = 'https://cran.r-project.org')" sh build-cran-package.sh Rdevel CMD INSTALL lightgbm_*.tar.gz || exit -1 - name: Run tests with sanitizers diff --git a/.github/workflows/r_valgrind.yml b/.github/workflows/r_valgrind.yml index e9af3ae6fdc5..602344343399 100644 --- a/.github/workflows/r_valgrind.yml +++ b/.github/workflows/r_valgrind.yml @@ -38,7 +38,7 @@ jobs: - name: Install packages shell: bash run: | - RDscriptvalgrind -e "install.packages(c('R6', 'data.table', 'jsonlite', 'testthat'), repos = 'https://cran.r-project.org')" + RDscriptvalgrind -e "install.packages(c('R6', 'data.table', 'jsonlite', 'testthat', 'knitr', 'rmarkdown'), repos = 'https://cran.r-project.org')" sh build-cran-package.sh RDvalgrind CMD INSTALL --preclean --install-tests lightgbm_*.tar.gz || exit -1 - name: Run tests with valgrind diff --git a/.github/workflows/static_analysis.yml b/.github/workflows/static_analysis.yml index 9f0faa12d1fd..6802a0ff2728 100644 --- a/.github/workflows/static_analysis.yml +++ b/.github/workflows/static_analysis.yml @@ -57,7 +57,7 @@ jobs: - name: Install packages shell: bash run: | - Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'roxygen2', 'testthat'), repos = 'https://cran.r-project.org')" + Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'roxygen2', 'testthat', 'knitr', 'rmarkdown'), repos = 'https://cran.r-project.org')" sh build-cran-package.sh || exit -1 R CMD INSTALL --with-keep.source lightgbm_*.tar.gz || exit -1 - name: Test documentation From 522bf2ee4bb6ce794fe7796e4695292befa46183 Mon Sep 17 00:00:00 2001 From: mayer79 Date: Fri, 12 Feb 2021 17:45:34 +0100 Subject: [PATCH 03/82] moving local gitignore to global --- .gitignore | 3 +++ R-package/vignettes/.gitignore | 3 --- 2 files changed, 3 insertions(+), 3 deletions(-) delete mode 100644 R-package/vignettes/.gitignore diff --git a/.gitignore b/.gitignore index 959462864825..a38395d7b879 100644 --- a/.gitignore +++ b/.gitignore @@ -437,6 +437,9 @@ miktex*.zip *.RData *.rds +# Files from interactively building R vignettes +**/vignettes/*.html + # Files generated by aspell **/*.bak diff --git a/R-package/vignettes/.gitignore b/R-package/vignettes/.gitignore deleted file mode 100644 index a9d285d9d065..000000000000 --- a/R-package/vignettes/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -*.html -*.R -*.model From ba5db214ffbb29dcc42ad3fe51aef38d61680fc8 Mon Sep 17 00:00:00 2001 From: Michael Mayer Date: Fri, 12 Feb 2021 17:46:58 +0100 Subject: [PATCH 04/82] Update R-package/vignettes/basic_walkthrough.Rmd Co-authored-by: James Lamb --- R-package/vignettes/basic_walkthrough.Rmd | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/R-package/vignettes/basic_walkthrough.Rmd b/R-package/vignettes/basic_walkthrough.Rmd index 577e99aaa448..62a04e2f065c 100644 --- a/R-package/vignettes/basic_walkthrough.Rmd +++ b/R-package/vignettes/basic_walkthrough.Rmd @@ -15,9 +15,9 @@ vignette: > ```{r, include = FALSE} knitr::opts_chunk$set( - collapse = TRUE, - comment = "#>", - warning = FALSE, + , collapse = TRUE + , comment = "#>" + , warning = FALSE message = FALSE ) ``` @@ -116,4 +116,3 @@ if (file.exists("lightgbm.model")) { ``` ## References - From 5ef063120b025d35cf4bc3dd5f4e53ddd0819d25 Mon Sep 17 00:00:00 2001 From: mayer79 Date: Fri, 12 Feb 2021 17:50:30 +0100 Subject: [PATCH 05/82] moved comma --- R-package/vignettes/basic_walkthrough.Rmd | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/R-package/vignettes/basic_walkthrough.Rmd b/R-package/vignettes/basic_walkthrough.Rmd index 62a04e2f065c..c776a83f722a 100644 --- a/R-package/vignettes/basic_walkthrough.Rmd +++ b/R-package/vignettes/basic_walkthrough.Rmd @@ -15,10 +15,10 @@ vignette: > ```{r, include = FALSE} knitr::opts_chunk$set( - , collapse = TRUE + collapse = TRUE , comment = "#>" , warning = FALSE - message = FALSE + , message = FALSE ) ``` From 3580df8b52f669c8338b6af0e5e37e9011f54da0 Mon Sep 17 00:00:00 2001 From: Michael Mayer Date: Fri, 12 Feb 2021 17:50:45 +0100 Subject: [PATCH 06/82] Update R-package/vignettes/basic_walkthrough.Rmd Co-authored-by: James Lamb --- R-package/vignettes/basic_walkthrough.Rmd | 1 - 1 file changed, 1 deletion(-) diff --git a/R-package/vignettes/basic_walkthrough.Rmd b/R-package/vignettes/basic_walkthrough.Rmd index 62a04e2f065c..d57e0f349602 100644 --- a/R-package/vignettes/basic_walkthrough.Rmd +++ b/R-package/vignettes/basic_walkthrough.Rmd @@ -54,7 +54,6 @@ The R package of LightGBM offers two ways to do so: - `lgb.train`: This is the main training logic. It offers full flexibility but requires data created by the `lgb.Dataset` function. - `lightgbm`: Simpler, but less flexible. Data can be passed without having to bother with `lgb.Dataset`. -Let's illustrate both logics: ### Using the `lightgbm` function From 9c462c7ed25a9aafa16469cdb68c9c7eedc8a1a6 Mon Sep 17 00:00:00 2001 From: Michael Mayer Date: Fri, 12 Feb 2021 17:51:03 +0100 Subject: [PATCH 07/82] Update R-package/DESCRIPTION Co-authored-by: James Lamb --- R-package/DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R-package/DESCRIPTION b/R-package/DESCRIPTION index 3fa5dcc435ae..6428ec8dba14 100755 --- a/R-package/DESCRIPTION +++ b/R-package/DESCRIPTION @@ -46,8 +46,8 @@ VignetteBuilder: knitr Suggests: knitr, - rmarkdown, processx, + rmarkdown, testthat Depends: R (>= 3.5), From 077cf60e03a898ad3101c2d403c84707311a767c Mon Sep 17 00:00:00 2001 From: Michael Mayer Date: Fri, 12 Feb 2021 17:52:05 +0100 Subject: [PATCH 08/82] Update R-package/vignettes/basic_walkthrough.Rmd Co-authored-by: James Lamb --- R-package/vignettes/basic_walkthrough.Rmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R-package/vignettes/basic_walkthrough.Rmd b/R-package/vignettes/basic_walkthrough.Rmd index d57e0f349602..355920a2cadf 100644 --- a/R-package/vignettes/basic_walkthrough.Rmd +++ b/R-package/vignettes/basic_walkthrough.Rmd @@ -105,7 +105,7 @@ fit <- lgb.train( ) ``` -Try it out! If you get stuck, visit LightGBM's [online docu](https://lightgbm.readthedocs.io/en/latest/) for more details. +Try it out! If you get stuck, visit LightGBM's [documentation](https://lightgbm.readthedocs.io/en/latest/R/index.html) for more details. ```{r, echo = FALSE, results = "hide"} # Cleanup From aee9712a193a33dd72eace74bf2f9922c5d2d50b Mon Sep 17 00:00:00 2001 From: Michael Mayer Date: Fri, 12 Feb 2021 17:52:34 +0100 Subject: [PATCH 09/82] Update R-package/vignettes/basic_walkthrough.Rmd Co-authored-by: James Lamb --- R-package/vignettes/basic_walkthrough.Rmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R-package/vignettes/basic_walkthrough.Rmd b/R-package/vignettes/basic_walkthrough.Rmd index 355920a2cadf..c7369710917f 100644 --- a/R-package/vignettes/basic_walkthrough.Rmd +++ b/R-package/vignettes/basic_walkthrough.Rmd @@ -2,7 +2,7 @@ title: "Basic Walkthrough" description: > - This vignette illustrates the basics of LightGBM. + This vignette describes how to train a LightGBM model for binary classification. bibliography: "biblio.bib" link-citations: true output: rmarkdown::html_vignette From 111ceab3ba034603452c64b17481bec56d4c1c4b Mon Sep 17 00:00:00 2001 From: Michael Mayer Date: Fri, 12 Feb 2021 17:52:53 +0100 Subject: [PATCH 10/82] Update R-package/vignettes/basic_walkthrough.Rmd Co-authored-by: James Lamb --- R-package/vignettes/basic_walkthrough.Rmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R-package/vignettes/basic_walkthrough.Rmd b/R-package/vignettes/basic_walkthrough.Rmd index c7369710917f..bd35d8a5ada0 100644 --- a/R-package/vignettes/basic_walkthrough.Rmd +++ b/R-package/vignettes/basic_walkthrough.Rmd @@ -52,7 +52,7 @@ Building a strong LightGBM model typically involves a little bit of hyperparamet The R package of LightGBM offers two ways to do so: - `lgb.train`: This is the main training logic. It offers full flexibility but requires data created by the `lgb.Dataset` function. -- `lightgbm`: Simpler, but less flexible. Data can be passed without having to bother with `lgb.Dataset`. +- `lightgbm()`: Simpler, but less flexible. Data can be passed without having to bother with `lgb.Dataset()`. ### Using the `lightgbm` function From 44d1a293967feba3b10a6ae4ca95d86cdb443408 Mon Sep 17 00:00:00 2001 From: Michael Mayer Date: Fri, 12 Feb 2021 17:53:10 +0100 Subject: [PATCH 11/82] Update R-package/vignettes/basic_walkthrough.Rmd Co-authored-by: James Lamb --- R-package/vignettes/basic_walkthrough.Rmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R-package/vignettes/basic_walkthrough.Rmd b/R-package/vignettes/basic_walkthrough.Rmd index bd35d8a5ada0..85f58e90f714 100644 --- a/R-package/vignettes/basic_walkthrough.Rmd +++ b/R-package/vignettes/basic_walkthrough.Rmd @@ -51,7 +51,7 @@ Building a strong LightGBM model typically involves a little bit of hyperparamet The R package of LightGBM offers two ways to do so: -- `lgb.train`: This is the main training logic. It offers full flexibility but requires data created by the `lgb.Dataset` function. +- `lgb.train()`: This is the main training logic. It offers full flexibility but requires a `Dataset` object created by the `lgb.Dataset()` function. - `lightgbm()`: Simpler, but less flexible. Data can be passed without having to bother with `lgb.Dataset()`. From 8373857268abfbc6514e4ebdd42afb4b0bc5748c Mon Sep 17 00:00:00 2001 From: Michael Mayer Date: Fri, 12 Feb 2021 17:53:57 +0100 Subject: [PATCH 12/82] Update R-package/vignettes/basic_walkthrough.Rmd Co-authored-by: James Lamb --- R-package/vignettes/basic_walkthrough.Rmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R-package/vignettes/basic_walkthrough.Rmd b/R-package/vignettes/basic_walkthrough.Rmd index 85f58e90f714..7d3fb5eb7d38 100644 --- a/R-package/vignettes/basic_walkthrough.Rmd +++ b/R-package/vignettes/basic_walkthrough.Rmd @@ -55,7 +55,7 @@ The R package of LightGBM offers two ways to do so: - `lightgbm()`: Simpler, but less flexible. Data can be passed without having to bother with `lgb.Dataset()`. -### Using the `lightgbm` function +### Using the `lightgbm()` function In a first step, we will make sure that all data is numeric. Then, we will fit the model by the `lightgbm` function and check how the distribution of the predictions looks like. From dc7a7f18900c36134585097cbe8418b2dd1810e8 Mon Sep 17 00:00:00 2001 From: mayer79 Date: Fri, 12 Feb 2021 17:58:06 +0100 Subject: [PATCH 13/82] add knitr, rmarkdown to ci testsuite --- .ci/test_r_package.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/test_r_package.sh b/.ci/test_r_package.sh index ebbaee38f57e..d1955d1672eb 100755 --- a/.ci/test_r_package.sh +++ b/.ci/test_r_package.sh @@ -99,7 +99,7 @@ fi # Manually install Depends and Imports libraries + 'testthat' # to avoid a CI-time dependency on devtools (for devtools::install_deps()) -packages="c('data.table', 'jsonlite', 'Matrix', 'R6', 'testthat')" +packages="c('data.table', 'jsonlite', 'Matrix', 'R6', 'testthat', 'knitr', 'rmarkdown')" if [[ $OS_NAME == "macos" ]]; then packages+=", type = 'both'" fi From ef4fcddc8d475c6b8e07897ce99e83cca0ea10c7 Mon Sep 17 00:00:00 2001 From: Michael Mayer Date: Fri, 12 Feb 2021 17:58:37 +0100 Subject: [PATCH 14/82] Update R-package/vignettes/basic_walkthrough.Rmd Co-authored-by: James Lamb --- R-package/vignettes/basic_walkthrough.Rmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R-package/vignettes/basic_walkthrough.Rmd b/R-package/vignettes/basic_walkthrough.Rmd index 7d3fb5eb7d38..33d90a1c252f 100644 --- a/R-package/vignettes/basic_walkthrough.Rmd +++ b/R-package/vignettes/basic_walkthrough.Rmd @@ -57,7 +57,7 @@ The R package of LightGBM offers two ways to do so: ### Using the `lightgbm()` function -In a first step, we will make sure that all data is numeric. Then, we will fit the model by the `lightgbm` function and check how the distribution of the predictions looks like. +In a first step, we will make sure that all data is numeric. Then, we will fit the model by the `lightgbm()` function and check how the distribution of the predictions looks like. ```{r} # Numeric response and feature matrix From 16383bd0c57149fd9808d58cd374fdfd27d5dd76 Mon Sep 17 00:00:00 2001 From: Michael Mayer Date: Fri, 12 Feb 2021 17:58:51 +0100 Subject: [PATCH 15/82] Update R-package/vignettes/basic_walkthrough.Rmd Co-authored-by: James Lamb --- R-package/vignettes/basic_walkthrough.Rmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R-package/vignettes/basic_walkthrough.Rmd b/R-package/vignettes/basic_walkthrough.Rmd index 33d90a1c252f..890e97719c14 100644 --- a/R-package/vignettes/basic_walkthrough.Rmd +++ b/R-package/vignettes/basic_walkthrough.Rmd @@ -81,7 +81,7 @@ summary(predict(fit, X)) It seems to have worked! And we can see that the predictions are indeed probabilities between 0 and 1. -### Using the `lgb.train` function +### Using the `lgb.train()` function Alternatively, we can go for the more flexible interface `lgb.train`. Here, as an additional step, we need to prepare `y` and `X` by the data API `lgb.Dataset` of LightGBM. Parameters are passed to `lgb.train` as a named list. From 14e7a643e42238b94039875f8515131c4212608b Mon Sep 17 00:00:00 2001 From: Michael Mayer Date: Fri, 12 Feb 2021 18:02:25 +0100 Subject: [PATCH 16/82] Update R-package/vignettes/basic_walkthrough.Rmd Co-authored-by: James Lamb --- R-package/vignettes/basic_walkthrough.Rmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R-package/vignettes/basic_walkthrough.Rmd b/R-package/vignettes/basic_walkthrough.Rmd index 890e97719c14..72bbd83d56dc 100644 --- a/R-package/vignettes/basic_walkthrough.Rmd +++ b/R-package/vignettes/basic_walkthrough.Rmd @@ -83,7 +83,7 @@ It seems to have worked! And we can see that the predictions are indeed probabil ### Using the `lgb.train()` function -Alternatively, we can go for the more flexible interface `lgb.train`. Here, as an additional step, we need to prepare `y` and `X` by the data API `lgb.Dataset` of LightGBM. Parameters are passed to `lgb.train` as a named list. +Alternatively, we can go for the more flexible interface `lgb.train()`. Here, as an additional step, we need to prepare `y` and `X` by the data API `lgb.Dataset` of LightGBM. Parameters are passed to `lgb.train()` as a named list. ```{r} # Data interface From ebd0637f4165a117337b842736bdeecc39242a89 Mon Sep 17 00:00:00 2001 From: mayer79 Date: Fri, 12 Feb 2021 18:04:21 +0100 Subject: [PATCH 17/82] we -> you --- R-package/vignettes/basic_walkthrough.Rmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R-package/vignettes/basic_walkthrough.Rmd b/R-package/vignettes/basic_walkthrough.Rmd index fc5770382e37..79eb367986b9 100644 --- a/R-package/vignettes/basic_walkthrough.Rmd +++ b/R-package/vignettes/basic_walkthrough.Rmd @@ -47,7 +47,7 @@ table(bank$y) ## Training the model -Building a strong LightGBM model typically involves a little bit of hyperparameter tuning. To keep things as simple as possible, we skip this step here and directly jump into fitting a first model. +Building a strong LightGBM model typically involves a little bit of hyperparameter tuning. Follow the steps below to train a simple {lightgbm} model for binary classification. The R package of LightGBM offers two ways to do so: From 22e576af9a175ffd8d92fbe8ba04e793fa8acbd6 Mon Sep 17 00:00:00 2001 From: mayer79 Date: Fri, 12 Feb 2021 18:09:20 +0100 Subject: [PATCH 18/82] removed knitr, rmarkdown from r_valgrind --- .github/workflows/r_valgrind.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/r_valgrind.yml b/.github/workflows/r_valgrind.yml index 602344343399..e9af3ae6fdc5 100644 --- a/.github/workflows/r_valgrind.yml +++ b/.github/workflows/r_valgrind.yml @@ -38,7 +38,7 @@ jobs: - name: Install packages shell: bash run: | - RDscriptvalgrind -e "install.packages(c('R6', 'data.table', 'jsonlite', 'testthat', 'knitr', 'rmarkdown'), repos = 'https://cran.r-project.org')" + RDscriptvalgrind -e "install.packages(c('R6', 'data.table', 'jsonlite', 'testthat'), repos = 'https://cran.r-project.org')" sh build-cran-package.sh RDvalgrind CMD INSTALL --preclean --install-tests lightgbm_*.tar.gz || exit -1 - name: Run tests with valgrind From 20ecbaf757cb2e22205c5714bb97d6e83d2aafb6 Mon Sep 17 00:00:00 2001 From: mayer79 Date: Fri, 12 Feb 2021 18:28:02 +0100 Subject: [PATCH 19/82] Revision of wording from "we" to "you" --- R-package/vignettes/basic_walkthrough.Rmd | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/R-package/vignettes/basic_walkthrough.Rmd b/R-package/vignettes/basic_walkthrough.Rmd index 86704e4d7944..8ff34ecbc4af 100644 --- a/R-package/vignettes/basic_walkthrough.Rmd +++ b/R-package/vignettes/basic_walkthrough.Rmd @@ -30,11 +30,11 @@ Welcome to the world of [LightGBM](https://lightgbm.readthedocs.io/en/latest/), library(lightgbm) ``` -This vignette will guide you through its basic usage. For illustration, we will use a subset of the `bank` dataset [@moro2014] to predict whether a client has subscribed a term deposit. As input features, we will use two numeric predictors, "age" and "balance". +This vignette will guide you through its basic usage. It will show how to build a simple binary classification model based on a subset of the `bank` dataset [@moro2014]. You will use the two input features "age" and "balance" to predict whether a client has subscribed a term deposit. ## The dataset -Let's have a look at the dataset first. +The dataset looks as follows. ```{r} data(bank, package = "lightgbm") @@ -47,9 +47,7 @@ table(bank$y) ## Training the model -Building a strong LightGBM model typically involves a little bit of hyperparameter tuning. Follow the steps below to train a simple {lightgbm} model for binary classification. - -The R package of LightGBM offers two ways to do so: +The R package of LightGBM offers two functions to train a model: - `lgb.train()`: This is the main training logic. It offers full flexibility but requires a `Dataset` object created by the `lgb.Dataset()` function. - `lightgbm()`: Simpler, but less flexible. Data can be passed without having to bother with `lgb.Dataset()`. @@ -57,7 +55,7 @@ The R package of LightGBM offers two ways to do so: ### Using the `lightgbm()` function -In a first step, we will make sure that all data is numeric. Then, we will fit the model by the `lightgbm()` function and check how the distribution of the predictions looks like. +In a first step, you need to convert data to numeric. Afterwards, you are ready to fit the model by the `lightgbm()` function. ```{r} # Numeric response and feature matrix @@ -79,11 +77,11 @@ fit <- lightgbm( summary(predict(fit, X)) ``` -It seems to have worked! And we can see that the predictions are indeed probabilities between 0 and 1. +It seems to have worked! And the predictions are indeed probabilities between 0 and 1. ### Using the `lgb.train()` function -Alternatively, we can go for the more flexible interface `lgb.train()`. Here, as an additional step, we need to prepare `y` and `X` by the data API `lgb.Dataset` of LightGBM. Parameters are passed to `lgb.train()` as a named list. +Alternatively, you can go for the more flexible interface `lgb.train()`. Here, as an additional step, you need to prepare `y` and `X` by the data API `lgb.Dataset()` of LightGBM. Parameters are passed to `lgb.train()` as a named list. ```{r} # Data interface @@ -105,7 +103,7 @@ fit <- lgb.train( ) ``` -Try it out! If you get stuck, visit LightGBM's [documentation](https://lightgbm.readthedocs.io/en/latest/R/index.html) for more details. +Try it out! If stuck, visit LightGBM's [documentation](https://lightgbm.readthedocs.io/en/latest/R/index.html) for more details. ```{r, echo = FALSE, results = "hide"} # Cleanup From f0426d48035a7dc6e00aa25f235affb3c257e1fa Mon Sep 17 00:00:00 2001 From: mayer79 Date: Sat, 13 Feb 2021 07:12:47 +0100 Subject: [PATCH 20/82] added rmarkdown, knitr to ci windows chain --- .ci/test_r_package_windows.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/test_r_package_windows.ps1 b/.ci/test_r_package_windows.ps1 index e49a638b10b7..41915401cc0e 100644 --- a/.ci/test_r_package_windows.ps1 +++ b/.ci/test_r_package_windows.ps1 @@ -96,7 +96,7 @@ Write-Output "Installing Rtools" Write-Output "Done installing Rtools" Write-Output "Installing dependencies" -$packages = "c('data.table', 'jsonlite', 'Matrix', 'processx', 'R6', 'testthat'), dependencies = c('Imports', 'Depends', 'LinkingTo')" +$packages = "c('data.table', 'jsonlite', 'Matrix', 'processx', 'R6', 'testthat', 'knitr', 'rmarkdown'), dependencies = c('Imports', 'Depends', 'LinkingTo')" Run-R-Code-Redirect-Stderr "options(install.packages.check.source = 'no'); install.packages($packages, repos = '$env:CRAN_MIRROR', type = 'binary', lib = '$env:R_LIB_PATH')" ; Check-Output $? # MiKTeX and pandoc can be skipped on non-MinGW builds, since we don't From e197ed4e4cd28d8d7bf0d6099f1b2d13bb9761a1 Mon Sep 17 00:00:00 2001 From: mayer79 Date: Sun, 14 Feb 2021 08:41:19 +0100 Subject: [PATCH 21/82] removed bibtex elements to simplify --- R-package/vignettes/basic_walkthrough.Rmd | 10 ++++++---- R-package/vignettes/biblio.bib | 18 ------------------ 2 files changed, 6 insertions(+), 22 deletions(-) delete mode 100644 R-package/vignettes/biblio.bib diff --git a/R-package/vignettes/basic_walkthrough.Rmd b/R-package/vignettes/basic_walkthrough.Rmd index 8ff34ecbc4af..117ef764e3b3 100644 --- a/R-package/vignettes/basic_walkthrough.Rmd +++ b/R-package/vignettes/basic_walkthrough.Rmd @@ -3,8 +3,6 @@ title: "Basic Walkthrough" description: > This vignette describes how to train a LightGBM model for binary classification. -bibliography: "biblio.bib" -link-citations: true output: rmarkdown::html_vignette vignette: > %\VignetteIndexEntry{Basic Walkthrough} @@ -24,13 +22,13 @@ knitr::opts_chunk$set( ## Introduction -Welcome to the world of [LightGBM](https://lightgbm.readthedocs.io/en/latest/), a hightly efficient gradient boosting implementation [@ke2017]. +Welcome to the world of [LightGBM](https://lightgbm.readthedocs.io/en/latest/), a hightly efficient gradient boosting implementation (Ke et al. 2017). ```{r setup} library(lightgbm) ``` -This vignette will guide you through its basic usage. It will show how to build a simple binary classification model based on a subset of the `bank` dataset [@moro2014]. You will use the two input features "age" and "balance" to predict whether a client has subscribed a term deposit. +This vignette will guide you through its basic usage. It will show how to build a simple binary classification model based on a subset of the `bank` dataset (Moro, Cortez, and Rita 2014). You will use the two input features "age" and "balance" to predict whether a client has subscribed a term deposit. ## The dataset @@ -113,3 +111,7 @@ if (file.exists("lightgbm.model")) { ``` ## References + +Ke, Guolin, Qi Meng, Thomas Finely, Taifeng Wang, Wei Chen, Weidong Ma, Qiwei Ye, and Tie-Yan Liu. 2017. "LightGBM: A Highly Efficient Gradient Boosting Decision Tree." In Advances in Neural Information Processing Systems 30 (Nip 2017). + +Moro, Sérgio, Paulo Cortez, and Paulo Rita. 2014. "A Data-Driven Approach to Predict the Success of Bank Telemarketing." Decision Support Systems 62: 22–31. diff --git a/R-package/vignettes/biblio.bib b/R-package/vignettes/biblio.bib deleted file mode 100644 index 7508fffb632c..000000000000 --- a/R-package/vignettes/biblio.bib +++ /dev/null @@ -1,18 +0,0 @@ -% Encoding: UTF-8 - -@InProceedings{ke2017, - author = {Ke, Guolin and Meng, Qi and Finely, Thomas and Wang, Taifeng and Chen, Wei and Ma, Weidong and Ye, Qiwei and Liu, Tie-Yan}, - title = {LightGBM: A Highly Efficient Gradient Boosting Decision Tree}, - booktitle = {Advances in Neural Information Processing Systems 30 (NIP 2017)}, - year = {2017} -} - -@article{moro2014, - title = {A data-driven approach to predict the success of bank telemarketing}, - journal = {Decision Support Systems}, - volume = {62}, - pages = {22-31}, - year = {2014}, - issn = {0167-9236}, - author = {Sérgio Moro and Paulo Cortez and Paulo Rita}, -} From 3e748be8a35e6ef41764af42becb4bb2add36347 Mon Sep 17 00:00:00 2001 From: Michael Mayer Date: Sun, 14 Feb 2021 18:40:51 +0100 Subject: [PATCH 22/82] Update R-package/vignettes/basic_walkthrough.Rmd Co-authored-by: James Lamb --- R-package/vignettes/basic_walkthrough.Rmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R-package/vignettes/basic_walkthrough.Rmd b/R-package/vignettes/basic_walkthrough.Rmd index 117ef764e3b3..192e4ee686f9 100644 --- a/R-package/vignettes/basic_walkthrough.Rmd +++ b/R-package/vignettes/basic_walkthrough.Rmd @@ -112,6 +112,6 @@ if (file.exists("lightgbm.model")) { ## References -Ke, Guolin, Qi Meng, Thomas Finely, Taifeng Wang, Wei Chen, Weidong Ma, Qiwei Ye, and Tie-Yan Liu. 2017. "LightGBM: A Highly Efficient Gradient Boosting Decision Tree." In Advances in Neural Information Processing Systems 30 (Nip 2017). +Ke, Guolin, Qi Meng, Thomas Finley, Taifeng Wang, Wei Chen, Weidong Ma, Qiwei Ye, and Tie-Yan Liu. 2017. "LightGBM: A Highly Efficient Gradient Boosting Decision Tree." In Advances in Neural Information Processing Systems 30 (Nip 2017). Moro, Sérgio, Paulo Cortez, and Paulo Rita. 2014. "A Data-Driven Approach to Predict the Success of Bank Telemarketing." Decision Support Systems 62: 22–31. From b660089133074c6cbdc07bf74c82cef3b0966fc9 Mon Sep 17 00:00:00 2001 From: mayer79 Date: Tue, 16 Feb 2021 07:37:43 +0100 Subject: [PATCH 23/82] changed .rbuildignore file --- R-package/.Rbuildignore | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/R-package/.Rbuildignore b/R-package/.Rbuildignore index 3e552132cf22..537a860c8dc3 100644 --- a/R-package/.Rbuildignore +++ b/R-package/.Rbuildignore @@ -1,3 +1,6 @@ +\.o$ +\.so$ +\.dll$ \.appveyor\.yml AUTOCONF_UBUNTU_VERSION ^autom4te.cache/.*$ @@ -6,17 +9,14 @@ AUTOCONF_UBUNTU_VERSION \.clang-format ^cran-comments\.md$ ^docs$ -^.*\.dll \.drone\.yml \.git \.gitkeep$ ^.*\.history ^Makefile$ -^.*\.o ^.*\.out ^pkgdown$ ^recreate-configure\.sh$ -^.*\.so ^src/build/.*$ ^src/CMakeLists.txt$ ^src/external_libs/compute/.appveyor.yml$ From 17e05808ea564c4dc00e534b5d52587ae75b771c Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sun, 14 Mar 2021 00:04:32 -0600 Subject: [PATCH 24/82] manually cleaning --- build-cran-package.sh | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/build-cran-package.sh b/build-cran-package.sh index 72e61f7c1c9f..e081147b6e93 100755 --- a/build-cran-package.sh +++ b/build-cran-package.sh @@ -145,4 +145,31 @@ R CMD build \ --keep-empty-dirs \ lightgbm_r +echo "removing object files created by vignettes" +mkdir _tmp +mv lightgbm*.tar.gz _tmp/ +TARBALL_NAME=lightgbm_${LGB_VERSION}.tar.gz +# cd _tmp +# # tar -xvf ${TARBALL_NAME} +# rm ${TARBALL_NAME} +# # find ./ -name \*.a -type f -exec rm -rf {} + +# # find ./ -name \*.dll -type f -exec rm -rf {} + +# # find ./ -name \*.o -type f -exec rm -rf {} + +# # find ./ -name \*.so -type f -exec rm -rf {} + +# cd .. + +cd _tmp + tar \ + -czvf ${TARBALL_NAME} \ + --exclude=*.a \ + --exclude=*.dll \ + --exclude=*.o \ + --exclude=*.so \ + --exclude=*.tar.gz \ + . + cp ${TARBALL_NAME} ../ +cd .. + +rm -rf ./_tmp + echo "Done building R package" From 0d328de3f23302c076c5b6cfde24e81065c298f3 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 5 Apr 2021 23:19:41 -0500 Subject: [PATCH 25/82] remove object files from tarball --- build-cran-package.sh | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/build-cran-package.sh b/build-cran-package.sh index e081147b6e93..11240f9c4e55 100755 --- a/build-cran-package.sh +++ b/build-cran-package.sh @@ -147,18 +147,12 @@ R CMD build \ echo "removing object files created by vignettes" mkdir _tmp -mv lightgbm*.tar.gz _tmp/ TARBALL_NAME=lightgbm_${LGB_VERSION}.tar.gz -# cd _tmp -# # tar -xvf ${TARBALL_NAME} -# rm ${TARBALL_NAME} -# # find ./ -name \*.a -type f -exec rm -rf {} + -# # find ./ -name \*.dll -type f -exec rm -rf {} + -# # find ./ -name \*.o -type f -exec rm -rf {} + -# # find ./ -name \*.so -type f -exec rm -rf {} + -# cd .. +mv ${TARBALL_NAME} _tmp/ cd _tmp + tar -xvf ${TARBALL_NAME} + rm -rf ${TARBALL_NAME} tar \ -czvf ${TARBALL_NAME} \ --exclude=*.a \ From 91741d5de88c6b293137fd3b2ce2d4bc7ab9ab09 Mon Sep 17 00:00:00 2001 From: mayer79 Date: Tue, 6 Apr 2021 07:51:50 +0200 Subject: [PATCH 26/82] add logo --- .../vignettes/LightGBM_logo_black_text.svg | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 R-package/vignettes/LightGBM_logo_black_text.svg diff --git a/R-package/vignettes/LightGBM_logo_black_text.svg b/R-package/vignettes/LightGBM_logo_black_text.svg new file mode 100644 index 000000000000..a3411280a70f --- /dev/null +++ b/R-package/vignettes/LightGBM_logo_black_text.svg @@ -0,0 +1,39 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + From 947c13041cabbd67b5e982e309744867fe6db071 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 26 Apr 2021 23:06:24 -0500 Subject: [PATCH 27/82] fix tar errors --- .appveyor.yml | 72 +-- .github/workflows/cuda.yml | 186 +++---- .github/workflows/python_package.yml | 152 +++--- .vsts-ci.yml | 762 +++++++++++++-------------- build-cran-package.sh | 20 +- 5 files changed, 597 insertions(+), 595 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index b7d9fe589176..ccb08022792a 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -1,43 +1,43 @@ -version: 3.2.1.99.{build} +# version: 3.2.1.99.{build} -image: Visual Studio 2015 -platform: x64 -configuration: # a trick to construct a build matrix with multiple Python versions - - 3.7 +# image: Visual Studio 2015 +# platform: x64 +# configuration: # a trick to construct a build matrix with multiple Python versions +# - 3.7 -# only build pull requests and -# commits to 'master' -branches: - only: - - master +# # only build pull requests and +# # commits to 'master' +# branches: +# only: +# - master -environment: - matrix: - - COMPILER: MSVC - TASK: python - - COMPILER: MINGW - TASK: python +# environment: +# matrix: +# - COMPILER: MSVC +# TASK: python +# - COMPILER: MINGW +# TASK: python -clone_depth: 5 +# clone_depth: 5 -install: - - git submodule update --init --recursive # get `external_libs` folder - - set PATH=%PATH:C:\Program Files\Git\usr\bin;=% # delete sh.exe from PATH (mingw32-make fix) - - set PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH% - - set PYTHON_VERSION=%CONFIGURATION% - - set CONDA_ENV="test-env" - - ps: | - switch ($env:PYTHON_VERSION) { - "3.6" {$env:MINICONDA = "C:\Miniconda36-x64"} - "3.7" {$env:MINICONDA = "C:\Miniconda37-x64"} - default {$env:MINICONDA = "C:\Miniconda37-x64"} - } - $env:PATH = "$env:MINICONDA;$env:MINICONDA\Scripts;$env:PATH" - $env:BUILD_SOURCESDIRECTORY = "$env:APPVEYOR_BUILD_FOLDER" - $env:LGB_VER = (Get-Content $env:APPVEYOR_BUILD_FOLDER\VERSION.txt).trim() +# install: +# - git submodule update --init --recursive # get `external_libs` folder +# - set PATH=%PATH:C:\Program Files\Git\usr\bin;=% # delete sh.exe from PATH (mingw32-make fix) +# - set PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH% +# - set PYTHON_VERSION=%CONFIGURATION% +# - set CONDA_ENV="test-env" +# - ps: | +# switch ($env:PYTHON_VERSION) { +# "3.6" {$env:MINICONDA = "C:\Miniconda36-x64"} +# "3.7" {$env:MINICONDA = "C:\Miniconda37-x64"} +# default {$env:MINICONDA = "C:\Miniconda37-x64"} +# } +# $env:PATH = "$env:MINICONDA;$env:MINICONDA\Scripts;$env:PATH" +# $env:BUILD_SOURCESDIRECTORY = "$env:APPVEYOR_BUILD_FOLDER" +# $env:LGB_VER = (Get-Content $env:APPVEYOR_BUILD_FOLDER\VERSION.txt).trim() -build: false +# build: false -test_script: - - conda init powershell - - powershell.exe -ExecutionPolicy Bypass -File %APPVEYOR_BUILD_FOLDER%\.ci\test_windows.ps1 +# test_script: +# - conda init powershell +# - powershell.exe -ExecutionPolicy Bypass -File %APPVEYOR_BUILD_FOLDER%\.ci\test_windows.ps1 diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 959a2c717edf..b93e155b934e 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -1,96 +1,96 @@ -name: CUDA Version +# name: CUDA Version -on: - push: - branches: - - master - pull_request: - branches: - - master +# on: +# push: +# branches: +# - master +# pull_request: +# branches: +# - master -env: - github_actions: 'true' - os_name: linux - task: cuda - conda_env: test-env +# env: +# github_actions: 'true' +# os_name: linux +# task: cuda +# conda_env: test-env -jobs: - test: - name: cuda ${{ matrix.cuda_version }} ${{ matrix.method }} (linux, ${{ matrix.compiler }}, Python ${{ matrix.python_version }}) - runs-on: [self-hosted, linux] - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - include: - - method: source - compiler: gcc - python_version: 3.7 - cuda_version: "11.2.2" - - method: pip - compiler: clang - python_version: 3.8 - cuda_version: "10.0" - - method: wheel - compiler: gcc - python_version: 3.9 - cuda_version: "9.0" - steps: - - name: Setup or update software on host machine - run: | - sudo apt-get update - sudo apt-get install --no-install-recommends -y \ - apt-transport-https \ - ca-certificates \ - curl \ - git \ - gnupg-agent \ - software-properties-common - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" -y - curl -sL https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add - - curl -sL https://nvidia.github.io/nvidia-docker/$(. /etc/os-release;echo $ID$VERSION_ID)/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list - sudo apt-get update - sudo apt-get install --no-install-recommends -y \ - containerd.io \ - docker-ce \ - docker-ce-cli \ - nvidia-docker2 - sudo chmod a+rw /var/run/docker.sock - sudo systemctl restart docker - - name: Remove old folder with repository - run: sudo rm -rf $GITHUB_WORKSPACE - - name: Checkout repository - uses: actions/checkout@v1 - with: - fetch-depth: 5 - submodules: true - - name: Setup and run tests - run: | - export ROOT_DOCKER_FOLDER=/LightGBM - cat > docker.env < docker-script.sh < docker.env < docker-script.sh < docker.env < docker-script.sh < docker.env < docker-script.sh < Date: Mon, 26 Apr 2021 23:42:15 -0500 Subject: [PATCH 28/82] fix installs, remove extra copy of logo --- .ci/test_r_package.sh | 2 +- .github/workflows/r_package.yml | 4 +- .github/workflows/r_valgrind.yml | 2 +- .../vignettes/LightGBM_logo_black_text.svg | 39 ------------------- build-cran-package.sh | 2 + build_r.R | 2 +- 6 files changed, 7 insertions(+), 44 deletions(-) delete mode 100644 R-package/vignettes/LightGBM_logo_black_text.svg diff --git a/.ci/test_r_package.sh b/.ci/test_r_package.sh index c210817ce454..94356e226614 100755 --- a/.ci/test_r_package.sh +++ b/.ci/test_r_package.sh @@ -94,7 +94,7 @@ fi # Manually install Depends and Imports libraries + 'testthat' # to avoid a CI-time dependency on devtools (for devtools::install_deps()) -packages="c('data.table', 'jsonlite', 'Matrix', 'R6', 'testthat', 'knitr', 'rmarkdown')" +packages="c('data.table', 'jsonlite', 'Matrix', 'R6', 'knitr', 'rmarkdown', 'testthat')" compile_from_source="both" if [[ $OS_NAME == "macos" ]]; then packages+=", type = 'binary'" diff --git a/.github/workflows/r_package.yml b/.github/workflows/r_package.yml index f932f48e73a3..3af413d5f691 100644 --- a/.github/workflows/r_package.yml +++ b/.github/workflows/r_package.yml @@ -183,7 +183,7 @@ jobs: - name: Install packages shell: bash run: | - Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'testthat', 'knitr', 'rmarkdown'), repos = 'https://cran.r-project.org', Ncpus = parallel::detectCores())" + Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'rmarkdown', 'testthat'), repos = 'https://cran.r-project.org', Ncpus = parallel::detectCores())" sh build-cran-package.sh Rdevel CMD INSTALL lightgbm_*.tar.gz || exit -1 - name: Run tests with sanitizers @@ -213,7 +213,7 @@ jobs: shell: bash run: | export PATH=/opt/R-devel/bin/:${PATH} - Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'testthat'), repos = 'https://cran.r-project.org', Ncpus = parallel::detectCores())" + Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'rmarkdown', 'testthat'), repos = 'https://cran.r-project.org', Ncpus = parallel::detectCores())" sh build-cran-package.sh R CMD check --as-cran --run-donttest lightgbm_*.tar.gz || exit -1 if grep -q -E "NOTE|WARNING|ERROR" lightgbm.Rcheck/00check.log; then diff --git a/.github/workflows/r_valgrind.yml b/.github/workflows/r_valgrind.yml index 678789af3bf0..2329307f52b6 100644 --- a/.github/workflows/r_valgrind.yml +++ b/.github/workflows/r_valgrind.yml @@ -38,7 +38,7 @@ jobs: - name: Install packages shell: bash run: | - RDscriptvalgrind -e "install.packages(c('R6', 'data.table', 'jsonlite', 'testthat'), repos = 'https://cran.r-project.org', Ncpus = parallel::detectCores())" + RDscriptvalgrind -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'rmarkdown', 'testthat'), repos = 'https://cran.r-project.org', Ncpus = parallel::detectCores())" sh build-cran-package.sh RDvalgrind CMD INSTALL --preclean --install-tests lightgbm_*.tar.gz || exit -1 - name: Run tests with valgrind diff --git a/R-package/vignettes/LightGBM_logo_black_text.svg b/R-package/vignettes/LightGBM_logo_black_text.svg deleted file mode 100644 index a3411280a70f..000000000000 --- a/R-package/vignettes/LightGBM_logo_black_text.svg +++ /dev/null @@ -1,39 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/build-cran-package.sh b/build-cran-package.sh index aa98359a84a4..d4a4ad84562a 100755 --- a/build-cran-package.sh +++ b/build-cran-package.sh @@ -25,6 +25,7 @@ LGB_VERSION=$(cat VERSION.txt | sed "s/rc/-/g") # move relevant files cp -R R-package/* ${TEMP_R_DIR} +cp docs/logo/LightGBM_logo_black_text.svg ${TEMP_R_DIR}/vignettes/ cp -R include ${TEMP_R_DIR}/src/ cp -R src/* ${TEMP_R_DIR}/src/ @@ -145,6 +146,7 @@ R CMD build \ lightgbm_r echo "removing object files created by vignettes" +rm -rf ./_tmp mkdir _tmp TARBALL_NAME=lightgbm_${LGB_VERSION}.tar.gz mv ${TARBALL_NAME} _tmp/ diff --git a/build_r.R b/build_r.R index a4eeee228c6b..fa4f99e375f9 100644 --- a/build_r.R +++ b/build_r.R @@ -372,7 +372,7 @@ writeLines(description_contents, DESCRIPTION_FILE) # NOTE: --keep-empty-dirs is necessary to keep the deep paths expected # by CMake while also meeting the CRAN req to create object files # on demand -.run_shell_command("R", c("CMD", "build", TEMP_R_DIR, "--keep-empty-dirs")) +.run_shell_command("R", c("CMD", "build", TEMP_R_DIR, "--keep-empty-dirs", "--no-build-vignettes")) # Install the package version <- gsub( From 10a54d46a1435166b4dcd02338693a11c6c25c59 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 27 Apr 2021 00:01:02 -0500 Subject: [PATCH 29/82] ignore vignettes checking CMake builds --- .ci/test_r_package.sh | 9 +++++++-- .ci/test_r_package_windows.ps1 | 3 ++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.ci/test_r_package.sh b/.ci/test_r_package.sh index 94356e226614..084bb884c006 100755 --- a/.ci/test_r_package.sh +++ b/.ci/test_r_package.sh @@ -140,13 +140,18 @@ elif [[ $R_BUILD_TYPE == "cran" ]]; then cd ${R_CMD_CHECK_DIR} fi +# vignettes are only built for CRAN builds +CHECK_ARGS="--as-cran --run-donttest" +if [[ $R_BUILD_TYPE != "cran" ]]; then + CHECK_ARGS="${CHECK_FLAGS} --ignore-vignettes" +fi + # fails tests if either ERRORs or WARNINGs are thrown by # R CMD CHECK check_succeeded="yes" ( R CMD check ${PKG_TARBALL} \ - --as-cran \ - --run-donttest \ + ${CHECK_ARGS} \ || check_succeeded="no" ) & diff --git a/.ci/test_r_package_windows.ps1 b/.ci/test_r_package_windows.ps1 index 137ac67f9e56..f51f328abe6b 100644 --- a/.ci/test_r_package_windows.ps1 +++ b/.ci/test_r_package_windows.ps1 @@ -168,7 +168,8 @@ if ($env:COMPILER -ne "MSVC") { # CRAN packages must pass without --no-multiarch (build on 64-bit and 32-bit) $check_args = "c('CMD', 'check', '--as-cran', '--run-donttest', '$PKG_FILE_NAME')" } else { - $check_args = "c('CMD', 'check', '--no-multiarch', '--as-cran', '--run-donttest', '$PKG_FILE_NAME')" + # vignettes are only built for CRAN builds + $check_args = "c('CMD', 'check', '--no-multiarch', '--as-cran', '--run-donttest', '--ignore-vignettes', $PKG_FILE_NAME')" } Run-R-Code-Redirect-Stderr "result <- processx::run(command = 'R.exe', args = $check_args, echo = TRUE, windows_verbatim_args = FALSE, error_on_status = TRUE)" ; $check_succeeded = $? From cfd9bee8f62121e767e589bf5ee4c658675d0f67 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 28 Apr 2021 22:58:24 -0500 Subject: [PATCH 30/82] try to fix Windows builds --- .ci/test_r_package_windows.ps1 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.ci/test_r_package_windows.ps1 b/.ci/test_r_package_windows.ps1 index f51f328abe6b..79016e1a4c3e 100644 --- a/.ci/test_r_package_windows.ps1 +++ b/.ci/test_r_package_windows.ps1 @@ -152,6 +152,7 @@ if ($env:COMPILER -ne "MSVC") { } Run-R-Code-Redirect-Stderr "commandArgs <- function(...){$env:BUILD_R_FLAGS}; source('build_r.R')"; Check-Output $? } elseif ($env:R_BUILD_TYPE -eq "cran") { + choco install --yes --no-progress --no-color gzip Run-R-Code-Redirect-Stderr "result <- processx::run(command = 'sh', args = 'build-cran-package.sh', echo = TRUE, windows_verbatim_args = FALSE, error_on_status = TRUE)" ; Check-Output $? # Test CRAN source .tar.gz in a directory that is not this repo or below it. # When people install.packages('lightgbm'), they won't have the LightGBM @@ -169,7 +170,7 @@ if ($env:COMPILER -ne "MSVC") { $check_args = "c('CMD', 'check', '--as-cran', '--run-donttest', '$PKG_FILE_NAME')" } else { # vignettes are only built for CRAN builds - $check_args = "c('CMD', 'check', '--no-multiarch', '--as-cran', '--run-donttest', '--ignore-vignettes', $PKG_FILE_NAME')" + $check_args = "c('CMD', 'check', '--no-multiarch', '--as-cran', '--run-donttest', '--ignore-vignettes', '$PKG_FILE_NAME')" } Run-R-Code-Redirect-Stderr "result <- processx::run(command = 'R.exe', args = $check_args, echo = TRUE, windows_verbatim_args = FALSE, error_on_status = TRUE)" ; $check_succeeded = $? From 3f8a685cfb7438ad5ef4b894628bf73cb4cee5b6 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 29 Apr 2021 23:24:53 -0500 Subject: [PATCH 31/82] more Windows things --- .ci/test_r_package_windows.ps1 | 7 ++++++- R-package/DESCRIPTION | 3 +-- build_r.R | 9 +++++++++ 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/.ci/test_r_package_windows.ps1 b/.ci/test_r_package_windows.ps1 index 79016e1a4c3e..b4175497aa67 100644 --- a/.ci/test_r_package_windows.ps1 +++ b/.ci/test_r_package_windows.ps1 @@ -36,6 +36,12 @@ function Remove-From-Path { $env:PATH = ($env:PATH.Split(';') | Where-Object { $_ -notmatch "$pattern_to_remove" }) -join ';' } +# gzip is needed to create a CRAN package on Windows. +# Install it here before chocolatey is removed from PATH. +if ($env:R_BUILD_TYPE -eq "cran") { + choco install --yes --no-progress --no-color gzip +} + # remove some details that exist in the GitHub Actions images which might # cause conflicts with R and other components installed by this script $env:RTOOLS40_HOME = "" @@ -152,7 +158,6 @@ if ($env:COMPILER -ne "MSVC") { } Run-R-Code-Redirect-Stderr "commandArgs <- function(...){$env:BUILD_R_FLAGS}; source('build_r.R')"; Check-Output $? } elseif ($env:R_BUILD_TYPE -eq "cran") { - choco install --yes --no-progress --no-color gzip Run-R-Code-Redirect-Stderr "result <- processx::run(command = 'sh', args = 'build-cran-package.sh', echo = TRUE, windows_verbatim_args = FALSE, error_on_status = TRUE)" ; Check-Output $? # Test CRAN source .tar.gz in a directory that is not this repo or below it. # When people install.packages('lightgbm'), they won't have the LightGBM diff --git a/R-package/DESCRIPTION b/R-package/DESCRIPTION index 6428ec8dba14..050008447622 100755 --- a/R-package/DESCRIPTION +++ b/R-package/DESCRIPTION @@ -42,8 +42,7 @@ URL: https://github.com/Microsoft/LightGBM BugReports: https://github.com/Microsoft/LightGBM/issues NeedsCompilation: yes Biarch: true -VignetteBuilder: - knitr +VignetteBuilder: knitr Suggests: knitr, processx, diff --git a/build_r.R b/build_r.R index fa4f99e375f9..3350d32516ce 100644 --- a/build_r.R +++ b/build_r.R @@ -367,6 +367,15 @@ description_contents <- gsub( , replacement = as.character(Sys.Date()) , x = description_contents ) + +# CMake-based builds don't get vignettes built. The +# VignetteBuilder field needs to be removed to avoid +# the R CMD CHECK note +# "Package has a VignetteBuilder field but no prebuilt vignette index" +description_contents <- description_contents[ + !grepl("^VignetteBuilder", description_contents) +] + writeLines(description_contents, DESCRIPTION_FILE) # NOTE: --keep-empty-dirs is necessary to keep the deep paths expected From e062a721285f1243d830e81a5089fe025e49246f Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 10 May 2021 18:25:05 -0500 Subject: [PATCH 32/82] try to fix Windows builds --- .ci/test_r_package_windows.ps1 | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.ci/test_r_package_windows.ps1 b/.ci/test_r_package_windows.ps1 index b4175497aa67..87515107b0f1 100644 --- a/.ci/test_r_package_windows.ps1 +++ b/.ci/test_r_package_windows.ps1 @@ -37,9 +37,12 @@ function Remove-From-Path { } # gzip is needed to create a CRAN package on Windows. -# Install it here before chocolatey is removed from PATH. +# Install it here before chocolatey is removed from PATH, then +# move it to a different location on PATH +$env:GZIP_INSTALL_LOCATION="C:\ProgramData\gzip" if ($env:R_BUILD_TYPE -eq "cran") { choco install --yes --no-progress --no-color gzip + Copy-Item -Path "C:\ProgramData\chocolatey\lib\gzip" -Destination "$env:GZIP_INSTALL_LOCATION" -Recurse } # remove some details that exist in the GitHub Actions images which might @@ -80,7 +83,7 @@ if ($env:R_MAJOR_VERSION -eq "3") { $env:R_LIB_PATH = "$env:BUILD_SOURCESDIRECTORY/RLibrary" -replace '[\\]', '/' $env:R_LIBS = "$env:R_LIB_PATH" -$env:PATH = "$env:RTOOLS_BIN;" + "$env:RTOOLS_MINGW_BIN;" + "$env:R_LIB_PATH/R/bin/x64;" + "$env:R_LIB_PATH/miktex/texmfs/install/miktex/bin/x64;" + $env:PATH +$env:PATH = "$env:RTOOLS_BIN;" + "$env:RTOOLS_MINGW_BIN;" + "$env:R_LIB_PATH/R/bin/x64;" + "$env:R_LIB_PATH/miktex/texmfs/install/miktex/bin/x64;" + "$env:GZIP_INSTALL_LOCATION;" + $env:PATH $env:CRAN_MIRROR = "https://cloud.r-project.org/" $env:CTAN_MIRROR = "https://ctan.math.illinois.edu/systems/win32/miktex" $env:CTAN_PACKAGE_ARCHIVE = "$env:CTAN_MIRROR/tm/packages/" From 5021729c5b827726d5482b63d766c05caaefd8f4 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 2 Aug 2021 03:49:36 +0100 Subject: [PATCH 33/82] use msys64 tar and gzip --- .ci/test_r_package_windows.ps1 | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.ci/test_r_package_windows.ps1 b/.ci/test_r_package_windows.ps1 index 87515107b0f1..9f327218ca56 100644 --- a/.ci/test_r_package_windows.ps1 +++ b/.ci/test_r_package_windows.ps1 @@ -36,13 +36,14 @@ function Remove-From-Path { $env:PATH = ($env:PATH.Split(';') | Where-Object { $_ -notmatch "$pattern_to_remove" }) -join ';' } -# gzip is needed to create a CRAN package on Windows. -# Install it here before chocolatey is removed from PATH, then -# move it to a different location on PATH -$env:GZIP_INSTALL_LOCATION="C:\ProgramData\gzip" +# gzip and tar are needed to create a CRAN package on Windows, but +# some flavors of tar.exe can fail in some settings on Windows +# +# copying the msys64 versions here and explicitly adding them to PATH +$env:GZIP_INSTALL_LOCATION="C:\ProgramData\" if ($env:R_BUILD_TYPE -eq "cran") { - choco install --yes --no-progress --no-color gzip - Copy-Item -Path "C:\ProgramData\chocolatey\lib\gzip" -Destination "$env:GZIP_INSTALL_LOCATION" -Recurse + Copy-Item -Path "C:\msys64\usr\bin\gzip" -Destination "$env:GZIP_INSTALL_LOCATION" -Recurse + Copy-Item -Path "C:\msys64\usr\bin\tar" -Destination "$env:GZIP_INSTALL_LOCATION" -Recurse } # remove some details that exist in the GitHub Actions images which might From eb45dff7e76ca2de34fb7c25c44b26183c7e19c8 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 2 Aug 2021 04:00:18 +0100 Subject: [PATCH 34/82] fix paths --- .ci/test_r_package_windows.ps1 | 3 +- .github/workflows/r_package.yml | 224 ++++++++++++++++---------------- 2 files changed, 113 insertions(+), 114 deletions(-) diff --git a/.ci/test_r_package_windows.ps1 b/.ci/test_r_package_windows.ps1 index bad4591bb9cd..8466d76768fc 100644 --- a/.ci/test_r_package_windows.ps1 +++ b/.ci/test_r_package_windows.ps1 @@ -42,8 +42,7 @@ function Remove-From-Path { # copying the msys64 versions here and explicitly adding them to PATH $env:GZIP_INSTALL_LOCATION="C:\ProgramData\" if ($env:R_BUILD_TYPE -eq "cran") { - Copy-Item -Path "C:\msys64\usr\bin\gzip" -Destination "$env:GZIP_INSTALL_LOCATION" -Recurse - Copy-Item -Path "C:\msys64\usr\bin\tar" -Destination "$env:GZIP_INSTALL_LOCATION" -Recurse + Copy-Item -Path "C:\msys64\usr\bin\tar.exe" -Destination "$env:GZIP_INSTALL_LOCATION" -Recurse } # remove some details that exist in the GitHub Actions images which might diff --git a/.github/workflows/r_package.yml b/.github/workflows/r_package.yml index 4261079ba156..e9e730b4f533 100644 --- a/.github/workflows/r_package.yml +++ b/.github/workflows/r_package.yml @@ -32,46 +32,46 @@ jobs: ################ # CMake builds # ################ - - os: ubuntu-latest - task: r-package - compiler: gcc - r_version: 3.6 - build_type: cmake - - os: ubuntu-latest - task: r-package - compiler: gcc - r_version: 4.0 - build_type: cmake - - os: ubuntu-latest - task: r-package - compiler: clang - r_version: 3.6 - build_type: cmake - - os: ubuntu-latest - task: r-package - compiler: clang - r_version: 4.0 - build_type: cmake - - os: macOS-latest - task: r-package - compiler: gcc - r_version: 3.6 - build_type: cmake - - os: macOS-latest - task: r-package - compiler: gcc - r_version: 4.0 - build_type: cmake - - os: macOS-latest - task: r-package - compiler: clang - r_version: 3.6 - build_type: cmake - - os: macOS-latest - task: r-package - compiler: clang - r_version: 4.0 - build_type: cmake + # - os: ubuntu-latest + # task: r-package + # compiler: gcc + # r_version: 3.6 + # build_type: cmake + # - os: ubuntu-latest + # task: r-package + # compiler: gcc + # r_version: 4.0 + # build_type: cmake + # - os: ubuntu-latest + # task: r-package + # compiler: clang + # r_version: 3.6 + # build_type: cmake + # - os: ubuntu-latest + # task: r-package + # compiler: clang + # r_version: 4.0 + # build_type: cmake + # - os: macOS-latest + # task: r-package + # compiler: gcc + # r_version: 3.6 + # build_type: cmake + # - os: macOS-latest + # task: r-package + # compiler: gcc + # r_version: 4.0 + # build_type: cmake + # - os: macOS-latest + # task: r-package + # compiler: clang + # r_version: 3.6 + # build_type: cmake + # - os: macOS-latest + # task: r-package + # compiler: clang + # r_version: 4.0 + # build_type: cmake - os: windows-latest task: r-package compiler: MINGW @@ -113,24 +113,24 @@ jobs: toolchain: MSYS r_version: 4.0 build_type: cran - - os: ubuntu-latest - task: r-package - compiler: gcc - r_version: 4.0 - build_type: cran - - os: macOS-latest - task: r-package - compiler: clang - r_version: 4.0 - build_type: cran + # - os: ubuntu-latest + # task: r-package + # compiler: gcc + # r_version: 4.0 + # build_type: cran + # - os: macOS-latest + # task: r-package + # compiler: clang + # r_version: 4.0 + # build_type: cran ################ # Other checks # ################ - - os: ubuntu-latest - task: r-rchk - compiler: gcc - r_version: 4.0 - build_type: cran + # - os: ubuntu-latest + # task: r-rchk + # compiler: gcc + # r_version: 4.0 + # build_type: cran steps: - name: Prevent conversion of line endings on Windows if: startsWith(matrix.os, 'windows') @@ -172,66 +172,66 @@ jobs: $env:GITHUB_ACTIONS = "true" $env:TASK = "${{ matrix.task }}" & "$env:GITHUB_WORKSPACE/.ci/test_windows.ps1" - test-r-sanitizers: - name: r-package (ubuntu-latest, R-devel, GCC ASAN/UBSAN) - timeout-minutes: 60 - runs-on: ubuntu-latest - container: rhub/rocker-gcc-san - steps: - - name: Install Git before checkout - shell: bash - run: | - apt-get update - apt-get install --no-install-recommends -y git - - name: Checkout repository - uses: actions/checkout@v2.3.4 - with: - fetch-depth: 5 - submodules: true - - name: Install packages - shell: bash - run: | - Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'rmarkdown', 'testthat'), repos = 'https://cran.r-project.org', Ncpus = parallel::detectCores())" - sh build-cran-package.sh - Rdevel CMD INSTALL lightgbm_*.tar.gz || exit -1 - - name: Run tests with sanitizers - shell: bash - run: | - cd R-package/tests - Rscriptdevel testthat.R 2>&1 > ubsan-tests.log - cat ubsan-tests.log - exit $(cat ubsan-tests.log | grep --count "runtime error") - test-r-debian-clang: - name: r-package (debian, R-devel, clang) - timeout-minutes: 60 - runs-on: ubuntu-latest - container: rhub/debian-clang-devel - steps: - - name: Install Git before checkout - shell: bash - run: | - apt-get update - apt-get install --no-install-recommends -y git - - name: Checkout repository - uses: actions/checkout@v2.3.4 - with: - fetch-depth: 5 - submodules: true - - name: Install packages and run tests - shell: bash - run: | - export PATH=/opt/R-devel/bin/:${PATH} - Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'rmarkdown', 'testthat'), repos = 'https://cran.r-project.org', Ncpus = parallel::detectCores())" - sh build-cran-package.sh - R CMD check --as-cran --run-donttest lightgbm_*.tar.gz || exit -1 - if grep -q -E "NOTE|WARNING|ERROR" lightgbm.Rcheck/00check.log; then - echo "NOTEs, WARNINGs, or ERRORs have been found by R CMD check" - exit -1 - fi + # test-r-sanitizers: + # name: r-package (ubuntu-latest, R-devel, GCC ASAN/UBSAN) + # timeout-minutes: 60 + # runs-on: ubuntu-latest + # container: rhub/rocker-gcc-san + # steps: + # - name: Install Git before checkout + # shell: bash + # run: | + # apt-get update + # apt-get install --no-install-recommends -y git + # - name: Checkout repository + # uses: actions/checkout@v2.3.4 + # with: + # fetch-depth: 5 + # submodules: true + # - name: Install packages + # shell: bash + # run: | + # Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'rmarkdown', 'testthat'), repos = 'https://cran.r-project.org', Ncpus = parallel::detectCores())" + # sh build-cran-package.sh + # Rdevel CMD INSTALL lightgbm_*.tar.gz || exit -1 + # - name: Run tests with sanitizers + # shell: bash + # run: | + # cd R-package/tests + # Rscriptdevel testthat.R 2>&1 > ubsan-tests.log + # cat ubsan-tests.log + # exit $(cat ubsan-tests.log | grep --count "runtime error") + # test-r-debian-clang: + # name: r-package (debian, R-devel, clang) + # timeout-minutes: 60 + # runs-on: ubuntu-latest + # container: rhub/debian-clang-devel + # steps: + # - name: Install Git before checkout + # shell: bash + # run: | + # apt-get update + # apt-get install --no-install-recommends -y git + # - name: Checkout repository + # uses: actions/checkout@v2.3.4 + # with: + # fetch-depth: 5 + # submodules: true + # - name: Install packages and run tests + # shell: bash + # run: | + # export PATH=/opt/R-devel/bin/:${PATH} + # Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'rmarkdown', 'testthat'), repos = 'https://cran.r-project.org', Ncpus = parallel::detectCores())" + # sh build-cran-package.sh + # R CMD check --as-cran --run-donttest lightgbm_*.tar.gz || exit -1 + # if grep -q -E "NOTE|WARNING|ERROR" lightgbm.Rcheck/00check.log; then + # echo "NOTEs, WARNINGs, or ERRORs have been found by R CMD check" + # exit -1 + # fi all-successful: # https://github.community/t/is-it-possible-to-require-all-github-actions-tasks-to-pass-without-enumerating-them/117957/4?u=graingert runs-on: ubuntu-latest - needs: [test, test-r-sanitizers, test-r-debian-clang] + needs: [test] steps: - name: Note that all tests succeeded run: echo "🎉" From 86b31a7189c6b5334c1c1857f87df10745ddb183 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 2 Aug 2021 04:14:23 +0100 Subject: [PATCH 35/82] put msys tar first --- .ci/test_r_package_windows.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/test_r_package_windows.ps1 b/.ci/test_r_package_windows.ps1 index 8466d76768fc..601c8ffcdc13 100644 --- a/.ci/test_r_package_windows.ps1 +++ b/.ci/test_r_package_windows.ps1 @@ -95,7 +95,7 @@ if ($env:R_MAJOR_VERSION -eq "3") { $env:R_LIB_PATH = "$env:BUILD_SOURCESDIRECTORY/RLibrary" -replace '[\\]', '/' $env:R_LIBS = "$env:R_LIB_PATH" -$env:PATH = "$env:RTOOLS_BIN;" + "$env:RTOOLS_MINGW_BIN;" + "$env:R_LIB_PATH/R/bin/x64;" + "$env:R_LIB_PATH/miktex/texmfs/install/miktex/bin/x64;" + "$env:GZIP_INSTALL_LOCATION;" + $env:PATH +$env:PATH = "$env:GZIP_INSTALL_LOCATION;" + "$env:RTOOLS_BIN;" + "$env:RTOOLS_MINGW_BIN;" + "$env:R_LIB_PATH/R/bin/x64;" + "$env:R_LIB_PATH/miktex/texmfs/install/miktex/bin/x64;" + $env:PATH $env:CRAN_MIRROR = "https://cloud.r-project.org/" $env:CTAN_MIRROR = "https://ctan.math.illinois.edu/systems/win32/miktex" $env:CTAN_PACKAGE_ARCHIVE = "$env:CTAN_MIRROR/tm/packages/" From ddd8b32058341be45176d30b61a9c0a3ce3b4c69 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sun, 1 Aug 2021 22:44:37 -0500 Subject: [PATCH 36/82] more fiddling with tar --- build-cran-package.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/build-cran-package.sh b/build-cran-package.sh index d4a4ad84562a..da1adb896603 100755 --- a/build-cran-package.sh +++ b/build-cran-package.sh @@ -151,11 +151,14 @@ mkdir _tmp TARBALL_NAME=lightgbm_${LGB_VERSION}.tar.gz mv ${TARBALL_NAME} _tmp/ +echo "untarring ${TARBALL_NAME}" cd _tmp tar -xvf ${TARBALL_NAME} rm -rf ${TARBALL_NAME} cd .. +echo "done untarring ${TARBALL_NAME}" +echo "re-tarring ${TARBALL_NAME}" tar \ -czv \ -C ./_tmp \ @@ -166,6 +169,7 @@ tar \ --exclude=*.tar.gz \ -f ${TARBALL_NAME} \ lightgbm +echo "Done creating ${TARBALL_NAME}" rm -rf ./_tmp From bfb1d58bd7d96f3edf2b4193296d86ca2a51b26b Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sun, 1 Aug 2021 23:12:15 -0500 Subject: [PATCH 37/82] more excludes and use quotes --- build-cran-package.sh | 54 ++++++++++++++++++++++--------------------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/build-cran-package.sh b/build-cran-package.sh index da1adb896603..086fbe5ebd02 100755 --- a/build-cran-package.sh +++ b/build-cran-package.sh @@ -12,10 +12,10 @@ set -e ORIG_WD=$(pwd) TEMP_R_DIR=$(pwd)/lightgbm_r -if test -d ${TEMP_R_DIR}; then - rm -r ${TEMP_R_DIR} +if test -d "${TEMP_R_DIR}"; then + rm -r "${TEMP_R_DIR}" fi -mkdir -p ${TEMP_R_DIR} +mkdir -p "${TEMP_R_DIR}" CURRENT_DATE=$(date +'%Y-%m-%d') @@ -24,42 +24,42 @@ CURRENT_DATE=$(date +'%Y-%m-%d') LGB_VERSION=$(cat VERSION.txt | sed "s/rc/-/g") # move relevant files -cp -R R-package/* ${TEMP_R_DIR} -cp docs/logo/LightGBM_logo_black_text.svg ${TEMP_R_DIR}/vignettes/ -cp -R include ${TEMP_R_DIR}/src/ -cp -R src/* ${TEMP_R_DIR}/src/ +cp -R R-package/* "${TEMP_R_DIR}" +cp docs/logo/LightGBM_logo_black_text.svg "${TEMP_R_DIR}/vignettes/" +cp -R include "${TEMP_R_DIR}/src/" +cp -R src/* "${TEMP_R_DIR}/src/" cp \ external_libs/fast_double_parser/include/fast_double_parser.h \ - ${TEMP_R_DIR}/src/include/LightGBM + "${TEMP_R_DIR}/src/include/LightGBM" -mkdir -p ${TEMP_R_DIR}/src/include/LightGBM/fmt +mkdir -p "${TEMP_R_DIR}/src/include/LightGBM/fmt" cp \ external_libs/fmt/include/fmt/*.h \ - ${TEMP_R_DIR}/src/include/LightGBM/fmt/ + "${TEMP_R_DIR}/src/include/LightGBM/fmt/" # including only specific files from Eigen, to keep the R package # small and avoid redistributing code with licenses incompatible with # LightGBM's license -EIGEN_R_DIR=${TEMP_R_DIR}/src/include/Eigen -mkdir -p ${EIGEN_R_DIR} +EIGEN_R_DIR="${TEMP_R_DIR}/src/include/Eigen" +mkdir -p "${EIGEN_R_DIR}" modules="Cholesky Core Dense Eigenvalues Geometry Householder Jacobi LU QR SVD" for eigen_module in ${modules}; do - cp external_libs/eigen/Eigen/${eigen_module} ${EIGEN_R_DIR}/${eigen_module} + cp external_libs/eigen/Eigen/${eigen_module} "${EIGEN_R_DIR}/${eigen_module}" if [ ${eigen_module} != "Dense" ]; then - mkdir -p ${EIGEN_R_DIR}/src/${eigen_module}/ - cp -R external_libs/eigen/Eigen/src/${eigen_module}/* ${EIGEN_R_DIR}/src/${eigen_module}/ + mkdir -p "${EIGEN_R_DIR}/src/${eigen_module}/" + cp -R external_libs/eigen/Eigen/src/${eigen_module}/* "${EIGEN_R_DIR}/src/${eigen_module}/" fi done -mkdir -p ${EIGEN_R_DIR}/src/misc -cp -R external_libs/eigen/Eigen/src/misc/* ${EIGEN_R_DIR}/src/misc/ +mkdir -p "${EIGEN_R_DIR}/src/misc" +cp -R external_libs/eigen/Eigen/src/misc/* "${EIGEN_R_DIR}/src/misc/" -mkdir -p ${EIGEN_R_DIR}/src/plugins -cp -R external_libs/eigen/Eigen/src/plugins/* ${EIGEN_R_DIR}/src/plugins/ +mkdir -p "${EIGEN_R_DIR}/src/plugins" +cp -R external_libs/eigen/Eigen/src/plugins/* "${EIGEN_R_DIR}/src/plugins/" -cd ${TEMP_R_DIR} +cd "${TEMP_R_DIR}" # Remove files not needed for CRAN echo "Removing files not needed for CRAN" @@ -139,7 +139,7 @@ cd ${TEMP_R_DIR} rm R/*.R.bak rm NAMESPACE.bak -cd ${ORIG_WD} +cd "${ORIG_WD}" R CMD build \ --keep-empty-dirs \ @@ -148,13 +148,13 @@ R CMD build \ echo "removing object files created by vignettes" rm -rf ./_tmp mkdir _tmp -TARBALL_NAME=lightgbm_${LGB_VERSION}.tar.gz -mv ${TARBALL_NAME} _tmp/ +TARBALL_NAME="lightgbm_${LGB_VERSION}.tar.gz" +mv "${TARBALL_NAME}" _tmp/ echo "untarring ${TARBALL_NAME}" cd _tmp - tar -xvf ${TARBALL_NAME} - rm -rf ${TARBALL_NAME} + tar -xvf "${TARBALL_NAME}" + rm -rf "${TARBALL_NAME}" cd .. echo "done untarring ${TARBALL_NAME}" @@ -167,7 +167,9 @@ tar \ --exclude=*.o \ --exclude=*.so \ --exclude=*.tar.gz \ - -f ${TARBALL_NAME} \ + --exclude=**/conftest.c \ + --exclude=**/conftest.exe \ + -f "${TARBALL_NAME}" \ lightgbm echo "Done creating ${TARBALL_NAME}" From b8a92160b2d44d3aa40796c256d350a9049ecd1a Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sun, 1 Aug 2021 23:29:33 -0500 Subject: [PATCH 38/82] trying to get more information about the available gzip --- .ci/test_r_package_windows.ps1 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.ci/test_r_package_windows.ps1 b/.ci/test_r_package_windows.ps1 index 601c8ffcdc13..62fe85faf502 100644 --- a/.ci/test_r_package_windows.ps1 +++ b/.ci/test_r_package_windows.ps1 @@ -40,6 +40,9 @@ function Remove-From-Path { # some flavors of tar.exe can fail in some settings on Windows # # copying the msys64 versions here and explicitly adding them to PATH +Write-Output "---- listing msys ----" +Get-ChildItem -Path "C:\msys64" -Recurse +Write-Output "---- done listing msys ----" $env:GZIP_INSTALL_LOCATION="C:\ProgramData\" if ($env:R_BUILD_TYPE -eq "cran") { Copy-Item -Path "C:\msys64\usr\bin\tar.exe" -Destination "$env:GZIP_INSTALL_LOCATION" -Recurse From da9fd20266e84013f7ea2b328039d33a9e52f88e Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sun, 1 Aug 2021 23:48:08 -0500 Subject: [PATCH 39/82] move both gzip and gunzip --- .ci/test_r_package_windows.ps1 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.ci/test_r_package_windows.ps1 b/.ci/test_r_package_windows.ps1 index 62fe85faf502..406c5da41d0c 100644 --- a/.ci/test_r_package_windows.ps1 +++ b/.ci/test_r_package_windows.ps1 @@ -46,6 +46,8 @@ Write-Output "---- done listing msys ----" $env:GZIP_INSTALL_LOCATION="C:\ProgramData\" if ($env:R_BUILD_TYPE -eq "cran") { Copy-Item -Path "C:\msys64\usr\bin\tar.exe" -Destination "$env:GZIP_INSTALL_LOCATION" -Recurse + Copy-Item -Path "C:\msys64\usr\bin\gzip.exe" -Destination "$env:GZIP_INSTALL_LOCATION" -Recurse + Copy-Item -Path "C:\msys64\usr\bin\gunzip" -Destination "$env:GZIP_INSTALL_LOCATION" -Recurse } # remove some details that exist in the GitHub Actions images which might From 6097769e53e03ca1f198aaaddb5b8a5ee6a3d15c Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 2 Aug 2021 14:14:08 -0500 Subject: [PATCH 40/82] use msys sh and try to get stderr in logs --- .ci/test_r_package_windows.ps1 | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.ci/test_r_package_windows.ps1 b/.ci/test_r_package_windows.ps1 index 406c5da41d0c..1e6997ad5aee 100644 --- a/.ci/test_r_package_windows.ps1 +++ b/.ci/test_r_package_windows.ps1 @@ -41,13 +41,14 @@ function Remove-From-Path { # # copying the msys64 versions here and explicitly adding them to PATH Write-Output "---- listing msys ----" -Get-ChildItem -Path "C:\msys64" -Recurse +#Get-ChildItem -Path "C:\msys64" -Recurse Write-Output "---- done listing msys ----" $env:GZIP_INSTALL_LOCATION="C:\ProgramData\" if ($env:R_BUILD_TYPE -eq "cran") { Copy-Item -Path "C:\msys64\usr\bin\tar.exe" -Destination "$env:GZIP_INSTALL_LOCATION" -Recurse Copy-Item -Path "C:\msys64\usr\bin\gzip.exe" -Destination "$env:GZIP_INSTALL_LOCATION" -Recurse Copy-Item -Path "C:\msys64\usr\bin\gunzip" -Destination "$env:GZIP_INSTALL_LOCATION" -Recurse + Copy-Item -Path "C:\msys64\usr\bin\sh.exe" -Destination "$env:GZIP_INSTALL_LOCATION" -Recurse } # remove some details that exist in the GitHub Actions images which might @@ -178,7 +179,8 @@ if ($env:COMPILER -ne "MSVC") { } Run-R-Code-Redirect-Stderr "commandArgs <- function(...){$env:BUILD_R_FLAGS}; source('build_r.R')"; Check-Output $? } elseif ($env:R_BUILD_TYPE -eq "cran") { - Run-R-Code-Redirect-Stderr "result <- processx::run(command = 'sh', args = 'build-cran-package.sh', echo = TRUE, windows_verbatim_args = FALSE, error_on_status = TRUE)" ; Check-Output $? + #Run-R-Code-Redirect-Stderr "result <- processx::run(command = 'sh', args = 'build-cran-package.sh', echo = TRUE, windows_verbatim_args = FALSE, error_on_status = TRUE)" ; Check-Output $? + sh build-cran-package.sh # Test CRAN source .tar.gz in a directory that is not this repo or below it. # When people install.packages('lightgbm'), they won't have the LightGBM # git repo around. This is to protect against the use of relative paths From 60b5373f0de35b2f2245519375876bc678b3a5f7 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 2 Aug 2021 14:26:31 -0500 Subject: [PATCH 41/82] trying to get more logs --- .ci/test_r_package_windows.ps1 | 3 +- .github/workflows/r_package.yml | 52 ++++++++++++++++----------------- 2 files changed, 28 insertions(+), 27 deletions(-) diff --git a/.ci/test_r_package_windows.ps1 b/.ci/test_r_package_windows.ps1 index 1e6997ad5aee..61e120e63288 100644 --- a/.ci/test_r_package_windows.ps1 +++ b/.ci/test_r_package_windows.ps1 @@ -180,7 +180,8 @@ if ($env:COMPILER -ne "MSVC") { Run-R-Code-Redirect-Stderr "commandArgs <- function(...){$env:BUILD_R_FLAGS}; source('build_r.R')"; Check-Output $? } elseif ($env:R_BUILD_TYPE -eq "cran") { #Run-R-Code-Redirect-Stderr "result <- processx::run(command = 'sh', args = 'build-cran-package.sh', echo = TRUE, windows_verbatim_args = FALSE, error_on_status = TRUE)" ; Check-Output $? - sh build-cran-package.sh + Write-Output "running build-cran-package.sh" + sh "build-cran-package.sh" # Test CRAN source .tar.gz in a directory that is not this repo or below it. # When people install.packages('lightgbm'), they won't have the LightGBM # git repo around. This is to protect against the use of relative paths diff --git a/.github/workflows/r_package.yml b/.github/workflows/r_package.yml index e9e730b4f533..68bd31f4630c 100644 --- a/.github/workflows/r_package.yml +++ b/.github/workflows/r_package.yml @@ -72,32 +72,32 @@ jobs: # compiler: clang # r_version: 4.0 # build_type: cmake - - os: windows-latest - task: r-package - compiler: MINGW - toolchain: MINGW - r_version: 3.6 - build_type: cmake - - os: windows-latest - task: r-package - compiler: MINGW - toolchain: MSYS - r_version: 4.0 - build_type: cmake - # Visual Studio 2017 - - os: windows-2016 - task: r-package - compiler: MSVC - toolchain: MSVC - r_version: 3.6 - build_type: cmake - # Visual Studio 2019 - - os: windows-2019 - task: r-package - compiler: MSVC - toolchain: MSVC - r_version: 4.0 - build_type: cmake + # - os: windows-latest + # task: r-package + # compiler: MINGW + # toolchain: MINGW + # r_version: 3.6 + # build_type: cmake + # - os: windows-latest + # task: r-package + # compiler: MINGW + # toolchain: MSYS + # r_version: 4.0 + # build_type: cmake + # # Visual Studio 2017 + # - os: windows-2016 + # task: r-package + # compiler: MSVC + # toolchain: MSVC + # r_version: 3.6 + # build_type: cmake + # # Visual Studio 2019 + # - os: windows-2019 + # task: r-package + # compiler: MSVC + # toolchain: MSVC + # r_version: 4.0 + # build_type: cmake ############### # CRAN builds # ############### From 2833387cac3043c17ba21b82fa6ac64ae3e2d42a Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 2 Aug 2021 14:37:02 -0500 Subject: [PATCH 42/82] what is happening --- .ci/test_r_package_windows.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/test_r_package_windows.ps1 b/.ci/test_r_package_windows.ps1 index 61e120e63288..2d5733dd9458 100644 --- a/.ci/test_r_package_windows.ps1 +++ b/.ci/test_r_package_windows.ps1 @@ -181,7 +181,7 @@ if ($env:COMPILER -ne "MSVC") { } elseif ($env:R_BUILD_TYPE -eq "cran") { #Run-R-Code-Redirect-Stderr "result <- processx::run(command = 'sh', args = 'build-cran-package.sh', echo = TRUE, windows_verbatim_args = FALSE, error_on_status = TRUE)" ; Check-Output $? Write-Output "running build-cran-package.sh" - sh "build-cran-package.sh" + sh.exe ".\build-cran-package.sh" # Test CRAN source .tar.gz in a directory that is not this repo or below it. # When people install.packages('lightgbm'), they won't have the LightGBM # git repo around. This is to protect against the use of relative paths From c46144cb924a4810bdb696ea20a56b6c92ac0c84 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 2 Aug 2021 20:23:57 -0500 Subject: [PATCH 43/82] ok, another option --- .ci/test_r_package_windows.ps1 | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/.ci/test_r_package_windows.ps1 b/.ci/test_r_package_windows.ps1 index 2d5733dd9458..7ec17f0b59f2 100644 --- a/.ci/test_r_package_windows.ps1 +++ b/.ci/test_r_package_windows.ps1 @@ -43,13 +43,14 @@ function Remove-From-Path { Write-Output "---- listing msys ----" #Get-ChildItem -Path "C:\msys64" -Recurse Write-Output "---- done listing msys ----" -$env:GZIP_INSTALL_LOCATION="C:\ProgramData\" -if ($env:R_BUILD_TYPE -eq "cran") { - Copy-Item -Path "C:\msys64\usr\bin\tar.exe" -Destination "$env:GZIP_INSTALL_LOCATION" -Recurse - Copy-Item -Path "C:\msys64\usr\bin\gzip.exe" -Destination "$env:GZIP_INSTALL_LOCATION" -Recurse - Copy-Item -Path "C:\msys64\usr\bin\gunzip" -Destination "$env:GZIP_INSTALL_LOCATION" -Recurse - Copy-Item -Path "C:\msys64\usr\bin\sh.exe" -Destination "$env:GZIP_INSTALL_LOCATION" -Recurse -} +$env:GZIP_INSTALL_LOCATION="C:\msys64\usr\bin" +# if ($env:R_BUILD_TYPE -eq "cran") { +# Copy-Item -Path "C:\msys64\usr\bin\tar.exe" -Destination "$env:GZIP_INSTALL_LOCATION" -Recurse +# Copy-Item -Path "C:\msys64\usr\bin\gzip.exe" -Destination "$env:GZIP_INSTALL_LOCATION" -Recurse +# Copy-Item -Path "C:\msys64\usr\bin\gunzip" -Destination "$env:GZIP_INSTALL_LOCATION" -Recurse +# Copy-Item -Path "C:\msys64\usr\bin\pwd.exe" -Destination "$env:GZIP_INSTALL_LOCATION" -Recurse +# Copy-Item -Path "C:\msys64\usr\bin\sh.exe" -Destination "$env:GZIP_INSTALL_LOCATION" -Recurse +# } # remove some details that exist in the GitHub Actions images which might # cause conflicts with R and other components installed by this script From 65f550b0a9b9f69c3f42e19e23c778539fe1fe8d Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 2 Aug 2021 22:59:21 -0500 Subject: [PATCH 44/82] start simplifying --- .ci/test_r_package_windows.ps1 | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/.ci/test_r_package_windows.ps1 b/.ci/test_r_package_windows.ps1 index 7ec17f0b59f2..3600b9c32186 100644 --- a/.ci/test_r_package_windows.ps1 +++ b/.ci/test_r_package_windows.ps1 @@ -36,22 +36,6 @@ function Remove-From-Path { $env:PATH = ($env:PATH.Split(';') | Where-Object { $_ -notmatch "$pattern_to_remove" }) -join ';' } -# gzip and tar are needed to create a CRAN package on Windows, but -# some flavors of tar.exe can fail in some settings on Windows -# -# copying the msys64 versions here and explicitly adding them to PATH -Write-Output "---- listing msys ----" -#Get-ChildItem -Path "C:\msys64" -Recurse -Write-Output "---- done listing msys ----" -$env:GZIP_INSTALL_LOCATION="C:\msys64\usr\bin" -# if ($env:R_BUILD_TYPE -eq "cran") { -# Copy-Item -Path "C:\msys64\usr\bin\tar.exe" -Destination "$env:GZIP_INSTALL_LOCATION" -Recurse -# Copy-Item -Path "C:\msys64\usr\bin\gzip.exe" -Destination "$env:GZIP_INSTALL_LOCATION" -Recurse -# Copy-Item -Path "C:\msys64\usr\bin\gunzip" -Destination "$env:GZIP_INSTALL_LOCATION" -Recurse -# Copy-Item -Path "C:\msys64\usr\bin\pwd.exe" -Destination "$env:GZIP_INSTALL_LOCATION" -Recurse -# Copy-Item -Path "C:\msys64\usr\bin\sh.exe" -Destination "$env:GZIP_INSTALL_LOCATION" -Recurse -# } - # remove some details that exist in the GitHub Actions images which might # cause conflicts with R and other components installed by this script $env:RTOOLS40_HOME = "" @@ -102,7 +86,11 @@ if ($env:R_MAJOR_VERSION -eq "3") { $env:R_LIB_PATH = "$env:BUILD_SOURCESDIRECTORY/RLibrary" -replace '[\\]', '/' $env:R_LIBS = "$env:R_LIB_PATH" -$env:PATH = "$env:GZIP_INSTALL_LOCATION;" + "$env:RTOOLS_BIN;" + "$env:RTOOLS_MINGW_BIN;" + "$env:R_LIB_PATH/R/bin/x64;" + "$env:R_LIB_PATH/miktex/texmfs/install/miktex/bin/x64;" + $env:PATH +# NOTE: gzip and tar are needed to create a CRAN package on Windows, but +# some flavors of tar.exe can fail in some settings on Windows. Putting +# the msys64 utilities at the beginning of PATH to be sure they're used +# for that purpose. +$env:PATH = "C:\msys64\usr\bin;" + "$env:RTOOLS_BIN;" + "$env:RTOOLS_MINGW_BIN;" + "$env:R_LIB_PATH/R/bin/x64;" + "$env:R_LIB_PATH/miktex/texmfs/install/miktex/bin/x64;" + $env:PATH $env:CRAN_MIRROR = "https://cloud.r-project.org/" $env:CTAN_MIRROR = "https://ctan.math.illinois.edu/systems/win32/miktex" $env:CTAN_PACKAGE_ARCHIVE = "$env:CTAN_MIRROR/tm/packages/" From 1d3e14abe5e609dbe3973c172d203a03373642d1 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 2 Aug 2021 23:19:26 -0500 Subject: [PATCH 45/82] undo unnecessary Rbuildignore changes --- R-package/.Rbuildignore | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/R-package/.Rbuildignore b/R-package/.Rbuildignore index 537a860c8dc3..3e552132cf22 100644 --- a/R-package/.Rbuildignore +++ b/R-package/.Rbuildignore @@ -1,6 +1,3 @@ -\.o$ -\.so$ -\.dll$ \.appveyor\.yml AUTOCONF_UBUNTU_VERSION ^autom4te.cache/.*$ @@ -9,14 +6,17 @@ AUTOCONF_UBUNTU_VERSION \.clang-format ^cran-comments\.md$ ^docs$ +^.*\.dll \.drone\.yml \.git \.gitkeep$ ^.*\.history ^Makefile$ +^.*\.o ^.*\.out ^pkgdown$ ^recreate-configure\.sh$ +^.*\.so ^src/build/.*$ ^src/CMakeLists.txt$ ^src/external_libs/compute/.appveyor.yml$ From 5c99c35552c4fce37034b6014d863fcc085d4292 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 3 Aug 2021 11:38:52 -0500 Subject: [PATCH 46/82] choose whether or not to build vignettes based on env variable --- .ci/test_r_package_windows.ps1 | 12 ++++- build-cran-package.sh | 83 ++++++++++++++++++++-------------- 2 files changed, 60 insertions(+), 35 deletions(-) diff --git a/.ci/test_r_package_windows.ps1 b/.ci/test_r_package_windows.ps1 index 3600b9c32186..afb5f35a67eb 100644 --- a/.ci/test_r_package_windows.ps1 +++ b/.ci/test_r_package_windows.ps1 @@ -84,6 +84,11 @@ if ($env:R_MAJOR_VERSION -eq "3") { Check-Output $false } +# only build vignettes for CRAN-style builds +if ($env:R_BUILD_TYPE -eq "cran") { + $env:LGB_BUILD_VIGNETTES = "true" +} + $env:R_LIB_PATH = "$env:BUILD_SOURCESDIRECTORY/RLibrary" -replace '[\\]', '/' $env:R_LIBS = "$env:R_LIB_PATH" # NOTE: gzip and tar are needed to create a CRAN package on Windows, but @@ -125,8 +130,11 @@ Start-Process -FilePath Rtools.exe -NoNewWindow -Wait -ArgumentList "/VERYSILENT Write-Output "Done installing Rtools" Write-Output "Installing dependencies" -$packages = "c('data.table', 'jsonlite', 'Matrix', 'processx', 'R6', 'testthat', 'knitr', 'rmarkdown'), dependencies = c('Imports', 'Depends', 'LinkingTo')" -Run-R-Code-Redirect-Stderr "options(install.packages.check.source = 'no'); install.packages($packages, repos = '$env:CRAN_MIRROR', type = 'binary', lib = '$env:R_LIB_PATH', Ncpus = parallel::detectCores())" ; Check-Output $? +$packages = "c('data.table', 'jsonlite', 'Matrix', 'processx', 'R6', 'testthat')" +if ($env:LGB_BUILD_VIGNETTES -eq "true") { + $packages = "$packages, 'knitr', 'rmarkdown'" +} +Run-R-Code-Redirect-Stderr "options(install.packages.check.source = 'no'); install.packages(c($packages), dependencies = c('Imports', 'Depends', 'LinkingTo'), repos = '$env:CRAN_MIRROR', type = 'binary', lib = '$env:R_LIB_PATH', Ncpus = parallel::detectCores())" ; Check-Output $? # MiKTeX and pandoc can be skipped on non-MinGW builds, since we don't # build the package documentation for those. diff --git a/build-cran-package.sh b/build-cran-package.sh index 5a803c36e642..b3a00a327cba 100755 --- a/build-cran-package.sh +++ b/build-cran-package.sh @@ -4,14 +4,24 @@ # Prepare a source distribution of the R package # to be submitted to CRAN. # +# Vignette building is skipped by default because it requires +# some shell features (tar and gzip) which can cause compatibility +# issues on Windows. +# +# To build a package with vignettes, set environment variable +# `LGB_BUILD_VIGNETTES=true`. +# # [usage] # sh build-cran-package.sh set -e +LGB_BUILD_VIGNETTES=${LGB_BUILD_VIGNETTES:-false} ORIG_WD="$(pwd)" TEMP_R_DIR="$(pwd)/lightgbm_r" +echo "Building R package (build vignettes: ${LGB_BUILD_VIGNETTES})" + if test -d "${TEMP_R_DIR}"; then rm -r "${TEMP_R_DIR}" fi @@ -141,38 +151,45 @@ cd "${TEMP_R_DIR}" cd "${ORIG_WD}" -R CMD build \ - --keep-empty-dirs \ - lightgbm_r - -echo "removing object files created by vignettes" -rm -rf ./_tmp -mkdir _tmp -TARBALL_NAME="lightgbm_${LGB_VERSION}.tar.gz" -mv "${TARBALL_NAME}" _tmp/ - -echo "untarring ${TARBALL_NAME}" -cd _tmp - tar -xvf "${TARBALL_NAME}" - rm -rf "${TARBALL_NAME}" -cd .. -echo "done untarring ${TARBALL_NAME}" - -echo "re-tarring ${TARBALL_NAME}" -tar \ - -czv \ - -C ./_tmp \ - --exclude=*.a \ - --exclude=*.dll \ - --exclude=*.o \ - --exclude=*.so \ - --exclude=*.tar.gz \ - --exclude=**/conftest.c \ - --exclude=**/conftest.exe \ - -f "${TARBALL_NAME}" \ - lightgbm -echo "Done creating ${TARBALL_NAME}" - -rm -rf ./_tmp +if [[ $LGB_BUILD_VIGNETTES == "false" ]]; then + R CMD build \ + --keep-empty-dirs \ + --no-build-vignettes \ + lightgbm_r +else + R CMD build \ + --keep-empty-dirs \ + lightgbm_r + + echo "removing object files created by vignettes" + rm -rf ./_tmp + mkdir _tmp + TARBALL_NAME="lightgbm_${LGB_VERSION}.tar.gz" + mv "${TARBALL_NAME}" _tmp/ + + echo "untarring ${TARBALL_NAME}" + cd _tmp + tar -xvf "${TARBALL_NAME}" + rm -rf "${TARBALL_NAME}" + cd .. + echo "done untarring ${TARBALL_NAME}" + + echo "re-tarring ${TARBALL_NAME}" + tar \ + -czv \ + -C ./_tmp \ + --exclude=*.a \ + --exclude=*.dll \ + --exclude=*.o \ + --exclude=*.so \ + --exclude=*.tar.gz \ + --exclude=**/conftest.c \ + --exclude=**/conftest.exe \ + -f "${TARBALL_NAME}" \ + lightgbm + echo "Done creating ${TARBALL_NAME}" + + rm -rf ./_tmp +fi echo "Done building R package" From 07b62571ad0b506ee4ec4496906ec81724edd0f6 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 3 Aug 2021 13:47:22 -0500 Subject: [PATCH 47/82] more windows job, and standard approach to running build script --- .ci/test_r_package_windows.ps1 | 4 +-- .github/workflows/r_package.yml | 52 ++++++++++++++++----------------- 2 files changed, 27 insertions(+), 29 deletions(-) diff --git a/.ci/test_r_package_windows.ps1 b/.ci/test_r_package_windows.ps1 index afb5f35a67eb..31d0e91f92ee 100644 --- a/.ci/test_r_package_windows.ps1 +++ b/.ci/test_r_package_windows.ps1 @@ -176,9 +176,7 @@ if ($env:COMPILER -ne "MSVC") { } Run-R-Code-Redirect-Stderr "commandArgs <- function(...){$env:BUILD_R_FLAGS}; source('build_r.R')"; Check-Output $? } elseif ($env:R_BUILD_TYPE -eq "cran") { - #Run-R-Code-Redirect-Stderr "result <- processx::run(command = 'sh', args = 'build-cran-package.sh', echo = TRUE, windows_verbatim_args = FALSE, error_on_status = TRUE)" ; Check-Output $? - Write-Output "running build-cran-package.sh" - sh.exe ".\build-cran-package.sh" + Run-R-Code-Redirect-Stderr "result <- processx::run(command = 'sh', args = 'build-cran-package.sh', echo = TRUE, windows_verbatim_args = FALSE, error_on_status = TRUE)" ; Check-Output $? # Test CRAN source .tar.gz in a directory that is not this repo or below it. # When people install.packages('lightgbm'), they won't have the LightGBM # git repo around. This is to protect against the use of relative paths diff --git a/.github/workflows/r_package.yml b/.github/workflows/r_package.yml index 68bd31f4630c..e9e730b4f533 100644 --- a/.github/workflows/r_package.yml +++ b/.github/workflows/r_package.yml @@ -72,32 +72,32 @@ jobs: # compiler: clang # r_version: 4.0 # build_type: cmake - # - os: windows-latest - # task: r-package - # compiler: MINGW - # toolchain: MINGW - # r_version: 3.6 - # build_type: cmake - # - os: windows-latest - # task: r-package - # compiler: MINGW - # toolchain: MSYS - # r_version: 4.0 - # build_type: cmake - # # Visual Studio 2017 - # - os: windows-2016 - # task: r-package - # compiler: MSVC - # toolchain: MSVC - # r_version: 3.6 - # build_type: cmake - # # Visual Studio 2019 - # - os: windows-2019 - # task: r-package - # compiler: MSVC - # toolchain: MSVC - # r_version: 4.0 - # build_type: cmake + - os: windows-latest + task: r-package + compiler: MINGW + toolchain: MINGW + r_version: 3.6 + build_type: cmake + - os: windows-latest + task: r-package + compiler: MINGW + toolchain: MSYS + r_version: 4.0 + build_type: cmake + # Visual Studio 2017 + - os: windows-2016 + task: r-package + compiler: MSVC + toolchain: MSVC + r_version: 3.6 + build_type: cmake + # Visual Studio 2019 + - os: windows-2019 + task: r-package + compiler: MSVC + toolchain: MSVC + r_version: 4.0 + build_type: cmake ############### # CRAN builds # ############### From 815163f5e1700332d2581fa6479425bdbbe7cdc2 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 3 Aug 2021 15:14:45 -0500 Subject: [PATCH 48/82] clean up Suggests in DESCRIPTION for CMake-based builds --- build_r.R | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/build_r.R b/build_r.R index b980f4629428..25b5308a92e5 100644 --- a/build_r.R +++ b/build_r.R @@ -376,6 +376,15 @@ description_contents <- description_contents[ !grepl("^VignetteBuilder", description_contents) ] +# {knitr} and {rmarkdown} can be removed, since we don't build vignettes +# for CMake-based installations +description_contents <- description_contents[ + !grepl("^ +knitr,$", description_contents) +] +description_contents <- description_contents[ + !grepl("^ +rmarkdown,$", description_contents) +] + writeLines(description_contents, DESCRIPTION_FILE) # CMake-based builds can't currently use R's builtin routine registration, From 3368ae6d278c979e36ec3f33cd27053753b38a8d Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 3 Aug 2021 15:45:46 -0500 Subject: [PATCH 49/82] get non-Windows builds working --- .ci/test_r_package.sh | 39 ++--- .github/workflows/r_package.yml | 242 ++++++++++++++++---------------- build-cran-package.sh | 7 +- 3 files changed, 148 insertions(+), 140 deletions(-) diff --git a/.ci/test_r_package.sh b/.ci/test_r_package.sh index 672657b6d72b..c0d09bc70bd6 100755 --- a/.ci/test_r_package.sh +++ b/.ci/test_r_package.sh @@ -7,10 +7,15 @@ mkdir -p $R_LIB_PATH export R_LIBS=$R_LIB_PATH export PATH="$R_LIB_PATH/R/bin:$PATH" -# don't fail builds for long-running examples unless they're very long. -# See https://github.com/microsoft/LightGBM/issues/4049#issuecomment-793412254. -if [[ $R_BUILD_TYPE != "cran" ]]; then +# vignettes are only built for CRAN builds +R_CMD_CHECK_ARGS="--as-cran --run-donttest" +if [[ $R_BUILD_TYPE == "cran" ]]; then + export LGB_BUILD_VIGNETTES="true" +elif + # don't fail builds for long-running examples unless they're very long. + # See https://github.com/microsoft/LightGBM/issues/4049#issuecomment-793412254. export _R_CHECK_EXAMPLE_TIMING_THRESHOLD_=30 + R_CMD_CHECK_ARGS="${R_CMD_CHECK_ARGS} --ignore-vignettes" fi # Get details needed for installing R components @@ -92,20 +97,26 @@ if [[ $OS_NAME == "macos" ]]; then fi fi -# Manually install Depends and Imports libraries + 'testthat' +# Manually install libraries # to avoid a CI-time dependency on devtools (for devtools::install_deps()) -# NOTE: testthat is not required when running rchk -if [[ "${TASK}" == "r-rchk" ]]; then - packages="c('data.table', 'jsonlite', 'Matrix', 'R6', 'knitr', 'rmarkdown')" -else - packages="c('data.table', 'jsonlite', 'Matrix', 'R6', 'testthat', 'knitr', 'rmarkdown')" +packages="'data.table', 'jsonlite', 'Matrix', 'R6'" + +# testthat is not required when running rchk +if [[ "${TASK}" != "r-rchk" ]]; then + packages="$packages, 'testthat'" +fi + +# only need testthat and knitr if building vignettes +if [[ "${LGB_BUILD_VIGNETTES}" == "true" ]]; then + packages="$packages, 'knitr', 'rmarkdown'" fi + compile_from_source="both" if [[ $OS_NAME == "macos" ]]; then packages+=", type = 'binary'" compile_from_source="never" fi -Rscript --vanilla -e "options(install.packages.compile.from.source = '${compile_from_source}'); install.packages(${packages}, repos = '${CRAN_MIRROR}', lib = '${R_LIB_PATH}', dependencies = c('Depends', 'Imports', 'LinkingTo'), Ncpus = parallel::detectCores())" || exit -1 +Rscript --vanilla -e "options(install.packages.compile.from.source = '${compile_from_source}'); install.packages(c(${packages}), repos = '${CRAN_MIRROR}', lib = '${R_LIB_PATH}', dependencies = c('Depends', 'Imports', 'LinkingTo'), Ncpus = parallel::detectCores())" || exit -1 cd ${BUILD_DIRECTORY} @@ -167,18 +178,12 @@ elif [[ $R_BUILD_TYPE == "cran" ]]; then cd ${R_CMD_CHECK_DIR} fi -# vignettes are only built for CRAN builds -CHECK_ARGS="--as-cran --run-donttest" -if [[ $R_BUILD_TYPE != "cran" ]]; then - CHECK_ARGS="${CHECK_FLAGS} --ignore-vignettes" -fi - # fails tests if either ERRORs or WARNINGs are thrown by # R CMD CHECK check_succeeded="yes" ( R CMD check ${PKG_TARBALL} \ - ${CHECK_ARGS} \ + ${R_CMD_CHECK_ARGS} \ || check_succeeded="no" ) & diff --git a/.github/workflows/r_package.yml b/.github/workflows/r_package.yml index e9e730b4f533..00dda0f7a3fc 100644 --- a/.github/workflows/r_package.yml +++ b/.github/workflows/r_package.yml @@ -32,87 +32,87 @@ jobs: ################ # CMake builds # ################ - # - os: ubuntu-latest - # task: r-package - # compiler: gcc - # r_version: 3.6 - # build_type: cmake - # - os: ubuntu-latest - # task: r-package - # compiler: gcc - # r_version: 4.0 - # build_type: cmake - # - os: ubuntu-latest - # task: r-package - # compiler: clang - # r_version: 3.6 - # build_type: cmake - # - os: ubuntu-latest - # task: r-package - # compiler: clang - # r_version: 4.0 - # build_type: cmake - # - os: macOS-latest - # task: r-package - # compiler: gcc - # r_version: 3.6 - # build_type: cmake - # - os: macOS-latest - # task: r-package - # compiler: gcc - # r_version: 4.0 - # build_type: cmake - # - os: macOS-latest - # task: r-package - # compiler: clang - # r_version: 3.6 - # build_type: cmake - # - os: macOS-latest - # task: r-package - # compiler: clang - # r_version: 4.0 - # build_type: cmake - - os: windows-latest + - os: ubuntu-latest task: r-package - compiler: MINGW - toolchain: MINGW + compiler: gcc r_version: 3.6 build_type: cmake - - os: windows-latest + - os: ubuntu-latest task: r-package - compiler: MINGW - toolchain: MSYS + compiler: gcc r_version: 4.0 build_type: cmake - # Visual Studio 2017 - - os: windows-2016 + - os: ubuntu-latest task: r-package - compiler: MSVC - toolchain: MSVC + compiler: clang r_version: 3.6 build_type: cmake - # Visual Studio 2019 - - os: windows-2019 + - os: ubuntu-latest task: r-package - compiler: MSVC - toolchain: MSVC + compiler: clang r_version: 4.0 build_type: cmake - ############### - # CRAN builds # - ############### - - os: windows-latest + - os: macOS-latest task: r-package - compiler: MINGW - toolchain: MINGW + compiler: gcc r_version: 3.6 - build_type: cran - - os: windows-latest + build_type: cmake + - os: macOS-latest task: r-package - compiler: MINGW - toolchain: MSYS + compiler: gcc r_version: 4.0 - build_type: cran + build_type: cmake + - os: macOS-latest + task: r-package + compiler: clang + r_version: 3.6 + build_type: cmake + - os: macOS-latest + task: r-package + compiler: clang + r_version: 4.0 + build_type: cmake + # - os: windows-latest + # task: r-package + # compiler: MINGW + # toolchain: MINGW + # r_version: 3.6 + # build_type: cmake + # - os: windows-latest + # task: r-package + # compiler: MINGW + # toolchain: MSYS + # r_version: 4.0 + # build_type: cmake + # # Visual Studio 2017 + # - os: windows-2016 + # task: r-package + # compiler: MSVC + # toolchain: MSVC + # r_version: 3.6 + # build_type: cmake + # # Visual Studio 2019 + # - os: windows-2019 + # task: r-package + # compiler: MSVC + # toolchain: MSVC + # r_version: 4.0 + # build_type: cmake + # ############### + # # CRAN builds # + # ############### + # - os: windows-latest + # task: r-package + # compiler: MINGW + # toolchain: MINGW + # r_version: 3.6 + # build_type: cran + # - os: windows-latest + # task: r-package + # compiler: MINGW + # toolchain: MSYS + # r_version: 4.0 + # build_type: cran # - os: ubuntu-latest # task: r-package # compiler: gcc @@ -172,62 +172,62 @@ jobs: $env:GITHUB_ACTIONS = "true" $env:TASK = "${{ matrix.task }}" & "$env:GITHUB_WORKSPACE/.ci/test_windows.ps1" - # test-r-sanitizers: - # name: r-package (ubuntu-latest, R-devel, GCC ASAN/UBSAN) - # timeout-minutes: 60 - # runs-on: ubuntu-latest - # container: rhub/rocker-gcc-san - # steps: - # - name: Install Git before checkout - # shell: bash - # run: | - # apt-get update - # apt-get install --no-install-recommends -y git - # - name: Checkout repository - # uses: actions/checkout@v2.3.4 - # with: - # fetch-depth: 5 - # submodules: true - # - name: Install packages - # shell: bash - # run: | - # Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'rmarkdown', 'testthat'), repos = 'https://cran.r-project.org', Ncpus = parallel::detectCores())" - # sh build-cran-package.sh - # Rdevel CMD INSTALL lightgbm_*.tar.gz || exit -1 - # - name: Run tests with sanitizers - # shell: bash - # run: | - # cd R-package/tests - # Rscriptdevel testthat.R 2>&1 > ubsan-tests.log - # cat ubsan-tests.log - # exit $(cat ubsan-tests.log | grep --count "runtime error") - # test-r-debian-clang: - # name: r-package (debian, R-devel, clang) - # timeout-minutes: 60 - # runs-on: ubuntu-latest - # container: rhub/debian-clang-devel - # steps: - # - name: Install Git before checkout - # shell: bash - # run: | - # apt-get update - # apt-get install --no-install-recommends -y git - # - name: Checkout repository - # uses: actions/checkout@v2.3.4 - # with: - # fetch-depth: 5 - # submodules: true - # - name: Install packages and run tests - # shell: bash - # run: | - # export PATH=/opt/R-devel/bin/:${PATH} - # Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'rmarkdown', 'testthat'), repos = 'https://cran.r-project.org', Ncpus = parallel::detectCores())" - # sh build-cran-package.sh - # R CMD check --as-cran --run-donttest lightgbm_*.tar.gz || exit -1 - # if grep -q -E "NOTE|WARNING|ERROR" lightgbm.Rcheck/00check.log; then - # echo "NOTEs, WARNINGs, or ERRORs have been found by R CMD check" - # exit -1 - # fi + test-r-sanitizers: + name: r-package (ubuntu-latest, R-devel, GCC ASAN/UBSAN) + timeout-minutes: 60 + runs-on: ubuntu-latest + container: rhub/rocker-gcc-san + steps: + - name: Install Git before checkout + shell: bash + run: | + apt-get update + apt-get install --no-install-recommends -y git + - name: Checkout repository + uses: actions/checkout@v2.3.4 + with: + fetch-depth: 5 + submodules: true + - name: Install packages + shell: bash + run: | + Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'rmarkdown', 'testthat'), repos = 'https://cran.r-project.org', Ncpus = parallel::detectCores())" + sh build-cran-package.sh + Rdevel CMD INSTALL lightgbm_*.tar.gz || exit -1 + - name: Run tests with sanitizers + shell: bash + run: | + cd R-package/tests + Rscriptdevel testthat.R 2>&1 > ubsan-tests.log + cat ubsan-tests.log + exit $(cat ubsan-tests.log | grep --count "runtime error") + test-r-debian-clang: + name: r-package (debian, R-devel, clang) + timeout-minutes: 60 + runs-on: ubuntu-latest + container: rhub/debian-clang-devel + steps: + - name: Install Git before checkout + shell: bash + run: | + apt-get update + apt-get install --no-install-recommends -y git + - name: Checkout repository + uses: actions/checkout@v2.3.4 + with: + fetch-depth: 5 + submodules: true + - name: Install packages and run tests + shell: bash + run: | + export PATH=/opt/R-devel/bin/:${PATH} + Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'rmarkdown', 'testthat'), repos = 'https://cran.r-project.org', Ncpus = parallel::detectCores())" + sh build-cran-package.sh + R CMD check --as-cran --run-donttest lightgbm_*.tar.gz || exit -1 + if grep -q -E "NOTE|WARNING|ERROR" lightgbm.Rcheck/00check.log; then + echo "NOTEs, WARNINGs, or ERRORs have been found by R CMD check" + exit -1 + fi all-successful: # https://github.community/t/is-it-possible-to-require-all-github-actions-tasks-to-pass-without-enumerating-them/117957/4?u=graingert runs-on: ubuntu-latest diff --git a/build-cran-package.sh b/build-cran-package.sh index b3a00a327cba..c333bfbb81a3 100755 --- a/build-cran-package.sh +++ b/build-cran-package.sh @@ -35,10 +35,13 @@ LGB_VERSION=$(cat VERSION.txt | sed "s/rc/-/g") # move relevant files cp -R R-package/* "${TEMP_R_DIR}" -cp docs/logo/LightGBM_logo_black_text.svg "${TEMP_R_DIR}/vignettes/" cp -R include "${TEMP_R_DIR}/src/" cp -R src/* "${TEMP_R_DIR}/src/" +if [[ ${LGB_BUILD_VIGNETTES} == "true" ]]; then + cp docs/logo/LightGBM_logo_black_text.svg "${TEMP_R_DIR}/vignettes/" +fi + cp \ external_libs/fast_double_parser/include/fast_double_parser.h \ "${TEMP_R_DIR}/src/include/LightGBM" @@ -151,7 +154,7 @@ cd "${TEMP_R_DIR}" cd "${ORIG_WD}" -if [[ $LGB_BUILD_VIGNETTES == "false" ]]; then +if [[ ${LGB_BUILD_VIGNETTES} == "false" ]]; then R CMD build \ --keep-empty-dirs \ --no-build-vignettes \ From 6ba58fb05edef8e1eef67be19d4a38ae4dfb6c3e Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 3 Aug 2021 16:27:34 -0500 Subject: [PATCH 50/82] fix incorrect if-else syntax --- .ci/test_r_package.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/test_r_package.sh b/.ci/test_r_package.sh index c0d09bc70bd6..3635576d9e75 100755 --- a/.ci/test_r_package.sh +++ b/.ci/test_r_package.sh @@ -11,7 +11,7 @@ export PATH="$R_LIB_PATH/R/bin:$PATH" R_CMD_CHECK_ARGS="--as-cran --run-donttest" if [[ $R_BUILD_TYPE == "cran" ]]; then export LGB_BUILD_VIGNETTES="true" -elif +else # don't fail builds for long-running examples unless they're very long. # See https://github.com/microsoft/LightGBM/issues/4049#issuecomment-793412254. export _R_CHECK_EXAMPLE_TIMING_THRESHOLD_=30 From 77ae12e3af5b08f4738b5240d33b82a4bd02e6bb Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 3 Aug 2021 17:09:11 -0500 Subject: [PATCH 51/82] try more specific msys64 placement --- .ci/test_r_package.sh | 2 +- .ci/test_r_package_windows.ps1 | 12 ++- .github/workflows/r_package.yml | 165 ++++++++++++++++---------------- 3 files changed, 91 insertions(+), 88 deletions(-) diff --git a/.ci/test_r_package.sh b/.ci/test_r_package.sh index 3635576d9e75..5c944add1c50 100755 --- a/.ci/test_r_package.sh +++ b/.ci/test_r_package.sh @@ -106,7 +106,7 @@ if [[ "${TASK}" != "r-rchk" ]]; then packages="$packages, 'testthat'" fi -# only need testthat and knitr if building vignettes +# only need knitr and rmarkdown if building vignettes if [[ "${LGB_BUILD_VIGNETTES}" == "true" ]]; then packages="$packages, 'knitr', 'rmarkdown'" fi diff --git a/.ci/test_r_package_windows.ps1 b/.ci/test_r_package_windows.ps1 index 31d0e91f92ee..524812938294 100644 --- a/.ci/test_r_package_windows.ps1 +++ b/.ci/test_r_package_windows.ps1 @@ -91,11 +91,7 @@ if ($env:R_BUILD_TYPE -eq "cran") { $env:R_LIB_PATH = "$env:BUILD_SOURCESDIRECTORY/RLibrary" -replace '[\\]', '/' $env:R_LIBS = "$env:R_LIB_PATH" -# NOTE: gzip and tar are needed to create a CRAN package on Windows, but -# some flavors of tar.exe can fail in some settings on Windows. Putting -# the msys64 utilities at the beginning of PATH to be sure they're used -# for that purpose. -$env:PATH = "C:\msys64\usr\bin;" + "$env:RTOOLS_BIN;" + "$env:RTOOLS_MINGW_BIN;" + "$env:R_LIB_PATH/R/bin/x64;" + "$env:R_LIB_PATH/miktex/texmfs/install/miktex/bin/x64;" + $env:PATH +$env:PATH = "$env:RTOOLS_BIN;" + "$env:RTOOLS_MINGW_BIN;" + "$env:R_LIB_PATH/R/bin/x64;" + "$env:R_LIB_PATH/miktex/texmfs/install/miktex/bin/x64;" + $env:PATH $env:CRAN_MIRROR = "https://cloud.r-project.org/" $env:CTAN_MIRROR = "https://ctan.math.illinois.edu/systems/win32/miktex" $env:CTAN_PACKAGE_ARCHIVE = "$env:CTAN_MIRROR/tm/packages/" @@ -176,7 +172,13 @@ if ($env:COMPILER -ne "MSVC") { } Run-R-Code-Redirect-Stderr "commandArgs <- function(...){$env:BUILD_R_FLAGS}; source('build_r.R')"; Check-Output $? } elseif ($env:R_BUILD_TYPE -eq "cran") { + # NOTE: gzip and tar are needed to create a CRAN package on Windows, but + # some flavors of tar.exe can fail in some settings on Windows. + # Putting the msys64 utilities at the beginning of PATH temporarily to be + # sure they're used for that purpose. + $env:PATH = "C:\msys64\usr\bin;" + $env:PATH Run-R-Code-Redirect-Stderr "result <- processx::run(command = 'sh', args = 'build-cran-package.sh', echo = TRUE, windows_verbatim_args = FALSE, error_on_status = TRUE)" ; Check-Output $? + Remove-From-Path ".*msys64.*" # Test CRAN source .tar.gz in a directory that is not this repo or below it. # When people install.packages('lightgbm'), they won't have the LightGBM # git repo around. This is to protect against the use of relative paths diff --git a/.github/workflows/r_package.yml b/.github/workflows/r_package.yml index 00dda0f7a3fc..c649f9a53d12 100644 --- a/.github/workflows/r_package.yml +++ b/.github/workflows/r_package.yml @@ -72,65 +72,65 @@ jobs: compiler: clang r_version: 4.0 build_type: cmake - # - os: windows-latest - # task: r-package - # compiler: MINGW - # toolchain: MINGW - # r_version: 3.6 - # build_type: cmake - # - os: windows-latest - # task: r-package - # compiler: MINGW - # toolchain: MSYS - # r_version: 4.0 - # build_type: cmake - # # Visual Studio 2017 - # - os: windows-2016 - # task: r-package - # compiler: MSVC - # toolchain: MSVC - # r_version: 3.6 - # build_type: cmake - # # Visual Studio 2019 - # - os: windows-2019 - # task: r-package - # compiler: MSVC - # toolchain: MSVC - # r_version: 4.0 - # build_type: cmake + - os: windows-latest + task: r-package + compiler: MINGW + toolchain: MINGW + r_version: 3.6 + build_type: cmake + - os: windows-latest + task: r-package + compiler: MINGW + toolchain: MSYS + r_version: 4.0 + build_type: cmake + # Visual Studio 2017 + - os: windows-2016 + task: r-package + compiler: MSVC + toolchain: MSVC + r_version: 3.6 + build_type: cmake + # Visual Studio 2019 + - os: windows-2019 + task: r-package + compiler: MSVC + toolchain: MSVC + r_version: 4.0 + build_type: cmake # ############### # # CRAN builds # # ############### - # - os: windows-latest - # task: r-package - # compiler: MINGW - # toolchain: MINGW - # r_version: 3.6 - # build_type: cran - # - os: windows-latest - # task: r-package - # compiler: MINGW - # toolchain: MSYS - # r_version: 4.0 - # build_type: cran - # - os: ubuntu-latest - # task: r-package - # compiler: gcc - # r_version: 4.0 - # build_type: cran - # - os: macOS-latest - # task: r-package - # compiler: clang - # r_version: 4.0 - # build_type: cran + - os: windows-latest + task: r-package + compiler: MINGW + toolchain: MINGW + r_version: 3.6 + build_type: cran + - os: windows-latest + task: r-package + compiler: MINGW + toolchain: MSYS + r_version: 4.0 + build_type: cran + - os: ubuntu-latest + task: r-package + compiler: gcc + r_version: 4.0 + build_type: cran + - os: macOS-latest + task: r-package + compiler: clang + r_version: 4.0 + build_type: cran ################ # Other checks # ################ - # - os: ubuntu-latest - # task: r-rchk - # compiler: gcc - # r_version: 4.0 - # build_type: cran + - os: ubuntu-latest + task: r-rchk + compiler: gcc + r_version: 4.0 + build_type: cran steps: - name: Prevent conversion of line endings on Windows if: startsWith(matrix.os, 'windows') @@ -172,35 +172,35 @@ jobs: $env:GITHUB_ACTIONS = "true" $env:TASK = "${{ matrix.task }}" & "$env:GITHUB_WORKSPACE/.ci/test_windows.ps1" - test-r-sanitizers: - name: r-package (ubuntu-latest, R-devel, GCC ASAN/UBSAN) - timeout-minutes: 60 - runs-on: ubuntu-latest - container: rhub/rocker-gcc-san - steps: - - name: Install Git before checkout - shell: bash - run: | - apt-get update - apt-get install --no-install-recommends -y git - - name: Checkout repository - uses: actions/checkout@v2.3.4 - with: - fetch-depth: 5 - submodules: true - - name: Install packages - shell: bash - run: | - Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'rmarkdown', 'testthat'), repos = 'https://cran.r-project.org', Ncpus = parallel::detectCores())" - sh build-cran-package.sh - Rdevel CMD INSTALL lightgbm_*.tar.gz || exit -1 - - name: Run tests with sanitizers - shell: bash - run: | - cd R-package/tests - Rscriptdevel testthat.R 2>&1 > ubsan-tests.log - cat ubsan-tests.log - exit $(cat ubsan-tests.log | grep --count "runtime error") + # test-r-sanitizers: + # name: r-package (ubuntu-latest, R-devel, GCC ASAN/UBSAN) + # timeout-minutes: 60 + # runs-on: ubuntu-latest + # container: rhub/rocker-gcc-san + # steps: + # - name: Install Git before checkout + # shell: bash + # run: | + # apt-get update + # apt-get install --no-install-recommends -y git + # - name: Checkout repository + # uses: actions/checkout@v2.3.4 + # with: + # fetch-depth: 5 + # submodules: true + # - name: Install packages + # shell: bash + # run: | + # Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'rmarkdown', 'testthat'), repos = 'https://cran.r-project.org', Ncpus = parallel::detectCores())" + # sh build-cran-package.sh + # Rdevel CMD INSTALL lightgbm_*.tar.gz || exit -1 + # - name: Run tests with sanitizers + # shell: bash + # run: | + # cd R-package/tests + # Rscriptdevel testthat.R 2>&1 > ubsan-tests.log + # cat ubsan-tests.log + # exit $(cat ubsan-tests.log | grep --count "runtime error") test-r-debian-clang: name: r-package (debian, R-devel, clang) timeout-minutes: 60 @@ -222,6 +222,7 @@ jobs: run: | export PATH=/opt/R-devel/bin/:${PATH} Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'rmarkdown', 'testthat'), repos = 'https://cran.r-project.org', Ncpus = parallel::detectCores())" + export LGB_BUILD_VIGNETTES=true sh build-cran-package.sh R CMD check --as-cran --run-donttest lightgbm_*.tar.gz || exit -1 if grep -q -E "NOTE|WARNING|ERROR" lightgbm.Rcheck/00check.log; then From 080d0d34a7a4a85c1267e3011ec852f7fabf474e Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 3 Aug 2021 17:33:01 -0500 Subject: [PATCH 52/82] try restoring all CI jobs --- .appveyor.yml | 72 +-- .github/workflows/cuda.yml | 186 +++---- .github/workflows/python_package.yml | 152 ++--- .github/workflows/r_package.yml | 63 ++- .github/workflows/r_valgrind.yml | 1 + .github/workflows/static_analysis.yml | 2 + .gitignore | 3 - .vsts-ci.yml | 768 +++++++++++++------------- 8 files changed, 625 insertions(+), 622 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index ccb08022792a..b7d9fe589176 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -1,43 +1,43 @@ -# version: 3.2.1.99.{build} +version: 3.2.1.99.{build} -# image: Visual Studio 2015 -# platform: x64 -# configuration: # a trick to construct a build matrix with multiple Python versions -# - 3.7 +image: Visual Studio 2015 +platform: x64 +configuration: # a trick to construct a build matrix with multiple Python versions + - 3.7 -# # only build pull requests and -# # commits to 'master' -# branches: -# only: -# - master +# only build pull requests and +# commits to 'master' +branches: + only: + - master -# environment: -# matrix: -# - COMPILER: MSVC -# TASK: python -# - COMPILER: MINGW -# TASK: python +environment: + matrix: + - COMPILER: MSVC + TASK: python + - COMPILER: MINGW + TASK: python -# clone_depth: 5 +clone_depth: 5 -# install: -# - git submodule update --init --recursive # get `external_libs` folder -# - set PATH=%PATH:C:\Program Files\Git\usr\bin;=% # delete sh.exe from PATH (mingw32-make fix) -# - set PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH% -# - set PYTHON_VERSION=%CONFIGURATION% -# - set CONDA_ENV="test-env" -# - ps: | -# switch ($env:PYTHON_VERSION) { -# "3.6" {$env:MINICONDA = "C:\Miniconda36-x64"} -# "3.7" {$env:MINICONDA = "C:\Miniconda37-x64"} -# default {$env:MINICONDA = "C:\Miniconda37-x64"} -# } -# $env:PATH = "$env:MINICONDA;$env:MINICONDA\Scripts;$env:PATH" -# $env:BUILD_SOURCESDIRECTORY = "$env:APPVEYOR_BUILD_FOLDER" -# $env:LGB_VER = (Get-Content $env:APPVEYOR_BUILD_FOLDER\VERSION.txt).trim() +install: + - git submodule update --init --recursive # get `external_libs` folder + - set PATH=%PATH:C:\Program Files\Git\usr\bin;=% # delete sh.exe from PATH (mingw32-make fix) + - set PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH% + - set PYTHON_VERSION=%CONFIGURATION% + - set CONDA_ENV="test-env" + - ps: | + switch ($env:PYTHON_VERSION) { + "3.6" {$env:MINICONDA = "C:\Miniconda36-x64"} + "3.7" {$env:MINICONDA = "C:\Miniconda37-x64"} + default {$env:MINICONDA = "C:\Miniconda37-x64"} + } + $env:PATH = "$env:MINICONDA;$env:MINICONDA\Scripts;$env:PATH" + $env:BUILD_SOURCESDIRECTORY = "$env:APPVEYOR_BUILD_FOLDER" + $env:LGB_VER = (Get-Content $env:APPVEYOR_BUILD_FOLDER\VERSION.txt).trim() -# build: false +build: false -# test_script: -# - conda init powershell -# - powershell.exe -ExecutionPolicy Bypass -File %APPVEYOR_BUILD_FOLDER%\.ci\test_windows.ps1 +test_script: + - conda init powershell + - powershell.exe -ExecutionPolicy Bypass -File %APPVEYOR_BUILD_FOLDER%\.ci\test_windows.ps1 diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index d3f5d27398af..98e820b6ca04 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -1,97 +1,97 @@ -# name: CUDA Version +name: CUDA Version -# on: - # push: - # branches: - # - master - # pull_request: - # branches: - # - master +on: + push: + branches: + - master + pull_request: + branches: + - master -# env: - # github_actions: 'true' - # os_name: linux - # task: cuda - # conda_env: test-env +env: + github_actions: 'true' + os_name: linux + task: cuda + conda_env: test-env -# jobs: - # test: - # name: cuda ${{ matrix.cuda_version }} ${{ matrix.method }} (linux, ${{ matrix.compiler }}, Python ${{ matrix.python_version }}) - # runs-on: [self-hosted, linux] - # timeout-minutes: 60 - # strategy: - # fail-fast: false - # matrix: - # include: - # - method: source - # compiler: gcc - # python_version: 3.7 - # cuda_version: "11.4.0" - # - method: pip - # compiler: clang - # python_version: 3.8 - # cuda_version: "10.0" - # - method: wheel - # compiler: gcc - # python_version: 3.9 - # cuda_version: "9.0" - # steps: - # - name: Setup or update software on host machine - # run: | - # sudo apt-get update - # sudo apt-get install --no-install-recommends -y \ - # apt-transport-https \ - # ca-certificates \ - # curl \ - # git \ - # gnupg-agent \ - # software-properties-common - # curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - - # sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" -y - # curl -sL https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add - - # curl -sL https://nvidia.github.io/nvidia-docker/$(. /etc/os-release;echo $ID$VERSION_ID)/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list - # sudo apt-get update - # sudo apt-get install --no-install-recommends -y \ - # containerd.io \ - # docker-ce \ - # docker-ce-cli \ - # nvidia-docker2 - # sudo chmod a+rw /var/run/docker.sock - # sudo systemctl restart docker - # - name: Remove old folder with repository - # run: sudo rm -rf $GITHUB_WORKSPACE - # - name: Checkout repository - # uses: actions/checkout@v1 - # with: - # fetch-depth: 5 - # submodules: true - # - name: Setup and run tests - # run: | - # export ROOT_DOCKER_FOLDER=/LightGBM - # cat > docker.env < docker-script.sh < docker.env < docker-script.sh <&1 > ubsan-tests.log - # cat ubsan-tests.log - # exit $(cat ubsan-tests.log | grep --count "runtime error") + test-r-sanitizers: + name: r-package (ubuntu-latest, R-devel, GCC ASAN/UBSAN) + timeout-minutes: 60 + runs-on: ubuntu-latest + container: rhub/rocker-gcc-san + env: + LGB_BUILD_VIGNETTES: "true" + steps: + - name: Install Git before checkout + shell: bash + run: | + apt-get update + apt-get install --no-install-recommends -y git + - name: Checkout repository + uses: actions/checkout@v2.3.4 + with: + fetch-depth: 5 + submodules: true + - name: Install packages + shell: bash + run: | + Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'rmarkdown', 'testthat'), repos = 'https://cran.r-project.org', Ncpus = parallel::detectCores())" + sh build-cran-package.sh + Rdevel CMD INSTALL lightgbm_*.tar.gz || exit -1 + - name: Run tests with sanitizers + shell: bash + run: | + cd R-package/tests + Rscriptdevel testthat.R 2>&1 > ubsan-tests.log + cat ubsan-tests.log + exit $(cat ubsan-tests.log | grep --count "runtime error") test-r-debian-clang: name: r-package (debian, R-devel, clang) timeout-minutes: 60 runs-on: ubuntu-latest container: rhub/debian-clang-devel + env: + LGB_BUILD_VIGNETTES: "true" steps: - name: Install Git before checkout shell: bash @@ -222,7 +226,6 @@ jobs: run: | export PATH=/opt/R-devel/bin/:${PATH} Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'rmarkdown', 'testthat'), repos = 'https://cran.r-project.org', Ncpus = parallel::detectCores())" - export LGB_BUILD_VIGNETTES=true sh build-cran-package.sh R CMD check --as-cran --run-donttest lightgbm_*.tar.gz || exit -1 if grep -q -E "NOTE|WARNING|ERROR" lightgbm.Rcheck/00check.log; then diff --git a/.github/workflows/r_valgrind.yml b/.github/workflows/r_valgrind.yml index 97c4d853564c..f5731815b18e 100644 --- a/.github/workflows/r_valgrind.yml +++ b/.github/workflows/r_valgrind.yml @@ -12,6 +12,7 @@ jobs: container: wch1/r-debug env: SECRETS_WORKFLOW: ${{ secrets.WORKFLOW }} + LGB_BUILD_VIGNETTES: "true" steps: - name: Install essential software before checkout shell: bash diff --git a/.github/workflows/static_analysis.yml b/.github/workflows/static_analysis.yml index 0c41949bad1f..1904ca509292 100644 --- a/.github/workflows/static_analysis.yml +++ b/.github/workflows/static_analysis.yml @@ -48,6 +48,8 @@ jobs: timeout-minutes: 60 runs-on: ubuntu-latest container: rocker/verse + env: + LGB_BUILD_VIGNETTES: "true" steps: - name: Checkout repository uses: actions/checkout@v2.3.4 diff --git a/.gitignore b/.gitignore index bbcd1cf1416f..5a90094850b9 100644 --- a/.gitignore +++ b/.gitignore @@ -446,9 +446,6 @@ tests/distributed/predict* *.RData *.rds -# Files from interactively building R vignettes -**/vignettes/*.html - # Files generated by aspell **/*.bak diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 0f083f037684..278dfee93d7e 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -1,386 +1,386 @@ -# trigger: - # branches: - # include: - # - master - # tags: - # include: - # - v* -# pr: -# - master -# variables: - # AZURE: 'true' - # PYTHON_VERSION: 3.9 - # CONDA_ENV: test-env -# resources: - # containers: - # - container: ubuntu1404 - # image: lightgbm/vsts-agent:ubuntu-14.04 - # - container: ubuntu-latest - # image: 'ubuntu:latest' - # options: "--name ci-container -v /usr/bin/docker:/tmp/docker:ro" - # - container: rbase - # image: rocker/r-base -# jobs: -# ########################################### -# - job: Linux -# ########################################### - # variables: - # COMPILER: gcc - # SETUP_CONDA: 'false' - # OS_NAME: 'linux' - # PRODUCES_ARTIFACTS: 'true' - # pool: sh-ubuntu - # container: ubuntu1404 - # strategy: - # matrix: - # regular: - # TASK: regular - # sdist: - # TASK: sdist - # PYTHON_VERSION: 3.7 - # bdist: - # TASK: bdist - # inference: - # TASK: if-else - # mpi_source: - # TASK: mpi - # METHOD: source - # PYTHON_VERSION: 3.8 - # gpu_source: - # TASK: gpu - # METHOD: source - # swig: - # TASK: swig - # steps: - # - script: | - # echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY" - # echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)" - # echo "##vso[task.prependpath]$CONDA/bin" - # AMDAPPSDK_PATH=$BUILD_SOURCESDIRECTORY/AMDAPPSDK - # echo "##vso[task.setvariable variable=AMDAPPSDK_PATH]$AMDAPPSDK_PATH" - # LD_LIBRARY_PATH=$AMDAPPSDK_PATH/lib/x86_64:$LD_LIBRARY_PATH - # echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$LD_LIBRARY_PATH" - # echo "##vso[task.setvariable variable=OPENCL_VENDOR_PATH]$AMDAPPSDK_PATH/etc/OpenCL/vendors" - # displayName: 'Set variables' - # - bash: $(Build.SourcesDirectory)/.ci/setup.sh - # displayName: Setup - # - bash: $(Build.SourcesDirectory)/.ci/test.sh - # displayName: Test - # - task: PublishBuildArtifacts@1 - # condition: and(succeeded(), in(variables['TASK'], 'regular', 'sdist', 'bdist', 'swig'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))) - # inputs: - # pathtoPublish: '$(Build.ArtifactStagingDirectory)' - # artifactName: PackageAssets - # artifactType: container -# ########################################### -# - job: Linux_latest -# ########################################### - # variables: - # COMPILER: clang - # DEBIAN_FRONTEND: 'noninteractive' - # IN_UBUNTU_LATEST_CONTAINER: 'true' - # OS_NAME: 'linux' - # SETUP_CONDA: 'true' - # pool: sh-ubuntu - # container: ubuntu-latest - # strategy: - # matrix: - # regular: - # TASK: regular - # PYTHON_VERSION: 3.6 - # sdist: - # TASK: sdist - # bdist: - # TASK: bdist - # PYTHON_VERSION: 3.8 - # inference: - # TASK: if-else - # mpi_source: - # TASK: mpi - # METHOD: source - # mpi_pip: - # TASK: mpi - # METHOD: pip - # PYTHON_VERSION: 3.8 - # mpi_wheel: - # TASK: mpi - # METHOD: wheel - # PYTHON_VERSION: 3.7 - # gpu_source: - # TASK: gpu - # METHOD: source - # gpu_pip: - # TASK: gpu - # METHOD: pip - # PYTHON_VERSION: 3.6 - # gpu_wheel: - # TASK: gpu - # METHOD: wheel - # PYTHON_VERSION: 3.7 - # cpp_tests: - # TASK: cpp-tests - # METHOD: with-sanitizers - # steps: - # - script: | - # echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY" - # echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)" - # CONDA=$HOME/miniconda - # echo "##vso[task.setvariable variable=CONDA]$CONDA" - # echo "##vso[task.prependpath]$CONDA/bin" - # AMDAPPSDK_PATH=$BUILD_SOURCESDIRECTORY/AMDAPPSDK - # echo "##vso[task.setvariable variable=AMDAPPSDK_PATH]$AMDAPPSDK_PATH" - # LD_LIBRARY_PATH=$AMDAPPSDK_PATH/lib/x86_64:$LD_LIBRARY_PATH - # echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$LD_LIBRARY_PATH" - # echo "##vso[task.setvariable variable=OPENCL_VENDOR_PATH]$AMDAPPSDK_PATH/etc/OpenCL/vendors" - # displayName: 'Set variables' - # # https://github.com/microsoft/azure-pipelines-agent/issues/2043#issuecomment-687983301 - # - script: | - # /tmp/docker exec -t -u 0 ci-container \ - # sh -c "apt-get update && apt-get -o Dpkg::Options::="--force-confold" -y install sudo" - # displayName: 'Install sudo' - # - bash: $(Build.SourcesDirectory)/.ci/setup.sh - # displayName: Setup - # - bash: $(Build.SourcesDirectory)/.ci/test.sh - # displayName: Test -# ########################################### -# - job: QEMU_multiarch -# ########################################### - # variables: - # COMPILER: gcc - # OS_NAME: 'linux' - # PRODUCES_ARTIFACTS: 'true' - # pool: - # vmImage: ubuntu-latest - # timeoutInMinutes: 180 - # strategy: - # matrix: - # bdist: - # TASK: bdist - # ARCH: aarch64 - # steps: - # - script: | - # sudo apt-get update - # sudo apt-get install --no-install-recommends -y \ - # binfmt-support \ - # qemu \ - # qemu-user \ - # qemu-user-static - # displayName: 'Install QEMU' - # - script: | - # docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - # displayName: 'Enable Docker multi-architecture support' - # - script: | - # export ROOT_DOCKER_FOLDER=/LightGBM - # cat > docker.env < docker-script.sh < docker.env < docker-script.sh < Date: Tue, 3 Aug 2021 17:34:18 -0500 Subject: [PATCH 53/82] remove more unnecessary diff --- .github/workflows/cuda.yml | 1 - .vsts-ci.yml | 1 - 2 files changed, 2 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 98e820b6ca04..87c8db2aa094 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -94,4 +94,3 @@ jobs: steps: - name: Note that all tests succeeded run: echo "🎉" - \ No newline at end of file diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 278dfee93d7e..0a444f8aca05 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -383,4 +383,3 @@ jobs: isDraft: true isPreRelease: false addChangeLog: false - \ No newline at end of file From e9725c53d045cae5d657136924127d9d55bf6a73 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 3 Aug 2021 17:35:57 -0500 Subject: [PATCH 54/82] restore clang and sanitizer jobs --- .github/workflows/r_package.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/r_package.yml b/.github/workflows/r_package.yml index 6f232a127044..c98d78db873e 100644 --- a/.github/workflows/r_package.yml +++ b/.github/workflows/r_package.yml @@ -98,9 +98,9 @@ jobs: toolchain: MSVC r_version: 4.0 build_type: cmake - # ############### - # # CRAN builds # - # ############### + ############### + # CRAN builds # + ############### - os: windows-latest task: r-package compiler: MINGW @@ -235,7 +235,7 @@ jobs: all-successful: # https://github.community/t/is-it-possible-to-require-all-github-actions-tasks-to-pass-without-enumerating-them/117957/4?u=graingert runs-on: ubuntu-latest - needs: [test] + needs: [test, test-r-sanitizers, test-r-debian-clang] steps: - name: Note that all tests succeeded run: echo "🎉" From 480a599507c2e880a99d2ec3bd346838096be249 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 3 Aug 2021 22:07:12 -0500 Subject: [PATCH 55/82] simmplify --- .appveyor.yml | 72 +-- .ci/test_r_package.sh | 22 +- .ci/test_r_package_windows.ps1 | 10 +- .github/workflows/cuda.yml | 186 +++---- .github/workflows/python_package.yml | 152 ++--- .github/workflows/r_package.yml | 4 - .github/workflows/r_valgrind.yml | 1 - .github/workflows/static_analysis.yml | 2 - .vsts-ci.yml | 768 +++++++++++++------------- build-cran-package.sh | 41 +- build_r.R | 7 + 11 files changed, 637 insertions(+), 628 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index b7d9fe589176..ccb08022792a 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -1,43 +1,43 @@ -version: 3.2.1.99.{build} +# version: 3.2.1.99.{build} -image: Visual Studio 2015 -platform: x64 -configuration: # a trick to construct a build matrix with multiple Python versions - - 3.7 +# image: Visual Studio 2015 +# platform: x64 +# configuration: # a trick to construct a build matrix with multiple Python versions +# - 3.7 -# only build pull requests and -# commits to 'master' -branches: - only: - - master +# # only build pull requests and +# # commits to 'master' +# branches: +# only: +# - master -environment: - matrix: - - COMPILER: MSVC - TASK: python - - COMPILER: MINGW - TASK: python +# environment: +# matrix: +# - COMPILER: MSVC +# TASK: python +# - COMPILER: MINGW +# TASK: python -clone_depth: 5 +# clone_depth: 5 -install: - - git submodule update --init --recursive # get `external_libs` folder - - set PATH=%PATH:C:\Program Files\Git\usr\bin;=% # delete sh.exe from PATH (mingw32-make fix) - - set PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH% - - set PYTHON_VERSION=%CONFIGURATION% - - set CONDA_ENV="test-env" - - ps: | - switch ($env:PYTHON_VERSION) { - "3.6" {$env:MINICONDA = "C:\Miniconda36-x64"} - "3.7" {$env:MINICONDA = "C:\Miniconda37-x64"} - default {$env:MINICONDA = "C:\Miniconda37-x64"} - } - $env:PATH = "$env:MINICONDA;$env:MINICONDA\Scripts;$env:PATH" - $env:BUILD_SOURCESDIRECTORY = "$env:APPVEYOR_BUILD_FOLDER" - $env:LGB_VER = (Get-Content $env:APPVEYOR_BUILD_FOLDER\VERSION.txt).trim() +# install: +# - git submodule update --init --recursive # get `external_libs` folder +# - set PATH=%PATH:C:\Program Files\Git\usr\bin;=% # delete sh.exe from PATH (mingw32-make fix) +# - set PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH% +# - set PYTHON_VERSION=%CONFIGURATION% +# - set CONDA_ENV="test-env" +# - ps: | +# switch ($env:PYTHON_VERSION) { +# "3.6" {$env:MINICONDA = "C:\Miniconda36-x64"} +# "3.7" {$env:MINICONDA = "C:\Miniconda37-x64"} +# default {$env:MINICONDA = "C:\Miniconda37-x64"} +# } +# $env:PATH = "$env:MINICONDA;$env:MINICONDA\Scripts;$env:PATH" +# $env:BUILD_SOURCESDIRECTORY = "$env:APPVEYOR_BUILD_FOLDER" +# $env:LGB_VER = (Get-Content $env:APPVEYOR_BUILD_FOLDER\VERSION.txt).trim() -build: false +# build: false -test_script: - - conda init powershell - - powershell.exe -ExecutionPolicy Bypass -File %APPVEYOR_BUILD_FOLDER%\.ci\test_windows.ps1 +# test_script: +# - conda init powershell +# - powershell.exe -ExecutionPolicy Bypass -File %APPVEYOR_BUILD_FOLDER%\.ci\test_windows.ps1 diff --git a/.ci/test_r_package.sh b/.ci/test_r_package.sh index 5c944add1c50..acd7b52d3cd1 100755 --- a/.ci/test_r_package.sh +++ b/.ci/test_r_package.sh @@ -7,15 +7,16 @@ mkdir -p $R_LIB_PATH export R_LIBS=$R_LIB_PATH export PATH="$R_LIB_PATH/R/bin:$PATH" -# vignettes are only built for CRAN builds -R_CMD_CHECK_ARGS="--as-cran --run-donttest" -if [[ $R_BUILD_TYPE == "cran" ]]; then - export LGB_BUILD_VIGNETTES="true" -else - # don't fail builds for long-running examples unless they're very long. - # See https://github.com/microsoft/LightGBM/issues/4049#issuecomment-793412254. +# don't fail builds for long-running examples unless they're very long. +# See https://github.com/microsoft/LightGBM/issues/4049#issuecomment-793412254. +if [[ $R_BUILD_TYPE != "cran" ]]; then + export _R_CHECK_EXAMPLE_TIMING_THRESHOLD_=30 +fi + +# don't fail builds for long-running examples unless they're very long. +# See https://github.com/microsoft/LightGBM/issues/4049#issuecomment-793412254. +if [[ $R_BUILD_TYPE != "cran" ]]; then export _R_CHECK_EXAMPLE_TIMING_THRESHOLD_=30 - R_CMD_CHECK_ARGS="${R_CMD_CHECK_ARGS} --ignore-vignettes" fi # Get details needed for installing R components @@ -107,7 +108,7 @@ if [[ "${TASK}" != "r-rchk" ]]; then fi # only need knitr and rmarkdown if building vignettes -if [[ "${LGB_BUILD_VIGNETTES}" == "true" ]]; then +if [[ $R_BUILD_TYPE == "cran" ]]; then packages="$packages, 'knitr', 'rmarkdown'" fi @@ -183,7 +184,8 @@ fi check_succeeded="yes" ( R CMD check ${PKG_TARBALL} \ - ${R_CMD_CHECK_ARGS} \ + --as-cran \ + --run-donttest \ || check_succeeded="no" ) & diff --git a/.ci/test_r_package_windows.ps1 b/.ci/test_r_package_windows.ps1 index 524812938294..dd95a6c32336 100644 --- a/.ci/test_r_package_windows.ps1 +++ b/.ci/test_r_package_windows.ps1 @@ -84,11 +84,6 @@ if ($env:R_MAJOR_VERSION -eq "3") { Check-Output $false } -# only build vignettes for CRAN-style builds -if ($env:R_BUILD_TYPE -eq "cran") { - $env:LGB_BUILD_VIGNETTES = "true" -} - $env:R_LIB_PATH = "$env:BUILD_SOURCESDIRECTORY/RLibrary" -replace '[\\]', '/' $env:R_LIBS = "$env:R_LIB_PATH" $env:PATH = "$env:RTOOLS_BIN;" + "$env:RTOOLS_MINGW_BIN;" + "$env:R_LIB_PATH/R/bin/x64;" + "$env:R_LIB_PATH/miktex/texmfs/install/miktex/bin/x64;" + $env:PATH @@ -127,7 +122,7 @@ Write-Output "Done installing Rtools" Write-Output "Installing dependencies" $packages = "c('data.table', 'jsonlite', 'Matrix', 'processx', 'R6', 'testthat')" -if ($env:LGB_BUILD_VIGNETTES -eq "true") { +if ($env:R_BUILD_TYPE -eq "cran") { $packages = "$packages, 'knitr', 'rmarkdown'" } Run-R-Code-Redirect-Stderr "options(install.packages.check.source = 'no'); install.packages(c($packages), dependencies = c('Imports', 'Depends', 'LinkingTo'), repos = '$env:CRAN_MIRROR', type = 'binary', lib = '$env:R_LIB_PATH', Ncpus = parallel::detectCores())" ; Check-Output $? @@ -194,8 +189,7 @@ if ($env:COMPILER -ne "MSVC") { # CRAN packages must pass without --no-multiarch (build on 64-bit and 32-bit) $check_args = "c('CMD', 'check', '--as-cran', '--run-donttest', '$PKG_FILE_NAME')" } else { - # vignettes are only built for CRAN builds - $check_args = "c('CMD', 'check', '--no-multiarch', '--as-cran', '--run-donttest', '--ignore-vignettes', '$PKG_FILE_NAME')" + $check_args = "c('CMD', 'check', '--no-multiarch', '--as-cran', '--run-donttest', '$PKG_FILE_NAME')" } Run-R-Code-Redirect-Stderr "result <- processx::run(command = 'R.exe', args = $check_args, echo = TRUE, windows_verbatim_args = FALSE, error_on_status = TRUE)" ; $check_succeeded = $? diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 87c8db2aa094..ea38b04d9c80 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -1,96 +1,96 @@ -name: CUDA Version +# name: CUDA Version -on: - push: - branches: - - master - pull_request: - branches: - - master +# on: +# push: +# branches: +# - master +# pull_request: +# branches: +# - master -env: - github_actions: 'true' - os_name: linux - task: cuda - conda_env: test-env +# env: +# github_actions: 'true' +# os_name: linux +# task: cuda +# conda_env: test-env -jobs: - test: - name: cuda ${{ matrix.cuda_version }} ${{ matrix.method }} (linux, ${{ matrix.compiler }}, Python ${{ matrix.python_version }}) - runs-on: [self-hosted, linux] - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - include: - - method: source - compiler: gcc - python_version: 3.7 - cuda_version: "11.4.0" - - method: pip - compiler: clang - python_version: 3.8 - cuda_version: "10.0" - - method: wheel - compiler: gcc - python_version: 3.9 - cuda_version: "9.0" - steps: - - name: Setup or update software on host machine - run: | - sudo apt-get update - sudo apt-get install --no-install-recommends -y \ - apt-transport-https \ - ca-certificates \ - curl \ - git \ - gnupg-agent \ - software-properties-common - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" -y - curl -sL https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add - - curl -sL https://nvidia.github.io/nvidia-docker/$(. /etc/os-release;echo $ID$VERSION_ID)/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list - sudo apt-get update - sudo apt-get install --no-install-recommends -y \ - containerd.io \ - docker-ce \ - docker-ce-cli \ - nvidia-docker2 - sudo chmod a+rw /var/run/docker.sock - sudo systemctl restart docker - - name: Remove old folder with repository - run: sudo rm -rf $GITHUB_WORKSPACE - - name: Checkout repository - uses: actions/checkout@v1 - with: - fetch-depth: 5 - submodules: true - - name: Setup and run tests - run: | - export ROOT_DOCKER_FOLDER=/LightGBM - cat > docker.env < docker-script.sh < docker.env < docker-script.sh < docker.env < docker-script.sh < docker.env < docker-script.sh < Date: Tue, 3 Aug 2021 22:42:25 -0500 Subject: [PATCH 56/82] restore all CI --- .appveyor.yml | 72 ++-- .ci/test_r_package.sh | 6 - .vsts-ci.yml | 768 +++++++++++++++++++++--------------------- 3 files changed, 420 insertions(+), 426 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index ccb08022792a..b7d9fe589176 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -1,43 +1,43 @@ -# version: 3.2.1.99.{build} +version: 3.2.1.99.{build} -# image: Visual Studio 2015 -# platform: x64 -# configuration: # a trick to construct a build matrix with multiple Python versions -# - 3.7 +image: Visual Studio 2015 +platform: x64 +configuration: # a trick to construct a build matrix with multiple Python versions + - 3.7 -# # only build pull requests and -# # commits to 'master' -# branches: -# only: -# - master +# only build pull requests and +# commits to 'master' +branches: + only: + - master -# environment: -# matrix: -# - COMPILER: MSVC -# TASK: python -# - COMPILER: MINGW -# TASK: python +environment: + matrix: + - COMPILER: MSVC + TASK: python + - COMPILER: MINGW + TASK: python -# clone_depth: 5 +clone_depth: 5 -# install: -# - git submodule update --init --recursive # get `external_libs` folder -# - set PATH=%PATH:C:\Program Files\Git\usr\bin;=% # delete sh.exe from PATH (mingw32-make fix) -# - set PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH% -# - set PYTHON_VERSION=%CONFIGURATION% -# - set CONDA_ENV="test-env" -# - ps: | -# switch ($env:PYTHON_VERSION) { -# "3.6" {$env:MINICONDA = "C:\Miniconda36-x64"} -# "3.7" {$env:MINICONDA = "C:\Miniconda37-x64"} -# default {$env:MINICONDA = "C:\Miniconda37-x64"} -# } -# $env:PATH = "$env:MINICONDA;$env:MINICONDA\Scripts;$env:PATH" -# $env:BUILD_SOURCESDIRECTORY = "$env:APPVEYOR_BUILD_FOLDER" -# $env:LGB_VER = (Get-Content $env:APPVEYOR_BUILD_FOLDER\VERSION.txt).trim() +install: + - git submodule update --init --recursive # get `external_libs` folder + - set PATH=%PATH:C:\Program Files\Git\usr\bin;=% # delete sh.exe from PATH (mingw32-make fix) + - set PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH% + - set PYTHON_VERSION=%CONFIGURATION% + - set CONDA_ENV="test-env" + - ps: | + switch ($env:PYTHON_VERSION) { + "3.6" {$env:MINICONDA = "C:\Miniconda36-x64"} + "3.7" {$env:MINICONDA = "C:\Miniconda37-x64"} + default {$env:MINICONDA = "C:\Miniconda37-x64"} + } + $env:PATH = "$env:MINICONDA;$env:MINICONDA\Scripts;$env:PATH" + $env:BUILD_SOURCESDIRECTORY = "$env:APPVEYOR_BUILD_FOLDER" + $env:LGB_VER = (Get-Content $env:APPVEYOR_BUILD_FOLDER\VERSION.txt).trim() -# build: false +build: false -# test_script: -# - conda init powershell -# - powershell.exe -ExecutionPolicy Bypass -File %APPVEYOR_BUILD_FOLDER%\.ci\test_windows.ps1 +test_script: + - conda init powershell + - powershell.exe -ExecutionPolicy Bypass -File %APPVEYOR_BUILD_FOLDER%\.ci\test_windows.ps1 diff --git a/.ci/test_r_package.sh b/.ci/test_r_package.sh index acd7b52d3cd1..954f5bba0e1b 100755 --- a/.ci/test_r_package.sh +++ b/.ci/test_r_package.sh @@ -13,12 +13,6 @@ if [[ $R_BUILD_TYPE != "cran" ]]; then export _R_CHECK_EXAMPLE_TIMING_THRESHOLD_=30 fi -# don't fail builds for long-running examples unless they're very long. -# See https://github.com/microsoft/LightGBM/issues/4049#issuecomment-793412254. -if [[ $R_BUILD_TYPE != "cran" ]]; then - export _R_CHECK_EXAMPLE_TIMING_THRESHOLD_=30 -fi - # Get details needed for installing R components R_MAJOR_VERSION=( ${R_VERSION//./ } ) if [[ "${R_MAJOR_VERSION}" == "3" ]]; then diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 37f64dd416cb..0a444f8aca05 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -1,385 +1,385 @@ -# trigger: -# branches: -# include: -# - master -# tags: -# include: -# - v* -# pr: -# - master -# variables: -# AZURE: 'true' -# PYTHON_VERSION: 3.9 -# CONDA_ENV: test-env -# resources: -# containers: -# - container: ubuntu1404 -# image: lightgbm/vsts-agent:ubuntu-14.04 -# - container: ubuntu-latest -# image: 'ubuntu:latest' -# options: "--name ci-container -v /usr/bin/docker:/tmp/docker:ro" -# - container: rbase -# image: rocker/r-base -# jobs: -# ########################################### -# - job: Linux -# ########################################### -# variables: -# COMPILER: gcc -# SETUP_CONDA: 'false' -# OS_NAME: 'linux' -# PRODUCES_ARTIFACTS: 'true' -# pool: sh-ubuntu -# container: ubuntu1404 -# strategy: -# matrix: -# regular: -# TASK: regular -# sdist: -# TASK: sdist -# PYTHON_VERSION: 3.7 -# bdist: -# TASK: bdist -# inference: -# TASK: if-else -# mpi_source: -# TASK: mpi -# METHOD: source -# PYTHON_VERSION: 3.8 -# gpu_source: -# TASK: gpu -# METHOD: source -# swig: -# TASK: swig -# steps: -# - script: | -# echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY" -# echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)" -# echo "##vso[task.prependpath]$CONDA/bin" -# AMDAPPSDK_PATH=$BUILD_SOURCESDIRECTORY/AMDAPPSDK -# echo "##vso[task.setvariable variable=AMDAPPSDK_PATH]$AMDAPPSDK_PATH" -# LD_LIBRARY_PATH=$AMDAPPSDK_PATH/lib/x86_64:$LD_LIBRARY_PATH -# echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$LD_LIBRARY_PATH" -# echo "##vso[task.setvariable variable=OPENCL_VENDOR_PATH]$AMDAPPSDK_PATH/etc/OpenCL/vendors" -# displayName: 'Set variables' -# - bash: $(Build.SourcesDirectory)/.ci/setup.sh -# displayName: Setup -# - bash: $(Build.SourcesDirectory)/.ci/test.sh -# displayName: Test -# - task: PublishBuildArtifacts@1 -# condition: and(succeeded(), in(variables['TASK'], 'regular', 'sdist', 'bdist', 'swig'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))) -# inputs: -# pathtoPublish: '$(Build.ArtifactStagingDirectory)' -# artifactName: PackageAssets -# artifactType: container -# ########################################### -# - job: Linux_latest -# ########################################### -# variables: -# COMPILER: clang -# DEBIAN_FRONTEND: 'noninteractive' -# IN_UBUNTU_LATEST_CONTAINER: 'true' -# OS_NAME: 'linux' -# SETUP_CONDA: 'true' -# pool: sh-ubuntu -# container: ubuntu-latest -# strategy: -# matrix: -# regular: -# TASK: regular -# PYTHON_VERSION: 3.6 -# sdist: -# TASK: sdist -# bdist: -# TASK: bdist -# PYTHON_VERSION: 3.8 -# inference: -# TASK: if-else -# mpi_source: -# TASK: mpi -# METHOD: source -# mpi_pip: -# TASK: mpi -# METHOD: pip -# PYTHON_VERSION: 3.8 -# mpi_wheel: -# TASK: mpi -# METHOD: wheel -# PYTHON_VERSION: 3.7 -# gpu_source: -# TASK: gpu -# METHOD: source -# gpu_pip: -# TASK: gpu -# METHOD: pip -# PYTHON_VERSION: 3.6 -# gpu_wheel: -# TASK: gpu -# METHOD: wheel -# PYTHON_VERSION: 3.7 -# cpp_tests: -# TASK: cpp-tests -# METHOD: with-sanitizers -# steps: -# - script: | -# echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY" -# echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)" -# CONDA=$HOME/miniconda -# echo "##vso[task.setvariable variable=CONDA]$CONDA" -# echo "##vso[task.prependpath]$CONDA/bin" -# AMDAPPSDK_PATH=$BUILD_SOURCESDIRECTORY/AMDAPPSDK -# echo "##vso[task.setvariable variable=AMDAPPSDK_PATH]$AMDAPPSDK_PATH" -# LD_LIBRARY_PATH=$AMDAPPSDK_PATH/lib/x86_64:$LD_LIBRARY_PATH -# echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$LD_LIBRARY_PATH" -# echo "##vso[task.setvariable variable=OPENCL_VENDOR_PATH]$AMDAPPSDK_PATH/etc/OpenCL/vendors" -# displayName: 'Set variables' -# # https://github.com/microsoft/azure-pipelines-agent/issues/2043#issuecomment-687983301 -# - script: | -# /tmp/docker exec -t -u 0 ci-container \ -# sh -c "apt-get update && apt-get -o Dpkg::Options::="--force-confold" -y install sudo" -# displayName: 'Install sudo' -# - bash: $(Build.SourcesDirectory)/.ci/setup.sh -# displayName: Setup -# - bash: $(Build.SourcesDirectory)/.ci/test.sh -# displayName: Test -# ########################################### -# - job: QEMU_multiarch -# ########################################### -# variables: -# COMPILER: gcc -# OS_NAME: 'linux' -# PRODUCES_ARTIFACTS: 'true' -# pool: -# vmImage: ubuntu-latest -# timeoutInMinutes: 180 -# strategy: -# matrix: -# bdist: -# TASK: bdist -# ARCH: aarch64 -# steps: -# - script: | -# sudo apt-get update -# sudo apt-get install --no-install-recommends -y \ -# binfmt-support \ -# qemu \ -# qemu-user \ -# qemu-user-static -# displayName: 'Install QEMU' -# - script: | -# docker run --rm --privileged multiarch/qemu-user-static --reset -p yes -# displayName: 'Enable Docker multi-architecture support' -# - script: | -# export ROOT_DOCKER_FOLDER=/LightGBM -# cat > docker.env < docker-script.sh < docker.env < docker-script.sh < Date: Tue, 3 Aug 2021 22:43:15 -0500 Subject: [PATCH 57/82] cuda and python --- .github/workflows/cuda.yml | 186 +++++++++++++-------------- .github/workflows/python_package.yml | 152 +++++++++++----------- 2 files changed, 169 insertions(+), 169 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index ea38b04d9c80..87c8db2aa094 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -1,96 +1,96 @@ -# name: CUDA Version +name: CUDA Version -# on: -# push: -# branches: -# - master -# pull_request: -# branches: -# - master +on: + push: + branches: + - master + pull_request: + branches: + - master -# env: -# github_actions: 'true' -# os_name: linux -# task: cuda -# conda_env: test-env +env: + github_actions: 'true' + os_name: linux + task: cuda + conda_env: test-env -# jobs: -# test: -# name: cuda ${{ matrix.cuda_version }} ${{ matrix.method }} (linux, ${{ matrix.compiler }}, Python ${{ matrix.python_version }}) -# runs-on: [self-hosted, linux] -# timeout-minutes: 60 -# strategy: -# fail-fast: false -# matrix: -# include: -# - method: source -# compiler: gcc -# python_version: 3.7 -# cuda_version: "11.4.0" -# - method: pip -# compiler: clang -# python_version: 3.8 -# cuda_version: "10.0" -# - method: wheel -# compiler: gcc -# python_version: 3.9 -# cuda_version: "9.0" -# steps: -# - name: Setup or update software on host machine -# run: | -# sudo apt-get update -# sudo apt-get install --no-install-recommends -y \ -# apt-transport-https \ -# ca-certificates \ -# curl \ -# git \ -# gnupg-agent \ -# software-properties-common -# curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - -# sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" -y -# curl -sL https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add - -# curl -sL https://nvidia.github.io/nvidia-docker/$(. /etc/os-release;echo $ID$VERSION_ID)/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list -# sudo apt-get update -# sudo apt-get install --no-install-recommends -y \ -# containerd.io \ -# docker-ce \ -# docker-ce-cli \ -# nvidia-docker2 -# sudo chmod a+rw /var/run/docker.sock -# sudo systemctl restart docker -# - name: Remove old folder with repository -# run: sudo rm -rf $GITHUB_WORKSPACE -# - name: Checkout repository -# uses: actions/checkout@v1 -# with: -# fetch-depth: 5 -# submodules: true -# - name: Setup and run tests -# run: | -# export ROOT_DOCKER_FOLDER=/LightGBM -# cat > docker.env < docker-script.sh < docker.env < docker-script.sh < Date: Tue, 3 Aug 2021 23:16:09 -0500 Subject: [PATCH 58/82] fix configs for Solaris and valgrind --- .github/workflows/r_solaris.yml | 1 + .github/workflows/r_valgrind.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/r_solaris.yml b/.github/workflows/r_solaris.yml index a297338ba2f6..a528b3d8464c 100644 --- a/.github/workflows/r_solaris.yml +++ b/.github/workflows/r_solaris.yml @@ -38,6 +38,7 @@ jobs: - name: Run tests on Solaris shell: bash run: | + Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'testthat', 'knitr', 'rmarkdown'), repos = 'https://cran.r-project.org', Ncpus = parallel::detectCores())" sh build-cran-package.sh || exit -1 apt-get install --no-install-recommends -y \ libcurl4-openssl-dev \ diff --git a/.github/workflows/r_valgrind.yml b/.github/workflows/r_valgrind.yml index 97c4d853564c..88dea07432d0 100644 --- a/.github/workflows/r_valgrind.yml +++ b/.github/workflows/r_valgrind.yml @@ -39,7 +39,7 @@ jobs: shell: bash run: | RDscriptvalgrind -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'rmarkdown', 'testthat'), repos = 'https://cran.r-project.org', Ncpus = parallel::detectCores())" - sh build-cran-package.sh + sh build-cran-package.sh --no-build-vignettes RDvalgrind CMD INSTALL --preclean --install-tests lightgbm_*.tar.gz || exit -1 - name: Run tests with valgrind shell: bash From 2353cc405245247f1f3a023a99716b6d8e00f35e Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sun, 8 Aug 2021 19:10:42 -0500 Subject: [PATCH 59/82] try TAR variable --- .appveyor.yml | 72 +-- .ci/test_r_package_windows.ps1 | 6 +- .github/workflows/cuda.yml | 186 +++---- .github/workflows/python_package.yml | 152 ++--- .github/workflows/r_package.yml | 224 ++++---- .github/workflows/static_analysis.yml | 164 +++--- .vsts-ci.yml | 768 +++++++++++++------------- 7 files changed, 787 insertions(+), 785 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index b7d9fe589176..ccb08022792a 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -1,43 +1,43 @@ -version: 3.2.1.99.{build} +# version: 3.2.1.99.{build} -image: Visual Studio 2015 -platform: x64 -configuration: # a trick to construct a build matrix with multiple Python versions - - 3.7 +# image: Visual Studio 2015 +# platform: x64 +# configuration: # a trick to construct a build matrix with multiple Python versions +# - 3.7 -# only build pull requests and -# commits to 'master' -branches: - only: - - master +# # only build pull requests and +# # commits to 'master' +# branches: +# only: +# - master -environment: - matrix: - - COMPILER: MSVC - TASK: python - - COMPILER: MINGW - TASK: python +# environment: +# matrix: +# - COMPILER: MSVC +# TASK: python +# - COMPILER: MINGW +# TASK: python -clone_depth: 5 +# clone_depth: 5 -install: - - git submodule update --init --recursive # get `external_libs` folder - - set PATH=%PATH:C:\Program Files\Git\usr\bin;=% # delete sh.exe from PATH (mingw32-make fix) - - set PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH% - - set PYTHON_VERSION=%CONFIGURATION% - - set CONDA_ENV="test-env" - - ps: | - switch ($env:PYTHON_VERSION) { - "3.6" {$env:MINICONDA = "C:\Miniconda36-x64"} - "3.7" {$env:MINICONDA = "C:\Miniconda37-x64"} - default {$env:MINICONDA = "C:\Miniconda37-x64"} - } - $env:PATH = "$env:MINICONDA;$env:MINICONDA\Scripts;$env:PATH" - $env:BUILD_SOURCESDIRECTORY = "$env:APPVEYOR_BUILD_FOLDER" - $env:LGB_VER = (Get-Content $env:APPVEYOR_BUILD_FOLDER\VERSION.txt).trim() +# install: +# - git submodule update --init --recursive # get `external_libs` folder +# - set PATH=%PATH:C:\Program Files\Git\usr\bin;=% # delete sh.exe from PATH (mingw32-make fix) +# - set PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH% +# - set PYTHON_VERSION=%CONFIGURATION% +# - set CONDA_ENV="test-env" +# - ps: | +# switch ($env:PYTHON_VERSION) { +# "3.6" {$env:MINICONDA = "C:\Miniconda36-x64"} +# "3.7" {$env:MINICONDA = "C:\Miniconda37-x64"} +# default {$env:MINICONDA = "C:\Miniconda37-x64"} +# } +# $env:PATH = "$env:MINICONDA;$env:MINICONDA\Scripts;$env:PATH" +# $env:BUILD_SOURCESDIRECTORY = "$env:APPVEYOR_BUILD_FOLDER" +# $env:LGB_VER = (Get-Content $env:APPVEYOR_BUILD_FOLDER\VERSION.txt).trim() -build: false +# build: false -test_script: - - conda init powershell - - powershell.exe -ExecutionPolicy Bypass -File %APPVEYOR_BUILD_FOLDER%\.ci\test_windows.ps1 +# test_script: +# - conda init powershell +# - powershell.exe -ExecutionPolicy Bypass -File %APPVEYOR_BUILD_FOLDER%\.ci\test_windows.ps1 diff --git a/.ci/test_r_package_windows.ps1 b/.ci/test_r_package_windows.ps1 index dd95a6c32336..e76bce907abc 100644 --- a/.ci/test_r_package_windows.ps1 +++ b/.ci/test_r_package_windows.ps1 @@ -171,9 +171,11 @@ if ($env:COMPILER -ne "MSVC") { # some flavors of tar.exe can fail in some settings on Windows. # Putting the msys64 utilities at the beginning of PATH temporarily to be # sure they're used for that purpose. - $env:PATH = "C:\msys64\usr\bin;" + $env:PATH + # $env:PATH = "C:\msys64\usr\bin;" + $env:PATH + $env:TAR = "C:\msys64\usr\bin\tar.exe" + $env:R_GZIPCMD = "C:\msys64\usr\bin\gzip.exe" Run-R-Code-Redirect-Stderr "result <- processx::run(command = 'sh', args = 'build-cran-package.sh', echo = TRUE, windows_verbatim_args = FALSE, error_on_status = TRUE)" ; Check-Output $? - Remove-From-Path ".*msys64.*" + #Remove-From-Path ".*msys64.*" # Test CRAN source .tar.gz in a directory that is not this repo or below it. # When people install.packages('lightgbm'), they won't have the LightGBM # git repo around. This is to protect against the use of relative paths diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 87c8db2aa094..ea38b04d9c80 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -1,96 +1,96 @@ -name: CUDA Version +# name: CUDA Version -on: - push: - branches: - - master - pull_request: - branches: - - master +# on: +# push: +# branches: +# - master +# pull_request: +# branches: +# - master -env: - github_actions: 'true' - os_name: linux - task: cuda - conda_env: test-env +# env: +# github_actions: 'true' +# os_name: linux +# task: cuda +# conda_env: test-env -jobs: - test: - name: cuda ${{ matrix.cuda_version }} ${{ matrix.method }} (linux, ${{ matrix.compiler }}, Python ${{ matrix.python_version }}) - runs-on: [self-hosted, linux] - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - include: - - method: source - compiler: gcc - python_version: 3.7 - cuda_version: "11.4.0" - - method: pip - compiler: clang - python_version: 3.8 - cuda_version: "10.0" - - method: wheel - compiler: gcc - python_version: 3.9 - cuda_version: "9.0" - steps: - - name: Setup or update software on host machine - run: | - sudo apt-get update - sudo apt-get install --no-install-recommends -y \ - apt-transport-https \ - ca-certificates \ - curl \ - git \ - gnupg-agent \ - software-properties-common - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" -y - curl -sL https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add - - curl -sL https://nvidia.github.io/nvidia-docker/$(. /etc/os-release;echo $ID$VERSION_ID)/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list - sudo apt-get update - sudo apt-get install --no-install-recommends -y \ - containerd.io \ - docker-ce \ - docker-ce-cli \ - nvidia-docker2 - sudo chmod a+rw /var/run/docker.sock - sudo systemctl restart docker - - name: Remove old folder with repository - run: sudo rm -rf $GITHUB_WORKSPACE - - name: Checkout repository - uses: actions/checkout@v1 - with: - fetch-depth: 5 - submodules: true - - name: Setup and run tests - run: | - export ROOT_DOCKER_FOLDER=/LightGBM - cat > docker.env < docker-script.sh < docker.env < docker-script.sh <&1 > ubsan-tests.log - cat ubsan-tests.log - exit $(cat ubsan-tests.log | grep --count "runtime error") - test-r-debian-clang: - name: r-package (debian, R-devel, clang) - timeout-minutes: 60 - runs-on: ubuntu-latest - container: rhub/debian-clang-devel - steps: - - name: Install Git before checkout - shell: bash - run: | - apt-get update - apt-get install --no-install-recommends -y git - - name: Checkout repository - uses: actions/checkout@v2.3.4 - with: - fetch-depth: 5 - submodules: true - - name: Install packages and run tests - shell: bash - run: | - export PATH=/opt/R-devel/bin/:${PATH} - Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'rmarkdown', 'testthat'), repos = 'https://cran.r-project.org', Ncpus = parallel::detectCores())" - sh build-cran-package.sh - R CMD check --as-cran --run-donttest lightgbm_*.tar.gz || exit -1 - if grep -q -E "NOTE|WARNING|ERROR" lightgbm.Rcheck/00check.log; then - echo "NOTEs, WARNINGs, or ERRORs have been found by R CMD check" - exit -1 - fi + # test-r-sanitizers: + # name: r-package (ubuntu-latest, R-devel, GCC ASAN/UBSAN) + # timeout-minutes: 60 + # runs-on: ubuntu-latest + # container: rhub/rocker-gcc-san + # steps: + # - name: Install Git before checkout + # shell: bash + # run: | + # apt-get update + # apt-get install --no-install-recommends -y git + # - name: Checkout repository + # uses: actions/checkout@v2.3.4 + # with: + # fetch-depth: 5 + # submodules: true + # - name: Install packages + # shell: bash + # run: | + # Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'rmarkdown', 'testthat'), repos = 'https://cran.r-project.org', Ncpus = parallel::detectCores())" + # sh build-cran-package.sh + # Rdevel CMD INSTALL lightgbm_*.tar.gz || exit -1 + # - name: Run tests with sanitizers + # shell: bash + # run: | + # cd R-package/tests + # Rscriptdevel testthat.R 2>&1 > ubsan-tests.log + # cat ubsan-tests.log + # exit $(cat ubsan-tests.log | grep --count "runtime error") + # test-r-debian-clang: + # name: r-package (debian, R-devel, clang) + # timeout-minutes: 60 + # runs-on: ubuntu-latest + # container: rhub/debian-clang-devel + # steps: + # - name: Install Git before checkout + # shell: bash + # run: | + # apt-get update + # apt-get install --no-install-recommends -y git + # - name: Checkout repository + # uses: actions/checkout@v2.3.4 + # with: + # fetch-depth: 5 + # submodules: true + # - name: Install packages and run tests + # shell: bash + # run: | + # export PATH=/opt/R-devel/bin/:${PATH} + # Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'rmarkdown', 'testthat'), repos = 'https://cran.r-project.org', Ncpus = parallel::detectCores())" + # sh build-cran-package.sh + # R CMD check --as-cran --run-donttest lightgbm_*.tar.gz || exit -1 + # if grep -q -E "NOTE|WARNING|ERROR" lightgbm.Rcheck/00check.log; then + # echo "NOTEs, WARNINGs, or ERRORs have been found by R CMD check" + # exit -1 + # fi all-successful: # https://github.community/t/is-it-possible-to-require-all-github-actions-tasks-to-pass-without-enumerating-them/117957/4?u=graingert runs-on: ubuntu-latest - needs: [test, test-r-sanitizers, test-r-debian-clang] + needs: [test] steps: - name: Note that all tests succeeded run: echo "🎉" diff --git a/.github/workflows/static_analysis.yml b/.github/workflows/static_analysis.yml index 0c41949bad1f..9851a3d771ba 100644 --- a/.github/workflows/static_analysis.yml +++ b/.github/workflows/static_analysis.yml @@ -1,85 +1,85 @@ -# contains non-functional tests, like checks on docs -# and code style -name: Static Analysis +# # contains non-functional tests, like checks on docs +# # and code style +# name: Static Analysis -on: - push: - branches: - - master - pull_request: - branches: - - master +# on: +# push: +# branches: +# - master +# pull_request: +# branches: +# - master -env: - COMPILER: 'gcc' - CONDA_ENV: test-env - GITHUB_ACTIONS: 'true' - OS_NAME: 'linux' - PYTHON_VERSION: 3.9 +# env: +# COMPILER: 'gcc' +# CONDA_ENV: test-env +# GITHUB_ACTIONS: 'true' +# OS_NAME: 'linux' +# PYTHON_VERSION: 3.9 -jobs: - test: - name: ${{ matrix.task }} - runs-on: ubuntu-latest - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - include: - - task: lint - - task: check-docs - steps: - - name: Checkout repository - uses: actions/checkout@v2.3.4 - with: - fetch-depth: 5 - submodules: false - - name: Setup and run tests - shell: bash - run: | - export TASK="${{ matrix.task }}" - export BUILD_DIRECTORY="$GITHUB_WORKSPACE" - export CONDA=${HOME}/miniconda - export PATH=${CONDA}/bin:$HOME/.local/bin:${PATH} - $GITHUB_WORKSPACE/.ci/setup.sh || exit -1 - $GITHUB_WORKSPACE/.ci/test.sh || exit -1 - r-check-docs: - name: r-package-check-docs - timeout-minutes: 60 - runs-on: ubuntu-latest - container: rocker/verse - steps: - - name: Checkout repository - uses: actions/checkout@v2.3.4 - with: - fetch-depth: 5 - submodules: true - - name: Install packages - shell: bash - run: | - Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'roxygen2', 'testthat', 'knitr', 'rmarkdown'), repos = 'https://cran.r-project.org', Ncpus = parallel::detectCores())" - sh build-cran-package.sh || exit -1 - R CMD INSTALL --with-keep.source lightgbm_*.tar.gz || exit -1 - - name: Test documentation - shell: bash --noprofile --norc {0} - run: | - Rscript --vanilla -e "roxygen2::roxygenize('R-package/', load = 'installed')" || exit -1 - num_doc_files_changed=$( - git diff --name-only | grep --count -E "\.Rd|NAMESPACE" - ) - if [[ ${num_doc_files_changed} -gt 0 ]]; then - echo "Some R documentation files have changed. Please re-generate them and commit those changes." - echo "" - echo " sh build-cran-package.sh" - echo " R CMD INSTALL --with-keep.source lightgbm_*.tar.gz" - echo " Rscript -e \"roxygen2::roxygenize('R-package/', load = 'installed')\"" - echo "" - exit -1 - fi - all-successful: - # https://github.community/t/is-it-possible-to-require-all-github-actions-tasks-to-pass-without-enumerating-them/117957/4?u=graingert - runs-on: ubuntu-latest - needs: [test, r-check-docs] - steps: - - name: Note that all tests succeeded - run: echo "🎉" +# jobs: +# test: +# name: ${{ matrix.task }} +# runs-on: ubuntu-latest +# timeout-minutes: 60 +# strategy: +# fail-fast: false +# matrix: +# include: +# - task: lint +# - task: check-docs +# steps: +# - name: Checkout repository +# uses: actions/checkout@v2.3.4 +# with: +# fetch-depth: 5 +# submodules: false +# - name: Setup and run tests +# shell: bash +# run: | +# export TASK="${{ matrix.task }}" +# export BUILD_DIRECTORY="$GITHUB_WORKSPACE" +# export CONDA=${HOME}/miniconda +# export PATH=${CONDA}/bin:$HOME/.local/bin:${PATH} +# $GITHUB_WORKSPACE/.ci/setup.sh || exit -1 +# $GITHUB_WORKSPACE/.ci/test.sh || exit -1 +# r-check-docs: +# name: r-package-check-docs +# timeout-minutes: 60 +# runs-on: ubuntu-latest +# container: rocker/verse +# steps: +# - name: Checkout repository +# uses: actions/checkout@v2.3.4 +# with: +# fetch-depth: 5 +# submodules: true +# - name: Install packages +# shell: bash +# run: | +# Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'roxygen2', 'testthat', 'knitr', 'rmarkdown'), repos = 'https://cran.r-project.org', Ncpus = parallel::detectCores())" +# sh build-cran-package.sh || exit -1 +# R CMD INSTALL --with-keep.source lightgbm_*.tar.gz || exit -1 +# - name: Test documentation +# shell: bash --noprofile --norc {0} +# run: | +# Rscript --vanilla -e "roxygen2::roxygenize('R-package/', load = 'installed')" || exit -1 +# num_doc_files_changed=$( +# git diff --name-only | grep --count -E "\.Rd|NAMESPACE" +# ) +# if [[ ${num_doc_files_changed} -gt 0 ]]; then +# echo "Some R documentation files have changed. Please re-generate them and commit those changes." +# echo "" +# echo " sh build-cran-package.sh" +# echo " R CMD INSTALL --with-keep.source lightgbm_*.tar.gz" +# echo " Rscript -e \"roxygen2::roxygenize('R-package/', load = 'installed')\"" +# echo "" +# exit -1 +# fi +# all-successful: +# # https://github.community/t/is-it-possible-to-require-all-github-actions-tasks-to-pass-without-enumerating-them/117957/4?u=graingert +# runs-on: ubuntu-latest +# needs: [test, r-check-docs] +# steps: +# - name: Note that all tests succeeded +# run: echo "🎉" diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 0a444f8aca05..37f64dd416cb 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -1,385 +1,385 @@ -trigger: - branches: - include: - - master - tags: - include: - - v* -pr: -- master -variables: - AZURE: 'true' - PYTHON_VERSION: 3.9 - CONDA_ENV: test-env -resources: - containers: - - container: ubuntu1404 - image: lightgbm/vsts-agent:ubuntu-14.04 - - container: ubuntu-latest - image: 'ubuntu:latest' - options: "--name ci-container -v /usr/bin/docker:/tmp/docker:ro" - - container: rbase - image: rocker/r-base -jobs: -########################################### -- job: Linux -########################################### - variables: - COMPILER: gcc - SETUP_CONDA: 'false' - OS_NAME: 'linux' - PRODUCES_ARTIFACTS: 'true' - pool: sh-ubuntu - container: ubuntu1404 - strategy: - matrix: - regular: - TASK: regular - sdist: - TASK: sdist - PYTHON_VERSION: 3.7 - bdist: - TASK: bdist - inference: - TASK: if-else - mpi_source: - TASK: mpi - METHOD: source - PYTHON_VERSION: 3.8 - gpu_source: - TASK: gpu - METHOD: source - swig: - TASK: swig - steps: - - script: | - echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY" - echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)" - echo "##vso[task.prependpath]$CONDA/bin" - AMDAPPSDK_PATH=$BUILD_SOURCESDIRECTORY/AMDAPPSDK - echo "##vso[task.setvariable variable=AMDAPPSDK_PATH]$AMDAPPSDK_PATH" - LD_LIBRARY_PATH=$AMDAPPSDK_PATH/lib/x86_64:$LD_LIBRARY_PATH - echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$LD_LIBRARY_PATH" - echo "##vso[task.setvariable variable=OPENCL_VENDOR_PATH]$AMDAPPSDK_PATH/etc/OpenCL/vendors" - displayName: 'Set variables' - - bash: $(Build.SourcesDirectory)/.ci/setup.sh - displayName: Setup - - bash: $(Build.SourcesDirectory)/.ci/test.sh - displayName: Test - - task: PublishBuildArtifacts@1 - condition: and(succeeded(), in(variables['TASK'], 'regular', 'sdist', 'bdist', 'swig'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))) - inputs: - pathtoPublish: '$(Build.ArtifactStagingDirectory)' - artifactName: PackageAssets - artifactType: container -########################################### -- job: Linux_latest -########################################### - variables: - COMPILER: clang - DEBIAN_FRONTEND: 'noninteractive' - IN_UBUNTU_LATEST_CONTAINER: 'true' - OS_NAME: 'linux' - SETUP_CONDA: 'true' - pool: sh-ubuntu - container: ubuntu-latest - strategy: - matrix: - regular: - TASK: regular - PYTHON_VERSION: 3.6 - sdist: - TASK: sdist - bdist: - TASK: bdist - PYTHON_VERSION: 3.8 - inference: - TASK: if-else - mpi_source: - TASK: mpi - METHOD: source - mpi_pip: - TASK: mpi - METHOD: pip - PYTHON_VERSION: 3.8 - mpi_wheel: - TASK: mpi - METHOD: wheel - PYTHON_VERSION: 3.7 - gpu_source: - TASK: gpu - METHOD: source - gpu_pip: - TASK: gpu - METHOD: pip - PYTHON_VERSION: 3.6 - gpu_wheel: - TASK: gpu - METHOD: wheel - PYTHON_VERSION: 3.7 - cpp_tests: - TASK: cpp-tests - METHOD: with-sanitizers - steps: - - script: | - echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY" - echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)" - CONDA=$HOME/miniconda - echo "##vso[task.setvariable variable=CONDA]$CONDA" - echo "##vso[task.prependpath]$CONDA/bin" - AMDAPPSDK_PATH=$BUILD_SOURCESDIRECTORY/AMDAPPSDK - echo "##vso[task.setvariable variable=AMDAPPSDK_PATH]$AMDAPPSDK_PATH" - LD_LIBRARY_PATH=$AMDAPPSDK_PATH/lib/x86_64:$LD_LIBRARY_PATH - echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$LD_LIBRARY_PATH" - echo "##vso[task.setvariable variable=OPENCL_VENDOR_PATH]$AMDAPPSDK_PATH/etc/OpenCL/vendors" - displayName: 'Set variables' - # https://github.com/microsoft/azure-pipelines-agent/issues/2043#issuecomment-687983301 - - script: | - /tmp/docker exec -t -u 0 ci-container \ - sh -c "apt-get update && apt-get -o Dpkg::Options::="--force-confold" -y install sudo" - displayName: 'Install sudo' - - bash: $(Build.SourcesDirectory)/.ci/setup.sh - displayName: Setup - - bash: $(Build.SourcesDirectory)/.ci/test.sh - displayName: Test -########################################### -- job: QEMU_multiarch -########################################### - variables: - COMPILER: gcc - OS_NAME: 'linux' - PRODUCES_ARTIFACTS: 'true' - pool: - vmImage: ubuntu-latest - timeoutInMinutes: 180 - strategy: - matrix: - bdist: - TASK: bdist - ARCH: aarch64 - steps: - - script: | - sudo apt-get update - sudo apt-get install --no-install-recommends -y \ - binfmt-support \ - qemu \ - qemu-user \ - qemu-user-static - displayName: 'Install QEMU' - - script: | - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - displayName: 'Enable Docker multi-architecture support' - - script: | - export ROOT_DOCKER_FOLDER=/LightGBM - cat > docker.env < docker-script.sh < docker.env < docker-script.sh < Date: Wed, 18 Aug 2021 21:10:21 -0500 Subject: [PATCH 60/82] uncomment CI --- .appveyor.yml | 72 +-- .github/workflows/cuda.yml | 186 +++---- .github/workflows/python_package.yml | 152 ++--- .github/workflows/r_package.yml | 110 ++-- .github/workflows/static_analysis.yml | 164 +++--- .vsts-ci.yml | 768 +++++++++++++------------- 6 files changed, 726 insertions(+), 726 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index ccb08022792a..b7d9fe589176 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -1,43 +1,43 @@ -# version: 3.2.1.99.{build} +version: 3.2.1.99.{build} -# image: Visual Studio 2015 -# platform: x64 -# configuration: # a trick to construct a build matrix with multiple Python versions -# - 3.7 +image: Visual Studio 2015 +platform: x64 +configuration: # a trick to construct a build matrix with multiple Python versions + - 3.7 -# # only build pull requests and -# # commits to 'master' -# branches: -# only: -# - master +# only build pull requests and +# commits to 'master' +branches: + only: + - master -# environment: -# matrix: -# - COMPILER: MSVC -# TASK: python -# - COMPILER: MINGW -# TASK: python +environment: + matrix: + - COMPILER: MSVC + TASK: python + - COMPILER: MINGW + TASK: python -# clone_depth: 5 +clone_depth: 5 -# install: -# - git submodule update --init --recursive # get `external_libs` folder -# - set PATH=%PATH:C:\Program Files\Git\usr\bin;=% # delete sh.exe from PATH (mingw32-make fix) -# - set PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH% -# - set PYTHON_VERSION=%CONFIGURATION% -# - set CONDA_ENV="test-env" -# - ps: | -# switch ($env:PYTHON_VERSION) { -# "3.6" {$env:MINICONDA = "C:\Miniconda36-x64"} -# "3.7" {$env:MINICONDA = "C:\Miniconda37-x64"} -# default {$env:MINICONDA = "C:\Miniconda37-x64"} -# } -# $env:PATH = "$env:MINICONDA;$env:MINICONDA\Scripts;$env:PATH" -# $env:BUILD_SOURCESDIRECTORY = "$env:APPVEYOR_BUILD_FOLDER" -# $env:LGB_VER = (Get-Content $env:APPVEYOR_BUILD_FOLDER\VERSION.txt).trim() +install: + - git submodule update --init --recursive # get `external_libs` folder + - set PATH=%PATH:C:\Program Files\Git\usr\bin;=% # delete sh.exe from PATH (mingw32-make fix) + - set PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH% + - set PYTHON_VERSION=%CONFIGURATION% + - set CONDA_ENV="test-env" + - ps: | + switch ($env:PYTHON_VERSION) { + "3.6" {$env:MINICONDA = "C:\Miniconda36-x64"} + "3.7" {$env:MINICONDA = "C:\Miniconda37-x64"} + default {$env:MINICONDA = "C:\Miniconda37-x64"} + } + $env:PATH = "$env:MINICONDA;$env:MINICONDA\Scripts;$env:PATH" + $env:BUILD_SOURCESDIRECTORY = "$env:APPVEYOR_BUILD_FOLDER" + $env:LGB_VER = (Get-Content $env:APPVEYOR_BUILD_FOLDER\VERSION.txt).trim() -# build: false +build: false -# test_script: -# - conda init powershell -# - powershell.exe -ExecutionPolicy Bypass -File %APPVEYOR_BUILD_FOLDER%\.ci\test_windows.ps1 +test_script: + - conda init powershell + - powershell.exe -ExecutionPolicy Bypass -File %APPVEYOR_BUILD_FOLDER%\.ci\test_windows.ps1 diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index ea38b04d9c80..87c8db2aa094 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -1,96 +1,96 @@ -# name: CUDA Version +name: CUDA Version -# on: -# push: -# branches: -# - master -# pull_request: -# branches: -# - master +on: + push: + branches: + - master + pull_request: + branches: + - master -# env: -# github_actions: 'true' -# os_name: linux -# task: cuda -# conda_env: test-env +env: + github_actions: 'true' + os_name: linux + task: cuda + conda_env: test-env -# jobs: -# test: -# name: cuda ${{ matrix.cuda_version }} ${{ matrix.method }} (linux, ${{ matrix.compiler }}, Python ${{ matrix.python_version }}) -# runs-on: [self-hosted, linux] -# timeout-minutes: 60 -# strategy: -# fail-fast: false -# matrix: -# include: -# - method: source -# compiler: gcc -# python_version: 3.7 -# cuda_version: "11.4.0" -# - method: pip -# compiler: clang -# python_version: 3.8 -# cuda_version: "10.0" -# - method: wheel -# compiler: gcc -# python_version: 3.9 -# cuda_version: "9.0" -# steps: -# - name: Setup or update software on host machine -# run: | -# sudo apt-get update -# sudo apt-get install --no-install-recommends -y \ -# apt-transport-https \ -# ca-certificates \ -# curl \ -# git \ -# gnupg-agent \ -# software-properties-common -# curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - -# sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" -y -# curl -sL https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add - -# curl -sL https://nvidia.github.io/nvidia-docker/$(. /etc/os-release;echo $ID$VERSION_ID)/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list -# sudo apt-get update -# sudo apt-get install --no-install-recommends -y \ -# containerd.io \ -# docker-ce \ -# docker-ce-cli \ -# nvidia-docker2 -# sudo chmod a+rw /var/run/docker.sock -# sudo systemctl restart docker -# - name: Remove old folder with repository -# run: sudo rm -rf $GITHUB_WORKSPACE -# - name: Checkout repository -# uses: actions/checkout@v1 -# with: -# fetch-depth: 5 -# submodules: true -# - name: Setup and run tests -# run: | -# export ROOT_DOCKER_FOLDER=/LightGBM -# cat > docker.env < docker-script.sh < docker.env < docker-script.sh < docker.env < docker-script.sh < docker.env < docker-script.sh < Date: Wed, 18 Aug 2021 21:11:53 -0500 Subject: [PATCH 61/82] more uncommenting --- .github/workflows/r_package.yml | 112 ++++++++++++++++---------------- 1 file changed, 56 insertions(+), 56 deletions(-) diff --git a/.github/workflows/r_package.yml b/.github/workflows/r_package.yml index e2ad3fbdad60..6e2e2c1f3b09 100644 --- a/.github/workflows/r_package.yml +++ b/.github/workflows/r_package.yml @@ -172,62 +172,62 @@ jobs: $env:GITHUB_ACTIONS = "true" $env:TASK = "${{ matrix.task }}" & "$env:GITHUB_WORKSPACE/.ci/test_windows.ps1" - # test-r-sanitizers: - # name: r-package (ubuntu-latest, R-devel, GCC ASAN/UBSAN) - # timeout-minutes: 60 - # runs-on: ubuntu-latest - # container: rhub/rocker-gcc-san - # steps: - # - name: Install Git before checkout - # shell: bash - # run: | - # apt-get update - # apt-get install --no-install-recommends -y git - # - name: Checkout repository - # uses: actions/checkout@v2.3.4 - # with: - # fetch-depth: 5 - # submodules: true - # - name: Install packages - # shell: bash - # run: | - # Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'rmarkdown', 'testthat'), repos = 'https://cran.r-project.org', Ncpus = parallel::detectCores())" - # sh build-cran-package.sh - # Rdevel CMD INSTALL lightgbm_*.tar.gz || exit -1 - # - name: Run tests with sanitizers - # shell: bash - # run: | - # cd R-package/tests - # Rscriptdevel testthat.R 2>&1 > ubsan-tests.log - # cat ubsan-tests.log - # exit $(cat ubsan-tests.log | grep --count "runtime error") - # test-r-debian-clang: - # name: r-package (debian, R-devel, clang) - # timeout-minutes: 60 - # runs-on: ubuntu-latest - # container: rhub/debian-clang-devel - # steps: - # - name: Install Git before checkout - # shell: bash - # run: | - # apt-get update - # apt-get install --no-install-recommends -y git - # - name: Checkout repository - # uses: actions/checkout@v2.3.4 - # with: - # fetch-depth: 5 - # submodules: true - # - name: Install packages and run tests - # shell: bash - # run: | - # export PATH=/opt/R-devel/bin/:${PATH} - # Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'rmarkdown', 'testthat'), repos = 'https://cran.r-project.org', Ncpus = parallel::detectCores())" - # sh build-cran-package.sh - # R CMD check --as-cran --run-donttest lightgbm_*.tar.gz || exit -1 - # if grep -q -E "NOTE|WARNING|ERROR" lightgbm.Rcheck/00check.log; then - # echo "NOTEs, WARNINGs, or ERRORs have been found by R CMD check" - # exit -1 - # fi + test-r-sanitizers: + name: r-package (ubuntu-latest, R-devel, GCC ASAN/UBSAN) + timeout-minutes: 60 + runs-on: ubuntu-latest + container: rhub/rocker-gcc-san + steps: + - name: Install Git before checkout + shell: bash + run: | + apt-get update + apt-get install --no-install-recommends -y git + - name: Checkout repository + uses: actions/checkout@v2.3.4 + with: + fetch-depth: 5 + submodules: true + - name: Install packages + shell: bash + run: | + Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'rmarkdown', 'testthat'), repos = 'https://cran.r-project.org', Ncpus = parallel::detectCores())" + sh build-cran-package.sh + Rdevel CMD INSTALL lightgbm_*.tar.gz || exit -1 + - name: Run tests with sanitizers + shell: bash + run: | + cd R-package/tests + Rscriptdevel testthat.R 2>&1 > ubsan-tests.log + cat ubsan-tests.log + exit $(cat ubsan-tests.log | grep --count "runtime error") + test-r-debian-clang: + name: r-package (debian, R-devel, clang) + timeout-minutes: 60 + runs-on: ubuntu-latest + container: rhub/debian-clang-devel + steps: + - name: Install Git before checkout + shell: bash + run: | + apt-get update + apt-get install --no-install-recommends -y git + - name: Checkout repository + uses: actions/checkout@v2.3.4 + with: + fetch-depth: 5 + submodules: true + - name: Install packages and run tests + shell: bash + run: | + export PATH=/opt/R-devel/bin/:${PATH} + Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'rmarkdown', 'testthat'), repos = 'https://cran.r-project.org', Ncpus = parallel::detectCores())" + sh build-cran-package.sh + R CMD check --as-cran --run-donttest lightgbm_*.tar.gz || exit -1 + if grep -q -E "NOTE|WARNING|ERROR" lightgbm.Rcheck/00check.log; then + echo "NOTEs, WARNINGs, or ERRORs have been found by R CMD check" + exit -1 + fi all-successful: # https://github.community/t/is-it-possible-to-require-all-github-actions-tasks-to-pass-without-enumerating-them/117957/4?u=graingert runs-on: ubuntu-latest From 985411b32b646c19645e13b5d1a1a8e6336a8f0f Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 18 Aug 2021 21:15:27 -0500 Subject: [PATCH 62/82] comment out cuda --- .github/workflows/cuda.yml | 186 ++++++++++++++++++------------------- 1 file changed, 93 insertions(+), 93 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 87c8db2aa094..ea38b04d9c80 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -1,96 +1,96 @@ -name: CUDA Version +# name: CUDA Version -on: - push: - branches: - - master - pull_request: - branches: - - master +# on: +# push: +# branches: +# - master +# pull_request: +# branches: +# - master -env: - github_actions: 'true' - os_name: linux - task: cuda - conda_env: test-env +# env: +# github_actions: 'true' +# os_name: linux +# task: cuda +# conda_env: test-env -jobs: - test: - name: cuda ${{ matrix.cuda_version }} ${{ matrix.method }} (linux, ${{ matrix.compiler }}, Python ${{ matrix.python_version }}) - runs-on: [self-hosted, linux] - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - include: - - method: source - compiler: gcc - python_version: 3.7 - cuda_version: "11.4.0" - - method: pip - compiler: clang - python_version: 3.8 - cuda_version: "10.0" - - method: wheel - compiler: gcc - python_version: 3.9 - cuda_version: "9.0" - steps: - - name: Setup or update software on host machine - run: | - sudo apt-get update - sudo apt-get install --no-install-recommends -y \ - apt-transport-https \ - ca-certificates \ - curl \ - git \ - gnupg-agent \ - software-properties-common - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" -y - curl -sL https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add - - curl -sL https://nvidia.github.io/nvidia-docker/$(. /etc/os-release;echo $ID$VERSION_ID)/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list - sudo apt-get update - sudo apt-get install --no-install-recommends -y \ - containerd.io \ - docker-ce \ - docker-ce-cli \ - nvidia-docker2 - sudo chmod a+rw /var/run/docker.sock - sudo systemctl restart docker - - name: Remove old folder with repository - run: sudo rm -rf $GITHUB_WORKSPACE - - name: Checkout repository - uses: actions/checkout@v1 - with: - fetch-depth: 5 - submodules: true - - name: Setup and run tests - run: | - export ROOT_DOCKER_FOLDER=/LightGBM - cat > docker.env < docker-script.sh < docker.env < docker-script.sh < Date: Wed, 18 Aug 2021 21:18:30 -0500 Subject: [PATCH 63/82] windows CI scripts --- .ci/test_r_package_windows.ps1 | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.ci/test_r_package_windows.ps1 b/.ci/test_r_package_windows.ps1 index e76bce907abc..c110f90dcd27 100644 --- a/.ci/test_r_package_windows.ps1 +++ b/.ci/test_r_package_windows.ps1 @@ -172,10 +172,11 @@ if ($env:COMPILER -ne "MSVC") { # Putting the msys64 utilities at the beginning of PATH temporarily to be # sure they're used for that purpose. # $env:PATH = "C:\msys64\usr\bin;" + $env:PATH - $env:TAR = "C:\msys64\usr\bin\tar.exe" - $env:R_GZIPCMD = "C:\msys64\usr\bin\gzip.exe" + if ($env:R_MAJOR_VERSION -eq "3") { + $env:PATH = "C:\msys64\usr\bin;" + $env:PATH + } Run-R-Code-Redirect-Stderr "result <- processx::run(command = 'sh', args = 'build-cran-package.sh', echo = TRUE, windows_verbatim_args = FALSE, error_on_status = TRUE)" ; Check-Output $? - #Remove-From-Path ".*msys64.*" + Remove-From-Path ".*msys64.*" # Test CRAN source .tar.gz in a directory that is not this repo or below it. # When people install.packages('lightgbm'), they won't have the LightGBM # git repo around. This is to protect against the use of relative paths From 1d5f8bd847193edd130d6de11c1e14ff3d2c7313 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sat, 30 Oct 2021 00:27:23 -0500 Subject: [PATCH 64/82] remove conftest files --- R-package/configure.win | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/R-package/configure.win b/R-package/configure.win index dfbaa3fc0377..d7f75f972771 100755 --- a/R-package/configure.win +++ b/R-package/configure.win @@ -37,6 +37,8 @@ int main() { EOL ${CXX} ${CXXFLAGS} ${CPPFLAGS} -o conftest conftest.cpp 2>/dev/null && ./conftest && ac_mm_prefetch="yes" +rm -f ./conftest +rm -f ./conftest.cpp echo "checking whether MM_PREFETCH works...${ac_mm_prefetch}" if test "${ac_mm_prefetch}" = "yes"; @@ -59,6 +61,8 @@ int main() { EOL ${CXX} ${CXXFLAGS} ${CPPFLAGS} -o conftest conftest.cpp 2>/dev/null && ./conftest && ac_mm_malloc="yes" +rm -f ./conftest +rm -f ./conftest.cpp echo "checking whether MM_MALLOC works...${ac_mm_malloc}" if test "${ac_mm_malloc}" = "yes"; From 78be0c4a40bafa1723ebafdd56fa896bd1373d82 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sat, 30 Oct 2021 00:27:48 -0500 Subject: [PATCH 65/82] restore cuda CI job --- .github/workflows/cuda.yml | 186 ++++++++++++++++++------------------- 1 file changed, 93 insertions(+), 93 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index ea38b04d9c80..87c8db2aa094 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -1,96 +1,96 @@ -# name: CUDA Version +name: CUDA Version -# on: -# push: -# branches: -# - master -# pull_request: -# branches: -# - master +on: + push: + branches: + - master + pull_request: + branches: + - master -# env: -# github_actions: 'true' -# os_name: linux -# task: cuda -# conda_env: test-env +env: + github_actions: 'true' + os_name: linux + task: cuda + conda_env: test-env -# jobs: -# test: -# name: cuda ${{ matrix.cuda_version }} ${{ matrix.method }} (linux, ${{ matrix.compiler }}, Python ${{ matrix.python_version }}) -# runs-on: [self-hosted, linux] -# timeout-minutes: 60 -# strategy: -# fail-fast: false -# matrix: -# include: -# - method: source -# compiler: gcc -# python_version: 3.7 -# cuda_version: "11.4.0" -# - method: pip -# compiler: clang -# python_version: 3.8 -# cuda_version: "10.0" -# - method: wheel -# compiler: gcc -# python_version: 3.9 -# cuda_version: "9.0" -# steps: -# - name: Setup or update software on host machine -# run: | -# sudo apt-get update -# sudo apt-get install --no-install-recommends -y \ -# apt-transport-https \ -# ca-certificates \ -# curl \ -# git \ -# gnupg-agent \ -# software-properties-common -# curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - -# sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" -y -# curl -sL https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add - -# curl -sL https://nvidia.github.io/nvidia-docker/$(. /etc/os-release;echo $ID$VERSION_ID)/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list -# sudo apt-get update -# sudo apt-get install --no-install-recommends -y \ -# containerd.io \ -# docker-ce \ -# docker-ce-cli \ -# nvidia-docker2 -# sudo chmod a+rw /var/run/docker.sock -# sudo systemctl restart docker -# - name: Remove old folder with repository -# run: sudo rm -rf $GITHUB_WORKSPACE -# - name: Checkout repository -# uses: actions/checkout@v1 -# with: -# fetch-depth: 5 -# submodules: true -# - name: Setup and run tests -# run: | -# export ROOT_DOCKER_FOLDER=/LightGBM -# cat > docker.env < docker-script.sh < docker.env < docker-script.sh < Date: Sat, 30 Oct 2021 00:31:11 -0500 Subject: [PATCH 66/82] remove non-R CI --- .appveyor.yml | 43 --- .github/workflows/cuda.yml | 103 ------- .github/workflows/python_package.yml | 79 ------ .vsts-ci.yml | 390 --------------------------- 4 files changed, 615 deletions(-) delete mode 100644 .appveyor.yml delete mode 100644 .github/workflows/cuda.yml delete mode 100644 .github/workflows/python_package.yml delete mode 100644 .vsts-ci.yml diff --git a/.appveyor.yml b/.appveyor.yml deleted file mode 100644 index ec87d24d613c..000000000000 --- a/.appveyor.yml +++ /dev/null @@ -1,43 +0,0 @@ -version: 3.3.1.99.{build} - -image: Visual Studio 2015 -platform: x64 -configuration: # a trick to construct a build matrix with multiple Python versions - - 3.7 - -# only build pull requests and -# commits to 'master' -branches: - only: - - master - -environment: - matrix: - - COMPILER: MSVC - TASK: python - - COMPILER: MINGW - TASK: python - -clone_depth: 5 - -install: - - git submodule update --init --recursive # get `external_libs` folder - - set PATH=%PATH:C:\Program Files\Git\usr\bin;=% # delete sh.exe from PATH (mingw32-make fix) - - set PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH% - - set PYTHON_VERSION=%CONFIGURATION% - - set CONDA_ENV="test-env" - - ps: | - switch ($env:PYTHON_VERSION) { - "3.6" {$env:MINICONDA = "C:\Miniconda36-x64"} - "3.7" {$env:MINICONDA = "C:\Miniconda37-x64"} - default {$env:MINICONDA = "C:\Miniconda37-x64"} - } - $env:PATH = "$env:MINICONDA;$env:MINICONDA\Scripts;$env:PATH" - $env:BUILD_SOURCESDIRECTORY = "$env:APPVEYOR_BUILD_FOLDER" - $env:LGB_VER = (Get-Content $env:APPVEYOR_BUILD_FOLDER\VERSION.txt).trim() - -build: false - -test_script: - - conda init powershell - - powershell.exe -ExecutionPolicy Bypass -File %APPVEYOR_BUILD_FOLDER%\.ci\test_windows.ps1 diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml deleted file mode 100644 index 822ea2d44b82..000000000000 --- a/.github/workflows/cuda.yml +++ /dev/null @@ -1,103 +0,0 @@ -name: CUDA Version - -on: - push: - branches: - - master - pull_request: - branches: - - master - -env: - github_actions: 'true' - os_name: linux - task: cuda - conda_env: test-env - -jobs: - test: - name: cuda ${{ matrix.cuda_version }} ${{ matrix.method }} (linux, ${{ matrix.compiler }}, Python ${{ matrix.python_version }}) - runs-on: [self-hosted, linux] - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - include: - - method: source - compiler: gcc - python_version: 3.7 - cuda_version: "11.4.2" - - method: pip - compiler: clang - python_version: 3.8 - cuda_version: "10.0" - - method: wheel - compiler: gcc - python_version: 3.9 - cuda_version: "9.0" - steps: - - name: Setup or update software on host machine - run: | - sudo apt-get update - sudo apt-get install --no-install-recommends -y \ - apt-transport-https \ - ca-certificates \ - curl \ - git \ - gnupg-agent \ - lsb-release \ - software-properties-common - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" -y - curl -sL https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add - - curl -sL https://nvidia.github.io/nvidia-docker/$(. /etc/os-release;echo $ID$VERSION_ID)/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list - sudo apt-get update - sudo apt-get install --no-install-recommends -y \ - containerd.io \ - docker-ce \ - docker-ce-cli \ - nvidia-docker2 - sudo chmod a+rw /var/run/docker.sock - sudo systemctl restart docker - - name: Remove old folder with repository - run: sudo rm -rf $GITHUB_WORKSPACE - - name: Checkout repository - uses: actions/checkout@v1 - with: - fetch-depth: 5 - submodules: true - - name: Setup and run tests - run: | - export ROOT_DOCKER_FOLDER=/LightGBM - cat > docker.env < docker-script.sh < docker.env < docker-script.sh < Date: Sat, 30 Oct 2021 00:43:47 -0500 Subject: [PATCH 67/82] run all R jobs --- .github/workflows/r_package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/r_package.yml b/.github/workflows/r_package.yml index 3607266bf71c..321f93d907d3 100644 --- a/.github/workflows/r_package.yml +++ b/.github/workflows/r_package.yml @@ -235,7 +235,7 @@ jobs: all-successful: # https://github.community/t/is-it-possible-to-require-all-github-actions-tasks-to-pass-without-enumerating-them/117957/4?u=graingert runs-on: ubuntu-latest - needs: [test] + needs: [test, test-r-sanitizers, test-r-debian-clang] steps: - name: Note that all tests succeeded run: echo "🎉" From 1eff95d458d27e91371a050ed323e95c9bf08111 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sat, 30 Oct 2021 01:17:09 -0500 Subject: [PATCH 68/82] use correct R executable in custom builds --- .ci/test_r_package_valgrind.sh | 3 ++- .github/workflows/r_package.yml | 3 ++- build-cran-package.sh | 8 ++++++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.ci/test_r_package_valgrind.sh b/.ci/test_r_package_valgrind.sh index 9af5559ba215..828189c328d6 100755 --- a/.ci/test_r_package_valgrind.sh +++ b/.ci/test_r_package_valgrind.sh @@ -1,7 +1,8 @@ #!/bin/bash RDscriptvalgrind -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'Matrix', 'rmarkdown', 'testthat'), repos = 'https://cran.r-project.org', Ncpus = parallel::detectCores())" || exit -1 -sh build-cran-package.sh || exit -1 +LGB_R_EXECUTABLE=RDvalgrind \ + sh build-cran-package.sh || exit -1 RDvalgrind CMD INSTALL --preclean --install-tests lightgbm_*.tar.gz || exit -1 cd R-package/tests diff --git a/.github/workflows/r_package.yml b/.github/workflows/r_package.yml index 321f93d907d3..d0b75fcfba65 100644 --- a/.github/workflows/r_package.yml +++ b/.github/workflows/r_package.yml @@ -195,7 +195,8 @@ jobs: shell: bash run: | RDscript${{ matrix.r_customization }} -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'Matrix', 'rmarkdown', 'testthat'), repos = 'https://cran.r-project.org', Ncpus = parallel::detectCores())" - sh build-cran-package.sh + LGB_R_EXECUTABLE=RD${{ matrix.r_customization }} \ + sh build-cran-package.sh RD${{ matrix.r_customization }} CMD INSTALL lightgbm_*.tar.gz || exit -1 - name: Run tests with sanitizers shell: bash diff --git a/build-cran-package.sh b/build-cran-package.sh index 9d73478583a1..91c49b43813d 100755 --- a/build-cran-package.sh +++ b/build-cran-package.sh @@ -19,6 +19,10 @@ set -e # Default values of arguments BUILD_VIGNETTES=true +# This option exists primarily so that certain CI jobs +# can use alternative builds of R +LGB_R_EXECUTABLE=${LGB_R_EXECUTABLE:-R} + # Loop through arguments and process them for arg in "$@" do @@ -168,7 +172,7 @@ cd "${TEMP_R_DIR}" cd "${ORIG_WD}" if ${BUILD_VIGNETTES} ; then - R CMD build \ + "${LGB_R_EXECUTABLE}" CMD build \ --keep-empty-dirs \ lightgbm_r @@ -202,7 +206,7 @@ if ${BUILD_VIGNETTES} ; then rm -rf ./_tmp else - R CMD build \ + "${LGB_R_EXECUTABLE}" CMD build \ --keep-empty-dirs \ --no-build-vignettes \ lightgbm_r From ce6004ada8b4711b22d77aca22ee1f2e16190009 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sat, 30 Oct 2021 23:30:02 -0500 Subject: [PATCH 69/82] Revert "use correct R executable in custom builds" This reverts commit 1eff95d458d27e91371a050ed323e95c9bf08111. --- .ci/test_r_package_valgrind.sh | 3 +-- .github/workflows/r_package.yml | 3 +-- build-cran-package.sh | 8 ++------ 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/.ci/test_r_package_valgrind.sh b/.ci/test_r_package_valgrind.sh index 828189c328d6..9af5559ba215 100755 --- a/.ci/test_r_package_valgrind.sh +++ b/.ci/test_r_package_valgrind.sh @@ -1,8 +1,7 @@ #!/bin/bash RDscriptvalgrind -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'Matrix', 'rmarkdown', 'testthat'), repos = 'https://cran.r-project.org', Ncpus = parallel::detectCores())" || exit -1 -LGB_R_EXECUTABLE=RDvalgrind \ - sh build-cran-package.sh || exit -1 +sh build-cran-package.sh || exit -1 RDvalgrind CMD INSTALL --preclean --install-tests lightgbm_*.tar.gz || exit -1 cd R-package/tests diff --git a/.github/workflows/r_package.yml b/.github/workflows/r_package.yml index d0b75fcfba65..321f93d907d3 100644 --- a/.github/workflows/r_package.yml +++ b/.github/workflows/r_package.yml @@ -195,8 +195,7 @@ jobs: shell: bash run: | RDscript${{ matrix.r_customization }} -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'Matrix', 'rmarkdown', 'testthat'), repos = 'https://cran.r-project.org', Ncpus = parallel::detectCores())" - LGB_R_EXECUTABLE=RD${{ matrix.r_customization }} \ - sh build-cran-package.sh + sh build-cran-package.sh RD${{ matrix.r_customization }} CMD INSTALL lightgbm_*.tar.gz || exit -1 - name: Run tests with sanitizers shell: bash diff --git a/build-cran-package.sh b/build-cran-package.sh index 91c49b43813d..9d73478583a1 100755 --- a/build-cran-package.sh +++ b/build-cran-package.sh @@ -19,10 +19,6 @@ set -e # Default values of arguments BUILD_VIGNETTES=true -# This option exists primarily so that certain CI jobs -# can use alternative builds of R -LGB_R_EXECUTABLE=${LGB_R_EXECUTABLE:-R} - # Loop through arguments and process them for arg in "$@" do @@ -172,7 +168,7 @@ cd "${TEMP_R_DIR}" cd "${ORIG_WD}" if ${BUILD_VIGNETTES} ; then - "${LGB_R_EXECUTABLE}" CMD build \ + R CMD build \ --keep-empty-dirs \ lightgbm_r @@ -206,7 +202,7 @@ if ${BUILD_VIGNETTES} ; then rm -rf ./_tmp else - "${LGB_R_EXECUTABLE}" CMD build \ + R CMD build \ --keep-empty-dirs \ --no-build-vignettes \ lightgbm_r From 85bf811c278ab09a18b0f31b68215be3295fd80b Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sat, 30 Oct 2021 23:40:07 -0500 Subject: [PATCH 70/82] whitespace and typos --- R-package/vignettes/basic_walkthrough.Rmd | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/R-package/vignettes/basic_walkthrough.Rmd b/R-package/vignettes/basic_walkthrough.Rmd index 192e4ee686f9..ad55a19f83ef 100644 --- a/R-package/vignettes/basic_walkthrough.Rmd +++ b/R-package/vignettes/basic_walkthrough.Rmd @@ -10,7 +10,6 @@ vignette: > %\VignetteEncoding{UTF-8} --- - ```{r, include = FALSE} knitr::opts_chunk$set( collapse = TRUE @@ -22,7 +21,7 @@ knitr::opts_chunk$set( ## Introduction -Welcome to the world of [LightGBM](https://lightgbm.readthedocs.io/en/latest/), a hightly efficient gradient boosting implementation (Ke et al. 2017). +Welcome to the world of [LightGBM](https://lightgbm.readthedocs.io/en/latest/), a highly efficient gradient boosting implementation (Ke et al. 2017). ```{r setup} library(lightgbm) From 832013f2e9493bfe382547c463fc0ed64ce8827c Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sat, 30 Oct 2021 23:40:19 -0500 Subject: [PATCH 71/82] lint vignettes --- .ci/lint_r_code.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/lint_r_code.R b/.ci/lint_r_code.R index cc83cb7c1cc9..d477a1a70b9c 100755 --- a/.ci/lint_r_code.R +++ b/.ci/lint_r_code.R @@ -8,7 +8,7 @@ SOURCE_DIR <- args[[1L]] FILES_TO_LINT <- list.files( path = SOURCE_DIR - , pattern = "\\.r$" + , pattern = "\\.r$|\\.rmd$" , all.files = TRUE , ignore.case = TRUE , full.names = TRUE From 4499c62ac767f625893e02db90aeb9459934865c Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sat, 30 Oct 2021 23:44:54 -0500 Subject: [PATCH 72/82] whitespace --- R-package/vignettes/basic_walkthrough.Rmd | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/R-package/vignettes/basic_walkthrough.Rmd b/R-package/vignettes/basic_walkthrough.Rmd index ad55a19f83ef..f74f214e659f 100644 --- a/R-package/vignettes/basic_walkthrough.Rmd +++ b/R-package/vignettes/basic_walkthrough.Rmd @@ -21,13 +21,13 @@ knitr::opts_chunk$set( ## Introduction -Welcome to the world of [LightGBM](https://lightgbm.readthedocs.io/en/latest/), a highly efficient gradient boosting implementation (Ke et al. 2017). +Welcome to the world of [LightGBM](https://lightgbm.readthedocs.io/en/latest/), a highly efficient gradient boosting implementation (Ke et al. 2017). ```{r setup} library(lightgbm) ``` -This vignette will guide you through its basic usage. It will show how to build a simple binary classification model based on a subset of the `bank` dataset (Moro, Cortez, and Rita 2014). You will use the two input features "age" and "balance" to predict whether a client has subscribed a term deposit. +This vignette will guide you through its basic usage. It will show how to build a simple binary classification model based on a subset of the `bank` dataset (Moro, Cortez, and Rita 2014). You will use the two input features "age" and "balance" to predict whether a client has subscribed a term deposit. ## The dataset @@ -49,7 +49,6 @@ The R package of LightGBM offers two functions to train a model: - `lgb.train()`: This is the main training logic. It offers full flexibility but requires a `Dataset` object created by the `lgb.Dataset()` function. - `lightgbm()`: Simpler, but less flexible. Data can be passed without having to bother with `lgb.Dataset()`. - ### Using the `lightgbm()` function In a first step, you need to convert data to numeric. Afterwards, you are ready to fit the model by the `lightgbm()` function. From 35e3a42b3d70794de088c1990ba3ba06a3d9bf8d Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sat, 30 Oct 2021 23:47:08 -0500 Subject: [PATCH 73/82] remove unnecessary log file --- build-cran-package.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/build-cran-package.sh b/build-cran-package.sh index 2a69ce7ad564..53bf50c9e390 100755 --- a/build-cran-package.sh +++ b/build-cran-package.sh @@ -54,8 +54,6 @@ echo "Building lightgbm with R executable: ${LGB_R_EXECUTABLE}" ORIG_WD="$(pwd)" TEMP_R_DIR="$(pwd)/lightgbm_r" -echo "Building R package (build vignettes: ${BUILD_VIGNETTES})" - if test -d "${TEMP_R_DIR}"; then rm -r "${TEMP_R_DIR}" fi From eebd89f7249f5800531160361dd2a080145ee049 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sat, 30 Oct 2021 23:52:46 -0500 Subject: [PATCH 74/82] Revert "remove non-R CI" This reverts commit 2075316e141809567f8b8b8c4fcd6c353a712ba3. --- .appveyor.yml | 43 +++ .github/workflows/cuda.yml | 103 +++++++ .github/workflows/python_package.yml | 79 ++++++ .vsts-ci.yml | 390 +++++++++++++++++++++++++++ 4 files changed, 615 insertions(+) create mode 100644 .appveyor.yml create mode 100644 .github/workflows/cuda.yml create mode 100644 .github/workflows/python_package.yml create mode 100644 .vsts-ci.yml diff --git a/.appveyor.yml b/.appveyor.yml new file mode 100644 index 000000000000..ec87d24d613c --- /dev/null +++ b/.appveyor.yml @@ -0,0 +1,43 @@ +version: 3.3.1.99.{build} + +image: Visual Studio 2015 +platform: x64 +configuration: # a trick to construct a build matrix with multiple Python versions + - 3.7 + +# only build pull requests and +# commits to 'master' +branches: + only: + - master + +environment: + matrix: + - COMPILER: MSVC + TASK: python + - COMPILER: MINGW + TASK: python + +clone_depth: 5 + +install: + - git submodule update --init --recursive # get `external_libs` folder + - set PATH=%PATH:C:\Program Files\Git\usr\bin;=% # delete sh.exe from PATH (mingw32-make fix) + - set PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH% + - set PYTHON_VERSION=%CONFIGURATION% + - set CONDA_ENV="test-env" + - ps: | + switch ($env:PYTHON_VERSION) { + "3.6" {$env:MINICONDA = "C:\Miniconda36-x64"} + "3.7" {$env:MINICONDA = "C:\Miniconda37-x64"} + default {$env:MINICONDA = "C:\Miniconda37-x64"} + } + $env:PATH = "$env:MINICONDA;$env:MINICONDA\Scripts;$env:PATH" + $env:BUILD_SOURCESDIRECTORY = "$env:APPVEYOR_BUILD_FOLDER" + $env:LGB_VER = (Get-Content $env:APPVEYOR_BUILD_FOLDER\VERSION.txt).trim() + +build: false + +test_script: + - conda init powershell + - powershell.exe -ExecutionPolicy Bypass -File %APPVEYOR_BUILD_FOLDER%\.ci\test_windows.ps1 diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml new file mode 100644 index 000000000000..822ea2d44b82 --- /dev/null +++ b/.github/workflows/cuda.yml @@ -0,0 +1,103 @@ +name: CUDA Version + +on: + push: + branches: + - master + pull_request: + branches: + - master + +env: + github_actions: 'true' + os_name: linux + task: cuda + conda_env: test-env + +jobs: + test: + name: cuda ${{ matrix.cuda_version }} ${{ matrix.method }} (linux, ${{ matrix.compiler }}, Python ${{ matrix.python_version }}) + runs-on: [self-hosted, linux] + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + include: + - method: source + compiler: gcc + python_version: 3.7 + cuda_version: "11.4.2" + - method: pip + compiler: clang + python_version: 3.8 + cuda_version: "10.0" + - method: wheel + compiler: gcc + python_version: 3.9 + cuda_version: "9.0" + steps: + - name: Setup or update software on host machine + run: | + sudo apt-get update + sudo apt-get install --no-install-recommends -y \ + apt-transport-https \ + ca-certificates \ + curl \ + git \ + gnupg-agent \ + lsb-release \ + software-properties-common + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" -y + curl -sL https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add - + curl -sL https://nvidia.github.io/nvidia-docker/$(. /etc/os-release;echo $ID$VERSION_ID)/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list + sudo apt-get update + sudo apt-get install --no-install-recommends -y \ + containerd.io \ + docker-ce \ + docker-ce-cli \ + nvidia-docker2 + sudo chmod a+rw /var/run/docker.sock + sudo systemctl restart docker + - name: Remove old folder with repository + run: sudo rm -rf $GITHUB_WORKSPACE + - name: Checkout repository + uses: actions/checkout@v1 + with: + fetch-depth: 5 + submodules: true + - name: Setup and run tests + run: | + export ROOT_DOCKER_FOLDER=/LightGBM + cat > docker.env < docker-script.sh < docker.env < docker-script.sh < Date: Tue, 2 Nov 2021 15:02:32 -0500 Subject: [PATCH 75/82] build dependencies before building lightgbm for Solaris submission --- .ci/test_r_package_solaris.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.ci/test_r_package_solaris.sh b/.ci/test_r_package_solaris.sh index c16c45e4ffe5..18ed6cb2f7ad 100755 --- a/.ci/test_r_package_solaris.sh +++ b/.ci/test_r_package_solaris.sh @@ -1,13 +1,15 @@ #!/bin/bash -sh build-cran-package.sh || exit -1 - apt-get install --no-install-recommends -y \ libcurl4-openssl-dev \ libxml2-dev \ libssl-dev -log_file="rhub_logs.txt" -Rscript -e "install.packages(c('knitr', 'rhub', 'rmarkdown'), dependencies = c('Depends', 'Imports', 'LinkingTo'), repos = 'https://cran.r-project.org', Ncpus = parallel::detectCores())" || exit -1 +# installation of dependencies needs to happen before building the package, +# since `R CMD build` needs to install the package to build vignettes +Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'Matrix', 'rmarkdown', 'rhub', 'testthat'), dependencies = c('Depends', 'Imports', 'LinkingTo'), repos = 'https://cran.r-project.org', Ncpus = parallel::detectCores())" || exit -1 +sh build-cran-package.sh || exit -1 + +log_file="rhub_logs.txt" Rscript ./.ci/run_rhub_solaris_checks.R lightgbm_*.tar.gz $log_file || exit -1 From 2eab091847f8b7b0745ef1e5d5ee0148befe7e81 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 2 Nov 2021 15:10:34 -0500 Subject: [PATCH 76/82] install necessary dependencies when building R package for releases --- .vsts-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 059ebf9d35b4..fce1edd477fc 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -300,6 +300,7 @@ jobs: steps: - script: | LGB_VER=$(head -n 1 VERSION.txt | sed "s/rc/-/g") + Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'Matrix', 'rmarkdown', 'testthat'), dependencies = c('Depends', 'Imports', 'LinkingTo'), repos = 'https://cran.r-project.org', Ncpus = parallel::detectCores())" || exit -1 sh build-cran-package.sh || exit -1 mv lightgbm_${LGB_VER}.tar.gz $(Build.ArtifactStagingDirectory)/lightgbm-${LGB_VER}-r-cran.tar.gz displayName: 'Build CRAN R-package' From d7d750bb446c4ca5dd01b5128655090534bc7bf6 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 2 Nov 2021 17:39:43 -0500 Subject: [PATCH 77/82] use wch1/r-debug image for solaris tests and artifact-building --- .github/workflows/r_solaris.yml | 3 +-- .vsts-ci.yml | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/r_solaris.yml b/.github/workflows/r_solaris.yml index 662307fce35d..da19394eedfd 100644 --- a/.github/workflows/r_solaris.yml +++ b/.github/workflows/r_solaris.yml @@ -9,7 +9,7 @@ jobs: name: solaris-cran timeout-minutes: 120 runs-on: ubuntu-latest - container: rocker/r-base + container: wch1/r-debug env: SECRETS_WORKFLOW: ${{ secrets.WORKFLOW }} steps: @@ -19,7 +19,6 @@ jobs: apt-get update apt-get install --no-install-recommends -y \ curl \ - git \ jq - name: Checkout repository uses: actions/checkout@v2.3.4 diff --git a/.vsts-ci.yml b/.vsts-ci.yml index fce1edd477fc..e3f350eba594 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -19,7 +19,7 @@ resources: image: 'ubuntu:latest' options: "--name ci-container -v /usr/bin/docker:/tmp/docker:ro" - container: rbase - image: rocker/r-base + image: wch1/r-debug jobs: ########################################### - job: Linux From f057e224c8ee48df41fad190d4be45c5eaa00d5a Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 3 Nov 2021 21:14:17 -0500 Subject: [PATCH 78/82] increase timeout --- .github/workflows/r_valgrind.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/r_valgrind.yml b/.github/workflows/r_valgrind.yml index d2e2fc0c2be9..56b9dac031fe 100644 --- a/.github/workflows/r_valgrind.yml +++ b/.github/workflows/r_valgrind.yml @@ -7,7 +7,7 @@ on: jobs: test-r-valgrind: name: r-package (ubuntu-latest, R-devel, valgrind) - timeout-minutes: 180 + timeout-minutes: 240 runs-on: ubuntu-latest container: wch1/r-debug env: From b3000a58c6cd4a51920b3ac4051c063f9b26d420 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 3 Nov 2021 21:59:41 -0500 Subject: [PATCH 79/82] pin knitr and rmarkdown in RTD builds --- docs/conf.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/conf.py b/docs/conf.py index c24143665909..cb81ee05698e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -272,8 +272,10 @@ def generate_r_docs(app: Sphinx) -> None: r-base=4.1.0=hb67fd72_2 \ r-data.table=1.14.0=r41hcfec24a_0 \ r-jsonlite=1.7.2=r41hcfec24a_0 \ + r-knitr=1.35=r41hc72bb7e_0 \ r-matrix=1.3_4=r41he454529_0 \ r-pkgdown=1.6.1=r41hc72bb7e_0 \ + r-rmarkdown=2.11=r41hc72bb7e_0 \ r-roxygen2=7.1.1=r41h03ef668_0 source /home/docs/.conda/bin/activate r_env export TAR=/bin/tar From 788897f02b338cd498ed69da060b91625f72e996 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 3 Nov 2021 23:30:32 -0500 Subject: [PATCH 80/82] build vignettes for CMake too --- .ci/test_r_package.sh | 20 +++++++------------ .ci/test_r_package_windows.ps1 | 7 ++----- R-package/README.md | 6 ++++++ build_r.R | 35 ++++++++-------------------------- 4 files changed, 23 insertions(+), 45 deletions(-) diff --git a/.ci/test_r_package.sh b/.ci/test_r_package.sh index cfe4b806e01d..aec25234c9fd 100755 --- a/.ci/test_r_package.sh +++ b/.ci/test_r_package.sh @@ -92,26 +92,20 @@ if [[ $OS_NAME == "macos" ]]; then fi fi -# Manually install libraries +# Manually install Depends and Imports libraries + 'knitr', 'rmarkdown', 'testthat' # to avoid a CI-time dependency on devtools (for devtools::install_deps()) -packages="'data.table', 'jsonlite', 'Matrix', 'R6'" - -# testthat is not required when running rchk -if [[ "${TASK}" != "r-rchk" ]]; then - packages="$packages, 'testthat'" -fi - -# only need knitr and rmarkdown if building vignettes -if [[ $R_BUILD_TYPE == "cran" ]]; then - packages="$packages, 'knitr', 'rmarkdown'" +# NOTE: testthat is not required when running rchk +if [[ "${TASK}" == "r-rchk" ]]; then + packages="c('data.table', 'jsonlite', 'knitr', 'Matrix', 'R6', 'rmarkdown')" +else + packages="c('data.table', 'jsonlite', 'knitr', 'Matrix', 'R6', 'rmarkdown', 'testthat')" fi - compile_from_source="both" if [[ $OS_NAME == "macos" ]]; then packages+=", type = 'binary'" compile_from_source="never" fi -Rscript --vanilla -e "options(install.packages.compile.from.source = '${compile_from_source}'); install.packages(c(${packages}), repos = '${CRAN_MIRROR}', lib = '${R_LIB_PATH}', dependencies = c('Depends', 'Imports', 'LinkingTo'), Ncpus = parallel::detectCores())" || exit -1 +Rscript --vanilla -e "options(install.packages.compile.from.source = '${compile_from_source}'); install.packages(${packages}, repos = '${CRAN_MIRROR}', lib = '${R_LIB_PATH}', dependencies = c('Depends', 'Imports', 'LinkingTo'), Ncpus = parallel::detectCores())" || exit -1 cd ${BUILD_DIRECTORY} diff --git a/.ci/test_r_package_windows.ps1 b/.ci/test_r_package_windows.ps1 index 01992264bc43..3045937c0f2b 100644 --- a/.ci/test_r_package_windows.ps1 +++ b/.ci/test_r_package_windows.ps1 @@ -122,11 +122,8 @@ Start-Process -FilePath Rtools.exe -NoNewWindow -Wait -ArgumentList "/VERYSILENT Write-Output "Done installing Rtools" Write-Output "Installing dependencies" -$packages = "c('data.table', 'jsonlite', 'Matrix', 'processx', 'R6', 'testthat')" -if ($env:R_BUILD_TYPE -eq "cran") { - $packages = "$packages, 'knitr', 'rmarkdown'" -} -Run-R-Code-Redirect-Stderr "options(install.packages.check.source = 'no'); install.packages(c($packages), dependencies = c('Imports', 'Depends', 'LinkingTo'), repos = '$env:CRAN_MIRROR', type = 'binary', lib = '$env:R_LIB_PATH', Ncpus = parallel::detectCores())" ; Check-Output $? +$packages = "c('data.table', 'jsonlite', 'knitr', 'Matrix', 'processx', 'R6', 'rmarkdown', 'testthat'), dependencies = c('Imports', 'Depends', 'LinkingTo')" +Run-R-Code-Redirect-Stderr "options(install.packages.check.source = 'no'); install.packages($packages, repos = '$env:CRAN_MIRROR', type = 'binary', lib = '$env:R_LIB_PATH', Ncpus = parallel::detectCores())" ; Check-Output $? # MiKTeX and pandoc can be skipped on non-MinGW builds, since we don't # build the package documentation for those. diff --git a/R-package/README.md b/R-package/README.md index 352071f0ad4b..d80a7f188025 100644 --- a/R-package/README.md +++ b/R-package/README.md @@ -152,6 +152,7 @@ Rscript build_r.R The `build_r.R` script builds the package in a temporary directory called `lightgbm_r`. It will destroy and recreate that directory each time you run the script. That script supports the following command-line options: +- `--no-build-vignettes`: Skip building vignettes. - `--skip-install`: Build the package tarball, but do not install it. - `--use-gpu`: Build a GPU-enabled version of the library. - `--use-mingw`: Force the use of MinGW toolchain, regardless of R version. @@ -266,6 +267,11 @@ sh build-cran-package.sh This will create a file `lightgbm_${VERSION}.tar.gz`, where `VERSION` is the version of `LightGBM`. +That script supports the following command-line options: + +- `--no-build-vignettes`: Skip building vignettes. +- `--r-executable=[path-to-executable]`: use an alternative build of R + Also, CRAN package is generated with every commit to any repo's branch and can be found in "Artifacts" section of the associated Azure Pipelines run. ### Standard Installation from CRAN Package diff --git a/build_r.R b/build_r.R index 0d462abf684d..dc3f61e6efa1 100644 --- a/build_r.R +++ b/build_r.R @@ -36,6 +36,7 @@ TEMP_SOURCE_DIR <- file.path(TEMP_R_DIR, "src") } parsed_args <- .parse_args(args) +SKIP_VIGNETTES <- "--no-build-vignettes" %in% parsed_args[["flags"]] USING_GPU <- "--use-gpu" %in% parsed_args[["flags"]] USING_MINGW <- "--use-mingw" %in% parsed_args[["flags"]] USING_MSYS2 <- "--use-msys2" %in% parsed_args[["flags"]] @@ -51,7 +52,8 @@ ARGS_TO_DEFINES <- c( ) recognized_args <- c( - "--skip-install" + "--no-build-vignettes" + , "--skip-install" , "--use-gpu" , "--use-mingw" , "--use-msys2" @@ -367,31 +369,6 @@ description_contents <- gsub( , replacement = as.character(Sys.Date()) , x = description_contents ) - -# CMake-based builds don't get vignettes built. The -# VignetteBuilder field needs to be removed to avoid -# the R CMD CHECK note -# "Package has a VignetteBuilder field but no prebuilt vignette index" -description_contents <- description_contents[ - !grepl("^VignetteBuilder", description_contents) -] - -# {knitr} and {rmarkdown} can be removed, since we don't build vignettes -# for CMake-based installations -description_contents <- description_contents[ - !grepl("^ +knitr,$", description_contents) -] -description_contents <- description_contents[ - !grepl("^ +rmarkdown,$", description_contents) -] - -# remove vignettes/ directory to avoid an R CMD CHECK note -# like "files in the 'vignettes' directory but no files in inst/doc" -unlink( - x = file.path(TEMP_R_DIR, "vignettes") - , recursive = TRUE -) - writeLines(description_contents, DESCRIPTION_FILE) # CMake-based builds can't currently use R's builtin routine registration, @@ -432,7 +409,11 @@ writeLines(namespace_contents, NAMESPACE_FILE) # NOTE: --keep-empty-dirs is necessary to keep the deep paths expected # by CMake while also meeting the CRAN req to create object files # on demand -.run_shell_command("R", c("CMD", "build", TEMP_R_DIR, "--keep-empty-dirs", "--no-build-vignettes")) +r_build_args <- c("CMD", "build", TEMP_R_DIR, "--keep-empty-dirs") +if (isTRUE(SKIP_VIGNETTES)) { + r_build_args <- c(r_build_args, "--no-build-vignettes") +} +.run_shell_command("R", r_build_args) # Install the package version <- gsub( From 3bbee12b5a2dceb80b6781d979c45522daa5d7af Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 5 Nov 2021 23:07:53 -0500 Subject: [PATCH 81/82] suppress noisy output from tar --- build-cran-package.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/build-cran-package.sh b/build-cran-package.sh index 53bf50c9e390..704edfbcc63e 100755 --- a/build-cran-package.sh +++ b/build-cran-package.sh @@ -199,7 +199,7 @@ if ${BUILD_VIGNETTES} ; then echo "untarring ${TARBALL_NAME}" cd _tmp - tar -xvf "${TARBALL_NAME}" + tar -xvf "${TARBALL_NAME}" > /dev/null 2>&1 rm -rf "${TARBALL_NAME}" cd .. echo "done untarring ${TARBALL_NAME}" @@ -216,7 +216,8 @@ if ${BUILD_VIGNETTES} ; then --exclude=**/conftest.c \ --exclude=**/conftest.exe \ -f "${TARBALL_NAME}" \ - lightgbm + lightgbm \ + > /dev/null 2>&1 echo "Done creating ${TARBALL_NAME}" rm -rf ./_tmp From ddd9a62b1fda06c12f89c87b4d5a493f59909a29 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sat, 6 Nov 2021 09:56:24 -0500 Subject: [PATCH 82/82] add link in pkgdown config --- R-package/pkgdown/_pkgdown.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/R-package/pkgdown/_pkgdown.yml b/R-package/pkgdown/_pkgdown.yml index b89ab96f3cd9..9e105d2c6bb5 100644 --- a/R-package/pkgdown/_pkgdown.yml +++ b/R-package/pkgdown/_pkgdown.yml @@ -41,6 +41,8 @@ navbar: href: ../ - icon: fa-home fa-lg href: index.html + - text: Articles + href: articles/index.html - text: Reference href: reference/index.html right: