diff --git a/.travis.yml b/.travis.yml index f7bf002c73c..20fc82ada9f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,86 +14,45 @@ env: - _R_CHECK_LENGTH_1_CONDITION_=true - _R_CHECK_LENGTH_1_LOGIC2_=true -_apt: &apt-base - - bc - - curl - - gdal-bin - - jags - - libgdal-dev - - libgl1-mesa-dev - - libglu1-mesa-dev - - libglpk-dev # indirectly needed by BayesianTools - - libgmp-dev - - libhdf5-dev - - liblapack-dev - - libnetcdf-dev - - libproj-dev - - librdf0-dev - - libudunits2-dev - - netcdf-bin - - pandoc - - python-dev - - qpdf - - tcl - - tcl-dev - - udunits-bin - -_c2d4u: &apt-r-binaries - - r-bioc-biocinstaller - - r-cran-ape - - r-cran-curl - - r-cran-data.table - - r-cran-devtools - - r-cran-dplyr - - r-cran-gap - - r-cran-ggplot2 - - r-cran-httr - - r-cran-igraph - - r-cran-lme4 - - r-cran-matrixstats - - r-cran-mcmcpack - - r-cran-raster - - r-cran-rcpp - - r-cran-rcurl - - r-cran-redland - - r-cran-rjags - - r-cran-rncl - - r-cran-roxygen2 - - r-cran-rsqlite - # - r-cran-sf - - r-cran-shiny - - r-cran-sirt - - r-cran-testthat - - r-cran-tidyverse - - r-cran-xml - - r-cran-xml2 - - r-cran-xts +addons: + apt: + sources: + - sourceline: 'ppa:ubuntugis/ppa' # for GDAL 2 binaries + packages: + - bc + - curl + - gdal-bin + - jags + - libgdal-dev + - libgl1-mesa-dev + - libglu1-mesa-dev + - libglpk-dev # indirectly needed by BayesianTools + - libgmp-dev + - libhdf5-dev + - liblapack-dev + - libnetcdf-dev + - libproj-dev + - librdf0-dev + - libudunits2-dev + - netcdf-bin + - pandoc + - python-dev + - qpdf + - tcl + - tcl-dev + - udunits-bin jobs: fast_finish: true include: - r: release - addons: &addons-c2d4u - apt: - sources: - - sourceline: 'ppa:ubuntugis/ppa' # for GDAL 2 binaries - packages: - - *apt-base - - *apt-r-binaries - r: devel - addons: &addons-base - apt: - sources: - - sourceline: 'ppa:ubuntugis/ppa' # for GDAL 2 binaries - packages: - - *apt-base - r: oldrel - addons: *addons-base + - r: 3.5 allow_failures: - r: devel - addons: *addons-base - r: oldrel - addons: *addons-base + - r: 3.5 cache: - directories: diff --git a/book_source/02_demos_tutorials_workflows/05_developer_workflows/04-testing.Rmd b/book_source/02_demos_tutorials_workflows/05_developer_workflows/04-testing.Rmd index 2f9bddb3291..2694dc56c57 100755 --- a/book_source/02_demos_tutorials_workflows/05_developer_workflows/04-testing.Rmd +++ b/book_source/02_demos_tutorials_workflows/05_developer_workflows/04-testing.Rmd @@ -122,22 +122,21 @@ The `batch_run.R` script can take the following command-line arguments: ### Continuous Integration -Every time anyone commits a change to the PEcAn code, the act of pushing to GitHub triggers an automated build and test of the full PEcAn codebase using [Travis CI](https://travis-ci.org/pecanProject/pecan), and all pull requests must report a successful CI build before they will be merged. This will sometimes feel like a burden when the build breaks on an issue that looks trivial, but anything that breaks the build is important enough to fix. It's much better to find errors early and fix them before they get incorporated into the released PEcAn code. +Every time anyone commits a change to the PEcAn code, the act of pushing to GitHub triggers an automated build and test of the full PEcAn codebase, and all pull requests must report a successful CI build before they will be merged. This will sometimes feel like a burden when the build breaks on an issue that looks trivial, but anything that breaks the build is important enough to fix. It's much better to find errors early and fix them before they get incorporated into the released PEcAn code. -At this writing (September 2019), all our Travis builds run on the same version of Linux (currently Ubuntu 16.04, `xenial`) using three different versions of R in parallel: previous release, current release, and nightly builds of the R development branch. In most cases the build should pass on all three versions, but only the current release (`R:release`) is truly mandatory. We aim to fix errors found on R:release before merging, errors found on R:devel before the next R release, and errors found on R:oldrel as developer time and forward compatibility allow. +At this writing PEcAn's CI builds primarily use [Travis CI](https://travis-ci.org/pecanProject/pecan) and the rest of this section assumes a Travis build, but as of May 2020 we also have an experimental GitHub Actions build, and if we switch completely to GitHub Actions then this guide will need to be rewritten. -Each build starts by launching three clean virtual machines (one for each R version) and performs roughly the following actions on all of them: +All our Travis builds run on the same version of Linux (currently Ubuntu 16.04, `xenial`) using four different versions of R in parallel: the two most recent previous releases, current release, and nightly builds of the R development branch. In most cases the build should pass on all four versions, but only the current release (`R:release`) is truly mandatory. We aim to fix errors found on R:release before merging, errors found on R:devel before the next R release, and errors found on older releases as developer time and forward compatibility allow. + +Each build starts by launching a separate clean virtual machine for each R version and performs roughly the following actions on all of them: * Installs binary system dependencies needed by PEcAn (NetCDF and HDF5 libraries, JAGS, udunits, etc). -* Installs prebuilt binaries of a few R packages from [cran2deb4ubuntu](https://launchpad.net/~marutter/+archive/ubuntu/c2d4u3.5). - - This is only a small subset of the PEcAn dependency list, mostly the ones that take a very long time to compile but have binaries small enough to download quickly. - - The rest are installed as needed during the installation of PEcAn packages. - - Because these packages are compiled for a specific version of R and we test on three different versions, in some cases the installed binary is incompatible with the installed R version and is overwritten by a locally-compiled one in a later step. At this writing this is only done on R 3.5 (the current R:oldrel), but this may change. +* Installs all the R packages that are declared as dependencies in any PEcAn package, as computed by `scripts/generate_dependencies.R`. * Clones the PEcAn repository from GitHub, and checks out the branch to be tested. * Retrieves any cached files available from previous Travis builds. - The main thing in the cache is previously-installed dependency R packages, to avoid recompiling them every time. - If the cache becomes stale or is preventing a package update needed by the build (e.g. to get a new version that contains a needed bug fix), delete the cache through the Travis web interface and it will be reconstructed on the next build. - - Because PEcAn has so many dependencies, builds with no cache will spend most of their time recompiling packages and will often run out of time before the tests complete. You can fix this by using `scripts/travis/cache_buildup.sh` and `scripts/travis/prime_travis_cache.sh` to build up the cache incrementally through one or more [custom builds](https://blog.travis-ci.com/2017-08-24-trigger-custom-build), each of which installs some dependencies and then uploads a freshened cache *without* running any tests. Once all dependencies have been cached, restart the standard full build. + - Because PEcAn has so many dependencies, builds with no cache will spend most of their time recompiling packages and will probably run out of time before the tests complete. You can fix this by using `scripts/travis/cache_buildup.sh` and `scripts/travis/prime_travis_cache.sh` to build up the cache incrementally through one or more [custom builds](https://blog.travis-ci.com/2017-08-24-trigger-custom-build), each of which installs some dependencies and then uploads a freshened cache *without* running any tests. Once all dependencies have been cached, restart the standard full build. * Initializes a skeleton version of the PEcAn database (BeTY) containing a few public records to be used by the test runs. * Installs all the PEcAn packages, recompiling documentation and installing dependencies as needed. * Runs package unit tests (the same ones you run locally with `make test` or `devtools::test(pkgname)`). @@ -150,7 +149,7 @@ Each build starts by launching three clean virtual machines (one for each R vers - Each line of the check log is considered a separate message, and the test requires exact matching, so a change from `Undocumented arguments in documentation object 'foo': 'x'` to `Undocumented arguments in documentation object 'foo': 'x', 'y'` will be counted as a new warning... and you should fix both of them while you're at it! - The idea here is to enforce good coding practice and catch likely errors in all new code while recognizing that we have a lot of legacy code whose warnings need to be fixed as we have time rather than all at once. - As we fix historic warnings, we will revoke their grandfathered status by removing them from the stored check results, so that they will break the build if they reappear. - - If your PR reports a failure in pre-existing code that you think ought to be grandfathered, please fix it anyway. The failures all need to be cleaned up eventually, and it's likely easier to fix the error than to figure out how to re-ignore it. + - If your PR reports a failure in pre-existing code that you think ought to be grandfathered, please fix it as part of your PR anyway. It's frustrating to see tests complain about code you didn't touch, but the failures all need to be cleaned up eventually and it's likely easier to fix the error than to figure out how to re-ignore it. * Runs a simple PEcAn workflow from beginning to end (three years of SIPNET at Niwot Ridge using stored weather data, meta-analysis on fixed effects only, sensitivity analysis on NPP), and verifies that the models complete successfully. * Checks whether any version-controlled files have been changed by the build and testing process, and marks a build failure if they have. - If your build fails at this step, the most common cause is that you forgot to Roxygenize before committing. diff --git a/book_source/04_appendix/04-package-dependencies.Rmd b/book_source/04_appendix/04-package-dependencies.Rmd index bc6e68a93f0..94d80ddf96e 100644 --- a/book_source/04_appendix/04-package-dependencies.Rmd +++ b/book_source/04_appendix/04-package-dependencies.Rmd @@ -98,8 +98,4 @@ If you think your package needs to load or attach code for any reason, please no ## Installing dependencies: Let the machines do it -In most cases you won't need to think about how dependencies get installed -- just declare them in your package's DESCRIPTION and the installation will be handled automatically by R and devtools during the build process. In PEcAn packages, the rare cases where this isn't enough will probably fall into one of two categories. - -First, some dependencies rely on non-R software that R does not know how to install automatically. For example, rjags relies on JAGS, which might be installed in a different place on every machine. If your dependency falls in this category, you will know (because your CI builds will keep failing until you figure out how to fix it), but the exact details of the fix will differ for every case. - -Second, some dependencies *will* install automatically, but they take a long time to compile or conflict with dependencies needed by other packages or are otherwise annoying to deal with. To save time during CI builds, PEcAn's Travis configuration file includes a manually curated list of the most-annoying dependencies and installs them from pre-compiled binaries before starting the normal installation process. If you suspect you're adding a dependency that will be "annoying", please *do not* add it to the Travis binary-install list right away; instead, focus on getting your package to work right using the default automatic installation. Then, if needed to keep build times reasonable, submit a separate pull request to change the Travis configuration. This two-step procedure makes it much easier to understand which merges changed package code and which ones changed the testing configuration without changing package functionality, and also lets you focus on what the code is supposed to do instead of on installation details. +In most cases you won't need to think about how dependencies get installed -- just declare them in your package's DESCRIPTION and the installation will be handled automatically by R and devtools during the build process. The main exception is when a dependency relies on non-R software that R does not know how to install automatically. For example, rjags relies on JAGS, which might be installed in a different place on every machine. If your dependency falls in this category, you will know (because your CI builds will keep failing until you figure out how to fix it), but the exact details of the fix will differ for every case. diff --git a/docker/depends/pecan.depends b/docker/depends/pecan.depends index 5494ad53436..cd041decf6e 100644 --- a/docker/depends/pecan.depends +++ b/docker/depends/pecan.depends @@ -7,6 +7,7 @@ set -e # Don't use X11 for rgl RGL_USE_NULL=TRUE +RLIB=${R_LIBS_USER:-/usr/local/lib/R/site-library} # install remotes first in case packages are references in dependencies installGithub.r \ @@ -17,7 +18,7 @@ installGithub.r \ ropensci/nneo # install all packages (depends, imports, suggests) -install2.r -e -s -n -1\ +install2.r -e -s -l "${RLIB}" -n -1\ abind \ BayesianTools \ binaryLogic \ diff --git a/modules/assim.batch/tests/Rcheck_reference.log b/modules/assim.batch/tests/Rcheck_reference.log index 40865f2a793..6072eb26a19 100644 --- a/modules/assim.batch/tests/Rcheck_reference.log +++ b/modules/assim.batch/tests/Rcheck_reference.log @@ -8,7 +8,11 @@ * this is package ‘PEcAn.assim.batch’ version ‘1.7.0’ * package encoding: UTF-8 * checking package namespace information ... OK -* checking package dependencies ... OK +* checking package dependencies ... NOTE +Imports includes 28 non-default packages. +Importing from so many packages makes the package vulnerable to any of +them becoming unavailable. Move as many as possible to Suggests and +use conditionally. * checking if this is a source package ... OK * checking if there is a namespace ... OK * checking for executable files ... OK diff --git a/modules/data.atmosphere/R/data.R b/modules/data.atmosphere/R/data.R new file mode 100644 index 00000000000..256b8573748 --- /dev/null +++ b/modules/data.atmosphere/R/data.R @@ -0,0 +1,150 @@ + +## Drafts of documentation for package datasets +## +## Written by CKB 2020-05-03, then commented out when I realized that as +## written we need to enable lazy-loading of package data to use these. +## TODO in this order: +## * Inspect all datasets, determine whether lazy-loading them into package +## namespace will cause any issues +## * make any changes needed to resolve issues identified above +## * Change DESCRIPTION line to read `LazyData: true` +## * Uncomment this file, delete this header block +## * run Roxygen, commit resulting Rd files + +# #' 2010 CRUNCEP weather data for Urbana, IL +# #' +# #' Hourly 2010 meteorology for the 0.5-degree grid cell containing the +# #' EBI Energy Farm (Urbana, IL), as obtained from the CRUNCEP +# #' 6-hourly product. +# #' Please see the `compare_narr_cruncep_met` vignette for details of a +# #' comparison between this and the `narr`, `narr3h`, and `ebifarm` datasets. +# #' +# #' @format A data frame with 8736 rows and 10 columns: +# #' \describe{ +# #' \item{date}{POSIXct timestamp} +# #' \item{year, doy, hour}{integer, extracted from `date`} +# #' \item{solarR}{solar radiation, in umol/h/m2} +# #' \item{DailyTemp.C}{air temperature, in degrees C} +# #' \item{RH}{relative humidity, in percent} +# #' \item{WindSpeed}{wind speed, in m/s} +# #' \item{precip}{precipitation rate, in mm/h} +# #' \item{source}{dataset identifier, in this case always "cruncep"}} +# #' @seealso \code{\link{narr}} \code{\link{narr3h}} \code{\link{ebifarm}} +# "cruncep" +# +# +# #' Global 0.5 degree land/water mask for the CRUNCEP dataset +# #' +# #' For details, please see the CRUNCEP scripts included with this package: +# #' `system.file("scripts/cruncep", package = "PEcAn.data.atmosphere")` +# #' +# #' @format a data frame with 259200 rows and 3 columns: +# #' \describe{ +# #' \item{lat}{latitude, in decimal degrees} +# #' \item{lon}{longitude, in decimal degrees} +# #' \item{land}{logical. TRUE = land, FALSE = water}} +# "cruncep_landmask" +# +# +# #' 2010 weather station data from near Urbana, IL +# #' +# #' Hourly 2010 weather data collected at the EBI Energy Farm (Urbana, IL). +# #' Please see the `compare_narr_cruncep_met` vignette for details of a +# #' comparison between this and the `narr`, `narr3h`, and `cruncep` datasets. +# #' +# #' @format A data frame with 8390 rows and 10 columns: +# #' \describe{ +# #' \item{date}{POSIXct timestamp} +# #' \item{year, doy, hour}{integer, extracted from `date`} +# #' \item{Temp}{air temperature, in degrees C} +# #' \item{RH}{relative humidity, in percent} +# #' \item{precip}{precipitation rate, in mm/h} +# #' \item{wind}{wind speed, in m/s} +# #' \item{solar}{solar radiation, in umol/h/m2} +# #' \item{source}{dataset identifier, in this case always "ebifarm"}} +# #' @seealso \code{\link{cruncep}} \code{\link{narr}} \code{\link{narr3h}} +# "ebifarm" +# +# +# #' Codes and BeTY IDs for sites in the FLUXNET network +# #' +# #' @format a data frame with 698 rows and 2 columns: +# #' \describe{ +# #' \item{FLUX.id}{character identifier used by FLUXNET, +# #' e.g. Niwot Ridge USA is `US-NR1`} +# #' \item{site.id}{identifier used in the `sites` table of the PEcAn +# #' database. Integer, but stored as character}} +# "FLUXNET.sitemap" +# +# +# #' Global land/water mask for the NCEP dataset +# #' +# #' For details, please see the NCEP scripts included with this package: +# #' `system.file("scripts/ncep", package = "PEcAn.data.atmosphere")` +# #' +# #' @format a data frame with 18048 rows and 3 columns: +# #' \describe{ +# #' \item{lat}{latitude, in decimal degrees} +# #' \item{lon}{longitude, in decimal degrees} +# #' \item{land}{logical. TRUE = land, FALSE = water}} +# "landmask" +# +# +# #' Latitudes of 94 sites from the NCEP dataset +# #' +# #' For details, please see the NCEP scripts included with this package: +# #' `system.file("scripts/ncep", package = "PEcAn.data.atmosphere")` +# #' +# #' @format a vector of 94 decimal values +# "Lat" +# +# +# #' Longitudes of 192 sites from the NCEP dataset +# #' +# #' For details, please see the NCEP scripts included with this package: +# #' `system.file("scripts/ncep", package = "PEcAn.data.atmosphere")` +# #' +# #' @format a vector of 192 decimal values +# "Lon" +# +# +# #' 2010 NARR weather data for Urbana, IL +# #' +# #' Hourly 2010 meteorology for the 0.3-degree grid cell containing the +# #' EBI Energy Farm (Urbana, IL), as obtained from the NARR daily product. +# #' Please see the `compare_narr_cruncep_met` vignette for details of a +# #' comparison between this and the `cruncep`, `narr3h`, and `ebifarm` datasets. +# #' +# #' @format A data frame with 8760 rows and 10 columns: +# #' \describe{ +# #' \item{date}{POSIXct timestamp} +# #' \item{year, doy, hour}{integer, extracted from `date`} +# #' \item{SolarR}{solar radiation, in umol/h/m2} +# #' \item{Temp}{air temperature, in degrees C} +# #' \item{RH}{relative humidity, in percent} +# #' \item{WS}{wind speed, in m/s} +# #' \item{precip}{precipitation rate, in mm/h} +# #' \item{source}{dataset identifier, in this case always "narr"}} +# #' @seealso \code{\link{cruncep}} \code{\link{ebifarm}} \code{\link{narr3h}} +# "narr" +# +# +# #' 2010 NARR 3-hourly weather data for Urbana, IL +# #' +# #' Hourly 2010 meteorology for the 0.25-degree grid cell containing the +# #' EBI Energy Farm (Urbana, IL), as obtained from the NARR 3-hourly product. +# #' Please see the `compare_narr_cruncep_met` vignette for details of a +# #' comparison between this and the `cruncep`, `narr`, and `ebifarm` datasets. +# #' +# #' @format A data frame with 8736 rows and 10 columns: +# #' \describe{ +# #' \item{date}{POSIXct timestamp} +# #' \item{year, doy, hour}{integer, extracted from `date`} +# #' \item{solarR}{solar radiation, in umol/h/m2} +# #' \item{DailyTemp.C}{air temperature, in degrees C} +# #' \item{RH}{relative humidity, in percent} +# #' \item{WindSpeed}{wind speed, in m/s} +# #' \item{precip}{precipitation rate, in mm/h} +# #' \item{source}{dataset identifier, in this case always "narr3h"}} +# #' @seealso \code{\link{cruncep}} \code{\link{ebifarm}} \code{\link{narr}} +# "narr3h" diff --git a/modules/data.atmosphere/data/FLUXNET.sitemap.R b/modules/data.atmosphere/data/FLUXNET.sitemap.R new file mode 100644 index 00000000000..823665ae51a --- /dev/null +++ b/modules/data.atmosphere/data/FLUXNET.sitemap.R @@ -0,0 +1,5 @@ + +FLUXNET.sitemap <- utils::read.csv( + file = "FLUXNET.sitemap.csv", + colClasses = "character", + row.names = 1) diff --git a/modules/data.atmosphere/tests/Rcheck_reference.log b/modules/data.atmosphere/tests/Rcheck_reference.log index 39e696f479a..bf1308c9e6f 100644 --- a/modules/data.atmosphere/tests/Rcheck_reference.log +++ b/modules/data.atmosphere/tests/Rcheck_reference.log @@ -8,7 +8,11 @@ * this is package ‘PEcAn.data.atmosphere’ version ‘1.7.0’ * package encoding: UTF-8 * checking package namespace information ... OK -* checking package dependencies ... OK +* checking package dependencies ... NOTE +Imports includes 36 non-default packages. +Importing from so many packages makes the package vulnerable to any of +them becoming unavailable. Move as many as possible to Suggests and +use conditionally. * checking if this is a source package ... OK * checking if there is a namespace ... OK * checking for executable files ... OK diff --git a/modules/rtm/R/fortran.datamodule.R b/modules/rtm/R/fortran.datamodule.R index 1941641830c..2e6eb1bd6ee 100644 --- a/modules/rtm/R/fortran.datamodule.R +++ b/modules/rtm/R/fortran.datamodule.R @@ -22,14 +22,16 @@ #' z <- seq(exp(1), pi, length.out=42) #' l <- list(x=x, y=y, z=z) ## NOTE that names must be explicitly declared #' l.types <- c('real','integer', 'real*4', 'real*8') -#' fortran_data_module(l, l.types, 'testmod') -#' +#' fortran_data_module(l, l.types, 'testmod', +#' file.path(tempdir(), "testmod.f90")) +#' #' x <- runif(10) #' y <- rnorm(10) #' z <- rgamma(10, 3) #' d <- data.frame(x,y,z) ## NOTE that data.frames are just named lists #' d.types <- rep('real*8', ncol(d)) -#' fortran_data_module(d, d.types, 'random') +#' fortran_data_module(d, d.types, 'random', +#' file.path(tempdir(), "random.f90")) #' @export fortran_data_module <- function(dat, types, modname, fname = paste0(modname, ".f90")) { if (!is.list(dat)) { diff --git a/modules/rtm/man/fortran_data_module.Rd b/modules/rtm/man/fortran_data_module.Rd index d0441cc3553..31b9abc88d7 100644 --- a/modules/rtm/man/fortran_data_module.Rd +++ b/modules/rtm/man/fortran_data_module.Rd @@ -36,14 +36,16 @@ Currently, only numeric data are supported (i.e. no characters). z <- seq(exp(1), pi, length.out=42) l <- list(x=x, y=y, z=z) ## NOTE that names must be explicitly declared l.types <- c('real','integer', 'real*4', 'real*8') - fortran_data_module(l, l.types, 'testmod') - + fortran_data_module(l, l.types, 'testmod', + file.path(tempdir(), "testmod.f90")) + x <- runif(10) y <- rnorm(10) z <- rgamma(10, 3) d <- data.frame(x,y,z) ## NOTE that data.frames are just named lists d.types <- rep('real*8', ncol(d)) - fortran_data_module(d, d.types, 'random') + fortran_data_module(d, d.types, 'random', + file.path(tempdir(), "random.f90")) } \author{ Alexey Shiklomanov diff --git a/modules/uncertainty/R/plots.R b/modules/uncertainty/R/plots.R index 7c773e29ec9..029d9328b1a 100644 --- a/modules/uncertainty/R/plots.R +++ b/modules/uncertainty/R/plots.R @@ -8,26 +8,14 @@ #------------------------------------------------------------------------------- ##--------------------------------------------------------------------------------------------------# -##' Plot results of variance decomposition +##' Variance Decomposition Plots ##' ##' Plots variance decomposition tryptich ##' @name plot_variance_decomposition -##' @title Variance Decomposition Plots -##' @export plot_variance_decomposition +##' @export ##' @author David LeBauer, Carl Davidson -##' @param ... Output from any number of sensitivity analyses. Output must be of the form +##' @param plot.inputs Output from a sensitivity analysis. Output must be of the form ##' given by sensitivity.results$variance.decomposition.output in model output -##' @param all.plot.inputs Optional argument allowing output from sensitivity analyses to be specified in a list -##' @param exclude vector of strings specifying parameters to omit from the variance decomposition graph -##' @param convert.var function transforming variances to the value displayed in the graph -##' @param var.label label to displayed over variance column -##' @param order.plot.input Output from a sensitivity analysis that is to be used to order parameters. -##' Parameters are ordered by variance. Defaults to the first sensitivity analysis output given -##' @param ticks.plot.input Output from a sensitivity analysis that is to be used. -##' Defaults to the first sensitivity analysis output given -##' @param col Color of each sensitivity analysis. Equivalent to col parameter of the plot function. -##' @param pch Shape of each sensitivity analysis. Equivalent to pch parameter of the plot function. -##' @param main Plot title. Useful for multi-pft variance decompositions. ##' @param fontsize list specifying the font size of the titles and axes of the graph ##' @examples ##' x <- list(trait.labels = c('a', 'b', 'c'), diff --git a/modules/uncertainty/man/plot_variance_decomposition.Rd b/modules/uncertainty/man/plot_variance_decomposition.Rd index 8a33da6acf0..9a8317c41e0 100644 --- a/modules/uncertainty/man/plot_variance_decomposition.Rd +++ b/modules/uncertainty/man/plot_variance_decomposition.Rd @@ -10,35 +10,12 @@ plot_variance_decomposition( ) } \arguments{ -\item{fontsize}{list specifying the font size of the titles and axes of the graph} - -\item{...}{Output from any number of sensitivity analyses. Output must be of the form +\item{plot.inputs}{Output from a sensitivity analysis. Output must be of the form given by sensitivity.results$variance.decomposition.output in model output} -\item{all.plot.inputs}{Optional argument allowing output from sensitivity analyses to be specified in a list} - -\item{exclude}{vector of strings specifying parameters to omit from the variance decomposition graph} - -\item{convert.var}{function transforming variances to the value displayed in the graph} - -\item{var.label}{label to displayed over variance column} - -\item{order.plot.input}{Output from a sensitivity analysis that is to be used to order parameters. -Parameters are ordered by variance. Defaults to the first sensitivity analysis output given} - -\item{ticks.plot.input}{Output from a sensitivity analysis that is to be used. -Defaults to the first sensitivity analysis output given} - -\item{col}{Color of each sensitivity analysis. Equivalent to col parameter of the plot function.} - -\item{pch}{Shape of each sensitivity analysis. Equivalent to pch parameter of the plot function.} - -\item{main}{Plot title. Useful for multi-pft variance decompositions.} +\item{fontsize}{list specifying the font size of the titles and axes of the graph} } \description{ -Plot results of variance decomposition -} -\details{ Plots variance decomposition tryptich } \examples{ diff --git a/modules/uncertainty/tests/Rcheck_reference.log b/modules/uncertainty/tests/Rcheck_reference.log index f62a8f5e7c8..a97db2893ca 100644 --- a/modules/uncertainty/tests/Rcheck_reference.log +++ b/modules/uncertainty/tests/Rcheck_reference.log @@ -277,12 +277,6 @@ Documented arguments not in \usage in documentation object 'plot_sensitivities': Undocumented arguments in documentation object 'plot_sensitivity' ‘linesize’ ‘dotsize’ -Undocumented arguments in documentation object 'plot_variance_decomposition' - ‘plot.inputs’ -Documented arguments not in \usage in documentation object 'plot_variance_decomposition': - ‘all.plot.inputs’ ‘exclude’ ‘convert.var’ ‘var.label’ - ‘order.plot.input’ ‘ticks.plot.input’ ‘col’ ‘pch’ ‘main’ - Undocumented arguments in documentation object 'read.ameriflux.L2' ‘file.name’ ‘year’ diff --git a/scripts/check_with_errors.R b/scripts/check_with_errors.R index 891bd5ffba0..534d542ffe3 100755 --- a/scripts/check_with_errors.R +++ b/scripts/check_with_errors.R @@ -1,9 +1,5 @@ #!/usr/bin/env Rscript -# Avoid some spurious failures from platform-dependent sorting -invisible(Sys.setlocale("LC_ALL", "en_US.UTF-8")) # sets sorting in this script -Sys.setenv(LC_ALL = "en_US.UTF-8") # sets sorting in rcmdcheck processes - arg <- commandArgs(trailingOnly = TRUE) pkg <- arg[1] @@ -108,7 +104,15 @@ msg_lines <- function(msg) { msg <- lapply(msg, function(x)x[x != ""]) # prepend message title (e.g. "checking Rd files ... NOTE") to each line - unlist(lapply(msg, function(x)paste(x[[1]], x[-1], sep = ": "))) + unlist(lapply( + msg, + function(x) { + if (length(x) > 1) { + paste(x[[1]], x[-1], sep = ": ") + } else { + x + } + })) } if (cmp$status != "+") { @@ -155,6 +159,33 @@ if (cmp$status != "+") { cur_msgs <- cur_msgs[!grepl("NOTE: Consider adding importFrom", cur_msgs)] lines_changed <- setdiff(cur_msgs, prev_msgs) + + # Crude hack: + # Some messages are locale-dependent in complex ways, + # e.g. the note about undocumented datasets concatenates CSV names + # (ordered in the current locale) and objects in RData files (always + # ordered in C locale), and so on. + # As a last effort, we look for pre-existing lines that contain the same + # words in a different order + if (length(lines_changed) > 0) { + prev_words <- strsplit(prev_msgs, " ") + changed_words <- strsplit(lines_changed, " ") + is_reordered <- function(v1, v2) { + length(v1[v1 != ""]) == length(v2[v2 != ""]) && setequal(v1, v2) + } + is_in_prev <- function(line) { + any(vapply( + X = prev_words, + FUN = is_reordered, + FUN.VALUE = logical(1), + line)) + } + in_prev <- vapply( + X = changed_words, + FUN = is_in_prev, + FUN.VALUE = logical(1)) + lines_changed <- lines_changed[!in_prev] + } if (length(lines_changed) > 0) { cat("R check of", pkg, "returned new problems:\n") cat(lines_changed, sep = "\n") diff --git a/scripts/generate_dependencies.R b/scripts/generate_dependencies.R index 5ea9ad772a1..ef824e8d4b7 100755 --- a/scripts/generate_dependencies.R +++ b/scripts/generate_dependencies.R @@ -1,7 +1,7 @@ -#!/usr/bin/env RScript +#!/usr/bin/env Rscript # force sorting -if(capabilities("ICU")) { +if (capabilities("ICU")) { icuSetCollate(locale = "en_US.UTF-8") } else { print("Can not force sorting, this could result in unpredicted results.") @@ -97,6 +97,7 @@ cat("#!/bin/bash", "", "# Don\'t use X11 for rgl", "RGL_USE_NULL=TRUE", + "RLIB=${R_LIBS_USER:-/usr/local/lib/R/site-library}", "", "# install remotes first in case packages are references in dependencies", paste0( @@ -105,6 +106,6 @@ cat("#!/bin/bash", "", "# install all packages (depends, imports, suggests)", paste0( - "install2.r -e -s -n -1\\\n ", + "install2.r -e -s -l \"${RLIB}\" -n -1\\\n ", paste(sort(docker), sep = "", collapse = " \\\n ")), file = "docker/depends/pecan.depends", sep = "\n", append = FALSE) diff --git a/scripts/travis/cache_buildup.sh b/scripts/travis/cache_buildup.sh index 7cdbde55745..b7e3e37b884 100755 --- a/scripts/travis/cache_buildup.sh +++ b/scripts/travis/cache_buildup.sh @@ -34,7 +34,7 @@ MAX_TIME=${1:-30m} CMDS=${2:-'scripts/travis/install.sh && make install'} # Spends up to $MAX_TIME installing packages, then sends HUP -timeout ${MAX_TIME} bash -c ${CMDS} +timeout ${MAX_TIME} bash -c "${CMDS}" if [[ $? -ne 0 ]]; then # Clean up any lock files left from killing install.packages diff --git a/scripts/travis/install.sh b/scripts/travis/install.sh index ff99c0670a5..eb062a3f249 100755 --- a/scripts/travis/install.sh +++ b/scripts/travis/install.sh @@ -3,16 +3,7 @@ set -e . $( dirname $0 )/func.sh -# FIXING R BINARIES -( - travis_time_start "pkg_version_check" "Checking R package binaries" - - Rscript scripts/travis/rebuild_pkg_binaries.R - - travis_time_end -) - -# ROLL A FEW R PACKAGES BACK TO SPECIFIED VERSIONS +# Install R packages that need specified versions ( travis_time_start "pecan_install_roxygen" "Installing Roxygen 7.0.2 to match comitted documentation version" # We keep Roxygen pinned to a known version, merely to avoid hassle / @@ -34,6 +25,22 @@ set -e fi ) +# Install R package dependencies +# N.B. we run this *after* installing packages that need pinned versions, +# relying on fact that pecan.depends calls littler with -s, +# so it will skip reinstalling packages that already exist. +# This way each package is only installed once. +( + travis_time_start "r_pkgs" "installing R packages" + # Seems like a lot of fiddling to set up littler and only use it once + # inside pecan.depends, but still easier than duplicating the script + Rscript -e 'if (!requireNamespace("littler", quietly = TRUE)) { install.packages(c("littler", "remotes", "docopt"), repos = "https://cloud.r-project.org") }' + LRPATHS=$(Rscript -e 'cat(system.file(c("examples", "bin"), package = "littler"), sep = ":")') + echo 'options(repos="https://cloud.r-project.org")' > ~/.littler.r + PATH=$LRPATHS:$PATH bash docker/depends/pecan.depends + travis_time_end +) + # INSTALLING SIPNET ( travis_time_start "install_sipnet" "Installing SIPNET for testing" diff --git a/scripts/travis/prime_travis_cache.sh b/scripts/travis/prime_travis_cache.sh index 695ed205158..bf9537ae7ee 100755 --- a/scripts/travis/prime_travis_cache.sh +++ b/scripts/travis/prime_travis_cache.sh @@ -23,7 +23,7 @@ BODY='{ "branch":"'${BRANCH}'", "config": { "install":"echo skipping", - "before-script":"echo skipping", + "before_script":"echo skipping", "script":"scripts/travis/cache_buildup.sh 30m"} }}' diff --git a/scripts/travis/rebuild_pkg_binaries.R b/scripts/travis/rebuild_pkg_binaries.R deleted file mode 100644 index e206ff8938f..00000000000 --- a/scripts/travis/rebuild_pkg_binaries.R +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env Rscript - -# ¡ugly hack! -# -# Travis setup uses many prebuilt R packages from c2d4u3.5, which despite its -# name now contains a mix of packages built for R 3.5 and R 3.6. When loaded in -# R 3.5.x, 3.6-built packages throw error -# `rbind(info, getNamespaceInfo(env, "S3methods")): -# number of columns of matrices must match`, -# and as of 2019-10-30 at least one package (data.table) refuses to load if its -# build version does not match the R version it is running on. -# -# We resolve this the slow brute-force way: By running this script before -# attempting to load any R packages, thereby reinstalling from source any -# package whose binary was built with a different R version. -# -# TODO: Remove this when c2d4u situation improves. - -is_wrong_build <- function(pkgname) { - - # lockfile implies incomplete previous installation => delete and rebuild - lock_path <- file.path(.libPaths(), paste0("00LOCK-", pkgname)) - if (any(file.exists(lock_path))) { - unlink(lock_path, recursive = TRUE) - return(TRUE) - } - - built_str <- tryCatch( - packageDescription(pkgname)$Built, - error = function(e)e) - if (inherits(built_str, "error")) { - # In the rare case we can't read the description, - # assume package is broken and needs rebuilding - return(TRUE) - } - - # Typical packageDescription(pkgname)$Built result: we only need chars 3-7 - # "R 3.4.4; x86_64-apple-darwin15.6.0; 2019-03-18 04:41:51 UTC; unix" - built_ver <- R_system_version(substr(built_str, start = 3, stop = 7)) - - # NB strict comparison: even patch level must agree - built_ver != getRversion() -} - -all_pkgs <- installed.packages()[, 1] -needs_rebuild <- vapply(all_pkgs, is_wrong_build, logical(1)) - -if (any(needs_rebuild)) { - print(paste( - "Found R packages that were built for a different R version.", - "Reinstalling these from source.")) - install.packages( - all_pkgs[needs_rebuild], - repos = "cloud.r-project.org", - dependencies = FALSE, - Ncpus = 2) -}