.dev-only: revdep updates and tweaks

.dev/.bash_aliases

@@ -8,14 +8,16 @@ alias gdm='git difftool master &> /dev/null'
 # If meld has scrolling issues, turn off GTK animation which I don't need:
 #   https://gitlab.gnome.org/GNOME/meld/-/issues/479#note_866040
 
+alias perfbar=~/build/gtk_perfbar/linux_perfbar   # revdep.R; https://github.com/tomkraljevic/gtk_perfbar
+
 alias Rdevel='~/build/R-devel/bin/R --vanilla'
 alias Rdevel-strict-gcc='~/build/R-devel-strict-gcc/bin/R --vanilla'
 alias Rdevel-strict-clang='~/build/R-devel-strict-clang/bin/R --vanilla'
 alias Rdevel-valgrind='~/build/R-devel-valgrind/bin/R --vanilla'
 alias Rdevel32='~/build/32bit/R-devel/bin/R --vanilla'
 alias R310='~/build/R-3.1.0/bin/R --vanilla'
 
-alias revdepsh='cd ~/build/revdeplib/ && export TZ=UTC && export R_LIBS_SITE=none && export R_LIBS=~/build/revdeplib/ && export _R_CHECK_FORCE_SUGGESTS_=true'
+alias revdepsh='cd ~/build/revdeplib/ && export TZ=UTC && export R_LIBS_SITE=NULL && export R_LIBS=~/build/revdeplib/ && export _R_CHECK_FORCE_SUGGESTS_=true'
 alias revdepr='revdepsh; R_PROFILE_USER=~/GitHub/data.table/.dev/revdep.R R'
 # use ~/build/R-devel/bin/R at the end of revdepr to use R-devel instead of R-release.
 # If so, doing a `rm -rf *` in revdeplib first to rebuild everything is easiest way to avoid potential problems later. A full rebuild is a good idea periodically anyway. Packages in

.dev/CRAN_Release.cmd

@@ -490,14 +490,15 @@ shutdown now   # doesn't return you to host prompt properly so just kill the win
 #  Downstream dependencies
 ###############################################
 
-# IF NOT ALREADY INSTALLED
+# IF NOT ALREADY INSTALLED, OR AFTER AN OS UPGRADE
+# No harm rerunning these commands; they do not reinstall if already latest version
 sudo apt-get update
 sudo apt-get -y install htop
 sudo apt-get -y install r-base r-base-dev
 sudo apt-get -y build-dep r-base-dev
 sudo apt-get -y build-dep qpdf
 sudo apt-get -y install aptitude
-sudo aptitude -y build-dep r-cran-rgl   # leads to libglu1-mesa-dev
+sudo apt-get -y build-dep r-cran-rgl   # leads to libglu1-mesa-dev
 sudo apt-get -y build-dep r-cran-rmpi
 sudo apt-get -y build-dep r-cran-cairodevice
 sudo apt-get -y build-dep r-cran-tkrplot
@@ -545,6 +546,8 @@ sudo apt-get -y install libgit2-dev  # for gert
 sudo apt-get -y install cmake  # for symengine for RxODE
 sudo apt-get -y install libxslt1-dev  # for xslt
 sudo apt-get -y install flex  # for RcppCWB
+sudo apt-get -y install libavfilter-dev libsodium-dev libgmp-dev libssh-dev librdf0-dev
+sudo apt-get -y install libmariadb-dev mariadb-client   # RMySQL for xQTLbiolinks
 sudo R CMD javareconf
 # ENDIF
 
@@ -553,6 +556,7 @@ inst()   # *** ensure latest dev version of data.table installed into revdeplib
 run()    # prints menu of options
 status() # includes timestamp of installed data.table that is being tested.
 log()    # cats all fail logs to ~/fail.log
+cran()   # compare packages with error or warning to their status on CRAN
 
 # Once all issues resolved with CRAN packages, tackle long-term unfixed bioconductor packages as follows.
 # 1. Note down all error and warning bioc packages

.dev/revdep.R

@@ -13,9 +13,10 @@ options(error=quote(utils::dump.frames()))
 options(width=200)      # for cran() output not to wrap
 
 # Check that env variables have been set correctly:
-#   export R_LIBS_SITE=none
+#   export R_LIBS_SITE=NULL   # R 4.2.0 changed to NULL but it doesn't appear to work
 #   export R_LIBS=~/build/revdeplib/
 #   export _R_CHECK_FORCE_SUGGESTS_=true
+if (length(.libPaths())==3L) .libPaths(.libPaths()[-2L], include.site=FALSE)  # workaround as I couldn't get R_LIBS_SITE=NULL to be effective
 stopifnot(identical(length(.libPaths()), 2L))  # revdeplib writeable by me, and the pre-installed recommended R library (sudo writeable)
 stopifnot(identical(.libPaths()[1L], getwd()))
 tt = file.info(.libPaths())[,"uname"]
@@ -96,10 +97,33 @@ update.packages(ask=FALSE, checkBuilt=TRUE)
 
 avail = available.packages()  # includes CRAN and Bioc, from getOption("repos") set above
 
-avail = avail[!rownames(avail) %in% "cplexAPI", ]
+avail = avail[!rownames(avail) %in% c("cplexAPI","Rcplex"), ]
 # cplexAPI is suggested by revdeps ivmte and prioritizr. I haven't succeeded to install IBM ILOG CPLEX which requires a license,
 # so consider cplexAPI not available when resolving missing suggests at the end of status().
-# Update: cplexAPI was removed from CRAN on 5 Nov 2021 so this is now redundant, but leave it in place for future use
+# Update: cplexAPI was removed from CRAN on 5 Nov 2021 so this is now redundant, but leave it in place for future use.
+# Update: Rcplex is on CRAN as of 20 Nov 2022 but with install errors, therefore treat it as not available.
+
+# The presence of packages here in revdeplib which no longer exist on CRAN could explain differences to CRAN. A revdep
+# could be running tests using that package when available and failing which may be the very reason that package was removed from CRAN.
+# When it is removed from revdeplib to match CRAN, then the revdep might then pass as it will skip its tests using that package.
+x = installed.packages()
+tt = match(rownames(x), rownames(avail))
+removed = rownames(x)[is.na(tt) & is.na(x[,"Priority"])]
+cat("Removing",length(removed),"packages which are no longer available on CRAN/Bioc:", paste(removed, collapse=","), "\n")
+stopifnot(all(x[removed,"LibPath"] == .libPaths()[1]))
+oldn = nrow(x)
+remove.packages(removed, .libPaths()[1])
+x = installed.packages()
+stopifnot(nrow(x) == oldn-length(removed))
+
+# Ensure all installed packages were built with this x.y release of R; i.e. that checkBuilt=TRUE worked above
+cat("This is R ",R.version$major,".",R.version$minor,"; ",R.version.string,"\n",sep="")
+cat("Previously installed packages were built using:\n")
+print(tt <- table(x[,"Built"], dnn=NULL))
+minorR = paste(strsplit(as.character(getRversion()), split="[.]")[[1]][c(1,2)], collapse=".")
+if (any(w<-names(tt)<minorR)) {
+  stop(sum(tt[w])," packages built with R<",minorR," failed to update: ", paste(paste0("'",rownames(x)[x[,"Built"]<minorR],"'"), collapse=","), "\n",sep="")
+}
 
 deps = tools::package_dependencies("data.table",
   db = available.packages(repos=getOption("repos")["CRAN"]),  # just CRAN revdeps though (not Bioc) from October 2020
@@ -133,28 +157,6 @@ for (p in deps) {
 cat("New downloaded:",new," Already had latest:", old, " TOTAL:", length(deps), "\n")
 update.packages(checkBuilt=TRUE, ask=FALSE)  # won't rebuild packages which are no longer available on CRAN
 
-# The presence of packages here in revdeplib which no longer exist on CRAN could explain differences to CRAN. A revdep
-# could be running tests using that package when available and failing which may be the very reason that package was removed from CRAN.
-# When it is removed from revdeplib to match CRAN, then the revdep might then pass as it will skip its tests using that package.
-x = installed.packages()
-tt = match(rownames(x), rownames(avail))
-removed = rownames(x)[is.na(tt) & is.na(x[,"Priority"])]
-cat("Removing",length(removed),"packages which are no longer available on CRAN/Bioc:", paste(removed, collapse=","), "\n")
-stopifnot(all(x[removed,"LibPath"] == .libPaths()[1]))
-oldn = nrow(x)
-remove.packages(removed, .libPaths()[1])
-x = installed.packages()
-stopifnot(nrow(x) == oldn-length(removed))
-
-# Ensure all installed packages were built with this x.y release of R
-cat("This is R ",R.version$major,".",R.version$minor,"; ",R.version.string,"\n",sep="")
-cat("Previously installed packages were built using:\n")
-print(tt <- table(x[,"Built"], dnn=NULL))
-minorR = paste(strsplit(as.character(getRversion()), split="[.]")[[1]][c(1,2)], collapse=".")
-stopifnot(all(substring(names(tt),1,nchar(minorR)) == minorR))
-# if not (e.g. when using R-devel for revdep testing) perhaps run the following manually replacing "4.0.0" as appropriate 
-# for (p in rownames(x)[x[,"Built"]=="4.0.0"]) install.packages(p)
-
 # Remove the tar.gz no longer needed :
 for (p in deps) {
   f = paste0(p, "_", avail[p,"Version"], ".tar.gz")  # keep this one
@@ -256,15 +258,22 @@ status = function(bioc=FALSE) {
       }
     }
     if (length(all_sugg_unavail)) {
-      cat('\nPackages for which all their missing suggests are not available, try:\n',
-          '  run("',paste(all_sugg_unavail,collapse=" "),'", R_CHECK_FORCE_SUGGESTS=FALSE)\n', sep="")
+      cat('\nPackages for which all their missing suggests are not available (',length(all_sugg_unavail),'): ', paste(all_sugg_unavail, collapse=" "), "\n", sep="")
+      cat('Rerunning them with R_CHECK_FORCE_SUGGESTS=FALSE ...\n')
+      run(all_sugg_unavail, R_CHECK_FORCE_SUGGESTS=FALSE, ask=FALSE)
+      # the main run() ran with TRUE in an attempt to check suggests where possible in case data.table usage is there. It does that as OS level using
+      # parallel linux command of `R CMD check`. Therefore it would be awkward to rerun with TRUE in that step. Instead we trigger the rerun from
+      # here in status() afterwards once we know all such packages and can count and log them (the cat() just above). Since run() is
+      # asynchronous we do have to wait again and run status() again when perfbar shows finished. However, currently there are only
+      # 37/1315 like this so it only takes a few minutes with my default of 6 at a time.
+      return(TRUE) # to indicate status() started a run() so that the run() afterwards can be avoided which would otherwise see these as not started and run them simultaneously but with _SUGGESTS=TRUE
     }
     # Otherwise, inspect manually each result in fail.log written by log()
   }
-  invisible()
+  invisible(FALSE)
 }
 
-cran = function()  # reports CRAN status of the .cran.fail packages
+cran = function()  # reports CRAN status of the .fail.cran packages
 {
   x = c(.fail.cran, .running.cran)
   if (!length(x)) {
@@ -286,7 +295,7 @@ cran = function()  # reports CRAN status of the .cran.fail packages
   setkey(ans, Package)[x,]
 }
 
-run = function(pkgs=NULL, R_CHECK_FORCE_SUGGESTS=TRUE, choose=NULL) {
+run = function(pkgs=NULL, R_CHECK_FORCE_SUGGESTS=TRUE, choose=NULL, ask=TRUE) {
   if (length(pkgs)==1) pkgs = strsplit(pkgs, split="[, ]")[[1]]
   if (anyDuplicated(pkgs)) stop("pkgs contains dups")
   if (!length(pkgs)) {
@@ -325,7 +334,7 @@ run = function(pkgs=NULL, R_CHECK_FORCE_SUGGESTS=TRUE, choose=NULL) {
     cat("Running",length(pkgs),"packages:", paste(pkgs), "\n")
     filter = paste0("| grep -E '", paste0(paste0(pkgs,"_"),collapse="|"), "' ")
   }
-  if (is.null(choose)) {
+  if (ask && is.null(choose)) {
     cat("Proceed? (ctrl-c or enter)\n")
     scan(quiet=TRUE)
   }
@@ -373,8 +382,7 @@ log = function(bioc=FALSE, fnam="~/fail.log", app="gedit") {
 }
 
 inst()
-status()
-run(choose=1)  # run not-started (i.e. updates to and new revdeps) automatically on revdep startup
+if (!status()) run(choose=1)  # run pkgs in not-started status; i.e. updates to and new revdeps. Unless status() found any all-suggests-unavail which it then started run() for with _SUGGESTS=FALSE
 
 # Now R prompt is ready to fix any problems with CRAN or Bioconductor updates.
 # Then run run(), status() and log() as per section in CRAN_Release.cmd