Merge branch 'master' into cube-missing-j

.Rbuildignore

@@ -1,6 +1,13 @@
 ^\.Rprofile$
 ^data\.table_.*\.tar\.gz$
 ^vignettes/plots/figures$
+^\.Renviron$
+^[^/]+\.R$
+^[^/]+\.csv$
+^[^/]+\.csvy$
+^[^/]+\.RDS$
+^[^/]+\.diff$
+^[^/]+\.patch$
 
 ^\.ci$
 ^\.dev$
@@ -13,7 +20,6 @@
 
 ^Makefile$
 ^NEWS\.0\.md$
-^README\.md$
 ^_pkgdown\.yml$
 ^src/Makevars$
 
@@ -30,3 +36,5 @@
 
 ^bus$
 ^pkgdown$
+^lib$
+^library$

.appveyor.yml

@@ -32,7 +32,7 @@ environment:
 
   - R_VERSION: release  # the single Windows.zip binary (both 32bit/64bit) that users following dev version of installation instructions should click
 
-#  - R_VERSION: devel   # When off it's to speed up dev cycle; R-devel is still checked but by GLCI on a roughly hourly cycle.
+#  - R_VERSION: devel   # Never turn back on. GLCI after merge covers latest daily R-devel very well, so we shouldn't confuse and slow down PR dev cycle by measuring PRs against daily R-devel too. If a change in R-devel yesterday breaks the PR, it's very unlikely to be due to something in the PR. So we should accept the PR if it passes R-release and fix separately anything related to R-devel which we'll see from GLCI.
 
 before_build:
   - cmd: ECHO no Revision metadata added to DESCRIPTION

.ci/README.md

@@ -1,6 +1,6 @@
 # data.table continuous integration and deployment
 
-On each Pull Request opened in GitHub we run Travis CI and Appveyor to provide prompt feedback about the status of PR. Our main CI pipeline runs on GitLab CI. GitLab repository automatically mirrors our GitHub repository and runs pipeline on `master` branch. It tests more environments and different configurations. It publish variety of artifacts. Windows jobs are being run on our private windows CI runner.
+On each Pull Request opened in GitHub we run Travis CI and Appveyor to provide prompt feedback about the status of PR. Our main CI pipeline runs on GitLab CI. GitLab repository automatically mirrors our GitHub repository and runs pipeline on `master` branch. It tests more environments and different configurations. It publish variety of artifacts.
 
 ## Environments
 
@@ -9,13 +9,14 @@ On each Pull Request opened in GitHub we run Travis CI and Appveyor to provide p
 Test jobs:
 - `test-rel-lin` - `r-release` on Linux, most comprehensive test environment, `-O3 -flto -fno-common -Wunused-result`, extra check for no compilation warnings, includes testing [_with other packages_](./../inst/tests/other.Rraw) ([extended suggests](./../inst/tests/tests-DESCRIPTION))
 - `test-rel-cran-lin` - `--as-cran` on Linux, `-g0`, extra check for final status of `R CMD check` where we allow one NOTE (_size of tarball_).
-- `test-dev-cran-lin` - `r-devel` and `--as-cran` on Linux, `--enable-strict-barrier --disable-long-double`
+- `test-dev-cran-lin` - `r-devel` and `--as-cran` on Linux, `--with-recommended-packages --enable-strict-barrier --disable-long-double`, tests for compilation warnings in pkg install and new NOTEs/Warnings in pkg check, and because it is R-devel it is marked as allow_failure
 - `test-rel-vanilla-lin` - `r-release` on Linux, no suggested deps, no OpenMP, `-O0`, tracks memory usage during tests
 - `test-310-cran-lin` - R 3.1.0 on Linux
 - `test-344-cran-lin` - R 3.4.4 on Linux
 - `test-350-cran-lin` - R 3.5.0 on Linux, no `r-recommended`
 - `test-rel-win` - `r-release` on Windows
 - `test-dev-win` - `r-devel` on Windows
+- `test-old-win` - `r-oldrel` on Windows
 - `test-rel-osx` - MacOSX build not yet deployed, see [#3326](https://github.com/Rdatatable/data.table/issues/3326) for status
 
 Artifacts:
@@ -25,9 +26,9 @@ Artifacts:
 - [html vignettes](https://rdatatable.gitlab.io/data.table/library/data.table/doc/index.html)
 - R packages repository for `data.table` and all _Suggests_ dependencies, url: `https://Rdatatable.gitlab.io/data.table`
   - sources
-  - Windows binaries for `r-release` and `r-devel`
+  - Windows binaries for `r-release`, `r-devel` and `r-oldrel`
 - [CRAN-like homepage](https://rdatatable.gitlab.io/data.table/web/packages/data.table/index.html)
-- [CRAN-like checks results](https://rdatatable.gitlab.io/data.table/web/checks/check_results_data.table.html) - note that all artifacts, including this page, are being published only when all test jobs successfully pass, thus one will not see an _ERROR_ status there (unless `allow_failure` option has been used in a job).
+- [CRAN-like checks results](https://rdatatable.gitlab.io/data.table/web/checks/check_results_data.table.html) - note that all artifacts, including check results page, are being published only when all test jobs successfully pass, thus one will not see an _ERROR_ status there (unless error happened on a job marked as `allow_failure`).
 - [docker images](https://gitlab.com/Rdatatable/data.table/container_registry) - copy/paste-able `docker pull` commands can be found at the bottom of our [CRAN-like homepage](https://rdatatable.gitlab.io/data.table/web/packages/data.table/index.html)
 
 ### [Travis CI](./../.travis.yml)
@@ -64,7 +65,7 @@ Base R implemented helper script to orchestrate generation of most artifacts. It
 Template file to produce `Dockerfile` for, as of now, three docker images. Docker images are being built and published in [_deploy_ stage in GitLab CI pipeline](./../.gitlab-ci.yml).
 - `r-base-dev` using `r-release`: publish docker image of `data.table` on R-release
 - `r-builder` using `r-release`: publish on R-release and OS dependencies for building Rmarkdown vignettes
-- `r-devel`: publish docker image of `data.table` on R-devel
+- `r-devel`: publish docker image of `data.table` on R-devel built with `--with-recommended-packages --enable-strict-barrier --disable-long-double`
 
 ### [`deploy.sh`](./deploy.sh)
 

.ci/publish.R

@@ -1,12 +1,17 @@
 format.deps <- function(file, which) {
   deps.raw = read.dcf(file, fields=which)[[1L]]
   if (all(is.na(deps.raw))) return(character())
+  deps.raw = gsub("\n", " ", deps.raw, fixed=TRUE)
   deps.full = trimws(strsplit(deps.raw, ", ", fixed=TRUE)[[1L]])
   deps = trimws(sapply(strsplit(deps.full, "(", fixed=TRUE), `[[`, 1L))
+  deps.full = gsub(">=", "&ge;", deps.full, fixed=TRUE)
+  deps.full = gsub("<=", "&le;", deps.full, fixed=TRUE)
+  if (any(grepl(">", deps.full, fixed=TRUE), grepl("<", deps.full, fixed=TRUE), grepl("=", deps.full, fixed=TRUE)))
+    stop("formatting dependencies version for CRAN-line package website failed because some dependencies have version defined using operators other than >= and <=")
   names(deps.full) <- deps
   base.deps = c("R", unlist(tools:::.get_standard_package_names(), use.names = FALSE))
   ans = sapply(deps, function(x) {
-    if (x %in% base.deps) deps.full[[x]]
+    if (x %in% base.deps) deps.full[[x]] ## base R packages are not linked
     else sprintf("<a href=\"../%s/index.html\">%s</a>", x, deps.full[[x]])
   })
   sprintf("<tr><td>%s:</td><td>%s</td></tr>", which, paste(ans, collapse=", "))
@@ -26,6 +31,39 @@ format.bins <- function(ver, bin_ver, cran.home, os.type, pkg, version, repodir)
   paste(ans[fe], collapse=", ")
 }
 
+format.entry <- function(field, dcf, url=FALSE) {
+  if (field %in% colnames(dcf)) {
+    value = gsub("\n", " ", dcf[,field], fixed=TRUE)
+    if (url) {
+      urls = trimws(strsplit(value, ",", fixed=TRUE)[[1L]])
+      value = paste(sprintf("<a href=\"%s\">%s</a>", urls, urls), collapse=", ")
+    }
+    sprintf("<tr><td>%s:</td><td>%s</td></tr>", field, value)
+  }
+}
+format.maintainer <- function(dcf) {
+  if ("Maintainer" %in% colnames(dcf)) {
+    text2html = function(x) {
+      # https://stackoverflow.com/a/64446320/2490497
+      splitted <- strsplit(x, "")[[1L]]
+      intvalues <- as.hexmode(utf8ToInt(enc2utf8(x)))
+      paste(paste0("&#x", intvalues, ";"), collapse = "")
+    }
+    tmp = gsub("@", " at ", dcf[,"Maintainer"], fixed=TRUE)
+    sep = regexpr("<", tmp, fixed=TRUE)
+    name = trimws(substr(tmp, 1L, sep-1L))
+    mail = text2html(trimws(substr(tmp, sep, nchar(tmp))))
+    sprintf("<tr><td>Maintainer:</td><td>%s  %s</td></tr>", name, mail)
+  }
+}
+format.materials <- function() {
+  return(NULL) ## TODO
+  value = NA
+  #NEWS
+  #README
+  sprintf("<tr><td>Materials:</td><td>%s</td></tr>", value)
+}
+
 package.index <- function(package, lib.loc, repodir="bus/integration/cran") {
   file = system.file("DESCRIPTION", package=package, lib.loc=lib.loc)
   dcf = read.dcf(file)
@@ -40,28 +78,38 @@ package.index <- function(package, lib.loc, repodir="bus/integration/cran") {
     format.deps(file, "LinkingTo"),
     format.deps(file, "Suggests"),
     format.deps(file, "Enhances"),
+    if ("Built" %in% colnames(dcf)) sprintf("<tr><td>Built:</td><td>%s</td></tr>", substr(trimws(strsplit(dcf[,"Built"], ";", fixed=TRUE)[[1L]][[3L]]), 1L, 10L)),
+    if ("Author" %in% colnames(dcf)) sprintf("<tr><td>Author:</td><td>%s</td></tr>", dcf[,"Author"]),
+    format.maintainer(dcf),
+    format.entry("BugReports", dcf, url=TRUE),
+    format.entry("License", dcf),
+    format.entry("URL", dcf, url=TRUE),
+    format.entry("NeedsCompilation", dcf),
+    format.entry("SystemRequirements", dcf),
+    format.materials(), ## TODO
     if (pkg=="data.table") sprintf("<tr><td>Checks:</td><td><a href=\"../../checks/check_results_%s.html\">%s results</a></td></tr>", pkg, pkg)
   )
   vign = tools::getVignetteInfo(pkg, lib.loc=lib.loc)
-  r_bin_ver = Sys.getenv("R_BIN_VERSION")
-  r_devel_bin_ver = Sys.getenv("R_DEVEL_BIN_VERSION")
-  stopifnot(nzchar(r_bin_ver), nzchar(r_devel_bin_ver))
+  r_rel_ver = Sys.getenv("R_REL_VERSION")
+  r_devel_ver = Sys.getenv("R_DEVEL_VERSION")
+  r_oldrel_ver = Sys.getenv("R_OLDREL_VERSION")
+  stopifnot(nzchar(r_rel_ver), nzchar(r_devel_ver), nzchar(r_oldrel_ver))
   cran.home = "../../.."
   tbl.dl = c(
     sprintf("<tr><td> Reference manual: </td><td> <a href=\"%s.pdf\">%s.pdf</a>, <a href=\"%s/library/%s/html/00Index.html\">00Index.html</a> </td></tr>", pkg, pkg, cran.home, pkg),
     if (nrow(vign)) sprintf("<tr><td>Vignettes:</td><td>%s</td></tr>", paste(sprintf("<a href=\"%s/library/data.table/doc/%s\">%s</a><br/>", cran.home, vign[,"PDF"], vign[,"Title"]), collapse="\n")), # location unline cran web/pkg/vignettes to not duplicate content, documentation is in ../../../library
     sprintf("<tr><td> Package source: </td><td> <a href=\"%s/src/contrib/%s_%s.tar.gz\"> %s_%s.tar.gz </a> </td></tr>", cran.home,pkg, version, pkg, version),
-    sprintf("<tr><td> Windows binaries: </td><td> %s </td></tr>", format.bins(ver=c("r-devel","r-release"), bin_ver=c(r_devel_bin_ver,r_bin_ver), cran.home=cran.home, os.type="windows", pkg=pkg, version=version, repodir=repodir)),
-    sprintf("<tr><td> OS X binaries: </td><td> %s </td></tr>", format.bins(ver=c("r-devel","r-release"), bin_ver=c(r_devel_bin_ver, r_bin_ver), cran.home=cran.home, os.type="macosx", pkg=pkg, version=version, repodir=repodir))
+    sprintf("<tr><td> Windows binaries: </td><td> %s </td></tr>", format.bins(ver=c("r-devel","r-release","r-oldrel"), bin_ver=c(r_devel_ver, r_rel_ver, r_oldrel_ver), cran.home=cran.home, os.type="windows", pkg=pkg, version=version, repodir=repodir)),
+    sprintf("<tr><td> macOS binaries: </td><td> %s </td></tr>", format.bins(ver=c("r-release","r-oldrel"), bin_ver=c(r_rel_ver, r_oldrel_ver), cran.home=cran.home, os.type="macosx", pkg=pkg, version=version, repodir=repodir))
   )
-  if (pkg=="data.table") {
+  if (pkg=="data.table") { ## docker images
     registry = Sys.getenv("CI_REGISTRY", "registry.gitlab.com")
     namespace = Sys.getenv("CI_PROJECT_NAMESPACE", "Rdatatable")
     project = Sys.getenv("CI_PROJECT_NAME", "data.table")
     images = c("r-release","r-devel","r-release-builder")
     images.title = c("Base R release", "Base R development", "R release package builder")
     tags = rep("latest", 3)
-    docker.dl = sprintf("<tr><td> %s: </td><td> <pre><code>docker pull %s/%s/%s/%s:%s</code></pre> </td></tr>", images.title, registry, namespace, project, images, tags)
+    docker.dl = sprintf("<tr><td> %s: </td><td> <pre><code>docker pull %s/%s/%s/%s:%s</code></pre> </td></tr>", images.title, tolower(registry), tolower(namespace), tolower(project), tolower(images), tags)
   }
   index.file = file.path(repodir, "web/packages", pkg, "index.html")
   if (!dir.exists(dirname(index.file))) dir.create(dirname(index.file), recursive=TRUE)
@@ -74,7 +122,7 @@ package.index <- function(package, lib.loc, repodir="bus/integration/cran") {
     "<style type=\"text/css\">table td { vertical-align: top; }</style>",
     "</head>",
     "<body>",
-    sprintf("<h2>%s</h2>", dcf[,"Title"]),
+    sprintf("<h2>%s: %s</h2>", pkg, dcf[,"Title"]),
     sprintf("<p>%s</p>", dcf[,"Description"]),
     sprintf("<table summary=\"Package %s summary\">", pkg),
     tbl,
@@ -117,7 +165,48 @@ doc.copy <- function(repodir="bus/integration/cran"){
   c(ans1, ans2)
 }
 
-plat <- function(x) if (grepl("^.*win", x)) "Windows" else if (grepl("^.*osx", x)) "Mac OS X" else "Linux"
+plat <- function(x) if (grepl("^.*win", x)) "Windows" else if (grepl("^.*mac", x)) "macOS" else "Linux"
+
+r.ver <- function(x) {
+  tmp = strsplit(x, "-", fixed=TRUE)[[1L]]
+  if (length(tmp) < 2L) stop("test job names must be test-[r.version]-...")
+  v = tmp[2L]
+  if (identical(v, "rel")) "r-release"
+  else if (identical(v, "dev")) "r-devel"
+  else if (identical(v, "old")) "r-oldrel"
+  else {
+    if (grepl("\\D", v)) stop("second word in test job name must be rel/dev/old or numbers of R version")
+    paste0("r-", paste(strsplit(v, "")[[1L]], collapse="."))
+  }
+}
+
+# this for now is constant but when we move to independent pipelines (commit, daily, weekly) those values can be different
+pkg.version <- function(job, pkg) {
+  dcf = read.dcf(file.path("bus", job, paste(pkg, "Rcheck", sep="."), pkg, "DESCRIPTION"))
+  dcf[,"Version"]
+}
+pkg.revision <- function(job, pkg) {
+  dcf = read.dcf(file.path("bus", job, paste(pkg, "Rcheck", sep="."), pkg, "DESCRIPTION"))
+  if ("Revision" %in% colnames(dcf)) {
+    proj.url = Sys.getenv("CI_PROJECT_URL", "")
+    if (!nzchar(proj.url)) {
+      warning("pkg.revision was designed to be run on GLCI where CI_PROJECT_URL var is set, links to commits will not be produced for checks table")
+      substr(dcf[,"Revision"], 1, 7)
+    } else {
+      sprintf("<a href=\"%s\">%s</a>", file.path(proj.url, "-", "commit", dcf[,"Revision"]), substr(dcf[,"Revision"], 1, 7))
+    }
+  } else ""
+}
+pkg.flags <- function(job, pkg) {
+  cc = file.path("bus", job, paste(pkg, "Rcheck", sep="."), pkg, "cc") ## data.table style cc file
+  if (file.exists(cc)) {
+    d = readLines(cc)
+    w.cflags = substr(d, 1, 7)=="CFLAGS="
+    if (sum(w.cflags)==1L)
+      return(sub("CFLAGS=", "", d[w.cflags], fixed=TRUE))
+  }
+  ""
+}
 
 check.copy <- function(job, repodir="bus/integration/cran"){
   dir.create(job.checks<-file.path(repodir, "web", "checks", pkg<-"data.table", job), recursive=TRUE);
@@ -146,6 +235,39 @@ check.copy <- function(job, repodir="bus/integration/cran"){
   setNames(file.exists(file.path(job.checks, c(inst.check, routs))), c(inst.check, routs))
 }
 
+check.flavors <- function(jobs, repodir="bus/integration/cran") {
+  th = "<th>Flavor</th><th>R Version</th><th>OS Type</th><th>CPU Type</th><th>OS Info</th><th>CPU Info</th><th>Compilers</th>"
+  tbl = sprintf(
+    "<tr><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td></tr>",
+    sub("test-", "", jobs, fixed=TRUE),
+    sapply(jobs, r.ver),
+    sapply(jobs, plat),
+    "", # "x86_64"
+    "", # "Debian GNU/Linux testing"
+    "", # "2x 8-core Intel(R) Xeon(R) CPU E5-2690 0 @ 2.90GHz"
+    "" # "GCC 10.2.0 (Debian 10.2.0-13)"
+  )
+  file = file.path(repodir, "web/checks", "check_flavors.html")
+  writeLines(c(
+    "<html>",
+    "<head>",
+    "<title>Package Check Flavors</title>",
+    "<link rel=\"stylesheet\" type=\"text/css\" href=\"../CRAN_web.css\"/>",
+    "<meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\"/>",
+    "</head>",
+    "<body lang=\"en\">",
+    "<h2>Package Check Flavors</h2>",
+    sprintf("<p>Last updated on %s.</p>", format(Sys.time(), usetz=TRUE)),
+    "<table border=\"1\" summary=\"CRAN check flavors.\">",
+    "<tr>",th,"</tr>",
+    tbl,
+    "</table>",
+    "</body>",
+    "</html>"
+  ), file)
+  setNames(file.exists(file), file)
+}
+
 check.index <- function(pkg, jobs, repodir="bus/integration/cran") {
   status = function(x) if (grepl("^.*ERROR", x)) "ERROR" else if (grepl("^.*WARNING", x)) "WARNING" else if (grepl("^.*NOTE", x)) "NOTE" else if (grepl("^.*OK", x)) "OK" else NA_character_
   test.files = function(job, files, trim.name=FALSE, trim.exts=0L, pkg="data.table") {
@@ -186,30 +308,36 @@ check.index <- function(pkg, jobs, repodir="bus/integration/cran") {
     }
     memouts
   })
-  tbl = sprintf("<tr><td>%s</td><td>%s</td><td><a href=\"%s/%s/00install.out\">out</a></td><td><a href=\"%s/%s/00check.log\">%s</a></td><td>%s</td><td>%s</td></tr>",
-                sub("test-", "", jobs, fixed=TRUE),
-                sapply(jobs, plat),
-                pkg, jobs,
-                pkg, jobs, sapply(sapply(jobs, check.test, pkg="data.table"), status),
-                mapply(test.files, jobs, routs, trim.exts=2L), # 1st fail, 2nd Rout, keep just: tests_x64/main
-                mapply(test.files, jobs, memouts, trim.name=TRUE))
+  th = "<th>Flavor</th><th>Version</th><th>Revision</th><th>Install</th><th>Status</th><th>Flags</th><th>Rout.fail</th><th>Memtest</th>"
+  tbl = sprintf(
+    "<tr><td><a href=\"check_flavors.html\">%s</a></td><td>%s</td><td>%s</td><td><a href=\"%s/%s/00install.out\">out</a></td><td><a href=\"%s/%s/00check.log\">%s</a></td><td>%s</td><td>%s</td><td>%s</td></tr>",
+    sub("test-", "", jobs, fixed=TRUE),
+    sapply(jobs, pkg.version, pkg),
+    sapply(jobs, pkg.revision, pkg),
+    pkg, jobs, ## install
+    pkg, jobs, sapply(sapply(jobs, check.test, pkg="data.table"), status), ## check
+    sapply(jobs, pkg.flags, pkg),
+    mapply(test.files, jobs, routs, trim.exts=2L), # 1st fail, 2nd Rout, keep just: tests_x64/main
+    mapply(test.files, jobs, memouts, trim.name=TRUE)
+  )
   file = file.path(repodir, "web/checks", sprintf("check_results_%s.html", pkg))
-  writeLines(c("<html>",
-               "<head>",
-               sprintf("<title>Package Check Results for Package %s</title>", pkg),
-               "<link rel=\"stylesheet\" type=\"text/css\" href=\"../CRAN_web.css\"/>",
-               "<meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\"/>",
-               "</head>",
-               "<body lang=\"en\">",
-               sprintf("<h2>Package Check Results for Package <a href=\"../packages/%s/index.html\"> %s </a> </h2>", pkg, pkg),
-               sprintf("<p>Last updated on %s.</p>", format(Sys.time(), usetz=TRUE)),
-               sprintf("<table border=\"1\" summary=\"CRAN check results for package %s\">", pkg),
-               "<tr><th>Test job</th><th>OS type</th><th>Install</th><th>Check</th><th>Rout.fail</th><th>Memtest</th></tr>",
-               tbl,
-               "</table>",
-               "</body>",
-               "</html>"),
-             file)
+  writeLines(c(
+    "<html>",
+    "<head>",
+    sprintf("<title>Package Check Results for Package %s</title>", pkg),
+    "<link rel=\"stylesheet\" type=\"text/css\" href=\"../CRAN_web.css\"/>",
+    "<meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\"/>",
+    "</head>",
+    "<body lang=\"en\">",
+    sprintf("<h2>Package Check Results for Package <a href=\"../packages/%s/index.html\"> %s </a> </h2>", pkg, pkg),
+    sprintf("<p>Last updated on %s.</p>", format(Sys.time(), usetz=TRUE)),
+    sprintf("<table border=\"1\" summary=\"CRAN check results for package %s\">", pkg),
+    "<tr>",th,"</tr>",
+    tbl,
+    "</table>",
+    "</body>",
+    "</html>"
+  ), file)
   setNames(file.exists(file), file)
 }