Merge branch 'master' into bmerge_verbose_coerce_col

.ci/atime/tests.R

@@ -231,7 +231,20 @@ test.list <- atime::atime_test_list(
     },
     expr = data.table:::melt(DT, measure.vars = measure.vars),
     Slow = "fd24a3105953f7785ea7414678ed8e04524e6955", # Parent of the merge commit (https://github.com/Rdatatable/data.table/commit/ed72e398df76a0fcfd134a4ad92356690e4210ea) of the PR (https://github.com/Rdatatable/data.table/pull/5054) that fixes the issue
-    Fast = "ed72e398df76a0fcfd134a4ad92356690e4210ea"), # Merge commit of the PR (https://github.com/Rdatatable/data.table/pull/5054) that fixes the issue
+    Fast = "ed72e398df76a0fcfd134a4ad92356690e4210ea"), # Merge commit of the PR (https://github.com/Rdatatable/data.table/pull/5054) that fixes the issue  # Test case created directly using the atime code below (not adapted from any other benchmark), based on the issue/fix PR https://github.com/Rdatatable/data.table/pull/5054#issue-930603663 "melt should be more efficient when there are missing input columns."
+
+  # Test case created from @tdhock's comment https://github.com/Rdatatable/data.table/pull/6393#issuecomment-2327396833, in turn adapted from @philippechataignon's comment https://github.com/Rdatatable/data.table/pull/6393#issuecomment-2326714012
+  "fwrite refactored in #6393" = atime::atime_test(
+    setup = {
+      set.seed(1)
+      NC = 10L
+      L <- data.table(i=1:N)
+      L[, paste0("V", 1:NC) := replicate(NC, rnorm(N), simplify=FALSE)]
+      out.csv <- tempfile()
+    },
+    expr = data.table::fwrite(L, out.csv, compress="gzip"),
+    Before = "f339aa64c426a9cd7cf2fcb13d91fc4ed353cd31", # Parent of the first commit https://github.com/Rdatatable/data.table/commit/fcc10d73a20837d0f1ad3278ee9168473afa5ff1 in the PR https://github.com/Rdatatable/data.table/pull/6393/commits with major change to fwrite with gzip.
+    PR = "3630413ae493a5a61b06c50e80d166924d2ef89a"), # Close-to-last merge commit in the PR.
 
   tests=extra.test.list)
 # nolint end: undesirable_operator_linter.

.ci/linters/c/alloc_linter.R

@@ -4,8 +4,9 @@
 #   2. Check the next line for a check like 'if (!x || !y)'
 alloc_linter = function(c_obj) {
   lines = c_obj$lines
-  # Be a bit more precise to avoid mentions in comments
-  alloc_lines = grep(R"{=\s*([(]\w+\s*[*][)])?[mc]alloc[(]}", lines)
+  # Be a bit more precise to avoid mentions in comments, and allow
+  #   malloc(0) to be used for convenience (e.g. #6757)
+  alloc_lines = grep(R"{=\s*([(]\w+\s*[*][)])?[mc]alloc[(][^0]}", lines)
   if (!length(alloc_lines)) return()
   # int *tmp=(int*)malloc(...); or just int tmp=malloc(...);
   alloc_keys = lines[alloc_lines] |>

.devcontainer/r-devel-alpine/Dockerfile

@@ -0,0 +1,22 @@
+FROM docker.io/rhub/r-minimal:devel
+
+RUN apk update \
+  && apk add --no-cache \
+     gcc git musl-dev openmp pkgconf tzdata zlib-dev \
+  && echo 'options("repos"="https://cloud.r-project.org")' >> /usr/local/lib/R/etc/Rprofile.site
+
+ENV TZDIR=/usr/share/zoneinfo
+
+COPY DESCRIPTION .
+
+RUN Rscript -e '                                          \ 
+read.dcf("DESCRIPTION", c("Imports", "Suggests")) |>      \
+  tools:::.split_dependencies() |>                        \
+  names() |>                                              \
+  setdiff(tools:::.get_standard_package_names()$base) |>  \
+  install.packages(repos="https://cloud.r-project.org")   \
+'
+
+# setup cc()
+WORKDIR /root
+COPY .devcontainer/.Rprofile .

.devcontainer/r-devel-alpine/devcontainer.json

@@ -0,0 +1,9 @@
+{
+    "build": { "dockerfile": "Dockerfile", "context": "../.." },
+    "customizations": { "vscode": {
+       "extensions": [
+        "REditorSupport.r",
+        "ms-vscode.cpptools-extension-pack"
+       ]
+    }}
+  }

.github/ISSUE_TEMPLATE/issue_template.md

@@ -5,13 +5,18 @@ about: Report a bug or describe a new requested feature
 
 Click preview tab ^^^ above!
 
-By continuing to file this new issue / feature request, I confirm I have :
+By continuing to file this new issue / feature request, I confirm I have:
 1. searched the [live NEWS file](https://github.com/Rdatatable/data.table/blob/master/NEWS.md) to see if it has been fixed in dev already. If so, I tried the [latest dev version](https://github.com/Rdatatable/data.table/wiki/Installation#windows).
 2. looked at the titles of all the issues in the [current milestones](https://github.com/Rdatatable/data.table/milestones) and am aware of those. (Adding new information to existing issues is very helpful and appreciated.)
 3. [searched all issues](https://github.com/Rdatatable/data.table/issues) (i.e. not in a milestone yet) for similar issues to mine and will include links to them explaining why mine is different.
 4. searched on [Stack Overflow's data.table tag](http://stackoverflow.com/questions/tagged/data.table) and there is nothing similar there.
 5. read the [Support](https://github.com/Rdatatable/data.table/wiki/Support) and [Contributing](https://github.com/Rdatatable/data.table/blob/master/.github/CONTRIBUTING.md) guides.
-6. please don't tag your issue with text in the title; project members will add the appropriate tags later.
+
+Some general advice on the title and description fields for your PR
+
+- Please don't tag your issue with text in the title like '[Joins]'; project members will add the appropriate tags later.
+- Don't write text like 'Closes #xxx' in the PR title either; GitHub does not recognize this text, whereas GitHub auto-links issues in the description, [see docs](https://docs.github.com/en/issues/tracking-your-work-with-issues/using-issues/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword).
+- Title and Description fields should try and be self-contained as much as possible. The title answers "what is this change" and the description provides necessary details/thought processes/things tried but abandoned. Imagine visiting your PR in 5 years' time and trying to glean what it's all about quickly and without needing to open 10 new tabs.
 
 #### Thanks! Please remove the text above and include the two items below.
 

.github/workflows/code-quality.yaml

@@ -70,6 +70,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+      - uses: r-lib/actions/setup-r@v2
       - name: Lint
         run: for (f in list.files('.ci/linters/md', full.names=TRUE)) source(f)
         shell: Rscript {0}

.github/workflows/pkgup.yaml

@@ -50,7 +50,7 @@ jobs:
           cp -R ${{ env.R_LIBS_USER }} library
           R CMD INSTALL --library="library" $(ls -1t data.table_*.tar.gz | head -n 1) --html
           mkdir -p doc/html
-          cp /usr/share/R/doc/html/{left.jpg,up.jpg,Rlogo.svg,R.css,index.html} doc/html
+          cp $(R RHOME)/doc/html/{left.jpg,up.jpg,Rlogo.svg,R.css,index.html} doc/html
           Rscript -e 'utils::make.packages.html("library", docdir="doc")'
           sed -i "s|file://|../..|g" doc/html/packages.html
           mkdir -p public

CODEOWNERS

@@ -68,6 +68,11 @@
 # docs
 /man/openmp-utils.Rd @Anirban166
 /Seal_of_Approval.md @tdhock
+/GOVERNANCE.md: @Rdatatable/committers
 
 # GLCI
 .gitlab-ci.yml @jangorecki @ben-schwen
+
+# C code tricks
+/src/chmatch.c @aitap
+/src/fread.c @aitap

GOVERNANCE.md

@@ -94,8 +94,9 @@ A pull request can be merged by any committer, if there is one approving review,
 
 ## Changing this GOVERNANCE.md document
 
-There is no special process for changing this document (submit a PR
-and ask for review).
+There is no special process for changing this document. Submit a PR and ask for review; the group `@Rdatatable/committers` will automatically be assigned to ensure all current Committers are aware of the change.
+
+Please also make a note in the change log under [`# Governance history`](#governance-history)
 
 # Code of conduct
 
@@ -123,6 +124,8 @@ data.table Version line in DESCRIPTION typically has the following meanings
 
 # Governance history
 
+Jan 2025: clarify that edits to governance should notify all committers.
+
 Feb 2024: change team name/link maintainers to committers, to be consistent with role defined in governance.
 
 Nov-Dec 2023: initial version drafted by Toby Dylan Hocking and

NAMESPACE

@@ -5,7 +5,7 @@ importFrom(methods, "S3Part<-", slotNames)
 exportClasses(data.table, IDate, ITime)
 ##
 
-export(data.table, tables, setkey, setkeyv, key, "key<-", haskey, CJ, SJ, copy)
+export(data.table, tables, setkey, setkeyv, key, haskey, CJ, SJ, copy)
 export(rowwiseDT)
 export(setindex, setindexv, indices)
 export(as.data.table,is.data.table,test.data.table)