better error msg when assign RHS is list column, closes #950 (#3457)

inst/tests/tests.Rraw

@@ -13783,7 +13783,7 @@ test(2005.1, truelength(NULL), 0L)
 DT = data.table(a=1:3, b=4:6)
 test(2005.2, set(DT, 4L, "b", NA), error="i[1] is 4 which is out of range [1,nrow=3]")
 test(2005.3, set(DT, 3L, 8i, NA), error="j is type 'complex'. Must be integer, character, or numeric is coerced with warning.")
-test(2005.4, set(DT, 1L, 2L, expression(x+2)), error="RHS of assignment is not NULL, not an an atomic vector (see ?is.atomic) and not a list column.")
+test(2005.4, set(DT, 1L, 2L, expression(x+2)), error="RHS of assignment is not NULL, not an atomic vector (see ?is.atomic) and not a list column.")
 DT[,foo:=factor(c("a","b","c"))]
 test(2005.5, DT[2, foo:=8i], error="Can't assign to column 'foo' (type 'factor') a value of type 'complex' (not character, factor, integer or numeric)")
 test(2005.6, DT[2, a:=9, verbose=TRUE], output="Coerced length-1 RHS from double to integer to match column's type. No precision was lost. If this")
@@ -13986,6 +13986,11 @@ setkey(DT, a, b)
 setorder(DT, b)
 test(2021.3, key(DT), NULL)
 
+# assign RHS list better error msg, #950
+d = data.table(id=c("a","b"), f=list(function(x) x*2, function(x) x^2), key="id")
+test(2022.1, d[.("a"), f:=list(function(x) x^3)], error="RHS of assignment is not NULL.*try wrapping it in an extra list.*")
+test(2022.2, d[.("a"), f:=list(list(function(x) x^3))], data.table(id=c("a","b"), f=list(function(x) x^3, function(x) x^2), key="id"))
+
 
 ###################################
 #  Add new tests above this line  #

src/assign.c

@@ -419,7 +419,7 @@ SEXP assign(SEXP dt, SEXP rows, SEXP cols, SEXP newcolnames, SEXP values, SEXP v
       // RHS of assignment to new column is zero length but we'll use its type to create all-NA column of that type
     }
     if (!(isVectorAtomic(thisvalue) || isNewList(thisvalue)))  // NULL had a continue earlier above
-      error("RHS of assignment is not NULL, not an an atomic vector (see ?is.atomic) and not a list column.");
+      error("RHS of assignment is not NULL, not an atomic vector (see ?is.atomic) and not a list column. Recall that structurally, a list column is a sequence of lists wrapped in list() or .(); if you're trying to create a list column, try wrapping it in an extra list().");
     if (isMatrix(thisvalue) && (j=INTEGER(getAttrib(thisvalue, R_DimSymbol))[1]) > 1)  // matrix passes above (considered atomic vector)
       warning("%d column matrix RHS of := will be treated as one vector", j);
     if ((coln+1)<=oldncol && isFactor(VECTOR_ELT(dt,coln)) &&