better error msg when assign RHS is list column, closes #950

inst/tests/tests.Rraw

@@ -13626,6 +13626,11 @@ dx = data.table(id = 1L, key = "id")
 di = list(z=c(2L, 1L))
 test(1999.2, key(dx[di]), NULL)
 
+# assign RHS list better error msg #950
+d = data.table(id=c("a","b"), f=list(function(x) x*2, function(x) x^2), key="id")
+test(2000.1, d[.("a"), f:=list(function(x) x^3)], error="RHS of assignment is not NULL.*try wrapping it in extra list.*")
+test(2000.2, d[.("a"), f:=list(list(function(x) x^3))], data.table(id=c("a","b"), f=list(function(x) x^3, function(x) x^2), key="id"))
+
 
 ###################################
 #  Add new tests above this line  #

src/assign.c

@@ -446,7 +446,7 @@ SEXP assign(SEXP dt, SEXP rows, SEXP cols, SEXP newcolnames, SEXP values, SEXP v
       }
     }
     if (!(isVectorAtomic(thisvalue) || isNewList(thisvalue)))  // NULL had a continue earlier above
-      error("RHS of assignment is not NULL, not an an atomic vector (see ?is.atomic) and not a list column.");
+      error("RHS of assignment is not NULL, not an atomic vector (see ?is.atomic) and not a list column. Recall that structurally, a list column is a sequence of lists wrapped in list() or .(); if you're trying to create a list column, try wrapping it in extra list().");
     if (isMatrix(thisvalue) && (j=INTEGER(getAttrib(thisvalue, R_DimSymbol))[1]) > 1)  // matrix passes above (considered atomic vector)
       warning("%d column matrix RHS of := will be treated as one vector", j);
     if ((coln+1)<=oldncol && isFactor(VECTOR_ELT(dt,coln)) &&