|
| 1 | +require(methods) |
| 2 | +if (exists("test.data.table", .GlobalEnv, inherits=FALSE)) { |
| 3 | + if ((tt<-compiler::enableJIT(-1))>0) |
| 4 | + cat("This is dev mode and JIT is enabled (level ", tt, ") so there will be a brief pause around the first test.\n", sep="") |
| 5 | +} else { |
| 6 | + require(data.table) |
| 7 | + test = data.table:::test |
| 8 | + null.data.table = data.table:::null.data.table |
| 9 | +} |
| 10 | + |
| 11 | +# characterization tests guarding the modularization of [.data.table, #852 |
| 12 | +# they pin down behaviour that depends on the frame [ is evaluated in, on missing() and on early returns |
| 13 | +# names in calling scope must be resolved in the caller's frame, not in the global env nor in [.data.table's frame |
| 14 | +DT = data.table(a=1:6, b=c("a","a","b","b","c","c"), v=c(10,20,30,40,50,60)) |
| 15 | +thr = 100L; col = "zzz"; cols = "zzz"; grp = "zzz"; fn = function(x) stop("wrong frame"); n = 0L; e = quote(stop("wrong frame")) |
| 16 | +f = function(DT) { |
| 17 | + thr = 3L; col = "v"; cols = c("a","v"); grp = "b"; fn = sum; n = 1L; e = quote(a > thr) |
| 18 | + list( |
| 19 | + DT[a > thr, a], |
| 20 | + DT[, ..cols], |
| 21 | + DT[, get(col)], |
| 22 | + DT[, mget(cols)], |
| 23 | + DT[, .(s=fn(v)), by=grp], |
| 24 | + DT[, .(s=fn(v)), keyby=grp], |
| 25 | + DT[, lapply(.SD, fn), .SDcols=cols], |
| 26 | + DT[, .(s=FUN(COL)), by=GRP, env=list(FUN="sum", COL=col, GRP=grp)], |
| 27 | + DT[eval(e), a], |
| 28 | + DT[.("b"), on=grp, v], |
| 29 | + DT[, head(.SD, n), by=b], |
| 30 | + DT[, .(s=sum(v)), by=.(g = a > thr)]) |
| 31 | +} |
| 32 | +ans = f(DT) |
| 33 | +test(1.001, ans[[1L]], DT[a > 3L, a]) |
| 34 | +test(1.002, ans[[2L]], DT[, c("a", "v")]) |
| 35 | +test(1.003, ans[[3L]], DT[, v]) |
| 36 | +test(1.004, ans[[4L]], DT[, mget(c("a", "v"))]) |
| 37 | +test(1.005, ans[[5L]], DT[, .(s=sum(v)), by=b]) |
| 38 | +test(1.006, ans[[6L]], DT[, .(s=sum(v)), keyby=b]) |
| 39 | +test(1.007, ans[[7L]], DT[, lapply(.SD, sum), .SDcols=c("a", "v")]) |
| 40 | +test(1.008, ans[[8L]], DT[, .(s=FUN(COL)), by=GRP, env=list(FUN="sum", COL="v", GRP="b")]) |
| 41 | +test(1.009, ans[[9L]], DT[a > 3L, a]) |
| 42 | +test(1.010, ans[[10L]], DT[.("b"), on="b", v]) |
| 43 | +test(1.011, ans[[11L]], DT[, head(.SD, 1L), by=b]) |
| 44 | +test(1.012, ans[[12L]], DT[, .(s=sum(v)), by=.(g = a > 3L)]) |
| 45 | +# two levels of closures, and a local data.table used as i |
| 46 | +f = function(DT) { g = function() { lim = 40; X = data.table(b="c"); list(DT[v > lim, a], DT[X, on="b", a]) }; g() } |
| 47 | +test(1.021, f(DT), list(5:6, 5:6)) |
| 48 | +# by= a vector in calling scope that is not a column |
| 49 | +f = function(DT) { g = c(1,1,1,2,2,2); DT[, sum(v), by=g] } |
| 50 | +test(1.022, f(DT), data.table(g=c(1,2), V1=c(60,150))) |
| 51 | +# arguments other than i/j/by taken from calling scope; 'q' rolls forward onto the last 'c' row |
| 52 | +f = function(DT) { r = TRUE; nm = NULL; m = "last"; o = "b"; X = data.table(b=c("b","q")); DT[X, on=o, mult=m, nomatch=nm, roll=r, a] } |
| 53 | +test(1.023, f(DT), c(4L, 6L)) |
| 54 | +rm(thr, col, cols, grp, fn, n, e, f, ans) |
| 55 | + |
| 56 | +# .N in i means nrow(x) even when the caller has a .N; the caller's .N is restored and none is left behind |
| 57 | +f = function(DT) { .N = 100L; ans = list(DT[.N, a], DT[a < .N, a], DT[, .N]); c(ans, list(.N)) } |
| 58 | +test(1.11, f(DT), list(6L, 1:5, 6L, 100L)) |
| 59 | +f = function(DT) { DT[.N, a]; exists(".N", inherits=FALSE) } |
| 60 | +test(1.12, f(DT), FALSE) |
| 61 | +rm(f) |
| 62 | + |
| 63 | +# early returns |
| 64 | +test(1.21, DT[NULL], null.data.table()) |
| 65 | +test(1.22, DT[, NULL], NULL) |
| 66 | +test(1.23, DT[0L], DT[integer()]) |
| 67 | +test(1.24, DT[NA], data.table(a=NA_integer_, b=NA_character_, v=NA_real_)) |
| 68 | +test(1.25, DT[a > 3L, which=TRUE], 4:6) |
| 69 | +test(1.26, DT[.(c("b","q")), on="b", which=TRUE], c(3L, 4L, NA)) |
| 70 | +test(1.27, DT[.(c("b","q")), on="b", which=NA], 2L) |
| 71 | +test(1.28, DT[.(c("b","q")), on="b", which=TRUE, nomatch=NULL], 3:4) |
| 72 | +test(1.29, DT[!.("b"), on="b", which=TRUE], c(1L,2L,5L,6L)) |
| 73 | +test(1.30, DT[, character(0L), with=FALSE], null.data.table()) |
| 74 | +test(1.31, DT[, !character(0L), with=FALSE], DT) |
| 75 | +test(1.32, DT[!(a > 2L), a], 1:2) |
| 76 | +test(1.33, DT[c(0L, 2L, 0L), a], 2L) |
| 77 | + |
| 78 | +# missing() semantics: an argument substituted to the empty symbol via env= counts as missing |
| 79 | +test(1.41, DT[, J, env=list(J=quote(expr=))], DT[,,]) |
| 80 | +test(1.42, DT[I, env=list(I=quote(expr=))], DT[,,]) |
| 81 | +test(1.43, DT[I, a, env=list(I=quote(expr=))], DT[, a]) |
| 82 | +test(1.44, DT[a > 4L, J, env=list(J=quote(expr=))], DT[5:6,]) |
| 83 | +test(1.45, DT[, a, by=B, env=list(B=quote(expr=))], 1:6) |
| 84 | +test(1.46, DT[, a, keyby=B, env=list(B=quote(expr=))], 1:6) |
| 85 | +test(1.47, DT[, a, by=NULL], data.table(a=1:6)) |
| 86 | +test(1.48, DT[, a, keyby=NULL], data.table(a=1:6)) |
| 87 | +test(1.49, DT[, a, .SDcols="b"], 1:6, warning="This j doesn't use .SD but .SDcols has been supplied") |
| 88 | +test(1.50, DT[, .(a, )], error="Item 2 of the .() or list() passed to j is missing") |
| 89 | +test(1.51, DT[on="b"], DT, warning=c("When on= is provided but not i=", "i and j are both missing so ignoring the other arguments")) |
| 90 | + |
| 91 | +# forms of i |
| 92 | +test(1.61, DT[matrix(1:2, ncol=1L)], DT[1:2,]) |
| 93 | +test(1.62, DT[list("b"), on="b"], DT[3:4,]) |
| 94 | +test(1.63, DT[list(b="c"), on="b"], DT[5:6,]) |
| 95 | +K = copy(DT); setkey(K, b) |
| 96 | +test(1.64, K["b"], K[3:4,]) |
| 97 | +test(1.65, K[factor("c")], K[5:6,]) |
| 98 | +test(1.66, DT["b"], error="When i is a data.table (or character vector), the columns to join by must be specified using the 'on=' argument") |
| 99 | +test(1.67, DT[c(TRUE, FALSE)], error="i evaluates to a logical vector length 2 but there are 6 rows") # note that DT[c(TRUE)] just shallow copies |
| 100 | +test(1.68, DT[.(c("a","b")), on="b", mult="first"], DT[c(1L, 3L)]) |
| 101 | +test(1.69, DT[.(c("a","b")), on="b", mult="last"], DT[c(2L, 4L)]) |
| 102 | +test(1.70, DT[.("a"), on="b", mult="error"], error="mult='error' and multiple matches during merge") |
| 103 | +test(1.71, DT[.(c(3L, 5L)), on="a", mult="error"], DT[c(3L, 5L)]) |
| 104 | +X = data.table(k=c(1L,1L), x=1:2) |
| 105 | +test(1.72, options=c(datatable.join.many=FALSE), X[data.table(k=c(1L, 1L)), on="k"], , error="Joining resulted in many-to-many join") |
| 106 | +test(1.73, options=c(datatable.join.many=TRUE), X[data.table(k=c(1L, 1L)), on="k"], X[c(1L, 2L, 1L, 2L)]) |
| 107 | +# := by group on a key column drops the key, with and without GForce |
| 108 | +K = data.table(k=c(2L,1L,2L), a=1:3, g=c(1L,1L,2L), key="k") |
| 109 | +test(1.74, optimize=c(0, Inf), key(copy(K)[, k := max(k), by=g]), NULL) |
| 110 | +test(1.75, optimize=c(0, Inf), key(copy(K)[, a := max(a), by=a]), "k") |
| 111 | +rm(K, X) |
| 112 | + |
| 113 | +# forms of j with with=FALSE, and their with=TRUE equivalents |
| 114 | +DT = data.table(x=1, y=2, z=3) |
| 115 | +test(1.81, DT[, c("x","z"), with=FALSE], DT[, .(x, z)]) |
| 116 | +test(1.82, DT[, !"y", with=FALSE], DT[, .(x, z)]) |
| 117 | +test(1.83, DT[, -c("y"), with=FALSE], DT[, .(x, z)]) |
| 118 | +test(1.84, DT[, z:x, with=FALSE], DT[, .(z, y, x)]) |
| 119 | +test(1.85, DT[, c(TRUE,FALSE,TRUE), with=FALSE], DT[, .(x, z)]) |
| 120 | +test(1.86, DT[, x:y], DT[, .(x, y)]) |
| 121 | +test(1.87, DT[, !c("x","y")], DT[, .(z)]) |
| 122 | +DT = data.table(x=1, y=2, x=3) |
| 123 | +test(1.88, DT[, !"x"], data.table(y=2)) |
| 124 | +test(1.89, DT[, c("x","x")], DT[, c(1L, 1L)]) # debatable if we want to test for this |
| 125 | +test(1.90, DT[, `:=`(z=1), with=FALSE], error="`:=` is only supported under with=TRUE") |
| 126 | +col = 5 |
| 127 | +test(1.91, DT[, col], error="j (the 2nd argument inside [...]) is a single symbol but column name 'col' is not found") |
| 128 | +rm(col, DT) |
| 129 | + |
| 130 | +# joins combined with by=.EACHI, :=, roll and x./i. prefixes |
| 131 | +X = data.table(id=c(1L,1L,2L,3L), t=c(1,5,2,4), v=1:4) |
| 132 | +Y = data.table(id=c(1L,2L,4L), t=c(3,3,3), w=c(10,20,30)) |
| 133 | +test(2.01, copy(X)[Y, on="id", s := sum(v) + i.w, by=.EACHI], data.table(id=c(1L,1L,2L,3L), t=c(1,5,2,4), v=1:4, s=c(13,13,23,NA))) |
| 134 | +test(2.02, X[Y, on="id", .(n=.N, s=sum(v)), by=.EACHI], data.table(id=c(1L,2L,4L), n=c(2L,1L,0L), s=c(3L,3L,NA))) |
| 135 | +test(2.03, X[Y, on="id", .(n=.N), by=.EACHI, nomatch=NULL], data.table(id=1:2, n=2:1)) |
| 136 | +test(2.04, copy(Y)[X, on=c("id","t"), roll=TRUE, xv := i.v], data.table(id=c(1L,2L,4L), t=c(3,3,3), w=c(10,20,30), xv=c(2L,NA,NA))) |
| 137 | +test(2.05, X[Y, on=c("id","t"), roll=-Inf, v], c(2L,NA,NA)) |
| 138 | +test(2.06, X[Y, on=c("id","t"), roll="nearest", v], c(1L,3L,NA)) |
| 139 | +test(2.07, X[Y, on=c("id","t"), roll=TRUE, rollends=c(TRUE,TRUE), v], c(1L,3L,NA)) |
| 140 | +test(2.08, X[Y, on=.(id, t<=t), .(id, x.t, v, w)], data.table(id=c(1L,2L,4L), x.t=c(1,2,NA), v=c(1L,3L,NA), w=c(10,20,30))) |
| 141 | +test(2.09, X[Y, on="id", .(x.id, i.id, x.t, i.t)], data.table(x.id=c(1L,1L,2L,NA), i.id=c(1L,1L,2L,4L), x.t=c(1,5,2,NA), i.t=c(3,3,3,3))) |
| 142 | +test(2.10, copy(X)[Y, on="id", w := i.w], data.table(id=c(1L,1L,2L,3L), t=c(1,5,2,4), v=1:4, w=c(10,10,20,NA))) |
| 143 | + |
| 144 | +# := forms |
| 145 | +test(2.21, copy(X)[, let(u = v*2L)], cbind(X, u=c(2L,4L,6L,8L))) |
| 146 | +test(2.22, copy(X)[, `:=`(a=1L, b=2L)], cbind(X, a=1L, b=2L)) |
| 147 | +cols = c("p","q") |
| 148 | +test(2.23, copy(X)[, (cols) := list(1L, 2L)], cbind(X, p=1L, q=2L)) |
| 149 | +rm(cols) |
| 150 | +test(2.24, copy(X)[, m := mean(v), by=id], cbind(X, m=c(1.5,1.5,3,4))) |
| 151 | +test(2.25, copy(X)[v > 2L, v := 0L], data.table(id=c(1L,1L,2L,3L), t=c(1,5,2,4), v=c(1L,2L,0L,0L))) |
| 152 | +test(2.26, copy(X)[v > 100L, nn := 1L], cbind(X, nn=NA_integer_)) |
| 153 | +test(2.27, copy(X)[, z := a*2L, keyby=id, env=list(a="v")], setkey(cbind(X, z=c(2L,4L,6L,8L)), id)) |
| 154 | +test(2.28, X[, .SD[, z := 1L], by=id], error=".SD is locked") |
| 155 | +test(2.29, X[, (function(d) d[, z := 1L])(.SD), by=id], error=".SD is locked") |
| 156 | + |
| 157 | +# special symbols in grouped j, and by= forms |
| 158 | +test(2.41, X[, .(.N, .GRP, .NGRP, first_I=.I[1L], b=.BY$id * 10L), by=id], data.table(id=1:3, N=c(2L,1L,1L), GRP=1:3, NGRP=3L, first_I=c(1L,3L,4L), b=c(10L,20L,30L))) |
| 159 | +test(2.42, X[, sum(v), keyby=.(g = -id)], data.table(g=-3:-1, V1=c(4L,3L,3L), key="g")) |
| 160 | +test(2.43, X[, sum(v), by=.(g = id %% 2L)], data.table(g=c(1L,0L), V1=c(7L,3L))) |
| 161 | +test(2.44, X[, sum(v), by="id,t"], data.table(id=c(1L,1L,2L,3L), t=c(1,5,2,4), V1=1:4)) |
| 162 | +test(2.45, X[, sum(v), by=character()], data.table(V1=10L)) |
| 163 | +DT = data.table(d=as.IDate(c("2020-01-05","2020-02-01","2020-01-20")), v=1:3, key="d") |
| 164 | +test(2.46, DT[, sum(v), by=month(d)], data.table(month=1:2, V1=c(4L,2L))) |
| 165 | +rm(DT) |
| 166 | +test(2.47, X[, lapply(.SD, sum), .SDcols=patterns("^[tv]$")], data.table(t=12, v=10L)) |
| 167 | +test(2.48, X[, lapply(.SD, sum), .SDcols=is.integer], data.table(id=7L, v=10L)) |
| 168 | +test(2.49, X[, lapply(.SD, sum), .SDcols=!"id"], data.table(t=12, v=10L)) |
| 169 | +test(2.50, X[, lapply(.SD, sum), .SDcols=-(id:t)], data.table(v=10L)) |
| 170 | +test(2.51, X[, lapply(.SD, function(x) x/v), .SDcols=!"v"], data.table(id=c(1,0.5,2/3,0.75), t=c(1,2.5,2/3,1))) |
0 commit comments