Skip to content

Commit b7fe976

Browse files
committed
move inline helper functions to top level of [
Functions defined inside [ capture its frame, so they block moving the code that uses them. Move them to internal top-level functions, named .dt_* so that conditions they raise still report the [.data.table call. No behaviour change.
1 parent 74c52e9 commit b7fe976

3 files changed

Lines changed: 143 additions & 106 deletions

File tree

‎R/data.table.R‎

Lines changed: 133 additions & 105 deletions
Original file line numberDiff line numberDiff line change
@@ -725,8 +725,6 @@ replace_dot_alias = function(e) {
725725
}
726726
}
727727

728-
# setdiff removes duplicate entries, which'll create issues with duplicated names. Use %chin% instead.
729-
dupdiff = function(x, y) x[!x %chin% y]
730728
isub = NULL
731729
if (!missing_i) {
732730
if (is.null(env)) isub = substitute(i) else {
@@ -1354,66 +1352,22 @@ replace_dot_alias = function(e) {
13541352
}
13551353

13561354
jvnames = NULL
1357-
drop_dot = function(x) {
1358-
if (length(x)!=1L) internal_error("drop_dot passed %d items", length(x)) # nocov
1359-
if (startsWith(x<-as.character(x), ".") && x %chin% c(".N", ".I", ".GRP", ".NGRP", ".BY"))
1360-
substr(x, 2L, nchar(x))
1361-
else
1362-
x
1363-
}
1364-
# handle auto-naming of last item of j (e.g. within {} or if/else, #2478)
1365-
# e.g. DT[, .(a=sum(v), v, .N), by=] should create columns named a, v, N
1366-
do_j_names = function(q) {
1367-
if (!is.call(q) || !is.name(q[[1L]])) return(q)
1368-
if (q[[1L]] %chin% c('list', '.')) {
1369-
q[[1L]] = quote(list)
1370-
qlen = length(q)
1371-
if (qlen>1L) {
1372-
nm = names(q[-1L]) # check list(a=sum(v),v)
1373-
if (is.null(nm)) nm = rep.int("", qlen-1L)
1374-
# attempt to auto-name unnamed columns
1375-
for (jj in which(!nzchar(nm))) {
1376-
thisq = q[[jj + 1L]]
1377-
if (missing(thisq)) stopf("Item %d of the .() or list() passed to j is missing", jj) #3507
1378-
if (is.name(thisq)) nm[jj] = drop_dot(thisq)
1379-
# TO DO: if call to a[1] for example, then call it 'a' too
1380-
}
1381-
if (!is.null(jvnames)) {
1382-
if (length(nm) != length(jvnames))
1383-
warningf("j may not evaluate to the same number of columns for each group; if you're sure this warning is in error, please put the branching logic outside of [ for efficiency")
1384-
else if (any(idx <- nm != jvnames))
1385-
warningf('Different branches of j expression produced different auto-named columns: %s; using the most "last" names. If this was intentional (e.g., you know only one branch will ever be used in a given query because the branch is controlled by a function argument), please (1) pull this branch out of the call; (2) explicitly provide missing defaults for each branch in all cases; or (3) use the same name for each branch and re-name it in a follow-up call.', brackify(sprintf('%s!=%s', nm[idx], jvnames[idx])))
1386-
}
1387-
# nolint next: undesirable_operator_linter. Workaround is clunkier, though a bigger refactor could be considered.
1388-
jvnames <<- nm # TODO: handle if() list(a, b) else list(b, a) better
1389-
setattr(q, "names", NULL) # drops the names from the list so it's faster to eval the j for each group; reinstated at the end on the result.
1390-
}
1391-
return(q) # else empty list is needed for test 468: adding an empty list column
1392-
}
1393-
if (q[[1L]] == '{') {
1394-
if (!is.null(q[[qlen<-length(q)]])) q[[qlen]] = do_j_names(q[[qlen]])
1395-
return(q)
1396-
}
1397-
if (q[[1L]] == 'if') {
1398-
#explicit NULL would return NULL, assigning NULL would delete that from the expression
1399-
if (!is.null(q[[3L]])) q[[3L]] = do_j_names(q[[3L]])
1400-
if (length(q) == 4L && !is.null(q[[4L]])) q[[4L]] = do_j_names(q[[4L]])
1401-
return(q)
1402-
}
1403-
q
1404-
}
14051355
if (is.name(jsub)) {
14061356
# j is a single unquoted column name
1407-
if (jsub!=".SD") jvnames = drop_dot(jsub)
1357+
if (jsub!=".SD") jvnames = .dt_drop_dot(jsub)
14081358
# jsub is list()ed after it's eval'd inside dogroups.
1409-
} else jsub = do_j_names(jsub) # else maybe a call to transform or something which returns a list.
1359+
} else { # else maybe a call to transform or something which returns a list.
1360+
jsub_names = .dt_do_j_names(jsub, jvnames)
1361+
jsub = jsub_names$jsub
1362+
jvnames = jsub_names$jvnames
1363+
}
14101364
av = all.vars(jsub,TRUE) # TRUE fixes bug #1294 which didn't see b in j=fns[[b]](c)
14111365
use.I = ".I" %chin% av
14121366
if (any(c(".SD","eval","get","mget") %chin% av)) {
14131367
if (missing_SDcols) {
1414-
# here we need to use 'dupdiff' instead of 'setdiff'. Ex: setdiff(c("x", "x"), NULL) will give 'x'.
1368+
# here we need to use '.dt_dupdiff' instead of 'setdiff'. Ex: setdiff(c("x", "x"), NULL) will give 'x'.
14151369
# slight memory efficiency boost if we only store sdvars if it differs from ansvars, but a bit tricky to examine all the uses here
1416-
ansvars = sdvars = dupdiff(names_x, union(bynames, allbyvars)) # TO DO: allbyvars here for vars used by 'by'. Document.
1370+
ansvars = sdvars = .dt_dupdiff(names_x, union(bynames, allbyvars)) # TO DO: allbyvars here for vars used by 'by'. Document.
14171371
# just using .SD in j triggers all non-by columns in the subset even if some of
14181372
# those columns are not used. It would be tricky to detect whether the j expression
14191373
# really does use all of the .SD columns or not, hence .SDcols for grouping
@@ -1465,11 +1419,11 @@ replace_dot_alias = function(e) {
14651419
ansvars = sdvars = names_x[ansvals]
14661420
} else if (is.numeric(.SDcols)) {
14671421
.SDcols = as.integer(.SDcols)
1468-
# if .SDcols is numeric, use 'dupdiff' instead of 'setdiff'
1422+
# if .SDcols is numeric, use '.dt_dupdiff' instead of 'setdiff'
14691423
if (length(unique(sign(.SDcols))) > 1L) stopf(".SDcols is numeric but has both +ve and -ve indices")
14701424
if (any(idx <- abs(.SDcols)>ncol(x) | abs(.SDcols)<1L))
14711425
stopf(".SDcols is numeric but out of bounds [1, %d] at: %s", ncol(x), brackify(which(idx)))
1472-
ansvars = sdvars = if (negate_sdcols) dupdiff(names_x[-.SDcols], bynames) else names_x[.SDcols]
1426+
ansvars = sdvars = if (negate_sdcols) .dt_dupdiff(names_x[-.SDcols], bynames) else names_x[.SDcols]
14731427
ansvals = if (negate_sdcols) setdiff(seq_along(names(x)), c(.SDcols, which(names(x) %chin% bynames))) else .SDcols
14741428
} else {
14751429
if (!is.character(.SDcols)) stopf(".SDcols should be column numbers or names")
@@ -1533,7 +1487,7 @@ replace_dot_alias = function(e) {
15331487
suppPrint = identity
15341488
if (length(av) && av[1L] == ":=") {
15351489
if (.Call(C_islocked, x)) stopf(".SD is locked. Using := in .SD's j is reserved for possible future use; a tortuously flexible way to modify by group. Use := in j directly to modify by group by reference.")
1536-
suppPrint = function(x) { .global$print=address(x); x }
1490+
suppPrint = .dt_suppPrint
15371491
# Suppress print when returns ok not on error, bug #2376. Thanks to: http://stackoverflow.com/a/13606880/403310
15381492
# All appropriate returns following this point are wrapped; i.e. return(suppPrint(x)).
15391493

@@ -1883,20 +1837,7 @@ replace_dot_alias = function(e) {
18831837
if (identical(class(jval), c('data.table', 'data.frame')))
18841838
setattr(jval, 'class', class(x)) # fix for #64
18851839
# can jval be sorted by the same key as x? improved for #4498
1886-
get_shared_keys = function(jsub, jvnames, sdvars, key) {
1887-
if (is.null(key)) return(NULL)
1888-
if (!((SD_only <- jsub == quote(.SD)) || jsub %iscall% "list")) return(NULL)
1889-
if (SD_only)
1890-
jvnames = jnames = sdvars
1891-
else
1892-
jnames = vapply_1c(jsub, function(x) if (is.name(x)) as.character(x) else NA_character_)[-1L]
1893-
key_idx = chmatch(key, jnames)
1894-
missing_keys = which(is.na(key_idx))
1895-
if (length(missing_keys) && missing_keys[1L] == 1L) return(NULL)
1896-
if (!length(missing_keys)) return(jvnames[key_idx])
1897-
jvnames[head(key_idx, missing_keys[1L] - 1L)]
1898-
}
1899-
shared_keys = get_shared_keys(jsub, jvnames, sdvars = sdvars, key(x))
1840+
shared_keys = .dt_get_shared_keys(jsub, jvnames, sdvars = sdvars, key(x))
19001841
if (is.null(irows) && !is.null(shared_keys)) {
19011842
if (!any(shared_keys %chin% duplicated_values(names(jval)))) setattr(jval, 'sorted', shared_keys)
19021843
# potentially inefficient backup -- check if jval is sorted by key(x)
@@ -2101,58 +2042,30 @@ replace_dot_alias = function(e) {
21012042
gi = if (length(o__)) o__[f__] else f__
21022043
g = lapply(grpcols, function(i) .Call(CsubsetVector, groups[[i]], gi)) # use CsubsetVector instead of [ to preserve attributes #5567
21032044

2104-
# returns all rows instead of one per group
2105-
nrow_funs = c("gshift")
2106-
.is_nrows = function(q) {
2107-
if (!is.call(q)) return(FALSE)
2108-
if (q[[1L]] == "list") {
2109-
any(vapply(q, .is_nrows, FALSE))
2110-
} else {
2111-
q[[1L]] %chin% nrow_funs
2112-
}
2113-
}
2114-
21152045
# adding ghead/gtail(n) support for n > 1 #5060 #523
21162046
q3 = 0L
21172047
if (!is.symbol(jsub)) {
2118-
headTail_arg = function(q) {
2119-
if (length(q)==3L && length(q3 <- q[[3L]])==1L && is.numeric(q3) &&
2120-
(q[[1L]]) %chin% c("ghead", "gtail") && q3!=1L) q3
2121-
else 0L
2122-
}
21232048
if (jsub %iscall% "list"){
2124-
q3 = max(sapply(jsub, headTail_arg))
2049+
q3 = max(sapply(jsub, .dt_headTail_arg))
21252050
} else if (length(jsub)==3L) {
2126-
q3 = headTail_arg(jsub)
2051+
q3 = .dt_headTail_arg(jsub)
21272052
}
21282053
}
21292054
if (q3 > 0L) {
21302055
grplens = pmin.int(q3, len__)
21312056
g = lapply(g, rep.int, times=grplens)
2132-
} else if (.is_nrows(jsub)) {
2057+
} else if (.dt_is_nrows(jsub)) {
21332058
g = lapply(g, rep.int, times=len__)
21342059
# unpack list of lists for nrows functions
2135-
zip_items = function(ll) do.call(mapply, c(list(FUN = c), ll, SIMPLIFY=FALSE, USE.NAMES=FALSE))
21362060
if (all(vapply_1b(ans, is.list))) {
2137-
ans = lapply(ans, zip_items)
2061+
ans = lapply(ans, .dt_zip_items)
21382062
}
21392063
}
21402064
ans = c(g, ans)
21412065
} else {
21422066
ans = .Call(Cdogroups, x, xcols, groups, grpcols, jiscols, xjiscols, grporder, o__, f__, len__, jsub, SDenv, cols, newnames, !missing_on, verbose, as.integer(showProgress))
21432067
}
2144-
# unlock any locked data.table components of the answer, #4159
2145-
# MAX_DEPTH prevents possible infinite recursion from truly recursive object, #4173
2146-
# TODO: is there an efficient way to get around this MAX_DEPTH limit?
2147-
MAX_DEPTH = 5L
2148-
runlock = function(x, current_depth = 1L) {
2149-
if (is.list(x) && current_depth <= MAX_DEPTH) { # is.list() used to be is.recursive(), #4814
2150-
if (inherits(x, 'data.table')) .Call(C_unlock, x)
2151-
else return(lapply(x, runlock, current_depth = current_depth + 1L))
2152-
}
2153-
invisible()
2154-
}
2155-
runlock(ans)
2068+
.dt_runlock(ans)
21562069
if (verbose) {cat(timetaken(last.started.at),"\n"); flush.console()}
21572070
# TO DO: xrows would be a better name for irows: irows means the rows of x that i joins to
21582071
# Grouping by i: icols the joins columns (might not need), isdcols (the non join i and used by j), all __ are length x
@@ -2167,7 +2080,7 @@ replace_dot_alias = function(e) {
21672080
if (length(o__)) jrows = o__[jrows]
21682081
if (length(irows)) jrows = irows[jrows]
21692082
if (length(jvals)==1L) jvals = jvals[[1L]] # unwrap single column jvals for assign
2170-
if (.is_nrows(jsub)) { # 5403 unwrap multicolumn jvals for gfunctions that can return lists
2083+
if (.dt_is_nrows(jsub)) { # 5403 unwrap multicolumn jvals for gfunctions that can return lists
21712084
jvals = if (length(jvals) != length(lhs)) split(unlist(jvals), rep(seq_along(jvals[[1L]]), length(jvals))) else lapply(jvals, unlist)
21722085
}
21732086
.Call(Cassign, x, jrows, lhs, newnames, jvals)
@@ -2226,6 +2139,121 @@ replace_dot_alias = function(e) {
22262139
setalloccol(ans) # TODO: overallocate in dogroups in the first place and remove this line
22272140
}
22282141

2142+
# setdiff removes duplicate entries, which'll create issues with duplicated names. Use %chin% instead.
2143+
.dt_dupdiff = function(x, y) x[!x %chin% y]
2144+
2145+
# auto-name a symbol in j: special symbols .N, .I, .GRP, .NGRP and .BY lose their leading dot
2146+
.dt_drop_dot = function(x) {
2147+
if (length(x)!=1L) internal_error("drop_dot passed %d items", length(x)) # nocov
2148+
if (startsWith(x<-as.character(x), ".") && x %chin% c(".N", ".I", ".GRP", ".NGRP", ".BY"))
2149+
substr(x, 2L, nchar(x))
2150+
else
2151+
x
2152+
}
2153+
2154+
# handle auto-naming of last item of j (e.g. within {} or if/else, #2478)
2155+
# e.g. DT[, .(a=sum(v), v, .N), by=] should create columns named a, v, N
2156+
# returns list(jsub=, jvnames=); jvnames is threaded through the recursion so that the names of different branches can be compared
2157+
.dt_do_j_names = function(q, jvnames) {
2158+
if (!is.call(q) || !is.name(q[[1L]])) return(list(jsub=q, jvnames=jvnames))
2159+
if (q[[1L]] %chin% c('list', '.')) {
2160+
q[[1L]] = quote(list)
2161+
qlen = length(q)
2162+
if (qlen>1L) {
2163+
nm = names(q[-1L]) # check list(a=sum(v),v)
2164+
if (is.null(nm)) nm = rep.int("", qlen-1L)
2165+
# attempt to auto-name unnamed columns
2166+
for (jj in which(!nzchar(nm))) {
2167+
thisq = q[[jj + 1L]]
2168+
if (missing(thisq)) stopf("Item %d of the .() or list() passed to j is missing", jj) #3507
2169+
if (is.name(thisq)) nm[jj] = .dt_drop_dot(thisq)
2170+
# TO DO: if call to a[1] for example, then call it 'a' too
2171+
}
2172+
if (!is.null(jvnames)) {
2173+
if (length(nm) != length(jvnames))
2174+
warningf("j may not evaluate to the same number of columns for each group; if you're sure this warning is in error, please put the branching logic outside of [ for efficiency")
2175+
else if (any(idx <- nm != jvnames))
2176+
warningf('Different branches of j expression produced different auto-named columns: %s; using the most "last" names. If this was intentional (e.g., you know only one branch will ever be used in a given query because the branch is controlled by a function argument), please (1) pull this branch out of the call; (2) explicitly provide missing defaults for each branch in all cases; or (3) use the same name for each branch and re-name it in a follow-up call.', brackify(sprintf('%s!=%s', nm[idx], jvnames[idx])))
2177+
}
2178+
jvnames = nm # TODO: handle if() list(a, b) else list(b, a) better
2179+
setattr(q, "names", NULL) # drops the names from the list so it's faster to eval the j for each group; reinstated at the end on the result.
2180+
}
2181+
return(list(jsub=q, jvnames=jvnames)) # else empty list is needed for test 468: adding an empty list column
2182+
}
2183+
if (q[[1L]] == '{') {
2184+
if (!is.null(q[[qlen<-length(q)]])) {
2185+
ans = .dt_do_j_names(q[[qlen]], jvnames)
2186+
q[[qlen]] = ans$jsub
2187+
jvnames = ans$jvnames
2188+
}
2189+
return(list(jsub=q, jvnames=jvnames))
2190+
}
2191+
if (q[[1L]] == 'if') {
2192+
#explicit NULL would return NULL, assigning NULL would delete that from the expression
2193+
if (!is.null(q[[3L]])) {
2194+
ans = .dt_do_j_names(q[[3L]], jvnames)
2195+
q[[3L]] = ans$jsub
2196+
jvnames = ans$jvnames
2197+
}
2198+
if (length(q) == 4L && !is.null(q[[4L]])) {
2199+
ans = .dt_do_j_names(q[[4L]], jvnames)
2200+
q[[4L]] = ans$jsub
2201+
jvnames = ans$jvnames
2202+
}
2203+
return(list(jsub=q, jvnames=jvnames))
2204+
}
2205+
list(jsub=q, jvnames=jvnames)
2206+
}
2207+
2208+
.dt_suppPrint = function(x) { .global$print=address(x); x }
2209+
2210+
# can jval be sorted by the same key as x? improved for #4498
2211+
.dt_get_shared_keys = function(jsub, jvnames, sdvars, key) {
2212+
if (is.null(key)) return(NULL)
2213+
if (!((SD_only <- jsub == quote(.SD)) || jsub %iscall% "list")) return(NULL)
2214+
if (SD_only)
2215+
jvnames = jnames = sdvars
2216+
else
2217+
jnames = vapply_1c(jsub, function(x) if (is.name(x)) as.character(x) else NA_character_)[-1L]
2218+
key_idx = chmatch(key, jnames)
2219+
missing_keys = which(is.na(key_idx))
2220+
if (length(missing_keys) && missing_keys[1L] == 1L) return(NULL)
2221+
if (!length(missing_keys)) return(jvnames[key_idx])
2222+
jvnames[head(key_idx, missing_keys[1L] - 1L)]
2223+
}
2224+
2225+
# GForce functions that return all rows instead of one per group
2226+
.dt_is_nrows = function(q) {
2227+
nrow_funs = c("gshift")
2228+
if (!is.call(q)) return(FALSE)
2229+
if (q[[1L]] == "list") {
2230+
any(vapply(q, .dt_is_nrows, FALSE))
2231+
} else {
2232+
q[[1L]] %chin% nrow_funs
2233+
}
2234+
}
2235+
2236+
# n of ghead/gtail(x, n) when n > 1, otherwise 0, #5060 #523
2237+
.dt_headTail_arg = function(q) {
2238+
if (length(q)==3L && length(q3 <- q[[3L]])==1L && is.numeric(q3) &&
2239+
(q[[1L]]) %chin% c("ghead", "gtail") && q3!=1L) q3
2240+
else 0L
2241+
}
2242+
2243+
# unpack list of lists for nrows functions
2244+
.dt_zip_items = function(ll) do.call(mapply, c(list(FUN = c), ll, SIMPLIFY=FALSE, USE.NAMES=FALSE))
2245+
2246+
# unlock any locked data.table components of the answer, #4159
2247+
# max_depth prevents possible infinite recursion from truly recursive object, #4173
2248+
# TODO: is there an efficient way to get around this max_depth limit?
2249+
.dt_runlock = function(x, current_depth = 1L, max_depth = 5L) {
2250+
if (is.list(x) && current_depth <= max_depth) { # is.list() used to be is.recursive(), #4814
2251+
if (inherits(x, 'data.table')) .Call(C_unlock, x)
2252+
else return(lapply(x, .dt_runlock, current_depth = current_depth + 1L, max_depth = max_depth))
2253+
}
2254+
invisible()
2255+
}
2256+
22292257
# can the specified merge of x and i be marked as sorted? return the columns for which this is true, otherwise NULL
22302258
.join_result_key <- function(x, i, ans, on_lhs, ansvars, leftcols, rightcols, names_i, irows, roll) {
22312259
x_key <- key(x)

‎R/translation.R‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,11 @@ catf = function(fmt, ..., sep=" ", domain="R-data.table") {
55
}
66

77
raise_condition = function(signal, message, classes, immediate=FALSE, appendLF=FALSE) {
8-
obj = list(message=message, call=sys.call(sys.nframe()-2L))
8+
n = sys.nframe() - 2L # the caller of stopf() etc.
9+
# helpers extracted from [.data.table are named .dt_*; their conditions report the call of [.data.table (or whoever called them)
10+
parents = sys.parents()
11+
while (n > 0L && parents[n] > 0L && .is_dt_helper_call(sys.call(n))) n = parents[n]
12+
obj = list(message=message, call=sys.call(n))
913
# NB: append _after_ translation
1014
if (appendLF) obj$message = paste0(obj$message, "\n")
1115
setattr(obj, "class", classes)
@@ -17,6 +21,8 @@ raise_condition = function(signal, message, classes, immediate=FALSE, appendLF=F
1721
signal(obj)
1822
}
1923

24+
.is_dt_helper_call = function(call) is.call(call) && is.name(f <- call[[1L]]) && startsWith(as.character(f), ".dt_")
25+
2026
stopf = function(fmt, ..., class=NULL, domain="R-data.table") {
2127
raise_condition(stop, gettextf(fmt, ..., domain=domain), c(class, "simpleError", "error", "condition"))
2228
}

‎inst/tests/bracket.Rraw‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,9 @@ test(1.48, DT[, a, keyby=NULL], data.table(a=1:6))
8787
test(1.49, DT[, a, .SDcols="b"], 1:6, warning="This j doesn't use .SD but .SDcols has been supplied")
8888
test(1.50, DT[, .(a, )], error="Item 2 of the .() or list() passed to j is missing")
8989
test(1.51, DT[on="b"], DT, warning=c("When on= is provided but not i=", "i and j are both missing so ignoring the other arguments"))
90+
# raised in helper but reported as coming from [.data.table
91+
test(1.52, tryCatch(DT[, .(a, )], error=conditionCall)[[1L]], quote(`[.data.table`))
92+
test(1.53, tryCatch(DT[, if (.N > 1L) .(a) else .(z=a), by=b], warning=conditionCall)[[1L]], quote(`[.data.table`))
9093

9194
# forms of i
9295
test(1.61, DT[matrix(1:2, ncol=1L)], DT[1:2,])

0 commit comments

Comments
 (0)