[med-svn] [r-cran-data.table] 01/05: New upstream version 1.10.4
Andreas Tille
tille at debian.org
Thu Sep 7 07:07:58 UTC 2017
This is an automated email from the git hooks/post-receive script.
tille pushed a commit to branch master
in repository r-cran-data.table.
commit 46c23ec89b5c37d1fa581af430ccfeba399d3b20
Author: Andreas Tille <tille at debian.org>
Date: Thu Sep 7 09:00:48 2017 +0200
New upstream version 1.10.4
---
DESCRIPTION | 12 +-
MD5 | 68 +--
NEWS.md | 56 ++
R/data.table.R | 57 +-
R/fread.R | 3 +-
R/fwrite.R | 7 +-
R/onAttach.R | 3 +-
R/onLoad.R | 5 +-
R/setkey.R | 7 +-
R/utils.R | 9 +
README.md | 2 +-
inst/doc/datatable-faq.html | 14 +-
inst/doc/datatable-intro.html | 8 +-
inst/doc/datatable-keys-fast-subset.html | 10 +-
inst/doc/datatable-reference-semantics.html | 4 +-
inst/doc/datatable-reshape.html | 4 +-
...atable-secondary-indices-and-auto-indexing.html | 8 +-
inst/tests/tests.Rraw | 62 +-
man/assign.Rd | 2 +-
man/duplicated.Rd | 2 +-
man/fwrite.Rd | 4 +-
man/setkey.Rd | 11 +-
src/assign.c | 4 +-
src/bmerge.c | 14 +-
src/data.table.h | 10 +-
src/dogroups.c | 4 +-
src/fcast.c | 32 +-
src/forder.c | 4 +-
src/fread.c | 10 +-
src/fwrite.c | 639 ++++++++-------------
src/gsumm.c | 36 +-
src/init.c | 32 +-
src/rbindlist.c | 21 +-
src/subset.c | 13 +-
src/wrappers.c | 4 +-
35 files changed, 584 insertions(+), 597 deletions(-)
diff --git a/DESCRIPTION b/DESCRIPTION
index 3d0eae6..94949f2 100644
--- a/DESCRIPTION
+++ b/DESCRIPTION
@@ -1,5 +1,5 @@
Package: data.table
-Version: 1.10.0
+Version: 1.10.4
Title: Extension of `data.frame`
Authors at R: c(
person("Matt","Dowle", role=c("aut","cre"), email="mattjdowle at gmail.com"),
@@ -10,9 +10,9 @@ Authors at R: c(
person("Eduard","Antonyan", role="ctb") )
Depends: R (>= 3.0.0)
Imports: methods
-Suggests: bit64, knitr, chron, ggplot2 (>= 0.9.0), plyr, reshape,
- reshape2, testthat (>= 0.4), hexbin, fastmatch, nlme, xts,
- gdata, GenomicRanges, caret, curl, zoo, plm, rmarkdown,
+Suggests: bit64, knitr, nanotime, chron, ggplot2 (>= 0.9.0), plyr,
+ reshape, reshape2, testthat (>= 0.4), hexbin, fastmatch, nlme,
+ xts, gdata, GenomicRanges, caret, curl, zoo, plm, rmarkdown,
parallel
Description: Fast aggregation of large data (e.g. 100GB in RAM), fast ordered joins, fast add/modify/delete of columns by group using no copies at all, list columns, a fast friendly file reader and parallel file writer. Offers a natural and flexible syntax, for faster development.
License: GPL-3 | file LICENSE
@@ -22,7 +22,7 @@ MailingList: datatable-help at lists.r-forge.r-project.org
VignetteBuilder: knitr
ByteCompile: TRUE
NeedsCompilation: yes
-Packaged: 2016-12-02 19:25:45.346 UTC; mdowle
+Packaged: 2017-02-01 00:15:31.074 UTC; mdowle
Author: Matt Dowle [aut, cre],
Arun Srinivasan [aut],
Jan Gorecki [ctb],
@@ -31,4 +31,4 @@ Author: Matt Dowle [aut, cre],
Eduard Antonyan [ctb]
Maintainer: Matt Dowle <mattjdowle at gmail.com>
Repository: CRAN
-Date/Publication: 2016-12-03 11:05:23
+Date/Publication: 2017-02-01 15:52:19
diff --git a/MD5 b/MD5
index e9c3114..d1decf1 100644
--- a/MD5
+++ b/MD5
@@ -1,7 +1,7 @@
-8c3e3e59b79bc6ff0826e6cb7fd46526 *DESCRIPTION
+ae468059bb64612611a8899452d6acfb *DESCRIPTION
d32239bcb673463ab874e80d47fae504 *LICENSE
5feca43f299bbbd001563b1033a59705 *NAMESPACE
-33a8be67e6ca2155dbb2b451af19d7db *NEWS.md
+3dc8bdf9e48d766984dd260692d5408f *NEWS.md
4377e4b917c44366f66862324c8fd32a *R/AllS4.R
3d35eb16da4271f72a9118200db76d65 *R/IDateTime.R
55b63a5d831071ec745ce7bd4ab4a336 *R/as.data.table.R
@@ -9,22 +9,22 @@ ad10acb3d99aab6ccf9dc37bb836d817 *R/between.R
9052f694cdd5de844433349a91f3bc7a *R/bmerge.R
0041b7a70824ed486983cf52ff902aaa *R/c.factor.R
b28ccef4d6291066b742fd3a3a5f5d39 *R/cedta.R
-3c6276faf6ce4d69984c537b4ec21f63 *R/data.table.R
+f370c168081fb7efddc89d88c5afb872 *R/data.table.R
4e672da0bdc96542f0713ba1da6bde69 *R/duplicated.R
8fdd31e066db641fe2da803bd2e20268 *R/fcast.R
f758580ab081803256534bd9ee22e250 *R/fmelt.R
8b4c18a8d933cc3a92dab206aec20e60 *R/foverlaps.R
ff31822b594f68c126528ff50f21897a *R/frank.R
-9c97a879ed5969be61b1d505abc40b96 *R/fread.R
-6d9b80bfa81f1cf0a7e204db48f0f5b4 *R/fwrite.R
+399312e5fdec2f6e0849275403159d9f *R/fread.R
+cd2b459de37c217870b1ada4d197d1c4 *R/fwrite.R
50f35848d63ae310c3b4a448bb170097 *R/getdots.R
9e49976bdb1d7b41a755a588fd9eb05b *R/last.R
a9a8ec624e5afd3067d8ea9cc33482f6 *R/like.R
8cace748741791eac2d81501c02a1488 *R/merge.R
-5a8e73bd84c6460446b9e43c2966e5f7 *R/onAttach.R
-e1b5de66ef3e976962065e029655f709 *R/onLoad.R
+9a127e9279e88bebc231dd038a62f537 *R/onAttach.R
+0354731d8708c60ba8bcc167c6428829 *R/onLoad.R
a00ff6edb3599fcf2801acfc12c4a7e4 *R/openmp-utils.R
-697ce1fba7cf18681bb3f58076bd32cc *R/setkey.R
+4cbd6f0cf26f1a7ecc7b2bd2fb5df8bb *R/setkey.R
b8d3020884ea456d3eeeecf5b06323ef *R/setops.R
b9a144d533820eb7529e86a9b794e661 *R/shift.R
1432e4ed7a2a3d6be052527b9005cd8d *R/tables.R
@@ -32,28 +32,28 @@ f8b4fc4d8738808c6a8540e290324a83 *R/test.data.table.R
20aa38be566f1dc0f21a8937944481c5 *R/timetaken.R
5ced66446db69ebb73eba28399336922 *R/transpose.R
085cbece46eb9e4689189ac6cda84457 *R/uniqlist.R
-fda476f057d8bc9cdd88f97265d4d162 *R/utils.R
+5d0affa6ab449271b8d6d076526ac75f *R/utils.R
8ff82535188f42a368314fddf610483a *R/xts.R
-dc6551c1e43c07a784519699e51586ae *README.md
+399a1d96153aa5721e8544fa53084c9b *README.md
52d0eaab5376b90d6f2e7e20e90e0e2a *build/vignette.rds
d20a5d50c2a2fae35659da9de05acea3 *inst/doc/datatable-faq.R
0b13b6d61aa41e7908a871653f63c755 *inst/doc/datatable-faq.Rmd
-929d8d437e0e91f721a958de1f7f4e17 *inst/doc/datatable-faq.html
+9092d0c955a3b47273c5b883f856a90b *inst/doc/datatable-faq.html
ea6aff9f9dc4f5f26a8cee86ba0e8393 *inst/doc/datatable-intro.R
e9fff1a46fdf96e3572b583bc89e8f86 *inst/doc/datatable-intro.Rmd
-ff2c63b3c8aa6b7b0f8a9069ce937e94 *inst/doc/datatable-intro.html
+2cc922bca1adcdce0f112d200c9e6e4a *inst/doc/datatable-intro.html
c2f4d1dc6234576bf0c1f071325d5b1d *inst/doc/datatable-keys-fast-subset.R
3f2980389baaff06c2d6b401b26d71bf *inst/doc/datatable-keys-fast-subset.Rmd
-937a425071ae4728cc6fada95ef75ecf *inst/doc/datatable-keys-fast-subset.html
+f77edced904971a84d23a9593746856b *inst/doc/datatable-keys-fast-subset.html
723df81331669d44c4cab1f541a3d956 *inst/doc/datatable-reference-semantics.R
531acab6260b82f65ab9048aee6fb331 *inst/doc/datatable-reference-semantics.Rmd
-80616e790d763e60b5592054f6976207 *inst/doc/datatable-reference-semantics.html
+1d2f06ba982c95bc983b51164a72794c *inst/doc/datatable-reference-semantics.html
7149288c1c45ff4e6dc0c89b71f81f72 *inst/doc/datatable-reshape.R
e8ef65c1d8424e390059b854cb18740e *inst/doc/datatable-reshape.Rmd
-eabf46e6fa6f72a90fb29ba97d7af33a *inst/doc/datatable-reshape.html
+bfca7571d9aba8825edf6565e02fadcf *inst/doc/datatable-reshape.html
22265ade65535db347b44213d4354772 *inst/doc/datatable-secondary-indices-and-auto-indexing.R
bcdc8c1716a1e3aa1ef831bad0d67715 *inst/doc/datatable-secondary-indices-and-auto-indexing.Rmd
-4ddddfdef09aeec8d1be3a2b043357a8 *inst/doc/datatable-secondary-indices-and-auto-indexing.html
+99d25b646960c7b3cf9d6799c9c7d7e0 *inst/doc/datatable-secondary-indices-and-auto-indexing.html
e48efd4babf364e97ff98e56b1980c8b *inst/tests/1206FUT.txt
28b57d31f67353c1192c6f65d69a12b1 *inst/tests/1680-fread-header-encoding.csv
fe198c1178f7db508ee0b10a94272e7e *inst/tests/2008head.csv
@@ -84,7 +84,7 @@ d5f1a4914ee94df080c787a3dcc530e3 *inst/tests/issue_785_fread.txt
0738c8cabf507aecd5b004fbbc45c2b4 *inst/tests/quoted_multiline.csv
278198d16e23ea87b8be7822d0af26e3 *inst/tests/russellCRCRLF.csv
2d8e8b64b59f727ae5bf5c527948be6a *inst/tests/russellCRLF.csv
-8befb61685660b818dfe830dba387af5 *inst/tests/tests.Rraw
+aa9ca3a13c9a76ee9eb18ad7afa00600 *inst/tests/tests.Rraw
9f0dd826cb0518ead716be2e932d4def *man/IDateTime.Rd
9cbcd078347c5a08402df1ba73aa44b5 *man/J.Rd
835019f8d7596f0e2f6482b725b10cbf *man/address.Rd
@@ -92,7 +92,7 @@ cba1f36488291b0fbfa133edf0b061c7 *man/all.equal.data.table.Rd
fcee579c860188bed47a04336af9f9c0 *man/as.data.table.Rd
ea442e905301b4305b5c0cdc5a7aa117 *man/as.data.table.xts.Rd
313e6cb9f227a39ff91a73b2a73f1674 *man/as.xts.data.table.Rd
-ba628c186b54d45d15bc4d6276edf731 *man/assign.Rd
+7e6ba73f8dfcc502ca60c017c5113a01 *man/assign.Rd
262b9b6611f9347776ec21b60cc654ad *man/between.Rd
e22276a8d899dc4ffb5f24cdc7d5a6ac *man/chmatch.Rd
3beb57267468fd528548ce3210506bf2 *man/copy.Rd
@@ -100,13 +100,13 @@ e22276a8d899dc4ffb5f24cdc7d5a6ac *man/chmatch.Rd
814bfbf9ec9e3f7d9fc214b3d337a87b *man/data.table.Rd
6a5dcb5da93a2cd9be8edb77b0aab182 *man/datatable-optimize.Rd
0f95b7876a3e66740fce419eba57ba46 *man/dcast.data.table.Rd
-75ef60fdf0a53a3f70eacfa67a094933 *man/duplicated.Rd
+61fa25c0cb40fb5ada455467f407c456 *man/duplicated.Rd
c313e9f95feafc04c75e7526105c9d8c *man/first.Rd
d073c3f067c5113a93cc91487b8b8d13 *man/foverlaps.Rd
14b89aaa05e22182e1eb28a82c125088 *man/frank.Rd
f0581f51c048bb08d7ea87a4b19cf62f *man/fread.Rd
f3ad1b3fee28abbb0010cf8e7ca526d2 *man/fsort.Rd
-d151acecbb8e4045eb7b5e47910797fd *man/fwrite.Rd
+f2209a520fc5e59a72a8255e1eb919e6 *man/fwrite.Rd
da2b630f6e90002b4e3c820185874d9a *man/last.Rd
eef491c35003ceefb9c522d1c53a3386 *man/like.Rd
330e05945e94fedd232564509fda1d3a *man/melt.data.table.Rd
@@ -123,7 +123,7 @@ cfd8e9966df5c3f856c80d2eac2ce225 *man/setDT.Rd
5dd46328a97acd449b00b72ade0f7e08 *man/setNumericRounding.Rd
e1f96224db1469216c98abe8ac01ceaa *man/setattr.Rd
5b9d6ef8696a28a89e21fb1db01fa0c3 *man/setcolorder.Rd
-856e0aaec6aa12d657211ad2cf8cec7c *man/setkey.Rd
+8c287e0253fbdddbfcdebd9dd5817cd4 *man/setkey.Rd
2b077a22ecf6b9ff2aa289f571531f3b *man/setops.Rd
f55745cbb4d5d4e72f89c7dbf4813d27 *man/setorder.Rd
3643b5e199ac96ca4223f53d7fbf1a8f *man/shift.Rd
@@ -139,35 +139,35 @@ a2a4eb530691106fb68f4851182b66fb *man/transpose.Rd
4d71404f8c7d5b3a9aa93195c22e8f97 *man/truelength.Rd
142dc7a616c9c546ffb9f0bea81cc2d7 *man/tstrsplit.Rd
7140c94fe69f182d6c111a8d40795e4f *src/Makevars
-ddf75bd43a58be15256ee2ea99e8c2e1 *src/assign.c
+52bc93d41bad403ca8fcc4f7878d4934 *src/assign.c
059a1ee027698e97d5757cdc6493ba2d *src/between.c
-5668430c2b9d513b544cfa6c9d93b664 *src/bmerge.c
+12239da57bc1754b2345486e48fe1d2b *src/bmerge.c
771f833144f4bcd530d36944c9ece29f *src/chmatch.c
-8c6af4083df6e5193d6c18e2f2c679b4 *src/data.table.h
-659f0e6641f909726724915818a6a2f2 *src/dogroups.c
+8977546957face8344131b9eaf04da32 *src/data.table.h
+530e5eb6ef7580c12385c9a964f5e7de *src/dogroups.c
e3d5b289ef4b511e4f89c9838b86cbd0 *src/fastmean.c
-b14b7f112e32f64e2389b4f6176783c6 *src/fcast.c
+87d693c023ca8935be4f1f4aa8aa979e *src/fcast.c
15f33d9fd7a31c7f288019c0d667f003 *src/fmelt.c
-9d4344ed7c33cc5e79a91c262f5af38e *src/forder.c
+5682cc0d46990d81a83ed0d540b31acf *src/forder.c
4cace1380fb006a5c786e9a7f4d12937 *src/frank.c
-f249407a5e5b24e1d1e9e5e29d41895c *src/fread.c
+dae009c0fbb1f7f1bb028d756417f406 *src/fread.c
8cc10403f23c6e018f26b0220b509a86 *src/fsort.c
-4eb633083629e9f2df7431000100418a *src/fwrite.c
+9d057220f1278fd15061b81dbb684d2e *src/fwrite.c
237a455e5212cf3358ab5aaca12fbd9c *src/fwriteLookups.h
-4ddc11c18a0ef9ca3153c64ba9ef9f68 *src/gsumm.c
+e7aae63b27c01a5acce45023ff436b69 *src/gsumm.c
47792eafb3cee1c03bbcb972d00c4aad *src/ijoin.c
-c286736030104e7222bf17b7b1d91a9f *src/init.c
+6d60952300a7cb70ad844aa06f3d2edd *src/init.c
520938944d8dbd58460bcf4ca44e9479 *src/inrange.c
a1237adc2f1ced2e4e9e3724a9434211 *src/openmp-utils.c
ab561ed83137b5b2d78d5d06030f7446 *src/quickselect.c
-9075a14cce153c75374d798fb1ea4bde *src/rbindlist.c
+7081147bbf13dd8d85c29a58a4a66e55 *src/rbindlist.c
416562e57a9368398d026ec1edc96313 *src/reorder.c
43566a73264aab49b4f4fb9ffcf77c0b *src/shift.c
-422ef9d84f83359beb206ffe087ef89c *src/subset.c
+eb145cdab68c4ea5d0285ffa8dcb7bb4 *src/subset.c
53304fe0233e11f15786cdbddf6c89f8 *src/transpose.c
cfe85d1626ba795d8edc0f098c1b7f12 *src/uniqlist.c
75a359ce5e8c6927d46dd9b2aa169da1 *src/vecseq.c
-edb7d033ad8f381276ef9600ec9ff8da *src/wrappers.c
+8fdbfefae387548d671c637eb28694bb *src/wrappers.c
441a393fe285e88a86e1126af8d6d7d8 *tests/autoprint.R
1ad409241d679d234e1a56ef06507e64 *tests/autoprint.Rout.save
5b9fc0d7c7ea64a9b1f60f9eba66327e *tests/knitr.R
diff --git a/NEWS.md b/NEWS.md
index 5e8cf64..9291dd2 100644
--- a/NEWS.md
+++ b/NEWS.md
@@ -1,4 +1,60 @@
+### Changes in v1.10.4 (on CRAN 01 Feb 2017)
+
+#### BUG FIXES
+
+1. The new specialized `nanotime` writer in `fwrite()` type punned using `*(long long *)&REAL(column)[i]` which, strictly, is undefined behavour under C standards. It passed a plethora of tests on linux (gcc 5.4 and clang 3.8), win-builder and 6 out 10 CRAN flavours using gcc. But failed (wrong data written) with the newest version of clang (3.9.1) as used by CRAN on the failing flavors, and solaris-sparc. Replaced with the union method and added a grep to CRAN_Release.cmd.
+
+
+### Changes in v1.10.2 (on CRAN 31 Jan 2017)
+
+#### NEW FEATURES
+
+1. When `j` is a symbol prefixed with `..` it will be looked up in calling scope and its value taken to be column names or numbers.
+ ```R
+ myCols = c("colA","colB")
+ DT[, myCols, with=FALSE]
+ DT[, ..myCols] # same
+ ```
+ When you see the `..` prefix think _one-level-up_ like the directory `..` in all operating systems meaning the parent directory. In future the `..` prefix could be made to work on all symbols apearing anywhere inside `DT[...]`. It is intended to be a convenient way to protect your code from accidentally picking up a column name. Similar to how `x.` and `i.` prefixes (analogous to SQL table aliases) can already be used to disambiguate the same column name present in both `x` and `i`. A [...]
+
+2. When `fread()` or `print()` see `integer64` columns are present, `bit64`'s namespace is now automatically loaded for convenience.
+
+3. `fwrite()` now supports the new [`nanotime`](https://cran.r-project.org/package=nanotime) type by Dirk Eddelbuettel, [#1982](https://github.com/Rdatatable/data.table/issues/1982). Aside: `data.table` already automatically supported `nanotime` in grouping and joining operations via longstanding support of its underlying `integer64` type.
+
+4. `indices()` gains a new argument `vectors`, default `FALSE`. This strsplits the index names by `__` for you, [#1589](https://github.com/Rdatatable/data.table/issues/1589).
+ ```R
+ DT = data.table(A=1:3, B=6:4)
+ setindex(DT, B)
+ setindex(DT, B, A)
+ indices(DT)
+ [1] "B" "B__A"
+ indices(DT, vectors=TRUE)
+ [[1]]
+ [1] "B"
+ [[2]]
+ [1] "B" "A"
+ ```
+
+#### BUG FIXES
+
+1. Some long-standing potential instability has been discovered and resolved many thanks to a detailed report from Bill Dunlap and Michael Sannella. At C level any call of the form `setAttrib(x, install(), allocVector())` can be unstable in any R package. Despite `setAttrib()` PROTECTing its inputs, the 3rd argument (`allocVector`) can be executed first only for its result to to be released by `install()`'s potential GC before reaching `setAttrib`'s PROTECTion of its inputs. Fixed by eit [...]
+
+2. A new potential instability introduced in the last release (v1.10.0) in GForce optimized grouping has been fixed by reverting one change from malloc to R_alloc. Thanks again to Michael Sannella for the detailed report.
+
+3. `fwrite()` could write floating point values incorrectly, [#1968](https://github.com/Rdatatable/data.table/issues/1968). A thread-local variable was incorrectly thread-global. This variable's usage lifetime is only a few clock cycles so it needed large data and many threads for several threads to overlap their usage of it and cause the problem. Many thanks to @mgahan and @jmosser for finding and reporting.
+
+#### NOTES
+
+1. `fwrite()`'s `..turbo` option has been removed as the warning message warned. If you've found a problem, please [report it](https://github.com/Rdatatable/data.table/issues).
+
+2. No known issues have arisen due to `DT[,1]` and `DT[,c("colA","colB")]` now returning columns as introduced in v1.9.8. However, as we've moved forward by setting `options('datatable.WhenJisSymbolThenCallingScope'=TRUE)` introduced then too, it has become clear a better solution is needed. All 340 CRAN and Bioconductor packages that use data.table have been checked with this option on. 331 lines would need to be changed in 59 packages. Their usage is elegant, correct and recommended, t [...]
+`options('datatable.WhenJisSymbolThenCallingScope')` is now removed. A migration timeline is no longer needed. The new strategy needs no code changes and has no breakage. It was proposed and discussed in point 2 [here](https://github.com/Rdatatable/data.table/issues/1188#issuecomment-127824969), as follows.<br>
+When `j` is a symbol (as in the quanteda and xgboost examples above) it will continue to be looked up as a column name and returned as a vector, as has always been the case. If it's not a column name however, it is now a helpful error explaining that data.table is different to data.frame and what to do instead (use `..` prefix or `with=FALSE`). The old behaviour of returning the symbol's value in calling scope can never have been useful to anybody and therefore not depended on. Just as [...]
+
+3. As before, and as we can see is in common use in CRAN and Bioconductor packages using data.table, `DT[,myCols,with=FALSE]` continues to lookup `myCols` in calling scope and take its value as column names or numbers. You can move to the new experimental convenience feature `DT[, ..myCols]` if you wish at leisure.
+
+
### Changes in v1.10.0 (on CRAN 3 Dec 2016)
#### BUG FIXES
diff --git a/R/data.table.R b/R/data.table.R
index 97da35e..1effe44 100644
--- a/R/data.table.R
+++ b/R/data.table.R
@@ -72,10 +72,8 @@ print.data.table <- function(x, topn=getOption("datatable.print.topn"),
printdots = FALSE
}
toprint=format.data.table(toprint, ...)
- # fix for #975.
- if (any(sapply(x, function(col) "integer64" %in% class(col))) && !"package:bit64" %in% search()) {
- warning("Some columns have been read as type 'integer64' but package bit64 isn't loaded. Those columns will display as strange looking floating point data. There is no need to reload the data. Just require(bit64) to obtain the integer64 print method and print the data again.")
- }
+ if ((!"bit64" %chin% loadedNamespaces()) && any(sapply(x,inherits,"integer64"))) require_bit64()
+ # When we depend on R 3.2.0 (Apr 2015) we can use isNamespaceLoaded() added then, instead of %chin% above
# FR #5020 - add row.names = logical argument to print.data.table
if (isTRUE(row.names)) rownames(toprint)=paste(format(rn,right=TRUE,scientific=FALSE),":",sep="") else rownames(toprint)=rep.int("", nrow(toprint))
if (is.null(names(x)) | all(names(x) == "")) colnames(toprint)=rep("", ncol(toprint)) # fixes bug #97 (RF#4934) and #545 (RF#5253)
@@ -399,34 +397,33 @@ chmatch2 <- function(x, table, nomatch=NA_integer_) {
( !length(all.vars(jsub)) &&
root %chin% c("","c","paste","paste0","-","!") &&
missing(by) )) { # test 763. TODO: likely that !missing(by) iff with==TRUE (so, with can be removed)
- # When no variable names occur in j, scope doesn't matter because there are no symbols to find.
- # Auto set with=FALSE in this case so that DT[,1], DT[,2:3], DT[,"someCol"] and DT[,c("colB","colD")]
+ # When no variable names (i.e. symbols) occur in j, scope doesn't matter because there are no symbols to find.
+ # Automatically set with=FALSE in this case so that DT[,1], DT[,2:3], DT[,"someCol"] and DT[,c("colB","colD")]
# work as expected. As before, a vector will never be returned, but a single column data.table
# for type consistency with >1 cases. To return a single vector use DT[["someCol"]] or DT[[3]].
- # The root==":" is to allow DT[,colC:colH] even though that contains two variable names
- # root either "-" or "!" is for tests 1504.11 and 1504.13 (a : with a ! or - modifier root)
+ # The root==":" is to allow DT[,colC:colH] even though that contains two variable names.
+ # root == "-" or "!" is for tests 1504.11 and 1504.13 (a : with a ! or - modifier root)
# This change won't break anything because it didn't do anything anyway; i.e. used to return the
- # j value straight back: DT[,1] == 1 which isn't possibly useful. It was that was for consistency
- # of learning, since it was simpler to state that j always gets eval'd within the scope of DT.
+ # j value straight back: DT[,1] == 1 which isn't possibly useful. It was that for consistency
+ # of learning since it was simpler to state that j always gets eval'd within the scope of DT.
# We don't want to evaluate j at all in making this decision because i) evaluating itself could
# increment some variable and not intended to be evaluated a 2nd time later on and ii) we don't
# want decisions like this to depend on the data or vector lengths since that can introduce
# inconistency reminiscent of drop in [.data.table that we seek to avoid.
- #if (verbose) cat("Auto with=FALSE because j
with=FALSE
- } else if (is.name(jsub) && isTRUE(getOption("datatable.WhenJisSymbolThenCallingScope"))) {
- # Allow future behaviour to be turned on. Current default is FALSE.
- # Use DT[["someCol"]] or DT$someCol to fetch that column as vector, regardless of this option.
- if (!missingwith && isTRUE(with)) {
- # unusual edge case only when future option has been turned on
- stop('j is a single symbol, WhenJisSymbol is turned on but with=TRUE has been passed explicitly. Please instead use DT[,"someVar"], DT[,.(someVar)] or DT[["someVar"]]')
- } else {
- with=FALSE
- }
+ } else if (is.name(jsub)) {
jsubChar = as.character(jsub)
- if (!exists(jsubChar, where=parent.frame()) && jsubChar %chin% names(x)) {
- # Would fail anyway with 'object 'a' not found' but give a more helpful error. Thanks to Jan's suggestion.
- stop("The option 'datatable.WhenJisSymbolThenCallingScope' is TRUE so looking for the variable '", jsubChar, "' in calling scope but it is not found there. It is a column name though. So, most likely, please wrap with quotes (i.e. DT[,'", jsubChar, "']) to return a 1-column data.table or if you need the column as a plain vector then DT[['",jsubChar,"']] or DT$",jsubChar)
+ if (substring(jsubChar,1,2) == "..") {
+ if (nchar(jsubChar)==2) stop("The symbol .. is invalid. The .. prefix must be followed by at least one character.")
+ if (!exists(jsubChar, where=parent.frame())) {
+ # We have recommended manual ".." prefix in the past so that needs to keep working and take precedence
+ jsub = as.name(jsubChar<-substring(jsubChar,3))
+ }
+ with = FALSE
+ }
+ if (!with && !exists(jsubChar, where=parent.frame())) {
+ # Would fail anyway with 'object 'a' not found' but give a more helpful error. Thanks to Jan's suggestion.
+ stop("Variable '",jsubChar,"' is not found in calling scope. Looking in calling scope because either you used the .. prefix or set with=FALSE")
}
}
if (root=="{") {
@@ -1434,10 +1431,14 @@ chmatch2 <- function(x, table, nomatch=NA_integer_) {
# Since .SD is inside SDenv, alongside its columns as variables, R finds .SD symbol more quickly, if used.
# There isn't a copy of the columns here, the xvar symbols point to the SD columns (copy-on-write).
- # Temp fix for #921 - check address and copy *after* evaluating 'jval'
+ if (is.name(jsub) && is.null(lhs) && !exists(jsubChar<-as.character(jsub), SDenv, inherits=FALSE)) {
+ stop("j (the 2nd argument inside [...]) is a single symbol but column name '",jsubChar,"' is not found. Perhaps you intended DT[,..",jsubChar,"] or DT[,",jsubChar,",with=FALSE]. This difference to data.frame is deliberate and explained in FAQ 1.1.")
+ }
+
jval = eval(jsub, SDenv, parent.frame())
# copy 'jval' when required
# More speedup - only check + copy if irows is NULL
+ # Temp fix for #921 - check address and copy *after* evaluating 'jval'
if (is.null(irows)) {
if (!is.list(jval)) { # performance improvement when i-arg is S4, but not list, #1438, Thanks @DCEmilberg.
jcpy = address(jval) %in% sapply(SDenv$.SD, address) # %chin% errors when RHS is list()
@@ -1878,10 +1879,7 @@ chmatch2 <- function(x, table, nomatch=NA_integer_) {
assign(".N", len__, thisEnv) # For #5760
#fix for #1683
if (use.I) assign(".I", seq_len(nrow(x)), thisEnv)
- gstart(o__, f__, len__, irows) # irows needed for #971.
- ans = eval(jsub, thisEnv)
- if (is.atomic(ans)) ans=list(ans) # won't copy named argument in new version of R, good
- gend()
+ ans = gforce(thisEnv, jsub, o__, f__, len__, irows) # irows needed for #971.
gi = if (length(o__)) o__[f__] else f__
g = lapply(grpcols, function(i) groups[[i]][gi])
ans = c(g, ans)
@@ -2860,8 +2858,7 @@ gmin <- function(x, na.rm=FALSE) .Call(Cgmin, x, na.rm)
gmax <- function(x, na.rm=FALSE) .Call(Cgmax, x, na.rm)
gvar <- function(x, na.rm=FALSE) .Call(Cgvar, x, na.rm)
gsd <- function(x, na.rm=FALSE) .Call(Cgsd, x, na.rm)
-gstart <- function(o, f, l, rows) .Call(Cgstart, o, f, l, rows)
-gend <- function() .Call(Cgend)
+gforce <- function(env, jsub, o, f, l, rows) .Call(Cgforce, env, jsub, o, f, l, rows)
isReallyReal <- function(x) {
.Call(CisReallyReal, x)
diff --git a/R/fread.R b/R/fread.R
index 8efb633..91cb5d4 100644
--- a/R/fread.R
+++ b/R/fread.R
@@ -99,8 +99,7 @@ fread <- function(input="",sep="auto",sep2="auto",nrows=-1L,header="auto",na.str
if (is.atomic(colClasses) && !is.null(names(colClasses))) colClasses = tapply(names(colClasses),colClasses,c,simplify=FALSE) # named vector handling
ans = .Call(Creadfile,input,sep,as.integer(nrows),header,na.strings,verbose,as.integer(autostart),skip,select,drop,colClasses,integer64,dec,encoding,quote,strip.white,blank.lines.skip,fill,showProgress)
nr = length(ans[[1]])
- if ( integer64=="integer64" && !exists("print.integer64") && any(sapply(ans,inherits,"integer64")) )
- warning("Some columns have been read as type 'integer64' but package bit64 isn't loaded. Those columns will display as strange looking floating point data. There is no need to reload the data. Just require(bit64) to obtain the integer64 print method and print the data again.")
+ if ((!"bit64" %chin% loadedNamespaces()) && any(sapply(ans,inherits,"integer64"))) require_bit64()
setattr(ans,"row.names",.set_row_names(nr))
if (isTRUE(data.table)) {
diff --git a/R/fwrite.R b/R/fwrite.R
index 90e2ffc..52a9f89 100644
--- a/R/fwrite.R
+++ b/R/fwrite.R
@@ -5,8 +5,7 @@ fwrite <- function(x, file="", append=FALSE, quote="auto",
logicalAsInt=FALSE, dateTimeAs = c("ISO","squash","epoch","write.csv"),
buffMB=8, nThread=getDTthreads(),
showProgress = getOption("datatable.showProgress"),
- verbose = getOption("datatable.verbose"),
- ..turbo=TRUE) {
+ verbose = getOption("datatable.verbose")) {
isLOGICAL = function(x) isTRUE(x) || identical(FALSE, x) # it seems there is no isFALSE in R?
na = as.character(na[1L]) # fix for #1725
if (missing(qmethod)) qmethod = qmethod[1L]
@@ -29,7 +28,6 @@ fwrite <- function(x, file="", append=FALSE, quote="auto",
isLOGICAL(col.names), isLOGICAL(append), isLOGICAL(row.names),
isLOGICAL(verbose), isLOGICAL(showProgress), isLOGICAL(logicalAsInt),
length(na) == 1L, #1725, handles NULL or character(0) input
- isLOGICAL(..turbo),
is.character(file) && length(file)==1 && !is.na(file),
length(buffMB)==1 && !is.na(buffMB) && 1<=buffMB && buffMB<=1024,
length(nThread)==1 && !is.na(nThread) && nThread>=1
@@ -37,7 +35,6 @@ fwrite <- function(x, file="", append=FALSE, quote="auto",
file <- path.expand(file) # "~/foo/bar"
if (append && missing(col.names) && (file=="" || file.exists(file)))
col.names = FALSE # test 1658.16 checks this
- if (!..turbo) warning("The ..turbo=FALSE option will be removed in future. Please report any problems with ..turbo=TRUE.")
if (identical(quote,"auto")) quote=NA # logical NA
if (file=="") {
# console output (Rprintf) isn't thread safe.
@@ -48,7 +45,7 @@ fwrite <- function(x, file="", append=FALSE, quote="auto",
.Call(Cwritefile, x, file, sep, sep2, eol, na, dec, quote, qmethod=="escape", append,
row.names, col.names, logicalAsInt, dateTimeAs, buffMB, nThread,
- showProgress, verbose, ..turbo)
+ showProgress, verbose)
invisible()
}
diff --git a/R/onAttach.R b/R/onAttach.R
index ae2d481..3d91493 100644
--- a/R/onAttach.R
+++ b/R/onAttach.R
@@ -17,10 +17,11 @@
if (dev && (Sys.Date() - as.Date(d))>28)
packageStartupMessage("**********\nThis development version of data.table was built more than 4 weeks ago. Please update.\n**********")
if (!.Call(ChasOpenMP))
- packageStartupMessage("**********\nThis installation of data.table has not detected OpenMP support. It will still work but in single-threaded mode.\n**********")
+ packageStartupMessage("**********\nThis installation of data.table has not detected OpenMP support. It will still work but in single-threaded mode. If this a Mac and you obtained the Mac binary of data.table from CRAN, CRAN's Mac does not yet support OpenMP. In the meantime please follow our Mac installation instructions on the data.table homepage. If it works and you observe benefits from multiple threads as others have reported, please convince Simon Ubanek by sending him evide [...]
packageStartupMessage(' The fastest way to learn (by data.table authors): https://www.datacamp.com/courses/data-analysis-the-data-table-way')
packageStartupMessage(' Documentation: ?data.table, example(data.table) and browseVignettes("data.table")')
packageStartupMessage(' Release notes, videos and slides: http://r-datatable.com')
}
}
+
diff --git a/R/onLoad.R b/R/onLoad.R
index 3f7dc08..e67a4e7 100644
--- a/R/onLoad.R
+++ b/R/onLoad.R
@@ -45,9 +45,8 @@
"datatable.fread.datatable"="TRUE",
"datatable.fread.dec.experiment"="TRUE", # temp. will remove once stable
"datatable.fread.dec.locale"=if (.Platform$OS.type=="unix") "'fr_FR.utf8'" else "'French_France.1252'",
- "datatable.prettyprint.char" = NULL, # FR #1091
- "datatable.old.unique.by.key" = "FALSE", # TODO: warn 1 year, remove after 2 years
- "datatable.WhenJisSymbolThenCallingScope" = "FALSE" # TODO: warn (asking user to change to DT[,"someCol"] or DT[["someCol"]], then change default, then remove.
+ "datatable.prettyprint.char" = NULL, # FR #1091
+ "datatable.old.unique.by.key" = "FALSE" # TODO: warn 1 year, remove after 2 years
)
for (i in setdiff(names(opts),names(options()))) {
eval(parse(text=paste("options(",i,"=",opts[i],")",sep="")))
diff --git a/R/setkey.R b/R/setkey.R
index f229511..c13a88f 100644
--- a/R/setkey.R
+++ b/R/setkey.R
@@ -86,10 +86,13 @@ key2 <- function(x) {
if (is.null(ans)) return(ans) # otherwise character() gets returned by next line
gsub("^__","",ans)
}
-indices <- function(x) {
+indices <- function(x, vectors = FALSE) {
ans = names(attributes(attr(x,"index",exact=TRUE)))
if (is.null(ans)) return(ans) # otherwise character() gets returned by next line
- gsub("^__","",ans)
+ ans <- gsub("^__","",ans)
+ if (isTRUE(vectors))
+ ans <- strsplit(ans, "__", fixed = TRUE)
+ ans
}
get2key <- function(x, col) attr(attr(x,"index",exact=TRUE),paste("__",col,sep=""),exact=TRUE) # work in progress, not yet exported
diff --git a/R/utils.R b/R/utils.R
index 0e4cbbd..07af702 100644
--- a/R/utils.R
+++ b/R/utils.R
@@ -47,3 +47,12 @@ UseMethod("%+%")
"%+%.default" <- function(x,y) paste(paste(x,collapse=","),paste(y,collapse=","),sep="")
# we often construct warning msgs with a msg followed by several items of a vector, so %+% is for convenience
+
+require_bit64 = function() {
+ # called in fread and print when they see integer64 columns are present
+ tt = try(requireNamespace("bit64",quietly=TRUE))
+ if (inherits(tt,"try-error"))
+ warning("Some columns are type 'integer64' but package bit64 is not installed. Those columns will print as strange looking floating point data. There is no need to reload the data. Simply install.packages('bit64') to obtain the integer64 print method and print the data again.")
+}
+
+
diff --git a/README.md b/README.md
index 3441f3f..7a4e128 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,4 @@
-### Project overview is on the GitHub Wiki tab, our [HOMEPAGE](https://github.com/Rdatatable/data.table/wiki)
+### Project overview is on the GitHub Wiki tab, our [HOMEPAGE](http://r-datatable.com)
diff --git a/inst/doc/datatable-faq.html b/inst/doc/datatable-faq.html
index 472b175..8d283c6 100644
--- a/inst/doc/datatable-faq.html
+++ b/inst/doc/datatable-faq.html
@@ -11,7 +11,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1">
-<meta name="date" content="2016-12-02" />
+<meta name="date" content="2017-01-31" />
<title>Frequently Asked Questions about data.table</title>
@@ -68,7 +68,7 @@ code > span.in { color: #60a0b0; font-weight: bold; font-style: italic; } /* Inf
<h1 class="title toc-ignore">Frequently Asked Questions about data.table</h1>
-<h4 class="date"><em>2016-12-02</em></h4>
+<h4 class="date"><em>2017-01-31</em></h4>
<div id="TOC">
@@ -727,11 +727,11 @@ DT[<span class="dv">2</span>, b :<span class="er">=</span><span class="st"> </sp
<span class="co"># [1] "integer"</span>
DT[ , b :<span class="er">=</span><span class="st"> </span><span class="kw">rnorm</span>(<span class="dv">5</span>)] <span class="co"># 'replace' integer column with a numeric column</span>
<span class="co"># a b</span>
-<span class="co"># 1: 1 0.3941705</span>
-<span class="co"># 2: 2 -0.5748240</span>
-<span class="co"># 3: 3 0.8167449</span>
-<span class="co"># 4: 4 -1.3435421</span>
-<span class="co"># 5: 5 -0.2106035</span></code></pre></div>
+<span class="co"># 1: 1 0.1552267</span>
+<span class="co"># 2: 2 1.0811211</span>
+<span class="co"># 3: 3 -0.2778669</span>
+<span class="co"># 4: 4 -0.7133567</span>
+<span class="co"># 5: 5 0.4300428</span></code></pre></div>
</div>
<div id="reading-data.table-from-rds-or-rdata-file" class="section level2">
<h2><span class="header-section-number">5.3</span> Reading data.table from RDS or RData file</h2>
diff --git a/inst/doc/datatable-intro.html b/inst/doc/datatable-intro.html
index c928e04..aab9cc9 100644
--- a/inst/doc/datatable-intro.html
+++ b/inst/doc/datatable-intro.html
@@ -11,7 +11,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1">
-<meta name="date" content="2016-12-02" />
+<meta name="date" content="2017-01-31" />
<title>Introduction to data.table</title>
@@ -68,7 +68,7 @@ code > span.in { color: #60a0b0; font-weight: bold; font-style: italic; } /* Inf
<h1 class="title toc-ignore">Introduction to data.table</h1>
-<h4 class="date"><em>2016-12-02</em></h4>
+<h4 class="date"><em>2017-01-31</em></h4>
@@ -217,10 +217,10 @@ ans
<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">odt =<span class="st"> </span><span class="kw">data.table</span>(<span class="dt">col =</span> <span class="kw">sample</span>(<span class="fl">1e7</span>))
(t1 <-<span class="st"> </span><span class="kw">system.time</span>(ans1 <-<span class="st"> </span>odt[base::<span class="kw">order</span>(col)])) ## uses order from base R
<span class="co"># user system elapsed </span>
-<span class="co"># 0.640 0.004 0.645</span>
+<span class="co"># 0.312 0.008 0.320</span>
(t2 <-<span class="st"> </span><span class="kw">system.time</span>(ans2 <-<span class="st"> </span>odt[<span class="kw">order</span>(col)])) ## uses data.table's forder
<span class="co"># user system elapsed </span>
-<span class="co"># 0.648 0.004 0.651</span>
+<span class="co"># 0.336 0.008 0.345</span>
(<span class="kw">identical</span>(ans1, ans2))
<span class="co"># [1] TRUE</span></code></pre></div></li>
</ul>
diff --git a/inst/doc/datatable-keys-fast-subset.html b/inst/doc/datatable-keys-fast-subset.html
index 2c2f003..4b90d19 100644
--- a/inst/doc/datatable-keys-fast-subset.html
+++ b/inst/doc/datatable-keys-fast-subset.html
@@ -11,7 +11,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1">
-<meta name="date" content="2016-12-02" />
+<meta name="date" content="2017-01-31" />
<title>Keys and fast binary search based subset</title>
@@ -68,7 +68,7 @@ code > span.in { color: #60a0b0; font-weight: bold; font-style: italic; } /* Inf
<h1 class="title toc-ignore">Keys and fast binary search based subset</h1>
-<h4 class="date"><em>2016-12-02</em></h4>
+<h4 class="date"><em>2017-01-31</em></h4>
@@ -574,7 +574,7 @@ DT =<span class="st"> </span><span class="kw">data.table</span>(<span class="dt"
t1 <-<span class="st"> </span><span class="kw">system.time</span>(ans1 <-<span class="st"> </span>DT[x ==<span class="st"> "g"</span> &<span class="st"> </span>y ==<span class="st"> </span>877L])
t1
<span class="co"># user system elapsed </span>
-<span class="co"># 0.172 0.016 0.188</span>
+<span class="co"># 0.124 0.016 0.140</span>
<span class="kw">head</span>(ans1)
<span class="co"># x y val</span>
<span class="co"># 1: g 877 0.3946652</span>
@@ -590,7 +590,7 @@ t1
t2 <-<span class="st"> </span><span class="kw">system.time</span>(ans2 <-<span class="st"> </span>DT[.(<span class="st">"g"</span>, 877L)])
t2
<span class="co"># user system elapsed </span>
-<span class="co"># 0.000 0.004 0.000</span>
+<span class="co"># 0.000 0.000 0.001</span>
<span class="kw">head</span>(ans2)
<span class="co"># x y val</span>
<span class="co"># 1: g 877 0.3946652</span>
@@ -605,7 +605,7 @@ t2
<span class="kw">identical</span>(ans1$val, ans2$val)
<span class="co"># [1] TRUE</span></code></pre></div>
<ul>
-<li>The speedup is <strong>~188x</strong>!</li>
+<li>The speedup is <strong>~140x</strong>!</li>
</ul>
</div>
<div id="b-why-does-keying-a-data.table-result-in-blazing-fast-susbets" class="section level3">
diff --git a/inst/doc/datatable-reference-semantics.html b/inst/doc/datatable-reference-semantics.html
index 6a3e201..542ed96 100644
--- a/inst/doc/datatable-reference-semantics.html
+++ b/inst/doc/datatable-reference-semantics.html
@@ -11,7 +11,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1">
-<meta name="date" content="2016-12-02" />
+<meta name="date" content="2017-01-31" />
<title>Reference semantics</title>
@@ -68,7 +68,7 @@ code > span.in { color: #60a0b0; font-weight: bold; font-style: italic; } /* Inf
<h1 class="title toc-ignore">Reference semantics</h1>
-<h4 class="date"><em>2016-12-02</em></h4>
+<h4 class="date"><em>2017-01-31</em></h4>
diff --git a/inst/doc/datatable-reshape.html b/inst/doc/datatable-reshape.html
index 28f3c87..e747b5c 100644
--- a/inst/doc/datatable-reshape.html
+++ b/inst/doc/datatable-reshape.html
@@ -11,7 +11,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1">
-<meta name="date" content="2016-12-02" />
+<meta name="date" content="2017-01-31" />
<title>Efficient reshaping using data.tables</title>
@@ -68,7 +68,7 @@ code > span.in { color: #60a0b0; font-weight: bold; font-style: italic; } /* Inf
<h1 class="title toc-ignore">Efficient reshaping using data.tables</h1>
-<h4 class="date"><em>2016-12-02</em></h4>
+<h4 class="date"><em>2017-01-31</em></h4>
diff --git a/inst/doc/datatable-secondary-indices-and-auto-indexing.html b/inst/doc/datatable-secondary-indices-and-auto-indexing.html
index eb6371d..2c67a8a 100644
--- a/inst/doc/datatable-secondary-indices-and-auto-indexing.html
+++ b/inst/doc/datatable-secondary-indices-and-auto-indexing.html
@@ -11,7 +11,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1">
-<meta name="date" content="2016-12-02" />
+<meta name="date" content="2017-01-31" />
<title>Secondary indices and auto indexing</title>
@@ -68,7 +68,7 @@ code > span.in { color: #60a0b0; font-weight: bold; font-style: italic; } /* Inf
<h1 class="title toc-ignore">Secondary indices and auto indexing</h1>
-<h4 class="date"><em>2016-12-02</em></h4>
+<h4 class="date"><em>2017-01-31</em></h4>
@@ -398,7 +398,7 @@ dt =<span class="st"> </span><span class="kw">data.table</span>(<span class="dt"
## run thefirst time
(t1 <-<span class="st"> </span><span class="kw">system.time</span>(ans <-<span class="st"> </span>dt[x ==<span class="st"> </span>989L]))
<span class="co"># user system elapsed </span>
-<span class="co"># 0.16 0.00 0.16</span>
+<span class="co"># 0.156 0.012 0.170</span>
<span class="kw">head</span>(ans)
<span class="co"># x y</span>
<span class="co"># 1: 989 0.5372007</span>
@@ -424,7 +424,7 @@ dt =<span class="st"> </span><span class="kw">data.table</span>(<span class="dt"
<span class="co"># user system elapsed </span>
<span class="co"># 0.000 0.000 0.001</span></code></pre></div>
<ul>
-<li><p>Running the first time took 0.160 seconds where as the second time took 0.001 seconds.</p></li>
+<li><p>Running the first time took 0.170 seconds where as the second time took 0.001 seconds.</p></li>
<li><p>Auto indexing can be disabled by setting the global argument <code>options(datatable.auto.index = FALSE)</code>.</p></li>
<li><p>Disabling auto indexing still allows to use indices created explicitly with <code>setindex</code> or <code>setindexv</code>. You can disable indices fully by setting global argument <code>options(datatable.use.index = FALSE)</code>.</p></li>
</ul>
diff --git a/inst/tests/tests.Rraw b/inst/tests/tests.Rraw
index 91b6fcf..8d26b54 100644
--- a/inst/tests/tests.Rraw
+++ b/inst/tests/tests.Rraw
@@ -3,7 +3,7 @@ if (!exists("test.data.table",.GlobalEnv,inherits=FALSE)) {
require(data.table) # in dev the package should not be loaded
options(warn=0) # use require() so it warns but doesn't halt if not available
inst_pkgs = rownames(installed.packages())
- sugg_pkgs = c("chron", "ggplot2", "plyr", "reshape", "reshape2", "testthat", "hexbin", "fastmatch", "nlme", "GenomicRanges", "xts", "bit64", "gdata", "caret", "knitr", "curl", "zoo", "plm", "rmarkdown", "parallel")
+ sugg_pkgs = c("bit64", "knitr", "nanotime", "chron", "ggplot2", "plyr", "reshape", "reshape2", "testthat", "hexbin", "fastmatch", "nlme", "GenomicRanges", "xts", "gdata", "caret", "curl", "zoo", "plm", "rmarkdown", "parallel")
lapply(setNames(sugg_pkgs, nm = sugg_pkgs), function(pkg) if(pkg %in% inst_pkgs) require(pkg, character.only=TRUE))
# reshape2 ahead of reshape ...
try(detach(package:reshape2),silent=TRUE)
@@ -38,7 +38,6 @@ started.at = Sys.time()
# Test default values in case user set global option. These are restored
# at the end of this file.
oldalloccol = options(datatable.alloccol=1024L)
-oldWhenJsymbol = options(datatable.WhenJisSymbolThenCallingScope=FALSE)
if (!.devtesting) {
test = data.table:::test
@@ -235,12 +234,10 @@ test(76, TESTDT[,2:3], TESTDT[,2:3,with=FALSE])
test(77, TESTDT[,2:3,with=FALSE], data.table(b=c("e","e","f","f","i","i","b"),v=1:7))
test(78, TESTDT[,c("b","v")], data.table(b=c("e","e","f","f","i","i","b"),v=1:7))
colsVar = c("b","v")
-test(79.1, TESTDT[,colsVar], colsVar)
-test(79.2, TESTDT[,colsVar,with=FALSE], data.table(b=c("e","e","f","f","i","i","b"),v=1:7))
-options(datatable.WhenJisSymbolThenCallingScope=TRUE)
-test(80.1, TESTDT[,colsVar], data.table(b=c("e","e","f","f","i","i","b"),v=1:7))
-test(80.2, TESTDT[,colsVar,with=TRUE], error="is turned on.*Please")
-options(datatable.WhenJisSymbolThenCallingScope=FALSE)
+test(79.1, TESTDT[,colsVar], error="column name 'colsVar' is not found")
+test(79.2, TESTDT[,colsVar,with=FALSE], ans<-data.table(b=c("e","e","f","f","i","i","b"),v=1:7))
+test(79.3, TESTDT[, ..colsVar], ans)
+
# works in test.data.table, but not eval(body(test.data.table)) when in R CMD check ... test(81, TESTDT[1:2,c(a,b)], factor(c("a","c","e","e")))
# It is expected the above to be common source of confusion. c(a,b) is evaluated within
@@ -9280,13 +9277,14 @@ test(1712, DT[!is.na(a) & d == "3"], DT[2])
test(1713, DT[d==3], DT[2])
# Test new helpful error message suggested by Jan
-test(1714, !exists("aColumnOfDT")) # use a long column name to be sure not used before in this test script
-DT = data.table(a=1:3, aColumnOfDT=4:6)
-options(datatable.WhenJisSymbolThenCallingScope=FALSE)
-test(1715, DT[,aColumnOfDT], 4:6) # old behaviour for sure tested before but here for context
-options(datatable.WhenJisSymbolThenCallingScope=TRUE)
-test(1716, DT[,aColumnOfDT], error=".*datatable.WhenJisSymbolThenCallingScope.*TRUE.*aColumnOfDT.*wrap with quotes.*aColumnOfDT.*aColumnOfDT.*aColumnOfDT") # The .* are to check the salient points of the error (e.g. that it names the variable correctly 4 times), without getting too picky over the exact wording and punctuation of the full error
-options(datatable.WhenJisSymbolThenCallingScope=FALSE)
+notAColName = 1
+test(1714.1, exists("notAColName")) # use a long column name to be sure it exists and unique
+test(1714.1, !exists("notInCallingScope")) # use a long column name to be sure it exists and unique
+DT = data.table(a=1:3, b=4:6)
+test(1715, DT[,b], 4:6) # old behaviour for sure tested before but here for context
+test(1716.1, DT[,notAColName], error="column name 'notAColName' is not found") # ensure it doesn't find it calling scope either
+test(1716.2, DT[, ..notInCallingScope], error="Variable 'notInCallingScope' is not found in calling scope")
+test(1716.3, DT[, notInCallingScope, with=FALSE], error="Variable 'notInCallingScope' is not found in calling scope")
# Test out-of-bounds error on numeric j
DT = data.table(a=1:3, b=4:6, c=7:9)
@@ -9484,7 +9482,7 @@ if ("package:bit64" %in% search()) {
test(1731.1, class(DT[[1L]]), "integer64")
test(1731.2, fwrite(DT,na="__NA__"), output=paste(ans,collapse=""))
f = tempfile()
- test(1731.3, fwrite(DT,f, na="__NA__",..turbo=FALSE), NULL, warning="turbo.*will be removed")
+ test(1731.3, fwrite(DT, f, na="__NA__"), NULL)
test(1731.4, readLines(f), ans)
unlink(f)
test(1731.5, write.csv(DT,na="__NA__",row.names=FALSE,quote=FALSE), output=paste(ans,collapse=""))
@@ -9780,7 +9778,35 @@ DT = as.data.table(mtcars[mtcars$cyl %in% c(6, 8), c("am", "vs", "hp")])
test(1748.3, DT[am==0, max(hp), by=vs], ans1)
test(1748.4, DT[am==0, max(hp), keyby=vs], ans2)
-
+# indices() can return list of vectors, #1589
+DT = data.table(A=5:1,B=letters[5:1])
+setindex(DT)
+setindex(DT, A)
+setindex(DT, B)
+indices(DT, vectors = TRUE)
+test(1749.1, indices(DT), c("A__B","A","B"))
+test(1749.2, indices(DT, vectors = TRUE), list(c("A","B"),"A","B"))
+
+# for completeness, added test for NA problem to close #1837. Fixed long ago before release to CRAN.
+test(1750, capture.output(fwrite(data.table(x=NA_integer_),verbose=FALSE)), c("x",""))
+
+if ("package:nanotime" %in% search()) {
+ DT = data.table(A=nanotime(tt<-c("2016-09-28T15:30:00.000000070Z",
+ "2016-09-29T23:59:00.000000001Z",
+ "2016-09-29T23:59:00.000000999Z",
+ "1970-01-01T00:01:01.000001000Z",
+ "1970-01-01T00:00:00.000000000Z",
+ "1969-12-31T23:59:59.999999999Z",
+ "1969-12-31T23:59:59.000000089Z",
+ "1969-12-31T12:13:14.000000000Z",
+ "1969-12-31T12:13:14.999999999Z",
+ "1969-12-31T12:13:14.000000001Z",
+ "1967-03-15T00:00:00.300000002Z",
+ "1967-03-15T23:59:59.300000002Z")))
+ test(1751, capture.output(fwrite(DT, verbose=FALSE))[-1], tt)
+} else {
+ cat("Test 1751 not run. If required call library(nanotime) first.\n")
+}
##########################
@@ -9814,7 +9840,7 @@ test(1748.4, DT[am==0, max(hp), keyby=vs], ans2)
options(warn=0)
setDTthreads(0)
options(oldalloccol) # set at top of this file
-options(oldWhenJsymbol)
+
plat = paste("endian==",.Platform$endian,", sizeof(long double)==",.Machine$sizeof.longdouble,
", sizeof(pointer)==",.Machine$sizeof.pointer, sep="")
if (nfail > 0) {
diff --git a/man/assign.Rd b/man/assign.Rd
index 06bc96c..afe9642 100644
--- a/man/assign.Rd
+++ b/man/assign.Rd
@@ -3,7 +3,7 @@
\alias{set}
\title{ Assignment by reference }
\description{
- Fast add, remove and update subsets of columns, by reference. \code{:=} opertor can be used in two ways: \code{LHS := RHS} form, and \code{Functional form}. See \code{Usage}.
+ Fast add, remove and update subsets of columns, by reference. \code{:=} operator can be used in two ways: \code{LHS := RHS} form, and \code{Functional form}. See \code{Usage}.
\code{set} is a low overhead loopable version of \code{:=}. It is particularly useful for repetitively updating rows of certain columns by reference (using a for-loop). See \code{Examples}. It can not perform grouping operations.
diff --git a/man/duplicated.Rd b/man/duplicated.Rd
index fa9052b..ddd7817 100644
--- a/man/duplicated.Rd
+++ b/man/duplicated.Rd
@@ -44,7 +44,7 @@ the reverse side, i.e., the last (or rightmost) of identical elements would
correspond to \code{duplicated = FALSE}.}
\item{by}{\code{character} or \code{integer} vector indicating which combinations
of columns from \code{x} to use for uniqueness checks. By default all columns are
-are being used. The was changed recently for consistency to data.frame methods.
+are being used. That was changed recently for consistency to data.frame methods.
In version \code{< 1.9.8} default was \code{key(x)}.}
\item{na.rm}{Logical (default is \code{FALSE}). Should missing values (including
\code{NaN}) be removed?}
diff --git a/man/fwrite.Rd b/man/fwrite.Rd
index a1251e9..e3ab4fd 100644
--- a/man/fwrite.Rd
+++ b/man/fwrite.Rd
@@ -15,8 +15,7 @@ fwrite(x, file = "", append = FALSE, quote = "auto",
logicalAsInt = FALSE, dateTimeAs = c("ISO","squash","epoch","write.csv"),
buffMB = 8L, nThread = getDTthreads(),
showProgress = getOption("datatable.showProgress"),
- verbose = getOption("datatable.verbose"),
- ..turbo=TRUE)
+ verbose = getOption("datatable.verbose"))
}
\arguments{
\item{x}{Any \code{list} of same length vectors; e.g. \code{data.frame} and \code{data.table}.}
@@ -51,7 +50,6 @@ fwrite(x, file = "", append = FALSE, quote = "auto",
\item{nThread}{The number of threads to use. Experiment to see what works best for your data on your hardware.}
\item{showProgress}{ Display a progress meter on the console? Ignored when \code{file==""}. }
\item{verbose}{Be chatty and report timings?}
- \item{..turbo}{Use specialized custom C code to format numeric, integer and integer64 columns. This reduces call overhead to the C library and avoids copies. Try with and without to see the difference it makes on your machine and please report any differences in output. If you do find cases where \code{..turbo=FALSE} is needed, please report them as bugs, since this option WILL BE REMOVED in future. Hence why it has the \code{..} prefix.}
}
\details{
\code{fwrite} began as a community contribution with \href{https://github.com/Rdatatable/data.table/pull/1613}{pull request #1613} by Otto Seiskari. This gave Matt Dowle the impetus to specialize the numeric formatting and to parallelize: \url{http://blog.h2o.ai/2016/04/fast-csv-writing-for-r/}. Final items were tracked in \href{https://github.com/Rdatatable/data.table/issues/1664}{issue #1664} such as automatic quoting, \code{bit64::integer64} support, decimal/scientific formatting exac [...]
diff --git a/man/setkey.Rd b/man/setkey.Rd
index 5116881..cdab41c 100644
--- a/man/setkey.Rd
+++ b/man/setkey.Rd
@@ -36,7 +36,7 @@ setkeyv(x, cols, verbose=getOption("datatable.verbose"), physical = TRUE)
setindex(...)
setindexv(...)
key(x)
-indices(x)
+indices(x, vectors = FALSE)
haskey(x)
key(x) <- value # DEPRECATED, please use setkey or setkeyv instead.
}
@@ -51,6 +51,8 @@ names.}
\item{verbose}{ Output status and information. }
\item{physical}{ TRUE changes the order of the data in RAM. FALSE adds a
secondary key a.k.a. index. }
+\item{vectors}{ logical scalar default \code{FALSE}, when set to \code{TRUE}
+then list of character vectors is returned, each vector refers to one index. }
}
\details{
\code{setkey} reorders (or sorts) the rows of a data.table by the columns
@@ -155,6 +157,13 @@ DT = data.table(A=5:1,B=letters[5:1])
DT2 = copy(DT) # explicit copy() needed to copy a data.table
setkey(DT2,B) # now just changes DT2
identical(DT,DT2) # FALSE. DT and DT2 are now different tables
+
+DT = data.table(A=5:1,B=letters[5:1])
+setindex(DT) # set indices
+setindex(DT, A)
+setindex(DT, B)
+indices(DT) # get indices single vector
+indices(DT, vectors = TRUE) # get indices list
}
\keyword{ data }
diff --git a/src/assign.c b/src/assign.c
index a1fb47d..4be58c1 100644
--- a/src/assign.c
+++ b/src/assign.c
@@ -611,7 +611,7 @@ SEXP assign(SEXP dt, SEXP rows, SEXP cols, SEXP newcolnames, SEXP values, SEXP v
}
memrecycle(targetcol, rows, 0, targetlen, RHS); // also called from dogroups where these arguments are used more
}
- key = getAttrib(dt,install("sorted"));
+ key = getAttrib(dt, sym_sorted);
if (length(key)) {
// if assigning to any key column, then drop the key. any() and subsetVector() don't seem to be
// exposed by R API at C level, so this is done here long hand.
@@ -624,7 +624,7 @@ SEXP assign(SEXP dt, SEXP rows, SEXP cols, SEXP newcolnames, SEXP values, SEXP v
for (i=0;i<LENGTH(tmp);i++) if (LOGICAL(tmp)[i]) {
// If a key column is being assigned to, clear the key, since it may change the row ordering.
// More likely that users will assign to non-key columns, though, most of the time.
- setAttrib(dt, install("sorted"), R_NilValue);
+ setAttrib(dt, sym_sorted, R_NilValue);
break;
}
}
diff --git a/src/bmerge.c b/src/bmerge.c
index 012ac9b..35e3788 100644
--- a/src/bmerge.c
+++ b/src/bmerge.c
@@ -142,9 +142,11 @@ SEXP bmerge(SEXP iArg, SEXP xArg, SEXP icolsArg, SEXP xcolsArg, SEXP isorted, SE
// isorted arg
o = NULL;
if (!LOGICAL(isorted)[0]) {
- SEXP order = PROTECT(vec_init(length(icolsArg), ScalarInteger(1))); // rep(1, length(icolsArg))
- SEXP oSxp = PROTECT(forder(i, icolsArg, ScalarLogical(FALSE), ScalarLogical(TRUE), order, ScalarLogical(FALSE)));
- protecti += 2;
+ SEXP order = PROTECT(int_vec_init(length(icolsArg), 1)); // rep(1L, length(icolsArg))
+ SEXP oSxp = PROTECT(forder(i, icolsArg, PROTECT(ScalarLogical(FALSE)),
+ PROTECT(ScalarLogical(TRUE)), order, PROTECT(ScalarLogical(FALSE))));
+ UNPROTECT(3); // The 3 logicals in line above. TODO - split head of forder into C-level callable
+ protecti += 2; // order and oSxp
if (!LENGTH(oSxp)) o = NULL; else o = INTEGER(oSxp);
}
@@ -180,7 +182,7 @@ SEXP bmerge(SEXP iArg, SEXP xArg, SEXP icolsArg, SEXP xcolsArg, SEXP isorted, SE
SET_VECTOR_ELT(ans, 2, retIndexArg);
SET_VECTOR_ELT(ans, 3, allLen1Arg);
SET_VECTOR_ELT(ans, 4, allGrp1Arg);
- SET_STRING_ELT(ansnames, 0, mkChar("starts"));
+ SET_STRING_ELT(ansnames, 0, char_starts); // changed from mkChar to char_ to pass the grep in CRAN_Release.cmd
SET_STRING_ELT(ansnames, 1, mkChar("lens"));
SET_STRING_ELT(ansnames, 2, mkChar("indices"));
SET_STRING_ELT(ansnames, 3, mkChar("allLen1"));
@@ -364,7 +366,7 @@ void bmerge_r(int xlowIn, int xuppIn, int ilowIn, int iuppIn, int col, int thisg
}
}
if (col>-1 && op[col] != EQ) {
- Rboolean isivalNA = !isInt64 ? ISNAN(REAL(ic)[ir]) : (*(long long *)&REAL(ic)[ir] == NAINT64);
+ Rboolean isivalNA = !isInt64 ? ISNAN(REAL(ic)[ir]) : (I64(REAL(ic)[ir]) == NAINT64);
switch (op[col]) {
case LE : if (!isivalNA) xlow = xlowIn; break;
case LT : xupp = xlow + 1; if (!isivalNA) xlow = xlowIn; break;
@@ -372,7 +374,7 @@ void bmerge_r(int xlowIn, int xuppIn, int ilowIn, int iuppIn, int col, int thisg
case GT : xlow = xupp - 1; if (!isivalNA) xupp = xuppIn; break;
}
// for LE/LT cases, we need to ensure xlow excludes NA indices, != EQ is checked above already
- if (op[col] <= 3 && xlow<xupp-1 && !isivalNA && (!isInt64 ? ISNAN(REAL(xc)[XIND(xlow+1)]) : (*(long long *)&REAL(xc)[XIND(xlow+1)] == NAINT64))) {
+ if (op[col] <= 3 && xlow<xupp-1 && !isivalNA && (!isInt64 ? ISNAN(REAL(xc)[XIND(xlow+1)]) : (I64(REAL(xc)[XIND(xlow+1)]) == NAINT64))) {
tmplow = xlow; tmpupp = xupp;
while (tmplow < tmpupp-1) {
mid = tmplow + (tmpupp-tmplow)/2;
diff --git a/src/data.table.h b/src/data.table.h
index 354b1e7..9bf6092 100644
--- a/src/data.table.h
+++ b/src/data.table.h
@@ -38,7 +38,13 @@ SEXP char_ITime;
SEXP char_IDate;
SEXP char_Date;
SEXP char_POSIXct;
-Rboolean INHERITS(SEXP x, SEXP char_);
+SEXP char_nanotime;
+SEXP sym_sorted;
+SEXP sym_BY;
+SEXP sym_starts, char_starts;
+SEXP sym_maxgrpn;
+Rboolean INHERITS(SEXP x, SEXP char_);
+long long I64(double x);
// dogroups.c
SEXP keepattr(SEXP to, SEXP from);
@@ -62,7 +68,7 @@ SEXP forder(SEXP DT, SEXP by, SEXP retGrp, SEXP sortStrArg, SEXP orderArg, SEXP
SEXP reorder(SEXP x, SEXP order);
// fcast.c
-SEXP vec_init(R_len_t n, SEXP val);
+SEXP int_vec_init(R_len_t n, int val);
// vecseq.c
SEXP vecseq(SEXP x, SEXP len, SEXP clamp);
diff --git a/src/dogroups.c b/src/dogroups.c
index 255d49c..4098695 100644
--- a/src/dogroups.c
+++ b/src/dogroups.c
@@ -44,7 +44,7 @@ SEXP dogroups(SEXP dt, SEXP dtcols, SEXP groups, SEXP grpcols, SEXP jiscols, SEX
// fix for longstanding FR/bug, #495. E.g., DT[, c(sum(v1), lapply(.SD, mean)), by=grp, .SDcols=v2:v3] resulted in error.. the idea is, 1) we create .SDall, which is normally == .SD. But if extra vars are detected in jexp other than .SD, then .SD becomes a shallow copy of .SDall with only .SDcols in .SD. Since internally, we don't make a copy, changing .SDall will reflect in .SD. Hopefully this'll workout :-).
SDall = findVar(install(".SDall"), env);
- defineVar(install(".BY"), BY = allocVector(VECSXP, ngrpcols), env);
+ defineVar(sym_BY, BY = allocVector(VECSXP, ngrpcols), env);
bynames = PROTECT(allocVector(STRSXP, ngrpcols)); protecti++; // TO DO: do we really need bynames, can we assign names afterwards in one step?
for (i=0; i<ngrpcols; i++) {
j = INTEGER(grpcols)[i]-1;
@@ -58,7 +58,7 @@ SEXP dogroups(SEXP dt, SEXP dtcols, SEXP groups, SEXP grpcols, SEXP jiscols, SEX
error("Unsupported type '%s' in column %d of 'by'", type2char(TYPEOF(VECTOR_ELT(BY, i))), i+1);
}
setAttrib(BY, R_NamesSymbol, bynames); // Fix for #5415 - BY doesn't retain names anymore
- R_LockBinding(install(".BY"), env);
+ R_LockBinding(sym_BY, env);
if (isNull(jiscols) && (length(bynames)!=length(groups) || length(bynames)!=length(grpcols))) error("!length(bynames)[%d]==length(groups)[%d]==length(grpcols)[%d]",length(bynames),length(groups),length(grpcols));
// TO DO: check this check above.
diff --git a/src/fcast.c b/src/fcast.c
index 96daeb8..848f050 100644
--- a/src/fcast.c
+++ b/src/fcast.c
@@ -96,30 +96,14 @@ SEXP fcast(SEXP lhs, SEXP val, SEXP nrowArg, SEXP ncolArg, SEXP idxArg, SEXP fil
}
// used in bmerge.c
-SEXP vec_init(R_len_t n, SEXP val) {
-
- SEXP ans;
- R_len_t i;
- if (n < 0) error("Input argument 'n' to 'vec_init' must be >= 0");
- ans = PROTECT(allocVector(TYPEOF(val), n));
- switch(TYPEOF(val)) {
- case INTSXP :
- for (i=0; i<n; i++) INTEGER(ans)[i] = INTEGER(val)[0];
- break;
- case REALSXP :
- for (i=0; i<n; i++) REAL(ans)[i] = REAL(val)[0];
- break;
- case LGLSXP :
- for (i=0; i<n; i++) LOGICAL(ans)[i] = LOGICAL(val)[0];
- break;
- case STRSXP :
- for (i=0; i<n; i++) SET_STRING_ELT(ans, i, STRING_ELT(val, 0));
- break;
- case VECSXP :
- for (i=0; i<n; i++) SET_VECTOR_ELT(ans, i, VECTOR_ELT(val, 0));
- default :
- error("Unknown input type '%s'", type2char(TYPEOF(val)));
- }
+SEXP int_vec_init(R_len_t n, int val) {
+ // Matt removed the SEXP val input because the ScalarInteger() input wasn't PROTECTed in caller
+ // and was potentially leaking/crashing.
+ // This function only used once and only for int, so to remove untested lines (code-coverage)
+ // decided to simplify it rather than add the PROTECT and UNPROTECT in caller.
+ if (n < 0) error("Input argument 'n' to 'int_vec_init' must be >= 0");
+ SEXP ans = PROTECT(allocVector(INTSXP, n));
+ for (R_len_t i=0; i<n; i++) INTEGER(ans)[i] = val;
UNPROTECT(1);
return(ans);
}
diff --git a/src/forder.c b/src/forder.c
index 32b99ae..d9d277e 100644
--- a/src/forder.c
+++ b/src/forder.c
@@ -1305,7 +1305,7 @@ SEXP forder(SEXP DT, SEXP by, SEXP retGrp, SEXP sortStrArg, SEXP orderArg, SEXP
}
if (LOGICAL(retGrp)[0]) {
ngrp = gsngrp[flip];
- setAttrib(ans, install("starts"), x = allocVector(INTSXP, ngrp));
+ setAttrib(ans, sym_starts, x = allocVector(INTSXP, ngrp));
//if (isSorted || LOGICAL(sort)[0])
for (INTEGER(x)[0]=1, i=1; i<ngrp; i++) INTEGER(x)[i] = INTEGER(x)[i-1] + gs[flip][i-1];
//else {
@@ -1314,7 +1314,7 @@ SEXP forder(SEXP DT, SEXP by, SEXP retGrp, SEXP sortStrArg, SEXP orderArg, SEXP
// for (i=0; i<ngrp; i++) { INTEGER(x)[i] = o[i+cumsum]; cumsum+=gs[flip][i]; }
// isort(INTEGER(x), ngrp);
//}
- setAttrib(ans, install("maxgrpn"), ScalarInteger(gsmax[flip]));
+ setAttrib(ans, sym_maxgrpn, ScalarInteger(gsmax[flip]));
}
gsfree();
diff --git a/src/fread.c b/src/fread.c
index 4283c7e..f5085c1 100644
--- a/src/fread.c
+++ b/src/fread.c
@@ -446,7 +446,7 @@ static SEXP coerceVectorSoFar(SEXP v, int oldtype, int newtype, R_len_t sofar, R
// This was 1.3s (all of tCoerce) when testing on 2008.csv; might have triggered a gc, included.
// Happily, mid read bumps are very rarely needed, due to testing types at the start, middle and end of the file, first.
}
- setAttrib(newv, R_ClassSymbol, newtype==SXP_INT64 ? ScalarString(mkChar("integer64")) : R_NilValue);
+ setAttrib(newv, R_ClassSymbol, newtype==SXP_INT64 ? ScalarString(char_integer64) : R_NilValue);
switch(newtype) {
case SXP_INT :
switch(oldtype) {
@@ -474,7 +474,7 @@ static SEXP coerceVectorSoFar(SEXP v, int oldtype, int newtype, R_len_t sofar, R
for (i=0; i<sofar; i++) REAL(newv)[i] = (INTEGER(v)[i]==NA_INTEGER ? NA_REAL : (double)INTEGER(v)[i]);
break;
case SXP_INT64 :
- for (i=0; i<sofar; i++) REAL(newv)[i] = ((*(long long *)&REAL(v)[i] == NAINT64) ? NA_REAL : (double)(*(long long *)&REAL(v)[i]));
+ for (i=0; i<sofar; i++) REAL(newv)[i] = (I64(REAL(v)[i]) == NAINT64) ? NA_REAL : (double)(I64(REAL(v)[i]));
break;
default :
STOP("Internal error: attempt to bump from type %d to type %d. Please report to datatable-help.", oldtype, newtype);
@@ -501,9 +501,9 @@ static SEXP coerceVectorSoFar(SEXP v, int oldtype, int newtype, R_len_t sofar, R
SET_STRING_ELT(newv,i,R_BlankString);
else {
#ifdef WIN32
- snprintf(buffer,128,"%" PRId64,*(long long *)&REAL(v)[i]);
+ snprintf(buffer,128,"%" PRId64, I64(REAL(v)[i]));
#else
- snprintf(buffer,128,"%lld", *(long long *)&REAL(v)[i]);
+ snprintf(buffer,128,"%lld", I64(REAL(v)[i]));
#endif
SET_STRING_ELT(newv, i, mkChar(buffer));
}
@@ -1236,7 +1236,7 @@ SEXP readfile(SEXP input, SEXP separg, SEXP nrowsarg, SEXP headerarg, SEXP nastr
if (type[i] == SXP_NULL) continue;
SEXP thiscol = allocVector(TypeSxp[ type[i] ], nrow);
SET_VECTOR_ELT(ans,resi++,thiscol); // no need to PROTECT thiscol, see R-exts 5.9.1
- if (type[i]==SXP_INT64) setAttrib(thiscol, R_ClassSymbol, ScalarString(mkChar("integer64")));
+ if (type[i]==SXP_INT64) setAttrib(thiscol, R_ClassSymbol, ScalarString(char_integer64));
SET_TRUELENGTH(thiscol, nrow);
}
clock_t tAlloc = clock();
diff --git a/src/fwrite.c b/src/fwrite.c
index 8962023..f8a377c 100644
--- a/src/fwrite.c
+++ b/src/fwrite.c
@@ -18,10 +18,11 @@
#define NUM_SF 15
#define SIZE_SF 1000000000000000ULL // 10^NUM_SF
-// Globals for this file only (written once to hold parameters passed from R level)
+// Globals for this file only. Written once to hold parameters passed from R level.
static const char *na; // by default "" or if set then usually "NA"
static char sep; // comma in .csv files
-static char sep2; // ; in list column vectors
+static char sep2; // '|' within list columns
+static const char *sep2start, *sep2end;
static char dec; // the '.' in the number 3.1416. In Europe often: 3,1416
static Rboolean verbose=FALSE; // be chatty?
static Rboolean quote=FALSE; // whether to surround fields with double quote ". NA means 'auto' (default)
@@ -31,47 +32,22 @@ static Rboolean squash=FALSE; // 0=ISO(yyyy-mm-dd) 1=squash(yyyymmdd)
static int dateTimeAs=0; // 0=ISO(yyyy-mm-dd) 1=squash(yyyymmdd), 2=epoch, 3=write.csv
#define DATETIMEAS_EPOCH 2
#define DATETIMEAS_WRITECSV 3
-#define ET_DATE 1 // extraType values
-#define ET_ITIME 2
-#define ET_INT64 3
-#define ET_POSIXCT 4
-#define ET_FACTOR 5
+typedef void (*writer_fun_t)(SEXP, int, char **);
-static inline void writeInteger(long long x, char **thisCh)
+static inline void write_chars(const char *x, char **thisCh)
{
- char *ch = *thisCh;
- // both integer and integer64 are passed to this function so careful
- // to test for NA_INTEGER in the calling code. INT_MIN (NA_INTEGER) is
- // a valid non-NA in integer64
- if (x == 0) {
- *ch++ = '0';
- } else {
- if (x<0) { *ch++ = '-'; x=-x; }
- // avoid log() call for speed. write backwards then reverse when we know how long
- int width = 0;
- while (x>0) { *ch++ = '0'+x%10; x /= 10; width++; }
- for (int i=width/2; i>0; i--) {
- char tmp=*(ch-i);
- *(ch-i) = *(ch-width+i-1);
- *(ch-width+i-1) = tmp;
- }
- }
- *thisCh = ch;
-}
-
-static inline void writeChars(const char *x, char **thisCh)
-{
- // similar to C's strcpy but i) doesn't copy \0 and ii) moves destination along
+ // similar to C's strcpy but i) doesn't include trailing \0 and ii) moves destination along
char *ch = *thisCh;
while (*x) *ch++=*x++;
*thisCh = ch;
}
-static inline void writeLogical(Rboolean x, char **thisCh)
+static void writeLogical(SEXP column, int i, char **thisCh)
{
+ Rboolean x = LOGICAL(column)[i];
char *ch = *thisCh;
if (x == NA_LOGICAL) {
- writeChars(na, &ch);
+ write_chars(na, &ch);
} else if (logicalAsInt) {
*ch++ = '0'+x;
} else if (x) {
@@ -82,6 +58,38 @@ static inline void writeLogical(Rboolean x, char **thisCh)
*thisCh = ch;
}
+static inline void write_positive_int(long long x, char **thisCh)
+{
+ // Avoid log() for speed. Write backwards then reverse when we know how long.
+ // Separate function just because it's used if row numbers are asked for, too
+ // x >= 1
+ char *ch = *thisCh;
+ int width = 0;
+ while (x>0) { *ch++ = '0'+x%10; x /= 10; width++; }
+ for (int i=width/2; i>0; i--) {
+ char tmp=*(ch-i);
+ *(ch-i) = *(ch-width+i-1);
+ *(ch-width+i-1) = tmp;
+ }
+ *thisCh = ch;
+}
+
+static void writeInteger(SEXP column, int i, char **thisCh)
+{
+ long long x = (TYPEOF(column)!=REALSXP) ? INTEGER(column)[i] : I64(REAL(column)[i]);
+ // != REALSXP rather than ==INTSXP to cover LGLSXP when logicalAsInt==TRUE
+ char *ch = *thisCh;
+ if (x == 0) {
+ *ch++ = '0';
+ } else if (x == ((TYPEOF(column)==INTSXP) ? NA_INTEGER : NAINT64)) {
+ write_chars(na, &ch);
+ } else {
+ if (x<0) { *ch++ = '-'; x=-x; }
+ write_positive_int(x, &ch);
+ }
+ *thisCh = ch;
+}
+
SEXP genLookups() {
Rprintf("genLookups commented out of the package so it's clear it isn't needed to build. The hooks are left in so it's easy to put back in development should we need to.\n");
// e.g. ldexpl may not be available on some platforms, or if it is it may not be accurate.
@@ -121,12 +129,7 @@ SEXP genLookups() {
}
*/
-static union {
- double d;
- unsigned long long ull;
-} u;
-
-static inline void writeNumeric(double x, char **thisCh)
+static void writeNumeric(SEXP column, int i, char **thisCh)
{
// hand-rolled / specialized for speed
// *thisCh is safely the output destination with enough space (ensured via calculating maxLineLen up front)
@@ -136,10 +139,11 @@ static inline void writeNumeric(double x, char **thisCh)
// ii) no C libary calls such as sprintf() where the fmt string has to be interpretted over and over
// iii) no need to return variables or flags. Just writes.
// iv) shorter, easier to read and reason with. In one self contained place.
+ double x = REAL(column)[i];
char *ch = *thisCh;
if (!R_FINITE(x)) {
if (ISNAN(x)) {
- writeChars(na, &ch);
+ write_chars(na, &ch);
} else if (x>0) {
*ch++ = 'I'; *ch++ = 'n'; *ch++ = 'f';
} else {
@@ -149,6 +153,7 @@ static inline void writeNumeric(double x, char **thisCh)
*ch++ = '0'; // and we're done. so much easier rather than passing back special cases
} else {
if (x < 0.0) { *ch++ = '-'; x = -x; } // and we're done on sign, already written. no need to pass back sign
+ union { double d; unsigned long long ull; } u;
u.d = x;
unsigned long long fraction = u.ull & 0xFFFFFFFFFFFFF; // (1ULL<<52)-1;
int exponent = (int)((u.ull>>52) & 0x7FF); // [0,2047]
@@ -249,18 +254,19 @@ static inline void writeNumeric(double x, char **thisCh)
*thisCh = ch;
}
-static inline void writeString(SEXP x, char **thisCh)
+static void writeString(SEXP column, int i, char **thisCh)
{
+ SEXP x = STRING_ELT(column, i);
char *ch = *thisCh;
if (x == NA_STRING) {
// NA is not quoted by write.csv even when quote=TRUE to distinguish from "NA"
- writeChars(na, &ch);
+ write_chars(na, &ch);
} else {
Rboolean q = quote;
if (q==NA_LOGICAL) { // quote="auto"
const char *tt = CHAR(x);
while (*tt!='\0' && *tt!=sep && *tt!=sep2 && *tt!='\n' && *tt!='"') *ch++ = *tt++;
- // windows includes \n in its \r\n so looking for \n only is sufficient
+ // Windows includes \n in its \r\n so looking for \n only is sufficient
// sep2 is set to '\0' when no list columns are present
if (*tt=='\0') {
// most common case: no sep, newline or " contained in string
@@ -271,7 +277,7 @@ static inline void writeString(SEXP x, char **thisCh)
q = TRUE;
}
if (q==FALSE) {
- writeChars(CHAR(x), &ch);
+ write_chars(CHAR(x), &ch);
} else {
*ch++ = '"';
const char *tt = CHAR(x);
@@ -293,13 +299,20 @@ static inline void writeString(SEXP x, char **thisCh)
*thisCh = ch;
}
+static void writeFactor(SEXP column, int i, char **thisCh) {
+ char *ch = *thisCh;
+ if (INTEGER(column)[i]==NA_INTEGER) write_chars(na, &ch);
+ else writeString(getAttrib(column, R_LevelsSymbol), INTEGER(column)[i]-1, &ch);
+ *thisCh = ch;
+}
+
// DATE/TIME
-static inline void writeITime(int x, char **thisCh)
+static inline void write_time(int x, char **thisCh)
{
char *ch = *thisCh;
- if (x<0) writeChars(na, &ch); // <0 covers NA_INTEGER too
- else if (dateTimeAs == DATETIMEAS_EPOCH) writeInteger(x, &ch);
- else {
+ if (x<0) { // <0 covers NA_INTEGER too (==INT_MIN checked in init.c)
+ write_chars(na, &ch);
+ } else {
int hh = x/3600;
int mm = (x - hh*3600) / 60;
int ss = x%60;
@@ -316,8 +329,11 @@ static inline void writeITime(int x, char **thisCh)
}
*thisCh = ch;
}
+static void writeITime(SEXP column, int i, char **thisCh) {
+ write_time(INTEGER(column)[i], thisCh);
+}
-static inline void writeDate(int x, char **thisCh)
+static inline void write_date(int x, char **thisCh)
{
// From base ?Date :
// " Dates are represented as the number of days since 1970-01-01, with negative values
@@ -325,10 +341,10 @@ static inline void writeDate(int x, char **thisCh)
// even though that calendar was not in use long ago (it was adopted in 1752 in Great Britain and its
// colonies) "
- // The algorithm here was taken from civil_from_days() here :
+ // The algorithm here in data.table::fwrite was taken from civil_from_days() here :
// http://howardhinnant.github.io/date_algorithms.html
- // donated to the public domain thanks to Howard Hinnant, 2013.
- // The rebase to 1 March 0000 is inspired; avoids needing isleap at all.
+ // which was donated to the public domain thanks to Howard Hinnant, 2013.
+ // The rebase to 1 March 0000 is inspired: avoids needing isleap() at all.
// The only small modifications here are :
// 1) no need for era
// 2) impose date range of [0000-03-01, 9999-12-31]. All 3,652,365 dates tested in test 1739
@@ -338,9 +354,10 @@ static inline void writeDate(int x, char **thisCh)
// as.integer(as.Date(c("0000-03-01","9999-12-31"))) == c(-719468,+2932896)
char *ch = *thisCh;
- if (x< -719468 || x>2932896) writeChars(na, &ch); // NA_INTEGER<(-719468) too (==INT_MIN checked in init.c)
- else if (dateTimeAs == DATETIMEAS_EPOCH) writeInteger(x, &ch);
- else {
+ if (x< -719468 || x>2932896) {
+ // NA_INTEGER<(-719468) (==INT_MIN checked in init.c)
+ write_chars(na, &ch);
+ } else {
x += 719468; // convert days from 1970-01-01 to days from 0000-03-01 (the day after 29 Feb 0000)
int y = (x - x/1461 + x/36525 - x/146097) / 365; // year of the preceeding March 1st
int z = x - y*365 - y/4 + y/100 - y/400 + 1; // days from March 1st in year y
@@ -364,12 +381,15 @@ static inline void writeDate(int x, char **thisCh)
}
*thisCh = ch;
}
+static void writeDateInt(SEXP column, int i, char **thisCh) {
+ write_date(INTEGER(column)[i], thisCh);
+}
+static void writeDateReal(SEXP column, int i, char **thisCh) {
+ write_date(R_FINITE(REAL(column)[i]) ? (int)REAL(column)[i] : NA_INTEGER, thisCh);
+}
-
-static inline void writePOSIXct(double x, char **thisCh)
+static void writePOSIXct(SEXP column, int i, char **thisCh)
{
- char *ch = *thisCh;
-
// Write ISO8601 UTC by default to encourage ISO standards, stymie ambiguity and for speed.
// R internally represents POSIX datetime in UTC always. Its 'tzone' attribute can be ignored.
// R's representation ignores leap seconds too which is POSIX compliant, convenient and fast.
@@ -377,9 +397,11 @@ static inline void writePOSIXct(double x, char **thisCh)
// All positive integers up to 2^53 (9e15) are exactly representable by double which is relied
// on in the ops here; number of seconds since epoch.
- if (!R_FINITE(x)) { writeChars(na, &ch); }
- else if (dateTimeAs==DATETIMEAS_EPOCH) writeNumeric(x, &ch);
- else {
+ double x = REAL(column)[i];
+ char *ch = *thisCh;
+ if (!R_FINITE(x)) {
+ write_chars(na, &ch);
+ } else {
int xi, d, t;
if (x>=0) {
xi = (int)x;
@@ -394,31 +416,31 @@ static inline void writePOSIXct(double x, char **thisCh)
int m = (int)((x-xi)*10000000); // 7th digit used to round up if 9
m += (m%10); // 9 is numerical accuracy, 8 or less then we truncate to last microsecond
m /= 10;
- writeDate(d, &ch);
+ write_date(d, &ch);
*ch++ = 'T';
ch -= squash;
- writeITime(t, &ch);
+ write_time(t, &ch);
if (squash || (m && m%1000==0)) {
- // when squash always write 3 digits of milliseconds even if 000, for consistent scale of squash integer64
- // don't use writeInteger() because it doesn't 0 pad which we need here
- // integer64 is big enough for squash with milli but not micro; trunc (not round) micro when squash
- m /= 1000;
- *ch++ = '.';
- ch -= squash;
- *(ch+2) = '0'+m%10; m/=10;
- *(ch+1) = '0'+m%10; m/=10;
- *ch = '0'+m;
- ch += 3;
+ // when squash always write 3 digits of milliseconds even if 000, for consistent scale of squash integer64
+ // don't use writeInteger() because it doesn't 0 pad which we need here
+ // integer64 is big enough for squash with milli but not micro; trunc (not round) micro when squash
+ m /= 1000;
+ *ch++ = '.';
+ ch -= squash;
+ *(ch+2) = '0'+m%10; m/=10;
+ *(ch+1) = '0'+m%10; m/=10;
+ *ch = '0'+m;
+ ch += 3;
} else if (m) {
- // microseconds are present and !squash
- *ch++ = '.';
- *(ch+5) = '0'+m%10; m/=10;
- *(ch+4) = '0'+m%10; m/=10;
- *(ch+3) = '0'+m%10; m/=10;
- *(ch+2) = '0'+m%10; m/=10;
- *(ch+1) = '0'+m%10; m/=10;
- *ch = '0'+m;
- ch += 6;
+ // microseconds are present and !squash
+ *ch++ = '.';
+ *(ch+5) = '0'+m%10; m/=10;
+ *(ch+4) = '0'+m%10; m/=10;
+ *(ch+3) = '0'+m%10; m/=10;
+ *(ch+2) = '0'+m%10; m/=10;
+ *(ch+1) = '0'+m%10; m/=10;
+ *ch = '0'+m;
+ ch += 6;
}
*ch++ = 'Z';
ch -= squash;
@@ -426,6 +448,84 @@ static inline void writePOSIXct(double x, char **thisCh)
*thisCh = ch;
}
+static void writeNanotime(SEXP column, int i, char **thisCh)
+{
+ long long x = I64(REAL(column)[i]);
+ char *ch = *thisCh;
+ if (x == NAINT64) {
+ write_chars(na, &ch);
+ } else {
+ int d/*days*/, s/*secs*/, n/*nanos*/;
+ n = x % 1000000000;
+ x /= 1000000000;
+ if (x>=0 && n>=0) {
+ d = x / 86400;
+ s = x % 86400;
+ } else {
+ // before 1970-01-01T00:00:00.000000000Z
+ if (n) { x--; n += 1000000000; }
+ d = (x+1)/86400 - 1;
+ s = x - d*86400; // x and d are both negative here; secs becomes the positive number of seconds into the day
+ }
+ write_date(d, &ch);
+ *ch++ = 'T';
+ ch -= squash;
+ write_time(s, &ch);
+ *ch++ = '.';
+ ch -= squash;
+ for (int i=8; i>=0; i--) { *(ch+i) = '0'+n%10; n/=10; } // always 9 digits for nanoseconds
+ ch += 9;
+ *ch++ = 'Z';
+ ch -= squash;
+ }
+ *thisCh = ch;
+}
+
+static void writeList(SEXP, int, char **); // prototype needed because it calls back to whichWriter too
+
+static writer_fun_t whichWriter(SEXP column) {
+ switch(TYPEOF(column)) {
+ case LGLSXP:
+ return logicalAsInt ? writeInteger : writeLogical;
+ case INTSXP:
+ if (isFactor(column)) return writeFactor;
+ if (dateTimeAs==DATETIMEAS_EPOCH) return writeInteger;
+ if (INHERITS(column, char_ITime)) return writeITime;
+ if (INHERITS(column, char_Date)) return writeDateInt;
+ return writeInteger;
+ case REALSXP:
+ if (INHERITS(column, char_integer64))
+ return (INHERITS(column, char_nanotime) && dateTimeAs!=DATETIMEAS_EPOCH) ? writeNanotime : writeInteger;
+ if (dateTimeAs==DATETIMEAS_EPOCH) return writeNumeric;
+ if (INHERITS(column, char_Date)) return writeDateReal;
+ if (INHERITS(column, char_POSIXct)) return writePOSIXct;
+ return writeNumeric;
+ case STRSXP:
+ return writeString;
+ case VECSXP:
+ return writeList;
+ default:
+ return NULL;
+ }
+}
+
+static void writeList(SEXP column, int i, char **thisCh) {
+ SEXP v = VECTOR_ELT(column,i);
+ writer_fun_t fun = whichWriter(v);
+ if (TYPEOF(v)==VECSXP || fun==NULL) {
+ error("Row %d of list column is type '%s' - not yet implemented. fwrite() can write list columns containing atomic vectors of type logical, integer, integer64, double, character and factor, currently.", i+1, type2char(TYPEOF(v)));
+ }
+ char *ch = *thisCh;
+ write_chars(sep2start, &ch);
+ for (int j=0; j<LENGTH(v); j++) {
+ (*fun)(v, j, &ch);
+ *ch++ = sep2;
+ }
+ if (LENGTH(v)) ch--; // backup over the last sep2 after the last item
+ write_chars(sep2end, &ch);
+ *thisCh = ch;
+}
+
static int failed = 0;
static int rowsPerBatch;
@@ -475,8 +575,7 @@ SEXP writefile(SEXP DFin, // any list of same length vectors; e.g.
SEXP buffMB_Arg, // [1-1024] default 8MB
SEXP nThread,
SEXP showProgress_Arg,
- SEXP verbose_Arg,
- SEXP turbo_Arg)
+ SEXP verbose_Arg)
{
if (!isNewList(DFin)) error("fwrite must be passed an object of type list; e.g. data.frame, data.table");
RLEN ncol = length(DFin);
@@ -491,12 +590,11 @@ SEXP writefile(SEXP DFin, // any list of same length vectors; e.g.
time_t next_time = start_time+2; // start printing progress meter in 2 sec if not completed by then
verbose = LOGICAL(verbose_Arg)[0];
- const Rboolean turbo = LOGICAL(turbo_Arg)[0];
sep = *CHAR(STRING_ELT(sep_Arg, 0)); // DO NOT DO: allow multichar separator (bad idea)
- const char *sep2start = CHAR(STRING_ELT(sep2_Arg, 0));
+ sep2start = CHAR(STRING_ELT(sep2_Arg, 0));
sep2 = *CHAR(STRING_ELT(sep2_Arg, 1));
- const char *sep2end = CHAR(STRING_ELT(sep2_Arg, 2));
+ sep2end = CHAR(STRING_ELT(sep2_Arg, 2));
const char *eol = CHAR(STRING_ELT(eol_Arg, 0));
// someone might want a trailer on every line so allow any length string as eol
@@ -538,41 +636,24 @@ SEXP writefile(SEXP DFin, // any list of same length vectors; e.g.
}
}
- int sameType = TYPEOF(VECTOR_ELT(DFin, 0)); // to avoid deep switch later
-
- // Store column type tests in lookup for efficiency
- // ET_INT64, ET_ITIME, ET_DATE, ET_POSIXCT, ET_FACTOR
- char *extraType = (char *)R_alloc(ncol, sizeof(char)); // not a VLA as ncol could be > 1e6 columns
-
+ // Allocate lookup vector to writer function for each column. For simplicity and robustness via many fewer lines
+ // of code and less logic need. Secondly, for efficiency to save deep switch and branches later.
+ // Don't use a VLA as ncol could be > 1e6 columns
+ writer_fun_t *fun = (writer_fun_t *)R_alloc(ncol, sizeof(writer_fun_t));
for (int j=0; j<ncol; j++) {
SEXP column = VECTOR_ELT(DF, j);
if (nrow != length(column))
error("Column %d's length (%d) is not the same as column 1's length (%d)", j+1, length(column), nrow);
- extraType[j] = 0;
- if (isFactor(column)) {
- extraType[j] = ET_FACTOR;
- } else if (INHERITS(column, char_integer64)) {
- if (TYPEOF(column)!=REALSXP) error("Column %d inherits from 'integer64' but is type '%s' not REALSXP", j+1, type2char(TYPEOF(column)));
- extraType[j] = ET_INT64;
- } else if (INHERITS(column, char_ITime)) {
- extraType[j] = ET_ITIME;
- } else if (INHERITS(column, char_Date)) { // including IDate which inherits from Date
- extraType[j] = ET_DATE;
- } else if (INHERITS(column, char_POSIXct)) {
- if (dateTimeAs==DATETIMEAS_WRITECSV) error("Internal error: column should have already been coerced to character");
- extraType[j] = ET_POSIXCT;
- }
- if (TYPEOF(column)!=sameType || getAttrib(column,R_ClassSymbol)!=R_NilValue) {
- sameType = 0; // only all plain INTSXP or all plain REALSXP save the deep switch() below.
- }
- if (firstListColumn==0 && TYPEOF(column)==VECSXP) firstListColumn = j+1;
+ if ((fun[j] = whichWriter(column)) == NULL)
+ error("Column %d's type is '%s' - not yet implemented.", j+1, type2char(TYPEOF(column)) );
+ if (TYPEOF(column)==VECSXP && firstListColumn==0) firstListColumn = j+1;
}
-
+
if (!firstListColumn) {
if (verbose) Rprintf("No list columns are present. Setting sep2='' otherwise quote='auto' would quote fields containing sep2.\n");
sep2='\0';
} else {
- if (verbose) Rprintf("If quote='auto' all fields will be quoted if the field contains either sep ('%c') or sep2[2] ('%c'). Column %d is a list column.\n", sep, sep2, firstListColumn );
+ if (verbose) Rprintf("If quote='auto', fields will be quoted if the field contains either sep ('%c') or sep2[2] ('%c') because column %d is a list column.\n", sep, sep2, firstListColumn );
if (dec==sep) error("Internal error: dec != sep was checked at R level");
if (dec==sep2 || sep==sep2)
error("sep ('%c'), sep2[2L] ('%c') and dec ('%c') must all be different when list columns are present. Column %d is a list column.", sep, sep2, dec, firstListColumn);
@@ -589,7 +670,7 @@ SEXP writefile(SEXP DFin, // any list of same length vectors; e.g.
// Estimate max line length of a 1000 row sample (100 rows in 10 places).
// 'Estimate' even of this sample because quote='auto' may add quotes and escape embedded quotes.
// Buffers will be resized later if there are too many line lengths outside the sample, anyway.
- // maxLineLen is just used to determine rowsPerBatch.
+ // maxLineLen is required to determine a reasonable rowsPerBatch.
int maxLineLen = 0;
int na_len = strlen(na);
int step = nrow<1000 ? 100 : nrow/10;
@@ -605,89 +686,39 @@ SEXP writefile(SEXP DFin, // any list of same length vectors; e.g.
}
for (int j=0; j<ncol; j++) {
SEXP column = VECTOR_ELT(DF, j);
- static char tmp[32]; // +- 15digits dec e +- nnn \0 = 23 + 9 safety = 32. Covers integer64 too (20 digits).
+ static char tmp[64]; // +- 15digits dec e +- nnn \0 = 23 + 9 safety = 32. Covers integer64 too (20 digits).
+ // nanotime could be 36 chars: yyyy-mm-ddTHH:MM:SS.000000000+00:00. So be safe with 64.
char *ch=tmp;
- switch(TYPEOF(column)) {
- case LGLSXP:
- thisLineLen += logicalAsInt ? 1/*0|1*/ : 5/*FALSE*/; // na_len might be 2 (>1) but ok; this is estimate
- break;
- case INTSXP: {
- int i32 = INTEGER(column)[i];
- if (i32 == NA_INTEGER) ch += na_len;
- else if (extraType[j] == ET_FACTOR) ch += LENGTH(STRING_ELT(getAttrib(column,R_LevelsSymbol), i32-1));
- else if (extraType[j] == ET_ITIME) ch += 8;
- else if (extraType[j] == ET_DATE) writeDate(i32, &ch);
- else writeInteger(i32, &ch); }
- thisLineLen += (int)(ch-tmp);
- break;
- case REALSXP:
- if (extraType[j] == ET_INT64) {
- long long i64 = *(long long *)&REAL(column)[i];
- if (i64==NAINT64) ch += na_len;
- else writeInteger(i64, &ch);
- }
- else if (extraType[j]==ET_DATE) writeDate((int)REAL(column)[i], &ch);
- else if (extraType[j]==ET_POSIXCT) writePOSIXct(REAL(column)[i], &ch);
- else writeNumeric(REAL(column)[i], &ch);
- thisLineLen += (int)(ch-tmp);
- break;
- case STRSXP:
- thisLineLen += LENGTH(STRING_ELT(column, i));
- break;
- case VECSXP: {
+ if (TYPEOF(column)==STRSXP) thisLineLen+=LENGTH(STRING_ELT(column, i));
+ else if (isFactor(column)) {
+ int k = INTEGER(column)[i];
+ thisLineLen += (k==NA_INTEGER) ? na_len : LENGTH(STRING_ELT(getAttrib(column,R_LevelsSymbol), k-1));
+ } else if (TYPEOF(column)==VECSXP) {
// a list column containing atomic vectors in each cell
+ // for thisLineLen calculation we don't want to allocate tmp wide enough for the potentially
+ // long field. Just to know its width we can add up the width of each item individually.
SEXP v = VECTOR_ELT(column,i);
thisLineLen += strlen(sep2start);
- switch(TYPEOF(v)) {
- case LGLSXP :
- thisLineLen += LENGTH(v) * (logicalAsInt ? 1 : 5);
- break;
- case INTSXP:
- if (isFactor(v)) {
- SEXP l = getAttrib(v, R_LevelsSymbol);
- for (int k=0; k<LENGTH(v); k++) {
- thisLineLen += INTEGER(v)[k]==NA_INTEGER ? na_len : LENGTH(STRING_ELT(l, INTEGER(v)[k]-1));
- }
- }
- else if (INHERITS(v, char_ITime)) thisLineLen += LENGTH(v) * (6 + 2*!squash);
- else if (INHERITS(v, char_Date)) thisLineLen += LENGTH(v) * (8 + 2*!squash);
- else {
- for (int k=0; k<LENGTH(v); k++) {
- if ( INTEGER(v)[k]==NA_INTEGER ) thisLineLen += na_len;
- else { writeInteger(INTEGER(v)[k], &ch); thisLineLen+=(int)(ch-tmp); ch=tmp; }
- }
+ if (TYPEOF(v)==STRSXP) for (int k=0; k<LENGTH(v); k++) thisLineLen += LENGTH(STRING_ELT(v, k));
+ else if (isFactor(v)) {
+ SEXP l = getAttrib(v, R_LevelsSymbol);
+ for (int k=0; k<LENGTH(v); k++) {
+ thisLineLen += INTEGER(v)[k]==NA_INTEGER ? na_len : LENGTH(STRING_ELT(l, INTEGER(v)[k]-1));
}
- break;
- case REALSXP:
- if (INHERITS(v, char_integer64)) {
- for (int k=0; k<LENGTH(v); k++) {
- long long i64 = *(long long *)&REAL(v)[k];
- if (i64==NAINT64) thisLineLen += na_len;
- else {
- writeInteger(i64, &ch);
- thisLineLen += (int)(ch-tmp);
- ch=tmp;
- }
- }
- }
- else if (INHERITS(v, char_Date)) thisLineLen += LENGTH(v) * (8 + 2*!squash);
- else if (INHERITS(v, char_POSIXct)) {
- for (int k=0; k<LENGTH(v); k++) { writePOSIXct(REAL(v)[k], &ch); thisLineLen+=(int)(ch-tmp); ch=tmp; }
- } else {
- for (int k=0; k<LENGTH(v); k++) { writeNumeric(REAL(v)[k], &ch); thisLineLen+=(int)(ch-tmp); ch=tmp; }
- }
- break;
- case STRSXP:
- for (int k=0; k<LENGTH(v); k++) thisLineLen += LENGTH(STRING_ELT(v, k));
- break;
- default:
- error("Column %d is a list column but on row %d is type '%s' - not yet implemented. fwrite() can write list columns containing atomic vectors of type logical, integer, integer64, double, character and factor, currently.", j+1, type2char(TYPEOF(v)));
- } // end switch on atomic vector type in a list column
- thisLineLen += LENGTH(v)/*sep2 after each field*/ +strlen(sep2end); }
- // LENGTH(v) could be 0 so we don't subtract one for the last sep2 (just an estimate anyway)
- break; // from case VECSXP for list column
- default:
- error("Column %d's type is '%s' - not yet implemented.", j+1, type2char(TYPEOF(column)) );
+ } else {
+ writer_fun_t fun = whichWriter(v);
+ if (fun==NULL) error("Column %d is a list column but on row %d is type '%s' - not yet implemented. fwrite() can write list columns containing atomic vectors of type logical, integer, integer64, double, character and factor, currently.", j+1, i+1, type2char(TYPEOF(v)));
+ for (int k=0; k<LENGTH(v); k++) {
+ (*fun)(v, k, &ch); thisLineLen+=(int)(ch-tmp); ch=tmp;
+ }
+ }
+ thisLineLen += LENGTH(v); // sep2 after each field. LENGTH(v) could be 0 so don't -1 (only estimate anyway)
+ thisLineLen += strlen(sep2end);
+ } else {
+ // regular atomic columns like integer, integer64, double, date and time
+ (*fun[j])(column, i, &ch);
+ thisLineLen += (int)(ch-tmp);
+ ch = tmp;
}
thisLineLen++; // column sep
} // next column
@@ -740,11 +771,11 @@ SEXP writefile(SEXP DFin, // any list of same length vectors; e.g.
*ch++ = sep;
}
for (int j=0; j<ncol; j++) {
- writeString(STRING_ELT(names, j), &ch);
+ writeString(names, j, &ch);
*ch++ = sep;
}
ch--; // backup onto the last sep after the last column
- writeChars(eol, &ch); // replace it with the newline
+ write_chars(eol, &ch); // replace it with the newline
if (f==-1) { *ch='\0'; Rprintf(buffer); }
else if (WRITE(f, buffer, (int)(ch-buffer))==-1) {
int errwrite=errno;
@@ -782,8 +813,8 @@ SEXP writefile(SEXP DFin, // any list of same length vectors; e.g.
int numBatches = (nrow-1)/rowsPerBatch + 1;
if (numBatches < nth) nth = numBatches;
if (verbose) {
- Rprintf("Writing %d rows in %d batches of %d rows (each buffer size %dMB, turbo=%d, showProgress=%d, nth=%d) ... ",
- nrow, numBatches, rowsPerBatch, buffMB, turbo, showProgress, nth);
+ Rprintf("Writing %d rows in %d batches of %d rows (each buffer size %dMB, showProgress=%d, nth=%d) ... ",
+ nrow, numBatches, rowsPerBatch, buffMB, showProgress, nth);
if (f==-1) Rprintf("\n");
}
t0 = clock();
@@ -824,199 +855,33 @@ SEXP writefile(SEXP DFin, // any list of same length vectors; e.g.
if (failed) continue; // Not break. See comments above about #omp cancel
int end = ((nrow-start)<rowsPerBatch) ? nrow : start+rowsPerBatch;
- // all-integer and all-double deep switch() avoidance. We could code up all-integer64
- // as well but that seems even less likely in practice than all-integer or all-double
- if (turbo && sameType==REALSXP && !doRowNames) {
- // avoid deep switch() on type. turbo switches on both sameType and specialized writeNumeric
- for (RLEN i=start; i<end; i++) {
- char *lineStart = ch;
- for (int j=0; j<ncol; j++) {
- SEXP column = VECTOR_ELT(DF, j);
- writeNumeric(REAL(column)[i], &ch);
- *ch++ = sep;
- }
- ch--; // backup onto the last sep after the last column
- writeChars(eol, &ch); // replace it with the newline.
-
- size_t thisLineLen = ch-lineStart;
- if (thisLineLen > myMaxLineLen) myMaxLineLen=thisLineLen;
- checkBuffer(&buffer, &myAlloc, &ch, myMaxLineLen);
- if (failed) break;
- }
- } else if (turbo && sameType==INTSXP && !doRowNames) {
- for (RLEN i=start; i<end; i++) {
- char *lineStart = ch;
- for (int j=0; j<ncol; j++) {
- SEXP column = VECTOR_ELT(DF, j);
- if (INTEGER(column)[i] == NA_INTEGER) {
- writeChars(na, &ch);
- } else {
- writeInteger(INTEGER(column)[i], &ch);
- }
- *ch++ = sep;
+ for (RLEN i=start; i<end; i++) {
+ char *lineStart = ch;
+ if (doRowNames) {
+ if (rowNames==NULL) {
+ if (quote!=FALSE) *ch++='"'; // default 'auto' will quote the row.name numbers
+ write_positive_int(i+1, &ch);
+ if (quote!=FALSE) *ch++='"';
+ } else {
+ writeString(rowNames, i, &ch);
}
- ch--;
- writeChars(eol, &ch);
-
- size_t thisLineLen = ch-lineStart;
- if (thisLineLen > myMaxLineLen) myMaxLineLen=thisLineLen;
- checkBuffer(&buffer, &myAlloc, &ch, myMaxLineLen);
- if (failed) break;
+ *ch++=sep;
}
- } else {
- // mixed types. switch() on every cell value since must write row-by-row
- for (RLEN i=start; i<end; i++) {
- char *lineStart = ch;
- if (doRowNames) {
- if (rowNames==NULL) {
- if (quote!=FALSE) *ch++='"'; // default 'auto' will quote the row.name numbers
- writeInteger(i+1, &ch);
- if (quote!=FALSE) *ch++='"';
- } else {
- writeString(STRING_ELT(rowNames, i), &ch);
- }
- *ch++=sep;
- }
- for (int j=0; j<ncol; j++) {
- SEXP column = VECTOR_ELT(DF, j);
- switch(TYPEOF(column)) {
- case LGLSXP:
- writeLogical(LOGICAL(column)[i], &ch);
- break;
- case INTSXP:
- if (INTEGER(column)[i] == NA_INTEGER) {
- writeChars(na, &ch);
- } else if (extraType[j] == ET_FACTOR) {
- writeString(STRING_ELT(getAttrib(column,R_LevelsSymbol), INTEGER(column)[i]-1), &ch);
- } else if (extraType[j] == ET_ITIME) {
- writeITime(INTEGER(column)[i], &ch);
- } else if (extraType[j] == ET_DATE) {
- writeDate(INTEGER(column)[i], &ch);
- } else {
- writeInteger(INTEGER(column)[i], &ch);
- }
- break;
- case REALSXP:
- if (extraType[j] == ET_INT64) {
- long long i64 = *(long long *)&REAL(column)[i];
- if (i64 == NAINT64) {
- writeChars(na, &ch);
- } else {
- writeInteger(i64, &ch);
- }
- } else {
- if (extraType[j] == ET_DATE) {
- writeDate( R_FINITE(REAL(column)[i]) ? (int)REAL(column)[i] : NA_INTEGER, &ch);
- } else if (extraType[j] == ET_POSIXCT) {
- writePOSIXct(REAL(column)[i], &ch);
- } else if (turbo) {
- writeNumeric(REAL(column)[i], &ch); // handles NA, Inf etc within it
- } else {
- // if there are any problems with the specialized writeNumeric, user can revert to (slower) standard library
- if (ISNAN(REAL(column)[i])) {
- writeChars(na, &ch);
- } else {
- ch += sprintf(ch, "%.15g", REAL(column)[i]);
- }
- }
- }
- break;
- case STRSXP:
- writeString(STRING_ELT(column, i), &ch);
- break;
-
- case VECSXP: {
- // a list column containing atomic vectors in each cell
- SEXP v = VECTOR_ELT(column,i);
- writeChars(sep2start, &ch);
- switch(TYPEOF(v)) {
- case LGLSXP :
- for (int k=0; k<LENGTH(v); k++) {
- writeLogical(LOGICAL(v)[k], &ch);
- *ch++ = sep2;
- }
- break;
- case INTSXP:
- if (isFactor(v)) {
- SEXP l = getAttrib(v, R_LevelsSymbol);
- for (int k=0; k<LENGTH(v); k++) {
- if (INTEGER(v)[k]==NA_INTEGER) writeChars(na, &ch);
- else writeString(STRING_ELT(l, INTEGER(v)[k]-1), &ch);
- *ch++ = sep2;
- }
- } else if (INHERITS(v, char_ITime)) {
- for (int k=0; k<LENGTH(v); k++) {
- writeITime(INTEGER(v)[k], &ch);
- *ch++ = sep2;
- }
- } else if (INHERITS(v, char_Date)) {
- for (int k=0; k<LENGTH(v); k++) {
- writeDate(INTEGER(v)[k], &ch);
- *ch++ = sep2;
- }
- } else {
- for (int k=0; k<LENGTH(v); k++) {
- if (INTEGER(v)[k]==NA_INTEGER ) writeChars(na, &ch);
- else writeInteger(INTEGER(v)[k], &ch);
- *ch++ = sep2;
- }
- }
- break;
- case REALSXP:
- if (INHERITS(v, char_integer64)) {
- for (int k=0; k<LENGTH(v); k++) {
- long long i64 = *(long long *)&REAL(v)[k];
- if (i64==NAINT64) writeChars(na, &ch);
- else writeInteger(i64, &ch);
- *ch++ = sep2;
- }
- } else if (INHERITS(v, char_Date)) {
- for (int k=0; k<LENGTH(v); k++) {
- writeDate(R_FINITE(REAL(v)[k]) ? (int)REAL(v)[k] : NA_INTEGER, &ch);
- *ch++ = sep2;
- }
- } else if (INHERITS(v, char_POSIXct)) {
- for (int k=0; k<LENGTH(v); k++) {
- writePOSIXct(REAL(v)[k], &ch);
- *ch++ = sep2;
- }
- } else {
- for (int k=0; k<LENGTH(v); k++) {
- writeNumeric(REAL(v)[k], &ch);
- *ch++ = sep2;
- }
- }
- break;
- case STRSXP:
- for (int k=0; k<LENGTH(v); k++) {
- writeString(STRING_ELT(v, k), &ch);
- *ch++ = sep2;
- }
- break;
- default:
- error("Column %d is a list column but on row %d is type '%s' - not yet implemented. fwrite() can write list columns containing atomic vectors of type logical, integer, integer64, double, character and factor, currently.", j+1, type2char(TYPEOF(v)));
- } // end switch on atomic vector type in a list column
- if (LENGTH(v)) ch--; // backup over the last sep2 after the last item
- writeChars(sep2end, &ch); }
- break; // from case VECSXP for list column
-
- default:
- error("Internal error: unsupported column type should have been thrown above when calculating maxLineLen");
- }
- *ch++ = sep; // next column
- }
- ch--; // backup onto the last sep after the last column. 0-columns was caught and returned earlier, so >=1 cols.
- writeChars(eol, &ch); // replace it with the newline.
-
- // Track longest line seen so far. If we start to see longer lines than we saw in the
- // sample, we'll realloc the buffer. The rowsPerBatch chosen based on the (very good) sample,
- // must fit in the buffer. Can't early write and reset buffer because the
- // file output would be out-of-order. Can't change rowsPerBatch after the 'parallel for' started.
- size_t thisLineLen = ch-lineStart;
- if (thisLineLen > myMaxLineLen) myMaxLineLen=thisLineLen;
- checkBuffer(&buffer, &myAlloc, &ch, myMaxLineLen);
- if (failed) break; // don't write any more rows, fall through to clear up and error() below
+ for (int j=0; j<ncol; j++) {
+ (*fun[j])(VECTOR_ELT(DF, j), i, &ch);
+ *ch++ = sep;
}
+ ch--; // backup onto the last sep after the last column. ncol>=1 because 0-columns was caught earlier.
+ write_chars(eol, &ch); // replace it with the newline.
+
+ // Track longest line seen so far. If we start to see longer lines than we saw in the
+ // sample, we'll realloc the buffer. The rowsPerBatch chosen based on the (very good) sample,
+ // must fit in the buffer. Can't early write and reset buffer because the
+ // file output would be out-of-order. Can't change rowsPerBatch after the 'parallel for' started.
+ size_t thisLineLen = ch-lineStart;
+ if (thisLineLen > myMaxLineLen) myMaxLineLen=thisLineLen;
+ checkBuffer(&buffer, &myAlloc, &ch, myMaxLineLen);
+ if (failed) break; // this thread stop writing rows; fall through to clear up and error() below
}
#pragma omp ordered
{
diff --git a/src/gsumm.c b/src/gsumm.c
index bfe4f50..c3b1364 100644
--- a/src/gsumm.c
+++ b/src/gsumm.c
@@ -23,19 +23,25 @@ static union {double d;
# define SQRTL sqrt
#endif
-SEXP gstart(SEXP o, SEXP f, SEXP l, SEXP irowsArg) {
+SEXP gforce(SEXP env, SEXP jsub, SEXP o, SEXP f, SEXP l, SEXP irowsArg) {
int i, j, g, *this;
// clock_t start = clock();
- if (!isInteger(o)) error("o is not integer vector");
- if (!isInteger(f)) error("f is not integer vector");
- if (!isInteger(l)) error("l is not integer vector");
+ if (TYPEOF(env) != ENVSXP) error("env is not an environment");
+ // The type of jsub is pretty flexbile in R, so leave checking to eval() below.
+ if (!isInteger(o)) error("o is not an integer vector");
+ if (!isInteger(f)) error("f is not an integer vector");
+ if (!isInteger(l)) error("l is not an integer vector");
+ if (!isInteger(irowsArg) && !isNull(irowsArg)) error("irowsArg is not an integer vector");
ngrp = LENGTH(l);
if (LENGTH(f) != ngrp) error("length(f)=%d != length(l)=%d", LENGTH(f), ngrp);
grpn=0;
- grpsize = INTEGER(l); // l will be protected in calling R scope until gend(), too
+ grpsize = INTEGER(l);
for (i=0; i<ngrp; i++) grpn+=grpsize[i];
if (LENGTH(o) && LENGTH(o)!=grpn) error("o has length %d but sum(l)=%d", LENGTH(o), grpn);
+
grp = (int *)R_alloc(grpn, sizeof(int));
+ // global grp because the g* functions (inside jsub) share this common memory
+
if (LENGTH(o)) {
isunsorted = 1; // for gmedian
for (g=0; g<ngrp; g++) {
@@ -56,14 +62,20 @@ SEXP gstart(SEXP o, SEXP f, SEXP l, SEXP irowsArg) {
irows = INTEGER(irowsArg);
if (!isNull(irowsArg)) irowslen = length(irowsArg);
-
- // Rprintf("gstart took %8.3f\n", 1.0*(clock()-start)/CLOCKS_PER_SEC);
- return(R_NilValue);
-}
-
-SEXP gend() {
+
+ SEXP ans = PROTECT( eval(jsub, env) );
+ // if this eval() fails with R error, R will release grp for us. Which is why we use R_alloc above.
+ if (isVectorAtomic(ans)) {
+ SEXP tt = ans;
+ ans = PROTECT(allocVector(VECSXP, 1));
+ SET_VECTOR_ELT(ans, 0, tt);
+ UNPROTECT(1);
+ }
ngrp = 0; maxgrpn = 0; irowslen = -1; isunsorted = 0;
- return(R_NilValue);
+
+ // Rprintf("gforce took %8.3f\n", 1.0*(clock()-start)/CLOCKS_PER_SEC);
+ UNPROTECT(1);
+ return(ans);
}
// long double usage here results in test 648 being failed when running with valgrind
diff --git a/src/init.c b/src/init.c
index eb7c35f..564b3d6 100644
--- a/src/init.c
+++ b/src/init.c
@@ -34,8 +34,7 @@ SEXP uniqlengths();
SEXP setrev();
SEXP forder();
SEXP fsorted();
-SEXP gstart();
-SEXP gend();
+SEXP gforce();
SEXP gsum();
SEXP gmean();
SEXP gmin();
@@ -114,8 +113,7 @@ R_CallMethodDef callMethods[] = {
{"Csetrev", (DL_FUNC) &setrev, -1},
{"Cforder", (DL_FUNC) &forder, -1},
{"Cfsorted", (DL_FUNC) &fsorted, -1},
-{"Cgstart", (DL_FUNC) &gstart, -1},
-{"Cgend", (DL_FUNC) &gend, -1},
+{"Cgforce", (DL_FUNC) &gforce, -1},
{"Cgsum", (DL_FUNC) &gsum, -1},
{"Cgmean", (DL_FUNC) &gmean, -1},
{"Cgmin", (DL_FUNC) &gmin, -1},
@@ -203,7 +201,8 @@ void attribute_visible R_init_datatable(DllInfo *info)
memset(&ld, 0, sizeof(long double));
if (ld != 0.0) error("Checking memset(&ld, 0, sizeof(long double)); ld == (long double)0.0 %s", msg);
- setNumericRounding(ScalarInteger(0)); // #1642, #1728, #1463, #485
+ setNumericRounding(PROTECT(ScalarInteger(0))); // #1642, #1728, #1463, #485
+ UNPROTECT(1);
// create needed strings in advance for speed, same techique as R_*Symbol
// Following R-exts 5.9.4; paragraph and example starting "Using install ..."
@@ -212,16 +211,28 @@ void attribute_visible R_init_datatable(DllInfo *info)
char_ITime = PRINTNAME(install("ITime"));
char_Date = PRINTNAME(install("Date")); // used for IDate too since IDate inherits from Date
char_POSIXct = PRINTNAME(install("POSIXct"));
+ char_nanotime = PRINTNAME(install("nanotime"));
+ char_starts = PRINTNAME(sym_starts = install("starts"));
if (TYPEOF(char_integer64) != CHARSXP) {
// checking one is enough in case of any R-devel changes
error("PRINTNAME(install(\"integer64\")) has returned %s not %s",
type2char(TYPEOF(char_integer64)), type2char(CHARSXP));
}
+ // create commonly used symbols, same as R_*Symbol but internal to DT
+ // Not really for speed but to avoid leak in situations like setAttrib(DT, install(), allocVector()) where
+ // the allocVector() can happen first and then the install() could gc and free it before it is protected
+ // within setAttrib. Thanks to Bill Dunlap finding and reporting. Using these symbols instead of install()
+ // avoids the gc without needing an extra PROTECT and immediate UNPROTECT after the setAttrib which would
+ // look odd (and devs in future might be tempted to remove them). Avoiding passing install() to API calls
+ // keeps the code neat and readable. Also see grep's added to CRAN_Release.cmd to find such calls.
+ sym_sorted = install("sorted");
+ sym_BY = install(".BY");
+ sym_maxgrpn = install("maxgrpn");
+
avoid_openmp_hang_within_fork();
}
-
inline Rboolean INHERITS(SEXP x, SEXP char_) {
// Thread safe inherits() by pre-calling install() above in init first then
// passing those char_* in here for simple and fast non-API pointer compare.
@@ -240,6 +251,15 @@ inline Rboolean INHERITS(SEXP x, SEXP char_) {
return FALSE;
}
+inline long long I64(double x) {
+ // type punning such as *(long long *)&REAL(column)[i] is undefined and I think was the
+ // cause of 1.10.2 failing on 31 Jan 2017 under clang 3.9.1 -O3 and solaris-sparc but
+ // not solaris-x86 or gcc. There is now a grep in CRAN_Release.cmd; use this union method instead.
+ union {double d; long long ll;} u;
+ u.d = x;
+ return u.ll;
+}
+
SEXP hasOpenMP() {
// Just for use by onAttach to avoid an RPRINTF from C level which isn't suppressable by CRAN
// There is now a 'grep' in CRAN_Release.cmd to detect any use of RPRINTF in init.c, which is
diff --git a/src/rbindlist.c b/src/rbindlist.c
index b5188ed..cc08300 100644
--- a/src/rbindlist.c
+++ b/src/rbindlist.c
@@ -404,11 +404,12 @@ static SEXP fast_order(SEXP dt, R_len_t byArg, R_len_t handleSorted) {
}
ans = PROTECT(forder(dt, by, retGrp, sortStr, order, na)); protecti++;
if (!length(ans) && handleSorted != 0) {
- starts = PROTECT(getAttrib(ans, mkString("starts"))); protecti++;
+ starts = getAttrib(ans, sym_starts);
// if cols are already sorted, 'forder' gives integer(0), got to replace it with 1:.N
ans = PROTECT(allocVector(INTSXP, length(VECTOR_ELT(dt, 0)))); protecti++;
for (i=0; i<length(ans); i++) INTEGER(ans)[i] = i+1;
- setAttrib(ans, install("starts"), starts);
+ // TODO: for loop appears redundant because length(ans)==0 due to if (!length(ans)) above
+ setAttrib(ans, sym_starts, starts);
}
UNPROTECT(protecti); // ans
return(ans);
@@ -441,7 +442,7 @@ static SEXP match_names(SEXP v) {
runid = VECTOR_ELT(dt, 2);
uorder = PROTECT(fast_order(dt, 2, 1)); protecti++; // byArg alone is set, everything else is set inside fast_order
- starts = PROTECT(getAttrib(uorder, mkString("starts"))); protecti++;
+ starts = getAttrib(uorder, sym_starts);
ulens = PROTECT(uniq_lengths(starts, length(lnames))); protecti++;
// seq_len(.N) for each group
@@ -452,7 +453,7 @@ static SEXP match_names(SEXP v) {
}
// order again
uorder = PROTECT(fast_order(dt, 2, 1)); protecti++; // byArg alone is set, everything else is set inside fast_order
- starts = PROTECT(getAttrib(uorder, mkString("starts"))); protecti++;
+ starts = getAttrib(uorder, sym_starts);
ulens = PROTECT(uniq_lengths(starts, length(lnames))); protecti++;
ncols = length(starts);
// check if order has to be changed (bysameorder = FALSE here by default - in `[.data.table` parlance)
@@ -827,7 +828,7 @@ SEXP chmatch2_old(SEXP x, SEXP table, SEXP nomatch) {
// order - first time
order = PROTECT(fast_order(dt, 2, 1));
- start = PROTECT(getAttrib(order, mkString("starts")));
+ start = getAttrib(order, sym_starts);
lens = PROTECT(uniq_lengths(start, length(order))); // length(order) = nrow(dt)
grpid = VECTOR_ELT(dt, 1);
index = VECTOR_ELT(dt, 2);
@@ -841,9 +842,9 @@ SEXP chmatch2_old(SEXP x, SEXP table, SEXP nomatch) {
k += j;
}
// order - again
- UNPROTECT(3); // order, start, lens
+ UNPROTECT(2); // order, lens
order = PROTECT(fast_order(dt, 2, 1));
- start = PROTECT(getAttrib(order, mkString("starts")));
+ start = getAttrib(order, sym_starts);
lens = PROTECT(uniq_lengths(start, length(order)));
ans = PROTECT(allocVector(INTSXP, nx));
@@ -855,7 +856,7 @@ SEXP chmatch2_old(SEXP x, SEXP table, SEXP nomatch) {
if (oi > nx-1) continue;
INTEGER(ans)[oi] = (li == 2) ? INTEGER(index)[INTEGER(order)[si+1]-1]+1 : INTEGER(nomatch)[0];
}
- UNPROTECT(5); // order, start, lens, ans
+ UNPROTECT(4); // order, lens, ans
return(ans);
}
@@ -868,7 +869,7 @@ static SEXP listlist(SEXP x) {
lx = PROTECT(allocVector(VECSXP, 1));
SET_VECTOR_ELT(lx, 0, x);
xo = PROTECT(fast_order(lx, 1, 1));
- xs = PROTECT(getAttrib(xo, mkString("starts")));
+ xs = getAttrib(xo, sym_starts);
xl = PROTECT(uniq_lengths(xs, length(x)));
ans0 = PROTECT(allocVector(STRSXP, length(xs)));
@@ -887,7 +888,7 @@ static SEXP listlist(SEXP x) {
ans = PROTECT(allocVector(VECSXP, 2));
SET_VECTOR_ELT(ans, 0, ans0);
SET_VECTOR_ELT(ans, 1, ans1);
- UNPROTECT(7);
+ UNPROTECT(6);
return(ans);
}
diff --git a/src/subset.c b/src/subset.c
index 1a12128..893740a 100644
--- a/src/subset.c
+++ b/src/subset.c
@@ -277,18 +277,21 @@ SEXP subsetDT(SEXP x, SEXP rows, SEXP cols) {
UNPROTECT(1);
// maintain key if ordered subset ...
- SEXP key = getAttrib(x, install("sorted"));
+ SEXP key = getAttrib(x, sym_sorted);
if (length(key)) {
SEXP in = PROTECT(chmatch(key,getAttrib(ans,R_NamesSymbol), 0, TRUE)); // (nomatch ignored when in=TRUE)
int i = 0; while(i<LENGTH(key) && LOGICAL(in)[i]) i++;
UNPROTECT(1);
// i is now the keylen that can be kept. 2 lines above much easier in C than R
if (i==0) {
- setAttrib(ans, install("sorted"), R_NilValue);
+ setAttrib(ans, sym_sorted, R_NilValue);
// clear key that was copied over by copyMostAttrib() above
- } else if (isOrderedSubset(rows, ScalarInteger(length(VECTOR_ELT(x,0))))) {
- setAttrib(ans, install("sorted"), tmp=allocVector(STRSXP, i));
- for (int j=0; j<i; j++) SET_STRING_ELT(tmp, j, STRING_ELT(key, j));
+ } else {
+ if (isOrderedSubset(rows, PROTECT(ScalarInteger(length(VECTOR_ELT(x,0)))))) {
+ setAttrib(ans, sym_sorted, tmp=allocVector(STRSXP, i));
+ for (int j=0; j<i; j++) SET_STRING_ELT(tmp, j, STRING_ELT(key, j));
+ }
+ UNPROTECT(1); // the ScalarInteger above. isOrderedSubset() is exposed at R level hence needs SEXP
}
}
setAttrib(ans, install(".data.table.locked"), R_NilValue);
diff --git a/src/wrappers.c b/src/wrappers.c
index f8fff6c..9f483ae 100644
--- a/src/wrappers.c
+++ b/src/wrappers.c
@@ -14,7 +14,7 @@ SEXP setattrib(SEXP x, SEXP name, SEXP value)
isString(value) && (strcmp(CHAR(STRING_ELT(value, 0)), "data.table") == 0 ||
strcmp(CHAR(STRING_ELT(value, 0)), "data.frame") == 0) )
error("Internal structure doesn't seem to be a list. Can't set class to be 'data.table' or 'data.frame'. Use 'as.data.table()' or 'as.data.frame()' methods instead.");
- if (isLogical(x) && x == ScalarLogical(TRUE)) {
+ if (isLogical(x) && x == ScalarLogical(TRUE)) { // ok not to protect this ScalarLogical() as not assigned or passed
x = PROTECT(duplicate(x));
setAttrib(x, name, NAMED(value) ? duplicate(value) : value);
UNPROTECT(1);
@@ -79,7 +79,7 @@ SEXP address(SEXP x)
// A better way than : http://stackoverflow.com/a/10913296/403310
char buffer[32];
snprintf(buffer, 32, "%p", (void *)x);
- return(ScalarString(mkChar(buffer)));
+ return(mkString(buffer));
}
SEXP copyNamedInList(SEXP x)
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/debian-med/r-cran-data.table.git
More information about the debian-med-commit
mailing list