[med-svn] [r-cran-future] 01/01: New upstream version 1.1.1
Michael Crusoe
misterc-guest at moszumanska.debian.org
Thu Nov 10 11:00:03 UTC 2016
This is an automated email from the git hooks/post-receive script.
misterc-guest pushed a commit to annotated tag upstream/1.1.1
in repository r-cran-future.
commit 57e780c6df6cc63c0eecb00ba59248fef07703f7
Author: Michael R. Crusoe <michael.crusoe at gmail.com>
Date: Thu Nov 10 02:53:52 2016 -0800
New upstream version 1.1.1
---
.Rbuildignore | 11 +
.gitignore | 3 +
.travis.yml | 35 +-
CONDUCT.md | 74 ++++
DESCRIPTION | 42 ++-
NAMESPACE | 12 +-
NEWS | 50 ++-
R/ClusterFuture-class.R | 70 +++-
R/ClusterRegistry.R | 34 +-
R/ConstantFuture-class.R | 10 +-
R/EagerFuture-class.R | 32 +-
R/Future-class.R | 200 ++++++-----
R/LazyFuture-class.R | 44 ++-
R/MulticoreFuture-class.R | 36 +-
R/MultiprocessFuture-class.R | 12 +-
R/MultisessionFuture-class.R | 6 +-
R/UniprocessFuture-class.R | 20 +-
R/availableCores.R | 13 +-
R/cluster.R | 26 +-
R/constant.R | 3 +-
R/eager.R | 33 +-
R/flapply.R | 45 ++-
R/future.R | 178 ++++++++-
R/futureAssign.R | 51 +--
R/futureAssign_OP.R | 36 +-
R/futureCall.R | 8 +-
R/futures.R | 4 +-
R/globals.R | 137 ++++---
R/globals_OP.R | 18 +
R/label_OP.R | 18 +
R/lazy.R | 34 +-
R/multicore.R | 29 +-
R/multiprocess.R | 28 +-
R/multisession.R | 22 +-
R/nbrOfWorkers.R | 4 +-
R/plan_OP.R | 2 +-
R/remote.R | 62 ++--
R/resolve.R | 2 +-
R/tweak.R | 2 +-
R/tweakExpression.R | 141 ++++++--
R/tweak_OP.R | 2 +-
R/tweak_parallel_PSOCK.R | 149 ++++++++
R/utils.R | 4 +-
R/values.R | 2 +-
README.md | 104 +++---
appveyor.yml | 10 +-
cran-comments.md | 36 +-
demo/mandelbrot.R | 3 +-
incl/future.R | 73 +++-
incl/futureAssign_OP.R | 32 --
incl/nbrOfWorkers.R | 8 +
inst/vignettes-static/future-1-overview.md.rsp.rsp | 12 +-
man/ClusterFuture-class.Rd | 39 +-
man/ConstantFuture-class.Rd | 18 +-
man/EagerFuture-class.Rd | 18 +-
man/Future-class.Rd | 12 +-
man/LazyFuture-class.Rd | 18 +-
man/MulticoreFuture-class.Rd | 12 +-
man/MultiprocessFuture-class.Rd | 9 +-
man/UniprocessFuture-class.Rd | 4 +-
man/availableCores.Rd | 15 +-
man/cluster.Rd | 44 ++-
man/eager.Rd | 29 +-
man/flapply.Rd | 23 ++
man/future.Rd | 276 ++++++++++++--
man/futureAssign.Rd | 88 -----
man/futures.Rd | 4 +-
man/getExpression.Rd | 2 +-
man/getGlobalsAndPackages.Rd | 35 ++
man/grapes-globals-grapes.Rd | 24 ++
man/grapes-label-grapes.Rd | 21 ++
man/grapes-plan-grapes.Rd | 2 +-
man/grapes-tweak-grapes.Rd | 2 +-
man/lazy.Rd | 21 +-
man/multicore.Rd | 31 +-
man/multiprocess.Rd | 29 +-
man/multisession.Rd | 31 +-
man/nbrOfWorkers.Rd | 14 +-
man/remote.Rd | 42 ++-
man/resolve.Rd | 2 +-
man/run.Rd | 27 ++
man/tweak.Rd | 6 +-
man/tweak_parallel_PSOCK.Rd | 68 ++++
man/values.Rd | 2 +-
revdep/README.md | 399 +++++++++++++++++++++
revdep/check.R | 20 ++
revdep/checks.rds | Bin 0 -> 8524 bytes
revdep/problems.md | 32 ++
revdep/timing.md | 25 ++
tests/ClusterRegistry.R | 12 +
tests/Future-class.R | 25 ++
tests/early-signaling.R | 10 +
tests/future,labels.R | 40 +++
tests/globals,formulas.R | 136 +++++++
tests/globals,manual.R | 141 ++++++++
tests/globals,subassignment.R | 143 ++++++++
tests/incl/end.R | 3 +
tests/incl/start,load-only.R | 5 +
tests/multicore.R | 2 +-
tests/multisession.R | 25 ++
tests/nested_futures.R | 86 +++++
tests/rng.R | 5 +
tests/tweak.R | 16 +
tests/utils.R | 10 +-
vignettes/future-1-overview.md.rsp | 94 ++---
vignettes/future-2-issues.md.rsp | 190 +++-------
vignettes/future-3-topologies.md.rsp | 14 +-
vignettes/future-4-startup.md.rsp | 76 ++++
108 files changed, 3354 insertions(+), 1145 deletions(-)
diff --git a/.Rbuildignore b/.Rbuildignore
index 968c558..cc3088f 100644
--- a/.Rbuildignore
+++ b/.Rbuildignore
@@ -7,6 +7,12 @@
INSTALL[.]md
OVERVIEW[.]md
README[.]md
+CONDUCT[.]md
+
+#----------------------------
+# devtools
+#----------------------------
+^revdep
#----------------------------
# Travis-CI et al.
@@ -44,4 +50,9 @@ Rplots.pdf$
^[.]BatchJobs[.]R$
[.]future
+#----------------------------
+# Miscellaneous
+#----------------------------
+^.ghi
+^.issues
diff --git a/.gitignore b/.gitignore
index 41e9bc6..86ab9cf 100644
--- a/.gitignore
+++ b/.gitignore
@@ -14,3 +14,6 @@
*.swp
.covr.rds
.future
+.ghi
+.issues
+
diff --git a/.travis.yml b/.travis.yml
index b022f0c..35447e1 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -10,29 +10,30 @@
language: r
sudo: false
cache: packages
-
-os:
- - linux
- - osx
-
-r:
- - oldrel
- - release
- - devel
+warnings_are_errors: false
+r_check_args: --as-cran
+latex: false
matrix:
- exclude:
- - os: osx
+ include:
+ - os: linux
+ r: oldrel
+ - os: linux
+ r: release
+ r_github_packages:
+ - jimhester/covr
+ after_success:
+ - Rscript -e 'covr::codecov(quiet=FALSE)'
+ env: NB='w/ covr' ## Just a label
+ - os: linux
r: devel
+ - os: osx
+ r: oldrel
+ - os: osx
+ r: release
-r_github_packages:
- - jimhester/covr
-
-after_success:
- - Rscript -e 'covr::codecov(quiet=FALSE)'
notifications:
email:
on_success: change
on_failure: change
-
diff --git a/CONDUCT.md b/CONDUCT.md
new file mode 100644
index 0000000..9c1c621
--- /dev/null
+++ b/CONDUCT.md
@@ -0,0 +1,74 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, gender identity and expression, level of experience,
+nationality, personal appearance, race, religion, or sexual identity and
+orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+ address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+ professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at [http://contributor-covenant.org/version/1/4][version]
+
+[homepage]: http://contributor-covenant.org
+[version]: http://contributor-covenant.org/version/1/4/
diff --git a/DESCRIPTION b/DESCRIPTION
index 7e4d793..83f705e 100644
--- a/DESCRIPTION
+++ b/DESCRIPTION
@@ -1,9 +1,9 @@
Package: future
-Version: 1.0.1
+Version: 1.1.1
Title: A Future API for R
Imports:
digest,
- globals (>= 0.6.1),
+ globals (>= 0.7.0),
listenv (>= 0.6.0),
parallel,
utils
@@ -15,26 +15,24 @@ Authors at R: c(person("Henrik", "Bengtsson", role=c("aut", "cre", "cph"),
email = "henrikb at braju.com"))
Description: A Future API for R is provided. In programming, a future is an
abstraction for a value that may be available at some point in the future.
- The state of a future can either be unresolved or resolved. As soon as it is
- resolved, the value is available. Futures are useful constructs in for instance
- concurrent evaluation, e.g. parallel processing and distributed processing on
- compute clusters. The purpose of this package is to provide a lightweight
- interface for using futures in R. Functions 'future()' and 'value()' exist for
- creating futures and requesting their values, e.g.
- 'f <- future({ mandelbrot(-0.75, 0, side=3) })' and 'v <- value(f)'.
- The 'resolved()' function can be used to check if a future is resolved or not.
- An infix assignment operator '%<-%' exists for creating futures whose values
- are accessible by the assigned variables (as promises), e.g.
- 'v %<-% { mandelbrot(-0.75, 0, side=3) }'.
- This package implements synchronous "lazy" and "eager" futures, and asynchronous
- "multicore", "multisession" and ad hoc "cluster" futures.
- Globals variables and functions are automatically identified and exported.
- Required packages are attached in external R sessions whenever needed.
- All types of futures are designed to behave the same such that the exact
- same code work regardless of futures used or number of cores, background
- sessions or cluster nodes available.
- Additional types of futures are provided by other packages enhancing
- this package.
+ The state of a future can either be unresolved or resolved. As soon as it
+ is resolved, the value is available. Futures are useful constructs in for
+ instance concurrent evaluation, e.g. parallel processing and distributed
+ processing on compute clusters. The purpose of this package is to provide
+ a lightweight interface for using futures in R. Functions 'future()' and
+ 'value()' exist for creating futures and requesting their values, e.g.
+ 'f <- future({ mandelbrot(-0.75, 0, side=3) })' and 'v <- value(f)'. The
+ 'resolved()' function can be used to check if a future is resolved or
+ not. An infix assignment operator '%<-%' exists for creating futures whose
+ values are accessible by the assigned variables (as promises), e.g. 'v %<-%
+ { mandelbrot(-0.75, 0, side=3) }'. This package implements synchronous "lazy"
+ and "eager" futures, and asynchronous "multicore", "multisession" and ad hoc
+ "cluster" futures. Globals variables and functions are automatically identified
+ and exported. Required packages are attached in external R sessions whenever
+ needed. All types of futures are designed to behave the same such that the
+ exact same code work regardless of futures used or number of cores, background
+ sessions or cluster nodes available. Additional types of futures are provided by
+ other packages enhancing this package.
License: LGPL (>= 2.1)
LazyLoad: TRUE
URL: https://github.com/HenrikBengtsson/future
diff --git a/NAMESPACE b/NAMESPACE
index b1902d8..157f2aa 100644
--- a/NAMESPACE
+++ b/NAMESPACE
@@ -29,6 +29,10 @@ S3method(resolved,MulticoreFuture)
S3method(resolved,default)
S3method(resolved,environment)
S3method(resolved,list)
+S3method(run,ClusterFuture)
+S3method(run,Future)
+S3method(run,MulticoreFuture)
+S3method(run,UniprocessFuture)
S3method(tweak,character)
S3method(tweak,future)
S3method(value,ClusterFuture)
@@ -43,6 +47,8 @@ export("%->%")
export("%<-%")
export("%<=%")
export("%=>%")
+export("%globals%")
+export("%label%")
export("%plan%")
export("%tweak%")
export(ClusterFuture)
@@ -77,15 +83,19 @@ export(plan)
export(remote)
export(resolve)
export(resolved)
+export(run)
export(supportsMulticore)
export(transparent)
export(tweak)
export(value)
export(values)
importFrom(digest,digest)
+importFrom(globals,as.Globals)
importFrom(globals,cleanup)
+importFrom(globals,globalsByName)
importFrom(globals,globalsOf)
importFrom(globals,packagesOf)
+importFrom(globals,walkAST)
importFrom(grDevices,as.raster)
importFrom(grDevices,hsv)
importFrom(graphics,par)
@@ -98,8 +108,8 @@ importFrom(parallel,clusterCall)
importFrom(parallel,clusterExport)
importFrom(parallel,makeCluster)
importFrom(parallel,stopCluster)
+importFrom(utils,assignInNamespace)
importFrom(utils,capture.output)
importFrom(utils,file_test)
importFrom(utils,head)
importFrom(utils,object.size)
-importFrom(utils,packageVersion)
diff --git a/NEWS b/NEWS
index 8062425..06ce6d5 100644
--- a/NEWS
+++ b/NEWS
@@ -1,12 +1,60 @@
Package: future
===============
+Version: 1.1.1 [2016-10-10]
+o FIX: For the special case where 'remote' futures use
+ workers = "localhost" they (again) use the exact same R executable
+ as the main / calling R session (in all other cases it uses
+ whatever 'Rscript' is found in the PATH). This was already indeed
+ implemented in 1.0.1, but with the added support for reverse SSH
+ tunnels in 1.1.0 this default behavior was lost.
+
+
+Version: 1.1.0 [2016-10-09]
+o REMOTE CLUSTERS: It is now very simple to use cluster() and remote()
+ to connect to remote clusters / machines. As long as you can connect
+ via ssh to those machines, it works also with these future. The new
+ code completely avoids incoming firewall and incoming port forwarding
+ issues previously needed. This is done by using reverse SSH tunneling.
+ There is also no need to worry about internal or external IP numbers.
+o GLOBALS: Global variables part of subassignments in future expressions
+ are recognized and exported (iff found), e.g. x$a <- value,
+ x[["a"]] <- value, and x[1,2,3] <- value.
+o GLOBALS: Global variables part of formulae in future expressions are
+ recognized and exported (iff found), e.g. y ~ x | z.
+o GLOBALS: As an alternative to the default automatic identification of
+ globals, it is now also possible to explicitly specify them either
+ by their names (as a character vector) or by their names and values
+ (as a named list), e.g. f <- future({ 2*a }, globals=c("a")) or
+ f <- future({ 2*a }, globals=list(a=42)). For future assignments one
+ can use the %globals% operator, e.g. y %<-% { 2*a } %globals% c("a").
+o Now all Future classes supports run() for launching the future and
+ value() calls run() if the future has not been launched.
+o Added optional argument 'label' to all futures, e.g.
+ f <- future(42, label="answer") and v %<-% { 42 } %label% "answer".
+o Added argument 'user' to cluster() and remote().
+o ROBUSTNESS: Now the default future strategy is explicitly set when
+ no strategies are set, e.g. when used nested futures. Previously,
+ only mc.cores was set so that only a single core was used, but now
+ also plan("default") set.
+o WORKAROUND: resolved() on cluster futures would block on Linux until
+ future was resolved. This is due to a bug in R. The workaround is
+ to use round the timeout (in seconds) to an integer, which seems to
+ always work / be respected.
+o DOCUMENTATION: Added vignette on command-line options and other
+ methods for controlling the default type of futures to use.
+o MEMORY: Now plan(cluster, gc=TRUE) causes the background
+ R session to be garbage collected immediately after the value
+ is collected. Since multisession and remote futures are special
+ cases of cluster futures, the same is true for these as well.
+
+
Version: 1.0.1 [2016-07-04]
o DOCUMENTATION: Adding section to vignette on globals in formulas
describing how they are currently not automatically detected
and how to explicitly export them.
o ROBUSTNESS: For the special case where 'remote' futures use
- workers = "localhost" they now uses the exact same R executable
+ workers = "localhost" they now use the exact same R executable
as the main / calling R session (in all other cases it uses
whatever 'Rscript' is found in the PATH).
o FutureError now extends simpleError and no longer the error
diff --git a/R/ClusterFuture-class.R b/R/ClusterFuture-class.R
index 2ee8f1f..f329fae 100644
--- a/R/ClusterFuture-class.R
+++ b/R/ClusterFuture-class.R
@@ -1,21 +1,26 @@
#' A cluster future is a future whose value will be resolved asynchroneously in a parallel process
#'
-#' @param expr An R \link[base]{expression}.
-#' @param envir The \link{environment} in which the evaluation
-#' is done (or inherits from if \code{local} is TRUE).
-#' @param substitute If TRUE, argument \code{expr} is
-#' \code{\link[base]{substitute}()}:ed, otherwise not.
-#' @param local If TRUE, the expression is evaluated such that
-#' all assignments are done to local temporary environment, otherwise
-#' the assignments are done in the global environment of the cluster node.
-#' @param gc If TRUE, the garbage collector run after the future
-#' is resolved (in the process that evaluated the future).
+#' @inheritParams MultiprocessFuture-class
+#' @param globals (optional) a logical, a character vector,
+#' or a named list for controlling how globals are handled.
+#' For details, see section 'Globals used by future expressions'
+#' in the help for \code{\link{future}()}.
#' @param persistent If FALSE, the evaluation environment is cleared
#' from objects prior to the evaluation of the future.
#' @param workers A \code{\link[parallel:makeCluster]{cluster}}.
#' Alternatively, a character vector of host names or a numeric scalar,
#' for creating a cluster via \code{\link[parallel]{makeCluster}(workers)}.
-#' @param \dots Additional named elements of the future.
+#' @param revtunnel If TRUE, reverse SSH tunneling is used for the
+#' PSOCK cluster nodes to connect back to the master R process. This
+#' avoids the hassle of firewalls, port forwarding and having to know
+#' the internal / public IP address of the master R session.
+#' @param user (optional) The user name to be used when communicating
+#' with another host.
+#' @param master (optional) The hostname or IP address of the master
+#' machine running this node.
+#' @param homogeneous If TRUE, all cluster nodes is assumed to use the
+#' same path to \file{Rscript} as the main R session. If FALSE, the
+#' it is assumed to be on the PATH for each node.
#'
#' @return An object of class \code{ClusterFuture}.
#'
@@ -29,7 +34,7 @@
#' @importFrom digest digest
#' @name ClusterFuture-class
#' @keywords internal
-ClusterFuture <- function(expr=NULL, envir=parent.frame(), substitute=FALSE, local=!persistent, gc=!persistent, persistent=FALSE, workers=NULL, ...) {
+ClusterFuture <- function(expr=NULL, envir=parent.frame(), substitute=FALSE, local=!persistent, globals=TRUE, gc=FALSE, persistent=FALSE, workers=NULL, user=NULL, master=NULL, revtunnel=TRUE, homogeneous=TRUE, ...) {
defaultCluster <- importParallel("defaultCluster")
## BACKWARD COMPATIBILITY
@@ -44,7 +49,7 @@ ClusterFuture <- function(expr=NULL, envir=parent.frame(), substitute=FALSE, loc
if (is.null(workers)) {
workers <- defaultCluster()
} else if (is.character(workers) || is.numeric(workers)) {
- workers <- ClusterRegistry("start", workers=workers)
+ workers <- ClusterRegistry("start", workers=workers, user=user, master=master, revtunnel=revtunnel, homogeneous=homogeneous)
} else if (!inherits(workers, "cluster")) {
stop("Argument 'workers' is not of class 'cluster': ", class(workers)[1])
}
@@ -58,16 +63,26 @@ ClusterFuture <- function(expr=NULL, envir=parent.frame(), substitute=FALSE, loc
attr(workers, "name") <- name
}
- gp <- getGlobalsAndPackages(expr, envir=envir, persistent=persistent)
+ ## Global objects
+ gp <- getGlobalsAndPackages(expr, envir=envir, persistent=persistent, globals=globals)
+ globals <- gp$globals
+ packages <- gp$packages
+ expr <- gp$expr
+ gp <- NULL
- f <- MultiprocessFuture(expr=gp$expr, envir=envir, substitute=FALSE, local=local, persistent=persistent, globals=gp$globals, packages=gp$packages, workers=workers, node=NA_integer_, ...)
+ f <- MultiprocessFuture(expr=expr, envir=envir, substitute=FALSE, local=local, gc=gc, persistent=persistent, globals=globals, packages=packages, workers=workers, node=NA_integer_, ...)
structure(f, class=c("ClusterFuture", class(f)))
}
#' @importFrom parallel clusterCall clusterExport
+#' @export
run.ClusterFuture <- function(future, ...) {
+ if (future$state != 'created') {
+ stop("A future can only be launched once.")
+ }
+
## Assert that the process that created the future is
## also the one that evaluates/resolves/queries it.
assertOwner(future)
@@ -174,6 +189,13 @@ resolved.ClusterFuture <- function(x, timeout=0.2, ...) {
node <- x$node
cl <- workers[node]
+ ## WORKAROUND: Non-integer timeouts (at least < 2.0 seconds) may
+ ## result in infinite waiting, cf.
+ ## https://stat.ethz.ch/pipermail/r-devel/2016-October/073218.html
+ if (.Platform$OS.type != "windows") {
+ timeout <- round(timeout, digits = 0L)
+ }
+
## Check if workers socket connection is available for reading
con <- cl[[1]]$con
res <- socketSelect(list(con), write=FALSE, timeout=timeout)
@@ -191,6 +213,10 @@ value.ClusterFuture <- function(future, ...) {
return(NextMethod("value"))
}
+ if (future$state == 'created') {
+ future <- run(future)
+ }
+
## Assert that the process that created the future is
## also the one that evaluates/resolves/queries it.
assertOwner(future)
@@ -238,6 +264,18 @@ value.ClusterFuture <- function(future, ...) {
## Remove from registry
FutureRegistry(reg, action="remove", future=future, earlySignal=FALSE)
+ ## Garbage collect cluster worker?
+ if (future$gc) {
+ ## Cleanup global environment while at it
+ if (!future$persistent) clusterCall(cl[1], fun=grmall)
+
+ ## WORKAROUND: Need to clear cluster worker before garbage collection,
+ ## cf. https://github.com/HenrikBengtsson/Wishlist-for-R/issues/27
+ clusterCall(cl[1], function() NULL)
+
+ clusterCall(cl[1], gc, verbose=FALSE, reset=FALSE)
+ }
+
NextMethod("value")
}
@@ -287,7 +325,7 @@ requestNode <- function(await, workers, times=getOption("future.wait.times", 600
## Find which node is available
avail <- rep(TRUE, times=length(workers))
futures <- FutureRegistry(reg, action="list", earlySignal=FALSE)
- nodes <- unlist(lapply(futures, FUN=function(f) f$node))
+ nodes <- unlist(lapply(futures, FUN=function(f) f$node), use.names=FALSE)
avail[nodes] <- FALSE
## Sanity check
diff --git a/R/ClusterRegistry.R b/R/ClusterRegistry.R
index 7d08bb7..109b2fd 100644
--- a/R/ClusterRegistry.R
+++ b/R/ClusterRegistry.R
@@ -4,11 +4,41 @@ ClusterRegistry <- local({
last <- NULL
cluster <- NULL
- .makeCluster <- function(workers, ...) {
+ .makeCluster <- function(workers, user=NULL, master=NULL, revtunnel=FALSE, ...) {
if (is.null(workers)) return(NULL)
+
+ debug <- getOption("future.debug", FALSE)
+ if (debug) mdebug("ClusterRegister:::.makeCluster() ...")
+
+ ## HACKS:
+ ## 1. Don't pass ssh option `-l <user>` unless `user` is specified
+ ## 2. Connect via reverse SSH tunneling.
+ if (debug) {
+ mdebug("tweak_parallel_PSOCK(user=%s, revtunnel=%s, rshopts=TRUE)",
+ is.null(user), revtunnel)
+ }
+ on.exit(tweak_parallel_PSOCK(reset=TRUE), add=TRUE)
+ tweak_parallel_PSOCK(user=is.null(user), revtunnel=revtunnel, rshopts=TRUE)
+
+ if (debug) {
+ on.exit(suppressMessages(untrace(system)), add=TRUE)
+ suppressMessages(
+ trace(system, print=FALSE, tracer=quote(message(command)))
+ )
+ }
+
+ if (debug) {
+ on.exit(mdebug("ClusterRegister:::.makeCluster() ... DONE"), add=TRUE)
+ }
+
+ ## This will _not_ pass `master` iff master=NULL
+ args <- list(workers, revtunnel=revtunnel, ...)
+ args$master <- master
+
capture.output({
- cluster <- makeCluster(workers, ...)
+ cluster <- do.call(makeCluster, args=args)
})
+
cluster
}
diff --git a/R/ConstantFuture-class.R b/R/ConstantFuture-class.R
index f0d7961..63a3a65 100644
--- a/R/ConstantFuture-class.R
+++ b/R/ConstantFuture-class.R
@@ -1,9 +1,9 @@
#' A future with a constant value
#'
#' A constant future is a future whose expression is a constant
-#' and therefore already resolved upon creation
+#' and therefore by definition already resolved upon creation.
#'
-#' @param expr An R value.
+#' @inheritParams Future-class
#' @param \dots Not used.
#'
#' @return An object of class \code{ConstantFuture}.
@@ -11,13 +11,11 @@
#' @export
#' @name ConstantFuture-class
#' @keywords internal
-ConstantFuture <- function(expr=NULL, envir=emptyenv(), substitute=FALSE, local=FALSE, gc=FALSE, ...) {
+ConstantFuture <- function(expr=NULL, envir=emptyenv(), substitute=FALSE, local=FALSE, ...) {
expr <- force(expr)
- f <- Future(expr=expr, envir=emptyenv(), substitute=FALSE, local=FALSE, gc=FALSE, ...)
+ f <- Future(expr=expr, envir=emptyenv(), substitute=FALSE, local=FALSE, ...)
f$value <- expr
f$state <- "finished"
structure(f, class=c("ConstantFuture", class(f)))
f
}
-
-evaluate.ConstantFuture <- function(future, ...) future
diff --git a/R/EagerFuture-class.R b/R/EagerFuture-class.R
index 4af322a..e5f90f2 100644
--- a/R/EagerFuture-class.R
+++ b/R/EagerFuture-class.R
@@ -1,15 +1,6 @@
#' An eager future is a future whose value will be resolved immediately
#'
-#' @param expr An R \link[base]{expression}.
-#' @param envir The \link{environment} in which the evaluation
-#' is done (or inherits from if \code{local} is TRUE).
-#' @param substitute If TRUE, argument \code{expr} is
-#' \code{\link[base]{substitute}()}:ed, otherwise not.
-#' @param local If TRUE, the expression is evaluated such that
-#' all assignments are done to local temporary environment, otherwise
-#' the assignments are done in the calling environment.
-#' @param gc If TRUE, the garbage collector run after the future is resolved.
-#' @param \dots Additional named elements of the future.
+#' @inheritParams UniprocessFuture-class
#'
#' @return An object of class \code{EagerFuture}.
#'
@@ -20,8 +11,25 @@
#' @export
#' @name EagerFuture-class
#' @keywords internal
-EagerFuture <- function(expr=NULL, envir=parent.frame(), substitute=FALSE, local=TRUE, gc=FALSE, ...) {
+EagerFuture <- function(expr=NULL, envir=parent.frame(), substitute=FALSE, globals=TRUE, local=TRUE, ...) {
if (substitute) expr <- substitute(expr)
- f <- UniprocessFuture(expr=expr, envir=envir, substitute=FALSE, local=local, gc=gc, ...)
+
+ ## Global objects
+ assignToTarget <- (is.list(globals) || inherits(globals, "Globals"))
+ gp <- getGlobalsAndPackages(expr, envir=envir, tweak=tweakExpression, globals=globals, resolve=TRUE)
+
+ ## Assign?
+ if (assignToTarget && length(gp) > 0) {
+ target <- new.env(parent=envir)
+ globalsT <- gp$globals
+ for (name in names(globalsT)) {
+ target[[name]] <- globalsT[[name]]
+ }
+ globalsT <- NULL
+ envir <- target
+ }
+ gp <- NULL
+
+ f <- UniprocessFuture(expr=expr, envir=envir, substitute=FALSE, local=local, ...)
structure(f, class=c("EagerFuture", class(f)))
}
diff --git a/R/Future-class.R b/R/Future-class.R
index f67d172..a48fd5c 100644
--- a/R/Future-class.R
+++ b/R/Future-class.R
@@ -15,10 +15,13 @@
#' @param local If TRUE, the expression is evaluated such that
#' all assignments are done to local temporary environment, otherwise
#' the assignments are done in the calling environment.
-#' @param gc If TRUE, the garbage collector run after the future
-#' is resolved (in the process that evaluated the future).
+#' @param gc If TRUE, the garbage collector run (in the process that
+#' evaluated the future) after the value of the future is collected.
+#' \emph{Some types of futures ignore this argument.}
#' @param earlySignal Specified whether conditions should be signaled
#' as soon as possible or not.
+#' @param label An optional character string label attached to the
+#' future.
#' @param \dots Additional named elements of the future.
#'
#' @return An object of class \code{Future}.
@@ -29,14 +32,14 @@
#' @seealso
#' One function that creates a Future is \code{\link{future}()}.
#' It returns a Future that evaluates an R expression in the future.
-#' An alternative approach is to use the \code{\link{\%<=\%}} infix
+#' An alternative approach is to use the \code{\link{\%<-\%}} infix
#' assignment operator, which creates a future from the
#' right-hand-side (RHS) R expression and assigns its future value
#' to a variable as a \emph{\link[base]{promise}}.
#'
#' @export
#' @name Future-class
-Future <- function(expr=NULL, envir=parent.frame(), substitute=FALSE, local=TRUE, gc=FALSE, earlySignal=FALSE, ...) {
+Future <- function(expr=NULL, envir=parent.frame(), substitute=FALSE, local=TRUE, gc=FALSE, earlySignal=FALSE, label=NULL, ...) {
if (substitute) expr <- substitute(expr)
args <- list(...)
@@ -47,6 +50,7 @@ Future <- function(expr=NULL, envir=parent.frame(), substitute=FALSE, local=TRUE
core$local <- local
core$gc <- gc
core$earlySignal <- earlySignal
+ core$label <- label
## The current state of the future, e.g.
## 'created', 'running', 'finished', 'failed', 'interrupted'.
@@ -64,6 +68,9 @@ Future <- function(expr=NULL, envir=parent.frame(), substitute=FALSE, local=TRUE
print.Future <- function(x, ...) {
class <- class(x)
cat(sprintf("%s:\n", class[1]))
+ label <- x$label
+ if (is.null(label)) label <- "<none>"
+ cat("Label: ", sQuote(label), "\n", sep="")
cat("Expression:\n")
print(x$expr)
cat(sprintf("Local evaluation: %s\n", x$local))
@@ -127,6 +134,35 @@ assertOwner <- function(future, ...) {
}
+#' Run a future
+#'
+#' @param future A \link{Future}.
+#' @param \dots Not used.
+#'
+#' @return The \link{Future} object.
+#'
+#' @details
+#' This function can only be called once per future.
+#' Further calls will result in an informative error.
+#' If a future is not run when its value is queried,
+#' then it is run at that point.
+#'
+#' @aliases run
+#' @rdname run
+#' @export
+#' @export run
+run.Future <- function(future, ...) {
+ if (future$state != 'created') {
+ stop("A future can only be launched once.")
+ }
+
+ future
+}
+
+run <- function(...) UseMethod("run")
+
+
+
#' The value of a future
#'
#' Gets the value of a future. If the future is unresolved, then
@@ -148,6 +184,10 @@ assertOwner <- function(future, ...) {
#' @export
#' @export value
value.Future <- function(future, signal=TRUE, ...) {
+ if (future$state == 'created') {
+ future <- run(future)
+ }
+
if (!future$state %in% c('finished', 'failed', 'interrupted')) {
msg <- sprintf("Internal error: value() called on a non-finished future: %s", class(future)[1])
mdebug(msg)
@@ -182,7 +222,7 @@ resolved.Future <- function(x, ...) {
#' Inject code for the next type of future to use for nested futures
#'
#' @param future Current future.
-#' @param ... Not used.
+#' @param \dots Not used.
#'
#' @return A future expression with code injected to set what
#' type of future to use for nested futures, iff any.
@@ -218,21 +258,11 @@ getExpression <- function(future, ...) UseMethod("getExpression")
#' @export
getExpression.Future <- function(future, mc.cores=NULL, ...) {
- strategies <- plan("list")
-
- ## If end of future stack, fall back to using single-core
- ## processing. In this case we don't have to rely
- ## on the future package. Instead, we can use the
- ## light-weight approach where we force the number of
- ## cores available to be one. This we achieve by
- ## setting the number of _additional_ cores to be
- ## zero (sic!).
- if (length(strategies) == 0) {
- mc.cores <- 0L
- }
+## mdebug("getExpression() ...")
## Should 'mc.cores' be set?
if (!is.null(mc.cores)) {
+## mdebug("getExpression(): setting mc.cores=%d inside future", mc.cores)
## FIXME: How can we guarantee that '...future.mc.cores.old'
## is not overwritten? /HB 2016-03-14
enter <- bquote({
@@ -249,64 +279,80 @@ getExpression.Future <- function(future, mc.cores=NULL, ...) {
enter <- exit <- NULL
}
- if (length(strategies) > 0) {
- exit <- bquote({
+ ## Reset future strategies upon exit of future
+ strategies <- plan("list")
+ stopifnot(length(strategies) >= 1L)
+ exit <- bquote({
+ ## covr: skip=2
+ .(exit)
+ future::plan(.(strategies))
+ })
+
+ ## Pass down the default or the remain set of future strategies?
+ strategiesR <- strategies[-1]
+## mdebug("Number of remaining strategies: %d\n", length(strategiesR))
+ if (length(strategiesR) == 0L) {
+## mdebug("Set plan('default') inside future")
+ ## Use default future strategy
+ enter <- bquote({
## covr: skip=2
- .(exit)
- future::plan(.(strategies))
+ .(enter)
+ future::plan("default")
})
- }
-
- ## Identify package namespaces for strategies
- pkgs <- lapply(strategies, FUN=environment)
- pkgs <- lapply(pkgs, FUN=environmentName)
- pkgs <- unique(unlist(pkgs))
- pkgs <- intersect(pkgs, loadedNamespaces())
- mdebug("Packages to be loaded by expression (n=%d): %s", length(pkgs), paste(sQuote(pkgs), collapse=", "))
+ } else {
+## mdebug("Set plan(<remaining strategies>) inside future")
+ ## Identify package namespaces for strategies
+ pkgs <- lapply(strategiesR, FUN=environment)
+ pkgs <- lapply(pkgs, FUN=environmentName)
+ pkgs <- unique(unlist(pkgs, use.names=FALSE))
+ pkgs <- intersect(pkgs, loadedNamespaces())
+## mdebug("Packages to be loaded by expression (n=%d): %s", length(pkgs), paste(sQuote(pkgs), collapse=", "))
+
+ if (length(pkgs) > 0L) {
+ ## Sanity check by verifying packages can be loaded already here
+ ## If there is somethings wrong in 'pkgs', we get the error
+ ## already before launching the future.
+ for (pkg in pkgs) loadNamespace(pkg)
- if (length(pkgs) > 0L) {
- ## Sanity check by verifying packages can be loaded already here
- ## If there is somethings wrong in 'pkgs', we get the error
- ## already before launching the future.
- for (pkg in pkgs) loadNamespace(pkg)
-
- enter <- bquote({
- ## covr: skip=3
- .(enter)
- ## TROUBLESHOOTING: If the package fails to load, then library()
- ## suppress that error and generates a generic much less
- ## informative error message. Because of this, we load the
- ## namespace first (to get a better error message) and then
- ## calls library(), which attaches the package. /HB 2016-06-16
- ## NOTE: We use local() here such that 'pkg' is not assigned
- ## to the future environment. /HB 2016-07-03
- local({
- for (pkg in .(pkgs)) {
- loadNamespace(pkg)
- library(pkg, character.only=TRUE)
- }
+ enter <- bquote({
+ ## covr: skip=3
+ .(enter)
+ ## TROUBLESHOOTING: If the package fails to load, then library()
+ ## suppress that error and generates a generic much less
+ ## informative error message. Because of this, we load the
+ ## namespace first (to get a better error message) and then
+ ## calls library(), which attaches the package. /HB 2016-06-16
+ ## NOTE: We use local() here such that 'pkg' is not assigned
+ ## to the future environment. /HB 2016-07-03
+ local({
+ for (pkg in .(pkgs)) {
+ loadNamespace(pkg)
+ library(pkg, character.only=TRUE)
+ }
+ })
})
- })
- } else {
+ }
+
+ ## Pass down future strategies
enter <- bquote({
## covr: skip=2
.(enter)
+ future::plan(.(strategiesR))
})
- }
+ } ## if (length(strategiesR) > 0L)
- if (length(strategies) >= 2L) {
- enter <- bquote({
- ## covr: skip=2
- .(enter)
- future::plan(.(strategies[-1]))
- })
+ expr <- makeExpression(expr=future$expr, local=future$local, enter=enter, exit=exit)
+ if (getOption("future.debug", FALSE)) {
+ print(expr)
}
- makeExpression(expr=future$expr, local=future$local, gc=future$gc, enter=enter, exit=exit)
+## mdebug("getExpression() ... DONE")
+
+ expr
} ## getExpression()
-makeExpression <- function(expr, local=TRUE, gc=FALSE, globals.onMissing=getOption("future.globals.onMissing", "error"), enter=NULL, exit=NULL) {
+makeExpression <- function(expr, local=TRUE, globals.onMissing=getOption("future.globals.onMissing", "error"), enter=NULL, exit=NULL) {
## Evaluate expression in a local() environment?
if (local) {
a <- NULL; rm(list="a") ## To please R CMD check
@@ -335,29 +381,15 @@ makeExpression <- function(expr, local=TRUE, gc=FALSE, globals.onMissing=getOpti
## evaluation in a local is optional, cf. argument 'local'.
## If this was mandatory, we could. Instead we use
## a tryCatch() statement. /HB 2016-03-14
- if (gc) {
- expr <- substitute({
- ## covr: skip=8
- enter
- ...future.value <- tryCatch({
- body
- }, finally = {
- exit
- })
- gc(verbose=FALSE, reset=FALSE)
- ...future.value
- }, env=list(enter=enter, body=expr, exit=exit, cleanup=cleanup))
- } else {
- expr <- substitute({
- ## covr: skip=6
- enter
- tryCatch({
- body
- }, finally = {
- exit
- })
- }, env=list(enter=enter, body=expr, exit=exit))
- }
+ expr <- substitute({
+ ## covr: skip=6
+ enter
+ tryCatch({
+ body
+ }, finally = {
+ exit
+ })
+ }, env=list(enter=enter, body=expr, exit=exit))
expr
} ## makeExpression()
diff --git a/R/LazyFuture-class.R b/R/LazyFuture-class.R
index 5bd7280..a449428 100644
--- a/R/LazyFuture-class.R
+++ b/R/LazyFuture-class.R
@@ -1,15 +1,6 @@
#' A lazy future is a future whose value will be resolved at the time when it is requested
#'
-#' @param expr An R \link[base]{expression}.
-#' @param envir The \link{environment} in which the evaluation
-#' is done (or inherits from if \code{local} is TRUE).
-#' @param substitute If TRUE, argument \code{expr} is
-#' \code{\link[base]{substitute}()}:ed, otherwise not.
-#' @param local If TRUE, the expression is evaluated such that
-#' all assignments are done to local temporary environment, otherwise
-#' the assignments are done in the calling environment.
-#' @param gc If TRUE, the garbage collector run after the future is resolved.
-#' @param \dots Additional named elements of the future.
+#' @inheritParams UniprocessFuture-class
#'
#' @return An object of class \code{LazyFuture}.
#'
@@ -20,9 +11,34 @@
#' @export
#' @name LazyFuture-class
#' @keywords internal
-LazyFuture <- function(expr=NULL, envir=parent.frame(), substitute=FALSE, local=TRUE, gc=FALSE, ...) {
+LazyFuture <- function(expr=NULL, envir=parent.frame(), substitute=FALSE, globals=TRUE, local=TRUE, ...) {
if (substitute) expr <- substitute(expr)
- f <- UniprocessFuture(expr=expr, envir=envir, substitute=FALSE, local=local, gc=gc, ...)
+
+ ## Evaluate in a local environment?
+ if (local) {
+ envir <- new.env(parent=envir)
+ } else {
+ if (!is.logical(globals) || globals) {
+ stop("Non-supported use of lazy futures: Whenever argument 'local' is FALSE, then argument 'globals' must also be FALSE. Lazy future evaluation in the calling environment (local=FALSE) can only be done if global objects are resolved at the same time.")
+ }
+ }
+
+ ## Global objects
+ gp <- getGlobalsAndPackages(expr, envir=envir, tweak=tweakExpression, globals=globals, resolve=TRUE)
+
+ ## Assign?
+ if (length(gp) > 0) {
+ target <- new.env(parent=envir)
+ globalsT <- gp$globals
+ for (name in names(globalsT)) {
+ target[[name]] <- globalsT[[name]]
+ }
+ globalsT <- NULL
+ envir <- target
+ }
+ gp <- NULL
+
+ f <- UniprocessFuture(expr=expr, envir=envir, substitute=FALSE, local=local, ...)
structure(f, class=c("LazyFuture", class(f)))
}
@@ -38,7 +54,9 @@ resolved.LazyFuture <- function(x, ...) {
#' @export
value.LazyFuture <- function(future, signal=TRUE, ...) {
- future <- evaluate(future)
+ if (future$state == 'created') {
+ future <- run(future)
+ }
value <- future$value
if (signal && future$state == 'failed') {
diff --git a/R/MulticoreFuture-class.R b/R/MulticoreFuture-class.R
index 8ba5211..d1ff1a8 100644
--- a/R/MulticoreFuture-class.R
+++ b/R/MulticoreFuture-class.R
@@ -1,11 +1,6 @@
#' An multicore future is a future whose value will be resolved asynchroneously in a parallel process
#'
-#' @param expr An R \link[base]{expression}.
-#' @param envir The \link{environment} in which the evaluation
-#' is done (or inherits from if \code{local} is TRUE).
-#' @param substitute If TRUE, argument \code{expr} is
-#' \code{\link[base]{substitute}()}:ed, otherwise not.
-#' @param \dots Additional named elements of the future.
+#' @inheritParams MultiprocessFuture-class
#'
#' @return An object of class \code{MulticoreFuture}.
#'
@@ -16,17 +11,36 @@
#' @export
#' @name MulticoreFuture-class
#' @keywords internal
-MulticoreFuture <- function(expr=NULL, envir=parent.frame(), substitute=FALSE, ...) {
+MulticoreFuture <- function(expr=NULL, envir=parent.frame(), substitute=FALSE, globals=TRUE, ...) {
if (substitute) expr <- substitute(expr)
+ ## Global objects
+ assignToTarget <- (is.list(globals) || inherits(globals, "Globals"))
+ gp <- getGlobalsAndPackages(expr, envir=envir, tweak=tweakExpression, globals=globals, resolve=TRUE)
+
+ ## Assign?
+ if (assignToTarget && length(gp) > 0) {
+ target <- new.env(parent=envir)
+ globalsT <- gp$globals
+ for (name in names(globalsT)) {
+ target[[name]] <- globalsT[[name]]
+ }
+ globalsT <- NULL
+ envir <- target
+ }
+ gp <- NULL
+
f <- MultiprocessFuture(expr=expr, envir=envir, substitute=FALSE, job=NULL, ...)
structure(f, class=c("MulticoreFuture", class(f)))
}
-run <- function(...) UseMethod("run")
-
+#' @export
run.MulticoreFuture <- function(future, ...) {
+ if (future$state != 'created') {
+ stop("A future can only be launched once.")
+ }
+
## Assert that the process that created the future is
## also the one that evaluates/resolves/queries it.
assertOwner(future)
@@ -89,6 +103,10 @@ value.MulticoreFuture <- function(future, signal=TRUE, ...) {
return(NextMethod("value"))
}
+ if (future$state == 'created') {
+ future <- run(future)
+ }
+
## Assert that the process that created the future is
## also the one that evaluates/resolves/queries it.
assertOwner(future)
diff --git a/R/MultiprocessFuture-class.R b/R/MultiprocessFuture-class.R
index 1bd4856..e3be606 100644
--- a/R/MultiprocessFuture-class.R
+++ b/R/MultiprocessFuture-class.R
@@ -1,11 +1,11 @@
#' An multiprocess future is a future whose value will be resolved asynchroneously in a parallel process
#'
-#' @param expr An R \link[base]{expression}.
-#' @param envir The \link{environment} in which the evaluation
-#' is done (or inherits from if \code{local} is TRUE).
-#' @param substitute If TRUE, argument \code{expr} is
-#' \code{\link[base]{substitute}()}:ed, otherwise not.
-#' @param \dots Additional named elements of the future.
+#' @inheritParams Future-class
+#' @param local If TRUE, the expression is evaluated such that
+#' all assignments are done to local temporary environment, otherwise
+#' the assignments are done to the global environment of the \R process
+#' evaluating the future.
+#' @param \dots Additional named elements passed to \code{\link{Future}()}.
#'
#' @return An object of class \code{MultiprocessFuture}.
#'
diff --git a/R/MultisessionFuture-class.R b/R/MultisessionFuture-class.R
index f5751cf..6becac6 100644
--- a/R/MultisessionFuture-class.R
+++ b/R/MultisessionFuture-class.R
@@ -1,8 +1,10 @@
+#' @inheritParams ClusterFuture-class
+#'
## Currently aliased in ClusterFuture
#' @export
-MultisessionFuture <- function(expr=NULL, envir=parent.frame(), substitute=FALSE, local=!persistent, persistent=FALSE, workers=NULL, ...) {
+MultisessionFuture <- function(expr=NULL, envir=parent.frame(), substitute=FALSE, local=!persistent, globals=TRUE, persistent=FALSE, workers=NULL, ...) {
if (substitute) expr <- substitute(expr)
- f <- ClusterFuture(expr=expr, envir=envir, substitute=FALSE, local=local, persistent=persistent, workers=workers, ...)
+ f <- ClusterFuture(expr=expr, envir=envir, substitute=FALSE, local=local, globals=globals, persistent=persistent, workers=workers, ...)
structure(f, class=c("MultisessionFuture", class(f)))
}
diff --git a/R/UniprocessFuture-class.R b/R/UniprocessFuture-class.R
index ffcb227..dbe72e6 100644
--- a/R/UniprocessFuture-class.R
+++ b/R/UniprocessFuture-class.R
@@ -1,14 +1,7 @@
#' An uniprocess future is a future whose value will be resolved synchroneously in the current process
#'
-#' @param expr An R \link[base]{expression}.
-#' @param envir The \link{environment} in which the evaluation
-#' is done (or inherits from if \code{local} is TRUE).
-#' @param substitute If TRUE, argument \code{expr} is
-#' \code{\link[base]{substitute}()}:ed, otherwise not.
-#' @param local If TRUE, the expression is evaluated such that
-#' all assignments are done to local temporary environment, otherwise
-#' the assignments are done in the calling environment.
-#' @param \dots Additional named elements of the future.
+#' @inheritParams Future-class
+#' @param \dots Additional named elements passed to \code{\link{Future}()}.
#'
#' @return An object of class \code{UniprocessFuture}.
#'
@@ -26,11 +19,10 @@ UniprocessFuture <- function(expr=NULL, envir=parent.frame(), substitute=FALSE,
}
-evaluate <- function(...) UseMethod("evaluate")
-
-evaluate.UniprocessFuture <- function(future, ...) {
- if (future$state %in% c('finished', 'failed', 'interrupted')) {
- return(invisible(future))
+#' @export
+run.UniprocessFuture <- function(future, ...) {
+ if (future$state != 'created') {
+ stop("A future can only be launched once.")
}
## Assert that the process that created the future is
diff --git a/R/availableCores.R b/R/availableCores.R
index 53d34ac..e942742 100644
--- a/R/availableCores.R
+++ b/R/availableCores.R
@@ -62,10 +62,21 @@
#' same name is queried. If that is not set, the system environment
#' variable is queried. If neither is set, a missing value is returned.
#'
+#' @section Advanced usage:
+#' It is possible to override the maximum number of cores on the machine
+#' as reported by \code{availableCores(methods="system")}. This can be
+#' done by first specifying
+#' \code{options(future.availableCores.methods="mc.cores+1")} and
+#' then the number of cores to use (in addition to the main R process),
+#' e.g. \code{options(mc.cores=8)} will cause the value of
+#' \code{availableCores()} to be 9 (=8+1).
+#' Having said this, it is almost always better to do this by explicitly
+#' setting the number of workers when specifying the future strategy,
+#' e.g. \code{plan(multiprocess, workers=9)}.
#'
#' @export
#' @keywords internal
-availableCores <- function(constraints=NULL, methods=getOption("future.availableCores.methods", c("system", "mc.cores+1", "_R_CHECK_LIMIT_CORES_", "Slurm", "PBS", "SGE")), na.rm=TRUE, default=c(current=1L), which=c("min", "max", "all")) {
+availableCores <- function(constraints=NULL, methods=getOption("future.availableCores.methods", c("system", "mc.cores+1", "_R_CHECK_LIMIT_CORES_", "PBS", "SGE", "Slurm")), na.rm=TRUE, default=c(current=1L), which=c("min", "max", "all")) {
## Local functions
getenv <- function(name) {
as.integer(trim(Sys.getenv(name, NA_character_)))
diff --git a/R/cluster.R b/R/cluster.R
index b173c8b..eff61bf 100644
--- a/R/cluster.R
+++ b/R/cluster.R
@@ -4,19 +4,21 @@
#' which means that its \emph{value is computed and resolved in
#' parallel in another process}.
#'
-#' @param expr An R \link[base]{expression}.
-#' @param envir The \link{environment} in which the evaluation
-#' is done and from which globals are obtained.
-#' @param substitute If TRUE, argument \code{expr} is
-#' \code{\link[base]{substitute}()}:ed, otherwise not.
-#' @param gc If TRUE, the garbage collector run after the future
-#' is resolved (in the process that evaluated the future).
+#' @inheritParams future
+#' @inheritParams multiprocess
#' @param persistent If FALSE, the evaluation environment is cleared
#' from objects prior to the evaluation of the future.
#' @param workers A cluster object created by
#' \code{\link[parallel]{makeCluster}()}.
-#' @param earlySignal Specified whether conditions should be signaled as soon as possible or not.
-#' @param \dots Not used.
+#' @param revtunnel If TRUE, reverse SSH tunneling is used for the
+#' PSOCK cluster nodes to connect back to the master R process. This
+#' avoids the hassle of firewalls, port forwarding and having to know
+#' the internal / public IP address of the master R session.
+#' @param user (optional) The user name to be used when communicating
+#' with another host.
+#' @param homogeneous If TRUE, all cluster nodes is assumed to use the
+#' same path to \file{Rscript} as the main R session. If FALSE, the
+#' it is assumed to be on the PATH for each node.
#'
#' @return A \link{ClusterFuture}.
#'
@@ -31,10 +33,10 @@
#' this function directly, but to register it via
#' \code{\link{plan}(cluster)} such that it becomes the default
#' mechanism for all futures. After this \code{\link{future}()}
-#' and \code{\link{\%<=\%}} will create \emph{cluster futures}.
+#' and \code{\link{\%<-\%}} will create \emph{cluster futures}.
#'
#' @export
-cluster <- function(expr, envir=parent.frame(), substitute=TRUE, persistent=FALSE, workers=NULL, gc=FALSE, earlySignal=FALSE, ...) {
+cluster <- function(expr, envir=parent.frame(), substitute=TRUE, globals=TRUE, persistent=FALSE, workers=NULL, user=NULL, revtunnel=TRUE, homogeneous=TRUE, gc=FALSE, earlySignal=FALSE, label=NULL, ...) {
## BACKWARD COMPATIBILITY
args <- list(...)
if ("cluster" %in% names(args)) {
@@ -44,7 +46,7 @@ cluster <- function(expr, envir=parent.frame(), substitute=TRUE, persistent=FALS
if (substitute) expr <- substitute(expr)
- future <- ClusterFuture(expr=expr, envir=envir, substitute=FALSE, persistent=persistent, workers=workers, gc=gc, earlySignal=earlySignal, ...)
+ future <- ClusterFuture(expr=expr, envir=envir, substitute=FALSE, globals=globals, persistent=persistent, workers=workers, user=user, revtunnel=revtunnel, homogeneous=homogeneous, gc=gc, earlySignal=earlySignal, label=label, ...)
run(future)
}
class(cluster) <- c("cluster", "multiprocess", "future", "function")
diff --git a/R/constant.R b/R/constant.R
index cd52553..53d611f 100644
--- a/R/constant.R
+++ b/R/constant.R
@@ -1,6 +1,5 @@
## Used only internally
constant <- function(value, ...) {
- future <- ConstantFuture(value=value, ...)
- evaluate(future)
+ ConstantFuture(value=value, ...)
}
class(constant) <- c("constant", "uniprocess", "future", "function")
diff --git a/R/eager.R b/R/eager.R
index 160d770..967cb3d 100644
--- a/R/eager.R
+++ b/R/eager.R
@@ -6,22 +6,11 @@
#' The only difference to R itself is that globals are validated
#' by default just as for all other types of futures in this package.
#'
-#' @param expr An R \link[base]{expression}.
-#' @param envir The \link{environment} in which the evaluation
-#' is done (or inherits from if \code{local} is TRUE).
-#' @param substitute If TRUE, argument \code{expr} is
-#' \code{\link[base]{substitute}()}:ed, otherwise not.
-#' @param globals If TRUE, global objects are validated at the point
-#' in time when the future is created (always before it is resolved),
-#' that is, they identified and located. If some globals fail to be
-#' located, an informative error is generated.
+#' @inheritParams future
+#' @inheritParams multiprocess
#' @param local If TRUE, the expression is evaluated such that
#' all assignments are done to local temporary environment, otherwise
#' the assignments are done in the calling environment.
-#' @param gc If TRUE, the garbage collector run after the future
-#' is resolved (in the process that evaluated the future).
-#' @param earlySignal Specified whether conditions should be signaled as soon as possible or not.
-#' @param \dots Not used.
#'
#' @return An \link{EagerFuture}.
#'
@@ -31,7 +20,7 @@
#' The preferred way to create an eager future is not to call this function
#' directly, but to register it via \code{\link{plan}(eager)} such that it
#' becomes the default mechanism for all futures. After this
-#' \code{\link{future}()} and \code{\link{\%<=\%}} will create
+#' \code{\link{future}()} and \code{\link{\%<-\%}} will create
#' \emph{eager futures}.
#'
#' @section transparent futures:
@@ -45,25 +34,19 @@
#' @aliases transparent
#' @export transparent
#' @export
-eager <- function(expr, envir=parent.frame(), substitute=TRUE, globals=TRUE, local=TRUE, gc=FALSE, earlySignal=FALSE, ...) {
+eager <- function(expr, envir=parent.frame(), substitute=TRUE, globals=TRUE, local=TRUE, earlySignal=FALSE, label=NULL, ...) {
if (substitute) expr <- substitute(expr)
- globals <- as.logical(globals)
local <- as.logical(local)
- ## Validate globals at this point in time?
- if (globals) {
- exportGlobals(expr, envir=envir, target=NULL, tweak=tweakExpression, resolve=TRUE)
- }
-
- future <- EagerFuture(expr=expr, envir=envir, substitute=FALSE, local=local, gc=gc, earlySignal=earlySignal)
- evaluate(future)
+ future <- EagerFuture(expr=expr, envir=envir, substitute=FALSE, globals=globals, local=local, earlySignal=earlySignal, label=label)
+ run(future)
}
class(eager) <- c("eager", "uniprocess", "future", "function")
-transparent <- function(expr, envir=parent.frame(), substitute=TRUE, globals=FALSE, local=FALSE, gc=FALSE, earlySignal=TRUE, ...) {
+transparent <- function(expr, envir=parent.frame(), substitute=TRUE, globals=FALSE, local=FALSE, earlySignal=TRUE, label=NULL, ...) {
if (substitute) expr <- substitute(expr)
- future <- eager(expr, envir=envir, substitute=FALSE, globals=globals, local=local, gc=gc, earlySignal=earlySignal)
+ future <- eager(expr, envir=envir, substitute=FALSE, globals=globals, local=local, earlySignal=earlySignal, label=label)
invisible(future)
}
class(transparent) <- c("transparent", "eager", "uniprocess", "future", "function")
diff --git a/R/flapply.R b/R/flapply.R
index c13bdf8..2e73e40 100644
--- a/R/flapply.R
+++ b/R/flapply.R
@@ -1,23 +1,36 @@
-## FIXME: Do we need an envir argument here?
-#' @importFrom listenv listenv
-fleapply <- function(x, FUN, ...) {
+
+#' Apply a Function over a List or Vector via Futures
+#'
+#' @param x A vector-like object to iterate over.
+#' @param FUN A function taking at least one argument.
+#' @param ... (optional) Additional arguments pass to \code{FUN()}.
+#'
+#' @return A list.
+#'
+#' @keywords internal
+flapply <- function(x, FUN, ...) {
stopifnot(is.function(FUN))
- res <- listenv()
+
+ ## 1. Collect all globals (once)
+ envir <- environment()
+ x_ii <- NULL
+ globals <- c("FUN", "x_ii", names(list(...)), "...")
+ globals <- globalsByName(globals, envir=envir, mustExist=FALSE)
+
+ ## 2. Apply function with fixed set of globals
+ fs <- vector("list", length = length(x))
+ names(fs) <- names(x)
+
for (ii in seq_along(x)) {
- x_ii <- x[[ii]]
- res[[ii]] <- future(FUN(x_ii, ...))
+ ## Subsetting outside future is more efficient
+ globals$x_ii <- x[[ii]]
+
+ fs[[ii]] <- future(FUN(x_ii, ...), envir=envir, globals=globals)
}
- names(res) <- names(x)
## Not needed anymore
- rm(list=c("x", "FUN", "ii"))
+ rm(list=c("x_ii", "globals", "ii", "envir"))
## Resolve and return as values
- values(res)
-} # fleapply()
-
-
-## FIXME: Do we need an envir argument here?
-flapply <- function(x, FUN, ...) {
- as.list(fleapply(x=x, FUN=FUN, ...))
-}
+ values(fs)
+} # flapply()
diff --git a/R/future.R b/R/future.R
index 52fe564..598319a 100644
--- a/R/future.R
+++ b/R/future.R
@@ -1,33 +1,181 @@
#' Create a future
#'
-#' Creates a future from an expression and returns it.
-#' The state of the future is either unresolved or resolved.
-#' When it becomes resolved, at some point in the future,
-#' its value can be retrieved.
+#' Creates a future that evaluates an \R expression or
+#' a future that calls an \R function with a set of arguments.
+#' How, when, and where these futures are evaluated can be configured
+#' using \code{\link{plan}()} such that it is evaluated in parallel on,
+#' for instance, the current machine, on a remote machine, or via a
+#' job queue on a compute cluster.
+#' Importantly, \R code using futures remains the same regardless
+#' on these settings.
#'
-#' @param expr An R \link[base]{expression}.
+#' @param expr An \R \link[base]{expression}.
#' @param envir The \link{environment} from where global
#' objects should be identified. Depending on "evaluator",
#' it may also be the environment in which the expression
#' is evaluated.
#' @param substitute If TRUE, argument \code{expr} is
#' \code{\link[base]{substitute}()}:ed, otherwise not.
+#' @param globals A logical, a character vector,
+#' or a named list for controlling how globals are handled.
+#' For details, see below section.
+#' This argument can be specified via the \dots arguments
+#' for \code{future()} and \code{futureCall()}.
#' @param evaluator The actual function that evaluates
-#' \code{expr} and returns a future. The evaluator function
-#' should accept all the same arguments as this function
-#' (except \code{evaluator}).
-#' @param \dots Additional arguments passed to the "evaluator".
+#' the future expression and returns a \link{Future}.
+#' The evaluator function should accept all of the same
+#' arguments as the ones listed here
+#' (except \code{evaluator}, \code{FUN} and \code{args}).
+#' The default evaluator function is the one that the user
+#' has specified via \code{\link{plan}()}.
+#' @param ... Additional arguments passed to the "evaluator".
#'
-#' @return A \link{Future}.
+#' @return
+#' \code{f <- future(expr)} creates a \link{Future} \code{f} that evaluates expression \code{expr}, the value of the future is retrieved using \code{v <- value(f)}.
+#'
+#' \code{f <- futureCall(FUN, args)} creates a \link{Future} \code{f} that calls function \code{FUN} with arguments \code{args}, where the value of the future is retrieved using \code{v <- value(f)}.
+#'
+#' \code{futureAssign("v", expr)} and \code{v \%<-\% expr} (a future assignment) create a \link{Future} that evaluates expression \code{expr} and binds its value (as a \link[base]{promise}) to a variable \code{v}. The value of the future is automatically retrieved when the assigned variable (promise) is queried.
+#' The future itself is returned invisibly, e.g.
+#' \code{f <- futureAssign("v", expr)} and \code{f <- (v \%<-\% expr)}.
+#' Alternatively, the future of a future variable \code{v} can be retrieved
+#' without blocking using \code{f <- \link{futureOf}(v)}.
+#' Both the future and the variable (promise) are assigned to environment
+#' \code{assign.env} where the name of the future is \code{.future_<name>}.
+#'
+#'
+#' @details
+#' The state of a future is either unresolved or resolved.
+#' The value of a future can be retrieved using \code{v <- \link{value}(f)}.
+#' Querying the value of a non-resolved future will \emph{block} the call
+#' until the future is resolved.
+#' It is possible to check whether a future is resolved or not
+#' without blocking by using \code{\link{resolved}(f)}.
+#'
+#' For a future created via a future assignment, the value is bound to
+#' a promise, which when queried will internally call \code{\link{value}()}
+#' on the future and which will then be resolved into a regular variable
+#' bound to that value. For example, with future assignment
+#' \code{v \%<-\% expr}, the first time variable \code{v} is queried
+#' the call blocks if (and only if) the future is not yet resolved. As soon
+#' as it is resolved, and any succeeding queries, querying \code{v} will
+#' immediately give the value.
+#'
+#' The future assignment construct \code{v \%<-\% expr} is not a formal
+#' assignment per se, but a binary infix operator on objects \code{v}
+#' and \code{expr}. However, by using non-standard evaluation, this
+#' constructs can emulate an assignment operator similar to
+#' \code{v <- expr}. Due to \R's precedence rules of operators,
+#' future expressions that contain multiple statements need to be
+#' explicitly bracketed, e.g. \code{v \%<-\% { a <- 2; a^2 }}.
+#'
+#'
+#' @section Globals used by future expressions:
+#' Global objects (short \emph{globals}) are objects (e.g. variables and
+#' functions) that are needed in order for the future expression to be
+#' evaluated while not being local objects that are defined by the future
+#' expression. For example, in
+#' \preformatted{
+#' a <- 42
+#' f <- future({ b <- 2; a * b })
+#' }
+#' variable \code{a} is a global of future assignment \code{f} whereas
+#' \code{b} is a local variable.
+#' In order for the future to be resolved successfully (and correctly),
+#' all globals need to be gathered when the future is created such that
+#' they are available whenever and wherever the future is resolved.
+#'
+#' The default behavior (\code{globals = TRUE}) of all evaluator functions,
+#' is that globals are automatically identified and gathered.
+#' More precisely, globals are identified via code inspection of the
+#' future expression \code{expr} and their values are retrieved with
+#' environment \code{envir} as the starting point (basically via
+#' \code{get(global, envir=envir, inherits=TRUE)}).
+#' \emph{In most cases, such automatic collection of globals is sufficient
+#' and less tedious and error prone than if they are manually specified}.
+#'
+#' However, for full control, it is also possible to explicitly specify
+#' exactly which the globals are by providing their names as a character
+#' vector.
+#' In the above example, we could use
+#' \preformatted{
+#' a <- 42
+#' f <- future({ b <- 2; a * b }, globals = "a")
+#' }
+#'
+#' Yet another alternative is to explicitly specify also their values
+#' using a named list as in
+#' \preformatted{
+#' a <- 42
+#' f <- future({ b <- 2; a * b }, globals = list(a = a))
+#' }
+#' or
+#' \preformatted{
+#' f <- future({ b <- 2; a * b }, globals = list(a = 42))
+#' }
+#'
+#' Specifying globals explicitly avoids the overhead added from
+#' automatically identifying the globals and gathering their values.
+#' Futhermore, if we know that the future expression does not make use
+#' of any global variables, we can disable the automatic search for
+#' globals by using
+#' \preformatted{
+#' f <- future({ a <- 42; b <- 2; a * b }, globals = FALSE)
+#' }
+#'
+#' Future expressions often make use of functions from one or more packages.
+#' As long as these functions are part of the set of globals, the future
+#' package will make sure that those packages are attached when the future
+#' is resolved. Because there is no need for such globals to be frozen
+#' or exported, the future package will not export them, which reduces
+#' the amount of transferred objects.
+#' For example, in
+#' \preformatted{
+#' x <- rnorm(1000)
+#' f <- future({ median(x) })
+#' }
+#' variable \code{x} and \code{median()} are globals, but only \code{x}
+#' is exported whereas \code{median()}, which is part of the \pkg{stats}
+#' package, is not exported. Instead it is made sure that the \pkg{stats}
+#' package is on the search path when the future expression is evaluated.
+#' Effectively, the above becomes
+#' \preformatted{
+#' x <- rnorm(1000)
+#' f <- future({
+#' library("stats")
+#' median(x)
+#' })
+#' }
+#' To manually specify this, one can either do
+#' \preformatted{
+#' x <- rnorm(1000)
+#' f <- future({
+#' median(x)
+#' }, globals = list(x = x, median = stats::median)
+#' }
+#' or
+#' \preformatted{
+#' x <- rnorm(1000)
+#' f <- future({
+#' library("stats")
+#' median(x)
+#' }, globals = list(x = x))
+#' }
+#' Both are effectively the same.
+#'
+#' When using future assignments, globals can be specified analogously
+#' using the \code{\link{\%globals\%}} operator, e.g.
+#' \preformatted{
+#' x <- rnorm(1000)
+#' y \%<-\% { median(x) } \%globals\% list(x = x, median = stats::median)
+#' }
#'
#' @example incl/future.R
#'
+#'
#' @seealso
-#' It is recommended that the evaluator is \emph{non-blocking}
-#' (returns immediately), but it is not required.
-#
-#' The default evaluator function is \code{\link{eager}()},
-#' but this can be changed via \code{\link{plan}()} function.
+#' How, when and where futures are resolved is given by the future strategy,
+#' which can be set by the \code{\link{plan}()} function.
#'
#' @aliases futureCall
#' @export
diff --git a/R/futureAssign.R b/R/futureAssign.R
index 06997f6..11e937a 100644
--- a/R/futureAssign.R
+++ b/R/futureAssign.R
@@ -1,52 +1,23 @@
-#' Create a future and assign its value to a variable as a promise
+#' @param x the name of a future variable.
+#' @param value the \R \link[base]{expression} to be evaluated in
+#' the future and whose value will be assigned to the variable.
+#' @param assign.env The \link[base]{environment} to which the variable
+#' should be assigned.
#'
-#' Method and infix operators for creating futures and assigning
-#' their values as variables using \link[base]{promise}s. Trying
-#' to access such a "future variable" will correspond to requesting
-#' the value of the underlying future. If the the future is already
-#' resolved at this time, then the value will be available
-#' instantaneously and the future variable will appear as any other
-#' variable. If the future is unresolved, then the current process
-#' will block until the future is resolved and the value is available.
-#'
-#' @param name the name of the variable (and the future) to assign.
-#' @param value the expression to be evaluated in the future and
-#' whose value will be assigned to the variable.
-#' @param envir The environment from which global variables used by
-#' the expression should be search for.
-#' @param assign.env The environment to which the variable should
-#' be assigned.
-#' @param substitute Controls whether \code{expr} should be
-#' \code{substitute()}:d or not.
-#'
-#' @return A \link{Future} invisibly.
-#'
-#' @example incl/futureAssign_OP.R
-#'
-#' @details
-#' This function creates a future and a corresponding
-#' \emph{\link[base]{promise}}, which hold the future's value.
-#' Both the future and the promise are assigned to environment
-#' \code{assign.env}. The name of the promise is given by \code{name}
-#' and the name of the future is \code{.future_<name>}.
-#' The future is also returned invisibly.
-#'
-#' @seealso
-#' The \code{\link{futureOf}()} function can be used to get the
-#' Future object of a future variable.
+#' @rdname future
#'
#' @aliases %<-% %->% %<=% %=>%
-#' @export
#' @export %<-% %->% %<=% %=>%
-futureAssign <- function(name, value, envir=parent.frame(), assign.env=envir, substitute=TRUE) {
- stopifnot(is.character(name), !is.na(name), nzchar(name))
+#' @export
+futureAssign <- function(x, value, envir=parent.frame(), assign.env=envir, substitute=TRUE) {
+ stopifnot(is.character(x), !is.na(x), nzchar(x))
if (substitute) value <- substitute(value)
## - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
## (1) Create future
## - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
## Name of "future" saved in parallel with the "promise"
- future_name <- sprintf(".future_%s", name)
+ future_name <- sprintf(".future_%s", x)
if (exists(future_name, envir=envir)) {
msg <- sprintf("A future with name %s already exists in environment %s: %s", sQuote(future_name), sQuote(environmentName(envir)), hpaste(ls(envir=envir, all.names=TRUE)))
## warning(msg)
@@ -73,7 +44,7 @@ futureAssign <- function(name, value, envir=parent.frame(), assign.env=envir, su
## retrieved.
env <- new.env()
env$job <- future
- delayedAssign(name, local({
+ delayedAssign(x, local({
value <- value(future)
## Remove internal future variable
rm(list=future_name, envir=assign.env)
diff --git a/R/futureAssign_OP.R b/R/futureAssign_OP.R
index 2014d00..c1747dd 100644
--- a/R/futureAssign_OP.R
+++ b/R/futureAssign_OP.R
@@ -1,3 +1,24 @@
+#' @usage x \%<-\% value
+#'
+#' @rdname future
+#'
+#' @aliases %<-% %->% %<=% %=>%
+#' @export %<-% %->% %<=% %=>%
+`%<-%` <- `%<=%` <- function(x, value) {
+ target <- substitute(x)
+ expr <- substitute(value)
+ envir <- parent.frame(1)
+ futureAssignInternal(target, expr, envir=envir, substitute=FALSE)
+}
+
+`%->%` <- `%=>%` <- function(value, x) {
+ target <- substitute(x)
+ expr <- substitute(value)
+ envir <- parent.frame(1)
+ futureAssignInternal(target, expr, envir=envir, substitute=FALSE)
+}
+
+
#' @importFrom listenv get_variable parse_env_subset
futureAssignInternal <- function(target, expr, envir=parent.frame(), substitute=FALSE) {
target <- parse_env_subset(target, envir=envir, substitute=substitute)
@@ -22,18 +43,3 @@ futureAssignInternal <- function(target, expr, envir=parent.frame(), substitute=
futureAssign(name, expr, envir=envir, assign.env=assign.env, substitute=FALSE)
} # futureAssignInternal()
-
-
-`%<-%` <- `%<=%` <- function(x, value) {
- target <- substitute(x)
- expr <- substitute(value)
- envir <- parent.frame(1)
- futureAssignInternal(target, expr, envir=envir, substitute=FALSE)
-}
-
-`%->%` <- `%=>%` <- function(value, x) {
- target <- substitute(x)
- expr <- substitute(value)
- envir <- parent.frame(1)
- futureAssignInternal(target, expr, envir=envir, substitute=FALSE)
-}
diff --git a/R/futureCall.R b/R/futureCall.R
index b15f151..852ea7b 100644
--- a/R/futureCall.R
+++ b/R/futureCall.R
@@ -1,10 +1,10 @@
-#' @param FUN A function object.
-#' @param args A list of arguments passed to function \code{FUN}.
+#' @param FUN A \link[base]{function} object.
+#' @param args A \link[base]{list} of arguments passed to function \code{FUN}.
#'
#' @rdname future
#'
#' @export
-futureCall <- function(FUN, args=NULL, envir=parent.frame(), evaluator=plan(), ...) {
+futureCall <- function(FUN, args=NULL, envir=parent.frame(), globals=TRUE, evaluator=plan(), ...) {
stopifnot(is.function(FUN))
stopifnot(is.list(args))
@@ -12,5 +12,5 @@ futureCall <- function(FUN, args=NULL, envir=parent.frame(), evaluator=plan(), .
envirT$FUN <- FUN
envirT$args <- args
- future(do.call(what=FUN, args=args), substitute=TRUE, envir=envirT, evaluator=evaluator)
+ future(do.call(what=FUN, args=args), substitute=TRUE, envir=envirT, globals=globals, evaluator=evaluator)
} # futureCall()
diff --git a/R/futures.R b/R/futures.R
index a2b3e15..2dea29b 100644
--- a/R/futures.R
+++ b/R/futures.R
@@ -5,14 +5,14 @@
#' Non-future elements are returned as is.
#'
#' @param x An environment, a list, or a list environment.
-#' @param ... Not used.
+#' @param \dots Not used.
#'
#' @return An object of same type as \code{x} and with the same names
#' and/or dimensions, if set.
#'
#' @details
#' This function is useful for retrieve futures that were created via
-#' future assignments (\code{\%<=\%}) and therefore stored as promises.
+#' future assignments (\code{\%<-\%}) and therefore stored as promises.
#' This function turns such promises into standard \code{Future}
#' objects.
#'
diff --git a/R/globals.R b/R/globals.R
index 078c604..9adabe0 100644
--- a/R/globals.R
+++ b/R/globals.R
@@ -1,44 +1,27 @@
-#' @importFrom globals globalsOf packagesOf cleanup
-#' @importFrom utils packageVersion
-exportGlobals <- function(expr, envir, target=envir, tweak=NULL, resolve=getOption("future.globals.resolve", FALSE), persistent=FALSE) {
- gp <- getGlobalsAndPackages(expr, envir=envir, tweak=tweak, resolve=resolve, persistent=persistent)
- globals <- gp$globals
-
- ## Inject global objects?
- if (!is.null(target)) {
- for (name in names(globals)) {
- target[[name]] <- globals[[name]]
- }
- }
-
- invisible(globals)
-} # exportGlobals()
-
-
-
-#' @importFrom globals globalsOf packagesOf cleanup
+#' Retrieves global variables of an expression and their associated packages
+#'
+#' @param expr An R expression whose globals should be found.
+#' @param envir The environment from which globals should be searched.
+#' @param tweak (optional) A function that takes an expression and returned a modified one.
+#' @param globals (optional) a logical, a character vector, a named list, or a \link[globals]{Globals} object. If TRUE, globals are identified by code inspection based on \code{expr} and \code{tweak} searching from environment \code{envir}. If FALSE, no globals are used. If a character vector, then globals are identified by lookup based their names \code{globals} searching from environment \code{envir}. If a named list or a Globals object, the globals are used as is.
+#' @param resolve If TRUE, any future that is a global variables (or part of one) is resolved and replaced by a "constant" future.
+#' persistent If TRUE, non-existing globals (= identified in expression but not found in memory) are always silently ignored and assumed to be existing in the evaluation environment. If FALSE, non-existing globals are by default ignored, but may also trigger an informative error if option \code{future.globals.onMissing == "error"}.
+#' @param \dots Not used.
+#'
+#' @return A named list with elements \code{expr} (the tweaked expression), \code{globals} (a named list) and \code{packages} (a character string).
+#'
+#' @seealso Internally, \code{\link[globals]{globalsOf}()} is used to identify globals and associated packages from the expression.
+#'
+#' @importFrom globals globalsOf globalsByName as.Globals packagesOf cleanup
#' @importFrom utils object.size
-getGlobalsAndPackages <- function(expr, envir=parent.frame(), tweak=tweakExpression, resolve=getOption("future.globals.resolve", FALSE), persistent=FALSE, ...) {
- ## Local functions
- attachedPackages <- function() {
- pkgs <- search()
- pkgs <- grep("^package:", pkgs, value=TRUE)
- pkgs <- gsub("^package:", "", pkgs)
- pkgs
+#'
+#' @keywords internal
+getGlobalsAndPackages <- function(expr, envir=parent.frame(), tweak=tweakExpression, globals=TRUE, resolve=getOption("future.globals.resolve", FALSE), persistent=FALSE, ...) {
+ ## Nothing to do?
+ if (is.logical(globals) && !globals) {
+ return(list(expr=expr, globals=list(), packages=character(0)))
}
- asPkgEnvironment <- function(pkg) {
- name <- sprintf("package:%s", pkg)
- if (!name %in% search()) return(emptyenv())
- as.environment(name)
- } ## asPkgEnvironment()
-
-
- ## Maximum size of globals (to prevent too large exports) = 500 MiB
- maxSizeOfGlobals <- getOption("future.globals.maxSize", 500*1024^2)
- maxSizeOfGlobals <- as.numeric(maxSizeOfGlobals)
- stopifnot(!is.na(maxSizeOfGlobals), maxSizeOfGlobals > 0)
-
## Assert that all identified globals exists when future is created?
if (persistent) {
## If future relies on persistent storage, then the globals may
@@ -54,22 +37,41 @@ getGlobalsAndPackages <- function(expr, envir=parent.frame(), tweak=tweakExpress
}
- ## Identify globals
- ## Algorithm for identifying globals
- globals.method <- getOption("future.globals.method", "ordered")
- globals <- globalsOf(expr, envir=envir, substitute=FALSE,
- tweak=tweak,
- dotdotdot="return",
- primitive=FALSE, base=FALSE,
- unlist=TRUE,
- mustExist=mustExist,
- method=globals.method
- )
+ ## Alt 1. Identify globals based on expr, envir and tweak
+ if (is.logical(globals)) {
+ stopifnot(length(globals) == 1, !is.na(globals))
+ ## Algorithm for identifying globals
+ globals.method <- getOption("future.globals.method", "ordered")
+ globals <- globalsOf(
+ ## Passed to globals::findGlobals()
+ expr, envir=envir, substitute=FALSE, tweak=tweak,
+ ## Passed to globals::findGlobals() via '...'
+ dotdotdot="return",
+ method=globals.method,
+ unlist=TRUE,
+ ## Passed to globals::globalsByName()
+ mustExist=mustExist
+ )
+ } else if (is.character(globals)) {
+ globals <- globalsByName(globals, envir=envir, mustExist=mustExist)
+ } else if (inherits(globals, "Globals")) {
+ ## Keep as is
+ } else if (is.list(globals)) {
+ globals <- as.Globals(globals)
+ } else {
+ stop("Argument 'globals' must be either a logical scalar or a character vector: ", mode(globals))
+ }
+ stopifnot(inherits(globals, "Globals"))
+ ## Nothing more to do?
+ if (length(globals) == 0) {
+ return(list(expr=expr, globals=list(), packages=character(0)))
+ }
+
exprOrg <- expr
## Tweak expression to be called with global ... arguments?
- if (inherits(globals$`...`, "DotDotDotList")) {
+ if (length(globals) > 0 && inherits(globals$`...`, "DotDotDotList")) {
## Missing global '...'?
if (!is.list(globals$`...`)) {
msg <- sprintf("Did you mean to create the future within a function? Invalid future expression tries to use global '...' variables that do not exist: %s", hexpr(exprOrg))
@@ -77,8 +79,10 @@ getGlobalsAndPackages <- function(expr, envir=parent.frame(), tweak=tweakExpress
stop(msg)
}
- globals$`future.call.arguments` <- globals$`...`
- globals$`...` <- NULL
+ names <- names(globals)
+ names[names == "..."] <- "future.call.arguments"
+ names(globals) <- names
+ names <- NULL
## To please R CMD check
a <- `future.call.arguments` <- NULL
@@ -95,7 +99,7 @@ getGlobalsAndPackages <- function(expr, envir=parent.frame(), tweak=tweakExpress
## or may not point to packages (include base R package)
if (resolve && length(globals) > 0L) {
mdebug("Resolving globals that are futures ...")
- idxs <- which(unlist(lapply(globals, FUN=inherits, "Future")))
+ idxs <- which(unlist(lapply(globals, FUN=inherits, "Future"), use.names=FALSE))
mdebug("Number of global futures: %d", length(idxs))
if (length(idxs) > 0) {
mdebug("Global futures: %s", hpaste(sQuote(names(globals[idxs]))))
@@ -104,12 +108,18 @@ getGlobalsAndPackages <- function(expr, envir=parent.frame(), tweak=tweakExpress
valuesF <- NULL ## Not needed anymore
}
idxs <- NULL ## Not needed anymore
- mdebug("Resolving global that are futures ... DONE")
+ mdebug("Resolving globals that are futures ... DONE")
}
pkgs <- NULL
if (length(globals) > 0L) {
+ asPkgEnvironment <- function(pkg) {
+ name <- sprintf("package:%s", pkg)
+ if (!name %in% search()) return(emptyenv())
+ as.environment(name)
+ } ## asPkgEnvironment()
+
## Append packages associated with globals
pkgs <- packagesOf(globals)
@@ -136,6 +146,11 @@ getGlobalsAndPackages <- function(expr, envir=parent.frame(), tweak=tweakExpress
## that are part of the base packages, which now are
## part of 'pkgs' if needed.
globals <- cleanup(globals)
+
+ ## Nothing more to do?
+ if (length(globals) == 0) {
+ return(list(expr=expr, globals=list(), packages=pkgs))
+ }
}
@@ -153,6 +168,10 @@ getGlobalsAndPackages <- function(expr, envir=parent.frame(), tweak=tweakExpress
## Protect against user error exporting too large objects?
+ ## Maximum size of globals (to prevent too large exports) = 500 MiB
+ maxSizeOfGlobals <- getOption("future.globals.maxSize", 500*1024^2)
+ maxSizeOfGlobals <- as.numeric(maxSizeOfGlobals)
+ stopifnot(!is.na(maxSizeOfGlobals), maxSizeOfGlobals > 0)
if (length(globals) > 0L && is.finite(maxSizeOfGlobals)) {
sizes <- lapply(globals, FUN=object.size)
sizes <- unlist(sizes, use.names=TRUE)
@@ -189,9 +208,17 @@ getGlobalsAndPackages <- function(expr, envir=parent.frame(), tweak=tweakExpress
## available for all R sessions / implementations.
pkgs <- setdiff(pkgs, "base")
if (length(pkgs) > 0L) {
+ ## Local functions
+ attachedPackages <- function() {
+ pkgs <- search()
+ pkgs <- grep("^package:", pkgs, value=TRUE)
+ pkgs <- gsub("^package:", "", pkgs)
+ pkgs
+ }
+
## Record which packages in 'pkgs' that are loaded and
## which of them are attached (at this point in time).
- isLoaded <- is.element(pkgs, loadedNamespaces())
+ ## isLoaded <- is.element(pkgs, loadedNamespaces())
isAttached <- is.element(pkgs, attachedPackages())
pkgs <- pkgs[isAttached]
}
diff --git a/R/globals_OP.R b/R/globals_OP.R
new file mode 100644
index 0000000..ed71159
--- /dev/null
+++ b/R/globals_OP.R
@@ -0,0 +1,18 @@
+#' Specify globals for a future assignment
+#'
+#' @usage fassignment \%globals\% globals
+#'
+#' @param fassignment The future assignment, e.g.
+#' \code{x \%<-\% \{ expr \}}.
+#' @inheritParams multiprocess
+#'
+#' @details
+#' \code{a \%globals\% b} is short for \code{a \%tweak\% list(globals = b)}.
+#'
+#' @export
+`%globals%` <- function(fassignment, globals) {
+ fassignment <- substitute(fassignment)
+ envir <- parent.frame(1)
+ args <- list(fassignment, list(globals=globals))
+ do.call(`%tweak%`, args=args, envir=envir)
+}
diff --git a/R/label_OP.R b/R/label_OP.R
new file mode 100644
index 0000000..528918a
--- /dev/null
+++ b/R/label_OP.R
@@ -0,0 +1,18 @@
+#' Specify label for a future assignment
+#'
+#' @usage fassignment \%label\% label
+#'
+#' @param fassignment The future assignment, e.g.
+#' \code{x \%<-\% \{ expr \}}.
+#' @inheritParams multiprocess
+#'
+#' @details
+#' \code{a \%label\% b} is short for \code{a \%tweak\% list(label = b)}.
+#'
+#' @export
+`%label%` <- function(fassignment, label) {
+ fassignment <- substitute(fassignment)
+ envir <- parent.frame(1)
+ args <- list(fassignment, list(label=label))
+ do.call(`%tweak%`, args=args, envir=envir)
+}
diff --git a/R/lazy.R b/R/lazy.R
index bdd88dc..ce0dfe7 100644
--- a/R/lazy.R
+++ b/R/lazy.R
@@ -5,22 +5,14 @@
#' value is requested}. This means that the future will not be resolved
#' if the value is never requested.
#'
-#' @param expr An R \link[base]{expression}.
-#' @param envir The \link{environment} in which the evaluation
-#' is done (or inherits from if \code{local} is TRUE)
-#' and from which globals are obtained.
-#' @param substitute If TRUE, argument \code{expr} is
-#' \code{\link[base]{substitute}()}:ed, otherwise not.
+#' @inheritParams future
+#' @inheritParams multiprocess
#' @param local If TRUE, the expression is evaluated such that
#' all assignments are done to local temporary environment, otherwise
#' the assignments are done in the calling environment.
#' @param globals If TRUE, global objects are resolved ("frozen") at
#' the point of time when the future is created, otherwise they are
#' resolved when the future is resolved.
-#' @param gc If TRUE, the garbage collector run after the future
-#' is resolved (in the process that evaluated the future).
-#' @param earlySignal Specified whether conditions should be signaled as soon as possible or not.
-#' @param \dots Not used.
#'
#' @return A \link{LazyFuture}.
#'
@@ -30,31 +22,15 @@
#' The preferred way to create a lazy future is not to call this function
#' directly, but to register it via \code{\link{plan}(lazy)} such that it
#' becomes the default mechanism for all futures. After this
-#' \code{\link{future}()} and \code{\link{\%<=\%}} will create
+#' \code{\link{future}()} and \code{\link{\%<-\%}} will create
#' \emph{lazy futures}.
#'
#' @export
-lazy <- function(expr, envir=parent.frame(), substitute=TRUE, globals=TRUE, local=TRUE, gc=FALSE, earlySignal=FALSE, ...) {
+lazy <- function(expr, envir=parent.frame(), substitute=TRUE, globals=TRUE, local=TRUE, earlySignal=FALSE, label=NULL, ...) {
if (substitute) expr <- substitute(expr)
- globals <- as.logical(globals)
local <- as.logical(local)
- if (!local && globals) {
- stop("Non-supported call to lazy(): Argument 'globals' must be FALSE whenever 'local' is FALSE. Lazy future evaluation in the calling environment (local=FALSE) can only be done if global objects are resolved at the same time (globals=FALSE).")
- }
-
- ## Evaluate in "local" environment?
- if (local || globals) {
- envir <- new.env(parent=envir)
- }
-
-
- ## Resolve globals at this point in time?
- if (globals) {
- exportGlobals(expr, envir=envir, target=envir, tweak=tweakExpression, resolve=TRUE)
- }
-
- LazyFuture(expr=expr, envir=envir, local=local, globals=globals, gc=gc, earlySignal=earlySignal)
+ LazyFuture(expr=expr, envir=envir, local=local, globals=globals, earlySignal=earlySignal, label=label)
}
class(lazy) <- c("lazy", "uniprocess", "future", "function")
diff --git a/R/multicore.R b/R/multicore.R
index ebff9aa..1f885da 100644
--- a/R/multicore.R
+++ b/R/multicore.R
@@ -4,21 +4,10 @@
#' which means that its \emph{value is computed and resolved in
#' parallel in another process}.
#'
-#' @param expr An R \link[base]{expression}.
-#' @param envir The \link{environment} in which the evaluation
-#' is done and from which globals are obtained.
-#' @param substitute If TRUE, argument \code{expr} is
-#' \code{\link[base]{substitute}()}:ed, otherwise not.
-#' @param globals If TRUE, global objects are validated at the point
-#' in time when the future is created (always before it is resolved),
-#' that is, they identified and located. If some globals fail to be
-#' located, an informative error is generated.
+#' @inheritParams future
+#' @inheritParams multiprocess
#' @param workers The maximum number of multicore futures that can
#' be active at the same time before blocking.
-#' @param gc If TRUE, the garbage collector run after the future
-#' is resolved (in the process that evaluated the future).
-#' @param earlySignal Specified whether conditions should be signaled as soon as possible or not.
-#' @param \dots Not used.
#'
#' @return A \link{MulticoreFuture}
#' If \code{workers == 1}, then all processing using done in the
@@ -46,7 +35,7 @@
#' this function directly, but to register it via
#' \code{\link{plan}(multicore)} such that it becomes the default
#' mechanism for all futures. After this \code{\link{future}()}
-#' and \code{\link{\%<=\%}} will create \emph{multicore futures}.
+#' and \code{\link{\%<-\%}} will create \emph{multicore futures}.
#'
#' @seealso
#' For processing in multiple background R sessions, see
@@ -61,7 +50,7 @@
#' system.
#'
#' @export
-multicore <- function(expr, envir=parent.frame(), substitute=TRUE, globals=TRUE, workers=availableCores(constraints="multicore"), gc=FALSE, earlySignal=FALSE, ...) {
+multicore <- function(expr, envir=parent.frame(), substitute=TRUE, globals=TRUE, workers=availableCores(constraints="multicore"), earlySignal=FALSE, label=NULL, ...) {
## BACKWARD COMPATIBILITY
args <- list(...)
if ("maxCores" %in% names(args)) {
@@ -70,7 +59,6 @@ multicore <- function(expr, envir=parent.frame(), substitute=TRUE, globals=TRUE,
}
if (substitute) expr <- substitute(expr)
- globals <- as.logical(globals)
workers <- as.integer(workers)
stopifnot(is.finite(workers), workers >= 1L)
@@ -79,18 +67,13 @@ multicore <- function(expr, envir=parent.frame(), substitute=TRUE, globals=TRUE,
## Eager futures best reflect how multicore futures handle globals.
if (workers == 1L || !supportsMulticore()) {
## covr: skip=1
- return(eager(expr, envir=envir, substitute=FALSE, globals=globals, local=TRUE, gc=gc))
- }
-
- ## Validate globals at this point in time?
- if (globals) {
- exportGlobals(expr, envir=envir, target=NULL, tweak=tweakExpression, resolve=TRUE)
+ return(eager(expr, envir=envir, substitute=FALSE, globals=globals, local=TRUE, label=label))
}
oopts <- options(mc.cores=workers)
on.exit(options(oopts))
- future <- MulticoreFuture(expr=expr, envir=envir, substitute=FALSE, workers=workers, gc=gc, earlySignal=earlySignal)
+ future <- MulticoreFuture(expr=expr, envir=envir, substitute=FALSE, globals=globals, workers=workers, earlySignal=earlySignal, label=label)
run(future)
}
class(multicore) <- c("multicore", "multiprocess", "future", "function")
diff --git a/R/multiprocess.R b/R/multiprocess.R
index 6fbc66a..50e4f45 100644
--- a/R/multiprocess.R
+++ b/R/multiprocess.R
@@ -5,21 +5,19 @@
#' Regardless, its \emph{value is computed and resolved in
#' parallel in another process}.
#'
-#' @param expr An R \link[base]{expression}.
-#' @param envir The \link{environment} in which the evaluation
-#' is done and from which globals are obtained.
-#' @param substitute If TRUE, argument \code{expr} is
-#' \code{\link[base]{substitute}()}:ed, otherwise not.
-#' @param globals If TRUE, global objects are validated at the point
-#' in time when the future is created (always before it is resolved),
-#' that is, they identified and located. If some globals fail to be
-#' located, an informative error is generated.
+#' @inheritParams future
+#' @param globals (optional) a logical, a character vector,
+#' or a named list for controlling how globals are handled.
+#' For details, see section 'Globals used by future expressions'
+#' in the help for \code{\link{future}()}.
#' @param workers The maximum number of multiprocess futures that
#' can be active at the same time before blocking.
-#' @param gc If TRUE, the garbage collector run after the future
-#' is resolved (in the process that evaluated the future).
-#' @param earlySignal Specified whether conditions should be signaled as soon as possible or not.
-#' @param \dots Not used.
+#' @param gc If TRUE, the garbage collector run (in the process that
+#' evaluated the future) after the value of the future is collected.
+#' @param earlySignal Specified whether conditions should be signaled
+#' as soon as possible or not.
+#' @param label An optional character string label attached to the future.
+#' @param ... Not used.
#'
#' @return A \link{MultiprocessFuture} implemented as either a
#' \link{MulticoreFuture} or a \link{MultisessionFuture}.
@@ -31,9 +29,9 @@
#' are used.
#'
#' @export
-multiprocess <- function(expr, envir=parent.frame(), substitute=TRUE, globals=TRUE, workers=availableCores(), gc=FALSE, earlySignal=FALSE, ...) {
+multiprocess <- function(expr, envir=parent.frame(), substitute=TRUE, globals=TRUE, workers=availableCores(), gc=FALSE, earlySignal=FALSE, label=NULL, ...) {
if (substitute) expr <- substitute(expr)
fun <- if (supportsMulticore()) multicore else multisession
- fun(expr=expr, envir=envir, substitute=FALSE, globals=globals, workers=workers, gc=gc, earlySignal=earlySignal, ...)
+ fun(expr=expr, envir=envir, substitute=FALSE, globals=globals, workers=workers, gc=gc, earlySignal=earlySignal, label=label, ...)
}
class(multiprocess) <- c("multiprocess", "future", "function")
diff --git a/R/multisession.R b/R/multisession.R
index ccd80fe..cb9809e 100644
--- a/R/multisession.R
+++ b/R/multisession.R
@@ -4,19 +4,11 @@
#' which means that its \emph{value is computed and resolved in
#' parallel in another R session}.
#'
-#' @param expr An R \link[base]{expression}.
-#' @param envir The \link{environment} in which the evaluation
-#' is done and from which globals are obtained.
-#' @param substitute If TRUE, argument \code{expr} is
-#' \code{\link[base]{substitute}()}:ed, otherwise not.
-#' @param persistent If FALSE, the evaluation environment is cleared
-#' from objects prior to the evaluation of the future.
+#' @inheritParams future
+#' @inheritParams multiprocess
+#' @inheritParams cluster
#' @param workers The maximum number of multisession futures that
#' can be active at the same time before blocking.
-#' @param gc If TRUE, the garbage collector run after the future
-#' is resolved (in the process that evaluated the future).
-#' @param earlySignal Specified whether conditions should be signaled as soon as possible or not.
-#' @param \dots Not used.
#'
#' @return A \link{MultisessionFuture}.
#' If \code{workers == 1}, then all processing using done in the
@@ -41,7 +33,7 @@
#' this function directly, but to register it via
#' \code{\link{plan}(multisession)} such that it becomes the default
#' mechanism for all futures. After this \code{\link{future}()}
-#' and \code{\link{\%<=\%}} will create \emph{multisession futures}.
+#' and \code{\link{\%<-\%}} will create \emph{multisession futures}.
#'
#' @section Known issues:
#' In the current implementation, \emph{all} background R sessions
@@ -62,7 +54,7 @@
#' cores that are available for the current R session.
#'
#' @export
-multisession <- function(expr, envir=parent.frame(), substitute=TRUE, persistent=FALSE, workers=availableCores(), gc=FALSE, earlySignal=FALSE, ...) {
+multisession <- function(expr, envir=parent.frame(), substitute=TRUE, globals=TRUE, persistent=FALSE, workers=availableCores(), gc=FALSE, earlySignal=FALSE, label=NULL, ...) {
## BACKWARD COMPATIBILITY
args <- list(...)
if ("maxCores" %in% names(args)) {
@@ -78,7 +70,7 @@ multisession <- function(expr, envir=parent.frame(), substitute=TRUE, persistent
## i.e. the use the current main R process.
if (workers == 1L) {
## FIXME: How to handle argument 'persistent'? /HB 2016-03-19
- return(lazy(expr, envir=envir, substitute=FALSE, globals=TRUE, local=TRUE))
+ return(lazy(expr, envir=envir, substitute=FALSE, globals=globals, local=TRUE, label=label))
}
## IMPORTANT: When we setup a multisession cluster, we need to
@@ -86,7 +78,7 @@ multisession <- function(expr, envir=parent.frame(), substitute=TRUE, persistent
## a cluster with one less process.
workers <- ClusterRegistry("start", workers=workers-1L)
- future <- MultisessionFuture(expr=expr, envir=envir, substitute=FALSE, persistent=persistent, workers=workers, gc=gc, earlySignal=earlySignal, ...)
+ future <- MultisessionFuture(expr=expr, envir=envir, substitute=FALSE, globals=globals, persistent=persistent, workers=workers, gc=gc, earlySignal=earlySignal, label=label, ...)
run(future)
}
class(multisession) <- c("multisession", "cluster", "multiprocess", "future", "function")
diff --git a/R/nbrOfWorkers.R b/R/nbrOfWorkers.R
index bbdfbf7..3ae0862 100644
--- a/R/nbrOfWorkers.R
+++ b/R/nbrOfWorkers.R
@@ -1,4 +1,4 @@
-#' Gets the number of workers
+#' Gets the number of workers available
#'
#' @param evaluator A future evaluator function.
#' If NULL (default), the current evaluator as returned
@@ -6,6 +6,8 @@
#'
#' @return A number in [1,Inf].
#'
+#' @example incl/nbrOfWorkers.R
+#'
#' @export
nbrOfWorkers <- function(evaluator=NULL) {
UseMethod("nbrOfWorkers")
diff --git a/R/plan_OP.R b/R/plan_OP.R
index f941c60..ab65a3b 100644
--- a/R/plan_OP.R
+++ b/R/plan_OP.R
@@ -3,7 +3,7 @@
#' @usage fassignment \%plan\% strategy
#'
#' @param fassignment The future assignment, e.g.
-#' \code{x \%<=\% \{ expr \}}.
+#' \code{x \%<-\% \{ expr \}}.
#' @param strategy The mechanism for how the future should be
#' resolved. See \code{\link{plan}()} for further details.
#'
diff --git a/R/remote.R b/R/remote.R
index b0c7452..2641114 100644
--- a/R/remote.R
+++ b/R/remote.R
@@ -4,21 +4,11 @@
#' which means that its \emph{value is computed and resolved
#' remotely in another process}.
#'
-#' @param expr An R \link[base]{expression}.
-#' @param envir The \link{environment} in which the evaluation
-#' is done and from which globals are obtained.
-#' @param substitute If TRUE, argument \code{expr} is
-#' \code{\link[base]{substitute}()}:ed, otherwise not.
-#' @param persistent If FALSE, the evaluation environment is cleared
-#' from objects prior to the evaluation of the future.
-#' @param workers A cluster object created by
-#' \code{\link[parallel]{makeCluster}()}.
-#' @param gc If TRUE, the garbage collector run after the future
-#' is resolved (in the process that evaluated the future).
-#' @param earlySignal Specified whether conditions should be signaled as soon as possible or not.
+#' @inheritParams future
+#' @inheritParams multiprocess
+#' @inheritParams cluster
#' @param myip The external IP address of this machine.
#' If NULL, then it is inferred using an online service (default).
-#' @param \dots Not used.
#'
#' @return A \link{ClusterFuture}.
#'
@@ -28,7 +18,7 @@
#' Note that remote futures use \code{persistent=TRUE} by default.
#'
#' @export
-remote <- function(expr, envir=parent.frame(), substitute=TRUE, persistent=TRUE, workers=NULL, gc=FALSE, earlySignal=FALSE, myip=NULL, ...) {
+remote <- function(expr, envir=parent.frame(), substitute=TRUE, globals=TRUE, persistent=TRUE, workers=NULL, user=NULL, revtunnel=TRUE, gc=FALSE, earlySignal=FALSE, myip=NULL, label=NULL, ...) {
if (substitute) expr <- substitute(expr)
stopifnot(length(workers) >= 1L, is.character(workers), !anyNA(workers))
@@ -36,22 +26,30 @@ remote <- function(expr, envir=parent.frame(), substitute=TRUE, persistent=TRUE,
if (is.character(workers)) {
homogeneous <- FALSE ## Calls plain 'Rscript'
- ## Guess what type of IP to use
- if (is.null(myip)) {
- if (all(workers %in% c("localhost", "127.0.0.1"))) {
- ## For conveniency, if all workers are on the localhost,
- ## then we know that only the local machine will be used.
-
- ## (a) We use 127.0.0.1 because it's slightly more generic
- ## than localhost.
- myip <- "127.0.0.1"
-
- ## (b) We can also use the pathname of the currently running
- ## R session, file.path(R.home("bin"), "Rscript"), rather
- ## than 'Rscript' which is the best guess we can make for
- ## a remote machine. This is controlled by 'homogeneous'.
- homogeneous <- TRUE
- } else {
+ if (all(workers %in% c("localhost", "127.0.0.1"))) {
+ ## For conveniency, if all workers are on the localhost,
+ ## then we know that only the local machine will be used.
+
+ ## (a) We can use the pathname of the currently running R
+ ## session, file.path(R.home("bin"), "Rscript"), rather
+ ## than 'Rscript' which is the best guess we can make for
+ ## a remote machine. This is controlled by 'homogeneous'.
+ homogeneous <- TRUE
+
+ ## (b) No need to request reverse SSH tunnel (won't happen
+ ## for 'localhost' anyways, but just in case)
+ revtunnel <- FALSE
+
+ ## (c) We use myip='127.0.0.1' because it's slightly more
+ ## generic than 'localhost'.
+ myip <- "127.0.0.1"
+ } else {
+ if (revtunnel) {
+ ## Default is that reverse tunnel uses 127.0.0.1 / localhost.
+ if (is.null(myip)) myip <- "127.0.0.1"
+ } else if (is.null(myip)) {
+ ## The best guess we can make here is that the workers need
+ ## to connect back using our exernal IP address.
myip <- "<external>"
}
}
@@ -62,13 +60,11 @@ remote <- function(expr, envir=parent.frame(), substitute=TRUE, persistent=TRUE,
} else if (myip == "<internal>") {
myip <- myInternalIP()
}
-
- workers <- ClusterRegistry("start", workers=workers, master=myip, homogeneous=homogeneous)
} else if (!inherits(workers, "cluster")) {
stop("Argument 'workers' is not of class 'cluster': ", class(workers)[1])
}
- future <- ClusterFuture(expr=expr, envir=envir, substitute=FALSE, persistent=persistent, workers=workers, gc=gc, earlySignal=earlySignal, ...)
+ future <- ClusterFuture(expr=expr, envir=envir, substitute=FALSE, globals=globals, persistent=persistent, workers=workers, user=user, master=myip, revtunnel=revtunnel, homogeneous=homogeneous, gc=gc, earlySignal=earlySignal, label=label, ...)
run(future)
}
class(remote) <- c("remote", "multiprocess", "future", "function")
diff --git a/R/resolve.R b/R/resolve.R
index c09ab2d..05709a3 100644
--- a/R/resolve.R
+++ b/R/resolve.R
@@ -18,7 +18,7 @@
#' @param progress If TRUE textual progress summary is outputted.
#' If a function, the it is called as \code{progress(done, total)}
#' every time a future is resolved.
-#' @param ... Not used
+#' @param \dots Not used
#'
#' @return Returns \code{x} (regardless of subsetting or not).
#'
diff --git a/R/tweak.R b/R/tweak.R
index 07898cc..75703de 100644
--- a/R/tweak.R
+++ b/R/tweak.R
@@ -1,7 +1,7 @@
#' Tweaks a future function by adjusting its default arguments
#'
#' @param strategy An existing future function or the name of one.
-#' @param ... Named arguments to replace the defaults of existing
+#' @param \dots Named arguments to replace the defaults of existing
#' arguments.
#' @param penvir The environment used when searching for a future
#' function by its name.
diff --git a/R/tweakExpression.R b/R/tweakExpression.R
index c71b5b1..541fa0a 100644
--- a/R/tweakExpression.R
+++ b/R/tweakExpression.R
@@ -1,27 +1,122 @@
-tweakExpression <- function(expr) {
- if (!is.language(expr)) return(expr)
+## Formula #1:
+## expression: ~ rhs
+## AST: (~ rhs)
+## tweaked expression: rhs; ~ rhs
+##
+## Formula #2:
+## expression: lhs ~ rhs
+## AST: (~ lhs rhs)
+## tweaked expression: rhs; lhs ~ rhs
+tweakFormulaCall <- function(expr) {
+ if (!is.call(expr)) return(expr)
+ op <- expr[[1]]
+ if (!is.symbol(op)) return(expr)
+ op <- as.character(op)
+ if (op != "~") return(expr)
+ n <- length(expr)
+ if (n != 2 && n != 3) return(expr)
+
+ if (n == 2) {
+ lhs <- NULL
+ rhs <- expr[[2]]
+ } else if (n == 3) {
+ lhs <- expr[[2]]
+ rhs <- expr[[3]]
+ }
+
+ ## covr: skip=1
+ substitute({ lhs; rhs; e }, list(lhs=lhs, rhs=rhs, e=expr))
+} ## tweakFormulaCall()
+
+
+## Subassignment #1:
+## expression: x$a <- value
+## AST: (<- ($ x a) value)
+## tweaked expression: x; x$a <- value
+##
+## expression: value -> x$a
+## AST: (<- ($ x a) value)
+## tweaked expression: x; value -> x$a
+##
+## Subassignment #2:
+## expression: x[["a"]] <- value
+## AST: (<- ($ [[ a) value)
+## tweaked expression: x; x[["a"]] <- value
+##
+## expression: value -> x[["a"]]
+## AST: (<- ([[ x a) value)
+## tweaked expression: x; value -> x[["a"]]
+##
+## Subassignment #3:
+## expression: x["a"] <- value
+## AST: (<- ([ x a) value)
+## tweaked expression: x; x["a"] <- value
+##
+## expression: x[1,2,3] <- value
+## AST: (<- ([ x 1 2 3) value)
+## tweaked expression: x; x[1,2,3] <- value
+tweakSubassignmentCall <- function(expr) {
+ if (!is.call(expr)) return(expr)
+ op <- expr[[1]]
+ if (!is.symbol(op)) return(expr)
+ op <- as.character(op)
+ if (op != "<-") return(expr)
+ n <- length(expr)
+ if (n != 3) return(expr)
+
+ ## expression #2: x$a, x[["a"]], or x[...]
+ ## AST #2: ($ x a), ([[ x a), or ([ x ...)
+ expr2 <- expr[[2]]
+ if (!is.call(expr2)) return(expr)
+ op2 <- expr2[[1]]
+ if (!is.symbol(op2)) return(expr)
+ op2 <- as.character(op2)
+ if (!op2 %in% c("$", "[[", "[")) return(expr)
+ n2 <- length(expr2)
+ if (n2 < 3) return(expr)
+
+ target <- expr2[[2]]
+
+ ## covr: skip=1
+ substitute({ target; e }, list(target=target, e=expr))
+} ## tweakSubassignmentCall()
- for (ii in seq_along(expr)) {
- # If expr[[ii]] is "missing", ignore the error. This
- # happens with for instance expressions like x[,1].
- # FIXME: Is there a better way?!? /HB 2014-05-08
- tryCatch({
- exprI <- expr[[ii]]
- op <- exprI[[1]]
- if (!is.symbol(op)) next
- op <- as.character(op)
- if (op %in% c("<<-", "%<-%", "%<=%")) {
- lhs <- exprI[[2]]
- rhs <- exprI[[3]]
- ## covr: skip=1
- expr[[ii]] <- substitute({a <- b; e}, list(a=lhs, b=rhs, e=exprI))
- } else if (op %in% c("->>", "%->%", "%=>%")) {
- lhs <- exprI[[3]]
- rhs <- exprI[[2]]
- ## covr: skip=1
- expr[[ii]] <- substitute({a <- b; e}, list(a=lhs, b=rhs, e=exprI))
- }
- }, error=function(ex) {})
+
+
+## Future assignment #1:
+## expression: lhs %<-% rhs
+## AST: (%<-% lhs rhs)
+## tweaked expression: lhs <- rhs; lhs %<-% rhs
+tweakFutureAssignmentCall <- function(expr) {
+ if (!is.call(expr)) return(expr)
+ op <- expr[[1]]
+ if (!is.symbol(op)) return(expr)
+ n <- length(expr)
+ if (n != 3) return(expr)
+
+ op <- as.character(op)
+ if (op %in% c("<<-", "%<-%", "%<=%")) {
+ lhs <- expr[[2]]
+ rhs <- expr[[3]]
+ ## covr: skip=1
+ expr <- substitute({a <- b; e}, list(a=lhs, b=rhs, e=expr))
+ } else if (op %in% c("->>", "%->%", "%=>%")) {
+ lhs <- expr[[3]]
+ rhs <- expr[[2]]
+ ## covr: skip=1
+ expr <- substitute({a <- b; e}, list(a=lhs, b=rhs, e=expr))
}
expr
+} ## tweakFutureAssignmentCall()
+
+
+#' @importFrom globals walkAST
+tweakExpression <- function(expr) {
+ if (!is.language(expr)) return(expr)
+
+ expr <- walkAST(expr, call=tweakFutureAssignmentCall)
+ expr <- walkAST(expr, call=tweakFormulaCall)
+ expr <- walkAST(expr, call=tweakSubassignmentCall)
+
+ expr
} # tweakExpression()
diff --git a/R/tweak_OP.R b/R/tweak_OP.R
index c56f20b..74711b8 100644
--- a/R/tweak_OP.R
+++ b/R/tweak_OP.R
@@ -3,7 +3,7 @@
#' @usage fassignment \%tweak\% tweaks
#'
#' @param fassignment The future assignment, e.g.
-#' \code{x \%<=\% \{ expr \}}.
+#' \code{x \%<-\% \{ expr \}}.
#' @param tweaks A named list (or vector) with arguments that
#' should be changed relative to the current strategy.
#'
diff --git a/R/tweak_parallel_PSOCK.R b/R/tweak_parallel_PSOCK.R
new file mode 100644
index 0000000..5617de5
--- /dev/null
+++ b/R/tweak_parallel_PSOCK.R
@@ -0,0 +1,149 @@
+#' Tweak PSOCK backend of the parallel package
+#'
+#' @param user If TRUE, parallel is tweaked to only pass username to SSH if it is specified via argument \code{user}.
+#' @param revtunnel If TRUE, parallel is tweaked to make use of reverse SSH tunneling.
+#' @param rshopts If TRUE, parallel is tweaked so it is possible to specify additional command-line options to the \code{rshcmd} executable.
+#' @param use127.0.0.1 If TRUE, \code{127.0.0.1} is used instead of \code{localhost}.
+#' @param reset If TRUE, all tweaks are undone.
+#'
+#' @return Nothing.
+#'
+#' @examples
+#' \donttest{\dontrun{
+#' trace(system, tracer = quote(stop("Command: ", command)), print = FALSE)
+#'
+#' ## Without tweaks
+#' try(cl <- parallel::makeCluster("remote.myserver.org", revtunnel=TRUE,
+#' master="localhost", homogeneous=FALSE))
+#' ### Error in eval(expr, envir, enclos) :
+#' ### Command:
+#' ### ssh
+#' ### -l hb
+#' ### remote.myserver.org
+#' ### "Rscript
+#' ### --default-packages=datasets,utils,grDevices,graphics,stats,methods
+#' ### -e 'parallel:::.slaveRSOCK()' MASTER=localhost PORT=11099
+#' ### OUT=/dev/null TIMEOUT=2592000 XDR=TRUE"
+#'
+#' ## With tweaks
+#' future:::tweak_parallel_PSOCK()
+#' try(cl <- parallel::makeCluster("remote.myserver.org", revtunnel=TRUE,
+#' master="localhost", homogeneous=FALSE))
+#' ### Error in eval(expr, envir, enclos) :
+#' ### Command:
+#' ### ssh
+#' ### -R 11624:localhost:11624
+#' ### remote.myserver.org
+#' ### "Rscript
+#' ### --default-packages=datasets,utils,grDevices,graphics,stats,methods
+#' ### -e 'parallel:::.slaveRSOCK()' MASTER=localhost PORT=11099
+#' ### OUT=/dev/null TIMEOUT=2592000 XDR=TRUE"
+#'
+#' ## Undo tweaks
+#' untrace(system)
+#' future:::tweak_parallel_PSOCK(reset=TRUE)
+#' }}
+#'
+#' @references
+#' \url{https://github.com/HenrikBengtsson/Wishlist-for-R/issues/32}\cr
+#'
+#' @importFrom utils assignInNamespace
+#' @keywords internal
+tweak_parallel_PSOCK <- local({
+ parallel <- NULL
+
+ ## The original newPSOCKnode() and defaultClusterOptions of parallel
+ newPSOCKnode_org <- defaultClusterOptions_org <- NULL
+
+ .assignInNamespace <- NULL
+
+ init <- function() {
+ ## Already initiated?
+ if (!is.null(parallel)) return()
+
+ parallel <<- getNamespace("parallel")
+
+ ## The original newPSOCKnode() and defaultClusterOptions of parallel
+ newPSOCKnode_org <<- get("newPSOCKnode", mode="function", envir=parallel)
+ defaultClusterOptions_org <<- get("defaultClusterOptions", envir=parallel)
+
+ ## HACK: Trick assignInNamespace() to not complain
+ .assignInNamespace <<- gsub_body("nf <- sys.nframe()", "nf <- 1L", assignInNamespace, fixed=TRUE)
+ } ## init()
+
+ gsub_body <- function(pattern, replacement, fun, ...) {
+ stopifnot(is.function(fun))
+ expr <- body(fun)
+ code <- deparse(expr)
+ code <- gsub(pattern=pattern, replacement=replacement, x=code, ...)
+ expr <- parse(text=code)
+ body(fun) <- expr
+ fun
+ } ## gsub_body()
+
+
+
+ ## tweak_parallel_PSOCK()
+ function(user=TRUE, revtunnel=TRUE, rshopts=TRUE, use127.0.0.1=FALSE, reset=FALSE) {
+ ## To please R CMD check
+ .assignInNamespace <- NULL; rm(list=".assignInNamespace")
+
+ ## Make sure to initiate
+ init()
+
+ ## Original parallel setup
+ newPSOCKnode <- newPSOCKnode_org
+ defaultClusterOptions <- defaultClusterOptions_org
+
+ if (!reset) {
+ defaultClusterOptions <- as.environment(as.list(defaultClusterOptions))
+
+ pattern <- 'cmd <- paste(rshcmd, "-l", user, machine, cmd)'
+ replacement <- 'opts <- NULL'
+
+ ## Only pass '-l <user>' if explicitly specified
+ ## https://github.com/HenrikBengtsson/Wishlist-for-R/issues/31
+ if (user) {
+ ## Drop default 'user' (clone to avoid overwriting the original environment)
+ defaultClusterOptions$user <- NULL
+
+ ## newPSOCKnode() tweaks
+ replacement <- c(replacement, 'if (!is.null(user)) opts <- c(opts, paste("-l", user))')
+ }
+
+ ## Reverse SSH tunneling (avoids all NAT / firewall issues)
+ ## https://github.com/HenrikBengtsson/Wishlist-for-R/issues/32
+ if (revtunnel) {
+ defaultClusterOptions$revtunnel <- FALSE
+
+ ## newPSOCKnode() tweaks
+ replacement <- c(replacement, 'revtunnel <- getClusterOption("revtunnel", options)')
+ replacement <- c(replacement, 'if (revtunnel) opts <- c(opts, sprintf("-R %d:%s:%d", port + (rank - 1L), master, port))')
+ }
+
+ ## Support for any type of command-line options (advanced usage)
+ ## https://github.com/HenrikBengtsson/Wishlist-for-R/issues/32
+ if (rshopts) {
+ defaultClusterOptions$rshopts <- NULL
+
+ ## newPSOCKnode() tweaks
+ replacement <- c(replacement, 'opts <- c(opts, getClusterOption("rshopts", options))')
+ }
+
+ replacement <- c(replacement, 'opts <- paste(opts, collapse = " ")')
+ replacement <- c(replacement, 'cmd <- paste(rshcmd, opts, machine, cmd)')
+ replacement <- paste(replacement, collapse="\n")
+
+ newPSOCKnode <- gsub_body(pattern, replacement, fun=newPSOCKnode, fixed=TRUE)
+
+ ## Use '127.0.0.1' instead of 'localhost'
+ if (use127.0.0.1) {
+ newPSOCKnode <- gsub_body('socketConnection("localhost"', 'socketConnection("127.0.0.1"', fun=newPSOCKnode, fixed=TRUE)
+ }
+ }
+
+ ## Update newPSOCKnode() and defaultClusterOptions of parallel
+ .assignInNamespace("newPSOCKnode", newPSOCKnode, ns=parallel)
+ .assignInNamespace("defaultClusterOptions", defaultClusterOptions, ns=parallel)
+ } ## tweak_parallel_PSOCK()
+}) ## local()
diff --git a/R/utils.R b/R/utils.R
index a87360a..e96c75c 100644
--- a/R/utils.R
+++ b/R/utils.R
@@ -330,7 +330,7 @@ myInternalIP <- local({
if (grepl("^linux", os)) {
res <- system2("hostname", args="-I", stdout=TRUE)
res <- grep(pattern, res, value=TRUE)
- res <- unlist(strsplit(res, split="[ ]+", fixed=FALSE))
+ res <- unlist(strsplit(res, split="[ ]+", fixed=FALSE), use.names=FALSE)
res <- grep(pattern, res, value=TRUE)
res <- unique(trim(res))
## Keep private network IPs only (just in case)
@@ -339,7 +339,7 @@ myInternalIP <- local({
res <- system2("ipconfig", stdout=TRUE)
res <- grep("IPv4", res, value=TRUE)
res <- grep(pattern, res, value=TRUE)
- res <- unlist(strsplit(res, split="[ ]+", fixed=FALSE))
+ res <- unlist(strsplit(res, split="[ ]+", fixed=FALSE), use.names=FALSE)
res <- grep(pattern, res, value=TRUE)
res <- unique(trim(res))
## Keep private network IPs only (just in case)
diff --git a/R/values.R b/R/values.R
index fbf31e3..27faad3 100644
--- a/R/values.R
+++ b/R/values.R
@@ -7,7 +7,7 @@
#' object is kept.
#'
#' @param x An environment, a list, or a list environment.
-#' @param ... Additional arguments passed to \code{value()} of each future.
+#' @param \dots Additional arguments passed to \code{value()} of each future.
#'
#' @return An object of same type as \code{x} and with the same names
#' and/or dimensions, if set.
diff --git a/README.md b/README.md
index 1909de3..39a5fbf 100644
--- a/README.md
+++ b/README.md
@@ -111,7 +111,7 @@ One of the designs of the Future API was to encapsulate any differences such tha
Because of this, the defaults of the different strategies are such that the results and side effects of evaluating a future expression are as similar as possible. More specifically, the following is true for all futures:
-* All _evaluation is done in a local environment_ (i.e. `local({ expr })`) such assignments do not affect the calling environment. This is natural when evaluating in an external R process, but is also enforced when evaluating in the current R session.
+* All _evaluation is done in a local environment_ (i.e. `local({ expr })`) so that assignments do not affect the calling environment. This is natural when evaluating in an external R process, but is also enforced when evaluating in the current R session.
* When a future is constructed, _global variables are identified_. For asynchronous evaluation, globals are exported to the R process/session that will be evaluating the future expression. For lazy futures, globals are "frozen" (cloned to a local environment of the future). Also, in order to protect against exporting too large objects by mistake, there is a built-in assertion that the total size of all globals is less than a given threshold (controllable via an option, cf. `help("futu [...]
@@ -146,7 +146,7 @@ Eager futures are the default unless otherwise specified. They were designed to
> plan(eager)
> pid <- Sys.getpid()
> pid
-[1] 28072
+[1] 14641
> a %<-% {
+ pid <- Sys.getpid()
+ cat("Resolving 'a' ...\n")
@@ -165,13 +165,13 @@ Resolving 'b' ...
+ }
Resolving 'c' ...
> b
-[1] 28072
+[1] 14641
> c
[1] 6.28
> a
[1] 3.14
> pid
-[1] 28072
+[1] 14641
```
Since eager evaluation is taking place, each of the three futures is resolved instantaneously in the moment it is created. Note also how `pid` in the calling environment, which was assigned the process ID of the current process, is neither overwritten nor removed. This is because futures are evaluated in a local environment. Since synchronous (uni-)processing is used, future `b` is resolved by the main R process (still in a local environment), which is why the value of `b` and `pid` a [...]
@@ -182,7 +182,7 @@ A lazy future evaluates its expression only if its value is queried. Evaluation
> plan(lazy)
> pid <- Sys.getpid()
> pid
-[1] 28072
+[1] 14641
> a %<-% {
+ pid <- Sys.getpid()
+ cat("Resolving 'a' ...\n")
@@ -200,14 +200,14 @@ A lazy future evaluates its expression only if its value is queried. Evaluation
Resolving 'a' ...
> b
Resolving 'b' ...
-[1] 28072
+[1] 14641
> c
Resolving 'c' ...
[1] 6.28
> a
[1] 3.14
> pid
-[1] 28072
+[1] 14641
```
As previously, variable `pid` is unaffected because all evaluation is done in a local environment. More interestingly, future `a` is no longer evaluated in the moment it is created, but instead when it is needed the first time, which happens when future `c` is created. This is because `a` is identified as a global variable that needs to be captured ("frozen" to `a == 3.14`) in order to set up future `c`. Later when `c` (the value of future `c`) is queried, `a` has already been resolve [...]
@@ -229,7 +229,7 @@ We start with multisession futures because they are supported by all operating s
> plan(multisession)
> pid <- Sys.getpid()
> pid
-[1] 28072
+[1] 14641
> a %<-% {
+ pid <- Sys.getpid()
+ cat("Resolving 'a' ...\n")
@@ -245,13 +245,13 @@ We start with multisession futures because they are supported by all operating s
+ 2 * a
+ }
> b
-[1] 28093
+[1] 14662
> c
[1] 6.28
> a
[1] 3.14
> pid
-[1] 28072
+[1] 14641
```
The first thing we observe is that the values of `a`, `c` and `pid` are the same as previously. However, we notice that `b` is different from before. This is because future `b` is evaluated in a different R process and therefore it returns a different process ID. Another difference is that the messages, generated by `cat()`, are no longer displayed. This is because they are outputted to the background sessions and not the calling session.
@@ -291,7 +291,7 @@ Cluster futures evaluate expressions on an ad-hoc cluster (as implemented by the
> plan(cluster, workers = c("n1", "n2", "n3"))
> pid <- Sys.getpid()
> pid
-[1] 28072
+[1] 14641
> a %<-% {
+ pid <- Sys.getpid()
+ cat("Resolving 'a' ...\n")
@@ -307,13 +307,13 @@ Cluster futures evaluate expressions on an ad-hoc cluster (as implemented by the
+ 2 * a
+ }
> b
-[1] 28115
+[1] 14684
> c
[1] 6.28
> a
[1] 3.14
> pid
-[1] 28072
+[1] 14641
```
Just as for the other asynchronous evaluation strategies, the output from `cat()` is not displayed on the current/calling machine.
@@ -335,7 +335,7 @@ Sometimes one may want to use an alternative evaluation strategy for a specific
> plan(eager)
> pid <- Sys.getpid()
> pid
-[1] 28072
+[1] 14641
> a %<-% {
+ Sys.getpid()
+ }
@@ -346,11 +346,11 @@ Sometimes one may want to use an alternative evaluation strategy for a specific
+ Sys.getpid()
+ } %plan% multiprocess
> a
-[1] 28072
+[1] 14641
> b
-[1] 28132
+[1] 14701
> c
-[1] 28133
+[1] 14702
```
As seen by the different process IDs, future `a` is evaluated eagerly using the same process as the calling environment whereas the other two are evaluated using multiprocess futures.
@@ -384,14 +384,14 @@ For instance, here is an example of two "top" futures (`a` and `b`) that uses mu
+ c(b.pid = Sys.getpid(), b1.pid = b1, b2.pid = b2)
+ }
> pid
-[1] 28072
+[1] 14641
> a
-[1] 28134
+[1] 14703
> b
b.pid b1.pid b2.pid
- 28135 28135 28135
+ 14704 14704 14704
```
-By inspection the process IDs, we see that there are in total three different processes involved for resolving the futures. There is the main R process (pid 28072), and there are the two processes used by `a` (pid 28134) and `b` (pid 28135). However, the two futures (`b1` and `b2`) that is nested by `b` are evaluated by the same R process as `b`. This is because nested futures use eager evaluation unless otherwise specified. There are a few reasons for this, but the main reason is th [...]
+By inspection the process IDs, we see that there are in total three different processes involved for resolving the futures. There is the main R process (pid 14641), and there are the two processes used by `a` (pid 14703) and `b` (pid 14704). However, the two futures (`b1` and `b2`) that is nested by `b` are evaluated by the same R process as `b`. This is because nested futures use eager evaluation unless otherwise specified. There are a few reasons for this, but the main reason is th [...]
@@ -404,12 +404,12 @@ We would actually get the same behavior if we try with multiple levels of multip
> plan(list(multiprocess, multiprocess))
[...]
> pid
-[1] 28072
+[1] 14641
> a
-[1] 28136
+[1] 14705
> b
b.pid b1.pid b2.pid
- 28137 28137 28137
+ 14706 14706 14706
```
The reason for this is, also here, to protect us from launching more processes than what the machine can support. Internally, this is done by setting `mc.cores` to zero ([sic!](https://github.com/HenrikBengtsson/Wishlist-for-R/issues/7)) such that no _additional_ parallel processes can be launched. This is the case for both multisession and multicore evaluation.
@@ -421,14 +421,14 @@ Continuing, if we start off by eager evaluation and then use multiprocess evalua
Resolving 'a' ...
Resolving 'b' ...
> pid
-[1] 28072
+[1] 14641
> a
-[1] 28072
+[1] 14641
> b
b.pid b1.pid b2.pid
- 28072 28138 28139
+ 14641 14707 14708
```
-which clearly show that `a` and `b` are resolved in the calling process (pid 28072) whereas the two nested futures (`b1` and `b2`) are resolved in two separate R processes (pids 28138 and 28139).
+which clearly show that `a` and `b` are resolved in the calling process (pid 14641) whereas the two nested futures (`b1` and `b2`) are resolved in two separate R processes (pids 14707 and 14708).
@@ -438,14 +438,14 @@ Having said this, it is indeed possible to use nested multiprocess evaluation st
+ workers = 3)))
[...]
> pid
-[1] 28072
+[1] 14641
> a
-[1] 28140
+[1] 14709
> b
b.pid b1.pid b2.pid
- 28141 28142 28145
+ 14710 14711 14712
```
-First, we see that both `a` and `b` are resolved in different processes (pids 28140 and 28141) than the calling process (pid 28072). Second, the two nested futures (`b1` and `b2`) are resolved in yet two other R processes (pids 28142 and 28145).
+First, we see that both `a` and `b` are resolved in different processes (pids 14709 and 14710) than the calling process (pid 14641). Second, the two nested futures (`b1` and `b2`) are resolved in yet two other R processes (pids 14711 and 14712).
To clarify, when we set up the two levels of multiprocess evaluation, we specified that in total 3 processes may be used at each level. We choose three parallel processes, not just two, because one is always consumed by the calling process leaving two to be used for the asynchronous futures. This is why we see that `pid`, `a` and `b` are all resolved by the same process. If we had allowed only two cores at the top level, `a` and `b` would have been resolved by the same background proc [...]
@@ -480,7 +480,7 @@ Waiting for 'a' to be resolved ...
> cat("Waiting for 'a' to be resolved ... DONE\n")
Waiting for 'a' to be resolved ... DONE
> a
-[1] 28146
+[1] 14713
```
@@ -547,9 +547,9 @@ There is one limitation with implicit futures that does not exist for explicit o
> v <- lapply(f, FUN = value)
> str(v)
List of 3
- $ : int 28147
- $ : int 28148
- $ : int 28149
+ $ : int 14714
+ $ : int 14715
+ $ : int 14716
```
This is _not_ possible to do when using implicit futures. This is because the `%<-%` assignment operator _cannot_ be used in all cases where the regular `<-` assignment operator can be used. It can only be used to assign future values to _environments_ (including the calling environment) much like how `assign(name, value, envir)` works. However, we can assign implicit futures to environments using _named indices_, e.g.
```r
@@ -563,9 +563,9 @@ This is _not_ possible to do when using implicit futures. This is because the `
> v <- as.list(v)
> str(v)
List of 3
- $ a: int 28150
- $ b: int 28151
- $ c: int 28152
+ $ a: int 14717
+ $ b: int 14718
+ $ c: int 14719
```
Here `as.list(v)` blocks until all futures in the environment `v` have been resolved. Then their values are collected and returned as a regular list.
@@ -582,9 +582,9 @@ If _numeric indices_ are required, then _list environments_ can be used. List e
> v <- as.list(v)
> str(v)
List of 3
- $ : int 28153
- $ : int 28154
- $ : int 28155
+ $ : int 14720
+ $ : int 14721
+ $ : int 14722
```
As previously, `as.list(v)` blocks until all futures are resolved.
@@ -622,11 +622,11 @@ demo("mandelbrot", package="future", ask=FALSE)
The goal of this package is to provide a standardized and unified API for using futures in R. What you are seeing right now is an early but sincere attempt to achieve this goal. If you have comments or ideas on how to improve the 'future' package, I would love to hear about them. The preferred way to get in touch is via the [GitHub repository](https://github.com/HenrikBengtsson/future/), where you also find the latest source code. I am also open to contributions and collaborations of [...]
-[BatchJobs]: http://cran.r-project.org/package=BatchJobs
-[future]: http://cran.r-project.org/package=future
-[future.BatchJobs]: http://cran.r-project.org/package=future.BatchJobs
-[globals]: http://cran.r-project.org/package=globals
-[listenv]: http://cran.r-project.org/package=listenv
+[BatchJobs]: https://cran.r-project.org/package=BatchJobs
+[future]: https://cran.r-project.org/package=future
+[future.BatchJobs]: https://cran.r-project.org/package=future.BatchJobs
+[globals]: https://cran.r-project.org/package=globals
+[listenv]: https://cran.r-project.org/package=listenv
[Futures in R: Common Issues with Solutions]: future-2-issues.html
[Futures in R: Future Topologies]: future-3-topologies.html
@@ -648,8 +648,8 @@ This will install the package from source.
## Software status
-| Resource: | CRAN | Travis CI | Appveyor |
-| ------------- | ------------------- | -------------- | ---------------- |
-| _Platforms:_ | _Multiple_ | _Linux & OS X_ | _Windows_ |
-| R CMD check | <a href="http://cran.r-project.org/web/checks/check_results_future.html"><img border="0" src="http://www.r-pkg.org/badges/version/future" alt="CRAN version"></a> | <a href="https://travis-ci.org/HenrikBengtsson/future"><img src="https://travis-ci.org/HenrikBengtsson/future.svg" alt="Build status"></a> | <a href="https://ci.appveyor.com/project/HenrikBengtsson/future"><img src="https://ci.appveyor.com/api/projects/status/github/HenrikBengtsson/future?svg=true" alt="Buil [...]
-| Test coverage | | <a href="https://codecov.io/gh/HenrikBengtsson/future"><img src="https://codecov.io/gh/HenrikBengtsson/future/branch/develop/graph/badge.svg" alt="Coverage Status"/></a> | |
+| Resource: | CRAN | Travis CI | Appveyor |
+| ------------- | ------------------- | --------------- | ---------------- |
+| _Platforms:_ | _Multiple_ | _Linux & macOS_ | _Windows_ |
+| R CMD check | <a href="http://cran.r-project.org/web/checks/check_results_future.html"><img border="0" src="http://www.r-pkg.org/badges/version/future" alt="CRAN version"></a> | <a href="https://travis-ci.org/HenrikBengtsson/future"><img src="https://travis-ci.org/HenrikBengtsson/future.svg" alt="Build status"></a> | <a href="https://ci.appveyor.com/project/HenrikBengtsson/future"><img src="https://ci.appveyor.com/api/projects/status/github/HenrikBengtsson/future?svg=true" alt="Bui [...]
+| Test coverage | | <a href="https://codecov.io/gh/HenrikBengtsson/future"><img src="https://codecov.io/gh/HenrikBengtsson/future/branch/develop/graph/badge.svg" alt="Coverage Status"/></a> | |
diff --git a/appveyor.yml b/appveyor.yml
index 73fcf83..bf6075d 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -10,7 +10,8 @@
#----------------------------------------------------------------
environment:
_R_CHECK_FORCE_SUGGESTS_: false
-
+ USE_RTOOLS: true
+
matrix:
- R_VERSION: devel
R_ARCH: x64
@@ -18,8 +19,8 @@ environment:
- R_VERSION: devel
R_ARCH: i386
-# - R_VERSION: release
-# R_ARCH: x64
+ - R_VERSION: release
+ R_ARCH: x64
# DO NOT CHANGE the "init" and "install" sections below
@@ -39,8 +40,7 @@ install:
build_script:
- echo Current directory=%CD%
- - travis-tool.sh install_r digest globals listenv markdown
- - travis-tool.sh install_r R.rsp
+ - travis-tool.sh install_r digest globals listenv markdown R.rsp
test_script:
- travis-tool.sh run_tests
diff --git a/cran-comments.md b/cran-comments.md
index 849884d..a89c92d 100644
--- a/cran-comments.md
+++ b/cran-comments.md
@@ -1,12 +1,14 @@
-# CRAN submission future 1.0.1
-on 2015-07-04
+# CRAN submission future 1.1.1
+on 2015-10-10
-Updates related to CRAN:
+Fixing R CMD check test error on CRAN r-devel-macos-x86_64-clang server that was (re-)introduced in future 1.1.0, which was submitted yesterday.
+
+Sorry about that and thanks in advance
+
+
+# CRAN submission future 1.1.0
+on 2015-10-09
-* Package failed on CRAN under r-devel-osx-x86_64-clang (only),
- because test code incorrectly assumed system("Rscript") would
- launch the same executable as the running R session.
-
Thanks in advance
@@ -20,30 +22,30 @@ The package has been verified using `R CMD check --as-cran` on:
* Platform x86_64-unknown-linux-gnu (64-bit) [Travis CI]:
- R version 3.2.5 (2016-04-14)
- R version 3.3.1 (2016-06-21)
- - R Under development (unstable) (2016-06-30 r70858)
+ - R Under development (unstable) (2016-10-07 r71466)
* Platform x86_64-pc-linux-gnu (64-bit):
+ - R version 3.0.3 (2014-03-06)
+ - R version 3.1.2 (2014-10-31)
- R version 3.1.3 (2015-03-09)
- R version 3.2.5 (2016-04-14)
- R version 3.3.0 (2016-05-03)
- R version 3.3.1 (2016-06-21)
- - R version 3.3.1 Patched (2016-06-28 r70858)
- - R Under development (unstable) (2016-06-30 r70858)
+ - R version 3.3.1 Patched (2016-10-05 r71468)
* Platform i686-pc-linux-gnu (32-bit):
- R version 3.3.1 (2016-06-21)
* Platform i386-w64-mingw32 (32-bit) [Appveyor CI]:
- - R Under development (unstable) (2016-06-22 r70818)
+ - R Under development (unstable) (2016-10-07 r71466)
* Platform x86_64-w64-mingw32/x64 (64-bit) [Appveyor CI]:
- - R Under development (unstable) (2016-06-22 r70818)
+ - R version 3.3.1 (2016-06-21)
+ - R Under development (unstable) (2016-10-07 r71466)
* Platform x86_64-w64-mingw32/x64 (64-bit) [win-builder]:
- R version 3.3.1 (2016-06-21)
- - R Under development (unstable) (2016-06-24 r70828)
+ - R Under development (unstable) (2016-10-08 r71472)
-The submitted updates cause no issues for any of the following
-7 reverse dependencies on CRAN and Bioconductor: PSCBS 0.61.0,
-QDNAseq 1.8.0, R.filesets 2.10.0, aroma.affymetrix 3.0.0,
-aroma.core 3.0.0, doFuture 0.2.0, and future.BatchJobs 0.12.1.
+The submitted updates cause no issues for any of the
+19 reverse dependencies on CRAN and Bioconductor.
diff --git a/demo/mandelbrot.R b/demo/mandelbrot.R
index 5561b42..5e2c1a2 100644
--- a/demo/mandelbrot.R
+++ b/demo/mandelbrot.R
@@ -33,6 +33,7 @@ if (!is.list(region)) {
}
}
nrow <- getOption("future.demo.mandelbrot.nrow", 3L)
+resolution <- getOption("future.demo.mandelbrot.resolution", 400L)
delay <- getOption("future.demo.mandelbrot.delay", interactive())
if (isTRUE(delay)) {
delay <- function(counts) Sys.sleep(rexp(1, rate=2))
@@ -42,7 +43,7 @@ if (isTRUE(delay)) {
## Generate Mandelbrot tiles to be computed
Cs <- mandelbrotTiles(xmid=region$xmid, ymid=region$ymid,
- side=region$side, nrow=nrow)
+ side=region$side, nrow=nrow, resolution=resolution)
if (interactive()) {
dev.new()
diff --git a/incl/future.R b/incl/future.R
index ed2a5da..339e3b9 100644
--- a/incl/future.R
+++ b/incl/future.R
@@ -1,13 +1,64 @@
-## Use lazy futures
-plan(lazy)
-
-f <- future({
- a <- 7
- b <- 3
- c <- 2
- a * b * c
+## Cluster futures gives an error on R CMD check on
+## Windows (but not Linux or OS X) for unknown reasons.
+## The same code works in package tests.
+\donttest{
+## Evaluate futures in parallel
+plan(multiprocess)
+}
+
+## Data
+x <- rnorm(100)
+y <- 2*x + 0.2 + rnorm(100)
+w <- 1 + x^2
+
+
+## (1) Regular assignments (evaluated sequentially)
+fitA <- lm(y ~ x, weights = w) ## with offset
+fitB <- lm(y ~ x - 1, weights = w) ## without offset
+fitC <- {
+ w <- 1 + abs(x) ## Different weights
+ lm(y ~ x, weights = w)
+}
+print(fitA)
+print(fitB)
+print(fitC)
+
+
+## (2) Future assignments (evaluated in parallel)
+fitA %<-% lm(y ~ x, weights = w) ## with offset
+fitB %<-% lm(y ~ x - 1, weights = w) ## without offset
+fitC %<-% {
+ w <- 1 + abs(x)
+ lm(y ~ x, weights = w)
+}
+print(fitA)
+print(fitB)
+print(fitC)
+
+
+## (3) Explicitly create futures (evaluated in parallel)
+## and retrieve their values
+fA <- future( lm(y ~ x, weights = w) )
+fB <- future( lm(y ~ x - 1, weights = w) )
+fC <- future({
+ w <- 1 + abs(x)
+ lm(y ~ x, weights = w)
})
+fitA <- value(fA)
+fitB <- value(fB)
+fitC <- value(fC)
+print(fitA)
+print(fitB)
+print(fitC)
-print(resolved(f))
-y <- value(f)
-print(y)
+
+## (4) Explit future assignments (evaluated in parallel)
+futureAssign("fitA", lm(y ~ x, weights = w))
+futureAssign("fitB", lm(y ~ x - 1, weights = w))
+futureAssign("fitC", {
+ w <- 1 + abs(x)
+ lm(y ~ x, weights = w)
+})
+print(fitA)
+print(fitB)
+print(fitC)
diff --git a/incl/futureAssign_OP.R b/incl/futureAssign_OP.R
deleted file mode 100644
index efee987..0000000
--- a/incl/futureAssign_OP.R
+++ /dev/null
@@ -1,32 +0,0 @@
-## Future assignment via "assign" function
-futureAssign("A", {
- x <- 3
- x^2
-})
-cat("Value 'A': ", A, sep="")
-
-
-## Equivalent via infix "assign" operator
-A %<-% {
- x <- 3
- x^2
-}
-cat("Value 'A': ", A, sep="")
-
-
-## A global variable
-a <- 1
-
-## Three future evaluations
-A %<-% { 0.1 }
-B %<-% { 0.2 }
-C %<-% { z <- a+0.3 }
-
-## Sleep until 'C' is available
-cat("Value 'C': ", C, sep="")
-
-## Sleep until 'A' is available
-cat("Value 'A': ", A, sep="")
-
-## Sleep until 'C' is available
-cat("Value 'B': ", B, sep="")
diff --git a/incl/nbrOfWorkers.R b/incl/nbrOfWorkers.R
new file mode 100644
index 0000000..f262ae6
--- /dev/null
+++ b/incl/nbrOfWorkers.R
@@ -0,0 +1,8 @@
+plan(multiprocess)
+nbrOfWorkers() ## == availableCores()
+
+plan(multiprocess, workers=2)
+nbrOfWorkers() ## == 2
+
+plan(lazy)
+nbrOfWorkers() ## == 1
diff --git a/inst/vignettes-static/future-1-overview.md.rsp.rsp b/inst/vignettes-static/future-1-overview.md.rsp.rsp
index e1be024..d5d7dec 100644
--- a/inst/vignettes-static/future-1-overview.md.rsp.rsp
+++ b/inst/vignettes-static/future-1-overview.md.rsp.rsp
@@ -161,7 +161,7 @@ One of the designs of the Future API was to encapsulate any differences such tha
Because of this, the defaults of the different strategies are such that the results and side effects of evaluating a future expression are as similar as possible. More specifically, the following is true for all futures:
-* All _evaluation is done in a local environment_ (i.e. `local({ expr })`) such assignments do not affect the calling environment. This is natural when evaluating in an external R process, but is also enforced when evaluating in the current R session.
+* All _evaluation is done in a local environment_ (i.e. `local({ expr })`) so that assignments do not affect the calling environment. This is natural when evaluating in an external R process, but is also enforced when evaluating in the current R session.
* When a future is constructed, _global variables are identified_. For asynchronous evaluation, globals are exported to the R process/session that will be evaluating the future expression. For lazy futures, globals are "frozen" (cloned to a local environment of the future). Also, in order to protect against exporting too large objects by mistake, there is a built-in assertion that the total size of all globals is less than a given threshold (controllable via an option, cf. `help("futu [...]
@@ -672,11 +672,11 @@ demo("mandelbrot", package="future", ask=FALSE)
The goal of this package is to provide a standardized and unified API for using futures in R. What you are seeing right now is an early but sincere attempt to achieve this goal. If you have comments or ideas on how to improve the 'future' package, I would love to hear about them. The preferred way to get in touch is via the [GitHub repository](https://github.com/HenrikBengtsson/future/), where you also find the latest source code. I am also open to contributions and collaborations of [...]
-[BatchJobs]: http://cran.r-project.org/package=BatchJobs
-[future]: http://cran.r-project.org/package=future
-[future.BatchJobs]: http://cran.r-project.org/package=future.BatchJobs
-[globals]: http://cran.r-project.org/package=globals
-[listenv]: http://cran.r-project.org/package=listenv
+[BatchJobs]: https://cran.r-project.org/package=BatchJobs
+[future]: https://cran.r-project.org/package=future
+[future.BatchJobs]: https://cran.r-project.org/package=future.BatchJobs
+[globals]: https://cran.r-project.org/package=globals
+[listenv]: https://cran.r-project.org/package=listenv
[Futures in R: Common Issues with Solutions]: future-2-issues.html
[Futures in R: Future Topologies]: future-3-topologies.html
---
diff --git a/man/ClusterFuture-class.Rd b/man/ClusterFuture-class.Rd
index ad469af..7b748d3 100644
--- a/man/ClusterFuture-class.Rd
+++ b/man/ClusterFuture-class.Rd
@@ -8,24 +8,20 @@
\title{A cluster future is a future whose value will be resolved asynchroneously in a parallel process}
\usage{
ClusterFuture(expr = NULL, envir = parent.frame(), substitute = FALSE,
- local = !persistent, gc = !persistent, persistent = FALSE,
- workers = NULL, ...)
+ local = !persistent, globals = TRUE, gc = FALSE, persistent = FALSE,
+ workers = NULL, user = NULL, master = NULL, revtunnel = TRUE,
+ homogeneous = TRUE, ...)
}
\arguments{
-\item{expr}{An R \link[base]{expression}.}
-
-\item{envir}{The \link{environment} in which the evaluation
-is done (or inherits from if \code{local} is TRUE).}
-
-\item{substitute}{If TRUE, argument \code{expr} is
-\code{\link[base]{substitute}()}:ed, otherwise not.}
-
\item{local}{If TRUE, the expression is evaluated such that
all assignments are done to local temporary environment, otherwise
-the assignments are done in the global environment of the cluster node.}
+the assignments are done to the global environment of the \R process
+evaluating the future.}
-\item{gc}{If TRUE, the garbage collector run after the future
-is resolved (in the process that evaluated the future).}
+\item{globals}{(optional) a logical, a character vector,
+or a named list for controlling how globals are handled.
+For details, see section 'Globals used by future expressions'
+in the help for \code{\link{future}()}.}
\item{persistent}{If FALSE, the evaluation environment is cleared
from objects prior to the evaluation of the future.}
@@ -34,7 +30,22 @@ from objects prior to the evaluation of the future.}
Alternatively, a character vector of host names or a numeric scalar,
for creating a cluster via \code{\link[parallel]{makeCluster}(workers)}.}
-\item{\dots}{Additional named elements of the future.}
+\item{user}{(optional) The user name to be used when communicating
+with another host.}
+
+\item{master}{(optional) The hostname or IP address of the master
+machine running this node.}
+
+\item{revtunnel}{If TRUE, reverse SSH tunneling is used for the
+PSOCK cluster nodes to connect back to the master R process. This
+avoids the hassle of firewalls, port forwarding and having to know
+the internal / public IP address of the master R session.}
+
+\item{homogeneous}{If TRUE, all cluster nodes is assumed to use the
+same path to \file{Rscript} as the main R session. If FALSE, the
+it is assumed to be on the PATH for each node.}
+
+\item{...}{Additional named elements passed to \code{\link{Future}()}.}
}
\value{
An object of class \code{ClusterFuture}.
diff --git a/man/ConstantFuture-class.Rd b/man/ConstantFuture-class.Rd
index 6e3e7b0..bd26f7e 100644
--- a/man/ConstantFuture-class.Rd
+++ b/man/ConstantFuture-class.Rd
@@ -6,10 +6,22 @@
\title{A future with a constant value}
\usage{
ConstantFuture(expr = NULL, envir = emptyenv(), substitute = FALSE,
- local = FALSE, gc = FALSE, ...)
+ local = FALSE, ...)
}
\arguments{
-\item{expr}{An R value.}
+\item{expr}{An R \link[base]{expression}.}
+
+\item{envir}{The \link{environment} in which the evaluation
+is done (or inherits from if \code{local} is TRUE).}
+
+\item{substitute}{If TRUE, argument \code{expr} is
+\code{\link[base]{substitute}()}:ed, otherwise not.}
+
+\item{local}{If TRUE, the expression is evaluated such that
+all assignments are done to local temporary environment, otherwise
+the assignments are done in the calling environment.}
+
+\item{...}{Additional named elements of the future.}
\item{\dots}{Not used.}
}
@@ -18,7 +30,7 @@ An object of class \code{ConstantFuture}.
}
\description{
A constant future is a future whose expression is a constant
-and therefore already resolved upon creation
+and therefore by definition already resolved upon creation.
}
\keyword{internal}
diff --git a/man/EagerFuture-class.Rd b/man/EagerFuture-class.Rd
index e31e997..4da57f3 100644
--- a/man/EagerFuture-class.Rd
+++ b/man/EagerFuture-class.Rd
@@ -6,24 +6,10 @@
\title{An eager future is a future whose value will be resolved immediately}
\usage{
EagerFuture(expr = NULL, envir = parent.frame(), substitute = FALSE,
- local = TRUE, gc = FALSE, ...)
+ globals = TRUE, local = TRUE, ...)
}
\arguments{
-\item{expr}{An R \link[base]{expression}.}
-
-\item{envir}{The \link{environment} in which the evaluation
-is done (or inherits from if \code{local} is TRUE).}
-
-\item{substitute}{If TRUE, argument \code{expr} is
-\code{\link[base]{substitute}()}:ed, otherwise not.}
-
-\item{local}{If TRUE, the expression is evaluated such that
-all assignments are done to local temporary environment, otherwise
-the assignments are done in the calling environment.}
-
-\item{gc}{If TRUE, the garbage collector run after the future is resolved.}
-
-\item{\dots}{Additional named elements of the future.}
+\item{...}{Additional named elements passed to \code{\link{Future}()}.}
}
\value{
An object of class \code{EagerFuture}.
diff --git a/man/Future-class.Rd b/man/Future-class.Rd
index 1d483fa..244d0aa 100644
--- a/man/Future-class.Rd
+++ b/man/Future-class.Rd
@@ -6,7 +6,7 @@
\title{A future represents a value that will be available at some point in the future}
\usage{
Future(expr = NULL, envir = parent.frame(), substitute = FALSE,
- local = TRUE, gc = FALSE, earlySignal = FALSE, ...)
+ local = TRUE, gc = FALSE, earlySignal = FALSE, label = NULL, ...)
}
\arguments{
\item{expr}{An R \link[base]{expression}.}
@@ -21,12 +21,16 @@ is done (or inherits from if \code{local} is TRUE).}
all assignments are done to local temporary environment, otherwise
the assignments are done in the calling environment.}
-\item{gc}{If TRUE, the garbage collector run after the future
-is resolved (in the process that evaluated the future).}
+\item{gc}{If TRUE, the garbage collector run (in the process that
+evaluated the future) after the value of the future is collected.
+\emph{Some types of futures ignore this argument.}}
\item{earlySignal}{Specified whether conditions should be signaled
as soon as possible or not.}
+\item{label}{An optional character string label attached to the
+future.}
+
\item{\dots}{Additional named elements of the future.}
}
\value{
@@ -46,7 +50,7 @@ A Future object is itself an \link{environment}.
\seealso{
One function that creates a Future is \code{\link{future}()}.
It returns a Future that evaluates an R expression in the future.
-An alternative approach is to use the \code{\link{\%<=\%}} infix
+An alternative approach is to use the \code{\link{\%<-\%}} infix
assignment operator, which creates a future from the
right-hand-side (RHS) R expression and assigns its future value
to a variable as a \emph{\link[base]{promise}}.
diff --git a/man/LazyFuture-class.Rd b/man/LazyFuture-class.Rd
index 51359aa..e24d1c4 100644
--- a/man/LazyFuture-class.Rd
+++ b/man/LazyFuture-class.Rd
@@ -6,24 +6,10 @@
\title{A lazy future is a future whose value will be resolved at the time when it is requested}
\usage{
LazyFuture(expr = NULL, envir = parent.frame(), substitute = FALSE,
- local = TRUE, gc = FALSE, ...)
+ globals = TRUE, local = TRUE, ...)
}
\arguments{
-\item{expr}{An R \link[base]{expression}.}
-
-\item{envir}{The \link{environment} in which the evaluation
-is done (or inherits from if \code{local} is TRUE).}
-
-\item{substitute}{If TRUE, argument \code{expr} is
-\code{\link[base]{substitute}()}:ed, otherwise not.}
-
-\item{local}{If TRUE, the expression is evaluated such that
-all assignments are done to local temporary environment, otherwise
-the assignments are done in the calling environment.}
-
-\item{gc}{If TRUE, the garbage collector run after the future is resolved.}
-
-\item{\dots}{Additional named elements of the future.}
+\item{...}{Additional named elements passed to \code{\link{Future}()}.}
}
\value{
An object of class \code{LazyFuture}.
diff --git a/man/MulticoreFuture-class.Rd b/man/MulticoreFuture-class.Rd
index ad199ec..811aeef 100644
--- a/man/MulticoreFuture-class.Rd
+++ b/man/MulticoreFuture-class.Rd
@@ -6,18 +6,10 @@
\title{An multicore future is a future whose value will be resolved asynchroneously in a parallel process}
\usage{
MulticoreFuture(expr = NULL, envir = parent.frame(), substitute = FALSE,
- ...)
+ globals = TRUE, ...)
}
\arguments{
-\item{expr}{An R \link[base]{expression}.}
-
-\item{envir}{The \link{environment} in which the evaluation
-is done (or inherits from if \code{local} is TRUE).}
-
-\item{substitute}{If TRUE, argument \code{expr} is
-\code{\link[base]{substitute}()}:ed, otherwise not.}
-
-\item{\dots}{Additional named elements of the future.}
+\item{...}{Additional named elements passed to \code{\link{Future}()}.}
}
\value{
An object of class \code{MulticoreFuture}.
diff --git a/man/MultiprocessFuture-class.Rd b/man/MultiprocessFuture-class.Rd
index 544e023..0396a02 100644
--- a/man/MultiprocessFuture-class.Rd
+++ b/man/MultiprocessFuture-class.Rd
@@ -17,7 +17,14 @@ is done (or inherits from if \code{local} is TRUE).}
\item{substitute}{If TRUE, argument \code{expr} is
\code{\link[base]{substitute}()}:ed, otherwise not.}
-\item{\dots}{Additional named elements of the future.}
+\item{...}{Additional named elements of the future.}
+
+\item{local}{If TRUE, the expression is evaluated such that
+all assignments are done to local temporary environment, otherwise
+the assignments are done to the global environment of the \R process
+evaluating the future.}
+
+\item{\dots}{Additional named elements passed to \code{\link{Future}()}.}
}
\value{
An object of class \code{MultiprocessFuture}.
diff --git a/man/UniprocessFuture-class.Rd b/man/UniprocessFuture-class.Rd
index 137a0fe..2c35072 100644
--- a/man/UniprocessFuture-class.Rd
+++ b/man/UniprocessFuture-class.Rd
@@ -21,7 +21,9 @@ is done (or inherits from if \code{local} is TRUE).}
all assignments are done to local temporary environment, otherwise
the assignments are done in the calling environment.}
-\item{\dots}{Additional named elements of the future.}
+\item{...}{Additional named elements of the future.}
+
+\item{\dots}{Additional named elements passed to \code{\link{Future}()}.}
}
\value{
An object of class \code{UniprocessFuture}.
diff --git a/man/availableCores.Rd b/man/availableCores.Rd
index c625e94..62f8e00 100644
--- a/man/availableCores.Rd
+++ b/man/availableCores.Rd
@@ -6,7 +6,7 @@
\usage{
availableCores(constraints = NULL,
methods = getOption("future.availableCores.methods", c("system",
- "mc.cores+1", "_R_CHECK_LIMIT_CORES_", "Slurm", "PBS", "SGE")),
+ "mc.cores+1", "_R_CHECK_LIMIT_CORES_", "PBS", "SGE", "Slurm")),
na.rm = TRUE, default = c(current = 1L), which = c("min", "max", "all"))
}
\arguments{
@@ -78,5 +78,18 @@ For any other value of a \code{methods} element, the R option with the
same name is queried. If that is not set, the system environment
variable is queried. If neither is set, a missing value is returned.
}
+\section{Advanced usage}{
+
+It is possible to override the maximum number of cores on the machine
+as reported by \code{availableCores(methods="system")}. This can be
+done by first specifying
+\code{options(future.availableCores.methods="mc.cores+1")} and
+then the number of cores to use (in addition to the main R process),
+e.g. \code{options(mc.cores=8)} will cause the value of
+\code{availableCores()} to be 9 (=8+1).
+Having said this, it is almost always better to do this by explicitly
+setting the number of workers when specifying the future strategy,
+e.g. \code{plan(multiprocess, workers=9)}.
+}
\keyword{internal}
diff --git a/man/cluster.Rd b/man/cluster.Rd
index d7d16c3..3ca2771 100644
--- a/man/cluster.Rd
+++ b/man/cluster.Rd
@@ -4,31 +4,55 @@
\alias{cluster}
\title{Create a cluster future whose value will be resolved asynchroneously in a parallel process}
\usage{
-cluster(expr, envir = parent.frame(), substitute = TRUE,
- persistent = FALSE, workers = NULL, gc = FALSE, earlySignal = FALSE,
+cluster(expr, envir = parent.frame(), substitute = TRUE, globals = TRUE,
+ persistent = FALSE, workers = NULL, user = NULL, revtunnel = TRUE,
+ homogeneous = TRUE, gc = FALSE, earlySignal = FALSE, label = NULL,
...)
}
\arguments{
-\item{expr}{An R \link[base]{expression}.}
+\item{expr}{An \R \link[base]{expression}.}
-\item{envir}{The \link{environment} in which the evaluation
-is done and from which globals are obtained.}
+\item{envir}{The \link{environment} from where global
+objects should be identified. Depending on "evaluator",
+it may also be the environment in which the expression
+is evaluated.}
\item{substitute}{If TRUE, argument \code{expr} is
\code{\link[base]{substitute}()}:ed, otherwise not.}
+\item{globals}{A logical, a character vector,
+or a named list for controlling how globals are handled.
+For details, see below section.
+This argument can be specified via the \dots arguments
+for \code{future()} and \code{futureCall()}.}
+
\item{persistent}{If FALSE, the evaluation environment is cleared
from objects prior to the evaluation of the future.}
\item{workers}{A cluster object created by
\code{\link[parallel]{makeCluster}()}.}
-\item{gc}{If TRUE, the garbage collector run after the future
-is resolved (in the process that evaluated the future).}
+\item{user}{(optional) The user name to be used when communicating
+with another host.}
+
+\item{revtunnel}{If TRUE, reverse SSH tunneling is used for the
+PSOCK cluster nodes to connect back to the master R process. This
+avoids the hassle of firewalls, port forwarding and having to know
+the internal / public IP address of the master R session.}
+
+\item{homogeneous}{If TRUE, all cluster nodes is assumed to use the
+same path to \file{Rscript} as the main R session. If FALSE, the
+it is assumed to be on the PATH for each node.}
+
+\item{gc}{If TRUE, the garbage collector run (in the process that
+evaluated the future) after the value of the future is collected.}
+
+\item{earlySignal}{Specified whether conditions should be signaled
+as soon as possible or not.}
-\item{earlySignal}{Specified whether conditions should be signaled as soon as possible or not.}
+\item{label}{An optional character string label attached to the future.}
-\item{\dots}{Not used.}
+\item{...}{Additional arguments passed to the "evaluator".}
}
\value{
A \link{ClusterFuture}.
@@ -47,7 +71,7 @@ The preferred way to create an cluster future is not to call
this function directly, but to register it via
\code{\link{plan}(cluster)} such that it becomes the default
mechanism for all futures. After this \code{\link{future}()}
-and \code{\link{\%<=\%}} will create \emph{cluster futures}.
+and \code{\link{\%<-\%}} will create \emph{cluster futures}.
}
\examples{
## Cluster futures gives an error on R CMD check on
diff --git a/man/eager.Rd b/man/eager.Rd
index ba20cf0..e3f5817 100644
--- a/man/eager.Rd
+++ b/man/eager.Rd
@@ -6,32 +6,35 @@
\title{Create an eager future whose value will be resolved immediately}
\usage{
eager(expr, envir = parent.frame(), substitute = TRUE, globals = TRUE,
- local = TRUE, gc = FALSE, earlySignal = FALSE, ...)
+ local = TRUE, earlySignal = FALSE, label = NULL, ...)
}
\arguments{
-\item{expr}{An R \link[base]{expression}.}
+\item{expr}{An \R \link[base]{expression}.}
-\item{envir}{The \link{environment} in which the evaluation
-is done (or inherits from if \code{local} is TRUE).}
+\item{envir}{The \link{environment} from where global
+objects should be identified. Depending on "evaluator",
+it may also be the environment in which the expression
+is evaluated.}
\item{substitute}{If TRUE, argument \code{expr} is
\code{\link[base]{substitute}()}:ed, otherwise not.}
-\item{globals}{If TRUE, global objects are validated at the point
-in time when the future is created (always before it is resolved),
-that is, they identified and located. If some globals fail to be
-located, an informative error is generated.}
+\item{globals}{A logical, a character vector,
+or a named list for controlling how globals are handled.
+For details, see below section.
+This argument can be specified via the \dots arguments
+for \code{future()} and \code{futureCall()}.}
\item{local}{If TRUE, the expression is evaluated such that
all assignments are done to local temporary environment, otherwise
the assignments are done in the calling environment.}
-\item{gc}{If TRUE, the garbage collector run after the future
-is resolved (in the process that evaluated the future).}
+\item{earlySignal}{Specified whether conditions should be signaled
+as soon as possible or not.}
-\item{earlySignal}{Specified whether conditions should be signaled as soon as possible or not.}
+\item{label}{An optional character string label attached to the future.}
-\item{\dots}{Not used.}
+\item{...}{Additional arguments passed to the "evaluator".}
}
\value{
An \link{EagerFuture}.
@@ -47,7 +50,7 @@ by default just as for all other types of futures in this package.
The preferred way to create an eager future is not to call this function
directly, but to register it via \code{\link{plan}(eager)} such that it
becomes the default mechanism for all futures. After this
-\code{\link{future}()} and \code{\link{\%<=\%}} will create
+\code{\link{future}()} and \code{\link{\%<-\%}} will create
\emph{eager futures}.
}
\section{transparent futures}{
diff --git a/man/flapply.Rd b/man/flapply.Rd
new file mode 100644
index 0000000..f127c62
--- /dev/null
+++ b/man/flapply.Rd
@@ -0,0 +1,23 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/flapply.R
+\name{flapply}
+\alias{flapply}
+\title{Apply a Function over a List or Vector via Futures}
+\usage{
+flapply(x, FUN, ...)
+}
+\arguments{
+\item{x}{A vector-like object to iterate over.}
+
+\item{FUN}{A function taking at least one argument.}
+
+\item{...}{(optional) Additional arguments pass to \code{FUN()}.}
+}
+\value{
+A list.
+}
+\description{
+Apply a Function over a List or Vector via Futures
+}
+\keyword{internal}
+
diff --git a/man/future.Rd b/man/future.Rd
index 6b352bd..18da2c6 100644
--- a/man/future.Rd
+++ b/man/future.Rd
@@ -1,18 +1,28 @@
% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/future.R, R/futureCall.R
+% Please edit documentation in R/future.R, R/futureAssign.R, R/futureAssign_OP.R, R/futureCall.R
\name{future}
+\alias{\%->\%}
+\alias{\%<-\%}
+\alias{\%<=\%}
+\alias{\%=>\%}
\alias{future}
+\alias{futureAssign}
\alias{futureCall}
\title{Create a future}
\usage{
future(expr, envir = parent.frame(), substitute = TRUE,
evaluator = plan(), ...)
-futureCall(FUN, args = NULL, envir = parent.frame(), evaluator = plan(),
- ...)
+futureAssign(x, value, envir = parent.frame(), assign.env = envir,
+ substitute = TRUE)
+
+x \%<-\% value
+
+futureCall(FUN, args = NULL, envir = parent.frame(), globals = TRUE,
+ evaluator = plan(), ...)
}
\arguments{
-\item{expr}{An R \link[base]{expression}.}
+\item{expr}{An \R \link[base]{expression}.}
\item{envir}{The \link{environment} from where global
objects should be identified. Depending on "evaluator",
@@ -23,44 +33,250 @@ is evaluated.}
\code{\link[base]{substitute}()}:ed, otherwise not.}
\item{evaluator}{The actual function that evaluates
-\code{expr} and returns a future. The evaluator function
-should accept all the same arguments as this function
-(except \code{evaluator}).}
+the future expression and returns a \link{Future}.
+The evaluator function should accept all of the same
+arguments as the ones listed here
+(except \code{evaluator}, \code{FUN} and \code{args}).
+The default evaluator function is the one that the user
+has specified via \code{\link{plan}()}.}
+
+\item{...}{Additional arguments passed to the "evaluator".}
+
+\item{x}{the name of a future variable.}
+
+\item{value}{the \R \link[base]{expression} to be evaluated in
+the future and whose value will be assigned to the variable.}
-\item{FUN}{A function object.}
+\item{assign.env}{The \link[base]{environment} to which the variable
+should be assigned.}
-\item{args}{A list of arguments passed to function \code{FUN}.}
+\item{FUN}{A \link[base]{function} object.}
-\item{\dots}{Additional arguments passed to the "evaluator".}
+\item{args}{A \link[base]{list} of arguments passed to function \code{FUN}.}
+
+\item{globals}{A logical, a character vector,
+or a named list for controlling how globals are handled.
+For details, see below section.
+This argument can be specified via the \dots arguments
+for \code{future()} and \code{futureCall()}.}
}
\value{
-A \link{Future}.
+\code{f <- future(expr)} creates a \link{Future} \code{f} that evaluates expression \code{expr}, the value of the future is retrieved using \code{v <- value(f)}.
+
+\code{f <- futureCall(FUN, args)} creates a \link{Future} \code{f} that calls function \code{FUN} with arguments \code{args}, where the value of the future is retrieved using \code{v <- value(f)}.
+
+\code{futureAssign("v", expr)} and \code{v \%<-\% expr} (a future assignment) create a \link{Future} that evaluates expression \code{expr} and binds its value (as a \link[base]{promise}) to a variable \code{v}. The value of the future is automatically retrieved when the assigned variable (promise) is queried.
+The future itself is returned invisibly, e.g.
+\code{f <- futureAssign("v", expr)} and \code{f <- (v \%<-\% expr)}.
+Alternatively, the future of a future variable \code{v} can be retrieved
+without blocking using \code{f <- \link{futureOf}(v)}.
+Both the future and the variable (promise) are assigned to environment
+\code{assign.env} where the name of the future is \code{.future_<name>}.
}
\description{
-Creates a future from an expression and returns it.
-The state of the future is either unresolved or resolved.
-When it becomes resolved, at some point in the future,
-its value can be retrieved.
+Creates a future that evaluates an \R expression or
+a future that calls an \R function with a set of arguments.
+How, when, and where these futures are evaluated can be configured
+using \code{\link{plan}()} such that it is evaluated in parallel on,
+for instance, the current machine, on a remote machine, or via a
+job queue on a compute cluster.
+Importantly, \R code using futures remains the same regardless
+on these settings.
+}
+\details{
+The state of a future is either unresolved or resolved.
+The value of a future can be retrieved using \code{v <- \link{value}(f)}.
+Querying the value of a non-resolved future will \emph{block} the call
+until the future is resolved.
+It is possible to check whether a future is resolved or not
+without blocking by using \code{\link{resolved}(f)}.
+
+For a future created via a future assignment, the value is bound to
+a promise, which when queried will internally call \code{\link{value}()}
+on the future and which will then be resolved into a regular variable
+bound to that value. For example, with future assignment
+\code{v \%<-\% expr}, the first time variable \code{v} is queried
+the call blocks if (and only if) the future is not yet resolved. As soon
+as it is resolved, and any succeeding queries, querying \code{v} will
+immediately give the value.
+
+The future assignment construct \code{v \%<-\% expr} is not a formal
+assignment per se, but a binary infix operator on objects \code{v}
+and \code{expr}. However, by using non-standard evaluation, this
+constructs can emulate an assignment operator similar to
+\code{v <- expr}. Due to \R's precedence rules of operators,
+future expressions that contain multiple statements need to be
+explicitly bracketed, e.g. \code{v \%<-\% { a <- 2; a^2 }}.
+}
+\section{Globals used by future expressions}{
+
+Global objects (short \emph{globals}) are objects (e.g. variables and
+functions) that are needed in order for the future expression to be
+evaluated while not being local objects that are defined by the future
+expression. For example, in
+\preformatted{
+ a <- 42
+ f <- future({ b <- 2; a * b })
+}
+variable \code{a} is a global of future assignment \code{f} whereas
+\code{b} is a local variable.
+In order for the future to be resolved successfully (and correctly),
+all globals need to be gathered when the future is created such that
+they are available whenever and wherever the future is resolved.
+
+The default behavior (\code{globals = TRUE}) of all evaluator functions,
+is that globals are automatically identified and gathered.
+More precisely, globals are identified via code inspection of the
+future expression \code{expr} and their values are retrieved with
+environment \code{envir} as the starting point (basically via
+\code{get(global, envir=envir, inherits=TRUE)}).
+\emph{In most cases, such automatic collection of globals is sufficient
+and less tedious and error prone than if they are manually specified}.
+
+However, for full control, it is also possible to explicitly specify
+exactly which the globals are by providing their names as a character
+vector.
+In the above example, we could use
+\preformatted{
+ a <- 42
+ f <- future({ b <- 2; a * b }, globals = "a")
+}
+
+Yet another alternative is to explicitly specify also their values
+using a named list as in
+\preformatted{
+ a <- 42
+ f <- future({ b <- 2; a * b }, globals = list(a = a))
+}
+or
+\preformatted{
+ f <- future({ b <- 2; a * b }, globals = list(a = 42))
+}
+
+Specifying globals explicitly avoids the overhead added from
+automatically identifying the globals and gathering their values.
+Futhermore, if we know that the future expression does not make use
+of any global variables, we can disable the automatic search for
+globals by using
+\preformatted{
+ f <- future({ a <- 42; b <- 2; a * b }, globals = FALSE)
+}
+
+Future expressions often make use of functions from one or more packages.
+As long as these functions are part of the set of globals, the future
+package will make sure that those packages are attached when the future
+is resolved. Because there is no need for such globals to be frozen
+or exported, the future package will not export them, which reduces
+the amount of transferred objects.
+For example, in
+\preformatted{
+ x <- rnorm(1000)
+ f <- future({ median(x) })
+}
+variable \code{x} and \code{median()} are globals, but only \code{x}
+is exported whereas \code{median()}, which is part of the \pkg{stats}
+package, is not exported. Instead it is made sure that the \pkg{stats}
+package is on the search path when the future expression is evaluated.
+Effectively, the above becomes
+\preformatted{
+ x <- rnorm(1000)
+ f <- future({
+ library("stats")
+ median(x)
+ })
+}
+To manually specify this, one can either do
+\preformatted{
+ x <- rnorm(1000)
+ f <- future({
+ median(x)
+ }, globals = list(x = x, median = stats::median)
+}
+or
+\preformatted{
+ x <- rnorm(1000)
+ f <- future({
+ library("stats")
+ median(x)
+ }, globals = list(x = x))
+}
+Both are effectively the same.
+
+When using future assignments, globals can be specified analogously
+using the \code{\link{\%globals\%}} operator, e.g.
+\preformatted{
+ x <- rnorm(1000)
+ y \%<-\% { median(x) } \%globals\% list(x = x, median = stats::median)
+}
}
\examples{
-## Use lazy futures
-plan(lazy)
-
-f <- future({
- a <- 7
- b <- 3
- c <- 2
- a * b * c
+## Cluster futures gives an error on R CMD check on
+## Windows (but not Linux or OS X) for unknown reasons.
+## The same code works in package tests.
+\donttest{
+## Evaluate futures in parallel
+plan(multiprocess)
+}
+
+## Data
+x <- rnorm(100)
+y <- 2*x + 0.2 + rnorm(100)
+w <- 1 + x^2
+
+
+## (1) Regular assignments (evaluated sequentially)
+fitA <- lm(y ~ x, weights = w) ## with offset
+fitB <- lm(y ~ x - 1, weights = w) ## without offset
+fitC <- {
+ w <- 1 + abs(x) ## Different weights
+ lm(y ~ x, weights = w)
+}
+print(fitA)
+print(fitB)
+print(fitC)
+
+
+## (2) Future assignments (evaluated in parallel)
+fitA \%<-\% lm(y ~ x, weights = w) ## with offset
+fitB \%<-\% lm(y ~ x - 1, weights = w) ## without offset
+fitC \%<-\% {
+ w <- 1 + abs(x)
+ lm(y ~ x, weights = w)
+}
+print(fitA)
+print(fitB)
+print(fitC)
+
+
+## (3) Explicitly create futures (evaluated in parallel)
+## and retrieve their values
+fA <- future( lm(y ~ x, weights = w) )
+fB <- future( lm(y ~ x - 1, weights = w) )
+fC <- future({
+ w <- 1 + abs(x)
+ lm(y ~ x, weights = w)
})
+fitA <- value(fA)
+fitB <- value(fB)
+fitC <- value(fC)
+print(fitA)
+print(fitB)
+print(fitC)
-print(resolved(f))
-y <- value(f)
-print(y)
+
+## (4) Explit future assignments (evaluated in parallel)
+futureAssign("fitA", lm(y ~ x, weights = w))
+futureAssign("fitB", lm(y ~ x - 1, weights = w))
+futureAssign("fitC", {
+ w <- 1 + abs(x)
+ lm(y ~ x, weights = w)
+})
+print(fitA)
+print(fitB)
+print(fitC)
}
\seealso{
-It is recommended that the evaluator is \emph{non-blocking}
-(returns immediately), but it is not required.
-The default evaluator function is \code{\link{eager}()},
-but this can be changed via \code{\link{plan}()} function.
+How, when and where futures are resolved is given by the future strategy,
+which can be set by the \code{\link{plan}()} function.
}
diff --git a/man/futureAssign.Rd b/man/futureAssign.Rd
deleted file mode 100644
index 039bc78..0000000
--- a/man/futureAssign.Rd
+++ /dev/null
@@ -1,88 +0,0 @@
-% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/futureAssign.R
-\name{futureAssign}
-\alias{\%->\%}
-\alias{\%<-\%}
-\alias{\%<=\%}
-\alias{\%=>\%}
-\alias{futureAssign}
-\title{Create a future and assign its value to a variable as a promise}
-\usage{
-futureAssign(name, value, envir = parent.frame(), assign.env = envir,
- substitute = TRUE)
-}
-\arguments{
-\item{name}{the name of the variable (and the future) to assign.}
-
-\item{value}{the expression to be evaluated in the future and
-whose value will be assigned to the variable.}
-
-\item{envir}{The environment from which global variables used by
-the expression should be search for.}
-
-\item{assign.env}{The environment to which the variable should
-be assigned.}
-
-\item{substitute}{Controls whether \code{expr} should be
-\code{substitute()}:d or not.}
-}
-\value{
-A \link{Future} invisibly.
-}
-\description{
-Method and infix operators for creating futures and assigning
-their values as variables using \link[base]{promise}s. Trying
-to access such a "future variable" will correspond to requesting
-the value of the underlying future. If the the future is already
-resolved at this time, then the value will be available
-instantaneously and the future variable will appear as any other
-variable. If the future is unresolved, then the current process
-will block until the future is resolved and the value is available.
-}
-\details{
-This function creates a future and a corresponding
-\emph{\link[base]{promise}}, which hold the future's value.
-Both the future and the promise are assigned to environment
-\code{assign.env}. The name of the promise is given by \code{name}
-and the name of the future is \code{.future_<name>}.
-The future is also returned invisibly.
-}
-\examples{
-## Future assignment via "assign" function
-futureAssign("A", {
- x <- 3
- x^2
-})
-cat("Value 'A': ", A, sep="")
-
-
-## Equivalent via infix "assign" operator
-A \%<-\% {
- x <- 3
- x^2
-}
-cat("Value 'A': ", A, sep="")
-
-
-## A global variable
-a <- 1
-
-## Three future evaluations
-A \%<-\% { 0.1 }
-B \%<-\% { 0.2 }
-C \%<-\% { z <- a+0.3 }
-
-## Sleep until 'C' is available
-cat("Value 'C': ", C, sep="")
-
-## Sleep until 'A' is available
-cat("Value 'A': ", A, sep="")
-
-## Sleep until 'C' is available
-cat("Value 'B': ", B, sep="")
-}
-\seealso{
-The \code{\link{futureOf}()} function can be used to get the
-Future object of a future variable.
-}
-
diff --git a/man/futures.Rd b/man/futures.Rd
index 197adea..8189631 100644
--- a/man/futures.Rd
+++ b/man/futures.Rd
@@ -9,7 +9,7 @@ futures(x, ...)
\arguments{
\item{x}{An environment, a list, or a list environment.}
-\item{...}{Not used.}
+\item{\dots}{Not used.}
}
\value{
An object of same type as \code{x} and with the same names
@@ -22,7 +22,7 @@ Non-future elements are returned as is.
}
\details{
This function is useful for retrieve futures that were created via
-future assignments (\code{\%<=\%}) and therefore stored as promises.
+future assignments (\code{\%<-\%}) and therefore stored as promises.
This function turns such promises into standard \code{Future}
objects.
}
diff --git a/man/getExpression.Rd b/man/getExpression.Rd
index bac6280..186cf8d 100644
--- a/man/getExpression.Rd
+++ b/man/getExpression.Rd
@@ -10,7 +10,7 @@ getExpression(future, ...)
\arguments{
\item{future}{Current future.}
-\item{...}{Not used.}
+\item{\dots}{Not used.}
}
\value{
A future expression with code injected to set what
diff --git a/man/getGlobalsAndPackages.Rd b/man/getGlobalsAndPackages.Rd
new file mode 100644
index 0000000..6757ecb
--- /dev/null
+++ b/man/getGlobalsAndPackages.Rd
@@ -0,0 +1,35 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/globals.R
+\name{getGlobalsAndPackages}
+\alias{getGlobalsAndPackages}
+\title{Retrieves global variables of an expression and their associated packages}
+\usage{
+getGlobalsAndPackages(expr, envir = parent.frame(), tweak = tweakExpression,
+ globals = TRUE, resolve = getOption("future.globals.resolve", FALSE),
+ persistent = FALSE, ...)
+}
+\arguments{
+\item{expr}{An R expression whose globals should be found.}
+
+\item{envir}{The environment from which globals should be searched.}
+
+\item{tweak}{(optional) A function that takes an expression and returned a modified one.}
+
+\item{globals}{(optional) a logical, a character vector, a named list, or a \link[globals]{Globals} object. If TRUE, globals are identified by code inspection based on \code{expr} and \code{tweak} searching from environment \code{envir}. If FALSE, no globals are used. If a character vector, then globals are identified by lookup based their names \code{globals} searching from environment \code{envir}. If a named list or a Globals object, the globals are used as is.}
+
+\item{resolve}{If TRUE, any future that is a global variables (or part of one) is resolved and replaced by a "constant" future.
+persistent If TRUE, non-existing globals (= identified in expression but not found in memory) are always silently ignored and assumed to be existing in the evaluation environment. If FALSE, non-existing globals are by default ignored, but may also trigger an informative error if option \code{future.globals.onMissing == "error"}.}
+
+\item{\dots}{Not used.}
+}
+\value{
+A named list with elements \code{expr} (the tweaked expression), \code{globals} (a named list) and \code{packages} (a character string).
+}
+\description{
+Retrieves global variables of an expression and their associated packages
+}
+\seealso{
+Internally, \code{\link[globals]{globalsOf}()} is used to identify globals and associated packages from the expression.
+}
+\keyword{internal}
+
diff --git a/man/grapes-globals-grapes.Rd b/man/grapes-globals-grapes.Rd
new file mode 100644
index 0000000..be73a23
--- /dev/null
+++ b/man/grapes-globals-grapes.Rd
@@ -0,0 +1,24 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/globals_OP.R
+\name{\%globals\%}
+\alias{\%globals\%}
+\title{Specify globals for a future assignment}
+\usage{
+fassignment \%globals\% globals
+}
+\arguments{
+\item{fassignment}{The future assignment, e.g.
+\code{x \%<-\% \{ expr \}}.}
+
+\item{globals}{(optional) a logical, a character vector,
+or a named list for controlling how globals are handled.
+For details, see section 'Globals used by future expressions'
+in the help for \code{\link{future}()}.}
+}
+\description{
+Specify globals for a future assignment
+}
+\details{
+\code{a \%globals\% b} is short for \code{a \%tweak\% list(globals = b)}.
+}
+
diff --git a/man/grapes-label-grapes.Rd b/man/grapes-label-grapes.Rd
new file mode 100644
index 0000000..438a7ac
--- /dev/null
+++ b/man/grapes-label-grapes.Rd
@@ -0,0 +1,21 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/label_OP.R
+\name{\%label\%}
+\alias{\%label\%}
+\title{Specify label for a future assignment}
+\usage{
+fassignment \%label\% label
+}
+\arguments{
+\item{fassignment}{The future assignment, e.g.
+\code{x \%<-\% \{ expr \}}.}
+
+\item{label}{An optional character string label attached to the future.}
+}
+\description{
+Specify label for a future assignment
+}
+\details{
+\code{a \%label\% b} is short for \code{a \%tweak\% list(label = b)}.
+}
+
diff --git a/man/grapes-plan-grapes.Rd b/man/grapes-plan-grapes.Rd
index cdc1adb..d304fbf 100644
--- a/man/grapes-plan-grapes.Rd
+++ b/man/grapes-plan-grapes.Rd
@@ -8,7 +8,7 @@ fassignment \%plan\% strategy
}
\arguments{
\item{fassignment}{The future assignment, e.g.
-\code{x \%<=\% \{ expr \}}.}
+\code{x \%<-\% \{ expr \}}.}
\item{strategy}{The mechanism for how the future should be
resolved. See \code{\link{plan}()} for further details.}
diff --git a/man/grapes-tweak-grapes.Rd b/man/grapes-tweak-grapes.Rd
index 6206212..3896ad2 100644
--- a/man/grapes-tweak-grapes.Rd
+++ b/man/grapes-tweak-grapes.Rd
@@ -8,7 +8,7 @@ fassignment \%tweak\% tweaks
}
\arguments{
\item{fassignment}{The future assignment, e.g.
-\code{x \%<=\% \{ expr \}}.}
+\code{x \%<-\% \{ expr \}}.}
\item{tweaks}{A named list (or vector) with arguments that
should be changed relative to the current strategy.}
diff --git a/man/lazy.Rd b/man/lazy.Rd
index 6ad09ed..4ff2597 100644
--- a/man/lazy.Rd
+++ b/man/lazy.Rd
@@ -5,14 +5,15 @@
\title{Create a lazy future whose value will be resolved at the time when requested}
\usage{
lazy(expr, envir = parent.frame(), substitute = TRUE, globals = TRUE,
- local = TRUE, gc = FALSE, earlySignal = FALSE, ...)
+ local = TRUE, earlySignal = FALSE, label = NULL, ...)
}
\arguments{
-\item{expr}{An R \link[base]{expression}.}
+\item{expr}{An \R \link[base]{expression}.}
-\item{envir}{The \link{environment} in which the evaluation
-is done (or inherits from if \code{local} is TRUE)
-and from which globals are obtained.}
+\item{envir}{The \link{environment} from where global
+objects should be identified. Depending on "evaluator",
+it may also be the environment in which the expression
+is evaluated.}
\item{substitute}{If TRUE, argument \code{expr} is
\code{\link[base]{substitute}()}:ed, otherwise not.}
@@ -25,12 +26,12 @@ resolved when the future is resolved.}
all assignments are done to local temporary environment, otherwise
the assignments are done in the calling environment.}
-\item{gc}{If TRUE, the garbage collector run after the future
-is resolved (in the process that evaluated the future).}
+\item{earlySignal}{Specified whether conditions should be signaled
+as soon as possible or not.}
-\item{earlySignal}{Specified whether conditions should be signaled as soon as possible or not.}
+\item{label}{An optional character string label attached to the future.}
-\item{\dots}{Not used.}
+\item{...}{Additional arguments passed to the "evaluator".}
}
\value{
A \link{LazyFuture}.
@@ -45,7 +46,7 @@ if the value is never requested.
The preferred way to create a lazy future is not to call this function
directly, but to register it via \code{\link{plan}(lazy)} such that it
becomes the default mechanism for all futures. After this
-\code{\link{future}()} and \code{\link{\%<=\%}} will create
+\code{\link{future}()} and \code{\link{\%<-\%}} will create
\emph{lazy futures}.
}
\examples{
diff --git a/man/multicore.Rd b/man/multicore.Rd
index c8b78bf..7599186 100644
--- a/man/multicore.Rd
+++ b/man/multicore.Rd
@@ -5,32 +5,35 @@
\title{Create a multicore future whose value will be resolved asynchroneously in a forked parallel process}
\usage{
multicore(expr, envir = parent.frame(), substitute = TRUE, globals = TRUE,
- workers = availableCores(constraints = "multicore"), gc = FALSE,
- earlySignal = FALSE, ...)
+ workers = availableCores(constraints = "multicore"), earlySignal = FALSE,
+ label = NULL, ...)
}
\arguments{
-\item{expr}{An R \link[base]{expression}.}
+\item{expr}{An \R \link[base]{expression}.}
-\item{envir}{The \link{environment} in which the evaluation
-is done and from which globals are obtained.}
+\item{envir}{The \link{environment} from where global
+objects should be identified. Depending on "evaluator",
+it may also be the environment in which the expression
+is evaluated.}
\item{substitute}{If TRUE, argument \code{expr} is
\code{\link[base]{substitute}()}:ed, otherwise not.}
-\item{globals}{If TRUE, global objects are validated at the point
-in time when the future is created (always before it is resolved),
-that is, they identified and located. If some globals fail to be
-located, an informative error is generated.}
+\item{globals}{A logical, a character vector,
+or a named list for controlling how globals are handled.
+For details, see below section.
+This argument can be specified via the \dots arguments
+for \code{future()} and \code{futureCall()}.}
\item{workers}{The maximum number of multicore futures that can
be active at the same time before blocking.}
-\item{gc}{If TRUE, the garbage collector run after the future
-is resolved (in the process that evaluated the future).}
+\item{earlySignal}{Specified whether conditions should be signaled
+as soon as possible or not.}
-\item{earlySignal}{Specified whether conditions should be signaled as soon as possible or not.}
+\item{label}{An optional character string label attached to the future.}
-\item{\dots}{Not used.}
+\item{...}{Additional arguments passed to the "evaluator".}
}
\value{
A \link{MulticoreFuture}
@@ -62,7 +65,7 @@ The preferred way to create an multicore future is not to call
this function directly, but to register it via
\code{\link{plan}(multicore)} such that it becomes the default
mechanism for all futures. After this \code{\link{future}()}
-and \code{\link{\%<=\%}} will create \emph{multicore futures}.
+and \code{\link{\%<-\%}} will create \emph{multicore futures}.
}
\examples{
## Use multicore futures
diff --git a/man/multiprocess.Rd b/man/multiprocess.Rd
index b99c552..573c235 100644
--- a/man/multiprocess.Rd
+++ b/man/multiprocess.Rd
@@ -6,31 +6,36 @@
\usage{
multiprocess(expr, envir = parent.frame(), substitute = TRUE,
globals = TRUE, workers = availableCores(), gc = FALSE,
- earlySignal = FALSE, ...)
+ earlySignal = FALSE, label = NULL, ...)
}
\arguments{
-\item{expr}{An R \link[base]{expression}.}
+\item{expr}{An \R \link[base]{expression}.}
-\item{envir}{The \link{environment} in which the evaluation
-is done and from which globals are obtained.}
+\item{envir}{The \link{environment} from where global
+objects should be identified. Depending on "evaluator",
+it may also be the environment in which the expression
+is evaluated.}
\item{substitute}{If TRUE, argument \code{expr} is
\code{\link[base]{substitute}()}:ed, otherwise not.}
-\item{globals}{If TRUE, global objects are validated at the point
-in time when the future is created (always before it is resolved),
-that is, they identified and located. If some globals fail to be
-located, an informative error is generated.}
+\item{globals}{(optional) a logical, a character vector,
+or a named list for controlling how globals are handled.
+For details, see section 'Globals used by future expressions'
+in the help for \code{\link{future}()}.}
\item{workers}{The maximum number of multiprocess futures that
can be active at the same time before blocking.}
-\item{gc}{If TRUE, the garbage collector run after the future
-is resolved (in the process that evaluated the future).}
+\item{gc}{If TRUE, the garbage collector run (in the process that
+evaluated the future) after the value of the future is collected.}
-\item{earlySignal}{Specified whether conditions should be signaled as soon as possible or not.}
+\item{earlySignal}{Specified whether conditions should be signaled
+as soon as possible or not.}
-\item{\dots}{Not used.}
+\item{label}{An optional character string label attached to the future.}
+
+\item{...}{Not used.}
}
\value{
A \link{MultiprocessFuture} implemented as either a
diff --git a/man/multisession.Rd b/man/multisession.Rd
index c3468a6..8f21e6f 100644
--- a/man/multisession.Rd
+++ b/man/multisession.Rd
@@ -5,30 +5,41 @@
\title{Create a multisession future whose value will be resolved asynchroneously in a parallel R session}
\usage{
multisession(expr, envir = parent.frame(), substitute = TRUE,
- persistent = FALSE, workers = availableCores(), gc = FALSE,
- earlySignal = FALSE, ...)
+ globals = TRUE, persistent = FALSE, workers = availableCores(),
+ gc = FALSE, earlySignal = FALSE, label = NULL, ...)
}
\arguments{
-\item{expr}{An R \link[base]{expression}.}
+\item{expr}{An \R \link[base]{expression}.}
-\item{envir}{The \link{environment} in which the evaluation
-is done and from which globals are obtained.}
+\item{envir}{The \link{environment} from where global
+objects should be identified. Depending on "evaluator",
+it may also be the environment in which the expression
+is evaluated.}
\item{substitute}{If TRUE, argument \code{expr} is
\code{\link[base]{substitute}()}:ed, otherwise not.}
+\item{globals}{A logical, a character vector,
+or a named list for controlling how globals are handled.
+For details, see below section.
+This argument can be specified via the \dots arguments
+for \code{future()} and \code{futureCall()}.}
+
\item{persistent}{If FALSE, the evaluation environment is cleared
from objects prior to the evaluation of the future.}
\item{workers}{The maximum number of multisession futures that
can be active at the same time before blocking.}
-\item{gc}{If TRUE, the garbage collector run after the future
-is resolved (in the process that evaluated the future).}
+\item{gc}{If TRUE, the garbage collector run (in the process that
+evaluated the future) after the value of the future is collected.}
+
+\item{earlySignal}{Specified whether conditions should be signaled
+as soon as possible or not.}
-\item{earlySignal}{Specified whether conditions should be signaled as soon as possible or not.}
+\item{label}{An optional character string label attached to the future.}
-\item{\dots}{Not used.}
+\item{...}{Additional arguments passed to the "evaluator".}
}
\value{
A \link{MultisessionFuture}.
@@ -54,7 +65,7 @@ The preferred way to create an multisession future is not to call
this function directly, but to register it via
\code{\link{plan}(multisession)} such that it becomes the default
mechanism for all futures. After this \code{\link{future}()}
-and \code{\link{\%<=\%}} will create \emph{multisession futures}.
+and \code{\link{\%<-\%}} will create \emph{multisession futures}.
}
\section{Known issues}{
diff --git a/man/nbrOfWorkers.Rd b/man/nbrOfWorkers.Rd
index 0ec612d..9a4eb0f 100644
--- a/man/nbrOfWorkers.Rd
+++ b/man/nbrOfWorkers.Rd
@@ -2,7 +2,7 @@
% Please edit documentation in R/nbrOfWorkers.R
\name{nbrOfWorkers}
\alias{nbrOfWorkers}
-\title{Gets the number of workers}
+\title{Gets the number of workers available}
\usage{
nbrOfWorkers(evaluator = NULL)
}
@@ -15,6 +15,16 @@ by \code{\link{plan}()} is used.}
A number in [1,Inf].
}
\description{
-Gets the number of workers
+Gets the number of workers available
+}
+\examples{
+plan(multiprocess)
+nbrOfWorkers() ## == availableCores()
+
+plan(multiprocess, workers=2)
+nbrOfWorkers() ## == 2
+
+plan(lazy)
+nbrOfWorkers() ## == 1
}
diff --git a/man/remote.Rd b/man/remote.Rd
index 9547c88..2cc5feb 100644
--- a/man/remote.Rd
+++ b/man/remote.Rd
@@ -4,33 +4,53 @@
\alias{remote}
\title{Create a remote future whose value will be resolved asynchroneously in a remote process}
\usage{
-remote(expr, envir = parent.frame(), substitute = TRUE, persistent = TRUE,
- workers = NULL, gc = FALSE, earlySignal = FALSE, myip = NULL, ...)
+remote(expr, envir = parent.frame(), substitute = TRUE, globals = TRUE,
+ persistent = TRUE, workers = NULL, user = NULL, revtunnel = TRUE,
+ gc = FALSE, earlySignal = FALSE, myip = NULL, label = NULL, ...)
}
\arguments{
-\item{expr}{An R \link[base]{expression}.}
+\item{expr}{An \R \link[base]{expression}.}
-\item{envir}{The \link{environment} in which the evaluation
-is done and from which globals are obtained.}
+\item{envir}{The \link{environment} from where global
+objects should be identified. Depending on "evaluator",
+it may also be the environment in which the expression
+is evaluated.}
\item{substitute}{If TRUE, argument \code{expr} is
\code{\link[base]{substitute}()}:ed, otherwise not.}
+\item{globals}{A logical, a character vector,
+or a named list for controlling how globals are handled.
+For details, see below section.
+This argument can be specified via the \dots arguments
+for \code{future()} and \code{futureCall()}.}
+
\item{persistent}{If FALSE, the evaluation environment is cleared
from objects prior to the evaluation of the future.}
-\item{workers}{A cluster object created by
-\code{\link[parallel]{makeCluster}()}.}
+\item{workers}{The maximum number of multiprocess futures that
+can be active at the same time before blocking.}
+
+\item{user}{(optional) The user name to be used when communicating
+with another host.}
-\item{gc}{If TRUE, the garbage collector run after the future
-is resolved (in the process that evaluated the future).}
+\item{revtunnel}{If TRUE, reverse SSH tunneling is used for the
+PSOCK cluster nodes to connect back to the master R process. This
+avoids the hassle of firewalls, port forwarding and having to know
+the internal / public IP address of the master R session.}
-\item{earlySignal}{Specified whether conditions should be signaled as soon as possible or not.}
+\item{gc}{If TRUE, the garbage collector run (in the process that
+evaluated the future) after the value of the future is collected.}
+
+\item{earlySignal}{Specified whether conditions should be signaled
+as soon as possible or not.}
\item{myip}{The external IP address of this machine.
If NULL, then it is inferred using an online service (default).}
-\item{\dots}{Not used.}
+\item{label}{An optional character string label attached to the future.}
+
+\item{...}{Additional arguments passed to the "evaluator".}
}
\value{
A \link{ClusterFuture}.
diff --git a/man/resolve.Rd b/man/resolve.Rd
index 94ec3f6..c064150 100644
--- a/man/resolve.Rd
+++ b/man/resolve.Rd
@@ -25,7 +25,7 @@ if futures have been resolved since last time.}
If a function, the it is called as \code{progress(done, total)}
every time a future is resolved.}
-\item{...}{Not used}
+\item{\dots}{Not used}
}
\value{
Returns \code{x} (regardless of subsetting or not).
diff --git a/man/run.Rd b/man/run.Rd
new file mode 100644
index 0000000..a2452bc
--- /dev/null
+++ b/man/run.Rd
@@ -0,0 +1,27 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/Future-class.R
+\name{run.Future}
+\alias{run}
+\alias{run.Future}
+\title{Run a future}
+\usage{
+\method{run}{Future}(future, ...)
+}
+\arguments{
+\item{future}{A \link{Future}.}
+
+\item{\dots}{Not used.}
+}
+\value{
+The \link{Future} object.
+}
+\description{
+Run a future
+}
+\details{
+This function can only be called once per future.
+Further calls will result in an informative error.
+If a future is not run when its value is queried,
+then it is run at that point.
+}
+
diff --git a/man/tweak.Rd b/man/tweak.Rd
index fd402e5..008a85e 100644
--- a/man/tweak.Rd
+++ b/man/tweak.Rd
@@ -9,11 +9,11 @@ tweak(strategy, ..., penvir = parent.frame())
\arguments{
\item{strategy}{An existing future function or the name of one.}
-\item{...}{Named arguments to replace the defaults of existing
-arguments.}
-
\item{penvir}{The environment used when searching for a future
function by its name.}
+
+\item{\dots}{Named arguments to replace the defaults of existing
+arguments.}
}
\value{
a future function.
diff --git a/man/tweak_parallel_PSOCK.Rd b/man/tweak_parallel_PSOCK.Rd
new file mode 100644
index 0000000..b742fdd
--- /dev/null
+++ b/man/tweak_parallel_PSOCK.Rd
@@ -0,0 +1,68 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/tweak_parallel_PSOCK.R
+\name{tweak_parallel_PSOCK}
+\alias{tweak_parallel_PSOCK}
+\title{Tweak PSOCK backend of the parallel package}
+\usage{
+tweak_parallel_PSOCK(user = TRUE, revtunnel = TRUE, rshopts = TRUE,
+ use127.0.0.1 = FALSE, reset = FALSE)
+}
+\arguments{
+\item{user}{If TRUE, parallel is tweaked to only pass username to SSH if it is specified via argument \code{user}.}
+
+\item{revtunnel}{If TRUE, parallel is tweaked to make use of reverse SSH tunneling.}
+
+\item{rshopts}{If TRUE, parallel is tweaked so it is possible to specify additional command-line options to the \code{rshcmd} executable.}
+
+\item{use127.0.0.1}{If TRUE, \code{127.0.0.1} is used instead of \code{localhost}.}
+
+\item{reset}{If TRUE, all tweaks are undone.}
+}
+\value{
+Nothing.
+}
+\description{
+Tweak PSOCK backend of the parallel package
+}
+\examples{
+\donttest{\dontrun{
+trace(system, tracer = quote(stop("Command: ", command)), print = FALSE)
+
+## Without tweaks
+try(cl <- parallel::makeCluster("remote.myserver.org", revtunnel=TRUE,
+ master="localhost", homogeneous=FALSE))
+### Error in eval(expr, envir, enclos) :
+### Command:
+### ssh
+### -l hb
+### remote.myserver.org
+### "Rscript
+### --default-packages=datasets,utils,grDevices,graphics,stats,methods
+### -e 'parallel:::.slaveRSOCK()' MASTER=localhost PORT=11099
+### OUT=/dev/null TIMEOUT=2592000 XDR=TRUE"
+
+## With tweaks
+future:::tweak_parallel_PSOCK()
+try(cl <- parallel::makeCluster("remote.myserver.org", revtunnel=TRUE,
+ master="localhost", homogeneous=FALSE))
+### Error in eval(expr, envir, enclos) :
+### Command:
+### ssh
+### -R 11624:localhost:11624
+### remote.myserver.org
+### "Rscript
+### --default-packages=datasets,utils,grDevices,graphics,stats,methods
+### -e 'parallel:::.slaveRSOCK()' MASTER=localhost PORT=11099
+### OUT=/dev/null TIMEOUT=2592000 XDR=TRUE"
+
+## Undo tweaks
+untrace(system)
+future:::tweak_parallel_PSOCK(reset=TRUE)
+}}
+
+}
+\references{
+\url{https://github.com/HenrikBengtsson/Wishlist-for-R/issues/32}\cr
+}
+\keyword{internal}
+
diff --git a/man/values.Rd b/man/values.Rd
index 128be90..ea98f6b 100644
--- a/man/values.Rd
+++ b/man/values.Rd
@@ -9,7 +9,7 @@ values(x, ...)
\arguments{
\item{x}{An environment, a list, or a list environment.}
-\item{...}{Additional arguments passed to \code{value()} of each future.}
+\item{\dots}{Additional arguments passed to \code{value()} of each future.}
}
\value{
An object of same type as \code{x} and with the same names
diff --git a/revdep/README.md b/revdep/README.md
new file mode 100644
index 0000000..8277f64
--- /dev/null
+++ b/revdep/README.md
@@ -0,0 +1,399 @@
+# Setup
+
+## Platform
+
+|setting |value |
+|:--------|:----------------------------|
+|version |R version 3.3.1 (2016-06-21) |
+|system |x86_64, linux-gnu |
+|ui |X11 |
+|language |en |
+|collate |en_US.UTF-8 |
+|tz |SystemV/PST8PDT |
+|date |2016-10-09 |
+
+## Packages
+
+|package |* |version |date |source |
+|:--------|:--|:-------|:----------|:--------------|
+|digest | |0.6.10 |2016-08-02 |cran (@0.6.10) |
+|future | |1.0.1 |2016-07-04 |cran (@1.0.1) |
+|globals | |0.7.0 |2016-09-09 |cran (@0.7.0) |
+|listenv | |0.6.0 |2015-12-28 |CRAN (R 3.3.1) |
+|markdown | |0.7.7 |2015-04-22 |CRAN (R 3.3.1) |
+|R.rsp | |0.30.0 |2016-05-15 |CRAN (R 3.3.1) |
+
+# Check results
+
+19 packages
+
+|package |version | errors| warnings| notes|
+|:----------------|:-------|------:|--------:|-----:|
+|ACNE |0.8.1 | 0| 0| 0|
+|aroma.affymetrix |3.0.0 | 0| 0| 0|
+|aroma.cn |1.6.1 | 0| 0| 0|
+|aroma.core |3.0.0 | 0| 0| 1|
+|calmate |0.12.1 | 0| 0| 0|
+|doFuture |0.2.1 | 0| 0| 0|
+|fiery |0.2.1 | 0| 0| 0|
+|future.BatchJobs |0.13.0 | 0| 0| 0|
+|GeneBreak |1.3.0 | 0| 0| 1|
+|MPAgenomics |1.1.2 | 0| 0| 2|
+|NSA |0.0.32 | 0| 0| 6|
+|pbmcapply |1.1.1 | 0| 0| 0|
+|PECA |1.9.0 | 0| 0| 1|
+|PSCBS |0.61.0 | 0| 0| 0|
+|PureCN |1.1.54 | 0| 0| 1|
+|QDNAseq |1.9.2 | 0| 0| 0|
+|Repitools |1.19.3 | 0| 0| 2|
+|R.filesets |2.10.0 | 0| 0| 0|
+|TIN |1.5.2 | 0| 0| 2|
+
+## ACNE (0.8.1)
+Maintainer: Henrik Bengtsson <henrikb at braju.com>
+Bug reports: https://github.com/HenrikBengtsson/ACNE/issues
+
+0 errors | 0 warnings | 0 notes
+
+## aroma.affymetrix (3.0.0)
+Maintainer: Henrik Bengtsson <henrikb at braju.com>
+Bug reports: https://github.com/HenrikBengtsson/aroma.affymetrix/issues
+
+0 errors | 0 warnings | 0 notes
+
+## aroma.cn (1.6.1)
+Maintainer: Henrik Bengtsson <henrikb at braju.com>
+Bug reports: https://github.com/HenrikBengtsson/aroma.cn/issues
+
+0 errors | 0 warnings | 0 notes
+
+## aroma.core (3.0.0)
+Maintainer: Henrik Bengtsson <henrikb at braju.com>
+Bug reports: https://github.com/HenrikBengtsson/aroma.core/issues
+
+0 errors | 0 warnings | 1 note
+
+```
+checking package dependencies ... NOTE
+Packages suggested but not available for checking:
+ ‘expectile’ ‘HaarSeg’ ‘mpcbs’
+```
+
+## calmate (0.12.1)
+Maintainer: Henrik Bengtsson <henrikb at braju.com>
+Bug reports: https://github.com/HenrikBengtsson/calmate/issues
+
+0 errors | 0 warnings | 0 notes
+
+## doFuture (0.2.1)
+Maintainer: Henrik Bengtsson <henrikb at braju.com>
+Bug reports: https://github.com/HenrikBengtsson/doFuture/issues
+
+0 errors | 0 warnings | 0 notes
+
+## fiery (0.2.1)
+Maintainer: Thomas Lin Pedersen <thomasp85 at gmail.com>
+Bug reports: https://github.com/thomasp85/fiery/issues
+
+0 errors | 0 warnings | 0 notes
+
+## future.BatchJobs (0.13.0)
+Maintainer: Henrik Bengtsson <henrikb at braju.com>
+Bug reports: https://github.com/HenrikBengtsson/future.BatchJobs/issues
+
+0 errors | 0 warnings | 0 notes
+
+## GeneBreak (1.3.0)
+Maintainer: Evert van den Broek <vandenbroek.evert at gmail.com>
+
+0 errors | 0 warnings | 1 note
+
+```
+checking R code for possible problems ... NOTE
+.glmbreak: no visible global function definition for ‘glm’
+.glmbreak: no visible global function definition for ‘predict’
+addGeneAnnotation,CopyNumberBreakPoints: no visible global function
+ definition for ‘head’
+bpStats,CopyNumberBreakPoints: no visible global function definition
+ for ‘sd’
+bpStats,CopyNumberBreakPoints: no visible global function definition
+ for ‘p.adjust’
+Undefined global functions or variables:
+ glm head p.adjust predict sd
+Consider adding
+ importFrom("stats", "glm", "p.adjust", "predict", "sd")
+ importFrom("utils", "head")
+to your NAMESPACE file.
+```
+
+## MPAgenomics (1.1.2)
+Maintainer: Samuel Blanck <samuel.blanck at inria.fr>
+
+0 errors | 0 warnings | 2 notes
+
+```
+checking dependencies in R code ... NOTE
+'library' or 'require' calls in package code:
+ ‘R.devices’ ‘R.filesets’ ‘R.methodsS3’ ‘R.oo’ ‘aroma.affymetrix’
+ ‘aroma.cn’ ‘aroma.core’ ‘aroma.light’ ‘matrixStats’ ‘snowfall’
+ Please use :: or requireNamespace() instead.
+ See section 'Suggested packages' in the 'Writing R Extensions' manual.
+Unexported object imported by a ':::' call: ‘cghseg:::segmeanCO’
+ See the note in ?`:::` about the use of this operator.
+
+checking R code for possible problems ... NOTE
+.varregtimescount: no visible global function definition for ‘var’
+CGHSEGaroma: no visible global function definition for ‘read.csv’
+CGHSEGaroma : <anonymous>: no visible global function definition for
+ ‘points’
+CGHSEGaroma : <anonymous>: no visible global function definition for
+ ‘lines’
+CGHSEGaroma : <anonymous>: no visible global function definition for
+ ‘write.table’
+CGHcall: no visible global function definition for ‘mad’
+... 43 lines ...
+tumorboostPlot: no visible global function definition for ‘par’
+tumorboostPlot: no visible global function definition for ‘axis’
+tumorboostPlot: no visible global function definition for ‘points’
+Undefined global functions or variables:
+ axis head lines lm mad median optim par points read.csv sd var
+ write.table
+Consider adding
+ importFrom("graphics", "axis", "lines", "par", "points")
+ importFrom("stats", "lm", "mad", "median", "optim", "sd", "var")
+ importFrom("utils", "head", "read.csv", "write.table")
+to your NAMESPACE file.
+```
+
+## NSA (0.0.32)
+Maintainer: Maria Ortiz-Estevez <mortizest at gmail.com>
+
+0 errors | 0 warnings | 6 notes
+
+```
+checking package dependencies ... NOTE
+Depends: includes the non-default packages:
+ ‘R.methodsS3’ ‘MASS’ ‘matrixStats’ ‘R.oo’ ‘R.utils’ ‘aroma.core’
+ ‘aroma.affymetrix’ ‘DNAcopy’
+Adding so many packages to the search path is excessive and importing
+selectively is preferable.
+
+checking top-level files ... NOTE
+Non-standard file/directory found at top level:
+ ‘incl’
+
+checking dependencies in R code ... NOTE
+Packages in Depends field not imported from:
+ ‘DNAcopy’ ‘MASS’ ‘R.methodsS3’ ‘R.oo’ ‘aroma.affymetrix’ ‘aroma.core’
+ ‘matrixStats’
+ These packages need to be imported from (in the NAMESPACE file)
+ for when this namespace is loaded but not attached.
+
+checking S3 generic/method consistency ... NOTE
+Found the following apparent S3 methods exported but not registered:
+ NSAByTotalAndFracB.matrix allocateOutputDataSets.NSANormalization
+ allocateOutputDataSets.SNPsNormalization
+ allocateOutputDataSets.SampleNormalization
+ as.character.NSANormalization as.character.SNPsNormalization
+ as.character.SampleNormalization findArraysTodo.NSANormalization
+ findArraysTodo.SampleNormalization findUnitsTodo.SNPsNormalization
+ fitNSA.matrix fitNSAcnPs.matrix getDataSets.NSANormalization
+ getDataSets.SNPsNormalization getDataSets.SampleNormalization
+ getName.NSANormalization getName.SNPsNormalization
+ getName.SampleNormalization getOutputDataSets.NSANormalization
+ getOutputDataSets.SNPsNormalization
+ getOutputDataSets.SampleNormalization getPath.NSANormalization
+ getPath.SNPsNormalization getPath.SampleNormalization
+ getRootPath.NSANormalization getRootPath.SNPsNormalization
+ getRootPath.SampleNormalization process.NSANormalization
+ process.SNPsNormalization process.SampleNormalization
+ sampleNByTotalAndFracB.numeric snpsNByTotalAndFracB.matrix
+See section ‘Registering S3 methods’ in the ‘Writing R Extensions’
+manual.
+
+checking R code for possible problems ... NOTE
+NB: .First.lib is obsolete and will not be used in R >= 3.0.0
+
+.First.lib: no visible global function definition for
+ ‘packageDescription’
+NSAByTotalAndFracB.matrix: no visible global function definition for
+ ‘throw’
+NSAByTotalAndFracB.matrix: no visible global function definition for
+ ‘str’
+NSANormalization: no visible global function definition for ‘throw’
+... 279 lines ...
+ extractMatrix findUnitsTodo getAsteriskTags getChipType getFile
+ getFullName getFullNames getGenomeInformation getName getNames
+ getPath getPathname getPathnames getPositions getRam getRootPath
+ getTags getUnitsOnChromosome hist median nbrOfFiles newInstance
+ packageDescription rowAlls rowMedians segment setTags str throw trim
+ verbose
+Consider adding
+ importFrom("graphics", "hist")
+ importFrom("stats", "approxfun", "median")
+ importFrom("utils", "packageDescription", "str")
+to your NAMESPACE file.
+
+checking Rd line widths ... NOTE
+Rd file 'NSANormalization.Rd':
+ \examples lines wider than 100 characters:
+ by <- 50e3; # 50kb bins; you may want to try with other amounts of smoothing xOut <- seq(from=xRange[1], to=xRange[2], by=by);
+ plot(getSignals(cnCNPS), getSignals(cnSNPS), xlim=Clim, ylim=Clim); abline(a=0, b=1, col="red", lwd=2);
+
+These lines will be truncated in the PDF manual.
+```
+
+## pbmcapply (1.1.1)
+Maintainer: Kevin kuang <kvn.kuang at mail.utoronto.ca>
+Bug reports: https://github.com/kvnkuang/pbmcapply/issues
+
+0 errors | 0 warnings | 0 notes
+
+## PECA (1.9.0)
+Maintainer: Tomi Suomi <tomi.suomi at utu.fi>
+
+0 errors | 0 warnings | 1 note
+
+```
+checking Rd line widths ... NOTE
+Rd file 'PECA.Rd':
+ \usage lines wider than 90 characters:
+ PECA_AffyBatch(affy=NULL, normalize=FALSE, test="t", type="median", paired=FALSE, progress=FALSE)
+
+These lines will be truncated in the PDF manual.
+```
+
+## PSCBS (0.61.0)
+Maintainer: Henrik Bengtsson <henrikb at braju.com>
+Bug reports: https://github.com/HenrikBengtsson/PSCBS/issues
+
+0 errors | 0 warnings | 0 notes
+
+## PureCN (1.1.54)
+Maintainer: Markus Riester <markus.riester at novartis.com>
+
+0 errors | 0 warnings | 1 note
+
+```
+checking R code for possible problems ... NOTE
+Found the following calls to data() loading into the global environment:
+File ‘PureCN/R/bootstrapResults.R’:
+ data(purecn.example.output)
+File ‘PureCN/R/callAlterations.R’:
+ data(purecn.example.output)
+ data(purecn.example.output)
+File ‘PureCN/R/callLOH.R’:
+ data(purecn.example.output)
+File ‘PureCN/R/createCurationFile.R’:
+ data(purecn.example.output)
+File ‘PureCN/R/curateResults.R’:
+ data(purecn.example.output)
+ data(purecn.example.output)
+File ‘PureCN/R/plotAbs.R’:
+ data(purecn.example.output)
+File ‘PureCN/R/predictSomatic.R’:
+ data(purecn.example.output)
+File ‘PureCN/R/readCurationFile.R’:
+ data(purecn.example.output)
+See section ‘Good practice’ in ‘?data’.
+```
+
+## QDNAseq (1.9.2)
+Maintainer: Daoud Sie <d.sie at vumc.nl>
+Bug reports: https://github.com/ccagc/QDNAseq/issues
+
+0 errors | 0 warnings | 0 notes
+
+## Repitools (1.19.3)
+Maintainer: Mark Robinson <mark.robinson at imls.uzh.ch>
+
+0 errors | 0 warnings | 2 notes
+
+```
+checking R code for possible problems ... NOTE
+Found an obsolete/platform-specific call in the following function:
+ ‘maskOut’
+Found the platform-specific device:
+ ‘windows’
+dev.new() is the preferred way to open a new device, in the unlikely
+event one is needed.
+.cpgBoxplots: no visible global function definition for ‘pdf’
+.cpgBoxplots: no visible global function definition for ‘par’
+.cpgBoxplots: no visible global function definition for ‘dev.off’
+... 291 lines ...
+ rainbow read.table rect str t.test text title verbose
+Consider adding
+ importFrom("grDevices", "dev.off", "pdf", "rainbow")
+ importFrom("graphics", "abline", "axis", "barplot", "bxp", "grid",
+ "layout", "legend", "lines", "matlines", "matplot", "mtext",
+ "par", "persp", "plot", "plot.new", "plot.window", "points",
+ "polygon", "rect", "text", "title")
+ importFrom("stats", "dbeta", "embed", "filter", "kmeans", "lm",
+ "lowess", "p.adjust", "predict", "pt", "qnorm", "t.test")
+ importFrom("utils", "read.table", "str")
+to your NAMESPACE file.
+
+checking Rd line widths ... NOTE
+Rd file 'ChromaBlocks.Rd':
+ \usage lines wider than 90 characters:
+ ChromaBlocks(rs.ip, rs.input, organism, chrs, ipWidth=100, inputWidth=500, preset=NULL, blockWidth=NULL, minBlocks=NULL, extend=NULL, c ... [TRUNCATED]
+
+Rd file 'GCbiasPlots.Rd':
+ \usage lines wider than 90 characters:
+ cex = 0.2, pch.col = "black", line.col = "red", lty = 1, lwd = 2, verbose = TRUE)
+
+Rd file 'absoluteCN.Rd':
+... 57 lines ...
+
+Rd file 'regionStats.Rd':
+ \usage lines wider than 90 characters:
+ regionStats(x, design = NULL, maxFDR=0.05, n.perm=5, window=600, mean.trim=.1, min.probes=10, max.gap=500, two.sides=TRUE, ndf, return. ... [TRUNCATED]
+ regionStats(x, design = NULL, maxFDR=0.05, n.perm=5, window=600, mean.trim=.1, min.probes=10, max.gap=500, two.sides=TRUE, ind=NULL, re ... [TRUNCATED]
+
+Rd file 'writeWig.Rd':
+ \usage lines wider than 90 characters:
+ writeWig(rs, seq.len = NULL, design=NULL, sample=20, drop.zero=TRUE, normalize=TRUE, verbose=TRUE)
+
+These lines will be truncated in the PDF manual.
+```
+
+## R.filesets (2.10.0)
+Maintainer: Henrik Bengtsson <henrikb at braju.com>
+Bug reports: https://github.com/HenrikBengtsson/R.filesets/issues
+
+0 errors | 0 warnings | 0 notes
+
+## TIN (1.5.2)
+Maintainer: Bjarne Johannessen <bjajoh at rr-research.no>
+
+0 errors | 0 warnings | 2 notes
+
+```
+checking top-level files ... NOTE
+Non-standard file/directory found at top level:
+ ‘doc’
+
+checking R code for possible problems ... NOTE
+aberrantExonUsage: no visible global function definition for ‘quantile’
+aberrantExonUsage: no visible global function definition for ‘ave’
+clusterPlot: no visible global function definition for ‘dist’
+clusterPlot: no visible global function definition for ‘hclust’
+clusterPlot: no visible global function definition for
+ ‘colorRampPalette’
+clusterPlot: no visible global function definition for ‘par’
+clusterPlot: no visible global function definition for ‘png’
+clusterPlot: no visible global function definition for ‘jpeg’
+... 50 lines ...
+ importFrom("stats", "ave", "dist", "hclust", "median", "quantile")
+ importFrom("utils", "data", "read.table")
+to your NAMESPACE file.
+
+Found the following assignments to the global environment:
+File ‘TIN/R/aberrantExonUsage.R’:
+ assign("quantiles", quantiles, envir = .GlobalEnv)
+ assign("aberrantExons", aberrantExons, envir = .GlobalEnv)
+File ‘TIN/R/correlationPlot.R’:
+ assign("randomGeneSetsDist", B, envir = .GlobalEnv)
+ assign("traPermutationsDist", L, envir = .GlobalEnv)
+```
+
diff --git a/revdep/check.R b/revdep/check.R
new file mode 100644
index 0000000..1eefa00
--- /dev/null
+++ b/revdep/check.R
@@ -0,0 +1,20 @@
+library("devtools")
+
+availableCores <- function() {
+ getenv <- function(name) {
+ as.integer(Sys.getenv(name, NA_character_))
+ }
+ getopt <- function(name) {
+ as.integer(getOption(name, NA_integer_))
+ }
+ if (is.finite(n <- getopt("mc.cores") + 1L)) return(n)
+ if (is.finite(n <- getopt("Ncpus") + 1L)) return(n)
+ if (is.finite(n <- getenv("PBS_NUM_PPN"))) return(n)
+ if (is.finite(n <- getenv("SLURM_CPUS_PER_TASK"))) return(n)
+ if (is.finite(n <- getenv("NSLOTS"))) return(n)
+ 1L
+}
+
+revdep_check(bioconductor = TRUE, recursive = TRUE, threads = availableCores())
+revdep_check_save_summary()
+revdep_check_print_problems()
diff --git a/revdep/checks.rds b/revdep/checks.rds
new file mode 100644
index 0000000..6693cae
Binary files /dev/null and b/revdep/checks.rds differ
diff --git a/revdep/problems.md b/revdep/problems.md
new file mode 100644
index 0000000..29435d4
--- /dev/null
+++ b/revdep/problems.md
@@ -0,0 +1,32 @@
+# Setup
+
+## Platform
+
+|setting |value |
+|:--------|:----------------------------|
+|version |R version 3.3.1 (2016-06-21) |
+|system |x86_64, linux-gnu |
+|ui |X11 |
+|language |en |
+|collate |en_US.UTF-8 |
+|tz |SystemV/PST8PDT |
+|date |2016-10-09 |
+
+## Packages
+
+|package |* |version |date |source |
+|:--------|:--|:-------|:----------|:--------------|
+|digest | |0.6.10 |2016-08-02 |cran (@0.6.10) |
+|future | |1.0.1 |2016-07-04 |cran (@1.0.1) |
+|globals | |0.7.0 |2016-09-09 |cran (@0.7.0) |
+|listenv | |0.6.0 |2015-12-28 |CRAN (R 3.3.1) |
+|markdown | |0.7.7 |2015-04-22 |CRAN (R 3.3.1) |
+|R.rsp | |0.30.0 |2016-05-15 |CRAN (R 3.3.1) |
+
+# Check results
+
+0 packages with problems
+
+
+
+
diff --git a/revdep/timing.md b/revdep/timing.md
new file mode 100644
index 0000000..f37edd1
--- /dev/null
+++ b/revdep/timing.md
@@ -0,0 +1,25 @@
+# Check times
+
+| |package |version | check_time|
+|:--|:----------------|:-------|----------:|
+|15 |PureCN |1.1.54 | 519.5|
+|17 |Repitools |1.19.3 | 324.9|
+|7 |fiery |0.2.1 | 252|
+|14 |PSCBS |0.61.0 | 192.5|
+|2 |aroma.affymetrix |3.0.0 | 156.5|
+|16 |QDNAseq |1.9.2 | 144.6|
+|19 |TIN |1.5.2 | 140.7|
+|9 |GeneBreak |1.3.0 | 121.1|
+|6 |doFuture |0.2.1 | 118.2|
+|13 |PECA |1.9.0 | 100.5|
+|8 |future.BatchJobs |0.13.0 | 92.7|
+|18 |R.filesets |2.10.0 | 91.1|
+|4 |aroma.core |3.0.0 | 90|
+|10 |MPAgenomics |1.1.2 | 46|
+|3 |aroma.cn |1.6.1 | 39.1|
+|5 |calmate |0.12.1 | 30.5|
+|1 |ACNE |0.8.1 | 28|
+|11 |NSA |0.0.32 | 25.8|
+|12 |pbmcapply |1.1.1 | 10.1|
+
+
diff --git a/tests/ClusterRegistry.R b/tests/ClusterRegistry.R
index c6ec62a..afa12b0 100644
--- a/tests/ClusterRegistry.R
+++ b/tests/ClusterRegistry.R
@@ -9,25 +9,37 @@ for (cores in 1:min(3L, availableCores())) {
message("Available cores: ", availableCores())
## In case sessions have been created in previous tests
+ message("Stopping any running cluster ...")
ClusterRegistry("stop")
+ cluster <- ClusterRegistry("get")
+ stopifnot(length(cluster) == 0)
+ message("Stopping any running cluster ... DONE")
+ message("Starting cluster ...")
res <- try({
cluster <- ClusterRegistry("set", workers=availableCores()-1L)
+ str(cluster)
print(cluster)
}, silent=TRUE)
if (cores == 1) stopifnot(inherits(res, "try-error"))
+ message("Starting cluster ... DONE")
+ message("Starting single-worker cluster ...")
cluster <- ClusterRegistry(action="start", workers=1L)
+ str(cluster)
print(cluster)
stopifnot(length(cluster) == 1L)
+ message("Starting single-worker cluster ... DONE")
cluster <- ClusterRegistry(action="get")
print(cluster)
stopifnot(length(cluster) == 1L)
+ message("Stopping single-worker cluster ...")
cluster <- ClusterRegistry(action="stop")
print(cluster)
stopifnot(length(cluster) == 0L)
+ message("Stopping single-worker cluster ... DONE")
message(sprintf("Testing with %d cores ... DONE", cores))
} ## for (cores ...)
diff --git a/tests/Future-class.R b/tests/Future-class.R
index 158ff83..4e9c311 100644
--- a/tests/Future-class.R
+++ b/tests/Future-class.R
@@ -22,6 +22,31 @@ expr <- getExpression(f)
print(expr)
stopifnot(is.call(expr))
+clazzes <- list(
+ eager = EagerFuture,
+ lazy = LazyFuture,
+ multisession = function(...) MultisessionFuture(..., workers=2L)
+)
+if (supportsMulticore()) clazzes$multicore = function(...) MulticoreFuture(..., workers=2L)
+
+for (clazz in clazzes) {
+ ## Calling run() more than once
+ f <- clazz({ 42L })
+ print(f)
+ run(f)
+ res <- tryCatch(run(f), error=identity)
+ stopifnot(inherits(res, "error"))
+ v <- value(f)
+ print(v)
+ stopifnot(v == 42L)
+
+ ## Call value() without run()
+ f <- clazz({ 42L })
+ v <- value(f)
+ print(v)
+ stopifnot(v == 42L)
+}
+
message("*** Future class - exception ... DONE")
message("*** Future class ... DONE")
diff --git a/tests/early-signaling.R b/tests/early-signaling.R
index 6caee07..d89bd2a 100644
--- a/tests/early-signaling.R
+++ b/tests/early-signaling.R
@@ -8,6 +8,7 @@ message("*** Early signaling of conditions with eager futures ...")
plan(eager)
f <- future({ stop("bang!") })
+Sys.sleep(1.0)
r <- resolved(f)
stopifnot(r)
v <- try(value(f), silent=TRUE)
@@ -24,6 +25,7 @@ message("*** Early signaling of conditions with lazy futures ...")
plan(lazy)
f <- future({ stop("bang!") })
+Sys.sleep(1.0)
r <- resolved(f)
stopifnot(r)
v <- try(value(f), silent=TRUE)
@@ -33,6 +35,7 @@ plan(lazy, earlySignal=TRUE)
## Errors
f <- future({ stop("bang!") })
+Sys.sleep(1.0)
r <- try(resolved(f), silent=TRUE)
stopifnot(inherits(r, "try-error"))
v <- try(value(f), silent=TRUE)
@@ -40,6 +43,7 @@ stopifnot(inherits(v, "try-error"))
## Warnings
f <- future({ warning("careful!") })
+Sys.sleep(1.0)
res <- tryCatch({
r <- resolved(f)
}, condition = function(w) w)
@@ -47,6 +51,7 @@ stopifnot(inherits(res, "warning"))
## Messages
f <- future({ message("hey!") })
+Sys.sleep(1.0)
res <- tryCatch({
r <- resolved(f)
}, condition = function(w) w)
@@ -54,6 +59,7 @@ stopifnot(inherits(res, "message"))
## Condition
f <- future({ signalCondition(simpleCondition("hmm")) })
+Sys.sleep(1.0)
res <- tryCatch({
r <- resolved(f)
}, condition = function(w) w)
@@ -67,6 +73,7 @@ message("*** Early signaling of conditions with multisession futures ...")
plan(multisession)
f <- future({ stop("bang!") })
+Sys.sleep(1.0)
r <- resolved(f)
stopifnot(r)
v <- try(value(f), silent=TRUE)
@@ -74,6 +81,7 @@ stopifnot(inherits(v, "try-error"))
plan(multisession, earlySignal=TRUE)
f <- future({ stop("bang!") })
+Sys.sleep(1.0)
print(f)
r <- try(resolved(f), silent=TRUE)
stopifnot(inherits(r, "try-error") || inherits(f, "UniprocessFuture"))
@@ -88,6 +96,7 @@ message("*** Early signaling of conditions with multiprocess futures ...")
plan(multiprocess)
f <- future({ stop("bang!") })
+Sys.sleep(1.0)
r <- resolved(f)
stopifnot(r)
v <- try(value(f), silent=TRUE)
@@ -95,6 +104,7 @@ stopifnot(inherits(v, "try-error"))
plan(multiprocess, earlySignal=TRUE)
f <- future({ stop("bang!") })
+Sys.sleep(1.0)
print(f)
r <- try(resolved(f), silent=TRUE)
stopifnot(inherits(r, "try-error") || inherits(f, "UniprocessFuture"))
diff --git a/tests/future,labels.R b/tests/future,labels.R
new file mode 100644
index 0000000..f1f6b62
--- /dev/null
+++ b/tests/future,labels.R
@@ -0,0 +1,40 @@
+source("incl/start.R")
+
+message("*** Futures - labels ...")
+
+strategies <- supportedStrategies()
+strategies <- setdiff(strategies, "multiprocess")
+
+for (strategy in strategies) {
+ message(sprintf("- plan('%s') ...", strategy))
+ plan(strategy)
+
+ for (label in list(NULL, sprintf("strategy=%s", strategy))) {
+ fcn <- get(strategy, mode="function")
+ stopifnot(inherits(fcn, strategy))
+ f <- fcn(42, label = label)
+ print(f)
+ stopifnot(identical(f$label, label))
+ v <- value(f)
+ stopifnot(v == 42)
+
+ f <- future(42, label = label)
+ print(f)
+ stopifnot(identical(f$label, label))
+ v <- value(f)
+ stopifnot(v == 42)
+
+ v %<-% { 42 } %label% label
+ f <- futureOf(v)
+ print(f)
+ stopifnot(identical(f$label, label))
+ stopifnot(v == 42)
+
+ } ## for (label ...)
+
+ message(sprintf("- plan('%s') ... DONE", strategy))
+} ## for (strategy ...)
+
+message("*** Futures - labels ... DONE")
+
+source("incl/end.R")
diff --git a/tests/globals,formulas.R b/tests/globals,formulas.R
new file mode 100644
index 0000000..b48abe3
--- /dev/null
+++ b/tests/globals,formulas.R
@@ -0,0 +1,136 @@
+source("incl/start.R")
+
+library("datasets") ## cars data set
+library("stats") ## lm(), poly(), xtabs()
+
+strategies <- supportedStrategies()
+strategies <- setdiff(strategies, "multiprocess")
+
+message("*** Globals - formulas ...")
+
+message("*** Globals - lm(<formula>) ...")
+
+## From example("lm", package="stats")
+ctl <- c(4.17,5.58,5.18,6.11,4.50,4.61,5.17,4.53,5.33,5.14)
+trt <- c(4.81,4.17,4.41,3.59,5.87,3.83,6.03,4.89,4.32,4.69)
+group <- gl(2, 10, 20, labels = c("Ctl","Trt"))
+weight <- c(ctl, trt)
+
+## Truth:
+fit0 <- lm(weight ~ group - 1)
+print(fit0)
+
+for (cores in 1:min(3L, availableCores())) {
+ message(sprintf("Testing with %d cores ...", cores))
+ options(mc.cores=cores-1L)
+
+ message("availableCores(): ", availableCores())
+
+ for (strategy in strategies) {
+ message(sprintf("- plan('%s') ...", strategy))
+ plan(strategy)
+
+ ## Explicit future
+ f <- future({ lm(weight ~ group - 1) })
+ fit <- value(f)
+ print(fit)
+ stopifnot(all.equal(fit, fit0))
+
+ ## Future assignment
+ fit %<-% { lm(weight ~ group - 1) }
+ print(fit)
+ stopifnot(all.equal(fit, fit0))
+ } ## for (strategy ...)
+
+ message(sprintf("Testing with %d cores ... DONE", cores))
+} ## for (cores ...)
+
+message("*** Globals - lm(<formula>) ... DONE")
+
+
+message("*** Globals - one-side formulas, e.g. xtabs(~ x) ...")
+
+x <- c(1, 1, 2, 2, 2)
+
+## Truth:
+tbl0 <- xtabs(~ x)
+print(tbl0)
+
+for (cores in 1:min(3L, availableCores())) {
+ message(sprintf("Testing with %d cores ...", cores))
+ options(mc.cores=cores-1L)
+
+ message("availableCores(): ", availableCores())
+
+ for (strategy in strategies) {
+ message(sprintf("- plan('%s') ...", strategy))
+ plan(strategy)
+
+ ## Explicit future
+ f <- future({ xtabs(~ x) })
+ tbl <- value(f)
+ print(tbl)
+ stopifnot(all.equal(tbl, tbl0))
+
+ ## Future assignment
+ tbl %<-% { xtabs(~ x) }
+ print(tbl)
+ stopifnot(all.equal(tbl, tbl0))
+ } ## for (strategy ...)
+
+ message(sprintf("Testing with %d cores ... DONE", cores))
+} ## for (cores ...)
+
+message("*** Globals - one-side formulas, e.g. xtabs(~ x) ... DONE")
+
+
+message("*** Globals - lm(<formula>, data=cars) ...")
+
+exprs <- list(
+ # "remove-intercept-term" form of no-intercept
+ a = substitute({ lm(dist ~ . -1, data=cars) }),
+ # "make-intercept-zero" form of no-intercept
+ b = substitute({ lm(dist ~ . +0, data=cars) }),
+ # doesn't do what we want here
+ c = substitute({ lm(dist ~ speed + speed^2, data=cars) }),
+ # gets us a quadratic term
+ d = substitute({ lm(dist ~ speed + I(speed^2), data=cars) }),
+ # avoid potential multicollinearity
+ e = substitute({ lm(dist ~ poly(speed,2), data=cars) })
+)
+
+for (kk in seq_along(exprs)) {
+ expr <- exprs[[kk]]
+ name <- names(exprs)[kk]
+ message(sprintf("- Globals - lm(<formula #%d (%s)>, data=cars) ...", kk, sQuote(name)))
+
+ fit0 <- eval(expr)
+ print(fit0)
+
+ for (cores in 1:min(3L, availableCores())) {
+ message(sprintf("Testing with %d cores ...", cores))
+ options(mc.cores=cores-1L)
+
+ message("availableCores(): ", availableCores())
+
+ for (strategy in strategies) {
+ message(sprintf("- plan('%s') ...", strategy))
+ plan(strategy)
+
+ f <- future(expr, substitute=FALSE)
+ fit <- value(f)
+ print(fit)
+
+ stopifnot(all.equal(fit, fit0))
+ } ## for (strategy ...)
+
+ message(sprintf("Testing with %d cores ... DONE", cores))
+ } ## for (cores ...)
+} ## for (kk ...)
+
+message("*** Globals - lm(<formula>, data=cars) ... DONE")
+
+
+message("*** Globals - formulas ... DONE")
+
+source("incl/end.R")
diff --git a/tests/globals,manual.R b/tests/globals,manual.R
new file mode 100644
index 0000000..97aa54c
--- /dev/null
+++ b/tests/globals,manual.R
@@ -0,0 +1,141 @@
+source("incl/start.R")
+
+message("*** Globals - manually ...")
+
+message("*** Globals manually specified as named list ...")
+
+globals <- list(
+ a=1,
+ b=2,
+ sumtwo=function(x) x[1] + x[2]
+)
+
+## Assign 'globals' globally
+attachLocally(globals)
+
+## Truth
+v0 <- local({
+ x <- 1:10
+ sumtwo(a + b*x)
+})
+
+
+message("*** Globals - automatic ...")
+
+for (strategy in supportedStrategies()) {
+ message(sprintf("- Strategy: %s ...", strategy))
+
+ plan(strategy)
+
+ attachLocally(globals)
+ f <- future({
+ x <- 1:10
+ sumtwo(a + b*x)
+ }, globals=TRUE)
+ print(f)
+ rm(list=names(globals))
+ y <- value(f)
+ print(y)
+ stopifnot(all.equal(y, v0))
+
+ attachLocally(globals)
+ y %<-% {
+ x <- 1:10
+ sumtwo(a + b*x)
+ } %globals% TRUE
+ rm(list=names(globals))
+ print(y)
+ stopifnot(all.equal(y, v0))
+
+ ## No need to search for globals
+ y %<-% { 1 } %globals% FALSE
+ print(y)
+ stopifnot(identical(y, 1))
+
+ ## Exception - missing global
+ attachLocally(globals)
+ f <- future({
+ x <- 1:10
+ sumtwo(a + b*x)
+ }, globals=FALSE)
+ print(f)
+ rm(list=names(globals))
+ y <- tryCatch(value(f), error = identity)
+ if (!inherits(f, c("EagerFuture", "MulticoreFuture"))) {
+ stopifnot(inherits(y, "simpleError"))
+ }
+
+ message(sprintf("- Strategy: %s ... DONE", strategy))
+}
+
+message("*** Globals - automatic ... DONE")
+
+
+message("*** Globals manually specified as named list ...")
+
+for (strategy in supportedStrategies()) {
+ message(sprintf("- Strategy: %s ...", strategy))
+
+ plan(strategy)
+
+ ## Make sure globals do not exist
+ rm(list=names(globals))
+
+ f <- future({
+ x <- 1:10
+ sumtwo(a + b*x)
+ }, globals=globals)
+ print(f)
+ v <- value(f)
+ print(v)
+ stopifnot(all.equal(v, v0))
+
+ y %<-% {
+ x <- 1:10
+ sumtwo(a + b*x)
+ } %globals% globals
+ print(y)
+ stopifnot(all.equal(y, v0))
+
+ message(sprintf("- Strategy: %s ... DONE", strategy))
+}
+
+message("*** Globals manually specified as named list ... DONE")
+
+
+message("*** Globals manually specified by their names ...")
+
+for (strategy in supportedStrategies()) {
+ message(sprintf("- Strategy: %s ...", strategy))
+
+ plan(strategy)
+
+ attachLocally(globals)
+ f <- future({
+ x <- 1:10
+ sumtwo(a + b*x)
+ }, globals=c("a", "b", "sumtwo"))
+ print(f)
+ rm(list=names(globals))
+ v <- value(f)
+ print(v)
+ stopifnot(all.equal(v, v0))
+
+ attachLocally(globals)
+ y %<-% {
+ x <- 1:10
+ sumtwo(a + b*x)
+ } %globals% c("a", "b", "sumtwo")
+ rm(list=names(globals))
+ print(y)
+ stopifnot(all.equal(y, v0))
+
+ message(sprintf("- Strategy: %s ... DONE", strategy))
+}
+
+message("*** Globals manually specified by their names ... DONE")
+
+
+message("*** Globals - manually ... DONE")
+
+source("incl/end.R")
diff --git a/tests/globals,subassignment.R b/tests/globals,subassignment.R
new file mode 100644
index 0000000..59ec507
--- /dev/null
+++ b/tests/globals,subassignment.R
@@ -0,0 +1,143 @@
+source("incl/start.R")
+
+oopts <- c(oopts, options(
+ future.globals.resolve=TRUE,
+ future.globals.onMissing="error"
+))
+
+strategies <- supportedStrategies()
+strategies <- setdiff(strategies, "multiprocess")
+
+message("*** Globals - subassignments ...")
+
+message("*** Globals - subassignments w/ x$a <- value ...")
+
+## Truth:
+x <- x0 <- list()
+y0 <- list(a = 1)
+str(list(x=x, y0=y0))
+
+y <- local({
+ x$a <- 1
+ x
+})
+stopifnot(identical(y, y0))
+
+y <- local({
+ x[["a"]] <- 1
+ x
+})
+stopifnot(identical(y, y0))
+
+y <- local({
+ x["a"] <- list(1)
+ x
+})
+stopifnot(identical(y, y0))
+
+stopifnot(identical(x, list()))
+
+for (cores in 1:min(3L, availableCores())) {
+ message(sprintf("Testing with %d cores ...", cores))
+ options(mc.cores=cores-1L)
+
+ message("availableCores(): ", availableCores())
+
+ for (strategy in strategies) {
+ message(sprintf("- plan('%s') ...", strategy))
+ plan(strategy)
+
+ ## Explicit future
+ x <- list()
+ f <- future({
+ x$a <- 1
+ x
+ })
+ rm(list="x")
+ y <- value(f)
+ print(y)
+ stopifnot(identical(y, y0))
+
+ ## Future assignment
+ x <- list()
+ y %<-% {
+ x$a <- 1
+ x
+ }
+ rm(list="x")
+ print(y)
+ stopifnot(identical(y, y0))
+
+ ## 'x' is _not_ a global variable here
+ x <- list()
+ y %<-% {
+ x <- list(b = 2)
+ x$a <- 1
+ x
+ }
+ rm(list="x")
+ print(y)
+ stopifnot(identical(y, list(b = 2, a = 1)))
+
+ ## Explicit future
+ x <- list()
+ f <- future({
+ x[["a"]] <- 1
+ x
+ })
+ rm(list="x")
+ y <- value(f)
+ print(y)
+ stopifnot(identical(y, y0))
+
+ ## Future assignment
+ x <- list()
+ y %<-% {
+ x[["a"]] <- 1
+ x
+ }
+ rm(list="x")
+ print(y)
+ stopifnot(identical(y, y0))
+
+ ## Explicit future
+ x <- list()
+ f <- future({
+ x["a"] <- list(1)
+ x
+ })
+ rm(list="x")
+ y <- value(f)
+ print(y)
+ stopifnot(identical(y, y0))
+
+ ## Future assignment
+ x <- list()
+ y %<-% {
+ x["a"] <- list(1)
+ x
+ }
+ rm(list="x")
+ print(y)
+ stopifnot(identical(y, y0))
+
+ ## Future assignment
+ x <- list()
+ name <- "a"
+ y %<-% {
+ x[name] <- list(1)
+ x
+ }
+ rm(list=c("x", "name"))
+ print(y)
+ stopifnot(identical(y, y0))
+ } ## for (strategy ...)
+
+ message(sprintf("Testing with %d cores ... DONE", cores))
+} ## for (cores ...)
+
+message("*** Globals - subassignments w/ x$a <- value ... DONE")
+
+message("*** Globals - subassignments ... DONE")
+
+source("incl/end.R")
diff --git a/tests/incl/end.R b/tests/incl/end.R
index 1441991..443ad87 100644
--- a/tests/incl/end.R
+++ b/tests/incl/end.R
@@ -21,6 +21,9 @@ added <- setdiff(names(cenvs), names(oenvs))
for (name in added) Sys.unsetenv(name)
## (b) Modified?
for (name in intersect(names(cenvs), names(oenvs))) {
+ ## WORKAROUND: On Linux Wine, base::Sys.getenv() may
+ ## return elements with empty names. /HB 2016-10-06
+ if (nchar(name) == 0) next
if (!identical(cenvs[[name]], oenvs[[name]])) {
Sys.setenv(name, oenvs[[name]])
}
diff --git a/tests/incl/start,load-only.R b/tests/incl/start,load-only.R
index 8bf631c..19f7ec8 100644
--- a/tests/incl/start,load-only.R
+++ b/tests/incl/start,load-only.R
@@ -41,3 +41,8 @@ whichIndex <- future:::whichIndex
## Local functions for test scripts
printf <- function(...) cat(sprintf(...))
mstr <- function(...) message(paste(capture.output(str(...)), collapse="\n"))
+attachLocally <- function(x, envir=parent.frame()) {
+ for (name in names(x)) {
+ assign(name, value=x[[name]], envir=envir)
+ }
+}
diff --git a/tests/multicore.R b/tests/multicore.R
index 6eddc62..41322a5 100644
--- a/tests/multicore.R
+++ b/tests/multicore.R
@@ -9,7 +9,7 @@ for (cores in 1:min(3L, availableCores("multicore"))) {
options(mc.cores=cores-1L)
if (!supportsMulticore()) {
- message(sprintf("Multicore futures are not supporting on '%s'. Falling back to use synchroneous eager futures", .Platform$OS))
+ message(sprintf("Multicore futures are not supporting on '%s'. Falling back to use synchroneous eager futures", .Platform$OS.type))
}
for (globals in c(FALSE, TRUE)) {
diff --git a/tests/multisession.R b/tests/multisession.R
index 8e8bc0c..142b278 100644
--- a/tests/multisession.R
+++ b/tests/multisession.R
@@ -125,6 +125,31 @@ for (cores in 1:min(3L, availableCores())) {
message("*** multisession(..., workers=1L) ... DONE")
+ message("*** multisession(..., gc=TRUE) ...")
+ plan(multisession, workers=2L)
+
+ f <- future({ gc() })
+ v <- value(f)
+ print(v)
+
+ f <- future({ integer(10e6) })
+ v <- value(f)
+ str(v)
+
+ f <- future({ gc() })
+ v <- value(f)
+ print(v)
+
+ f <- future({ integer(10e6) }, gc=TRUE)
+ v <- value(f)
+ str(v)
+
+ f <- future({ gc() })
+ v <- value(f)
+ print(v)
+
+ message("*** multisession(..., gc=TRUE) ... TRUE")
+
message(sprintf("Testing with %d cores ... DONE", cores))
} ## for (cores ...)
diff --git a/tests/nested_futures.R b/tests/nested_futures.R
new file mode 100644
index 0000000..a57ea5e
--- /dev/null
+++ b/tests/nested_futures.R
@@ -0,0 +1,86 @@
+source("incl/start.R")
+
+strategies <- future:::supportedStrategies()
+strategies <- setdiff(strategies, "multiprocess")
+
+message("*** Nested futures ...")
+
+for (strategy1 in strategies) {
+ for (strategy2 in strategies) {
+ message(sprintf("- plan(list('%s', '%s')) ...", strategy1, strategy2))
+ plan(list(a=strategy1, b=strategy2))
+
+ nested <- plan("list")
+ stopifnot(
+ length(nested) == 2L,
+ all(names(nested) == c("a", "b")),
+ inherits(plan(), strategy1)
+ )
+
+ x %<-% {
+ a <- 1L
+
+ ## IMPORTANT: Use future::plan() - not just plan() - otherwise
+ ## we're exporting the plan() function including its local stack!
+ plan_a <- future::plan("list")
+ nested_a <- nested[-1]
+
+ stopifnot(
+ length(nested_a) == 1L,
+ length(plan_a) == 1L,
+ inherits(plan_a[[1]], "future"),
+ all.equal(plan_a, nested_a),
+ inherits(future::plan(), strategy2)
+ )
+
+ y %<-% {
+ b <- 2L
+
+ ## IMPORTANT: Use future::plan() - not just plan() - otherwise
+ ## we're exporting the plan() function including its local stack!
+ plan_b <- future::plan("list")
+ nested_b <- nested_a[-1]
+ stopifnot(
+ length(nested_b) == 0L,
+ length(plan_b) == 1L,
+ inherits(plan_b[[1]], "future"),
+ inherits(future::plan(), getOption("future.default", "eager"))
+ )
+
+ list(a = a, nested_a = nested_a, plan_a = plan_a,
+ b = b, nested_b = nested_b, plan_b = plan_b)
+ }
+ y
+ }
+
+ str(x)
+
+ stopifnot(
+ length(x) == 3 * length(nested),
+ all(names(x) == c("a", "nested_a", "plan_a",
+ "b", "nested_b", "plan_b")),
+
+ x$a == 1L,
+ length(x$nested_a) == 1L,
+ is.list(x$plan_a),
+ length(x$plan_a) == 1L,
+ inherits(x$plan_a[[1]], "future"),
+ all.equal(x$plan_a, nested[-1L]),
+
+ x$b == 2L,
+ length(x$nested_b) == 0L,
+ is.list(x$plan_b),
+ length(x$plan_b) == 1L,
+ inherits(x$plan_b[[1]], "future"),
+ inherits(x$plan_b[[1]], getOption("future.default", "eager"))
+ )
+
+ rm(list=c("nested", "x"))
+
+ message(sprintf("- plan(list('%s', '%s')) ... DONE", strategy1, strategy2))
+ }
+}
+
+message("*** Nested futures ... DONE")
+
+source("incl/end.R")
diff --git a/tests/rng.R b/tests/rng.R
index 2a89ff2..ab739ae 100644
--- a/tests/rng.R
+++ b/tests/rng.R
@@ -6,6 +6,9 @@ message("*** rng ...")
## See Section 6 on 'Random-number generation' in
## vignette("parallel", package="parallel")
fsample <- function(x, size=4L, seed=NULL) {
+ ## Must use session-specific '.GlobalEnv' here
+ .GlobalEnv <- globalenv()
+
oseed <- .GlobalEnv$.Random.seed
orng <- RNGkind("L'Ecuyer-CMRG")[1L]
on.exit(RNGkind(orng))
@@ -28,6 +31,8 @@ fsample <- function(x, size=4L, seed=NULL) {
for (ii in seq_len(size)) {
.seed <- parallel::nextRNGStream(.seed)
res[[ii]] %<-% {
+ ## Must use session-specific '.GlobalEnv' here
+ .GlobalEnv <- globalenv()
.GlobalEnv$.Random.seed <- .seed
sample(x, size=1L)
}
diff --git a/tests/tweak.R b/tests/tweak.R
index 209a860..2ad177d 100644
--- a/tests/tweak.R
+++ b/tests/tweak.R
@@ -105,6 +105,22 @@ stopifnot(a == 2, x == 2)
message("*** y %<-% { expr } %tweak% tweaks ... DONE")
+message("*** tweak() - gc=TRUE ...")
+
+res <- tryCatch(tweak(multisession, gc=TRUE), condition=identity)
+stopifnot(inherits(res, "tweaked"))
+
+## Argument 'gc' is unknown
+res <- tryCatch(tweak(eager, gc=TRUE), condition=identity)
+stopifnot(inherits(res, "warning"))
+
+res <- tryCatch(tweak(multicore, gc=TRUE), condition=identity)
+stopifnot(inherits(res, "warning"))
+
+message("*** tweak() - gc=TRUE ... DONE")
+
+
+
message("*** tweak() - exceptions ...")
res <- try(tweak("<unknown-future-strategy>"), silent=TRUE)
diff --git a/tests/utils.R b/tests/utils.R
index 00c7afa..864fa2a 100644
--- a/tests/utils.R
+++ b/tests/utils.R
@@ -37,14 +37,16 @@ printf("y = %s.\n", hpaste(y, maxHead=Inf))
printf("y = %s.\n", paste(y, collapse=", "))
## y = 10, 9, 8, 7, 6, 5, 4, 3, 2, 1
-
-# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-# Adding a special separator before the last element
-# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# Change last separator
printf("x = %s.\n", hpaste(x, lastCollapse=" and "))
## x = 1, 2, 3, 4, 5 and 6.
+# No collapse
+stopifnot(all(hpaste(x, collapse=NULL) == x))
+
+# Empty input
+stopifnot(identical(hpaste(character(0)), character(0)))
+
message("*** hpaste() ...")
diff --git a/vignettes/future-1-overview.md.rsp b/vignettes/future-1-overview.md.rsp
index ef864c8..6b21a6c 100644
--- a/vignettes/future-1-overview.md.rsp
+++ b/vignettes/future-1-overview.md.rsp
@@ -130,7 +130,7 @@ One of the designs of the Future API was to encapsulate any differences such tha
Because of this, the defaults of the different strategies are such that the results and side effects of evaluating a future expression are as similar as possible. More specifically, the following is true for all futures:
-* All _evaluation is done in a local environment_ (i.e. `local({ expr })`) such assignments do not affect the calling environment. This is natural when evaluating in an external R process, but is also enforced when evaluating in the current R session.
+* All _evaluation is done in a local environment_ (i.e. `local({ expr })`) so that assignments do not affect the calling environment. This is natural when evaluating in an external R process, but is also enforced when evaluating in the current R session.
* When a future is constructed, _global variables are identified_. For asynchronous evaluation, globals are exported to the R process/session that will be evaluating the future expression. For lazy futures, globals are "frozen" (cloned to a local environment of the future). Also, in order to protect against exporting too large objects by mistake, there is a built-in assertion that the total size of all globals is less than a given threshold (controllable via an option, cf. `help("futu [...]
@@ -165,7 +165,7 @@ Eager futures are the default unless otherwise specified. They were designed to
> plan(eager)
> pid <- Sys.getpid()
> pid
-[1] 28072
+[1] 14641
> a %<-% {
+ pid <- Sys.getpid()
+ cat("Resolving 'a' ...\n")
@@ -184,13 +184,13 @@ Resolving 'b' ...
+ }
Resolving 'c' ...
> b
-[1] 28072
+[1] 14641
> c
[1] 6.28
> a
[1] 3.14
> pid
-[1] 28072
+[1] 14641
```
Since eager evaluation is taking place, each of the three futures is resolved instantaneously in the moment it is created. Note also how `pid` in the calling environment, which was assigned the process ID of the current process, is neither overwritten nor removed. This is because futures are evaluated in a local environment. Since synchronous (uni-)processing is used, future `b` is resolved by the main R process (still in a local environment), which is why the value of `b` and `pid` a [...]
@@ -201,7 +201,7 @@ A lazy future evaluates its expression only if its value is queried. Evaluation
> plan(lazy)
> pid <- Sys.getpid()
> pid
-[1] 28072
+[1] 14641
> a %<-% {
+ pid <- Sys.getpid()
+ cat("Resolving 'a' ...\n")
@@ -219,14 +219,14 @@ A lazy future evaluates its expression only if its value is queried. Evaluation
Resolving 'a' ...
> b
Resolving 'b' ...
-[1] 28072
+[1] 14641
> c
Resolving 'c' ...
[1] 6.28
> a
[1] 3.14
> pid
-[1] 28072
+[1] 14641
```
As previously, variable `pid` is unaffected because all evaluation is done in a local environment. More interestingly, future `a` is no longer evaluated in the moment it is created, but instead when it is needed the first time, which happens when future `c` is created. This is because `a` is identified as a global variable that needs to be captured ("frozen" to `a == 3.14`) in order to set up future `c`. Later when `c` (the value of future `c`) is queried, `a` has already been resolve [...]
@@ -248,7 +248,7 @@ We start with multisession futures because they are supported by all operating s
> plan(multisession)
> pid <- Sys.getpid()
> pid
-[1] 28072
+[1] 14641
> a %<-% {
+ pid <- Sys.getpid()
+ cat("Resolving 'a' ...\n")
@@ -264,13 +264,13 @@ We start with multisession futures because they are supported by all operating s
+ 2 * a
+ }
> b
-[1] 28093
+[1] 14662
> c
[1] 6.28
> a
[1] 3.14
> pid
-[1] 28072
+[1] 14641
```
The first thing we observe is that the values of `a`, `c` and `pid` are the same as previously. However, we notice that `b` is different from before. This is because future `b` is evaluated in a different R process and therefore it returns a different process ID. Another difference is that the messages, generated by `cat()`, are no longer displayed. This is because they are outputted to the background sessions and not the calling session.
@@ -310,7 +310,7 @@ Cluster futures evaluate expressions on an ad-hoc cluster (as implemented by the
> plan(cluster, workers = c("n1", "n2", "n3"))
> pid <- Sys.getpid()
> pid
-[1] 28072
+[1] 14641
> a %<-% {
+ pid <- Sys.getpid()
+ cat("Resolving 'a' ...\n")
@@ -326,13 +326,13 @@ Cluster futures evaluate expressions on an ad-hoc cluster (as implemented by the
+ 2 * a
+ }
> b
-[1] 28115
+[1] 14684
> c
[1] 6.28
> a
[1] 3.14
> pid
-[1] 28072
+[1] 14641
```
Just as for the other asynchronous evaluation strategies, the output from `cat()` is not displayed on the current/calling machine.
@@ -354,7 +354,7 @@ Sometimes one may want to use an alternative evaluation strategy for a specific
> plan(eager)
> pid <- Sys.getpid()
> pid
-[1] 28072
+[1] 14641
> a %<-% {
+ Sys.getpid()
+ }
@@ -365,11 +365,11 @@ Sometimes one may want to use an alternative evaluation strategy for a specific
+ Sys.getpid()
+ } %plan% multiprocess
> a
-[1] 28072
+[1] 14641
> b
-[1] 28132
+[1] 14701
> c
-[1] 28133
+[1] 14702
```
As seen by the different process IDs, future `a` is evaluated eagerly using the same process as the calling environment whereas the other two are evaluated using multiprocess futures.
@@ -403,14 +403,14 @@ For instance, here is an example of two "top" futures (`a` and `b`) that uses mu
+ c(b.pid = Sys.getpid(), b1.pid = b1, b2.pid = b2)
+ }
> pid
-[1] 28072
+[1] 14641
> a
-[1] 28134
+[1] 14703
> b
b.pid b1.pid b2.pid
- 28135 28135 28135
+ 14704 14704 14704
```
-By inspection the process IDs, we see that there are in total three different processes involved for resolving the futures. There is the main R process (pid 28072), and there are the two processes used by `a` (pid 28134) and `b` (pid 28135). However, the two futures (`b1` and `b2`) that is nested by `b` are evaluated by the same R process as `b`. This is because nested futures use eager evaluation unless otherwise specified. There are a few reasons for this, but the main reason is th [...]
+By inspection the process IDs, we see that there are in total three different processes involved for resolving the futures. There is the main R process (pid 14641), and there are the two processes used by `a` (pid 14703) and `b` (pid 14704). However, the two futures (`b1` and `b2`) that is nested by `b` are evaluated by the same R process as `b`. This is because nested futures use eager evaluation unless otherwise specified. There are a few reasons for this, but the main reason is th [...]
@@ -423,12 +423,12 @@ We would actually get the same behavior if we try with multiple levels of multip
> plan(list(multiprocess, multiprocess))
[...]
> pid
-[1] 28072
+[1] 14641
> a
-[1] 28136
+[1] 14705
> b
b.pid b1.pid b2.pid
- 28137 28137 28137
+ 14706 14706 14706
```
The reason for this is, also here, to protect us from launching more processes than what the machine can support. Internally, this is done by setting `mc.cores` to zero ([sic!](https://github.com/HenrikBengtsson/Wishlist-for-R/issues/7)) such that no _additional_ parallel processes can be launched. This is the case for both multisession and multicore evaluation.
@@ -440,14 +440,14 @@ Continuing, if we start off by eager evaluation and then use multiprocess evalua
Resolving 'a' ...
Resolving 'b' ...
> pid
-[1] 28072
+[1] 14641
> a
-[1] 28072
+[1] 14641
> b
b.pid b1.pid b2.pid
- 28072 28138 28139
+ 14641 14707 14708
```
-which clearly show that `a` and `b` are resolved in the calling process (pid 28072) whereas the two nested futures (`b1` and `b2`) are resolved in two separate R processes (pids 28138 and 28139).
+which clearly show that `a` and `b` are resolved in the calling process (pid 14641) whereas the two nested futures (`b1` and `b2`) are resolved in two separate R processes (pids 14707 and 14708).
@@ -457,14 +457,14 @@ Having said this, it is indeed possible to use nested multiprocess evaluation st
+ workers = 3)))
[...]
> pid
-[1] 28072
+[1] 14641
> a
-[1] 28140
+[1] 14709
> b
b.pid b1.pid b2.pid
- 28141 28142 28145
+ 14710 14711 14712
```
-First, we see that both `a` and `b` are resolved in different processes (pids 28140 and 28141) than the calling process (pid 28072). Second, the two nested futures (`b1` and `b2`) are resolved in yet two other R processes (pids 28142 and 28145).
+First, we see that both `a` and `b` are resolved in different processes (pids 14709 and 14710) than the calling process (pid 14641). Second, the two nested futures (`b1` and `b2`) are resolved in yet two other R processes (pids 14711 and 14712).
To clarify, when we set up the two levels of multiprocess evaluation, we specified that in total 3 processes may be used at each level. We choose three parallel processes, not just two, because one is always consumed by the calling process leaving two to be used for the asynchronous futures. This is why we see that `pid`, `a` and `b` are all resolved by the same process. If we had allowed only two cores at the top level, `a` and `b` would have been resolved by the same background proc [...]
@@ -499,7 +499,7 @@ Waiting for 'a' to be resolved ...
> cat("Waiting for 'a' to be resolved ... DONE\n")
Waiting for 'a' to be resolved ... DONE
> a
-[1] 28146
+[1] 14713
```
@@ -566,9 +566,9 @@ There is one limitation with implicit futures that does not exist for explicit o
> v <- lapply(f, FUN = value)
> str(v)
List of 3
- $ : int 28147
- $ : int 28148
- $ : int 28149
+ $ : int 14714
+ $ : int 14715
+ $ : int 14716
```
This is _not_ possible to do when using implicit futures. This is because the `%<-%` assignment operator _cannot_ be used in all cases where the regular `<-` assignment operator can be used. It can only be used to assign future values to _environments_ (including the calling environment) much like how `assign(name, value, envir)` works. However, we can assign implicit futures to environments using _named indices_, e.g.
```r
@@ -582,9 +582,9 @@ This is _not_ possible to do when using implicit futures. This is because the `
> v <- as.list(v)
> str(v)
List of 3
- $ a: int 28150
- $ b: int 28151
- $ c: int 28152
+ $ a: int 14717
+ $ b: int 14718
+ $ c: int 14719
```
Here `as.list(v)` blocks until all futures in the environment `v` have been resolved. Then their values are collected and returned as a regular list.
@@ -601,9 +601,9 @@ If _numeric indices_ are required, then _list environments_ can be used. List e
> v <- as.list(v)
> str(v)
List of 3
- $ : int 28153
- $ : int 28154
- $ : int 28155
+ $ : int 14720
+ $ : int 14721
+ $ : int 14722
```
As previously, `as.list(v)` blocks until all futures are resolved.
@@ -641,11 +641,11 @@ demo("mandelbrot", package="future", ask=FALSE)
The goal of this package is to provide a standardized and unified API for using futures in R. What you are seeing right now is an early but sincere attempt to achieve this goal. If you have comments or ideas on how to improve the 'future' package, I would love to hear about them. The preferred way to get in touch is via the [GitHub repository](https://github.com/HenrikBengtsson/future/), where you also find the latest source code. I am also open to contributions and collaborations of [...]
-[BatchJobs]: http://cran.r-project.org/package=BatchJobs
-[future]: http://cran.r-project.org/package=future
-[future.BatchJobs]: http://cran.r-project.org/package=future.BatchJobs
-[globals]: http://cran.r-project.org/package=globals
-[listenv]: http://cran.r-project.org/package=listenv
+[BatchJobs]: https://cran.r-project.org/package=BatchJobs
+[future]: https://cran.r-project.org/package=future
+[future.BatchJobs]: https://cran.r-project.org/package=future.BatchJobs
+[globals]: https://cran.r-project.org/package=globals
+[listenv]: https://cran.r-project.org/package=listenv
[Futures in R: Common Issues with Solutions]: future-2-issues.html
[Futures in R: Future Topologies]: future-3-topologies.html
---
diff --git a/vignettes/future-2-issues.md.rsp b/vignettes/future-2-issues.md.rsp
index 68a6641..6316ffc 100644
--- a/vignettes/future-2-issues.md.rsp
+++ b/vignettes/future-2-issues.md.rsp
@@ -20,91 +20,35 @@ _If you identify other cases, please consider [reporting](https://github.com/Hen
## Issues with globals
-
### Missing or incorrect globals (false negatives)
-#### Global variable obscured by subassignment
-When a global variable is a vector, a matrix, a list, a data frame, an environment, or any other type of object that can be assigned via subsetting, the global package fails to identify it as a global variable if its first occurrence in the future expression is as part of a subsetting assignment.
-For example,
-```r
-> library("future")
-> plan(lazy)
-> x <- matrix(1:12, nrow=3, ncol=4)
-> y %<-% {
-+ x[1,1] <- 3
-+ 42
-+ }
-> rm(x)
-> y
-Error in x[1, 1] <- 3 : object 'x' not found
-```
-Another example is
+#### Missing globals due to self assignment
+
+If a global variable is (only) part of a "self" assignment, such as `x <- x + 1`, then the global is not identified and therefore neither exported / frozen. This explains the error in the following example:
```r
> library("future")
-> plan(lazy)
-> x <- list(a=1, b=2)
-> y %<-% {
-+ x$c <- 3
-+ 42
-+ }
+> plan("lazy")
+> x <- 0
+> y %<-% { x <- x + 1 }
> rm(x)
> y
-Error in x$c <- 3 : object 'x' not found
+Error: object 'x' not found
```
-A workaround is to explicitly tell the future package about the global variable by simply listing it at the beginning of the expression, e.g.
+
+The workaround is to guide R to find the missing global by explicitly listing it, e.g.
```r
> library("future")
-> plan(lazy)
-> x <- list(a=1, b=2)
-> y %<-% {
-+ x ## Force 'x' to be global
-+ x$c <- 3
-+ 42
-+ }
+> plan("lazy")
+> x <- 0
+> y %<-% { x; x <- x + 1 }
> rm(x)
> y
-[1] 42
+[1] 1
```
-_Comment_: The goal is that the [globals] package will also handle the above case in a future version.
+_Comment:_ The goal is to in a future version of the package detect globals also in such self-assignment constructs.
-#### Global variable specified as part of formula
-Global variables that are part of formulas are currently not identified automatically. For example,
-```r
-> library("future")
-> plan(multisession)
->
-> x <- 1:10
-> y <- rnorm(10)
-> fit %<-% lm(y ~ x)
-> fit
-Received error on future: 'Error: object 'y' not found
-'
-Future state: 'failed'
-Future value: 'Error: object 'y' not found
-'
-Error: object 'y' not found
-```
-
-The workaround is to guide R by specifying such variables explicitly, e.g.
-```r
-> fit %<-% {
-+ x; y ## Force 'x' and 'y' to be global
-+ lm(y ~ x)
-+ }
-> fit
-
-Call:
-lm(formula = y ~ x)
-
-Coefficients:
-(Intercept) x
- -0.40506 0.01751
-```
-
-_Comment_: The goal is that the [globals] package will also handle the above case in a future version.
-
#### do.call() - function not found
When calling a function using `do.call()` make sure to specify the function as the object itself and not by name. This will help identify the function as a global object in the future expression. For instance, use
@@ -116,7 +60,7 @@ instead of
do.call("file_ext", list("foo.txt"))
```
so that `file_ext()` is properly located and exported. Although you may not notice a difference when evaluating futures in the same R session, it may become a problem if you use a character string instead of a function object when futures are evaluated in external R sessions, such as on a cluster.
-It may also become a problem with lazy futures if the intended function is redefined after the future is resolved. For example,
+It may also become a problem with lazy uniprocess futures if the intended function is redefined after the future is resolved. For example,
```r
> library("future")
> library("listenv")
@@ -132,82 +76,6 @@ It may also become a problem with lazy futures if the intended function is redef
[1] "haha!" "haha!" "haha!"
```
-<%---
-### False globals due to non-standard evaluation (false positives)
-
-#### subset(data, x < 3)
-
-Consider the following use of `subset()`:
-```r
-> data <- data.frame(x=1:5, y=1:5)
-> v <- subset(data, x < 3)$y
-> v
-[1] 1 2
-```
-From a static code inspection point of view, the expression `x < 3` asks for variable `x` to be compared to 3, and there is nothing specifying that `x` is part of `data` and not the global environment. That `x` is indeed part of the `data` object can only safely be inferred at runtime when `subset()` is called. This is not a problem in the above snippet, but when using futures all global/unknown variables need to be captured when the future is created (it is too late to do it when the [...]
-```r
-> library("future")
-> data <- data.frame(x=1:5, y=1:5)
-> v %<-% subset(data, x < 3)$y
-Error in globalsOf(expr, envir = envir, tweak = tweakExpression, dotdotdot = "return", :
- Identified a global by static code inspection, but failed to locate the corresponding
- object in the relevant environments: 'x'
-```
-Above, code inspection of the future expression `subset(data, x < 3)$y` incorrectly identifies `x` as a global variables that needs to be captured ("frozen") for the (lazy) future. Since no such variable `x` exists, we get an error.
-
-A clearer and backward-compatible solution to this problem is to explicitly specify the context of `x`, i.e.
-```r
-> data <- data.frame(x=1:5, y=1:5)
-> v %<-% subset(data, data$x < 3)$y
-> v
-[1] 1 2
-```
-
-An alternative is to use a dummy variable. In contrast to the code-inspection algorithm used to identify globals, _we know from reading the documentation_ that `subset()` will look for `x` in the `data` object, not in the parent environments. Armed with this knowledge, we can trick the [future] package (more precisely the [globals] package) to pick up a dummy variable `x` instead, e.g.
-```r
-> data <- data.frame(x=1:5, y=1:5)
-> x <- NULL ## To please future et al.
-> v %<-% subset(data, x < 3)$y
-> v
-[1] 1 2
-```
-
-#### ggplot2
-Another common use case for non-standard evaluation is when creating ggplot2 figures. For instance, in
-```r
-> library("ggplot2")
-> p <- ggplot(mtcars, aes(wt, mpg)) + geom_point()
-> p
-```
-fields `mpg` and `wt` of the `mtcars` data object are plotted against each other. That `mpg` and `wt` are fields of `mtcars` cannot be inferred from code inspection alone, but you need to know that that is how ggplot2 works. Analogously to the above `subset()` example, this explains why we get the following error:
-```r
-> library("future")
-> library("ggplot2")
-> p %<-% { ggplot(mtcars, aes(wt, mpg)) + geom_point() }
-Error in globalsOf(expr, envir = envir, tweak = tweakExpression, dotdotdot = "return", :
- Identified a global by static code inspection, but failed to locate the corresponding
- object in the relevant environments: 'wt'
-```
-A few comments are needed here. First of all, because `%<-%` has higher precedence than `+`, we need to place all of the ggplot2 expression within curly brackets, otherwise, we get an error. Second, the reason for only `wt` being listed as a missing global variable and not `mpg` is because the latter is (incorrectly) located to be `ggplot2::mpg`.
-
-One workaround is to make use of the `*_string()` functions of ggplot2, e.g.
-```r
-> p %<-% { ggplot(mtcars, aes_string('wt', 'mpg')) + geom_point() }
-> p
-```
-Another one, is to explicitly specify `mtcars$wt` and `mtcars$mpg`, which may become a bit tedious.
-A third alternative is to make use of dummy variables `wt` and `mpg`, i.e.
-```r
-> p %<-% {
-+ wt <- mpg <- NULL
-+ ggplot(mtcars, aes(wt, mpg)) + geom_point()
-+ }
-> p
-```
-By the way, since all futures are evaluated in a local environment, the dummy variables are _not_ assigned to the calling environment.
----%>
-
-
## Trying to pass an unresolved future to another future
It is not possible for a future to resolve another one unless it was created by the future trying to resolve it. For instance, the following gives an error:
```r
@@ -265,7 +133,7 @@ Error: Invalid usage of futures: A future whose value has not yet been collected
```
As previously, this can be avoided by making sure `x$a` is resolved first, which can be one in various ways, e.g. `dummy <- x$a`, `resolve(x$a)` and `force(x$a)`.
-_Footnote_: (*) Although certain types of futures, such as eager and lazy ones, could be passed on to other futures and be resolved there because they share the same evaluation process, by definition of the Future API it is invalid to do so regardless of future type. This conservative approach is taken in order to make future expressions behave consistently regardless of the type of future used.
+_Footnote_: (*) Although uniprocess futures could be passed on to other futures part of the same R process and be resolved there because they share the same evaluation process, by definition of the Future API it is invalid to do so regardless of future type. This conservative approach is taken in order to make future expressions behave consistently regardless of the type of future used.
## Miscellaneous
@@ -349,6 +217,32 @@ What effectively is happening here is that because of the higher priority of `%<
Parentheses will also do. For details on precedence on operators in R, see Section 'Infix and prefix operators' in the 'R Language Definition' document.
+### R CMD check NOTEs
+The code inspection run by `R CMD check` will not recognize the future assignment operator `%<-%` as an assignment operator, which is not surprising because `%<-%` is technically an infix operator. This means that if you for instance use the following code in your package:
+```r
+foo <- function() {
+ b <- 3.14
+ a %<-% { b + 1 }
+ a
+}
+```
+then `R CMD check` will produce a NOTE saying:
+```sh
+* checking R code for possible problems ... NOTE
+foo: no visible binding for global variable 'a'
+Undefined global functions or variables:
+ a
+```
+In order to avoid this, we can add a dummy assignment of the missing global at the top of the function, i.e.
+```r
+foo <- function() {
+ a <- NULL ## To please R CMD check
+ b <- 3.14
+ a %<-% { b + 1 }
+ a
+}
+```
+
[future]: https://cran.r-project.org/package=future
[globals]: https://cran.r-project.org/package=globals
diff --git a/vignettes/future-3-topologies.md.rsp b/vignettes/future-3-topologies.md.rsp
index 7b1ba5c..c2bd7c5 100644
--- a/vignettes/future-3-topologies.md.rsp
+++ b/vignettes/future-3-topologies.md.rsp
@@ -34,7 +34,7 @@ for (ii in 1:3) {
unlist(x)
## [1] 1.1 1.2 1.3 2.1 2.2 2.3 3.1 3.2 3.3
```
-The default is to use eager futures unless otherwise specified, which is also true for nested futures. If we for instance specify, `plan(multiprocess)`, the first layer of futures (`x[[ii]] %<-% { expr }`) will be processed asynchronously in background R processes, and the futures in the second layer of futures (`y[[jj]] %<-% { expr }`) will be processed synchronously in the current R processes (one of the background processes). If we wish to use a different strategy for the second lay [...]
+The default is to use synchronous futures unless otherwise specified, which is also true for nested futures. If we for instance specify, `plan(multiprocess)`, the first layer of futures (`x[[ii]] %<-% { expr }`) will be processed asynchronously in background R processes, and the futures in the second layer of futures (`y[[jj]] %<-% { expr }`) will be processed synchronously in the current R processes (one of the background processes). If we wish to use a different strategy for the seco [...]
## Example: High-Throughput Sequencing
@@ -62,23 +62,23 @@ for (ss in seq_along(fqs)) {
bams <- as.list(bams)
```
-The default is to use eager futures, so without further specifications, the above will process each sample and each chromosome sequentially. Next, we will consider what can be done with the following two computer setups:
+The default is to use synchronous futures, so without further specifications, the above will process each sample and each chromosome sequentially. Next, we will consider what can be done with the following two computer setups:
* A single machine with 8 cores
* A compute cluster with 3 machines each with 16 cores
### One multi-core machine
-With a single machine of 8 cores, we could choose to process multiple samples at the same while processing chromosomes sequentially. In other words, we would like to evaluate the outer layer of futures using multiprocess futures and the inner ones as regular (uniprocess) eager futures. This can be specified as:
+With a single machine of 8 cores, we could choose to process multiple samples at the same time while processing chromosomes sequentially. In other words, we would like to evaluate the outer layer of futures using multiprocess futures and the inner ones as eager synchronous futures. This can be specified as:
```r
plan(list(multiprocess, eager))
```
The internals for processing multiprocess future queries `availableCores()` to infer how many cores can be used simultaneously, so there is no need to explicitly specify that there are 8 cores available.
-_Comment_: Since eager is the default future, we could skip trailing eager futures in the setup, e.g. `plan(list(multiprocess))` or just `plan(multiprocess)`. However, it does hurt to be explicit.
+_Comment_: Since synchronous is the default future, we could skip trailing synchronous eager futures in the setup, e.g. `plan(list(multiprocess))` or just `plan(uniprocess)`. However, it does not hurt to be explicit.
If we instead would like to process the sample sequentially and the chromosomes in parallel, we can use:
```r
-plan(list(eager, multiprocess))
+plan(list(uniprocess, multiprocess))
```
We could also process the data such that we allocate two cores for processing two samples in parallel each using four cores for processing four chromosomes in parallel:
@@ -166,8 +166,8 @@ print(x)
Try the above `x %<-% { ... }` future with, say, `plan(list(lazy, multiprocess))` or `plan(list(eager, lazy, multiprocess))` and see what the output will be.
-[listenv]: http://cran.r-project.org/package=listenv
-[globals]: http://cran.r-project.org/package=globals
+[listenv]: https://cran.r-project.org/package=listenv
+[globals]: https://cran.r-project.org/package=globals
[Futures in R: Common issues with solutions]: future-issues.html
---
diff --git a/vignettes/future-4-startup.md.rsp b/vignettes/future-4-startup.md.rsp
new file mode 100644
index 0000000..e8637d8
--- /dev/null
+++ b/vignettes/future-4-startup.md.rsp
@@ -0,0 +1,76 @@
+<%@meta language="R-vignette" content="--------------------------------
+%\VignetteIndexEntry{A Future for R: Controlling Default Future Strategy}
+%\VignetteAuthor{Henrik Bengtsson}
+%\VignetteKeyword{R}
+%\VignetteKeyword{package}
+%\VignetteKeyword{vignette}
+%\VignetteKeyword{future}
+%\VignetteKeyword{promise}
+%\VignetteEngine{R.rsp::rsp}
+%\VignetteTangle{FALSE}
+--------------------------------------------------------------------"%>
+<%
+library("R.utils")
+`%<-%` <- future::`%<-%`
+options("withCapture/newline"=FALSE)
+%>
+
+# <%@meta name="title"%>
+
+The default is to use synchronous futures, but this _default_ can be overridden via R options, system environment variables and command-line options as explained below as well as in `help("future.options", package="future")`.
+
+
+## R options
+The default strategy for resolving futures can be controlled via R option `future.plan`. For instance, if we add
+```r
+options(future.plan="multiprocess")
+```
+to our `~/.Rprofile` startup script, the future package will resolve futures in parallel (asynchronously using all available cores), i.e.
+```sh
+$ Rscript -e "class(future::plan())"
+[1] "multiprocess" "future" "function"
+```
+
+Option `future.plan` is ignored if command-line option `--parallel` (`-p`) is specified.
+
+
+## Environment variables
+An alternative to using `options()` for setting option `future.plan` is to specify system environment variable `R_FUTURE_PLAN`. If set, then the future package will set `future.plan` accordingly _when loaded_. For example,
+```sh
+$ export R_FUTURE_PLAN=multiprocess
+$ Rscript -e "class(future::plan())"
+[1] "multiprocess" "future" "function"
+```
+Environment variable `R_FUTURE_PLAN` is ignored if either option `future.plan` or command-line option `--parallel` (`-p`) is specified.
+
+
+## Command-line options
+
+When loaded, the future package checks for the command-line option `--parallel=ncores` (short `-p ncores`) and sets the future strategy (via option `future.plan`) and the number of available cores (via option `mc.cores`) accordingly. This provides a convenient mechanism for specifying parallel future processing from the command line. For example, if we start R with
+```sh
+$ R --quiet --args --parallel=2
+```
+then future will interpret this as we wish to resolve futures in parallel using 2 cores. More specifically, we get that
+```r
+> future::availableCores()
+mc.cores+1
+ 2
+> class(future::plan())
+[1] "tweaked" "multiprocess" "future" "function"
+```
+
+We can use this command-line option also with `Rscript`, which provides a convenient mechanism for launching future-enhanced R scripts such that they run in parallel, e.g.
+```sh
+$ Rscript analysis.R --parallel=4
+```
+This does, of course, require that the script uses futures and the future package.
+
+If `--parallel=1` is specified, or equivalently `-p 1`, then futures are resolved using a single process.
+
+Specifying these command-line options override any other startup settings.
+
+
+[future]: https://cran.r-project.org/package=future
+
+---
+Copyright Henrik Bengtsson, 2015-2016
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/debian-med/r-cran-future.git
More information about the debian-med-commit
mailing list